brock/torrentio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[scraper] update seeders for tpb from trackers not website

Browse Source
This commit is contained in:
TheBeastLT
2021-01-11 09:56:00 +01:00
parent 0522a5e8ab
commit 4de4801b13
3 changed files with 19 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
scraper/lib/torrentEntries.js
View File

@@ -152,7 +152,7 @@ async function createTorrentContents(torrent) {
async function updateTorrentSeeders(torrent) {
if (!(torrent.infoHash || (torrent.provider && torrent.torrentId)) || !Number.isInteger(torrent.seeders)) {
return;
return torrent;
}
return repository.setTorrentSeeders(torrent, torrent.seeders)

2
scraper/scrapers/thepiratebay/thepiratebay_api.js
View File

@@ -117,7 +117,7 @@ function toTorrent(result) {
name: escapeHTML(result.name),
infoHash: result.info_hash.toLowerCase(),
size: parseInt(result.size),
seeders: parseInt(result.seeders),
// seeders: parseInt(result.seeders),
leechers: parseInt(result.leechers),
subcategory: parseInt(result.category),
uploadDate: new Date(result.added * 1000),

32
scraper/scrapers/thepiratebay/thepiratebay_scraper.js
View File

@@ -4,17 +4,17 @@ const thepiratebay = require('./thepiratebay_api.js');
const { Type } = require('../../lib/types');
const repository = require('../../lib/repository');
const Promises = require('../../lib/promises');
const { updateCurrentSeeders } = require('../../lib/torrent');
const { createTorrentEntry, checkAndUpdateTorrent } = require('../../lib/torrentEntries');
const NAME = 'ThePirateBay';
const UNTIL_PAGE = 5;
const limiter = new Bottleneck({ maxConcurrent: 10 });
const limiter = new Bottleneck({ maxConcurrent: 5 });
const allowedCategories = [
thepiratebay.Categories.VIDEO.MOVIES,
thepiratebay.Categories.VIDEO.MOVIES_HD,
thepiratebay.Categories.VIDEO.MOVIES_DVDR,
thepiratebay.Categories.VIDEO.MOVIES_3D,
thepiratebay.Categories.VIDEO.TV_SHOWS,
thepiratebay.Categories.VIDEO.TV_SHOWS_HD
@@ -38,7 +38,8 @@ async function scrape() {
}
async function updateSeeders(torrent) {
return limiter.schedule(() => thepiratebay.torrent(torrent.torrentId));
// return limiter.schedule(() => thepiratebay.torrent(torrent.torrentId));
return Promise.resolve([]);
}
async function scrapeLatestTorrents() {
@@ -64,23 +65,24 @@ async function processTorrentRecord(record) {
return record;
}
const torrentFound = await thepiratebay.torrent(record.torrentId).catch(() => undefined);
if (!torrentFound || !allowedCategories.includes(torrentFound.subcategory)) {
if (!record || !allowedCategories.includes(record.subcategory)) {
return Promise.resolve('Invalid torrent record');
}
if (record.seeders === null || record.seeders === undefined) {
await updateCurrentSeeders(record);
}
const torrent = {
infoHash: torrentFound.infoHash,
infoHash: record.infoHash,
provider: NAME,
torrentId: torrentFound.torrentId,
title: torrentFound.name.replace(/\t|\s+/g, ' '),
type: seriesCategories.includes(torrentFound.subcategory) ? Type.SERIES : Type.MOVIE,
size: torrentFound.size,
seeders: torrentFound.seeders,
uploadDate: torrentFound.uploadDate,
imdbId: seriesCategories.includes(torrentFound.subcategory) && torrentFound.imdbId || undefined,
languages: torrentFound.languages && torrentFound.languages.trim() || undefined
torrentId: record.torrentId,
title: record.name.replace(/\t|\s+/g, ' '),
type: seriesCategories.includes(record.subcategory) ? Type.SERIES : Type.MOVIE,
size: record.size,
seeders: record.seeders,
uploadDate: record.uploadDate,
imdbId: seriesCategories.includes(record.subcategory) && record.imdbId || undefined,
languages: record.languages && record.languages.trim() || undefined
};
return createTorrentEntry(torrent);