tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] update seeders for tpb from trackers not website

Browse Source
This commit is contained in:
TheBeastLT
2021-01-11 09:56:00 +01:00
parent 0522a5e8ab
commit 4de4801b13
3 changed files with 19 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
scraper/lib/torrentEntries.js
View File

@@ -152,7 +152,7 @@ async function createTorrentContents(torrent) {
async function updateTorrentSeeders(torrent) { async function updateTorrentSeeders(torrent) {
if (!(torrent.infoHash || (torrent.provider && torrent.torrentId)) || !Number.isInteger(torrent.seeders)) { if (!(torrent.infoHash || (torrent.provider && torrent.torrentId)) || !Number.isInteger(torrent.seeders)) {
return; return torrent;
} }
return repository.setTorrentSeeders(torrent, torrent.seeders) return repository.setTorrentSeeders(torrent, torrent.seeders)

2
scraper/scrapers/thepiratebay/thepiratebay_api.js
View File

@@ -117,7 +117,7 @@ function toTorrent(result) {
name: escapeHTML(result.name), name: escapeHTML(result.name),
infoHash: result.info_hash.toLowerCase(), infoHash: result.info_hash.toLowerCase(),
size: parseInt(result.size), size: parseInt(result.size),
seeders: parseInt(result.seeders), // seeders: parseInt(result.seeders),
leechers: parseInt(result.leechers), leechers: parseInt(result.leechers),
subcategory: parseInt(result.category), subcategory: parseInt(result.category),
uploadDate: new Date(result.added * 1000), uploadDate: new Date(result.added * 1000),

32
scraper/scrapers/thepiratebay/thepiratebay_scraper.js
View File

@@ -4,17 +4,17 @@ const thepiratebay = require('./thepiratebay_api.js');
const { Type } = require('../../lib/types'); const { Type } = require('../../lib/types');
const repository = require('../../lib/repository'); const repository = require('../../lib/repository');
const Promises = require('../../lib/promises'); const Promises = require('../../lib/promises');
const { updateCurrentSeeders } = require('../../lib/torrent');
const { createTorrentEntry, checkAndUpdateTorrent } = require('../../lib/torrentEntries'); const { createTorrentEntry, checkAndUpdateTorrent } = require('../../lib/torrentEntries');
const NAME = 'ThePirateBay'; const NAME = 'ThePirateBay';
const UNTIL_PAGE = 5; const UNTIL_PAGE = 5;
const limiter = new Bottleneck({ maxConcurrent: 10 }); const limiter = new Bottleneck({ maxConcurrent: 5 });
const allowedCategories = [ const allowedCategories = [
thepiratebay.Categories.VIDEO.MOVIES, thepiratebay.Categories.VIDEO.MOVIES,
thepiratebay.Categories.VIDEO.MOVIES_HD, thepiratebay.Categories.VIDEO.MOVIES_HD,
thepiratebay.Categories.VIDEO.MOVIES_DVDR,
thepiratebay.Categories.VIDEO.MOVIES_3D, thepiratebay.Categories.VIDEO.MOVIES_3D,
thepiratebay.Categories.VIDEO.TV_SHOWS, thepiratebay.Categories.VIDEO.TV_SHOWS,
thepiratebay.Categories.VIDEO.TV_SHOWS_HD thepiratebay.Categories.VIDEO.TV_SHOWS_HD
@@ -38,7 +38,8 @@ async function scrape() {
} }
async function updateSeeders(torrent) { async function updateSeeders(torrent) {
return limiter.schedule(() => thepiratebay.torrent(torrent.torrentId)); // return limiter.schedule(() => thepiratebay.torrent(torrent.torrentId));
return Promise.resolve([]);
} }
async function scrapeLatestTorrents() { async function scrapeLatestTorrents() {
@@ -64,23 +65,24 @@ async function processTorrentRecord(record) {
return record; return record;
} }
const torrentFound = await thepiratebay.torrent(record.torrentId).catch(() => undefined); if (!record || !allowedCategories.includes(record.subcategory)) {
if (!torrentFound || !allowedCategories.includes(torrentFound.subcategory)) {
return Promise.resolve('Invalid torrent record'); return Promise.resolve('Invalid torrent record');
} }
if (record.seeders === null || record.seeders === undefined) {
await updateCurrentSeeders(record);
}
const torrent = { const torrent = {
infoHash: torrentFound.infoHash, infoHash: record.infoHash,
provider: NAME, provider: NAME,
torrentId: torrentFound.torrentId, torrentId: record.torrentId,
title: torrentFound.name.replace(/\t|\s+/g, ' '), title: record.name.replace(/\t|\s+/g, ' '),
type: seriesCategories.includes(torrentFound.subcategory) ? Type.SERIES : Type.MOVIE, type: seriesCategories.includes(record.subcategory) ? Type.SERIES : Type.MOVIE,
size: torrentFound.size, size: record.size,
seeders: torrentFound.seeders, seeders: record.seeders,
uploadDate: torrentFound.uploadDate, uploadDate: record.uploadDate,
imdbId: seriesCategories.includes(torrentFound.subcategory) && torrentFound.imdbId || undefined, imdbId: seriesCategories.includes(record.subcategory) && record.imdbId || undefined,
languages: torrentFound.languages && torrentFound.languages.trim() || undefined languages: record.languages && record.languages.trim() || undefined
}; };
return createTorrentEntry(torrent); return createTorrentEntry(torrent);