tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] checks for torrent contents when updating existing torrent

Browse Source
This commit is contained in:
TheBeastLT
2020-05-06 11:59:53 +02:00
parent 0ad7324a6a
commit 9ce3f72e5d
12 changed files with 125 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
scraper/scrapers/1337x/1337x_scraper.js
View File

@@ -4,7 +4,7 @@ const leetx = require('./1337x_api');
const { Type } = require('../../lib/types');
const repository = require('../../lib/repository');
const Promises = require('../../lib/promises');
const { createTorrentEntry, getStoredTorrentEntry, updateTorrentSeeders } = require('../../lib/torrentEntries');
const { createTorrentEntry, checkAndUpdateTorrent } = require('../../lib/torrentEntries');
const NAME = '1337x';
const UNTIL_PAGE = 10;
@@ -65,8 +65,8 @@ async function processTorrentRecord(record) {
console.warn(`Incorrect upload date for [${torrentFound.infoHash}] ${torrentFound.name}`);
return;
}
if (await getStoredTorrentEntry(torrentFound)) {
return updateTorrentSeeders(torrentFound);
if (await checkAndUpdateTorrent(torrentFound)) {
return torrentFound;
}
const torrent = {

6
scraper/scrapers/eztv/eztv_scraper.js
View File

@@ -3,7 +3,7 @@ const Bottleneck = require('bottleneck');
const eztv = require('./eztv_api');
const { Type } = require('../../lib/types');
const repository = require('../../lib/repository');
const { createTorrentEntry, getStoredTorrentEntry, updateTorrentSeeders } = require('../../lib/torrentEntries');
const { createTorrentEntry, checkAndUpdateTorrent } = require('../../lib/torrentEntries');
const NAME = 'EZTV';
const UNTIL_PAGE = 10;
@@ -50,8 +50,8 @@ async function scrapeLatestTorrentsForCategory(page = 1) {
}
async function processTorrentRecord(record) {
if (await getStoredTorrentEntry(record)) {
return updateTorrentSeeders(record);
if (await checkAndUpdateTorrent(record)) {
return record;
}
if (!record || !record.size) {

4
scraper/scrapers/horriblesubs/horriblesubs_scraper.js
View File

@@ -6,7 +6,7 @@ const horriblesubs = require('./horriblesubs_api.js');
const repository = require('../../lib/repository');
const { Type } = require('../../lib/types');
const { updateCurrentSeeders, updateTorrentSize } = require('../../lib/torrent');
const { createTorrentEntry, updateTorrentSeeders } = require('../../lib/torrentEntries');
const { createTorrentEntry, checkAndUpdateTorrent } = require('../../lib/torrentEntries');
const { getMetadata, getKitsuId } = require('../../lib/metadata');
const showMappings = require('./horriblesubs_mapping.json');
@@ -172,7 +172,7 @@ async function processTorrentRecord(torrent, updateSeeders = true) {
if (existingTorrent && existingTorrent.provider === NAME) {
if (updateSeeders) {
return updateCurrentSeeders(torrent).then(updatedSeeders => updateTorrentSeeders(updatedSeeders))
return updateCurrentSeeders(torrent).then(updatedSeeders => checkAndUpdateTorrent(updatedSeeders))
}
return Promise.resolve(torrent)
}

6
scraper/scrapers/kickass/kickass_scraper.js
View File

@@ -4,7 +4,7 @@ const kickass = require('./kickass_api');
const { Type } = require('../../lib/types');
const repository = require('../../lib/repository');
const Promises = require('../../lib/promises');
const { createTorrentEntry, getStoredTorrentEntry, updateTorrentSeeders } = require('../../lib/torrentEntries');
const { createTorrentEntry, checkAndUpdateTorrent } = require('../../lib/torrentEntries');
const NAME = 'KickassTorrents';
const UNTIL_PAGE = 10;
@@ -54,8 +54,8 @@ async function scrapeLatestTorrentsForCategory(category, page = 1) {
}
async function processTorrentRecord(record) {
if (await getStoredTorrentEntry(record)) {
return updateTorrentSeeders(record);
if (await checkAndUpdateTorrent(record)) {
return record;
}
const torrentFound = await kickass.torrent(record.torrentId).catch(() => undefined);

11
scraper/scrapers/rarbg/rarbg_dump_scraper.js
View File

@@ -3,12 +3,7 @@ const Bottleneck = require('bottleneck');
const rarbg = require('rarbg-api');
const decode = require('magnet-uri');
const { Type } = require('../../lib/types');
const {
createTorrentEntry,
getStoredTorrentEntry,
updateTorrentSeeders
} = require('../../lib/torrentEntries');
const { createTorrentEntry, checkAndUpdateTorrent } = require('../../lib/torrentEntries');
const NAME = 'RARBG';
const limiter = new Bottleneck({ maxConcurrent: 1, minTime: 2500 });
@@ -54,8 +49,8 @@ async function getTorrentsForImdbId(imdbId, retries = 5) {
}
async function processTorrentRecord(record) {
if (await getStoredTorrentEntry(record)) {
return updateTorrentSeeders(record);
if (await checkAndUpdateTorrent(record)) {
return record;
}
const torrent = {

6
scraper/scrapers/rarbg/rarbg_scraper.js
View File

@@ -5,7 +5,7 @@ const decode = require('magnet-uri');
const { Type } = require('../../lib/types');
const repository = require('../../lib/repository');
const Promises = require('../../lib/promises');
const { createTorrentEntry, getStoredTorrentEntry, updateTorrentSeeders } = require('../../lib/torrentEntries');
const { createTorrentEntry, checkAndUpdateTorrent } = require('../../lib/torrentEntries');
const NAME = 'RARBG';
const SEARCH_OPTIONS = { limit: 100, sort: 'seeders', format: 'json_extended', ranked: 0 };
@@ -70,8 +70,8 @@ async function scrapeLatestTorrentsForCategory(category, retries = 5) {
}
async function processTorrentRecord(record) {
if (await getStoredTorrentEntry(record)) {
return updateTorrentSeeders(record);
if (await checkAndUpdateTorrent(record)) {
return record;
}
const torrent = {

6
scraper/scrapers/thepiratebay/thepiratebay_scraper.js
View File

@@ -4,7 +4,7 @@ const thepiratebay = require('./thepiratebay_api.js');
const { Type } = require('../../lib/types');
const repository = require('../../lib/repository');
const Promises = require('../../lib/promises');
const { createTorrentEntry, getStoredTorrentEntry, updateTorrentSeeders } = require('../../lib/torrentEntries');
const { createTorrentEntry, checkAndUpdateTorrent } = require('../../lib/torrentEntries');
const NAME = 'ThePirateBay';
const UNTIL_PAGE = 5;
@@ -60,8 +60,8 @@ async function scrapeLatestTorrentsForCategory(category, page = 1) {
}
async function processTorrentRecord(record) {
if (await getStoredTorrentEntry(record)) {
return updateTorrentSeeders(record);
if (await checkAndUpdateTorrent(record)) {
return record;
}
const torrentFound = await thepiratebay.torrent(record.torrentId).catch(() => undefined);

8
scraper/scrapers/yts/yts_scraper.js
View File

@@ -3,12 +3,12 @@ const Bottleneck = require('bottleneck');
const yts = require('./yts_api');
const { Type } = require('../../lib/types');
const repository = require('../../lib/repository');
const { createTorrentEntry, getStoredTorrentEntry, updateTorrentSeeders } = require('../../lib/torrentEntries');
const { createTorrentEntry, checkAndUpdateTorrent } = require('../../lib/torrentEntries');
const NAME = 'YTS';
const UNTIL_PAGE = 2;
const limiter = new Bottleneck({ maxConcurrent: 20 });
const limiter = new Bottleneck({ maxConcurrent: 10 });
async function scrape() {
const scrapeStart = moment();
@@ -45,8 +45,8 @@ async function scrapeLatestTorrentsForCategory(page = 1) {
}
async function processTorrentRecord(record) {
if (await getStoredTorrentEntry(record)) {
return updateTorrentSeeders(record);
if (await checkAndUpdateTorrent(record)) {
return record;
}
if (!record || !record.size) {