tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] reduces number of calls to 1337x

Browse Source
This commit is contained in:
TheBeastLT
2020-10-02 12:49:45 +02:00
parent 42714d19a9
commit 5426707dd9
5 changed files with 18 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
scraper/lib/repository.js
View File

@@ -163,13 +163,10 @@ function getProvider(provider) {
} }
function getTorrent(torrent) { function getTorrent(torrent) {
return Torrent.findByPk(torrent.infoHash) const where = torrent.infoHash
.then((result) => { ? { infoHash: torrent.infoHash }
if (!result) { : { provider: torrent.provider, torrentId: torrent.torrentId }
throw new Error(`torrent not found: ${torrent.infoHash}`); return Torrent.findOne({ where: where });
}
return result;
})
} }
function getTorrentsBasedOnTitle(titleQuery, type) { function getTorrentsBasedOnTitle(titleQuery, type) {
@@ -212,10 +209,13 @@ function createTorrent(torrent) {
.then(() => createSubtitles(torrent.infoHash, torrent.subtitles)); .then(() => createSubtitles(torrent.infoHash, torrent.subtitles));
} }
function setTorrentSeeders(infoHash, seeders) { function setTorrentSeeders(torrent, seeders) {
const where = torrent.infoHash
? { infoHash: torrent.infoHash }
: { provider: torrent.provider, torrentId: torrent.torrentId }
return Torrent.update( return Torrent.update(
{ seeders: seeders }, { seeders: seeders },
{ where: { infoHash: infoHash } } { where: where }
); );
} }

4
scraper/lib/torrentEntries.js
View File

@@ -151,11 +151,11 @@ async function createTorrentContents(torrent) {
} }
async function updateTorrentSeeders(torrent) { async function updateTorrentSeeders(torrent) {
if (!torrent.infoHash || !Number.isInteger(torrent.seeders)) { if (!(torrent.infoHash || (torrent.provider && torrent.torrentId)) || !Number.isInteger(torrent.seeders)) {
return; return;
} }
return repository.setTorrentSeeders(torrent.infoHash, torrent.seeders) return repository.setTorrentSeeders(torrent, torrent.seeders)
.catch(error => { .catch(error => {
console.warn('Failed updating seeders:', error); console.warn('Failed updating seeders:', error);
return undefined; return undefined;

4
scraper/scrapers/1337x/1337x_api.js
View File

@@ -4,7 +4,7 @@ const Sugar = require('sugar-date');
const decode = require('magnet-uri'); const decode = require('magnet-uri');
const Promises = require('../../lib/promises'); const Promises = require('../../lib/promises');
const { escapeHTML } = require('../../lib/metadata'); const { escapeHTML } = require('../../lib/metadata');
const { defaultOptionsWithProxy } = require('../../lib/request_helper'); const { getRandomUserAgent, defaultOptionsWithProxy } = require('../../lib/request_helper');
const defaultProxies = [ const defaultProxies = [
'https://1337x.to' 'https://1337x.to'
@@ -80,7 +80,7 @@ function browse(config = {}, retries = 2) {
function singleRequest(requestUrl, config = {}) { function singleRequest(requestUrl, config = {}) {
const timeout = config.timeout || defaultTimeout; const timeout = config.timeout || defaultTimeout;
const options = { ...defaultOptionsWithProxy(), open_timeout: timeout, follow: 2 }; const options = { userAgent: getRandomUserAgent(), open_timeout: timeout, follow: 2 };
return needle('get', requestUrl, options) return needle('get', requestUrl, options)
.then((response) => { .then((response) => {

7
scraper/scrapers/1337x/1337x_scraper.js
View File

@@ -56,6 +56,10 @@ async function scrapeLatestTorrentsForCategory(category, page = 1) {
} }
async function processTorrentRecord(record) { async function processTorrentRecord(record) {
if (await checkAndUpdateTorrent({ provider: NAME, ...record })) {
return record;
}
const torrentFound = await leetx.torrent(record.torrentId).catch(() => undefined); const torrentFound = await leetx.torrent(record.torrentId).catch(() => undefined);
if (!torrentFound || !TYPE_MAPPING[torrentFound.category]) { if (!torrentFound || !TYPE_MAPPING[torrentFound.category]) {
@@ -65,9 +69,6 @@ async function processTorrentRecord(record) {
console.warn(`Incorrect upload date for [${torrentFound.infoHash}] ${torrentFound.name}`); console.warn(`Incorrect upload date for [${torrentFound.infoHash}] ${torrentFound.name}`);
return; return;
} }
if (await checkAndUpdateTorrent(torrentFound)) {
return torrentFound;
}
const torrent = { const torrent = {
infoHash: torrentFound.infoHash, infoHash: torrentFound.infoHash,

2
scraper/scrapers/horriblesubs/horriblesubs_mapping.json
View File

@@ -639,7 +639,7 @@
}, },
"Grisaia no Meikyuu": { "Grisaia no Meikyuu": {
"showId": "159", "showId": "159",
"kitsu_id": "11236" "kitsu_id": "10154"
}, },
"Grisaia no Rakuen": { "Grisaia no Rakuen": {
"showId": "160", "showId": "160",