tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] clean up torrent galaxy provider

Browse Source
This commit is contained in:
TheBeastLT
2021-02-01 19:46:46 +01:00
parent 473747bd33
commit 31b0c85fcf
3 changed files with 39 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
scraper/scrapers/torrentgalaxy/torrentgalaxy_scraper.js
View File

@@ -9,6 +9,7 @@ const { createTorrentEntry, checkAndUpdateTorrent } = require('../../lib/torrent
const NAME = 'TorrentGalaxy';
const TYPE_MAPPING = typeMapping();
const api_limiter = new Bottleneck({ maxConcurrent: 1, minTime: 5000 });
const limiter = new Bottleneck({ maxConcurrent: 10 });
const allowedCategories = [
torrentGalaxy.Categories.ANIME,
@@ -21,7 +22,6 @@ const allowedCategories = [
torrentGalaxy.Categories.TV_SD,
torrentGalaxy.Categories.TV_HD,
torrentGalaxy.Categories.TV_PACKS,
torrentGalaxy.Categories.TV_SPORT,
torrentGalaxy.Categories.DOCUMENTARIES,
];
const packCategories = [
@@ -57,7 +57,7 @@ async function scrapeLatestTorrents() {
async function scrapeLatestTorrentsForCategory(category, page = 1) {
console.log(`Scrapping ${NAME} ${category} category page ${page}`);
return torrentGalaxy.browse(({ category, page }))
return api_limiter.schedule(() => torrentGalaxy.browse({ category, page }))
.catch(error => {
console.warn(`Failed ${NAME} scrapping for [${page}] ${category} due: `, error);
return Promise.resolve([]);
@@ -73,10 +73,6 @@ async function processTorrentRecord(record) {
return Promise.resolve('Invalid torrent record');
}
if (await checkAndUpdateTorrent(record)) {
return record;
}
const torrent = {
provider: NAME,
infoHash: record.infoHash,
@@ -92,6 +88,15 @@ async function processTorrentRecord(record) {
languages: !(record.languages || '').includes('Other') ? record.languages : undefined
};
if (await checkAndUpdateTorrent(torrent)) {
return torrent;
}
const isOld = moment(torrent.uploadDate).isBefore(moment().subtract(18, 'month'));
if (torrent.seeders === 0 && isOld && !torrent.pack) {
console.log(`Skipping old unseeded torrent [${torrent.infoHash}] ${torrent.title}`)
return torrent;
}
return createTorrentEntry(torrent).then(() => torrent);
}
@@ -116,7 +121,6 @@ function getMaxPage(category) {
switch (category) {
case torrentGalaxy.Categories.TV_SD:
case torrentGalaxy.Categories.TV_HD:
return 10;
case torrentGalaxy.Categories.MOVIE_SD:
case torrentGalaxy.Categories.MOVIE_HD:
return 5;