tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] clean up torrent galaxy provider

Browse Source
This commit is contained in:
TheBeastLT
2021-02-01 19:46:46 +01:00
parent 473747bd33
commit 31b0c85fcf
3 changed files with 39 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
scraper/scrapers/1337x/1337x_api.js
View File

@@ -4,7 +4,7 @@ const Sugar = require('sugar-date');
const decode = require('magnet-uri');
const Promises = require('../../lib/promises');
const { escapeHTML } = require('../../lib/metadata');
const { getRandomUserAgent, defaultOptionsWithProxy } = require('../../lib/requestHelper');
const { getRandomUserAgent } = require('../../lib/requestHelper');
const defaultProxies = [
'https://1337x.to'

49
scraper/scrapers/torrentgalaxy/torrentgalaxy_api.js
View File

@@ -3,11 +3,12 @@ const needle = require('needle');
const moment = require('moment');
const decode = require('magnet-uri');
const Promises = require('../../lib/promises');
const { getRandomUserAgent } = require('../../lib/requestHelper');
const defaultProxies = [
'https://torrentgalaxy.to',
// 'https://torrentgalaxy.to',
// 'https://torrentgalaxy.mx',
// 'https://torrentgalaxy.su'
'https://torrentgalaxy.su'
];
const defaultTimeout = 10000;
@@ -53,9 +54,9 @@ function search(keyword, config = {}, retries = 2) {
.catch(() => search(keyword, config, retries - 1));
}
function browse(config = {}, retries = 2) {
function browse(config = {}, retries = 2, error = null) {
if (retries === 0) {
return Promise.reject(new Error(`Failed browse request`));
return Promise.reject(error || new Error(`Failed browse request`));
}
const proxyList = config.proxyList || defaultProxies;
const page = config.page || 1;
@@ -64,17 +65,17 @@ function browse(config = {}, retries = 2) {
return Promises.first(proxyList
.map((proxyUrl) => singleRequest(`${proxyUrl}/torrents.php?cat=${category}&page=${page - 1}`)))
.then((body) => parseTableBody(body))
.catch(() => browse(config, retries - 1));
.catch((err) => browse(config, retries - 1, err));
}
function singleRequest(requestUrl) {
const options = { open_timeout: defaultTimeout, follow: 2 };
const options = { userAgent: getRandomUserAgent(), open_timeout: defaultTimeout, follow: 2 };
return needle('get', requestUrl, options)
.then((response) => {
const body = response.body;
if (!body) {
throw new Error(`No body: ${requestUrl}`);
throw new Error(`No body: ${requestUrl} with status ${response.statusCode}`);
} else if (body.includes('Access Denied')) {
console.log(`Access Denied: ${requestUrl}`);
throw new Error(`Access Denied: ${requestUrl}`);
@@ -102,21 +103,25 @@ function parseTableBody(body) {
const row = $(element);
const magnetLink = row.find('div:nth-of-type(n+2) .collapsehide > a:nth-of-type(2)').attr('href');
const imdbIdMatch = row.html().match(/search=(tt\d+)/i);
torrents.push({
name: row.find('.tgxtablecell div a[title]').first().text(),
infoHash: decode(magnetLink).infoHash,
magnetLink: magnetLink,
torrentLink: row.find('div:nth-of-type(n+2) .collapsehide > a:nth-of-type(1)').first().attr('href'),
torrentId: row.find('.tgxtablecell div a[title]').first().attr('href').match(/torrent\/(\d+)/)[1],
verified: !!row.find('i.fa-check').length,
category: row.find('div:nth-of-type(n+2) .shrink a').first().attr('href').match(/cat=(\d+)$/)[1],
seeders: parseInt(row.find('div:nth-of-type(n+2) .collapsehide [color=\'green\'] b').first().text()),
leechers: parseInt(row.find('div:nth-of-type(n+2) .collapsehide [color=\'#ff0000\'] b').first().text()),
languages: row.find('.tgxtablecell img[title]').first().attr('title'),
size: parseSize(row.find('.collapsehide span.badge-secondary').first().text()),
uploadDate: parseDate(row.find('div.collapsehide:nth-of-type(12)').first().text()),
imdbId: imdbIdMatch && imdbIdMatch[1],
});
try {
torrents.push({
name: row.find('.tgxtablecell div a[title]').first().text(),
infoHash: decode(magnetLink).infoHash,
magnetLink: magnetLink,
torrentLink: row.find('div:nth-of-type(n+2) .collapsehide > a:nth-of-type(1)').first().attr('href'),
torrentId: row.find('.tgxtablecell div a[title]').first().attr('href').match(/torrent\/(\d+)/)[1],
verified: !!row.find('i.fa-check').length,
category: row.find('div:nth-of-type(n+2) .shrink a').first().attr('href').match(/cat=(\d+)$/)[1],
seeders: parseInt(row.find('div:nth-of-type(n+2) .collapsehide [color=\'green\'] b').first().text()),
leechers: parseInt(row.find('div:nth-of-type(n+2) .collapsehide [color=\'#ff0000\'] b').first().text()),
languages: row.find('.tgxtablecell img[title]').first().attr('title'),
size: parseSize(row.find('.collapsehide span.badge-secondary').first().text()),
uploadDate: parseDate(row.find('div.collapsehide:nth-of-type(12)').first().text()),
imdbId: imdbIdMatch && imdbIdMatch[1],
});
} catch (e) {
console.error('Failed parsing TorrentGalaxy row: ', e);
}
});
resolve(torrents);

18
scraper/scrapers/torrentgalaxy/torrentgalaxy_scraper.js
View File

@@ -9,6 +9,7 @@ const { createTorrentEntry, checkAndUpdateTorrent } = require('../../lib/torrent
const NAME = 'TorrentGalaxy';
const TYPE_MAPPING = typeMapping();
const api_limiter = new Bottleneck({ maxConcurrent: 1, minTime: 5000 });
const limiter = new Bottleneck({ maxConcurrent: 10 });
const allowedCategories = [
torrentGalaxy.Categories.ANIME,
@@ -21,7 +22,6 @@ const allowedCategories = [
torrentGalaxy.Categories.TV_SD,
torrentGalaxy.Categories.TV_HD,
torrentGalaxy.Categories.TV_PACKS,
torrentGalaxy.Categories.TV_SPORT,
torrentGalaxy.Categories.DOCUMENTARIES,
];
const packCategories = [
@@ -57,7 +57,7 @@ async function scrapeLatestTorrents() {
async function scrapeLatestTorrentsForCategory(category, page = 1) {
console.log(`Scrapping ${NAME} ${category} category page ${page}`);
return torrentGalaxy.browse(({ category, page }))
return api_limiter.schedule(() => torrentGalaxy.browse({ category, page }))
.catch(error => {
console.warn(`Failed ${NAME} scrapping for [${page}] ${category} due: `, error);
return Promise.resolve([]);
@@ -73,10 +73,6 @@ async function processTorrentRecord(record) {
return Promise.resolve('Invalid torrent record');
}
if (await checkAndUpdateTorrent(record)) {
return record;
}
const torrent = {
provider: NAME,
infoHash: record.infoHash,
@@ -92,6 +88,15 @@ async function processTorrentRecord(record) {
languages: !(record.languages || '').includes('Other') ? record.languages : undefined
};
if (await checkAndUpdateTorrent(torrent)) {
return torrent;
}
const isOld = moment(torrent.uploadDate).isBefore(moment().subtract(18, 'month'));
if (torrent.seeders === 0 && isOld && !torrent.pack) {
console.log(`Skipping old unseeded torrent [${torrent.infoHash}] ${torrent.title}`)
return torrent;
}
return createTorrentEntry(torrent).then(() => torrent);
}
@@ -116,7 +121,6 @@ function getMaxPage(category) {
switch (category) {
case torrentGalaxy.Categories.TV_SD:
case torrentGalaxy.Categories.TV_HD:
return 10;
case torrentGalaxy.Categories.MOVIE_SD:
case torrentGalaxy.Categories.MOVIE_HD:
return 5;