tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] fixes update seeders method

Browse Source
This commit is contained in:
TheBeastLT
2020-03-30 14:42:00 +02:00
parent 04b3d8a774
commit 97cbe5d6c1
8 changed files with 53 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
scraper/scrapers/1337x/1337x_api.js
View File

@@ -130,7 +130,7 @@ function parseTorrentPage(body) {
category: details.find('strong:contains(\'Category\')').next().text(),
language: details.find('strong:contains(\'Language\')').next().text(),
size: parseSize(details.find('strong:contains(\'Total size\')').next().text()),
uploadDate: Sugar.Date.create(details.find('strong:contains(\'Date uploaded\')').next().text()),
uploadDate: parseDate(details.find('strong:contains(\'Date uploaded\')').next().text()),
imdbId: imdbIdMatch && imdbIdMatch[1],
files: details.find('div[id=\'files\']').first().find('li')
.map((i, elem) => $(elem).text())
@@ -145,6 +145,13 @@ function parseTorrentPage(body) {
});
}
function parseDate(dateString) {
if (/decade.*ago/i.test(dateString)) {
return Sugar.Date.create('10 years ago');
}
return Sugar.Date.create(dateString);
}
function parseSize(sizeText) {
if (!sizeText) {
return undefined;

9
scraper/scrapers/1337x/1337x_scraper.js
View File

@@ -80,6 +80,15 @@ async function processTorrentRecord(record) {
return createTorrentEntry(torrent);
}
async function updateSeeders() {
const startDate = moment().subtract(7, 'day').toDate();
const endDate = moment().subtract(1, 'day').toDate();
return repository.getTorrentsUpdatedBetween(NAME, startDate, endDate)
.then(torrents => Promise.all(torrents.map(torrent => limiter.schedule(() => leetx.torrent(torrent.torrentId)
.then(foundTorrent => updateTorrentSeeders(foundTorrent))
.catch(error => console.warn(error))))))
}
function typeMapping() {
const mapping = {};
mapping[leetx.Categories.MOVIE] = Type.MOVIE;

2
scraper/scrapers/horriblesubs/horriblesubs_scraper.js
View File

@@ -203,7 +203,7 @@ async function checkIfExists(torrent) {
} else if (existingTorrent.provider === NAME) {
return undefined; // torrent by this provider already exists
}
return { ...torrent, size: existingTorrent.size, seeders: existingTorrent.seeders };
return { ...torrent.get(), size: existingTorrent.size, seeders: existingTorrent.seeders };
}
module.exports = { scrape, NAME };

9
scraper/scrapers/thepiratebay/thepiratebay_api.js
View File

@@ -7,10 +7,13 @@ const Promises = require('../../lib/promises');
const defaultProxies = [
// 'https://thepiratebay.org',
// 'https://proxybay.pro',
'https://ukpiratebayproxy.com',
'https://thepiratebayproxy.info',
'https://mypiratebay.co',
// 'https://ukpiratebayproxy.com',
// 'https://thepiratebayproxy.info',
// 'https://mypiratebay.co',
'https://thepiratebay.asia',
'https://thepiratebay10.org',
'https://thepiratebay0.org',
'https://proxtpb.art',
];
const dumpUrl = '/static/dump/csv/';
const defaultTimeout = 10000;