tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] get torrent size for horriblesubs

Browse Source
This commit is contained in:
TheBeastLT
2020-04-02 20:58:08 +02:00
parent 6b36667b7f
commit db78f5873b
5 changed files with 71 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
scraper/scrapers/horriblesubs/horriblesubs_scraper.js
View File

@@ -5,7 +5,7 @@ const decode = require('magnet-uri');
const horriblesubs = require('./horriblesubs_api.js');
const repository = require('../../lib/repository');
const { Type } = require('../../lib/types');
const { updateCurrentSeeders } = require('../../lib/torrent');
const { updateCurrentSeeders, updateTorrentSize } = require('../../lib/torrent');
const { parseTorrentFiles } = require('../../lib/torrentFiles');
const { getMetadata, getKitsuId } = require('../../lib/metadata');
const showMappings = require('./horriblesubs_mapping.json');
@@ -153,7 +153,6 @@ async function _parseShowData(showData) {
infoHash: decode(mirror.magnetLink).infoHash,
trackers: decode(mirror.magnetLink).tr.join(','),
title: formatTitle(episodeInfo, mirror),
size: 300000000,
type: Type.ANIME,
kitsuId: getKitsuId(episodeInfo.episode),
uploadDate: episodeInfo.uploadDate,
@@ -161,12 +160,14 @@ async function _parseShowData(showData) {
.reduce((a, b) => a.concat(b), [])
.filter((incompleteTorrent) => incompleteTorrent.kitsuId)
.map((incompleteTorrent) => entryLimiter.schedule(() => checkIfExists(incompleteTorrent)
.then((torrent) => torrent && updateTorrentSize(torrent))
.then((torrent) => torrent && updateCurrentSeeders(torrent))
.then((torrent) => torrent && parseTorrentFiles(torrent)
.then((files) => verifyFiles(torrent, files))
.then((files) => repository.createTorrent(torrent)
.then(() => files.forEach(file => repository.createFile(file)))
.then(() => console.log(`Created entry for ${torrent.title}`)))))))
.then(() => console.log(`Created entry for ${torrent.title}`))))
.catch(error => console.warn(`Failed creating entry for ${incompleteTorrent.title}:`, error)))))
.then(() => console.log(`${NAME}: finished scrapping ${showData.title} data`));
}

14
scraper/scrapers/thepiratebay/thepiratebay_api.js
View File

@@ -6,14 +6,12 @@ const Promises = require('../../lib/promises');
const defaultProxies = [
// 'https://thepiratebay.org',
// 'https://proxybay.pro',
// 'https://ukpiratebayproxy.com',
// 'https://thepiratebayproxy.info',
// 'https://mypiratebay.co',
'https://thepiratebay.asia',
'https://thepiratebay10.org',
'https://thepiratebay0.org',
'https://proxtpb.art',
'https://proxybay.pro',
'https://ukpiratebayproxy.com',
'https://thepiratebayproxy.info',
'https://mypiratebay.co',
// 'https://thepiratebay10.org',
// 'https://thepiratebay0.org',
];
const dumpUrl = '/static/dump/csv/';
const defaultTimeout = 10000;