tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

updates id search input arguments

Browse Source
This commit is contained in:
TheBeastLT
2020-03-03 20:36:10 +01:00
parent 30d1a60d00
commit cbfcea65f8
8 changed files with 30 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
scrapers/horriblesubs/horriblesubs_scraper.js
View File

@@ -1,6 +1,5 @@
const fs = require('fs');
const Bottleneck = require('bottleneck');
const { parse } = require('parse-torrent-title');
const decode = require('magnet-uri');
const horriblesubs = require('./horriblesubs_api.js');
const repository = require('../../lib/repository');
@@ -54,8 +53,8 @@ async function initMapping() {
async function enrichShow(show) {
console.log(`${NAME}: getting show info for ${show.title}...`);
const showId = await horriblesubs._getShowId(show.url)
.catch((error) => show.title);
const metadata = await getKitsuId({ name: show.title })
.catch(() => show.title);
const metadata = await getKitsuId({ title: show.title })
.then((kitsuId) => getMetadata(kitsuId))
.catch((error) => {
console.log(`Failed getting kitsu meta: ${error.message}`);

20
scrapers/thepiratebay/thepiratebay_dump_scraper.js
View File

@@ -18,12 +18,12 @@ const limiter = new Bottleneck({ maxConcurrent: 40 });
async function scrape() {
const lastScraped = await repository.getProvider({ name: NAME });
const lastDump = { updatedAt: 2147000000 };
const checkPoint = moment('2016-06-17 00:00:00', 'YYYY-MMM-DD HH:mm:ss').toDate();
//const checkPoint = moment('2016-06-17 00:00:00', 'YYYY-MMM-DD HH:mm:ss').toDate();
//const lastDump = await thepiratebay.dumps().then((dumps) => dumps.sort((a, b) => b.updatedAt - a.updatedAt)[0]);
if (!lastScraped.lastScraped || lastScraped.lastScraped < lastDump.updatedAt) {
console.log(`starting to scrape tpb dump: ${JSON.stringify(lastDump)}`);
//await downloadDump(lastDump);
await downloadDump(lastDump);
let entriesProcessed = 0;
const lr = new LineByLineReader(CSV_FILE_PATH);
@@ -51,10 +51,10 @@ async function scrape() {
size: parseInt(row[3], 10)
};
if (torrent.uploadDate > checkPoint) {
entriesProcessed++;
return;
}
// if (torrent.uploadDate > checkPoint) {
// entriesProcessed++;
// return;
// }
if (lastScraped.lastScraped && lastScraped.lastScraped > torrent.uploadDate) {
// torrent was already scraped previously, skipping
@@ -75,8 +75,8 @@ async function scrape() {
console.log(err);
});
lr.on('end', () => {
fs.unlink(CSV_FILE_PATH);
repository.updateProvider({ name: NAME, lastScraped: lastDump.updatedAt });
fs.unlink(CSV_FILE_PATH, (error) => console.warn(error));
//repository.updateProvider({ name: NAME, lastScraped: lastDump.updatedAt });
console.log(`finished to scrape tpb dump: ${JSON.stringify(lastDump)}!`);
});
}
@@ -144,8 +144,8 @@ async function findTorrentInSource(record) {
async function findTorrentViaBing(record) {
return bing.web(`${record.infoHash}`)
.then((results) => results
.find(result => result.description.includes('Direct download via magnet link') || result.description.includes(
'Get this torrent')))
.find(result => result.description.includes('Direct download via magnet link') ||
result.description.includes('Get this torrent')))
.then((result) => {
if (!result) {
throw new Error(`Failed to find torrent ${record.title}`);

2
scrapers/thepiratebay/thepiratebay_unofficial_dump_scraper.js
View File

@@ -15,7 +15,7 @@ const limiter = new Bottleneck({ maxConcurrent: 40 });
async function scrape() {
console.log(`starting to scrape tpb dump...`);
//const checkPoint = moment('2013-06-16 00:00:00', 'YYYY-MMM-DD HH:mm:ss').toDate();
const checkPoint = 951000;
const checkPoint = 4115000;
let entriesProcessed = 0;
const lr = new LineByLineReader(CSV_FILE_PATH);