tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

remove scraper module

Browse Source
This commit is contained in:
TheBeastLT
2022-06-16 17:44:38 +03:00
parent a253e8e0b0
commit 554c07d636
69 changed files with 0 additions and 61771 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
scraper/scrapers/nyaasi/nyaa_si_api.js
View File

@@ -1,66 +0,0 @@
const { si } = require('nyaapi')
const { parseSize } = require("../scraperHelper");
const Categories = {
ANIME: {
ALL: '1_0',
MUSIC_VIDEO: '1_1',
ENGLISH: '1_2',
NON_ENGLISH: '1_3',
RAW: '1_4'
},
LIVE_ACTION: {
ALL: '4_0',
ENGLISH: '4_1',
PROMOTIONAL_VIDEO: '4_2',
NON_ENGLISH: '4_3',
RAW: '4_4'
}
}
function torrent(torrentId) {
if (!torrentId) {
return Promise.reject(new Error(`Failed ${torrentId} search`));
}
return si.infoRequest(torrentId)
.then(result => parseTorrent(result))
.then(result => ({ ...result, torrentId }))
.catch(error => {
if (error.statusCode && error.statusCode === 404) {
return Promise.reject(new Error(`404: [${torrentId}] not found on NyaaSi`));
}
return Promise.reject(error);
});
}
function search(query) {
return si.search(query, null, { category: Categories.ANIME.ENGLISH})
.then(results => results.map(torrent => parseTorrent(torrent)));
}
function browse(config = {}) {
const page = config.page || 1;
const category = config.category || Categories.ANIME.ENGLISH;
const sort = config.sort || 'id'
return si.list(category, page, { sort })
.then(response => response.results || [])
.then(results => results.map(torrent => parseTorrent(torrent)));
}
function parseTorrent(torrent) {
return {
title: torrent.name.replace(/\t|\s+/g, ' ').trim(),
torrentId: torrent.id,
infoHash: torrent.hash.trim().toLowerCase(),
magnetLink: torrent.magnet,
torrentLink: torrent.torrent,
seeders: parseInt(torrent.seeders),
size: parseSize(torrent.filesize),
uploadDate: new Date(torrent.date),
category: torrent.sub_category,
}
}
module.exports = { torrent, search, browse, Categories };

87
scraper/scrapers/nyaasi/nyaa_si_scraper.js
View File

@@ -1,87 +0,0 @@
const moment = require('moment');
const Bottleneck = require('bottleneck');
const nyaasi = require('./nyaa_si_api');
const { Type } = require('../../lib/types');
const Promises = require('../../lib/promises');
const repository = require('../../lib/repository');
const { createTorrentEntry, checkAndUpdateTorrent } = require('../../lib/torrentEntries');
const NAME = 'NyaaSi';
const UNTIL_PAGE = 10
const limiter = new Bottleneck({ maxConcurrent: 10 });
async function scrape() {
const scrapeStart = moment();
const lastScrape = await repository.getProvider({ name: NAME });
console.log(`[${scrapeStart}] starting ${NAME} scrape...`);
// const ids = ['1292786'];
// return Promise.all(ids.map(id => limiter.schedule(() => nyaasi.torrent(id)
// .then(torrent => processTorrentRecord(torrent)))))
// .then(() => console.log(`[${moment()}] finished ${NAME} scrape`));
// const queries = ['Sagrada Reset', 'Sakurada Reset'];
// return Promise.all(queries.map(query => limiter.schedule(() => nyaasi.search(query)
// .then(torrents => Promise.all(torrents.map(torrent => limiter.schedule(() => processTorrentRecord(torrent))))))))
// .then(() => console.log(`[${moment()}] finished ${NAME} scrape`));
return scrapeLatestTorrents()
.then(() => {
lastScrape.lastScraped = scrapeStart;
return lastScrape.save();
})
.then(() => console.log(`[${moment()}] finished ${NAME} scrape`));
}
async function updateSeeders(torrent) {
// return limiter.schedule(() => nyaasi.torrent(torrent.torrentId))
// .then(foundTorrent => Number.isInteger(foundTorrent.seeders) ? [foundTorrent] : []);
return Promise.resolve([]);
}
async function scrapeLatestTorrents() {
const allowedCategories = [
nyaasi.Categories.ANIME.ENGLISH
];
return Promises.sequence(allowedCategories.map(category => () => scrapeLatestTorrentsForCategory(category)))
.then(entries => entries.reduce((a, b) => a.concat(b), []));
}
async function scrapeLatestTorrentsForCategory(category, page = 1) {
console.log(`Scrapping ${NAME} ${category} category page ${page}`);
return nyaasi.browse({ page, sort: 'id' })
.catch(error => {
console.warn(`Failed ${NAME} scrapping for [${page}] ${category} due: `, error);
return Promise.resolve([]);
})
.then(torrents => Promise.all(torrents.map(torrent => limiter.schedule(() => processTorrentRecord(torrent)
.catch(error => {
console.warn(`Failed processing [${torrent.infoHash}] ${torrent.title} due: `, error);
return Promise.resolve();
})))))
.then(resolved => resolved.length > 0 && page < UNTIL_PAGE
? scrapeLatestTorrentsForCategory(category, page + 1)
: Promise.resolve());
}
async function processTorrentRecord(record) {
if (!record || await checkAndUpdateTorrent(record)) {
return record;
}
const torrent = {
infoHash: record.infoHash,
torrentLink: record.torrentLink,
provider: NAME,
torrentId: record.torrentId,
title: record.title,
type: Type.ANIME,
size: record.size,
seeders: record.seeders,
uploadDate: record.uploadDate,
};
return createTorrentEntry(torrent).then(() => torrent);
}
module.exports = { scrape, updateSeeders, NAME };