tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] updates hs scraper to update seeders for existing latest entries

Browse Source
This commit is contained in:
TheBeastLT
2020-04-11 17:57:30 +02:00
parent f3952dcc3e
commit 31e3970272
2 changed files with 46 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
scraper/lib/torrentEntries.js
View File

@@ -4,7 +4,7 @@ const repository = require('./repository');
const { getImdbId, getKitsuId } = require('./metadata');
const { parseTorrentFiles } = require('./torrentFiles');
async function createTorrentEntry(torrent) {
async function createTorrentEntry(torrent, overwrite = false) {
const titleInfo = parse(torrent.title);
if (titleInfo.seasons && torrent.type === Type.MOVIE) {
@@ -33,7 +33,8 @@ async function createTorrentEntry(torrent) {
return;
}
const files = await parseTorrentFiles(torrent);
const files = await parseTorrentFiles(torrent)
.then(files => overwrite ? overwriteExistingFiles(torrent, files) : files);
if (!files || !files.length) {
console.log(`no video files found for [${torrent.infoHash}] ${torrent.title}`);
return;
@@ -44,6 +45,34 @@ async function createTorrentEntry(torrent) {
.then(() => console.log(`Created entry for [${torrent.infoHash}] ${torrent.title}`));
}
async function overwriteExistingFiles(torrent, files) {
if (files && files.length) {
const existingFiles = await repository.getFiles({ infoHash: files[0].infoHash })
.then((existing) => existing
.reduce((map, next) => {
const fileIndex = next.fileIndex !== undefined ? next.fileIndex : null;
map[fileIndex] = (map[fileIndex] || []).concat(next);
return map;
}, {}))
.catch(() => undefined);
if (existingFiles && Object.keys(existingFiles).length) {
return files
.map(file => {
const mapping = files.length === 1 && Object.keys(existingFiles).length === 1
? Object.values(existingFiles)[0]
: existingFiles[file.fileIndex !== undefined ? file.fileIndex : null];
if (mapping) {
const originalFile = mapping.shift();
return { ...file, id: originalFile.id, size: originalFile.size || file.size };
}
return file;
})
}
return files;
}
return Promise.reject(`No video files found for: ${torrent.title}`);
}
async function createSkipTorrentEntry(torrent) {
return repository.createSkipTorrent(torrent);
}

64
scraper/scrapers/horriblesubs/horriblesubs_scraper.js
View File

@@ -6,8 +6,7 @@ const horriblesubs = require('./horriblesubs_api.js');
const repository = require('../../lib/repository');
const { Type } = require('../../lib/types');
const { updateCurrentSeeders, updateTorrentSize } = require('../../lib/torrent');
const { parseTorrentFiles } = require('../../lib/torrentFiles');
const { updateTorrentSeeders } = require('../../lib/torrentEntries');
const { createTorrentEntry, updateTorrentSeeders } = require('../../lib/torrentEntries');
const { getMetadata, getKitsuId } = require('../../lib/metadata');
const showMappings = require('./horriblesubs_mapping.json');
@@ -151,9 +150,9 @@ async function _parseShowData(showData) {
};
return Promise.all([].concat(showData.singleEpisodes || []).concat(showData.packEpisodes || [])
.map((episodeInfo) => episodeInfo.mirrors
.filter((mirror) => mirror.magnetLink && mirror.magnetLink.length)
.map((mirror) => ({
.map(episodeInfo => episodeInfo.mirrors
.filter(mirror => mirror.magnetLink && mirror.magnetLink.length)
.map(mirror => ({
provider: NAME,
...mirror,
infoHash: decode(mirror.magnetLink).infoHash,
@@ -164,55 +163,22 @@ async function _parseShowData(showData) {
uploadDate: episodeInfo.uploadDate,
})))
.reduce((a, b) => a.concat(b), [])
.filter((incompleteTorrent) => incompleteTorrent.kitsuId)
.map((incompleteTorrent) => entryLimiter.schedule(() => checkIfExists(incompleteTorrent)
.then((torrent) => torrent && updateTorrentSize(torrent))
.then((torrent) => torrent && updateCurrentSeeders(torrent))
.then((torrent) => torrent && parseTorrentFiles(torrent)
.then((files) => verifyFiles(torrent, files))
.then((files) => repository.createTorrent(torrent)
.then(() => files.forEach(file => repository.createFile(file)))
.then(() => console.log(`Created entry for [${torrent.infoHash}] ${torrent.title}`))))
.catch(error => console.warn(`Failed creating entry for ${incompleteTorrent.title}:`, error)))))
.filter(torrent => torrent.kitsuId)
.map(torrent => entryLimiter.schedule(() => processTorrentRecord(torrent))))
.then(() => console.log(`${NAME}: finished scrapping ${showData.title} data`));
}
async function verifyFiles(torrent, files) {
if (files && files.length) {
const existingFiles = await repository.getFiles({ infoHash: files[0].infoHash })
.then((existing) => existing
.reduce((map, next) => {
const fileIndex = next.fileIndex !== undefined ? next.fileIndex : null;
map[fileIndex] = (map[fileIndex] || []).concat(next);
return map;
}, {}))
.catch(() => undefined);
if (existingFiles && Object.keys(existingFiles).length) {
return files
.map(file => {
const mapping = files.length === 1 && Object.keys(existingFiles).length === 1
? Object.values(existingFiles)[0]
: existingFiles[file.fileIndex !== undefined ? file.fileIndex : null];
if (mapping) {
const originalFile = mapping.shift();
return { ...file, id: originalFile.id, size: originalFile.size || file.size };
}
return file;
})
}
return files;
}
return Promise.reject(`No video files found for: ${torrent.title}`);
}
async function checkIfExists(torrent) {
async function processTorrentRecord(torrent) {
const existingTorrent = await repository.getTorrent(torrent).catch(() => undefined);
if (!existingTorrent) {
return torrent; // no torrent exists yet
} else if (existingTorrent.provider === NAME) {
return undefined; // torrent by this provider already exists
if (existingTorrent && existingTorrent.provider === NAME) {
return updateCurrentSeeders(torrent).then(updatedSeeders => updateTorrentSeeders(updatedSeeders))
}
return { ...torrent, size: existingTorrent.size, seeders: existingTorrent.seeders };
return updateTorrentSize(torrent)
.then(updated => updateCurrentSeeders(updated))
.then(updated => createTorrentEntry(updated, true))
.catch(error => console.warn(`Failed creating entry for ${torrent.title}:`, error));
}
module.exports = { scrape, updateSeeders, NAME };