tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] fixes torrent size update method for horriblesubs

Browse Source
This commit is contained in:
TheBeastLT
2020-04-03 14:39:23 +02:00
parent 201ec02260
commit 7dac31ae61
2 changed files with 17 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
scraper/lib/torrent.js
View File

@@ -31,56 +31,15 @@ module.exports.updateCurrentSeeders = function (torrent) {
ready(); ready();
}); });
}, callback); }, callback);
}).then((seeders) => ({ ...torrent, seeders: torrent.seeders || seeders })); }).then(seeders => ({ ...torrent, seeders: torrent.seeders || seeders }));
}; };
module.exports.updateTorrentSize = function (torrent) { module.exports.updateTorrentSize = function (torrent) {
if (!torrent.infoHash && !torrent.magnetLink) { return filesAndSizeFromTorrentStream(torrent, SEEDS_CHECK_TIMEOUT)
return Promise.reject(new Error("no infoHash or magnetLink")); .then(result => ({ ...torrent, size: result.size, files: result.files }));
}
const magnetLink = torrent.magnetLink || decode.encode({ infoHash: torrent.infoHash });
return new Promise((resolve, rejected) => {
const engine = new torrentStream(magnetLink, { connections: MAX_PEER_CONNECTIONS });
engine.ready(() => {
const size = engine.torrent.length;
engine.destroy();
resolve({ size });
});
setTimeout(() => {
engine.destroy();
rejected(new Error('No available connections for torrent!'));
}, SEEDS_CHECK_TIMEOUT);
}).then((size) => ({ ...torrent, size }));
}; };
module.exports.sizeAndFiles = function (torrent) { module.exports.sizeAndFiles = torrent => filesAndSizeFromTorrentStream(torrent, 20000);
if (!torrent.infoHash && !torrent.magnetLink) {
return Promise.reject(new Error("no infoHash or magnetLink"));
}
// const magnet = decode.encode({ infoHash: torrent.infoHash, announce: torrent.trackers });
return new Promise((resolve, rejected) => {
const engine = new torrentStream(torrent.infoHash, { connections: MAX_PEER_CONNECTIONS });
engine.ready(() => {
const files = engine.files
.map((file, fileId) => ({
fileIndex: fileId,
name: file.name,
path: file.path.replace(/^[^\/]+\//, ''),
size: file.length
}));
const size = engine.torrent.length;
engine.destroy();
resolve({ files, size });
});
setTimeout(() => {
engine.destroy();
rejected(new Error('No available connections for torrent!'));
}, 20000);
});
};
module.exports.torrentFiles = function (torrent) { module.exports.torrentFiles = function (torrent) {
return getFilesFromObject(torrent) return getFilesFromObject(torrent)
@@ -131,11 +90,16 @@ async function filesFromTorrentFile(torrent) {
} }
async function filesFromTorrentStream(torrent) { async function filesFromTorrentStream(torrent) {
return filesAndSizeFromTorrentStream(torrent, 60000).then(result => result.files);
}
function filesAndSizeFromTorrentStream(torrent, timeout = 60000) {
if (!torrent.infoHash && !torrent.magnetLink) { if (!torrent.infoHash && !torrent.magnetLink) {
return Promise.reject(new Error("no infoHash or magnetLink")); return Promise.reject(new Error("no infoHash or magnetLink"));
} }
// const magnet = decode.encode({ infoHash: torrent.infoHash, announce: torrent.trackers });
return new Promise((resolve, rejected) => { return new Promise((resolve, rejected) => {
const engine = new torrentStream(torrent.magnetLink || torrent.infoHash, { connections: MAX_PEER_CONNECTIONS }); const engine = new torrentStream(torrent.infoHash, { connections: MAX_PEER_CONNECTIONS });
engine.ready(() => { engine.ready(() => {
const files = engine.files const files = engine.files
@@ -145,14 +109,15 @@ async function filesFromTorrentStream(torrent) {
path: file.path.replace(/^[^\/]+\//, ''), path: file.path.replace(/^[^\/]+\//, ''),
size: file.length size: file.length
})); }));
const size = engine.torrent.length;
engine.destroy(); engine.destroy();
resolve(files); resolve({ files, size });
}); });
setTimeout(() => { setTimeout(() => {
engine.destroy(); engine.destroy();
rejected(new Error('No available connections for torrent!')); rejected(new Error('No available connections for torrent!'));
}, 60000); }, timeout);
}); });
} }

6
scraper/scrapers/horriblesubs/horriblesubs_scraper.js
View File

@@ -166,7 +166,7 @@ async function _parseShowData(showData) {
.then((files) => verifyFiles(torrent, files)) .then((files) => verifyFiles(torrent, files))
.then((files) => repository.createTorrent(torrent) .then((files) => repository.createTorrent(torrent)
.then(() => files.forEach(file => repository.createFile(file))) .then(() => files.forEach(file => repository.createFile(file)))
.then(() => console.log(`Created entry for ${torrent.title}`)))) .then(() => console.log(`Created entry for [${torrent.infoHash}] ${torrent.title}`))))
.catch(error => console.warn(`Failed creating entry for ${incompleteTorrent.title}:`, error))))) .catch(error => console.warn(`Failed creating entry for ${incompleteTorrent.title}:`, error)))))
.then(() => console.log(`${NAME}: finished scrapping ${showData.title} data`)); .then(() => console.log(`${NAME}: finished scrapping ${showData.title} data`));
} }
@@ -184,7 +184,9 @@ async function verifyFiles(torrent, files) {
if (existingFiles && Object.keys(existingFiles).length) { if (existingFiles && Object.keys(existingFiles).length) {
return files return files
.map(file => { .map(file => {
const mapping = existingFiles[file.fileIndex !== undefined ? file.fileIndex : null]; const mapping = files.length === 1 && Object.keys(existingFiles).length === 1
? Object.values(existingFiles)[0]
: existingFiles[file.fileIndex !== undefined ? file.fileIndex : null];
if (mapping) { if (mapping) {
const originalFile = mapping.shift(); const originalFile = mapping.shift();
return { ...file, id: originalFile.id, size: originalFile.size || file.size }; return { ...file, id: originalFile.id, size: originalFile.size || file.size };