mirror of
https://github.com/knightcrawler-stremio/knightcrawler.git
synced 2024-12-20 03:29:51 +00:00
Initial commit
This commit is contained in:
TheBeastLT
2
.gitignore
vendored
Normal file
2
.gitignore
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
/.idea
|
||||
/node_modules
|
||||
16
docker-compose.yml
Normal file
16
docker-compose.yml
Normal file
@@ -0,0 +1,16 @@
|
||||
version: '3'
|
||||
services:
|
||||
database:
|
||||
image: postgres
|
||||
volumes:
|
||||
- db-data:/var/lib/postgresql/data
|
||||
ports:
|
||||
- 5432:5432
|
||||
environment:
|
||||
POSTGRES_USER: torrentio
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: torrentio
|
||||
|
||||
volumes:
|
||||
db-data:
|
||||
driver: local
|
||||
24
index.js
Normal file
24
index.js
Normal file
@@ -0,0 +1,24 @@
|
||||
const express = require("express");
|
||||
const server = express();
|
||||
const { connect } = require('./lib/repository');
|
||||
const tpbDump = require('./scrapers/piratebay_dump');
|
||||
const horribleSubs = require('./scrapers/api/horriblesubs');
|
||||
|
||||
const providers = [tpbDump];
|
||||
|
||||
async function scrape() {
|
||||
providers.forEach((provider) => provider.scrape());
|
||||
}
|
||||
|
||||
server.post('/scrape', function(req, res) {
|
||||
scrape();
|
||||
res.send(200);
|
||||
});
|
||||
|
||||
server.listen(7000, async function () {
|
||||
await connect();
|
||||
console.log('Scraper started');
|
||||
const shows = await horribleSubs.allShows();
|
||||
console.log(shows)
|
||||
//scrape();
|
||||
});
|
||||
94
lib/metadata.js
Normal file
94
lib/metadata.js
Normal file
@@ -0,0 +1,94 @@
|
||||
const _ = require('lodash');
|
||||
const needle = require('needle');
|
||||
const nameToImdb = require('name-to-imdb');
|
||||
|
||||
const CINEMETA_URL = 'https://v3-cinemeta.strem.io';
|
||||
|
||||
function getMetadata(imdbId, type) {
|
||||
return needle('get', `${CINEMETA_URL}/meta/${type}/${imdbId}.json`, { open_timeout: 1000 })
|
||||
.then((response) => response.body)
|
||||
.then((body) => {
|
||||
if (body && body.meta && body.meta.name) {
|
||||
return {
|
||||
title: body.meta.name,
|
||||
year: body.meta.year,
|
||||
genres: body.meta.genres,
|
||||
episodeCount: body.meta.videos && _.chain(body.meta.videos)
|
||||
.countBy('season')
|
||||
.toPairs()
|
||||
.filter((pair) => pair[0] !== '0')
|
||||
.sortBy((pair) => parseInt(pair[0], 10))
|
||||
.map((pair) => pair[1])
|
||||
.value()
|
||||
};
|
||||
} else {
|
||||
console.log(`failed cinemeta query: Empty Body`);
|
||||
throw new Error('failed cinemeta query');
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function escapeTitle(title, hyphenEscape = true) {
|
||||
return title.toLowerCase()
|
||||
.normalize('NFKD') // normalize non-ASCII characters
|
||||
.replace(/[\u0300-\u036F]/g, '')
|
||||
.replace(/&/g, 'and')
|
||||
.replace(hyphenEscape ? /[.,_+ -]+/g : /[.,_+ ]+/g, ' ') // replace dots, commas or underscores with spaces
|
||||
.replace(/[^\w- ()]/gi, '') // remove all non-alphanumeric chars
|
||||
.trim();
|
||||
}
|
||||
|
||||
const hardcodedTitles = {
|
||||
'tt0388629': 'one piece',
|
||||
'tt0182629': 'rurouni kenshin',
|
||||
'tt2098220': 'hunter x hunter 2011',
|
||||
'tt1409055': 'dragon ball kai',
|
||||
'tt7441658': 'black clover tv'
|
||||
};
|
||||
|
||||
async function seriesMetadata(id) {
|
||||
const idInfo = id.split(':');
|
||||
const imdbId = idInfo[0];
|
||||
const season = parseInt(idInfo[1], 10);
|
||||
const episode = parseInt(idInfo[2], 10);
|
||||
|
||||
const metadata = await getMetadata(imdbId, 'series');
|
||||
const title = escapeTitle(metadata.title);
|
||||
const hasEpisodeCount = metadata.episodeCount && metadata.episodeCount.length >= season;
|
||||
|
||||
return {
|
||||
imdb: imdbId,
|
||||
title: hardcodedTitles[imdbId] || title,
|
||||
season: season,
|
||||
episode: episode,
|
||||
absoluteEpisode: hasEpisodeCount && metadata.episodeCount.slice(0, season - 1).reduce((a, b) => a + b, episode),
|
||||
genres: metadata.genres,
|
||||
isAnime: !metadata.genres.length || metadata.genres.includes('Animation')
|
||||
};
|
||||
}
|
||||
|
||||
async function movieMetadata(id) {
|
||||
const metadata = await getMetadata(id, 'movie');
|
||||
|
||||
return {
|
||||
imdb: id,
|
||||
title: escapeTitle(metadata.title),
|
||||
year: metadata.year,
|
||||
genres: metadata.genres,
|
||||
isAnime: !metadata.genres.length || metadata.genres.includes('Animation')
|
||||
};
|
||||
}
|
||||
|
||||
async function getImdbId(info) {
|
||||
return new Promise((resolve, reject) => {
|
||||
nameToImdb(info, function(err, res) {
|
||||
if (res) {
|
||||
resolve(res);
|
||||
} else {
|
||||
reject(err || new Error('failed imdbId search'));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = { movieMetadata, seriesMetadata, getImdbId };
|
||||
84
lib/repository.js
Normal file
84
lib/repository.js
Normal file
@@ -0,0 +1,84 @@
|
||||
const { Sequelize }= require('sequelize');
|
||||
|
||||
const POSTGRES_URI = process.env.POSTGRES_URI || 'postgres://torrentio:postgres@localhost:5432/torrentio';
|
||||
|
||||
const database = new Sequelize(POSTGRES_URI, { logging: false });
|
||||
|
||||
const Provider = database.define('provider', {
|
||||
name: { type: Sequelize.STRING(16), primaryKey: true},
|
||||
lastScraped: { type: Sequelize.DATE }
|
||||
});
|
||||
|
||||
const Torrent = database.define('torrent', {
|
||||
infoHash: { type: Sequelize.STRING(64), primaryKey: true },
|
||||
provider: { type: Sequelize.STRING(16), allowNull: false },
|
||||
title: { type: Sequelize.STRING(128), allowNull: false },
|
||||
imdbId: { type: Sequelize.STRING(12) },
|
||||
uploadDate: { type: Sequelize.DATE, allowNull: false },
|
||||
seeders: { type: Sequelize.SMALLINT },
|
||||
files: { type: Sequelize.ARRAY(Sequelize.TEXT) },
|
||||
});
|
||||
|
||||
const SkipTorrent = database.define('skip_torrent', {
|
||||
infoHash: {type: Sequelize.STRING(64), primaryKey: true},
|
||||
});
|
||||
|
||||
const FailedImdbTorrent = database.define('failed_imdb_torrent', {
|
||||
infoHash: {type: Sequelize.STRING(64), primaryKey: true},
|
||||
});
|
||||
|
||||
function connect() {
|
||||
return database.sync({ alter: true });
|
||||
}
|
||||
|
||||
function getProvider(provider) {
|
||||
return Provider.findOrCreate({ where: { name: provider.name }, defaults: provider });
|
||||
}
|
||||
|
||||
function updateProvider(provider) {
|
||||
return Provider.update(provider);
|
||||
}
|
||||
|
||||
function getTorrent(torrent) {
|
||||
return Torrent.findByPk(torrent.infoHash)
|
||||
.then((result) =>{
|
||||
if (!result) {
|
||||
throw new Error(`torrent not found: ${torrent.infoHash}`);
|
||||
}
|
||||
return result.dataValues;
|
||||
})
|
||||
}
|
||||
|
||||
function updateTorrent(torrent) {
|
||||
return Torrent.upsert(torrent);
|
||||
}
|
||||
|
||||
function getSkipTorrent(torrent) {
|
||||
return SkipTorrent.findByPk(torrent.infoHash)
|
||||
.then((result) =>{
|
||||
if (!result) {
|
||||
return getFailedImdbTorrent(torrent);
|
||||
}
|
||||
return result.dataValues;
|
||||
})
|
||||
}
|
||||
|
||||
function createSkipTorrent(torrent) {
|
||||
return SkipTorrent.upsert({ infoHash: torrent.infoHash });
|
||||
}
|
||||
|
||||
function getFailedImdbTorrent(torrent) {
|
||||
return FailedImdbTorrent.findByPk(torrent.infoHash)
|
||||
.then((result) =>{
|
||||
if (!result) {
|
||||
throw new Error(`torrent not found: ${torrent.infoHash}`);
|
||||
}
|
||||
return result.dataValues;
|
||||
})
|
||||
}
|
||||
|
||||
function createFailedImdbTorrent(torrent) {
|
||||
return FailedImdbTorrent.upsert({ infoHash: torrent.infoHash });
|
||||
}
|
||||
|
||||
module.exports = { connect, getProvider, updateProvider, getTorrent, updateTorrent, getSkipTorrent, createSkipTorrent, createFailedImdbTorrent };
|
||||
109
lib/torrent.js
Normal file
109
lib/torrent.js
Normal file
@@ -0,0 +1,109 @@
|
||||
const torrentStream = require('torrent-stream');
|
||||
const cheerio = require('cheerio');
|
||||
const needle = require('needle');
|
||||
const parseTorrent = require('parse-torrent');
|
||||
const cloudscraper = require('cloudscraper');
|
||||
|
||||
const MAX_PEER_CONNECTIONS = process.env.MAX_PEER_CONNECTIONS || 20;
|
||||
const EXTENSIONS = ["3g2", "3gp", "avi", "flv", "mkv", "mov", "mp2", "mp4", "mpe", "mpeg", "mpg", "mpv", "webm", "wmv"];
|
||||
|
||||
module.exports.torrentFiles = function(torrent) {
|
||||
return filesFromKat(torrent.infoHash)
|
||||
.catch(() => filesFromTorrentStream(torrent))
|
||||
.then((files) => files
|
||||
.filter((file) => isVideo(file))
|
||||
.map((file) => `${file.fileIndex}@@${file.path}`));
|
||||
};
|
||||
|
||||
// async function filesFromBtSeeds(infoHash) {
|
||||
// const url = `https://www.btseed.net/show/${infoHash}`;
|
||||
// return needle('get', url, { open_timeout: 2000 })
|
||||
// .then((response) => response.body)
|
||||
// .then((body) => body.match(/<script id="__NEXT_DATA__"[^>]+>(.*?)<\/script>/)[1])
|
||||
// .then((match) => JSON.parse(match).props.pageProps.result.torrent.files)
|
||||
// }
|
||||
|
||||
function filesFromKat(infoHash) {
|
||||
const url = `http://kat.rip/torrent/${infoHash}.html`;
|
||||
return needle('get', url, { open_timeout: 2000 })
|
||||
.then((response) => {
|
||||
if (!response.body) {
|
||||
throw new Error('torrent not found in kat')
|
||||
}
|
||||
return response.body
|
||||
})
|
||||
.then((body) => {
|
||||
const $ = cheerio.load(body);
|
||||
const files = [];
|
||||
|
||||
$('table[id=\'ul_top\'] tr').each((index, row) => {
|
||||
files.push({
|
||||
fileIndex: index,
|
||||
path: $(row).find('td[class=\'torFileName\']').text(),
|
||||
size: convertToBytes($(row).find('td[class=\'torFileSize\']').text())
|
||||
});
|
||||
});
|
||||
return files;
|
||||
})
|
||||
}
|
||||
|
||||
async function filesFromTorrentStream(torrent) {
|
||||
return new Promise((resolve, rejected) => {
|
||||
const engine = new torrentStream(torrent.infoHash, { connections: MAX_PEER_CONNECTIONS });
|
||||
|
||||
engine.ready(() => {
|
||||
const files = engine.files
|
||||
.map((file, fileId) => ({
|
||||
fileIndex: fileId,
|
||||
name: file.name,
|
||||
path: file.path.replace(/^[^\/]+\//, ''),
|
||||
size: file.length
|
||||
}));
|
||||
|
||||
engine.destroy();
|
||||
resolve(files);
|
||||
});
|
||||
setTimeout(() => {
|
||||
engine.destroy();
|
||||
rejected(new Error('No available connections for torrent!'));
|
||||
}, dynamicTimeout(torrent));
|
||||
});
|
||||
}
|
||||
|
||||
function isVideo(title) {
|
||||
return EXTENSIONS.includes(title.path.match(/\.(\w{2,4})$/)[1]);
|
||||
}
|
||||
|
||||
|
||||
function convertToBytes(sizeString) {
|
||||
if (!sizeString) {
|
||||
return;
|
||||
}
|
||||
const prefix = sizeString.match(/\w+$/)[0].toLowerCase();
|
||||
let multiplier = 1;
|
||||
if (prefix === 'gb') multiplier = 1024 * 1024 * 1024;
|
||||
else if (prefix === 'mb') multiplier = 1024 * 1024;
|
||||
else if (prefix === 'kb') multiplier = 1024;
|
||||
|
||||
return Math.floor(parseFloat(sizeString) * multiplier);
|
||||
}
|
||||
|
||||
|
||||
function dynamicTimeout(torrent) {
|
||||
if (torrent.seeders < 5) {
|
||||
return 2000;
|
||||
} else if (torrent.seeders < 10) {
|
||||
return 3000;
|
||||
} else if (torrent.seeders < 20) {
|
||||
return 4000;
|
||||
} else if (torrent.seeders < 30) {
|
||||
return 5000;
|
||||
} else if (torrent.seeders < 50) {
|
||||
return 7000;
|
||||
} else if (torrent.seeders < 100) {
|
||||
return 10000;
|
||||
} else {
|
||||
return 15000;
|
||||
}
|
||||
}
|
||||
|
||||
2317
package-lock.json
generated
Normal file
2317
package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
35
package.json
Normal file
35
package.json
Normal file
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"name": "stremio-torrention",
|
||||
"version": "1.0.0",
|
||||
"main": "index.js",
|
||||
"engines": {
|
||||
"npm": "6.x",
|
||||
"node": "10.x"
|
||||
},
|
||||
"scripts": {
|
||||
"start": "node index.js"
|
||||
},
|
||||
"author": "TheBeastLT <pauliox@beyond.lt>",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"bottleneck": "^2.16.2",
|
||||
"cheerio": "^0.22.0",
|
||||
"cloudscraper": "^3.0.0",
|
||||
"express": "^4.16.4",
|
||||
"imdb": "^1.1.0",
|
||||
"is-video": "^1.0.1",
|
||||
"line-by-line": "^0.1.6",
|
||||
"lodash": "^4.17.11",
|
||||
"magnet-uri": "^5.1.7",
|
||||
"moment": "^2.24.0",
|
||||
"name-to-imdb": "^2.3.0",
|
||||
"needle": "^2.2.4",
|
||||
"node-gzip": "^1.1.2",
|
||||
"parse-torrent": "^6.1.2",
|
||||
"parse-torrent-title": "git://github.com/TheBeastLT/parse-torrent-title.git#master",
|
||||
"pg": "^7.8.2",
|
||||
"pg-hstore": "^2.3.2",
|
||||
"sequelize": "^4.43.0",
|
||||
"torrent-stream": "^1.1.0"
|
||||
}
|
||||
}
|
||||
42
scrapers/api/horriblesubs.js
Normal file
42
scrapers/api/horriblesubs.js
Normal file
@@ -0,0 +1,42 @@
|
||||
const cheerio = require('cheerio');
|
||||
const needle = require('needle');
|
||||
const moment = require('moment');
|
||||
|
||||
const defaultUrl = 'https://horriblesubs.info';
|
||||
const defaultTimeout = 5000;
|
||||
|
||||
function _getContent(url, config = {},) {
|
||||
const baseUrl = config.proxyUrl || defaultUrl;
|
||||
const timeout = config.timeout || defaultTimeout;
|
||||
|
||||
return needle('get', `${baseUrl}${url}`, { open_timeout: timeout, follow: 2 })
|
||||
.then((response) => response.body)
|
||||
.then((body) => cheerio.load(body))
|
||||
}
|
||||
|
||||
function _getAnimeId(showInfo) {
|
||||
return _getContent(showInfo.url).then($ => {
|
||||
const text = $('div.entry-content').find('script').html();
|
||||
showInfo.id = text.match(/var hs_showid = (\d+)/)[1];
|
||||
return showInfo
|
||||
})
|
||||
}
|
||||
|
||||
function allShows(config = {}) {
|
||||
return _getContent('/shows', config)
|
||||
.then(($) => $('div[class=\'ind-show\']')
|
||||
.map((index, element) => $(element).children('a'))
|
||||
.map((index, element) => ({
|
||||
title: element.attr('title'),
|
||||
url: `${config.proxyUrl || defaultUrl}${element.attr('href')}`
|
||||
})).get());
|
||||
}
|
||||
|
||||
function showData(showInfo) {
|
||||
return _getAnimeId(showInfo)
|
||||
.then((showInfo) => )
|
||||
|
||||
}
|
||||
|
||||
module.exports = { allShows };
|
||||
|
||||
179
scrapers/api/thepiratebay.js
Normal file
179
scrapers/api/thepiratebay.js
Normal file
@@ -0,0 +1,179 @@
|
||||
const cheerio = require('cheerio');
|
||||
const needle = require('needle');
|
||||
const moment = require('moment');
|
||||
|
||||
const defaultProxies = ['https://pirateproxy.sh', 'https://thepiratebay.org'];
|
||||
const dumpUrl = '/static/dump/csv/';
|
||||
const defaultTimeout = 5000;
|
||||
|
||||
const errors = {
|
||||
REQUEST_ERROR: { code: 'REQUEST_ERROR' },
|
||||
PARSER_ERROR: { code: 'PARSER_ERROR' }
|
||||
};
|
||||
|
||||
Categories = {
|
||||
AUDIO: {
|
||||
ALL: 100,
|
||||
MUSIC: 101,
|
||||
AUDIO_BOOKS: 102,
|
||||
SOUND_CLIPS: 103,
|
||||
FLAC: 104,
|
||||
OTHER: 199
|
||||
},
|
||||
VIDEO: {
|
||||
ALL: 200,
|
||||
MOVIES: 201,
|
||||
MOVIES_DVDR: 202,
|
||||
MUSIC_VIDEOS: 203,
|
||||
MOVIE_CLIPS: 204,
|
||||
TV_SHOWS: 205,
|
||||
HANDHELD: 206,
|
||||
MOVIES_HD: 207,
|
||||
TV_SHOWS_HD: 208,
|
||||
MOVIES_3D: 209,
|
||||
OTHER: 299
|
||||
},
|
||||
APPS: {
|
||||
ALL: 300,
|
||||
WINDOWS: 301,
|
||||
MAC: 302,
|
||||
UNIX: 303,
|
||||
HANDHELD: 304,
|
||||
IOS: 305,
|
||||
ANDROID: 306,
|
||||
OTHER_OS: 399
|
||||
},
|
||||
GAMES: {
|
||||
ALL: 400,
|
||||
PC: 401,
|
||||
MAC: 402,
|
||||
PSx: 403,
|
||||
XBOX360: 404,
|
||||
Wii: 405,
|
||||
HANDHELD: 406,
|
||||
IOS: 407,
|
||||
ANDROID: 408,
|
||||
OTHER: 499
|
||||
},
|
||||
PORN: {
|
||||
ALL: 500,
|
||||
MOVIES: 501,
|
||||
MOVIES_DVDR: 502,
|
||||
PICTURES: 503,
|
||||
GAMES: 504,
|
||||
MOVIES_HD: 505,
|
||||
MOVIE_CLIPS: 506,
|
||||
OTHER: 599
|
||||
},
|
||||
OTHER: {
|
||||
ALL: 600,
|
||||
E_BOOKS: 601,
|
||||
COMICS: 602,
|
||||
PICTURES: 603,
|
||||
COVERS: 604,
|
||||
PHYSIBLES: 605,
|
||||
OTHER: 699
|
||||
}
|
||||
};
|
||||
|
||||
function search(keyword, config = {}, retries = 2) {
|
||||
if (!keyword || retries === 0) {
|
||||
return Promise.reject(new Error(`Failed ${keyword} search`));
|
||||
}
|
||||
const proxyList = config.proxyList || defaultProxies;
|
||||
const page = config.page || 0;
|
||||
const category = config.cat || 0;
|
||||
|
||||
return raceFirstSuccessful(proxyList
|
||||
.map((proxyUrl) => singleRequest(`${proxyUrl}/search/${keyword}/${page}/99/${category}`, config)))
|
||||
.then((body) => parseBody(body))
|
||||
.catch(() => search(keyword, config, retries - 1));
|
||||
}
|
||||
|
||||
function dumps(config = {}, retries = 2) {
|
||||
if (retries === 0) {
|
||||
return Promise.reject(new Error(`Failed dump search`));
|
||||
}
|
||||
const proxyList = config.proxyList || defaultProxies;
|
||||
|
||||
return raceFirstSuccessful(proxyList
|
||||
.map((proxyUrl) => singleRequest(`${proxyUrl}${dumpUrl}`, config)
|
||||
.then((body) => body.match(/(<a href="[^"]+">[^<]+<\/a>.+\d)/g)
|
||||
.map((group) => ({
|
||||
url: `${proxyUrl}${dumpUrl}` + group.match(/<a href="([^"]+)">/)[1],
|
||||
updatedAt: moment(group.match(/\s+([\w-]+\s+[\d:]+)\s+\d+$/)[1], 'DD-MMM-YYYY HH:mm').toDate()
|
||||
})))))
|
||||
.catch(() => dumps(config, retries - 1));
|
||||
}
|
||||
|
||||
function singleRequest(requestUrl, config = {}) {
|
||||
const timeout = config.timeout || defaultTimeout;
|
||||
|
||||
return new Promise(((resolve, reject) => {
|
||||
needle.get(requestUrl,
|
||||
{ open_timeout: timeout, follow: 2 },
|
||||
(err, res, body) => {
|
||||
if (err || !body) {
|
||||
reject(err || errors.REQUEST_ERROR);
|
||||
} else if (body.includes('Access Denied') && !body.includes('<title>The Pirate Bay')) {
|
||||
console.log(`Access Denied: ${url}`);
|
||||
reject(new Error(`Access Denied: ${url}`));
|
||||
} else if (body.includes('502: Bad gateway') ||
|
||||
body.includes('403 Forbidden') ||
|
||||
body.includes('Database maintenance') ||
|
||||
body.includes('Origin DNS error') ||
|
||||
!body.includes('<title>The Pirate Bay')) {
|
||||
reject(errors.REQUEST_ERROR);
|
||||
}
|
||||
|
||||
resolve(body);
|
||||
});
|
||||
}));
|
||||
}
|
||||
|
||||
function parseBody(body) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const $ = cheerio.load(body);
|
||||
|
||||
if (!$) {
|
||||
reject(new Error(errors.PARSER_ERROR));
|
||||
}
|
||||
|
||||
const torrents = [];
|
||||
|
||||
$('table[id=\'searchResult\'] tr').each(function() {
|
||||
const name = $(this).find('.detLink').text();
|
||||
if (!name) {
|
||||
return;
|
||||
}
|
||||
torrents.push({
|
||||
name: name,
|
||||
seeders: parseInt($(this).find('td[align=\'right\']').eq(0).text(), 10),
|
||||
leechers: parseInt($(this).find('td[align=\'right\']').eq(1).text(), 10),
|
||||
magnetLink: $(this).find('a[title=\'Download this torrent using magnet\']').attr('href'),
|
||||
category: parseInt($(this).find('a[title=\'More from this category\']').eq(0).attr('href').match(/\d+$/)[0], 10),
|
||||
subcategory: parseInt($(this).find('a[title=\'More from this category\']').eq(1).attr('href').match(/\d+$/)[0], 10)
|
||||
});
|
||||
});
|
||||
resolve(torrents);
|
||||
});
|
||||
}
|
||||
|
||||
function raceFirstSuccessful(promises) {
|
||||
return Promise.all(promises.map((p) => {
|
||||
// If a request fails, count that as a resolution so it will keep
|
||||
// waiting for other possible successes. If a request succeeds,
|
||||
// treat it as a rejection so Promise.all immediately bails out.
|
||||
return p.then(
|
||||
(val) => Promise.reject(val),
|
||||
(err) => Promise.resolve(err)
|
||||
);
|
||||
})).then(
|
||||
// If '.all' resolved, we've just got an array of errors.
|
||||
(errors) => Promise.reject(errors),
|
||||
// If '.all' rejected, we've got the result we wanted.
|
||||
(val) => Promise.resolve(val)
|
||||
);
|
||||
}
|
||||
|
||||
module.exports = { search, dumps, Categories };
|
||||
167
scrapers/piratebay_dump.js
Normal file
167
scrapers/piratebay_dump.js
Normal file
@@ -0,0 +1,167 @@
|
||||
const moment = require('moment');
|
||||
const needle = require('needle');
|
||||
const Bottleneck = require('bottleneck');
|
||||
const { ungzip } = require('node-gzip');
|
||||
const LineByLineReader = require('line-by-line');
|
||||
const fs = require('fs');
|
||||
const { parse } = require('parse-torrent-title');
|
||||
const pirata = require('./api/thepiratebay');
|
||||
const { torrentFiles } = require('../lib/torrent');
|
||||
const repository = require('../lib/repository');
|
||||
const { getImdbId } = require('../lib/metadata');
|
||||
|
||||
const NAME = 'thepiratebay';
|
||||
const CSV_FILE_PATH = '/tmp/tpb_dump.csv';
|
||||
|
||||
const limiter = new Bottleneck({maxConcurrent: 40});
|
||||
|
||||
async function scrape() {
|
||||
const title = 'Я'
|
||||
+ '+(2014)_1280x720-raroch.mp4'
|
||||
.replace(/^"|"$/g, '')
|
||||
.normalize('NFKD') // normalize non-ASCII characters
|
||||
.replace(/[\u0300-\u036F]/g, '')
|
||||
.replace(/&\w{2,6};/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.replace(/[\W\s]+/, ' ');
|
||||
const titleInfo = parse(title);
|
||||
const imdbId = await getImdbId({
|
||||
name: titleInfo.title.toLowerCase(),
|
||||
year: titleInfo.year
|
||||
});
|
||||
|
||||
|
||||
const lastScraped = await repository.getProvider({ name: NAME });
|
||||
const lastDump = await pirata.dumps().then((dumps) => dumps.sort((a, b) => b.updatedAt - a.updatedAt)[0]);
|
||||
|
||||
if (!lastScraped.lastScraped || lastScraped.lastScraped < lastDump.updatedAt) {
|
||||
console.log(`starting to scrape tpb dump: ${JSON.stringify(lastDump)}`);
|
||||
//await downloadDump(lastDump);
|
||||
|
||||
const lr = new LineByLineReader(CSV_FILE_PATH);
|
||||
lr.on('line', (line) => {
|
||||
if (line.includes("#ADDED")) {
|
||||
return;
|
||||
}
|
||||
const row = line.match(/(?<=^|;)(".*"|[^;]+)(?=;|$)/g);
|
||||
const torrent = {
|
||||
uploadDate: moment(row[0], 'YYYY-MMM-DD HH:mm:ss').toDate(),
|
||||
infoHash: Buffer.from(row[1], 'base64').toString('hex'),
|
||||
title: row[2]
|
||||
.replace(/^"|"$/g, '')
|
||||
.replace(/&\w{2,6};/g, ' ')
|
||||
.replace(/\s+/g, ' '),
|
||||
size: parseInt(row[3], 10)
|
||||
};
|
||||
|
||||
if (!limiter.empty()) {
|
||||
lr.pause()
|
||||
}
|
||||
|
||||
limiter.schedule(() => processTorrentRecord(torrent)
|
||||
.catch((error) => console.log(`failed ${torrent.title} due: ${error}`)))
|
||||
.then(() => limiter.empty())
|
||||
.then((empty) => empty && lr.resume());
|
||||
});
|
||||
lr.on('error', (err) => {
|
||||
console.log(err);
|
||||
});
|
||||
lr.on('end', () => {
|
||||
fs.unlink(CSV_FILE_PATH);
|
||||
updateProvider({ name: NAME, lastScraped: lastDump.updatedAt.toDate() });
|
||||
console.log(`finished to scrape tpb dump: ${JSON.stringify(lastDump)}!`);
|
||||
});
|
||||
}
|
||||
}
|
||||
const allowedCategories = [
|
||||
pirata.Categories.VIDEO.MOVIES,
|
||||
pirata.Categories.VIDEO.MOVIES_HD,
|
||||
pirata.Categories.VIDEO.MOVIES_DVDR,
|
||||
pirata.Categories.VIDEO.MOVIES_3D,
|
||||
pirata.Categories.VIDEO.TV_SHOWS,
|
||||
pirata.Categories.VIDEO.TV_SHOWS_HD
|
||||
];
|
||||
const seriesCategories = [
|
||||
pirata.Categories.VIDEO.TV_SHOWS,
|
||||
pirata.Categories.VIDEO.TV_SHOWS_HD
|
||||
];
|
||||
async function processTorrentRecord(record) {
|
||||
const persisted = await repository.getSkipTorrent(record)
|
||||
.catch(() => repository.getTorrent(record)).catch(() => undefined);
|
||||
if (persisted) {
|
||||
return;
|
||||
}
|
||||
|
||||
let page = 0;
|
||||
let torrentFound;
|
||||
while (!torrentFound && page < 5) {
|
||||
const torrents = await pirata.search(record.title.replace(/[\W\s]+/, ' '), { page: page });
|
||||
torrentFound = torrents.
|
||||
filter(torrent => torrent.magnetLink.toLowerCase().includes(record.infoHash))[0];
|
||||
page = torrents.length === 0 ? 1000 : page + 1;
|
||||
}
|
||||
|
||||
if (!torrentFound) {
|
||||
console.log(`not found: ${JSON.stringify(record)}`);
|
||||
repository.createSkipTorrent(record);
|
||||
return;
|
||||
}
|
||||
if (!allowedCategories.includes(torrentFound.subcategory)) {
|
||||
console.log(`wrong category: ${torrentFound.name}`);
|
||||
repository.createSkipTorrent(record);
|
||||
return;
|
||||
}
|
||||
|
||||
const type = seriesCategories.includes(torrentFound.subcategory) ? 'series' : 'movie';
|
||||
console.log(`imdbId search: ${torrentFound.name}`);
|
||||
const titleInfo = parse(torrentFound.name);
|
||||
const imdbId = await getImdbId({
|
||||
name: titleInfo.title.toLowerCase(),
|
||||
year: titleInfo.year,
|
||||
type: type
|
||||
}).catch(() => undefined);
|
||||
|
||||
if (!imdbId) {
|
||||
console.log(`imdbId not found: ${torrentFound.name}`);
|
||||
repository.createFailedImdbTorrent(record);
|
||||
return;
|
||||
}
|
||||
|
||||
if (type === 'movie' || titleInfo.episode) {
|
||||
repository.updateTorrent({
|
||||
infoHash: record.infoHash,
|
||||
provider: NAME,
|
||||
title: torrentFound.name,
|
||||
imdbId: imdbId,
|
||||
uploadDate: record.uploadDate,
|
||||
seeders: torrentFound.seeders,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const files = await torrentFiles(record).catch(() => []);
|
||||
if (!files || !files.length) {
|
||||
console.log(`no video files found: ${torrentFound.name}`);
|
||||
return;
|
||||
}
|
||||
|
||||
repository.updateTorrent({
|
||||
infoHash: record.infoHash,
|
||||
provider: NAME,
|
||||
title: torrentFound.name,
|
||||
imdbId: imdbId,
|
||||
uploadDate: record.uploadDate,
|
||||
seeders: torrentFound.seeders,
|
||||
files: files
|
||||
})
|
||||
}
|
||||
|
||||
function downloadDump(dump) {
|
||||
console.log('downloading dump file...');
|
||||
return needle('get', dump.url, { open_timeout: 2000, output: '/home/paulius/Downloads/tpb_dump.gz' })
|
||||
.then((response) => response.body)
|
||||
.then((body) => { console.log('unzipping dump file...'); return ungzip(body); })
|
||||
.then((unzipped) => { console.log('writing dump file...'); return fs.promises.writeFile(CSV_FILE_PATH, unzipped); })
|
||||
}
|
||||
|
||||
module.exports = { scrape };
|
||||
Reference in New Issue
Block a user