tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

adds kickass scrapper

Browse Source
This commit is contained in:
TheBeastLT
2020-03-08 20:06:32 +01:00
parent 853c21472a
commit 6cc0c5dc64
11 changed files with 404 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
scrapers/1337x/1337x_api.js
View File

@@ -6,7 +6,7 @@ const decode = require('magnet-uri');
const defaultProxies = [
'https://1337x.to'
];
const defaultTimeout = 30000;
const defaultTimeout = 10000;
const Categories = {
MOVIE: 'Movies',
@@ -20,12 +20,12 @@ const Categories = {
OTHER: 'Other',
};
function torrent(torrentSlug, config = {}, retries = 2) {
if (!torrentSlug || retries === 0) {
return Promise.reject(new Error(`Failed ${torrentSlug} query`));
function torrent(torrentId, config = {}, retries = 2) {
if (!torrentId || retries === 0) {
return Promise.reject(new Error(`Failed ${torrentId} query`));
}
const proxyList = config.proxyList || defaultProxies;
const slug = torrentSlug.startsWith('/torrent/') ? torrentSlug.replace('/torrent/', '') : torrentSlug;
const slug = torrentId.startsWith('/torrent/') ? torrentId.replace('/torrent/', '') : torrentId;
return raceFirstSuccessful(proxyList
.map((proxyUrl) => singleRequest(`${proxyUrl}/torrent/${slug}`, config)))
@@ -53,7 +53,7 @@ function browse(config = {}, retries = 2) {
}
const proxyList = config.proxyList || defaultProxies;
const page = config.page || 1;
const category = config.category || 0;
const category = config.category;
return raceFirstSuccessful(proxyList
.map((proxyUrl) => singleRequest(`${proxyUrl}/cat/${category}/${page}/`, config)))
@@ -92,7 +92,7 @@ function parseTableBody(body) {
const row = $(element);
torrents.push({
name: row.find('a').eq(1).text(),
slug: row.find('a').eq(1).attr('href').replace('/torrent/', ''),
torrentId: row.find('a').eq(1).attr('href').replace('/torrent/', ''),
seeders: parseInt(row.children('td.coll-2').text()),
leechers: parseInt(row.children('td.coll-3').text()),
size: parseSize(row.children('td.coll-4').text())
@@ -116,9 +116,9 @@ function parseTorrentPage(body) {
const imdbIdMatch = details.find('div[id=\'description\']').html().match(/imdb\.com\/title\/tt(\d+)/i);
const torrent = {
name: decode(magnetLink).dn,
name: decode(magnetLink).name.replace(/\+/g, ' '),
infoHash: decode(magnetLink).infoHash,
magnetLink: magnetLink,
infoHash: details.find('strong:contains(\'Infohash\')').next().text(),
seeders: parseInt(details.find('strong:contains(\'Seeders\')').next().text(), 10),
leechers: parseInt(details.find('strong:contains(\'Leechers\')').next().text(), 10),
category: details.find('strong:contains(\'Category\')').next().text(),

46
scrapers/1337x/1337x_scraper.js
View File

@@ -3,16 +3,16 @@ const Bottleneck = require('bottleneck');
const leetx = require('./1337x_api');
const { Type } = require('../../lib/types');
const repository = require('../../lib/repository');
const { createTorrentEntry, createSkipTorrentEntry, getStoredTorrentEntry } = require('../../lib/torrentEntries');
const {
createTorrentEntry,
createSkipTorrentEntry,
getStoredTorrentEntry,
updateTorrentSeeders
} = require('../../lib/torrentEntries');
const NAME = '1337x';
const UNTIL_PAGE = 1;
const TYPE_MAPPING = {
'Movies': Type.MOVIE,
'Documentaries': Type.MOVIE,
'TV': Type.SERIES,
'Anime': Type.ANIME
};
const TYPE_MAPPING = typeMapping();
const limiter = new Bottleneck({ maxConcurrent: 40 });
@@ -26,16 +26,21 @@ async function scrape() {
.then(() => {
lastScrape.lastScraped = scrapeStart;
lastScrape.lastScrapedId = latestTorrents.length && latestTorrents[latestTorrents.length - 1].torrentId;
return lastScrape.save();
});
return repository.updateProvider(lastScrape);
})
.then(() => console.log(`[${moment()}] finished ${NAME} scrape`));
}
async function getLatestTorrents() {
const movies = await getLatestTorrentsForCategory(leetx.Categories.MOVIE);
const series = await getLatestTorrentsForCategory(leetx.Categories.TV);
const anime = await getLatestTorrentsForCategory(leetx.Categories.ANIME);
const docs = await getLatestTorrentsForCategory(leetx.Categories.DOCUMENTARIES);
return movies.concat(series).concat(anime).concat(docs);
const allowedCategories = [
leetx.Categories.MOVIE,
leetx.Categories.TV,
leetx.Categories.ANIME,
leetx.Categories.DOCUMENTARIES
];
return Promise.all(allowedCategories.map(category => getLatestTorrentsForCategory(category)))
.then(entries => entries.reduce((a, b) => a.concat(b), []));
}
async function getLatestTorrentsForCategory(category, page = 1) {
@@ -48,10 +53,10 @@ async function getLatestTorrentsForCategory(category, page = 1) {
async function processTorrentRecord(record) {
if (await getStoredTorrentEntry(record)) {
return;
return updateTorrentSeeders(record);
}
const torrentFound = await leetx.torrent(record.slug).catch(() => undefined);
const torrentFound = await leetx.torrent(record.torrentId).catch(() => undefined);
if (!torrentFound || !TYPE_MAPPING[torrentFound.category]) {
return createSkipTorrentEntry(record);
@@ -72,4 +77,13 @@ async function processTorrentRecord(record) {
return createTorrentEntry(torrent);
}
function typeMapping() {
const mapping = {};
mapping[leetx.Categories.MOVIE] = Type.MOVIE;
mapping[leetx.Categories.DOCUMENTARIES] = Type.MOVIE;
mapping[leetx.Categories.TV] = Type.SERIES;
mapping[leetx.Categories.ANIME] = Type.ANIME;
return mapping;
}
module.exports = { scrape };