tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Updated scrapers to latest available commit

Browse Source
This commit is contained in:
Gabisonfire
2024-01-17 16:43:58 -05:00
parent cab7f38c66
commit 909ade0d8e
39 changed files with 311 additions and 299 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
scraper/scrapers/rarbg/rarbg_api.js
View File

@@ -1,10 +1,12 @@
const needle = require('needle');
const axios = require('axios');
const decode = require('magnet-uri');
const Promises = require('../../lib/promises');
const { getRandomUserAgent } = require("../../lib/requestHelper");
const baseUrl = 'https://torrentapi.org/pubapi_v2.php';
const appId = 'torrentio-addon';
const defaultTimeout = 30000;
const retryDelay = 3000;
let token;
@@ -22,6 +24,7 @@ const Options = {
MOVIES_X265_4K_HDR: [52],
MOVIES_FULL_BD: [42],
MOVIES_BD_REMUX: [46],
MOVIES_HIGH_RES: [47, 50, 51, 52, 46],
TV_EPISODES: [18],
TV_UHD_EPISODES: [49],
TV_HD_EPISODES: [41],
@@ -84,33 +87,43 @@ function browse(params = {}) {
return singleRequest(parameters).then(results => parseResults(results));
}
async function singleRequest(params = {}, config = {}, retries = 10) {
async function singleRequest(params = {}, config = {}, retries = 15) {
const timeout = config.timeout || defaultTimeout;
const options = { open_timeout: timeout, follow: 2 };
const headers = {
'user-agent': getRandomUserAgent(),
'accept-encoding': 'gzip, deflate',
'accept-language': 'en-GB,en;q=0.9,en-US;q=0.8,lt;q=0.7,ar;q=0.6,fr;q=0.5,de;q=0.4'
};
params.token = await getToken();
params.app_id = appId;
Object.keys(params)
.filter(key => params[key] === undefined || params[key] === null)
.forEach(key => delete params[key]);
return needle('get', baseUrl, params, options)
const options = { headers, timeout, params };
return axios.get(baseUrl, options)
.then(response => {
if (response.body && response.body.error_code === 4) {
if (response.data && response.data.error_code === 4) {
// token expired
token = undefined;
return singleRequest(params, config);
}
if ((!response.body || !response.body.length || [5, 20].includes(response.body.error_code)) && retries > 0) {
if ((!response.data || !response.data.length || [5, 20].includes(response.data.error_code)) && retries > 0) {
// too many requests
return Promises.delay(3000).then(() => singleRequest(params, config, retries - 1));
return Promises.delay(retryDelay).then(() => singleRequest(params, config, retries - 1));
}
if (response.statusCode !== 200 || (response.body && response.body.error)) {
if (response.status !== 200 || (response.data && response.data.error)) {
// something went wrong
return Promise.reject(response.body || `Failed RARGB request with status=${response.statusCode}`);
return Promise.reject(response.data || `Failed RARGB request with status=${response.status}`);
}
return response.body;
return response.data;
})
.catch(error => {
if (error.response && [429].includes(error.response.status) && retries > 0) {
return Promises.delay(retryDelay).then(() => singleRequest(params, config, retries - 1));
}
return Promise.reject(error.message || error);
});
}
@@ -137,9 +150,10 @@ function parseResult(result) {
async function getToken() {
if (!token) {
const options = { open_timeout: defaultTimeout };
token = await needle('get', baseUrl, { get_token: 'get_token', app_id: appId }, options)
.then(response => response.body.token);
const params = { get_token: 'get_token', app_id: appId };
const options = { timeout: defaultTimeout, params };
token = await axios.get(baseUrl, options)
.then(response => response.data.token);
}
return token;
}

10
scraper/scrapers/rarbg/rarbg_dump_scraper.js
View File

@@ -12,14 +12,10 @@ const allowedCategories = [
rarbg.Options.category.MOVIES_XVID,
rarbg.Options.category.MOVIES_XVID_720P,
rarbg.Options.category.MOVIES_X265_1080P,
rarbg.Options.category.MOVIES_X265_4K,
rarbg.Options.category.MOVIES_X265_4K_HDR,
rarbg.Options.category.MOVIES_X264,
rarbg.Options.category.MOVIES_X264_720P,
rarbg.Options.category.MOVIES_X264_1080P,
rarbg.Options.category.MOVIES_X264_3D,
rarbg.Options.category.MOVIES_X264_4K,
rarbg.Options.category.MOVIES_BD_REMUX,
rarbg.Options.category.MOVIES_HIGH_RES,
rarbg.Options.category.TV_EPISODES,
rarbg.Options.category.TV_UHD_EPISODES,
rarbg.Options.category.TV_HD_EPISODES
@@ -32,8 +28,8 @@ async function scrape() {
//const allImdbIds = [].concat(movieImdbIds).concat(seriesImdbIds);
return Promise.all(
seriesImdbIds.map(imdbId => limiter.schedule(() => getTorrentsForImdbId(imdbId))
.then(torrents => Promise.all(torrents.map(t => entryLimiter.schedule(() => processTorrentRecord(t)))))))
seriesImdbIds.map(imdbId => limiter.schedule(() => getTorrentsForImdbId(imdbId))
.then(torrents => Promise.all(torrents.map(t => entryLimiter.schedule(() => processTorrentRecord(t)))))))
.then(() => console.log(`[${moment()}] finished ${NAME} dump scrape`));
}

8
scraper/scrapers/rarbg/rarbg_scraper.js
View File

@@ -36,21 +36,17 @@ async function scrapeLatestTorrents() {
rarbg.Options.category.MOVIES_XVID,
rarbg.Options.category.MOVIES_XVID_720P,
rarbg.Options.category.MOVIES_X265_1080P,
rarbg.Options.category.MOVIES_X265_4K,
rarbg.Options.category.MOVIES_X265_4K_HDR,
rarbg.Options.category.MOVIES_X264,
rarbg.Options.category.MOVIES_X264_720P,
rarbg.Options.category.MOVIES_X264_1080P,
rarbg.Options.category.MOVIES_X264_3D,
rarbg.Options.category.MOVIES_X264_4K,
rarbg.Options.category.MOVIES_BD_REMUX,
rarbg.Options.category.MOVIES_HIGH_RES,
rarbg.Options.category.TV_EPISODES,
rarbg.Options.category.TV_UHD_EPISODES,
rarbg.Options.category.TV_HD_EPISODES
];
return Promises.sequence(allowedCategories
.map(category => () => limiter.schedule(() => scrapeLatestTorrentsForCategory(category))))
.map(category => () => limiter.schedule(() => scrapeLatestTorrentsForCategory(category))))
.then(entries => entries.reduce((a, b) => a.concat(b), []));
}