tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] adds helper class for proxy and user agent

Browse Source
This commit is contained in:
TheBeastLT
2020-04-02 22:47:52 +02:00
parent db78f5873b
commit 2833c9d3a2
6 changed files with 76 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
scraper/lib/metadata.js
View File

@@ -5,6 +5,7 @@ const bing = require('nodejs-bing');
const he = require('he');
const { cacheWrapImdbId, cacheWrapKitsuId, cacheWrapMetadata } = require('./cache');
const { Type } = require('./types');
const { getRandomUserAgent } = require('./request_helper');
const CINEMETA_URL = 'https://v3-cinemeta.strem.io';
const KITSU_URL = 'https://anime-kitsu.strem.fun';
@@ -107,7 +108,7 @@ async function getImdbId(info, type) {
reject(err || new Error('failed imdbId search'));
}
});
}).catch(() => googleIt({ query, disableConsole: true })
}).catch(() => googleIt({ query, userAgent: getRandomUserAgent(), disableConsole: true })
.catch(() => bing.web(query))
.then(results => results
.map(result => result.link)

26
scraper/lib/request_helper.js Normal file
View File

@@ -0,0 +1,26 @@
const UserAgent = require('user-agents');
const PROXY_HOSTS = process.env.PROXY_HOST && process.env.PROXY_HOST.split(',');
const PROXY_USERNAME = process.env.PROXY_USERNAME;
const PROXY_PASSWORD = process.env.PROXY_PASSWORD;
const userAgent = new UserAgent();
function getRandomUserAgent() {
return userAgent.random().toString();
}
function getRandomProxy() {
if (PROXY_HOSTS && PROXY_HOSTS.length && PROXY_USERNAME && PROXY_PASSWORD) {
return `http://${PROXY_USERNAME}:${PROXY_PASSWORD}@${PROXY_HOSTS[Math.floor(Math.random() * PROXY_HOSTS.length)]}`;
}
return undefined;
}
function getProxy() {
if (PROXY_HOSTS && PROXY_HOSTS.length && PROXY_USERNAME && PROXY_PASSWORD) {
return `http://${PROXY_USERNAME}:${PROXY_PASSWORD}@${PROXY_HOSTS[0]}`;
}
return undefined;
}
module.exports = { getRandomUserAgent, getRandomProxy, getProxy };

12
scraper/moch/realdebrid.js
View File

@@ -2,10 +2,7 @@ const { encode } = require('magnet-uri');
const RealDebridClient = require('real-debrid-api');
const namedQueue = require('named-queue');
const { cacheWrapResolvedUrl } = require('../lib/cache');
const PROXY_HOST = process.env.PROXY_HOST;
const PROXY_USERNAME = process.env.PROXY_USERNAME;
const PROXY_PASSWORD = process.env.PROXY_PASSWORD;
const { getProxy } = require('../lib/request_helper');
const unrestrictQueue = new namedQueue((task, callback) => task.method()
.then(result => callback(false, result))
@@ -70,11 +67,4 @@ async function _unrestrictLink(RD, link) {
// });
}
function getProxy() {
if (PROXY_HOST && PROXY_USERNAME && PROXY_PASSWORD) {
return `http://${PROXY_USERNAME}:${PROXY_PASSWORD}@${PROXY_HOST}`;
}
return undefined;
}
module.exports = { resolve };

4
scraper/scrapers/kickass/kickass_api.js
View File

@@ -3,6 +3,7 @@ const needle = require('needle');
const moment = require('moment');
const decode = require('magnet-uri');
const Promises = require('../../lib/promises');
const { getRandomProxy, getRandomUserAgent } = require('./../../lib/request_helper');
const defaultProxies = [
'https://katcr.co'
@@ -64,8 +65,9 @@ function browse(config = {}, retries = 2) {
function singleRequest(requestUrl, config = {}) {
const timeout = config.timeout || defaultTimeout;
const options = { userAgent: getRandomUserAgent(), proxy: getRandomProxy(), open_timeout: timeout, follow: 2 };
return needle('get', requestUrl, { open_timeout: timeout, follow: 2 })
return needle('get', requestUrl, options)
.then((response) => {
const body = response.body;
if (!body) {