tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] removes proxy for katcr

Browse Source
This commit is contained in:
TheBeastLT
2020-09-17 21:33:38 +02:00
parent e9b88dd0ad
commit d9eb5f3b35
2 changed files with 2 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
scraper/lib/request_helper.js
View File

@@ -1,21 +1,10 @@
const UserAgent = require('user-agents'); const UserAgent = require('user-agents');
const PROXY_HOSTS = process.env.PROXY_HOSTS && process.env.PROXY_HOSTS.split(',');
const PROXY_USERNAME = process.env.PROXY_USERNAME;
const PROXY_PASSWORD = process.env.PROXY_PASSWORD;
const userAgent = new UserAgent(); const userAgent = new UserAgent();
function getRandomUserAgent() { function getRandomUserAgent() {
return userAgent.random().toString(); return userAgent.random().toString();
} }
function getRandomProxy() {
if (PROXY_HOSTS && PROXY_HOSTS.length && PROXY_USERNAME && PROXY_PASSWORD) {
return `http://${PROXY_USERNAME}:${PROXY_PASSWORD}@${PROXY_HOSTS[Math.floor(Math.random() * PROXY_HOSTS.length)]}`;
}
return undefined;
}
function defaultOptionsWithProxy() { function defaultOptionsWithProxy() {
if (process.env.PROXY_HOST && process.env.PROXY_TYPE) { if (process.env.PROXY_HOST && process.env.PROXY_TYPE) {
return { return {
@@ -29,4 +18,4 @@ function defaultOptionsWithProxy() {
return { userAgent: getRandomUserAgent() }; return { userAgent: getRandomUserAgent() };
} }
module.exports = { getRandomUserAgent, getRandomProxy, defaultOptionsWithProxy }; module.exports = { getRandomUserAgent, defaultOptionsWithProxy };

3
scraper/scrapers/kickass/kickass_api.js
View File

@@ -3,7 +3,6 @@ const needle = require('needle');
const moment = require('moment'); const moment = require('moment');
const decode = require('magnet-uri'); const decode = require('magnet-uri');
const Promises = require('../../lib/promises'); const Promises = require('../../lib/promises');
const { defaultOptionsWithProxy } = require('./../../lib/request_helper');
const defaultProxies = [ const defaultProxies = [
'https://katcr.co' 'https://katcr.co'
@@ -65,7 +64,7 @@ function browse(config = {}, retries = 2) {
function singleRequest(requestUrl, config = {}) { function singleRequest(requestUrl, config = {}) {
const timeout = config.timeout || defaultTimeout; const timeout = config.timeout || defaultTimeout;
const options = { ...defaultOptionsWithProxy(), open_timeout: timeout, follow: 2 }; const options = { open_timeout: timeout, follow: 2 };
return needle('get', requestUrl, options) return needle('get', requestUrl, options)
.then((response) => { .then((response) => {