tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] removes proxy for katcr

Browse Source
This commit is contained in:
TheBeastLT
2020-09-17 21:33:38 +02:00
parent e9b88dd0ad
commit d9eb5f3b35
2 changed files with 2 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
scraper/lib/request_helper.js
View File

@@ -1,21 +1,10 @@
const UserAgent = require('user-agents');
const PROXY_HOSTS = process.env.PROXY_HOSTS && process.env.PROXY_HOSTS.split(',');
const PROXY_USERNAME = process.env.PROXY_USERNAME;
const PROXY_PASSWORD = process.env.PROXY_PASSWORD;
const userAgent = new UserAgent();
function getRandomUserAgent() {
return userAgent.random().toString();
}
function getRandomProxy() {
if (PROXY_HOSTS && PROXY_HOSTS.length && PROXY_USERNAME && PROXY_PASSWORD) {
return `http://${PROXY_USERNAME}:${PROXY_PASSWORD}@${PROXY_HOSTS[Math.floor(Math.random() * PROXY_HOSTS.length)]}`;
}
return undefined;
}
function defaultOptionsWithProxy() {
if (process.env.PROXY_HOST && process.env.PROXY_TYPE) {
return {
@@ -29,4 +18,4 @@ function defaultOptionsWithProxy() {
return { userAgent: getRandomUserAgent() };
}
module.exports = { getRandomUserAgent, getRandomProxy, defaultOptionsWithProxy };
module.exports = { getRandomUserAgent, defaultOptionsWithProxy };