tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] adds helper class for proxy and user agent

Browse Source
This commit is contained in:
TheBeastLT
2020-04-02 22:47:52 +02:00
parent db78f5873b
commit 2833c9d3a2
6 changed files with 76 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
scraper/scrapers/kickass/kickass_api.js
View File

@@ -3,6 +3,7 @@ const needle = require('needle');
const moment = require('moment');
const decode = require('magnet-uri');
const Promises = require('../../lib/promises');
const { getRandomProxy, getRandomUserAgent } = require('./../../lib/request_helper');
const defaultProxies = [
'https://katcr.co'
@@ -64,8 +65,9 @@ function browse(config = {}, retries = 2) {
function singleRequest(requestUrl, config = {}) {
const timeout = config.timeout || defaultTimeout;
const options = { userAgent: getRandomUserAgent(), proxy: getRandomProxy(), open_timeout: timeout, follow: 2 };
return needle('get', requestUrl, { open_timeout: timeout, follow: 2 })
return needle('get', requestUrl, options)
.then((response) => {
const body = response.body;
if (!body) {