diff --git a/docker-compose.yml b/docker-compose.yml index 6a22fdc..d611076 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -53,3 +53,8 @@ services: volumes: mongo-data: postgres-data: + flaresolverr: + image: ghcr.io/flaresolverr/flaresolverr:latest + restart: unless-stopped + ports: + - 8191:8191 \ No newline at end of file diff --git a/scraper/scrapers/1337x/1337x_api.js b/scraper/scrapers/1337x/1337x_api.js index fa9dad6..1e16f1e 100644 --- a/scraper/scrapers/1337x/1337x_api.js +++ b/scraper/scrapers/1337x/1337x_api.js @@ -83,9 +83,16 @@ function singleRequest(requestUrl, config = {}) { const timeout = config.timeout || defaultTimeout; const options = { headers: { 'User-Agent': getRandomUserAgent() }, timeout: timeout }; - return axios.get(requestUrl, options) + return axios.post('http://flaresolverr:8191/v1', { + cmd: 'request.get', + url: requestUrl, + }, options) .then((response) => { - const body = response.data; + if (response.data.status !== 'ok'){ + throw new Error(`FlareSolverr did not return status 'ok': ${response.data.message}`) + } + + const body = response.data.solution.response; if (!body) { throw new Error(`No body: ${requestUrl}`); } else if (body.includes('502: Bad gateway') ||