brock/torrentio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

updates horriblesubs api to filter entries based on title

Browse Source
This commit is contained in:
TheBeastLT
2020-02-17 14:26:27 +01:00
parent 683b58b4b6
commit 30421815d7
3 changed files with 52 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
scrapers/horriblesubs/horriblesubs_api.js
View File

@@ -1,7 +1,6 @@
const cheerio = require('cheerio');
const needle = require('needle');
const moment = require('moment');
const decode = require('magnet-uri');
const defaultUrl = 'https://horriblesubs.info';
const defaultTimeout = 5000;
@@ -18,10 +17,10 @@ function allShows(config = {}) {
async function showData(showInfo, config = {}) {
const showEndpoint = (showInfo.url || showInfo).match(/\/show.+/)[0];
const title = showInfo.title;
const showId = await _getShowId(showEndpoint);
const packEntries = await _getAllEntries(showId, 'batch', config);
const singleEntries = await _getAllEntries(showId, 'show', config);
const title = showInfo.title || singleEntries[0] && singleEntries[0].title;
const packEntries = await _getShowEntries(showId, title, 'batch', config);
const singleEntries = await _getShowEntries(showId, title, 'show', config);
return {
title: title,
@@ -55,12 +54,17 @@ function _getShowId(showEndpoint) {
.then($ => $('div.entry-content').find('script').html().match(/var hs_showid = (\d+)/)[1]);
}
function _getShowEntries(animeId, animeTitle, type, config) {
return _getAllEntries(animeId, type, config)
.then((entries) => entries.filter((entry) => entry.title === animeTitle));
}
function _getAllEntries(animeId, type, config, page = 0, autoExtend = true) {
const entriesEndpoint = `/api.php?method=getshows&type=${type}&showid=${animeId}&nextid=${page}`;
return _getEntries(entriesEndpoint, config)
.then((entries) => !autoExtend || entries.length < 12 ? entries :
.then((entries) => !autoExtend || !entries.length ? entries :
_getAllEntries(animeId, type, config, page + 1)
.then((nextEntries) => entries.concat(nextEntries)))
.then((nextEntries) => entries.concat(nextEntries)));
}
function _getEntries(endpoint, config) {
@@ -74,11 +78,10 @@ function _getEntries(endpoint, config) {
mirrors: $(element).find('div[class="rls-links-container"]').children()
.map((indexLink, elementLink) => ({
resolution: $(elementLink).attr('id').match(/\d+p$/)[0],
infoHash: decode($(elementLink).find('a[title="Magnet Link"]').attr('href')).infoHash,
magnetLink: $(elementLink).find('a[title="Magnet Link"]').attr('href'),
torrentLink: $(elementLink).find('a[title="Torrent Link"]').attr('href')
})).get()
})).get())
})).get());
}
function _getAllLatestEntries(config, page = 0) {
@@ -93,7 +96,7 @@ function _getAllLatestEntries(config, page = 0) {
.then((entries) => entries.length < 12
? entries
: _getAllLatestEntries(config, page + 1)
.then((nextEntries) => entries.concat(nextEntries)))
.then((nextEntries) => entries.concat(nextEntries)));
}
async function _findLatestEntry(entry, config) {

7
scrapers/horriblesubs/horriblesubs_scraper.js
View File

@@ -1,6 +1,7 @@
const fs = require('fs');
const Bottleneck = require('bottleneck');
const { parse } = require('parse-torrent-title');
const decode = require('magnet-uri');
const horriblesubs = require('./horriblesubs_api.js');
const repository = require('../../lib/repository');
const { Type } = require('../../lib/types');
@@ -12,7 +13,7 @@ const showMappings = require('./horriblesubs_mapping.json');
const NAME = 'HorribleSubs';
const limiter = new Bottleneck({ maxConcurrent: 5 });
const entryLimiter = new Bottleneck({ maxConcurrent: 20 });
const entryLimiter = new Bottleneck({ maxConcurrent: 10 });
async function scrape() {
const lastScraped = await repository.getProvider({ name: NAME });
@@ -28,7 +29,7 @@ async function _scrapeAllShows() {
const shows = await horriblesubs.allShows();
return Promise.all(shows
.slice(0, 6)
.slice(0, 5)
.map((show) => limiter.schedule(() => horriblesubs.showData(show)
.then((showData) => _parseShowData(showData))
.catch((err) => console.log(err)))));
@@ -86,6 +87,8 @@ async function _parseShowData(showData) {
.map((mirror) => ({
provider: NAME,
...mirror,
infoHash: decode(mirror.magnetLink).infoHash,
trackers: decode(mirror.magnetLink).tr.join(','),
title: `${episodeInfo.title} - ${episodeInfo.episode} [${mirror.resolution}]`,
size: 300000000,
type: Type.ANIME,