brock/torrentio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

format animestorrent provider

Browse Source
This commit is contained in:
TheBeastLT
2021-09-14 15:09:37 +02:00
committed by TheBeastLT
parent a0fae89b81
commit 4b4cefb3c2
2 changed files with 93 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
scraper/scrapers/animestorrent/animestorrent_api.js
View File

@@ -8,12 +8,12 @@ const defaultTimeout = 10000;
const maxSearchPage = 50; const maxSearchPage = 50;
const defaultProxies = [ const defaultProxies = [
"https://animestorrent.com" 'https://animestorrent.com'
]; ];
const Categories = { const Categories = {
MOVIE: "filme", MOVIE: 'filme',
ANIME: "tv", ANIME: 'tv',
OVA: 'ova' OVA: 'ova'
}; };
@@ -23,9 +23,7 @@ function torrent(torrentId, config = {}, retries = 2) {
} }
const proxyList = config.proxyList || defaultProxies; const proxyList = config.proxyList || defaultProxies;
const slug = torrentId.split("/")[3]; const slug = torrentId.split("/")[3];
return Promises.first( return Promises.first(proxyList.map((proxyUrl) => singleRequest(`${proxyUrl}/${slug}`, config)))
proxyList.map((proxyUrl) => singleRequest(`${proxyUrl}/${slug}`, config))
)
.then((body) => parseTorrentPage(body)) .then((body) => parseTorrentPage(body))
.then((torrent) => torrent.map((el) => ({ torrentId: slug, ...el }))) .then((torrent) => torrent.map((el) => ({ torrentId: slug, ...el })))
.catch((err) => torrent(slug, config, retries - 1)); .catch((err) => torrent(slug, config, retries - 1));
@@ -40,17 +38,14 @@ function search(keyword, config = {}, retries = 2) {
const extendToPage = Math.min(maxSearchPage, config.extendToPage || 1); const extendToPage = Math.min(maxSearchPage, config.extendToPage || 1);
const requestUrl = (proxyUrl) => `${proxyUrl}/page/${page}/?s=${keyword}`; const requestUrl = (proxyUrl) => `${proxyUrl}/page/${page}/?s=${keyword}`;
return Promises.first( return Promises.first(proxyList.map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config)))
proxyList.map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config))
)
.then((body) => parseTableBody(body)) .then((body) => parseTableBody(body))
.then((torrents) => .then((torrents) =>
torrents.length === 40 && page < extendToPage torrents.length === 40 && page < extendToPage
? search(keyword, { ...config, page: page + 1 }) ? search(keyword, { ...config, page: page + 1 })
.catch(() => []) .catch(() => [])
.then((nextTorrents) => torrents.concat(nextTorrents)) .then((nextTorrents) => torrents.concat(nextTorrents))
: torrents : torrents)
)
.catch((err) => search(keyword, config, retries - 1)); .catch((err) => search(keyword, config, retries - 1));
} }
@@ -66,9 +61,7 @@ function browse(config = {}, retries = 2) {
? `${proxyUrl}/tipo/${category}/page/${page}/` ? `${proxyUrl}/tipo/${category}/page/${page}/`
: `${proxyUrl}/page/${page}/`; : `${proxyUrl}/page/${page}/`;
return Promises.first( return Promises.first(proxyList.map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config)))
proxyList.map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config))
)
.then((body) => parseTableBody(body)) .then((body) => parseTableBody(body))
.catch((err) => browse(config, retries - 1)); .catch((err) => browse(config, retries - 1));
} }
@@ -129,18 +122,20 @@ function parseTorrentPage(body) {
magnets.push(magnet); magnets.push(magnet);
}); });
const details = $('div.infox') const details = $('div.infox')
const torrent = magnets.map((magnetLink) => { const torrents = magnets.map((magnetLink) => {
return { return {
title: decode(magnetLink).name, title: decode(magnetLink).name,
original_name: details.find('h1.entry-title').text(), originalName: details.find('h1.entry-title').text(),
year: details.find('b:contains(\'Lançamento:\')')[0] ? details.find('b:contains(\'Lançamento:\')')[0].nextSibling.nodeValue.trim() : '', year: details.find('b:contains(\'Lançamento:\')')[0]
? details.find('b:contains(\'Lançamento:\')')[0].nextSibling.nodeValue.trim()
: '',
infoHash: decode(magnetLink).infoHash, infoHash: decode(magnetLink).infoHash,
magnetLink: magnetLink, magnetLink: magnetLink,
category: details.find('b:contains(\'Tipo:\')').next().attr('href').split('/')[4], category: details.find('b:contains(\'Tipo:\')').next().attr('href').split('/')[4],
uploadDate: new Date($("time[itemprop=dateModified]").attr("datetime")), uploadDate: new Date($("time[itemprop=dateModified]").attr("datetime")),
}; };
}) })
resolve(torrent.filter((x) => x)); resolve(torrents);
}); });
} }

111
scraper/scrapers/animestorrent/animestorrent_scraper.js
View File

@@ -1,16 +1,15 @@
const moment = require("moment"); const moment = require("moment");
const Bottleneck = require("bottleneck"); const Bottleneck = require("bottleneck");
const leetx = require("./animestorrent_api"); const animetorrrent = require("./animestorrent_api");
const { Type } = require("../../lib/types"); const { Type } = require("../../lib/types");
const repository = require("../../lib/repository"); const repository = require("../../lib/repository");
const Promises = require("../../lib/promises"); const Promises = require("../../lib/promises");
const { createTorrentEntry, checkAndUpdateTorrent } = require("../../lib/torrentEntries"); const { createTorrentEntry, checkAndUpdateTorrent } = require("../../lib/torrentEntries");
const { updateCurrentSeeders, updateTorrentSize } = require("../../lib/torrent"); const { updateCurrentSeeders, updateTorrentSize } = require("../../lib/torrent");
const { getImdbId } = require("../../lib/metadata"); const { getKitsuId } = require("../../lib/metadata");
const NAME = "AnimesTorrent"; const NAME = "AnimesTorrent";
const UNTIL_PAGE = 5; const UNTIL_PAGE = 5;
const TYPE_MAPPING = typeMapping();
const limiter = new Bottleneck({ maxConcurrent: 5 }); const limiter = new Bottleneck({ maxConcurrent: 5 });
@@ -28,101 +27,79 @@ async function scrape() {
} }
async function updateSeeders(torrent) { async function updateSeeders(torrent) {
return limiter.schedule(() => leetx.torrent(torrent.torrentId)); return limiter.schedule(() => animetorrrent.torrent(torrent.torrentId));
} }
async function scrapeLatestTorrents() { async function scrapeLatestTorrents() {
const allowedCategories = [ const allowedCategories = [
leetx.Categories.MOVIE, animetorrrent.Categories.MOVIE,
leetx.Categories.ANIME, animetorrrent.Categories.ANIME,
leetx.Categories.OVA animetorrrent.Categories.OVA
]; ];
return Promises.sequence( return Promises.sequence(allowedCategories
allowedCategories.map( .map((category) => () => scrapeLatestTorrentsForCategory(category)))
(category) => () => scrapeLatestTorrentsForCategory(category) .then((entries) => entries.reduce((a, b) => a.concat(b), []));
)
).then((entries) => entries.reduce((a, b) => a.concat(b), []));
} }
async function scrapeLatestTorrentsForCategory(category, page = 1) { async function scrapeLatestTorrentsForCategory(category, page = 1) {
console.log({Scraper: `Scrapping ${NAME} ${category} category page ${page}`}); console.log(`Scrapping ${NAME} ${category} category page ${page}`);
return leetx return animetorrrent
.browse({ category, page }) .browse({ category, page })
.catch((error) => { .catch((error) => {
console.warn( console.warn(`Failed ${NAME} scrapping for [${page}] ${category} due: `, error);
`Failed ${NAME} scrapping for [${page}] ${category} due: `,
error
);
return Promise.resolve([]); return Promise.resolve([]);
}) })
.then((torrents) => Promise.all(torrents.map((torrent) => limiter.schedule(() => processTorrentRecord(torrent))))) .then((torrents) => Promise.all(torrents.map((torrent) => limiter.schedule(() => processEntry(torrent)))))
.then((resolved) => resolved.length > 0 && page < untilPage(category) ? scrapeLatestTorrentsForCategory(category, page + 1) : Promise.resolve()); .then((resolved) => resolved.length > 0 && page < untilPage(category)
? scrapeLatestTorrentsForCategory(category, page + 1)
: Promise.resolve());
} }
async function processTorrentRecord(record) { async function processEntry(entry) {
if (await checkAndUpdateTorrent({ provider: NAME, ...record })) { return animetorrrent.torrent(entry.torrentId)
return record; .then(records => Promises.sequence(records.map(record => () => processTorrentRecord(record))))
}
const torrentEntrys = await leetx
.torrent(record.torrentId)
.catch(() => undefined); .catch(() => undefined);
if (torrentEntrys === undefined) {
return Promise.resolve([])
} }
return Promise.allSettled(
torrentEntrys.map(async (torrentFound) => { async function processTorrentRecord(foundTorrent) {
if (!torrentFound || !TYPE_MAPPING[torrentFound.category]) { if (await checkAndUpdateTorrent({ provider: NAME, ...foundTorrent })) {
return Promise.resolve("Invalid torrent record"); return foundTorrent;
} }
if (isNaN(torrentFound.uploadDate)) {
console.warn( if (!foundTorrent.size) {
`Incorrect upload date for [${torrentFound.infoHash}] ${torrentFound.name}` await updateTorrentSize(foundTorrent);
);
return;
} }
if (await checkAndUpdateTorrent(torrentFound)) { if (!Number.isInteger(foundTorrent.seeders)) {
return torrentFound; await updateCurrentSeeders(foundTorrent);
} }
if (!torrentFound.size) { if (!foundTorrent.imdbId && !foundTorrent.kitsuId) {
await updateTorrentSize(torrentFound) const info = { title: foundTorrent.originalName, year: foundTorrent.year };
.catch((err) => Promise.resolve(err)) foundTorrent.kitsuId = await getKitsuId(info).catch(() => undefined);
}
if (!torrentFound.seeders) {
await updateCurrentSeeders(torrentFound)
.then(response => response.seeders === 0 ? delete response.seeders : response)
}
if (!torrentFound.imdbId) {
torrentFound.imdbId = await getImdbId(torrentFound.original_name, torrentFound.year, TYPE_MAPPING[torrentFound.category])
} }
const torrent = { const torrent = {
infoHash: torrentFound.infoHash, infoHash: foundTorrent.infoHash,
provider: NAME, provider: NAME,
torrentId: torrentFound.torrentId, torrentId: foundTorrent.torrentId,
title: torrentFound.title.replace(/\t|\s+/g, " ").trim(), title: foundTorrent.title,
type: Type.ANIME, type: Type.ANIME,
imdbId: torrentFound.imdbId, imdbId: foundTorrent.imdbId,
uploadDate: torrentFound.uploadDate, kitsuId: foundTorrent.kitsuId,
seeders: torrentFound.seeders, uploadDate: foundTorrent.uploadDate,
seeders: foundTorrent.seeders,
size: foundTorrent.size,
files: foundTorrent.files,
languages: foundTorrent.languages
}; };
return createTorrentEntry(torrent); return createTorrentEntry(torrent);
})
);
}
function typeMapping() {
const mapping = {};
mapping[leetx.Categories.MOVIE] = Type.MOVIE;
mapping[leetx.Categories.ANIME] = Type.SERIES;
mapping[leetx.Categories.OVA] = Type.ANIME
return mapping;
} }
function untilPage(category) { function untilPage(category) {
if (leetx.Categories.ANIME === category) { if (animetorrrent.Categories.ANIME === category) {
return 5; return 5;
} }
if (leetx.Categories.OVA === category) { if (animetorrrent.Categories.OVA === category) {
return 3; return 3;
} }
return UNTIL_PAGE; return UNTIL_PAGE;