updates tpb dump scrapper

This commit is contained in:
TheBeastLT
2019-12-29 20:07:15 +01:00
parent 7925f18064
commit e7f46d2adc
12 changed files with 689 additions and 165 deletions

View File

@@ -46,7 +46,7 @@ function filesFromKat(infoHash) {
if (!infoHash) {
return Promise.reject(new Error("no infoHash"));
}
const url = `http://kat.rip/torrent/${infoHash}.html`;
const url = `https://kat.rip/torrent/${infoHash}.html`;
return needle('get', url, { open_timeout: 2000 })
.then((response) => {
if (!response.body || response.statusCode !== 200) {
@@ -61,6 +61,7 @@ function filesFromKat(infoHash) {
$('table[id=\'ul_top\'] tr').each((index, row) => {
files.push({
fileIndex: index,
name: $(row).find('td[class=\'torFileName\']').text().replace(/.*\//, ''),
path: $(row).find('td[class=\'torFileName\']').text(),
size: convertToBytes($(row).find('td[class=\'torFileSize\']').text())
});
@@ -139,24 +140,22 @@ function convertToBytes(sizeString) {
function dynamicTimeout(torrent) {
if (torrent.seeders < 5) {
return 3000;
} else if (torrent.seeders < 10) {
return 4000;
} else if (torrent.seeders < 20) {
return 5000;
} else if (torrent.seeders < 30) {
} else if (torrent.seeders < 10) {
return 7000;
} else if (torrent.seeders < 50) {
return 9000;
} else if (torrent.seeders < 100) {
return 12000;
} else {
} else if (torrent.seeders < 20) {
return 10000;
} else if (torrent.seeders < 30) {
return 15000;
} else if (torrent.seeders < 50) {
return 20000;
} else {
return 30000;
}
}
function getTrackerList() {
return needle('get', 'https://torrents.me/tracker-list/', { open_timeout: 2000 })
return needle('get', 'https://torrents.me/tracker-list/', { open_timeout: 2000, follow_max: 2 })
.then((response) => {
if (!response.body || response.statusCode !== 200) {
throw new Error('tracker list not found')