diff --git a/addon/package-lock.json b/addon/package-lock.json index 1869cca..19833b3 100644 --- a/addon/package-lock.json +++ b/addon/package-lock.json @@ -1553,8 +1553,8 @@ } }, "parse-torrent-title": { - "version": "git://github.com/TheBeastLT/parse-torrent-title.git#7259b01bfe6e1fbc3879ba68d9c58ebac84029e9", - "from": "git://github.com/TheBeastLT/parse-torrent-title.git#7259b01bfe6e1fbc3879ba68d9c58ebac84029e9", + "version": "git://github.com/TheBeastLT/parse-torrent-title.git#afd4a374276420c13c52df8e3d07ae7699c46b60", + "from": "git://github.com/TheBeastLT/parse-torrent-title.git#afd4a374276420c13c52df8e3d07ae7699c46b60", "requires": { "moment": "^2.24.0" } diff --git a/addon/package.json b/addon/package.json index 1b6381d..923eed9 100644 --- a/addon/package.json +++ b/addon/package.json @@ -12,7 +12,7 @@ "cache-manager-mongodb": "^0.2.1", "express-rate-limit": "^5.1.1", "needle": "^2.2.4", - "parse-torrent-title": "git://github.com/TheBeastLT/parse-torrent-title.git#7259b01bfe6e1fbc3879ba68d9c58ebac84029e9", + "parse-torrent-title": "git://github.com/TheBeastLT/parse-torrent-title.git#afd4a374276420c13c52df8e3d07ae7699c46b60", "pg": "^7.8.2", "pg-hstore": "^2.3.2", "real-debrid-api": "^1.0.1", diff --git a/package-lock.json b/package-lock.json index ac96b72..374b22f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1385,8 +1385,8 @@ } }, "parse-torrent-title": { - "version": "git://github.com/TheBeastLT/parse-torrent-title.git#7259b01bfe6e1fbc3879ba68d9c58ebac84029e9", - "from": "git://github.com/TheBeastLT/parse-torrent-title.git#7259b01bfe6e1fbc3879ba68d9c58ebac84029e9", + "version": "git://github.com/TheBeastLT/parse-torrent-title.git#afd4a374276420c13c52df8e3d07ae7699c46b60", + "from": "git://github.com/TheBeastLT/parse-torrent-title.git#afd4a374276420c13c52df8e3d07ae7699c46b60", "requires": { "moment": "^2.24.0" } diff --git a/package.json b/package.json index e5ad7b8..8ce95b0 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "node-schedule": "^1.3.2", "nodejs-bing": "^0.1.0", "parse-torrent": "^6.1.2", - "parse-torrent-title": "git://github.com/TheBeastLT/parse-torrent-title.git#7259b01bfe6e1fbc3879ba68d9c58ebac84029e9", + "parse-torrent-title": "git://github.com/TheBeastLT/parse-torrent-title.git#afd4a374276420c13c52df8e3d07ae7699c46b60", "peer-search": "^0.6.x", "pg": "^7.8.2", "pg-hstore": "^2.3.2", diff --git a/scraper/lib/repository.js b/scraper/lib/repository.js index 3fe73d3..785e7c5 100644 --- a/scraper/lib/repository.js +++ b/scraper/lib/repository.js @@ -128,7 +128,7 @@ function getFiles(torrent) { } function getFilesBasedOnTitle(titleQuery) { - return File.findAll({ where: { title: { [Op.iLike]: `%${titleQuery}%` } } }); + return File.findAll({ where: { title: { [Op.regexp]: `${titleQuery}` } } }); } function deleteFile(file) { diff --git a/scraper/scrapers/thepiratebay/thepiratebay_dump_scraper.js b/scraper/scrapers/thepiratebay/thepiratebay_dump_scraper.js index 8e7ff83..e5b8605 100644 --- a/scraper/scrapers/thepiratebay/thepiratebay_dump_scraper.js +++ b/scraper/scrapers/thepiratebay/thepiratebay_dump_scraper.js @@ -19,7 +19,7 @@ async function scrape() { const lastDump = { updatedAt: 2147000000 }; //const checkPoint = moment('2016-06-17 00:00:00', 'YYYY-MMM-DD HH:mm:ss').toDate(); //const lastDump = await thepiratebay.dumps().then((dumps) => dumps.sort((a, b) => b.updatedAt - a.updatedAt)[0]); - const checkPoint = 611000; + const checkPoint = 0; if (lastDump) { console.log(`starting to scrape tpb dump: ${JSON.stringify(lastDump)}`); @@ -55,11 +55,6 @@ async function scrape() { size: parseInt(row[3], 10) }; - // if (torrent.uploadDate > checkPoint) { - // entriesProcessed++; - // return; - // } - if (!limiter.empty()) { lr.pause() } @@ -74,8 +69,6 @@ async function scrape() { console.log(err); }); lr.on('end', () => { - fs.unlink(CSV_FILE_PATH, (error) => console.warn(error)); - //repository.updateProvider({ name: NAME, lastScraped: lastDump.updatedAt }); console.log(`finished to scrape tpb dump: ${JSON.stringify(lastDump)}!`); }); } @@ -106,13 +99,13 @@ async function processTorrentRecord(record) { } const torrent = { - infoHash: record.infoHash, + infoHash: torrentFound.infoHash, provider: NAME, - torrentId: record.torrentId, + torrentId: torrentFound.torrentId, title: torrentFound.name, size: torrentFound.size, type: seriesCategories.includes(torrentFound.subcategory) ? Type.SERIES : Type.MOVIE, - imdbId: torrentFound.imdbId, + imdbId: seriesCategories.includes(torrentFound.subcategory) && torrentFound.imdbId || undefined, uploadDate: torrentFound.uploadDate || record.uploadDate, seeders: torrentFound.seeders, }; diff --git a/scraper/scrapers/thepiratebay/thepiratebay_scraper.js b/scraper/scrapers/thepiratebay/thepiratebay_scraper.js index 0ff3648..592e555 100644 --- a/scraper/scrapers/thepiratebay/thepiratebay_scraper.js +++ b/scraper/scrapers/thepiratebay/thepiratebay_scraper.js @@ -73,7 +73,7 @@ async function processTorrentRecord(record) { title: torrentFound.name.replace(/\t|\s+/g, ' '), size: torrentFound.size, type: seriesCategories.includes(torrentFound.subcategory) ? Type.SERIES : Type.MOVIE, - imdbId: torrentFound.imdbId, + imdbId: seriesCategories.includes(torrentFound.subcategory) && torrentFound.imdbId || undefined, uploadDate: torrentFound.uploadDate, seeders: torrentFound.seeders, };