diff --git a/.github/workflows/deploy_scraper.yml b/.github/workflows/deploy_scraper.yml index ee2293e..a031ed1 100644 --- a/.github/workflows/deploy_scraper.yml +++ b/.github/workflows/deploy_scraper.yml @@ -40,5 +40,5 @@ jobs: docker load -i /tmp/docker/torrentio_scraper_latest.tar docker stop torrentio-scraper docker rm torrentio-scraper - docker run -p 80:7000 -d --name torrentio-scraper --restart always -e MONGODB_URI=${{ secrets.SCRAPER_MONGODB_URI }} -e DATABASE_URI=${{ secrets.DATABASE_URI }} -e ENABLE_SCHEDULING=true -e PROXY_HOST=${{ secrets.SCRAPER_PROXY_HOST }} -e PROXY_TYPE=${{ secrets.SCRAPER_PROXY_TYPE }} torrentio-scraper:latest + docker run -p 80:7000 -d --name torrentio-scraper --restart always -e MONGODB_URI=${{ secrets.SCRAPER_MONGODB_URI }} -e DATABASE_URI=${{ secrets.DATABASE_URI }} -e ENABLE_SCHEDULING=true -e PROXY_HOST=${{ secrets.SCRAPER_PROXY_HOST }} -e PROXY_TYPE=${{ secrets.SCRAPER_PROXY_TYPE }} -e ERAI_RSS_KEY=${{ secrets.SCRAPER_ERAI_RSS_KEY }} torrentio-scraper:latest docker image prune -f diff --git a/addon/lib/landingTemplate.js b/addon/lib/landingTemplate.js index dfc6462..811d1e5 100644 --- a/addon/lib/landingTemplate.js +++ b/addon/lib/landingTemplate.js @@ -211,7 +211,7 @@ function landingTemplate(manifest, config = {}) { `

Contact ${manifest.name} creator:

${manifest.contactEmail} -
` : ''; + ` : '
'; const providersHTML = Providers.options .map(provider => ``) .join('\n'); diff --git a/package-lock.json b/package-lock.json index 971d611..1b66d9d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -33,15 +33,12 @@ "parse-torrent-title": "git://github.com/TheBeastLT/parse-torrent-title.git#8fd87b3bfc7aa04143edcd4f0112868bb77dce0f", "pg": "^8.7.1", "pg-hstore": "^2.3.4", + "rss-parser": "^3.12.0", "rutracker-api-2": "^1.10.0", "sequelize": "^5.21.5", "sugar-date": "^2.0.6", "torrent-stream": "^1.2.1", "user-agents": "^1.0.773" - }, - "engines": { - "node": "12.x", - "npm": "6.x" } }, "node_modules/@types/node": { @@ -2882,6 +2879,23 @@ "rimraf": "bin.js" } }, + "node_modules/rss-parser": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/rss-parser/-/rss-parser-3.12.0.tgz", + "integrity": "sha512-aqD3E8iavcCdkhVxNDIdg1nkBI17jgqF+9OqPS1orwNaOgySdpvq6B+DoONLhzjzwV8mWg37sb60e4bmLK117A==", + "dependencies": { + "entities": "^2.0.3", + "xml2js": "^0.4.19" + } + }, + "node_modules/rss-parser/node_modules/entities": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz", + "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==", + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", @@ -3031,6 +3045,11 @@ "node": ">=6" } }, + "node_modules/sax": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" + }, "node_modules/send": { "version": "0.16.2", "resolved": "https://registry.npmjs.org/send/-/send-0.16.2.tgz", @@ -3853,6 +3872,26 @@ "node": ">=8.3.0" } }, + "node_modules/xml2js": { + "version": "0.4.23", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz", + "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==", + "dependencies": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", + "engines": { + "node": ">=4.0" + } + }, "node_modules/xtend": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", @@ -6333,6 +6372,22 @@ "glob": "7.1.7" } }, + "rss-parser": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/rss-parser/-/rss-parser-3.12.0.tgz", + "integrity": "sha512-aqD3E8iavcCdkhVxNDIdg1nkBI17jgqF+9OqPS1orwNaOgySdpvq6B+DoONLhzjzwV8mWg37sb60e4bmLK117A==", + "requires": { + "entities": "^2.0.3", + "xml2js": "^0.4.19" + }, + "dependencies": { + "entities": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz", + "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==" + } + } + }, "run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", @@ -6477,6 +6532,11 @@ "sparse-bitfield": "3.0.3" } }, + "sax": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" + }, "send": { "version": "0.16.2", "resolved": "https://registry.npmjs.org/send/-/send-0.16.2.tgz", @@ -7209,6 +7269,20 @@ "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.5.tgz", "integrity": "sha512-BAkMFcAzl8as1G/hArkxOxq3G7pjUqQ3gzYbLL0/5zNkph70e+lCoxBGnm6AW1+/aiNeV4fnKqZ8m4GZewmH2w==" }, + "xml2js": { + "version": "0.4.23", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz", + "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==", + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + } + }, + "xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==" + }, "xtend": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", diff --git a/package.json b/package.json index d077f1d..ca5b172 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "parse-torrent-title": "git://github.com/TheBeastLT/parse-torrent-title.git#8fd87b3bfc7aa04143edcd4f0112868bb77dce0f", "pg": "^8.7.1", "pg-hstore": "^2.3.4", + "rss-parser": "^3.12.0", "rutracker-api-2": "^1.10.0", "sequelize": "^5.21.5", "sugar-date": "^2.0.6", diff --git a/scraper/scheduler/scrapers.js b/scraper/scheduler/scrapers.js index cf931fa..aba31f6 100644 --- a/scraper/scheduler/scrapers.js +++ b/scraper/scheduler/scrapers.js @@ -37,7 +37,7 @@ module.exports = [ { scraper: OndeBaixa, name: OndeBaixa.NAME, cron: '0 0 */4 ? * *' }, // { scraper: AnimesTorrent, name: AnimesTorrent.NAME, cron: '0 0 */4 ? * *' }, // { scraper: DarkMahou, name: DarkMahou.NAME, cron: '0 0 */4 ? * *' }, - { scraper: erairawsScraper, name: erairawsScraper.NAME, cron: '0 0 */24 ? * *' }, + { scraper: erairawsScraper, name: erairawsScraper.NAME, cron: '0 0 */4 ? * *' }, // { scraper: require('../scrapers/rarbg/rarbg_dump_scraper') } // { scraper: require('../scrapers/1337x/1337x_search_scraper') } // { scraper: require('../scrapers/rarbg/rarbg_dump_scraper') } diff --git a/scraper/scrapers/erairaws/erairaws_rss_api.js b/scraper/scrapers/erairaws/erairaws_rss_api.js new file mode 100644 index 0000000..98c5745 --- /dev/null +++ b/scraper/scrapers/erairaws/erairaws_rss_api.js @@ -0,0 +1,50 @@ +const Parser = require('rss-parser'); +const decode = require("magnet-uri"); + +const parser = new Parser({ + customFields: { + item: [['erai:subtitles', 'subtitles']] + } +}); +const baseUrl = 'https://www.erai-raws.info'; +const rssKey = process.env.ERAI_RSS_KEY; + +const Categories = { + ANIMES: 'anime', + EPISODES: 'episodes' +}; + +function browse() { + return parser.parseURL(`${baseUrl}/feed/?type=magnet&${rssKey}`) + .then(result => result.items + .map(item => { + const decodedMagnet = decode(item.link); + const languages = parseLanguages(item.subtitles); + return { + title: decodedMagnet.name, + infoHash: decodedMagnet.infoHash, + trackers: decodedMagnet.tr, + languages: languages + } + })); +} + +const languageMapping = { + 'us': 'English', + 'br': 'Portuguese(Brazil)', + 'mx': 'Spanish(Latin_America)', + 'es': 'Spanish', + 'sa': 'Arabic', + 'fr': 'French', + 'de': 'German', + 'it': 'Italian', + 'ru': 'Russian' +} +function parseLanguages(languages) { + return languages.split('][') + .map(lang => lang.replace(/[\[\]]/g, '')) + .map(lang => languageMapping[lang] || lang) + .join('/'); +} + +module.exports = { browse, Categories }; diff --git a/scraper/scrapers/erairaws/erairaws_scraper.js b/scraper/scrapers/erairaws/erairaws_scraper.js index 3db4474..20c454a 100644 --- a/scraper/scrapers/erairaws/erairaws_scraper.js +++ b/scraper/scrapers/erairaws/erairaws_scraper.js @@ -1,6 +1,6 @@ const moment = require('moment'); const Bottleneck = require('bottleneck'); -const erairaws = require('./erairaws_api'); +const erairaws = require('./erairaws_rss_api'); const { checkAndUpdateTorrent } = require('../../lib/torrentEntries'); const NAME = 'EraiRaws'; @@ -41,7 +41,7 @@ function untilPage(category) { if (category === erairaws.Categories.ANIMES) { return 45; } - return 3; + return 1; } module.exports = { scrape, NAME };