add erai rss scraper

2022-02-19 13:04:47 +01:00
parent 30fb5624ea
commit 47c05c465c
7 changed files with 134 additions and 9 deletions
--- a/.github/workflows/deploy_scraper.yml
+++ b/.github/workflows/deploy_scraper.yml
@@ -40,5 +40,5 @@ jobs:
            docker load -i /tmp/docker/torrentio_scraper_latest.tar
            docker stop torrentio-scraper
            docker rm torrentio-scraper
-            docker run -p 80:7000 -d --name torrentio-scraper --restart always -e MONGODB_URI=${{ secrets.SCRAPER_MONGODB_URI }} -e DATABASE_URI=${{ secrets.DATABASE_URI }} -e ENABLE_SCHEDULING=true -e PROXY_HOST=${{ secrets.SCRAPER_PROXY_HOST }} -e PROXY_TYPE=${{ secrets.SCRAPER_PROXY_TYPE }} torrentio-scraper:latest
+            docker run -p 80:7000 -d --name torrentio-scraper --restart always -e MONGODB_URI=${{ secrets.SCRAPER_MONGODB_URI }} -e DATABASE_URI=${{ secrets.DATABASE_URI }} -e ENABLE_SCHEDULING=true -e PROXY_HOST=${{ secrets.SCRAPER_PROXY_HOST }} -e PROXY_TYPE=${{ secrets.SCRAPER_PROXY_TYPE }} -e ERAI_RSS_KEY=${{ secrets.SCRAPER_ERAI_RSS_KEY }} torrentio-scraper:latest
            docker image prune -f
--- a/addon/lib/landingTemplate.js
+++ b/addon/lib/landingTemplate.js
@@ -211,7 +211,7 @@ function landingTemplate(manifest, config = {}) {
      `<div class="contact">
         <p>Contact ${manifest.name} creator:</p>
         <a href="mailto:${manifest.contactEmail}">${manifest.contactEmail}</a>
-      </div>` : '';
+      </div>` : '<div class="separator"></div>';
  const providersHTML = Providers.options
      .map(provider => `<option value="${provider.key}">${provider.foreign || ''}${provider.label}</option>`)
      .join('\n');
--- a/package-lock.json
+++ b/package-lock.json
@@ -33,15 +33,12 @@
        "parse-torrent-title": "git://github.com/TheBeastLT/parse-torrent-title.git#8fd87b3bfc7aa04143edcd4f0112868bb77dce0f",
        "pg": "^8.7.1",
        "pg-hstore": "^2.3.4",
+        "rss-parser": "^3.12.0",
        "rutracker-api-2": "^1.10.0",
        "sequelize": "^5.21.5",
        "sugar-date": "^2.0.6",
        "torrent-stream": "^1.2.1",
        "user-agents": "^1.0.773"
-      },
-      "engines": {
-        "node": "12.x",
-        "npm": "6.x"
      }
    },
    "node_modules/@types/node": {
@@ -2882,6 +2879,23 @@
        "rimraf": "bin.js"
      }
    },
+    "node_modules/rss-parser": {
+      "version": "3.12.0",
+      "resolved": "https://registry.npmjs.org/rss-parser/-/rss-parser-3.12.0.tgz",
+      "integrity": "sha512-aqD3E8iavcCdkhVxNDIdg1nkBI17jgqF+9OqPS1orwNaOgySdpvq6B+DoONLhzjzwV8mWg37sb60e4bmLK117A==",
+      "dependencies": {
+        "entities": "^2.0.3",
+        "xml2js": "^0.4.19"
+      }
+    },
+    "node_modules/rss-parser/node_modules/entities": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz",
+      "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==",
+      "funding": {
+        "url": "https://github.com/fb55/entities?sponsor=1"
+      }
+    },
    "node_modules/run-parallel": {
      "version": "1.2.0",
      "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz",
@@ -3031,6 +3045,11 @@
        "node": ">=6"
      }
    },
+    "node_modules/sax": {
+      "version": "1.2.4",
+      "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
+      "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw=="
+    },
    "node_modules/send": {
      "version": "0.16.2",
      "resolved": "https://registry.npmjs.org/send/-/send-0.16.2.tgz",
@@ -3853,6 +3872,26 @@
        "node": ">=8.3.0"
      }
    },
+    "node_modules/xml2js": {
+      "version": "0.4.23",
+      "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz",
+      "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==",
+      "dependencies": {
+        "sax": ">=0.6.0",
+        "xmlbuilder": "~11.0.0"
+      },
+      "engines": {
+        "node": ">=4.0.0"
+      }
+    },
+    "node_modules/xmlbuilder": {
+      "version": "11.0.1",
+      "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz",
+      "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==",
+      "engines": {
+        "node": ">=4.0"
+      }
+    },
    "node_modules/xtend": {
      "version": "4.0.1",
      "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz",
@@ -6333,6 +6372,22 @@
        "glob": "7.1.7"
      }
    },
+    "rss-parser": {
+      "version": "3.12.0",
+      "resolved": "https://registry.npmjs.org/rss-parser/-/rss-parser-3.12.0.tgz",
+      "integrity": "sha512-aqD3E8iavcCdkhVxNDIdg1nkBI17jgqF+9OqPS1orwNaOgySdpvq6B+DoONLhzjzwV8mWg37sb60e4bmLK117A==",
+      "requires": {
+        "entities": "^2.0.3",
+        "xml2js": "^0.4.19"
+      },
+      "dependencies": {
+        "entities": {
+          "version": "2.2.0",
+          "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz",
+          "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A=="
+        }
+      }
+    },
    "run-parallel": {
      "version": "1.2.0",
      "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz",
@@ -6477,6 +6532,11 @@
        "sparse-bitfield": "3.0.3"
      }
    },
+    "sax": {
+      "version": "1.2.4",
+      "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
+      "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw=="
+    },
    "send": {
      "version": "0.16.2",
      "resolved": "https://registry.npmjs.org/send/-/send-0.16.2.tgz",
@@ -7209,6 +7269,20 @@
      "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.5.tgz",
      "integrity": "sha512-BAkMFcAzl8as1G/hArkxOxq3G7pjUqQ3gzYbLL0/5zNkph70e+lCoxBGnm6AW1+/aiNeV4fnKqZ8m4GZewmH2w=="
    },
+    "xml2js": {
+      "version": "0.4.23",
+      "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz",
+      "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==",
+      "requires": {
+        "sax": ">=0.6.0",
+        "xmlbuilder": "~11.0.0"
+      }
+    },
+    "xmlbuilder": {
+      "version": "11.0.1",
+      "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz",
+      "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA=="
+    },
    "xtend": {
      "version": "4.0.1",
      "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz",
--- a/package.json
+++ b/package.json
@@ -32,6 +32,7 @@
    "parse-torrent-title": "git://github.com/TheBeastLT/parse-torrent-title.git#8fd87b3bfc7aa04143edcd4f0112868bb77dce0f",
    "pg": "^8.7.1",
    "pg-hstore": "^2.3.4",
+    "rss-parser": "^3.12.0",
    "rutracker-api-2": "^1.10.0",
    "sequelize": "^5.21.5",
    "sugar-date": "^2.0.6",
--- a/scraper/scheduler/scrapers.js
+++ b/scraper/scheduler/scrapers.js
@@ -37,7 +37,7 @@ module.exports = [
  { scraper: OndeBaixa, name: OndeBaixa.NAME, cron: '0 0 */4 ? * *' },
  // { scraper: AnimesTorrent, name: AnimesTorrent.NAME, cron: '0 0 */4 ? * *' },
  // { scraper: DarkMahou, name: DarkMahou.NAME, cron: '0 0 */4 ? * *' },
-  { scraper: erairawsScraper, name: erairawsScraper.NAME, cron: '0 0 */24 ? * *' },
+  { scraper: erairawsScraper, name: erairawsScraper.NAME, cron: '0 0 */4 ? * *' },
  // { scraper: require('../scrapers/rarbg/rarbg_dump_scraper') }
  // { scraper: require('../scrapers/1337x/1337x_search_scraper') }
  // { scraper: require('../scrapers/rarbg/rarbg_dump_scraper') }
--- a/scraper/scrapers/erairaws/erairaws_rss_api.js
+++ b/scraper/scrapers/erairaws/erairaws_rss_api.js
@@ -0,0 +1,50 @@
+const Parser = require('rss-parser');
+const decode = require("magnet-uri");
+
+const parser = new Parser({
+  customFields: {
+    item: [['erai:subtitles', 'subtitles']]
+  }
+});
+const baseUrl = 'https://www.erai-raws.info';
+const rssKey = process.env.ERAI_RSS_KEY;
+
+const Categories = {
+  ANIMES: 'anime',
+  EPISODES: 'episodes'
+};
+
+function browse() {
+  return parser.parseURL(`${baseUrl}/feed/?type=magnet&${rssKey}`)
+      .then(result => result.items
+          .map(item => {
+            const decodedMagnet = decode(item.link);
+            const languages = parseLanguages(item.subtitles);
+            return {
+              title: decodedMagnet.name,
+              infoHash: decodedMagnet.infoHash,
+              trackers: decodedMagnet.tr,
+              languages: languages
+            }
+          }));
+}
+
+const languageMapping = {
+  'us': 'English',
+  'br': 'Portuguese(Brazil)',
+  'mx': 'Spanish(Latin_America)',
+  'es': 'Spanish',
+  'sa': 'Arabic',
+  'fr': 'French',
+  'de': 'German',
+  'it': 'Italian',
+  'ru': 'Russian'
+}
+function parseLanguages(languages) {
+    return languages.split('][')
+        .map(lang => lang.replace(/[\[\]]/g, ''))
+        .map(lang => languageMapping[lang] || lang)
+        .join('/');
+}
+
+module.exports = { browse, Categories };
--- a/scraper/scrapers/erairaws/erairaws_scraper.js
+++ b/scraper/scrapers/erairaws/erairaws_scraper.js
@@ -1,6 +1,6 @@
 const moment = require('moment');
 const Bottleneck = require('bottleneck');
-const erairaws = require('./erairaws_api');
+const erairaws = require('./erairaws_rss_api');
 const { checkAndUpdateTorrent } = require('../../lib/torrentEntries');

 const NAME = 'EraiRaws';
@@ -41,7 +41,7 @@ function untilPage(category) {
  if (category === erairaws.Categories.ANIMES) {
    return 45;
  }
-  return 3;
+  return 1;
 }

 module.exports = { scrape, NAME };