Big rewrite - distributed consumers for ingestion / scraping(scalable) - single producer written in c#.

Changed from page scraping to rss xml scraping
Includes RealDebridManager hashlist decoding (requires a github readonly PAT as requests must be authenticated) - This allows ingestion of 200k+ entries in a few hours.
Simplifies a lot of torrentio to deal with new data
This commit is contained in:
iPromKnight
2024-02-01 16:38:45 +00:00
parent 6fb4ddcf23
commit ab17ef81be
255 changed files with 18489 additions and 69074 deletions

View File

@@ -0,0 +1,34 @@
{
"name": "consumer",
"version": "1.0.0",
"exports": "./index.js",
"type": "module",
"scripts": {
"start": "node index.js",
"lint": "eslint . --ext .js"
},
"author": "A Dude",
"license": "MIT",
"dependencies": {
"amqplib": "^0.10.3",
"axios": "^1.6.1",
"bluebird": "^3.7.2",
"bottleneck": "^2.19.5",
"cache-manager": "^3.4.4",
"cache-manager-mongodb": "^0.3.0",
"google-sr": "^3.2.1",
"jaro-winkler": "^0.2.8",
"magnet-uri": "^6.2.0",
"moment": "^2.30.1",
"name-to-imdb": "^3.0.4",
"parse-torrent-title": "git://github.com/TheBeastLT/parse-torrent-title.git#022408972c2a040f846331a912a6a8487746a654",
"pg": "^8.11.3",
"sequelize": "^6.31.1",
"torrent-stream": "^1.2.1",
"user-agents": "^1.0.1444"
},
"devDependencies": {
"eslint": "^8.56.0",
"eslint-plugin-import": "^2.29.1"
}
}