tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] improves title based decomposer

Browse Source
This commit is contained in:
TheBeastLT
2020-03-13 14:33:34 +01:00
parent a47495281d
commit 2b73b56106
4 changed files with 21 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
addon/package.json
View File

@@ -3,8 +3,7 @@
"version": "1.0.0", "version": "1.0.0",
"main": "addon.js", "main": "addon.js",
"scripts": { "scripts": {
"start": "node index.js", "start": "node index.js"
"deploy": "now && now alias"
}, },
"author": "TheBeastLT <pauliox@beyond.lt>", "author": "TheBeastLT <pauliox@beyond.lt>",
"license": "MIT", "license": "MIT",
@@ -13,10 +12,10 @@
"cache-manager-mongodb": "^0.2.1", "cache-manager-mongodb": "^0.2.1",
"express-rate-limit": "^5.1.1", "express-rate-limit": "^5.1.1",
"parse-torrent-title": "git://github.com/TheBeastLT/parse-torrent-title.git#master", "parse-torrent-title": "git://github.com/TheBeastLT/parse-torrent-title.git#master",
"stremio-addon-sdk": "^1.6.1",
"pg": "^7.8.2", "pg": "^7.8.2",
"pg-hstore": "^2.3.2", "pg-hstore": "^2.3.2",
"sequelize": "^4.43.0" "sequelize": "^4.43.0",
"stremio-addon-sdk": "^1.6.1"
}, },
"devDependencies": { "devDependencies": {
"eslint": "^6.4.0", "eslint": "^6.4.0",

5
package-lock.json generated
View File

@@ -924,6 +924,11 @@
"resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
"integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo="
}, },
"jaro-winkler": {
"version": "0.2.8",
"resolved": "https://registry.npmjs.org/jaro-winkler/-/jaro-winkler-0.2.8.tgz",
"integrity": "sha1-Zyfg0LcJHiQ2+TVt6b+I+tI+U0o="
},
"jsbn": { "jsbn": {
"version": "0.1.1", "version": "0.1.1",
"resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",

1
package.json
View File

@@ -19,6 +19,7 @@
"cheerio": "^0.22.0", "cheerio": "^0.22.0",
"dotenv": "^8.2.0", "dotenv": "^8.2.0",
"express": "^4.16.4", "express": "^4.16.4",
"jaro-winkler": "^0.2.8",
"line-by-line": "^0.1.6", "line-by-line": "^0.1.6",
"magnet-uri": "^5.1.7", "magnet-uri": "^5.1.7",
"moment": "^2.24.0", "moment": "^2.24.0",

23
scraper/lib/torrentFiles.js
View File

@@ -1,4 +1,5 @@
const moment = require('moment'); const moment = require('moment');
const distance = require('jaro-winkler');
const { parse } = require('parse-torrent-title'); const { parse } = require('parse-torrent-title');
const { torrentFiles } = require('../lib/torrent'); const { torrentFiles } = require('../lib/torrent');
const { getMetadata, getImdbId } = require('../lib/metadata'); const { getMetadata, getImdbId } = require('../lib/metadata');
@@ -61,7 +62,8 @@ async function parseTorrentFiles(torrent) {
} }
async function getSeriesFiles(torrent, parsedTorrentName) { async function getSeriesFiles(torrent, parsedTorrentName) {
if (parsedTorrentName.episode || (!parsedTorrentName.episodes && parsedTorrentName.date)) { if ((parsedTorrentName.episode && (!parsedTorrentName.seasons || parsedTorrentName.seasons.length <= 1)) ||
(!parsedTorrentName.episodes && parsedTorrentName.date)) {
return [{ return [{
name: torrent.title, name: torrent.title,
path: torrent.title, path: torrent.title,
@@ -160,6 +162,7 @@ async function decomposeEpisodes(torrent, files, metadata = { episodeCount: [] }
.every(ep => metadata.episodeCount[file.season - 1] < ep)).length > Math.ceil(files.length / 5)) { .every(ep => metadata.episodeCount[file.season - 1] < ep)).length > Math.ceil(files.length / 5)) {
decomposeAbsoluteEpisodeFiles(torrent, files, metadata); decomposeAbsoluteEpisodeFiles(torrent, files, metadata);
} }
// decomposeEpisodeTitleFiles(torrent, files, metadata);
return files; return files;
} }
@@ -228,19 +231,17 @@ function decomposeDateEpisodeFiles(torrent, files, metadata) {
} }
function decomposeEpisodeTitleFiles(torrent, files, metadata) { function decomposeEpisodeTitleFiles(torrent, files, metadata) {
const titleMapping = metadata.videos
.reduce((map, video) => {
map[video.name.toLowerCase()] = video;
return map;
}, {});
files files
.filter(file => !file.season) .filter(file => !file.season)
.map(file => { .map(file => {
const episodeTitle = file.name.replace(/^.*-\s?(.+)\.\w{1,4}$/, '$1').toLowerCase(); const episodeTitle = file.name.replace(/^.*-\s?(.+)\.\w{1,4}$/, '$1').trim();
const mapping = titleMapping[episodeTitle]; const foundEpisode = metadata.videos
if (mapping) { .map(video => ({ ...video, distance: distance(episodeTitle, video.name) }))
file.season = mapping.season; .sort((a, b) => b.distance - a.distance)[0];
file.episodes = [mapping.episode]; if (foundEpisode) {
file.isMovie = false;
file.season = foundEpisode.season;
file.episodes = [foundEpisode.episode];
} }
}) })
} }