tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] date handling

Browse Source
This commit is contained in:
TheBeastLT
2020-03-13 15:46:06 +01:00
parent 2b73b56106
commit 21338b3aee
4 changed files with 14 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
scraper/lib/repository.js
View File

@@ -15,8 +15,8 @@ const Torrent = database.define('torrent',
{ {
infoHash: { type: Sequelize.STRING(64), primaryKey: true }, infoHash: { type: Sequelize.STRING(64), primaryKey: true },
provider: { type: Sequelize.STRING(32), allowNull: false }, provider: { type: Sequelize.STRING(32), allowNull: false },
torrentId: { type: Sequelize.STRING(128) }, torrentId: { type: Sequelize.STRING(512) },
title: { type: Sequelize.STRING(256), allowNull: false }, title: { type: Sequelize.STRING(512), allowNull: false },
size: { type: Sequelize.BIGINT }, size: { type: Sequelize.BIGINT },
type: { type: Sequelize.STRING(16), allowNull: false }, type: { type: Sequelize.STRING(16), allowNull: false },
uploadDate: { type: Sequelize.DATE, allowNull: false }, uploadDate: { type: Sequelize.DATE, allowNull: false },
@@ -37,7 +37,7 @@ const File = database.define('file',
onDelete: 'CASCADE' onDelete: 'CASCADE'
}, },
fileIndex: { type: Sequelize.INTEGER }, fileIndex: { type: Sequelize.INTEGER },
title: { type: Sequelize.STRING(256), allowNull: false }, title: { type: Sequelize.STRING(512), allowNull: false },
size: { type: Sequelize.BIGINT }, size: { type: Sequelize.BIGINT },
imdbId: { type: Sequelize.STRING(32) }, imdbId: { type: Sequelize.STRING(32) },
imdbSeason: { type: Sequelize.INTEGER }, imdbSeason: { type: Sequelize.INTEGER },
@@ -67,7 +67,9 @@ function connect() {
} }
function getProvider(provider) { function getProvider(provider) {
return Provider.findOrCreate({ where: { name: { [Op.eq]: provider.name } }, defaults: provider }); return Provider.findOrCreate({ where: { name: { [Op.eq]: provider.name } }, defaults: provider })
.then((result) => result[0])
.catch(() => provider);
} }
function updateProvider(provider) { function updateProvider(provider) {

2
scraper/lib/torrentFiles.js
View File

@@ -122,7 +122,7 @@ function parseSeriesFile(file, parsedTorrentName) {
fileInfo.season = pathInfo.season; fileInfo.season = pathInfo.season;
} }
// force episode to any found number if it was not parsed // force episode to any found number if it was not parsed
if (!fileInfo.episodes) { if (!fileInfo.episodes && !fileInfo.date) {
const epMatcher = fileInfo.title.match(/(?<!movie\W*|film\W*)(?:^|\W)(\d{1,4})(?:a|b|v\d)?(?:\W|$)(?!movie|film)/i); const epMatcher = fileInfo.title.match(/(?<!movie\W*|film\W*)(?:^|\W)(\d{1,4})(?:a|b|v\d)?(?:\W|$)(?!movie|film)/i);
fileInfo.episodes = epMatcher && [parseInt(epMatcher[1], 10)]; fileInfo.episodes = epMatcher && [parseInt(epMatcher[1], 10)];
fileInfo.episode = fileInfo.episodes && fileInfo.episodes[0]; fileInfo.episode = fileInfo.episodes && fileInfo.episodes[0];

3
scraper/manual/manual.js
View File

@@ -83,6 +83,9 @@ async function reapplyEpisodeDecomposing(infoHash, includeSourceFiles = true) {
if (fileIndexMap[file.fileIndex]) { if (fileIndexMap[file.fileIndex]) {
const originalFile = fileIndexMap[file.fileIndex].shift(); const originalFile = fileIndexMap[file.fileIndex].shift();
if (originalFile) { if (originalFile) {
if (!originalFile.imdbId) {
originalFile.imdbId = file.imdbId
}
originalFile.imdbSeason = file.imdbSeason; originalFile.imdbSeason = file.imdbSeason;
originalFile.imdbEpisode = file.imdbEpisode; originalFile.imdbEpisode = file.imdbEpisode;
originalFile.kitsuId = file.kitsuId; originalFile.kitsuId = file.kitsuId;

4
scraper/scrapers/1337x/1337x_scraper.js
View File

@@ -61,6 +61,10 @@ async function processTorrentRecord(record) {
if (!torrentFound || !TYPE_MAPPING[torrentFound.category]) { if (!torrentFound || !TYPE_MAPPING[torrentFound.category]) {
return createSkipTorrentEntry(record); return createSkipTorrentEntry(record);
} }
if (isNaN(torrentFound.uploadDate)) {
console.warn(`Incorrect upload date for [${torrentFound.infoHash}] ${torrentFound.name}`);
return;
}
const torrent = { const torrent = {
infoHash: torrentFound.infoHash, infoHash: torrentFound.infoHash,