diff --git a/README.md b/README.md index a1237c6..4dc31b0 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,26 @@ -# Selfhostio +# Knight Crawler +isolated A self-hosted Stremio addon for streaming torrents via a debrid service. ## Contents -- [Selfhostio](#selfhostio) +**Note: Until we reach `v1.0.0`, please consider releases as alpha.** + +**Important: The latest change renames the project and requires a [small migration](#selfhostio-to-knightcrawler-migration).** +- [Knight Crawler](#knight-crawler) - [Contents](#contents) - [Overview](#overview) - [Using](#using) - [Initial setup (optional)](#initial-setup-optional) - [Run the project](#run-the-project) - [Monitoring with Grafana and Prometheus (Optional)](#monitoring-with-grafana-and-prometheus-optional) + - [Accessing RabbitMQ Management](#accessing-rabbitmq-management) + - [Using Grafana and Prometheus](#using-grafana-and-prometheus) - [Importing external dumps](#importing-external-dumps) - [Import data into database](#import-data-into-database) - [INSERT INTO ingested\_torrents](#insert-into-ingested_torrents) + - [Selfhostio to KnightCrawler Migration](#selfhostio-to-knightcrawler-migration) - [To-do](#to-do) @@ -39,7 +46,7 @@ We can search DebridMediaManager hash lists which are hosted on GitHub. This all 3. Fill out the form (example data below): ``` Token name: - Selfhostio + KnightCrawler Expiration: 90 days Description: @@ -112,7 +119,7 @@ I created a file `db.load` as follows.. ``` load database from sqlite:///tmp/rarbg_db.sqlite - into postgresql://postgres:postgres@localhost/selfhostio + into postgresql://postgres:postgres@localhost/knightcrawler with include drop, create tables, create indexes, reset sequences @@ -131,6 +138,18 @@ SELECT title, 'RARBG', cat, hash, size, NULL, NULL, imdb, false, current_timesta FROM items where cat='tv' OR cat='movies'; ``` +## Selfhostio to KnightCrawler Migration + +With the renaming of the project, you will have to change your database name in order to keep your existing data. + +**With your existing stack still running**, run: +``` +docker exec -it torrentio-selfhostio-postgres-1 psql -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid() AND datname = 'selfhostio'; ALTER DATABASE selfhostio RENAME TO knightcrawler;" +``` +Make sure your postgres container is named `torrentio-selfhostio-postgres-1`, otherwise, adjust accordingly. + +This command should return: `ALTER DATABASE`. This means your database is now renamed. You can now pull the new changes if you haven't already and run `docker-compose up -d`. + ## To-do - [ ] Add a section on external access diff --git a/docker-compose-metrics.yml b/docker-compose-metrics.yml index d944246..27f58b9 100644 --- a/docker-compose-metrics.yml +++ b/docker-compose-metrics.yml @@ -1,5 +1,5 @@ version: '3.8' -name: torrentio-metrics +name: knightcrawler-metrics services: prometheus: @@ -11,7 +11,7 @@ services: ports: - "9090:9090" networks: - - torrentio-network + - knightcrawler-network grafana: image: grafana/grafana:latest @@ -25,17 +25,17 @@ services: depends_on: - prometheus networks: - - torrentio-network + - knightcrawler-network postgres-exporter: image: prometheuscommunity/postgres-exporter ports: - "9187:9187" environment: - DATA_SOURCE_NAME: "postgresql://postgres:postgres@postgres:5432/selfhostio?sslmode=disable" + DATA_SOURCE_NAME: "postgresql://postgres:postgres@postgres:5432/knightcrawler?sslmode=disable" networks: - - torrentio-network + - knightcrawler-network networks: - torrentio-network: + knightcrawler-network: external: true diff --git a/docker-compose.yaml b/docker-compose.yaml index 772800d..be018e5 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,5 +1,5 @@ version: '3.8' -name: torrentio-selfhostio +name: knightcrawler x-restart: &restart-policy "unless-stopped" @@ -22,7 +22,7 @@ x-postgreshealth: &postgresdb-health test: pg_isready <<: *base-health -x-apps: &selfhostio-app +x-apps: &knightcrawler-app depends_on: mongodb: condition: service_healthy @@ -38,7 +38,7 @@ services: environment: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres - POSTGRES_DB: selfhostio + POSTGRES_DB: knightcrawler PGUSER: postgres # needed for healthcheck. ports: - "5432:5432" @@ -47,7 +47,7 @@ services: healthcheck: *postgresdb-health restart: *restart-policy networks: - - torrentio-network + - knightcrawler-network mongodb: image: mongo:latest @@ -61,7 +61,7 @@ services: restart: *restart-policy healthcheck: *mongodb-health networks: - - torrentio-network + - knightcrawler-network rabbitmq: image: rabbitmq:3-management @@ -74,7 +74,7 @@ services: restart: *restart-policy healthcheck: *rabbitmq-health networks: - - torrentio-network + - knightcrawler-network producer: build: @@ -82,9 +82,9 @@ services: dockerfile: Dockerfile env_file: - env/producer.env - <<: *selfhostio-app + <<: *knightcrawler-app networks: - - torrentio-network + - knightcrawler-network consumer: build: @@ -94,9 +94,9 @@ services: - env/consumer.env deploy: replicas: 3 - <<: *selfhostio-app + <<: *knightcrawler-app networks: - - torrentio-network + - knightcrawler-network addon: build: @@ -106,14 +106,14 @@ services: - "7000:7000" env_file: - env/addon.env - <<: *selfhostio-app + <<: *knightcrawler-app networks: - - torrentio-network + - knightcrawler-network networks: - torrentio-network: + knightcrawler-network: driver: bridge - name: torrentio-network + name: knightcrawler-network volumes: postgres: diff --git a/env/addon.env b/env/addon.env index 5bd2f3c..5b5fb6b 100644 --- a/env/addon.env +++ b/env/addon.env @@ -1,4 +1,4 @@ TZ=London/Europe -DATABASE_URI=postgres://postgres:postgres@postgres/selfhostio -MONGODB_URI=mongodb://mongo:mongo@mongodb/selfhostio?tls=false&authSource=admin +DATABASE_URI=postgres://postgres:postgres@postgres/knightcrawler +MONGODB_URI=mongodb://mongo:mongo@mongodb/knightcrawler?tls=false&authSource=admin DEBUG_MODE=false \ No newline at end of file diff --git a/env/consumer.env b/env/consumer.env index 3ea17a8..555a938 100644 --- a/env/consumer.env +++ b/env/consumer.env @@ -1,6 +1,6 @@ TZ=London/Europe -MONGODB_URI=mongodb://mongo:mongo@mongodb/selfhostio?tls=false&authSource=admin -DATABASE_URI=postgres://postgres:postgres@postgres/selfhostio +MONGODB_URI=mongodb://mongo:mongo@mongodb/knightcrawler?tls=false&authSource=admin +DATABASE_URI=postgres://postgres:postgres@postgres/knightcrawler RABBIT_URI=amqp://guest:guest@rabbitmq:5672/?heartbeat=30 QUEUE_NAME=ingested JOB_CONCURRENCY=5 diff --git a/env/producer.env b/env/producer.env index 4d6a52f..d9d8ba9 100644 --- a/env/producer.env +++ b/env/producer.env @@ -1,4 +1,4 @@ -ScrapeConfiguration__StorageConnectionString=host=postgres;username=postgres;password=postgres;database=selfhostio; +ScrapeConfiguration__StorageConnectionString=host=postgres;username=postgres;password=postgres;database=knightcrawler; RabbitMqConfiguration__Host=rabbitmq RabbitMqConfiguration__QueueName=ingested RabbitMqConfiguration__Username=guest diff --git a/src/node/addon-jackett/src/moch/alldebrid.js b/src/node/addon-jackett/src/moch/alldebrid.js index 6af313a..ac431f0 100644 --- a/src/node/addon-jackett/src/moch/alldebrid.js +++ b/src/node/addon-jackett/src/moch/alldebrid.js @@ -6,7 +6,7 @@ import { BadTokenError, AccessDeniedError, sameFilename } from './mochHelper.js' import StaticResponse from './static.js'; const KEY = 'alldebrid'; -const AGENT = 'torrentio'; +const AGENT = 'knightcrawler'; export async function getCachedStreams(streams, apiKey) { const options = await getDefaultOptions(); diff --git a/src/node/addon-jackett/src/moch/moch.js b/src/node/addon-jackett/src/moch/moch.js index c9e66dd..7da07ec 100644 --- a/src/node/addon-jackett/src/moch/moch.js +++ b/src/node/addon-jackett/src/moch/moch.js @@ -224,14 +224,14 @@ function blackListToken(token, mochKey) { function errorStreamResponse(mochKey, error, config) { if (error === BadTokenError) { return { - name: `Torrentio\n${MochOptions[mochKey].shortName} error`, + name: `Knightcrawler\n${MochOptions[mochKey].shortName} error`, title: `Invalid ${MochOptions[mochKey].name} ApiKey/Token!`, url: `${config.host}/${StaticResponse.FAILED_ACCESS}` }; } if (error === AccessDeniedError) { return { - name: `Torrentio\n${MochOptions[mochKey].shortName} error`, + name: `Knightcrawler\n${MochOptions[mochKey].shortName} error`, title: `Expired/invalid ${MochOptions[mochKey].name} subscription!`, url: `${config.host}/${StaticResponse.FAILED_ACCESS}` }; diff --git a/src/node/addon/ecosystem.config.cjs b/src/node/addon/ecosystem.config.cjs index dff410d..4becc22 100644 --- a/src/node/addon/ecosystem.config.cjs +++ b/src/node/addon/ecosystem.config.cjs @@ -1,7 +1,7 @@ module.exports = { apps: [ { - name: "torrentio-selfhostio", + name: "knightcrawler", script: "npm start", cwd: "/app", watch: ["./dist/index.cjs"], diff --git a/src/node/addon/package-lock.json b/src/node/addon/package-lock.json index 6b64666..32cb4a5 100644 --- a/src/node/addon/package-lock.json +++ b/src/node/addon/package-lock.json @@ -1,11 +1,11 @@ { - "name": "selfhostio-addon", + "name": "knightcrawler-addon", "version": "0.0.1", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "selfhostio-addon", + "name": "knightcrawler-addon", "version": "0.0.1", "dependencies": { "@putdotio/api-client": "^8.42.0", diff --git a/src/node/addon/package.json b/src/node/addon/package.json index 0d2d0a7..d062dd2 100644 --- a/src/node/addon/package.json +++ b/src/node/addon/package.json @@ -1,5 +1,5 @@ { - "name": "selfhostio-addon", + "name": "knightcrawler-addon", "version": "0.0.1", "type": "module", "scripts": { diff --git a/src/node/addon/src/addon.js b/src/node/addon/src/addon.js index 85a2f07..92baf7a 100644 --- a/src/node/addon/src/addon.js +++ b/src/node/addon/src/addon.js @@ -44,7 +44,7 @@ builder.defineStreamHandler((args) => { builder.defineCatalogHandler((args) => { - const mochKey = args.id.replace("selfhostio-", ''); + const mochKey = args.id.replace("knightcrawler-", ''); console.log(`Incoming catalog ${args.id} request with skip=${args.extra.skip || 0}`) return getMochCatalog(mochKey, args.extra) .then(metas => ({ diff --git a/src/node/addon/src/lib/cache.js b/src/node/addon/src/lib/cache.js index 00400c7..1c6c438 100644 --- a/src/node/addon/src/lib/cache.js +++ b/src/node/addon/src/lib/cache.js @@ -2,7 +2,7 @@ import cacheManager from 'cache-manager'; import mangodbStore from 'cache-manager-mongodb'; import { isStaticUrl } from '../moch/static.js'; -const GLOBAL_KEY_PREFIX = 'selfhostio-addon'; +const GLOBAL_KEY_PREFIX = 'knightcrawler-addon'; const STREAM_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|stream`; const AVAILABILITY_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|availability`; const RESOLVED_URL_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|resolved`; @@ -28,7 +28,7 @@ function initiateRemoteCache() { store: mangodbStore, uri: MONGO_URI, options: { - collection: 'selfhostio_addon_collection', + collection: 'knightcrawler_addon_collection', socketTimeoutMS: 120000, useNewUrlParser: true, useUnifiedTopology: false, diff --git a/src/node/addon/src/lib/manifest.js b/src/node/addon/src/lib/manifest.js index 00dd4cf..a06d43b 100644 --- a/src/node/addon/src/lib/manifest.js +++ b/src/node/addon/src/lib/manifest.js @@ -6,13 +6,15 @@ const CatalogMochs = Object.values(MochOptions).filter(moch => moch.catalog); export function manifest(config = {}) { return { - id: 'com.stremio.selfhostio.selfhostio', - version: '0.0.1', + id: 'com.stremio.knightcrawler.knightcrawler', + version: 'v0.0.1', name: getName(config), - description: getDescription(config), + description: getDescription(), catalogs: getCatalogs(config), resources: getResources(config), types: [Type.MOVIE, Type.SERIES, Type.ANIME, Type.OTHER], + background: "https://i.ibb.co/9pXGycn/logo-color.png", + logo: "https://i.ibb.co/hYJPLdP/logo-only.png", behaviorHints: { configurable: true, configurationRequired: false, @@ -28,7 +30,7 @@ export function dummyManifest() { } function getName(config) { - const rootName = 'selfhostio'; + const rootName = 'Knight Crawler'; const mochSuffix = Object.values(MochOptions) .filter(moch => config[moch.key]) .map(moch => moch.shortName) @@ -36,15 +38,15 @@ function getName(config) { return [rootName, mochSuffix].filter(v => v).join(' '); } -function getDescription(config) { - return 'Selfhostio the Torrentio brings you much Funio'; +function getDescription() { + return 'Selfhost the Torrentio brings you much Funio'; } function getCatalogs(config) { return CatalogMochs .filter(moch => showDebridCatalog(config) && config[moch.key]) .map(moch => ({ - id: `selfhostio-${moch.key}`, + id: `knightcrawler-${moch.key}`, name: `${moch.name}`, type: 'other', extra: [{ name: 'skip' }], diff --git a/src/node/addon/src/lib/streamInfo.js b/src/node/addon/src/lib/streamInfo.js index 7bf7d61..949db7f 100644 --- a/src/node/addon/src/lib/streamInfo.js +++ b/src/node/addon/src/lib/streamInfo.js @@ -4,7 +4,7 @@ import { getSources } from './magnetHelper.js'; import { getSubtitles } from './subtitles.js'; import { Type } from './types.js'; -const ADDON_NAME = 'selfhostio'; +const ADDON_NAME = 'knightcrawler'; const SIZE_DELTA = 0.02; const UNKNOWN_SIZE = 300000000; const CAM_SOURCES = ['CAM', 'TeleSync', 'TeleCine', 'SCR']; @@ -38,7 +38,7 @@ export function toStreamInfo(record) { '\n' ); const bingeGroupParts = getBingeGroupParts(record, sameInfo, quality, torrentInfo, fileInfo); - const bingeGroup = joinDetailParts(bingeGroupParts, "selfhostio|", "|") + const bingeGroup = joinDetailParts(bingeGroupParts, "knightcrawler|", "|") const behaviorHints = bingeGroup ? { bingeGroup } : undefined; return cleanOutputObject({ diff --git a/src/node/addon/src/moch/alldebrid.js b/src/node/addon/src/moch/alldebrid.js index 9d0ad6c..eba383e 100644 --- a/src/node/addon/src/moch/alldebrid.js +++ b/src/node/addon/src/moch/alldebrid.js @@ -6,7 +6,7 @@ import { BadTokenError, AccessDeniedError, sameFilename } from './mochHelper.js' import StaticResponse from './static.js'; const KEY = 'alldebrid'; -const AGENT = 'selfhostio'; +const AGENT = 'knightcrawler'; export async function getCachedStreams(streams, apiKey) { const options = await getDefaultOptions(); diff --git a/src/node/addon/src/moch/moch.js b/src/node/addon/src/moch/moch.js index 602d89c..18e1925 100644 --- a/src/node/addon/src/moch/moch.js +++ b/src/node/addon/src/moch/moch.js @@ -224,14 +224,14 @@ function blackListToken(token, mochKey) { function errorStreamResponse(mochKey, error, config) { if (error === BadTokenError) { return { - name: `Selfhostio\n${MochOptions[mochKey].shortName} error`, + name: `KnightCrawler\n${MochOptions[mochKey].shortName} error`, title: `Invalid ${MochOptions[mochKey].name} ApiKey/Token!`, url: `${config.host}/${StaticResponse.FAILED_ACCESS}` }; } if (error === AccessDeniedError) { return { - name: `Selfhostio\n${MochOptions[mochKey].shortName} error`, + name: `KnightCrawler\n${MochOptions[mochKey].shortName} error`, title: `Expired/invalid ${MochOptions[mochKey].name} subscription!`, url: `${config.host}/${StaticResponse.FAILED_ACCESS}` }; diff --git a/src/node/consumer/src/lib/cache.js b/src/node/consumer/src/lib/cache.js index f7d5b80..7b9be26 100644 --- a/src/node/consumer/src/lib/cache.js +++ b/src/node/consumer/src/lib/cache.js @@ -4,7 +4,7 @@ import { cacheConfig } from './config.js'; import { logger } from './logger.js'; import { CacheType } from "./types.js"; -const GLOBAL_KEY_PREFIX = 'selfhostio-consumer'; +const GLOBAL_KEY_PREFIX = 'knightcrawler-consumer'; const IMDB_ID_PREFIX = `${GLOBAL_KEY_PREFIX}|imdb_id`; const KITSU_ID_PREFIX = `${GLOBAL_KEY_PREFIX}|kitsu_id`; const METADATA_PREFIX = `${GLOBAL_KEY_PREFIX}|metadata`; @@ -26,7 +26,7 @@ const initiateMongoCache = () => { url: cacheConfig.MONGO_URI, mongoConfig:{ socketTimeoutMS: 120000, - appName: 'selfhostio-consumer', + appName: 'knightcrawler-consumer', } }); diff --git a/src/node/consumer/src/lib/config.js b/src/node/consumer/src/lib/config.js index 9ad6491..ea1b55a 100644 --- a/src/node/consumer/src/lib/config.js +++ b/src/node/consumer/src/lib/config.js @@ -4,13 +4,13 @@ } export const cacheConfig = { - MONGO_URI: process.env.MONGODB_URI || 'mongodb://mongo:mongo@localhost:27017/selfhostio?authSource=admin', + MONGO_URI: process.env.MONGODB_URI || 'mongodb://mongo:mongo@localhost:27017/knightcrawler?authSource=admin', NO_CACHE: parseBool(process.env.NO_CACHE, false), - COLLECTION_NAME: process.env.MONGODB_COLLECTION || 'selfhostio_consumer_collection' + COLLECTION_NAME: process.env.MONGODB_COLLECTION || 'knightcrawler_consumer_collection' } export const databaseConfig = { - DATABASE_URI: process.env.DATABASE_URI || 'postgres://postgres:postgres@localhost:5432/selfhostio', + DATABASE_URI: process.env.DATABASE_URI || 'postgres://postgres:postgres@localhost:5432/knightcrawler', ENABLE_SYNC: parseBool(process.env.ENABLE_SYNC, true) }