diff --git a/.env.example b/.env.example index 4b6fe81..9fd3eee 100644 --- a/.env.example +++ b/.env.example @@ -3,3 +3,7 @@ SONARR_API_KEY= RADARR_API_KEY= SONARR4K_API_KEY= RADARR4K_API_KEY= +PROWLARR_API_KEY= +BAZARR_API_KEY= +SEERR_API_KEY= +PLEX_TOKEN= diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..ec64857 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,122 @@ +name: ci + +on: + push: + branches: ["**"] + pull_request: + +jobs: + # ── 1. Unit tests ──────────────────────────────────────────────────────── + test: + name: Unit tests (Python 3.12) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Run test suite + run: python3 -m unittest discover -s tests -v + + # ── 2. Syntax / lint gate ──────────────────────────────────────────────── + lint: + name: Syntax + lint (compileall + ruff) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Compile check (syntax errors) + run: python3 -m compileall -q doctor/ + + - name: Install ruff + run: pip install --quiet ruff + + - name: Ruff — undefined names and unused imports (F-only) + run: ruff check doctor/ tests/ --output-format=github + + # ── 3. Startup + /healthz smoke ───────────────────────────────────────── + smoke: + name: Startup smoke + /healthz + runs-on: ubuntu-latest + needs: [test, lint] + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Start daemon (no checks, UI enabled) + run: | + ENABLE_QUEUE=false \ + ENABLE_MULTIPACK=false \ + ENABLE_UI=true \ + DOCTOR_UI_PORT=12345 \ + DOCTOR_STATE_FILE=/tmp/doctor_state.json \ + python3 -m doctor & + echo $! > /tmp/doctor.pid + + - name: Wait for HTTP server to be ready + run: | + for i in $(seq 1 10); do + if curl -sf http://127.0.0.1:12345/healthz; then + echo " — /healthz OK" + exit 0 + fi + sleep 0.5 + done + echo "ERROR: /healthz did not respond within 5s" + exit 1 + + - name: Tear down daemon + if: always() + run: kill $(cat /tmp/doctor.pid) 2>/dev/null || true + + # ── 4. Docker publish (main + tags only, gated on test + lint + smoke) ── + publish: + name: Build and push image + runs-on: ubuntu-latest + needs: [test, lint, smoke] + if: | + github.event_name == 'push' && + (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')) + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v4 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository }} + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=ref,event=tag + type=sha,format=short + + - name: Set up Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + push: true + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 15cea65..f40f296 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -1,45 +1,17 @@ -name: publish image +# This workflow has been superseded by ci.yml which runs tests, lint, and smoke +# before publishing. The publish job now lives in ci.yml as the final gated step. +# This file is retained for reference only and has no triggers. +name: publish image (retired) on: - push: - branches: [ main ] - tags: [ 'v*' ] workflow_dispatch: + inputs: + reason: + description: "Reason (this workflow is retired — use ci.yml)" + required: false jobs: - build-and-push: + notice: runs-on: ubuntu-latest - permissions: - contents: read - packages: write steps: - - uses: actions/checkout@v4 - - - name: Log in to GHCR - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Docker metadata - id: meta - uses: docker/metadata-action@v5 - with: - images: ghcr.io/${{ github.repository }} - tags: | - type=raw,value=latest,enable={{is_default_branch}} - type=ref,event=tag - type=sha,format=short - - - name: Set up Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build and push - uses: docker/build-push-action@v6 - with: - context: . - push: true - platforms: linux/amd64,linux/arm64 - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + - run: echo "This workflow is retired. Publishing is handled by ci.yml." diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..3534cf1 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,26 @@ +# Agent Notes + +This project is a pure-Python package (no third-party dependencies). + +## How to run + +```bash +python3 -m doctor +``` + +## How to run tests + +```bash +python3 -m unittest discover -s tests -v +``` + +## Quick smoke test + +```bash +ENABLE_QUEUE=false ENABLE_UI=true DOCTOR_UI_PORT=12345 DOCTOR_STATE_FILE=/tmp/doctor_state.json timeout 3 python3 -m doctor +``` + +## How to deploy + +- Docker: `COPY doctor /app/doctor` and `ENTRYPOINT ["python3", "-m", "doctor"]`. +- Host systemd: copy the `doctor/` package directory, set `WorkingDirectory` to that directory, and run `python3 -m doctor`. diff --git a/DEPLOY.md b/DEPLOY.md index 678af3f..02591d3 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -1,6 +1,6 @@ # Deployment guide -stack-doctor is one small file (`doctor.py`, pure Python standard library, no dependencies). +stack-doctor is a small pure-Python package (standard library only, no third-party dependencies). Pick the path that matches your setup: - **Docker** is easiest. It runs the `queue` / `providers` / `plex` / `resources` checks and the @@ -69,8 +69,9 @@ Run this on the machine that runs decypharr. ```bash mkdir -p /opt/stack-doctor -curl -fsSL https://raw.githubusercontent.com/Neoo-Blue/stack-doctor/main/doctor.py \ - -o /opt/stack-doctor/doctor.py +cd /opt/stack-doctor +curl -fsSL https://github.com/Neoo-Blue/stack-doctor/archive/refs/heads/main.tar.gz \ + | tar -xz --strip-components=1 stack-doctor-main/doctor ``` 2. Install the service template and edit the values (URLs, API keys, paths): @@ -115,6 +116,6 @@ and open `http://:12345/?token=something`. ## Updating - **Docker**: `docker compose pull && docker compose up -d` -- **Host**: re-download `doctor.py` (step 1 above) and `systemctl restart stack-doctor` +- **Host**: re-download/extract the `doctor` package (step 1 above) and `systemctl restart stack-doctor` Your saved settings live in `DOCTOR_CONFIG_FILE` (`/data/config.json` by default) and survive updates. diff --git a/Dockerfile b/Dockerfile index 1de6e52..268ebce 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,19 +9,26 @@ ENV PYTHONUNBUFFERED=1 \ DOCTOR_STATE_FILE=/data/state.json WORKDIR /app -COPY doctor.py /app/doctor.py +COPY doctor /app/doctor -# No Python dependencies (standard library only). openssh-client lets a restart -# hook reach a *host* service (e.g. DECYPHARR_RESTART_CMD="ssh root@host systemctl restart decypharr"). +# The doctor package uses only the Python standard library. openssh-client lets a +# restart hook reach a *host* service (e.g. DECYPHARR_RESTART_CMD="ssh root@host systemctl restart decypharr"). +# docker-ce-cli lets a restart hook control local containers via a bind-mounted /var/run/docker.sock. # Runs as root so a bind-mounted /data (and an optional rw /mnt/library for the # janitor) is always writable regardless of host ownership. RUN apt-get update \ - && apt-get install -y --no-install-recommends openssh-client \ - && rm -rf /var/lib/apt/lists/* \ + && apt-get install -y --no-install-recommends openssh-client ca-certificates curl gnupg \ + && install -m 0755 -d /etc/apt/keyrings \ + && curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc \ + && chmod a+r /etc/apt/keyrings/docker.asc \ + && echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian bookworm stable" > /etc/apt/sources.list.d/docker.list \ + && apt-get update \ + && apt-get install -y --no-install-recommends docker-ce-cli \ + && rm -rf /var/lib/apt/lists/* /etc/apt/keyrings /etc/apt/sources.list.d/docker.list \ && mkdir -p /data VOLUME /data # webhook port (event mode) + web dashboard (ENABLE_UI) EXPOSE 8088 12345 -ENTRYPOINT ["python3", "/app/doctor.py"] +ENTRYPOINT ["python3", "-m", "doctor"] diff --git a/README.md b/README.md index 6a1d359..9fe74d1 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,10 @@ the failure modes: downloads that finish but never import, dead grabs stuck as hung decypharr FUSE mount that takes Plex down, memory/load pressure that OOMs your arrs. You only notice when something's "missing" or the family complains. -stack-doctor runs a set of **modular checks** on an interval (or on Sonarr/Radarr webhooks), -detects these, and fixes the safe ones automatically. No third-party dependencies, one small -container, everything configured by env vars. +stack-doctor runs a set of **modular checks** on independent schedules (or on Sonarr/Radarr webhooks), +detects these, and fixes the safe ones automatically. Fast checks like queue and providers run every +few minutes; slow checks like repair and missing_seasons run every 30 minutes. No third-party +dependencies, one small container, everything configured by env vars. > Born out of a long night of hand-fixing exactly these problems on a usenet *arr stack. > Now it's a daemon so you never have to do it by hand again. @@ -26,10 +27,14 @@ container, everything configured by env vars. | **providers** | failed indexers / download clients (sonarr/radarr/**prowlarr**) | runs the **Test** on them to re-validate + clear the failure | | **decypharr** | hung FUSE mount (read-test) + API down | runs your restart hook (`DECYPHARR_RESTART_CMD`) | | **plex** | Plex unresponsive | alerts (optional library refresh) | +| **plexscan** | a Plex library scan wedged with no progress (usually a hung mount) | restarts the hung mount, cancels the stuck scan, last-resort `PLEX_RESTART_CMD` | | **resources** | host load / low memory / swap pressure | reports; optional `drop_caches` relief | | **janitor** | permanently-dead usenet releases (from decypharr's log) | quarantines those library symlinks (reversible) | +| **repair** | dead library symlinks (debrid link gone); optionally MissingFromDisk history entries (usenet/direct downloads) | removes the dead file record + re-searches the owning *arr (API-first, mount-safe, SeasonSearch-first) | | **bazarr** | Bazarr unreachable | alerts | | **seerr** | Overseerr/Jellyseerr/Seerr requests stuck **FAILED** (the arr add timed out under load) | re-drives them so a transient blip self-heals (attempt-capped) | +| **missing_seasons** | monitored Sonarr seasons that have been around long enough but have zero episode files (skips still-airing seasons) | triggers a `SeasonSearch` so Sonarr re-tries (cooldown-gated, action-capped) | +| **no_upgrade_profile** | ended + fully-collected Sonarr series still on an upgrading quality profile | moves them to a no-upgrade profile so Sonarr stops searching for a better copy | | **warmer** | what a viewer is about to watch (Plex On Deck + next episode) | precaches the file head so playback starts instantly | Safe by design: risky actions (restart, drop_caches) are **opt-in**, the queue fixer only @@ -50,7 +55,7 @@ stack-doctor scales to the access it's given: janitor (`JANITOR_LOG_CMD=journalctl -u decypharr ...`), and touches the library directly, no container-to-host bridge needed. The *arr/Plex instances are still reached over the LAN. -Same `doctor.py`, same env vars; you just enable more checks where it has more power. +Same `python -m doctor`, same env vars; you just enable more checks where it has more power. --- @@ -85,7 +90,9 @@ services: restart: unless-stopped environment: DOCTOR_MODE: cron # cron | event - DOCTOR_INTERVAL: "900" + DOCTOR_INTERVAL: "900" # fallback/default interval (kept for compatibility) + DOCTOR_FAST_INTERVAL: "180s" # queue, providers, plex, plexscan, resources, bazarr, seerr + DOCTOR_SLOW_INTERVAL: "1800s" # repair, janitor, missing_seasons, no_upgrade_profile DOCTOR_DRY_RUN: "true" # start safe: log only, change nothing. flip to false when happy ENABLE_UI: "true" # web dashboard on :12345 (status, per-service health, warmer, config, logs) @@ -163,8 +170,13 @@ LAN isn't trusted. In event mode the webhook listener (`DOCTOR_PORT`) and the da | var | default | meaning | |---|---|---| -| `DOCTOR_MODE` | `cron` | `cron` (interval sweeps) or `event` (Sonarr/Radarr webhook) | -| `DOCTOR_INTERVAL` | `900` | cron: seconds between sweeps | +| `DOCTOR_MODE` | `cron` | `cron` (scheduled checks) or `event` (Sonarr/Radarr webhook + scheduled checks) | +| `DOCTOR_INTERVAL` | `900` | fallback interval used when a check has no explicit interval set | +| `DOCTOR_FAST_INTERVAL` | `180s` | interval for fast checks: queue, providers, decypharr, plex, plexscan, resources, bazarr, seerr | +| `DOCTOR_SLOW_INTERVAL` | `1800s` | interval for slow checks: repair, janitor, missing_seasons, no_upgrade_profile | +| `DOCTOR_SCHEDULER_TICK` | `30s` | how often the scheduler wakes to check which checks are due | +| `DOCTOR_SCHEDULER_CONCURRENCY` | `3` | max parallel scheduled checks | +| `_INTERVAL` | *(none)* | override a specific check's interval, e.g. `QUEUE_INTERVAL=60s` or `REPAIR_INTERVAL=1h` | | `DOCTOR_MIN_STRIKES` | `2` | item must be stuck this many consecutive checks before action (ignores transient blips like a download-client restart) | | `DOCTOR_MAX_ACTIONS` | `20` | max removals per sweep (rate limit, keeps re-searches gentle) | | `DOCTOR_BLOCKLIST` | `true` | blocklist removed grabs so a *different* release is fetched | @@ -174,7 +186,7 @@ LAN isn't trusted. In event mode the webhook listener (`DOCTOR_PORT`) and the da | `DOCTOR_REMOVE_FROM_CLIENT` | `true` | also remove from the download client | | `DOCTOR_DRY_RUN` | `false` | `true` = log only, change nothing | | `DOCTOR_CONDITIONS` | *all* | comma list of conditions to act on (see table above) | -| `DOCTOR_LOAD_MAX` | `0` | if > 0, skip a sweep when host 1-min load exceeds it (mount `/proc/loadavg:ro`) | +| `DOCTOR_LOAD_MAX` | `0` | if > 0, skip a scheduled check when host 1-min load exceeds it (mount `/proc/loadavg:ro`) | | `DOCTOR_HEALTH_REPORT` | `true` | log *arr `/health` warnings at debug level | | `DOCTOR_STATE_FILE` | `/data/state.json` | where strike counts persist | | `DOCTOR_PORT` | `8088` | webhook port (event mode) | @@ -204,6 +216,62 @@ real reason (dead TMDB id, removed title). Honors `DOCTOR_DRY_RUN` (logs what it would retry, changes nothing). +### Repair (dead-file re-grab) + +`repair` is now API-first: it queries Sonarr/Radarr for every file record, then checks the symlink's `readlink` target with `os.path.exists()` (fast, no FUSE read). Dead files are grouped by season or movie, the *arr file records are deleted, the season/movie monitor is toggled off+on, and a search is triggered. This catches dead debrid symlinks instantly without the slow filesystem walk, sampling, strike counting, or abort logic of the previous read-probe approach. + +**Two detection modes** (both feed the same re-grab action): + +- **Symlink check** (always on when `ENABLE_REPAIR=true`): API-first `readlink`/`exists` check for dead debrid symlinks. Processes the whole library every sweep; no `REPAIR_MIN_STRIKES`/`REPAIR_MAX_SCAN` needed. +- **MissingFromDisk history** (`REPAIR_MISSING_FROM_DISK=true`): queries Sonarr/Radarr download history for items the arr knows are missing from disk (`reason=MissingFromDisk`). Catches files that were never symlinks (usenet direct downloads, files removed by an external tool). Useful when you want doctor to cover a standard usenet stack. + +**Re-search priority** (Sonarr): `SeasonSearch` is used for every affected season so Sonarr has the chance to grab a season pack. + +**Post-repair verification** (`REPAIR_VERIFY=true`): after each re-search, doctor stores the command ID and a timestamp. On the next repair check it polls the search command status, then checks *arr download history for a new `grabbed` event. When a grab is confirmed it logs the indexer and exact release name (`[repair:verify] GRABBED 'Show Name' via NZBgeek: Show.S01E01.1080p...`). If no grab appears within `REPAIR_VERIFY_DEADLINE` it logs a warning so you know the search stalled. This is interval-based (checked once per `REPAIR_INTERVAL` or `DOCTOR_SLOW_INTERVAL`), not real-time. + +| var | default | meaning | +|---|---|---| +| `ENABLE_REPAIR` | `false` | turn the check on (needs a Sonarr/Radarr instance) | +| `REPAIR_LIBRARY_PATHS` | *(none)* | optional comma-separated library roots to limit which *arr file records are checked, e.g. `/mnt/library/movies,/mnt/library/tv` | +| `REPAIR_MAX_ACTIONS` | `20` | max search commands (seasons/movies) re-grabbed per sweep; caps indexer load | +| `REPAIR_MAX_SYMLINKS` | `100` | max total dead symlinks processed per sweep; caps the actual file-deletion workload | +| `REPAIR_LOAD_MAX` | `0` | skip the sweep when host 1-min load exceeds this (`0` = off) | +| `REPAIR_DEBRID_MOUNT` | *(none)* | debrid mount root (e.g. `/mnt/remote/realdebrid/__all__`); if set, only symlinks pointing here are checked, and the sweep is skipped when the mount is empty or missing (debrid-down guard) | +| `REPAIR_ITEM_INTERVAL` | `0` | seconds to wait between each re-grab action (`0` = no delay) | +| `REPAIR_SEASON_PACKS` | `false` | after the symlink check, flag Sonarr seasons whose files span multiple directories (individual episode grabs instead of a season pack) and trigger a `SeasonSearch` | +| `REPAIR_UNMONITORED` | `false` | include unmonitored series/movies in both the symlink and MissingFromDisk sweeps | +| `REPAIR_MISSING_FROM_DISK` | `false` | also scan *arr history for `MissingFromDisk` entries and re-search (usenet / direct-download mode) | +| `REPAIR_MFD_RECHECK` | `24h` | cooldown before re-searching the same MissingFromDisk item | +| `REPAIR_VERIFY` | `false` | after triggering a re-search, track the command ID and watch *arr history for a new `grabbed` event; logs the indexer name and release title when confirmed, or warns if no grab lands before the deadline | +| `REPAIR_VERIFY_DEADLINE` | `4h` | give up waiting for a grab confirmation after this long | + +Honors `DOCTOR_DRY_RUN`. + +### Missing seasons + +`missing_seasons` walks all monitored Sonarr series and finds seasons that have been around long enough (`MISSING_SEASONS_MIN_AGE_HOURS`) but still have zero episode files. Seasons that are still airing (have episodes with future air dates) are automatically skipped to avoid triggering searches for incomplete seasons. It triggers a `SeasonSearch` for each eligible season, with a per-season cooldown to avoid hammering the same season every sweep. + +| var | default | meaning | +|---|---|---| +| `ENABLE_MISSING_SEASONS` | `false` | turn the check on (needs a Sonarr instance) | +| `MISSING_SEASONS_MIN_AGE_HOURS` | `1` | ignore seasons added less than this long ago (gives Sonarr time to grab normally) | +| `MISSING_SEASONS_MAX_ACTIONS` | `5` | max `SeasonSearch` commands per sweep | +| `MISSING_SEASONS_RECHECK` | `24h` | cooldown before re-searching the same season | + +Honors `DOCTOR_DRY_RUN`. + +### No-upgrade profile + +`no_upgrade_profile` moves ended Sonarr series that are fully collected (all monitored episodes have files) onto a no-upgrade quality profile so Sonarr stops burning indexer quota searching for a better copy that will never come. + +| var | default | meaning | +|---|---|---| +| `ENABLE_NO_UPGRADE_PROFILE` | `false` | turn the check on (needs a Sonarr instance) | +| `NO_UPGRADE_PROFILE_NAME` | `WEB-1080p (No Upgrade)` | exact name of the quality profile to move qualifying series to (must already exist in Sonarr) | +| `NO_UPGRADE_PROFILE_ID` | `0` | use the profile's numeric ID instead of its name (`0` = resolve by name) | + +Honors `DOCTOR_DRY_RUN`. + ### Instances Add as many as you want, numbered from 1: @@ -219,15 +287,25 @@ Add as many as you want, numbered from 1: ## Cron vs Event mode -**Cron** (default): a daemon that sweeps every `DOCTOR_INTERVAL` seconds. Simple, reliable, -catches everything within ~`INTERVAL × MIN_STRIKES`. +**Cron** (default): the scheduler runs each enabled check on its own interval. +Fast checks run every `DOCTOR_FAST_INTERVAL`; slow checks run every `DOCTOR_SLOW_INTERVAL`. +You can override any check with `_INTERVAL`, e.g. `QUEUE_INTERVAL=60s`. -**Event**: stack-doctor runs a tiny webhook server. Point each *arr at it +**Event**: stack-doctor also runs a tiny webhook server. Point each *arr at it (*Settings → Connect → Webhook*, URL `http://stack-doctor:8088`, enable **On Grab / On Import / On Manual Interaction Required**) and it sweeps the moment the *arr reports trouble. -A slow safety-net sweep still runs in the background in case a webhook is missed. In event +The scheduler still runs in the background so nothing is missed if a webhook is lost. In event mode you'll usually set `DOCTOR_MIN_STRIKES: "1"` to act immediately, the event already confirms the item is stuck. +### Manual triggers (UI / HTTP) + +If `ENABLE_UI=true`, the dashboard can trigger checks on demand: + +- `POST /api/sweep` — run all enabled checks immediately +- `POST /api/check/` — run a specific check (e.g. `/api/check/queue`) immediately + +Both require the same authentication as the dashboard (`DOCTOR_UI_TOKEN`). + --- ## How the strike system works @@ -324,7 +402,7 @@ own concurrency lane, even during playback. ## Extending -Conditions are just predicates in `doctor.py` (`CONDITIONS` dict). Adding a new +Conditions are just predicates in `doctor/checks/queue.py` (`CONDITIONS` dict). Adding a new detect/fix rule is a couple of lines. PRs welcome. ## License diff --git a/docker-compose.example.yml b/docker-compose.example.yml index cd54ca4..8700aec 100644 --- a/docker-compose.example.yml +++ b/docker-compose.example.yml @@ -7,7 +7,11 @@ services: environment: # ---------- mode ---------- DOCTOR_MODE: cron # cron | event - DOCTOR_INTERVAL: "900" # cron: seconds between sweeps + DOCTOR_INTERVAL: "900" # fallback/default interval + DOCTOR_FAST_INTERVAL: "180s" # queue, providers, plex, plexscan, resources, bazarr, seerr + DOCTOR_SLOW_INTERVAL: "1800s" # repair, janitor, missing_seasons, no_upgrade_profile + DOCTOR_SCHEDULER_TICK: "30s" # how often the scheduler wakes + DOCTOR_SCHEDULER_CONCURRENCY: "3" # max parallel scheduled checks DOCTOR_DRY_RUN: "false" # true = log only, change nothing DOCTOR_LOG_LEVEL: INFO DOCTOR_LOG_FILE: /data/doctor.log # rotating file log (also logs to stdout) @@ -17,8 +21,10 @@ services: ENABLE_PROVIDERS: "true" # auto-Test failed indexers/download clients (sonarr/radarr/prowlarr) ENABLE_DECYPHARR: "true" # decypharr mount-hang watchdog ENABLE_PLEX: "true" # Plex reachability + ENABLE_PLEX_SCAN: "false" # recover a Plex library scan wedged with no progress (needs PLEX_URL/PLEX_TOKEN) ENABLE_RESOURCES: "true" # host load / mem / swap ENABLE_JANITOR: "false" # usenet dead-file quarantine (needs a decypharr log file) + ENABLE_REPAIR: "false" # probe library for dead files -> remove + re-search the owning *arr ENABLE_BAZARR: "false" # Bazarr reachability (set BAZARR_URL/BAZARR_APIKEY) ENABLE_SEERR: "false" # Overseerr/Jellyseerr/Seerr: auto-retry FAILED requests (set SEERR_URL/SEERR_APIKEY) ENABLE_WARMER: "false" # precache the head of likely-next media (needs PLEX_URL/PLEX_TOKEN) @@ -82,6 +88,11 @@ services: PLEX_TOKEN: ${PLEX_TOKEN} PLEX_SCAN_ON_CHECK: "false" # also trigger a library refresh when Plex is up + # ---------- plexscan (ENABLE_PLEX_SCAN: recover a wedged library scan; reuses the decypharr hook) ---------- + PLEX_SCAN_STUCK_AFTER: "30m" # a scan making no progress this long is wedged (usually a hung mount) + PLEX_SCAN_CANCEL: "true" # cancel the stuck scan via the Plex activities API + # PLEX_RESTART_CMD: "ssh root@192.168.1.20 systemctl restart plexmediaserver" # last resort if it stays wedged + # ---------- warmer (instant playback start; reuses PLEX_URL/PLEX_TOKEN) ---------- WARMER_PRECACHE_MB: "24" # head pulled per title (small = fast warm; the mount's read-ahead does the rest) WARMER_TAIL_MB: "4" # also pull the tail (mkv cues / Plex end-probe); 0 = off @@ -111,6 +122,26 @@ services: # ---------- janitor (optional) ---------- JANITOR_LIBRARY_PATHS: /mnt/library/tv,/mnt/library/movies JANITOR_DECYPHARR_LOG: /logs/decypharr.log # mount decypharr's error log here + + # ---------- repair (ENABLE_REPAIR: API-first dead-symlink detection -> remove + re-search) ---------- + # REPAIR_LIBRARY_PATHS: /mnt/library/tv,/mnt/library/movies # optional: only check *arr file records under these roots + REPAIR_MAX_ACTIONS: "20" # max search commands (seasons/movies) re-grabbed per sweep + REPAIR_MAX_SYMLINKS: "100" # max total dead symlinks processed per sweep + REPAIR_LOAD_MAX: "0" # skip the repair sweep above this host 1-min load (0 = off) + # REPAIR_DEBRID_MOUNT: /mnt/remote/realdebrid/__all__ # only check symlinks pointing here; skip if mount empty/missing + # REPAIR_ITEM_INTERVAL: "0" # seconds to wait between each re-grab (0 = no delay; gentle on providers) + # REPAIR_SEASON_PACKS: "false" # flag sonarr seasons spread across multiple dirs and search for a season pack + # REPAIR_UNMONITORED: "false" # include unmonitored series/movies in the repair sweep + # REPAIR_MISSING_FROM_DISK: "false" # also scan *arr history for MissingFromDisk items and re-search (usenet/direct downloads) + # REPAIR_MFD_RECHECK: "24h" # cooldown before re-searching the same MissingFromDisk item + # REPAIR_VERIFY: "false" # track re-searches and log confirmed grabs (indexer + release name) + # REPAIR_VERIFY_DEADLINE: "4h" # give up waiting for a grab confirmation after this long + + # ---------- missing_seasons (ENABLE_MISSING_SEASONS: find sonarr seasons with 0 files -> SeasonSearch) ---------- + # MISSING_SEASONS_MIN_AGE_HOURS: "1" # ignore seasons added less than this long ago (avoid triggering on new shows) + # MISSING_SEASONS_MAX_ACTIONS: "5" # max SeasonSearches per sweep + # MISSING_SEASONS_RECHECK: "24h" # cooldown before re-searching the same season + volumes: - ./data:/data # state + file log + quarantine manifests - /mnt/library:/mnt/library # for mount read-test + janitor (read/write for janitor) diff --git a/doctor.py b/doctor.py deleted file mode 100644 index 419370d..0000000 --- a/doctor.py +++ /dev/null @@ -1,1480 +0,0 @@ -#!/usr/bin/env python3 -""" -stack-doctor - auto-detect and fix recurring issues across a Sonarr/Radarr + -decypharr + Plex media stack. - -Modular checks, each toggled and configured by environment variables: - - queue *arr download queues - clear stuck/dead/blocked items -> re-search - providers *arr/prowlarr providers - auto-Test failed indexers/download clients to clear them - decypharr decypharr mount + API - detect a hung FUSE mount -> run a restart hook - plex Plex Media Server - detect unresponsive Plex (+ optional library scan) - resources host load / memory / swap - report pressure, optional drop_caches relief - janitor usenet dead files - quarantine library symlinks for permanently-dead - releases (reversible) from a decypharr log file - bazarr Bazarr - reachability check - seerr Overseerr/Jellyseerr/Seerr - auto-retry FAILED requests (arr add timed out under load) - warmer Plex-driven precache - read the head of likely-next media so playback starts - instantly (next episode + On Deck); thread, not a sweep - -Runs as a cron-style interval loop OR reacts to Sonarr/Radarr webhook events. -Pure Python standard library, no dependencies. -""" -import json -import logging -import logging.handlers -import os -import re -import signal -import subprocess -import sys -import threading -import time -import urllib.request -import urllib.error -import xml.etree.ElementTree as ET - -VERSION = "0.3" - -# --------------------------------------------------------------------------- # -# config helpers -# --------------------------------------------------------------------------- # - -def _b(name, default=False): - return os.environ.get(name, str(default)).strip().lower() in ("1", "true", "yes", "on") - -def _i(name, default): - try: - return int(os.environ.get(name, default)) - except (TypeError, ValueError): - return default - -def _f(name, default): - try: - return float(os.environ.get(name, default)) - except (TypeError, ValueError): - return default - -def _dur(tok, default=0): - """Parse a duration token: 30s / 10m / 2h / 1d, or a bare number of seconds.""" - t = str(tok).strip().lower() - if not t: - return default - mult = {"s": 1, "m": 60, "h": 3600, "d": 86400} - try: - return int(float(t[:-1]) * mult[t[-1]]) if t[-1] in mult else int(float(t)) - except (ValueError, KeyError): - return default - -def _human(sec): - sec = int(sec) - for size, suf in ((86400, "d"), (3600, "h"), (60, "m")): - if sec >= size and sec % size == 0: - return "%d%s" % (sec // size, suf) - return "%ds" % sec - -# UI-saved overrides: merge a JSON overlay over the inherited env BEFORE config is read, so edits win. -CONFIG_FILE = os.environ.get("DOCTOR_CONFIG_FILE", "/data/config.json") - -def _load_overrides(): - try: - with open(CONFIG_FILE) as f: - for k, v in json.load(f).items(): - if v is not None: - os.environ[str(k)] = str(v) - except Exception: - pass - -_load_overrides() - -MODE = os.environ.get("DOCTOR_MODE", "cron").strip().lower() # cron | event -INTERVAL = _i("DOCTOR_INTERVAL", 900) -PORT = _i("DOCTOR_PORT", 8088) # webhook port (event mode) -UI_PORT = _i("DOCTOR_UI_PORT", 12345) # web dashboard port -EN_UI = _b("ENABLE_UI", False) -UI_TOKEN = os.environ.get("DOCTOR_UI_TOKEN", "") # optional ?token= / X-Doctor-Token gate -LOG_LEVEL = os.environ.get("DOCTOR_LOG_LEVEL", "INFO").upper() -LOG_FILE = os.environ.get("DOCTOR_LOG_FILE", "") -TIMEOUT = _i("DOCTOR_HTTP_TIMEOUT", 60) -DRY_RUN = _b("DOCTOR_DRY_RUN", False) - -# which checks are on -EN_QUEUE = _b("ENABLE_QUEUE", True) -EN_DECYPHARR = _b("ENABLE_DECYPHARR", False) -EN_PLEX = _b("ENABLE_PLEX", False) -EN_RESOURCES = _b("ENABLE_RESOURCES", False) -EN_JANITOR = _b("ENABLE_JANITOR", False) -EN_PROVIDERS = _b("ENABLE_PROVIDERS", False) # auto-test failed indexers/download clients (sonarr/radarr/prowlarr) -EN_BAZARR = _b("ENABLE_BAZARR", False) # Bazarr reachability -EN_SEERR = _b("ENABLE_SEERR", False) # Overseerr/Jellyseerr/Seerr: auto-retry FAILED requests -EN_WESTREPAIR = _b("ENABLE_WESTREPAIR", False) # symlink repair via repair.py subprocess - -# westrepair config -WR_SCRIPT = os.environ.get("WESTREPAIR_SCRIPT", "/app/westrepair/repair.py") -WR_RUN_INTERVAL = os.environ.get("WESTREPAIR_RUN_INTERVAL", "6h") -WR_REPAIR_INTERVAL = os.environ.get("WESTREPAIR_REPAIR_INTERVAL", "1m") - -BAZARR_URL = os.environ.get("BAZARR_URL", "") -BAZARR_APIKEY = os.environ.get("BAZARR_APIKEY", "") - -# seerr (Overseerr / Jellyseerr / Seerr) failed-request auto-retry. -# When the arr API is briefly slow (e.g. under a heavy search load), seerr's add call times out and -# it marks the request FAILED - it never auto-retries, so the title silently never reaches the arr. -# We periodically re-drive those FAILED requests so a transient blip self-heals, with an attempt cap -# so a genuinely-bad request (dead tmdb id, etc.) doesn't get retried forever. -SEERR_URL = os.environ.get("SEERR_URL", "") -SEERR_APIKEY = os.environ.get("SEERR_APIKEY", "") -SEERR_MAX = _i("SEERR_RETRY_MAX", 10) # max requests retried per sweep (rate-limit the re-adds) -SEERR_MAX_TRIES = _i("SEERR_MAX_ATTEMPTS", 5) # give up on a request after this many auto-retries (0 = never give up) - -# queue check -MIN_STRIKES = _i("DOCTOR_MIN_STRIKES", 2) -MAX_ACTIONS = _i("DOCTOR_MAX_ACTIONS", 20) -BLOCKLIST = _b("DOCTOR_BLOCKLIST", True) -REMOVE_CLIENT = _b("DOCTOR_REMOVE_FROM_CLIENT", True) -STATE_FILE = os.environ.get("DOCTOR_STATE_FILE", "/data/state.json") -# churn brake: a title that keeps grabbing dead releases (re-grabbed despite blocklist, or only -# dead releases exist) never imports and just burns cycles. After CHURN_LIMIT failed grabs of the -# SAME episode/movie, stop the loop. action: report (log only) | park (un-monitor) | backoff -# (un-monitor, then auto re-monitor on an escalating schedule for a fresh attempt). -CHURN_LIMIT = _i("DOCTOR_CHURN_LIMIT", 0) # 0 = brake off -CHURN_ACTION = os.environ.get("DOCTOR_CHURN_ACTION", "report").strip().lower() -# backoff retry schedule: each park steps to the next delay; the last entry repeats forever. -# default "10m,1h,24h" = retry 10m after the 1st park, 1h after the 2nd, every 24h thereafter. -CHURN_BACKOFF = [_dur(x) for x in os.environ.get("DOCTOR_CHURN_BACKOFF", "").split(",") if x.strip()] -if not CHURN_BACKOFF: - _legacy = os.environ.get("DOCTOR_CHURN_COOLDOWN") # back-compat with the old single fixed cooldown - CHURN_BACKOFF = [_dur(_legacy)] if _legacy else [600, 3600, 86400] -DEFAULT_CONDITIONS = "downloadClientUnavailable,importBlocked,importFailed,importPending_warning,failedPending,stalled" -ENABLED_CONDITIONS = [c.strip() for c in os.environ.get("DOCTOR_CONDITIONS", DEFAULT_CONDITIONS).split(",") if c.strip()] - -# resource thresholds (host load uses /proc/loadavg if mounted) -LOAD_MAX = _f("DOCTOR_LOAD_MAX", 0) # queue check pauses above this (0=off) -RES_LOAD_WARN = _f("RES_LOAD_WARN", 40) -RES_SWAP_WARN = _i("RES_SWAP_WARN_MB", 7000) -RES_MEM_MIN = _i("RES_MEM_MIN_MB", 800) -RES_DROP_CACHES = _b("RES_DROP_CACHES", False) # echo 1 > drop_caches on memory pressure (needs privilege) - -# decypharr -DECY_URL = os.environ.get("DECYPHARR_URL", "") # e.g. http://192.168.50.202:8282 -DECY_MOUNT_TEST = os.environ.get("DECYPHARR_MOUNT_TEST", "") # a dir on the FUSE mount to read-test -DECY_READ_TIMEOUT = _i("DECYPHARR_READ_TIMEOUT", 25) -DECY_RESTART_CMD = os.environ.get("DECYPHARR_RESTART_CMD", "") # shell cmd to recover a hung mount - -# plex -PLEX_URL = os.environ.get("PLEX_URL", "") -PLEX_TOKEN = os.environ.get("PLEX_TOKEN", "") -PLEX_SCAN = _b("PLEX_SCAN_ON_CHECK", False) - -# warmer (Plex-driven precache of the heads of likely-next media -> instant playback start) -EN_WARMER = _b("ENABLE_WARMER", False) -WARM_HEAD_MB = _i("WARMER_PRECACHE_MB", 64) # how much of the file head to pull into cache -WARM_TAIL_MB = _i("WARMER_TAIL_MB", 8) # also pull the tail (mkv cues / Plex end-probe); 0=off -WARM_INTERVAL = _i("WARMER_INTERVAL", 120) # seconds between session polls (next-episode prefetch) -WARM_ONDECK_EVERY = _i("WARMER_ONDECK_EVERY", 600) # seconds between on-deck / recent warms -WARM_NEXT_EPS = _i("WARMER_NEXT_EPISODES", 1) # warm this many upcoming episodes of an active show -WARM_RECENT_COUNT = _i("WARMER_RECENT_COUNT", 0) # warm N most-recently-added per library (0=off) -WARM_MAX_CYCLE = _i("WARMER_MAX_PER_CYCLE", 12) # cap warms per cycle (rate-limit the usenet fetch) -WARM_COOLDOWN = _i("WARMER_COOLDOWN", 3600) # do not re-warm the same file within this many seconds -WARM_LOAD_MAX = _f("WARMER_LOAD_MAX", 0) # skip warming if host 1-min load above this (protect Plex); 0=off -WARM_READ_TIMEOUT = _i("WARMER_READ_TIMEOUT", 60) # abandon a single warm read after this long (hung mount guard) -WARM_CONCURRENCY = _i("WARMER_CONCURRENCY", 2) # simultaneous BACKGROUND (on-deck/recent) warm reads -WARM_OPEN_CONC = _i("WARMER_OPEN_CONCURRENCY", 4) # dedicated lane for the title you OPEN, so it starts instantly and never queues behind background warming -WARM_PARTS = _i("WARMER_PARTS", 1) # how many versions per title to warm (1 = highest-res only; 0 = all). Avoids warming a 1080p you'll never play next to the 4K -# low-cache mode: for small / RAM-backed caches. Skips On Deck (Continue Watching) warming entirely and -# only warms the NEXT episode as the current one nears its end, so almost nothing sits in cache early. -WARM_LOW_CACHE = _b("WARMER_LOW_CACHE", False) -WARM_NEXT_REMAIN = _i("WARMER_NEXT_REMAINING_MIN", 0) # warm the next episode only when <= this many minutes remain (0 = as soon as playback is seen) -WARM_NEXT_NEAR_END = WARM_NEXT_REMAIN if WARM_NEXT_REMAIN > 0 else (10 if WARM_LOW_CACHE else 0) -WARM_SOURCES = [s.strip().lower() for s in os.environ.get("WARMER_SOURCES", "ondeck,next").split(",") if s.strip()] -WARM_ONDECK = _b("WARMER_ONDECK", True) # quick on/off for Continue Watching (On Deck) warming -WARM_PATH_MAP = os.environ.get("WARMER_PATH_MAP", "") # "plexPrefix:hostPrefix" if Plex's file path != this host's -# detail-page warming: tail Plex's server log and warm the exact title a viewer opens (the one true -# pre-play signal Plex emits). Give it a streaming command (tail -F, or `pct exec ... tail -F`) OR a file. -WARM_PLEXLOG_CMD = os.environ.get("WARMER_PLEXLOG_CMD", "") -WARM_PLEXLOG_FILE = os.environ.get("WARMER_PLEXLOG_FILE", "") - -# janitor (give it decypharr's error log via a file OR a command, e.g. journalctl when on-host) -JAN_LIBS = [p.strip() for p in os.environ.get("JANITOR_LIBRARY_PATHS", "").split(",") if p.strip()] -JAN_LOG = os.environ.get("JANITOR_DECYPHARR_LOG", "") # log file path -JAN_LOG_CMD = os.environ.get("JANITOR_LOG_CMD", "") # cmd printing the log, e.g. "journalctl -u decypharr -n 10000 --no-hostname" -JAN_QUAR = os.environ.get("JANITOR_QUARANTINE_DIR", "/data/quarantine") -JAN_PATTERNS = os.environ.get("JANITOR_DEAD_PATTERNS", "ARTICLE_NOT_FOUND,still missing").split(",") - -TRIGGER_EVENTS = set(e.strip() for e in os.environ.get( - "DOCTOR_TRIGGER_EVENTS", "Download,ManualInteractionRequired,DownloadFailed,Grab").split(",") if e.strip()) - -# --------------------------------------------------------------------------- # -# logging -# --------------------------------------------------------------------------- # -handlers = [logging.StreamHandler(sys.stdout)] -if LOG_FILE: - try: - os.makedirs(os.path.dirname(LOG_FILE) or ".", exist_ok=True) - handlers.append(logging.handlers.RotatingFileHandler(LOG_FILE, maxBytes=5_000_000, backupCount=3)) - except Exception: - pass -logging.basicConfig(level=getattr(logging, LOG_LEVEL, logging.INFO), - format="%(asctime)s | %(levelname)-7s | %(name)s | %(message)s", - datefmt="%Y-%m-%d %H:%M:%S", handlers=handlers) -log = logging.getLogger("doctor") - -# --------------------------------------------------------------------------- # -# small helpers -# --------------------------------------------------------------------------- # - -def http_code(url, headers=None, t=10): - try: - r = urllib.request.urlopen(urllib.request.Request(url, headers=headers or {}), timeout=t) - return r.status - except urllib.error.HTTPError as e: - return e.code - except Exception: - return 0 - -def run_cmd(cmd): - if not cmd: - return None - try: - p = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=180) - return (p.returncode, (p.stdout + p.stderr).strip()[:300]) - except Exception as e: - return (1, "cmd error: " + str(e)[:120]) - -def run_output(cmd, t=120): - try: - p = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=t) - return p.stdout - except Exception as e: - log.warning("log cmd failed: %s", str(e)[:80]) - return "" - -def host_load(): - try: - with open("/proc/loadavg") as f: - return float(f.read().split()[0]) - except Exception: - return 0.0 - -# =========================================================================== # -# CHECK: queue -# =========================================================================== # - -def _msgs(rec): - out = [] - for sm in (rec.get("statusMessages") or []): - out += [m for m in (sm.get("messages") or [])] - if rec.get("errorMessage"): - out.append(rec["errorMessage"]) - return out - -CONDITIONS = { - "downloadClientUnavailable": lambda r: r.get("status") == "downloadClientUnavailable", - "importBlocked": lambda r: r.get("trackedDownloadState") == "importBlocked", - "importFailed": lambda r: r.get("trackedDownloadState") == "importFailed", - "importPending_warning": lambda r: r.get("trackedDownloadState") == "importPending" - and r.get("trackedDownloadStatus") in ("warning", "error"), - "failedPending": lambda r: r.get("trackedDownloadState") == "failedPending", - "stalled": lambda r: r.get("trackedDownloadStatus") == "warning" - and any("stall" in m.lower() or "no files" in m.lower() for m in _msgs(r)), -} - -def stuck_reason(rec): - for name in ENABLED_CONDITIONS: - pred = CONDITIONS.get(name) - if pred and pred(rec): - return name - return None - -class Arr: - def __init__(self, name, kind, url, apikey): - self.name, self.kind = name, kind # sonarr | radarr | prowlarr - self.base = url.rstrip("/") + ("/api/v1" if kind == "prowlarr" else "/api/v3") - self.apikey = apikey - self.unknown = "includeUnknownSeriesItems=true" if kind == "sonarr" else "includeUnknownMovieItems=true" - - def _req(self, method, path, data=None, t=None): - req = urllib.request.Request(self.base + path, data=data, method=method, - headers={"X-Api-Key": self.apikey, "Content-Type": "application/json"}) - return urllib.request.urlopen(req, timeout=t or TIMEOUT) - - def queue(self): - if self.kind == "prowlarr": - return [] # prowlarr has no download queue - try: - return json.load(self._req("GET", "/queue?page=1&pageSize=1000&" + self.unknown)).get("records", []) - except Exception as e: - log.warning("[%s] queue fetch failed: %s", self.name, e); return None - - def health(self): - try: - return json.load(self._req("GET", "/health")) - except Exception: - return [] - - def remove(self, item_id): - q = "removeFromClient=%s&blocklist=%s" % (str(REMOVE_CLIENT).lower(), str(BLOCKLIST).lower()) - self._req("DELETE", "/queue/%d?%s" % (item_id, q)) - - def post(self, path, t=150): - """POST with empty body (used for /indexer/testall, /downloadclient/testall). Returns parsed JSON or [].""" - try: - body = self._req("POST", path, data=b"", t=t).read() - return json.loads(body) if body else [] - except urllib.error.HTTPError as e: - try: return json.loads(e.read()) - except Exception: return [] - except Exception as ex: - log.debug("[%s] POST %s err %s", self.name, path, str(ex)[:50]); return [] - - def set_monitored(self, ids, monitored): - """Bulk toggle monitoring for episodes (sonarr) / movies (radarr). Used by the churn brake.""" - if self.kind == "sonarr": - path, body = "/episode/monitor", {"episodeIds": list(ids), "monitored": monitored} - elif self.kind == "radarr": - path, body = "/movie/editor", {"movieIds": list(ids), "monitored": monitored} - else: - return False - try: - self._req("PUT", path, data=json.dumps(body).encode()); return True - except Exception as e: - log.warning("[churn:%s] monitor %s failed: %s", self.name, "on" if monitored else "off", str(e)[:70]) - return False - - def queue_target_id(self, rec): - """Stable id of what a queue record is FOR (episode for sonarr, movie for radarr).""" - return rec.get("episodeId") if self.kind == "sonarr" else rec.get("movieId") if self.kind == "radarr" else None - -def load_instances(): - out = [] - for n in range(1, 51): - url = os.environ.get("INSTANCE_%d_URL" % n) - if not url: - continue - key = os.environ.get("INSTANCE_%d_APIKEY" % n, "") - kind = os.environ.get("INSTANCE_%d_TYPE" % n, "").strip().lower() - if kind not in ("sonarr", "radarr", "prowlarr"): - kind = ("radarr" if "radarr" in url.lower() else - "prowlarr" if "prowlarr" in url.lower() else "sonarr") - name = os.environ.get("INSTANCE_%d_NAME" % n, "%s-%d" % (kind, n)) - if not key: - log.warning("INSTANCE_%d has no APIKEY, skipping", n); continue - out.append(Arr(name, kind, url, key)) - return out - -INSTANCES = [] - -def _load_state(): - try: - return json.load(open(STATE_FILE)) - except Exception: - return {} - -def _save_state(s): - try: - os.makedirs(os.path.dirname(STATE_FILE) or ".", exist_ok=True) - json.dump(s, open(STATE_FILE, "w")) - except Exception: - pass - -def _offenders(state): - return state.setdefault("__offenders__", {}) - -def _churn_record(state, arr, rec, title): - """Count a dead grab for this episode/movie; brake if it's over the limit. - Returns True if it un-monitored the target (so the caller knows the blocklist-remove won't re-search).""" - if CHURN_LIMIT <= 0: - return False - tid = arr.queue_target_id(rec) - if not tid: - return False - off = _offenders(state).setdefault(arr.name, {}) - o = off.setdefault(str(tid), {"fails": 0, "until": 0, "level": 0, "title": title}) - o["fails"] += 1; o["title"] = title - if o["fails"] < CHURN_LIMIT or o["until"] != 0: # below limit, or already parked/reported - return False - if CHURN_ACTION == "report": - log.warning("[churn:%s] REPEAT-OFFENDER (%d dead grabs, still retrying): %s", arr.name, o["fails"], title) - o["until"] = -1 - return False - if CHURN_ACTION in ("park", "backoff") and arr.set_monitored([int(tid)], False): - o["fails"] = 0 - if CHURN_ACTION == "backoff": - lvl = o.get("level", 0) - delay = CHURN_BACKOFF[min(lvl, len(CHURN_BACKOFF) - 1)] - o["until"] = time.time() + delay; o["level"] = lvl + 1 - log.warning("[churn:%s] REPEAT-OFFENDER parked (retry #%d in %s) -> un-monitored: %s", - arr.name, lvl + 1, _human(delay), title) - else: # park: no auto-retry - o["until"] = -1 - log.warning("[churn:%s] REPEAT-OFFENDER parked (un-monitored, manual re-monitor): %s", arr.name, title) - return True - return False - -def _churn_remonitor(state): - """Re-monitor parked titles whose backoff delay has elapsed, giving them a fresh attempt.""" - if CHURN_LIMIT <= 0 or CHURN_ACTION != "backoff": - return - now = time.time(); off_all = state.get("__offenders__", {}) - for arr in INSTANCES: - for tid, o in list(off_all.get(arr.name, {}).items()): - until = o.get("until", 0) - if isinstance(until, (int, float)) and until > 0 and now >= until: - if arr.set_monitored([int(tid)], True): - log.info("[churn:%s] backoff #%d elapsed, re-monitoring for a fresh attempt: %s", - arr.name, o.get("level", 0), o.get("title", "")) - o["fails"] = 0; o["until"] = 0 # keep level so the next park escalates - -def check_queue(only=None): - if LOAD_MAX > 0 and host_load() > LOAD_MAX: - log.info("[queue] host load > %.0f -> skipping", LOAD_MAX); return - state = _load_state(); actions = 0 - _churn_remonitor(state) - for arr in INSTANCES: - if only and arr.name.lower() != only.lower(): - continue - recs = arr.queue() - if recs is None: - continue - strikes = state.get(arr.name, {}); new = {}; stuck = 0 - for r in recs: - reason = stuck_reason(r) - if not reason: - continue - stuck += 1; iid = str(r.get("id")); cnt = strikes.get(iid, 0) + 1; new[iid] = cnt - if cnt >= MIN_STRIKES and actions < MAX_ACTIONS: - title = (r.get("title") or "")[:70] - if DRY_RUN: - log.info("[queue:%s] WOULD remove (%s strike %d): %s", arr.name, reason, cnt, title) - else: - parked = _churn_record(state, arr, r, title) # un-monitor first so the remove can't re-search - try: - arr.remove(r["id"]); actions += 1; new.pop(iid, None) - log.info("[queue:%s] removed (%s, blocklist=%s)%s: %s", arr.name, reason, BLOCKLIST, - " [parked, no re-search]" if parked else " -> re-search", title) - except Exception as e: - log.warning("[queue:%s] remove failed: %s", arr.name, e) - state[arr.name] = new - if stuck: - log.info("[queue:%s] %d stuck tracked, %d acted", arr.name, stuck, actions) - for h in arr.health(): - if h.get("type") in ("error", "warning"): - log.debug("[queue:%s] health %s: %s", arr.name, h.get("type"), (h.get("message") or "")[:90]) - _save_state(state) - -# =========================================================================== # -# CHECK: decypharr (mount hang -> restart hook) -# =========================================================================== # - -def _read_test(path, timeout): - """Return True if a file under path read its first bytes within timeout, else False (hung/failed).""" - result = {"ok": False} - target = {"f": None} - try: - for root, _, files in os.walk(path): - for fn in files: - if fn.lower().endswith((".mkv", ".mp4", ".avi", ".m4v", ".ts")): - target["f"] = os.path.join(root, fn); break - if target["f"]: - break - except Exception: - return None # cannot even list -> unknown - if not target["f"]: - return None - def _do(): - try: - with open(target["f"], "rb") as fh: - fh.read(65536) - result["ok"] = True - except Exception: - result["ok"] = False - th = threading.Thread(target=_do, daemon=True); th.start(); th.join(timeout) - if th.is_alive(): - return False # hung - return result["ok"] - -_decy_last_restart = [0.0] - -def check_decypharr(): - if DECY_URL: - c = http_code(DECY_URL, t=10) - log.info("[decypharr] api %s -> %s", DECY_URL, c if c else "DOWN") - if not DECY_MOUNT_TEST: - return - ok = _read_test(DECY_MOUNT_TEST, DECY_READ_TIMEOUT) - if ok is None: - log.warning("[decypharr] mount %s: no test file found / unlistable", DECY_MOUNT_TEST); return - if ok: - log.info("[decypharr] mount %s read OK", DECY_MOUNT_TEST); return - log.error("[decypharr] mount %s READ HUNG (FUSE stall)", DECY_MOUNT_TEST) - if DRY_RUN or not DECY_RESTART_CMD: - log.error("[decypharr] no restart cmd set (or dry-run) -> alert only"); return - if time.time() - _decy_last_restart[0] < 300: - log.warning("[decypharr] restarted <5m ago, holding off"); return - log.error("[decypharr] running restart hook: %s", DECY_RESTART_CMD) - rc = run_cmd(DECY_RESTART_CMD); _decy_last_restart[0] = time.time() - log.error("[decypharr] restart hook rc=%s %s", rc[0] if rc else "?", rc[1] if rc else "") - -# =========================================================================== # -# CHECK: plex -# =========================================================================== # - -def check_plex(): - if not PLEX_URL: - return - sep = "&" if "?" in PLEX_URL else "?" - url = PLEX_URL.rstrip("/") + "/identity" - c = http_code(url + (sep + "X-Plex-Token=" + PLEX_TOKEN if PLEX_TOKEN else ""), t=10) - if c == 200: - log.info("[plex] %s -> 200 OK", PLEX_URL) - else: - log.error("[plex] %s -> %s (unresponsive)", PLEX_URL, c if c else "DOWN") - if PLEX_SCAN and PLEX_TOKEN and c == 200: - try: - urllib.request.urlopen(PLEX_URL.rstrip("/") + "/library/sections/all/refresh?X-Plex-Token=" + PLEX_TOKEN, timeout=10) - log.info("[plex] triggered library refresh") - except Exception as e: - log.debug("[plex] refresh failed: %s", e) - -# =========================================================================== # -# CHECK: resources -# =========================================================================== # - -def _meminfo(): - d = {} - try: - for line in open("/proc/meminfo"): - k, _, v = line.partition(":") - d[k.strip()] = int(v.split()[0]) // 1024 # MB - except Exception: - pass - return d - -def check_resources(): - l1 = host_load() - mi = _meminfo() - avail = mi.get("MemAvailable", -1) - swap_used = mi.get("SwapTotal", 0) - mi.get("SwapFree", 0) - msg = "[resources] load=%.1f memAvail=%sMB swapUsed=%sMB" % (l1, avail, swap_used) - crit = (l1 >= RES_LOAD_WARN) or (0 <= avail < RES_MEM_MIN) or (swap_used >= RES_SWAP_WARN) - (log.warning if crit else log.info)(msg + (" <-- PRESSURE" if crit else "")) - if crit and RES_DROP_CACHES and not DRY_RUN: - rc = run_cmd("sync; echo 1 > /proc/sys/vm/drop_caches") - log.warning("[resources] dropped page cache rc=%s", rc[0] if rc else "?") - -# =========================================================================== # -# CHECK: janitor (usenet dead-file quarantine, from a decypharr log file) -# =========================================================================== # - -def check_janitor(): - has_log = JAN_LOG_CMD or (JAN_LOG and os.path.exists(JAN_LOG)) - if not (JAN_LIBS and has_log): - log.debug("[janitor] need JANITOR_LIBRARY_PATHS + (JANITOR_LOG_CMD or a readable JANITOR_DECYPHARR_LOG)") - return - bad = set() - try: - if JAN_LOG_CMD: - data = run_output(JAN_LOG_CMD) # e.g. journalctl when running on-host - else: - data = open(JAN_LOG, errors="ignore").read()[-2_000_000:] - except Exception as e: - log.warning("[janitor] cannot read log: %s", e); return - pat = re.compile(r"Error streaming file: (.+?) error=\"([^\"]*)\"") - for m in pat.finditer(data): - path, err = m.group(1), m.group(2) - if any(p.strip() and p.strip() in err for p in JAN_PATTERNS): - bad.add(path.strip().split("/")[0]) - if not bad: - log.debug("[janitor] no dead releases in log tail"); return - moved = 0 - qroot = os.path.join(JAN_QUAR, time.strftime("%Y%m%d-%H%M%S")) - manifest = [] - for libp in JAN_LIBS: - for root, _, files in os.walk(libp): - for fn in files: - fp = os.path.join(root, fn) - if not os.path.islink(fp): - continue - try: - tgt = os.readlink(fp) - except Exception: - continue - mm = re.search(r"/__all__/([^/]+)(?:/|$)", tgt) - if mm and mm.group(1) in bad: - if DRY_RUN: - log.info("[janitor] WOULD quarantine: %s", fp); continue - try: - dst = os.path.join(qroot, os.path.relpath(fp, "/")) - os.makedirs(os.path.dirname(dst), exist_ok=True) - os.symlink(tgt, dst); os.unlink(fp) - manifest.append({"orig": fp, "target": tgt}); moved += 1 - except Exception as e: - log.warning("[janitor] move failed %s: %s", fp, e) - if manifest: - try: - os.makedirs(qroot, exist_ok=True); json.dump(manifest, open(qroot + "/manifest.json", "w"), indent=1) - except Exception: - pass - if moved: - log.info("[janitor] quarantined %d dead-file symlink(s) across %d release(s) -> %s", moved, len(bad), qroot) - -# =========================================================================== # -# CHECK: providers (radarr/sonarr/prowlarr indexers + download clients that errored -> Test) -# =========================================================================== # - -_PROVIDER_KEYWORDS = ("indexer", "download client", "applications unavailable", "applications are unavailable") - -def check_providers(): - for arr in INSTANCES: - if arr.kind not in ("sonarr", "radarr", "prowlarr"): - continue - issues = [h for h in arr.health() - if h.get("type") in ("warning", "error") - and any(k in (h.get("message") or "").lower() for k in _PROVIDER_KEYWORDS)] - if not issues: - continue - log.warning("[providers:%s] %d provider issue(s): %s", arr.name, len(issues), - " | ".join((h.get("message") or "")[:60] for h in issues[:2])) - if DRY_RUN: - continue - # re-test everything; a passing test clears the failure status and re-enables recovered ones - for ep, label in (("/indexer/testall", "indexers"), ("/downloadclient/testall", "download-clients")): - res = arr.post(ep) - if isinstance(res, list) and res: - ok = sum(1 for r in res if r.get("isValid")) - still = [r.get("id") for r in res if not r.get("isValid")] - log.info("[providers:%s] tested %s: %d ok, %d still failing %s", - arr.name, label, ok, len(still), still or "") - -# =========================================================================== # -# CHECK: bazarr (reachability) -# =========================================================================== # - -def check_bazarr(): - if not BAZARR_URL: - return - c = http_code(BAZARR_URL.rstrip("/") + "/api/system/status", - headers={"X-API-KEY": BAZARR_APIKEY} if BAZARR_APIKEY else None, t=10) - (log.info if c == 200 else log.error)("[bazarr] %s -> %s", BAZARR_URL, c if c else "DOWN") - -# =========================================================================== # -# CHECK: seerr (Overseerr / Jellyseerr / Seerr) - auto-retry FAILED requests -# -# seerr hands an approved request to Radarr/Sonarr with a fixed ~10s API timeout -# and NO retry of its own. If the arr is briefly slow (heavy search load, host -# contention) the add times out, the request is marked FAILED, and the title -# silently never lands in the arr. We re-drive those FAILED requests each sweep -# so a transient blip self-heals; an attempt cap stops us looping on a request -# that fails for a real reason (dead tmdb id, removed title). -# =========================================================================== # - -class Seerr: - def __init__(self, url, apikey): - self.base = url.rstrip("/") + "/api/v1" - self.apikey = apikey - - def _req(self, method, path, data=None, t=None): - req = urllib.request.Request(self.base + path, data=data, method=method, - headers={"X-Api-Key": self.apikey, "Content-Type": "application/json"}) - return urllib.request.urlopen(req, timeout=t or TIMEOUT) - - def failed(self): - """Requests currently in the FAILED state (seerr could not hand them to the arr).""" - try: - d = json.load(self._req("GET", "/request?take=100&skip=0&filter=failed&sort=added", t=15)) - return d.get("results", []) - except Exception as e: - log.warning("[seerr] failed-list fetch error: %s", str(e)[:80]); return None - - def retry(self, rid): - self._req("POST", "/request/%d/retry" % int(rid), data=b"", t=30) - -def check_seerr(): - if not SEERR_URL or not SEERR_APIKEY: - return - s = Seerr(SEERR_URL, SEERR_APIKEY) - reqs = s.failed() - if reqs is None: # fetch errored -> seerr down/unreachable - log.error("[seerr] %s unreachable", SEERR_URL); return - if not reqs: - log.info("[seerr] no failed requests"); return - state = _load_state() - tries = state.setdefault("__seerr__", {}) - log.warning("[seerr] %d failed request(s)", len(reqs)) - acted = 0 - for r in reqs: - if acted >= SEERR_MAX: - break - rid = r.get("id") - if rid is None: - continue - md = r.get("media") or {} - label = "%s tmdb=%s req#%s" % (md.get("mediaType", "?"), md.get("tmdbId", "?"), rid) - n = int(tries.get(str(rid), 0)) - if SEERR_MAX_TRIES and n >= SEERR_MAX_TRIES: # keeps failing -> stop, leave it for a human - log.error("[seerr] giving up on %s after %d retries (persistent failure)", label, n) - continue - if DRY_RUN: - log.info("[seerr] DRY-RUN would retry %s", label); acted += 1; continue - try: - s.retry(rid) - tries[str(rid)] = n + 1 - acted += 1 - log.info("[seerr] retried %s (attempt %d)", label, n + 1) - except Exception as e: - log.warning("[seerr] retry %s failed: %s", label, str(e)[:80]) - # a recovered request drops off the failed list; forget its counter so a future fresh fail starts clean - live = set(str(r.get("id")) for r in reqs) - for k in [k for k in tries if k not in live]: - tries.pop(k, None) - _save_state(state) - if acted: - log.info("[seerr] re-drove %d failed request(s)", acted) - -# =========================================================================== # -# WARMER: precache the head of likely-next media so playback starts instantly -# -# On a usenet/debrid FUSE mount the slow part of pressing Play is decypharr -# fetching the first segments from the provider. We ask Plex what a viewer is -# about to watch (the next episode of whatever is playing, plus everything in -# their On Deck / Continue Watching row) and read the first WARMER_PRECACHE_MB -# of each through the mount, which pulls those bytes into decypharr's on-disk -# cache. By the time Play is pressed, the head is already warm. -# -# Plex exposes no "user opened the detail page" event, so we approximate intent -# with the high-hit-rate signals it DOES expose (active sessions + On Deck). -# We do not force-delete warmed bytes: decypharr's cache is itself the speed -# win and it already evicts by age/LRU; instead we keep speculative cost low -# (small head, a per-cycle cap, a re-warm cooldown, and a host-load guard). -# =========================================================================== # - -class Plex: - def __init__(self, url, token): - self.url = url.rstrip("/"); self.token = token - - def _get(self, path): - sep = "&" if "?" in path else "?" - with urllib.request.urlopen(self.url + path + sep + "X-Plex-Token=" + self.token, timeout=15) as r: - return ET.fromstring(r.read()) - - def sessions(self): - try: return list(self._get("/status/sessions").iter("Video")) - except Exception: return [] - - def ondeck(self): - try: return list(self._get("/library/onDeck").iter("Video")) - except Exception: return [] - - def leaves(self, show_rk): - try: return list(self._get("/library/metadata/%s/allLeaves" % show_rk).iter("Video")) - except Exception: return [] - - def parts(self, rk): - """File paths for this item, highest-resolution version first (so we can warm just the top one).""" - out = [] - try: - for m in self._get("/library/metadata/%s" % rk).iter("Media"): - try: res = int(m.get("height") or 0) * 1000000 + int(m.get("bitrate") or 0) - except Exception: res = 0 - for p in m.iter("Part"): - if p.get("file"): - out.append((res, p.get("file"))) - out.sort(key=lambda x: x[0], reverse=True) - except Exception: - return [] - return [f for _, f in out] - - def recent(self, n): - out = [] - try: - for d in self._get("/library/sections").iter("Directory"): - if d.get("type") in ("movie", "show"): - ra = self._get("/library/sections/%s/recentlyAdded?X-Plex-Container-Start=0&X-Plex-Container-Size=%d" % (d.get("key"), n)) - out += list(ra.iter("Video"))[:n] - except Exception: pass - return out - -_warm_state = {} # host_path -> last_warm_ts -_warm_lock = threading.Lock() -_warm_sem = threading.Semaphore(max(1, WARM_CONCURRENCY)) # background warming lane -_warm_sem_open = threading.Semaphore(max(1, WARM_OPEN_CONC)) # detail-page (you opened it) lane - separate so opens never wait -_warm_last_ondeck = [0.0] -_warm_count = [0] # total warms since start (for the UI) -_warm_recent = [] # recent warms for the UI: [{"ts","title","why"}] - -def _warm_record(title, why): - _warm_count[0] += 1 - _warm_recent.append({"ts": time.time(), "title": title, "why": why}) - if len(_warm_recent) > 80: - del _warm_recent[:len(_warm_recent) - 80] - -def _limit_parts(files): - return files if WARM_PARTS <= 0 else files[:WARM_PARTS] - -def _host_path(f): - if WARM_PATH_MAP and ":" in WARM_PATH_MAP: - a, b = WARM_PATH_MAP.split(":", 1) - if f.startswith(a): - return b + f[len(a):] - return f - -def _warm_file(path, reason="cycle"): - p = _host_path(path) - # a title you actively opened tolerates more load (2x) than speculative background warming, but - # both still yield before meltdown; concurrency stays capped either way so a burst can't flood. - guard = (WARM_LOAD_MAX * 2) if reason == "detail-page" else WARM_LOAD_MAX - if guard > 0 and host_load() > guard: - return False - with _warm_lock: # atomic claim: one warm per file per cooldown - if time.time() - _warm_state.get(p, 0) < WARM_COOLDOWN: - return False - _warm_state[p] = time.time() - try: - sz = os.path.getsize(p) - except Exception as e: - _warm_state.pop(p, None) # release so it can be retried - log.debug("[warmer] stat fail %s: %s", p, str(e)[:60]); return False - head = min(WARM_HEAD_MB << 20, sz) - tail = WARM_TAIL_MB > 0 and sz > head + (WARM_TAIL_MB << 20) - res = {"got": 0, "err": None} - def _do(): - try: - with open(p, "rb", buffering=0) as fh: - while res["got"] < head: - b = fh.read(min(4 << 20, head - res["got"])) - if not b: break - res["got"] += len(b) - if tail: - fh.seek(sz - (WARM_TAIL_MB << 20)) - while fh.read(4 << 20): - pass - except Exception as e: - res["err"] = str(e)[:60] - t0 = time.time() - sem = _warm_sem_open if reason == "detail-page" else _warm_sem # opens get their own lane (instant) - with sem: # cap concurrent usenet pulls so warming never floods decypharr - th = threading.Thread(target=_do, daemon=True); th.start(); th.join(WARM_READ_TIMEOUT) - if th.is_alive(): - _warm_state.pop(p, None) - log.warning("[warmer] read timed out (%ds, mount slow/hung?): %s", WARM_READ_TIMEOUT, os.path.basename(p)) - return False - if res["err"]: - _warm_state.pop(p, None) - log.warning("[warmer] read fail %s: %s", os.path.basename(p), res["err"]); return False - _warm_record(os.path.basename(p), reason) - log.info("[warmer] warmed %dMB head%s in %.1fs: %s", - res["got"] >> 20, "+%dMB tail" % WARM_TAIL_MB if tail else "", - time.time() - t0, os.path.basename(p)) - return True - -def _warm_targets(plex): - """Ordered, de-duped list of (reason, plex_file_path) to warm this cycle.""" - targets, seen = [], set() - def add(reason, path): - if path and path not in seen: - seen.add(path); targets.append((reason, path)) - sessions = plex.sessions() - if "next" in WARM_SOURCES: # next episode(s) of anything playing - for v in sessions: - if v.get("type") != "episode" or not v.get("grandparentRatingKey"): - continue - if WARM_NEXT_NEAR_END > 0: # only warm the next ep once the current one nears the end - try: - remain_min = (int(v.get("duration", 0)) - int(v.get("viewOffset", 0))) / 60000.0 - except Exception: - remain_min = 0 - if remain_min > WARM_NEXT_NEAR_END: - continue - eps = plex.leaves(v.get("grandparentRatingKey")) - idx = next((i for i, e in enumerate(eps) if e.get("ratingKey") == v.get("ratingKey")), -1) - if idx >= 0: - for e in eps[idx + 1: idx + 1 + WARM_NEXT_EPS]: - for f in _limit_parts(plex.parts(e.get("ratingKey"))): - add("next-ep", f) - # Plex-first: speculative On Deck / recent warming pauses while ANYONE is watching (never competes - # with a live stream), and is skipped entirely in low-cache mode (keep almost nothing pre-warmed). - if not WARM_LOW_CACHE and not sessions and time.time() - _warm_last_ondeck[0] >= WARM_ONDECK_EVERY: - _warm_last_ondeck[0] = time.time() - if WARM_ONDECK and "ondeck" in WARM_SOURCES: # Continue Watching / Up Next (WARMER_ONDECK is the on/off) - for v in plex.ondeck(): - for f in _limit_parts(plex.parts(v.get("ratingKey"))): - add("ondeck", f) - if "recent" in WARM_SOURCES and WARM_RECENT_COUNT > 0: - for v in plex.recent(WARM_RECENT_COUNT): - for f in _limit_parts(plex.parts(v.get("ratingKey"))): - add("recent", f) - return targets - -def warm_cycle(): - if WARM_LOAD_MAX > 0 and host_load() > WARM_LOAD_MAX: - log.info("[warmer] host load > %.0f -> skip cycle", WARM_LOAD_MAX); return - targets = _warm_targets(Plex(PLEX_URL, PLEX_TOKEN)) - done = 0 - for reason, path in targets: - if done >= WARM_MAX_CYCLE: - break - if _warm_file(path, reason): - done += 1 - if done: - log.info("[warmer] cycle warmed %d (of %d candidate paths)", done, len(targets)) - -def warmer_loop(stop): - mode = (" | LOW-CACHE: no On Deck, next ep @<=%dmin left" % WARM_NEXT_NEAR_END) if WARM_LOW_CACHE \ - else ((" | next ep @<=%dmin left" % WARM_NEXT_NEAR_END) if WARM_NEXT_NEAR_END else "") - log.info("[warmer] started: head=%dMB tail=%dMB sources=%s poll=%ds ondeck-every=%ds%s", - WARM_HEAD_MB, WARM_TAIL_MB, ",".join(WARM_SOURCES) or "-", WARM_INTERVAL, WARM_ONDECK_EVERY, mode) - while not stop.is_set(): - try: - warm_cycle() - except Exception as e: - log.error("[warmer] cycle error: %s", e) - if stop.wait(WARM_INTERVAL): - break - -# opening a title's detail page fetches its extras (/extras, every client incl. Infuse) and, on the -# native Plex app, a rich includeExtras=1 metadata request. Match either -> works for Plex + Infuse. -_PLEXLOG_RE = re.compile(r"/library/metadata/(\d+)(?:/extras|\?[^\s]*includeExtras=1)") - -_playing = {"ts": 0.0, "rks": set()} - -def _playing_rks(plex): - """ratingKeys with an active Plex session, cached ~10s (Plex sends the same metadata query while - you browse a title AND while you play it, so this tells the two apart).""" - if time.time() - _playing["ts"] > 10: - try: _playing["rks"] = set(v.get("ratingKey") for v in plex.sessions()) - except Exception: pass - _playing["ts"] = time.time() - return _playing["rks"] - -def _warm_opened(plex, rk): - if rk in _playing_rks(plex): # already playing (so already cached) -> not a new open - return - for f in _limit_parts(plex.parts(rk)): # warm just the top version(s) you'd actually play - if _warm_file(f, "detail-page"): - log.info("[warmer] you opened rk=%s -> warmed: %s", rk, os.path.basename(_host_path(f))) - -def plexlog_loop(stop): - """Tail Plex's server log; warm the exact title a viewer opens (true pre-play intent).""" - cmd = WARM_PLEXLOG_CMD or ("tail -n0 -F %r" % WARM_PLEXLOG_FILE if WARM_PLEXLOG_FILE else "") - if not cmd: - return - plex = Plex(PLEX_URL, PLEX_TOKEN) - seen = {} # ratingKey -> last-handled ts - log.info("[warmer] detail-page warming enabled (tailing Plex log)") - while not stop.is_set(): - proc = None - try: - proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, text=True, bufsize=1) - for line in proc.stdout: - if stop.is_set(): - break - m = _PLEXLOG_RE.search(line) - if not m: - continue - rk = m.group(1); now = time.time() - if now - seen.get(rk, 0) < 300: # a detail page is polled repeatedly while open -> react once per item / 5 min - continue - seen[rk] = now # warm off-thread so the tailer stays responsive - threading.Thread(target=_warm_opened, args=(plex, rk), daemon=True).start() - except Exception as e: - log.warning("[warmer] plexlog tail error: %s", str(e)[:80]) - finally: - if proc: - try: proc.terminate() - except Exception: pass - if stop.wait(10): # tail died/rotated -> reconnect - break - -# =========================================================================== # -# sweep / loop -# =========================================================================== # - -# =========================================================================== # -# westrepair - symlink repair subprocess + background monitor thread -# =========================================================================== # - -_wr_lock = threading.Lock() -_wr_state = { - "running": False, "pid": None, - "current_item": None, "current_mode": None, - "items_processed": 0, "items_broken": 0, "items_fixed": 0, - "last_action": None, "last_run_start": None, "next_run_in": None, - "recent_log": [], - "exit_code": None, -} -_wr_proc = None - -_RE_WR_PROCESSING = re.compile(r'\[(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\] \[(\w+)\] \[DEBUG\] Processing: (.+)') -_RE_WR_BROKEN = re.compile(r'\[DEBUG\] .*(broken|missing|not found|unreachable)', re.IGNORECASE) -_RE_WR_FIXED = re.compile(r'\[(INFO|SUCCESS)\] .*(search|trigger|fix|repair|restor)', re.IGNORECASE) -_RE_WR_SLEEPING = re.compile(r'[Ss]leeping for ([^\n]+)') -_RE_WR_START = re.compile(r'Running repair') - - -def _wr_parse_line(line): - s = _wr_state - s["recent_log"].append(line.rstrip()) - if len(s["recent_log"]) > 20: - s["recent_log"].pop(0) - m = _RE_WR_PROCESSING.search(line) - if m: - s["current_item"] = m.group(3).strip() - s["current_mode"] = m.group(2) - s["items_processed"] += 1 - return - if _RE_WR_BROKEN.search(line): - s["items_broken"] += 1; s["last_action"] = line.strip(); return - if _RE_WR_FIXED.search(line): - s["items_fixed"] += 1; s["last_action"] = line.strip(); return - m2 = _RE_WR_SLEEPING.search(line) - if m2: - s["next_run_in"] = m2.group(1).strip(); s["current_item"] = None; return - if _RE_WR_START.search(line): - s["last_run_start"] = line.strip() - s["items_processed"] = s["items_broken"] = s["items_fixed"] = 0 - - -def westrepair_loop(stop): - """Run repair.py as a long-lived subprocess; restart on unexpected exit.""" - global _wr_proc - if not os.path.exists(WR_SCRIPT): - log.error("[westrepair] script not found: %s", WR_SCRIPT) - return - log.info("[westrepair] starting %s | run_interval=%s repair_interval=%s", - WR_SCRIPT, WR_RUN_INTERVAL, WR_REPAIR_INTERVAL) - while not stop.is_set(): - cmd = ["python", "-u", WR_SCRIPT, "--no-confirm", - "--run-interval", WR_RUN_INTERVAL, - "--repair-interval", WR_REPAIR_INTERVAL] - try: - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - text=True, bufsize=1, cwd=os.path.dirname(WR_SCRIPT)) - _wr_proc = proc - with _wr_lock: - _wr_state.update({"running": True, "pid": proc.pid, "exit_code": None}) - for line in proc.stdout: - log.info("[westrepair] %s", line.rstrip()) - with _wr_lock: - _wr_parse_line(line) - if stop.is_set(): - break - proc.wait() - with _wr_lock: - _wr_state.update({"running": False, "exit_code": proc.returncode}) - if stop.is_set(): - break - log.warning("[westrepair] exited (code %d), restarting in 30s", proc.returncode) - stop.wait(30) - except Exception as e: - log.error("[westrepair] error: %s", e) - stop.wait(30) - if _wr_proc and _wr_proc.poll() is None: - try: _wr_proc.terminate() - except Exception: pass - log.info("[westrepair] stopped") - - -def check_westrepair(): - """No-op periodic check — westrepair runs continuously in its own thread.""" - with _wr_lock: - s = dict(_wr_state) - if s["running"]: - log.debug("[westrepair] running pid=%s processed=%d broken=%d fixed=%d", - s["pid"], s["items_processed"], s["items_broken"], s["items_fixed"]) - else: - log.warning("[westrepair] repair.py not running (exit_code=%s)", s["exit_code"]) - - -def _wr_plex_rescan(): - """Trigger a Plex library refresh for all sections. Returns (ok, message).""" - plex_url = os.environ.get("PLEX_URL", "").rstrip("/") - plex_token = os.environ.get("PLEX_TOKEN", "") - if not plex_url or not plex_token: - return False, "PLEX_URL or PLEX_TOKEN not set" - # Get library sections - sections_url = "%s/library/sections?X-Plex-Token=%s" % (plex_url, plex_token) - try: - with urllib.request.urlopen(urllib.request.Request(sections_url), timeout=10) as r: - import xml.etree.ElementTree as ET - root = ET.fromstring(r.read()) - except Exception as e: - return False, "could not fetch sections: %s" % str(e)[:80] - keys = [d.get("key") for d in root.findall(".//Directory") if d.get("key")] - if not keys: - return False, "no library sections found" - triggered = [] - for key in keys: - scan_url = "%s/library/sections/%s/refresh?X-Plex-Token=%s" % (plex_url, key, plex_token) - try: - urllib.request.urlopen(urllib.request.Request(scan_url), timeout=10) - triggered.append(key) - except Exception as e: - log.warning("[westrepair] plex scan section %s failed: %s", key, e) - log.info("[westrepair] triggered Plex rescan for %d section(s): %s", len(triggered), triggered) - return True, "triggered %d section(s)" % len(triggered) - - -CHECKS = [("queue", EN_QUEUE, check_queue), ("providers", EN_PROVIDERS, check_providers), - ("decypharr", EN_DECYPHARR, check_decypharr), ("plex", EN_PLEX, check_plex), - ("resources", EN_RESOURCES, check_resources), ("janitor", EN_JANITOR, check_janitor), - ("bazarr", EN_BAZARR, check_bazarr), ("seerr", EN_SEERR, check_seerr), - ("westrepair", EN_WESTREPAIR, check_westrepair)] - -_lock = threading.Lock() - -def sweep(only=None): - if not _lock.acquire(blocking=False): - log.debug("sweep already running"); return - try: - for cid, en, fn in CHECKS: - if not en: - continue - try: - fn(only) if cid == "queue" else fn() - except Exception as e: - log.error("[%s] check error: %s", cid, e) - finally: - _lock.release() - -# =========================================================================== # -# web dashboard (optional, no dependencies): status + per-service health + -# warmer stats + editable tuning config + live logs. Secrets stay masked. -# =========================================================================== # - -_SECRET_HINT = ("APIKEY", "API_KEY", "TOKEN", "PASSWORD", "PASS", "SECRET") - -UI_SCHEMA = [ - ("Mode", [("DOCTOR_MODE", "cron|event"), ("DOCTOR_INTERVAL", "900"), - ("DOCTOR_DRY_RUN", "false"), ("DOCTOR_LOG_LEVEL", "INFO")]), - ("Checks (on/off)", [("ENABLE_QUEUE", ""), ("ENABLE_PROVIDERS", ""), ("ENABLE_DECYPHARR", ""), - ("ENABLE_PLEX", ""), ("ENABLE_RESOURCES", ""), ("ENABLE_JANITOR", ""), - ("ENABLE_BAZARR", ""), ("ENABLE_SEERR", ""), ("ENABLE_WARMER", ""), ("ENABLE_WESTREPAIR", "")]), - ("Westrepair", [("WESTREPAIR_SCRIPT", "/app/westrepair/repair.py"), - ("WESTREPAIR_RUN_INTERVAL", "6h"), ("WESTREPAIR_REPAIR_INTERVAL", "1m")]), - ("Queue / churn brake", [("DOCTOR_MIN_STRIKES", "2"), ("DOCTOR_MAX_ACTIONS", "20"), ("DOCTOR_BLOCKLIST", "true"), - ("DOCTOR_CHURN_LIMIT", "0"), ("DOCTOR_CHURN_ACTION", "report|park|backoff"), ("DOCTOR_CHURN_BACKOFF", "10m,1h,24h")]), - ("Warmer", [("WARMER_PRECACHE_MB", "64"), ("WARMER_TAIL_MB", "8"), ("WARMER_SOURCES", "ondeck,next"), - ("WARMER_ONDECK", "true|false"), ("WARMER_MAX_PER_CYCLE", "40"), ("WARMER_NEXT_EPISODES", "1"), - ("WARMER_COOLDOWN", "3600"), ("WARMER_LOAD_MAX", "0")]), - ("Resources", [("RES_LOAD_WARN", "40"), ("RES_SWAP_WARN_MB", "7000"), ("RES_MEM_MIN_MB", "800")]), - ("Seerr (failed-request retry)", [("SEERR_URL", "http://seerr:5055"), ("SEERR_RETRY_MAX", "10"), ("SEERR_MAX_ATTEMPTS", "5")]), -] -UI_KEYS = set(k for _, items in UI_SCHEMA for k, _ in items) - -def _is_secret(k): - ku = k.upper() - return any(h in ku for h in _SECRET_HINT) - -def _ui_health(): - """Quick reachability of every monitored service, probed in parallel (short timeouts).""" - def arr_probe(a): - def f(): - st = json.load(a._req("GET", "/system/status", t=5)) - warns = [h for h in a.health() if h.get("type") in ("warning", "error")] - return True, ("v%s" % st.get("version", "?")) + (", %d health warn" % len(warns) if warns else "") - return f - jobs = [(a.name, a.kind, arr_probe(a)) for a in INSTANCES] - if DECY_URL: - jobs.append(("decypharr", "mount", lambda: (http_code(DECY_URL, t=5) == 200, DECY_URL))) - if PLEX_URL: - jobs.append(("plex", "plex", lambda: ( - http_code(PLEX_URL.rstrip("/") + "/identity" + ("?X-Plex-Token=" + PLEX_TOKEN if PLEX_TOKEN else ""), t=5) == 200, ""))) - if BAZARR_URL: - jobs.append(("bazarr", "bazarr", lambda: (http_code(BAZARR_URL.rstrip("/") + "/api/system/status", - headers={"X-API-KEY": BAZARR_APIKEY} if BAZARR_APIKEY else None, t=5) == 200, ""))) - if SEERR_URL: - jobs.append(("seerr", "seerr", lambda: (http_code(SEERR_URL.rstrip("/") + "/api/v1/status", - headers={"X-Api-Key": SEERR_APIKEY} if SEERR_APIKEY else None, t=5) == 200, ""))) - out = [None] * len(jobs) - def run(i, name, kind, fn): - try: - up, detail = fn() - except Exception as e: - up, detail = False, str(e)[:46] - out[i] = {"name": name, "kind": kind, "up": up, "detail": detail} - ths = [threading.Thread(target=run, args=(i, n, k, fn), daemon=True) for i, (n, k, fn) in enumerate(jobs)] - for t in ths: t.start() - for t in ths: t.join(7) - return [r for r in out if r] - -def _ui_status(): - checks = [{"name": n, "on": bool(e)} for n, e, _ in CHECKS] - checks.append({"name": "warmer", "on": _b("ENABLE_WARMER", False) and bool(PLEX_URL)}) - checks.append({"name": "detail-page warm", "on": bool(WARM_PLEXLOG_CMD or WARM_PLEXLOG_FILE)}) - return {"version": VERSION, "mode": MODE, "dry_run": DRY_RUN, "load": round(host_load(), 2), "checks": checks} - -def _ui_warmer(): - rec = [{"title": r["title"], "why": r["why"], "ago": int(time.time() - r["ts"])} for r in reversed(_warm_recent)] - return {"enabled": _b("ENABLE_WARMER", False) and bool(PLEX_URL), - "detail_page": bool(WARM_PLEXLOG_CMD or WARM_PLEXLOG_FILE), - "total": _warm_count[0], "recent": rec[:40]} - -def _ui_westrepair(): - with _wr_lock: - s = dict(_wr_state) - s["recent_log"] = list(_wr_state["recent_log"]) - s["enabled"] = EN_WESTREPAIR - return s - -def _ui_config(): - groups = [] - for g, items in UI_SCHEMA: - rows = [{"key": k, "val": ("" if _is_secret(k) else os.environ.get(k, "")), "ph": ph, "secret": _is_secret(k)} - for k, ph in items] - groups.append({"group": g, "rows": rows}) - return {"groups": groups, "file": CONFIG_FILE} - -def _ui_save(body): - try: - incoming = json.loads(body or b"{}") - except Exception: - return False, "bad json" - try: - ov = json.load(open(CONFIG_FILE)) - except Exception: - ov = {} - n = 0 - for k, v in incoming.items(): - if k in UI_KEYS and not _is_secret(k): - ov[k] = v; os.environ[str(k)] = str(v); n += 1 - try: - os.makedirs(os.path.dirname(CONFIG_FILE) or ".", exist_ok=True) - json.dump(ov, open(CONFIG_FILE, "w"), indent=1) - except Exception as e: - return False, str(e)[:80] - return True, "saved %d (restart to apply)" % n - -def _ui_logs(n): - if not LOG_FILE: - return "(set DOCTOR_LOG_FILE to view logs here)" - try: - return "".join(open(LOG_FILE, errors="ignore").readlines()[-n:]) - except Exception as e: - return "log read error: " + str(e)[:80] - -UI_HTML = r""" -stack-doctor -

stack-doctor

loading
- -
-
-

Checks

-

Monitored services

-

Warmer

- -
- - -
-""" - -def _build_server(port): - from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer - from urllib.parse import urlparse, parse_qs - class H(BaseHTTPRequestHandler): - def _send(self, code, ctype, body): - if isinstance(body, str): - body = body.encode("utf-8") - self.send_response(code); self.send_header("Content-Type", ctype) - self.send_header("Content-Length", str(len(body))); self.end_headers() - try: self.wfile.write(body) - except Exception: pass - def _authed(self): - if not UI_TOKEN: - return True - q = parse_qs(urlparse(self.path).query) - return self.headers.get("X-Doctor-Token") == UI_TOKEN or q.get("token", [""])[0] == UI_TOKEN - def do_GET(self): - path = urlparse(self.path).path - if path in ("/health", "/healthz"): - return self._send(200, "text/plain", "ok") - if not EN_UI: - return self._send(404, "text/plain", "nf") - if not self._authed(): - return self._send(401, "text/plain", "unauthorized") - if path in ("/", "/ui", "/index.html"): - return self._send(200, "text/html; charset=utf-8", UI_HTML) - if path == "/api/status": return self._send(200, "application/json", json.dumps(_ui_status())) - if path == "/api/health": return self._send(200, "application/json", json.dumps(_ui_health())) - if path == "/api/warmer": return self._send(200, "application/json", json.dumps(_ui_warmer())) - if path == "/api/westrepair": return self._send(200, "application/json", json.dumps(_ui_westrepair())) - if path == "/api/config": return self._send(200, "application/json", json.dumps(_ui_config())) - if path == "/api/logs": - try: n = min(int(parse_qs(urlparse(self.path).query).get("n", ["300"])[0]), 3000) - except Exception: n = 300 - return self._send(200, "text/plain; charset=utf-8", _ui_logs(n)) - return self._send(404, "text/plain", "nf") - def do_POST(self): - path = urlparse(self.path).path - length = int(self.headers.get("Content-Length", 0) or 0) - body = self.rfile.read(length) if length else b"" - if path in ("/api/config", "/api/restart", "/api/westrepair/rescan"): - if not EN_UI or not self._authed(): - return self._send(401, "text/plain", "unauthorized") - if path == "/api/config": - ok, msg = _ui_save(body) - return self._send(200 if ok else 400, "application/json", json.dumps({"ok": ok, "msg": msg})) - if path == "/api/westrepair/rescan": - threading.Thread(target=lambda: _wr_plex_rescan(), daemon=True).start() - return self._send(200, "application/json", json.dumps({"ok": True, "msg": "Plex rescan triggered"})) - self._send(200, "application/json", json.dumps({"ok": True, "msg": "restarting"})) - log.info("[ui] restart requested"); threading.Thread(target=lambda: (time.sleep(0.4), os._exit(0)), daemon=True).start() - return - if MODE == "event": # arr webhook - try: p = json.loads(body or b"{}") - except Exception: p = {} - ev = p.get("eventType") or p.get("EventType") or "?"; inst = p.get("instanceName") or p.get("InstanceName") - self._send(200, "text/plain", "ok") - if ev == "Test": - log.info("webhook Test from %s", inst or "?"); return - if TRIGGER_EVENTS and ev not in TRIGGER_EVENTS: - return - log.info("event '%s' from %s -> sweep", ev, inst or "all") - threading.Thread(target=sweep, kwargs={"only": inst}, daemon=True).start(); return - self._send(404, "text/plain", "nf") - def log_message(self, *a): - pass - return ThreadingHTTPServer(("0.0.0.0", port), H) - -def main(): - global INSTANCES - INSTANCES = load_instances() - enabled = [c for c, e, _ in CHECKS if e] - warmer_on = EN_WARMER and bool(PLEX_URL) - if EN_WARMER and not PLEX_URL: - log.warning("ENABLE_WARMER set but PLEX_URL is empty -> warmer disabled") - if EN_QUEUE and not INSTANCES: - log.error("queue check enabled but no instances. Set INSTANCE_1_URL / _APIKEY / _TYPE.") - sys.exit(2) - if not enabled and not warmer_on and not EN_UI: - log.error("nothing enabled. Set ENABLE_QUEUE / ENABLE_DECYPHARR / ENABLE_PLEX / ENABLE_RESOURCES / ENABLE_JANITOR / ENABLE_WARMER / ENABLE_UI.") - sys.exit(2) - log.info("stack-doctor v%s | mode=%s | checks=[%s]%s%s | instances=%s | dry_run=%s", - VERSION, MODE, ",".join(enabled), " +warmer" if warmer_on else "", " +ui" if EN_UI else "", - ", ".join(a.name for a in INSTANCES) or "-", DRY_RUN) - - stop = threading.Event() - signal.signal(signal.SIGTERM, lambda *a: stop.set()) - signal.signal(signal.SIGINT, lambda *a: stop.set()) - - if warmer_on: - threading.Thread(target=warmer_loop, args=(stop,), daemon=True).start() - if WARM_PLEXLOG_CMD or WARM_PLEXLOG_FILE: - threading.Thread(target=plexlog_loop, args=(stop,), daemon=True).start() - - if EN_WESTREPAIR: - threading.Thread(target=westrepair_loop, args=(stop,), daemon=True).start() - - # http server(s): arr webhooks (event mode) and/or the web dashboard (ENABLE_UI) - servers, wanted = [], {} - if MODE == "event": - wanted[PORT] = "webhooks" - if EN_UI: - wanted[UI_PORT] = (wanted.get(UI_PORT, "") + "+dashboard").lstrip("+") - for pnum, what in wanted.items(): - try: - s = _build_server(pnum) - threading.Thread(target=s.serve_forever, daemon=True).start() - servers.append(s); log.info("http on :%d (%s)", pnum, what) - except Exception as e: - log.error("http bind :%d failed: %s", pnum, e) - - sweep() - interval = max(INTERVAL, 1800) if MODE == "event" else INTERVAL - while not stop.wait(interval): - sweep() - for s in servers: - try: s.shutdown() - except Exception: pass - log.info("stack-doctor stopped") - -if __name__ == "__main__": - main() diff --git a/doctor/__init__.py b/doctor/__init__.py new file mode 100644 index 0000000..b78018d --- /dev/null +++ b/doctor/__init__.py @@ -0,0 +1,2 @@ +"""stack-doctor package.""" +from .config import VERSION # noqa: F401 diff --git a/doctor/__main__.py b/doctor/__main__.py new file mode 100644 index 0000000..3359099 --- /dev/null +++ b/doctor/__main__.py @@ -0,0 +1,76 @@ +"""Entry point: python -m doctor.""" +import sys +import signal +import threading +from .config import ( + DRY_RUN, + EN_UI, + EN_WARMER, + MODE, + PLEX_URL, + PORT, + UI_PORT, + VERSION, + WARM_PLEXLOG_CMD, + WARM_PLEXLOG_FILE, + log, +) +from .clients import INSTANCES, load_instances +from .checks import backfill_missing_seasons, plexlog_loop, warmer_loop +from .scheduler import CHECKS, scheduler_loop +from .webui import _build_server + +def main(): + import doctor.clients as _clients + _clients.INSTANCES[:] = load_instances() + if "--backfill-missing-seasons" in sys.argv: + sys.argv.remove("--backfill-missing-seasons") + backfill_missing_seasons() + enabled = [c for c, e, _, _, _, _ in CHECKS if e] + warmer_on = EN_WARMER and bool(PLEX_URL) + if EN_WARMER and not PLEX_URL: + log.warning("ENABLE_WARMER set but PLEX_URL is empty -> warmer disabled") + _needs_instances = [cid for cid, en, _, _, _, needs in CHECKS if en and needs] + if _needs_instances and not INSTANCES: + log.error("checks %s require at least one instance. Set INSTANCE_1_URL / _APIKEY / _TYPE.", + _needs_instances) + sys.exit(2) + if not enabled and not warmer_on and not EN_UI: + log.error("nothing enabled. Set ENABLE_QUEUE / ENABLE_DECYPHARR / ENABLE_PLEX / ENABLE_PLEX_SCAN / " + "ENABLE_RESOURCES / ENABLE_JANITOR / ENABLE_REPAIR / ENABLE_WARMER / ENABLE_UI.") + sys.exit(2) + log.info("stack-doctor v%s | mode=%s | checks=[%s]%s%s | instances=%s | dry_run=%s", VERSION, + MODE, ",".join(enabled), " +warmer" if warmer_on else "", " +ui" if EN_UI else "", + ", ".join(a.name for a in INSTANCES) or "-", DRY_RUN) + + stop = threading.Event() + signal.signal(signal.SIGTERM, lambda *a: stop.set()) + signal.signal(signal.SIGINT, lambda *a: stop.set()) + + if warmer_on: + threading.Thread(target=warmer_loop, args=(stop,), daemon=True).start() + if WARM_PLEXLOG_CMD or WARM_PLEXLOG_FILE: + threading.Thread(target=plexlog_loop, args=(stop,), daemon=True).start() + + # http server(s): arr webhooks (event mode) and/or the web dashboard (ENABLE_UI) + servers, wanted = [], {} + if MODE == "event": + wanted[PORT] = "webhooks" + if EN_UI: + wanted[UI_PORT] = (wanted.get(UI_PORT, "") + "+dashboard").lstrip("+") + for pnum, what in wanted.items(): + try: + s = _build_server(pnum) + threading.Thread(target=s.serve_forever, daemon=True).start() + servers.append(s); log.info("http on :%d (%s)", pnum, what) + except Exception as e: + log.error("http bind :%d failed: %s", pnum, e) + + scheduler_loop(stop) + for s in servers: + try: s.shutdown() + except Exception: pass + log.info("stack-doctor stopped") + +if __name__ == "__main__": + main() diff --git a/doctor/checks/__init__.py b/doctor/checks/__init__.py new file mode 100644 index 0000000..24ccde6 --- /dev/null +++ b/doctor/checks/__init__.py @@ -0,0 +1,54 @@ +"""stack-doctor checks package. + +This module explicitly re-exports the check entry points and the small number +of auxiliary functions used by the rest of the package. The wildcard exports +from individual submodules are no longer re-exported here, so the public surface +of doctor.checks is now well-defined. +""" +from .queue import check_queue +from .providers import check_providers +from .decypharr import check_decypharr +from .decypharr_providers import check_decypharr_providers +from .plex import check_plex +from .plexscan import check_plex_scan +from .rescan import check_rescan +from .resources import check_resources +from .janitor import check_janitor +from .bazarr import check_bazarr +from .seerr import check_seerr +from .repair import check_repair +from .debridlink_migration import check_debridlink_migration +from .force_import import check_force_import +from .warmer import warmer_loop, plexlog_loop +from .missing_seasons import check_missing_seasons, backfill_missing_seasons +from .no_upgrade import check_no_upgrade_profile +from .multipack import check_multipack +from .maintainer import check_maintainer + +__all__ = [ + "check_bazarr", + "check_decypharr", + "check_decypharr_providers", + "check_debridlink_migration", + "check_force_import", + "check_janitor", + "check_maintainer", + "check_missing_seasons", + "check_maintainer", + "check_multipack", + "check_no_upgrade_profile", + "check_plex", + "check_plex_scan", + "check_providers", + "check_rescan", + "check_queue", + "check_repair", + "check_resources", + "check_seerr", + "backfill_missing_seasons", + "warmer_loop", + "backfill_missing_seasons", + "check_maintainer", + "warmer_loop", + "plexlog_loop", +] diff --git a/doctor/checks/bazarr.py b/doctor/checks/bazarr.py new file mode 100644 index 0000000..436f994 --- /dev/null +++ b/doctor/checks/bazarr.py @@ -0,0 +1,9 @@ +"""Check: bazarr.""" +from ..config import BAZARR_APIKEY, BAZARR_URL, http_code, log + +def check_bazarr(): + if not BAZARR_URL: + return + c = http_code(BAZARR_URL.rstrip("/") + "/api/system/status", + headers={"X-API-KEY": BAZARR_APIKEY} if BAZARR_APIKEY else None, t=10) + (log.info if c == 200 else log.error)("[bazarr] %s -> %s", BAZARR_URL, c if c else "DOWN") diff --git a/doctor/checks/decypharr.py b/doctor/checks/decypharr.py new file mode 100644 index 0000000..bc2e642 --- /dev/null +++ b/doctor/checks/decypharr.py @@ -0,0 +1,442 @@ +"""Check: decypharr + FUSE mount health. + +Three-layer health probe for a FUSE (rclone/zurg) mount: + + 1. Kernel mount table - /proc/mounts confirms the mountpoint is still + registered with the kernel (walks ancestors so + DECYPHARR_MOUNT_TEST can be a sub-path like + /mnt/zurg/__all__ when the mount is at /mnt/zurg). + 2. statvfs liveness - os.statvfs() on the mountpoint returns instantly + with ENOTCONN / EIO when FUSE is dead; it does NOT + hang like open() can. + 3. File read test - actually reads a few bytes from a real media file so + we know data flows end-to-end. Runs in a thread + with a configurable timeout so a hung mount doesn't + block the check. + +A configurable strike counter (DECYPHARR_FUSE_STRIKES, default 2) requires +consecutive failures before the restart hook is called, avoiding restarts +on single transient errors. +""" +import os +import threading +import time +import re + +from ..config import ( + DECY_FUSE_STRIKES, DECY_LINK_ERR_LOG_CMD, DECY_LINK_ERR_RESTART, + DECY_LINK_ERR_THRESHOLD, DECY_LINK_ERR_WINDOW, + DECY_MOUNT_TEST, DECY_READ_TIMEOUT, + DECY_RESTART_CMD, DECY_URL, DRY_RUN, + JAN_LOG, JAN_LOG_CMD, + http_code, run_cmd, run_output, log, +) + +# --------------------------------------------------------------------------- +# errno values that signal a dead/stuck FUSE mount +# --------------------------------------------------------------------------- +_FUSE_ERRNOS = frozenset({ + 5, # EIO - Input/output error + 6, # ENXIO - No such device or address + 107, # ENOTCONN - Transport endpoint is not connected +}) + +class _State: + """Tiny reset-able mutable cell used for module-level check state.""" + def __init__(self, default): + self._default = default + self.value = default + def reset(self): + self.value = self._default + +def _is_fuse_errno(exc): + """Return True if *exc* looks like a dead FUSE transport.""" + if not isinstance(exc, OSError): + return False + if exc.errno in _FUSE_ERRNOS: + return True + msg = str(exc).lower() + return any(s in msg for s in ( + "socket not connected", + "transport endpoint is not connected", + "input/output error", + "no such device", + )) + +# --------------------------------------------------------------------------- +# Layer 1 - kernel mount table +# --------------------------------------------------------------------------- +def _mount_registered(path): + """Return True if *path* or any of its parent directories appears in + /proc/mounts as a FUSE mountpoint. + + DECYPHARR_MOUNT_TEST is typically a subdirectory of the actual mountpoint + (e.g. /mnt/zurg/__all__ when the FUSE is mounted at /mnt/zurg), so we + walk up the path looking for a registered FUSE mount entry rather than + requiring an exact match.""" + real = os.path.realpath(path) + # Collect all FUSE mountpoints from /proc/mounts. + fuse_mounts = set() + try: + with open("/proc/mounts") as f: + for line in f: + parts = line.split() + if len(parts) >= 3 and "fuse" in parts[2]: + fuse_mounts.add(parts[1]) + except Exception: + pass + if not fuse_mounts: + return False + # Check if real path or any ancestor is a registered FUSE mount. + check = real + while True: + if check in fuse_mounts: + return True + parent = os.path.dirname(check) + if parent == check: # reached filesystem root + break + check = parent + return False + +# --------------------------------------------------------------------------- +# Layer 2 - statvfs liveness (fast, non-blocking on dead FUSE) +# --------------------------------------------------------------------------- +class _FuseStatus: + """Result of a FUSE health probe.""" + OK = "ok" + DEAD = "dead" # FUSE transport gone (ENOTCONN/EIO) + UNMOUNTED = "unmounted" # not in /proc/mounts + HUNG = "hung" # statvfs timed out + EMPTY = "empty" # mounted but no test file found + UNKNOWN = "unknown" # unexpected error + +def _probe_statvfs(path, timeout=5): + """Call os.statvfs(path) in a thread. Returns (_FuseStatus, detail_str).""" + result = {"status": _FuseStatus.UNKNOWN, "detail": ""} + def _do(): + try: + os.statvfs(path) + result["status"] = _FuseStatus.OK + except OSError as e: + if _is_fuse_errno(e): + result["status"] = _FuseStatus.DEAD + result["detail"] = "statvfs errno=%d (%s)" % (e.errno or 0, e.strerror or str(e)) + else: + result["status"] = _FuseStatus.UNKNOWN + result["detail"] = str(e) + except Exception as e: + result["status"] = _FuseStatus.UNKNOWN + result["detail"] = str(e) + th = threading.Thread(target=_do, daemon=True) + th.start(); th.join(timeout) + if th.is_alive(): + result["status"] = _FuseStatus.HUNG + result["detail"] = "statvfs blocked for >%ds" % timeout + return result["status"], result["detail"] + +# --------------------------------------------------------------------------- +# Layer 3 - file read test +# --------------------------------------------------------------------------- +def _find_media_file(path): + """Return the first media file found under *path*, or None.""" + exts = (".mkv", ".mp4", ".avi", ".m4v", ".ts") + try: + for root, _dirs, files in os.walk(path): + for fn in files: + if fn.lower().endswith(exts): + return os.path.join(root, fn) + except OSError as e: + if _is_fuse_errno(e): + raise # let the caller handle FUSE dead errors from os.walk + return None + +def _read_file(fpath, timeout): + """Read 64 KiB from *fpath* in a thread within *timeout* seconds. + Returns (_FuseStatus, detail_str).""" + result = {"status": _FuseStatus.UNKNOWN, "detail": ""} + def _do(): + try: + with open(fpath, "rb") as fh: + fh.read(65536) + result["status"] = _FuseStatus.OK + except OSError as e: + if _is_fuse_errno(e): + result["status"] = _FuseStatus.DEAD + result["detail"] = "read errno=%d (%s)" % (e.errno or 0, e.strerror or str(e)) + else: + result["status"] = _FuseStatus.UNKNOWN + result["detail"] = str(e) + except Exception as e: + result["status"] = _FuseStatus.UNKNOWN + result["detail"] = str(e) + th = threading.Thread(target=_do, daemon=True) + th.start(); th.join(timeout) + if th.is_alive(): + result["status"] = _FuseStatus.HUNG + result["detail"] = "read blocked for >%ds" % timeout + return result["status"], result["detail"] + +def _probe_mount(path, read_timeout): + """Run all three layers. Returns (_FuseStatus, detail_str).""" + # Layer 1 - kernel mount table + if not _mount_registered(path): + return _FuseStatus.UNMOUNTED, "not in /proc/mounts" + log.debug("[decypharr] mount probe layer 1 OK: %s registered in /proc/mounts", path) + + # Layer 2 - statvfs (fast dead-FUSE detector, does not hang) + status, detail = _probe_statvfs(path, timeout=5) + if status != _FuseStatus.OK: + log.debug("[decypharr] mount probe layer 2 FAIL: statvfs %s -> %s (%s)", path, status, detail) + return status, detail + log.debug("[decypharr] mount probe layer 2 OK: statvfs %s responsive", path) + + # Layer 3 - real file read + try: + fpath = _find_media_file(path) + except OSError as e: + return _FuseStatus.DEAD, "os.walk errno=%d (%s)" % (e.errno or 0, e.strerror or str(e)) + + if fpath is None: + return _FuseStatus.EMPTY, "no media file found under %s" % path + + log.debug("[decypharr] mount probe layer 3: reading %s (timeout=%ds)", fpath, read_timeout) + return _read_file(fpath, read_timeout) + +# --------------------------------------------------------------------------- +# Strike counter - require N consecutive failures before acting +# --------------------------------------------------------------------------- +_fuse_strikes = _State(0) # mutable cell updated by check_decypharr + +def _record_fuse_result(status): + """Increment/reset strike counter. Returns (strikes, needs_action).""" + if status in (_FuseStatus.OK, _FuseStatus.EMPTY): + _fuse_strikes.reset() + return 0, False + _fuse_strikes.value += 1 + return _fuse_strikes.value, _fuse_strikes.value >= DECY_FUSE_STRIKES + +# --------------------------------------------------------------------------- +# Restart hook +# --------------------------------------------------------------------------- +_decy_last_restart = _State(0.0) + +def _decy_restart(reason=""): + """Run the decypharr restart hook, rate-limited to once per 5 minutes.""" + tag = (" (%s)" % reason) if reason else "" + if DRY_RUN or not DECY_RESTART_CMD: + log.error("[decypharr] FUSE unhealthy but no restart cmd (or dry-run) -> alert only%s", tag) + return False + if time.time() - _decy_last_restart.value < 300: + log.warning("[decypharr] restart attempted <5m ago, holding off%s", tag) + return False + log.error("[decypharr] running restart hook%s: %s", tag, DECY_RESTART_CMD) + rc = run_cmd(DECY_RESTART_CMD) + _decy_last_restart.value = time.time() + log.error("[decypharr] restart hook rc=%s %s", + rc[0] if rc else "?", rc[1].strip() if (rc and rc[1]) else "") + return True + + +# --------------------------------------------------------------------------- +# Link-error cache poisoning detector +# --------------------------------------------------------------------------- +# decypharr's link/service.go caches every error from validateLink() in an +# in-memory map (s.validated). Errors returned by ErrorCodeToLinkError() for +# unknown codes (e.g. RealDebrid CDN errors: read_pxy_timeout, read_timeout, +# hoster_timeout) are classified as CategoryPermanent, so they are cached +# forever and never retried. The only way to clear the cache is a restart. +# +# This sub-check reads the decypharr log tail, counts webdav "Error streaming +# file" lines that contain known transient-but-mis-classified error strings +# within DECY_LINK_ERR_WINDOW seconds, and triggers a restart via +# DECY_RESTART_CMD when the count exceeds DECY_LINK_ERR_THRESHOLD. +# +# Patterns that indicate a poisoned cache (transient RD/debrid CDN errors that +# decypharr incorrectly caches as permanent): +_LINK_ERR_PATTERNS = re.compile( + r"Error streaming file:.*" + r"(?:read_pxy_timeout|read_timeout|hoster_timeout|hoster_unavailable" + r"|unknown error code)", + re.I, +) + +# Log-line timestamp formats decypharr uses: +# 2026-07-01 00:22:18 (space-separated date + time, no TZ) +_LOG_TS_RE = re.compile( + r"^(?:\[[0-9;]*m)?" # optional ANSI colour prefix + r"(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})" # group 1: timestamp +) + +_link_err_last_restart = _State(0.0) + + +def _parse_log_ts(line): + """Return a unix timestamp float from a decypharr log line, or None.""" + m = _LOG_TS_RE.match(line) + if not m: + return None + ts_str = m.group(1) + try: + import datetime + dt = datetime.datetime.strptime(ts_str, "%Y-%m-%d %H:%M:%S") + return dt.timestamp() + except ValueError: + return None + + +def _read_decy_log(): + """Return the decypharr log tail as a string (up to ~2 MB). + + Source priority: + 1. DECYPHARR_LINK_ERR_LOG_CMD (dedicated override) + 2. JAN_LOG_CMD (shared janitor log command) + 3. JAN_LOG (shared janitor log file path) + Falls back to "" if nothing is configured. + """ + cmd = DECY_LINK_ERR_LOG_CMD or JAN_LOG_CMD + if cmd: + return run_output(cmd) + if JAN_LOG: + try: + with open(JAN_LOG, "rb") as fh: + fh.seek(0, 2) + size = fh.tell() + fh.seek(max(0, size - 2 * 1024 * 1024)) + return fh.read().decode("utf-8", errors="replace") + except Exception as e: + log.debug("[decypharr] link_err: could not read %s: %s", JAN_LOG, e) + return "" + + +def _count_link_errors_in_window(log_data, window_secs): + """Count matching error lines whose timestamp falls within the last + *window_secs* seconds of wall-clock time. + + Returns (count, newest_ts_or_None). + """ + now = time.time() + cutoff = now - window_secs + count = 0 + newest_ts = None + for line in log_data.splitlines(): + if not _LINK_ERR_PATTERNS.search(line): + continue + ts = _parse_log_ts(line) + if ts is None or ts < cutoff: + continue + count += 1 + if newest_ts is None or ts > newest_ts: + newest_ts = ts + return count, newest_ts + + +def check_link_errors(): + """Detect a poisoned link-validation cache and restart decypharr if needed. + + Reads the decypharr log tail and counts webdav streaming errors caused by + transient provider errors (read_pxy_timeout etc.) that decypharr wrongly + caches as permanent. When the count in the rolling window exceeds + DECY_LINK_ERR_THRESHOLD the restart hook fires (if configured and enabled). + + Returns True if a restart was triggered, False otherwise. + """ + # Need a log source AND the restart cmd to do anything useful + if not (DECY_LINK_ERR_LOG_CMD or JAN_LOG_CMD or JAN_LOG): + return False + + log_data = _read_decy_log() + if not log_data: + return False + + count, newest_ts = _count_link_errors_in_window(log_data, DECY_LINK_ERR_WINDOW) + + if count == 0: + log.debug("[decypharr] link_err: 0 cached-error streaming failures in last %ds window", DECY_LINK_ERR_WINDOW) + return False + + log.info( + "[decypharr] link_err: %d transient-but-cached streaming error(s) in last %ds " + "(threshold=%d, newest=%.0fs ago)", + count, DECY_LINK_ERR_WINDOW, DECY_LINK_ERR_THRESHOLD, + (time.time() - newest_ts) if newest_ts else -1, + ) + + if count < DECY_LINK_ERR_THRESHOLD: + return False + + # Threshold exceeded — the validated-link cache is likely poisoned. + log.warning( + "[decypharr] link_err: %d errors >= threshold %d in %ds window -> " + "link validation cache is poisoned by transient provider errors", + count, DECY_LINK_ERR_THRESHOLD, DECY_LINK_ERR_WINDOW, + ) + + if not DECY_LINK_ERR_RESTART: + log.warning("[decypharr] link_err: DECYPHARR_LINK_ERR_RESTART=false, alert only") + return False + + if not DECY_RESTART_CMD: + log.warning("[decypharr] link_err: no DECYPHARR_RESTART_CMD configured, alert only") + return False + + if DRY_RUN: + log.warning("[decypharr] link_err: dry-run, would restart (reason=link_err_cache_poisoned)") + return False + + if time.time() - _link_err_last_restart.value < 300: + log.warning("[decypharr] link_err: restart attempted <5m ago, holding off") + return False + + log.error("[decypharr] link_err: restarting decypharr to flush poisoned link cache: %s", DECY_RESTART_CMD) + rc = run_cmd(DECY_RESTART_CMD) + _link_err_last_restart.value = time.time() + log.error("[decypharr] link_err: restart rc=%s %s", + rc[0] if rc else "?", rc[1].strip() if (rc and rc[1]) else "") + return True + + +# --------------------------------------------------------------------------- +# Main check entry point +# --------------------------------------------------------------------------- +_STATUS_LABELS = { + _FuseStatus.DEAD: "DEAD (transport/socket not connected)", + _FuseStatus.UNMOUNTED: "UNMOUNTED", + _FuseStatus.HUNG: "HUNG (read/statvfs blocked)", + _FuseStatus.UNKNOWN: "ERROR", +} + +def check_decypharr(): + # --- API health --- + if DECY_URL: + c = http_code(DECY_URL, t=10) + log.info("[decypharr] api %s -> %s", DECY_URL, c if c else "DOWN") + + # --- Link-error cache poisoning detector --- + check_link_errors() + + if not DECY_MOUNT_TEST: + return + + # --- FUSE mount health (3-layer probe) --- + status, detail = _probe_mount(DECY_MOUNT_TEST, DECY_READ_TIMEOUT) + + if status == _FuseStatus.OK: + _fuse_strikes.reset() + log.info("[decypharr] mount %s OK (statvfs + read)", DECY_MOUNT_TEST) + return + + if status == _FuseStatus.EMPTY: + _fuse_strikes.reset() + log.warning("[decypharr] mount %s: %s", DECY_MOUNT_TEST, detail) + return + + strikes, act = _record_fuse_result(status) + label = _STATUS_LABELS.get(status, str(status)) + + if act: + log.error("[decypharr] mount %s %s -- %s (strike %d/%d) -> restarting", + DECY_MOUNT_TEST, label, detail, strikes, DECY_FUSE_STRIKES) + _decy_restart(status) + else: + log.warning("[decypharr] mount %s %s -- %s (strike %d/%d, need %d to act)", + DECY_MOUNT_TEST, label, detail, strikes, DECY_FUSE_STRIKES, DECY_FUSE_STRIKES) diff --git a/doctor/checks/force_import.py b/doctor/checks/force_import.py new file mode 100644 index 0000000..e037c3c --- /dev/null +++ b/doctor/checks/force_import.py @@ -0,0 +1,241 @@ +"""Check: force_import (importarr-style). + +Sonarr/Radarr sometimes refuse to auto-import a release that matches the series/movie +by ID but whose title does not match any known alias. This is common with obfuscated +release names, bad metadata, or anime where TVDB/TMDb aliases are missing. + +This check scans the queue for that exact error, fetches the manual-import candidates +for the series/movie, and submits a ManualImport command. If the import fails (or no +candidate is found) we optionally fall back to the standard remove+re-search flow. +""" +import time +from ..config import ( + DRY_RUN, FI_FALLBACK, FI_IMPORT_MODE, FI_MAX_ACTIONS, FI_MIN_STRIKES, + FI_RECHECK, BLOCKLIST, REMOVE_CLIENT, log, +) +from ..clients import INSTANCES +from ..state import _churn_record, state_transaction + +# The exact message text varies slightly between Sonarr and Radarr, but all of +# these result in a release sitting in the queue that Sonarr/Radarr won't auto-import +# but that a ManualImport command can usually push through successfully. +_MATCH_PHRASES = ( + # obfuscated/misnamed release — matched by TVDB/TMDb ID but title doesn't match + "matched to series by id", + "matched to movie by id", + "automatic import is not possible", + "found matching series via grab history", + "found matching movie via grab history", + # sample-detection failure — common on FUSE/rclone mounts where reads are slow + "unable to determine if file is a sample", + "unable to determine if", # covers slight wording variations +) + + +def _is_matched_by_id(rec): + """Return True if this queue record is the target failure type.""" + msgs = [] + for sm in (rec.get("statusMessages") or []): + msgs += [m for m in (sm.get("messages") or [])] + if rec.get("errorMessage"): + msgs.append(rec["errorMessage"]) + joined = " ".join(msgs).lower() + return any(p in joined for p in _MATCH_PHRASES) + + +def _target_id(arr, rec): + """Episode id (sonarr) or movie id (radarr) this queue record is for.""" + if arr.kind == "sonarr": + return rec.get("episodeId") + if arr.kind == "radarr": + return rec.get("movieId") + return None + + +def _media_id(arr, rec): + """Series id (sonarr) or movie id (radarr) for the manualimport lookup.""" + if arr.kind == "sonarr": + return rec.get("seriesId") + if arr.kind == "radarr": + return rec.get("movieId") + return None + + +def _release_folder(rec): + """Best guess at the download folder the files are sitting in.""" + # outputPath is the top-level download folder in the *arr queue record. + return rec.get("outputPath") or rec.get("downloadPath") or "" + + +def _pick_candidates(arr, rec, candidates): + """Return the candidate(s) that belong to this queue record. + + For Sonarr we match by episodeId(s). For Radarr we prefer the candidate whose + path matches the queue item's outputPath; otherwise we take all returned files + for the movie (Radarr manualimport already filters by movieId). + """ + target = _target_id(arr, rec) + folder = _release_folder(rec) + out = [] + for cand in candidates: + if arr.kind == "sonarr": + ep_ids = cand.get("episodeIds") or [] + if target and ep_ids: + if target in ep_ids: + out.append(cand) + elif folder and folder in (cand.get("path") or cand.get("folderName", "")): + out.append(cand) + elif arr.kind == "radarr": + cand_path = cand.get("path") or cand.get("relativePath") or "" + if folder and cand_path: + if folder in cand_path: + out.append(cand) + else: + out.append(cand) + return out + + +def _dedupe_candidates(candidates): + """Remove duplicate paths (manualimport can return the same file twice).""" + seen = set() + out = [] + for c in candidates: + p = c.get("path") or c.get("relativePath") + if p in seen: + continue + seen.add(p) + out.append(c) + return out + + +def _check_force_import(only=None): + with state_transaction() as state: + fi_state = state.setdefault("force_import", {}) + now = time.time() + actions = 0 + + for arr in INSTANCES: + if only and arr.name.lower() != only.lower(): + continue + if arr.kind not in ("sonarr", "radarr"): + continue + + recs = arr.queue() + if recs is None: + continue + + hits = 0 + for rec in recs: + if not _is_matched_by_id(rec): + continue + hits += 1 + + iid = str(rec.get("id")) + media_id = _media_id(arr, rec) + target = _target_id(arr, rec) + title = (rec.get("title") or rec.get("sourceTitle") or "unknown")[:70] + + if not media_id: + log.debug("[force_import:%s] no series/movie id for %s", arr.name, title) + continue + + # recheck cooldown + key = "%s:%s:%s" % (arr.name, media_id, target or iid) + last = fi_state.get(key, 0) + if now - last < FI_RECHECK: + log.debug("[force_import:%s] %s still in recheck cooldown", arr.name, title) + continue + + strikes_key = "%s:strikes:%s" % (arr.name, key) + strikes = fi_state.get(strikes_key, 0) + 1 + fi_state[strikes_key] = strikes + if strikes < FI_MIN_STRIKES: + log.info("[force_import:%s] matched-by-ID strike %d/%d: %s", + arr.name, strikes, FI_MIN_STRIKES, title) + continue + + # fetch manualimport candidates + candidates = arr.manualimport( + series_id=media_id if arr.kind == "sonarr" else None, + movie_id=media_id if arr.kind == "radarr" else None, + ) + if not candidates: + log.info("[force_import:%s] no manualimport candidates for %s", arr.name, title) + if FI_FALLBACK: + _fallback(state, arr, rec, title, fi_state, key) + continue + + picked = _pick_candidates(arr, rec, candidates) + picked = _dedupe_candidates(picked) + if not picked: + log.info("[force_import:%s] no matching candidate for %s", arr.name, title) + if FI_FALLBACK: + _fallback(state, arr, rec, title, fi_state, key) + continue + + if actions >= FI_MAX_ACTIONS: + log.info("[force_import:%s] max actions reached (%d), skipping %s", + arr.name, FI_MAX_ACTIONS, title) + continue + + if DRY_RUN: + log.info("[force_import:%s] WOULD manual import %d file(s): %s", + arr.name, len(picked), title) + actions += 1 + fi_state[key] = now + fi_state.pop(strikes_key, None) + continue + + # Pass the manualimport candidate back to the command endpoint almost + # untouched. Remove only the UI-only fields that are known to be rejected. + files = [] + for c in picked: + f = dict(c) + for k in ("id", "rejections", "customFormatScore", "isCustomFormatScoreCalculated"): + f.pop(k, None) + files.append(f) + + cmd_id = arr.manualimport_command(files, import_mode=FI_IMPORT_MODE) + if cmd_id: + actions += 1 + fi_state[key] = now + fi_state.pop(strikes_key, None) + log.info("[force_import:%s] manual import command %s (%d file(s)): %s", + arr.name, cmd_id, len(files), title) + else: + log.warning("[force_import:%s] manual import command failed: %s", arr.name, title) + if FI_FALLBACK: + _fallback(state, arr, rec, title, fi_state, key) + + if hits: + log.info("[force_import:%s] %d matched-by-ID item(s), %d acted", + arr.name, hits, actions) + else: + log.debug("[force_import:%s] no matched-by-ID items", arr.name) + + +def _fallback(state, arr, rec, title, fi_state, key): + """Remove the queue item and trigger a re-search as a last resort.""" + if DRY_RUN: + log.info("[force_import:%s] WOULD fallback remove (blocklist=%s): %s", + arr.name, BLOCKLIST, title) + fi_state[key] = time.time() + return + parked = _churn_record(state, arr, rec, title) + try: + q = "removeFromClient=%s&blocklist=%s" % (str(REMOVE_CLIENT).lower(), str(BLOCKLIST).lower()) + arr._req("DELETE", "/queue/%d?%s" % (rec["id"], q)) + fi_state[key] = time.time() + log.info("[force_import:%s] fallback remove (blocklist=%s)%s: %s", + arr.name, BLOCKLIST, + " [parked, no re-search]" if parked else " -> re-search", title) + except Exception as e: + log.warning("[force_import:%s] fallback remove failed: %s", arr.name, e) + + +def check_force_import(only=None): + """Entry point used by the scheduler.""" + try: + _check_force_import(only) + except Exception as e: + log.error("[force_import] check error: %s", e) diff --git a/doctor/checks/janitor.py b/doctor/checks/janitor.py new file mode 100644 index 0000000..5fa9b7c --- /dev/null +++ b/doctor/checks/janitor.py @@ -0,0 +1,222 @@ +"""Check: janitor. + +1. Reads the decypharr log tail and quarantines library symlinks that point to dead files + (ARTICLE_NOT_FOUND, still missing, marked as bad, empty_link, etc.). Only the exact + file reported in the log is quarantined; other files in the same release are left alone. +2. Records the dead file paths in persistent state so the repair check can trigger a + re-search even after the symlink has been removed (important on FUSE mounts where a + "dead" file may still appear to exist). +3. Scans the same log tail for operational/infra error patterns (panic, fatal, rate-limit, + cloudflare, auth, network timeouts) and logs a summary, throttled so it doesn't spam. +4. Optionally probes the decypharr HTTP API (if DECY_URL is set) and logs when it returns + errors or becomes unreachable. +""" +import os +import json +import re +import time +from ..config import ( + DECY_URL, DRY_RUN, JAN_ALERT_COOLDOWN, JAN_ERROR_PATTERNS, + JAN_LIBS, JAN_LOG, JAN_LOG_CMD, JAN_PATTERNS, JAN_QUAR, + http_code, run_output, log, +) +from ..state import state_transaction + +# Operational-error categories we scan for in the decypharr log. +# Each regex is case-insensitive and matches a whole word / short phrase. +_JAN_OP_PATTERNS = [ + ("panic/fatal", re.compile(r"\b(panic|fatal|runtime error)\b", re.I)), + ("rate-limit", re.compile(r"\b(rate limit|rate limited|too many requests|429)\b", re.I)), + ("cloudflare/blocked", re.compile(r"\b(cloudflare|cf-ray|blocked|403)\b", re.I)), + ("auth", re.compile(r"\b(unauthorized|token expired|401)\b", re.I)), + ("network/timeout", re.compile(r"\b(context deadline exceeded|connection refused|i/o timeout|timeout)\b", re.I)), +] +# User-configurable extra patterns added to the scan. +# Wrap each pattern with word boundaries so short numeric codes (401, 403, 429) +# and other tokens do not match inside hex hashes, alldebrid IDs, etc. +_JAN_USER_PATTERNS = [(p, re.compile(r"\b" + re.escape(p) + r"\b", re.I)) for p in JAN_ERROR_PATTERNS] + +# Throttle repeated operational/API alerts so we don't log the same thing every 3 minutes. +_jan_alert_last = {} + +def _jan_alert(name, msg, *args): + now = time.time() + if now - _jan_alert_last.get(name, 0) < JAN_ALERT_COOLDOWN: + return + _jan_alert_last[name] = now + log.warning(msg, *args) + +def _scan_operational_errors(data): + """Return {category: count} for operational error lines in the log tail.""" + counts = {} + for line in data.splitlines(): + for label, pat in _JAN_OP_PATTERNS + _JAN_USER_PATTERNS: + if pat.search(line): + counts[label] = counts.get(label, 0) + 1 + break # count a line only once, under the first matching category + return counts + +def _probe_decy_api(): + """Probe the decypharr API root and /api/status. Log only on problems.""" + if not DECY_URL: + return + base = DECY_URL.rstrip("/") + for path in ("", "/api/status"): + url = base + (path or "/") + try: + code = http_code(url, t=5) + except Exception as e: + _jan_alert("decy_api:%s" % path, "[janitor] decypharr API %s unreachable: %s", url, str(e)[:60]) + continue + if code >= 500: + _jan_alert("decy_api:%s" % path, "[janitor] decypharr API %s returned HTTP %d", url, code) + elif code in (401, 403): + _jan_alert("decy_api:%s" % path, "[janitor] decypharr API %s returned HTTP %d (auth/blocked)", url, code) + elif code == 0: + _jan_alert("decy_api:%s" % path, "[janitor] decypharr API %s unreachable (no response)", url) + elif 200 <= code < 300: + log.debug("[janitor] decypharr API %s -> HTTP %d OK", url, code) + else: + log.debug("[janitor] decypharr API %s -> HTTP %d (unexpected but non-critical)", url, code) + +def _read_log_tail(): + """Return the last ~2MB of the decypharr log as a string.""" + if JAN_LOG_CMD: + return run_output(JAN_LOG_CMD) + if JAN_LOG and os.path.exists(JAN_LOG): + with open(JAN_LOG, errors="ignore") as f: + f.seek(0, os.SEEK_END) + size = f.tell() + f.seek(max(0, size - 2_000_000)) + return f.read() + return None + +def _release_rel(target): + """Return the path of a symlink target relative to the /__all__ or /complete root. + + Example: /mnt/zurg/__all__/RELEASE/file.mkv -> RELEASE/file.mkv + """ + mm = re.search(r"/(?:__all__|complete)/(.+)$", target) + if not mm: + return None + return mm.group(1).lstrip("/") + +def _dead_file_matches(rel_path, bad_files): + """Return True if a symlink relative path matches a known dead file. + + bad_files is a dict keyed by exact relative path (RELEASE/file.mkv). If only the + filename was available from the log, the key may be just the filename. + """ + if rel_path in bad_files: + return True + return os.path.basename(rel_path) in bad_files + +def check_janitor(): + data = _read_log_tail() + if data is None: + log.debug("[janitor] need JANITOR_LOG_CMD or a readable JANITOR_DECYPHARR_LOG") + return + log.debug("[janitor] scanning %d bytes of log tail", len(data)) + + bad_files = {} + + # Pattern 1: [webdav] Error streaming file: error="" + # Catches: ARTICLE_NOT_FOUND, still missing, marked as bad, etc. + # path is "RELEASE/filename.mkv" relative to the debrid root. + pat_stream = re.compile(r"Error streaming file: (.+?) error=\"([^\"]*)\"") + for m in pat_stream.finditer(data): + path, err = m.group(1), m.group(2) + if any(p.strip() and p.strip() in err for p in JAN_PATTERNS): + bad_files[path.strip()] = True + + # Pattern 2: [link] Giving up on entry ... filename= name= reason=empty_link + # filename alone is recorded if the release name is not on the same line. + pat_filename = re.compile( + r"(?:Giving up on entry|empty_link).*?\bfilename=(\S+)(?:.*?\bname=(\S+))?" + ) + for m in pat_filename.finditer(data): + filename = m.group(1) + release = m.group(2) + if release: + bad_files["%s/%s" % (release, filename)] = True + else: + bad_files[filename] = True + + # Operational errors that don't necessarily map to a single dead release. + op_counts = _scan_operational_errors(data) + if op_counts: + summary = ", ".join("%dx %s" % (n, k) for k, n in sorted(op_counts.items(), key=lambda x: -x[1])) + _jan_alert("janitor:ops", "[janitor] operational errors in log tail: %s", summary) + + # Probe the decypharr API for correlated health issues. + _probe_decy_api() + + if not bad_files: + log.debug("[janitor] no dead releases in log tail") + return + log.debug("[janitor] found %d dead file(s): %s", len(bad_files), ", ".join(sorted(bad_files)[:10])) + + if not JAN_LIBS: + _jan_alert("janitor:dead", "[janitor] %d dead file(s) in log but no JANITOR_LIBRARY_PATHS to quarantine", len(bad_files)) + return + + with state_transaction() as state: + janitor_dead = state.setdefault("__janitor_dead_files__", {}) + now = time.time() + for bf in bad_files: + if bf not in janitor_dead: + janitor_dead[bf] = {"ts": now, "orig": None, "target": None} + + moved = 0 + qroot = os.path.join(JAN_QUAR, time.strftime("%Y%m%d-%H%M%S")) + manifest = [] + for libp in JAN_LIBS: + libp = os.path.abspath(libp) + for root, _, files in os.walk(libp): + for fn in files: + fp = os.path.abspath(os.path.join(root, fn)) + if not os.path.islink(fp): + continue + try: + tgt = os.readlink(fp) + except Exception: + continue + rel = _release_rel(tgt) + if rel is None or not _dead_file_matches(rel, bad_files): + continue + if DRY_RUN: + log.info("[janitor] WOULD quarantine: %s", fp) + continue + try: + dst = os.path.join(qroot, fp.lstrip("/")) + if os.path.exists(dst) or os.path.islink(dst): + continue + os.makedirs(os.path.dirname(dst), exist_ok=True) + os.symlink(tgt, dst) + os.unlink(fp) + manifest.append({"orig": fp, "target": tgt}) + moved += 1 + # Update the state entry with the orig path so repair can act on it. + if rel in janitor_dead: + janitor_dead[rel]["orig"] = fp + janitor_dead[rel]["target"] = tgt + else: + # fallback when only the filename was known + base = os.path.basename(rel) + if base in janitor_dead: + janitor_dead[base]["orig"] = fp + janitor_dead[base]["target"] = tgt + except Exception as e: + log.warning("[janitor] move failed %s: %s", fp, e) + + if manifest: + try: + os.makedirs(qroot, exist_ok=True) + with open(os.path.join(qroot, "manifest.json"), "w") as f: + json.dump(manifest, f, indent=1) + except Exception: + pass + + if moved: + log.info("[janitor] quarantined %d dead-file symlink(s) across %d release(s) -> %s", + moved, len({b.split("/")[0] for b in bad_files}), qroot) diff --git a/doctor/checks/maintainer.py b/doctor/checks/maintainer.py new file mode 100644 index 0000000..88a1f41 --- /dev/null +++ b/doctor/checks/maintainer.py @@ -0,0 +1,236 @@ +"""Check: maintainer. + +Deletes pulsarr-requested TV shows that have ended, were released before a configurable +year threshold, and haven't been watched in N days (per Tautulli). + +For each deleted series: + 1. Creates per-user Pulsarr watchlist exclusions so the show won't be + re-added on future Plex watchlist syncs. + 2. Severs from the Sonarr library (delete files + series record). + +Users can remove the exclusion through Pulsarr to request the show again. +""" +import os +import time +from datetime import datetime, timezone +from ..config import ( + DRY_RUN, EN_MAINTAINER, MAINTAINER_LIBRARY_TITLE, + MAINTAINER_MAX_ACTIONS, MAINTAINER_MIN_AGE_DAYS, MAINTAINER_MIN_YEAR, + MAINTAINER_MODE, MAINTAINER_PLEX_SECTION_KEY, + MAINTAINER_PULSARR_TAG_PREFIX, MAINTAINER_RECHECK, + MAINTAINER_ROOT_FOLDER_PATHS, MAINTAINER_UNWATCHED_DAYS, + PLEX_TOKEN, PLEX_URL, PULSARR_APIKEY, PULSARR_URL, + TAUTULLI_APIKEY, TAUTULLI_URL, log, +) +from ..clients import INSTANCES +from ..clients.pulsarr import Pulsarr +from ..clients.tautulli import Tautulli +from ..state import state_transaction + + +def _pulsarr_tags(series, tag_map, prefix): + out = set() + for tid in series.get("tags", []) or []: + label = tag_map.get(tid, "") + if label and label.lower().startswith(prefix.lower()): + out.add(label) + return out + + +def _pulsarr_tagged_show(series, tag_map, prefix): + return bool(_pulsarr_tags(series, tag_map, prefix)) + + +def _tag_users(tag_labels, prefix): + users = set() + for label in tag_labels: + rest = label[len(prefix):].strip() + if rest.startswith("-user-"): + username = rest[6:].strip() + elif rest.startswith("user-"): + username = rest[5:].strip() + else: + continue + if username: + users.add(username) + return users + + +def _series_is_eligible(series, recently_watched, tag_map, prefix, now, mode): + if series.get("year", 9999) >= MAINTAINER_MIN_YEAR: + return False + if series.get("status") != "ended": + return False + added = series.get("added", "") + if added: + try: + added_dt = datetime.fromisoformat(added.replace("Z", "+00:00")) + age_days = (now - added_dt).total_seconds() / 86400 + if age_days < MAINTAINER_MIN_AGE_DAYS: + return False + except (ValueError, TypeError): + pass + if mode == "tagged" and not _pulsarr_tagged_show(series, tag_map, prefix): + return False + if series.get("title", "").strip() in recently_watched: + return False + return True + + +def _exclude_from_pulsarr(series, users, pulsarr): + """Create per-user Pulsarr exclusions so the show won't be re-added.""" + tmdb_id = series.get("tmdbId") + title = series.get("title", "?") + if not tmdb_id: + log.debug("[maintainer] no tmdbId for %s, cannot create exclusions", title) + return 0 + + user_ids = [] + for username in users: + uid = pulsarr.user_id_for_plex_username(username) + if uid: + try: + uid = int(uid) + except (ValueError, TypeError): + continue + user_ids.append(uid) + + if user_ids: + pulsarr.create_watchlist_exclusion(tmdb_id, media_type="tv", + users=user_ids, title=title) + return len(user_ids) + + +def check_maintainer(): + if not EN_MAINTAINER: + return + if not TAUTULLI_URL or not TAUTULLI_APIKEY: + log.debug("[maintainer] TAUTULLI_URL/TAUTULLI_APIKEY not set") + return + + tautulli = Tautulli(TAUTULLI_URL, TAUTULLI_APIKEY) + recently_watched = tautulli.recently_watched_shows(MAINTAINER_UNWATCHED_DAYS) + log.debug("[maintainer] Tautulli: %d show(s) watched in %d days", + len(recently_watched), MAINTAINER_UNWATCHED_DAYS) + + pulsarr = None + if PULSARR_URL and PULSARR_APIKEY: + pulsarr = Pulsarr(PULSARR_URL, PULSARR_APIKEY, + db_path=os.environ.get("PULSARR_DB_PATH", "")) + + with state_transaction() as state: + maintainer_state = state.setdefault("__maintainer__", {}) + deleted = 0 + excluded = 0 + candidates_skipped = 0 + + for arr in INSTANCES: + if arr.kind != "sonarr": + continue + if MAINTAINER_LIBRARY_TITLE not in arr.name: + log.debug("[maintainer] skipping %s (not matching library title filter '%s')", + arr.name, MAINTAINER_LIBRARY_TITLE) + continue + + series_list = arr.series() + if not series_list: + continue + + tag_map = arr.tag_map() + if MAINTAINER_MODE == "tagged" and not tag_map: + log.debug("[maintainer:%s] no tags found", arr.name) + continue + + allowed_roots = [p.strip() for p in MAINTAINER_ROOT_FOLDER_PATHS.split(",") if p.strip()] + + log.debug("[maintainer:%s] %d tag(s), %d show(s) watched recently", + arr.name, len(tag_map), len(recently_watched)) + + for series in series_list: + if deleted >= MAINTAINER_MAX_ACTIONS: + log.info("[maintainer:%s] action cap (%d) reached", + arr.name, MAINTAINER_MAX_ACTIONS) + break + + sid = series.get("id") + if not sid: + continue + + if MAINTAINER_MODE == "tagged" and not _pulsarr_tagged_show( + series, tag_map, MAINTAINER_PULSARR_TAG_PREFIX): + continue + + if allowed_roots: + rfp = (series.get("rootFolderPath") or "").rstrip("/") + if not any(rfp == p.rstrip("/") for p in allowed_roots): + continue + + if not _series_is_eligible(series, recently_watched, tag_map, + MAINTAINER_PULSARR_TAG_PREFIX, + datetime.now(timezone.utc), + MAINTAINER_MODE): + continue + + title = series.get("title", "?") + tag_labels = _pulsarr_tags(series, tag_map, MAINTAINER_PULSARR_TAG_PREFIX) + users = _tag_users(tag_labels, MAINTAINER_PULSARR_TAG_PREFIX) + + state_key = "%s:%s" % (arr.name, sid) + last_eval = maintainer_state.get(state_key) + now = time.time() + if last_eval and now - last_eval < MAINTAINER_RECHECK: + candidates_skipped += 1 + continue + + if DRY_RUN: + excl_msg = "" + if users and pulsarr: + excl_msg = ", excluded=%d" % len(users) + log.info("[maintainer:%s] WOULD delete: %s (ended, year=%s, unwatched %s%s%s)", + arr.name, title, series.get("year", "?"), + ">=%dd" % MAINTAINER_UNWATCHED_DAYS, + excl_msg, + (", users=" + ",".join(sorted(users))) if users else "") + maintainer_state[state_key] = now + deleted += 1 + continue + + if users and pulsarr: + n = _exclude_from_pulsarr(series, users, pulsarr) + if n: + excluded += n + log.info("[maintainer:%s] Pulsarr: %d exclusion(s) for %s", + arr.name, n, title) + + try: + arr._req("DELETE", "/series/%d?deleteFiles=true" % sid) + log.info("[maintainer:%s] deleted: %s (id=%d, year=%s%s)", + arr.name, title, sid, series.get("year", "?"), + (", users=" + ",".join(sorted(users))) if users else "") + deleted += 1 + except Exception as e: + log.warning("[maintainer:%s] delete failed for %s: %s", + arr.name, title, str(e)[:70]) + + maintainer_state[state_key] = time.time() + + if MAINTAINER_MODE == "all" and deleted and MAINTAINER_PLEX_SECTION_KEY: + if not DRY_RUN: + try: + import urllib.request + url = "%s/library/sections/%d/emptyTrash?X-Plex-Token=%s" % ( + PLEX_URL.rstrip("/"), MAINTAINER_PLEX_SECTION_KEY, PLEX_TOKEN) + urllib.request.urlopen(urllib.request.Request(url, method="PUT"), timeout=120) + log.info("[maintainer] Plex: emptyTrash section %d", MAINTAINER_PLEX_SECTION_KEY) + except Exception as e: + log.warning("[maintainer] Plex emptyTrash failed: %s", str(e)[:60]) + + report = ("[maintainer] sweep complete: %d deleted, %d excluded, " + "%d skipped (cooldown), %d cap" % + (deleted, excluded, candidates_skipped, MAINTAINER_MAX_ACTIONS)) + if DRY_RUN: + report = "[maintainer DRY-RUN] " + report + if deleted or candidates_skipped: + log.info(report) + else: + log.debug(report) diff --git a/doctor/checks/missing_seasons.py b/doctor/checks/missing_seasons.py new file mode 100644 index 0000000..211b7f3 --- /dev/null +++ b/doctor/checks/missing_seasons.py @@ -0,0 +1,216 @@ +"""Check: missing_seasons.""" +import time +import email.utils +from datetime import datetime, timezone +from ..config import ( + DRY_RUN, MS_BACKFILL_BATCH, MS_BACKFILL_DELAY, MS_MAX_ACTIONS, + MS_MIN_AGE_HOURS, MS_PARTIAL, MS_RECHECK, MS_SORT_BY, log, +) +from ..clients import INSTANCES +from ..state import state_transaction + +def _season_still_airing(episodes, season_number): + """Return True if *season_number* has at least one episode whose air date is in the future. + This prevents triggering a SeasonSearch for a season that is still actively airing + (only some episodes have been released so far).""" + now = datetime.now(timezone.utc) + for ep in episodes: + if ep.get("seasonNumber") != season_number: + continue + air = ep.get("airDateUtc") or "" + if not air: + continue + try: + dt = datetime.fromisoformat(air.replace("Z", "+00:00")) + if dt > now: + return True + except (ValueError, TypeError): + pass + return False + +def _series_added_ts(ser): + """Parse Sonarr's 'added' timestamp into a Unix epoch, or 0 if unknown.""" + added_str = ser.get("added") or "" + try: + return email.utils.parsedate_to_datetime(added_str).timestamp() if added_str else 0 + except Exception: + return 0 + +def _priority_key(c): + """Sort key for missing-season candidates. + + 'added' -> oldest series first (smallest timestamp), then largest seasons. + 'episodes' -> largest seasons first, then oldest series. + 'mixed' -> oldest series first, then largest seasons (the default). + Unknown added dates are pushed to the end.""" + added = c.get("added_ts", 0) or float("inf") + total = c.get("total_episodes", 0) + if MS_SORT_BY == "episodes": + return (-total, added) + # mixed and added both prioritize age, then size + return (added, -total) + +def _gather_candidates(ms, now, min_age_secs, recheck, backfill): + """Walk every Sonarr instance and collect seasons that need a SeasonSearch. + + A season is a candidate when ALL of the following are true: + - Series and season are monitored + - Season has at least one episode (totalEpisodeCount > 0) + - Series was added long enough ago (min_age_secs) + - Not still actively airing (no future episode air dates) + - Not on recheck cooldown (or backfill mode) + - AND one of: + a) episodeFileCount == 0 (nothing grabbed at all), OR + b) MS_PARTIAL is True AND episodeFileCount < totalEpisodeCount + (partial: some files present but season is incomplete and + fully aired, so the missing episodes can be searched for) + + Returns (candidates, skipped_cooldown, skipped_airing).""" + sonarr_instances = [a for a in INSTANCES if a.kind == "sonarr"] + candidates = [] + skipped = 0 + airing = 0 + for arr in sonarr_instances: + try: + all_series = arr.series() + except Exception as e: + log.warning("[missing_seasons:%s] failed to fetch series: %s", arr.name, str(e)[:60]) + continue + for ser in all_series: + if not ser.get("monitored"): + continue + sid = ser.get("id") + title = (ser.get("title") or "")[:60] + added_ts = _series_added_ts(ser) + if added_ts and (now - added_ts) < min_age_secs: + log.debug("[missing_seasons:%s] skipping %s (added %.1fh ago, min=%.1fh)", + arr.name, title, (now - added_ts) / 3600, min_age_secs / 3600) + continue # too new, give Sonarr time to grab it first + ep_cache = None + for season in (ser.get("seasons") or []): + sn = season.get("seasonNumber", 0) + if sn == 0: + continue + if not season.get("monitored"): + continue + stats = season.get("statistics") or {} + fc = stats.get("episodeFileCount", 0) + tc = stats.get("totalEpisodeCount", 0) + if tc == 0: + continue + if fc >= tc: + continue # season is complete, nothing to do + is_partial = fc > 0 # True = some files present; False = totally empty + if is_partial and not MS_PARTIAL: + continue # partial-season searching is disabled + key = "%s:%d:%d" % (arr.name, sid, sn) + if not backfill and (now - ms.get(key, 0) < recheck): + skipped += 1 + continue + # Fetch episode list once per series (shared across all its seasons). + if ep_cache is None: + try: + ep_cache = arr.episodes(sid) + except Exception: + ep_cache = [] + if _season_still_airing(ep_cache, sn): + airing += 1 + log.debug("[missing_seasons:%s] skipping still-airing season: %s S%02d", + arr.name, title, sn) + continue + candidates.append({ + "arr": arr, + "title": title, + "sid": sid, + "sn": sn, + "key": key, + "added_ts": added_ts, + "total_episodes": tc, + "file_count": fc, + "is_partial": is_partial, + }) + return candidates, skipped, airing + +def _process_candidates(ms, candidates, now, backfill): + """Issue SeasonSearch commands for up to MS_MAX_ACTIONS candidates (or all in backfill mode). + + Returns the number of searches issued.""" + max_actions = 0 if backfill else MS_MAX_ACTIONS + acted = 0 + batch_size = MS_BACKFILL_BATCH if backfill else 0 + for c in candidates: + if max_actions and acted >= max_actions: + break + if DRY_RUN: + if c.get("is_partial"): + log.info("[missing_seasons:%s] DRY-RUN would search partial (%d/%d): %s S%02d", + c["arr"].name, c["file_count"], c["total_episodes"], c["title"], c["sn"]) + else: + log.info("[missing_seasons:%s] DRY-RUN would search: %s S%02d", + c["arr"].name, c["title"], c["sn"]) + ms[c["key"]] = now + acted += 1 + continue + if c["arr"].command("SeasonSearch", seriesId=c["sid"], seasonNumber=c["sn"]): + if c.get("is_partial"): + log.warning("[missing_seasons:%s] partial season (%d/%d files) -> SeasonSearch: %s S%02d", + c["arr"].name, c["file_count"], c["total_episodes"], c["title"], c["sn"]) + else: + log.warning("[missing_seasons:%s] 0 files in monitored season -> SeasonSearch: %s S%02d", + c["arr"].name, c["title"], c["sn"]) + ms[c["key"]] = now + acted += 1 + if backfill and batch_size > 0 and acted > 0 and acted % batch_size == 0: + time.sleep(MS_BACKFILL_DELAY) + return acted + +def _run_missing_seasons(backfill=False): + """Core implementation shared between the scheduled check and the one-shot backfill.""" + sonarr_instances = [a for a in INSTANCES if a.kind == "sonarr"] + if not sonarr_instances: + log.warning("[missing_seasons] no Sonarr instances configured") + return + with state_transaction() as state: + ms = state.setdefault("__missing_seasons__", {}) + now = time.time() + recheck = 0 if backfill else MS_RECHECK + candidates, skipped, airing = _gather_candidates(ms, now, MS_MIN_AGE_HOURS * 3600, recheck, backfill) + label = "missing_seasons:backfill" if backfill else "missing_seasons" + log.debug("[%s] gathered %d candidate(s): %d on cooldown, %d still airing", + label, len(candidates), skipped, airing) + candidates.sort(key=_priority_key) + acted = _process_candidates(ms, candidates, now, backfill) + log.info("[%s] searched %d season(s), skipped %d (cooldown), %d (still airing)", + label, acted, skipped, airing) + +def searched_series(state, arr_name): + """Return the set of series IDs that missing_seasons has SeasonSearch-ed at least once. + + The state key format is ``::``. + Exposed as a public helper so multipack can gate on the same set without + coupling to the raw state key format. + """ + ms = state.get("__missing_seasons__", {}) + prefix = arr_name + ":" + ids = set() + for key in ms: + if key.startswith(prefix): + parts = key.split(":", 2) # split at most twice -> [arr_name, sid, sn] + if len(parts) == 3: + try: + ids.add(int(parts[1])) + except ValueError: + pass + return ids + + +def check_missing_seasons(): + """Scheduled check: capped by MS_MAX_ACTIONS and respects MS_RECHECK cooldown.""" + _run_missing_seasons(backfill=False) + +def backfill_missing_seasons(): + """One-shot backfill: ignore cap and recheck, search every eligible missing season once. + + Useful for clearing a large backlog. Normal scheduling resumes afterwards (if this was + invoked via `python -m doctor --backfill-missing-seasons`).""" + _run_missing_seasons(backfill=True) diff --git a/doctor/checks/multipack.py b/doctor/checks/multipack.py new file mode 100644 index 0000000..e4352c8 --- /dev/null +++ b/doctor/checks/multipack.py @@ -0,0 +1,249 @@ +"""Check: multipack — find cached multi-season torrent packs and push them to the download client. + +Sonarr's automatic SeasonSearch only grabs single-season releases; it hard-rejects any torrent +whose title spans multiple seasons (e.g. "Show S01-S05"). When you search manually in the +Sonarr UI and pick such a pack, Sonarr bypasses its own rejection logic via /api/v3/release/push. +This check automates exactly that: + + 1. Only consider series that missing_seasons has already SeasonSearch-ed at least once — + meaning Sonarr tried per-season searches but the show is still incomplete. + 2. Call Sonarr's release-search API for Season 1 (same data the UI shows). + 3. Filter to multi-season packs: fullSeason=True AND title matches S\\d+-S\\d+. + 4. Discard packs whose season range has zero overlap with the show's incomplete seasons + (e.g. S01-S04 pack when only S05-S07 are missing is useless). + 5. Sort remaining packs by coverage of incomplete seasons (most covered first), then by + total pack width as a tie-breaker — prefer S01-S07 over S01-S06. + 6. For each candidate (best first), verify it is debrid-cached by checking whether its + folder name exists in a normalised snapshot of the zurg __all__ mount taken once per + sweep (O(1) lookup rather than O(n) directory scan per pack). + 7. Push the first cached pack via Sonarr's /api/v3/release/push, which bypasses + quality/format rejection and delivers it straight to decypharr. + +By scoping to already-searched series, the check avoids hammering Prowlarr with +release searches for shows that missing_seasons hasn't tried yet. +""" +import os +import re +import time +from ..config import ( + DECY_MOUNT_TEST, DRY_RUN, MULTIPACK_ENABLED, MULTIPACK_ITEM_INTERVAL, + MULTIPACK_MAX_ACTIONS, MULTIPACK_RECHECK, log, +) +from ..clients import INSTANCES +from ..state import state_transaction +from .missing_seasons import searched_series as _ms_searched_series + +# Detects multi-season pack titles: S01-S05, S1-S3, S01-S02, etc. +_MULTI_SEASON_RE = re.compile(r'\bS(\d+)[.\-]S(\d+)\b', re.IGNORECASE) + + +def _pack_season_range(title): + """Parse the season range from a multi-season pack title. + + Returns (first_season, last_season) as ints, or None if not parseable. + E.g. "Show S01-S05 BluRay" -> (1, 5) + """ + m = _MULTI_SEASON_RE.search(title) + if not m: + return None + s1, s2 = int(m.group(1)), int(m.group(2)) + if s1 > s2: + s1, s2 = s2, s1 + return s1, s2 + + +def _incomplete_seasons_covered(pack_range, incomplete_season_numbers): + """Return the count of incomplete seasons that fall within the pack's range. + + pack_range: (first, last) ints from _pack_season_range() + incomplete_season_numbers: set of int season numbers that are missing/partial + """ + first, last = pack_range + return sum(1 for sn in incomplete_season_numbers if first <= sn <= last) + + +def _normalize(s): + """Strip all non-alphanumeric chars for fuzzy folder-name matching.""" + return re.sub(r'[^a-z0-9]', '', s.lower()) + + +def _zurg_cache_set(mount_path): + """Return a frozenset of normalised folder names from the zurg __all__ mount. + + Built once per sweep so individual cache lookups are O(1) set membership + tests rather than O(n) directory scans. Returns None if the mount is not + accessible. + """ + if not mount_path or not os.path.isdir(mount_path): + return None + try: + return frozenset(_normalize(e) for e in os.listdir(mount_path)) + except OSError: + return None + + +def _is_cached(pack_title, cache_set): + """Return True if pack_title matches any entry in cache_set (O(1)).""" + return _normalize(pack_title) in cache_set + + +def _incomplete_series(arr): + """Return dict of series_id -> (title, incomplete_season_numbers) for monitored series + that have at least one incomplete/missing season. + + incomplete_season_numbers is a frozenset of int season numbers with + episodeFileCount < totalEpisodeCount. + """ + try: + all_series = arr.series() + except Exception as e: + log.warning("[multipack:%s] failed to fetch series: %s", arr.name, str(e)[:60]) + return {} + result = {} + for ser in all_series: + if not ser.get("monitored"): + continue + incomplete = set() + for season in (ser.get("seasons") or []): + sn = season.get("seasonNumber", 0) + if sn == 0 or not season.get("monitored"): + continue + st = season.get("statistics") or {} + tc = st.get("totalEpisodeCount", 0) + fc = st.get("episodeFileCount", 0) + if tc > 0 and fc < tc: + incomplete.add(sn) + if incomplete: + result[ser["id"]] = ((ser.get("title") or "")[:60], frozenset(incomplete)) + return result + + +def _rank_packs(packs, incomplete_seasons): + """Sort packs best-first and return (pack, range, covered) triples. + + Each triple contains the raw release dict, its parsed (first, last) season + range, and the count of incomplete seasons it covers. Storing these avoids + re-parsing the title in the caller's push loop. + + Sort order: + Primary: most incomplete seasons covered (descending) + Secondary: widest pack range (descending) — S01-S07 beats S01-S06 + Tertiary: highest qualityWeight (descending) + + Packs that cover zero incomplete seasons or have an unparseable title are + excluded entirely. + """ + ranked = [] + for pack in packs: + pr = _pack_season_range(pack.get("title", "")) + if pr is None: + continue + covered = _incomplete_seasons_covered(pr, incomplete_seasons) + if covered == 0: + continue + width = pr[1] - pr[0] + qw = pack.get("qualityWeight", 0) + ranked.append((-covered, -width, -qw, pack, pr, covered)) + ranked.sort(key=lambda x: (x[0], x[1], x[2])) + return [(x[3], x[4], x[5]) for x in ranked] + + +def check_multipack(): + """For incomplete Sonarr series that missing_seasons has already tried, search for and push + cached multi-season packs that Sonarr would normally reject.""" + if not MULTIPACK_ENABLED: + return + sonarr_instances = [a for a in INSTANCES if a.kind == "sonarr"] + if not sonarr_instances: + return + + # Build zurg cache set once for the entire sweep (O(1) lookups per pack) + cache_set = _zurg_cache_set(DECY_MOUNT_TEST) + if cache_set is None: + log.warning("[multipack] DECY_MOUNT_TEST not accessible — cannot verify debrid cache") + return + + with state_transaction() as state: + mp = state.setdefault("__multipack__", {}) + now = time.time() + acted = 0 + + for arr in sonarr_instances: + if acted >= MULTIPACK_MAX_ACTIONS: + break + + # Only target series missing_seasons has already tried + already_searched = _ms_searched_series(state, arr.name) + if not already_searched: + log.debug("[multipack:%s] no series searched by missing_seasons yet — skipping", arr.name) + continue + + incomplete_map = _incomplete_series(arr) + # Intersection: incomplete AND already tried by missing_seasons + candidates = {sid: info for sid, info in incomplete_map.items() + if sid in already_searched} + log.debug("[multipack:%s] %d series targeted (incomplete + already SeasonSearched)", + arr.name, len(candidates)) + + for sid, (title, incomplete_seasons) in candidates.items(): + if acted >= MULTIPACK_MAX_ACTIONS: + break + + state_key = "%s:%d" % (arr.name, sid) + if now - mp.get(state_key, 0) < MULTIPACK_RECHECK: + log.debug("[multipack:%s] cooldown: %s", arr.name, title) + continue + + log.debug("[multipack:%s] searching releases for: %s (missing S%s)", + arr.name, title, + "+S".join(str(s) for s in sorted(incomplete_seasons))) + releases = arr.release_search(sid, season_number=1) + if not releases: + mp[state_key] = now + continue + + # Filter to multi-season packs, rank by coverage of incomplete seasons + raw_packs = [r for r in releases + if r.get("fullSeason") and _MULTI_SEASON_RE.search(r.get("title", ""))] + ranked = _rank_packs(raw_packs, incomplete_seasons) + + log.debug("[multipack:%s] %s — %d release(s), %d multi-season pack(s), " + "%d with overlap to missing seasons", + arr.name, title, len(releases), len(raw_packs), len(ranked)) + + pushed = False + for pack, pr, covered in ranked: + pack_title = pack.get("title", "") + if not _is_cached(pack_title, cache_set): + log.debug("[multipack:%s] not cached (covers %d missing season(s)): %s", + arr.name, covered, pack_title[:70]) + continue + if DRY_RUN: + log.info("[multipack:%s] DRY-RUN would push (covers %d/%d missing season(s)): " + "%s -> %s", + arr.name, covered, len(incomplete_seasons), + title, pack_title[:70]) + mp[state_key] = now + acted += 1 + pushed = True + break + if arr.release_push(pack): + log.warning("[multipack:%s] pushed cached pack (covers %d/%d missing " + "season(s)): %s -> %s", + arr.name, covered, len(incomplete_seasons), + title, pack_title[:70]) + mp[state_key] = now + acted += 1 + pushed = True + if MULTIPACK_ITEM_INTERVAL > 0: + time.sleep(MULTIPACK_ITEM_INTERVAL) + break + + if not pushed: + log.debug("[multipack:%s] no cached pack with overlap found: %s", arr.name, title) + mp[state_key] = now + + if acted: + log.info("[multipack] pushed %d cached multi-season pack(s) this sweep", acted) + else: + log.debug("[multipack] no cached multi-season packs found this sweep") diff --git a/doctor/checks/no_upgrade.py b/doctor/checks/no_upgrade.py new file mode 100644 index 0000000..d3ebfb5 --- /dev/null +++ b/doctor/checks/no_upgrade.py @@ -0,0 +1,69 @@ +"""Check: no_upgrade.""" +from ..config import EN_NO_UPGRADE_PROFILE, NO_UPGRADE_PROFILE_ID, NO_UPGRADE_PROFILE_NAME, log +from ..clients import INSTANCES + +def check_no_upgrade_profile(): + """Find ended Sonarr series that are 100% complete and move them to the no-upgrade profile.""" + if not EN_NO_UPGRADE_PROFILE: + return + + sonarr_instances = [a for a in INSTANCES if a.kind == "sonarr"] + if not sonarr_instances: + log.warning("[no_upgrade_profile] no Sonarr instances configured") + return + + for arr in sonarr_instances: + # Resolve target profile id per-instance — each Sonarr may have different profile IDs + target_id = NO_UPGRADE_PROFILE_ID + try: + if not target_id: + profiles = arr.quality_profiles() + match = next((p for p in profiles if p["name"] == NO_UPGRADE_PROFILE_NAME), None) + if not match: + log.warning("[no_upgrade_profile:%s] profile %r not found — skipping", arr.name, NO_UPGRADE_PROFILE_NAME) + continue + target_id = match["id"] + log.info("[no_upgrade_profile:%s] resolved profile %r -> id %d", arr.name, NO_UPGRADE_PROFILE_NAME, target_id) + + # Fetch all series + all_series = arr.series() + except Exception as e: + log.warning("[no_upgrade_profile:%s] fetch failed: %s", arr.name, e) + continue + log.debug("[no_upgrade_profile:%s] scanning %d series (target profile id=%d)", + arr.name, len(all_series), target_id) + + to_move = [] + for s in all_series: + if s.get("status") != "ended": + continue + if s.get("qualityProfileId") == target_id: + log.debug("[no_upgrade_profile:%s] already on target profile: %s", arr.name, s.get("title", "")[:60]) + continue + stats = s.get("statistics", {}) + ep_count = stats.get("episodeCount", 0) + pct = stats.get("percentOfEpisodes", 0) + if ep_count > 0 and pct >= 100: + to_move.append(s) + else: + log.debug("[no_upgrade_profile:%s] ended but incomplete (%.0f%%): %s", + arr.name, pct, s.get("title", "")[:60]) + + if not to_move: + log.debug("[no_upgrade_profile:%s] no newly completed ended shows found", arr.name) + continue + + log.info("[no_upgrade_profile:%s] moving %d completed ended show(s) to profile %d (%s)", + arr.name, len(to_move), target_id, NO_UPGRADE_PROFILE_NAME) + moved, failed = 0, 0 + for s in to_move: + try: + s["qualityProfileId"] = target_id + arr.update_series(s) + log.info("[no_upgrade_profile:%s] -> %s", arr.name, s["title"]) + moved += 1 + except Exception as e: + log.warning("[no_upgrade_profile:%s] failed to update %s: %s", arr.name, s["title"], e) + failed += 1 + + log.info("[no_upgrade_profile:%s] done — moved:%d failed:%d", arr.name, moved, failed) diff --git a/doctor/checks/plex.py b/doctor/checks/plex.py new file mode 100644 index 0000000..62ab02c --- /dev/null +++ b/doctor/checks/plex.py @@ -0,0 +1,83 @@ +"""Check: plex.""" +import urllib.request +import urllib.error +import xml.etree.ElementTree as ET +from ..config import http_code, PLEX_SCAN, PLEX_TOKEN, PLEX_URL, log + +def check_plex(): + if not PLEX_URL: + return + sep = "&" if "?" in PLEX_URL else "?" + url = PLEX_URL.rstrip("/") + "/identity" + c = http_code(url + (sep + "X-Plex-Token=" + PLEX_TOKEN if PLEX_TOKEN else ""), t=10) + if c == 200: + log.info("[plex] %s -> 200 OK", PLEX_URL) + else: + log.error("[plex] %s -> %s (unresponsive)", PLEX_URL, c if c else "DOWN") + if PLEX_SCAN and PLEX_TOKEN and c == 200: + try: + urllib.request.urlopen(PLEX_URL.rstrip("/") + "/library/sections/all/refresh?X-Plex-Token=" + PLEX_TOKEN, timeout=10) + log.info("[plex] triggered library refresh") + except Exception as e: + log.debug("[plex] refresh failed: %s", e) +def _plex_sections(): + """Return list of (key, title) for all Plex library sections. Raises on error.""" + plex_url = PLEX_URL + plex_token = PLEX_TOKEN + if not plex_url or not plex_token: + raise ValueError("PLEX_URL or PLEX_TOKEN not set") + with urllib.request.urlopen( + urllib.request.Request("%s/library/sections?X-Plex-Token=%s" % (plex_url, plex_token)), + timeout=10) as r: + root = ET.fromstring(r.read()) + sections = [(d.get("key"), d.get("title", d.get("key"))) + for d in root.findall("Directory") if d.get("key")] + if not sections: + raise ValueError("no library sections found") + return plex_url, plex_token, sections +def _plex_rescan(): + """Trigger a Plex library scan (refresh) for all sections. Returns (ok, message).""" + try: + plex_url, plex_token, sections = _plex_sections() + except Exception as e: + return False, str(e) + ok, failed = [], [] + for key, title in sections: + try: + urllib.request.urlopen( + urllib.request.Request( + "%s/library/sections/%s/refresh?X-Plex-Token=%s" % (plex_url, key, plex_token), + method="GET"), + timeout=10) + ok.append(title) + except Exception as e: + log.warning("[plex] rescan section %s (%s) failed: %s", key, title, e) + failed.append(title) + msg = "rescanned %d section(s): %s" % (len(ok), ", ".join(ok)) + if failed: + msg += " | failed: %s" % ", ".join(failed) + log.info("[plex] %s", msg) + return len(failed) == 0, msg +def _plex_empty_trash(): + """Empty trash in all Plex library sections. Returns (ok, message).""" + try: + plex_url, plex_token, sections = _plex_sections() + except Exception as e: + return False, str(e) + ok, failed = [], [] + for key, title in sections: + try: + urllib.request.urlopen( + urllib.request.Request( + "%s/library/sections/%s/emptyTrash?X-Plex-Token=%s" % (plex_url, key, plex_token), + method="PUT"), + timeout=10) + ok.append(title) + except Exception as e: + log.warning("[plex] empty trash section %s (%s) failed: %s", key, title, e) + failed.append(title) + msg = "emptied trash for %d section(s): %s" % (len(ok), ", ".join(ok)) + if failed: + msg += " | failed: %s" % ", ".join(failed) + log.info("[plex] %s", msg) + return len(failed) == 0, msg diff --git a/doctor/checks/plexscan.py b/doctor/checks/plexscan.py new file mode 100644 index 0000000..49bd48a --- /dev/null +++ b/doctor/checks/plexscan.py @@ -0,0 +1,89 @@ +"""Check: plexscan.""" +import time +from ..config import ( + DECY_MOUNT_TEST, DECY_READ_TIMEOUT, DRY_RUN, + PLEX_RESTART_CMD, PLEX_SCAN_CANCEL, PLEX_SCAN_STUCK, + PLEX_TOKEN, PLEX_URL, run_cmd, log, +) +from ..clients import Plex +from .decypharr import _decy_restart, _probe_mount, _FuseStatus + +class _State: + """Tiny reset-able mutable cell used for module-level check state.""" + def __init__(self, default): + self._default = default + self.value = default + def reset(self): + self.value = self._default + +_scan_seen = _State({}) # activity uuid -> {first, prog, prog_ts, title, acted_ts} +_plex_last_restart = _State(0.0) +def _is_scan_activity(a): + t = (a.get("type") or "").lower() + txt = ((a.get("title") or "") + " " + (a.get("subtitle") or "")).lower() + if "scan" in txt: + return True + return t.startswith("library.update") or t.startswith("library.refresh") +def check_plex_scan(): + if not (PLEX_URL and PLEX_TOKEN): + return + plex = Plex(PLEX_URL, PLEX_TOKEN) + acts = plex.activities() + log.debug("[plexscan] fetched %d Plex activit%s", len(acts), "y" if len(acts)==1 else "ies") + now = time.time(); cur = set(); stuck = [] + for a in acts: + if not _is_scan_activity(a): + log.debug("[plexscan] non-scan activity: type=%s title=%s", + a.get("type"), (a.get("title") or "")[:50]) + continue + uuid = a.get("uuid") or "" + if not uuid: + continue + cur.add(uuid) + try: prog = int(float(a.get("progress") or 0)) + except Exception: prog = 0 + title = (a.get("title") or a.get("subtitle") or "library scan")[:80] + s = _scan_seen.value.setdefault(uuid, {"first": now, "prog": -1, "prog_ts": now, "title": title, "acted_ts": 0}) + if prog > s["prog"]: + s["prog"] = prog; s["prog_ts"] = now # progress advanced -> not stuck, reset the clock + s["title"] = title + if now - s["prog_ts"] >= PLEX_SCAN_STUCK: + stuck.append((uuid, a, s)) + for u in list(_scan_seen.value): # forget scans that finished / disappeared + if u not in cur: + _scan_seen.value.pop(u, None) + if not stuck: + if cur: + log.info("[plexscan] %d scan(s) running, progressing", len(cur)) + else: + log.debug("[plexscan] no active scans") + return + for uuid, a, s in stuck: + if now - s.get("acted_ts", 0) < PLEX_SCAN_STUCK: # one recovery attempt per stuck-window; don't hammer + continue + s["acted_ts"] = now + mins = int((now - s["prog_ts"]) / 60) + log.error("[plexscan] STUCK scan '%s' (no progress for %dm, stalled at %d%%)", s["title"], mins, max(s["prog"], 0)) + if DRY_RUN: + log.info("[plexscan] DRY-RUN: would fix mount + cancel scan"); continue + # 1) root cause: a hung/dead decypharr mount blocks the scanner on I/O + if DECY_MOUNT_TEST: + _ps_status, _ps_detail = _probe_mount(DECY_MOUNT_TEST, DECY_READ_TIMEOUT) + if _ps_status in (_FuseStatus.DEAD, _FuseStatus.HUNG, _FuseStatus.UNMOUNTED): + log.error("[plexscan] decypharr mount %s (%s) -> restarting it (usual cause of wedged scan)", + _ps_status, _ps_detail) + _decy_restart("plex scan wedged on %s mount" % _ps_status) + # 2) cancel the wedged scan so Plex stops blocking on the bad item + cancelled = False + if PLEX_SCAN_CANCEL and (a.get("cancellable") in ("1", "true", None)): + if plex.cancel_activity(uuid): + log.warning("[plexscan] cancelled stuck scan '%s'", s["title"]) + cancelled = True + else: + log.warning("[plexscan] cancel failed for '%s'", s["title"]) + # 3) last resort: restart Plex if a scan stays wedged well past the threshold AND cancellation didn't succeed + if (PLEX_RESTART_CMD and now - s["first"] >= PLEX_SCAN_STUCK * 2 and + now - _plex_last_restart.value > 1800 and not cancelled): + log.error("[plexscan] scan still wedged -> restarting Plex: %s", PLEX_RESTART_CMD) + rc = run_cmd(PLEX_RESTART_CMD); _plex_last_restart.value = time.time() + log.error("[plexscan] Plex restart rc=%s %s", rc[0] if rc else "?", rc[1] if rc else "") diff --git a/doctor/checks/providers.py b/doctor/checks/providers.py new file mode 100644 index 0000000..b3e2e3d --- /dev/null +++ b/doctor/checks/providers.py @@ -0,0 +1,27 @@ +"""Check: providers.""" +from ..config import DRY_RUN, log +from ..clients import INSTANCES + +_PROVIDER_KEYWORDS = ("indexer", "download client", "applications unavailable", "applications are unavailable") +def check_providers(): + for arr in INSTANCES: + if arr.kind not in ("sonarr", "radarr", "prowlarr"): + continue + issues = [h for h in arr.health() + if h.get("type") in ("warning", "error") + and any(k in (h.get("message") or "").lower() for k in _PROVIDER_KEYWORDS)] + if not issues: + log.debug("[providers:%s] all providers healthy", arr.name) + continue + log.warning("[providers:%s] %d provider issue(s): %s", arr.name, len(issues), + " | ".join((h.get("message") or "")[:60] for h in issues[:2])) + if DRY_RUN: + continue + # re-test everything; a passing test clears the failure status and re-enables recovered ones + for ep, label in (("/indexer/testall", "indexers"), ("/downloadclient/testall", "download-clients")): + res = arr.post(ep) + if isinstance(res, list) and res: + ok = sum(1 for r in res if r.get("isValid")) + still = [r.get("id") for r in res if not r.get("isValid")] + log.info("[providers:%s] tested %s: %d ok, %d still failing %s", + arr.name, label, ok, len(still), still or "") diff --git a/doctor/checks/queue.py b/doctor/checks/queue.py new file mode 100644 index 0000000..4a39e42 --- /dev/null +++ b/doctor/checks/queue.py @@ -0,0 +1,75 @@ +"""Check: queue.""" +from ..config import ( + BLOCKLIST, DRY_RUN, ENABLED_CONDITIONS, host_load, + LOAD_MAX, MAX_ACTIONS, MIN_STRIKES, log, +) +from ..clients import INSTANCES +from ..state import _churn_record, _churn_remonitor, state_transaction + +def _msgs(rec): + out = [] + for sm in (rec.get("statusMessages") or []): + out += [m for m in (sm.get("messages") or [])] + if rec.get("errorMessage"): + out.append(rec["errorMessage"]) + return out +CONDITIONS = { + "downloadClientUnavailable": lambda r: r.get("status") == "downloadClientUnavailable", + "importBlocked": lambda r: r.get("trackedDownloadState") == "importBlocked", + "importFailed": lambda r: r.get("trackedDownloadState") == "importFailed", + "importPending_warning": lambda r: r.get("trackedDownloadState") == "importPending" + and r.get("trackedDownloadStatus") in ("warning", "error"), + "failedPending": lambda r: r.get("trackedDownloadState") == "failedPending", + "stalled": lambda r: r.get("trackedDownloadStatus") == "warning" + and any("stall" in m.lower() or "no files" in m.lower() for m in _msgs(r)), +} +def stuck_reason(rec): + for name in ENABLED_CONDITIONS: + pred = CONDITIONS.get(name) + if pred and pred(rec): + return name + return None +def check_queue(only=None): + if LOAD_MAX > 0 and host_load() > LOAD_MAX: + log.info("[queue] host load > %.0f -> skipping", LOAD_MAX); return + with state_transaction() as state: + actions = 0 + _churn_remonitor(state) + for arr in INSTANCES: + if only and arr.name.lower() != only.lower(): + continue + recs = arr.queue() + if recs is None: + continue + log.debug("[queue:%s] fetched %d queue item(s)", arr.name, len(recs)) + strikes = state.get(arr.name, {}); new = {}; stuck = 0 + for r in recs: + reason = stuck_reason(r) + if not reason: + log.debug("[queue:%s] item ok: %s (state=%s status=%s)", + arr.name, (r.get("title") or "")[:60], + r.get("trackedDownloadState"), r.get("trackedDownloadStatus")) + continue + stuck += 1; iid = str(r.get("id")); cnt = strikes.get(iid, 0) + 1; new[iid] = cnt + log.debug("[queue:%s] stuck item (reason=%s strike=%d): %s", + arr.name, reason, cnt, (r.get("title") or "")[:60]) + if cnt >= MIN_STRIKES and actions < MAX_ACTIONS: + title = (r.get("title") or "")[:70] + if DRY_RUN: + log.info("[queue:%s] WOULD remove (%s strike %d): %s", arr.name, reason, cnt, title) + else: + parked = _churn_record(state, arr, r, title) # un-monitor first so the remove can't re-search + try: + arr.remove(r["id"]); actions += 1; new.pop(iid, None) + log.info("[queue:%s] removed (%s, blocklist=%s)%s: %s", arr.name, reason, BLOCKLIST, + " [parked, no re-search]" if parked else " -> re-search", title) + except Exception as e: + log.warning("[queue:%s] remove failed: %s", arr.name, e) + state[arr.name] = new + if stuck: + log.info("[queue:%s] %d stuck tracked, %d acted", arr.name, stuck, actions) + else: + log.debug("[queue:%s] queue clean (0 stuck items)", arr.name) + for h in arr.health(): + if h.get("type") in ("error", "warning"): + log.debug("[queue:%s] health %s: %s", arr.name, h.get("type"), (h.get("message") or "")[:90]) diff --git a/doctor/checks/repair/__init__.py b/doctor/checks/repair/__init__.py new file mode 100644 index 0000000..2cad112 --- /dev/null +++ b/doctor/checks/repair/__init__.py @@ -0,0 +1,3 @@ +"""Repair check package.""" +from .common import _dead_symlink # noqa: F401 +from .main import check_repair # noqa: F401 diff --git a/doctor/checks/repair/common.py b/doctor/checks/repair/common.py new file mode 100644 index 0000000..24aa675 --- /dev/null +++ b/doctor/checks/repair/common.py @@ -0,0 +1,35 @@ +"""Helpers for the repair check.""" +import os +from ...config import REPAIR_DEBRID_MOUNT, log + +def _debrid_mount_ok(): + """Return True if the debrid mount looks live (path exists and has at least one child entry). + An empty or missing mount means the debrid service is down or the FUSE mount dropped — we must + not run repair in that state or we'd mass-delete + mass-regrab every file in the library.""" + p = REPAIR_DEBRID_MOUNT + if not p: + return True # not configured -> no check, proceed + try: + children = os.listdir(p) + if children: + log.debug("[repair] debrid mount %s OK (%d entries)", p, len(children)) + return True + log.warning("[repair] debrid mount %s exists but is empty -> service down? skipping sweep", p) + return False + except Exception as e: + log.warning("[repair] debrid mount %s not accessible (%s) -> skipping sweep", p, str(e)[:60]) + return False +def _dead_symlink(fp): + """True if fp is a symlink whose target no longer exists. If REPAIR_DEBRID_MOUNT is set, only + symlinks whose target lives under that root are considered (avoids acting on local files).""" + try: + if not os.path.islink(fp): + return False + target = os.readlink(fp) + if not os.path.isabs(target): + target = os.path.join(os.path.dirname(fp), target) + if REPAIR_DEBRID_MOUNT and not target.startswith(REPAIR_DEBRID_MOUNT): + return False + return not os.path.exists(target) + except Exception: + return False diff --git a/doctor/checks/repair/dead_symlinks.py b/doctor/checks/repair/dead_symlinks.py new file mode 100644 index 0000000..3a0301f --- /dev/null +++ b/doctor/checks/repair/dead_symlinks.py @@ -0,0 +1,326 @@ +"""Dead symlink detection and repair actions.""" +import os +import re +from datetime import datetime, timezone +from ...config import ( + DRY_RUN, REPAIR_LIBS, REPAIR_UNMONITORED, REPAIR_VERIFY, + REPAIR_HIERARCHICAL_SEARCH, REPAIR_SEASON_ENDED_THRESHOLD, log, +) +from .common import _dead_symlink +from .verify import _repair_record_verify, _repair_verify_key + +def _release_rel(target): + """Return the path of a symlink target relative to the /__all__ or /complete root. + + Example: /mnt/zurg/__all__/RELEASE/file.mkv -> RELEASE/file.mkv + """ + mm = re.search(r"/(?:__all__|complete)/(.+)$", target) + if not mm: + return None + return mm.group(1).lstrip("/") + +def _is_janitor_dead(fp, janitor_dead): + """Return True if the symlink target is recorded as dead by the janitor. + + janitor_dead is a dict keyed by relative path (RELEASE/file.mkv) or by filename. + """ + if not janitor_dead or not os.path.islink(fp): + return False + try: + target = os.readlink(fp) + except Exception: + return False + rel = _release_rel(target) + if rel is None: + return False + if rel in janitor_dead: + return True + return os.path.basename(rel) in janitor_dead + +def _parse_janitor_dead_path(orig_path, series): + """Parse a quarantined library path into (series, season_number, episode_numbers). + + Expected path layout: /.../Series Name (Year) {imdb-xxx}/Season 01/Series Name - S01E02.mkv + """ + for ser in series: + sp = ser.get("path") + if not sp or not orig_path.startswith(sp + "/"): + continue + rel = orig_path[len(sp) + 1:] + parts = rel.split("/", 1) + if len(parts) != 2: + continue + season_folder, filename = parts + sm = re.match(r"Season\s+(\d+)", season_folder, re.I) + if not sm: + continue + sn = int(sm.group(1)) + eps = _parse_episodes_from_filename(filename, sn) + if eps: + return ser, sn, eps + return None + + +def _parse_episodes_from_filename(filename, season_number): + """Extract episode numbers for a given season from a filename. + + Handles S01E05, S01E01-E02, S01E01E02, etc. + """ + episodes = [] + for m in re.finditer(r"[Ss](\d+)[Ee](\d+)", filename): + s, e = m.group(1), m.group(2) + if int(s) != season_number: + continue + start = int(e) + episodes.append(start) + # Look for a trailing continuation: S01E01-E02 or S01E01E02 + rest = filename[m.end():] + extra_m = re.match(r"(?:[-Ee][Ee]?)(\d+)", rest) + if extra_m: + end = int(extra_m.group(1)) + if end > start: + episodes.extend(range(start + 1, end + 1)) + elif end != start: + episodes.append(end) + return episodes + + +def _normalize_title(title): + """Normalize a title for fuzzy matching by removing punctuation and lowercasing.""" + return re.sub(r"\s+", " ", title.replace(".", "").replace("'", "").replace("-", " ")).strip().lower() + + +def _guess_series_from_release(release_name, series): + """Guess a Sonarr series from a release name like Mr.Robot.S01-S04.1080p....""" + # Strip season/episode ranges and everything after the first Sxx or year marker. + cleaned = re.sub(r"[Ss]\d+([-Ee]\d+)?.*$", "", release_name) + cleaned = re.sub(r"\s+\d{4}\s+.*$", "", cleaned) + cleaned = cleaned.replace(".", " ").replace("_", " ").strip() + cleaned_norm = _normalize_title(cleaned) + best = None + for ser in series: + title = _normalize_title(ser.get("title", "")) + sort_title = _normalize_title(ser.get("sortTitle", "")) + if title == cleaned_norm or sort_title == cleaned_norm: + return ser + if cleaned_norm.startswith(title + " ") or cleaned_norm.startswith(sort_title + " "): + best = ser + return best + +def _radarr_dead_files(movies, state=None, processed=None): + """Yield (movie_id, title, movie_file_id) for monitored movies whose on-disk symlink is dead. + Skips unmonitored movies unless REPAIR_UNMONITORED. + Also considers files that the janitor has recorded as dead in the persistent state. + """ + janitor_dead = (state or {}).get("__janitor_dead_files__", {}) + for m in movies: + if not m.get("monitored", True) and not REPAIR_UNMONITORED: + continue + mid = m.get("id") + mf = m.get("movieFile") or {} + fp = mf.get("path") + if not mid or not fp: + continue + if REPAIR_LIBS and not any(fp.startswith(p) for p in REPAIR_LIBS): + continue + if _dead_symlink(fp) or _is_janitor_dead(fp, janitor_dead): + if processed is not None: + for key, info in list(janitor_dead.items()): + if info.get("orig") == fp: + processed.append(key) + yield mid, (m.get("title") or "")[:70], mf.get("id") + +def _sonarr_dead_files(arr, series, state=None, processed=None): + """Yield (series_id, title, season_number, [episode_file_ids], series, [episode_ids]) + per season that has dead symlinks or files flagged dead by the janitor. + + Skips unmonitored series unless REPAIR_UNMONITORED. + episode_ids is populated when the janitor has already removed the file and we know the + specific missing episode(s) from the quarantined library path. + """ + janitor_dead = (state or {}).get("__janitor_dead_files__", {}) + for ser in series: + if not ser.get("monitored", True) and not REPAIR_UNMONITORED: + continue + sid = ser.get("id") + if not sid: + continue + title = (ser.get("title") or "")[:70] + try: + efiles = arr.episode_files(sid) + eps = arr.episodes(sid) + except Exception: + continue + # episodeFile objects may not include seasonNumber, so cross-reference with episodes + efid_to_season = {} + efid_to_epid = {} + for ep in eps: + if ep.get("episodeFileId"): + efid_to_season[ep["episodeFileId"]] = ep.get("seasonNumber") + efid_to_epid[ep["episodeFileId"]] = ep.get("id") + dead_by_season = {} + for ef in efiles: + fp = ef.get("path") + if not fp: + continue + if REPAIR_LIBS and not any(fp.startswith(p) for p in REPAIR_LIBS): + continue + if not _dead_symlink(fp) and not _is_janitor_dead(fp, janitor_dead): + continue + efid = ef.get("id") + if not efid: + continue + sn = ef.get("seasonNumber") if ef.get("seasonNumber") is not None else efid_to_season.get(efid) + if sn is None: + continue + entry = dead_by_season.setdefault(sn, {"efids": [], "epids": set(), "series": ser}) + entry["efids"].append(efid) + if efid_to_epid.get(efid): + entry["epids"].add(efid_to_epid[efid]) + # Mark the janitor entry as processed if it matches this file. + if processed is not None: + for key, info in list(janitor_dead.items()): + if info.get("orig") == fp: + processed.append(key) + # Handle files the janitor already removed (no episodeFile record left). + if janitor_dead: + for key, info in list(janitor_dead.items()): + orig = info.get("orig") + parsed = None + if orig: + parsed = _parse_janitor_dead_path(orig, series) + if not parsed: + # Fallback: guess the series from the release name and parse SxxEyy from filename. + if "/" in key: + release_name, filename = key.rsplit("/", 1) + else: + release_name, filename = "", key + jser = _guess_series_from_release(release_name, series) + if jser: + jsn = None + for m in re.finditer(r"[Ss](\d+)[Ee](\d+)", filename): + jsn = int(m.group(1)) + break + if jsn is not None: + jepisodes = _parse_episodes_from_filename(filename, jsn) + parsed = (jser, jsn, jepisodes) + if not parsed: + continue + jser, jsn, jepisodes = parsed + jsid = jser.get("id") + if jsid != sid: + continue + try: + jeps = arr.episodes(jsid) + except Exception: + continue + jepids = [e.get("id") for e in jeps + if e.get("seasonNumber") == jsn + and e.get("episodeNumber") in jepisodes + and e.get("id")] + if not jepids: + continue + entry = dead_by_season.setdefault(jsn, {"efids": [], "epids": set(), "series": jser}) + entry["epids"].update(jepids) + if processed is not None and key not in processed: + processed.append(key) + for sn, data in dead_by_season.items(): + yield sid, title, sn, data["efids"], data["series"], sorted(data["epids"]) + +def _repair_radarr_movie(arr, mid, title, mfid, state=None): + """Delete a dead movie file record, toggle the movie monitor off+on, and re-search.""" + if DRY_RUN: + log.info("[repair:%s] DRY-RUN would delete dead file + re-search movie: %s", arr.name, title) + return True + if mfid: + arr.delete_file(mfid) + # toggle monitor off+on to force the arr to refresh the title's availability state + try: + arr.set_monitored([mid], False) + arr.set_monitored([mid], True) + except Exception as e: + log.warning("[repair:%s] monitor toggle failed for movie %s: %s", arr.name, title, str(e)[:70]) + cmd_id = arr.command("MoviesSearch", movieIds=[mid]) + log.warning("[repair:%s] dead symlink -> deleted file + re-searching movie: %s", arr.name, title) + if REPAIR_VERIFY and state is not None: + _repair_record_verify(state, arr, title, cmd_id, mid, [mid], hierarchical=False) + return True + +def _sonarr_search_strategy(series, season_number): + """Choose the broadest Sonarr search for a dead season based on airing status. + + Returns (command_name, command_kwargs, strategy_tag). + - Ended show -> SeriesSearch (multi-season / complete-series packs). + - Ended season (and show still continuing) -> SeasonSearch (season packs). + - Ongoing season -> EpisodeSearch (specific episode(s)). + """ + if not REPAIR_HIERARCHICAL_SEARCH: + return "SeasonSearch", {"seriesId": series["id"], "seasonNumber": season_number}, "season" + + # Show ended -> try for a complete/multi-season pack first + if series.get("ended") or series.get("status") == "ended": + return "SeriesSearch", {"seriesId": series["id"]}, "series" + + # Season ended -> season pack + seasons = series.get("seasons", []) + season = next((s for s in seasons if s.get("seasonNumber") == season_number), None) + if season: + prev_air = (season.get("statistics") or {}).get("previousAiring") + if prev_air: + try: + last_air = datetime.fromisoformat(prev_air.replace("Z", "+00:00")) + age = (datetime.now(timezone.utc) - last_air).total_seconds() + if age >= REPAIR_SEASON_ENDED_THRESHOLD: + return "SeasonSearch", {"seriesId": series["id"], "seasonNumber": season_number}, "season" + except Exception: + pass + + # Ongoing season -> search only the affected episodes + return "EpisodeSearch", {}, "episode" + + +def _repair_sonarr_season(arr, sid, title, season_number, efids, state=None, series=None, epids=None): + """Delete all dead episode file records for a season, toggle the season's episodes off+on, and + trigger the appropriate search command (Series/Season/Episode) based on airing status. + + epids may be the specific episode IDs that are missing; if not provided, all episodes in the + season are used for EpisodeSearch. + """ + if DRY_RUN: + log.info("[repair:%s] DRY-RUN would delete %d dead file(s) + re-search season: %s S%02d", + arr.name, len(efids), title, season_number) + return True + for efid in efids: + arr.delete_file(efid) + # toggle every episode in this season off then on to force a fresh availability state + all_epids = [] + if epids is None: + try: + eps = arr.episodes(sid) + epids = [e.get("id") for e in eps if e.get("seasonNumber") == season_number and e.get("id")] + all_epids = epids + except Exception as e: + log.warning("[repair:%s] monitor toggle failed for %s S%02d: %s", arr.name, title, season_number, str(e)[:70]) + else: + all_epids = epids + if all_epids: + try: + arr.set_monitored(all_epids, False) + arr.set_monitored(all_epids, True) + except Exception as e: + log.warning("[repair:%s] monitor toggle failed for %s S%02d: %s", arr.name, title, season_number, str(e)[:70]) + cmd_name, cmd_kwargs, strategy = _sonarr_search_strategy(series or {"id": sid}, season_number) + if cmd_name == "EpisodeSearch": + cmd_kwargs = {"episodeIds": epids or all_epids} + elif cmd_name == "SeasonSearch": + cmd_kwargs = {"seriesId": sid, "seasonNumber": season_number} + elif cmd_name == "SeriesSearch": + cmd_kwargs = {"seriesId": sid} + cmd_id = arr.command(cmd_name, **cmd_kwargs) + log.warning("[repair:%s] dead symlinks -> deleted %d file(s) + re-searching %s: %s S%02d (strategy=%s)", + arr.name, len(efids), cmd_name, title, season_number, strategy) + if REPAIR_VERIFY and state is not None: + _repair_record_verify(state, arr, title, cmd_id, sid, epids or all_epids, + strategy=strategy, season_number=season_number, series_id=sid, + hierarchical=REPAIR_HIERARCHICAL_SEARCH) + return True diff --git a/doctor/checks/repair/main.py b/doctor/checks/repair/main.py new file mode 100644 index 0000000..5603bed --- /dev/null +++ b/doctor/checks/repair/main.py @@ -0,0 +1,128 @@ +"""Main repair check orchestrator.""" +import time +from ...config import ( + DRY_RUN, host_load, REPAIR_ITEM_INTERVAL, REPAIR_LOAD_MAX, REPAIR_MAX_ACTIONS, + REPAIR_MAX_SYMLINKS, REPAIR_MISSING_FROM_DISK, REPAIR_ORPHAN_SCAN, + REPAIR_SEASON_PACKS, REPAIR_VERIFY, log, +) +from ...clients import INSTANCES +from ...state import state_transaction +from .common import _debrid_mount_ok +from .dead_symlinks import _radarr_dead_files, _sonarr_dead_files, _repair_radarr_movie, _repair_sonarr_season +from .season_pack import _sonarr_season_pack_check +from .missing_from_disk import _missing_from_disk_check +from .verify import _repair_verify_pending, _repair_process_fallbacks, _repair_verify_key +from .orphan import _orphan_dead_symlink_scan + +def check_repair(): + if not INSTANCES: + log.warning("[repair] no Sonarr/Radarr instances configured"); return + if REPAIR_LOAD_MAX > 0 and host_load() > REPAIR_LOAD_MAX: + log.info("[repair] host load > %.0f -> skip sweep", REPAIR_LOAD_MAX); return + if not _debrid_mount_ok(): + return + with state_transaction() as state: + # verify pending searches from previous sweeps before starting a new one + if REPAIR_VERIFY: + _repair_verify_pending(state) + # issue any fallback searches that were scheduled by the verify step + fb_issued = _repair_process_fallbacks(state) + if fb_issued: + log.info("[repair] issued %d hierarchical fallback search(es)", fb_issued) + acted = 0 # search commands issued (groups) + symlinks = 0 # total dead symlinks deleted + cap_hit = None + # Track which janitor-reported dead files are being handled by this sweep. + janitor_processed = [] + for arr in INSTANCES: + if arr.kind not in ("sonarr", "radarr"): + continue + if acted >= REPAIR_MAX_ACTIONS or symlinks >= REPAIR_MAX_SYMLINKS: + break + try: + if arr.kind == "sonarr": + series = arr.series() + log.debug("[repair:%s] scanning %d series for dead symlinks", arr.name, len(series)) + for sid, title, sn, efids, series, epids in _sonarr_dead_files(arr, series, state=state, processed=janitor_processed): + if acted >= REPAIR_MAX_ACTIONS: + cap_hit = "REPAIR_MAX_ACTIONS"; break + if symlinks >= REPAIR_MAX_SYMLINKS: + cap_hit = "REPAIR_MAX_SYMLINKS"; break + count = len(efids) + if symlinks + count > REPAIR_MAX_SYMLINKS: + cap_hit = "REPAIR_MAX_SYMLINKS"; break + # Skip if this season already has a pending repair search in flight + if REPAIR_VERIFY: + key = _repair_verify_key(arr.name, title, sn) + if state.get("__repair_verify__", {}).get(key): + log.debug("[repair:%s] skipping %s S%02d: pending repair search in flight", + arr.name, title, sn) + continue + log.debug("[repair:%s] dead symlink(s) found: %s S%02d (%d file(s))", + arr.name, title, sn, count) + if _repair_sonarr_season(arr, sid, title, sn, efids, state, series=series, epids=epids): + acted += 1 + symlinks += count + if REPAIR_ITEM_INTERVAL > 0: + time.sleep(REPAIR_ITEM_INTERVAL) + else: + movies = arr.movies() + log.debug("[repair:%s] scanning %d movies for dead symlinks", arr.name, len(movies)) + for mid, title, mfid in _radarr_dead_files(movies, state=state, processed=janitor_processed): + if acted >= REPAIR_MAX_ACTIONS: + cap_hit = "REPAIR_MAX_ACTIONS"; break + if symlinks >= REPAIR_MAX_SYMLINKS: + cap_hit = "REPAIR_MAX_SYMLINKS"; break + log.debug("[repair:%s] dead symlink found: %s", arr.name, title) + if _repair_radarr_movie(arr, mid, title, mfid, state): + acted += 1 + symlinks += 1 + if REPAIR_ITEM_INTERVAL > 0: + time.sleep(REPAIR_ITEM_INTERVAL) + except Exception as e: + log.warning("[repair:%s] sweep error: %s", arr.name, str(e)[:70]) + if acted or symlinks: + log.info("[repair] symlink sweep: %d season(s)/movie(s), %d dead symlink(s) re-grabbed%s", + acted, symlinks, " (capped by %s)" % cap_hit if cap_hit else "") + else: + log.debug("[repair] symlink sweep: no dead symlinks found") + # Clean up janitor dead-file entries that were processed this sweep. + if janitor_processed: + janitor_dead = state.get("__janitor_dead_files__", {}) + removed = 0 + for key in janitor_processed: + if janitor_dead.pop(key, None): + removed += 1 + if removed: + log.debug("[repair] cleared %d processed janitor dead-file record(s)", removed) + # season-pack check: find sonarr seasons that are fully downloaded but spread across multiple + # parent dirs (individual episode grabs, not a season pack). Trigger a season search to upgrade. + if REPAIR_SEASON_PACKS and acted < REPAIR_MAX_ACTIONS: + sp_budget = REPAIR_MAX_ACTIONS - acted + for arr in INSTANCES: + if arr.kind != "sonarr" or sp_budget <= 0: + break + try: + series = arr.series() + except Exception: + continue + for title, sn, sid, a in _sonarr_season_pack_check(arr, series): + if sp_budget <= 0: + break + log.debug("[repair:season_pack:%s] multi-dir season detected: %s S%02d", arr.name, title, sn) + if DRY_RUN: + log.info("[repair:season_pack] DRY-RUN would search season pack: %s S%02d", title, sn); sp_budget -= 1; continue + if a.command("SeasonSearch", seriesId=sid, seasonNumber=sn): + log.warning("[repair:season_pack] non-season-pack detected -> searching season pack: %s S%02d", title, sn) + sp_budget -= 1 + if REPAIR_ITEM_INTERVAL > 0: + time.sleep(REPAIR_ITEM_INTERVAL) + # MissingFromDisk check: query *arr history for items Sonarr/Radarr knows are gone from disk. + # Runs after the symlink sweep so both modes share the REPAIR_MAX_ACTIONS budget. + if REPAIR_MISSING_FROM_DISK and acted < REPAIR_MAX_ACTIONS: + acted = _missing_from_disk_check(state, acted, REPAIR_MAX_ACTIONS - acted) + # Orphan scan runs outside the state_transaction: it is a read-only filesystem walk that + # neither reads nor writes the state dict, and can take seconds on large libraries. Holding + # STATE_LOCK for the entire walk would block every other concurrent check unnecessarily. + if REPAIR_ORPHAN_SCAN: + _orphan_dead_symlink_scan() diff --git a/doctor/checks/repair/missing_from_disk.py b/doctor/checks/repair/missing_from_disk.py new file mode 100644 index 0000000..0ef02d0 --- /dev/null +++ b/doctor/checks/repair/missing_from_disk.py @@ -0,0 +1,74 @@ +"""Re-trigger searches for items *arr reports as MissingFromDisk.""" +import time +from ...config import DRY_RUN, REPAIR_ITEM_INTERVAL, REPAIR_MFD_RECHECK, REPAIR_UNMONITORED, log +from ...clients import INSTANCES + +def _missing_from_disk_check(state, acted, budget): + """Query *arr download history for items with reason=MissingFromDisk and re-trigger searches. + This catches files that Sonarr/Radarr knows are gone but which have no on-disk symlink to probe + (e.g. usenet direct downloads, or files cleaned up by an external tool). Shares the REPAIR_MAX_ACTIONS + budget with the filesystem sweep so the two modes together never exceed the cap in one sweep.""" + mfd = state.setdefault("__repair_mfd__", {}) + now = time.time() + for arr in INSTANCES: + if arr.kind not in ("sonarr", "radarr"): + continue # skip non-media instances (prowlarr, etc.) + if budget <= 0: + break # budget exhausted: stop processing all instances + try: + all_media = arr.series() if arr.kind == "sonarr" else arr.movies() + except Exception as e: + log.warning("[repair:mfd:%s] failed to fetch media list: %s", arr.name, str(e)[:60]); continue + log.debug("[repair:mfd:%s] scanning %d item(s) for MissingFromDisk history", arr.name, len(all_media)) + for item in all_media: + if budget <= 0: + break + if not item.get("monitored") and not REPAIR_UNMONITORED: + continue + mid = item.get("id") + title = (item.get("title") or "")[:60] + try: + records = arr.history(mid) + except Exception as e: + log.warning("[repair:mfd:%s] history fetch failed for %s: %s", arr.name, title, str(e)[:60]); continue + # sonarr returns a list directly; radarr wraps in {"records": [...]} + if isinstance(records, dict): + records = records.get("records") or [] + # find the most recent grabbed record that is now MissingFromDisk + # group by season (sonarr) or movie so we only search once per parent + searched = set() + for rec in records: + if rec.get("eventType") != "grabbed": + continue + data = rec.get("data") or {} + if data.get("reason") != "MissingFromDisk": + continue + log.debug("[repair:mfd:%s] MissingFromDisk history entry for: %s", arr.name, title) + if arr.kind == "sonarr": + ep = rec.get("episode") or {} + season_number = ep.get("seasonNumber") + series_id = ep.get("seriesId") or mid + key = "%s:%d:s%s" % (arr.name, series_id, season_number) + else: + key = "%s:%d" % (arr.name, mid) + if key in searched: + continue + if now - mfd.get(key, 0) < REPAIR_MFD_RECHECK: + continue # searched recently, wait for cooldown + if budget <= 0: + break + if DRY_RUN: + log.info("[repair:mfd:%s] DRY-RUN would re-search MissingFromDisk: %s", arr.name, title) + mfd[key] = now; searched.add(key); acted += 1; budget -= 1; continue + if arr.kind == "sonarr" and season_number is not None: + arr.command("SeasonSearch", seriesId=series_id, seasonNumber=season_number) + log.warning("[repair:mfd:%s] MissingFromDisk -> SeasonSearch: %s S%02d", arr.name, title, season_number) + elif arr.kind == "radarr": + arr.command("MoviesSearch", movieIds=[mid]) + log.warning("[repair:mfd:%s] MissingFromDisk -> MoviesSearch: %s", arr.name, title) + else: + continue + mfd[key] = now; searched.add(key); acted += 1; budget -= 1 + if REPAIR_ITEM_INTERVAL > 0: + time.sleep(REPAIR_ITEM_INTERVAL) + return acted diff --git a/doctor/checks/repair/orphan.py b/doctor/checks/repair/orphan.py new file mode 100644 index 0000000..20735de --- /dev/null +++ b/doctor/checks/repair/orphan.py @@ -0,0 +1,63 @@ +"""Filesystem-only orphan dead-symlink scanner.""" +import os +from ...config import REPAIR_LIBS, log +from ...clients import INSTANCES +from .common import _dead_symlink + +def _collect_known_paths(): + """Return a set of all file paths currently tracked by Sonarr/Radarr.""" + known = set() + for arr in INSTANCES: + if arr.kind not in ("sonarr", "radarr"): + continue + try: + if arr.kind == "sonarr": + for ser in arr.series(): + sid = ser.get("id") + if not sid: + continue + for ef in arr.episode_files(sid): + fp = ef.get("path") + if fp: + known.add(fp) + else: + for m in arr.movies(): + mf = m.get("movieFile") or {} + fp = mf.get("path") + if fp: + known.add(fp) + except Exception as e: + log.warning("[repair:orphan] failed to collect paths from %s: %s", arr.name, str(e)[:70]) + return known + + +def _orphan_dead_symlink_scan(): + """Walk REPAIR_LIBRARY_PATHS and report dead symlinks that are not tracked by *arr.""" + if not REPAIR_LIBS: + return + known = _collect_known_paths() + log.debug("[repair:orphan] collected %d known path(s) from *arr, scanning %d library root(s)", + len(known), len(REPAIR_LIBS)) + orphans = [] + for root in REPAIR_LIBS: + if not os.path.isdir(root): + log.warning("[repair:orphan] library path not a directory: %s", root) + continue + for dirpath, _dirs, files in os.walk(root): + for name in files: + fp = os.path.join(dirpath, name) + if fp in known: + continue + if not _dead_symlink(fp): + continue + orphans.append(fp) + if orphans: + log.warning("[repair:orphan] found %d dead symlink(s) not tracked by *arr; manual cleanup may be needed", len(orphans)) + for fp in orphans[:20]: + log.warning("[repair:orphan] %s", fp) + if len(orphans) > 20: + log.warning("[repair:orphan] ... and %d more", len(orphans) - 20) + else: + log.debug("[repair:orphan] no orphan dead symlinks found in %d library path(s)", len(REPAIR_LIBS)) + + diff --git a/doctor/checks/repair/season_pack.py b/doctor/checks/repair/season_pack.py new file mode 100644 index 0000000..96c30a3 --- /dev/null +++ b/doctor/checks/repair/season_pack.py @@ -0,0 +1,38 @@ +"""Detect seasons spread across multiple dirs and upgrade them to season packs.""" +import os + +def _sonarr_season_pack_check(arr, series): + """Yield (series_title, season_number, arr) for any fully-available sonarr season whose episode + files are spread across more than one parent directory — a sign that individual episode grabs + replaced what should be a season pack. Only emits seasons where every episode is monitored.""" + for ser in series: + if not ser.get("monitored", True): + continue + sid = ser.get("id") + title = (ser.get("title") or "")[:60] + try: + seasons = {s["seasonNumber"]: s for s in (ser.get("seasons") or []) if s.get("seasonNumber", 0) > 0} + efiles = arr.episode_files(sid) + eps = arr.episodes(sid) + except Exception: + continue + # group episode files by season + ef_by_season = {} + for ef in efiles: + sn = ef.get("seasonNumber") + if sn: + ef_by_season.setdefault(sn, []).append(ef) + ep_by_season = {} + for ep in eps: + sn = ep.get("seasonNumber") + if sn: + ep_by_season.setdefault(sn, []).append(ep) + for sn, efs in ef_by_season.items(): + season_meta = seasons.get(sn, {}) + stats = season_meta.get("statistics") or {} + # only act when the season is fully downloaded + if stats.get("episodeFileCount", 0) < stats.get("totalEpisodeCount", 1): + continue + parent_dirs = set(os.path.dirname(ef.get("path", "")) for ef in efs if ef.get("path")) + if len(parent_dirs) > 1: + yield title, sn, sid, arr diff --git a/doctor/checks/repair/verify.py b/doctor/checks/repair/verify.py new file mode 100644 index 0000000..cd31d3b --- /dev/null +++ b/doctor/checks/repair/verify.py @@ -0,0 +1,147 @@ +"""Post-repair search verification.""" +import re +import time +from datetime import datetime, timezone +from ...config import REPAIR_VERIFY_DEADLINE, REPAIR_HIERARCHICAL_FALLBACK, log +from ...clients import INSTANCES + +def _repair_verify_key(arr_name, title, season_number=None): + """Stable key for a pending repair search entry.""" + slug = re.sub(r"[^a-z0-9]+", "_", title.lower())[:40] + if season_number is not None: + return "%s:%s:s%02d" % (arr_name, slug, season_number) + return "%s:%s" % (arr_name, slug) + + +def _repair_verify_pending(state): + """Check any in-flight repair searches from previous sweeps. + State entry per pending item (keyed by ':[:sNN]'): + {cmd_id, media_id, entity_ids, title, search_ts, arr_name, strategy, season_number, needs_fallback} + Flow per item each sweep: + 1. If command_id present, poll /command/{id} — log when done/failed. + 2. Poll /history for a new 'grabbed' event after search_ts. + 3. On confirmed grab: log indexer + sourceTitle, remove from pending. + 4. On deadline exceeded without grab: log warning, remove from pending. + """ + pv = state.setdefault("__repair_verify__", {}) + if not pv: + log.debug("[repair:verify] no pending searches to verify") + return + log.debug("[repair:verify] checking %d pending search(es)", len(pv)) + now = time.time() + arr_map = {a.name: a for a in INSTANCES} + expired = [] + for key, v in list(pv.items()): + arr = arr_map.get(v.get("arr_name")) + if not arr: + expired.append(key); continue + title = v.get("title", key) + search_ts = v.get("search_ts", "") + deadline = v.get("deadline", 0) + cmd_id = v.get("cmd_id") + media_id = v.get("media_id") + entity_ids = v.get("entity_ids") or [] + + # step 1: poll command status if we haven't confirmed it finished yet + if cmd_id and not v.get("cmd_done"): + status = arr.command_status(cmd_id) + if status in ("completed", "failed", "aborted"): + log.info("[repair:verify:%s] search command %s: %s", arr.name, cmd_id, status) + v["cmd_done"] = True + elif status is None: + v["cmd_done"] = True # endpoint gone, assume finished + + # step 2: check history for a new grab + if media_id: + rec = arr.history_grabbed(media_id, search_ts, entity_ids if arr.kind == "sonarr" else None) + if rec: + src = rec.get("sourceTitle") or "?" + indexer = (rec.get("data") or {}).get("indexer") or "?" + log.warning("[repair:verify:%s] GRABBED '%s' via %s: %s", arr.name, title, indexer, src) + expired.append(key); continue + + # step 3: deadline check + if now > deadline: + strategy = v.get("strategy", "season") + # Hierarchical fallback: series -> season -> episode (only for entries created by hierarchical search) + if v.get("hierarchical") and REPAIR_HIERARCHICAL_FALLBACK and strategy in ("series", "season"): + next_strategy = "season" if strategy == "series" else "episode" + log.warning("[repair:verify:%s] no grab for '%s' (%s search) within deadline -> falling back to %s search", + arr.name, title, strategy, next_strategy) + v["strategy"] = next_strategy + v["needs_fallback"] = True + v["deadline"] = time.time() + REPAIR_VERIFY_DEADLINE + v["cmd_done"] = True # old command is done; wait for new one + v.pop("cmd_id", None) + continue + log.warning("[repair:verify:%s] no grab confirmed for '%s' within deadline — search may have stalled", + arr.name, title) + expired.append(key) + + for key in expired: + pv.pop(key, None) +def _repair_record_verify(state, arr, title, cmd_id, media_id, entity_ids, + strategy="season", season_number=None, series_id=None, + hierarchical=False): + """Store a pending verification entry so the next sweep can check if the grab landed.""" + pv = state.setdefault("__repair_verify__", {}) + key = _repair_verify_key(arr.name, title, season_number) + pv[key] = { + "arr_name": arr.name, + "title": title, + "cmd_id": cmd_id if isinstance(cmd_id, int) else None, + "media_id": media_id, + "entity_ids": entity_ids or [], + "search_ts": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S") + "Z", + "deadline": time.time() + REPAIR_VERIFY_DEADLINE, + "strategy": strategy, + "season_number": season_number, + "series_id": series_id if series_id is not None else media_id, + "hierarchical": hierarchical, + } + + +def _repair_process_fallbacks(state): + """Issue search commands for any pending repairs that have fallen back to a narrower strategy. + Returns the number of fallback commands issued.""" + pv = state.get("__repair_verify__", {}) + if not pv: + return 0 + arr_map = {a.name: a for a in INSTANCES} + issued = 0 + for key, v in list(pv.items()): + if not v.get("needs_fallback"): + continue + arr = arr_map.get(v.get("arr_name")) + if not arr or arr.kind != "sonarr": + v.pop("needs_fallback", None) + continue + strategy = v.get("strategy", "season") + sid = v.get("series_id") + sn = v.get("season_number") + epids = v.get("entity_ids", []) + if strategy == "series": + cmd_id = arr.command("SeriesSearch", seriesId=sid) + elif strategy == "season" and sn is not None: + cmd_id = arr.command("SeasonSearch", seriesId=sid, seasonNumber=sn) + elif strategy == "episode" and epids: + cmd_id = arr.command("EpisodeSearch", episodeIds=epids) + else: + log.debug("[repair:verify:%s] cannot issue fallback for %s (strategy=%s, sn=%s, epids=%s)", + arr.name, key, strategy, sn, epids) + v.pop("needs_fallback", None) + continue + if cmd_id: + v["cmd_id"] = cmd_id if isinstance(cmd_id, int) else None + v["needs_fallback"] = False + v["cmd_done"] = False + v["search_ts"] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S") + "Z" + v["deadline"] = time.time() + REPAIR_VERIFY_DEADLINE + log.warning("[repair:verify:%s] fallback %s search issued for %s S%02d", + arr.name, strategy, v.get("title", key), sn or 0) + issued += 1 + else: + # command failed; leave needs_fallback set so next sweep retries + log.warning("[repair:verify:%s] fallback %s search failed for %s, will retry", + arr.name, strategy, v.get("title", key)) + return issued diff --git a/doctor/checks/resources.py b/doctor/checks/resources.py new file mode 100644 index 0000000..0169a5d --- /dev/null +++ b/doctor/checks/resources.py @@ -0,0 +1,23 @@ +"""Check: resources.""" +from ..config import DRY_RUN, host_load, RES_DROP_CACHES, RES_LOAD_WARN, RES_MEM_MIN, RES_SWAP_WARN, run_cmd, log + +def _meminfo(): + d = {} + try: + for line in open("/proc/meminfo"): + k, _, v = line.partition(":") + d[k.strip()] = int(v.split()[0]) // 1024 # MB + except Exception: + pass + return d +def check_resources(): + l1 = host_load() + mi = _meminfo() + avail = mi.get("MemAvailable", -1) + swap_used = mi.get("SwapTotal", 0) - mi.get("SwapFree", 0) + msg = "[resources] load=%.1f memAvail=%sMB swapUsed=%sMB" % (l1, avail, swap_used) + crit = (l1 >= RES_LOAD_WARN) or (0 <= avail < RES_MEM_MIN) or (swap_used >= RES_SWAP_WARN) + (log.warning if crit else log.info)(msg + (" <-- PRESSURE" if crit else "")) + if crit and RES_DROP_CACHES and not DRY_RUN: + rc = run_cmd("sync; echo 1 > /proc/sys/vm/drop_caches") + log.warning("[resources] dropped page cache rc=%s", rc[0] if rc else "?") diff --git a/doctor/checks/seerr.py b/doctor/checks/seerr.py new file mode 100644 index 0000000..4327739 --- /dev/null +++ b/doctor/checks/seerr.py @@ -0,0 +1,44 @@ +"""Check: seerr.""" +from ..config import DRY_RUN, SEERR_APIKEY, SEERR_MAX, SEERR_MAX_TRIES, SEERR_URL, log +from ..clients import Seerr +from ..state import state_transaction + +def check_seerr(): + if not SEERR_URL or not SEERR_APIKEY: + return + s = Seerr(SEERR_URL, SEERR_APIKEY) + reqs = s.failed() + if reqs is None: + log.error("[seerr] %s unreachable", SEERR_URL); return + if not reqs: + log.info("[seerr] no failed requests"); return + with state_transaction() as state: + tries = state.setdefault("__seerr__", {}) + log.warning("[seerr] %d failed request(s)", len(reqs)) + acted = 0 + for r in reqs: + if acted >= SEERR_MAX: + break + rid = r.get("id") + if rid is None: + continue + md = r.get("media") or {} + label = "%s tmdb=%s req#%s" % (md.get("mediaType", "?"), md.get("tmdbId", "?"), rid) + n = int(tries.get(str(rid), 0)) + if SEERR_MAX_TRIES and n >= SEERR_MAX_TRIES: + log.error("[seerr] giving up on %s after %d retries (persistent failure)", label, n) + continue + if DRY_RUN: + log.info("[seerr] DRY-RUN would retry %s", label); acted += 1; continue + try: + s.retry(rid) + tries[str(rid)] = n + 1 + acted += 1 + log.info("[seerr] retried %s (attempt %d)", label, n + 1) + except Exception as e: + log.warning("[seerr] retry %s failed: %s", label, str(e)[:80]) + live = set(str(r.get("id")) for r in reqs if r.get("id") is not None) + for k in [k for k in tries if k not in live]: + tries.pop(k, None) + if acted: + log.info("[seerr] re-drove %d failed request(s)", acted) diff --git a/doctor/checks/warmer.py b/doctor/checks/warmer.py new file mode 100644 index 0000000..8d66f0e --- /dev/null +++ b/doctor/checks/warmer.py @@ -0,0 +1,210 @@ +"""Check: warmer.""" +import os +import re +import time +import subprocess +import threading +from ..config import ( + host_load, log, + PLEX_TOKEN, PLEX_URL, + WARM_CONCURRENCY, WARM_COOLDOWN, WARM_HEAD_MB, WARM_INTERVAL, + WARM_LOAD_MAX, WARM_LOW_CACHE, WARM_MAX_CYCLE, WARM_NEXT_EPS, + WARM_NEXT_NEAR_END, WARM_ONDECK, WARM_ONDECK_EVERY, WARM_OPEN_CONC, + WARM_PARTS, WARM_PATH_MAP, WARM_PLEXLOG_CMD, WARM_PLEXLOG_FILE, + WARM_READ_TIMEOUT, WARM_RECENT_COUNT, WARM_SOURCES, WARM_TAIL_MB, +) +from ..clients import Plex + +_warm_state = {} # host_path -> last_warm_ts +_warm_lock = threading.Lock() +_warm_sem = threading.Semaphore(max(1, WARM_CONCURRENCY)) # background warming lane +_warm_sem_open = threading.Semaphore(max(1, WARM_OPEN_CONC)) # detail-page (you opened it) lane - separate so opens never wait +_warm_last_ondeck = [0.0] +_warm_count = [0] # total warms since start (for the UI) +_warm_recent = [] # recent warms for the UI: [{"ts","title","why"}] + +# Per-cycle metadata (single-element lists so tests can reset them by index) +_last_cycle_ts = [0.0] # unix ts when the last warm_cycle() started +_last_cycle_duration = [0.0] # seconds taken by the last cycle +_last_cycle_warmed = [0] # files warmed in the last cycle +_last_cycle_candidates = [0] # candidate paths considered in the last cycle +_last_cycle_skipped_load = [False] # True if the last cycle was skipped due to host load +def _warm_record(title, why): + _warm_count[0] += 1 + _warm_recent.append({"ts": time.time(), "title": title, "why": why}) + if len(_warm_recent) > 80: + del _warm_recent[:len(_warm_recent) - 80] +def _limit_parts(files): + return files if WARM_PARTS <= 0 else files[:WARM_PARTS] +def _host_path(f): + if WARM_PATH_MAP and ":" in WARM_PATH_MAP: + a, b = WARM_PATH_MAP.split(":", 1) + if f.startswith(a): + return b + f[len(a):] + return f +def _warm_file(path, reason="cycle"): + p = _host_path(path) + # a title you actively opened tolerates more load (2x) than speculative background warming, but + # both still yield before meltdown; concurrency stays capped either way so a burst can't flood. + guard = (WARM_LOAD_MAX * 2) if reason == "detail-page" else WARM_LOAD_MAX + if guard > 0 and host_load() > guard: + return False + with _warm_lock: # atomic claim: one warm per file per cooldown + if time.time() - _warm_state.get(p, 0) < WARM_COOLDOWN: + return False + _warm_state[p] = time.time() + try: + sz = os.path.getsize(p) + except Exception as e: + _warm_state.pop(p, None) # release so it can be retried + log.debug("[warmer] stat fail %s: %s", p, str(e)[:60]); return False + head = min(WARM_HEAD_MB << 20, sz) + tail = WARM_TAIL_MB > 0 and sz > head + (WARM_TAIL_MB << 20) + res = {"got": 0, "err": None} + def _do(): + try: + with open(p, "rb", buffering=0) as fh: + while res["got"] < head: + b = fh.read(min(4 << 20, head - res["got"])) + if not b: break + res["got"] += len(b) + if tail: + fh.seek(sz - (WARM_TAIL_MB << 20)) + while fh.read(4 << 20): + pass + except Exception as e: + res["err"] = str(e)[:60] + t0 = time.time() + sem = _warm_sem_open if reason == "detail-page" else _warm_sem # opens get their own lane (instant) + with sem: # cap concurrent usenet pulls so warming never floods decypharr + th = threading.Thread(target=_do, daemon=True); th.start(); th.join(WARM_READ_TIMEOUT) + if th.is_alive(): + _warm_state.pop(p, None) + log.warning("[warmer] read timed out (%ds, mount slow/hung?): %s", WARM_READ_TIMEOUT, os.path.basename(p)) + return False + if res["err"]: + _warm_state.pop(p, None) + log.warning("[warmer] read fail %s: %s", os.path.basename(p), res["err"]); return False + _warm_record(os.path.basename(p), reason) + log.info("[warmer] warmed %dMB head%s in %.1fs: %s", + res["got"] >> 20, "+%dMB tail" % WARM_TAIL_MB if tail else "", + time.time() - t0, os.path.basename(p)) + return True +def _warm_targets(plex): + """Ordered, de-duped list of (reason, plex_file_path) to warm this cycle.""" + targets, seen = [], set() + def add(reason, path): + if path and path not in seen: + seen.add(path); targets.append((reason, path)) + sessions = plex.sessions() + if "next" in WARM_SOURCES: # next episode(s) of anything playing + for v in sessions: + if v.get("type") != "episode" or not v.get("grandparentRatingKey"): + continue + if WARM_NEXT_NEAR_END > 0: # only warm the next ep once the current one nears the end + try: + remain_min = (int(v.get("duration", 0)) - int(v.get("viewOffset", 0))) / 60000.0 + except Exception: + remain_min = 0 + if remain_min > WARM_NEXT_NEAR_END: + continue + eps = plex.leaves(v.get("grandparentRatingKey")) + idx = next((i for i, e in enumerate(eps) if e.get("ratingKey") == v.get("ratingKey")), -1) + if idx >= 0: + for e in eps[idx + 1: idx + 1 + WARM_NEXT_EPS]: + for f in _limit_parts(plex.parts(e.get("ratingKey"))): + add("next-ep", f) + # Plex-first: speculative On Deck / recent warming pauses while ANYONE is watching (never competes + # with a live stream), and is skipped entirely in low-cache mode (keep almost nothing pre-warmed). + if not WARM_LOW_CACHE and not sessions and time.time() - _warm_last_ondeck[0] >= WARM_ONDECK_EVERY: + _warm_last_ondeck[0] = time.time() + if WARM_ONDECK and "ondeck" in WARM_SOURCES: # Continue Watching / Up Next (WARMER_ONDECK is the on/off) + for v in plex.ondeck(): + for f in _limit_parts(plex.parts(v.get("ratingKey"))): + add("ondeck", f) + if "recent" in WARM_SOURCES and WARM_RECENT_COUNT > 0: + for v in plex.recent(WARM_RECENT_COUNT): + for f in _limit_parts(plex.parts(v.get("ratingKey"))): + add("recent", f) + return targets +def warm_cycle(): + _t0 = time.time() + _last_cycle_ts[0] = _t0 + if WARM_LOAD_MAX > 0 and host_load() > WARM_LOAD_MAX: + log.info("[warmer] host load > %.0f -> skip cycle", WARM_LOAD_MAX) + _last_cycle_skipped_load[0] = True + _last_cycle_duration[0] = round(time.time() - _t0, 3) + return + _last_cycle_skipped_load[0] = False + targets = _warm_targets(Plex(PLEX_URL, PLEX_TOKEN)) + _last_cycle_candidates[0] = len(targets) + done = 0 + for reason, path in targets: + if done >= WARM_MAX_CYCLE: + break + if _warm_file(path, reason): + done += 1 + _last_cycle_warmed[0] = done + _last_cycle_duration[0] = round(time.time() - _t0, 3) + if done: + log.info("[warmer] cycle warmed %d (of %d candidate paths)", done, len(targets)) +def warmer_loop(stop): + mode = (" | LOW-CACHE: no On Deck, next ep @<=%dmin left" % WARM_NEXT_NEAR_END) if WARM_LOW_CACHE \ + else ((" | next ep @<=%dmin left" % WARM_NEXT_NEAR_END) if WARM_NEXT_NEAR_END else "") + log.info("[warmer] started: head=%dMB tail=%dMB sources=%s poll=%ds ondeck-every=%ds%s", + WARM_HEAD_MB, WARM_TAIL_MB, ",".join(WARM_SOURCES) or "-", WARM_INTERVAL, WARM_ONDECK_EVERY, mode) + while not stop.is_set(): + try: + warm_cycle() + except Exception as e: + log.error("[warmer] cycle error: %s", e) + if stop.wait(WARM_INTERVAL): + break +_PLEXLOG_RE = re.compile(r"/library/metadata/(\d+)(?:/extras|\?[^\s]*includeExtras=1)") +_playing = {"ts": 0.0, "rks": set()} +def _playing_rks(plex): + """ratingKeys with an active Plex session, cached ~10s (Plex sends the same metadata query while + you browse a title AND while you play it, so this tells the two apart).""" + if time.time() - _playing["ts"] > 10: + try: _playing["rks"] = set(v.get("ratingKey") for v in plex.sessions()) + except Exception: pass + _playing["ts"] = time.time() + return _playing["rks"] +def _warm_opened(plex, rk): + if rk in _playing_rks(plex): # already playing (so already cached) -> not a new open + return + for f in _limit_parts(plex.parts(rk)): # warm just the top version(s) you'd actually play + if _warm_file(f, "detail-page"): + log.info("[warmer] you opened rk=%s -> warmed: %s", rk, os.path.basename(_host_path(f))) +def plexlog_loop(stop): + """Tail Plex's server log; warm the exact title a viewer opens (true pre-play intent).""" + cmd = WARM_PLEXLOG_CMD or ("tail -n0 -F %r" % WARM_PLEXLOG_FILE if WARM_PLEXLOG_FILE else "") + if not cmd: + return + plex = Plex(PLEX_URL, PLEX_TOKEN) + seen = {} # ratingKey -> last-handled ts + log.info("[warmer] detail-page warming enabled (tailing Plex log)") + while not stop.is_set(): + proc = None + try: + proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, text=True, bufsize=1) + for line in proc.stdout: + if stop.is_set(): + break + m = _PLEXLOG_RE.search(line) + if not m: + continue + rk = m.group(1); now = time.time() + if now - seen.get(rk, 0) < 300: # a detail page is polled repeatedly while open -> react once per item / 5 min + continue + seen[rk] = now # warm off-thread so the tailer stays responsive + threading.Thread(target=_warm_opened, args=(plex, rk), daemon=True).start() + except Exception as e: + log.warning("[warmer] plexlog tail error: %s", str(e)[:80]) + finally: + if proc: + try: proc.terminate() + except Exception: pass + if stop.wait(10): # tail died/rotated -> reconnect + break diff --git a/doctor/clients.py b/doctor/clients.py new file mode 100644 index 0000000..fc19456 --- /dev/null +++ b/doctor/clients.py @@ -0,0 +1,377 @@ +"""HTTP API clients: Arr (Sonarr/Radarr/Prowlarr), Plex, Seerr + instance loader.""" +import os +import json +from datetime import datetime +import time +import urllib.request +import urllib.error +import socket +import xml.etree.ElementTree as ET +from typing import Optional +from .config import BLOCKLIST, REMOVE_CLIENT, TIMEOUT, log + +class Arr: + def __init__(self, name: str, kind: str, url: str, apikey: str): + self.name, self.kind = name, kind # sonarr | radarr | prowlarr + self.base = url.rstrip("/") + ("/api/v1" if kind == "prowlarr" else "/api/v3") + self.apikey = apikey + self.unknown = "includeUnknownSeriesItems=true" if kind == "sonarr" else "includeUnknownMovieItems=true" + + def _req(self, method: str, path: str, data: Optional[bytes] = None, + t: Optional[float] = None, retries: int = 3): + """Make an HTTP request with retry/backoff for transient failures. + + Retries on: 5xx, 429, timeout, connection reset. + Does not retry on: 4xx (except 429), 2xx/3xx responses. + """ + t = t or TIMEOUT + last_exc = None + for attempt in range(retries + 1): + try: + req = urllib.request.Request(self.base + path, data=data, method=method, + headers={"X-Api-Key": self.apikey, "Content-Type": "application/json"}) + return urllib.request.urlopen(req, timeout=t) + except urllib.error.HTTPError as e: + if e.code in (429, 502, 503, 504) and attempt < retries: + wait = 2 ** attempt + (0.5 if e.code == 429 else 0) + log.debug("[%s] %s %s -> %d, retrying in %.1fs (attempt %d/%d)", + self.name, method, path, e.code, wait, attempt + 1, retries) + time.sleep(wait) + last_exc = e + continue + raise + except (socket.timeout, urllib.error.URLError, ConnectionResetError, BrokenPipeError) as e: + if attempt < retries: + wait = 2 ** attempt + log.debug("[%s] %s %s -> %s, retrying in %.1fs (attempt %d/%d)", + self.name, method, path, str(e)[:50], wait, attempt + 1, retries) + time.sleep(wait) + last_exc = e + continue + raise + raise last_exc + + def queue(self): + if self.kind == "prowlarr": + return [] # prowlarr has no download queue + try: + return json.load(self._req("GET", "/queue?page=1&pageSize=1000&" + self.unknown)).get("records", []) + except Exception as e: + log.warning("[%s] queue fetch failed: %s", self.name, e); return None + + def health(self): + try: + return json.load(self._req("GET", "/health")) + except Exception: + return [] + + def remove(self, item_id): + q = "removeFromClient=%s&blocklist=%s" % (str(REMOVE_CLIENT).lower(), str(BLOCKLIST).lower()) + self._req("DELETE", "/queue/%d?%s" % (item_id, q)) + + def post(self, path, t=150): + """POST with empty body (used for /indexer/testall, /downloadclient/testall). Returns parsed JSON or [].""" + try: + body = self._req("POST", path, data=b"", t=t).read() + return json.loads(body) if body else [] + except urllib.error.HTTPError as e: + try: return json.loads(e.read()) + except Exception: return [] + except Exception as ex: + log.debug("[%s] POST %s err %s", self.name, path, str(ex)[:50]); return [] + + def set_monitored(self, ids, monitored): + """Bulk toggle monitoring for episodes (sonarr) / movies (radarr). Used by the churn brake.""" + if self.kind == "sonarr": + path, body = "/episode/monitor", {"episodeIds": list(ids), "monitored": monitored} + elif self.kind == "radarr": + path, body = "/movie/editor", {"movieIds": list(ids), "monitored": monitored} + else: + return False + try: + self._req("PUT", path, data=json.dumps(body).encode()); return True + except Exception as e: + log.warning("[churn:%s] monitor %s failed: %s", self.name, "on" if monitored else "off", str(e)[:70]) + return False + + def queue_target_id(self, rec): + """Stable id of what a queue record is FOR (episode for sonarr, movie for radarr).""" + return rec.get("episodeId") if self.kind == "sonarr" else rec.get("movieId") if self.kind == "radarr" else None + + # ---- repair helpers (map a dead library file -> *arr item, then remove + re-search) ---- + def _jget(self, path, t=30): + try: + return json.load(self._req("GET", path, t=t)) + except Exception as e: + log.warning("[%s] GET %s failed: %s", self.name, path, str(e)[:70]); return None + + def movies(self): + return self._jget("/movie") or [] # radarr: each has movieFile.path + + def series(self): + return self._jget("/series") or [] # sonarr + + def quality_profiles(self): + return self._jget("/qualityprofile") or [] # sonarr/radarr + + def update_series(self, series_dict): + """PUT the full series dict back (used to change qualityProfileId etc.).""" + return self._req("PUT", "/series/%d" % series_dict["id"], + data=json.dumps(series_dict).encode()) + + def episode_files(self, sid): + return self._jget("/episodefile?seriesId=%d" % sid) or [] + + def episodes(self, sid): + return self._jget("/episode?seriesId=%d" % sid) or [] + + def delete_file(self, file_id): + """Delete a movieFile/episodeFile record (removes the dead library symlink so it can be re-grabbed).""" + ep = "/moviefile/%d" % file_id if self.kind == "radarr" else "/episodefile/%d" % file_id + try: + self._req("DELETE", ep); return True + except Exception as e: + log.warning("[%s] delete file %s failed: %s", self.name, file_id, str(e)[:70]); return False + + def command(self, name, **kw): + """POST /command and return the command ID (int) on success, or None on failure.""" + body = {"name": name}; body.update(kw) + try: + resp = json.load(self._req("POST", "/command", data=json.dumps(body).encode())) + return resp.get("id") or True # return id if present, else True for compat + except Exception as e: + log.warning("[%s] command %s failed: %s", self.name, name, str(e)[:70]); return None + + def command_status(self, command_id): + """Poll GET /command/{id}. Returns the status string, or None on error.""" + try: + resp = json.load(self._req("GET", "/command/%d" % command_id)) + return resp.get("status") + except Exception: + return None + + def manualimport(self, series_id=None, movie_id=None, folder=None, t=30): + """Fetch manual-import candidates for a series/movie or folder. + Returns a list of file dicts, each with path, episodeIds/movieId, quality, etc.""" + import urllib.parse + params = [] + if self.kind == "sonarr" and series_id: + params.append("seriesId=%d" % series_id) + elif self.kind == "radarr" and movie_id: + params.append("movieId=%d" % movie_id) + if folder: + params.append("folder=%s" % urllib.parse.quote(folder)) + if not params: + return [] + path = "/manualimport?%s" % "&".join(params) + try: + return self._jget(path, t=t) or [] + except Exception as e: + log.warning("[%s] manualimport fetch failed: %s", self.name, str(e)[:70]) + return [] + + def manualimport_command(self, files, import_mode="auto"): + """POST a ManualImport command with the supplied file list. + Returns the command id on success, or None on failure.""" + if not files: + return None + body = {"name": "ManualImport", "files": files, "importMode": import_mode} + try: + resp = json.load(self._req("POST", "/command", data=json.dumps(body).encode())) + return resp.get("id") or True + except Exception as e: + log.warning("[%s] manualimport command failed: %s", self.name, str(e)[:70]) + return None + + def release_search(self, series_id, season_number=1, timeout=45): + """GET /release?seriesId=&seasonNumber= — returns list of release dicts (same as Sonarr UI). + Returns [] on failure.""" + try: + resp = self._req("GET", "/release?seriesId=%d&seasonNumber=%d" % (series_id, season_number), + t=timeout) + return json.load(resp) or [] + except Exception as e: + log.debug("[%s] release_search(%d, %d) failed: %s", self.name, series_id, season_number, str(e)[:60]) + return [] + + def release_push(self, release): + """POST /release/push — bypasses Sonarr's rejection logic and pushes directly to download client. + Returns True on success.""" + try: + self._req("POST", "/release/push", data=json.dumps(release).encode()) + return True + except urllib.error.HTTPError as e: + log.warning("[%s] release_push failed HTTP %d: %s", self.name, e.code, e.read()[:80]) + return False + except Exception as e: + log.warning("[%s] release_push failed: %s", self.name, str(e)[:60]) + return False + + def history_grabbed(self, media_id, since_ts, entity_ids=None): + """Return the most recent 'grabbed' history record for media_id posted after since_ts. + For sonarr, optionally filter to specific episode IDs. Returns None if nothing found.""" + records = self.history(media_id, page_size=50) + if isinstance(records, dict): + records = records.get("records") or [] + # Parse both timestamps to floats for a reliable "strictly after" comparison. + # Lexicographic string comparison breaks when *arr dates include milliseconds + # (e.g. "T04:55:30.5Z") or +00:00 offsets — different suffixes sort differently + # than the "Z" suffix stored in search_ts. + try: + since_epoch = datetime.fromisoformat(since_ts.replace("Z", "+00:00")).timestamp() + except Exception: + since_epoch = None + for rec in records: + if rec.get("eventType") != "grabbed": + continue + rec_date = rec.get("date") or "" + if since_epoch is not None: + try: + rec_epoch = datetime.fromisoformat(rec_date.replace("Z", "+00:00")).timestamp() + if rec_epoch <= since_epoch: + continue + except Exception: + continue # skip records with unparseable dates + elif not rec_date or rec_date <= since_ts: + continue # fallback: string compare (since_epoch parse failed) + if entity_ids and self.kind == "sonarr": + if rec.get("episodeId") not in entity_ids: + continue + return rec + return None + + def history(self, media_id, page_size=100): + """Fetch download history for a specific series (sonarr) or movie (radarr). + Returns a list of history records, each with eventType, sourceTitle, data dict, etc.""" + if self.kind == "sonarr": + path = "/history/series?seriesId=%d&pageSize=%d&includeSeries=false&includeEpisode=true" % (media_id, page_size) + elif self.kind == "radarr": + path = "/history/movie?movieId=%d&pageSize=%d" % (media_id, page_size) + else: + return [] + return self._jget(path) or [] +def load_instances(): + out = [] + for n in range(1, 51): + url = os.environ.get("INSTANCE_%d_URL" % n) + if not url: + continue + key = os.environ.get("INSTANCE_%d_APIKEY" % n, "") + kind = os.environ.get("INSTANCE_%d_TYPE" % n, "").strip().lower() + if kind not in ("sonarr", "radarr", "prowlarr"): + kind = ("radarr" if "radarr" in url.lower() else + "prowlarr" if "prowlarr" in url.lower() else "sonarr") + name = os.environ.get("INSTANCE_%d_NAME" % n, "%s-%d" % (kind, n)) + if not key: + log.warning("INSTANCE_%d has no APIKEY, skipping", n); continue + out.append(Arr(name, kind, url, key)) + return out +INSTANCES = [] +class Plex: + def __init__(self, url, token): + self.url = url.rstrip("/"); self.token = token + + def _get(self, path): + sep = "&" if "?" in path else "?" + with urllib.request.urlopen(self.url + path + sep + "X-Plex-Token=" + self.token, timeout=15) as r: + return ET.fromstring(r.read()) + + def sessions(self): + try: return list(self._get("/status/sessions").iter("Video")) + except Exception: return [] + + def ondeck(self): + try: return list(self._get("/library/onDeck").iter("Video")) + except Exception: return [] + + def leaves(self, show_rk): + try: return list(self._get("/library/metadata/%s/allLeaves" % show_rk).iter("Video")) + except Exception: return [] + + def parts(self, rk): + """File paths for this item, highest-resolution version first (so we can warm just the top one).""" + out = [] + try: + for m in self._get("/library/metadata/%s" % rk).iter("Media"): + try: res = int(m.get("height") or 0) * 1000000 + int(m.get("bitrate") or 0) + except Exception: res = 0 + for p in m.iter("Part"): + if p.get("file"): + out.append((res, p.get("file"))) + out.sort(key=lambda x: x[0], reverse=True) + except Exception: + return [] + return [f for _, f in out] + + def recent(self, n): + out = [] + try: + for d in self._get("/library/sections").iter("Directory"): + if d.get("type") in ("movie", "show"): + ra = self._get("/library/sections/%s/recentlyAdded?X-Plex-Container-Start=0&X-Plex-Container-Size=%d" % (d.get("key"), n)) + out += list(ra.iter("Video"))[:n] + except Exception: pass + return out + + def activities(self): + """Running background activities (library scans, analysis...). Used by the plexscan check.""" + try: return list(self._get("/activities").iter("Activity")) + except Exception: return [] + + def cancel_activity(self, uuid): + try: + req = urllib.request.Request(self.url + "/activities/" + uuid + "?X-Plex-Token=" + self.token, method="DELETE") + urllib.request.urlopen(req, timeout=10); return True + except Exception: + return False +class Seerr: + def __init__(self, url, apikey): + self.base = url.rstrip("/") + "/api/v1" + self.apikey = apikey + + def _req(self, method: str, path: str, data: Optional[bytes] = None, + t: Optional[float] = None, retries: int = 3): + """Make an HTTP request with retry/backoff for transient failures. + + Retries on: 5xx, 429, timeout, connection reset. + Does not retry on: 4xx (except 429), 2xx/3xx responses. + """ + t = t or TIMEOUT + last_exc = None + for attempt in range(retries + 1): + try: + req = urllib.request.Request(self.base + path, data=data, method=method, + headers={"X-Api-Key": self.apikey, "Content-Type": "application/json"}) + return urllib.request.urlopen(req, timeout=t) + except urllib.error.HTTPError as e: + if e.code in (429, 502, 503, 504) and attempt < retries: + wait = 2 ** attempt + (0.5 if e.code == 429 else 0) + log.debug("[%s] %s %s -> %d, retrying in %.1fs (attempt %d/%d)", + self.name, method, path, e.code, wait, attempt + 1, retries) + time.sleep(wait) + last_exc = e + continue + raise + except (socket.timeout, urllib.error.URLError, ConnectionResetError, BrokenPipeError) as e: + if attempt < retries: + wait = 2 ** attempt + log.debug("[%s] %s %s -> %s, retrying in %.1fs (attempt %d/%d)", + self.name, method, path, str(e)[:50], wait, attempt + 1, retries) + time.sleep(wait) + last_exc = e + continue + raise + raise last_exc + + def failed(self): + """Requests currently in the FAILED state (seerr could not hand them to the arr).""" + try: + d = json.load(self._req("GET", "/request?take=100&skip=0&filter=failed&sort=added", t=15)) + return d.get("results", []) + except Exception as e: + log.warning("[seerr] failed-list fetch error: %s", str(e)[:80]); return None + + def retry(self, rid): + self._req("POST", "/request/%d/retry" % int(rid), data=b"", t=30) + +__all__ = [n for n in dir() if not n.startswith("__") and not isinstance(globals()[n], type(os))] diff --git a/doctor/clients/__init__.py b/doctor/clients/__init__.py new file mode 100644 index 0000000..ab9ed00 --- /dev/null +++ b/doctor/clients/__init__.py @@ -0,0 +1,10 @@ +"""HTTP API clients package: Arr (Sonarr/Radarr/Prowlarr), Plex, Seerr.""" +from .arr import Arr +from .plex import Plex +from .seerr import Seerr +from .decypharr import Decypharr +from .tautulli import Tautulli +from .pulsarr import Pulsarr +from .loader import load_instances, INSTANCES + +__all__ = ["Arr", "Plex", "Seerr", "Decypharr", "Tautulli", "Pulsarr", "load_instances", "INSTANCES"] diff --git a/doctor/clients/arr.py b/doctor/clients/arr.py new file mode 100644 index 0000000..c5b1e33 --- /dev/null +++ b/doctor/clients/arr.py @@ -0,0 +1,258 @@ +"""Arr client: Sonarr/Radarr/Prowlarr.""" +import json +from datetime import datetime +import time +import urllib.request +import urllib.error +import urllib.parse +import socket +from typing import Optional +from ..config import BLOCKLIST, REMOVE_CLIENT, TIMEOUT, log + +class Arr: + def __init__(self, name: str, kind: str, url: str, apikey: str): + self.name, self.kind = name, kind # sonarr | radarr | prowlarr + self.base = url.rstrip("/") + ("/api/v1" if kind == "prowlarr" else "/api/v3") + self.apikey = apikey + self.unknown = "includeUnknownSeriesItems=true" if kind == "sonarr" else "includeUnknownMovieItems=true" + + def _req(self, method: str, path: str, data: Optional[bytes] = None, + t: Optional[float] = None, retries: int = 3): + """Make an HTTP request with retry/backoff for transient failures. + + Retries on: 5xx, 429, timeout, connection reset. + Does not retry on: 4xx (except 429), 2xx/3xx responses. + """ + t = t or TIMEOUT + last_exc = None + for attempt in range(retries + 1): + try: + req = urllib.request.Request(self.base + path, data=data, method=method, + headers={"X-Api-Key": self.apikey, "Content-Type": "application/json"}) + return urllib.request.urlopen(req, timeout=t) + except urllib.error.HTTPError as e: + if e.code in (429, 502, 503, 504) and attempt < retries: + wait = 2 ** attempt + (0.5 if e.code == 429 else 0) + log.debug("[%s] %s %s -> %d, retrying in %.1fs (attempt %d/%d)", + self.name, method, path, e.code, wait, attempt + 1, retries) + time.sleep(wait) + last_exc = e + continue + raise + except (socket.timeout, urllib.error.URLError, ConnectionResetError, BrokenPipeError) as e: + if attempt < retries: + wait = 2 ** attempt + log.debug("[%s] %s %s -> %s, retrying in %.1fs (attempt %d/%d)", + self.name, method, path, str(e)[:50], wait, attempt + 1, retries) + time.sleep(wait) + last_exc = e + continue + raise + raise last_exc + + def queue(self): + if self.kind == "prowlarr": + return [] # prowlarr has no download queue + try: + return json.load(self._req("GET", "/queue?page=1&pageSize=1000&" + self.unknown)).get("records", []) + except Exception as e: + log.warning("[%s] queue fetch failed: %s", self.name, e); return None + + def health(self): + try: + return json.load(self._req("GET", "/health")) + except Exception: + return [] + + def remove(self, item_id): + q = "removeFromClient=%s&blocklist=%s" % (str(REMOVE_CLIENT).lower(), str(BLOCKLIST).lower()) + self._req("DELETE", "/queue/%d?%s" % (item_id, q)) + + def post(self, path, t=150): + """POST with empty body (used for /indexer/testall, /downloadclient/testall). Returns parsed JSON or [].""" + try: + body = self._req("POST", path, data=b"", t=t).read() + return json.loads(body) if body else [] + except urllib.error.HTTPError as e: + try: return json.loads(e.read()) + except Exception: return [] + except Exception as ex: + log.debug("[%s] POST %s err %s", self.name, path, str(ex)[:50]); return [] + + def set_monitored(self, ids, monitored): + """Bulk toggle monitoring for episodes (sonarr) / movies (radarr). Used by the churn brake.""" + if self.kind == "sonarr": + path, body = "/episode/monitor", {"episodeIds": list(ids), "monitored": monitored} + elif self.kind == "radarr": + path, body = "/movie/editor", {"movieIds": list(ids), "monitored": monitored} + else: + return False + try: + self._req("PUT", path, data=json.dumps(body).encode()); return True + except Exception as e: + log.warning("[churn:%s] monitor %s failed: %s", self.name, "on" if monitored else "off", str(e)[:70]) + return False + + def queue_target_id(self, rec): + """Stable id of what a queue record is FOR (episode for sonarr, movie for radarr).""" + return rec.get("episodeId") if self.kind == "sonarr" else rec.get("movieId") if self.kind == "radarr" else None + + # ---- repair helpers (map a dead library file -> *arr item, then remove + re-search) ---- + def _jget(self, path, t=30): + try: + return json.load(self._req("GET", path, t=t)) + except Exception as e: + log.warning("[%s] GET %s failed: %s", self.name, path, str(e)[:70]); return None + + def movies(self): + return self._jget("/movie") or [] # radarr: each has movieFile.path + + def series(self): + return self._jget("/series") or [] # sonarr + + def tag_map(self): + """Return {tag_id: label} mapping for all Sonarr tags, or {} on failure.""" + try: + tags = self._jget("/tag") or [] + return {t["id"]: t["label"] for t in tags} + except Exception: + return {} + + def quality_profiles(self): + return self._jget("/qualityprofile") or [] # sonarr/radarr + + def update_series(self, series_dict): + """PUT the full series dict back (used to change qualityProfileId etc.).""" + return self._req("PUT", "/series/%d" % series_dict["id"], + data=json.dumps(series_dict).encode()) + + def episode_files(self, sid): + return self._jget("/episodefile?seriesId=%d" % sid) or [] + + def episodes(self, sid): + return self._jget("/episode?seriesId=%d" % sid) or [] + + def delete_file(self, file_id): + """Delete a movieFile/episodeFile record (removes the dead library symlink so it can be re-grabbed).""" + ep = "/moviefile/%d" % file_id if self.kind == "radarr" else "/episodefile/%d" % file_id + try: + self._req("DELETE", ep); return True + except Exception as e: + log.warning("[%s] delete file %s failed: %s", self.name, file_id, str(e)[:70]); return False + + def command(self, name, **kw): + """POST /command and return the command ID (int) on success, or None on failure.""" + body = {"name": name}; body.update(kw) + try: + resp = json.load(self._req("POST", "/command", data=json.dumps(body).encode())) + return resp.get("id") or True # return id if present, else True for compat + except Exception as e: + log.warning("[%s] command %s failed: %s", self.name, name, str(e)[:70]); return None + + def command_status(self, command_id): + """Poll GET /command/{id}. Returns the status string, or None on error.""" + try: + resp = json.load(self._req("GET", "/command/%d" % command_id)) + return resp.get("status") + except Exception: + return None + + def manualimport(self, series_id=None, movie_id=None, folder=None, t=30): + """Fetch manual-import candidates for a series/movie or folder. + Returns a list of file dicts, each with path, episodeIds/movieId, quality, etc.""" + params = [] + if self.kind == "sonarr" and series_id: + params.append("seriesId=%d" % series_id) + elif self.kind == "radarr" and movie_id: + params.append("movieId=%d" % movie_id) + if folder: + params.append("folder=%s" % urllib.parse.quote(folder)) + if not params: + return [] + path = "/manualimport?%s" % "&".join(params) + try: + return self._jget(path, t=t) or [] + except Exception as e: + log.warning("[%s] manualimport fetch failed: %s", self.name, str(e)[:70]) + return [] + + def manualimport_command(self, files, import_mode="auto"): + """POST a ManualImport command with the supplied file list. + Returns the command id on success, or None on failure.""" + if not files: + return None + body = {"name": "ManualImport", "files": files, "importMode": import_mode} + try: + resp = json.load(self._req("POST", "/command", data=json.dumps(body).encode())) + return resp.get("id") or True + except Exception as e: + log.warning("[%s] manualimport command failed: %s", self.name, str(e)[:70]) + return None + + def release_search(self, series_id, season_number=1, timeout=45): + """GET /release?seriesId=&seasonNumber= — returns list of release dicts (same as Sonarr UI). + Returns [] on failure.""" + try: + resp = self._req("GET", "/release?seriesId=%d&seasonNumber=%d" % (series_id, season_number), + t=timeout) + return json.load(resp) or [] + except Exception as e: + log.debug("[%s] release_search(%d, %d) failed: %s", self.name, series_id, season_number, str(e)[:60]) + return [] + + def release_push(self, release): + """POST /release/push — bypasses Sonarr's rejection logic and pushes directly to download client. + Returns True on success.""" + try: + self._req("POST", "/release/push", data=json.dumps(release).encode()) + return True + except urllib.error.HTTPError as e: + log.warning("[%s] release_push failed HTTP %d: %s", self.name, e.code, e.read()[:80]) + return False + except Exception as e: + log.warning("[%s] release_push failed: %s", self.name, str(e)[:60]) + return False + + def history_grabbed(self, media_id, since_ts, entity_ids=None): + """Return the most recent 'grabbed' history record for media_id posted after since_ts. + For sonarr, optionally filter to specific episode IDs. Returns None if nothing found.""" + records = self.history(media_id, page_size=50) + if isinstance(records, dict): + records = records.get("records") or [] + # Parse both timestamps to floats for a reliable "strictly after" comparison. + # Lexicographic string comparison breaks when *arr dates include milliseconds + # (e.g. "T04:55:30.5Z") or +00:00 offsets — different suffixes sort differently + # than the "Z" suffix stored in search_ts. + try: + since_epoch = datetime.fromisoformat(since_ts.replace("Z", "+00:00")).timestamp() + except Exception: + since_epoch = None + for rec in records: + if rec.get("eventType") != "grabbed": + continue + rec_date = rec.get("date") or "" + if since_epoch is not None: + try: + rec_epoch = datetime.fromisoformat(rec_date.replace("Z", "+00:00")).timestamp() + if rec_epoch <= since_epoch: + continue + except Exception: + continue # skip records with unparseable dates + elif not rec_date or rec_date <= since_ts: + continue # fallback: string compare (since_epoch parse failed) + if entity_ids and self.kind == "sonarr": + if rec.get("episodeId") not in entity_ids: + continue + return rec + return None + + def history(self, media_id, page_size=100): + """Fetch download history for a specific series (sonarr) or movie (radarr). + Returns a list of history records, each with eventType, sourceTitle, data dict, etc.""" + if self.kind == "sonarr": + path = "/history/series?seriesId=%d&pageSize=%d&includeSeries=false&includeEpisode=true" % (media_id, page_size) + elif self.kind == "radarr": + path = "/history/movie?movieId=%d&pageSize=%d" % (media_id, page_size) + else: + return [] + return self._jget(path) or [] diff --git a/doctor/clients/pulsarr.py b/doctor/clients/pulsarr.py new file mode 100644 index 0000000..6ec3a61 --- /dev/null +++ b/doctor/clients/pulsarr.py @@ -0,0 +1,75 @@ +"""Pulsarr API client (watchlist exclusions, health check).""" +import json +import os +import sqlite3 +import urllib.request +import urllib.error +from typing import Optional +from ..config import log + + +class Pulsarr: + def __init__(self, url: str, apikey: str, db_path: str = ""): + self.base = url.rstrip("/") + self.apikey = apikey + self.db_path = db_path + + def _req(self, method: str, path: str, data: Optional[bytes] = None, t: int = 10): + headers = {"x-api-key": self.apikey, "Content-Type": "application/json"} + req = urllib.request.Request(self.base + path, data=data, method=method, headers=headers) + return urllib.request.urlopen(req, timeout=t) + + def _jpost(self, path: str, body: dict = None, t: int = 10) -> Optional[dict]: + data = json.dumps(body or {}).encode() + try: + with self._req("POST", path, data=data, t=t) as r: + return json.loads(r.read()) + except urllib.error.HTTPError as e: + log.debug("[pulsarr] POST %s -> HTTP %d: %s", path, e.code, str(e.read())[:80]) + return None + except Exception as e: + log.warning("[pulsarr] POST %s failed: %s", path, str(e)[:70]) + return None + + def create_watchlist_exclusion(self, tmdb_id: int, media_type: str = "tv", + users: list = None, title: str = "") -> bool: + """Create per-user watchlist exclusions so Pulsarr won't re-add.""" + body = { + "key": str(tmdb_id), + "type": media_type, + "userIds": users or [], + "title": title, + "guids": [], + } + resp = self._jpost("/v1/watchlist-exclusions", body) + return resp is not None + + def _db_users(self) -> list: + """Fallback: query the Pulsarr SQLite DB directly for user list.""" + if not self.db_path or not os.path.exists(self.db_path): + return [] + try: + with sqlite3.connect(self.db_path) as conn: + rows = conn.execute("SELECT id, name FROM users").fetchall() + return [{"id": r[0], "plexUsername": r[1]} for r in rows] + except Exception as e: + log.debug("[pulsarr] DB users query failed: %s", str(e)[:60]) + return [] + + def user_id_for_plex_username(self, plex_username: str) -> str: + """Return the Pulsarr user ID for a given Plex username, or '' if not found.""" + target = plex_username.strip().lower() + if not target: + return "" + for u in self._db_users(): + if (u.get("plexUsername") or "").strip().lower() == target: + return str(u.get("id", "")) + return "" + + def health(self) -> bool: + """Ping Pulsarr to verify it's reachable.""" + try: + with self._req("GET", "/health", t=5) as r: + return r.getcode() < 500 + except Exception: + return False diff --git a/doctor/clients/tautulli.py b/doctor/clients/tautulli.py new file mode 100644 index 0000000..3f3136b --- /dev/null +++ b/doctor/clients/tautulli.py @@ -0,0 +1,51 @@ +"""Tautulli API client (watch history queries).""" +import json +import time +import urllib.request +import urllib.parse +from typing import Optional +from ..config import log + + +class Tautulli: + def __init__(self, url: str, apikey: str): + self.base = url.rstrip("/") + "/api/v2" + self.apikey = apikey + + def _get(self, cmd: str, **params) -> Optional[dict]: + params["apikey"] = self.apikey + params["cmd"] = cmd + qs = urllib.parse.urlencode(params) + url = self.base + "?" + qs + try: + with urllib.request.urlopen(url, timeout=15) as r: + body = r.read() + if not body: + return None + data = json.loads(body) + if data.get("response", {}).get("result") != "success": + return None + return data.get("response", {}).get("data") + except Exception as e: + log.warning("[tautulli] %s failed: %s", cmd, str(e)[:70]) + return None + + def recently_watched_shows(self, since_days: int = 30) -> set: + """Return a set of show titles with watch activity in the last N days. + + Queries Tautulli's get_history with media_type=episode. Because + Tautulli's after parameter is unreliable with large length values, + we fetch all recent history and filter client-side. + """ + cutoff = time.time() - since_days * 86400 + data = self._get("get_history", media_type="episode", length=20000) + if not data: + return set() + records = data if isinstance(data, list) else data.get("data", []) + titles = set() + for rec in records: + if rec.get("date", 0) >= cutoff: + title = (rec.get("grandparent_title") or "").strip() + if title: + titles.add(title) + return titles diff --git a/doctor/config.py b/doctor/config.py new file mode 100644 index 0000000..a4eb0c2 --- /dev/null +++ b/doctor/config.py @@ -0,0 +1,299 @@ +"""Configuration and small generic helpers (env-driven).""" +import os +import json + +VERSION = "0.3" +def _b(name: str, default: bool = False) -> bool: + return os.environ.get(name, str(default)).strip().lower() in ("1", "true", "yes", "on") +def _i(name: str, default: int) -> int: + try: + return int(os.environ.get(name, default)) + except (TypeError, ValueError): + return default +def _f(name: str, default: float) -> float: + try: + return float(os.environ.get(name, default)) + except (TypeError, ValueError): + return default +def _dur(tok, default: int = 0) -> int: + """Parse a duration token: 30s / 10m / 2h / 1d, or a bare number of seconds.""" + t = str(tok).strip().lower() + if not t: + return default + mult = {"s": 1, "m": 60, "h": 3600, "d": 86400} + try: + return int(float(t[:-1]) * mult[t[-1]]) if t[-1] in mult else int(float(t)) + except (ValueError, KeyError): + return default +def _human(sec: int) -> str: + sec = int(sec) + for size, suf in ((86400, "d"), (3600, "h"), (60, "m")): + if sec >= size and sec % size == 0: + return "%d%s" % (sec // size, suf) + return "%ds" % sec +CONFIG_FILE = os.environ.get("DOCTOR_CONFIG_FILE", "/data/config.json") +def _load_overrides(): + """Load config.json into os.environ. Environment variables already set + (e.g. from docker-compose) take priority over config.json values so that + compose overrides always win.""" + try: + with open(CONFIG_FILE) as f: + for k, v in json.load(f).items(): + if v is not None and str(k) not in os.environ: + os.environ[str(k)] = str(v) + except Exception: + pass +_load_overrides() +MODE = os.environ.get("DOCTOR_MODE", "cron").strip().lower() # cron | event +INTERVAL = _i("DOCTOR_INTERVAL", 900) # default/fallback interval; kept for compatibility +PORT = _i("DOCTOR_PORT", 8088) # webhook port (event mode) +UI_PORT = _i("DOCTOR_UI_PORT", 12345) # web dashboard port +EN_UI = _b("ENABLE_UI", False) +UI_TOKEN = os.environ.get("DOCTOR_UI_TOKEN", "") # optional ?token= / X-Doctor-Token gate +LOG_LEVEL = os.environ.get("DOCTOR_LOG_LEVEL", "INFO").upper() +LOG_FILE = os.environ.get("DOCTOR_LOG_FILE", "") +# Respect NO_COLOR and add explicit opt-out. Default to colored output for humans. +LOG_COLORS = _b("DOCTOR_LOG_COLORS", True) and not _b("NO_COLOR", False) +TIMEOUT = _i("DOCTOR_HTTP_TIMEOUT", 60) +DRY_RUN = _b("DOCTOR_DRY_RUN", False) +FAST_INTERVAL = _dur(os.environ.get("DOCTOR_FAST_INTERVAL", "180s"), 180) # 3 min +SLOW_INTERVAL = _dur(os.environ.get("DOCTOR_SLOW_INTERVAL", "1800s"), 1800) # 30 min +SCHEDULER_TICK = _dur(os.environ.get("DOCTOR_SCHEDULER_TICK", "30s"), 30) # how often scheduler wakes +SCHEDULER_CONCURRENCY = _i("DOCTOR_SCHEDULER_CONCURRENCY", 3) # max parallel scheduled checks +def _check_interval(cid, speed, default_iv=None): + """Return the run interval in seconds for a check. + + Resolution order (first match wins): + 1. _INTERVAL env var (or config.json key) + 2. default_iv argument (per-check override from CHECKS table) + 3. FAST_INTERVAL / SLOW_INTERVAL based on speed tag + """ + per = os.environ.get("%s_INTERVAL" % cid.upper()) + if per: + return _dur(per, INTERVAL) + if default_iv is not None: + return int(default_iv) + return FAST_INTERVAL if speed == "fast" else SLOW_INTERVAL +EN_QUEUE = _b("ENABLE_QUEUE", True) +EN_DECYPHARR = _b("ENABLE_DECYPHARR", False) +EN_DECYPHARR_PROVIDERS = _b("ENABLE_DECYPHARR_PROVIDERS", False) +EN_PLEX = _b("ENABLE_PLEX", False) +EN_RESOURCES = _b("ENABLE_RESOURCES", False) +EN_JANITOR = _b("ENABLE_JANITOR", False) +EN_PROVIDERS = _b("ENABLE_PROVIDERS", False) +EN_BAZARR = _b("ENABLE_BAZARR", False) +EN_SEERR = _b("ENABLE_SEERR", False) # Overseerr/Jellyseerr/Seerr: auto-retry FAILED requests +EN_PLEX_SCAN = _b("ENABLE_PLEX_SCAN", False) # detect + recover a wedged Plex library scan +EN_REPAIR = _b("ENABLE_REPAIR", False) # probe library for dead files -> remove + re-search +EN_MISSING_SEASONS = _b("ENABLE_MISSING_SEASONS", False) +MS_MIN_AGE_HOURS = _f("MISSING_SEASONS_MIN_AGE_HOURS", 1) # ignore seasons added less than this long ago +MS_MAX_ACTIONS = _i("MISSING_SEASONS_MAX_ACTIONS", 25) # SeasonSearches per sweep +MS_RECHECK = _dur(os.environ.get("MISSING_SEASONS_RECHECK", "6h"), 21600) # cooldown between re-searching same season +MS_SORT_BY = os.environ.get("MISSING_SEASONS_SORT_BY", "mixed").strip().lower() # mixed | added | episodes +MS_BACKFILL_BATCH = _i("MISSING_SEASONS_BACKFILL_BATCH", 50) # sleep after this many SeasonSearches in backfill mode +MS_BACKFILL_DELAY = _f("MISSING_SEASONS_BACKFILL_DELAY", 0) # seconds to pause between backfill batches +MS_PARTIAL = _b("MISSING_SEASONS_PARTIAL", True) # also search seasons that are partially complete (some files, not all) when the season has fully aired +# ---- multipack ---- +MULTIPACK_ENABLED = _b("ENABLE_MULTIPACK", True) # push cached multi-season packs that Sonarr would normally reject +MULTIPACK_MAX_ACTIONS = _i("MULTIPACK_MAX_ACTIONS", 3) # max packs pushed per sweep +MULTIPACK_RECHECK = _f("MULTIPACK_RECHECK", 7 * 86400) # seconds before re-checking a series for new packs (default 7 days) +MULTIPACK_ITEM_INTERVAL = _f("MULTIPACK_ITEM_INTERVAL", 2) # seconds between pushes +# ---- force_import (importarr-style: force import matched-by-ID releases) ---- +EN_FORCE_IMPORT = _b("ENABLE_FORCE_IMPORT", False) # try manual import of obfuscated/misnamed releases +FI_MAX_ACTIONS = _i("FORCE_IMPORT_MAX_ACTIONS", 10) # max manual imports per sweep +FI_MIN_STRIKES = _i("FORCE_IMPORT_MIN_STRIKES", 1) # consecutive hits before acting (often safe at 1) +FI_FALLBACK = _b("FORCE_IMPORT_FALLBACK", True) # remove + re-search if force import fails +FI_IMPORT_MODE = os.environ.get("FORCE_IMPORT_MODE", "auto").strip().lower() # auto | copy | move +FI_RECHECK = _dur(os.environ.get("FORCE_IMPORT_RECHECK", "1h"), 3600) # cooldown per item +# missing_seasons runs on a tighter default interval than other slow checks; +# the scheduler handles this via its per-check default_interval column. +EN_NO_UPGRADE_PROFILE = _b("ENABLE_NO_UPGRADE_PROFILE", False) +NO_UPGRADE_PROFILE_ID = _i("NO_UPGRADE_PROFILE_ID", 0) # target quality profile id in Sonarr +NO_UPGRADE_PROFILE_NAME = os.environ.get("NO_UPGRADE_PROFILE_NAME", "WEB-1080p (No Upgrade)") +BAZARR_URL = os.environ.get("BAZARR_URL", "") +BAZARR_APIKEY = os.environ.get("BAZARR_APIKEY", "") +SEERR_URL = os.environ.get("SEERR_URL", "") +SEERR_APIKEY = os.environ.get("SEERR_APIKEY", "") +SEERR_MAX = _i("SEERR_RETRY_MAX", 10) # max requests retried per sweep +SEERR_MAX_TRIES = _i("SEERR_MAX_ATTEMPTS", 5) # give up after this many auto-retries (0 = never) +MIN_STRIKES = _i("DOCTOR_MIN_STRIKES", 2) +MAX_ACTIONS = _i("DOCTOR_MAX_ACTIONS", 20) +BLOCKLIST = _b("DOCTOR_BLOCKLIST", True) +REMOVE_CLIENT = _b("DOCTOR_REMOVE_FROM_CLIENT", True) +STATE_FILE = os.environ.get("DOCTOR_STATE_FILE", "/data/state.json") +CHURN_LIMIT = _i("DOCTOR_CHURN_LIMIT", 0) # 0 = brake off +CHURN_ACTION = os.environ.get("DOCTOR_CHURN_ACTION", "report").strip().lower() +CHURN_BACKOFF = [_dur(x) for x in os.environ.get("DOCTOR_CHURN_BACKOFF", "").split(",") if x.strip()] +if not CHURN_BACKOFF: + _legacy = os.environ.get("DOCTOR_CHURN_COOLDOWN") # back-compat with the old single fixed cooldown + CHURN_BACKOFF = [_dur(_legacy)] if _legacy else [600, 3600, 86400] +DEFAULT_CONDITIONS = "downloadClientUnavailable,importBlocked,importFailed,importPending_warning,failedPending,stalled" +ENABLED_CONDITIONS = [c.strip() for c in os.environ.get("DOCTOR_CONDITIONS", DEFAULT_CONDITIONS).split(",") if c.strip()] +LOAD_MAX = _f("DOCTOR_LOAD_MAX", 0) # queue check pauses above this (0=off) +RES_LOAD_WARN = _f("RES_LOAD_WARN", 40) +RES_SWAP_WARN = _i("RES_SWAP_WARN_MB", 7000) +RES_MEM_MIN = _i("RES_MEM_MIN_MB", 800) +RES_DROP_CACHES = _b("RES_DROP_CACHES", False) # echo 1 > drop_caches on memory pressure (needs privilege) +DECY_URL = os.environ.get("DECYPHARR_URL", "") # e.g. http://192.168.50.202:8282 +DECY_MOUNT_TEST = os.environ.get("DECYPHARR_MOUNT_TEST", "") # a dir on the FUSE mount to read-test +DECY_READ_TIMEOUT = _i("DECYPHARR_READ_TIMEOUT", 25) +DECY_RESTART_CMD = os.environ.get("DECYPHARR_RESTART_CMD", "") # shell cmd to recover a hung mount +DECY_FUSE_STRIKES = _i("DECYPHARR_FUSE_STRIKES", 2) # consecutive failures before restart hook fires +# ---- decypharr repair trigger (ask decypharr to run its own repair sweep) ---- +DECY_REPAIR_TRIGGER = _b("DECYPHARR_REPAIR_TRIGGER", True) # ask decypharr to repair on stack-doctor sweep +DECY_REPAIR_INTERVAL = _dur(os.environ.get("DECYPHARR_REPAIR_INTERVAL", "2h"), 7200) # min seconds between triggers +# ---- decypharr link-error cache poisoning detector ---- +# decypharr caches ALL provider errors (including transient RD CDN errors like +# read_pxy_timeout) as permanent in-memory validation failures. Once poisoned +# the only fix is a restart. stack-doctor detects this by counting these +# error lines in the log tail over a rolling window. +DECY_LINK_ERR_LOG_CMD = os.environ.get("DECYPHARR_LINK_ERR_LOG_CMD", "") # cmd to fetch log; falls back to JAN_LOG_CMD / JAN_LOG +DECY_LINK_ERR_THRESHOLD = _i("DECYPHARR_LINK_ERR_THRESHOLD", 20) # errors in window before acting (default 20) +DECY_LINK_ERR_WINDOW = _dur(os.environ.get("DECYPHARR_LINK_ERR_WINDOW", "10m"), 600) # rolling window in seconds (default 10m) +DECY_LINK_ERR_RESTART = _b("DECYPHARR_LINK_ERR_RESTART", True) # restart decypharr when threshold hit (uses DECY_RESTART_CMD) +# ---- decypharr provider health / auto-disable ---- +# Watches decypharr's log for per-provider seedbox/add failures and, when a provider +# is consistently failing, removes it from decypharr/config.json and restarts decypharr. +# The original provider block is saved in stack-doctor state for later re-enable. +DCP_LOG_CMD = os.environ.get("DECYPHARR_PROVIDERS_LOG_CMD", "") # log source; falls back to janitor log cmd +DCP_LOG = os.environ.get("DECYPHARR_PROVIDERS_LOG", "") # or a plain log file path +DCP_CONFIG_PATH = os.environ.get("DECYPHARR_CONFIG_PATH", "/data/decypharr/config.json") +DCP_THRESHOLD = _i("DECYPHARR_PROVIDERS_THRESHOLD", 5) # failed submissions in window before provider is considered broken +DCP_WINDOW = _dur(os.environ.get("DECYPHARR_PROVIDERS_WINDOW", "10m"), 600) +DCP_AUTO_DISABLE = _b("DECYPHARR_PROVIDERS_AUTO_DISABLE", True) +DCP_COOLDOWN = _dur(os.environ.get("DECYPHARR_PROVIDERS_COOLDOWN", "1h"), 3600) # before attempting re-enable +DCP_REENABLE = _b("DECYPHARR_PROVIDERS_REENABLE", True) # test provider API and re-add after cooldown +DCP_PROVIDERS_RESTART_CMD = os.environ.get("DECYPHARR_PROVIDERS_RESTART_CMD", DECY_RESTART_CMD or "docker restart decypharr") +PLEX_URL = os.environ.get("PLEX_URL", "") +PLEX_TOKEN = os.environ.get("PLEX_TOKEN", "") +PLEX_SCAN = _b("PLEX_SCAN_ON_CHECK", False) +PLEX_SCAN_STUCK = _dur(os.environ.get("PLEX_SCAN_STUCK_AFTER", "30m"), 1800) # no-progress time before "stuck" +PLEX_SCAN_CANCEL = _b("PLEX_SCAN_CANCEL", True) # cancel the wedged scan via the activities API +PLEX_RESTART_CMD = os.environ.get("PLEX_RESTART_CMD", "") # last-resort hook if the scan stays wedged +EN_WARMER = _b("ENABLE_WARMER", False) +WARM_HEAD_MB = _i("WARMER_PRECACHE_MB", 64) # how much of the file head to pull into cache +WARM_TAIL_MB = _i("WARMER_TAIL_MB", 8) # also pull the tail (mkv cues / Plex end-probe); 0=off +WARM_INTERVAL = _i("WARMER_INTERVAL", 120) # seconds between session polls (next-episode prefetch) +WARM_ONDECK_EVERY = _i("WARMER_ONDECK_EVERY", 600) # seconds between on-deck / recent warms +WARM_NEXT_EPS = _i("WARMER_NEXT_EPISODES", 1) # warm this many upcoming episodes of an active show +WARM_RECENT_COUNT = _i("WARMER_RECENT_COUNT", 0) # warm N most-recently-added per library (0=off) +WARM_MAX_CYCLE = _i("WARMER_MAX_PER_CYCLE", 12) # cap warms per cycle (rate-limit the usenet fetch) +WARM_COOLDOWN = _i("WARMER_COOLDOWN", 3600) # do not re-warm the same file within this many seconds +WARM_LOAD_MAX = _f("WARMER_LOAD_MAX", 0) # skip warming if host 1-min load above this (protect Plex); 0=off +WARM_READ_TIMEOUT = _i("WARMER_READ_TIMEOUT", 60) # abandon a single warm read after this long (hung mount guard) +WARM_CONCURRENCY = _i("WARMER_CONCURRENCY", 2) # simultaneous BACKGROUND (on-deck/recent) warm reads +WARM_OPEN_CONC = _i("WARMER_OPEN_CONCURRENCY", 4) # dedicated lane for the title you OPEN, so it starts instantly and never queues behind background warming +WARM_PARTS = _i("WARMER_PARTS", 1) # how many versions per title to warm (1 = highest-res only; 0 = all). Avoids warming a 1080p you'll never play next to the 4K +WARM_LOW_CACHE = _b("WARMER_LOW_CACHE", False) +WARM_NEXT_REMAIN = _i("WARMER_NEXT_REMAINING_MIN", 0) # warm the next episode only when <= this many minutes remain (0 = as soon as playback is seen) +WARM_NEXT_NEAR_END = WARM_NEXT_REMAIN if WARM_NEXT_REMAIN > 0 else (10 if WARM_LOW_CACHE else 0) +WARM_SOURCES = [s.strip().lower() for s in os.environ.get("WARMER_SOURCES", "ondeck,next").split(",") if s.strip()] +WARM_ONDECK = _b("WARMER_ONDECK", True) # quick on/off for Continue Watching (On Deck) warming +WARM_PATH_MAP = os.environ.get("WARMER_PATH_MAP", "") # "plexPrefix:hostPrefix" if Plex's file path != this host's +WARM_PLEXLOG_CMD = os.environ.get("WARMER_PLEXLOG_CMD", "") +WARM_PLEXLOG_FILE = os.environ.get("WARMER_PLEXLOG_FILE", "") +JAN_LIBS = [p.strip() for p in os.environ.get("JANITOR_LIBRARY_PATHS", "").split(",") if p.strip()] +JAN_LOG = os.environ.get("JANITOR_DECYPHARR_LOG", "") # log file path +JAN_LOG_CMD = os.environ.get("JANITOR_LOG_CMD", "") # cmd printing the log, e.g. "journalctl -u decypharr -n 10000 --no-hostname" +JAN_QUAR = os.environ.get("JANITOR_QUARANTINE_DIR", "/data/quarantine") +JAN_PATTERNS = os.environ.get("JANITOR_DEAD_PATTERNS", "ARTICLE_NOT_FOUND,still missing,marked as bad").split(",") +JAN_ERROR_PATTERNS = [p.strip() for p in os.environ.get( + "JANITOR_ERROR_PATTERNS", + "panic,fatal,runtime error,rate limit,rate limited,too many requests,cloudflare,cf-ray,blocked,unauthorized,token expired,context deadline exceeded,connection refused,timeout,i/o timeout" +).split(",") if p.strip()] +JAN_ALERT_COOLDOWN = _dur(os.environ.get("JANITOR_ALERT_COOLDOWN", "5m"), 300) +REPAIR_LIBS = [p.strip() for p in os.environ.get("REPAIR_LIBRARY_PATHS", + os.environ.get("JANITOR_LIBRARY_PATHS", "")).split(",") if p.strip()] +REPAIR_MAX_ACTIONS = _i("REPAIR_MAX_ACTIONS", 20) # re-grab/search commands per sweep +REPAIR_MAX_SYMLINKS = _i("REPAIR_MAX_SYMLINKS", 100) # dead symlinks processed per sweep +REPAIR_LOAD_MAX = _f("REPAIR_LOAD_MAX", 0) # skip the whole repair sweep above this host 1-min load (0=off) +REPAIR_DEBRID_MOUNT = os.environ.get("REPAIR_DEBRID_MOUNT", "") # debrid mount root; non-empty means "check it's live before sweep" +REPAIR_ITEM_INTERVAL = _dur(os.environ.get("REPAIR_ITEM_INTERVAL", "0"), 0) # seconds to wait between each re-grab (0=off) +REPAIR_SEASON_PACKS = _b("REPAIR_SEASON_PACKS", False) # flag sonarr seasons spread across multiple dirs (non-season-pack) +REPAIR_UNMONITORED = _b("REPAIR_UNMONITORED", False) # include unmonitored series/movies in the repair sweep +REPAIR_MISSING_FROM_DISK = _b("REPAIR_MISSING_FROM_DISK", False) # enable history-based missing-file re-grab +REPAIR_MFD_RECHECK = _dur(os.environ.get("REPAIR_MFD_RECHECK", "24h"), 86400) # cooldown per item before re-searching +REPAIR_VERIFY = _b("REPAIR_VERIFY", False) # enable post-repair grab verification +REPAIR_VERIFY_DEADLINE = _dur(os.environ.get("REPAIR_VERIFY_DEADLINE", "4h"), 14400) # give up after this long +REPAIR_ORPHAN_SCAN = _b("REPAIR_ORPHAN_SCAN", True) # report dead symlinks not tracked by *arr +REPAIR_HIERARCHICAL_SEARCH = _b("REPAIR_HIERARCHICAL_SEARCH", False) # prefer series/season/episode searches based on airing status +REPAIR_HIERARCHICAL_FALLBACK = _b("REPAIR_HIERARCHICAL_FALLBACK", True) # fall back to narrower search if wider search finds nothing +REPAIR_SEASON_ENDED_THRESHOLD = _dur(os.environ.get("REPAIR_SEASON_ENDED_THRESHOLD", "7d"), 604800) # how long after last aired date to treat a season as ended +EN_RESCAN = _b("ENABLE_RESCAN", False) +RESCAN_LIBRARY_PATHS = [p.strip() for p in os.environ.get("RESCAN_LIBRARY_PATHS", + os.environ.get("REPAIR_LIBRARY_PATHS", "")).split(",") if p.strip()] +RESCAN_MAX_ACTIONS = _i("RESCAN_MAX_ACTIONS", 5) # partial Plex scans per sweep (keep low to avoid DB hammering) +RESCAN_SCAN_DELAY = _i("RESCAN_SCAN_DELAY", 60) # seconds between partial scans +RESCAN_MAX_WAIT = _dur(os.environ.get("RESCAN_MAX_WAIT", "10m"), 600) # max time to wait for Plex to finish a scan before giving up +RESCAN_COOLDOWN = _dur(os.environ.get("RESCAN_COOLDOWN", "1h"), 3600) # don't rescan same missing folder within this window +RESCAN_INTERVAL = _dur(os.environ.get("RESCAN_INTERVAL", "15m"), 900) # seconds between rescan sweeps +RESCAN_LOAD_MAX = _f("RESCAN_LOAD_MAX", 0) # skip sweep if 1-min load above this (0=off) +RESCAN_PLEX_RESPONSIVE_TIMEOUT = _f("RESCAN_PLEX_RESPONSIVE_TIMEOUT", 3.0) # abort if Plex root ping takes longer than this +RESCAN_DECYPHARR_REPAIR_BACKOFF = _b("RESCAN_DECYPHARR_REPAIR_BACKOFF", True) # skip sweep while decypharr repair is active +RESCAN_INCREMENTAL = _b("RESCAN_INCREMENTAL", True) # only scan files/parents changed since last sweep +RESCAN_FULL_INTERVAL = _dur(os.environ.get("RESCAN_FULL_INTERVAL", "24h"), 86400) # do a full walk this often +RESCAN_SECTIONS_CACHE_TTL = _dur(os.environ.get("RESCAN_SECTIONS_CACHE_TTL", "5m"), 300) # cache Plex sections/locations +RESCAN_JANITOR_CANDIDATES = _b("RESCAN_JANITOR_CANDIDATES", True) # use janitor dead-files as candidate parents +RESCAN_ARR_QUEUE_BACKOFF = _b("RESCAN_ARR_QUEUE_BACKOFF", True) # skip sweep while *arr has active queue items +RESCAN_ARR_QUEUE_MAX = _i("RESCAN_ARR_QUEUE_MAX", 0) # skip if any *arr queue exceeds this (0=any) +RESCAN_FULL_REFRESH_THRESHOLD = _i("RESCAN_FULL_REFRESH_THRESHOLD", 0) # full-section refresh if >N parents missing (0=off) +TRIGGER_EVENTS = set(e.strip() for e in os.environ.get( + "DOCTOR_TRIGGER_EVENTS", "Download,ManualInteractionRequired,DownloadFailed,Grab").split(",") if e.strip()) +# Logging is configured in a separate module so config.py stays focused on env constants. +from .logging_config import log + +# Re-export utils helpers for backward compatibility with any code that imports them +# from doctor.config. The canonical home is doctor.utils. +from doctor.utils import http_code, run_cmd, run_output, host_load # noqa: F401 + +# Public surface for ``from doctor.config import *``: uppercase constants, the logger, +# and the backward-compatible utils re-exports. +__all__ = [n for n in dir() if n.isupper()] +__all__ += ["log", "http_code", "run_cmd", "run_output", "host_load"] + +# ---- debridlink migration ---- +EN_DEBRIDLINK_MIGRATION = _b("ENABLE_DEBRIDLINK_MIGRATION", False) +DBR_PROWLARR_URL = os.environ.get("DBR_PROWLARR_URL", "") +DBR_PROWLARR_APIKEY = os.environ.get("DBR_PROWLARR_APIKEY", "") +DBR_QBT_URL = os.environ.get("DBR_QBT_URL", "") +DBR_QBT_CATEGORY = os.environ.get("DBR_QBT_CATEGORY", "sonarr") +DBR_MAX_ACTIONS = _i("DBR_MAX_ACTIONS", 10) +DBR_RECHECK = _dur(os.environ.get("DBR_RECHECK", "12h"), 43200) +DBR_MIN_SEEDS = _i("DBR_MIN_SEEDS", 1) +DBR_SCAN_DELAY = _f("DBR_SCAN_DELAY", 10) +# ---- library maintainer ---- +EN_MAINTAINER = _b("ENABLE_MAINTAINER", False) +MAINTAINER_MAX_ACTIONS = _i("MAINTAINER_MAX_ACTIONS", 5) +MAINTAINER_UNWATCHED_DAYS = _i("MAINTAINER_UNWATCHED_DAYS", 30) +MAINTAINER_MIN_YEAR = _i("MAINTAINER_MIN_YEAR", 2024) +MAINTAINER_MIN_AGE_DAYS = _i("MAINTAINER_MIN_AGE_DAYS", 30) +MAINTAINER_LIBRARY_TITLE = os.environ.get("MAINTAINER_LIBRARY_TITLE", "shows") +MAINTAINER_PULSARR_TAG_PREFIX = os.environ.get("MAINTAINER_PULSARR_TAG_PREFIX", "pulsarr") +MAINTAINER_MODE = os.environ.get("MAINTAINER_MODE", "tagged").strip().lower() +MAINTAINER_PLEX_SECTION_KEY = _i("MAINTAINER_PLEX_SECTION_KEY", 0) +MAINTAINER_RECHECK = _dur(os.environ.get("MAINTAINER_RECHECK", "24h"), 86400) +MAINTAINER_ROOT_FOLDER_PATHS = os.environ.get("MAINTAINER_ROOT_FOLDER_PATHS", "") +TAUTULLI_URL = os.environ.get("TAUTULLI_URL", "") +TAUTULLI_APIKEY = os.environ.get("TAUTULLI_APIKEY", "") +PULSARR_URL = os.environ.get("PULSARR_URL", "") +PULSARR_APIKEY = os.environ.get("PULSARR_APIKEY", "") +PULSARR_DB_PATH = os.environ.get("PULSARR_DB_PATH", "") + +DBR_MIGRATE_MODE = os.environ.get("DBR_MIGRATE_MODE", "continuous").strip().lower() +# ---- library maintainer ---- +EN_MAINTAINER = _b("ENABLE_MAINTAINER", False) +MAINTAINER_MAX_ACTIONS = _i("MAINTAINER_MAX_ACTIONS", 5) # max shows deleted per sweep +MAINTAINER_UNWATCHED_DAYS = _i("MAINTAINER_UNWATCHED_DAYS", 30) # must be unwatched for at least this many days +MAINTAINER_MIN_YEAR = _i("MAINTAINER_MIN_YEAR", 2024) # shows released before this year are eligible +MAINTAINER_MIN_AGE_DAYS = _i("MAINTAINER_MIN_AGE_DAYS", 30) # series must have been added to Sonarr at least this long ago +MAINTAINER_LIBRARY_TITLE = os.environ.get("MAINTAINER_LIBRARY_TITLE", "shows") # only delete from this Sonarr instance whose name contains this +MAINTAINER_PULSARR_TAG_PREFIX = os.environ.get("MAINTAINER_PULSARR_TAG_PREFIX", "pulsarr") +MAINTAINER_MODE = os.environ.get("MAINTAINER_MODE", "tagged").strip().lower() # tagged | all +MAINTAINER_PLEX_SECTION_KEY = _i("MAINTAINER_PLEX_SECTION_KEY", 0) # Plex section to empty trash on (all mode) +MAINTAINER_RECHECK = _dur(os.environ.get("MAINTAINER_RECHECK", "24h"), 86400) # cooldown before a previously-flagged series is reconsidered +MAINTAINER_ROOT_FOLDER_PATHS = os.environ.get("MAINTAINER_ROOT_FOLDER_PATHS", "") +TAUTULLI_URL = os.environ.get("TAUTULLI_URL", "") +TAUTULLI_APIKEY = os.environ.get("TAUTULLI_APIKEY", "") +PULSARR_URL = os.environ.get("PULSARR_URL", "") +PULSARR_APIKEY = os.environ.get("PULSARR_APIKEY", "") +PULSARR_DB_PATH = os.environ.get("PULSARR_DB_PATH", "") # direct sqlite3 access to pulsarr.db for watchlist record deletion diff --git a/doctor/scheduler.py b/doctor/scheduler.py new file mode 100644 index 0000000..00e2859 --- /dev/null +++ b/doctor/scheduler.py @@ -0,0 +1,161 @@ +"""Scheduler: per-check intervals, bounded concurrency, and the full sweep.""" +import time +import threading +from collections import namedtuple +from typing import Optional, Callable, Any +from .config import ( + EN_BAZARR, EN_DECYPHARR, EN_DECYPHARR_PROVIDERS, EN_DEBRIDLINK_MIGRATION, EN_FORCE_IMPORT, EN_JANITOR, + EN_MAINTAINER, EN_MISSING_SEASONS, + EN_NO_UPGRADE_PROFILE, EN_PLEX, EN_PLEX_SCAN, EN_PROVIDERS, EN_RESCAN, + EN_QUEUE, EN_REPAIR, EN_RESOURCES, EN_SEERR, + FAST_INTERVAL, MULTIPACK_ENABLED, SCHEDULER_CONCURRENCY, + SCHEDULER_TICK, SLOW_INTERVAL, + _check_interval, _human, log, +) +from .checks import ( # check_* functions referenced by CHECKS + check_debridlink_migration, + check_bazarr, + check_decypharr, + check_decypharr_providers, + check_force_import, + check_janitor, + check_maintainer, + check_missing_seasons, + check_multipack, + check_no_upgrade_profile, + check_plex, + check_plex_scan, + check_providers, + check_queue, + check_repair, + check_rescan, + check_resources, + check_seerr, +) + +# Descriptor for a scheduled check. +# Fields: +# cid – unique string id; used for logging and env-var lookup (_INTERVAL) +# enabled – bool from config (EN_* constant); False means the check never runs +# fn – the check_* function to call each cycle +# speed – "fast" or "slow"; selects FAST_INTERVAL / SLOW_INTERVAL when no override +# default_iv – optional int (seconds) that overrides speed without touching os.environ; +# still overrideable by a _INTERVAL env var +# +# Using a namedtuple makes field access self-documenting and turns wrong-width table +# edits into a TypeError at import time rather than a ValueError mid-sweep. +CheckEntry = namedtuple("CheckEntry", ["cid", "enabled", "fn", "speed", "default_iv", "needs_instances"]) + +CHECKS = [CheckEntry("queue", EN_QUEUE, check_queue, "fast", None, True), + CheckEntry("providers", EN_PROVIDERS, check_providers, "fast", None, True), + CheckEntry("decypharr", EN_DECYPHARR, check_decypharr, "fast", None, False), + CheckEntry("decypharr_providers", EN_DECYPHARR_PROVIDERS, check_decypharr_providers, "fast", 300, False), + CheckEntry("debridlink_migration", EN_DEBRIDLINK_MIGRATION, check_debridlink_migration, "slow", 3600, False), + CheckEntry("plex", EN_PLEX, check_plex, "fast", None, False), + CheckEntry("plexscan", EN_PLEX_SCAN, check_plex_scan, "fast", None, False), + CheckEntry("resources", EN_RESOURCES, check_resources, "fast", None, False), + CheckEntry("janitor", EN_JANITOR, check_janitor, "slow", None, False), + CheckEntry("repair", EN_REPAIR, check_repair, "slow", None, True), + CheckEntry("rescan", EN_RESCAN, check_rescan, "slow", 3600, False), # 1h default, no instances needed + CheckEntry("force_import", EN_FORCE_IMPORT, check_force_import, "slow", None, True), # importarr-style manual import + CheckEntry("bazarr", EN_BAZARR, check_bazarr, "fast", None, False), + CheckEntry("seerr", EN_SEERR, check_seerr, "fast", None, False), + CheckEntry("missing_seasons", EN_MISSING_SEASONS, check_missing_seasons, "slow", 900, True), # 15 min default + CheckEntry("no_upgrade_profile", EN_NO_UPGRADE_PROFILE, check_no_upgrade_profile, "slow", None, True), + CheckEntry("multipack", MULTIPACK_ENABLED, check_multipack, "slow", None, True), + CheckEntry("maintainer", EN_MAINTAINER, check_maintainer, "slow", None, True)] +_check_locks = {cid: threading.Lock() for cid, _, _, _, _, _ in CHECKS} +_scheduler_sem = threading.Semaphore(max(1, SCHEDULER_CONCURRENCY)) +_lock = threading.Lock() + +# Per-check run metadata. Keyed by cid; populated by _run_scheduled_check and sweep(). +# Shape: {last_start, last_end, last_duration, last_outcome, last_error, run_count, error_count} +# last_outcome values: "ok" | "error" | "skipped" | "deferred" +_check_runs: dict = {} + +def _record_run(cid: str, start: float, end: float, outcome: str, error: str = "") -> None: + """Update _check_runs for cid in-place (thread-safe: GIL-atomic dict update).""" + r = _check_runs.get(cid) + if r is None: + r = {"run_count": 0, "error_count": 0} + _check_runs[cid] = r + r["last_start"] = start + r["last_end"] = end + r["last_duration"] = round(end - start, 3) + r["last_outcome"] = outcome + r["last_error"] = error + if outcome in ("ok", "error"): + r["run_count"] += 1 + if outcome == "error": + r["error_count"] += 1 + +__all__ = ["CHECKS", "CheckEntry", "_check_runs", "scheduler_loop", "sweep", "_run_scheduled_check"] + +def sweep(only: Optional[Any] = None) -> None: + if not _lock.acquire(blocking=False): + log.debug("sweep already running"); return + log.info("[sweep] starting initial sweep of %d enabled check(s)", sum(1 for _, e, _, _, _, _ in CHECKS if e)) + try: + for cid, en, fn, _, _, _ in CHECKS: + if not en: + continue + log.info("[sweep] running %s", cid) + _t0 = time.time() + try: + fn(only) if cid == "queue" else fn() + _record_run(cid, _t0, time.time(), "ok") + except Exception as e: + _record_run(cid, _t0, time.time(), "error", str(e)[:200]) + log.error("[%s] check error: %s", cid, e) + log.info("[sweep] finished %s", cid) + finally: + _lock.release() + log.info("[sweep] initial sweep complete") +def _run_scheduled_check(cid: str, fn: Callable[[], None]) -> None: + """Run a single scheduled check with per-check locking and bounded concurrency.""" + lock = _check_locks.get(cid) + if lock and not lock.acquire(blocking=False): + log.debug("[%s] already running, skipping scheduled run", cid) + _record_run(cid, time.time(), time.time(), "skipped") + return + acquired = False + t0 = time.time() + try: + if not _scheduler_sem.acquire(blocking=False): + log.info("[%s] scheduler concurrency full, deferring", cid) + _record_run(cid, t0, time.time(), "deferred") + return + acquired = True + log.info("[%s] running scheduled check", cid) + fn() + _record_run(cid, t0, time.time(), "ok") + except Exception as e: + _record_run(cid, t0, time.time(), "error", str(e)[:200]) + log.error("[%s] scheduled check error: %s", cid, e) + finally: + if acquired: + log.info("[%s] scheduled check finished", cid) + _scheduler_sem.release() + if lock: + lock.release() +def scheduler_loop(stop: threading.Event) -> None: + """Background loop that runs each enabled check on its own interval. + An initial full sweep runs on startup, then checks are dispatched independently + so fast checks (queue, providers, plex, ...) run every few minutes while slow + checks (repair, janitor, missing_seasons, no_upgrade_profile) run every 30 min.""" + log.info("[scheduler] fast=%s, slow=%s, tick=%s, concurrency=%d", + _human(FAST_INTERVAL), _human(SLOW_INTERVAL), _human(SCHEDULER_TICK), SCHEDULER_CONCURRENCY) + sweep() + now = time.time() + last_run = {cid: now for cid, en, _, _, _, _ in CHECKS if en} + while not stop.wait(SCHEDULER_TICK): + now = time.time() + for cid, en, fn, speed, default_iv, _ in CHECKS: + if not en: + continue + interval = _check_interval(cid, speed, default_iv) + if now - last_run.get(cid, 0) >= interval: + elapsed = now - last_run.get(cid, 0) + last_run[cid] = now + log.info("[scheduler] dispatching %s (interval=%s, last=%.0fs ago)", cid, _human(interval), elapsed) + threading.Thread(target=_run_scheduled_check, args=(cid, fn), daemon=True).start() diff --git a/doctor/state.py b/doctor/state.py new file mode 100644 index 0000000..a0d94fa --- /dev/null +++ b/doctor/state.py @@ -0,0 +1,103 @@ +"""Persistent JSON state with an atomic transaction lock + churn-brake bookkeeping.""" +import contextlib +import json +import os +import time +import threading +from .config import CHURN_ACTION, CHURN_BACKOFF, CHURN_LIMIT, STATE_FILE, _human, log +from .clients import INSTANCES + + +# Single process-wide lock guarding read-modify-write cycles on the shared state file. +# The scheduler runs checks concurrently; without this, two checks could each load the +# state, modify their own slice, and the second save would clobber the first. +STATE_LOCK = threading.RLock() + +@contextlib.contextmanager +def state_transaction(): + """Load the persistent state, yield it for modification, then atomically save it. + + Any code that reads the state, makes decisions based on it, and writes it back + should use this context manager. A per-call lock around _load_state/_save_state + is not enough because the lock is released between load and save, allowing a + concurrent check to overwrite the changes. + """ + with STATE_LOCK: + state = _load_state_unlocked() + try: + yield state + except Exception: + # Don't persist a partially modified state if the check failed. + raise + else: + _save_state_unlocked(state) + +def _load_state(): + with STATE_LOCK: + return _load_state_unlocked() + +def _save_state(s): + with STATE_LOCK: + _save_state_unlocked(s) + +def _load_state_unlocked(): + try: + with open(STATE_FILE) as f: + return json.load(f) + except Exception: + return {} + +def _save_state_unlocked(s): + try: + os.makedirs(os.path.dirname(STATE_FILE) or ".", exist_ok=True) + with open(STATE_FILE, "w") as f: + json.dump(s, f) + except Exception as e: + log.warning("state save failed: %s", e) +def _offenders(state): + return state.setdefault("__offenders__", {}) +def _churn_record(state, arr, rec, title): + """Count a dead grab for this episode/movie; brake if it's over the limit. + Returns True if it un-monitored the target (so the caller knows the blocklist-remove won't re-search).""" + if CHURN_LIMIT <= 0: + return False + tid = arr.queue_target_id(rec) + if not tid: + return False + off = _offenders(state).setdefault(arr.name, {}) + o = off.setdefault(str(tid), {"fails": 0, "until": 0, "level": 0, "title": title}) + o["fails"] += 1; o["title"] = title + if o["fails"] < CHURN_LIMIT or o["until"] != 0: # below limit, or already parked/reported + return False + if CHURN_ACTION == "report": + log.warning("[churn:%s] REPEAT-OFFENDER (%d dead grabs, still retrying): %s", arr.name, o["fails"], title) + o["until"] = -1 + return False + if CHURN_ACTION in ("park", "backoff") and arr.set_monitored([int(tid)], False): + o["fails"] = 0 + if CHURN_ACTION == "backoff": + lvl = o.get("level", 0) + delay = CHURN_BACKOFF[min(lvl, len(CHURN_BACKOFF) - 1)] + o["until"] = time.time() + delay; o["level"] = lvl + 1 + log.warning("[churn:%s] REPEAT-OFFENDER parked (retry #%d in %s) -> un-monitored: %s", + arr.name, lvl + 1, _human(delay), title) + else: # park: no auto-retry + o["until"] = -1 + log.warning("[churn:%s] REPEAT-OFFENDER parked (un-monitored, manual re-monitor): %s", arr.name, title) + return True + return False +def _churn_remonitor(state): + """Re-monitor parked titles whose backoff delay has elapsed, giving them a fresh attempt.""" + if CHURN_LIMIT <= 0 or CHURN_ACTION != "backoff": + return + now = time.time(); off_all = state.get("__offenders__", {}) + for arr in INSTANCES: + for tid, o in list(off_all.get(arr.name, {}).items()): + until = o.get("until", 0) + if isinstance(until, (int, float)) and until > 0 and now >= until: + if arr.set_monitored([int(tid)], True): + log.info("[churn:%s] backoff #%d elapsed, re-monitoring for a fresh attempt: %s", + arr.name, o.get("level", 0), o.get("title", "")) + o["fails"] = 0; o["until"] = 0 # keep level so the next park escalates + +__all__ = [n for n in dir() if not n.startswith("__") and not isinstance(globals()[n], type(os))] diff --git a/doctor/ui.html b/doctor/ui.html new file mode 100644 index 0000000..f7dba1b --- /dev/null +++ b/doctor/ui.html @@ -0,0 +1,103 @@ + +stack-doctor +

stack-doctor

loading
+ +
+
+

Checks

+

Monitored services

+ +

Warmer

+
+ + +
+ \ No newline at end of file diff --git a/doctor/utils.py b/doctor/utils.py new file mode 100644 index 0000000..a63924d --- /dev/null +++ b/doctor/utils.py @@ -0,0 +1,58 @@ +"""Small, stateless utility helpers used across the doctor package. + +These functions were extracted from doctor.config.py in Phase 2 so that +doctor.config.py can focus on environment parsing and logging setup while +still re-exporting them for backward compatibility. +""" +import logging +import subprocess +import urllib.error +import urllib.request + +log = logging.getLogger("doctor") + +__all__ = ["http_code", "run_cmd", "run_output", "host_load"] + + +def http_code(url, headers=None, t=10): + """Return the HTTP status code for *url*, or 0 on any failure.""" + try: + r = urllib.request.urlopen(urllib.request.Request(url, headers=headers or {}), timeout=t) + return r.status + except urllib.error.HTTPError as e: + return e.code + except Exception: + return 0 + + +def run_cmd(cmd): + """Run *cmd* in a shell and return (returncode, combined_output[:300]). + + Returns None if *cmd* is empty. + """ + if not cmd: + return None + try: + p = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=180) + return (p.returncode, (p.stdout + p.stderr).strip()[:300]) + except Exception as e: + return (1, "cmd error: " + str(e)[:120]) + + +def run_output(cmd, t=120): + """Run *cmd* in a shell and return stdout; return "" on failure.""" + try: + p = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=t) + return p.stdout + except Exception as e: + log.warning("log cmd failed: %s", str(e)[:80]) + return "" + + +def host_load(): + """Return the 1-minute host load from /proc/loadavg, or 0.0 on failure.""" + try: + with open("/proc/loadavg") as f: + return float(f.read().split()[0]) + except Exception: + return 0.0 diff --git a/doctor/webui.py b/doctor/webui.py new file mode 100644 index 0000000..93af146 --- /dev/null +++ b/doctor/webui.py @@ -0,0 +1,342 @@ +"""Optional web dashboard: status, health, warmer stats, config editor, logs.""" +import os +import json +import time +import threading +from .config import ( + BAZARR_APIKEY, BAZARR_URL, CONFIG_FILE, DECY_URL, DRY_RUN, EN_UI, + LOG_FILE, MODE, PLEX_TOKEN, PLEX_URL, PULSARR_APIKEY, PULSARR_URL, + RESCAN_LIBRARY_PATHS, SEERR_APIKEY, SEERR_URL, TAUTULLI_APIKEY, TAUTULLI_URL, + TRIGGER_EVENTS, UI_TOKEN, VERSION, WARM_PLEXLOG_CMD, WARM_PLEXLOG_FILE, + _b, host_load, http_code, log, +) +from .clients import INSTANCES +from .actions.plex import plex_rescan, plex_empty_trash +from .checks.rescan import rescan_backlog +from .checks import warmer as _warmer + +from .scheduler import CHECKS, _check_runs, sweep, _run_scheduled_check +from .state import _load_state + + +UI_HTML = open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "ui.html"), encoding="utf-8").read() + +_SECRET_HINT = ("APIKEY", "API_KEY", "TOKEN", "PASSWORD", "PASS", "SECRET") +UI_SCHEMA = [ + ("Mode", [("DOCTOR_MODE", "cron|event"), ("DOCTOR_INTERVAL", "900"), + ("DOCTOR_FAST_INTERVAL", "180s"), ("DOCTOR_SLOW_INTERVAL", "1800s"), + ("DOCTOR_SCHEDULER_TICK", "30s"), ("DOCTOR_SCHEDULER_CONCURRENCY", "3"), + ("DOCTOR_DRY_RUN", "false"), ("DOCTOR_LOG_LEVEL", "INFO")]), + ("Checks (on/off)", [("ENABLE_QUEUE", ""), ("ENABLE_PROVIDERS", ""), ("ENABLE_DECYPHARR", ""), + ("ENABLE_DECYPHARR_PROVIDERS", ""), + ("ENABLE_PLEX", ""), ("ENABLE_PLEX_SCAN", ""), ("ENABLE_RESOURCES", ""), + ("ENABLE_JANITOR", ""), ("ENABLE_REPAIR", ""), ("ENABLE_BAZARR", ""), + ("ENABLE_SEERR", ""), ("ENABLE_WARMER", ""), + ("ENABLE_MISSING_SEASONS", ""), ("ENABLE_NO_UPGRADE_PROFILE", ""), + ("ENABLE_MAINTAINER", ""), ("ENABLE_FORCE_IMPORT", "")]), + ("Decypharr provider health", [("DECYPHARR_PROVIDERS_THRESHOLD", "5"), ("DECYPHARR_PROVIDERS_WINDOW", "10m"), + ("DECYPHARR_PROVIDERS_COOLDOWN", "1h"), ("DECYPHARR_PROVIDERS_AUTO_DISABLE", "true|false"), + ("DECYPHARR_PROVIDERS_REENABLE", "true|false"), ("DECYPHARR_PROVIDERS_RESTART_CMD", "docker restart decypharr")]), + ("Plex scan recovery", [("PLEX_SCAN_STUCK_AFTER", "30m"), ("PLEX_SCAN_CANCEL", "true|false")]), + ("Plex rescan (missing files)", [("ENABLE_RESCAN", ""), ("RESCAN_LIBRARY_PATHS", "/mnt/library/movies,/mnt/library/tv"), + ("RESCAN_MAX_ACTIONS", "20"), ("RESCAN_SCAN_DELAY", "5")]), + ("Repair (dead-file re-grab)", [("REPAIR_LIBRARY_PATHS", "/mnt/library/movies,/mnt/library/tv"), + ("REPAIR_MAX_ACTIONS", "20"), ("REPAIR_MAX_SYMLINKS", "100"), ("REPAIR_LOAD_MAX", "0"), + ("REPAIR_DEBRID_MOUNT", ""), + ("REPAIR_ITEM_INTERVAL", "0"), ("REPAIR_SEASON_PACKS", "false"), + ("REPAIR_UNMONITORED", "false"), + ("REPAIR_MISSING_FROM_DISK", "false"), ("REPAIR_MFD_RECHECK", "24h"), + ("REPAIR_VERIFY", "false"), ("REPAIR_VERIFY_DEADLINE", "4h")]), + ("Missing Seasons", [("MISSING_SEASONS_MIN_AGE_HOURS", "1"), ("MISSING_SEASONS_MAX_ACTIONS", "5"), + ("MISSING_SEASONS_RECHECK", "24h")]), + ("No-Upgrade Profile", [("NO_UPGRADE_PROFILE_NAME", "WEB-1080p (No Upgrade)"), + ("NO_UPGRADE_PROFILE_ID", "0")]), + ("Seerr (failed-request retry)", [("SEERR_URL", "http://overseerr:5055"), ("SEERR_APIKEY", ""), + ("SEERR_RETRY_MAX", "10"), ("SEERR_MAX_ATTEMPTS", "5")]), + ("Library Maintainer", [("TAUTULLI_URL", "http://tautulli:8181"), ("TAUTULLI_APIKEY", ""), + ("PULSARR_URL", "http://pulsarr:3003"), ("PULSARR_APIKEY", ""), + ("PULSARR_DB_PATH", "/pulsarr-data/db/pulsarr.db"), + ("MAINTAINER_MAX_ACTIONS", "5"), ("MAINTAINER_UNWATCHED_DAYS", "30"), + ("MAINTAINER_MIN_YEAR", "2024"), ("MAINTAINER_MIN_AGE_DAYS", "30"), + ("MAINTAINER_MODE", "tagged|all"), ("MAINTAINER_PLEX_SECTION_KEY", "0"), + ("MAINTAINER_LIBRARY_TITLE", "shows"), + ("MAINTAINER_PULSARR_TAG_PREFIX", "pulsarr-"), + ("MAINTAINER_RECHECK", "24h")]), + + ("Queue / churn brake", [("DOCTOR_MIN_STRIKES", "2"), ("DOCTOR_MAX_ACTIONS", "20"), ("DOCTOR_BLOCKLIST", "true"), + ("DOCTOR_CHURN_LIMIT", "0"), ("DOCTOR_CHURN_ACTION", "report|park|backoff"), ("DOCTOR_CHURN_BACKOFF", "10m,1h,24h")]), + ("Warmer", [("WARMER_PRECACHE_MB", "64"), ("WARMER_TAIL_MB", "8"), ("WARMER_SOURCES", "ondeck,next"), + ("WARMER_ONDECK", "true|false"), ("WARMER_MAX_PER_CYCLE", "40"), ("WARMER_NEXT_EPISODES", "1"), + ("WARMER_COOLDOWN", "3600"), ("WARMER_LOAD_MAX", "0")]), + ("Resources", [("RES_LOAD_WARN", "40"), ("RES_SWAP_WARN_MB", "7000"), ("RES_MEM_MIN_MB", "800")]), +] +UI_KEYS = set(k for _, items in UI_SCHEMA for k, _ in items) + +__all__ = ["_build_server"] + +def _is_secret(k): + ku = k.upper() + return any(h in ku for h in _SECRET_HINT) +def _ui_health(): + """Quick reachability of every monitored service, probed in parallel (short timeouts).""" + def arr_probe(a): + def f(): + st = json.load(a._req("GET", "/system/status", t=5)) + warns = [h for h in a.health() if h.get("type") in ("warning", "error")] + return True, ("v%s" % st.get("version", "?")) + (", %d health warn" % len(warns) if warns else "") + return f + jobs = [(a.name, a.kind, arr_probe(a)) for a in INSTANCES] + if DECY_URL: + jobs.append(("decypharr", "mount", lambda: (http_code(DECY_URL, t=5) == 200, DECY_URL))) + if PLEX_URL: + jobs.append(("plex", "plex", lambda: ( + http_code(PLEX_URL.rstrip("/") + "/identity" + ("?X-Plex-Token=" + PLEX_TOKEN if PLEX_TOKEN else ""), t=5) == 200, ""))) + if BAZARR_URL: + jobs.append(("bazarr", "bazarr", lambda: (http_code(BAZARR_URL.rstrip("/") + "/api/system/status", + headers={"X-API-KEY": BAZARR_APIKEY} if BAZARR_APIKEY else None, t=5) == 200, ""))) + if SEERR_URL: + jobs.append(("seerr", "seerr", lambda: (http_code(SEERR_URL.rstrip("/") + "/api/v1/status", + headers={"X-Api-Key": SEERR_APIKEY} if SEERR_APIKEY else None, t=5) == 200, ""))) + if TAUTULLI_URL: + jobs.append(("tautulli", "tautulli", lambda: (http_code( + TAUTULLI_URL.rstrip("/") + "/api/v2?apikey=" + TAUTULLI_APIKEY + "&cmd=get_activity", + t=5) == 200, ""))) + if PULSARR_URL: + jobs.append(("pulsarr", "pulsarr", lambda: (http_code( + PULSARR_URL.rstrip("/") + "/health", + headers={"x-api-key": PULSARR_APIKEY}, t=5) == 200, ""))) + out = [None] * len(jobs) + def run(i, name, kind, fn): + try: + up, detail = fn() + except Exception as e: + up, detail = False, str(e)[:46] + out[i] = {"name": name, "kind": kind, "up": up, "detail": detail} + ths = [threading.Thread(target=run, args=(i, n, k, fn), daemon=True) for i, (n, k, fn) in enumerate(jobs)] + for t in ths: t.start() + for t in ths: t.join(7) + return [r for r in out if r] +_RUN_NULL = {"last_start": None, "last_end": None, "last_duration": None, + "last_outcome": None, "last_error": None, "run_count": 0, "error_count": 0} + +def _ui_status(): + def _run_info(cid): + """Return run metadata for cid, or nulled defaults if the check has never run.""" + r = _check_runs.get(cid) or {} + return { + "last_start": r.get("last_start"), + "last_end": r.get("last_end"), + "last_duration": r.get("last_duration"), + "last_outcome": r.get("last_outcome"), + "last_error": r.get("last_error", ""), + "run_count": r.get("run_count", 0), + "error_count": r.get("error_count", 0), + } + checks = [{"name": n, "on": bool(e), **_run_info(n)} for n, e, _, _, _, _ in CHECKS] + # Synthetic entries: warmer and detail-page warm are not in CHECKS but appear in the UI. + # They have no run metadata in _check_runs so we always emit nulled defaults. + checks.append({"name": "warmer", "on": _b("ENABLE_WARMER", False) and bool(PLEX_URL), **_RUN_NULL}) + checks.append({"name": "detail-page warm", "on": bool(WARM_PLEXLOG_CMD or WARM_PLEXLOG_FILE), **_RUN_NULL}) + return {"version": VERSION, "mode": MODE, "dry_run": DRY_RUN, "load": round(host_load(), 2), "checks": checks, "rescan_backlog": rescan_backlog()} +def _ui_warmer(): + stats = _warmer.get_stats() + stats["enabled"] = _b("ENABLE_WARMER", False) and bool(PLEX_URL) + stats["detail_page"] = bool(WARM_PLEXLOG_CMD or WARM_PLEXLOG_FILE) + stats["recent"] = stats["recent"][:40] + return stats +def _ui_config(): + groups = [] + for g, items in UI_SCHEMA: + rows = [{"key": k, "val": ("" if _is_secret(k) else os.environ.get(k, "")), "ph": ph, "secret": _is_secret(k)} + for k, ph in items] + groups.append({"group": g, "rows": rows}) + return {"groups": groups, "file": CONFIG_FILE} +def _ui_save(body): + try: + incoming = json.loads(body or b"{}") + except Exception: + return False, "bad json" + try: + ov = json.load(open(CONFIG_FILE)) + except Exception: + ov = {} + n = 0 + for k, v in incoming.items(): + if k in UI_KEYS and not _is_secret(k): + ov[k] = v; os.environ[str(k)] = str(v); n += 1 + try: + os.makedirs(os.path.dirname(CONFIG_FILE) or ".", exist_ok=True) + json.dump(ov, open(CONFIG_FILE, "w"), indent=1) + except Exception as e: + return False, str(e)[:80] + return True, "saved %d (restart to apply)" % n +def _ui_logs(n): + if not LOG_FILE: + return "(set DOCTOR_LOG_FILE to view logs here)" + try: + return "".join(open(LOG_FILE, errors="ignore").readlines()[-n:]) + except Exception as e: + return "log read error: " + str(e)[:80] +def _ui_state(): + """Return the full state.json as a dict for operator inspection.""" + return _load_state() + + +def _is_under_roots(target, roots): + target = os.path.realpath(target) + for r in roots: + if target == os.path.realpath(r) or target.startswith(os.path.realpath(r) + os.sep): + return True + return False + + +def _rescan_folders(parent=None): + roots = [r for r in (RESCAN_LIBRARY_PATHS or []) if os.path.isdir(r)] + if not roots: + return [] + if parent: + parent = os.path.realpath(parent) + if not _is_under_roots(parent, roots): + return [] + base = parent + else: + base = None + if base is None: + out = [] + for r in roots: + out.append({"name": os.path.basename(r) or r, "path": r, "is_root": True}) + return sorted(out, key=lambda x: x["name"].lower()) + try: + entries = [] + for name in os.listdir(base): + p = os.path.join(base, name) + if os.path.isdir(p): + entries.append({"name": name, "path": p, "is_root": False}) + return sorted(entries, key=lambda x: x["name"].lower()) + except OSError: + return [] + + +def _trigger_rescan_path(folder_path): + from .clients import Plex + if not (PLEX_URL and PLEX_TOKEN): + return False, "PLEX_URL/PLEX_TOKEN not set" + roots = [r for r in (RESCAN_LIBRARY_PATHS or []) if os.path.isdir(r)] + folder_path = os.path.realpath(folder_path) + if not _is_under_roots(folder_path, roots): + return False, "path not in RESCAN_LIBRARY_PATHS" + plex = Plex(PLEX_URL, PLEX_TOKEN) + sections = [] + raw = plex.sections() + for sec in raw or []: + sec["locations"] = plex.section_locations(sec["key"]) or [] + sections.append(sec) + best = None + best_len = 0 + for sec in sections: + for loc in sec.get("locations", []): + loc = os.path.realpath(loc) + if folder_path == loc or folder_path.startswith(loc + os.sep): + if len(loc) > best_len: + best = sec + best_len = len(loc) + if not best: + return False, "no Plex section for path" + log.info("[ui] manual partial scan for %s in section %s", folder_path, best["title"]) + if plex.scan_path(best["key"], folder_path): + return True, "scan triggered" + return False, "Plex scan_path failed" + + +def _build_server(port): + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + from urllib.parse import urlparse, parse_qs + class H(BaseHTTPRequestHandler): + def _send(self, code, ctype, body): + if isinstance(body, str): + body = body.encode("utf-8") + self.send_response(code); self.send_header("Content-Type", ctype) + self.send_header("Content-Length", str(len(body))); self.end_headers() + try: self.wfile.write(body) + except Exception: pass + def _authed(self): + if not UI_TOKEN: + return True + q = parse_qs(urlparse(self.path).query) + return self.headers.get("X-Doctor-Token") == UI_TOKEN or q.get("token", [""])[0] == UI_TOKEN + def do_GET(self): + path = urlparse(self.path).path + if path in ("/health", "/healthz"): + return self._send(200, "text/plain", "ok") + if not EN_UI: + return self._send(404, "text/plain", "nf") + if not self._authed(): + return self._send(401, "text/plain", "unauthorized") + if path in ("/", "/ui", "/index.html"): + return self._send(200, "text/html; charset=utf-8", UI_HTML) + if path == "/api/status": return self._send(200, "application/json", json.dumps(_ui_status())) + if path == "/api/health": return self._send(200, "application/json", json.dumps(_ui_health())) + if path == "/api/warmer": return self._send(200, "application/json", json.dumps(_ui_warmer())) + + if path == "/api/config": return self._send(200, "application/json", json.dumps(_ui_config())) + if path == "/api/state": return self._send(200, "application/json", json.dumps(_ui_state())) + if path == "/api/rescan/folders": + parent = parse_qs(urlparse(self.path).query).get("parent", [""])[0] or None + return self._send(200, "application/json", json.dumps({"roots": [r for r in (RESCAN_LIBRARY_PATHS or []) if os.path.isdir(r)], "folders": _rescan_folders(parent)})) + if path == "/api/logs": + try: n = min(int(parse_qs(urlparse(self.path).query).get("n", ["300"])[0]), 3000) + except Exception: n = 300 + return self._send(200, "text/plain; charset=utf-8", _ui_logs(n)) + return self._send(404, "text/plain", "nf") + def do_POST(self): + path = urlparse(self.path).path + length = int(self.headers.get("Content-Length", 0) or 0) + body = self.rfile.read(length) if length else b"" + if path in ("/api/config", "/api/restart", "/api/rescan/scan", + "/api/plex/rescan", "/api/plex/emptytrash", "/api/sweep") or path.startswith("/api/check/"): + if not EN_UI or not self._authed(): + return self._send(401, "text/plain", "unauthorized") + if path == "/api/config": + ok, msg = _ui_save(body) + return self._send(200 if ok else 400, "application/json", json.dumps({"ok": ok, "msg": msg})) + if path == "/api/plex/rescan": + threading.Thread(target=plex_rescan, daemon=True).start() + return self._send(202, "application/json", json.dumps({"ok": True, "msg": "Plex rescan started"})) + if path == "/api/plex/emptytrash": + threading.Thread(target=plex_empty_trash, daemon=True).start() + return self._send(202, "application/json", json.dumps({"ok": True, "msg": "Plex empty trash started"})) + if path == "/api/sweep": + threading.Thread(target=sweep, daemon=True).start() + return self._send(202, "application/json", json.dumps({"ok": True, "msg": "sweep started"})) + if path == "/api/rescan/scan": + try: p = json.loads(body or b"{"); folder = p.get("path", "") + except Exception: folder = "" + if not folder: + return self._send(400, "application/json", json.dumps({"ok": False, "msg": "missing path"})) + ok, msg = _trigger_rescan_path(folder) + return self._send(202 if ok else 500, "application/json", json.dumps({"ok": ok, "msg": msg})) + if path.startswith("/api/check/"): + cid = path.split("/api/check/", 1)[1] + for name, en, fn, _, _, _ in CHECKS: + if name == cid and en: + threading.Thread(target=_run_scheduled_check, args=(cid, fn), daemon=True).start() + return self._send(202, "application/json", json.dumps({"ok": True, "msg": "check %s started" % cid})) + return self._send(400, "application/json", json.dumps({"ok": False, "msg": "unknown or disabled check"})) + self._send(200, "application/json", json.dumps({"ok": True, "msg": "restarting"})) + log.info("[ui] restart requested"); threading.Thread(target=lambda: (time.sleep(0.4), os._exit(0)), daemon=True).start() + return + if MODE == "event": # arr webhook + try: p = json.loads(body or b"{}") + except Exception: p = {} + ev = p.get("eventType") or p.get("EventType") or "?"; inst = p.get("instanceName") or p.get("InstanceName") + self._send(200, "text/plain", "ok") + if ev == "Test": + log.info("webhook Test from %s", inst or "?"); return + if TRIGGER_EVENTS and ev not in TRIGGER_EVENTS: + return + log.info("event '%s' from %s -> sweep", ev, inst or "all") + threading.Thread(target=sweep, kwargs={"only": inst}, daemon=True).start(); return + self._send(404, "text/plain", "nf") + def log_message(self, *a): + pass + return ThreadingHTTPServer(("0.0.0.0", port), H) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c6b452e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,15 @@ +[tool.ruff] +target-version = "py312" + +[tool.ruff.lint] +# F = pyflakes: undefined names, unused imports, redefined names. +# E and W are available for local runs but CI only gates on F (see ci.yml). +select = ["F"] + +# The compat re-export in config.py is intentional — silence it per-file +# rather than globally, so future real F401s in other files still fail. +[tool.ruff.lint.per-file-ignores] +"doctor/config.py" = ["F401"] # backward-compat re-export of utils helpers +"doctor/__init__.py" = ["F401"] # re-export of VERSION +"doctor/checks/repair/__init__.py" = ["F401"] # re-export of check_repair, _dead_symlink +"doctor/checks/__init__.py" = ["F401"] # explicit re-exports for checks package diff --git a/stack-doctor-patch.py b/stack-doctor-patch.py new file mode 100644 index 0000000..00338c2 --- /dev/null +++ b/stack-doctor-patch.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +import os + +WEBUI = "/data/stack-doctor-src/doctor/webui.py" +JANITOR = "/data/stack-doctor-src/doctor/checks/janitor.py" + +def patch_webui(): + if not os.path.exists(WEBUI): + print(f"ERROR: {WEBUI} not found") + return False + with open(WEBUI) as f: + content = f.read() + orig = content + content = content.replace('for r in reversed(_warm_recent)', 'for r in reversed(_warmer._warm_recent)') + content = content.replace('"total": _warm_count[0]', '"total": _warmer._warm_count[0]') + if content != orig: + with open(WEBUI, 'w') as f: + f.write(content) + print("PATCHED webui.py warmer variables") + else: + print("No webui patch needed") + return True + +def patch_janitor(): + if not os.path.exists(JANITOR): + print(f"ERROR: {JANITOR} not found") + return False + with open(JANITOR) as f: + content = f.read() + orig = content + old = 'mm = re.search(r"/__all__/([^/]+)(?:/|$)", tgt)' + new = 'mm = re.search(r"/(?:__all__|complete)/([^/]+)(?:/|$)", tgt)' + content = content.replace(old, new) + if content != orig: + with open(JANITOR, 'w') as f: + f.write(content) + print("PATCHED janitor.py altmount/complete regex") + else: + print("No janitor patch needed") + return True + +patch_webui() +patch_janitor() diff --git a/stack-doctor.service.example b/stack-doctor.service.example index 3949d89..cb54d63 100644 --- a/stack-doctor.service.example +++ b/stack-doctor.service.example @@ -2,7 +2,8 @@ # so it has native power (restart decypharr locally, read its journal, touch the library) # with no container-to-host bridge / SSH key needed. # -# cp doctor.py /opt/stack-doctor/doctor.py +# mkdir -p /opt/stack-doctor +# cp -r doctor /opt/stack-doctor/doctor # cp stack-doctor.service.example /etc/systemd/system/stack-doctor.service # edit values # systemctl daemon-reload && systemctl enable --now stack-doctor.service # journalctl -u stack-doctor -f @@ -13,7 +14,8 @@ After=network-online.target decypharr.service [Service] Type=simple -ExecStart=/usr/bin/python3 /opt/stack-doctor/doctor.py +WorkingDirectory=/opt/stack-doctor +ExecStart=/usr/bin/python3 -m doctor Restart=always RestartSec=30 # --- mode --- diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..1d432aa --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for stack-doctor.""" diff --git a/tests/test_checks_exports.py b/tests/test_checks_exports.py new file mode 100644 index 0000000..fe32ed3 --- /dev/null +++ b/tests/test_checks_exports.py @@ -0,0 +1,100 @@ +"""Compatibility tests for the doctor.checks public export boundary. + +These tests lock in the names that doctor.checks is expected to expose before +doctor/checks/__init__.py is converted from star-imports to explicit re-exports. + +The consumers we must preserve are: + - doctor/scheduler.py: imports 13 check_* functions explicitly + - doctor/__main__.py: imports backfill_missing_seasons, warmer_loop, plexlog_loop + - doctor/webui.py: imports warmer (as _warmer) explicitly +""" +import unittest + +import doctor.checks as _checks + + +class CheckFunctionsExportTest(unittest.TestCase): + """All check functions referenced by CHECKS must be importable from doctor.checks.""" + + def test_check_queue(self): + self.assertTrue(callable(_checks.check_queue)) + + def test_check_providers(self): + self.assertTrue(callable(_checks.check_providers)) + + def test_check_decypharr(self): + self.assertTrue(callable(_checks.check_decypharr)) + + def test_check_plex(self): + self.assertTrue(callable(_checks.check_plex)) + + def test_check_plex_scan(self): + self.assertTrue(callable(_checks.check_plex_scan)) + + def test_check_resources(self): + self.assertTrue(callable(_checks.check_resources)) + + def test_check_janitor(self): + self.assertTrue(callable(_checks.check_janitor)) + + def test_check_repair(self): + self.assertTrue(callable(_checks.check_repair)) + + def test_check_bazarr(self): + self.assertTrue(callable(_checks.check_bazarr)) + + def test_check_seerr(self): + self.assertTrue(callable(_checks.check_seerr)) + + def test_check_missing_seasons(self): + self.assertTrue(callable(_checks.check_missing_seasons)) + + def test_check_no_upgrade_profile(self): + self.assertTrue(callable(_checks.check_no_upgrade_profile)) + + def test_check_multipack(self): + self.assertTrue(callable(_checks.check_multipack)) + + +class AuxiliaryExportsTest(unittest.TestCase): + """Names used by __main__.py and webui.py must remain available.""" + + def test_backfill_missing_seasons(self): + self.assertTrue(callable(_checks.backfill_missing_seasons)) + + def test_warmer_loop(self): + self.assertTrue(callable(_checks.warmer_loop)) + + def test_plexlog_loop(self): + self.assertTrue(callable(_checks.plexlog_loop)) + + def test_warmer_module(self): + """webui.py imports warmer from doctor.checks as _warmer.""" + self.assertTrue(hasattr(_checks, "warmer")) + self.assertTrue(callable(_checks.warmer.warmer_loop)) + + +class ConsumerImportPatternsTest(unittest.TestCase): + """Simulate the exact import patterns used by the three consumers.""" + + def test_scheduler_import_pattern(self): + from doctor.checks import ( + check_queue, + check_repair, + ) + self.assertTrue(callable(check_queue)) + self.assertTrue(callable(check_repair)) + + def test_main_import_pattern(self): + from doctor.checks import backfill_missing_seasons, plexlog_loop, warmer_loop + self.assertTrue(callable(backfill_missing_seasons)) + self.assertTrue(callable(plexlog_loop)) + self.assertTrue(callable(warmer_loop)) + + def test_webui_import_pattern(self): + from doctor.checks import warmer + self.assertTrue(hasattr(warmer, "warmer_loop")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_churn.py b/tests/test_churn.py new file mode 100644 index 0000000..ef6dbd2 --- /dev/null +++ b/tests/test_churn.py @@ -0,0 +1,390 @@ +"""Unit tests for the churn-brake logic in doctor.state. + +These tests exercise _churn_record and _churn_remonitor in isolation using +only an in-memory state dict and a mocked Arr so no network or filesystem +access is required. + +The churn-brake constants (CHURN_LIMIT, CHURN_ACTION, CHURN_BACKOFF) are +module-level names in doctor.state (imported via star from doctor.config at +module load time). We patch them on the state module directly so the tests +are independent of env-var parsing order and don't affect each other. +""" +import time +import unittest +from unittest.mock import MagicMock, patch + +from doctor.state import _churn_record, _churn_remonitor, _offenders + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _make_arr(kind="sonarr", name="sonarr", target_id=42): + arr = MagicMock() + arr.name = name + arr.kind = kind + arr.set_monitored.return_value = True # success by default + # queue_target_id is a real method on Arr that inspects rec; since we use MagicMock + # we must return the target_id explicitly so _churn_record writes the right state key. + arr.queue_target_id.return_value = target_id + return arr + + +def _make_rec(episode_id=42, movie_id=None): + """A minimal queue record (content not inspected by tests; queue_target_id is mocked).""" + rec = {"id": 1} + if episode_id is not None: + rec["episodeId"] = episode_id + if movie_id is not None: + rec["movieId"] = movie_id + return rec + + +def _patched(**kwargs): + """Return a unittest.mock._patch context manager stack for churn constants.""" + defaults = { + "doctor.state.CHURN_LIMIT": 3, + "doctor.state.CHURN_ACTION": "report", + "doctor.state.CHURN_BACKOFF": [600, 3600, 86400], + } + defaults.update({"doctor.state." + k: v for k, v in kwargs.items()}) + # Build a single patcher using patch.multiple + return patch.multiple("doctor.state", **{k.replace("doctor.state.", ""): v for k, v in defaults.items()}) + + +# --------------------------------------------------------------------------- +# _churn_record — CHURN_LIMIT=0 (disabled) +# --------------------------------------------------------------------------- + +class ChurnDisabledTest(unittest.TestCase): + def test_disabled_when_limit_zero(self): + """CHURN_LIMIT=0 means the brake is off; _churn_record must return False immediately.""" + state = {} + arr = _make_arr() + rec = _make_rec() + with patch("doctor.state.CHURN_LIMIT", 0): + result = _churn_record(state, arr, rec, "Show S01E01") + self.assertFalse(result) + # No state should have been written + self.assertEqual(state, {}) + + def test_disabled_when_no_target_id(self): + """If queue_target_id returns None, _churn_record must return False.""" + arr = _make_arr(kind="prowlarr") # queue_target_id returns None for prowlarr + rec = _make_rec(episode_id=None, movie_id=None) + state = {} + with patch("doctor.state.CHURN_LIMIT", 3): + result = _churn_record(state, arr, rec, "Some Title") + self.assertFalse(result) + + +# --------------------------------------------------------------------------- +# _churn_record — accumulation below limit +# --------------------------------------------------------------------------- + +class ChurnAccumulationTest(unittest.TestCase): + def _run(self, n_fails, limit=3, action="report"): + arr = _make_arr(target_id=99) + rec = _make_rec(episode_id=99) + state = {} + with patch("doctor.state.CHURN_LIMIT", limit), \ + patch("doctor.state.CHURN_ACTION", action), \ + patch("doctor.state.CHURN_BACKOFF", [600]): + results = [] + for _ in range(n_fails): + results.append(_churn_record(state, arr, rec, "Show S01E01")) + return state, results + + def test_fails_accumulate_below_limit(self): + state, results = self._run(n_fails=2, limit=3) + self.assertTrue(all(r is False for r in results)) + offs = _offenders(state).get("sonarr", {}).get("99", {}) + self.assertEqual(offs["fails"], 2) + + def test_no_action_at_limit_minus_one(self): + _, results = self._run(n_fails=2, limit=3, action="park") + # At fail #2 (< limit 3), still no action + self.assertFalse(any(results)) + + def test_counters_accumulate_across_calls(self): + arr = _make_arr(target_id=7) + rec = _make_rec(episode_id=7) + state = {} + with patch("doctor.state.CHURN_LIMIT", 5), \ + patch("doctor.state.CHURN_ACTION", "report"), \ + patch("doctor.state.CHURN_BACKOFF", [600]): + for i in range(4): + _churn_record(state, arr, rec, "Movie") + offs = _offenders(state)["sonarr"]["7"] + self.assertEqual(offs["fails"], 4) + + +# --------------------------------------------------------------------------- +# _churn_record — action=report +# --------------------------------------------------------------------------- + +class ChurnReportActionTest(unittest.TestCase): + def _hit_limit(self, limit=3): + arr = _make_arr(target_id=10) + rec = _make_rec(episode_id=10) + state = {} + with patch("doctor.state.CHURN_LIMIT", limit), \ + patch("doctor.state.CHURN_ACTION", "report"), \ + patch("doctor.state.CHURN_BACKOFF", [600]): + results = [_churn_record(state, arr, rec, "Show") for _ in range(limit)] + return arr, state, results + + def test_report_returns_false(self): + """report action must not un-monitor, so the caller's re-search still fires.""" + _, _, results = self._hit_limit() + self.assertFalse(results[-1]) # last call hits limit + + def test_report_does_not_call_set_monitored(self): + arr, _, _ = self._hit_limit() + arr.set_monitored.assert_not_called() + + def test_report_sets_until_to_sentinel(self): + """until=-1 signals 'reported, no backoff scheduled'.""" + _, state, _ = self._hit_limit() + offs = _offenders(state)["sonarr"]["10"] + self.assertEqual(offs["until"], -1) + + def test_report_only_fires_once(self): + """Subsequent calls after the first report must short-circuit (until != 0).""" + arr = _make_arr() + rec = _make_rec(episode_id=10) + state = {} + with patch("doctor.state.CHURN_LIMIT", 2), \ + patch("doctor.state.CHURN_ACTION", "report"), \ + patch("doctor.state.CHURN_BACKOFF", [600]): + for _ in range(5): + _churn_record(state, arr, rec, "Show") + # set_monitored must never have been called + arr.set_monitored.assert_not_called() + + +# --------------------------------------------------------------------------- +# _churn_record — action=park +# --------------------------------------------------------------------------- + +class ChurnParkActionTest(unittest.TestCase): + def _park(self, limit=3): + arr = _make_arr(target_id=20) + rec = _make_rec(episode_id=20) + state = {} + with patch("doctor.state.CHURN_LIMIT", limit), \ + patch("doctor.state.CHURN_ACTION", "park"), \ + patch("doctor.state.CHURN_BACKOFF", [600]): + results = [_churn_record(state, arr, rec, "Movie") for _ in range(limit)] + return arr, state, results + + def test_park_returns_true_at_limit(self): + _, _, results = self._park() + self.assertTrue(results[-1]) + + def test_park_calls_set_monitored_false(self): + arr, _, _ = self._park() + arr.set_monitored.assert_called_once_with([20], False) + + def test_park_sets_until_to_sentinel(self): + _, state, _ = self._park() + offs = _offenders(state)["sonarr"]["20"] + self.assertEqual(offs["until"], -1) + + def test_park_resets_fails_counter(self): + _, state, _ = self._park() + offs = _offenders(state)["sonarr"]["20"] + self.assertEqual(offs["fails"], 0) + + def test_park_no_action_if_set_monitored_fails(self): + arr = _make_arr(target_id=20) + arr.set_monitored.return_value = False # API failure + rec = _make_rec(episode_id=20) + state = {} + with patch("doctor.state.CHURN_LIMIT", 3), \ + patch("doctor.state.CHURN_ACTION", "park"), \ + patch("doctor.state.CHURN_BACKOFF", [600]): + results = [_churn_record(state, arr, rec, "X") for _ in range(3)] + self.assertFalse(results[-1]) + + +# --------------------------------------------------------------------------- +# _churn_record — action=backoff +# --------------------------------------------------------------------------- + +class ChurnBackoffActionTest(unittest.TestCase): + def _backoff(self, limit=3, backoff_levels=None): + arr = _make_arr(target_id=30) + rec = _make_rec(episode_id=30) + state = {} + levels = backoff_levels or [600, 3600, 86400] + with patch("doctor.state.CHURN_LIMIT", limit), \ + patch("doctor.state.CHURN_ACTION", "backoff"), \ + patch("doctor.state.CHURN_BACKOFF", levels): + results = [_churn_record(state, arr, rec, "Series") for _ in range(limit)] + return arr, state, results + + def test_backoff_returns_true_at_limit(self): + _, _, results = self._backoff() + self.assertTrue(results[-1]) + + def test_backoff_schedules_remonitor_timestamp(self): + t_before = time.time() + _, state, _ = self._backoff(backoff_levels=[600]) + t_after = time.time() + offs = _offenders(state)["sonarr"]["30"] + self.assertGreater(offs["until"], t_before + 590) + self.assertLess(offs["until"], t_after + 610) + + def test_backoff_level_increments(self): + _, state, _ = self._backoff() + offs = _offenders(state)["sonarr"]["30"] + self.assertEqual(offs["level"], 1) + + def test_backoff_level_escalates_on_repeated_breach(self): + """Each successive park cycle uses the next backoff tier.""" + arr = _make_arr(target_id=30) + rec = _make_rec(episode_id=30) + state = {} + levels = [600, 3600, 86400] + + def _run_cycle(n): + with patch("doctor.state.CHURN_LIMIT", 2), \ + patch("doctor.state.CHURN_ACTION", "backoff"), \ + patch("doctor.state.CHURN_BACKOFF", levels): + for _ in range(n): + _churn_record(state, arr, rec, "S") + # Simulate re-monitor elapsed: reset until so next cycle can park again + offs = _offenders(state)["sonarr"]["30"] + offs["until"] = 0 + + _run_cycle(2) # first park: level 0 -> 1 + _run_cycle(2) # second park: level 1 -> 2 + offs = _offenders(state)["sonarr"]["30"] + self.assertEqual(offs["level"], 2) + + def test_backoff_clamps_to_last_tier(self): + """Level beyond the backoff list clamps to the last entry.""" + arr = _make_arr(target_id=31) + rec = _make_rec(episode_id=31) + state = {} + levels = [600] # only one tier + + def _cycle(): + with patch("doctor.state.CHURN_LIMIT", 2), \ + patch("doctor.state.CHURN_ACTION", "backoff"), \ + patch("doctor.state.CHURN_BACKOFF", levels): + for _ in range(2): + _churn_record(state, arr, rec, "T") + _offenders(state)["sonarr"]["31"]["until"] = 0 + + for _ in range(3): # three park cycles + _cycle() + + # After 3 park cycles with a single-tier [600] backoff, level should be 3 + # (level keeps incrementing even when clamped to last tier). + offs = _offenders(state)["sonarr"]["31"] + self.assertEqual(offs["level"], 3) + # until was reset to 0 by the last _cycle iteration so we only check level here. + + +# --------------------------------------------------------------------------- +# _churn_remonitor +# --------------------------------------------------------------------------- + +class ChurnRemonitorTest(unittest.TestCase): + def test_noop_when_limit_zero(self): + """If CHURN_LIMIT=0, remonitor must do nothing.""" + arr = _make_arr() + state = {"__offenders__": {"sonarr": {"5": {"until": 1, "fails": 0, "level": 0, "title": "X"}}}} + with patch("doctor.state.CHURN_LIMIT", 0), \ + patch("doctor.state.CHURN_ACTION", "backoff"), \ + patch("doctor.state.INSTANCES", [arr]): + _churn_remonitor(state) + arr.set_monitored.assert_not_called() + + def test_noop_when_action_is_park(self): + """remonitor only runs for action=backoff; park is manual.""" + arr = _make_arr() + state = {"__offenders__": {"sonarr": {"5": {"until": 1, "fails": 0, "level": 0, "title": "X"}}}} + with patch("doctor.state.CHURN_LIMIT", 3), \ + patch("doctor.state.CHURN_ACTION", "park"), \ + patch("doctor.state.INSTANCES", [arr]): + _churn_remonitor(state) + arr.set_monitored.assert_not_called() + + def test_noop_when_until_not_elapsed(self): + far_future = time.time() + 9999 + arr = _make_arr() + state = {"__offenders__": {"sonarr": {"5": {"until": far_future, "fails": 0, "level": 1, "title": "X"}}}} + with patch("doctor.state.CHURN_LIMIT", 3), \ + patch("doctor.state.CHURN_ACTION", "backoff"), \ + patch("doctor.state.INSTANCES", [arr]): + _churn_remonitor(state) + arr.set_monitored.assert_not_called() + + def test_remonitor_fires_when_elapsed(self): + past = time.time() - 1 # already elapsed + arr = _make_arr() + state = {"__offenders__": {"sonarr": {"42": {"until": past, "fails": 0, "level": 1, "title": "Show"}}}} + with patch("doctor.state.CHURN_LIMIT", 3), \ + patch("doctor.state.CHURN_ACTION", "backoff"), \ + patch("doctor.state.INSTANCES", [arr]): + _churn_remonitor(state) + arr.set_monitored.assert_called_once_with([42], True) + + def test_remonitor_resets_fails_and_until(self): + past = time.time() - 1 + arr = _make_arr() + state = {"__offenders__": {"sonarr": {"42": {"until": past, "fails": 3, "level": 1, "title": "Show"}}}} + with patch("doctor.state.CHURN_LIMIT", 3), \ + patch("doctor.state.CHURN_ACTION", "backoff"), \ + patch("doctor.state.INSTANCES", [arr]): + _churn_remonitor(state) + offs = state["__offenders__"]["sonarr"]["42"] + self.assertEqual(offs["fails"], 0) + self.assertEqual(offs["until"], 0) + + def test_remonitor_preserves_level(self): + """Level must survive remonitor so the next park escalates.""" + past = time.time() - 1 + arr = _make_arr() + state = {"__offenders__": {"sonarr": {"42": {"until": past, "fails": 0, "level": 2, "title": "X"}}}} + with patch("doctor.state.CHURN_LIMIT", 3), \ + patch("doctor.state.CHURN_ACTION", "backoff"), \ + patch("doctor.state.INSTANCES", [arr]): + _churn_remonitor(state) + self.assertEqual(state["__offenders__"]["sonarr"]["42"]["level"], 2) + + def test_remonitor_skips_sentinel_until(self): + """until=-1 (permanent park) must not be re-monitored automatically.""" + arr = _make_arr() + state = {"__offenders__": {"sonarr": {"7": {"until": -1, "fails": 0, "level": 0, "title": "X"}}}} + with patch("doctor.state.CHURN_LIMIT", 3), \ + patch("doctor.state.CHURN_ACTION", "backoff"), \ + patch("doctor.state.INSTANCES", [arr]): + _churn_remonitor(state) + arr.set_monitored.assert_not_called() + + def test_remonitor_multiple_instances(self): + """remonitor iterates over all INSTANCES, not just the first.""" + past = time.time() - 1 + arr1 = _make_arr(name="sonarr") + arr2 = _make_arr(name="radarr", kind="radarr") + state = { + "__offenders__": { + "sonarr": {"1": {"until": past, "fails": 0, "level": 0, "title": "A"}}, + "radarr": {"2": {"until": past, "fails": 0, "level": 0, "title": "B"}}, + } + } + with patch("doctor.state.CHURN_LIMIT", 3), \ + patch("doctor.state.CHURN_ACTION", "backoff"), \ + patch("doctor.state.INSTANCES", [arr1, arr2]): + _churn_remonitor(state) + arr1.set_monitored.assert_called_once_with([1], True) + arr2.set_monitored.assert_called_once_with([2], True) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_clients_history_grabbed.py b/tests/test_clients_history_grabbed.py new file mode 100644 index 0000000..b6ea82f --- /dev/null +++ b/tests/test_clients_history_grabbed.py @@ -0,0 +1,147 @@ +"""Tests for Arr.history_grabbed — timestamp comparison correctness. + +These tests target the ISO-8601 timestamp comparison bug (A3) where string +lexicographic comparison fails when *arr API dates include milliseconds or +timezone offsets in a different format than the stored search_ts. +""" +from unittest.mock import patch +import unittest +from doctor.clients import Arr + + +def _make_arr(kind="sonarr"): + arr = Arr.__new__(Arr) + arr.name = "test" + arr.kind = kind + arr.base = "http://localhost/api/v3" + arr.apikey = "test" + return arr + + +def _make_rec(date, event_type="grabbed", episode_id=None): + rec = {"eventType": event_type, "date": date, "sourceTitle": "Foo.S01E01"} + if episode_id is not None: + rec["episodeId"] = episode_id + return rec + + +class HistoryGrabbedTimestampTest(unittest.TestCase): + """history_grabbed must correctly identify records *after* since_ts across + all common *arr date formats.""" + + def _call(self, arr, records, since_ts, entity_ids=None): + with patch.object(arr, "history", return_value=records): + return arr.history_grabbed(1, since_ts, entity_ids) + + # --- Formats that must be recognised as AFTER the search timestamp --- + + def test_same_format_z_suffix_after(self): + """Record with matching Z-suffix format, newer than search_ts.""" + arr = _make_arr() + rec = _make_rec("2026-06-23T05:00:00Z") + result = self._call(arr, [rec], "2026-06-23T04:55:00Z") + self.assertIsNotNone(result) + + def test_milliseconds_after(self): + """Record with milliseconds (.123Z), newer than plain search_ts.""" + arr = _make_arr() + rec = _make_rec("2026-06-23T05:00:00.123Z") + result = self._call(arr, [rec], "2026-06-23T04:55:00Z") + self.assertIsNotNone(result) + + def test_plus_offset_after(self): + """Record with +00:00 offset (common *arr format), newer than search_ts.""" + arr = _make_arr() + # 05:00:00+00:00 is 5 minutes after 04:55:00Z — must be found + rec = _make_rec("2026-06-23T05:00:00+00:00") + result = self._call(arr, [rec], "2026-06-23T04:55:00Z") + self.assertIsNotNone(result) + + def test_milliseconds_same_second_after(self): + """A record at T04:55:30.5Z is newer than search_ts T04:55:30Z. + String compare bug: ord('.')=46 < ord('Z')=90, so '30.5Z' < '30Z' + lexicographically -> record is incorrectly skipped without the fix.""" + arr = _make_arr() + rec = _make_rec("2026-06-23T04:55:30.500Z") + result = self._call(arr, [rec], "2026-06-23T04:55:30Z") + self.assertIsNotNone(result) + + def test_plus_offset_with_ms_after(self): + """Record with milliseconds AND +00:00 offset, newer than search_ts.""" + arr = _make_arr() + rec = _make_rec("2026-06-23T05:00:00.456+00:00") + result = self._call(arr, [rec], "2026-06-23T04:55:00Z") + self.assertIsNotNone(result) + + # --- Formats that must be recognised as BEFORE/EQUAL and skipped --- + + def test_same_format_z_suffix_before(self): + arr = _make_arr() + rec = _make_rec("2026-06-23T04:50:00Z") + result = self._call(arr, [rec], "2026-06-23T04:55:00Z") + self.assertIsNone(result) + + def test_milliseconds_before(self): + arr = _make_arr() + rec = _make_rec("2026-06-23T04:50:00.999Z") + result = self._call(arr, [rec], "2026-06-23T04:55:00Z") + self.assertIsNone(result) + + def test_plus_offset_before(self): + """04:50:00+00:00 is before 04:55:00Z — must be skipped.""" + arr = _make_arr() + rec = _make_rec("2026-06-23T04:50:00+00:00") + result = self._call(arr, [rec], "2026-06-23T04:55:00Z") + self.assertIsNone(result) + + def test_equal_timestamp_is_skipped(self): + """Equal timestamp must NOT be returned (strictly after only).""" + arr = _make_arr() + rec = _make_rec("2026-06-23T04:55:00Z") + result = self._call(arr, [rec], "2026-06-23T04:55:00Z") + self.assertIsNone(result) + + # --- Non-grabbed events always skipped --- + + def test_non_grabbed_event_skipped(self): + arr = _make_arr() + rec = _make_rec("2026-06-23T05:00:00Z", event_type="downloadFolderImported") + result = self._call(arr, [rec], "2026-06-23T04:55:00Z") + self.assertIsNone(result) + + # --- entity_ids filter (sonarr) --- + + def test_sonarr_entity_id_filter_match(self): + arr = _make_arr(kind="sonarr") + rec = _make_rec("2026-06-23T05:00:00Z", episode_id=42) + result = self._call(arr, [rec], "2026-06-23T04:55:00Z", entity_ids=[42]) + self.assertIsNotNone(result) + + def test_sonarr_entity_id_filter_no_match(self): + arr = _make_arr(kind="sonarr") + rec = _make_rec("2026-06-23T05:00:00Z", episode_id=99) + result = self._call(arr, [rec], "2026-06-23T04:55:00Z", entity_ids=[42]) + self.assertIsNone(result) + + def test_radarr_ignores_entity_ids(self): + """Radarr does not filter by episode IDs; any grab after ts is returned.""" + arr = _make_arr(kind="radarr") + rec = _make_rec("2026-06-23T05:00:00Z", episode_id=99) + result = self._call(arr, [rec], "2026-06-23T04:55:00Z", entity_ids=[42]) + self.assertIsNotNone(result) + + # --- Defensive fallback for unparseable dates --- + + def test_unparseable_date_falls_back_gracefully(self): + """A record with a garbage date field should not crash; should be skipped.""" + arr = _make_arr() + rec = _make_rec("not-a-date") + result = self._call(arr, [rec], "2026-06-23T04:55:00Z") + self.assertIsNone(result) + + def test_missing_date_field_skipped(self): + """A record with no date field should be skipped.""" + arr = _make_arr() + rec = {"eventType": "grabbed", "sourceTitle": "Foo"} + result = self._call(arr, [rec], "2026-06-23T04:55:00Z") + self.assertIsNone(result) diff --git a/tests/test_decypharr.py b/tests/test_decypharr.py new file mode 100644 index 0000000..ced353e --- /dev/null +++ b/tests/test_decypharr.py @@ -0,0 +1,464 @@ +"""Unit tests for the FUSE mount health check (decypharr.py).""" +import errno +import os +import tempfile +import threading +import time +import unittest + +from doctor.checks.decypharr import ( + _is_fuse_errno, + _mount_registered, + _probe_statvfs, + _probe_mount, + _record_fuse_result, + _fuse_strikes, + _FuseStatus, +) + + +class IsFuseErrnoTest(unittest.TestCase): + """_is_fuse_errno identifies FUSE-dead errors by errno and message.""" + + def test_eio_errno(self): + self.assertTrue(_is_fuse_errno(OSError(5, "Input/output error"))) + + def test_enotconn_errno(self): + self.assertTrue(_is_fuse_errno(OSError(107, "Transport endpoint is not connected"))) + + def test_enxio_errno(self): + self.assertTrue(_is_fuse_errno(OSError(6, "No such device or address"))) + + def test_message_transport(self): + self.assertTrue(_is_fuse_errno(OSError(0, "transport endpoint is not connected"))) + + def test_message_socket(self): + self.assertTrue(_is_fuse_errno(OSError(0, "Socket not connected"))) + + def test_ordinary_enoent(self): + self.assertFalse(_is_fuse_errno(OSError(errno.ENOENT, "No such file or directory"))) + + def test_ordinary_eperm(self): + self.assertFalse(_is_fuse_errno(OSError(errno.EPERM, "Operation not permitted"))) + + def test_non_oserror(self): + self.assertFalse(_is_fuse_errno(ValueError("nope"))) + + +class MountRegisteredTest(unittest.TestCase): + """_mount_registered reads /proc/mounts to find a FUSE mount for a path.""" + + def test_nonexistent_path_not_under_fuse(self): + # A completely invented path cannot be under any real FUSE mount. + self.assertFalse(_mount_registered("/this/path/definitely/does/not/exist/xyz")) + + def test_zurg_path_registered(self): + # /mnt/zurg is the actual FUSE mount inside the container; + # /mnt/zurg/__all__ is a subdirectory – both should register as True. + # If this test runs outside the container or without the mount, skip it. + try: + with open("/proc/mounts") as f: + text = f.read() + if "/mnt/zurg" not in text: + self.skipTest("/mnt/zurg not mounted in this environment") + except Exception: + self.skipTest("cannot read /proc/mounts") + self.assertTrue(_mount_registered("/mnt/zurg")) + self.assertTrue(_mount_registered("/mnt/zurg/__all__")) + + def test_ancestor_walk(self): + """A child of a FUSE mount should return True even without exact match.""" + # We'll temporarily fake /proc/mounts by monkey-patching the open call. + import builtins + original_open = builtins.open + fake_mounts = "rclone /mnt/fake fuse.rclone rw 0 0\n" + class FakeFile: + def __enter__(self): return self + def __exit__(self, *a): pass + def __iter__(self): return iter(fake_mounts.splitlines(keepends=True)) + def fake_open(path, *a, **kw): + if path == "/proc/mounts": + return FakeFile() + return original_open(path, *a, **kw) + builtins.open = fake_open + try: + self.assertTrue(_mount_registered("/mnt/fake/subdir/deep")) + self.assertTrue(_mount_registered("/mnt/fake")) + self.assertFalse(_mount_registered("/mnt/other")) + finally: + builtins.open = original_open + + +class ProbeStatvfsTest(unittest.TestCase): + """_probe_statvfs returns OK for real accessible paths.""" + + def test_real_path_ok(self): + status, _ = _probe_statvfs("/tmp", timeout=5) + self.assertEqual(status, _FuseStatus.OK) + + def test_nonexistent_path_unknown(self): + status, detail = _probe_statvfs("/no/such/path/xyz", timeout=5) + self.assertEqual(status, _FuseStatus.UNKNOWN) + self.assertTrue(len(detail) > 0) + + def test_timeout_returns_hung(self): + """Monkey-patch os.statvfs to block; probe should return HUNG after timeout.""" + original = os.statvfs + barrier = threading.Event() + def _blocking(path): + barrier.wait(10) + return original(path) + os.statvfs = _blocking + try: + t0 = time.monotonic() + status, _ = _probe_statvfs("/tmp", timeout=1) + elapsed = time.monotonic() - t0 + self.assertEqual(status, _FuseStatus.HUNG) + self.assertLess(elapsed, 5) + finally: + os.statvfs = original + barrier.set() + + +class StrikeCounterTest(unittest.TestCase): + """_record_fuse_result increments / resets the strike counter correctly.""" + + def setUp(self): + _fuse_strikes.reset() + + def test_ok_resets_strikes(self): + _fuse_strikes.value = 3 + strikes, act = _record_fuse_result(_FuseStatus.OK) + self.assertEqual(strikes, 0) + self.assertFalse(act) + self.assertEqual(_fuse_strikes.value, 0) + + def test_empty_resets_strikes(self): + _fuse_strikes.value = 2 + strikes, act = _record_fuse_result(_FuseStatus.EMPTY) + self.assertEqual(strikes, 0) + self.assertFalse(act) + + def test_dead_increments_strikes(self): + s, a = _record_fuse_result(_FuseStatus.DEAD) + self.assertEqual(s, 1) + self.assertFalse(a) # default DECY_FUSE_STRIKES=2, 1 hit is not enough + + def test_consecutive_dead_triggers_action(self): + from doctor.config import DECY_FUSE_STRIKES + for i in range(DECY_FUSE_STRIKES - 1): + strikes, act = _record_fuse_result(_FuseStatus.DEAD) + self.assertFalse(act, "should not act on strike %d/%d" % (i + 1, DECY_FUSE_STRIKES)) + strikes, act = _record_fuse_result(_FuseStatus.DEAD) + self.assertTrue(act, "should act after %d consecutive failures" % DECY_FUSE_STRIKES) + + def test_reset_between_failures_prevents_action(self): + _record_fuse_result(_FuseStatus.DEAD) # strike 1 + _record_fuse_result(_FuseStatus.OK) # reset + strikes, act = _record_fuse_result(_FuseStatus.DEAD) # strike 1 again + self.assertEqual(strikes, 1) + self.assertFalse(act) + + def test_hung_also_increments(self): + s, _ = _record_fuse_result(_FuseStatus.HUNG) + self.assertEqual(s, 1) + + def test_unknown_also_increments(self): + s, _ = _record_fuse_result(_FuseStatus.UNKNOWN) + self.assertEqual(s, 1) + + def test_unmounted_also_increments(self): + s, _ = _record_fuse_result(_FuseStatus.UNMOUNTED) + self.assertEqual(s, 1) + + +class ProbeMountTest(unittest.TestCase): + """_probe_mount integration tests using real local filesystem.""" + + def setUp(self): + _fuse_strikes.reset() + + def test_nonexistent_path_unmounted(self): + # Completely invented path with no matching FUSE ancestor -> UNMOUNTED + status, detail = _probe_mount("/no/such/mount/point/xyz/abc", read_timeout=5) + self.assertEqual(status, _FuseStatus.UNMOUNTED) + + def test_statvfs_layer_works_on_tmp(self): + status, detail = _probe_statvfs("/tmp", timeout=5) + self.assertEqual(status, _FuseStatus.OK, detail) + + def test_read_layer_with_real_file(self): + """Layer 3: create a real .mkv, confirm _read_file returns OK.""" + from doctor.checks.decypharr import _read_file + with tempfile.NamedTemporaryFile(suffix=".mkv", delete=False) as fh: + fh.write(b"\x00" * 65536) + fpath = fh.name + try: + status, detail = _read_file(fpath, timeout=5) + self.assertEqual(status, _FuseStatus.OK, detail) + finally: + os.unlink(fpath) + + def test_read_layer_nonexistent_file_unknown(self): + from doctor.checks.decypharr import _read_file + status, detail = _read_file("/no/such/file.mkv", timeout=5) + self.assertEqual(status, _FuseStatus.UNKNOWN) + + + + +class ParseLogTsTest(unittest.TestCase): + """_parse_log_ts extracts unix timestamps from decypharr log lines.""" + + def test_plain_timestamp(self): + from doctor.checks.decypharr import _parse_log_ts + import datetime + line = "2026-07-01 00:22:18 | ERROR | [webdav] Error streaming file: foo.mkv" + ts = _parse_log_ts(line) + self.assertIsNotNone(ts) + dt = datetime.datetime.fromtimestamp(ts) + self.assertEqual(dt.year, 2026) + self.assertEqual(dt.month, 7) + self.assertEqual(dt.day, 1) + self.assertEqual(dt.hour, 0) + self.assertEqual(dt.minute, 22) + + def test_ansi_prefixed_timestamp(self): + from doctor.checks.decypharr import _parse_log_ts + line = "2026-06-30 15:54:13 | ERROR | [webdav] Error streaming file" + ts = _parse_log_ts(line) + self.assertIsNotNone(ts) + + def test_no_timestamp_returns_none(self): + from doctor.checks.decypharr import _parse_log_ts + self.assertIsNone(_parse_log_ts("no timestamp here")) + self.assertIsNone(_parse_log_ts("")) + + +class CountLinkErrorsTest(unittest.TestCase): + """_count_link_errors_in_window counts matching errors within the window.""" + + def _make_line(self, offset_secs, error_code="read_pxy_timeout"): + """Return a log line whose timestamp is now - offset_secs.""" + import datetime + ts = datetime.datetime.fromtimestamp(time.time() - offset_secs) + ts_str = ts.strftime("%Y-%m-%d %H:%M:%S") + return ( + '%s | ERROR | [webdav] Error streaming file: Show/Episode.mkv ' + 'error="failed to get download link: %s: unknown error code: %s"' + % (ts_str, error_code, error_code) + ) + + def test_empty_log(self): + from doctor.checks.decypharr import _count_link_errors_in_window + count, _ = _count_link_errors_in_window("", 600) + self.assertEqual(count, 0) + + def test_no_matching_lines(self): + from doctor.checks.decypharr import _count_link_errors_in_window + log = "2026-07-01 00:00:00 | INFO | [manager] all good\n" + count, _ = _count_link_errors_in_window(log, 600) + self.assertEqual(count, 0) + + def test_recent_errors_counted(self): + from doctor.checks.decypharr import _count_link_errors_in_window + lines = "\n".join(self._make_line(i * 30) for i in range(5)) + count, newest = _count_link_errors_in_window(lines, 600) + self.assertEqual(count, 5) + self.assertIsNotNone(newest) + + def test_old_errors_excluded(self): + from doctor.checks.decypharr import _count_link_errors_in_window + # All lines are older than the window + lines = "\n".join(self._make_line(700 + i * 10) for i in range(5)) + count, _ = _count_link_errors_in_window(lines, 600) + self.assertEqual(count, 0) + + def test_mixed_age_only_recent_counted(self): + from doctor.checks.decypharr import _count_link_errors_in_window + recent = [self._make_line(60), self._make_line(120)] + old = [self._make_line(900), self._make_line(1200)] + lines = "\n".join(recent + old) + count, _ = _count_link_errors_in_window(lines, 600) + self.assertEqual(count, 2) + + def test_hoster_timeout_pattern(self): + from doctor.checks.decypharr import _count_link_errors_in_window + line = self._make_line(60, error_code="hoster_timeout") + count, _ = _count_link_errors_in_window(line, 600) + self.assertEqual(count, 1) + + def test_unknown_error_code_pattern(self): + from doctor.checks.decypharr import _count_link_errors_in_window + import datetime + ts = datetime.datetime.fromtimestamp(time.time() - 30).strftime("%Y-%m-%d %H:%M:%S") + line = ('%s | ERROR | [webdav] Error streaming file: foo error=' + '"failed to get download link: xyz: unknown error code: xyz"' % ts) + count, _ = _count_link_errors_in_window(line, 600) + self.assertEqual(count, 1) + + def test_hoster_unavailable_pattern(self): + from doctor.checks.decypharr import _count_link_errors_in_window + line = self._make_line(60, error_code="hoster_unavailable") + count, _ = _count_link_errors_in_window(line, 600) + self.assertEqual(count, 1) + + def test_ansi_coloured_line(self): + from doctor.checks.decypharr import _count_link_errors_in_window + import datetime + ts = datetime.datetime.fromtimestamp(time.time() - 10).strftime("%Y-%m-%d %H:%M:%S") + line = ( + "%s | ERROR | [webdav] Error streaming file: foo " + 'error="failed to get download link: read_pxy_timeout: unknown error code: read_pxy_timeout"' + % ts + ) + count, _ = _count_link_errors_in_window(line, 600) + self.assertEqual(count, 1) + + +class CheckLinkErrorsTest(unittest.TestCase): + """check_link_errors() integration: patching log source and restart hook.""" + + def _make_line(self, offset_secs): + import datetime + ts = datetime.datetime.fromtimestamp(time.time() - offset_secs).strftime("%Y-%m-%d %H:%M:%S") + return ( + '%s | ERROR | [webdav] Error streaming file: Show/Ep.mkv ' + 'error="failed to get download link: read_pxy_timeout: unknown error code: read_pxy_timeout"' + % ts + ) + + def setUp(self): + import doctor.checks.decypharr as m + self._orig_read = m._read_decy_log + self._orig_restart_ts = m._link_err_last_restart.value + m._link_err_last_restart.value = 0.0 # reset cooldown + + def tearDown(self): + import doctor.checks.decypharr as m + m._read_decy_log = self._orig_read + m._link_err_last_restart.value = self._orig_restart_ts + + def _patch_log(self, lines): + import doctor.checks.decypharr as m + m._read_decy_log = lambda: lines + + def test_below_threshold_no_restart(self): + import doctor.checks.decypharr as m + import doctor.config as cfg + orig_log_cmd = m.DECY_LINK_ERR_LOG_CMD + try: + m.DECY_LINK_ERR_LOG_CMD = "notempty" # pass the guard + # Provide fewer errors than the threshold + below = "\n".join(self._make_line(i * 10) for i in range(max(1, cfg.DECY_LINK_ERR_THRESHOLD - 1))) + self._patch_log(below) + called = [] + orig_run_cmd = m.run_cmd + m.run_cmd = lambda cmd: called.append(cmd) or (0, "ok") + result = m.check_link_errors() + self.assertFalse(result) + self.assertEqual(called, []) + finally: + m.DECY_LINK_ERR_LOG_CMD = orig_log_cmd + m.run_cmd = orig_run_cmd + + def test_above_threshold_triggers_restart(self): + import doctor.checks.decypharr as m + import doctor.config as cfg + orig_restart_cmd = m.DECY_RESTART_CMD + orig_dry_run = m.DRY_RUN + orig_log_cmd = m.DECY_LINK_ERR_LOG_CMD + orig_restart = m.DECY_LINK_ERR_RESTART + try: + m.DECY_RESTART_CMD = "echo restart" + m.DRY_RUN = False + m.DECY_LINK_ERR_LOG_CMD = "notempty" # any truthy value passes the guard + m.DECY_LINK_ERR_RESTART = True + lines = "\n".join(self._make_line(i * 10) for i in range(cfg.DECY_LINK_ERR_THRESHOLD + 5)) + self._patch_log(lines) + called = [] + orig_run_cmd = m.run_cmd + m.run_cmd = lambda cmd: called.append(cmd) or (0, "ok") + result = m.check_link_errors() + self.assertTrue(result) + self.assertEqual(len(called), 1) + self.assertIn("restart", called[0]) + finally: + m.DECY_RESTART_CMD = orig_restart_cmd + m.DRY_RUN = orig_dry_run + m.DECY_LINK_ERR_LOG_CMD = orig_log_cmd + m.DECY_LINK_ERR_RESTART = orig_restart + m.run_cmd = orig_run_cmd + + def test_dry_run_no_restart(self): + import doctor.checks.decypharr as m + import doctor.config as cfg + orig_restart_cmd = m.DECY_RESTART_CMD + orig_dry_run = m.DRY_RUN + orig_log_cmd = m.DECY_LINK_ERR_LOG_CMD + orig_restart = m.DECY_LINK_ERR_RESTART + try: + m.DECY_RESTART_CMD = "echo restart" + m.DRY_RUN = True + m.DECY_LINK_ERR_LOG_CMD = "notempty" + m.DECY_LINK_ERR_RESTART = True + lines = "\n".join(self._make_line(i * 10) for i in range(cfg.DECY_LINK_ERR_THRESHOLD + 5)) + self._patch_log(lines) + called = [] + orig_run_cmd = m.run_cmd + m.run_cmd = lambda cmd: called.append(cmd) or (0, "ok") + result = m.check_link_errors() + self.assertFalse(result) + self.assertEqual(called, []) + finally: + m.DECY_RESTART_CMD = orig_restart_cmd + m.DRY_RUN = orig_dry_run + m.DECY_LINK_ERR_LOG_CMD = orig_log_cmd + m.DECY_LINK_ERR_RESTART = orig_restart + m.run_cmd = orig_run_cmd + + def test_cooldown_prevents_second_restart(self): + import doctor.checks.decypharr as m + import doctor.config as cfg + orig_restart_cmd = m.DECY_RESTART_CMD + orig_dry_run = m.DRY_RUN + orig_log_cmd = m.DECY_LINK_ERR_LOG_CMD + orig_restart = m.DECY_LINK_ERR_RESTART + try: + m.DECY_RESTART_CMD = "echo restart" + m.DRY_RUN = False + m.DECY_LINK_ERR_LOG_CMD = "notempty" + m.DECY_LINK_ERR_RESTART = True + m._link_err_last_restart.value = time.time() # simulate recent restart + lines = "\n".join(self._make_line(i * 10) for i in range(cfg.DECY_LINK_ERR_THRESHOLD + 5)) + self._patch_log(lines) + called = [] + orig_run_cmd = m.run_cmd + m.run_cmd = lambda cmd: called.append(cmd) or (0, "ok") + result = m.check_link_errors() + self.assertFalse(result) + self.assertEqual(called, []) + finally: + m.DECY_RESTART_CMD = orig_restart_cmd + m.DRY_RUN = orig_dry_run + m.DECY_LINK_ERR_LOG_CMD = orig_log_cmd + m.DECY_LINK_ERR_RESTART = orig_restart + m.run_cmd = orig_run_cmd + + def test_no_log_source_skips(self): + import doctor.checks.decypharr as m + orig_log_cmd = m.DECY_LINK_ERR_LOG_CMD + orig_jan_cmd = m.JAN_LOG_CMD + orig_jan_log = m.JAN_LOG + try: + m.DECY_LINK_ERR_LOG_CMD = "" + m.JAN_LOG_CMD = "" + m.JAN_LOG = "" + result = m.check_link_errors() + self.assertFalse(result) + finally: + m.DECY_LINK_ERR_LOG_CMD = orig_log_cmd + m.JAN_LOG_CMD = orig_jan_cmd + m.JAN_LOG = orig_jan_log +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_helpers.py b/tests/test_helpers.py new file mode 100644 index 0000000..9a48d96 --- /dev/null +++ b/tests/test_helpers.py @@ -0,0 +1,123 @@ +"""Unit tests for pure helpers that have no external dependencies.""" +import os +import tempfile +import unittest +from datetime import datetime, timezone, timedelta + +from doctor.config import _dur, _human +from doctor.checks.queue import stuck_reason, _msgs +from doctor.checks.missing_seasons import _season_still_airing +from doctor.checks.repair import _dead_symlink + + +class DurationParsingTest(unittest.TestCase): + def test_dur_seconds(self): + self.assertEqual(_dur("30s"), 30) + + def test_dur_minutes(self): + self.assertEqual(_dur("5m"), 300) + + def test_dur_hours(self): + self.assertEqual(_dur("2h"), 7200) + + def test_dur_days(self): + self.assertEqual(_dur("1d"), 86400) + + def test_dur_bare_number(self): + self.assertEqual(_dur("900"), 900) + + def test_dur_empty_uses_default(self): + self.assertEqual(_dur(""), 0) + self.assertEqual(_dur("garbage", 42), 42) + + +class HumanReadableTest(unittest.TestCase): + def test_human_seconds(self): + self.assertEqual(_human(45), "45s") + + def test_human_minutes(self): + self.assertEqual(_human(180), "3m") + + def test_human_hours(self): + self.assertEqual(_human(7200), "2h") + + def test_human_days(self): + self.assertEqual(_human(86400), "1d") + + +class QueuePredicateTest(unittest.TestCase): + def test_stuck_reason_download_client_unavailable(self): + self.assertEqual(stuck_reason({"status": "downloadClientUnavailable"}), + "downloadClientUnavailable") + + def test_stuck_reason_import_blocked(self): + rec = {"trackedDownloadState": "importBlocked"} + self.assertEqual(stuck_reason(rec), "importBlocked") + + def test_stuck_reason_import_failed(self): + rec = {"trackedDownloadState": "importFailed"} + self.assertEqual(stuck_reason(rec), "importFailed") + + def test_stuck_reason_import_pending_warning(self): + rec = {"trackedDownloadState": "importPending", + "trackedDownloadStatus": "warning"} + self.assertEqual(stuck_reason(rec), "importPending_warning") + + def test_stuck_reason_stalled(self): + rec = {"trackedDownloadStatus": "warning", + "statusMessages": [{"messages": ["download is stalled"]}]} + self.assertEqual(stuck_reason(rec), "stalled") + + def test_stuck_reason_no_match(self): + self.assertIsNone(stuck_reason({"status": "ok"})) + + def test_msgs_extracts_messages(self): + rec = { + "statusMessages": [{"messages": ["m1", "m2"]}, {"messages": ["m3"]}], + "errorMessage": "top" + } + self.assertEqual(_msgs(rec), ["m1", "m2", "m3", "top"]) + + +class SeasonAiringTest(unittest.TestCase): + def test_still_airing_future_episode(self): + future = (datetime.now(timezone.utc) + timedelta(days=1)).isoformat() + eps = [{"seasonNumber": 1, "airDateUtc": future}] + self.assertTrue(_season_still_airing(eps, 1)) + + def test_not_airing_all_past(self): + past = (datetime.now(timezone.utc) - timedelta(days=1)).isoformat() + eps = [{"seasonNumber": 2, "airDateUtc": past}] + self.assertFalse(_season_still_airing(eps, 2)) + + def test_wrong_season_ignored(self): + future = (datetime.now(timezone.utc) + timedelta(days=1)).isoformat() + eps = [{"seasonNumber": 1, "airDateUtc": future}] + self.assertFalse(_season_still_airing(eps, 2)) + + +class DeadSymlinkTest(unittest.TestCase): + def test_dead_symlink_detected(self): + with tempfile.TemporaryDirectory() as d: + target = os.path.join(d, "missing") + link = os.path.join(d, "link") + os.symlink(target, link) + self.assertTrue(_dead_symlink(link)) + + def test_live_symlink_not_dead(self): + with tempfile.TemporaryDirectory() as d: + target = os.path.join(d, "real") + open(target, "w").close() + link = os.path.join(d, "link") + os.symlink(target, link) + self.assertFalse(_dead_symlink(link)) + + def test_regular_file_not_dead(self): + with tempfile.TemporaryDirectory() as d: + f = os.path.join(d, "file") + open(f, "w").close() + self.assertFalse(_dead_symlink(f)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_janitor.py b/tests/test_janitor.py new file mode 100644 index 0000000..476408c --- /dev/null +++ b/tests/test_janitor.py @@ -0,0 +1,450 @@ +"""Characterization tests for doctor.checks.janitor. + +Lock in the current behavior of: + - _scan_operational_errors(): regex matching on log text + - _read_log_tail(): file read and command fallback + - _jan_alert(): throttled alerting with cooldown + - _probe_decy_api(): HTTP health probing with throttled alerts + - _JAN_OP_PATTERNS: built-in pattern matching (panic, rate-limit, etc.) + - _JAN_USER_PATTERNS: user-configurable extra patterns with word-boundary wrapping + +All I/O is mocked or uses temp files. Config globals are patched on the +janitor module directly (they arrive via star-import). +""" +import os +import tempfile +import time +import unittest +from unittest.mock import patch, MagicMock + +from doctor.checks.janitor import ( + _scan_operational_errors, + _read_log_tail, + _jan_alert, + _jan_alert_last, + _probe_decy_api, +) + +_MOD = "doctor.checks.janitor" + + +# --------------------------------------------------------------------------- +# _scan_operational_errors +# --------------------------------------------------------------------------- + +class ScanOperationalErrorsTest(unittest.TestCase): + """Characterize _scan_operational_errors: regex matching and counting.""" + + def test_empty_log_returns_empty(self): + self.assertEqual(_scan_operational_errors(""), {}) + + def test_panic_detected(self): + data = "2024-01-01 some panic happened here\n" + counts = _scan_operational_errors(data) + self.assertIn("panic/fatal", counts) + self.assertEqual(counts["panic/fatal"], 1) + + def test_fatal_detected(self): + data = "goroutine 1: fatal error\n" + counts = _scan_operational_errors(data) + self.assertIn("panic/fatal", counts) + + def test_runtime_error_detected(self): + data = "runtime error: index out of range\n" + counts = _scan_operational_errors(data) + self.assertIn("panic/fatal", counts) + + def test_rate_limit_detected(self): + data = "API returned rate limit exceeded\n" + counts = _scan_operational_errors(data) + self.assertIn("rate-limit", counts) + + def test_rate_limited_detected(self): + data = "provider rate limited us\n" + counts = _scan_operational_errors(data) + self.assertIn("rate-limit", counts) + + def test_too_many_requests_detected(self): + data = "HTTP 429 too many requests\n" + counts = _scan_operational_errors(data) + # "429" matches rate-limit; "too many requests" also matches rate-limit + # The line matches the first pattern that hits + self.assertIn("rate-limit", counts) + + def test_429_as_word_detected(self): + data = "server returned 429\n" + counts = _scan_operational_errors(data) + self.assertIn("rate-limit", counts) + + def test_cloudflare_detected(self): + data = "blocked by cloudflare challenge\n" + counts = _scan_operational_errors(data) + self.assertIn("cloudflare/blocked", counts) + + def test_403_as_word_detected(self): + data = "server returned 403 forbidden\n" + counts = _scan_operational_errors(data) + self.assertIn("cloudflare/blocked", counts) + + def test_unauthorized_detected(self): + data = "request returned unauthorized\n" + counts = _scan_operational_errors(data) + self.assertIn("auth", counts) + + def test_401_as_word_detected(self): + data = "HTTP 401 from server\n" + counts = _scan_operational_errors(data) + self.assertIn("auth", counts) + + def test_token_expired_detected(self): + data = "token expired, renewing\n" + counts = _scan_operational_errors(data) + self.assertIn("auth", counts) + + def test_timeout_detected(self): + data = "context deadline exceeded while fetching\n" + counts = _scan_operational_errors(data) + self.assertIn("network/timeout", counts) + + def test_connection_refused_detected(self): + data = "dial tcp: connection refused\n" + counts = _scan_operational_errors(data) + self.assertIn("network/timeout", counts) + + def test_io_timeout_detected(self): + data = "i/o timeout reading body\n" + counts = _scan_operational_errors(data) + self.assertIn("network/timeout", counts) + + def test_line_counted_only_once(self): + """A line matching multiple categories should only be counted under the first match.""" + # "panic" matches panic/fatal; if it also contained "timeout", only panic/fatal should count + data = "panic: context deadline exceeded\n" + counts = _scan_operational_errors(data) + # Should only have panic/fatal + self.assertEqual(counts.get("panic/fatal", 0), 1) + total = sum(counts.values()) + self.assertEqual(total, 1, "Line should be counted exactly once") + + def test_multiple_lines_accumulate(self): + data = "panic error\npanic again\nrate limit hit\n" + counts = _scan_operational_errors(data) + self.assertEqual(counts.get("panic/fatal", 0), 2) + self.assertEqual(counts.get("rate-limit", 0), 1) + + def test_case_insensitive(self): + data = "PANIC in goroutine\n" + counts = _scan_operational_errors(data) + self.assertIn("panic/fatal", counts) + + def test_clean_log_no_matches(self): + data = "INFO: everything is fine\nDEBUG: all good\n" + self.assertEqual(_scan_operational_errors(data), {}) + + def test_hash_ids_not_false_positive(self): + """Hex hashes and alldebrid IDs containing '401' or '403' should NOT match + because patterns use word boundaries.""" + data = "downloading hash=a401b9f3c2 from provider\n" + counts = _scan_operational_errors(data) + # "401" is embedded in a hex string -> word boundary should prevent match + self.assertEqual(counts.get("auth", 0), 0) + + def test_false_positive_suppression_429_in_hash(self): + """'429' embedded in a hash should not match rate-limit.""" + data = "item id=abc429def status=ok\n" + counts = _scan_operational_errors(data) + self.assertEqual(counts.get("rate-limit", 0), 0) + + +# --------------------------------------------------------------------------- +# _read_log_tail +# --------------------------------------------------------------------------- + +class ReadLogTailTest(unittest.TestCase): + + @patch(_MOD + ".JAN_LOG_CMD", "") + @patch(_MOD + ".JAN_LOG", "") + def test_returns_none_when_no_source_configured(self): + self.assertIsNone(_read_log_tail()) + + @patch(_MOD + ".JAN_LOG_CMD", "") + def test_reads_from_file(self): + with tempfile.NamedTemporaryFile(mode="w", suffix=".log", delete=False) as f: + f.write("line 1\nline 2\nline 3\n") + f.flush() + fname = f.name + try: + with patch(_MOD + ".JAN_LOG", fname): + data = _read_log_tail() + self.assertIn("line 1", data) + self.assertIn("line 3", data) + finally: + os.unlink(fname) + + @patch(_MOD + ".JAN_LOG_CMD", "") + def test_reads_tail_of_large_file(self): + """For files > 2MB, only the last ~2MB should be returned.""" + with tempfile.NamedTemporaryFile(mode="w", suffix=".log", delete=False) as f: + # Write 3MB of data + chunk = "x" * 1000 + "\n" + for _ in range(3000): + f.write(chunk) + f.write("TAIL_MARKER\n") + f.flush() + fname = f.name + try: + with patch(_MOD + ".JAN_LOG", fname): + data = _read_log_tail() + self.assertIn("TAIL_MARKER", data) + # Should be approximately 2MB, not 3MB + self.assertLess(len(data), 2_100_000) + finally: + os.unlink(fname) + + @patch(_MOD + ".run_output", return_value="cmd output here") + @patch(_MOD + ".JAN_LOG_CMD", "some command") + def test_reads_from_command_when_configured(self, mock_run): + data = _read_log_tail() + self.assertEqual(data, "cmd output here") + mock_run.assert_called_once_with("some command") + + @patch(_MOD + ".run_output", return_value="cmd output") + @patch(_MOD + ".JAN_LOG_CMD", "some command") + @patch(_MOD + ".JAN_LOG", "/some/file.log") + def test_command_takes_priority_over_file(self, mock_run): + """When both JAN_LOG_CMD and JAN_LOG are set, command takes priority.""" + data = _read_log_tail() + self.assertEqual(data, "cmd output") + + @patch(_MOD + ".JAN_LOG_CMD", "") + def test_returns_none_for_nonexistent_file(self): + with patch(_MOD + ".JAN_LOG", "/nonexistent/file.log"): + self.assertIsNone(_read_log_tail()) + + +# --------------------------------------------------------------------------- +# _jan_alert (throttled alerting) +# --------------------------------------------------------------------------- + +class JanAlertTest(unittest.TestCase): + + def setUp(self): + # Save and clear the throttle state + self._saved = dict(_jan_alert_last) + _jan_alert_last.clear() + + def tearDown(self): + _jan_alert_last.clear() + _jan_alert_last.update(self._saved) + + @patch(_MOD + ".JAN_ALERT_COOLDOWN", 300) + @patch(_MOD + ".log") + def test_first_alert_fires(self, mock_log): + _jan_alert("test_key", "message %s", "arg1") + mock_log.warning.assert_called_once_with("message %s", "arg1") + + @patch(_MOD + ".JAN_ALERT_COOLDOWN", 300) + @patch(_MOD + ".log") + def test_second_alert_within_cooldown_suppressed(self, mock_log): + _jan_alert("test_key", "first") + mock_log.warning.reset_mock() + _jan_alert("test_key", "second") + mock_log.warning.assert_not_called() + + @patch(_MOD + ".JAN_ALERT_COOLDOWN", 0) + @patch(_MOD + ".log") + def test_zero_cooldown_always_fires(self, mock_log): + _jan_alert("test_key", "first") + _jan_alert("test_key", "second") + self.assertEqual(mock_log.warning.call_count, 2) + + @patch(_MOD + ".JAN_ALERT_COOLDOWN", 300) + @patch(_MOD + ".log") + def test_different_keys_not_throttled(self, mock_log): + _jan_alert("key_a", "msg a") + _jan_alert("key_b", "msg b") + self.assertEqual(mock_log.warning.call_count, 2) + + @patch(_MOD + ".JAN_ALERT_COOLDOWN", 1) + @patch(_MOD + ".log") + def test_alert_fires_after_cooldown_expires(self, mock_log): + _jan_alert("test_key", "first") + # Fake expiry by backdating the timestamp + _jan_alert_last["test_key"] = time.time() - 2 + _jan_alert("test_key", "second") + self.assertEqual(mock_log.warning.call_count, 2) + + +# --------------------------------------------------------------------------- +# _probe_decy_api +# --------------------------------------------------------------------------- + +class ProbeDecyApiTest(unittest.TestCase): + + def setUp(self): + self._saved = dict(_jan_alert_last) + _jan_alert_last.clear() + + def tearDown(self): + _jan_alert_last.clear() + _jan_alert_last.update(self._saved) + + @patch(_MOD + ".DECY_URL", "") + @patch(_MOD + ".http_code") + def test_noop_when_no_url(self, mock_http): + _probe_decy_api() + mock_http.assert_not_called() + + @patch(_MOD + ".DECY_URL", "http://decy:8282") + @patch(_MOD + ".http_code", return_value=200) + @patch(_MOD + ".log") + def test_ok_logs_debug_only(self, mock_log, mock_http): + _probe_decy_api() + # Should call http_code for both "" and "/api/status" paths + self.assertEqual(mock_http.call_count, 2) + # No warning for 200 + mock_log.warning.assert_not_called() + + @patch(_MOD + ".JAN_ALERT_COOLDOWN", 0) + @patch(_MOD + ".DECY_URL", "http://decy:8282") + @patch(_MOD + ".http_code", return_value=500) + @patch(_MOD + ".log") + def test_500_triggers_alert(self, mock_log, mock_http): + _probe_decy_api() + # Should log warnings for 500 + self.assertTrue(mock_log.warning.called) + + @patch(_MOD + ".JAN_ALERT_COOLDOWN", 0) + @patch(_MOD + ".DECY_URL", "http://decy:8282") + @patch(_MOD + ".http_code", return_value=401) + @patch(_MOD + ".log") + def test_401_triggers_auth_alert(self, mock_log, mock_http): + _probe_decy_api() + self.assertTrue(mock_log.warning.called) + + @patch(_MOD + ".JAN_ALERT_COOLDOWN", 0) + @patch(_MOD + ".DECY_URL", "http://decy:8282") + @patch(_MOD + ".http_code", return_value=0) + @patch(_MOD + ".log") + def test_zero_code_triggers_unreachable_alert(self, mock_log, mock_http): + _probe_decy_api() + self.assertTrue(mock_log.warning.called) + + @patch(_MOD + ".DECY_URL", "http://decy:8282") + @patch(_MOD + ".http_code", side_effect=Exception("DNS failure")) + @patch(_MOD + ".JAN_ALERT_COOLDOWN", 0) + @patch(_MOD + ".log") + def test_exception_triggers_unreachable_alert(self, mock_log, mock_http): + _probe_decy_api() + self.assertTrue(mock_log.warning.called) + + @patch(_MOD + ".DECY_URL", "http://decy:8282") + @patch(_MOD + ".http_code", return_value=302) + @patch(_MOD + ".log") + def test_unexpected_non_critical_code_logs_debug(self, mock_log, mock_http): + _probe_decy_api() + # 302 is not 2xx, not 5xx, not auth => debug only, no warning + mock_log.warning.assert_not_called() + + +if __name__ == "__main__": + unittest.main() + + +# --------------------------------------------------------------------------- +# File-level quarantine and state recording +# --------------------------------------------------------------------------- + +class ReleaseRelTest(unittest.TestCase): + + def test_under_all(self): + from doctor.checks.janitor import _release_rel + self.assertEqual(_release_rel("/mnt/zurg/__all__/RELEASE/file.mkv"), "RELEASE/file.mkv") + + def test_under_complete(self): + from doctor.checks.janitor import _release_rel + self.assertEqual(_release_rel("/mnt/zurg/complete/RELEASE/file.mkv"), "RELEASE/file.mkv") + + def test_no_match(self): + from doctor.checks.janitor import _release_rel + self.assertIsNone(_release_rel("/some/other/path/file.mkv")) + + +class DeadFileMatchesTest(unittest.TestCase): + + def test_exact_rel_match(self): + from doctor.checks.janitor import _dead_file_matches + self.assertTrue(_dead_file_matches("RELEASE/file.mkv", {"RELEASE/file.mkv": True})) + + def test_basename_match(self): + from doctor.checks.janitor import _dead_file_matches + self.assertTrue(_dead_file_matches("RELEASE/file.mkv", {"file.mkv": True})) + + def test_no_match(self): + from doctor.checks.janitor import _dead_file_matches + self.assertFalse(_dead_file_matches("RELEASE/file.mkv", {"OTHER/file.mkv": True})) + + +class CheckJanitorFileLevelTest(unittest.TestCase): + + def setUp(self): + self._saved_alert = dict(_jan_alert_last) + _jan_alert_last.clear() + + def tearDown(self): + _jan_alert_last.clear() + _jan_alert_last.update(self._saved_alert) + + @patch(_MOD + "._read_log_tail", return_value=""" +[webdav] Error streaming file: Mr.Robot.S01-S04.1080p.BluRay.DD5.1.x264-MIXED/Mr.Robot.S04E13.1080p.BluRay.DTS.x264-SbR.mkv error="marked as bad" +""") + @patch(_MOD + ".JAN_LIBS", ["/lib"]) + @patch(_MOD + ".JAN_QUAR", "/quarantine") + @patch(_MOD + ".DRY_RUN", False) + @patch(_MOD + ".state_transaction") + def test_only_quarantines_specific_dead_file(self, mock_tx, *_): + from doctor.checks.janitor import check_janitor + import os + with tempfile.TemporaryDirectory() as libdir: + os.makedirs(os.path.join(libdir, "shows")) + # Create two symlinks in the same release + good = os.path.join(libdir, "shows", "good.mkv") + bad = os.path.join(libdir, "shows", "bad.mkv") + os.symlink("/mnt/zurg/__all__/Mr.Robot.S01-S04.1080p.BluRay.DD5.1.x264-MIXED/Mr.Robot.S04E12.1080p.BluRay.DTS.x264-SbR.mkv", good) + os.symlink("/mnt/zurg/__all__/Mr.Robot.S01-S04.1080p.BluRay.DD5.1.x264-MIXED/Mr.Robot.S04E13.1080p.BluRay.DTS.x264-SbR.mkv", bad) + + state = {"__janitor_dead_files__": {}} + mock_tx.return_value.__enter__ = lambda s: state + mock_tx.return_value.__exit__ = MagicMock(return_value=False) + with patch(_MOD + ".JAN_LIBS", [os.path.join(libdir, "shows")]): + check_janitor() + + self.assertTrue(os.path.islink(good), "good file should not be quarantined") + self.assertFalse(os.path.exists(bad), "bad file should be quarantined") + self.assertIn("Mr.Robot.S01-S04.1080p.BluRay.DD5.1.x264-MIXED/Mr.Robot.S04E13.1080p.BluRay.DTS.x264-SbR.mkv", state["__janitor_dead_files__"]) + self.assertIsNotNone(state["__janitor_dead_files__"]["Mr.Robot.S01-S04.1080p.BluRay.DD5.1.x264-MIXED/Mr.Robot.S04E13.1080p.BluRay.DTS.x264-SbR.mkv"]["orig"]) + + @patch(_MOD + "._read_log_tail", return_value=""" +[link] Giving up on entry after repeated failed re-insertions attempts=3 filename=Mr.Robot.S04E13.1080p.BluRay.DTS.x264-SbR.mkv infohash=abc name=Mr.Robot.S01-S04.1080p.BluRay.DD5.1.x264-MIXED reason=empty_link +""") + @patch(_MOD + ".JAN_LIBS", ["/lib"]) + @patch(_MOD + ".JAN_QUAR", "/quarantine") + @patch(_MOD + ".DRY_RUN", False) + @patch(_MOD + ".state_transaction") + def test_quarantines_from_link_pattern(self, mock_tx, *_): + from doctor.checks.janitor import check_janitor + import os + with tempfile.TemporaryDirectory() as libdir: + os.makedirs(os.path.join(libdir, "shows")) + fp = os.path.join(libdir, "shows", "file.mkv") + os.symlink("/mnt/zurg/__all__/Mr.Robot.S01-S04.1080p.BluRay.DD5.1.x264-MIXED/Mr.Robot.S04E13.1080p.BluRay.DTS.x264-SbR.mkv", fp) + + state = {"__janitor_dead_files__": {}} + mock_tx.return_value.__enter__ = lambda s: state + mock_tx.return_value.__exit__ = MagicMock(return_value=False) + with patch(_MOD + ".JAN_LIBS", [os.path.join(libdir, "shows")]): + check_janitor() + + self.assertFalse(os.path.exists(fp)) + key = "Mr.Robot.S01-S04.1080p.BluRay.DD5.1.x264-MIXED/Mr.Robot.S04E13.1080p.BluRay.DTS.x264-SbR.mkv" + self.assertIn(key, state["__janitor_dead_files__"]) diff --git a/tests/test_maintainer.py b/tests/test_maintainer.py new file mode 100644 index 0000000..9b010f2 --- /dev/null +++ b/tests/test_maintainer.py @@ -0,0 +1,351 @@ +"""Unit tests for the maintainer check - eligibility logic.""" +import time +import unittest +from datetime import datetime, timedelta, timezone +from unittest.mock import MagicMock, patch + +from doctor.checks.maintainer import (_series_is_eligible, _pulsarr_tagged_show, + _pulsarr_tags, _tag_users) + + +class PulsarrTagsTest(unittest.TestCase): + def test_extracts_all_matching_tags(self): + series = {"tags": [1, 2, 3]} + tag_map = {1: "pulsarr-alice", 2: "pulsarr-bob", 3: "other"} + result = _pulsarr_tags(series, tag_map, "pulsarr-") + self.assertEqual(result, {"pulsarr-alice", "pulsarr-bob"}) + + def test_returns_empty_when_no_match(self): + series = {"tags": [1]} + tag_map = {1: "something-else"} + self.assertEqual(_pulsarr_tags(series, tag_map, "pulsarr-"), set()) + + def test_returns_empty_when_no_tags(self): + self.assertEqual(_pulsarr_tags({"tags": []}, {1: "pulsarr-x"}, "pulsarr-"), set()) + + def test_empty_tag_map(self): + self.assertEqual(_pulsarr_tags({"tags": [1]}, {}, "pulsarr-"), set()) + + +class TagUsersTest(unittest.TestCase): + def test_extracts_username_from_user_tag(self): + self.assertEqual(_tag_users({"pulsarr-user-alice"}, "pulsarr"), {"alice"}) + + def test_multiple_users(self): + self.assertEqual(_tag_users({"pulsarr-user-alice", "pulsarr-user-bob"}, "pulsarr"), + {"alice", "bob"}) + + def test_base_pulsarr_tag_ignored(self): + self.assertEqual(_tag_users({"pulsarr"}, "pulsarr"), set()) + + def test_empty_suffix_handled(self): + self.assertEqual(_tag_users({"pulsarr-user-"}, "pulsarr"), set()) + + def test_empty_labels(self): + self.assertEqual(_tag_users(set(), "pulsarr"), set()) + + +class SeriesEligibilityTest(unittest.TestCase): + def _now(self): + return datetime.now(timezone.utc) + + def _make_series(self, **kw): + defaults = { + "id": 1, "title": "Test Show", "status": "ended", + "monitored": True, "year": 2020, "tags": [1], + "added": "2020-01-15T00:00:00Z", + } + defaults.update(kw) + return defaults + + def test_eligible_ended_old_unwatched_pulsarr_tagged(self): + series = self._make_series() + tag_map = {1: "pulsarr-user-test"} + with patch("doctor.checks.maintainer.MAINTAINER_MIN_YEAR", 2024), \ + patch("doctor.checks.maintainer.MAINTAINER_MIN_AGE_DAYS", 0): + self.assertTrue(_series_is_eligible(series, set(), tag_map, "pulsarr-", self._now(), "tagged")) + + def test_not_eligible_watched_recently(self): + series = self._make_series() + tag_map = {1: "pulsarr-test"} + with patch("doctor.checks.maintainer.MAINTAINER_MIN_YEAR", 2024), \ + patch("doctor.checks.maintainer.MAINTAINER_MIN_AGE_DAYS", 0): + self.assertFalse(_series_is_eligible(series, {"Test Show"}, tag_map, "pulsarr-", self._now(), "tagged")) + + def test_not_eligible_continuing(self): + series = self._make_series(status="continuing") + tag_map = {1: "pulsarr-test"} + with patch("doctor.checks.maintainer.MAINTAINER_MIN_YEAR", 2024), \ + patch("doctor.checks.maintainer.MAINTAINER_MIN_AGE_DAYS", 0): + self.assertFalse(_series_is_eligible(series, set(), tag_map, "pulsarr-", self._now(), "tagged")) + + def test_not_eligible_year_too_recent(self): + series = self._make_series(year=2025) + tag_map = {1: "pulsarr-test"} + with patch("doctor.checks.maintainer.MAINTAINER_MIN_YEAR", 2024), \ + patch("doctor.checks.maintainer.MAINTAINER_MIN_AGE_DAYS", 0): + self.assertFalse(_series_is_eligible(series, set(), tag_map, "pulsarr-", self._now(), "tagged")) + + def test_not_eligible_no_pulsarr_tag(self): + series = self._make_series(tags=[2]) + tag_map = {1: "other", 2: "something-else"} + with patch("doctor.checks.maintainer.MAINTAINER_MIN_YEAR", 2024), \ + patch("doctor.checks.maintainer.MAINTAINER_MIN_AGE_DAYS", 0): + self.assertFalse(_series_is_eligible(series, set(), tag_map, "pulsarr-", self._now(), "tagged")) + + def test_not_eligible_no_tags(self): + series = self._make_series(tags=[]) + tag_map = {1: "pulsarr-test"} + with patch("doctor.checks.maintainer.MAINTAINER_MIN_YEAR", 2024), \ + patch("doctor.checks.maintainer.MAINTAINER_MIN_AGE_DAYS", 0): + self.assertFalse(_series_is_eligible(series, set(), tag_map, "pulsarr-", self._now(), "tagged")) + + def test_eligible_year_exactly_at_threshold(self): + series = self._make_series(year=2023) + tag_map = {1: "pulsarr-test"} + with patch("doctor.checks.maintainer.MAINTAINER_MIN_YEAR", 2024), \ + patch("doctor.checks.maintainer.MAINTAINER_MIN_AGE_DAYS", 0): + self.assertTrue(_series_is_eligible(series, set(), tag_map, "pulsarr-", self._now(), "tagged")) + + def test_not_eligible_added_recently(self): + series = self._make_series(added=(datetime.now(timezone.utc) - timedelta(days=5)).isoformat()) + tag_map = {1: "pulsarr-test"} + with patch("doctor.checks.maintainer.MAINTAINER_MIN_YEAR", 2024), \ + patch("doctor.checks.maintainer.MAINTAINER_MIN_AGE_DAYS", 30): + self.assertFalse(_series_is_eligible(series, set(), tag_map, "pulsarr-", self._now(), "tagged")) + + def test_eligible_added_long_ago(self): + series = self._make_series(added="2020-01-01T00:00:00Z") + tag_map = {1: "pulsarr-test"} + with patch("doctor.checks.maintainer.MAINTAINER_MIN_YEAR", 2024), \ + patch("doctor.checks.maintainer.MAINTAINER_MIN_AGE_DAYS", 30): + self.assertTrue(_series_is_eligible(series, set(), tag_map, "pulsarr-", self._now(), "tagged")) + + def test_unparseable_added_date_let_through(self): + series = self._make_series(added="garbage-date") + tag_map = {1: "pulsarr-test"} + with patch("doctor.checks.maintainer.MAINTAINER_MIN_YEAR", 2024), \ + patch("doctor.checks.maintainer.MAINTAINER_MIN_AGE_DAYS", 30): + self.assertTrue(_series_is_eligible(series, set(), tag_map, "pulsarr-", self._now(), "tagged")) + + def test_missing_added_date_let_through(self): + series = self._make_series() + del series["added"] + tag_map = {1: "pulsarr-test"} + with patch("doctor.checks.maintainer.MAINTAINER_MIN_YEAR", 2024), \ + patch("doctor.checks.maintainer.MAINTAINER_MIN_AGE_DAYS", 30): + self.assertTrue(_series_is_eligible(series, set(), tag_map, "pulsarr-", self._now(), "tagged")) + + +class CheckMaintainerIntegrationTest(unittest.TestCase): + """Integration test for check_maintainer with mocked INSTANCES and API clients.""" + + @staticmethod + def _make_sonarr_instance(name="sonarr-shows", series_list=None): + arr = MagicMock() + arr.name = name + arr.kind = "sonarr" + arr.tag_map.return_value = {1: "pulsarr-user-plexuser"} + arr.series.return_value = series_list or [] + return arr + + @classmethod + def _make_series(cls, sid=1, title="Old Show", status="ended", year=2020, + monitored=True, tags=None): + return { + "id": sid, "title": title, "status": status, + "monitored": monitored, "year": year, + "tags": tags if tags is not None else [1], + "tvdbId": sid * 100, + "added": "2020-01-15T00:00:00Z", + } + + def test_dry_run_logs_but_does_not_delete(self): + series = [self._make_series()] + arr = self._make_sonarr_instance(series_list=series) + + with patch("doctor.checks.maintainer.EN_MAINTAINER", True), \ + patch("doctor.checks.maintainer.TAUTULLI_URL", "http://taut:8181"), \ + patch("doctor.checks.maintainer.TAUTULLI_APIKEY", "k"), \ + patch("doctor.checks.maintainer.DRY_RUN", True), \ + patch("doctor.checks.maintainer.MAINTAINER_MIN_YEAR", 2024), \ + patch("doctor.checks.maintainer.MAINTAINER_MAX_ACTIONS", 5), \ + patch("doctor.checks.maintainer.MAINTAINER_LIBRARY_TITLE", "shows"), \ + patch("doctor.checks.maintainer.MAINTAINER_UNWATCHED_DAYS", 30), \ + patch("doctor.checks.maintainer.MAINTAINER_MIN_AGE_DAYS", 0), \ + patch("doctor.checks.maintainer.MAINTAINER_PULSARR_TAG_PREFIX", "pulsarr"), \ + patch("doctor.checks.maintainer.MAINTAINER_RECHECK", 86400), \ + patch("doctor.checks.maintainer.MAINTAINER_MODE", "tagged"), \ + patch("doctor.checks.maintainer.MAINTAINER_PLEX_SECTION_KEY", 0), \ + patch("doctor.checks.maintainer.PULSARR_URL", ""), \ + patch("doctor.checks.maintainer.PULSARR_APIKEY", ""), \ + patch("doctor.checks.maintainer.INSTANCES", [arr]), \ + patch("doctor.clients.tautulli.Tautulli.recently_watched_shows", + return_value=set()) as _mock_taut, \ + patch("doctor.checks.maintainer.log") as mock_log: + from doctor.checks.maintainer import check_maintainer + with patch("builtins.open"): # state file mock + check_maintainer() + + # Should NOT call DELETE + arr._req.assert_not_called() + # Should log WOULD delete + log_calls = [c[0][0] for c in mock_log.info.call_args_list if c[0]] + dry_call = [m for m in log_calls if isinstance(m, str) and "WOULD delete" in m] + self.assertTrue(len(dry_call) > 0) + + def test_non_matching_library_skipped(self): + series = [self._make_series()] + arr = self._make_sonarr_instance(name="sonarr-anime", series_list=series) + + with patch("doctor.checks.maintainer.EN_MAINTAINER", True), \ + patch("doctor.checks.maintainer.TAUTULLI_URL", "http://taut:8181"), \ + patch("doctor.checks.maintainer.TAUTULLI_APIKEY", "k"), \ + patch("doctor.checks.maintainer.DRY_RUN", True), \ + patch("doctor.checks.maintainer.MAINTAINER_MIN_YEAR", 2024), \ + patch("doctor.checks.maintainer.MAINTAINER_MAX_ACTIONS", 5), \ + patch("doctor.checks.maintainer.MAINTAINER_LIBRARY_TITLE", "shows"), \ + patch("doctor.checks.maintainer.MAINTAINER_UNWATCHED_DAYS", 30), \ + patch("doctor.checks.maintainer.MAINTAINER_MIN_AGE_DAYS", 0), \ + patch("doctor.checks.maintainer.MAINTAINER_PULSARR_TAG_PREFIX", "pulsarr"), \ + patch("doctor.checks.maintainer.MAINTAINER_RECHECK", 86400), \ + patch("doctor.checks.maintainer.MAINTAINER_MODE", "tagged"), \ + patch("doctor.checks.maintainer.MAINTAINER_PLEX_SECTION_KEY", 0), \ + patch("doctor.checks.maintainer.PULSARR_URL", ""), \ + patch("doctor.checks.maintainer.PULSARR_APIKEY", ""), \ + patch("doctor.checks.maintainer.INSTANCES", [arr]), \ + patch("doctor.clients.tautulli.Tautulli.recently_watched_shows", + return_value=set()), \ + patch("doctor.checks.maintainer.log") as mock_log: + from doctor.checks.maintainer import check_maintainer + with patch("builtins.open"): + check_maintainer() + + arr.tag_map.assert_not_called() + debug_calls = [c[0][0] for c in mock_log.debug.call_args_list if c[0]] + skip_call = [m for m in debug_calls if isinstance(m, str) and "not matching library title" in m] + self.assertTrue(len(skip_call) > 0) + + def test_watched_show_skipped(self): + series = [self._make_series(title="Watched Show")] + arr = self._make_sonarr_instance(series_list=series) + + with patch("doctor.checks.maintainer.EN_MAINTAINER", True), \ + patch("doctor.checks.maintainer.TAUTULLI_URL", "http://taut:8181"), \ + patch("doctor.checks.maintainer.TAUTULLI_APIKEY", "k"), \ + patch("doctor.checks.maintainer.DRY_RUN", True), \ + patch("doctor.checks.maintainer.MAINTAINER_MIN_YEAR", 2024), \ + patch("doctor.checks.maintainer.MAINTAINER_MAX_ACTIONS", 5), \ + patch("doctor.checks.maintainer.MAINTAINER_LIBRARY_TITLE", "shows"), \ + patch("doctor.checks.maintainer.MAINTAINER_UNWATCHED_DAYS", 30), \ + patch("doctor.checks.maintainer.MAINTAINER_MIN_AGE_DAYS", 0), \ + patch("doctor.checks.maintainer.MAINTAINER_PULSARR_TAG_PREFIX", "pulsarr"), \ + patch("doctor.checks.maintainer.MAINTAINER_RECHECK", 86400), \ + patch("doctor.checks.maintainer.MAINTAINER_MODE", "tagged"), \ + patch("doctor.checks.maintainer.MAINTAINER_PLEX_SECTION_KEY", 0), \ + patch("doctor.checks.maintainer.PULSARR_URL", ""), \ + patch("doctor.checks.maintainer.PULSARR_APIKEY", ""), \ + patch("doctor.checks.maintainer.INSTANCES", [arr]), \ + patch("doctor.clients.tautulli.Tautulli.recently_watched_shows", + return_value={"Watched Show"}) as _mock_taut, \ + patch("doctor.checks.maintainer.log") as mock_log: + from doctor.checks.maintainer import check_maintainer + with patch("builtins.open"): + check_maintainer() + + arr._req.assert_not_called() + log_calls = [c[0][0] for c in mock_log.info.call_args_list if c[0]] + delete_calls = [m for m in log_calls if isinstance(m, str) and "WOULD delete" in m] + self.assertEqual(len(delete_calls), 0) + + def test_enabled_false_returns_immediately(self): + with patch("doctor.checks.maintainer.EN_MAINTAINER", False), \ + patch("doctor.checks.maintainer.log") as mock_log: + from doctor.checks.maintainer import check_maintainer + check_maintainer() + mock_log.debug.assert_not_called() + + def test_arrow_show_scenario(self): + """Simulate 'Arrow' — ended 2012 show, pulsarr-tagged, no Tautulli watches.""" + series = [self._make_series( + sid=42, title="Arrow", status="ended", year=2012, tags=[1, 2], + )] + arr = self._make_sonarr_instance(name="sonarr-shows", series_list=series) + arr.tag_map.return_value = {1: "pulsarr-user-alice", 2: "pulsarr-user-bob"} + + with patch("doctor.checks.maintainer.EN_MAINTAINER", True), \ + patch("doctor.checks.maintainer.TAUTULLI_URL", "http://taut:8181"), \ + patch("doctor.checks.maintainer.TAUTULLI_APIKEY", "k"), \ + patch("doctor.checks.maintainer.DRY_RUN", True), \ + patch("doctor.checks.maintainer.MAINTAINER_MIN_YEAR", 2024), \ + patch("doctor.checks.maintainer.MAINTAINER_MAX_ACTIONS", 5), \ + patch("doctor.checks.maintainer.MAINTAINER_LIBRARY_TITLE", "shows"), \ + patch("doctor.checks.maintainer.MAINTAINER_UNWATCHED_DAYS", 30), \ + patch("doctor.checks.maintainer.MAINTAINER_MIN_AGE_DAYS", 0), \ + patch("doctor.checks.maintainer.MAINTAINER_PULSARR_TAG_PREFIX", "pulsarr"), \ + patch("doctor.checks.maintainer.MAINTAINER_RECHECK", 86400), \ + patch("doctor.checks.maintainer.MAINTAINER_MODE", "tagged"), \ + patch("doctor.checks.maintainer.MAINTAINER_PLEX_SECTION_KEY", 0), \ + patch("doctor.checks.maintainer.PULSARR_URL", ""), \ + patch("doctor.checks.maintainer.PULSARR_APIKEY", ""), \ + patch("doctor.checks.maintainer.INSTANCES", [arr]), \ + patch("doctor.clients.tautulli.Tautulli.recently_watched_shows", + return_value=set()) as _mock_taut, \ + patch("doctor.checks.maintainer.log") as mock_log: + from doctor.checks.maintainer import check_maintainer + with patch("builtins.open"): + check_maintainer() + + arr._req.assert_not_called() + log_calls = [(c[0], c[1]) for c in mock_log.info.call_args_list] + delete_calls = [(args, kwargs) for (args, kwargs) in log_calls + if args and "WOULD delete" in str(args[0])] + self.assertEqual(len(delete_calls), 1) + args = delete_calls[0][0] + msg = " ".join(str(a) for a in args) + self.assertIn("Arrow", msg) + self.assertIn("ended", msg) + self.assertIn("2012", msg) + self.assertIn("alice", msg) + self.assertIn("bob", msg) + self.assertIn("30d", msg) + + def test_arrow_show_watched_skipped(self): + """Arrow is watched — should NOT be deleted.""" + series = [self._make_series( + sid=42, title="Arrow", status="ended", year=2012, tags=[1], + )] + arr = self._make_sonarr_instance(name="sonarr-shows", series_list=series) + arr.tag_map.return_value = {1: "pulsarr-alice"} + + with patch("doctor.checks.maintainer.EN_MAINTAINER", True), \ + patch("doctor.checks.maintainer.TAUTULLI_URL", "http://taut:8181"), \ + patch("doctor.checks.maintainer.TAUTULLI_APIKEY", "k"), \ + patch("doctor.checks.maintainer.DRY_RUN", True), \ + patch("doctor.checks.maintainer.MAINTAINER_MIN_YEAR", 2024), \ + patch("doctor.checks.maintainer.MAINTAINER_MAX_ACTIONS", 5), \ + patch("doctor.checks.maintainer.MAINTAINER_LIBRARY_TITLE", "shows"), \ + patch("doctor.checks.maintainer.MAINTAINER_UNWATCHED_DAYS", 30), \ + patch("doctor.checks.maintainer.MAINTAINER_MIN_AGE_DAYS", 0), \ + patch("doctor.checks.maintainer.MAINTAINER_PULSARR_TAG_PREFIX", "pulsarr"), \ + patch("doctor.checks.maintainer.MAINTAINER_RECHECK", 86400), \ + patch("doctor.checks.maintainer.MAINTAINER_MODE", "tagged"), \ + patch("doctor.checks.maintainer.MAINTAINER_PLEX_SECTION_KEY", 0), \ + patch("doctor.checks.maintainer.PULSARR_URL", ""), \ + patch("doctor.checks.maintainer.PULSARR_APIKEY", ""), \ + patch("doctor.checks.maintainer.INSTANCES", [arr]), \ + patch("doctor.clients.tautulli.Tautulli.recently_watched_shows", + return_value={"Arrow"}) as _mock_taut, \ + patch("doctor.checks.maintainer.log") as mock_log: + from doctor.checks.maintainer import check_maintainer + with patch("builtins.open"): + check_maintainer() + + arr._req.assert_not_called() + log_calls = [c[0][0] for c in mock_log.info.call_args_list if c[0]] + delete_msgs = [m for m in log_calls if isinstance(m, str) and "WOULD delete" in m] + self.assertEqual(len(delete_msgs), 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_missing_seasons.py b/tests/test_missing_seasons.py new file mode 100644 index 0000000..7df9180 --- /dev/null +++ b/tests/test_missing_seasons.py @@ -0,0 +1,193 @@ +"""Unit tests for the missing_seasons check - candidate gathering logic.""" +import time +import unittest +from datetime import datetime, timezone, timedelta +from unittest.mock import MagicMock, patch + +from doctor.checks.missing_seasons import _gather_candidates + + +def _make_season(sn, monitored=True, file_count=0, total=10): + return { + "seasonNumber": sn, + "monitored": monitored, + "statistics": { + "episodeFileCount": file_count, + "totalEpisodeCount": total, + "episodeCount": total, + }, + } + +def _make_series(sid, title, status="ended", monitored=True, seasons=None): + return { + "id": sid, + "title": title, + "status": status, + "monitored": monitored, + "added": "Mon, 01 Jan 2020 00:00:00 +0000", + "seasons": seasons or [], + } + +def _make_arr(series_list, episodes_by_sid=None): + arr = MagicMock() + arr.name = "sonarr" + arr.kind = "sonarr" + arr.series.return_value = series_list + episodes_by_sid = episodes_by_sid or {} + arr.episodes.side_effect = lambda sid: episodes_by_sid.get(sid, []) + arr.command.return_value = True + return arr + +def _run(series_list, episodes_by_sid=None, ms=None, recheck=0, partial=True): + """Helper: run _gather_candidates with patched INSTANCES and MS_PARTIAL.""" + if ms is None: + ms = {} + arr = _make_arr(series_list, episodes_by_sid) + now = time.time() + with patch("doctor.checks.missing_seasons.INSTANCES", [arr]), \ + patch("doctor.checks.missing_seasons.MS_PARTIAL", partial): + cands, skipped, airing = _gather_candidates(ms, now, 0, recheck, backfill=False) + return cands, skipped, airing, now + + +class ZeroFileSeasonTest(unittest.TestCase): + """Original behaviour: zero-file seasons are always candidates.""" + + def test_zero_file_ended_season_is_candidate(self): + s = _make_series(1, "Show A", status="ended", seasons=[_make_season(1, file_count=0)]) + cands, _, _, _ = _run([s]) + self.assertEqual(len(cands), 1) + self.assertFalse(cands[0]["is_partial"]) + self.assertEqual(cands[0]["file_count"], 0) + + def test_zero_file_continuing_not_airing_is_candidate(self): + past = (datetime.now(timezone.utc) - timedelta(days=1)).isoformat() + eps = {1: [{"seasonNumber": 1, "airDateUtc": past}]} + s = _make_series(1, "Show B", status="continuing", seasons=[_make_season(1, file_count=0)]) + cands, _, _, _ = _run([s], eps) + self.assertEqual(len(cands), 1) + + def test_complete_season_not_a_candidate(self): + s = _make_series(1, "Show C", seasons=[_make_season(1, file_count=10, total=10)]) + cands, _, _, _ = _run([s]) + self.assertEqual(len(cands), 0) + + def test_unmonitored_season_skipped(self): + s = _make_series(1, "Show D", seasons=[_make_season(1, monitored=False, file_count=0)]) + cands, _, _, _ = _run([s]) + self.assertEqual(len(cands), 0) + + def test_unmonitored_series_skipped(self): + s = _make_series(1, "Show E", monitored=False, seasons=[_make_season(1, file_count=0)]) + cands, _, _, _ = _run([s]) + self.assertEqual(len(cands), 0) + + def test_season_zero_skipped(self): + s = _make_series(1, "Show F", seasons=[_make_season(0, file_count=0)]) + cands, _, _, _ = _run([s]) + self.assertEqual(len(cands), 0) + + def test_still_airing_skipped(self): + future = (datetime.now(timezone.utc) + timedelta(days=1)).isoformat() + eps = {1: [{"seasonNumber": 1, "airDateUtc": future}]} + s = _make_series(1, "Show G", status="continuing", seasons=[_make_season(1, file_count=0)]) + cands, _, airing, _ = _run([s], eps) + self.assertEqual(len(cands), 0) + self.assertEqual(airing, 1) + + def test_cooldown_skips(self): + ms = {} + now = time.time() + ms["sonarr:1:1"] = now - 100 # searched 100s ago, recheck=3600 + s = _make_series(1, "Show H", seasons=[_make_season(1, file_count=0)]) + cands, skipped, _, _ = _run([s], ms=ms, recheck=3600) + self.assertEqual(len(cands), 0) + self.assertEqual(skipped, 1) + + def test_cooldown_expired_is_candidate(self): + recheck = 3600 + ms = {"sonarr:1:1": time.time() - recheck - 1} + s = _make_series(1, "Show I", seasons=[_make_season(1, file_count=0)]) + cands, _, _, _ = _run([s], ms=ms, recheck=recheck) + self.assertEqual(len(cands), 1) + + def test_total_episodes_zero_skipped(self): + s = _make_series(1, "Show J", seasons=[_make_season(1, file_count=0, total=0)]) + cands, _, _, _ = _run([s]) + self.assertEqual(len(cands), 0) + + +class PartialSeasonTest(unittest.TestCase): + """New behaviour: partial seasons (some files, not complete) on fully-aired seasons.""" + + def test_partial_ended_season_is_candidate_when_enabled(self): + s = _make_series(1, "Partial Show", status="ended", + seasons=[_make_season(1, file_count=5, total=10)]) + cands, _, _, _ = _run([s], partial=True) + self.assertEqual(len(cands), 1) + self.assertTrue(cands[0]["is_partial"]) + self.assertEqual(cands[0]["file_count"], 5) + self.assertEqual(cands[0]["total_episodes"], 10) + + def test_partial_ended_season_skipped_when_disabled(self): + s = _make_series(1, "Partial Show", status="ended", + seasons=[_make_season(1, file_count=5, total=10)]) + cands, _, _, _ = _run([s], partial=False) + self.assertEqual(len(cands), 0) + + def test_partial_continuing_not_airing_is_candidate(self): + past = (datetime.now(timezone.utc) - timedelta(days=1)).isoformat() + eps = {1: [{"seasonNumber": 2, "airDateUtc": past}]} + s = _make_series(1, "Cont Show", status="continuing", + seasons=[_make_season(2, file_count=3, total=8)]) + cands, _, _, _ = _run([s], eps, partial=True) + self.assertEqual(len(cands), 1) + self.assertTrue(cands[0]["is_partial"]) + + def test_partial_continuing_still_airing_skipped(self): + future = (datetime.now(timezone.utc) + timedelta(days=1)).isoformat() + eps = {1: [{"seasonNumber": 2, "airDateUtc": future}]} + s = _make_series(1, "Airing Show", status="continuing", + seasons=[_make_season(2, file_count=3, total=8)]) + cands, _, airing, _ = _run([s], eps, partial=True) + self.assertEqual(len(cands), 0) + self.assertEqual(airing, 1) + + def test_complete_season_never_a_candidate_even_with_partial_on(self): + s = _make_series(1, "Complete Show", seasons=[_make_season(1, file_count=10, total=10)]) + cands, _, _, _ = _run([s], partial=True) + self.assertEqual(len(cands), 0) + + def test_partial_respects_cooldown(self): + recheck = 3600 + ms = {"sonarr:1:1": time.time() - 60} # searched 60s ago + s = _make_series(1, "Recent Show", seasons=[_make_season(1, file_count=3, total=10)]) + cands, skipped, _, _ = _run([s], ms=ms, recheck=recheck, partial=True) + self.assertEqual(len(cands), 0) + self.assertEqual(skipped, 1) + + def test_mixed_zero_and_partial_returned_together(self): + seasons = [ + _make_season(1, file_count=0, total=10), # zero -> candidate + _make_season(2, file_count=5, total=10), # partial -> candidate + _make_season(3, file_count=10, total=10), # complete -> skip + ] + s = _make_series(1, "Mixed Show", status="ended", seasons=seasons) + cands, _, _, _ = _run([s], partial=True) + self.assertEqual(len(cands), 2) + by_sn = {c["sn"]: c for c in cands} + self.assertFalse(by_sn[1]["is_partial"]) + self.assertTrue(by_sn[2]["is_partial"]) + + def test_one_file_out_of_many_is_partial(self): + s = _make_series(1, "Sparse Show", status="ended", + seasons=[_make_season(1, file_count=1, total=24)]) + cands, _, _, _ = _run([s], partial=True) + self.assertEqual(len(cands), 1) + self.assertTrue(cands[0]["is_partial"]) + + + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_multipack.py b/tests/test_multipack.py new file mode 100644 index 0000000..a7b3ccb --- /dev/null +++ b/tests/test_multipack.py @@ -0,0 +1,166 @@ +"""Unit tests for check_multipack helper functions.""" +import unittest + +from doctor.checks.multipack import ( + _pack_season_range, + _incomplete_seasons_covered, + _rank_packs, +) +from doctor.checks.missing_seasons import searched_series as _series_searched_by_missing_seasons + + +class PackSeasonRangeTest(unittest.TestCase): + def test_standard_dash(self): + self.assertEqual(_pack_season_range("Show S01-S05 BluRay"), (1, 5)) + + def test_dot_separator(self): + self.assertEqual(_pack_season_range("Show.S01.S03.1080p"), (1, 3)) + + def test_single_digit_seasons(self): + self.assertEqual(_pack_season_range("Show S1-S3 WEB"), (1, 3)) + + def test_reversed_order_normalized(self): + # S05-S01 should normalize to (1, 5) + self.assertEqual(_pack_season_range("Show S05-S01 Pack"), (1, 5)) + + def test_no_match_returns_none(self): + self.assertIsNone(_pack_season_range("Show S01E01 Episode")) + self.assertIsNone(_pack_season_range("Show Season 1 Complete")) + self.assertIsNone(_pack_season_range("")) + + def test_two_season_range(self): + self.assertEqual(_pack_season_range("Billions S01-S02 1080p"), (1, 2)) + + def test_parentheses_in_title(self): + self.assertEqual( + _pack_season_range("The Last Ship (2014) S01-S05 (1080p BluRay x265)"), + (1, 5) + ) + + def test_complete_keyword_prefix(self): + self.assertEqual(_pack_season_range("Show Complete S01-S07 WEB"), (1, 7)) + + +class IncompleteSeasonsCovers(unittest.TestCase): + def test_full_overlap(self): + # Pack S01-S05, all 5 seasons incomplete + self.assertEqual(_incomplete_seasons_covered((1, 5), {1, 2, 3, 4, 5}), 5) + + def test_partial_overlap(self): + # Pack S01-S06, only S06+S07 incomplete -> covers 1 + self.assertEqual(_incomplete_seasons_covered((1, 6), {6, 7}), 1) + + def test_no_overlap(self): + # Pack S01-S04, only S05-S07 incomplete -> covers 0 + self.assertEqual(_incomplete_seasons_covered((1, 4), {5, 6, 7}), 0) + + def test_exact_match(self): + # Pack S06-S07, exactly those two missing + self.assertEqual(_incomplete_seasons_covered((6, 7), {6, 7}), 2) + + def test_wider_than_needed(self): + # Pack S01-S07, only S03 and S05 incomplete -> covers 2 + self.assertEqual(_incomplete_seasons_covered((1, 7), {3, 5}), 2) + + def test_single_season_pack_matches(self): + # Edge: S03-S03 range (shouldn't normally appear, but safe) + self.assertEqual(_incomplete_seasons_covered((3, 3), {3, 5}), 1) + + +def _make_pack(title, quality_weight=1000): + return {"title": title, "fullSeason": True, "qualityWeight": quality_weight} + + +class RankPacksTest(unittest.TestCase): + def _titles(self, packs, incomplete): + return [p["title"] for p, _pr, _cov in _rank_packs(packs, incomplete)] + + def test_zero_overlap_excluded(self): + # S01-S04 pack is useless when only S05-S07 are missing + packs = [_make_pack("Show S01-S04 1080p")] + self.assertEqual(_rank_packs(packs, {5, 6, 7}), []) + + def test_most_coverage_first(self): + # S01-S07 covers more missing seasons than S01-S06 + packs = [ + _make_pack("Show S01-S06 1080p"), + _make_pack("Show S01-S07 1080p"), + ] + incomplete = {6, 7} + titles = self._titles(packs, incomplete) + self.assertEqual(titles[0], "Show S01-S07 1080p") # covers both S06+S07 + + def test_wider_pack_preferred_on_equal_coverage(self): + # Both cover the same 1 missing season (S06), but S01-S07 is wider + packs = [ + _make_pack("Show S01-S06 1080p"), + _make_pack("Show S01-S07 1080p"), + ] + incomplete = {6} # only S06 missing + titles = self._titles(packs, incomplete) + # Both cover 1 missing season; S01-S07 is wider -> comes first + self.assertEqual(titles[0], "Show S01-S07 1080p") + + def test_quality_weight_tiebreaker(self): + # Same coverage, same width -> higher qualityWeight wins + packs = [ + _make_pack("Show S01-S05 WEB", quality_weight=800), + _make_pack("Show S01-S05 BluRay", quality_weight=1200), + ] + incomplete = {3, 5} + titles = self._titles(packs, incomplete) + self.assertEqual(titles[0], "Show S01-S05 BluRay") + + def test_mixed_useful_and_useless(self): + # One pack covers nothing, two cover different amounts + packs = [ + _make_pack("Show S01-S02 1080p"), # useless: only S05-S07 missing + _make_pack("Show S01-S06 1080p"), # covers S05+S06 + _make_pack("Show S01-S07 1080p"), # covers S05+S06+S07 + ] + incomplete = {5, 6, 7} + titles = self._titles(packs, incomplete) + self.assertEqual(len(titles), 2) # S01-S02 excluded + self.assertEqual(titles[0], "Show S01-S07 1080p") # most coverage + + def test_empty_packs(self): + self.assertEqual(_rank_packs([], {1, 2, 3}), []) + + def test_empty_incomplete(self): + packs = [_make_pack("Show S01-S05 1080p")] + self.assertEqual(_rank_packs(packs, set()), []) + + +class SeriesSearchedByMissingSeasons(unittest.TestCase): + def test_parses_state_keys(self): + state = { + "__missing_seasons__": { + "sonarr:17:1": 1000.0, + "sonarr:17:2": 1000.0, + "sonarr:42:3": 1000.0, + "sonarr:99:1": 1000.0, + } + } + ids = _series_searched_by_missing_seasons(state, "sonarr") + self.assertEqual(ids, {17, 42, 99}) + + def test_different_arr_name_ignored(self): + state = { + "__missing_seasons__": { + "sonarr:17:1": 1000.0, + "radarr:99:1": 1000.0, + } + } + ids = _series_searched_by_missing_seasons(state, "sonarr") + self.assertEqual(ids, {17}) + + def test_empty_state(self): + self.assertEqual(_series_searched_by_missing_seasons({}, "sonarr"), set()) + + def test_no_missing_seasons_key(self): + state = {"__multipack__": {}} + self.assertEqual(_series_searched_by_missing_seasons(state, "sonarr"), set()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_no_upgrade.py b/tests/test_no_upgrade.py new file mode 100644 index 0000000..e2db946 --- /dev/null +++ b/tests/test_no_upgrade.py @@ -0,0 +1,144 @@ +"""Unit tests for the no_upgrade_profile check. + +Covers profile lookup, series filtering, and the PUT update call via mocked Arr. +The check's three _req calls are now all behind public Arr methods (quality_profiles, +series, update_series) so we can mock them cleanly without touching _req. +""" +import unittest +from unittest.mock import MagicMock, patch + +from doctor.checks.no_upgrade import check_no_upgrade_profile + + +def _make_arr(name="sonarr", kind="sonarr"): + arr = MagicMock() + arr.name = name + arr.kind = kind + arr.quality_profiles.return_value = [] + arr.series.return_value = [] + arr.update_series.return_value = None + return arr + + +def _series(sid, title, status="ended", pct=100, ep_count=10, profile_id=1): + return { + "id": sid, + "title": title, + "status": status, + "qualityProfileId": profile_id, + "statistics": {"episodeCount": ep_count, "percentOfEpisodes": pct}, + } + + +_BASE = dict( + INSTANCES=[], # overridden per test + EN_NO_UPGRADE_PROFILE=True, + NO_UPGRADE_PROFILE_NAME="No Upgrade", + NO_UPGRADE_PROFILE_ID=0, +) + +def _patch(**overrides): + kw = {**_BASE, **overrides} + return patch.multiple("doctor.checks.no_upgrade", **kw) + + +class ProfileLookupTest(unittest.TestCase): + def test_skips_when_profile_not_found(self): + arr = _make_arr() + arr.quality_profiles.return_value = [{"id": 5, "name": "Other"}] + arr.series.return_value = [] + with _patch(INSTANCES=[arr], NO_UPGRADE_PROFILE_ID=0): + check_no_upgrade_profile() + arr.update_series.assert_not_called() + + def test_resolves_profile_by_name(self): + arr = _make_arr() + arr.quality_profiles.return_value = [{"id": 7, "name": "No Upgrade"}] + arr.series.return_value = [_series(1, "Show A", profile_id=2)] + with _patch(INSTANCES=[arr], NO_UPGRADE_PROFILE_ID=0): + check_no_upgrade_profile() + updated = arr.update_series.call_args[0][0] + self.assertEqual(updated["qualityProfileId"], 7) + + def test_uses_explicit_profile_id_without_lookup(self): + arr = _make_arr() + arr.series.return_value = [_series(1, "Show A", profile_id=2)] + with _patch(INSTANCES=[arr], NO_UPGRADE_PROFILE_ID=9): + check_no_upgrade_profile() + arr.quality_profiles.assert_not_called() + updated = arr.update_series.call_args[0][0] + self.assertEqual(updated["qualityProfileId"], 9) + + +class SeriesFilterTest(unittest.TestCase): + def _run(self, series_list, profile_id=5): + arr = _make_arr() + arr.quality_profiles.return_value = [{"id": profile_id, "name": "No Upgrade"}] + arr.series.return_value = series_list + with _patch(INSTANCES=[arr], NO_UPGRADE_PROFILE_ID=0): + check_no_upgrade_profile() + return arr + + def test_skips_continuing_series(self): + arr = self._run([_series(1, "Ongoing", status="continuing")]) + arr.update_series.assert_not_called() + + def test_skips_already_on_target_profile(self): + arr = self._run([_series(1, "Done", profile_id=5)], profile_id=5) + arr.update_series.assert_not_called() + + def test_skips_incomplete_ended_series(self): + arr = self._run([_series(1, "Partial", pct=80, ep_count=10)]) + arr.update_series.assert_not_called() + + def test_skips_ended_with_zero_episodes(self): + arr = self._run([_series(1, "Empty", pct=100, ep_count=0)]) + arr.update_series.assert_not_called() + + def test_moves_complete_ended_series(self): + arr = self._run([_series(1, "Complete", status="ended", pct=100, ep_count=10)]) + arr.update_series.assert_called_once() + + def test_moves_multiple_eligible_series(self): + series = [ + _series(1, "Show A", status="ended", pct=100, ep_count=5), + _series(2, "Show B", status="ended", pct=100, ep_count=12), + ] + arr = self._run(series) + self.assertEqual(arr.update_series.call_count, 2) + + +class UpdateSeriesTest(unittest.TestCase): + def test_update_sets_quality_profile_id(self): + arr = _make_arr() + arr.quality_profiles.return_value = [{"id": 3, "name": "No Upgrade"}] + s = _series(42, "My Show", profile_id=1) + arr.series.return_value = [s] + with _patch(INSTANCES=[arr], NO_UPGRADE_PROFILE_ID=0): + check_no_upgrade_profile() + updated = arr.update_series.call_args[0][0] + self.assertEqual(updated["id"], 42) + self.assertEqual(updated["qualityProfileId"], 3) + + def test_update_failure_does_not_abort(self): + """update_series() raising must not stop the rest of the series from being processed.""" + arr = _make_arr() + arr.quality_profiles.return_value = [{"id": 3, "name": "No Upgrade"}] + arr.series.return_value = [ + _series(1, "Fail Show", profile_id=1), + _series(2, "Good Show", profile_id=1), + ] + arr.update_series.side_effect = [Exception("timeout"), None] + with _patch(INSTANCES=[arr], NO_UPGRADE_PROFILE_ID=0): + check_no_upgrade_profile() + self.assertEqual(arr.update_series.call_count, 2) + + def test_skips_non_sonarr_instances(self): + arr = _make_arr(kind="radarr") + with _patch(INSTANCES=[arr]): + check_no_upgrade_profile() + arr.series.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_plexscan.py b/tests/test_plexscan.py new file mode 100644 index 0000000..f1fd5b1 --- /dev/null +++ b/tests/test_plexscan.py @@ -0,0 +1,435 @@ +"""Characterization tests for doctor.checks.plexscan. + +Lock in the current behavior of: + - _is_scan_activity(): activity classification predicate + - check_plex_scan(): stuck-scan detection, progress tracking, and + multi-step recovery (mount probe -> cancel -> restart) + +Plex is fully mocked. Time is patched so tests are deterministic. +Module-level mutable state (_scan_seen, _plex_last_restart) is reset +between tests. +""" +import time +import unittest +from unittest.mock import patch, MagicMock + +from doctor.checks.plexscan import ( + _is_scan_activity, + check_plex_scan, + _scan_seen, + _plex_last_restart, +) + +_MOD = "doctor.checks.plexscan" + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _activity(uuid, title="Library scan", atype="library.update.section", + progress=0, subtitle="", cancellable="1"): + """Minimal Plex activity dict.""" + a = { + "uuid": uuid, + "type": atype, + "title": title, + "subtitle": subtitle, + "progress": str(progress), + "cancellable": cancellable, + } + return a + + +def _make_plex(activities=None): + plex = MagicMock() + plex.activities.return_value = activities or [] + plex.cancel_activity.return_value = True + return plex + + +# --------------------------------------------------------------------------- +# _is_scan_activity +# --------------------------------------------------------------------------- + +class IsScanActivityTest(unittest.TestCase): + + def test_library_update_type(self): + a = {"type": "library.update.section", "title": "", "subtitle": ""} + self.assertTrue(_is_scan_activity(a)) + + def test_library_refresh_type(self): + a = {"type": "library.refresh", "title": "", "subtitle": ""} + self.assertTrue(_is_scan_activity(a)) + + def test_scan_in_title(self): + a = {"type": "something", "title": "Scanning Movies", "subtitle": ""} + self.assertTrue(_is_scan_activity(a)) + + def test_scan_in_subtitle(self): + a = {"type": "something", "title": "", "subtitle": "Library scan in progress"} + self.assertTrue(_is_scan_activity(a)) + + def test_non_scan_activity(self): + a = {"type": "media.play", "title": "Playing Movie", "subtitle": ""} + self.assertFalse(_is_scan_activity(a)) + + def test_empty_activity(self): + a = {"type": "", "title": "", "subtitle": ""} + self.assertFalse(_is_scan_activity(a)) + + def test_missing_fields_default_to_empty(self): + a = {} + self.assertFalse(_is_scan_activity(a)) + + def test_case_insensitive(self): + a = {"type": "Library.Update.Section", "title": "", "subtitle": ""} + self.assertTrue(_is_scan_activity(a)) + + +# --------------------------------------------------------------------------- +# check_plex_scan - basic flow +# --------------------------------------------------------------------------- + +class CheckPlexScanBasicTest(unittest.TestCase): + """Tests for the basic flow: no scans, progressing scans, early return.""" + + def setUp(self): + _scan_seen.value.clear() + _plex_last_restart.reset() + + def tearDown(self): + _scan_seen.value.clear() + _plex_last_restart.reset() + + @patch(_MOD + ".PLEX_URL", "") + @patch(_MOD + ".PLEX_TOKEN", "") + def test_noop_when_no_plex_configured(self): + """Early return when PLEX_URL or PLEX_TOKEN are empty.""" + check_plex_scan() # should not raise + + @patch(_MOD + ".Plex") + @patch(_MOD + ".PLEX_URL", "http://plex:32400") + @patch(_MOD + ".PLEX_TOKEN", "token123") + def test_no_activities_clears_seen(self, MockPlex): + plex = _make_plex([]) + MockPlex.return_value = plex + + check_plex_scan() + self.assertEqual(len(_scan_seen.value), 0) + + @patch(_MOD + ".PLEX_SCAN_STUCK", 1800) + @patch(_MOD + ".Plex") + @patch(_MOD + ".PLEX_URL", "http://plex:32400") + @patch(_MOD + ".PLEX_TOKEN", "token123") + def test_progressing_scan_tracked_not_stuck(self, MockPlex): + plex = _make_plex([_activity("uuid-1", progress=10)]) + MockPlex.return_value = plex + + check_plex_scan() + self.assertIn("uuid-1", _scan_seen.value) + # Not stuck yet (just started) + self.assertEqual(_scan_seen.value["uuid-1"]["prog"], 10) + + @patch(_MOD + ".PLEX_SCAN_STUCK", 1800) + @patch(_MOD + ".Plex") + @patch(_MOD + ".PLEX_URL", "http://plex:32400") + @patch(_MOD + ".PLEX_TOKEN", "token123") + def test_finished_scan_removed_from_seen(self, MockPlex): + """When a scan disappears from activities, it's removed from _scan_seen.""" + plex = _make_plex([_activity("uuid-1")]) + MockPlex.return_value = plex + + check_plex_scan() + self.assertIn("uuid-1", _scan_seen.value) + + # Second call: scan is gone + plex.activities.return_value = [] + check_plex_scan() + self.assertNotIn("uuid-1", _scan_seen.value) + + @patch(_MOD + ".PLEX_SCAN_STUCK", 1800) + @patch(_MOD + ".Plex") + @patch(_MOD + ".PLEX_URL", "http://plex:32400") + @patch(_MOD + ".PLEX_TOKEN", "token123") + def test_non_scan_activity_ignored(self, MockPlex): + a = {"uuid": "uuid-play", "type": "media.play", + "title": "Movie", "subtitle": "", "progress": "0"} + plex = _make_plex([a]) + MockPlex.return_value = plex + + check_plex_scan() + self.assertNotIn("uuid-play", _scan_seen.value) + + @patch(_MOD + ".PLEX_SCAN_STUCK", 1800) + @patch(_MOD + ".Plex") + @patch(_MOD + ".PLEX_URL", "http://plex:32400") + @patch(_MOD + ".PLEX_TOKEN", "token123") + def test_activity_without_uuid_skipped(self, MockPlex): + a = {"type": "library.update.section", "title": "Scan", "subtitle": "", + "progress": "0"} # no uuid + plex = _make_plex([a]) + MockPlex.return_value = plex + + check_plex_scan() + self.assertEqual(len(_scan_seen.value), 0) + + +# --------------------------------------------------------------------------- +# check_plex_scan - stuck detection & recovery +# --------------------------------------------------------------------------- + +class CheckPlexScanStuckTest(unittest.TestCase): + """Tests for stuck-scan detection and the 3-step recovery.""" + + def setUp(self): + _scan_seen.value.clear() + _plex_last_restart.reset() + + def tearDown(self): + _scan_seen.value.clear() + _plex_last_restart.reset() + + @patch(_MOD + ".PLEX_RESTART_CMD", "") + @patch(_MOD + ".PLEX_SCAN_CANCEL", True) + @patch(_MOD + ".DECY_MOUNT_TEST", "") + @patch(_MOD + ".DRY_RUN", False) + @patch(_MOD + ".PLEX_SCAN_STUCK", 1800) + @patch(_MOD + ".Plex") + @patch(_MOD + ".PLEX_URL", "http://plex:32400") + @patch(_MOD + ".PLEX_TOKEN", "token123") + def test_stuck_scan_detected_and_cancelled(self, MockPlex): + """A scan with no progress for >= PLEX_SCAN_STUCK is detected and cancelled.""" + now = time.time() + plex = _make_plex([_activity("uuid-stuck", progress=50)]) + MockPlex.return_value = plex + + # Seed the scan as already tracked and stale + _scan_seen.value["uuid-stuck"] = { + "first": now - 3600, + "prog": 50, + "prog_ts": now - 2000, # stale for 2000s > 1800s threshold + "title": "Library scan", + "acted_ts": 0, + } + + check_plex_scan() + plex.cancel_activity.assert_called_once_with("uuid-stuck") + + @patch(_MOD + ".PLEX_RESTART_CMD", "") + @patch(_MOD + ".PLEX_SCAN_CANCEL", True) + @patch(_MOD + ".DECY_MOUNT_TEST", "") + @patch(_MOD + ".DRY_RUN", False) + @patch(_MOD + ".PLEX_SCAN_STUCK", 1800) + @patch(_MOD + ".Plex") + @patch(_MOD + ".PLEX_URL", "http://plex:32400") + @patch(_MOD + ".PLEX_TOKEN", "token123") + def test_progress_advance_resets_stuck_timer(self, MockPlex): + """When progress advances, prog_ts resets and the scan is not stuck.""" + now = time.time() + plex = _make_plex([_activity("uuid-prog", progress=60)]) + MockPlex.return_value = plex + + # Previously at progress 50, stale timing + _scan_seen.value["uuid-prog"] = { + "first": now - 3600, + "prog": 50, # current progress 60 > 50 -> advances + "prog_ts": now - 2000, + "title": "Library scan", + "acted_ts": 0, + } + + check_plex_scan() + # Progress advanced -> not stuck -> no cancel + plex.cancel_activity.assert_not_called() + # prog_ts should be refreshed to approximately now + self.assertGreater(_scan_seen.value["uuid-prog"]["prog_ts"], now - 5) + + @patch(_MOD + ".PLEX_RESTART_CMD", "") + @patch(_MOD + ".PLEX_SCAN_CANCEL", True) + @patch(_MOD + ".DECY_MOUNT_TEST", "") + @patch(_MOD + ".DRY_RUN", False) + @patch(_MOD + ".PLEX_SCAN_STUCK", 1800) + @patch(_MOD + ".Plex") + @patch(_MOD + ".PLEX_URL", "http://plex:32400") + @patch(_MOD + ".PLEX_TOKEN", "token123") + def test_acted_ts_prevents_repeated_action(self, MockPlex): + """After acting on a stuck scan, the acted_ts throttle prevents re-acting within the window.""" + now = time.time() + plex = _make_plex([_activity("uuid-acted", progress=50)]) + MockPlex.return_value = plex + + _scan_seen.value["uuid-acted"] = { + "first": now - 3600, + "prog": 50, + "prog_ts": now - 2000, + "title": "Library scan", + "acted_ts": now - 100, # acted 100s ago, within the PLEX_SCAN_STUCK window + } + + check_plex_scan() + plex.cancel_activity.assert_not_called() + + @patch(_MOD + ".PLEX_RESTART_CMD", "") + @patch(_MOD + ".PLEX_SCAN_CANCEL", True) + @patch(_MOD + ".DECY_MOUNT_TEST", "") + @patch(_MOD + ".DRY_RUN", True) + @patch(_MOD + ".PLEX_SCAN_STUCK", 1800) + @patch(_MOD + ".Plex") + @patch(_MOD + ".PLEX_URL", "http://plex:32400") + @patch(_MOD + ".PLEX_TOKEN", "token123") + def test_dry_run_does_not_cancel(self, MockPlex): + now = time.time() + plex = _make_plex([_activity("uuid-dry", progress=50)]) + MockPlex.return_value = plex + + _scan_seen.value["uuid-dry"] = { + "first": now - 3600, + "prog": 50, + "prog_ts": now - 2000, + "title": "Library scan", + "acted_ts": 0, + } + + check_plex_scan() + plex.cancel_activity.assert_not_called() + + @patch(_MOD + "._decy_restart") + @patch(_MOD + "._probe_mount") + @patch(_MOD + ".PLEX_RESTART_CMD", "") + @patch(_MOD + ".PLEX_SCAN_CANCEL", True) + @patch(_MOD + ".DECY_MOUNT_TEST", "/mnt/zurg") + @patch(_MOD + ".DRY_RUN", False) + @patch(_MOD + ".PLEX_SCAN_STUCK", 1800) + @patch(_MOD + ".Plex") + @patch(_MOD + ".PLEX_URL", "http://plex:32400") + @patch(_MOD + ".PLEX_TOKEN", "token123") + def test_stuck_scan_probes_mount_on_dead(self, MockPlex, mock_probe, mock_restart): + """When mount is DEAD, _decy_restart is called before cancelling.""" + from doctor.checks.decypharr import _FuseStatus + mock_probe.return_value = (_FuseStatus.DEAD, "transport gone") + now = time.time() + plex = _make_plex([_activity("uuid-mount", progress=50)]) + MockPlex.return_value = plex + + _scan_seen.value["uuid-mount"] = { + "first": now - 3600, + "prog": 50, + "prog_ts": now - 2000, + "title": "Library scan", + "acted_ts": 0, + } + + check_plex_scan() + mock_restart.assert_called_once() + plex.cancel_activity.assert_called_once_with("uuid-mount") + + @patch(_MOD + ".run_cmd", return_value=(0, "ok")) + @patch(_MOD + ".PLEX_RESTART_CMD", "systemctl restart plex") + @patch(_MOD + ".PLEX_SCAN_CANCEL", True) + @patch(_MOD + ".DECY_MOUNT_TEST", "") + @patch(_MOD + ".DRY_RUN", False) + @patch(_MOD + ".PLEX_SCAN_STUCK", 1800) + @patch(_MOD + ".Plex") + @patch(_MOD + ".PLEX_URL", "http://plex:32400") + @patch(_MOD + ".PLEX_TOKEN", "token123") + def test_restart_fires_when_scan_wedged_long_and_cancel_fails(self, MockPlex, mock_cmd): + """Plex restart fires when: wedged >= 2*threshold, cancel fails, and no recent restart.""" + now = time.time() + plex = _make_plex([_activity("uuid-restart", progress=50)]) + plex.cancel_activity.return_value = False # cancel fails + MockPlex.return_value = plex + + _scan_seen.value["uuid-restart"] = { + "first": now - 7200, # started 2h ago, well past 2*1800=3600 + "prog": 50, + "prog_ts": now - 4000, + "title": "Library scan", + "acted_ts": 0, + } + + check_plex_scan() + mock_cmd.assert_called_once_with("systemctl restart plex") + + @patch(_MOD + ".run_cmd", return_value=(0, "ok")) + @patch(_MOD + ".PLEX_RESTART_CMD", "systemctl restart plex") + @patch(_MOD + ".PLEX_SCAN_CANCEL", True) + @patch(_MOD + ".DECY_MOUNT_TEST", "") + @patch(_MOD + ".DRY_RUN", False) + @patch(_MOD + ".PLEX_SCAN_STUCK", 1800) + @patch(_MOD + ".Plex") + @patch(_MOD + ".PLEX_URL", "http://plex:32400") + @patch(_MOD + ".PLEX_TOKEN", "token123") + def test_restart_suppressed_when_cancelled_successfully(self, MockPlex, mock_cmd): + """Successful cancel suppresses the restart, even if timing qualifies.""" + now = time.time() + plex = _make_plex([_activity("uuid-norest", progress=50)]) + plex.cancel_activity.return_value = True # cancel succeeds + MockPlex.return_value = plex + + _scan_seen.value["uuid-norest"] = { + "first": now - 7200, + "prog": 50, + "prog_ts": now - 4000, + "title": "Library scan", + "acted_ts": 0, + } + + check_plex_scan() + mock_cmd.assert_not_called() + + @patch(_MOD + ".run_cmd", return_value=(0, "ok")) + @patch(_MOD + ".PLEX_RESTART_CMD", "systemctl restart plex") + @patch(_MOD + ".PLEX_SCAN_CANCEL", True) + @patch(_MOD + ".DECY_MOUNT_TEST", "") + @patch(_MOD + ".DRY_RUN", False) + @patch(_MOD + ".PLEX_SCAN_STUCK", 1800) + @patch(_MOD + ".Plex") + @patch(_MOD + ".PLEX_URL", "http://plex:32400") + @patch(_MOD + ".PLEX_TOKEN", "token123") + def test_restart_rate_limited_to_30min(self, MockPlex, mock_cmd): + """Restart should not fire if _plex_last_restart was < 1800s ago.""" + now = time.time() + _plex_last_restart.value = now - 600 # restarted 10 min ago + plex = _make_plex([_activity("uuid-rl", progress=50)]) + plex.cancel_activity.return_value = False + MockPlex.return_value = plex + + _scan_seen.value["uuid-rl"] = { + "first": now - 7200, + "prog": 50, + "prog_ts": now - 4000, + "title": "Library scan", + "acted_ts": 0, + } + + check_plex_scan() + mock_cmd.assert_not_called() + + @patch(_MOD + ".PLEX_RESTART_CMD", "") + @patch(_MOD + ".PLEX_SCAN_CANCEL", False) + @patch(_MOD + ".DECY_MOUNT_TEST", "") + @patch(_MOD + ".DRY_RUN", False) + @patch(_MOD + ".PLEX_SCAN_STUCK", 1800) + @patch(_MOD + ".Plex") + @patch(_MOD + ".PLEX_URL", "http://plex:32400") + @patch(_MOD + ".PLEX_TOKEN", "token123") + def test_cancel_skipped_when_disabled(self, MockPlex): + now = time.time() + plex = _make_plex([_activity("uuid-nocancel", progress=50)]) + MockPlex.return_value = plex + + _scan_seen.value["uuid-nocancel"] = { + "first": now - 3600, + "prog": 50, + "prog_ts": now - 2000, + "title": "Library scan", + "acted_ts": 0, + } + + check_plex_scan() + plex.cancel_activity.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_queue.py b/tests/test_queue.py new file mode 100644 index 0000000..8daa500 --- /dev/null +++ b/tests/test_queue.py @@ -0,0 +1,313 @@ +"""Unit tests for the queue check. + +Tests cover: + - stuck_reason(): all six condition predicates + - check_queue(): strike accumulation, removal, DRY_RUN, action counting, + health warnings, and per-arr filtering via `only=` + +All Arr HTTP interactions are replaced with MagicMock so no network is needed. +Constants consumed from the star-import (MIN_STRIKES, MAX_ACTIONS, DRY_RUN, +LOAD_MAX, BLOCKLIST, INSTANCES, ENABLED_CONDITIONS) are patched on the +doctor.checks.queue module directly, matching the pattern used in +test_missing_seasons.py. +""" +import os +import tempfile +import unittest +from unittest.mock import MagicMock, patch + +import doctor.state as _state +from doctor.checks.queue import stuck_reason, _msgs, check_queue + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _make_arr(name="sonarr", kind="sonarr"): + arr = MagicMock() + arr.name = name + arr.kind = kind + arr.queue.return_value = [] + arr.health.return_value = [] + arr.remove.return_value = None + arr.queue_target_id.return_value = None # churn brake disabled by default in tests + return arr + + +def _rec(iid, *, status=None, tracked_state=None, tracked_status=None, messages=None): + """Minimal queue record.""" + r = {"id": iid, "title": "Show S01E01"} + if status: + r["status"] = status + if tracked_state: + r["trackedDownloadState"] = tracked_state + if tracked_status: + r["trackedDownloadStatus"] = tracked_status + if messages: + r["statusMessages"] = [{"messages": messages}] + return r + + +# --------------------------------------------------------------------------- +# stuck_reason predicate tests (these use no patching — pure function) +# --------------------------------------------------------------------------- + +class StuckReasonTest(unittest.TestCase): + def test_download_client_unavailable(self): + r = _rec(1, status="downloadClientUnavailable") + self.assertEqual(stuck_reason(r), "downloadClientUnavailable") + + def test_import_blocked(self): + r = _rec(1, tracked_state="importBlocked") + self.assertEqual(stuck_reason(r), "importBlocked") + + def test_import_failed(self): + r = _rec(1, tracked_state="importFailed") + self.assertEqual(stuck_reason(r), "importFailed") + + def test_import_pending_warning(self): + r = _rec(1, tracked_state="importPending", tracked_status="warning") + self.assertEqual(stuck_reason(r), "importPending_warning") + + def test_import_pending_error(self): + r = _rec(1, tracked_state="importPending", tracked_status="error") + self.assertEqual(stuck_reason(r), "importPending_warning") + + def test_import_pending_ok_not_stuck(self): + r = _rec(1, tracked_state="importPending", tracked_status="ok") + self.assertIsNone(stuck_reason(r)) + + def test_failed_pending(self): + r = _rec(1, tracked_state="failedPending") + self.assertEqual(stuck_reason(r), "failedPending") + + def test_stalled_by_message(self): + r = _rec(1, tracked_status="warning", messages=["download is stalled with no connections"]) + self.assertEqual(stuck_reason(r), "stalled") + + def test_stalled_no_files(self): + r = _rec(1, tracked_status="warning", messages=["no files found are eligible for import"]) + self.assertEqual(stuck_reason(r), "stalled") + + def test_warning_without_stall_message(self): + r = _rec(1, tracked_status="warning", messages=["something else"]) + self.assertIsNone(stuck_reason(r)) + + def test_clean_item_is_none(self): + r = _rec(1, status="ok") + self.assertIsNone(stuck_reason(r)) + + def test_empty_record_is_none(self): + self.assertIsNone(stuck_reason({})) + + def test_conditions_respect_enabled_set(self): + """Only conditions in ENABLED_CONDITIONS should be tested.""" + r = _rec(1, status="downloadClientUnavailable") + with patch("doctor.checks.queue.ENABLED_CONDITIONS", ["importBlocked"]): + # downloadClientUnavailable is not enabled → should return None + self.assertIsNone(stuck_reason(r)) + + +class MsgsTest(unittest.TestCase): + def test_extracts_nested_messages(self): + r = {"statusMessages": [{"messages": ["a", "b"]}, {"messages": ["c"]}], "errorMessage": "top"} + self.assertEqual(_msgs(r), ["a", "b", "c", "top"]) + + def test_empty_status_messages(self): + self.assertEqual(_msgs({}), []) + + def test_only_error_message(self): + r = {"errorMessage": "boom"} + self.assertEqual(_msgs(r), ["boom"]) + + +# --------------------------------------------------------------------------- +# check_queue integration-style tests (Arr fully mocked, state in temp file) +# --------------------------------------------------------------------------- + +_Q_PATCHES = dict( + LOAD_MAX=0, # don't skip on load + MIN_STRIKES=2, + MAX_ACTIONS=10, + DRY_RUN=False, + BLOCKLIST=True, +) + +def _patch_queue(**overrides): + """Return a patch.multiple context for doctor.checks.queue constants.""" + kw = {**_Q_PATCHES, **overrides} + return patch.multiple("doctor.checks.queue", **kw) + + +class CheckQueueStrikeTest(unittest.TestCase): + """Strike accumulation: items must be stuck for MIN_STRIKES before removal.""" + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + _state.STATE_FILE = os.path.join(self.tmp.name, "state.json") + + def _run(self, arr, **kw): + with _patch_queue(**kw), \ + patch("doctor.checks.queue.INSTANCES", [arr]), \ + patch("doctor.state.CHURN_LIMIT", 0): + check_queue() + + def test_no_removal_on_first_strike(self): + arr = _make_arr() + arr.queue.return_value = [_rec(1, status="downloadClientUnavailable")] + self._run(arr, MIN_STRIKES=2) + arr.remove.assert_not_called() + + def test_removal_on_second_strike(self): + arr = _make_arr() + arr.queue.return_value = [_rec(1, status="downloadClientUnavailable")] + self._run(arr, MIN_STRIKES=2) + self._run(arr, MIN_STRIKES=2) + arr.remove.assert_called_once_with(1) + + def test_removal_on_first_strike_when_min_strikes_is_one(self): + arr = _make_arr() + arr.queue.return_value = [_rec(1, status="downloadClientUnavailable")] + self._run(arr, MIN_STRIKES=1) + arr.remove.assert_called_once_with(1) + + def test_strike_counter_resets_after_removal(self): + """After an item is removed, its strike count is cleared from state.""" + arr = _make_arr() + arr.queue.return_value = [_rec(1, status="downloadClientUnavailable")] + self._run(arr, MIN_STRIKES=2) # strike 1 + self._run(arr, MIN_STRIKES=2) # strike 2 → remove + arr.remove.reset_mock() + # Next sweep: item is back (re-grabbed); strike counter should start fresh + self._run(arr, MIN_STRIKES=2) # strike 1 again + arr.remove.assert_not_called() + + def test_clean_item_ignored(self): + arr = _make_arr() + arr.queue.return_value = [_rec(1, status="ok")] + self._run(arr) + arr.remove.assert_not_called() + + +class CheckQueueDryRunTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + _state.STATE_FILE = os.path.join(self.tmp.name, "state.json") + + def test_dry_run_does_not_remove(self): + arr = _make_arr() + arr.queue.return_value = [_rec(1, status="downloadClientUnavailable")] + with _patch_queue(MIN_STRIKES=1, DRY_RUN=True), \ + patch("doctor.checks.queue.INSTANCES", [arr]), \ + patch("doctor.state.CHURN_LIMIT", 0): + check_queue() + arr.remove.assert_not_called() + + def test_dry_run_does_not_update_state(self): + """DRY_RUN still accumulates strike counts (so we don't re-act on restart).""" + # Note: check_queue updates state[arr.name] regardless of DRY_RUN. + # This test documents the current behavior. + arr = _make_arr() + arr.queue.return_value = [_rec(7, status="downloadClientUnavailable")] + with _patch_queue(MIN_STRIKES=2, DRY_RUN=True), \ + patch("doctor.checks.queue.INSTANCES", [arr]), \ + patch("doctor.state.CHURN_LIMIT", 0): + check_queue() + with _state.state_transaction() as s: + self.assertIn("7", s.get("sonarr", {})) + + +class CheckQueueMaxActionsTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + _state.STATE_FILE = os.path.join(self.tmp.name, "state.json") + + def test_max_actions_caps_removals(self): + """With MAX_ACTIONS=2, only two items are removed per sweep.""" + arr = _make_arr() + arr.queue.return_value = [ + _rec(i, status="downloadClientUnavailable") for i in range(1, 6) + ] + # pre-fill strikes so all 5 items are at MIN_STRIKES on the second run + with _patch_queue(MIN_STRIKES=2, MAX_ACTIONS=2), \ + patch("doctor.checks.queue.INSTANCES", [arr]), \ + patch("doctor.state.CHURN_LIMIT", 0): + check_queue() # strike 1 for all + arr.remove.reset_mock() + check_queue() # strike 2 → eligible, but capped at 2 + self.assertEqual(arr.remove.call_count, 2) + + +class CheckQueueOnlyFilterTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + _state.STATE_FILE = os.path.join(self.tmp.name, "state.json") + + def test_only_filters_to_named_instance(self): + arr1 = _make_arr(name="sonarr") + arr2 = _make_arr(name="radarr", kind="radarr") + arr1.queue.return_value = [_rec(1, status="downloadClientUnavailable")] + arr2.queue.return_value = [_rec(2, status="downloadClientUnavailable")] + with _patch_queue(MIN_STRIKES=1), \ + patch("doctor.checks.queue.INSTANCES", [arr1, arr2]), \ + patch("doctor.state.CHURN_LIMIT", 0): + check_queue(only="sonarr") + arr1.remove.assert_called_once_with(1) + arr2.remove.assert_not_called() + + def test_only_is_case_insensitive(self): + arr = _make_arr(name="Sonarr") + arr.queue.return_value = [_rec(1, status="downloadClientUnavailable")] + with _patch_queue(MIN_STRIKES=1), \ + patch("doctor.checks.queue.INSTANCES", [arr]), \ + patch("doctor.state.CHURN_LIMIT", 0): + check_queue(only="sonarr") + arr.remove.assert_called_once_with(1) + + +class CheckQueueRemoveFailureTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + _state.STATE_FILE = os.path.join(self.tmp.name, "state.json") + + def test_remove_exception_is_caught(self): + """A remove() exception must not abort the entire sweep.""" + arr = _make_arr() + arr.queue.return_value = [ + _rec(1, status="downloadClientUnavailable"), + _rec(2, status="downloadClientUnavailable"), + ] + arr.remove.side_effect = [Exception("network error"), None] + with _patch_queue(MIN_STRIKES=1), \ + patch("doctor.checks.queue.INSTANCES", [arr]), \ + patch("doctor.state.CHURN_LIMIT", 0): + check_queue() + # Both removes were attempted despite the first failure + self.assertEqual(arr.remove.call_count, 2) + + +class CheckQueueQueueNoneTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + _state.STATE_FILE = os.path.join(self.tmp.name, "state.json") + + def test_none_queue_response_skips_arr(self): + """arr.queue() returning None (API down) must be handled gracefully.""" + arr = _make_arr() + arr.queue.return_value = None + with _patch_queue(MIN_STRIKES=1), \ + patch("doctor.checks.queue.INSTANCES", [arr]), \ + patch("doctor.state.CHURN_LIMIT", 0): + check_queue() # must not raise + arr.remove.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_repair_dead_symlinks.py b/tests/test_repair_dead_symlinks.py new file mode 100644 index 0000000..5625c0c --- /dev/null +++ b/tests/test_repair_dead_symlinks.py @@ -0,0 +1,717 @@ +"""Characterization tests for doctor.checks.repair.dead_symlinks. + +Lock in the current behavior of: + - _radarr_dead_files(): filtering / yielding dead movie files + - _sonarr_dead_files(): filtering / yielding dead episode files per season + - _repair_radarr_movie(): delete + toggle + search orchestration + - _repair_sonarr_season(): delete + toggle + SeasonSearch orchestration + +All filesystem access (_dead_symlink) is patched so tests run without real +symlinks. Arr instances are MagicMock. Config globals are patched on the +dead_symlinks module (they arrive via star-import). +""" +import unittest +from datetime import datetime, timedelta, timezone +from unittest.mock import MagicMock, patch + +from doctor.checks.repair.dead_symlinks import ( + _radarr_dead_files, + _sonarr_dead_files, + _repair_radarr_movie, + _repair_sonarr_season, +) + +# Module path prefix for patching star-imported names +_MOD = "doctor.checks.repair.dead_symlinks" + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _movie(mid, title, path, monitored=True, mfid=None): + """Minimal Radarr movie dict.""" + mf = {"path": path} + if mfid is not None: + mf["id"] = mfid + return {"id": mid, "title": title, "monitored": monitored, "movieFile": mf} + + +def _series(sid, title, monitored=True): + return {"id": sid, "title": title, "monitored": monitored} + + +def _efile(efid, path, season_number=None): + """Minimal Sonarr episode file dict.""" + ef = {"id": efid, "path": path} + if season_number is not None: + ef["seasonNumber"] = season_number + return ef + + +def _episode(epid, efid, season_number, episode_number=None): + """Minimal Sonarr episode dict.""" + return {"id": epid, "episodeFileId": efid, "seasonNumber": season_number, "episodeNumber": episode_number} + + +def _make_arr(name="sonarr-1", kind="sonarr"): + arr = MagicMock() + arr.name = name + arr.kind = kind + return arr + + +# --------------------------------------------------------------------------- +# _radarr_dead_files +# --------------------------------------------------------------------------- + +class RadarrDeadFilesTest(unittest.TestCase): + + @patch(_MOD + "._dead_symlink", return_value=True) + def test_yields_dead_movie(self, _ds): + movies = [_movie(1, "Dead Movie", "/lib/dead.mkv", mfid=10)] + result = list(_radarr_dead_files(movies)) + self.assertEqual(result, [(1, "Dead Movie", 10)]) + + @patch(_MOD + "._dead_symlink", return_value=False) + def test_skips_live_symlink(self, _ds): + movies = [_movie(1, "Live Movie", "/lib/live.mkv", mfid=10)] + self.assertEqual(list(_radarr_dead_files(movies)), []) + + @patch(_MOD + "._dead_symlink", return_value=True) + @patch(_MOD + ".REPAIR_UNMONITORED", False) + def test_skips_unmonitored_when_flag_off(self, _ds): + movies = [_movie(1, "Unmon Movie", "/lib/x.mkv", monitored=False, mfid=10)] + self.assertEqual(list(_radarr_dead_files(movies)), []) + + @patch(_MOD + "._dead_symlink", return_value=True) + @patch(_MOD + ".REPAIR_UNMONITORED", True) + def test_includes_unmonitored_when_flag_on(self, _ds): + movies = [_movie(1, "Unmon Movie", "/lib/x.mkv", monitored=False, mfid=10)] + result = list(_radarr_dead_files(movies)) + self.assertEqual(len(result), 1) + self.assertEqual(result[0][0], 1) + + @patch(_MOD + "._dead_symlink", return_value=True) + @patch(_MOD + ".REPAIR_LIBS", ["/allowed/"]) + def test_repair_libs_filters_path(self, _ds): + movies = [ + _movie(1, "Allowed", "/allowed/a.mkv", mfid=10), + _movie(2, "Blocked", "/other/b.mkv", mfid=20), + ] + result = list(_radarr_dead_files(movies)) + self.assertEqual(len(result), 1) + self.assertEqual(result[0][0], 1) + + @patch(_MOD + "._dead_symlink", return_value=True) + @patch(_MOD + ".REPAIR_LIBS", []) + def test_empty_repair_libs_allows_all(self, _ds): + """Empty REPAIR_LIBS means no path filter -> all paths pass.""" + movies = [_movie(1, "Any", "/any/path.mkv", mfid=10)] + self.assertEqual(len(list(_radarr_dead_files(movies))), 1) + + @patch(_MOD + "._dead_symlink", return_value=True) + def test_skips_movie_without_id(self, _ds): + movies = [{"title": "No ID", "movieFile": {"path": "/x.mkv"}}] + self.assertEqual(list(_radarr_dead_files(movies)), []) + + @patch(_MOD + "._dead_symlink", return_value=True) + def test_skips_movie_without_file_path(self, _ds): + movies = [{"id": 1, "title": "No Path", "movieFile": {}}] + self.assertEqual(list(_radarr_dead_files(movies)), []) + + @patch(_MOD + "._dead_symlink", return_value=True) + def test_skips_movie_with_no_movie_file(self, _ds): + movies = [{"id": 1, "title": "No File"}] + self.assertEqual(list(_radarr_dead_files(movies)), []) + + @patch(_MOD + "._dead_symlink", return_value=True) + def test_title_truncated_to_70_chars(self, _ds): + long_title = "A" * 100 + movies = [_movie(1, long_title, "/lib/x.mkv", mfid=10)] + result = list(_radarr_dead_files(movies)) + self.assertEqual(len(result[0][1]), 70) + + +# --------------------------------------------------------------------------- +# _sonarr_dead_files +# --------------------------------------------------------------------------- + +class SonarrDeadFilesTest(unittest.TestCase): + + def _setup_arr(self, efiles, eps): + arr = _make_arr() + arr.episode_files.return_value = efiles + arr.episodes.return_value = eps + return arr + + @patch(_MOD + "._dead_symlink", return_value=True) + def test_yields_dead_episode_files_grouped_by_season(self, _ds): + efiles = [_efile(100, "/lib/S01E01.mkv", season_number=1), + _efile(101, "/lib/S01E02.mkv", season_number=1)] + eps = [_episode(10, 100, 1), _episode(11, 101, 1)] + arr = self._setup_arr(efiles, eps) + series = [_series(5, "Show")] + + result = list(_sonarr_dead_files(arr, series)) + self.assertEqual(len(result), 1) + sid, title, sn, efids, _series_dict, _epids = result[0] + self.assertEqual(sid, 5) + self.assertEqual(sn, 1) + self.assertCountEqual(efids, [100, 101]) + + @patch(_MOD + "._dead_symlink", return_value=True) + def test_multiple_seasons_yield_separately(self, _ds): + efiles = [_efile(100, "/lib/S01E01.mkv", season_number=1), + _efile(200, "/lib/S02E01.mkv", season_number=2)] + eps = [_episode(10, 100, 1), _episode(20, 200, 2)] + arr = self._setup_arr(efiles, eps) + series = [_series(5, "Show")] + + result = list(_sonarr_dead_files(arr, series)) + self.assertEqual(len(result), 2) + seasons = {r[2] for r in result} + self.assertEqual(seasons, {1, 2}) + + @patch(_MOD + "._dead_symlink", return_value=False) + def test_skips_live_files(self, _ds): + efiles = [_efile(100, "/lib/S01E01.mkv", season_number=1)] + eps = [_episode(10, 100, 1)] + arr = self._setup_arr(efiles, eps) + series = [_series(5, "Show")] + self.assertEqual(list(_sonarr_dead_files(arr, series)), []) + + @patch(_MOD + "._dead_symlink", return_value=True) + @patch(_MOD + ".REPAIR_UNMONITORED", False) + def test_skips_unmonitored_series(self, _ds): + efiles = [_efile(100, "/lib/S01E01.mkv", season_number=1)] + eps = [_episode(10, 100, 1)] + arr = self._setup_arr(efiles, eps) + series = [_series(5, "UnmonShow", monitored=False)] + self.assertEqual(list(_sonarr_dead_files(arr, series)), []) + + @patch(_MOD + "._dead_symlink", return_value=True) + @patch(_MOD + ".REPAIR_UNMONITORED", True) + def test_includes_unmonitored_series_when_flag_on(self, _ds): + efiles = [_efile(100, "/lib/S01E01.mkv", season_number=1)] + eps = [_episode(10, 100, 1)] + arr = self._setup_arr(efiles, eps) + series = [_series(5, "UnmonShow", monitored=False)] + self.assertEqual(len(list(_sonarr_dead_files(arr, series))), 1) + + @patch(_MOD + "._dead_symlink", return_value=True) + @patch(_MOD + ".REPAIR_LIBS", ["/allowed/"]) + def test_repair_libs_filters_episode_paths(self, _ds): + efiles = [_efile(100, "/allowed/S01E01.mkv", season_number=1), + _efile(101, "/other/S01E02.mkv", season_number=1)] + eps = [_episode(10, 100, 1), _episode(11, 101, 1)] + arr = self._setup_arr(efiles, eps) + series = [_series(5, "Show")] + + result = list(_sonarr_dead_files(arr, series)) + # Only the /allowed/ file should be included + self.assertEqual(len(result), 1) + self.assertEqual(result[0][3], [100]) + self.assertEqual(result[0][5], [10]) + + @patch(_MOD + "._dead_symlink", return_value=True) + def test_season_from_episode_cross_reference(self, _ds): + """Episode file without seasonNumber falls back to episode cross-reference.""" + efiles = [_efile(100, "/lib/S01E01.mkv")] # no seasonNumber on the file + eps = [_episode(10, 100, 1)] # episode knows season 1 + arr = self._setup_arr(efiles, eps) + series = [_series(5, "Show")] + + result = list(_sonarr_dead_files(arr, series)) + self.assertEqual(len(result), 1) + self.assertEqual(result[0][2], 1) # season number from cross-reference + self.assertEqual(result[0][5], [10]) + + @patch(_MOD + "._dead_symlink", return_value=True) + def test_skips_file_without_id(self, _ds): + efiles = [{"path": "/lib/S01E01.mkv", "seasonNumber": 1}] # no "id" + eps = [] + arr = self._setup_arr(efiles, eps) + series = [_series(5, "Show")] + self.assertEqual(list(_sonarr_dead_files(arr, series)), []) + + @patch(_MOD + "._dead_symlink", return_value=True) + def test_skips_file_without_season(self, _ds): + """File with no seasonNumber and no cross-reference is skipped.""" + efiles = [_efile(100, "/lib/orphan.mkv")] # no seasonNumber + eps = [] # no episode cross-reference either + arr = self._setup_arr(efiles, eps) + series = [_series(5, "Show")] + self.assertEqual(list(_sonarr_dead_files(arr, series)), []) + + @patch(_MOD + "._dead_symlink", return_value=True) + def test_continues_on_episode_files_exception(self, _ds): + """If arr.episode_files() raises, the series is skipped silently.""" + arr = _make_arr() + arr.episode_files.side_effect = Exception("API error") + arr.episodes.return_value = [] + series = [_series(5, "Show"), _series(6, "Show2")] + + # The generator should not raise; it just skips the broken series + result = list(_sonarr_dead_files(arr, series)) + self.assertEqual(result, []) + + @patch(_MOD + "._dead_symlink", return_value=True) + def test_skips_series_without_id(self, _ds): + series = [{"title": "No ID", "monitored": True}] + arr = _make_arr() + self.assertEqual(list(_sonarr_dead_files(arr, series)), []) + + +# --------------------------------------------------------------------------- +# _repair_radarr_movie +# --------------------------------------------------------------------------- + +class RepairRadarrMovieTest(unittest.TestCase): + + @patch(_MOD + ".REPAIR_VERIFY", False) + @patch(_MOD + ".DRY_RUN", False) + def test_deletes_file_toggles_monitor_and_searches(self): + arr = _make_arr("radarr-1", "radarr") + arr.command.return_value = 42 + + result = _repair_radarr_movie(arr, mid=1, title="Movie", mfid=10) + self.assertTrue(result) + arr.delete_file.assert_called_once_with(10) + arr.set_monitored.assert_any_call([1], False) + arr.set_monitored.assert_any_call([1], True) + arr.command.assert_called_once_with("MoviesSearch", movieIds=[1]) + + @patch(_MOD + ".REPAIR_VERIFY", False) + @patch(_MOD + ".DRY_RUN", False) + def test_skips_delete_when_no_mfid(self): + arr = _make_arr("radarr-1", "radarr") + arr.command.return_value = 42 + + _repair_radarr_movie(arr, mid=1, title="Movie", mfid=None) + arr.delete_file.assert_not_called() + # toggle and search still happen + arr.command.assert_called_once() + + @patch(_MOD + ".REPAIR_VERIFY", False) + @patch(_MOD + ".DRY_RUN", True) + def test_dry_run_does_not_call_arr(self): + arr = _make_arr("radarr-1", "radarr") + + result = _repair_radarr_movie(arr, mid=1, title="Movie", mfid=10) + self.assertTrue(result) + arr.delete_file.assert_not_called() + arr.set_monitored.assert_not_called() + arr.command.assert_not_called() + + @patch(_MOD + ".REPAIR_VERIFY", False) + @patch(_MOD + ".DRY_RUN", False) + def test_monitor_toggle_failure_does_not_abort(self): + """If set_monitored raises, repair still issues the search command.""" + arr = _make_arr("radarr-1", "radarr") + arr.set_monitored.side_effect = Exception("API down") + arr.command.return_value = 42 + + result = _repair_radarr_movie(arr, mid=1, title="Movie", mfid=10) + self.assertTrue(result) + arr.command.assert_called_once_with("MoviesSearch", movieIds=[1]) + + @patch(_MOD + "._repair_record_verify") + @patch(_MOD + ".REPAIR_VERIFY", True) + @patch(_MOD + ".DRY_RUN", False) + def test_verify_records_when_enabled(self, mock_verify): + arr = _make_arr("radarr-1", "radarr") + arr.command.return_value = 42 + state = {} + + _repair_radarr_movie(arr, mid=1, title="Movie", mfid=10, state=state) + mock_verify.assert_called_once_with(state, arr, "Movie", 42, 1, [1], hierarchical=False) + + @patch(_MOD + "._repair_record_verify") + @patch(_MOD + ".REPAIR_VERIFY", True) + @patch(_MOD + ".DRY_RUN", False) + def test_verify_skipped_when_state_is_none(self, mock_verify): + arr = _make_arr("radarr-1", "radarr") + arr.command.return_value = 42 + + _repair_radarr_movie(arr, mid=1, title="Movie", mfid=10, state=None) + mock_verify.assert_not_called() + + +# --------------------------------------------------------------------------- +# _repair_sonarr_season +# --------------------------------------------------------------------------- + +class RepairSonarrSeasonTest(unittest.TestCase): + + @patch(_MOD + ".REPAIR_VERIFY", False) + @patch(_MOD + ".DRY_RUN", False) + def test_deletes_all_efids_and_searches_season(self): + arr = _make_arr("sonarr-1", "sonarr") + arr.episodes.return_value = [ + {"id": 10, "seasonNumber": 1}, + {"id": 11, "seasonNumber": 1}, + {"id": 20, "seasonNumber": 2}, + ] + arr.command.return_value = 99 + + result = _repair_sonarr_season(arr, sid=5, title="Show", season_number=1, + efids=[100, 101]) + self.assertTrue(result) + # Both episode files deleted + self.assertEqual(arr.delete_file.call_count, 2) + arr.delete_file.assert_any_call(100) + arr.delete_file.assert_any_call(101) + # Monitor toggle on episodes for season 1 only + arr.set_monitored.assert_any_call([10, 11], False) + arr.set_monitored.assert_any_call([10, 11], True) + # Season search + arr.command.assert_called_once_with("SeasonSearch", seriesId=5, seasonNumber=1) + + @patch(_MOD + ".REPAIR_VERIFY", False) + @patch(_MOD + ".DRY_RUN", True) + def test_dry_run_does_not_call_arr(self): + arr = _make_arr("sonarr-1", "sonarr") + + result = _repair_sonarr_season(arr, sid=5, title="Show", season_number=1, + efids=[100, 101]) + self.assertTrue(result) + arr.delete_file.assert_not_called() + arr.set_monitored.assert_not_called() + arr.command.assert_not_called() + + @patch(_MOD + ".REPAIR_VERIFY", False) + @patch(_MOD + ".DRY_RUN", False) + def test_monitor_toggle_failure_does_not_abort(self): + arr = _make_arr("sonarr-1", "sonarr") + arr.episodes.return_value = [{"id": 10, "seasonNumber": 1}] + arr.set_monitored.side_effect = Exception("API down") + arr.command.return_value = 99 + + result = _repair_sonarr_season(arr, sid=5, title="Show", season_number=1, + efids=[100]) + self.assertTrue(result) + arr.command.assert_called_once_with("SeasonSearch", seriesId=5, seasonNumber=1) + + @patch(_MOD + "._repair_record_verify") + @patch(_MOD + ".REPAIR_VERIFY", True) + @patch(_MOD + ".DRY_RUN", False) + def test_verify_records_when_enabled(self, mock_verify): + arr = _make_arr("sonarr-1", "sonarr") + arr.episodes.return_value = [{"id": 10, "seasonNumber": 1}] + arr.command.return_value = 99 + state = {} + + _repair_sonarr_season(arr, sid=5, title="Show", season_number=1, + efids=[100], state=state) + mock_verify.assert_called_once_with(state, arr, "Show", 99, 5, [10], + strategy='season', season_number=1, series_id=5, + hierarchical=False) + + @patch(_MOD + "._repair_record_verify") + @patch(_MOD + ".REPAIR_VERIFY", True) + @patch(_MOD + ".DRY_RUN", False) + def test_verify_skipped_when_state_is_none(self, mock_verify): + arr = _make_arr("sonarr-1", "sonarr") + arr.episodes.return_value = [] + arr.command.return_value = 99 + + _repair_sonarr_season(arr, sid=5, title="Show", season_number=1, + efids=[100], state=None) + mock_verify.assert_not_called() + + @patch(_MOD + ".REPAIR_VERIFY", False) + @patch(_MOD + ".DRY_RUN", False) + def test_empty_epids_still_searches(self): + """If no episodes match the season (edge case), toggle is skipped but search runs.""" + arr = _make_arr("sonarr-1", "sonarr") + arr.episodes.return_value = [] # no episodes for this season + arr.command.return_value = 99 + + result = _repair_sonarr_season(arr, sid=5, title="Show", season_number=1, + efids=[100]) + self.assertTrue(result) + arr.set_monitored.assert_not_called() + arr.command.assert_called_once() + + +if __name__ == "__main__": + unittest.main() + + +# --------------------------------------------------------------------------- +# Hierarchical search strategy +# --------------------------------------------------------------------------- + +class HierarchicalSearchStrategyTest(unittest.TestCase): + """Test the smart command selection for dead seasons based on airing status.""" + + @patch(_MOD + ".REPAIR_HIERARCHICAL_SEARCH", True) + def test_ended_show_uses_series_search(self): + arr = _make_arr("sonarr-1", "sonarr") + arr.episodes.return_value = [{"id": 10, "seasonNumber": 1}] + arr.command.return_value = 99 + series = {"id": 5, "title": "Ended Show", "ended": True} + + _repair_sonarr_season(arr, sid=5, title="Ended Show", season_number=1, + efids=[100], state=None, series=series) + arr.command.assert_called_once_with("SeriesSearch", seriesId=5) + + @patch(_MOD + ".REPAIR_HIERARCHICAL_SEARCH", True) + def test_continuing_show_with_ended_season_uses_season_search(self): + arr = _make_arr("sonarr-1", "sonarr") + arr.episodes.return_value = [{"id": 10, "seasonNumber": 1}] + arr.command.return_value = 99 + # previousAiring 30 days ago -> season ended + prev = (datetime.now(timezone.utc) - timedelta(days=30)).strftime("%Y-%m-%dT%H:%M:%SZ") + series = { + "id": 5, "title": "Continuing Show", "ended": False, + "seasons": [{"seasonNumber": 1, "statistics": {"previousAiring": prev}}] + } + + _repair_sonarr_season(arr, sid=5, title="Continuing Show", season_number=1, + efids=[100], state=None, series=series) + arr.command.assert_called_once_with("SeasonSearch", seriesId=5, seasonNumber=1) + + @patch(_MOD + ".REPAIR_HIERARCHICAL_SEARCH", True) + def test_continuing_show_with_ongoing_season_uses_episode_search(self): + arr = _make_arr("sonarr-1", "sonarr") + arr.episodes.return_value = [{"id": 10, "seasonNumber": 1}, {"id": 11, "seasonNumber": 1}] + arr.command.return_value = 99 + # previousAiring 1 day ago -> ongoing season + prev = (datetime.now(timezone.utc) - timedelta(days=1)).strftime("%Y-%m-%dT%H:%M:%SZ") + series = { + "id": 5, "title": "Continuing Show", "ended": False, + "seasons": [{"seasonNumber": 1, "statistics": {"previousAiring": prev}}] + } + + _repair_sonarr_season(arr, sid=5, title="Continuing Show", season_number=1, + efids=[100], state=None, series=series) + arr.command.assert_called_once_with("EpisodeSearch", episodeIds=[10, 11]) + + @patch(_MOD + ".REPAIR_HIERARCHICAL_SEARCH", False) + def test_disabled_hierarchical_defaults_to_season_search(self): + arr = _make_arr("sonarr-1", "sonarr") + arr.episodes.return_value = [{"id": 10, "seasonNumber": 1}] + arr.command.return_value = 99 + series = {"id": 5, "title": "Show", "ended": True} + + _repair_sonarr_season(arr, sid=5, title="Show", season_number=1, + efids=[100], state=None, series=series) + arr.command.assert_called_once_with("SeasonSearch", seriesId=5, seasonNumber=1) + + @patch(_MOD + ".REPAIR_HIERARCHICAL_SEARCH", True) + @patch(_MOD + ".REPAIR_VERIFY", True) + @patch(_MOD + ".DRY_RUN", False) + def test_records_strategy_and_season_in_verify(self, *_): + from doctor.checks.repair.verify import _repair_record_verify + arr = _make_arr("sonarr-1", "sonarr") + arr.episodes.return_value = [{"id": 10, "seasonNumber": 1}] + arr.command.return_value = 99 + state = {} + series = {"id": 5, "title": "Ended Show", "ended": True} + + _repair_sonarr_season(arr, sid=5, title="Ended Show", season_number=1, + efids=[100], state=state, series=series) + arr.command.assert_called_once_with("SeriesSearch", seriesId=5) + # verify state should record strategy and season + pv = state.get("__repair_verify__", {}) + key = "sonarr-1:ended_show:s01" + self.assertIn(key, pv) + self.assertEqual(pv[key]["strategy"], "series") + self.assertEqual(pv[key]["season_number"], 1) + self.assertEqual(pv[key]["series_id"], 5) + self.assertTrue(pv[key].get("hierarchical")) + + +# --------------------------------------------------------------------------- +# Janitor-reported dead files +# --------------------------------------------------------------------------- + +class JanitorDeadFilesTest(unittest.TestCase): + + def _setup_arr(self, efiles, eps): + arr = _make_arr() + arr.episode_files.return_value = efiles + arr.episodes.return_value = eps + return arr + + @patch(_MOD + "._dead_symlink", return_value=False) + def test_janitor_dead_file_without_episode_file_record(self, _ds): + """When the janitor has already removed the file, _sonarr_dead_files should still + detect the missing episode from the quarantined orig path in the state.""" + efiles = [] # episode file already deleted + eps = [_episode(10, None, 1, episode_number=2)] # no episodeFileId + arr = self._setup_arr(efiles, eps) + series = [_series(5, "Show", monitored=True)] + series[0]["path"] = "/lib/shows/Show" + state = { + "__janitor_dead_files__": { + "RELEASE/Show.S01E02.1080p.mkv": { + "ts": 0, + "orig": "/lib/shows/Show/Season 01/Show - S01E02.mkv", + "target": "/mnt/zurg/__all__/RELEASE/Show.S01E02.1080p.mkv", + } + } + } + + result = list(_sonarr_dead_files(arr, series, state=state)) + self.assertEqual(len(result), 1) + sid, title, sn, efids, _series_dict, epids = result[0] + self.assertEqual(sid, 5) + self.assertEqual(sn, 1) + self.assertEqual(efids, []) + self.assertEqual(epids, [10]) + + @patch(_MOD + "._dead_symlink", return_value=False) + def test_janitor_dead_file_fallback_by_release_name(self, _ds): + """When the janitor entry has no orig path, the repair check can still find the + episode by guessing the series from the release name and parsing the filename.""" + efiles = [] + eps = [_episode(10, None, 1, episode_number=2)] + arr = self._setup_arr(efiles, eps) + series = [_series(5, "Mr. Robot", monitored=True)] + series[0]["sortTitle"] = "mrrobot" + state = { + "__janitor_dead_files__": { + "Mr.Robot.S01-S04.1080p.BluRay.DD5.1.x264-MIXED/Mr.Robot.S01E02.1080p.BluRay.DTS.x264-SbR.mkv": { + "ts": 0, + "orig": None, + "target": None, + } + } + } + result = list(_sonarr_dead_files(arr, series, state=state)) + self.assertEqual(len(result), 1) + sid, title, sn, efids, _series_dict, epids = result[0] + self.assertEqual(sid, 5) + self.assertEqual(sn, 1) + self.assertEqual(efids, []) + self.assertEqual(epids, [10]) + + @patch(_MOD + "._dead_symlink", return_value=False) + def test_janitor_dead_file_ignored_without_orig_path(self, _ds): + efiles = [] + eps = [] + arr = self._setup_arr(efiles, eps) + series = [_series(5, "Show")] + state = { + "__janitor_dead_files__": { + "RELEASE/Show.S01E02.1080p.mkv": { + "ts": 0, + "orig": None, + "target": None, + } + } + } + result = list(_sonarr_dead_files(arr, series, state=state)) + self.assertEqual(result, []) + + @patch(_MOD + "._dead_symlink", return_value=False) + def test_janitor_dead_file_matches_symlink_target(self, _ds): + """A live-looking symlink whose target is recorded as dead by the janitor is treated as dead.""" + import os + import tempfile + with tempfile.TemporaryDirectory() as libdir: + fp = os.path.join(libdir, "Show - S01E02.mkv") + os.symlink("/mnt/zurg/__all__/RELEASE/Show.S01E02.1080p.mkv", fp) + efiles = [_efile(100, fp, season_number=1)] + eps = [_episode(10, 100, 1, episode_number=2)] + arr = self._setup_arr(efiles, eps) + series = [_series(5, "Show")] + series[0]["path"] = libdir + state = { + "__janitor_dead_files__": { + "RELEASE/Show.S01E02.1080p.mkv": { + "ts": 0, + "orig": fp, + "target": "/mnt/zurg/__all__/RELEASE/Show.S01E02.1080p.mkv", + } + } + } + result = list(_sonarr_dead_files(arr, series, state=state)) + self.assertEqual(len(result), 1) + self.assertEqual(result[0][3], [100]) + self.assertEqual(result[0][5], [10]) + + +class ParseJanitorDeadPathTest(unittest.TestCase): + + def test_parses_standard_path(self): + from doctor.checks.repair.dead_symlinks import _parse_janitor_dead_path + series = [{"id": 5, "title": "Show", "path": "/lib/shows/Show (2020) {imdb-tt123}"}] + orig = "/lib/shows/Show (2020) {imdb-tt123}/Season 02/Show (2020) - S02E05.mkv" + ser, sn, eps = _parse_janitor_dead_path(orig, series) + self.assertEqual(ser["id"], 5) + self.assertEqual(sn, 2) + self.assertEqual(eps, [5]) + + def test_multi_episode_filename(self): + from doctor.checks.repair.dead_symlinks import _parse_janitor_dead_path + series = [{"id": 5, "title": "Show", "path": "/lib/shows/Show"}] + orig = "/lib/shows/Show/Season 01/Show - S01E01-E02.mkv" + ser, sn, eps = _parse_janitor_dead_path(orig, series) + self.assertEqual(sn, 1) + self.assertEqual(eps, [1, 2]) + + def test_no_match(self): + from doctor.checks.repair.dead_symlinks import _parse_janitor_dead_path + series = [{"id": 5, "title": "Show", "path": "/lib/shows/Show"}] + self.assertIsNone(_parse_janitor_dead_path("/other/path/Show/Season 01/Show - S01E01.mkv", series)) + + +class RepairSonarrSeasonEpidsTest(unittest.TestCase): + + @patch(_MOD + ".REPAIR_HIERARCHICAL_SEARCH", True) + @patch(_MOD + ".REPAIR_VERIFY", False) + @patch(_MOD + ".DRY_RUN", False) + def test_passed_epids_used_for_episode_search(self): + arr = _make_arr("sonarr-1", "sonarr") + arr.episodes.return_value = [ + {"id": 10, "seasonNumber": 1}, + {"id": 11, "seasonNumber": 1}, + ] + arr.command.return_value = 99 + series = {"id": 5, "title": "Show", "ended": False, "seasons": []} + + _repair_sonarr_season(arr, sid=5, title="Show", season_number=1, + efids=[], state=None, series=series, epids=[10]) + arr.command.assert_called_once_with("EpisodeSearch", episodeIds=[10]) + + + +class GuessSeriesFromReleaseTest(unittest.TestCase): + + def test_exact_match(self): + from doctor.checks.repair.dead_symlinks import _guess_series_from_release + series = [{"id": 5, "title": "Mr. Robot", "sortTitle": "mrrobot"}] + ser = _guess_series_from_release("Mr.Robot.S01-S04.1080p.BluRay.DD5.1.x264-MIXED", series) + self.assertEqual(ser["id"], 5) + + def test_dotted_title_normalized(self): + from doctor.checks.repair.dead_symlinks import _guess_series_from_release + series = [{"id": 5, "title": "My Dress-Up Darling", "sortTitle": "my dress up darling"}] + ser = _guess_series_from_release("My.Dress-Up.Darling.S02.1080p.BluRay.Remux.DUAL.FLAC.2.0.AVC-DemiHuman", series) + self.assertEqual(ser["id"], 5) + + def test_no_match(self): + from doctor.checks.repair.dead_symlinks import _guess_series_from_release + series = [{"id": 5, "title": "Other Show", "sortTitle": "other show"}] + ser = _guess_series_from_release("Mr.Robot.S01.1080p.mkv", series) + self.assertIsNone(ser) + + +class ParseEpisodesFromFilenameTest(unittest.TestCase): + + def test_single_episode(self): + from doctor.checks.repair.dead_symlinks import _parse_episodes_from_filename + self.assertEqual(_parse_episodes_from_filename("Show.S01E05.1080p.mkv", 1), [5]) + + def test_episode_range(self): + from doctor.checks.repair.dead_symlinks import _parse_episodes_from_filename + self.assertEqual(_parse_episodes_from_filename("Show.S01E01-E02.1080p.mkv", 1), [1, 2]) + self.assertEqual(_parse_episodes_from_filename("Show.S01E01E02.1080p.mkv", 1), [1, 2]) + + def test_different_season_ignored(self): + from doctor.checks.repair.dead_symlinks import _parse_episodes_from_filename + self.assertEqual(_parse_episodes_from_filename("Show.S02E05.1080p.mkv", 1), []) diff --git a/tests/test_repair_main.py b/tests/test_repair_main.py new file mode 100644 index 0000000..ad7b4a1 --- /dev/null +++ b/tests/test_repair_main.py @@ -0,0 +1,268 @@ +"""Unit tests for doctor.checks.repair.main.check_repair() orchestrator. + +Tests cover: + - returns early when INSTANCES is empty + - returns early when host load exceeds REPAIR_LOAD_MAX + - returns early when debrid mount is not OK + - calls _repair_verify_pending when REPAIR_VERIFY is True + - does NOT call _repair_verify_pending when REPAIR_VERIFY is False + - processes Sonarr dead symlinks and counts acted/symlinks + - processes Radarr dead symlinks + - stops at REPAIR_MAX_ACTIONS cap (symlink sweep) + - stops at REPAIR_MAX_SYMLINKS cap + - skips REPAIR_SEASON_PACKS sub-check when disabled + - runs REPAIR_SEASON_PACKS sub-check and calls SeasonSearch + - skips REPAIR_MISSING_FROM_DISK sub-check when disabled + - calls _missing_from_disk_check when REPAIR_MISSING_FROM_DISK is True + - calls _orphan_dead_symlink_scan when REPAIR_ORPHAN_SCAN is True + - skips _orphan_dead_symlink_scan when disabled + - handles per-arr sweep exceptions without crashing + +No real filesystem or network access. +""" +import unittest +from unittest.mock import MagicMock, patch + +_MOD = "doctor.checks.repair.main" + + +def _make_arr(name="sonarr-1", kind="sonarr"): + arr = MagicMock() + arr.name = name + arr.kind = kind + arr.series.return_value = [] + arr.movies.return_value = [] + return arr + + +def _run(instances, state=None, *, + repair_load_max=0, + host_load_val=0.0, + mount_ok=True, + repair_verify=False, + repair_max_actions=10, + repair_max_symlinks=50, + repair_season_packs=False, + repair_missing_from_disk=False, + repair_orphan_scan=False, + repair_item_interval=0, + dry_run=False, + # sub-function stubs + sonarr_dead_files=None, + radarr_dead_files=None, + repair_sonarr_season_ret=True, + repair_radarr_movie_ret=True, + season_pack_entries=None, + mfd_acted=0, + verify_pending_mock=None, + orphan_mock=None, + ): + if state is None: + state = {} + if sonarr_dead_files is None: + sonarr_dead_files = [] + if radarr_dead_files is None: + radarr_dead_files = [] + if season_pack_entries is None: + season_pack_entries = [] + + mock_sonarr_dead = MagicMock(return_value=iter(sonarr_dead_files)) + mock_radarr_dead = MagicMock(return_value=iter(radarr_dead_files)) + mock_repair_sonarr = MagicMock(return_value=repair_sonarr_season_ret) + mock_repair_radarr = MagicMock(return_value=repair_radarr_movie_ret) + mock_season_pack = MagicMock(return_value=iter(season_pack_entries)) + mock_mfd = MagicMock(return_value=mfd_acted) + mock_verify = verify_pending_mock or MagicMock() + mock_orphan = orphan_mock or MagicMock() + + from doctor.checks.repair.main import check_repair + + with patch(_MOD + ".INSTANCES", instances), \ + patch(_MOD + ".REPAIR_LOAD_MAX", repair_load_max), \ + patch(_MOD + ".host_load", return_value=host_load_val), \ + patch(_MOD + "._debrid_mount_ok", return_value=mount_ok), \ + patch(_MOD + ".REPAIR_VERIFY", repair_verify), \ + patch(_MOD + ".REPAIR_MAX_ACTIONS", repair_max_actions), \ + patch(_MOD + ".REPAIR_MAX_SYMLINKS", repair_max_symlinks), \ + patch(_MOD + ".REPAIR_SEASON_PACKS", repair_season_packs), \ + patch(_MOD + ".REPAIR_MISSING_FROM_DISK", repair_missing_from_disk), \ + patch(_MOD + ".REPAIR_ORPHAN_SCAN", repair_orphan_scan), \ + patch(_MOD + ".REPAIR_ITEM_INTERVAL", repair_item_interval), \ + patch(_MOD + ".DRY_RUN", dry_run), \ + patch(_MOD + "._sonarr_dead_files", mock_sonarr_dead), \ + patch(_MOD + "._radarr_dead_files", mock_radarr_dead), \ + patch(_MOD + "._repair_sonarr_season", mock_repair_sonarr), \ + patch(_MOD + "._repair_radarr_movie", mock_repair_radarr), \ + patch(_MOD + "._sonarr_season_pack_check", mock_season_pack), \ + patch(_MOD + "._missing_from_disk_check", mock_mfd), \ + patch(_MOD + "._repair_verify_pending", mock_verify), \ + patch(_MOD + "._orphan_dead_symlink_scan", mock_orphan), \ + patch(_MOD + ".state_transaction") as mock_tx: + mock_tx.return_value.__enter__ = lambda s: state + mock_tx.return_value.__exit__ = MagicMock(return_value=False) + check_repair() + + return { + "sonarr_dead": mock_sonarr_dead, + "radarr_dead": mock_radarr_dead, + "repair_sonarr": mock_repair_sonarr, + "repair_radarr": mock_repair_radarr, + "season_pack": mock_season_pack, + "mfd": mock_mfd, + "verify": mock_verify, + "orphan": mock_orphan, + } + + +class RepairMainEarlyExitTest(unittest.TestCase): + + def test_returns_when_no_instances(self): + mocks = _run([]) + mocks["sonarr_dead"].assert_not_called() + mocks["radarr_dead"].assert_not_called() + + def test_returns_when_load_too_high(self): + arr = _make_arr() + mocks = _run([arr], repair_load_max=1, host_load_val=5.0) + mocks["sonarr_dead"].assert_not_called() + + def test_passes_when_load_ok(self): + arr = _make_arr() + arr.series.return_value = [] + mocks = _run([arr], repair_load_max=10, host_load_val=1.0) + # Not called because series() returned [] + mocks["sonarr_dead"].assert_called_once() + + def test_returns_when_mount_not_ok(self): + arr = _make_arr() + mocks = _run([arr], mount_ok=False) + mocks["sonarr_dead"].assert_not_called() + + def test_skips_non_sonarr_radarr_instances(self): + arr = _make_arr(kind="prowlarr") + mocks = _run([arr]) + mocks["sonarr_dead"].assert_not_called() + mocks["radarr_dead"].assert_not_called() + + +class RepairVerifyTest(unittest.TestCase): + + def test_calls_verify_when_enabled(self): + arr = _make_arr() + mocks = _run([arr], repair_verify=True) + mocks["verify"].assert_called_once() + + def test_skips_verify_when_disabled(self): + arr = _make_arr() + mocks = _run([arr], repair_verify=False) + mocks["verify"].assert_not_called() + + +class RepairSonarrTest(unittest.TestCase): + + def test_calls_repair_sonarr_for_dead_files(self): + arr = _make_arr(kind="sonarr") + arr.series.return_value = [{"id": 1, "title": "Show"}] + dead = [(1, "Show", 1, [10, 11], {"id": 1, "title": "Show"}, [100, 101])] # sid, title, season, efids + mocks = _run([arr], sonarr_dead_files=dead) + mocks["repair_sonarr"].assert_called_once() + + def test_sonarr_increments_acted_and_symlinks(self): + arr = _make_arr(kind="sonarr") + arr.series.return_value = [{}] + # Two seasons with 1 and 2 files respectively + dead = [(1, "Show", 1, [10], {"id": 1, "title": "Show"}, [100]), (1, "Show", 2, [11, 12], {"id": 1, "title": "Show"}, [101, 102])] + mocks = _run([arr], sonarr_dead_files=dead, repair_max_actions=10, repair_max_symlinks=50) + self.assertEqual(mocks["repair_sonarr"].call_count, 2) + + def test_sonarr_stops_at_max_actions(self): + arr = _make_arr(kind="sonarr") + arr.series.return_value = [{}] + dead = [(1, "Show", 1, [10], {"id": 1, "title": "Show"}, [100]), (1, "Show", 2, [11], {"id": 1, "title": "Show"}, [101])] + mocks = _run([arr], sonarr_dead_files=dead, repair_max_actions=1) + self.assertEqual(mocks["repair_sonarr"].call_count, 1) + + def test_sonarr_stops_at_max_symlinks(self): + arr = _make_arr(kind="sonarr") + arr.series.return_value = [{}] + # 3 files in the group but symlink cap is 2 — group is too big + dead = [(1, "Show", 1, [10, 11, 12], {"id": 1, "title": "Show"}, [100, 101, 102])] + mocks = _run([arr], sonarr_dead_files=dead, repair_max_symlinks=2) + mocks["repair_sonarr"].assert_not_called() + + +class RepairRadarrTest(unittest.TestCase): + + def test_calls_repair_radarr_for_dead_files(self): + arr = _make_arr(name="radarr-1", kind="radarr") + arr.movies.return_value = [{}] + dead = [(5, "Movie", 50)] # mid, title, mfid + mocks = _run([arr], radarr_dead_files=dead) + mocks["repair_radarr"].assert_called_once() + + def test_radarr_stops_at_max_actions(self): + arr = _make_arr(name="radarr-1", kind="radarr") + arr.movies.return_value = [{}] + dead = [(5, "Movie A", 50), (6, "Movie B", 60)] + mocks = _run([arr], radarr_dead_files=dead, repair_max_actions=1) + self.assertEqual(mocks["repair_radarr"].call_count, 1) + + +class RepairSeasonPackTest(unittest.TestCase): + + def test_skips_season_pack_when_disabled(self): + arr = _make_arr() + mocks = _run([arr], repair_season_packs=False) + mocks["season_pack"].assert_not_called() + + def test_runs_season_pack_when_enabled(self): + arr = _make_arr(kind="sonarr") + arr.series.return_value = [] + entries = [("Show", 1, 10, arr)] + arr.command.return_value = True + mocks = _run([arr], repair_season_packs=True, season_pack_entries=entries) + mocks["season_pack"].assert_called_once() + arr.command.assert_called_once_with("SeasonSearch", seriesId=10, seasonNumber=1) + + def test_season_pack_dry_run_does_not_call_command(self): + arr = _make_arr(kind="sonarr") + arr.series.return_value = [] + entries = [("Show", 1, 10, arr)] + _run([arr], repair_season_packs=True, season_pack_entries=entries, dry_run=True) + arr.command.assert_not_called() + + +class RepairMfdTest(unittest.TestCase): + + def test_skips_mfd_when_disabled(self): + arr = _make_arr() + mocks = _run([arr], repair_missing_from_disk=False) + mocks["mfd"].assert_not_called() + + def test_calls_mfd_when_enabled(self): + arr = _make_arr() + mocks = _run([arr], repair_missing_from_disk=True) + mocks["mfd"].assert_called_once() + + +class RepairOrphanTest(unittest.TestCase): + + def test_skips_orphan_scan_when_disabled(self): + arr = _make_arr() + mocks = _run([arr], repair_orphan_scan=False) + mocks["orphan"].assert_not_called() + + def test_calls_orphan_scan_when_enabled(self): + arr = _make_arr() + mocks = _run([arr], repair_orphan_scan=True) + mocks["orphan"].assert_called_once() + + +class RepairExceptionHandlingTest(unittest.TestCase): + + def test_arr_sweep_exception_is_swallowed(self): + arr = _make_arr(kind="sonarr") + arr.series.side_effect = RuntimeError("API down") + # Should not raise + mocks = _run([arr]) + mocks["repair_sonarr"].assert_not_called() diff --git a/tests/test_repair_missing_from_disk.py b/tests/test_repair_missing_from_disk.py new file mode 100644 index 0000000..50abbbe --- /dev/null +++ b/tests/test_repair_missing_from_disk.py @@ -0,0 +1,285 @@ +"""Unit tests for doctor.checks.repair.missing_from_disk._missing_from_disk_check(). + +Tests cover: + - skips instance if kind is not sonarr/radarr + - skips when budget <= 0 on entry + - skips unmonitored items when REPAIR_UNMONITORED=False + - skips items within MFD recheck cooldown + - triggers SeasonSearch for a Sonarr MissingFromDisk entry + - triggers MoviesSearch for a Radarr MissingFromDisk entry + - does NOT trigger search for non-grabbed history events + - does NOT trigger search for grabbed events without reason=MissingFromDisk + - only searches each (arr, season/movie) key once per sweep + - records the timestamp in state after a search + - decrements budget after each action + - DRY_RUN: logs intent but does not call arr.command() + - handles arr.history() raising an exception (skips, does not crash) + - handles arr.series()/movies() raising an exception (skips, does not crash) + - radarr records dict unwrapped from {"records": [...]} correctly + +All external I/O is replaced with MagicMock. +""" +import time +import unittest +from unittest.mock import MagicMock, patch + +_MOD = "doctor.checks.repair.missing_from_disk" + + +def _make_arr(name="sonarr-1", kind="sonarr"): + arr = MagicMock() + arr.name = name + arr.kind = kind + arr.series.return_value = [] + arr.movies.return_value = [] + arr.history.return_value = [] + arr.command.return_value = None + return arr + + +def _series(sid, title="Show", monitored=True): + return {"id": sid, "title": title, "monitored": monitored} + + +def _movie(mid, title="Movie", monitored=True): + return {"id": mid, "title": title, "monitored": monitored} + + +def _grabbed_mfd_ep(series_id, season_number): + """Sonarr grabbed + MissingFromDisk history record.""" + return { + "eventType": "grabbed", + "episode": {"seriesId": series_id, "seasonNumber": season_number}, + "data": {"reason": "MissingFromDisk"}, + } + + +def _grabbed_mfd_movie(): + """Radarr grabbed + MissingFromDisk history record.""" + return { + "eventType": "grabbed", + "data": {"reason": "MissingFromDisk"}, + } + + +def _grabbed_ok(): + """Grabbed but NOT MissingFromDisk — should be ignored.""" + return {"eventType": "grabbed", "data": {"reason": "SomethingElse"}} + + +def _run(instances, state=None, *, dry_run=False, repair_unmonitored=True, + repair_mfd_recheck=0, repair_item_interval=0, budget=10): + """Call _missing_from_disk_check with fully patched config globals.""" + if state is None: + state = {} + + from doctor.checks.repair.missing_from_disk import _missing_from_disk_check + + with patch(_MOD + ".INSTANCES", instances), \ + patch(_MOD + ".DRY_RUN", dry_run), \ + patch(_MOD + ".REPAIR_UNMONITORED", repair_unmonitored), \ + patch(_MOD + ".REPAIR_MFD_RECHECK", repair_mfd_recheck), \ + patch(_MOD + ".REPAIR_ITEM_INTERVAL", repair_item_interval): + acted = _missing_from_disk_check(state, acted=0, budget=budget) + + return acted, state + + +class MfdSkipTest(unittest.TestCase): + + def test_skips_non_sonarr_radarr_instance(self): + arr = _make_arr(kind="prowlarr") + acted, _ = _run([arr]) + arr.series.assert_not_called() + arr.movies.assert_not_called() + self.assertEqual(acted, 0) + + def test_skips_when_budget_zero(self): + arr = _make_arr(kind="sonarr") + arr.series.return_value = [_series(1)] + arr.history.return_value = [_grabbed_mfd_ep(1, 1)] + acted, _ = _run([arr], budget=0) + arr.command.assert_not_called() + self.assertEqual(acted, 0) + + def test_skips_unmonitored_when_flag_off(self): + arr = _make_arr(kind="sonarr") + arr.series.return_value = [_series(1, monitored=False)] + arr.history.return_value = [_grabbed_mfd_ep(1, 1)] + acted, _ = _run([arr], repair_unmonitored=False) + arr.command.assert_not_called() + self.assertEqual(acted, 0) + + def test_includes_unmonitored_when_flag_on(self): + arr = _make_arr(kind="sonarr") + arr.series.return_value = [_series(1, monitored=False)] + arr.history.return_value = [_grabbed_mfd_ep(1, 1)] + acted, _ = _run([arr], repair_unmonitored=True) + arr.command.assert_called_once() + self.assertEqual(acted, 1) + + def test_skips_item_within_recheck_cooldown(self): + arr = _make_arr(kind="sonarr") + arr.series.return_value = [_series(1)] + arr.history.return_value = [_grabbed_mfd_ep(1, 1)] + state = {"__repair_mfd__": {"sonarr-1:1:s1": time.time()}} # just searched + acted, _ = _run([arr], state=state, repair_mfd_recheck=3600) + arr.command.assert_not_called() + self.assertEqual(acted, 0) + + +class MfdSonarrTest(unittest.TestCase): + + def test_triggers_season_search(self): + arr = _make_arr(kind="sonarr") + arr.series.return_value = [_series(10)] + arr.history.return_value = [_grabbed_mfd_ep(10, 2)] + acted, state = _run([arr]) + arr.command.assert_called_once_with("SeasonSearch", seriesId=10, seasonNumber=2) + self.assertEqual(acted, 1) + self.assertIn("sonarr-1:10:s2", state["__repair_mfd__"]) + + def test_only_searches_each_season_once_per_sweep(self): + arr = _make_arr(kind="sonarr") + arr.series.return_value = [_series(10)] + # Two records for the same series+season + arr.history.return_value = [_grabbed_mfd_ep(10, 2), _grabbed_mfd_ep(10, 2)] + acted, _ = _run([arr]) + self.assertEqual(arr.command.call_count, 1) + self.assertEqual(acted, 1) + + def test_does_not_trigger_for_non_grabbed_events(self): + arr = _make_arr(kind="sonarr") + arr.series.return_value = [_series(1)] + arr.history.return_value = [{"eventType": "downloadFolderImported", "data": {}}] + acted, _ = _run([arr]) + arr.command.assert_not_called() + self.assertEqual(acted, 0) + + def test_does_not_trigger_for_grabbed_without_mfd_reason(self): + arr = _make_arr(kind="sonarr") + arr.series.return_value = [_series(1)] + arr.history.return_value = [_grabbed_ok()] + acted, _ = _run([arr]) + arr.command.assert_not_called() + self.assertEqual(acted, 0) + + def test_decrements_budget(self): + arr = _make_arr(kind="sonarr") + arr.series.return_value = [_series(1), _series(2)] + arr.history.side_effect = [ + [_grabbed_mfd_ep(1, 1)], + [_grabbed_mfd_ep(2, 1)], + ] + acted, _ = _run([arr], budget=1) + # Budget of 1 → only one search + self.assertEqual(arr.command.call_count, 1) + self.assertEqual(acted, 1) + + +class MfdRadarrTest(unittest.TestCase): + + def test_triggers_movies_search(self): + arr = _make_arr(name="radarr-1", kind="radarr") + arr.movies.return_value = [_movie(5)] + arr.history.return_value = [_grabbed_mfd_movie()] + acted, state = _run([arr]) + arr.command.assert_called_once_with("MoviesSearch", movieIds=[5]) + self.assertEqual(acted, 1) + self.assertIn("radarr-1:5", state["__repair_mfd__"]) + + def test_unwraps_radarr_records_dict(self): + """Radarr wraps history in {"records": [...]}.""" + arr = _make_arr(name="radarr-1", kind="radarr") + arr.movies.return_value = [_movie(5)] + arr.history.return_value = {"records": [_grabbed_mfd_movie()]} + acted, _ = _run([arr]) + arr.command.assert_called_once() + self.assertEqual(acted, 1) + + def test_only_searches_each_movie_once_per_sweep(self): + arr = _make_arr(name="radarr-1", kind="radarr") + arr.movies.return_value = [_movie(5)] + arr.history.return_value = [_grabbed_mfd_movie(), _grabbed_mfd_movie()] + acted, _ = _run([arr]) + self.assertEqual(arr.command.call_count, 1) + self.assertEqual(acted, 1) + + +class MfdDryRunTest(unittest.TestCase): + + def test_dry_run_sonarr_does_not_call_command(self): + arr = _make_arr(kind="sonarr") + arr.series.return_value = [_series(1)] + arr.history.return_value = [_grabbed_mfd_ep(1, 3)] + acted, state = _run([arr], dry_run=True) + arr.command.assert_not_called() + self.assertEqual(acted, 1) + self.assertIn("sonarr-1:1:s3", state["__repair_mfd__"]) + + def test_dry_run_radarr_does_not_call_command(self): + arr = _make_arr(name="radarr-1", kind="radarr") + arr.movies.return_value = [_movie(5)] + arr.history.return_value = [_grabbed_mfd_movie()] + acted, state = _run([arr], dry_run=True) + arr.command.assert_not_called() + self.assertEqual(acted, 1) + + +class MfdErrorHandlingTest(unittest.TestCase): + + def test_history_exception_is_swallowed(self): + arr = _make_arr(kind="sonarr") + arr.series.return_value = [_series(1)] + arr.history.side_effect = RuntimeError("API error") + acted, _ = _run([arr]) + arr.command.assert_not_called() + self.assertEqual(acted, 0) + + def test_series_fetch_exception_is_swallowed(self): + arr = _make_arr(kind="sonarr") + arr.series.side_effect = RuntimeError("connection error") + acted, _ = _run([arr]) + self.assertEqual(acted, 0) + + def test_movies_fetch_exception_is_swallowed(self): + arr = _make_arr(name="radarr-1", kind="radarr") + arr.movies.side_effect = RuntimeError("timeout") + acted, _ = _run([arr]) + self.assertEqual(acted, 0) + + +class MfdBreakContinueBugTest(unittest.TestCase): + """Regression tests for the break/continue bug on line 14. + + When a non-sonarr/radarr instance (e.g. prowlarr) appears BEFORE a valid + Sonarr instance in INSTANCES, the old `break` would stop processing all + remaining instances. The fix uses `continue` for the kind-guard so only + that one instance is skipped. + """ + + def test_prowlarr_before_sonarr_does_not_block_sonarr(self): + """Prowlarr first, then Sonarr with an MFD entry -> Sonarr must still be processed.""" + prowlarr = _make_arr(name="prowlarr-1", kind="prowlarr") + sonarr = _make_arr(name="sonarr-1", kind="sonarr") + sonarr.series.return_value = [_series(1)] + sonarr.history.return_value = [_grabbed_mfd_ep(1, 1)] + acted, _ = _run([prowlarr, sonarr]) + # Prowlarr should be skipped (not touched), Sonarr should act + prowlarr.series.assert_not_called() + prowlarr.movies.assert_not_called() + sonarr.command.assert_called_once() + self.assertEqual(acted, 1) + + def test_budget_zero_still_breaks_out(self): + """budget=0 must still prevent any work even across multiple instances.""" + sonarr = _make_arr(name="sonarr-1", kind="sonarr") + sonarr.series.return_value = [_series(1)] + sonarr.history.return_value = [_grabbed_mfd_ep(1, 1)] + radarr = _make_arr(name="radarr-1", kind="radarr") + radarr.movies.return_value = [_movie(5)] + radarr.history.return_value = [_grabbed_mfd_movie()] + acted, _ = _run([sonarr, radarr], budget=0) + sonarr.command.assert_not_called() + radarr.command.assert_not_called() + self.assertEqual(acted, 0) diff --git a/tests/test_repair_orphan.py b/tests/test_repair_orphan.py new file mode 100644 index 0000000..a139a83 --- /dev/null +++ b/tests/test_repair_orphan.py @@ -0,0 +1,189 @@ +"""Unit tests for doctor.checks.repair.orphan. + +Tests cover: + _collect_known_paths(): + - returns empty set when INSTANCES is empty + - collects Sonarr episode file paths + - collects Radarr movie file paths + - skips instances of other kinds + - skips Sonarr series/episode_files that have no path field + - handles arr.series() / arr.movies() exceptions gracefully + + _orphan_dead_symlink_scan(): + - returns immediately when REPAIR_LIBS is empty + - logs a warning for library roots that are not directories + - does NOT report a path that is tracked by *arr (known path) + - does NOT report a path that is a live file (_dead_symlink=False) + - reports a dead symlink that is not tracked by *arr (orphan) + - caps the per-run log output at 20 individual paths + +All filesystem access (os.walk, os.path.isdir, _dead_symlink) is patched. +""" +import unittest +from unittest.mock import MagicMock, patch + +_MOD = "doctor.checks.repair.orphan" + + +def _make_arr(name="sonarr-1", kind="sonarr"): + arr = MagicMock() + arr.name = name + arr.kind = kind + return arr + + +class CollectKnownPathsTest(unittest.TestCase): + + def _run(self, instances): + from doctor.checks.repair.orphan import _collect_known_paths + with patch(_MOD + ".INSTANCES", instances): + return _collect_known_paths() + + def test_empty_when_no_instances(self): + result = self._run([]) + self.assertEqual(result, set()) + + def test_collects_sonarr_episode_paths(self): + arr = _make_arr(kind="sonarr") + arr.series.return_value = [{"id": 1}] + arr.episode_files.return_value = [{"path": "/lib/Show/ep.mkv"}] + result = self._run([arr]) + self.assertIn("/lib/Show/ep.mkv", result) + + def test_collects_radarr_movie_paths(self): + arr = _make_arr(name="radarr-1", kind="radarr") + arr.movies.return_value = [{"movieFile": {"path": "/lib/Movie/movie.mkv"}}] + result = self._run([arr]) + self.assertIn("/lib/Movie/movie.mkv", result) + + def test_skips_other_instance_kinds(self): + arr = _make_arr(kind="prowlarr") + result = self._run([arr]) + self.assertEqual(result, set()) + + def test_skips_sonarr_episode_without_path(self): + arr = _make_arr(kind="sonarr") + arr.series.return_value = [{"id": 1}] + arr.episode_files.return_value = [{"path": None}, {"no_path_key": True}] + result = self._run([arr]) + self.assertEqual(result, set()) + + def test_skips_sonarr_series_without_id(self): + arr = _make_arr(kind="sonarr") + arr.series.return_value = [{"title": "No ID here"}] + arr.episode_files.return_value = [{"path": "/lib/ep.mkv"}] + result = self._run([arr]) + # episode_files should not be called without an id + arr.episode_files.assert_not_called() + self.assertEqual(result, set()) + + def test_handles_series_exception(self): + arr = _make_arr(kind="sonarr") + arr.series.side_effect = RuntimeError("API gone") + result = self._run([arr]) + self.assertEqual(result, set()) + + def test_handles_movies_exception(self): + arr = _make_arr(name="radarr-1", kind="radarr") + arr.movies.side_effect = RuntimeError("timeout") + result = self._run([arr]) + self.assertEqual(result, set()) + + +class OrphanScanTest(unittest.TestCase): + + def _run(self, repair_libs, known_paths=None, walk_files=None, dead_symlinks=None): + """ + repair_libs: list of library root paths + known_paths: set of paths known to *arr + walk_files: dict {root: [filename, ...]} for os.walk simulation + dead_symlinks: set of paths that _dead_symlink() returns True for + """ + if known_paths is None: + known_paths = set() + if walk_files is None: + walk_files = {} + if dead_symlinks is None: + dead_symlinks = set() + + def fake_isdir(p): + return p in repair_libs + + def fake_walk(root): + files = walk_files.get(root, []) + if files: + yield root, [], files + # No recursion needed for unit tests + + def fake_dead_symlink(fp): + return fp in dead_symlinks + + from doctor.checks.repair.orphan import _orphan_dead_symlink_scan + + with patch(_MOD + ".REPAIR_LIBS", repair_libs), \ + patch(_MOD + "._collect_known_paths", return_value=known_paths), \ + patch(_MOD + "._dead_symlink", side_effect=fake_dead_symlink), \ + patch("os.path.isdir", side_effect=fake_isdir), \ + patch("os.walk", side_effect=fake_walk): + _orphan_dead_symlink_scan() + + def test_returns_immediately_when_no_repair_libs(self): + # Should not raise, should not call os.walk + with patch("os.walk") as mock_walk: + self._run([]) + mock_walk.assert_not_called() + + def test_skips_root_that_is_not_a_directory(self): + # /bad/path doesn't exist → log a warning, don't walk + with patch("os.walk") as mock_walk, \ + patch(_MOD + ".REPAIR_LIBS", ["/bad/path"]), \ + patch(_MOD + "._collect_known_paths", return_value=set()), \ + patch("os.path.isdir", return_value=False): + from doctor.checks.repair.orphan import _orphan_dead_symlink_scan + _orphan_dead_symlink_scan() + mock_walk.assert_not_called() + + def test_does_not_report_known_path(self): + known = {"/lib/Show/ep.mkv"} + walk = {"/lib": ["ep.mkv"]} + dead = {"/lib/ep.mkv"} + # The path is known → not an orphan even if dead + with patch(_MOD + ".log") as mock_log: + self._run(["/lib"], known_paths=known, walk_files=walk, dead_symlinks=dead) + # Warning about found orphans should NOT be called + calls_str = str(mock_log.warning.call_args_list) + self.assertNotIn("orphan", calls_str.lower().replace("repair:orphan", "")) + + def test_does_not_report_live_file(self): + walk = {"/lib": ["live.mkv"]} + dead = set() # live file + with patch(_MOD + ".log") as mock_log: + self._run(["/lib"], walk_files=walk, dead_symlinks=dead) + calls_str = str(mock_log.warning.call_args_list) + self.assertNotIn("found 1 dead", calls_str) + + def test_reports_orphan_dead_symlink(self): + walk = {"/lib": ["orphan.mkv"]} + dead = {"/lib/orphan.mkv"} + with patch(_MOD + ".log") as mock_log: + self._run(["/lib"], walk_files=walk, dead_symlinks=dead) + # Check that warning was called with count=1 + counts = [a[0][1] for a in mock_log.warning.call_args_list + if "dead symlink(s) not tracked" in (a[0][0] or "")] + self.assertTrue(any(c == 1 for c in counts)) + + def test_caps_individual_path_log_at_20(self): + """More than 20 orphans → logs first 20 + summary line.""" + files = [f"f{i}.mkv" for i in range(25)] + walk = {"/lib": files} + dead = {"/lib/" + f for f in files} + with patch(_MOD + ".log") as mock_log: + self._run(["/lib"], walk_files=walk, dead_symlinks=dead) + # Check summary count=25 + counts = [a[0][1] for a in mock_log.warning.call_args_list + if "dead symlink(s) not tracked" in (a[0][0] or "")] + self.assertTrue(any(c == 25 for c in counts)) + # Check overflow "and N more" + overflow = [a[0][1] for a in mock_log.warning.call_args_list + if "and %d more" in (a[0][0] or "")] + self.assertTrue(any(c == 5 for c in overflow)) diff --git a/tests/test_repair_verify.py b/tests/test_repair_verify.py new file mode 100644 index 0000000..5372dbf --- /dev/null +++ b/tests/test_repair_verify.py @@ -0,0 +1,367 @@ +"""Unit tests for doctor.checks.repair.verify. + +Tests cover: + _repair_verify_pending(state): + - returns immediately when no pending entries + - removes entry when the arr instance is unknown + - polls command_status when cmd_id is present and not yet done + - marks cmd_done=True when command reaches terminal state + - marks cmd_done=True when command_status returns None (endpoint gone) + - detects a successful grab via history_grabbed and removes the entry + - removes entry when deadline has passed without a grab + - leaves entry in place when within deadline and no grab yet + + _repair_record_verify(state, arr, title, cmd_id, media_id, entity_ids): + - creates a new pending entry with correct fields + - uses a string key derived from arr.name + title slug + - cmd_id is stored only if it is an int (None otherwise) + - deadline is approximately time.time() + REPAIR_VERIFY_DEADLINE + - search_ts is a UTC datetime string + - entity_ids defaults to [] when None is passed + +All time, Arr API calls, and state are provided by the test. +""" +import time +import unittest +from unittest.mock import MagicMock, patch + +_MOD = "doctor.checks.repair.verify" + + +def _make_arr(name="sonarr-1"): + arr = MagicMock() + arr.name = name + arr.kind = "sonarr" + arr.command_status.return_value = None + arr.history_grabbed.return_value = None + return arr + + +def _pending(arr_name, title="Show", cmd_id=None, media_id=1, + entity_ids=None, deadline=None, cmd_done=False, search_ts="2026-01-01T00:00:00Z"): + return { + "arr_name": arr_name, + "title": title, + "cmd_id": cmd_id, + "cmd_done": cmd_done, + "media_id": media_id, + "entity_ids": entity_ids or [], + "search_ts": search_ts, + "deadline": deadline if deadline is not None else time.time() + 3600, + } + + +def _state_with(*entries): + """Build a state dict with given pending verify entries keyed by index.""" + pv = {} + for i, e in enumerate(entries): + pv[f"key{i}"] = e + return {"__repair_verify__": pv} + + +def _run_pending(state, instances): + from doctor.checks.repair.verify import _repair_verify_pending + with patch(_MOD + ".INSTANCES", instances): + _repair_verify_pending(state) + return state + + +class VerifyPendingNoOpTest(unittest.TestCase): + + def test_no_op_when_no_pending_entries(self): + state = {} + arr = _make_arr() + _run_pending(state, [arr]) + # state should have the key created by setdefault, but it should be empty + self.assertEqual(state.get("__repair_verify__", {}), {}) + + def test_removes_entry_for_unknown_arr(self): + arr = _make_arr(name="known") + entry = _pending("unknown-arr") # not in INSTANCES + state = _state_with(entry) + _run_pending(state, [arr]) + self.assertEqual(state["__repair_verify__"], {}) + + +class VerifyPendingCommandPollTest(unittest.TestCase): + + def test_marks_cmd_done_on_terminal_status(self): + arr = _make_arr() + arr.command_status.return_value = "completed" + entry = _pending(arr.name, cmd_id=42, cmd_done=False) + state = _state_with(entry) + _run_pending(state, [arr]) + arr.command_status.assert_called_once_with(42) + remaining = state["__repair_verify__"] + if remaining: + self.assertTrue(list(remaining.values())[0].get("cmd_done")) + + def test_marks_cmd_done_when_status_is_none(self): + arr = _make_arr() + arr.command_status.return_value = None # endpoint gone + entry = _pending(arr.name, cmd_id=99, cmd_done=False) + state = _state_with(entry) + _run_pending(state, [arr]) + remaining = state["__repair_verify__"] + if remaining: + self.assertTrue(list(remaining.values())[0].get("cmd_done")) + + def test_skips_command_poll_when_already_done(self): + arr = _make_arr() + entry = _pending(arr.name, cmd_id=7, cmd_done=True) + state = _state_with(entry) + _run_pending(state, [arr]) + arr.command_status.assert_not_called() + + +class VerifyPendingGrabTest(unittest.TestCase): + + def test_removes_entry_on_successful_grab(self): + arr = _make_arr() + arr.history_grabbed.return_value = { + "sourceTitle": "Show.S01E01.mkv", + "data": {"indexer": "SomeIndexer"}, + } + entry = _pending(arr.name, media_id=1, cmd_done=True) + state = _state_with(entry) + _run_pending(state, [arr]) + self.assertEqual(state["__repair_verify__"], {}) + + def test_leaves_entry_when_no_grab_within_deadline(self): + arr = _make_arr() + arr.history_grabbed.return_value = None + entry = _pending(arr.name, media_id=1, deadline=time.time() + 3600, cmd_done=True) + state = _state_with(entry) + _run_pending(state, [arr]) + # Entry should still be present + self.assertEqual(len(state["__repair_verify__"]), 1) + + def test_removes_entry_when_deadline_exceeded(self): + arr = _make_arr() + arr.history_grabbed.return_value = None + past_deadline = time.time() - 1 # already expired + entry = _pending(arr.name, media_id=1, deadline=past_deadline, cmd_done=True) + state = _state_with(entry) + _run_pending(state, [arr]) + self.assertEqual(state["__repair_verify__"], {}) + + def test_multiple_entries_handled_independently(self): + arr = _make_arr() + # Entry 0: grabbed → removed + # Entry 1: past deadline → removed + # Entry 2: within deadline, no grab → kept + arr.history_grabbed.side_effect = [ + {"sourceTitle": "A", "data": {}}, # entry 0 grabbed + None, # entry 1 not grabbed (expired) + None, # entry 2 not grabbed (alive) + ] + e0 = _pending(arr.name, media_id=1, deadline=time.time() + 3600, cmd_done=True) + e1 = _pending(arr.name, media_id=2, deadline=time.time() - 1, cmd_done=True) + e2 = _pending(arr.name, media_id=3, deadline=time.time() + 3600, cmd_done=True) + state = {"__repair_verify__": {"k0": e0, "k1": e1, "k2": e2}} + _run_pending(state, [arr]) + remaining = state["__repair_verify__"] + self.assertNotIn("k0", remaining) + self.assertNotIn("k1", remaining) + self.assertIn("k2", remaining) + + +class RecordVerifyTest(unittest.TestCase): + + def _run_record(self, arr, title, cmd_id, media_id, entity_ids, + deadline_delta=3600, now=None): + if now is None: + now = time.time() + state = {} + from doctor.checks.repair.verify import _repair_record_verify + with patch(_MOD + ".REPAIR_VERIFY_DEADLINE", deadline_delta), \ + patch(_MOD + ".time") as mock_time: + mock_time.time.return_value = now + _repair_record_verify(state, arr, title, cmd_id, media_id, entity_ids) + return state + + def test_creates_pending_entry(self): + arr = _make_arr() + state = self._run_record(arr, "My Show", 42, 10, [1, 2]) + pv = state.get("__repair_verify__", {}) + self.assertEqual(len(pv), 1) + entry = list(pv.values())[0] + self.assertEqual(entry["arr_name"], arr.name) + self.assertEqual(entry["title"], "My Show") + self.assertEqual(entry["cmd_id"], 42) + self.assertEqual(entry["media_id"], 10) + self.assertEqual(entry["entity_ids"], [1, 2]) + + def test_key_is_stable_slug(self): + arr = _make_arr(name="sonarr-1") + state = self._run_record(arr, "My Show!", 1, 1, []) + pv = state["__repair_verify__"] + key = list(pv.keys())[0] + self.assertTrue(key.startswith("sonarr-1:")) + # Key should only contain safe chars + slug = key.split(":", 1)[1] + self.assertRegex(slug, r"^[a-z0-9_]+$") + + def test_deadline_is_now_plus_delta(self): + arr = _make_arr() + now = 1_000_000.0 + state = self._run_record(arr, "Show", None, 1, [], deadline_delta=7200, now=now) + entry = list(state["__repair_verify__"].values())[0] + self.assertAlmostEqual(entry["deadline"], now + 7200, places=0) + + def test_non_int_cmd_id_stored_as_none(self): + arr = _make_arr() + state = self._run_record(arr, "Show", "not-an-int", 1, []) + entry = list(state["__repair_verify__"].values())[0] + self.assertIsNone(entry["cmd_id"]) + + def test_none_entity_ids_stored_as_empty_list(self): + arr = _make_arr() + state = self._run_record(arr, "Show", None, 1, None) + entry = list(state["__repair_verify__"].values())[0] + self.assertEqual(entry["entity_ids"], []) + + def test_search_ts_is_utc_string(self): + arr = _make_arr() + state = self._run_record(arr, "Show", None, 1, []) + entry = list(state["__repair_verify__"].values())[0] + ts = entry["search_ts"] + self.assertRegex(ts, r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$") + + +class HierarchicalFallbackTest(unittest.TestCase): + """Test fallback to narrower search strategies when a wider search stalls.""" + + def _pending_fb(self, strategy, hierarchical=True, sn=1, sid=5, epids=None, **kw): + e = _pending("sonarr-1", media_id=sid, entity_ids=epids or [10], **kw) + e["strategy"] = strategy + e["season_number"] = sn + e["series_id"] = sid + e["hierarchical"] = hierarchical + return e + + @patch(_MOD + ".REPAIR_HIERARCHICAL_FALLBACK", True) + def test_series_search_falls_back_to_season(self): + arr = _make_arr() + arr.history_grabbed.return_value = None + past = time.time() - 1 + entry = self._pending_fb("series", deadline=past, cmd_done=True) + state = {"__repair_verify__": {"k1": entry}} + from doctor.checks.repair.verify import _repair_verify_pending + with patch(_MOD + ".INSTANCES", [arr]): + _repair_verify_pending(state) + pv = state["__repair_verify__"] + self.assertIn("k1", pv) + self.assertEqual(pv["k1"]["strategy"], "season") + self.assertTrue(pv["k1"].get("needs_fallback")) + + @patch(_MOD + ".REPAIR_HIERARCHICAL_FALLBACK", True) + def test_season_search_falls_back_to_episode(self): + arr = _make_arr() + arr.history_grabbed.return_value = None + past = time.time() - 1 + entry = self._pending_fb("season", deadline=past, cmd_done=True) + state = {"__repair_verify__": {"k1": entry}} + from doctor.checks.repair.verify import _repair_verify_pending + with patch(_MOD + ".INSTANCES", [arr]): + _repair_verify_pending(state) + pv = state["__repair_verify__"] + self.assertIn("k1", pv) + self.assertEqual(pv["k1"]["strategy"], "episode") + self.assertTrue(pv["k1"].get("needs_fallback")) + + @patch(_MOD + ".REPAIR_HIERARCHICAL_FALLBACK", True) + def test_episode_search_does_not_fall_back(self): + arr = _make_arr() + arr.history_grabbed.return_value = None + past = time.time() - 1 + entry = self._pending_fb("episode", deadline=past, cmd_done=True) + state = {"__repair_verify__": {"k1": entry}} + from doctor.checks.repair.verify import _repair_verify_pending + with patch(_MOD + ".INSTANCES", [arr]): + _repair_verify_pending(state) + self.assertEqual(state["__repair_verify__"], {}) + + @patch(_MOD + ".REPAIR_HIERARCHICAL_FALLBACK", False) + def test_fallback_disabled_removes_on_deadline(self): + arr = _make_arr() + arr.history_grabbed.return_value = None + past = time.time() - 1 + entry = self._pending_fb("series", deadline=past, cmd_done=True) + state = {"__repair_verify__": {"k1": entry}} + from doctor.checks.repair.verify import _repair_verify_pending + with patch(_MOD + ".INSTANCES", [arr]): + _repair_verify_pending(state) + self.assertEqual(state["__repair_verify__"], {}) + + def test_non_hierarchical_does_not_fall_back(self): + arr = _make_arr() + arr.history_grabbed.return_value = None + past = time.time() - 1 + entry = self._pending_fb("series", hierarchical=False, deadline=past, cmd_done=True) + state = {"__repair_verify__": {"k1": entry}} + from doctor.checks.repair.verify import _repair_verify_pending + with patch(_MOD + ".INSTANCES", [arr]): + _repair_verify_pending(state) + self.assertEqual(state["__repair_verify__"], {}) + + +class ProcessFallbacksTest(unittest.TestCase): + """Test _repair_process_fallbacks issues the correct narrowed search commands.""" + + def _make_state(self, strategy, sn=1, sid=5, epids=None): + return {"__repair_verify__": { + "k1": { + "arr_name": "sonarr-1", "title": "Show", "strategy": strategy, + "needs_fallback": True, "series_id": sid, "season_number": sn, + "entity_ids": epids or [10], "hierarchical": True, + } + }} + + def test_series_fallback_issues_season_search(self): + arr = _make_arr() + arr.command.return_value = 123 + # state contains the *target* fallback strategy (season) + state = self._make_state("season", sn=2, sid=5) + from doctor.checks.repair.verify import _repair_process_fallbacks + with patch(_MOD + ".INSTANCES", [arr]): + issued = _repair_process_fallbacks(state) + self.assertEqual(issued, 1) + arr.command.assert_called_once_with("SeasonSearch", seriesId=5, seasonNumber=2) + entry = state["__repair_verify__"]["k1"] + self.assertEqual(entry["cmd_id"], 123) + self.assertFalse(entry.get("needs_fallback")) + self.assertFalse(entry.get("cmd_done")) + + def test_season_fallback_issues_episode_search(self): + arr = _make_arr() + arr.command.return_value = 124 + # state contains the *target* fallback strategy (episode) + state = self._make_state("episode", sn=1, sid=5, epids=[10, 11]) + from doctor.checks.repair.verify import _repair_process_fallbacks + with patch(_MOD + ".INSTANCES", [arr]): + issued = _repair_process_fallbacks(state) + self.assertEqual(issued, 1) + arr.command.assert_called_once_with("EpisodeSearch", episodeIds=[10, 11]) + entry = state["__repair_verify__"]["k1"] + self.assertEqual(entry["cmd_id"], 124) + self.assertFalse(entry.get("needs_fallback")) + + def test_no_needs_fallback_skips(self): + arr = _make_arr() + state = {"__repair_verify__": {"k1": {"arr_name": "sonarr-1", "needs_fallback": False}}} + from doctor.checks.repair.verify import _repair_process_fallbacks + with patch(_MOD + ".INSTANCES", [arr]): + issued = _repair_process_fallbacks(state) + self.assertEqual(issued, 0) + arr.command.assert_not_called() + + def test_command_failure_keeps_needs_fallback(self): + arr = _make_arr() + arr.command.return_value = None + state = self._make_state("season", sn=1, sid=5, epids=[10]) + from doctor.checks.repair.verify import _repair_process_fallbacks + with patch(_MOD + ".INSTANCES", [arr]): + issued = _repair_process_fallbacks(state) + self.assertEqual(issued, 0) + self.assertTrue(state["__repair_verify__"]["k1"].get("needs_fallback")) diff --git a/tests/test_scheduler_run_metadata.py b/tests/test_scheduler_run_metadata.py new file mode 100644 index 0000000..202fbec --- /dev/null +++ b/tests/test_scheduler_run_metadata.py @@ -0,0 +1,131 @@ +"""Tests for per-check run metadata in scheduler._check_runs (B1). + +These tests verify that _run_scheduled_check and sweep() correctly populate +the _check_runs dict with timing, outcome, and counter information. +""" +import time +import unittest +from unittest.mock import MagicMock + +import doctor.scheduler as sched_mod +from doctor.scheduler import _run_scheduled_check, _check_runs + + +def _reset_runs(): + """Clear _check_runs between tests.""" + _check_runs.clear() + + +class RunMetadataBasicTest(unittest.TestCase): + + def setUp(self): + _reset_runs() + + def test_successful_run_records_ok_outcome(self): + fn = MagicMock() + _run_scheduled_check("queue", fn) + r = _check_runs.get("queue") + self.assertIsNotNone(r, "_check_runs should have 'queue' entry") + self.assertEqual(r["last_outcome"], "ok") + self.assertEqual(r["last_error"], "") + + def test_successful_run_increments_run_count(self): + fn = MagicMock() + _run_scheduled_check("queue", fn) + _run_scheduled_check("queue", fn) + self.assertEqual(_check_runs["queue"]["run_count"], 2) + + def test_error_run_records_error_outcome(self): + fn = MagicMock(side_effect=RuntimeError("disk full")) + _run_scheduled_check("queue", fn) + r = _check_runs["queue"] + self.assertEqual(r["last_outcome"], "error") + self.assertIn("disk full", r["last_error"]) + + def test_error_run_increments_error_count(self): + fn = MagicMock(side_effect=RuntimeError("oops")) + _run_scheduled_check("queue", fn) + _run_scheduled_check("queue", fn) + r = _check_runs["queue"] + self.assertEqual(r["error_count"], 2) + self.assertEqual(r["run_count"], 2) + + def test_successful_run_does_not_increment_error_count(self): + fn = MagicMock() + _run_scheduled_check("queue", fn) + self.assertEqual(_check_runs["queue"]["error_count"], 0) + + def test_records_last_start_and_end_timestamps(self): + before = time.time() + fn = MagicMock() + _run_scheduled_check("queue", fn) + after = time.time() + r = _check_runs["queue"] + self.assertGreaterEqual(r["last_start"], before) + self.assertLessEqual(r["last_end"], after) + self.assertGreater(r["last_end"], r["last_start"] - 0.001) # end >= start + + def test_records_duration(self): + fn = MagicMock() + _run_scheduled_check("queue", fn) + r = _check_runs["queue"] + self.assertGreaterEqual(r["last_duration"], 0.0) + self.assertIsInstance(r["last_duration"], float) + + def test_different_checks_tracked_separately(self): + fn_q = MagicMock() + fn_r = MagicMock(side_effect=RuntimeError("boom")) + _run_scheduled_check("queue", fn_q) + _run_scheduled_check("repair", fn_r) + self.assertEqual(_check_runs["queue"]["last_outcome"], "ok") + self.assertEqual(_check_runs["repair"]["last_outcome"], "error") + + def test_run_count_accumulates_across_ok_and_error(self): + fn_ok = MagicMock() + fn_err = MagicMock(side_effect=RuntimeError("x")) + _run_scheduled_check("queue", fn_ok) + _run_scheduled_check("queue", fn_err) + _run_scheduled_check("queue", fn_ok) + r = _check_runs["queue"] + self.assertEqual(r["run_count"], 3) + self.assertEqual(r["error_count"], 1) + + +class RunMetadataConcurrencyTest(unittest.TestCase): + + def setUp(self): + _reset_runs() + + def test_deferred_when_semaphore_full(self): + """When _scheduler_sem cannot be acquired, outcome is 'deferred'.""" + fn = MagicMock() + # Drain the semaphore completely + acquired = [] + for _ in range(sched_mod._scheduler_sem._value if hasattr(sched_mod._scheduler_sem, '_value') else 3): + if sched_mod._scheduler_sem.acquire(blocking=False): + acquired.append(True) + try: + _run_scheduled_check("queue", fn) + r = _check_runs.get("queue") + self.assertIsNotNone(r) + self.assertEqual(r["last_outcome"], "deferred") + fn.assert_not_called() + finally: + for _ in acquired: + sched_mod._scheduler_sem.release() + + def test_skipped_when_check_already_running(self): + """When the per-check lock is already held, outcome is 'skipped'.""" + lock = sched_mod._check_locks.get("queue") + if lock is None: + self.skipTest("no lock for 'queue'") + lock.acquire() + try: + fn = MagicMock() + _run_scheduled_check("queue", fn) + r = _check_runs.get("queue") + self.assertIsNotNone(r) + self.assertEqual(r["last_outcome"], "skipped") + fn.assert_not_called() + finally: + lock.release() diff --git a/tests/test_seerr.py b/tests/test_seerr.py new file mode 100644 index 0000000..c8f9874 --- /dev/null +++ b/tests/test_seerr.py @@ -0,0 +1,194 @@ +"""Unit tests for doctor.checks.seerr.check_seerr(). + +Tests cover: + - skips when SEERR_URL or SEERR_APIKEY is unset + - skips individual requests with no id field + - logs and returns when Seerr.failed() returns None (unreachable) + - logs and returns when there are no failed requests + - retries a failed request and updates the state counter + - respects SEERR_MAX_TRIES: stops retrying after n attempts + - respects SEERR_MAX: caps total actions per sweep + - DRY_RUN: logs intent but does not call s.retry() + - cleans up state keys for requests no longer in the failed list + - handles s.retry() raising an exception gracefully + +All external I/O is replaced with MagicMock. Config globals are patched on +doctor.checks.seerr directly. +""" +import unittest +from unittest.mock import MagicMock, patch + +_MOD = "doctor.checks.seerr" + + +def _make_seerr_client(failed=None): + s = MagicMock() + s.failed.return_value = [] if failed is None else failed + s.retry.return_value = None + return s + + +def _req(rid, media_type="movie", tmdb_id=123): + return {"id": rid, "media": {"mediaType": media_type, "tmdbId": tmdb_id}} + + +def _run(failed_reqs, *, state=None, seerr_url="http://seerr", + seerr_apikey="key", seerr_max=10, seerr_max_tries=3, + dry_run=False, client=None): + if state is None: + state = {} + if client is None: + client = _make_seerr_client(failed_reqs) + + from doctor.checks.seerr import check_seerr + + with patch(_MOD + ".SEERR_URL", seerr_url), \ + patch(_MOD + ".SEERR_APIKEY", seerr_apikey), \ + patch(_MOD + ".SEERR_MAX", seerr_max), \ + patch(_MOD + ".SEERR_MAX_TRIES", seerr_max_tries), \ + patch(_MOD + ".DRY_RUN", dry_run), \ + patch(_MOD + ".Seerr", return_value=client), \ + patch(_MOD + ".state_transaction") as mock_tx: + mock_tx.return_value.__enter__ = lambda s: state + mock_tx.return_value.__exit__ = MagicMock(return_value=False) + check_seerr() + + return state, client + + +class SeerrSkipTest(unittest.TestCase): + + def test_skips_when_no_url(self): + _, client = _run([], seerr_url="") + client.failed.assert_not_called() + + def test_skips_when_no_apikey(self): + _, client = _run([], seerr_apikey="") + client.failed.assert_not_called() + + def test_returns_when_unreachable(self): + client = _make_seerr_client(failed=None) + _, c = _run(None, client=client) + c.retry.assert_not_called() + + def test_returns_when_no_failed_requests(self): + _, client = _run([]) + client.retry.assert_not_called() + + +class SeerrRetryTest(unittest.TestCase): + + def test_retries_one_request(self): + reqs = [_req(1)] + state, client = _run(reqs) + client.retry.assert_called_once_with(1) + self.assertEqual(state["__seerr__"]["1"], 1) + + def test_increments_counter_on_successive_runs(self): + reqs = [_req(42)] + state = {"__seerr__": {"42": 1}} + _, client = _run(reqs, state=state, seerr_max_tries=5) + client.retry.assert_called_once_with(42) + self.assertEqual(state["__seerr__"]["42"], 2) + + def test_skips_request_without_id(self): + reqs = [{"media": {"mediaType": "movie"}}] + state, client = _run(reqs) + client.retry.assert_not_called() + + +class SeerrMaxTriesTest(unittest.TestCase): + + def test_stops_retrying_at_max_tries(self): + reqs = [_req(7)] + state = {"__seerr__": {"7": 3}} + _, client = _run(reqs, state=state, seerr_max_tries=3) + client.retry.assert_not_called() + + def test_retries_when_below_max_tries(self): + reqs = [_req(7)] + state = {"__seerr__": {"7": 2}} + _, client = _run(reqs, state=state, seerr_max_tries=3) + client.retry.assert_called_once_with(7) + + def test_zero_max_tries_means_unlimited(self): + reqs = [_req(9)] + state = {"__seerr__": {"9": 999}} + _, client = _run(reqs, state=state, seerr_max_tries=0) + client.retry.assert_called_once_with(9) + + +class SeerrCapTest(unittest.TestCase): + + def test_caps_at_seerr_max(self): + reqs = [_req(i) for i in range(5)] + _, client = _run(reqs, seerr_max=2) + self.assertEqual(client.retry.call_count, 2) + + def test_zero_max_breaks_immediately(self): + # acted >= SEERR_MAX is True when both are 0: no retries + reqs = [_req(i) for i in range(4)] + _, client = _run(reqs, seerr_max=0) + client.retry.assert_not_called() + + +class SeerrDryRunTest(unittest.TestCase): + + def test_dry_run_does_not_call_retry(self): + reqs = [_req(1)] + _, client = _run(reqs, dry_run=True) + client.retry.assert_not_called() + + def test_dry_run_respects_seerr_max(self): + reqs = [_req(1), _req(2)] + _, client = _run(reqs, seerr_max=1, dry_run=True) + client.retry.assert_not_called() + + +class SeerrStateCleanupTest(unittest.TestCase): + + def test_removes_stale_state_key(self): + reqs = [_req(1)] + state = {"__seerr__": {"1": 0, "99": 2}} + _run(reqs, state=state) + self.assertNotIn("99", state["__seerr__"]) + self.assertIn("1", state["__seerr__"]) + + +class SeerrRetryExceptionTest(unittest.TestCase): + + def test_retry_exception_is_swallowed(self): + client = _make_seerr_client([_req(1)]) + client.retry.side_effect = RuntimeError("connection refused") + state, _ = _run([_req(1)], client=client) + tries = state.get("__seerr__", {}) + self.assertEqual(tries.get("1", 0), 0) + + +class SeerrNoneIdCleanupTest(unittest.TestCase): + """Regression tests for the None-id state cleanup bug (A4). + + When a request has id=None, str(None)="None" was added to the `live` set, + preventing cleanup of a state key literally named "None". + """ + + def test_none_id_does_not_pollute_live_set(self): + """A request with id=None must not add 'None' to the live set, + so stale state keys that happen to be named 'None' get cleaned up.""" + # Simulate: one real request, one with id=None, and a stale 'None' key in state + reqs = [_req(1), {"id": None, "media": {}}] + state = {"__seerr__": {"1": 0, "None": 3}} + _run(reqs, state=state) + # "None" must be cleaned up — it was not a real request id + self.assertNotIn("None", state["__seerr__"]) + # "1" must stay (it's still in the live failed requests) + self.assertIn("1", state["__seerr__"]) + + def test_real_requests_with_none_id_in_mix_still_retry(self): + """A None-id request is skipped but real requests around it still work.""" + reqs = [{"id": None, "media": {}}, _req(2), {"id": None, "media": {}}] + state, client = _run(reqs) + tries = state.get("__seerr__", {}) + # Only req #2 should have been retried + self.assertEqual(tries.get("2", 0), 1) + self.assertNotIn("None", tries) diff --git a/tests/test_state.py b/tests/test_state.py new file mode 100644 index 0000000..a565b49 --- /dev/null +++ b/tests/test_state.py @@ -0,0 +1,45 @@ +"""Regression tests for persistent state concurrency.""" +import os +import tempfile +import threading +import unittest + +import doctor.state + + +class StateTransactionTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + # Point the state module at a temp file for this test. + doctor.state.STATE_FILE = os.path.join(self.tmp.name, "state.json") + + def test_transaction_loads_missing_state(self): + with doctor.state.state_transaction() as state: + self.assertEqual(state, {}) + + def test_transaction_persists_changes(self): + with doctor.state.state_transaction() as state: + state["x"] = 1 + with doctor.state.state_transaction() as state: + self.assertEqual(state.get("x"), 1) + + def test_no_lost_updates_under_concurrent_transactions(self): + """Two threads incrementing the same counter must not clobber each other.""" + def bump(): + for _ in range(50): + with doctor.state.state_transaction() as state: + state["counter"] = state.get("counter", 0) + 1 + + threads = [threading.Thread(target=bump) for _ in range(2)] + for t in threads: + t.start() + for t in threads: + t.join() + + with doctor.state.state_transaction() as state: + self.assertEqual(state.get("counter"), 100) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..47ef12d --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,143 @@ +"""Unit tests for the small utility helpers in doctor.utils. + +These helpers were extracted from doctor.config.py in Phase 2. +doctor.config still re-exports them for backward compatibility, but the tests target the new home. + +All external side effects (urllib, subprocess, /proc/loadavg) are mocked so +the tests run without network access or a real /proc filesystem. +""" +import subprocess +import unittest +from unittest.mock import MagicMock, patch + +from doctor.utils import http_code, run_cmd, run_output, host_load + + +class HttpCodeTest(unittest.TestCase): + """http_code(url, headers=None, t=10) returns the HTTP status or 0.""" + + @patch("doctor.utils.urllib.request.urlopen") + def test_returns_status_on_success(self, mock_open): + resp = MagicMock() + resp.status = 200 + mock_open.return_value = resp + self.assertEqual(http_code("http://example.com"), 200) + mock_open.assert_called_once() + req = mock_open.call_args[0][0] + self.assertEqual(req.full_url, "http://example.com") + self.assertEqual(req.headers, {}) + self.assertEqual(mock_open.call_args.kwargs.get("timeout"), 10) + + @patch("doctor.utils.urllib.request.urlopen") + def test_returns_error_code_on_http_error(self, mock_open): + from urllib.error import HTTPError + mock_open.side_effect = HTTPError( + url="http://example.com", code=503, msg="busy", hdrs=None, fp=None + ) + self.assertEqual(http_code("http://example.com"), 503) + + @patch("doctor.utils.urllib.request.urlopen") + def test_returns_zero_on_generic_exception(self, mock_open): + mock_open.side_effect = OSError("no route") + self.assertEqual(http_code("http://example.com"), 0) + + @patch("doctor.utils.urllib.request.urlopen") + def test_passes_headers_and_timeout(self, mock_open): + resp = MagicMock() + resp.status = 204 + mock_open.return_value = resp + self.assertEqual(http_code("http://example.com", headers={"X": "Y"}, t=5), 204) + req = mock_open.call_args[0][0] + self.assertEqual(req.headers, {"X": "Y"}) + self.assertEqual(mock_open.call_args.kwargs.get("timeout"), 5) + + +class RunCmdTest(unittest.TestCase): + """run_cmd(cmd) returns (rc, combined_output[:300]) or None.""" + + @patch("doctor.utils.subprocess.run") + def test_returns_none_for_empty_cmd(self, mock_run): + self.assertIsNone(run_cmd("")) + mock_run.assert_not_called() + + @patch("doctor.utils.subprocess.run") + def test_returns_rc_and_output(self, mock_run): + p = MagicMock() + p.returncode = 0 + p.stdout = "out\n" + p.stderr = "err\n" + mock_run.return_value = p + self.assertEqual(run_cmd("echo hi"), (0, "out\nerr")) + mock_run.assert_called_once_with( + "echo hi", shell=True, capture_output=True, text=True, timeout=180 + ) + + @patch("doctor.utils.subprocess.run") + def test_trims_output_to_300_chars(self, mock_run): + p = MagicMock() + p.returncode = 0 + p.stdout = "x" * 400 + p.stderr = "" + mock_run.return_value = p + self.assertEqual(run_cmd("x"), (0, "x" * 300)) + + @patch("doctor.utils.subprocess.run") + def test_returns_error_tuple_on_exception(self, mock_run): + mock_run.side_effect = subprocess.TimeoutExpired("echo hi", 180) + rc, out = run_cmd("echo hi") + self.assertEqual(rc, 1) + self.assertTrue(out.startswith("cmd error:")) + self.assertLessEqual(len(out), 120 + len("cmd error: ")) + + +class RunOutputTest(unittest.TestCase): + """run_output(cmd, t=120) returns stdout or empty string on failure.""" + + @patch("doctor.utils.subprocess.run") + def test_returns_stdout(self, mock_run): + p = MagicMock() + p.stdout = "log line\n" + mock_run.return_value = p + self.assertEqual(run_output("cat log"), "log line\n") + mock_run.assert_called_once_with( + "cat log", shell=True, capture_output=True, text=True, timeout=120 + ) + + @patch("doctor.utils.subprocess.run") + def test_returns_empty_string_on_exception(self, mock_run): + mock_run.side_effect = subprocess.TimeoutExpired("cat log", 120) + self.assertEqual(run_output("cat log"), "") + + @patch("doctor.utils.subprocess.run") + def test_passes_custom_timeout(self, mock_run): + p = MagicMock() + p.stdout = "" + mock_run.return_value = p + run_output("cat log", t=30) + mock_run.assert_called_once_with( + "cat log", shell=True, capture_output=True, text=True, timeout=30 + ) + + +class HostLoadTest(unittest.TestCase): + """host_load() returns the 1-min load from /proc/loadavg or 0.0.""" + + @patch("builtins.open") + def test_returns_first_load_value(self, mock_open): + mock_open.return_value.__enter__.return_value.read.return_value = "2.34 1.23 0.45 4/512 12345" + self.assertEqual(host_load(), 2.34) + mock_open.assert_called_once_with("/proc/loadavg") + + @patch("builtins.open") + def test_returns_zero_when_read_fails(self, mock_open): + mock_open.side_effect = OSError("no /proc") + self.assertEqual(host_load(), 0.0) + + @patch("builtins.open") + def test_returns_zero_when_parse_fails(self, mock_open): + mock_open.return_value.__enter__.return_value.read.return_value = "garbage" + self.assertEqual(host_load(), 0.0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_warmer.py b/tests/test_warmer.py new file mode 100644 index 0000000..55ea9a0 --- /dev/null +++ b/tests/test_warmer.py @@ -0,0 +1,490 @@ +"""Unit tests for doctor.checks.warmer. + +Tests cover the core logic that can be exercised without a real filesystem or +Plex server: + + _host_path(): + - returns the path unchanged when no WARM_PATH_MAP + - rewrites the prefix when WARM_PATH_MAP = "a:b" and path starts with a + - returns the path unchanged when path does not start with the map prefix + + _limit_parts(): + - returns all files when WARM_PARTS <= 0 + - truncates to WARM_PARTS when set + + _warm_record(): + - increments _warm_count[0] + - appends an entry to _warm_recent + - caps _warm_recent at 80 entries + + _warm_file(): + - skips (returns False) when host load exceeds WARM_LOAD_MAX + - skips (returns False) when file was warmed within WARM_COOLDOWN + - skips (returns False) when os.path.getsize raises + - returns True and records state when file is read successfully + - returns False when the read thread times out (is_alive=True) + - reads head bytes and optionally tail bytes based on config + + warm_cycle(): + - skips the cycle when host load exceeds WARM_LOAD_MAX + - calls _warm_file for each target up to WARM_MAX_CYCLE + - stops after WARM_MAX_CYCLE successful warms + + _warm_targets(): + - returns empty list when no sessions and no on-deck sources + - adds next-ep targets when "next" is in WARM_SOURCES and episode is near end + - skips next-ep when remaining time exceeds WARM_NEXT_NEAR_END + - adds ondeck targets when sessions is empty and ondeck is enabled + - skips ondeck when sessions are active (someone is watching) + +Patching strategy: module-level config constants are patched on the warmer +module directly. Filesystem calls (os.path.getsize, open, threading.Thread) +are patched per-test. The Plex client is always mocked. +""" +import threading +import time +import unittest +from unittest.mock import MagicMock, patch, mock_open + +_MOD = "doctor.checks.warmer" + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _reset_warmer_state(): + """Reset module-level mutable state between tests.""" + import doctor.checks.warmer as w + w._warm_state.clear() + w._warm_last_ondeck[0] = 0.0 + w._warm_count[0] = 0 + w._warm_recent.clear() + + +def _make_plex(sessions=None, ondeck=None, recent=None, parts=None, leaves=None): + p = MagicMock() + p.sessions.return_value = sessions or [] + p.ondeck.return_value = ondeck or [] + p.recent.return_value = recent or [] + p.parts.return_value = parts or [] + p.leaves.return_value = leaves or [] + return p + + +# --------------------------------------------------------------------------- +# _host_path +# --------------------------------------------------------------------------- + +class HostPathTest(unittest.TestCase): + + def setUp(self): + _reset_warmer_state() + + def test_no_map_returns_unchanged(self): + from doctor.checks.warmer import _host_path + with patch(_MOD + ".WARM_PATH_MAP", ""): + self.assertEqual(_host_path("/mnt/lib/file.mkv"), "/mnt/lib/file.mkv") + + def test_rewrites_matching_prefix(self): + from doctor.checks.warmer import _host_path + with patch(_MOD + ".WARM_PATH_MAP", "/mnt/lib:/data/lib"): + self.assertEqual(_host_path("/mnt/lib/Show/ep.mkv"), "/data/lib/Show/ep.mkv") + + def test_non_matching_prefix_unchanged(self): + from doctor.checks.warmer import _host_path + with patch(_MOD + ".WARM_PATH_MAP", "/mnt/lib:/data/lib"): + self.assertEqual(_host_path("/other/path/ep.mkv"), "/other/path/ep.mkv") + + def test_no_colon_in_map_returns_unchanged(self): + from doctor.checks.warmer import _host_path + with patch(_MOD + ".WARM_PATH_MAP", "nocolon"): + self.assertEqual(_host_path("/mnt/lib/file.mkv"), "/mnt/lib/file.mkv") + + +# --------------------------------------------------------------------------- +# _limit_parts +# --------------------------------------------------------------------------- + +class LimitPartsTest(unittest.TestCase): + + def test_returns_all_when_parts_zero(self): + from doctor.checks.warmer import _limit_parts + with patch(_MOD + ".WARM_PARTS", 0): + files = ["a", "b", "c"] + self.assertEqual(_limit_parts(files), files) + + def test_returns_all_when_parts_negative(self): + from doctor.checks.warmer import _limit_parts + with patch(_MOD + ".WARM_PARTS", -1): + files = ["a", "b", "c"] + self.assertEqual(_limit_parts(files), files) + + def test_truncates_to_warm_parts(self): + from doctor.checks.warmer import _limit_parts + with patch(_MOD + ".WARM_PARTS", 2): + self.assertEqual(_limit_parts(["a", "b", "c"]), ["a", "b"]) + + +# --------------------------------------------------------------------------- +# _warm_record +# --------------------------------------------------------------------------- + +class WarmRecordTest(unittest.TestCase): + + def setUp(self): + _reset_warmer_state() + + def test_increments_warm_count(self): + from doctor.checks.warmer import _warm_record + _warm_record("Show S01E01.mkv", "cycle") + import doctor.checks.warmer as w + self.assertEqual(w._warm_count[0], 1) + + def test_appends_to_warm_recent(self): + from doctor.checks.warmer import _warm_record + _warm_record("Show.mkv", "ondeck") + import doctor.checks.warmer as w + self.assertEqual(len(w._warm_recent), 1) + self.assertEqual(w._warm_recent[0]["title"], "Show.mkv") + self.assertEqual(w._warm_recent[0]["why"], "ondeck") + + def test_caps_warm_recent_at_80(self): + from doctor.checks.warmer import _warm_record + for i in range(90): + _warm_record(f"file{i}.mkv", "cycle") + import doctor.checks.warmer as w + self.assertEqual(len(w._warm_recent), 80) + + +# --------------------------------------------------------------------------- +# _warm_file +# --------------------------------------------------------------------------- + +class WarmFileLoadGuardTest(unittest.TestCase): + + def setUp(self): + _reset_warmer_state() + + def test_skips_when_load_exceeds_max(self): + from doctor.checks.warmer import _warm_file + with patch(_MOD + ".WARM_LOAD_MAX", 1.0), \ + patch(_MOD + ".WARM_COOLDOWN", 0), \ + patch(_MOD + ".host_load", return_value=5.0), \ + patch(_MOD + ".WARM_PATH_MAP", ""): + result = _warm_file("/some/file.mkv") + self.assertFalse(result) + + def test_passes_when_load_under_max(self): + from doctor.checks.warmer import _warm_file + with patch(_MOD + ".WARM_LOAD_MAX", 10.0), \ + patch(_MOD + ".WARM_COOLDOWN", 0), \ + patch(_MOD + ".host_load", return_value=1.0), \ + patch(_MOD + ".WARM_PATH_MAP", ""), \ + patch(_MOD + ".WARM_HEAD_MB", 1), \ + patch(_MOD + ".WARM_TAIL_MB", 0), \ + patch(_MOD + ".WARM_READ_TIMEOUT", 30), \ + patch("os.path.getsize", return_value=10 << 20), \ + patch("builtins.open", mock_open(read_data=b"x" * 4096)): + result = _warm_file("/some/file.mkv") + self.assertTrue(result) + + +class WarmFileCooldownTest(unittest.TestCase): + + def setUp(self): + _reset_warmer_state() + + def test_skips_file_within_cooldown(self): + import doctor.checks.warmer as w + from doctor.checks.warmer import _warm_file + # Seed state so file was "just" warmed + w._warm_state["/some/file.mkv"] = time.time() + with patch(_MOD + ".WARM_LOAD_MAX", 0), \ + patch(_MOD + ".WARM_COOLDOWN", 3600), \ + patch(_MOD + ".WARM_PATH_MAP", ""): + result = _warm_file("/some/file.mkv") + self.assertFalse(result) + + def test_warms_file_after_cooldown_expires(self): + import doctor.checks.warmer as w + from doctor.checks.warmer import _warm_file + # Seed state with an old timestamp (cooldown expired) + w._warm_state["/some/file.mkv"] = time.time() - 7200 + with patch(_MOD + ".WARM_LOAD_MAX", 0), \ + patch(_MOD + ".WARM_COOLDOWN", 3600), \ + patch(_MOD + ".WARM_PATH_MAP", ""), \ + patch(_MOD + ".WARM_HEAD_MB", 1), \ + patch(_MOD + ".WARM_TAIL_MB", 0), \ + patch(_MOD + ".WARM_READ_TIMEOUT", 30), \ + patch("os.path.getsize", return_value=10 << 20), \ + patch("builtins.open", mock_open(read_data=b"x" * 4096)): + result = _warm_file("/some/file.mkv") + self.assertTrue(result) + + +class WarmFileStatFailTest(unittest.TestCase): + + def setUp(self): + _reset_warmer_state() + + def test_returns_false_when_stat_fails(self): + from doctor.checks.warmer import _warm_file + with patch(_MOD + ".WARM_LOAD_MAX", 0), \ + patch(_MOD + ".WARM_COOLDOWN", 0), \ + patch(_MOD + ".WARM_PATH_MAP", ""), \ + patch("os.path.getsize", side_effect=OSError("no such file")): + result = _warm_file("/nonexistent.mkv") + self.assertFalse(result) + + +class WarmFileReadTimeoutTest(unittest.TestCase): + + def setUp(self): + _reset_warmer_state() + + def test_returns_false_when_read_times_out(self): + from doctor.checks.warmer import _warm_file + + # Simulate a thread that never finishes (is_alive stays True) + class HangingThread(threading.Thread): + def __init__(self, *a, **kw): + super().__init__(*a, **kw) + self.daemon = True + def start(self): pass + def join(self, timeout=None): pass + def is_alive(self): return True + + with patch(_MOD + ".WARM_LOAD_MAX", 0), \ + patch(_MOD + ".WARM_COOLDOWN", 0), \ + patch(_MOD + ".WARM_PATH_MAP", ""), \ + patch(_MOD + ".WARM_HEAD_MB", 1), \ + patch(_MOD + ".WARM_TAIL_MB", 0), \ + patch(_MOD + ".WARM_READ_TIMEOUT", 1), \ + patch("os.path.getsize", return_value=10 << 20), \ + patch("threading.Thread", HangingThread): + result = _warm_file("/some/file.mkv") + self.assertFalse(result) + + +# --------------------------------------------------------------------------- +# warm_cycle +# --------------------------------------------------------------------------- + +class WarmCycleTest(unittest.TestCase): + + def setUp(self): + _reset_warmer_state() + + def test_skips_cycle_when_load_too_high(self): + from doctor.checks.warmer import warm_cycle + with patch(_MOD + ".WARM_LOAD_MAX", 1.0), \ + patch(_MOD + ".host_load", return_value=5.0), \ + patch(_MOD + "._warm_targets") as mock_targets, \ + patch(_MOD + ".Plex"): + warm_cycle() + mock_targets.assert_not_called() + + def test_warms_up_to_max_cycle(self): + from doctor.checks.warmer import warm_cycle + targets = [("cycle", f"/lib/f{i}.mkv") for i in range(5)] + with patch(_MOD + ".WARM_LOAD_MAX", 0), \ + patch(_MOD + ".host_load", return_value=0.0), \ + patch(_MOD + ".WARM_MAX_CYCLE", 2), \ + patch(_MOD + ".PLEX_URL", "http://plex"), \ + patch(_MOD + ".PLEX_TOKEN", "tok"), \ + patch(_MOD + ".Plex"), \ + patch(_MOD + "._warm_targets", return_value=targets), \ + patch(_MOD + "._warm_file", return_value=True) as mock_warm: + warm_cycle() + # Called for up to WARM_MAX_CYCLE=2 successful warms + self.assertEqual(mock_warm.call_count, 2) + + def test_continues_past_failed_warms(self): + """Failed warms (False) do not count against WARM_MAX_CYCLE.""" + from doctor.checks.warmer import warm_cycle + # 4 targets: first 2 fail, next 2 succeed → should warm 2 + targets = [("cycle", f"/lib/f{i}.mkv") for i in range(4)] + side_effects = [False, False, True, True] + with patch(_MOD + ".WARM_LOAD_MAX", 0), \ + patch(_MOD + ".host_load", return_value=0.0), \ + patch(_MOD + ".WARM_MAX_CYCLE", 2), \ + patch(_MOD + ".PLEX_URL", "http://plex"), \ + patch(_MOD + ".PLEX_TOKEN", "tok"), \ + patch(_MOD + ".Plex"), \ + patch(_MOD + "._warm_targets", return_value=targets), \ + patch(_MOD + "._warm_file", side_effect=side_effects) as mock_warm: + warm_cycle() + self.assertEqual(mock_warm.call_count, 4) + + +# --------------------------------------------------------------------------- +# _warm_targets +# --------------------------------------------------------------------------- + +class WarmTargetsTest(unittest.TestCase): + + def setUp(self): + _reset_warmer_state() + + def _run_targets(self, plex, *, sources=None, load_max=0, max_cycle=99, + ondeck=True, ondeck_every=0, low_cache=False, + next_near_end=0, next_eps=1, recent_count=0, parts=0): + from doctor.checks.warmer import _warm_targets + with patch(_MOD + ".WARM_SOURCES", sources or []), \ + patch(_MOD + ".WARM_LOAD_MAX", load_max), \ + patch(_MOD + ".WARM_MAX_CYCLE", max_cycle), \ + patch(_MOD + ".WARM_ONDECK", ondeck), \ + patch(_MOD + ".WARM_ONDECK_EVERY", ondeck_every), \ + patch(_MOD + ".WARM_LOW_CACHE", low_cache), \ + patch(_MOD + ".WARM_NEXT_NEAR_END", next_near_end), \ + patch(_MOD + ".WARM_NEXT_EPS", next_eps), \ + patch(_MOD + ".WARM_RECENT_COUNT", recent_count), \ + patch(_MOD + ".WARM_PARTS", parts): + return _warm_targets(plex) + + def test_empty_when_no_sources(self): + plex = _make_plex() + targets = self._run_targets(plex, sources=[]) + self.assertEqual(targets, []) + + def test_ondeck_added_when_no_sessions(self): + plex = _make_plex( + sessions=[], + ondeck=[{"ratingKey": "rk1"}], + parts=["/lib/file.mkv"], + ) + plex.parts.return_value = ["/lib/file.mkv"] + targets = self._run_targets(plex, sources=["ondeck"], ondeck=True, ondeck_every=0) + reasons = [r for r, _ in targets] + self.assertIn("ondeck", reasons) + + def test_ondeck_skipped_when_sessions_active(self): + """On Deck is never pre-warmed while someone is actively watching.""" + plex = _make_plex( + sessions=[{"ratingKey": "live", "type": "episode"}], + ondeck=[{"ratingKey": "rk1"}], + ) + targets = self._run_targets(plex, sources=["ondeck"], ondeck=True, ondeck_every=0) + reasons = [r for r, _ in targets] + self.assertNotIn("ondeck", reasons) + + def test_next_ep_added_when_episode_near_end(self): + """next-ep sources added when remaining play time < WARM_NEXT_NEAR_END.""" + duration = 60 * 60 * 1000 # 60 min in ms + offset = 55 * 60 * 1000 # 55 min watched → 5 min remain + session = { + "type": "episode", + "grandparentRatingKey": "show1", + "ratingKey": "ep1", + "duration": duration, + "viewOffset": offset, + } + next_ep = {"ratingKey": "ep2"} + plex = _make_plex(sessions=[session], leaves=[{"ratingKey": "ep1"}, next_ep]) + plex.parts.return_value = ["/lib/next.mkv"] + targets = self._run_targets(plex, sources=["next"], next_near_end=10, next_eps=1) + reasons = [r for r, _ in targets] + self.assertIn("next-ep", reasons) + + def test_next_ep_skipped_when_too_much_remaining(self): + """next-ep NOT added when remaining play time > WARM_NEXT_NEAR_END.""" + duration = 60 * 60 * 1000 # 60 min + offset = 10 * 60 * 1000 # only 10 min watched → 50 min remain + session = { + "type": "episode", + "grandparentRatingKey": "show1", + "ratingKey": "ep1", + "duration": duration, + "viewOffset": offset, + } + plex = _make_plex(sessions=[session], leaves=[{"ratingKey": "ep1"}, {"ratingKey": "ep2"}]) + plex.parts.return_value = ["/lib/next.mkv"] + targets = self._run_targets(plex, sources=["next"], next_near_end=10, next_eps=1) + reasons = [r for r, _ in targets] + self.assertNotIn("next-ep", reasons) + + def test_duplicate_paths_deduplicated(self): + """The same path appearing in multiple sources is only added once.""" + plex = _make_plex( + sessions=[], + ondeck=[{"ratingKey": "rk1"}, {"ratingKey": "rk2"}], + ) + plex.parts.return_value = ["/lib/same.mkv"] # both return the same path + targets = self._run_targets(plex, sources=["ondeck"], ondeck=True, ondeck_every=0) + paths = [p for _, p in targets] + self.assertEqual(len(paths), len(set(paths))) + + +class WarmCyclemetadataTest(unittest.TestCase): + """Tests for per-cycle metadata vars added in B4.""" + + def _import_warmer(self): + import importlib + import doctor.checks.warmer as w + importlib.reload(w) + return w + + def test_last_cycle_ts_updated_after_cycle(self): + import doctor.checks.warmer as w + # Reset + w._last_cycle_ts[0] = 0.0 + with patch("doctor.checks.warmer.WARM_LOAD_MAX", 0), \ + patch("doctor.checks.warmer._warm_targets", return_value=[]), \ + patch("doctor.checks.warmer.Plex"): + before = time.time() + w.warm_cycle() + after = time.time() + self.assertGreaterEqual(w._last_cycle_ts[0], before) + self.assertLessEqual(w._last_cycle_ts[0], after) + + def test_last_cycle_warmed_count(self): + import doctor.checks.warmer as w + w._last_cycle_warmed[0] = 0 + targets = [("ondeck", "/fake/a.mkv"), ("ondeck", "/fake/b.mkv")] + with patch("doctor.checks.warmer.WARM_LOAD_MAX", 0), \ + patch("doctor.checks.warmer.WARM_MAX_CYCLE", 10), \ + patch("doctor.checks.warmer._warm_targets", return_value=targets), \ + patch("doctor.checks.warmer._warm_file", return_value=True), \ + patch("doctor.checks.warmer.Plex"): + w.warm_cycle() + self.assertEqual(w._last_cycle_warmed[0], 2) + + def test_last_cycle_candidates_count(self): + import doctor.checks.warmer as w + w._last_cycle_candidates[0] = 0 + targets = [("ondeck", "/a"), ("ondeck", "/b"), ("next-ep", "/c")] + with patch("doctor.checks.warmer.WARM_LOAD_MAX", 0), \ + patch("doctor.checks.warmer.WARM_MAX_CYCLE", 10), \ + patch("doctor.checks.warmer._warm_targets", return_value=targets), \ + patch("doctor.checks.warmer._warm_file", return_value=False), \ + patch("doctor.checks.warmer.Plex"): + w.warm_cycle() + self.assertEqual(w._last_cycle_candidates[0], 3) + + def test_last_cycle_skipped_load_true_when_load_exceeded(self): + import doctor.checks.warmer as w + w._last_cycle_skipped_load[0] = False + with patch("doctor.checks.warmer.WARM_LOAD_MAX", 1.0), \ + patch("doctor.checks.warmer.host_load", return_value=5.0): + w.warm_cycle() + self.assertTrue(w._last_cycle_skipped_load[0]) + + def test_last_cycle_skipped_load_false_when_load_ok(self): + import doctor.checks.warmer as w + w._last_cycle_skipped_load[0] = True + with patch("doctor.checks.warmer.WARM_LOAD_MAX", 0), \ + patch("doctor.checks.warmer._warm_targets", return_value=[]), \ + patch("doctor.checks.warmer.Plex"): + w.warm_cycle() + self.assertFalse(w._last_cycle_skipped_load[0]) + + def test_last_cycle_duration_is_nonnegative_float(self): + import doctor.checks.warmer as w + w._last_cycle_duration[0] = -1.0 + with patch("doctor.checks.warmer.WARM_LOAD_MAX", 0), \ + patch("doctor.checks.warmer._warm_targets", return_value=[]), \ + patch("doctor.checks.warmer.Plex"): + w.warm_cycle() + self.assertGreaterEqual(w._last_cycle_duration[0], 0.0) + self.assertIsInstance(w._last_cycle_duration[0], float) diff --git a/tests/test_webui_state_endpoint.py b/tests/test_webui_state_endpoint.py new file mode 100644 index 0000000..aab8b3c --- /dev/null +++ b/tests/test_webui_state_endpoint.py @@ -0,0 +1,99 @@ +"""Tests for the /api/state endpoint (B3). + +GET /api/state returns the full contents of state.json as JSON, +authenticated via the existing UI_TOKEN mechanism. +""" +import json +import threading +import time +import unittest +from http.client import HTTPConnection +from unittest.mock import patch + + +def _get(port, path, token=None): + conn = HTTPConnection("localhost", port, timeout=3) + headers = {} + if token: + headers["X-Doctor-Token"] = token + conn.request("GET", path, headers=headers) + resp = conn.getresponse() + body = resp.read() + conn.close() + return resp.status, body + + +class ApiStateEndpointTest(unittest.TestCase): + """Integration tests for GET /api/state.""" + + def _start(self, port, state_data, ui_token="", en_ui=True): + """Start the server with patches held for the test's lifetime.""" + self._patches = [ + patch("doctor.webui.EN_UI", en_ui), + patch("doctor.webui.UI_TOKEN", ui_token), + patch("doctor.state._load_state_unlocked", return_value=state_data), + ] + for p in self._patches: + p.start() + from doctor.webui import _build_server + srv = _build_server(port) + threading.Thread(target=srv.serve_forever, daemon=True).start() + time.sleep(0.05) # let the server bind + return srv + + def tearDown(self): + for p in getattr(self, "_patches", []): + try: p.stop() + except Exception: pass + + def test_returns_state_json_when_no_token_required(self): + """Without UI_TOKEN, /api/state is accessible and returns state data.""" + state = {"__seerr__": {"1": 2}, "__repair_verify__": {}} + srv = self._start(19100, state) + try: + code, body = _get(19100, "/api/state") + self.assertEqual(code, 200) + data = json.loads(body) + self.assertEqual(data.get("__seerr__"), {"1": 2}) + finally: + srv.shutdown() + + def test_returns_401_without_token_when_token_required(self): + """With UI_TOKEN set, unauthenticated request returns 401.""" + srv = self._start(19101, {}, ui_token="secret") + try: + code, _ = _get(19101, "/api/state") + self.assertEqual(code, 401) + finally: + srv.shutdown() + + def test_returns_state_with_valid_token(self): + """With correct X-Doctor-Token header, /api/state returns state data.""" + state = {"__repair_mfd__": {"arr:1": 1234}} + srv = self._start(19102, state, ui_token="secret") + try: + code, body = _get(19102, "/api/state", token="secret") + self.assertEqual(code, 200) + data = json.loads(body) + self.assertIn("__repair_mfd__", data) + finally: + srv.shutdown() + + def test_returns_404_when_ui_disabled(self): + """When EN_UI=False, /api/state returns 404 (same as all other UI endpoints).""" + srv = self._start(19103, {}, en_ui=False) + try: + code, _ = _get(19103, "/api/state") + self.assertEqual(code, 404) + finally: + srv.shutdown() + + def test_empty_state_returns_empty_object(self): + """An empty state file returns an empty JSON object.""" + srv = self._start(19104, {}) + try: + code, body = _get(19104, "/api/state") + self.assertEqual(code, 200) + self.assertEqual(json.loads(body), {}) + finally: + srv.shutdown() diff --git a/tests/test_webui_status.py b/tests/test_webui_status.py new file mode 100644 index 0000000..b419f4d --- /dev/null +++ b/tests/test_webui_status.py @@ -0,0 +1,145 @@ +"""Tests for _ui_status() run metadata exposure (B2). + +These tests verify that the /api/status response includes per-check run +metadata from scheduler._check_runs, without breaking existing fields. +""" +import time +import unittest + + +def _call_ui_status(): + from doctor.webui import _ui_status + return _ui_status() + + +class UiStatusFieldsTest(unittest.TestCase): + """Existing /api/status contract must remain intact.""" + + def test_top_level_keys_present(self): + r = _call_ui_status() + for key in ("version", "mode", "dry_run", "load", "checks"): + self.assertIn(key, r) + + def test_checks_is_a_list(self): + r = _call_ui_status() + self.assertIsInstance(r["checks"], list) + + def test_each_check_has_name_and_on(self): + r = _call_ui_status() + for c in r["checks"]: + self.assertIn("name", c) + self.assertIn("on", c) + self.assertIsInstance(c["on"], bool) + + +class UiStatusRunMetadataTest(unittest.TestCase): + """Per-check run metadata keys must appear in each check entry.""" + + RUN_FIELDS = ("last_start", "last_end", "last_duration", + "last_outcome", "last_error", "run_count", "error_count") + + def test_run_fields_present_on_unrun_check(self): + """Checks with no run record yet should still have the metadata keys (None/0).""" + import doctor.scheduler as sched + # Ensure the check has no record + sched._check_runs.pop("queue", None) + r = _call_ui_status() + queue_entry = next((c for c in r["checks"] if c["name"] == "queue"), None) + self.assertIsNotNone(queue_entry) + for field in self.RUN_FIELDS: + self.assertIn(field, queue_entry, "missing field: %s" % field) + + def test_run_fields_null_when_never_run(self): + import doctor.scheduler as sched + sched._check_runs.pop("queue", None) + r = _call_ui_status() + q = next(c for c in r["checks"] if c["name"] == "queue") + self.assertIsNone(q["last_start"]) + self.assertIsNone(q["last_outcome"]) + self.assertEqual(q["run_count"], 0) + self.assertEqual(q["error_count"], 0) + + def test_run_fields_populated_after_run(self): + import doctor.scheduler as sched + now = time.time() + sched._check_runs["queue"] = { + "last_start": now - 1.5, + "last_end": now, + "last_duration": 1.5, + "last_outcome": "ok", + "last_error": "", + "run_count": 3, + "error_count": 0, + } + r = _call_ui_status() + q = next(c for c in r["checks"] if c["name"] == "queue") + self.assertEqual(q["last_outcome"], "ok") + self.assertEqual(q["run_count"], 3) + self.assertAlmostEqual(q["last_duration"], 1.5, places=2) + # Cleanup + sched._check_runs.pop("queue", None) + + def test_error_metadata_propagated(self): + import doctor.scheduler as sched + now = time.time() + sched._check_runs["repair"] = { + "last_start": now - 5, + "last_end": now, + "last_duration": 5.0, + "last_outcome": "error", + "last_error": "connection refused", + "run_count": 1, + "error_count": 1, + } + r = _call_ui_status() + rep = next(c for c in r["checks"] if c["name"] == "repair") + self.assertEqual(rep["last_outcome"], "error") + self.assertIn("connection", rep["last_error"]) + self.assertEqual(rep["error_count"], 1) + sched._check_runs.pop("repair", None) + + def test_warmer_synthetic_entry_still_has_name_and_on(self): + """The synthetic warmer entry appended at the end must still have name + on.""" + r = _call_ui_status() + warmer = next((c for c in r["checks"] if c["name"] == "warmer"), None) + self.assertIsNotNone(warmer) + self.assertIn("on", warmer) + + +class UiWarmerMetadataTest(unittest.TestCase): + """_ui_warmer() must include per-cycle metadata fields.""" + + def _call(self): + from doctor.webui import _ui_warmer + return _ui_warmer() + + def test_cycle_metadata_fields_present(self): + r = self._call() + for field in ("last_cycle_ts", "last_cycle_ago", "last_cycle_duration_s", + "last_cycle_warmed", "last_cycle_candidates", "last_cycle_skipped_load"): + self.assertIn(field, r, "missing field: %s" % field) + + def test_null_when_never_run(self): + import doctor.checks.warmer as w + w._last_cycle_ts[0] = 0.0 + r = self._call() + self.assertIsNone(r["last_cycle_ts"]) + self.assertIsNone(r["last_cycle_ago"]) + + def test_populated_when_run(self): + import time, doctor.checks.warmer as w + now = time.time() + w._last_cycle_ts[0] = now - 60 + w._last_cycle_duration[0] = 2.5 + w._last_cycle_warmed[0] = 4 + w._last_cycle_candidates[0] = 10 + w._last_cycle_skipped_load[0] = False + r = self._call() + self.assertIsNotNone(r["last_cycle_ts"]) + self.assertAlmostEqual(r["last_cycle_ago"], 60, delta=2) + self.assertEqual(r["last_cycle_duration_s"], 2.5) + self.assertEqual(r["last_cycle_warmed"], 4) + self.assertEqual(r["last_cycle_candidates"], 10) + self.assertFalse(r["last_cycle_skipped_load"]) + # Reset + w._last_cycle_ts[0] = 0.0