From 0f7fc6f032458a7ade927fc6c33c63503e61dcee Mon Sep 17 00:00:00 2001 From: Daniel Gradman-Svendsen Date: Wed, 29 Jul 2026 13:06:00 +0200 Subject: [PATCH] Install Importarr from repository checkout --- Makefile | 27 +++++++++++++++++++++ README.md | 47 +++++++++++++++++++++++++++++++----- deploy/importarr.env.example | 20 +++++++++------ deploy/importarr.service | 5 ++-- deploy/repo-upgrade.sh | 35 +++++++++++++++++++++++++++ deploy/systemd-install.sh | 14 ++++++++--- importarr/config.py | 15 +++++++++--- importarr/main.py | 17 ++++++++++--- tests/test_config.py | 17 +++++++++++++ 9 files changed, 171 insertions(+), 26 deletions(-) create mode 100644 Makefile create mode 100644 deploy/repo-upgrade.sh create mode 100644 tests/test_config.py diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..4eda6ed --- /dev/null +++ b/Makefile @@ -0,0 +1,27 @@ +PYTHON ?= .venv/bin/python +PIP ?= .venv/bin/pip +SERVICE ?= importarr.service +LIVE_URL ?= http://127.0.0.1:8095 + +.PHONY: test install-systemd install-editable upgrade-local repo-upgrade verify-live + +test: + $(PYTHON) -m pytest + +install-systemd: + sudo -n sh deploy/systemd-install.sh + +install-editable: + sudo -n /opt/importarr/venv/bin/pip install --upgrade --editable /srv/opencode-workspace/importarr + +upgrade-local: + sudo -n /opt/importarr/venv/bin/pip install --upgrade --editable /srv/opencode-workspace/importarr + sudo -n systemctl restart $(SERVICE) + +repo-upgrade: + sudo -n sh /opt/importarr/repo-upgrade.sh + +verify-live: + curl -fsS $(LIVE_URL)/health + curl -fsS $(LIVE_URL)/api/status + curl -fsS $(LIVE_URL)/api/preview diff --git a/README.md b/README.md index f9f4b0a..f0eb27d 100644 --- a/README.md +++ b/README.md @@ -29,21 +29,41 @@ docker build -t importarr:local . git clone https://gitea.delphas.dk/daniels/importarr.git cd importarr sudo sh deploy/systemd-install.sh -sudo install -o importarr -g importarr -d /var/lib/importarr sudo ${EDITOR:-vi} /etc/importarr/importarr.env sudo systemctl start importarr.service ``` -The installer creates the `importarr` system user when needed. +The installer creates the `importarr` system user when needed, installs a virtualenv at `/opt/importarr/venv`, and installs the package in editable mode from the checked-out repository. The repository is therefore the source of truth: pull or edit the repo, reinstall/restart from the repo, and the service runs that code. + +For local upgrades from a checked-out repo on dgsserver1, use the repo workflow instead of editing live scripts: + +```sh +cd /srv/opencode-workspace/importarr +.venv/bin/python -m pytest +git status --short --branch +sudo -n /opt/importarr/venv/bin/pip install --upgrade --editable /srv/opencode-workspace/importarr +sudo -n systemctl restart importarr.service +make verify-live +``` + +For a machine that should stay current with the repository, use the installed repo-upgrade helper: + +```sh +sudo -n sh /opt/importarr/repo-upgrade.sh +``` + +The helper refuses to run when the checkout has uncommitted changes, then performs `git pull --ff-only`, refreshes the editable install, restarts `importarr.service`, and prints service status. Use it after changes have been committed and pushed to `main`. + +Release-worthy changes should be committed, tagged with SemVer (`v0.1.1`, `v0.2.0`, ...), pushed with tags, then reinstalled from the tagged checkout or artifact. Do not hand-edit `/usr/local/sbin/importarr-status.py` or `/usr/local/sbin/manual-media-import.py` except for a documented emergency hotfix that is immediately backported here. ### Required setup 1. In SABnzbd, create or confirm category `manual`. 2. Set its completed folder to the same path mounted as `IMPORTARR_DOWNLOAD_ROOT`. -3. Set `IMPORTARR_SAB_URL` and `IMPORTARR_SAB_API_KEY`. +3. Set `IMPORTARR_SAB_URL` and `IMPORTARR_SAB_API_KEY_FILE` or `IMPORTARR_SAB_API_KEY`. 4. Mount/configure `IMPORTARR_MOVIES_ROOT` and `IMPORTARR_TV_ROOT` read/write. 5. Set `IMPORTARR_AUTH_TOKEN` unless write endpoints are protected by a reverse proxy. -6. Check `GET /health`, then inspect `/api/jobs` before running imports. +6. Check `GET /health`, then inspect `/api/preview` before running imports. ## Safety model @@ -57,13 +77,14 @@ The installer creates the `importarr` system user when needed. - `GET /health` - `GET /api/status` - `GET /api/jobs` +- `GET /api/preview` - `GET /api/history` - `GET /api/manual-batches` - `POST /api/manual-batches` with `{ "path": "relative/or/absolute/path" }` - `DELETE /api/manual-batches/{id}` - `POST /api/import/run-now` -Set `IMPORTARR_AUTH_TOKEN` to require `Authorization: Bearer ` for write endpoints. +Set `IMPORTARR_AUTH_TOKEN_FILE` or `IMPORTARR_AUTH_TOKEN` to require `Authorization: Bearer ` for write endpoints. ## Development @@ -77,4 +98,18 @@ uvicorn importarr.main:app --reload ## Migration notes for dgsserver1 -Export the existing script settings into `IMPORTARR_*` env vars, add historical folders as explicit manual batches, run a dry-run/inspection through `/api/jobs`, then switch the systemd service or Compose route after the ready set matches expectations. +Export the existing script settings into `IMPORTARR_*` env vars, add historical folders as explicit manual batches, run a dry-run/inspection through `/api/preview`, then switch the systemd service or Compose route after the ready set matches expectations. + +On dgsserver1, the packaged service is the only intended active entrypoint after cutover. Keep `manual-media-import.timer` disabled unless a repo-managed worker/timer replaces it later. + +## Repository-as-install workflow + +Importarr should not drift into host-local scripts. Treat the checked-out repository as the install source: + +1. Make changes in `/srv/opencode-workspace/importarr`. +2. Run tests: `make test`. +3. Commit and push the repo change. +4. Install/restart from the same repo: `make upgrade-local` for local changes, or `make repo-upgrade` to pull the latest pushed `main` and restart. +5. Verify the live service: `make verify-live`. + +Do not edit `/usr/local/sbin/importarr-status.py`, `/usr/local/sbin/manual-media-import.py`, or files copied out of the repo as the normal workflow. If an emergency live hotfix is unavoidable, backport it to this repository immediately and run the repo install workflow again. diff --git a/deploy/importarr.env.example b/deploy/importarr.env.example index 8b2136d..7e4a3fd 100644 --- a/deploy/importarr.env.example +++ b/deploy/importarr.env.example @@ -1,12 +1,16 @@ -IMPORTARR_SAB_URL=http://sabnzbd:8080 -IMPORTARR_SAB_API_KEY=change-me +IMPORTARR_SAB_URL=http://127.0.0.1:8080 +# Prefer *_FILE for secrets. Plain env vars still work for local/dev installs. +# IMPORTARR_SAB_API_KEY=change-me +# IMPORTARR_SAB_API_KEY_FILE=/etc/importarr/sab-api-key IMPORTARR_SAB_CATEGORY=manual -IMPORTARR_DOWNLOAD_ROOT=/data/downloads/manual -IMPORTARR_MOVIES_ROOT=/data/movies -IMPORTARR_TV_ROOT=/data/tv -IMPORTARR_STATE_PATH=/config/importarr.db +IMPORTARR_DOWNLOAD_ROOT=/srv/scrypted/sabnzbd-data/downloads/manual +IMPORTARR_MOVIES_ROOT=/srv/media/movies +IMPORTARR_TV_ROOT=/srv/media/tv +IMPORTARR_STATE_PATH=/var/lib/importarr/importarr.db IMPORTARR_LOG_LEVEL=info -IMPORTARR_AUTH_TOKEN=change-me +# IMPORTARR_AUTH_TOKEN=change-me +# IMPORTARR_AUTH_TOKEN_FILE=/etc/importarr/auth-token IMPORTARR_BIND_HOST=0.0.0.0 -IMPORTARR_BIND_PORT=8765 +IMPORTARR_BIND_PORT=8095 IMPORTARR_POLL_SECONDS=60 +IMPORTARR_REPO_DIR=/srv/opencode-workspace/importarr diff --git a/deploy/importarr.service b/deploy/importarr.service index 70e6cb6..dd381e2 100644 --- a/deploy/importarr.service +++ b/deploy/importarr.service @@ -1,15 +1,16 @@ [Unit] Description=Importarr manual media importer After=network-online.target +Wants=network-online.target [Service] EnvironmentFile=/etc/importarr/importarr.env -ExecStart=/usr/local/bin/importarr +ExecStart=/opt/importarr/venv/bin/importarr Restart=on-failure +RestartSec=5s User=importarr Group=importarr StateDirectory=importarr -ReadWritePaths=/var/lib/importarr /etc/importarr [Install] WantedBy=multi-user.target diff --git a/deploy/repo-upgrade.sh b/deploy/repo-upgrade.sh new file mode 100644 index 0000000..46146d7 --- /dev/null +++ b/deploy/repo-upgrade.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env sh +set -eu + +if [ "$(id -u)" -ne 0 ]; then + echo "Run as root: sudo sh /opt/importarr/repo-upgrade.sh" >&2 + exit 1 +fi + +ENV_FILE=${IMPORTARR_ENV_FILE:-/etc/importarr/importarr.env} +if [ -f "$ENV_FILE" ]; then + # shellcheck disable=SC1090 + . "$ENV_FILE" +fi + +REPO_DIR=${IMPORTARR_REPO_DIR:-/srv/opencode-workspace/importarr} +SERVICE=${IMPORTARR_SERVICE:-importarr.service} +VENV=${IMPORTARR_VENV:-/opt/importarr/venv} + +if [ ! -d "$REPO_DIR/.git" ]; then + echo "Importarr repo not found at $REPO_DIR" >&2 + exit 1 +fi + +cd "$REPO_DIR" +if [ -n "$(git status --porcelain)" ]; then + echo "Refusing to upgrade: $REPO_DIR has uncommitted changes." >&2 + git status --short >&2 + exit 1 +fi + +git fetch --prune origin +git pull --ff-only +"$VENV/bin/pip" install --upgrade --editable "$REPO_DIR" +systemctl restart "$SERVICE" +systemctl --no-pager --full status "$SERVICE" diff --git a/deploy/systemd-install.sh b/deploy/systemd-install.sh index 0b6c0ba..7904820 100644 --- a/deploy/systemd-install.sh +++ b/deploy/systemd-install.sh @@ -7,16 +7,24 @@ if [ "$(id -u)" -ne 0 ]; then fi install -d -m 0755 /etc/importarr /var/lib/importarr +install -d -m 0755 /opt/importarr +REPO_DIR="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" if ! id importarr >/dev/null 2>&1; then useradd --system --home /var/lib/importarr --shell /usr/sbin/nologin importarr fi chown importarr:importarr /var/lib/importarr +python3 -m venv /opt/importarr/venv +/opt/importarr/venv/bin/pip install --upgrade pip +/opt/importarr/venv/bin/pip install --upgrade --editable "$REPO_DIR" if [ ! -f /etc/importarr/importarr.env ]; then - install -m 0600 deploy/importarr.env.example /etc/importarr/importarr.env + install -m 0600 "$REPO_DIR/deploy/importarr.env.example" /etc/importarr/importarr.env echo "Created /etc/importarr/importarr.env; edit it before starting the service." fi -install -m 0644 deploy/importarr.service /etc/systemd/system/importarr.service -python3 -m pip install --upgrade . +if ! grep -q '^IMPORTARR_REPO_DIR=' /etc/importarr/importarr.env; then + printf '\nIMPORTARR_REPO_DIR=%s\n' "$REPO_DIR" >> /etc/importarr/importarr.env +fi +install -m 0644 "$REPO_DIR/deploy/importarr.service" /etc/systemd/system/importarr.service +install -m 0755 "$REPO_DIR/deploy/repo-upgrade.sh" /opt/importarr/repo-upgrade.sh systemctl daemon-reload systemctl enable importarr.service echo "Edit /etc/importarr/importarr.env, then run: systemctl start importarr.service" diff --git a/importarr/config.py b/importarr/config.py index 7e1c92f..57bb377 100644 --- a/importarr/config.py +++ b/importarr/config.py @@ -28,7 +28,7 @@ class Settings(BaseModel): def from_env(cls) -> "Settings": return cls( sab_url=os.getenv("IMPORTARR_SAB_URL", cls.model_fields["sab_url"].default), - sab_api_key=os.getenv("IMPORTARR_SAB_API_KEY"), + sab_api_key=_env_secret("IMPORTARR_SAB_API_KEY"), sab_category=os.getenv("IMPORTARR_SAB_CATEGORY", "manual"), download_root=Path(os.getenv("IMPORTARR_DOWNLOAD_ROOT", "/data/downloads/manual")), movies_root=Path(os.getenv("IMPORTARR_MOVIES_ROOT", "/data/movies")), @@ -36,10 +36,10 @@ class Settings(BaseModel): state_path=Path(os.getenv("IMPORTARR_STATE_PATH", "/config/importarr.db")), log_level=os.getenv("IMPORTARR_LOG_LEVEL", "info"), radarr_url=os.getenv("IMPORTARR_RADARR_URL"), - radarr_api_key=os.getenv("IMPORTARR_RADARR_API_KEY"), + radarr_api_key=_env_secret("IMPORTARR_RADARR_API_KEY"), sonarr_url=os.getenv("IMPORTARR_SONARR_URL"), - sonarr_api_key=os.getenv("IMPORTARR_SONARR_API_KEY"), - auth_token=os.getenv("IMPORTARR_AUTH_TOKEN"), + sonarr_api_key=_env_secret("IMPORTARR_SONARR_API_KEY"), + auth_token=_env_secret("IMPORTARR_AUTH_TOKEN"), bind_host=os.getenv("IMPORTARR_BIND_HOST", "127.0.0.1"), bind_port=int(os.getenv("IMPORTARR_BIND_PORT", "8765")), poll_seconds=int(os.getenv("IMPORTARR_POLL_SECONDS", "60")), @@ -54,3 +54,10 @@ class Settings(BaseModel): if resolved != root and root not in resolved.parents: raise ValueError("path must resolve under IMPORTARR_DOWNLOAD_ROOT") return resolved + + +def _env_secret(name: str) -> str | None: + file_value = os.getenv(f"{name}_FILE") + if file_value: + return Path(file_value).read_text(encoding="utf-8").strip() + return os.getenv(name) diff --git a/importarr/main.py b/importarr/main.py index e61bb1f..8b36a2e 100644 --- a/importarr/main.py +++ b/importarr/main.py @@ -97,18 +97,29 @@ def history() -> list[dict[str, object]]: @app.get("/api/jobs") async def jobs() -> dict[str, object]: + return await preview() + + +@app.get("/api/preview") +async def preview() -> dict[str, object]: client = SabnzbdClient(settings.sab_url, settings.sab_api_key) try: active = await client.active_nzo_ids() data = await client.history() except Exception as exc: # do not leak keys in URL/params - return {"sab_status": "error", "error": exc.__class__.__name__, "jobs": manual_batch_jobs()} + jobs = manual_batch_jobs() + return {"sab_status": "error", "error": exc.__class__.__name__, "jobs": jobs, "would_import": len(jobs)} slots = data.get("history", {}).get("slots", []) rows = [] for item in slots: readiness = classify_history_item(item, active, settings.sab_category, settings.download_root) - rows.append({"name": item.get("name"), "state": readiness.state, "reason": readiness.reason, "storage": str(readiness.storage) if readiness.storage else None}) - return {"sab_status": "ok", "jobs": rows + manual_batch_jobs()} + if readiness.ready and readiness.storage: + for video in scan_videos(readiness.storage): + rows.append({"name": video.path.name, "state": "ready", "relative_path": str(video.relative_path), "storage": str(readiness.storage), "size": video.size}) + else: + rows.append({"name": item.get("name"), "state": readiness.state, "reason": readiness.reason, "storage": str(readiness.storage) if readiness.storage else None}) + jobs = rows + manual_batch_jobs() + return {"sab_status": "ok", "jobs": jobs, "would_import": sum(1 for row in jobs if row["state"] in {"ready", "manual_batch"})} def manual_batch_jobs() -> list[dict[str, object]]: diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..a61d1e2 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,17 @@ +from importarr.config import Settings + + +def test_file_secret_env_vars_are_supported(tmp_path, monkeypatch): + sab_key = tmp_path / "sab-api-key" + auth_token = tmp_path / "auth-token" + sab_key.write_text("sab-secret\n", encoding="utf-8") + auth_token.write_text("auth-secret\n", encoding="utf-8") + + monkeypatch.setenv("IMPORTARR_SAB_API_KEY_FILE", str(sab_key)) + monkeypatch.setenv("IMPORTARR_AUTH_TOKEN_FILE", str(auth_token)) + monkeypatch.setenv("IMPORTARR_STATE_PATH", str(tmp_path / "state.db")) + + settings = Settings.from_env() + + assert settings.sab_api_key == "sab-secret" + assert settings.auth_token == "auth-secret"