Initial productized Importarr service

importarr/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

importarr/arr_hints.py

@@ -0,0 +1,4 @@
+"""Optional Radarr/Sonarr hint clients will live here.
+
+Importarr owns file movement; Arr services are only future lookup helpers.
+"""

importarr/build_info.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+import os
+import platform
+from datetime import UTC, datetime
+
+from . import __version__
+
+
+def build_info() -> dict[str, str]:
+    return {
+        "name": "Importarr",
+        "version": os.getenv("IMPORTARR_VERSION", __version__),
+        "build_date": os.getenv("IMPORTARR_BUILD_DATE", "development"),
+        "git_sha": os.getenv("IMPORTARR_GIT_SHA", "development"),
+        "python": platform.python_version(),
+        "started_at": STARTED_AT,
+    }
+
+
+STARTED_AT = datetime.now(UTC).isoformat(timespec="seconds")

importarr/config.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+
+class Settings(BaseModel):
+    sab_url: str = "http://sabnzbd:8080"
+    sab_api_key: str | None = None
+    sab_category: str = "manual"
+    download_root: Path = Path("/data/downloads/manual")
+    movies_root: Path = Path("/data/movies")
+    tv_root: Path = Path("/data/tv")
+    state_path: Path = Path("/config/importarr.db")
+    log_level: str = "info"
+    radarr_url: str | None = None
+    radarr_api_key: str | None = None
+    sonarr_url: str | None = None
+    sonarr_api_key: str | None = None
+    auth_token: str | None = None
+    bind_host: str = "127.0.0.1"
+    bind_port: int = 8765
+    poll_seconds: int = Field(default=60, ge=5)
+
+    @classmethod
+    def from_env(cls) -> "Settings":
+        return cls(
+            sab_url=os.getenv("IMPORTARR_SAB_URL", cls.model_fields["sab_url"].default),
+            sab_api_key=os.getenv("IMPORTARR_SAB_API_KEY"),
+            sab_category=os.getenv("IMPORTARR_SAB_CATEGORY", "manual"),
+            download_root=Path(os.getenv("IMPORTARR_DOWNLOAD_ROOT", "/data/downloads/manual")),
+            movies_root=Path(os.getenv("IMPORTARR_MOVIES_ROOT", "/data/movies")),
+            tv_root=Path(os.getenv("IMPORTARR_TV_ROOT", "/data/tv")),
+            state_path=Path(os.getenv("IMPORTARR_STATE_PATH", "/config/importarr.db")),
+            log_level=os.getenv("IMPORTARR_LOG_LEVEL", "info"),
+            radarr_url=os.getenv("IMPORTARR_RADARR_URL"),
+            radarr_api_key=os.getenv("IMPORTARR_RADARR_API_KEY"),
+            sonarr_url=os.getenv("IMPORTARR_SONARR_URL"),
+            sonarr_api_key=os.getenv("IMPORTARR_SONARR_API_KEY"),
+            auth_token=os.getenv("IMPORTARR_AUTH_TOKEN"),
+            bind_host=os.getenv("IMPORTARR_BIND_HOST", "127.0.0.1"),
+            bind_port=int(os.getenv("IMPORTARR_BIND_PORT", "8765")),
+            poll_seconds=int(os.getenv("IMPORTARR_POLL_SECONDS", "60")),
+        )
+
+    def resolve_under_download_root(self, user_path: str) -> Path:
+        root = self.download_root.resolve()
+        candidate = Path(user_path)
+        if not candidate.is_absolute():
+            candidate = root / candidate
+        resolved = candidate.resolve()
+        if resolved != root and root not in resolved.parents:
+            raise ValueError("path must resolve under IMPORTARR_DOWNLOAD_ROOT")
+        return resolved

importarr/importer.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import os
+import shutil
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass
+class ImportResult:
+    source: Path
+    target: Path
+    bytes: int
+
+
+class Importer:
+    def __init__(self, movies_root: Path, tv_root: Path):
+        self.movies_root = movies_root
+        self.tv_root = tv_root
+
+    def target_for(self, source: Path) -> Path:
+        target_root = self.tv_root if _looks_like_tv(source) else self.movies_root
+        return _unique_path(target_root / source.name)
+
+    def import_file(self, source: Path) -> ImportResult:
+        target = self.target_for(source)
+        target.parent.mkdir(parents=True, exist_ok=True)
+        partial = target.with_name(target.name + ".partial")
+        with source.open("rb") as src, partial.open("wb") as dst:
+            shutil.copyfileobj(src, dst, length=1024 * 1024)
+            dst.flush()
+            os.fsync(dst.fileno())
+        if partial.stat().st_size != source.stat().st_size:
+            raise IOError("partial copy size mismatch")
+        partial.rename(target)
+        source.unlink()
+        _remove_empty_parents(source.parent)
+        return ImportResult(source=source, target=target, bytes=target.stat().st_size)
+
+
+def _looks_like_tv(path: Path) -> bool:
+    text = str(path).lower()
+    return any(marker in text for marker in ("s01", "season", "episode"))
+
+
+def _unique_path(path: Path) -> Path:
+    if not path.exists() and not path.with_name(path.name + ".partial").exists():
+        return path
+    stem, suffix = path.stem, path.suffix
+    for index in range(1, 10000):
+        candidate = path.with_name(f"{stem} ({index}){suffix}")
+        if not candidate.exists() and not candidate.with_name(candidate.name + ".partial").exists():
+            return candidate
+    raise RuntimeError(f"could not choose unique target for {path}")
+
+
+def _remove_empty_parents(path: Path) -> None:
+    while True:
+        try:
+            path.rmdir()
+        except OSError:
+            return
+        path = path.parent

importarr/main.py

@@ -0,0 +1,172 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated
+
+import uvicorn
+from fastapi import Depends, FastAPI, Header, HTTPException, Request
+from fastapi.responses import HTMLResponse
+from fastapi.staticfiles import StaticFiles
+from fastapi.templating import Jinja2Templates
+from pydantic import BaseModel
+
+from .build_info import build_info
+from .config import Settings
+from .importer import Importer
+from .sabnzbd import SabnzbdClient
+from .readiness import classify_history_item
+from .scanner import scan_videos
+from .state import State
+
+settings = Settings.from_env()
+state = State(settings.state_path)
+templates = Jinja2Templates(directory=str(Path(__file__).parent / "templates"))
+app = FastAPI(title="Importarr")
+app.mount("/static", StaticFiles(directory=str(Path(__file__).parent / "static")), name="static")
+
+
+class ManualBatchCreate(BaseModel):
+    path: str
+
+
+def require_write_auth(authorization: Annotated[str | None, Header()] = None) -> None:
+    if not settings.auth_token:
+        return
+    if authorization != f"Bearer {settings.auth_token}":
+        raise HTTPException(status_code=401, detail="write endpoint requires bearer token")
+
+
+@app.get("/health")
+def health() -> dict[str, str]:
+    return {"status": "ok", "name": "Importarr", "version": build_info()["version"]}
+
+
+@app.get("/", response_class=HTMLResponse)
+def index(request: Request) -> HTMLResponse:
+    return templates.TemplateResponse("index.html", {"request": request, "status": status(), "batches": state.list_manual_batches()})
+
+
+@app.get("/api/status")
+def status() -> dict[str, object]:
+    history = state.list_history()
+    return {
+        "app": "Importarr",
+        "build": build_info(),
+        "category": settings.sab_category,
+        "download_root": str(settings.download_root),
+        "movies_root": str(settings.movies_root),
+        "tv_root": str(settings.tv_root),
+        "sab_url": settings.sab_url,
+        "auth_enabled": bool(settings.auth_token),
+        "bind": f"{settings.bind_host}:{settings.bind_port}",
+        "manual_batches": len(state.list_manual_batches(active_only=True)),
+        "imported_total": sum(1 for row in history if row["status"] == "imported"),
+        "failed_total": sum(1 for row in history if row["status"] == "failed"),
+        "current": None,
+    }
+
+
+@app.get("/api/manual-batches")
+def manual_batches() -> list[dict[str, object]]:
+    rows = []
+    for batch in state.list_manual_batches():
+        videos = scan_videos(Path(batch["path"])) if batch["status"] == "active" else []
+        rows.append({**batch, "videos": [{"file": v.path.name, "relative_path": str(v.relative_path), "size": v.size} for v in videos]})
+    return rows
+
+
+@app.post("/api/manual-batches")
+def add_manual_batch(payload: ManualBatchCreate, _: None = Depends(require_write_auth)) -> dict[str, object]:
+    try:
+        resolved = settings.resolve_under_download_root(payload.path)
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+    return state.add_manual_batch(resolved)
+
+
+@app.delete("/api/manual-batches/{batch_id}")
+def delete_manual_batch(batch_id: int, _: None = Depends(require_write_auth)) -> dict[str, str]:
+    state.delete_manual_batch(batch_id)
+    return {"status": "deleted"}
+
+
+@app.get("/api/history")
+def history() -> list[dict[str, object]]:
+    return state.list_history()
+
+
+@app.get("/api/jobs")
+async def jobs() -> dict[str, object]:
+    client = SabnzbdClient(settings.sab_url, settings.sab_api_key)
+    try:
+        active = await client.active_nzo_ids()
+        data = await client.history()
+    except Exception as exc:  # do not leak keys in URL/params
+        return {"sab_status": "error", "error": exc.__class__.__name__, "jobs": manual_batch_jobs()}
+    slots = data.get("history", {}).get("slots", [])
+    rows = []
+    for item in slots:
+        readiness = classify_history_item(item, active, settings.sab_category, settings.download_root)
+        rows.append({"name": item.get("name"), "state": readiness.state, "reason": readiness.reason, "storage": str(readiness.storage) if readiness.storage else None})
+    return {"sab_status": "ok", "jobs": rows + manual_batch_jobs()}
+
+
+def manual_batch_jobs() -> list[dict[str, object]]:
+    rows: list[dict[str, object]] = []
+    root = settings.download_root.resolve()
+    for batch in state.list_manual_batches(active_only=True):
+        for video in scan_videos(Path(batch["path"])):
+            rows.append({"name": video.path.name, "state": "manual_batch", "relative_path": str(video.path.relative_to(root)), "size": video.size})
+    return rows
+
+
+@app.post("/api/import/run-now")
+async def run_now(_: None = Depends(require_write_auth)) -> dict[str, object]:
+    importer = Importer(settings.movies_root, settings.tv_root)
+    imported = 0
+    imported += await _import_ready_sab_jobs(importer)
+    imported += _import_manual_batches(importer)
+    return {"imported": imported}
+
+
+async def _import_ready_sab_jobs(importer: Importer) -> int:
+    client = SabnzbdClient(settings.sab_url, settings.sab_api_key)
+    try:
+        active = await client.active_nzo_ids()
+        data = await client.history()
+    except Exception:
+        return 0
+    imported = 0
+    for item in data.get("history", {}).get("slots", []):
+        readiness = classify_history_item(item, active, settings.sab_category, settings.download_root)
+        if not readiness.ready or readiness.storage is None:
+            continue
+        for video in scan_videos(readiness.storage):
+            try:
+                result = importer.import_file(video.path)
+                state.add_history(result.source, result.target, "imported", result.bytes)
+                imported += 1
+            except Exception as exc:
+                state.add_history(video.path, video.path, "failed", 0, exc.__class__.__name__)
+    return imported
+
+
+def _import_manual_batches(importer: Importer) -> int:
+    imported = 0
+    for batch in state.list_manual_batches(active_only=True):
+        path = Path(batch["path"])
+        videos = scan_videos(path)
+        for video in videos:
+            try:
+                result = importer.import_file(video.path)
+                state.add_history(result.source, result.target, "imported", result.bytes)
+                imported += 1
+            except Exception as exc:
+                state.add_history(video.path, video.path, "failed", 0, exc.__class__.__name__)
+        if not scan_videos(path):
+            state.complete_manual_batch(batch["id"])
+    return imported
+
+
+def run() -> None:
+    uvicorn.run("importarr.main:app", host=settings.bind_host, port=settings.bind_port, reload=False)

importarr/readiness.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+TRANSIENT_PARTS = {"_UNPACK_", "__UNPACK__", "_FAILED_", "_ADMIN_"}
+NOT_READY_STATUSES = {
+    "Queued", "QuickCheck", "Verifying", "Repairing", "Fetching", "Extracting",
+    "Moving", "Running", "Downloading", "Paused", "Propagating",
+}
+
+
+@dataclass(frozen=True)
+class Readiness:
+    state: str
+    reason: str
+    storage: Path | None = None
+
+    @property
+    def ready(self) -> bool:
+        return self.state == "ready"
+
+
+def has_transient_part(path: Path) -> bool:
+    return any(part in TRANSIENT_PARTS or any(token in part for token in TRANSIENT_PARTS) for part in path.parts)
+
+
+def classify_history_item(item: dict[str, Any], active_nzo_ids: set[str], category: str, download_root: Path) -> Readiness:
+    nzo_id = str(item.get("nzo_id") or item.get("nzoid") or "")
+    if nzo_id and nzo_id in active_nzo_ids:
+        return Readiness("processing", "SAB job is still present in queue")
+    if str(item.get("category") or "") != category:
+        return Readiness("ignored", "SAB category is not owned by Importarr")
+    status = str(item.get("status") or "")
+    if status == "Failed":
+        return Readiness("failed", "SAB history reports failure")
+    if status in NOT_READY_STATUSES or status != "Completed":
+        return Readiness("processing", f"SAB status is {status or 'unknown'}")
+    storage_value = str(item.get("storage") or "")
+    if not storage_value:
+        return Readiness("unknown", "SAB completed item has no final storage")
+    storage = Path(storage_value).resolve()
+    root = download_root.resolve()
+    if storage != root and root not in storage.parents:
+        return Readiness("ignored", "SAB storage is outside configured download root", storage)
+    if has_transient_part(storage):
+        return Readiness("processing", "SAB storage path contains transient unpack/admin marker", storage)
+    return Readiness("ready", "SAB completed in owned category with final storage", storage)

importarr/sabnzbd.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from typing import Any
+
+import httpx
+
+
+class SabnzbdClient:
+    def __init__(self, base_url: str, api_key: str | None):
+        self.base_url = base_url.rstrip("/")
+        self.api_key = api_key
+
+    async def _get(self, mode: str) -> dict[str, Any]:
+        params = {"mode": mode, "output": "json"}
+        if self.api_key:
+            params["apikey"] = self.api_key
+        async with httpx.AsyncClient(timeout=15) as client:
+            response = await client.get(f"{self.base_url}/api", params=params)
+            response.raise_for_status()
+            return response.json()
+
+    async def queue(self) -> dict[str, Any]:
+        return await self._get("queue")
+
+    async def history(self) -> dict[str, Any]:
+        return await self._get("history")
+
+    async def active_nzo_ids(self) -> set[str]:
+        data = await self.queue()
+        slots = data.get("queue", {}).get("slots", [])
+        return {str(slot.get("nzo_id") or slot.get("nzoid")) for slot in slots if slot.get("nzo_id") or slot.get("nzoid")}

importarr/scanner.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from .readiness import has_transient_part
+
+VIDEO_EXTENSIONS = {".mkv", ".mp4", ".avi", ".mov", ".m4v", ".wmv", ".ts"}
+
+
+@dataclass(frozen=True)
+class VideoFile:
+    path: Path
+    relative_path: Path
+    size: int
+
+
+def is_sample(path: Path) -> bool:
+    lowered = "/".join(path.parts).lower()
+    return "sample" in lowered or path.name.lower().startswith("sample")
+
+
+def scan_videos(root: Path, *, include_transient: bool = False) -> list[VideoFile]:
+    root = root.resolve()
+    if not root.exists():
+        return []
+    results: list[VideoFile] = []
+    for path in root.rglob("*"):
+        if not path.is_file() or path.suffix.lower() not in VIDEO_EXTENSIONS:
+            continue
+        if is_sample(path):
+            continue
+        if not include_transient and has_transient_part(path):
+            continue
+        results.append(VideoFile(path=path, relative_path=path.relative_to(root), size=path.stat().st_size))
+    return sorted(results, key=lambda item: str(item.relative_path))

importarr/state.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+import sqlite3
+from pathlib import Path
+from typing import Any
+
+
+class State:
+    def __init__(self, path: Path):
+        self.path = path
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self.conn = sqlite3.connect(self.path, check_same_thread=False)
+        self.conn.row_factory = sqlite3.Row
+        self.migrate()
+
+    def migrate(self) -> None:
+        self.conn.executescript(
+            """
+            create table if not exists manual_batches (
+              id integer primary key autoincrement,
+              path text not null unique,
+              status text not null default 'active',
+              created_at text not null default current_timestamp,
+              completed_at text
+            );
+            create table if not exists import_history (
+              id integer primary key autoincrement,
+              source text not null,
+              target text not null,
+              status text not null,
+              bytes integer not null default 0,
+              created_at text not null default current_timestamp,
+              completed_at text,
+              error text
+            );
+            create table if not exists app_state (key text primary key, value text not null);
+            """
+        )
+        self.conn.commit()
+
+    def add_manual_batch(self, path: Path) -> dict[str, Any]:
+        self.conn.execute("insert or ignore into manual_batches(path) values (?)", (str(path),))
+        self.conn.commit()
+        return self.get_manual_batch_by_path(path)
+
+    def get_manual_batch_by_path(self, path: Path) -> dict[str, Any]:
+        row = self.conn.execute("select * from manual_batches where path = ?", (str(path),)).fetchone()
+        return dict(row)
+
+    def list_manual_batches(self, active_only: bool = False) -> list[dict[str, Any]]:
+        sql = "select * from manual_batches"
+        if active_only:
+            sql += " where status = 'active'"
+        sql += " order by created_at desc"
+        return [dict(row) for row in self.conn.execute(sql)]
+
+    def delete_manual_batch(self, batch_id: int) -> None:
+        self.conn.execute("delete from manual_batches where id = ?", (batch_id,))
+        self.conn.commit()
+
+    def complete_manual_batch(self, batch_id: int) -> None:
+        self.conn.execute("update manual_batches set status='completed', completed_at=current_timestamp where id=?", (batch_id,))
+        self.conn.commit()
+
+    def add_history(self, source: Path, target: Path, status: str, bytes_count: int = 0, error: str | None = None) -> None:
+        self.conn.execute(
+            "insert into import_history(source,target,status,bytes,error,completed_at) values (?,?,?,?,?,case when ? in ('imported','failed') then current_timestamp else null end)",
+            (str(source), str(target), status, bytes_count, error, status),
+        )
+        self.conn.commit()
+
+    def list_history(self, limit: int = 100) -> list[dict[str, Any]]:
+        return [dict(row) for row in self.conn.execute("select * from import_history order by id desc limit ?", (limit,))]

importarr/static/importarr.css

@@ -0,0 +1 @@
+body{font-family:system-ui,sans-serif;margin:0;background:#111827;color:#e5e7eb}header,main{max-width:1100px;margin:auto;padding:1rem}.topbar{display:flex;justify-content:space-between;gap:1rem;align-items:center;background:#0f172a}.build{text-align:right}.build strong{font-size:1.2rem}.cards{display:grid;grid-template-columns:repeat(auto-fit,minmax(12rem,1fr));gap:1rem}.cards article,.panel{background:#1f2937;border-radius:.75rem;padding:1rem;margin-top:1rem}strong{display:block;font-size:2rem}span,small,dd{color:#9ca3af}table{width:100%;border-collapse:collapse;background:#1f2937;margin-top:1rem}th,td{padding:.6rem;border-bottom:1px solid #374151;text-align:left}input,button{padding:.6rem;border-radius:.4rem;border:1px solid #374151}button{background:#38bdf8;color:#082f49;font-weight:700}.info{display:grid;grid-template-columns:10rem 1fr;gap:.4rem 1rem}.info dt{font-weight:700}.info dd{margin:0;overflow-wrap:anywhere}.state{background:#0f172a;border:1px solid #374151;border-radius:999px;padding:.15rem .5rem}

importarr/templates/index.html

@@ -0,0 +1,54 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>📥 - idle · Importarr {{ status.build.version }}</title>
+  <link rel="stylesheet" href="/static/importarr.css">
+</head>
+<body>
+  <header class="topbar">
+    <div><h1>Importarr</h1><p>Manual SABnzbd imports, safely gated by SAB completion.</p></div>
+    <div class="build"><strong>{{ status.build.version }}</strong><span>{{ status.build.git_sha[:12] }} · {{ status.build.build_date }}</span></div>
+  </header>
+  <main>
+    <section class="cards">
+      <article><strong>{{ status.imported_total }}</strong><span>Imported total</span></article>
+      <article><strong>{{ status.failed_total }}</strong><span>Failed total</span></article>
+      <article><strong>{{ status.manual_batches }}</strong><span>Manual batches</span></article>
+      <article><strong>{{ status.category }}</strong><span>SAB category</span></article>
+    </section>
+    <section class="panel">
+      <h2>Service info</h2>
+      <dl class="info">
+        <dt>Name</dt><dd>{{ status.build.name }}</dd>
+        <dt>Version</dt><dd>{{ status.build.version }}</dd>
+        <dt>Build date</dt><dd>{{ status.build.build_date }}</dd>
+        <dt>Git SHA</dt><dd>{{ status.build.git_sha }}</dd>
+        <dt>Started</dt><dd>{{ status.build.started_at }}</dd>
+        <dt>Python</dt><dd>{{ status.build.python }}</dd>
+        <dt>SAB URL</dt><dd>{{ status.sab_url }}</dd>
+        <dt>Download root</dt><dd>{{ status.download_root }}</dd>
+        <dt>Movies root</dt><dd>{{ status.movies_root }}</dd>
+        <dt>TV root</dt><dd>{{ status.tv_root }}</dd>
+        <dt>Write auth</dt><dd>{{ 'enabled' if status.auth_enabled else 'disabled' }}</dd>
+      </dl>
+    </section>
+    <section>
+      <h2>Manual batches</h2>
+      <form id="batch-form"><input name="path" placeholder="folder under download root"><button>Add batch</button></form>
+      <table><thead><tr><th>ID</th><th>Status</th><th>Path</th></tr></thead><tbody>
+      {% for batch in batches %}<tr><td>{{ batch.id }}</td><td>{{ batch.status }}</td><td>{{ batch.path }}</td></tr>{% endfor %}
+      </tbody></table>
+    </section>
+    <section>
+      <h2>Jobs</h2><div id="jobs">Loading…</div>
+    </section>
+  </main>
+  <script>
+  async function refresh(){ const r=await fetch('/api/jobs'); const d=await r.json(); document.title=d.jobs.length?`📥 ${d.jobs.length} jobs - Importarr`:'📥 - idle · Importarr'; document.getElementById('jobs').innerHTML='<table><tr><th>Name</th><th>State</th><th>Context</th></tr>'+d.jobs.map(j=>`<tr><td>${j.name||''}</td><td><span class="state">${j.state}</span></td><td><small>${j.relative_path||j.storage||j.reason||''}</small></td></tr>`).join('')+'</table>'; }
+  document.getElementById('batch-form').addEventListener('submit', async e=>{ e.preventDefault(); await fetch('/api/manual-batches',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({path:e.target.path.value})}); location.reload(); });
+  refresh(); setInterval(refresh, 10000);
+  </script>
+</body>
+</html>