scaffold — FastAPI + SQLite + HTMX dashboard, ingest + dispatcher

- app/main.py : dashboard /, partials /partials/{jobs,monitor} (htmx polling)
- app/templates/ : index, jobs table, monitor card par worker
- app/static/style.css : thème sombre cohérent
- scripts/ingest.py : scan SSD d'acquisition, EXIF CreateDate → segments
  continus par (AUV, GoPro serial) avec seuil configurable
- scripts/dispatcher.py : polling queue, pick worker selon VRAM free,
  extraction ffmpeg + lingbot-map windowed --offload_to_cpu, progression DB
- DB : SQLite (acquisitions + jobs), lifecycle queued→extracting→running→done
- Workers par défaut : .87 (3060 12GB) + .84 (3090 24GB)

Contexte : QC terrain le jour-même (avant photogrammétrie à 30 jours),
plusieurs heures × 2 GoPros × 2-3 AUVs d'enregistrement à traiter en parallèle.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

scripts/ingest.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+"""Scan an acquisition directory, group GoPro MP4s into continuous segments,
+and insert jobs into the cosma-qc DB.
+
+Usage:
+    python3 ingest.py /mnt/portablessd/COSMA-<date>/ --name "La Ciotat 8 avril" [--gap-min 5]
+
+Directory layout expected (we saw this from the real SSD):
+
+    <root>/media/gopro{1,2}/GP{1,2}_AUV{209,210}/GX*.MP4
+
+The AUV tag and GoPro id come from folder names. The serial is read via
+exiftool (falls back to folder name if unavailable). Continuous segments are
+derived from EXIF CreateDate timestamps with a configurable gap threshold.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import sqlite3
+import subprocess
+from datetime import datetime, timedelta
+from pathlib import Path
+
+DB_PATH = Path(os.environ.get("COSMA_QC_DB", "/var/lib/cosma-qc/jobs.db"))
+FOLDER_RE = re.compile(r"GP(?P<gopro>\d+)_AUV(?P<auv>\d+)", re.I)
+
+
+def exif_create_date(path: Path) -> datetime | None:
+    try:
+        out = subprocess.check_output(
+            ["exiftool", "-s3", "-CreateDate", "-api", "QuickTimeUTC=1", str(path)],
+            stderr=subprocess.DEVNULL, text=True, timeout=10,
+        ).strip()
+        return datetime.strptime(out, "%Y:%m:%d %H:%M:%S") if out else None
+    except Exception:
+        return None
+
+
+def exif_duration_s(path: Path) -> float | None:
+    try:
+        out = subprocess.check_output(
+            ["exiftool", "-s3", "-Duration#", str(path)],
+            stderr=subprocess.DEVNULL, text=True, timeout=10,
+        ).strip()
+        return float(out) if out else None
+    except Exception:
+        return None
+
+
+def exif_serial(path: Path) -> str | None:
+    try:
+        out = subprocess.check_output(
+            ["exiftool", "-s3", "-SerialNumber", "-CameraSerialNumber", str(path)],
+            stderr=subprocess.DEVNULL, text=True, timeout=10,
+        ).strip().splitlines()
+        for line in out:
+            line = line.strip()
+            if line:
+                return line
+    except Exception:
+        pass
+    return None
+
+
+def group_segments(videos: list[dict], gap_min: int) -> list[dict]:
+    """Group consecutive videos into segments when gap between end-of-A and
+    start-of-B is below `gap_min` minutes."""
+    videos = sorted(videos, key=lambda v: v["start"])
+    segments: list[list[dict]] = []
+    for v in videos:
+        if not segments:
+            segments.append([v]); continue
+        last = segments[-1][-1]
+        last_end = last["start"] + timedelta(seconds=last["duration"] or 0)
+        if (v["start"] - last_end) <= timedelta(minutes=gap_min):
+            segments[-1].append(v)
+        else:
+            segments.append([v])
+    out = []
+    for seg in segments:
+        start = seg[0]["start"]
+        end = seg[-1]["start"] + timedelta(seconds=seg[-1]["duration"] or 0)
+        out.append({
+            "start": start, "end": end,
+            "label": f"{start.strftime('%H:%M')}–{end.strftime('%H:%M')}",
+            "videos": [str(v["path"]) for v in seg],
+        })
+    return out
+
+
+def scan(root: Path) -> dict:
+    """Return {(auv, gopro_tag): {serial, videos[]}}"""
+    grouped: dict[tuple[str, str], dict] = {}
+    for mp4 in root.rglob("*.MP4"):
+        m = FOLDER_RE.search(str(mp4.parent))
+        if not m:
+            continue
+        auv = f"AUV{m.group('auv')}"
+        gopro_tag = f"GP{m.group('gopro')}"
+        key = (auv, gopro_tag)
+        start = exif_create_date(mp4)
+        dur = exif_duration_s(mp4)
+        if not start:
+            print(f"  [skip] no CreateDate: {mp4}"); continue
+        serial = exif_serial(mp4)
+        slot = grouped.setdefault(key, {"serial": serial, "videos": []})
+        if serial and not slot["serial"]:
+            slot["serial"] = serial
+        slot["videos"].append({"path": mp4, "start": start, "duration": dur or 0})
+    return grouped
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("root", type=Path)
+    ap.add_argument("--name", required=True, help="Acquisition name")
+    ap.add_argument("--gap-min", type=int, default=5, help="Max gap between videos in one segment")
+    ap.add_argument("--dry-run", action="store_true")
+    args = ap.parse_args()
+
+    if not args.root.exists():
+        raise SystemExit(f"root not found: {args.root}")
+
+    print(f"Scanning {args.root}...")
+    grouped = scan(args.root)
+    if not grouped:
+        print("No (auv, gopro) folders found — expected GPx_AUVyyy layout."); return
+
+    DB_PATH.parent.mkdir(parents=True, exist_ok=True)
+    conn = sqlite3.connect(DB_PATH, isolation_level=None)
+    conn.execute("PRAGMA foreign_keys=ON")
+    conn.row_factory = sqlite3.Row
+
+    if args.dry_run:
+        acq_id = -1
+    else:
+        cur = conn.execute(
+            "INSERT INTO acquisitions (name, source_path) VALUES (?, ?)",
+            (args.name, str(args.root)),
+        )
+        acq_id = cur.lastrowid
+        print(f"Created acquisition id={acq_id}")
+
+    total_jobs = 0
+    for (auv, gopro_tag), info in sorted(grouped.items()):
+        serial = info["serial"] or gopro_tag
+        segs = group_segments(info["videos"], args.gap_min)
+        print(f"\n{auv} / {gopro_tag} (serial={serial}) — {len(info['videos'])} videos → {len(segs)} segments")
+        for seg in segs:
+            dur_min = (seg["end"] - seg["start"]).total_seconds() / 60
+            print(f"  · {seg['label']}  ({dur_min:.1f} min, {len(seg['videos'])} files)")
+            if args.dry_run:
+                continue
+            conn.execute("""
+                INSERT INTO jobs (acquisition_id, auv, gopro_serial, segment_label,
+                                  video_paths, status)
+                VALUES (?, ?, ?, ?, ?, 'queued')
+            """, (acq_id, auv, serial, seg["label"], json.dumps(seg["videos"])))
+            total_jobs += 1
+
+    print(f"\nInserted {total_jobs} jobs.")
+
+
+if __name__ == "__main__":
+    main()