feat: closes #2,#3,#4 — scripts pre_decimate + archive_job + check_jobs + tests

scripts/archive_job.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# archive_job.sh <job_id> [frames_base] [nas_base]
+# Rsyncs job frames + PLY to NAS .156
+set -euo pipefail
+
+JOB_ID="${1:?Usage: archive_job.sh <job_id> [frames_base] [nas_base]}"
+FRAMES_BASE="${2:-/root/cosma-qc-frames}"
+NAS_BASE="${3:-/mnt/nas-cosma/cosma-archive}"
+
+SRC="${FRAMES_BASE}/job_${JOB_ID}"
+DST="${NAS_BASE}/job_${JOB_ID}"
+
+if [ ! -d "${SRC}" ]; then
+    echo "Job dir not found: ${SRC}"
+    exit 1
+fi
+
+mkdir -p "${DST}"
+echo "[$(date)] Archivage job_${JOB_ID} vers NAS..."
+
+rsync -av --progress "${SRC}/" "${DST}/" \
+    --include="frame_*.jpg" \
+    --include="*.ply" \
+    --include="*.npz" \
+    --include="*.log" \
+    --exclude="*" \
+    2>&1 | tail -5
+
+echo "[$(date)] Archive job_${JOB_ID} done: ${DST}"

scripts/check_jobs.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+"""Check integrity of processed jobs (PLY + poses present)."""
+import argparse
+import glob
+import json
+import os
+from pathlib import Path
+from typing import Any
+
+REQUIRED_FILES = ["reconstruction.ply", "lingbot_poses.npz"]
+OPTIONAL_FILES = ["model_decimated.ply"]
+
+
+def check_job(job_id: int, frames_base: str = "/root/cosma-qc-frames") -> dict[str, Any]:
+    job_dir = Path(frames_base) / f"job_{job_id}"
+    if not job_dir.exists():
+        return {"job_id": job_id, "status": "missing", "missing": [], "details": {}}
+
+    missing = [f for f in REQUIRED_FILES if not (job_dir / f).exists()]
+    details: dict[str, Any] = {}
+
+    ply = job_dir / "reconstruction.ply"
+    if ply.exists():
+        details["ply_size_gb"] = round(ply.stat().st_size / 1e9, 2)
+
+    poses = job_dir / "lingbot_poses.npz"
+    if poses.exists():
+        try:
+            import numpy as np
+            d = np.load(str(poses))
+            n = d["poses"].shape[0] if "poses" in d else 0
+            details["n_poses"] = n
+        except Exception as e:
+            details["poses_error"] = str(e)
+
+    decimated = job_dir / "model_decimated.ply"
+    details["decimated"] = decimated.exists()
+
+    return {
+        "job_id": job_id,
+        "status": "ok" if not missing else "incomplete",
+        "missing": missing,
+        "details": details,
+    }
+
+
+def main() -> None:
+    p = argparse.ArgumentParser()
+    p.add_argument("job_ids", nargs="*", type=int)
+    p.add_argument("--frames-base", default="/root/cosma-qc-frames")
+    p.add_argument("--all", action="store_true", help="Check all job dirs")
+    args = p.parse_args()
+
+    base = Path(args.frames_base)
+    if args.all:
+        ids = sorted(
+            int(d.name.replace("job_", ""))
+            for d in base.iterdir()
+            if d.is_dir() and d.name.startswith("job_") and d.name[4:].isdigit()
+        )
+    else:
+        ids = args.job_ids
+
+    results = [check_job(jid, args.frames_base) for jid in ids]
+    print(json.dumps(results, indent=2))
+
+
+if __name__ == "__main__":
+    main()

scripts/pre_decimate.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+"""Decimate PLY and SCP to cosma-vm after a job completes."""
+import argparse
+import subprocess
+import sys
+from pathlib import Path
+
+COSMA_VM = "cosma@192.168.0.83"
+COSMA_DATA = "/data/cosma"
+MAX_PTS = 300_000
+
+
+def find_ply(frames_dir: Path) -> Path | None:
+    for candidate in ["model.ply", "output.ply", "reconstruction.ply"]:
+        p = frames_dir / candidate
+        if p.exists():
+            return p
+    plys = list(frames_dir.glob("*.ply"))
+    return plys[0] if plys else None
+
+
+def decimate_ply(src: str, dst: str, max_pts: int = MAX_PTS) -> None:
+    import open3d as o3d
+    import numpy as np
+    src_path = Path(src)
+    if not src_path.exists():
+        raise FileNotFoundError(src)
+    pcd = o3d.io.read_point_cloud(str(src_path))
+    n = len(pcd.points)
+    if n > max_pts:
+        vol = float(np.prod(pcd.get_max_bound() - pcd.get_min_bound()))
+        vox = max((vol / max_pts) ** (1 / 3), 0.02)
+        pcd = pcd.voxel_down_sample(vox)
+    o3d.io.write_point_cloud(dst, pcd)
+    print(f"Decimated {n} -> {len(pcd.points)} pts -> {dst}", flush=True)
+
+
+def main() -> None:
+    p = argparse.ArgumentParser()
+    p.add_argument("job_id", type=int)
+    p.add_argument("--frames-dir", required=True)
+    p.add_argument("--cosma-vm", default=COSMA_VM)
+    p.add_argument("--cosma-data", default=COSMA_DATA)
+    args = p.parse_args()
+
+    frames_dir = Path(args.frames_dir)
+    ply_src = find_ply(frames_dir)
+    if ply_src is None:
+        print(f"No PLY found in {frames_dir}", flush=True)
+        sys.exit(0)
+
+    ply_dec = frames_dir / "model_decimated.ply"
+    decimate_ply(str(ply_src), str(ply_dec))
+
+    remote_dir = f"{args.cosma_data}/{args.job_id}"
+    subprocess.run(["ssh", args.cosma_vm, f"mkdir -p {remote_dir}"], check=True)
+    subprocess.run(["scp", str(ply_dec), f"{args.cosma_vm}:{remote_dir}/model_decimated.ply"], check=True)
+    print(f"SCP done -> {args.cosma_vm}:{remote_dir}/model_decimated.ply", flush=True)
+
+
+if __name__ == "__main__":
+    main()