From 621f4e63e8914d4fe41d3f221fc151613ed2b8c7 Mon Sep 17 00:00:00 2001
From: Flag <flag@laboratoire>
Date: Wed, 22 Apr 2026 08:44:52 +0000
Subject: [PATCH] =?UTF-8?q?dispatcher=20=C2=97=20fix=20VRAM=20estimate=20+?=
 =?UTF-8?q?=20adaptive=20stride=20pour=20=C3=A9viter=20OOM=20RAM?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- estimate_vram_mib: 3500+13*frames (absurde 110GB pour 8k frames) ? 6000 MiB fixe
  (windowed+offload_to_cpu plafonne la VRAM indépendamment du total frames)
- do_reconstruct: stride adaptatif basé sur RAM worker (23GB .87, 62GB .84)
  load_fn.py stack le tensor complet en CPU RAM (~3.15 MB/frame)
  budget 55% RAM ? stride 2-3 pour jobs >4k frames

Debloque tous les jobs — avant aucun ne pouvait passer pick_worker car
estimate dépassait la VRAM totale des GPUs.
---
 scripts/dispatcher.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/scripts/dispatcher.py b/scripts/dispatcher.py
index c073485..f740c46 100644
--- a/scripts/dispatcher.py
+++ b/scripts/dispatcher.py
@@ -109,7 +109,9 @@ def release_worker(worker: dict, estimated_vram_mib: int):
 
 
 def estimate_vram_mib(frame_count: int) -> int:
-    return int(3500 + 13 * frame_count)
+    # windowed mode + offload_to_cpu caps VRAM usage regardless of total frames.
+    # Observed: ~3.5 GB model + ~1.5 GB working set for window_size=16. Safe budget: 6 GB.
+    return 6000
 
 
 def set_status(job_id: int, **fields):
@@ -230,6 +232,14 @@ def do_reconstruct(job: sqlite3.Row, worker: dict, frames_dir: str) -> tuple[str
     log = f"/tmp/cosma-qc-job-{job['id']}.log"
     ckpt = f"{worker['lingbot_path']}/checkpoints/lingbot-map/lingbot-map-long.pt"
     ply_path = f"{frames_dir}/reconstruction.ply"
+    # Adaptive stride to fit CPU RAM: load_fn stacks full image tensor ~3.15 MB/frame @ 512x512x3 fp32.
+    # .87 has 23 GB RAM, .84 has 62 GB. Keep effective frame count ~4k to stay safe.
+    frame_count = job["frame_count"] or 0
+    ram_gb = 23 if worker["host"] == "192.168.0.87" else 62
+    ram_budget_gb = ram_gb * 0.55
+    stride = 1
+    while frame_count * 3.15 / 1024 / stride > ram_budget_gb:
+        stride += 1
     # demo.py starts a viser web server after saving the PLY and never exits.
     # Wrap it: launch in bg, wait for "PLY saved" marker in the log, kill, exit 0.
     # Match on the unique job frames_dir to identify our demo.py among all children/threads.
@@ -238,7 +248,7 @@ def do_reconstruct(job: sqlite3.Row, worker: dict, frames_dir: str) -> tuple[str
         f"cd {shlex.quote(worker['lingbot_path'])} && source .venv/bin/activate && "
         f"setsid python3 demo.py --model_path {shlex.quote(ckpt)} "
         f"--image_folder {shlex.quote(frames_dir)} --port {port} "
-        f"--use_sdpa --mode windowed --window_size 16 --overlap_size 2 --offload_to_cpu "
+        f"--stride {stride} --use_sdpa --mode windowed --window_size 16 --overlap_size 2 --offload_to_cpu "
         f"--save_ply {shlex.quote(ply_path)} > {log} 2>&1 & "
         f"DEMO_PID=$!; "
         f"for i in $(seq 1 3600); do "