feat(viewer): v5 grid 2x2 + datebar calendar + mission label
- NAV viewer v5: grid 2x2 (map + charts), trail slider, play, layer toggles - Datebar: date picker + datalist, fetches /api/data-dates from :8766 - Mission label shows #NN-folder (X sessions) in green or grey - Tools: parse_usv_nav, extract_mcap_signals, extract_usv_pwm, merge_nav_usbl, usbl_to_json, check_sync, parse_kogger_usbl - Vendor: Kogger-Protocol docs
This commit is contained in:
Flag
@@ -1,24 +1,39 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Parse USV long-format CSV → track.geojson + points.json"""
|
||||
"""Parse USV long-format CSV → track.geojson + points.json + manifest.json
|
||||
v2: multi-session support via --input-dir, retro-compat with --input (single file)
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timezone
|
||||
|
||||
MAX_SLIDER_POINTS = 5000
|
||||
MAX_SLIDER_POINTS = 10000
|
||||
|
||||
|
||||
def parse_args():
|
||||
p = argparse.ArgumentParser(description="Parse USV nav CSV")
|
||||
p.add_argument("--input", required=True, help="CSV navigation log")
|
||||
p = argparse.ArgumentParser(description="Parse USV nav CSV v2")
|
||||
g = p.add_mutually_exclusive_group(required=True)
|
||||
g.add_argument("--input", help="Single CSV navigation log (v1 compat)")
|
||||
g.add_argument("--input-dir", help="Directory: glob *navigation_log*.csv")
|
||||
p.add_argument("--output", required=True, help="Output directory")
|
||||
return p.parse_args()
|
||||
|
||||
|
||||
def find_csvs(input_dir):
|
||||
pattern = os.path.join(input_dir, "*navigation_log*.csv")
|
||||
files = sorted(glob.glob(pattern))
|
||||
if not files:
|
||||
print(f"No navigation_log CSVs found in {input_dir}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
return files
|
||||
|
||||
|
||||
def load_csv(path):
|
||||
"""Load long-format CSV into {timestamp: {field: value}}"""
|
||||
"""Load long-format CSV → {timestamp: {field: value}}"""
|
||||
rows_by_ts = defaultdict(dict)
|
||||
with open(path, newline="", encoding="utf-8") as f:
|
||||
reader = csv.DictReader(f)
|
||||
@@ -41,13 +56,26 @@ def get_float(d, *keys):
|
||||
return None
|
||||
|
||||
|
||||
def build_points(rows_by_ts):
|
||||
"""Build sorted list of {t, lat, lon, heading} where lat/lon valid."""
|
||||
# We need to track last known lat/lon/heading per timestamp cluster.
|
||||
# Strategy: walk timestamps in order, emit a point each time we see a Lat or Lon update.
|
||||
# Accumulate state across timestamps.
|
||||
timestamps = sorted(rows_by_ts.keys())
|
||||
def ts_to_ms(ts_str):
|
||||
"""Convert ISO-like timestamp string to epoch ms (UTC)."""
|
||||
# Try formats: '2026-03-24 09:04:07.123456' or '2026-03-24T09:04:07.123456'
|
||||
for fmt in (
|
||||
"%Y-%m-%dT%H:%M:%S.%f",
|
||||
"%Y-%m-%dT%H:%M:%S",
|
||||
"%Y-%m-%d %H:%M:%S.%f",
|
||||
"%Y-%m-%d %H:%M:%S",
|
||||
):
|
||||
try:
|
||||
dt = datetime.strptime(ts_str, fmt).replace(tzinfo=timezone.utc)
|
||||
return int(dt.timestamp() * 1000)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def build_points(rows_by_ts, source_name):
|
||||
"""Build sorted list of {t, t_ms, lat, lon, heading, source}."""
|
||||
timestamps = sorted(rows_by_ts.keys())
|
||||
state = {}
|
||||
points = []
|
||||
|
||||
@@ -55,7 +83,6 @@ def build_points(rows_by_ts):
|
||||
updates = rows_by_ts[ts]
|
||||
state.update(updates)
|
||||
|
||||
# Only emit point if we have both Lat and Lon from this or earlier ts
|
||||
lat = get_float(state, "Lat", "RAW_Lat")
|
||||
lon = get_float(state, "Lon", "RAW_Lon")
|
||||
heading = get_float(state, "Heading", "Yaw")
|
||||
@@ -64,7 +91,6 @@ def build_points(rows_by_ts):
|
||||
continue
|
||||
if lat == 0.0 and lon == 0.0:
|
||||
continue
|
||||
# GPS_RAW_INT fallback (1e-7 degrees)
|
||||
if abs(lat) < 1 and abs(lon) < 1:
|
||||
raw_lat = get_float(state, "GPS_RAW_INT_lat")
|
||||
raw_lon = get_float(state, "GPS_RAW_INT_lon")
|
||||
@@ -74,87 +100,200 @@ def build_points(rows_by_ts):
|
||||
else:
|
||||
continue
|
||||
|
||||
# Only emit if Lat or Lon just updated (reduce duplicate consecutive points)
|
||||
if "Lat" in updates or "Lon" in updates or "RAW_Lat" in updates or "RAW_Lon" in updates:
|
||||
t_ms = ts_to_ms(ts)
|
||||
points.append({
|
||||
"t": ts,
|
||||
"t_ms": t_ms,
|
||||
"lat": round(lat, 8),
|
||||
"lon": round(lon, 8),
|
||||
"heading": round(heading, 2) if heading is not None else None,
|
||||
"source": source_name,
|
||||
})
|
||||
|
||||
return points
|
||||
|
||||
|
||||
def sample_points(points, max_n):
|
||||
if len(points) <= max_n:
|
||||
def sample_points_session(points, max_total, n_sessions):
|
||||
"""Sample per session, always keeping first+last point of each session."""
|
||||
if not points:
|
||||
return points
|
||||
step = len(points) / max_n
|
||||
return [points[int(i * step)] for i in range(max_n)]
|
||||
quota = max(10, max_total // max(n_sessions, 1))
|
||||
if len(points) <= quota:
|
||||
return points
|
||||
|
||||
step = (len(points) - 2) / max(quota - 2, 1)
|
||||
sampled = [points[0]]
|
||||
for i in range(1, quota - 1):
|
||||
sampled.append(points[min(int(1 + i * step), len(points) - 2)])
|
||||
sampled.append(points[-1])
|
||||
return sampled
|
||||
|
||||
|
||||
def write_geojson(points, path):
|
||||
coords = [[p["lon"], p["lat"]] for p in points]
|
||||
geojson = {
|
||||
"type": "FeatureCollection",
|
||||
"features": [{
|
||||
def session_bbox(points):
|
||||
lats = [p["lat"] for p in points]
|
||||
lons = [p["lon"] for p in points]
|
||||
return [min(lons), min(lats), max(lons), max(lats)]
|
||||
|
||||
|
||||
def write_outputs(all_sessions, output_dir):
|
||||
"""Write track.geojson, points.json, manifest.json."""
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
# Colors for multi-track
|
||||
COLORS = ["#00b4d8", "#e94560", "#06d6a0", "#ffd166", "#a855f7", "#f97316"]
|
||||
|
||||
# ── track.geojson (MultiLineString per session) ──
|
||||
features = []
|
||||
for i, sess in enumerate(all_sessions):
|
||||
coords = [[p["lon"], p["lat"]] for p in sess["points"]]
|
||||
features.append({
|
||||
"type": "Feature",
|
||||
"geometry": {"type": "LineString", "coordinates": coords},
|
||||
"properties": {
|
||||
"start": points[0]["t"] if points else None,
|
||||
"end": points[-1]["t"] if points else None,
|
||||
"n_points": len(points),
|
||||
"source_file": sess["source_file"],
|
||||
"source_name": sess["source_name"],
|
||||
"start_iso": sess["t_start"],
|
||||
"end_iso": sess["t_end"],
|
||||
"n_points": len(coords),
|
||||
"color": COLORS[i % len(COLORS)],
|
||||
"session_index": i,
|
||||
}
|
||||
}]
|
||||
}
|
||||
with open(path, "w") as f:
|
||||
})
|
||||
|
||||
geojson = {"type": "FeatureCollection", "features": features}
|
||||
geo_path = os.path.join(output_dir, "track.geojson")
|
||||
with open(geo_path, "w") as f:
|
||||
json.dump(geojson, f)
|
||||
print(f" track.geojson: {len(coords)} coords → {path}")
|
||||
print(f" track.geojson: {len(features)} sessions → {geo_path}")
|
||||
|
||||
# ── points.json (all sampled, sorted by t_ms) ──
|
||||
all_points = []
|
||||
n_sessions = len(all_sessions)
|
||||
for sess in all_sessions:
|
||||
sampled = sample_points_session(sess["points"], MAX_SLIDER_POINTS, n_sessions)
|
||||
all_points.extend(sampled)
|
||||
|
||||
# Sort by t_ms (sessions may overlap in time)
|
||||
all_points.sort(key=lambda p: (p["t_ms"] or 0))
|
||||
|
||||
pts_path = os.path.join(output_dir, "points.json")
|
||||
with open(pts_path, "w") as f:
|
||||
json.dump(all_points, f)
|
||||
print(f" points.json: {len(all_points)} points (sampled) → {pts_path}")
|
||||
|
||||
# ── manifest.json ──
|
||||
all_lats = [p["lat"] for s in all_sessions for p in s["points"]]
|
||||
all_lons = [p["lon"] for s in all_sessions for p in s["points"]]
|
||||
global_bbox = [min(all_lons), min(all_lats), max(all_lons), max(all_lats)]
|
||||
|
||||
all_t_ms = [p["t_ms"] for s in all_sessions for p in s["points"] if p["t_ms"]]
|
||||
t_min_ms = min(all_t_ms) if all_t_ms else None
|
||||
t_max_ms = max(all_t_ms) if all_t_ms else None
|
||||
|
||||
sessions_meta = []
|
||||
for sess in all_sessions:
|
||||
sessions_meta.append({
|
||||
"file": sess["source_file"],
|
||||
"source_name": sess["source_name"],
|
||||
"n_points": sess["n_points_raw"],
|
||||
"t_start": sess["t_start"],
|
||||
"t_end": sess["t_end"],
|
||||
"t_start_ms": sess["t_start_ms"],
|
||||
"t_end_ms": sess["t_end_ms"],
|
||||
"bbox": sess["bbox"],
|
||||
})
|
||||
|
||||
manifest = {
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"n_sessions": len(all_sessions),
|
||||
"sessions": sessions_meta,
|
||||
"global_bbox": global_bbox,
|
||||
"t_min": all_sessions[0]["t_start"] if all_sessions else None,
|
||||
"t_max": all_sessions[-1]["t_end"] if all_sessions else None,
|
||||
"t_min_ms": t_min_ms,
|
||||
"t_max_ms": t_max_ms,
|
||||
"n_points_total_raw": sum(s["n_points_raw"] for s in all_sessions),
|
||||
"n_points_sampled": len(all_points),
|
||||
}
|
||||
|
||||
mf_path = os.path.join(output_dir, "manifest.json")
|
||||
with open(mf_path, "w") as f:
|
||||
json.dump(manifest, f, indent=2)
|
||||
print(f" manifest.json → {mf_path}")
|
||||
|
||||
return manifest
|
||||
|
||||
|
||||
def write_points_json(points, path):
|
||||
with open(path, "w") as f:
|
||||
json.dump(points, f)
|
||||
print(f" points.json: {len(points)} points → {path}")
|
||||
def print_global_stats(manifest, all_sessions):
|
||||
print(f"\n=== Stats globales ===")
|
||||
print(f" Sessions: {manifest['n_sessions']}")
|
||||
print(f" Points bruts: {manifest['n_points_total_raw']}")
|
||||
print(f" Points sampled: {manifest['n_points_sampled']}")
|
||||
print(f" t_min: {manifest['t_min']}")
|
||||
print(f" t_max: {manifest['t_max']}")
|
||||
bb = manifest["global_bbox"]
|
||||
print(f" Bbox: lon [{bb[0]:.5f}, {bb[2]:.5f}] lat [{bb[1]:.5f}, {bb[3]:.5f}]")
|
||||
if manifest["t_min_ms"] and manifest["t_max_ms"]:
|
||||
dur_s = (manifest["t_max_ms"] - manifest["t_min_ms"]) / 1000
|
||||
h, rem = divmod(int(dur_s), 3600)
|
||||
m, s = divmod(rem, 60)
|
||||
print(f" Durée totale: {h}h{m:02d}m{s:02d}s")
|
||||
for i, sess in enumerate(all_sessions):
|
||||
print(f" Session {i+1}: {sess['source_name']} {sess['n_points_raw']} pts {sess['t_start']} → {sess['t_end']}")
|
||||
|
||||
|
||||
def print_stats(points):
|
||||
def process_file(path):
|
||||
source_name = os.path.basename(path)
|
||||
print(f"\nChargement {source_name} ...")
|
||||
rows = load_csv(path)
|
||||
print(f" {len(rows)} timestamps uniques")
|
||||
points = build_points(rows, source_name)
|
||||
if not points:
|
||||
print("No valid points found!")
|
||||
return
|
||||
print(f" WARNING: aucun point GPS valide dans {source_name}")
|
||||
return None
|
||||
|
||||
# Filter points without t_ms
|
||||
points = [p for p in points if p["t_ms"] is not None]
|
||||
lats = [p["lat"] for p in points]
|
||||
lons = [p["lon"] for p in points]
|
||||
print(f"\n=== Stats ===")
|
||||
print(f" N points (full): {len(points)}")
|
||||
print(f" First ts: {points[0]['t']}")
|
||||
print(f" Last ts: {points[-1]['t']}")
|
||||
print(f" Bbox lat: {min(lats):.6f} → {max(lats):.6f}")
|
||||
print(f" Bbox lon: {min(lons):.6f} → {max(lons):.6f}")
|
||||
headings = [p["heading"] for p in points if p["heading"] is not None]
|
||||
print(f" Heading data: {'yes' if headings else 'no'} ({len(headings)} values)")
|
||||
return {
|
||||
"source_file": path,
|
||||
"source_name": source_name,
|
||||
"points": points,
|
||||
"n_points_raw": len(points),
|
||||
"t_start": points[0]["t"],
|
||||
"t_end": points[-1]["t"],
|
||||
"t_start_ms": points[0]["t_ms"],
|
||||
"t_end_ms": points[-1]["t_ms"],
|
||||
"bbox": [min(lons), min(lats), max(lons), max(lats)],
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
os.makedirs(args.output, exist_ok=True)
|
||||
|
||||
print(f"Loading {args.input} ...")
|
||||
rows = load_csv(args.input)
|
||||
print(f" {len(rows)} unique timestamps")
|
||||
if args.input:
|
||||
csv_files = [args.input]
|
||||
else:
|
||||
csv_files = find_csvs(args.input_dir)
|
||||
|
||||
points = build_points(rows)
|
||||
print_stats(points)
|
||||
print(f"Fichiers trouvés: {len(csv_files)}")
|
||||
for f in csv_files:
|
||||
print(f" {os.path.basename(f)}")
|
||||
|
||||
if not points:
|
||||
all_sessions = []
|
||||
for path in csv_files:
|
||||
sess = process_file(path)
|
||||
if sess:
|
||||
all_sessions.append(sess)
|
||||
|
||||
if not all_sessions:
|
||||
print("Aucune session valide.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
write_geojson(points, os.path.join(args.output, "track.geojson"))
|
||||
|
||||
sampled = sample_points(points, MAX_SLIDER_POINTS)
|
||||
if len(sampled) < len(points):
|
||||
print(f" Sampled {len(sampled)} points for slider (from {len(points)})")
|
||||
write_points_json(sampled, os.path.join(args.output, "points.json"))
|
||||
|
||||
manifest = write_outputs(all_sessions, args.output)
|
||||
print_global_stats(manifest, all_sessions)
|
||||
print("\nDone.")
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user