276 lines
15 KiB
HTML
276 lines
15 KiB
HTML
|
||
|
||
<!DOCTYPE html>
|
||
<html class="writer-html5" lang="en" data-content_root="./">
|
||
<head>
|
||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||
<title>Pipeline cosma-qc — cosma-qc 1.0 documentation</title>
|
||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=9edc463e" />
|
||
|
||
|
||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||
<script src="_static/documentation_options.js?v=f2a433a1"></script>
|
||
<script src="_static/doctools.js?v=fd6eb6e6"></script>
|
||
<script src="_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||
<script src="_static/js/theme.js"></script>
|
||
<link rel="index" title="Index" href="genindex.html" />
|
||
<link rel="search" title="Search" href="search.html" />
|
||
<link rel="next" title="Infrastructure" href="infrastructure.html" />
|
||
<link rel="prev" title="cosma-qc — Documentation" href="index.html" />
|
||
</head>
|
||
|
||
<body class="wy-body-for-nav">
|
||
<div class="wy-grid-for-nav">
|
||
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
|
||
<div class="wy-side-scroll">
|
||
<div class="wy-side-nav-search" >
|
||
|
||
|
||
|
||
<a href="index.html" class="icon icon-home">
|
||
cosma-qc
|
||
</a>
|
||
<div role="search">
|
||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||
<input type="hidden" name="check_keywords" value="yes" />
|
||
<input type="hidden" name="area" value="default" />
|
||
</form>
|
||
</div>
|
||
</div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
|
||
<p class="caption" role="heading"><span class="caption-text">Contenu</span></p>
|
||
<ul class="current">
|
||
<li class="toctree-l1 current"><a class="current reference internal" href="#">Pipeline cosma-qc</a><ul>
|
||
<li class="toctree-l2"><a class="reference internal" href="#vue-d-ensemble">Vue d’ensemble</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="#flux-de-donnees-global">Flux de données global</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="#etape-1-ingest">Étape 1 — Ingest</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="#etape-2-extraction-des-frames">Étape 2 — Extraction des frames</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="#etape-3-reconstruction-3d">Étape 3 — Reconstruction 3D</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="#etape-4-stitch-par-auv">Étape 4 — Stitch par AUV</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="#etape-5-stitch-cross-auv">Étape 5 — Stitch cross-AUV</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="#etape-6-export-glb-a-la-demande">Étape 6 — Export GLB (à la demande)</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="#statuts-de-jobs">Statuts de jobs</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l1"><a class="reference internal" href="infrastructure.html">Infrastructure</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="data.html">Données — Stockage et budget disque</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="usage.html">Utilisation</a></li>
|
||
</ul>
|
||
|
||
</div>
|
||
</div>
|
||
</nav>
|
||
|
||
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
|
||
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
|
||
<a href="index.html">cosma-qc</a>
|
||
</nav>
|
||
|
||
<div class="wy-nav-content">
|
||
<div class="rst-content">
|
||
<div role="navigation" aria-label="Page navigation">
|
||
<ul class="wy-breadcrumbs">
|
||
<li><a href="index.html" class="icon icon-home" aria-label="Home"></a></li>
|
||
<li class="breadcrumb-item active">Pipeline cosma-qc</li>
|
||
<li class="wy-breadcrumbs-aside">
|
||
<a href="_sources/pipeline.rst.txt" rel="nofollow"> View page source</a>
|
||
</li>
|
||
</ul>
|
||
<hr/>
|
||
</div>
|
||
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
|
||
<div itemprop="articleBody">
|
||
|
||
<section id="pipeline-cosma-qc">
|
||
<h1>Pipeline cosma-qc<a class="headerlink" href="#pipeline-cosma-qc" title="Link to this heading"></a></h1>
|
||
<section id="vue-d-ensemble">
|
||
<h2>Vue d’ensemble<a class="headerlink" href="#vue-d-ensemble" title="Link to this heading"></a></h2>
|
||
<p>Le pipeline cosma-qc traite les vidéos GoPro brutes acquises par les drones sous-marins
|
||
AUV COSMA pour produire des nuages de points 3D denses (PLY) et des exports web (GLB).</p>
|
||
</section>
|
||
<section id="flux-de-donnees-global">
|
||
<h2>Flux de données global<a class="headerlink" href="#flux-de-donnees-global" title="Link to this heading"></a></h2>
|
||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>z620 (/mnt/portablessd)
|
||
├── GX*.MP4 (brut)
|
||
│
|
||
▼ [1. Ingest]
|
||
SQLite DB (jobs)
|
||
│
|
||
▼ [2. Extraction ffmpeg]
|
||
GPU worker /cosma-qc-frames/job_{id}/frame_%06d.jpg
|
||
│
|
||
▼ [3. Reconstruction lingbot-map]
|
||
GPU worker /cosma-qc-frames/job_{id}/reconstruction.ply
|
||
│
|
||
▼ [4. Stitch per_auv]
|
||
GPU worker /cosma-qc-frames/stitch_{N}.ply (par AUV)
|
||
│
|
||
▼ [5. Stitch cross_auv]
|
||
GPU worker /cosma-qc-frames/stitch_global.ply
|
||
│
|
||
▼ [6. Export GLB (à la demande)]
|
||
GPU worker /cosma-qc-frames/job_{id}/reconstruction.glb
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="etape-1-ingest">
|
||
<h2>Étape 1 — Ingest<a class="headerlink" href="#etape-1-ingest" title="Link to this heading"></a></h2>
|
||
<p><strong>Script :</strong> scripts/ingest.py</p>
|
||
<p><strong>Rôle :</strong> Scanner le SSD de z620, regrouper les MP4 GoPro par AUV/GoPro/segment
|
||
temporel et écrire les jobs dans la base SQLite.</p>
|
||
<p>Logique de regroupement :</p>
|
||
<ul class="simple">
|
||
<li><p>Les fichiers sont groupés par numéro de caméra GoPro (GXXX).</p></li>
|
||
<li><p>Un nouveau segment est créé si l’écart entre deux fichiers dépasse <strong>5 minutes</strong>.</p></li>
|
||
<li><p>Chaque segment → un job en base avec statut pending.</p></li>
|
||
</ul>
|
||
<p>Structure d’un job en base :</p>
|
||
<div class="highlight-sql notranslate"><div class="highlight"><pre><span></span><span class="k">CREATE</span><span class="w"> </span><span class="k">TABLE</span><span class="w"> </span><span class="n">jobs</span><span class="w"> </span><span class="p">(</span>
|
||
<span class="w"> </span><span class="n">id</span><span class="w"> </span><span class="nb">INTEGER</span><span class="w"> </span><span class="k">PRIMARY</span><span class="w"> </span><span class="k">KEY</span><span class="p">,</span>
|
||
<span class="w"> </span><span class="n">auv</span><span class="w"> </span><span class="nb">TEXT</span><span class="p">,</span>
|
||
<span class="w"> </span><span class="n">gopro</span><span class="w"> </span><span class="nb">TEXT</span><span class="p">,</span>
|
||
<span class="w"> </span><span class="n">segment</span><span class="w"> </span><span class="nb">INTEGER</span><span class="p">,</span>
|
||
<span class="w"> </span><span class="n">files</span><span class="w"> </span><span class="nb">TEXT</span><span class="p">,</span><span class="w"> </span><span class="c1">-- JSON list of z620:/path/GX*.MP4</span>
|
||
<span class="w"> </span><span class="n">status</span><span class="w"> </span><span class="nb">TEXT</span><span class="p">,</span><span class="w"> </span><span class="c1">-- pending / running / done / failed</span>
|
||
<span class="w"> </span><span class="n">worker</span><span class="w"> </span><span class="nb">TEXT</span><span class="p">,</span>
|
||
<span class="w"> </span><span class="n">created_at</span><span class="w"> </span><span class="k">TIMESTAMP</span><span class="p">,</span>
|
||
<span class="w"> </span><span class="n">updated_at</span><span class="w"> </span><span class="k">TIMESTAMP</span>
|
||
<span class="p">);</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Les chemins vidéo sont stockés sous la forme z620:/chemin/absolu/GX*.MP4
|
||
pour indiquer que la source est sur z620, pas sur le worker GPU.</p>
|
||
</section>
|
||
<section id="etape-2-extraction-des-frames">
|
||
<h2>Étape 2 — Extraction des frames<a class="headerlink" href="#etape-2-extraction-des-frames" title="Link to this heading"></a></h2>
|
||
<p><strong>Fonction :</strong> do_extract dans le dispatcher (core .82)</p>
|
||
<p><strong>Outil :</strong> ffmpeg</p>
|
||
<p>Paramètres d’extraction :</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>ffmpeg<span class="w"> </span>-i<span class="w"> </span>input.mp4<span class="w"> </span>-vf<span class="w"> </span><span class="nv">fps</span><span class="o">=</span><span class="m">2</span>,scale<span class="o">=</span><span class="m">1920</span>:1080<span class="w"> </span>-q:v<span class="w"> </span><span class="m">2</span><span class="w"> </span><span class="o">{</span>frames_dir<span class="o">}</span>/job_<span class="o">{</span>id<span class="o">}</span>/frame_%06d.jpg
|
||
</pre></div>
|
||
</div>
|
||
<p><strong>Particularité z620 :</strong> lorsque la source est z620:/…, ffmpeg s’exécute
|
||
<strong>directement sur z620</strong> via SSH — seuls les JPEG voyagent sur le réseau.
|
||
Les MP4 bruts ne quittent jamais z620.</p>
|
||
<p>Filtre hors-eau (heuristique de luminosité) :</p>
|
||
<ul class="simple">
|
||
<li><p>Les frames dont la luminosité moyenne est au-dessus d’un seuil sont supprimées
|
||
(ciel, surface, hors-eau).</p></li>
|
||
<li><p>Ce filtre réduit le bruit dans la reconstruction.</p></li>
|
||
</ul>
|
||
<p>Reprise sur crash :</p>
|
||
<ul class="simple">
|
||
<li><p>Un fichier marqueur .video_N.done est créé après chaque MP4 traité.</p></li>
|
||
<li><p>En cas de redémarrage, les vidéos déjà traitées sont ignorées.</p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="etape-3-reconstruction-3d">
|
||
<h2>Étape 3 — Reconstruction 3D<a class="headerlink" href="#etape-3-reconstruction-3d" title="Link to this heading"></a></h2>
|
||
<p><strong>Fonction :</strong> do_reconstruct</p>
|
||
<p><strong>Outil :</strong> lingbot-map (demo.py) sur le worker GPU</p>
|
||
<p><strong>Entrée :</strong> répertoire de frames {worker_frames_dir}/job_{id}/</p>
|
||
<p><strong>Sortie :</strong> {worker_frames_dir}/job_{id}/reconstruction.ply
|
||
(nuage de points dense, jusqu’à <strong>150 millions de points</strong>)</p>
|
||
<p>Un viewer Viser est automatiquement démarré sur le port 8100 + job_id
|
||
pendant la reconstruction pour visualisation en temps réel.</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="c1"># Visualiser pendant reconstruction</span>
|
||
<span class="c1"># naviguer vers http://{worker_ip}:{8100+job_id}</span>
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="etape-4-stitch-par-auv">
|
||
<h2>Étape 4 — Stitch par AUV<a class="headerlink" href="#etape-4-stitch-par-auv" title="Link to this heading"></a></h2>
|
||
<p><strong>Fonction :</strong> do_stitch_per_auv</p>
|
||
<p><strong>Déclenchement :</strong> automatique quand <strong>tous les jobs d’un AUV</strong> sont en statut done.</p>
|
||
<p><strong>Outil :</strong> cosma-stitch.py</p>
|
||
<p><strong>Opération :</strong> alignement et fusion des PLY de tous les segments d’un même AUV.</p>
|
||
<p><strong>Sortie :</strong> {worker_frames_dir}/stitch_{N}.ply</p>
|
||
<p>Ce stitch s’exécute sur le worker qui détient les PLY (pas de copie inter-workers).</p>
|
||
</section>
|
||
<section id="etape-5-stitch-cross-auv">
|
||
<h2>Étape 5 — Stitch cross-AUV<a class="headerlink" href="#etape-5-stitch-cross-auv" title="Link to this heading"></a></h2>
|
||
<p><strong>Fonction :</strong> do_stitch_cross_auv</p>
|
||
<p><strong>Déclenchement :</strong> automatique quand <strong>tous les stitches per_auv</strong> sont validés.</p>
|
||
<p><strong>Opération :</strong> fusion de tous les PLY par AUV en un nuage de points global final.</p>
|
||
<p><strong>Sortie :</strong> {worker_frames_dir}/stitch_global.ply</p>
|
||
</section>
|
||
<section id="etape-6-export-glb-a-la-demande">
|
||
<h2>Étape 6 — Export GLB (à la demande)<a class="headerlink" href="#etape-6-export-glb-a-la-demande" title="Link to this heading"></a></h2>
|
||
<p><strong>Outil :</strong> trimesh (conversion PLY → GLB)</p>
|
||
<p><strong>Sous-échantillonnage :</strong> 5 millions de points (adapté web/navigateur)</p>
|
||
<p><strong>Sortie :</strong> {worker_frames_dir}/job_{id}/reconstruction.glb</p>
|
||
<p>Un serveur HTTP minimal est lancé sur le worker (port <strong>8300</strong>) pour le téléchargement :</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="c1"># Sur le worker</span>
|
||
python3<span class="w"> </span>-m<span class="w"> </span>http.server<span class="w"> </span><span class="m">8300</span><span class="w"> </span>--directory<span class="w"> </span><span class="o">{</span>worker_frames_dir<span class="o">}</span>
|
||
|
||
<span class="c1"># Télécharger depuis un PC</span>
|
||
wget<span class="w"> </span>http://<span class="o">{</span>worker_ip<span class="o">}</span>:8300/job_<span class="o">{</span>id<span class="o">}</span>/reconstruction.glb
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="statuts-de-jobs">
|
||
<h2>Statuts de jobs<a class="headerlink" href="#statuts-de-jobs" title="Link to this heading"></a></h2>
|
||
<table class="docutils align-default">
|
||
<colgroup>
|
||
<col style="width: 20.0%" />
|
||
<col style="width: 80.0%" />
|
||
</colgroup>
|
||
<thead>
|
||
<tr class="row-odd"><th class="head"><p>Statut</p></th>
|
||
<th class="head"><p>Description</p></th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr class="row-even"><td><p>pending</p></td>
|
||
<td><p>Job créé, en attente de dispatch</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p>running</p></td>
|
||
<td><p>En cours de traitement (extraction ou reconstruction)</p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p>done</p></td>
|
||
<td><p>Reconstruction terminée, PLY disponible</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p>failed</p></td>
|
||
<td><p>Erreur — voir logs du dispatcher</p></td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</section>
|
||
</section>
|
||
|
||
|
||
</div>
|
||
</div>
|
||
<footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
|
||
<a href="index.html" class="btn btn-neutral float-left" title="cosma-qc — Documentation" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
|
||
<a href="infrastructure.html" class="btn btn-neutral float-right" title="Infrastructure" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
|
||
</div>
|
||
|
||
<hr/>
|
||
|
||
<div role="contentinfo">
|
||
<p>© Copyright .</p>
|
||
</div>
|
||
|
||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||
<a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
|
||
provided by <a href="https://readthedocs.org">Read the Docs</a>.
|
||
|
||
|
||
</footer>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
</div>
|
||
<script>
|
||
jQuery(function () {
|
||
SphinxRtdTheme.Navigation.enable(true);
|
||
});
|
||
</script>
|
||
|
||
</body>
|
||
</html> |