288 lines
15 KiB
HTML
288 lines
15 KiB
HTML
|
||
|
||
<!DOCTYPE html>
|
||
<html class="writer-html5" lang="en" data-content_root="./">
|
||
<head>
|
||
<meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||
<title>Données — Stockage et budget disque — cosma-qc 1.0 documentation</title>
|
||
<link rel="stylesheet" type="text/css" href="_static/pygments.css?v=b86133f3" />
|
||
<link rel="stylesheet" type="text/css" href="_static/css/theme.css?v=9edc463e" />
|
||
|
||
|
||
<script src="_static/jquery.js?v=5d32c60e"></script>
|
||
<script src="_static/_sphinx_javascript_frameworks_compat.js?v=2cd50e6c"></script>
|
||
<script src="_static/documentation_options.js?v=f2a433a1"></script>
|
||
<script src="_static/doctools.js?v=fd6eb6e6"></script>
|
||
<script src="_static/sphinx_highlight.js?v=6ffebe34"></script>
|
||
<script src="_static/js/theme.js"></script>
|
||
<link rel="index" title="Index" href="genindex.html" />
|
||
<link rel="search" title="Search" href="search.html" />
|
||
<link rel="next" title="Utilisation" href="usage.html" />
|
||
<link rel="prev" title="Infrastructure" href="infrastructure.html" />
|
||
</head>
|
||
|
||
<body class="wy-body-for-nav">
|
||
<div class="wy-grid-for-nav">
|
||
<nav data-toggle="wy-nav-shift" class="wy-nav-side">
|
||
<div class="wy-side-scroll">
|
||
<div class="wy-side-nav-search" >
|
||
|
||
|
||
|
||
<a href="index.html" class="icon icon-home">
|
||
cosma-qc
|
||
</a>
|
||
<div role="search">
|
||
<form id="rtd-search-form" class="wy-form" action="search.html" method="get">
|
||
<input type="text" name="q" placeholder="Search docs" aria-label="Search docs" />
|
||
<input type="hidden" name="check_keywords" value="yes" />
|
||
<input type="hidden" name="area" value="default" />
|
||
</form>
|
||
</div>
|
||
</div><div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="Navigation menu">
|
||
<p class="caption" role="heading"><span class="caption-text">Contenu</span></p>
|
||
<ul class="current">
|
||
<li class="toctree-l1"><a class="reference internal" href="pipeline.html">Pipeline cosma-qc</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="infrastructure.html">Infrastructure</a></li>
|
||
<li class="toctree-l1 current"><a class="current reference internal" href="#">Données — Stockage et budget disque</a><ul>
|
||
<li class="toctree-l2"><a class="reference internal" href="#ou-sont-stockees-les-donnees">Où sont stockées les données</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="#budget-disque-observe">Budget disque observé</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="#politique-de-nettoyage">Politique de nettoyage</a><ul>
|
||
<li class="toctree-l3"><a class="reference internal" href="#frames-jpeg">Frames JPEG</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="#ply-intermediaires">PLY intermédiaires</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="#verification-espace-disque">Vérification espace disque</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l2"><a class="reference internal" href="#export-glb">Export GLB</a></li>
|
||
<li class="toctree-l2"><a class="reference internal" href="#reprise-sur-crash-marqueurs-done">Reprise sur crash — marqueurs .done</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l1"><a class="reference internal" href="usage.html">Utilisation</a></li>
|
||
</ul>
|
||
|
||
</div>
|
||
</div>
|
||
</nav>
|
||
|
||
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"><nav class="wy-nav-top" aria-label="Mobile navigation menu" >
|
||
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
|
||
<a href="index.html">cosma-qc</a>
|
||
</nav>
|
||
|
||
<div class="wy-nav-content">
|
||
<div class="rst-content">
|
||
<div role="navigation" aria-label="Page navigation">
|
||
<ul class="wy-breadcrumbs">
|
||
<li><a href="index.html" class="icon icon-home" aria-label="Home"></a></li>
|
||
<li class="breadcrumb-item active">Données — Stockage et budget disque</li>
|
||
<li class="wy-breadcrumbs-aside">
|
||
<a href="_sources/data.rst.txt" rel="nofollow"> View page source</a>
|
||
</li>
|
||
</ul>
|
||
<hr/>
|
||
</div>
|
||
<div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
|
||
<div itemprop="articleBody">
|
||
|
||
<section id="donnees-stockage-et-budget-disque">
|
||
<h1>Données — Stockage et budget disque<a class="headerlink" href="#donnees-stockage-et-budget-disque" title="Link to this heading"></a></h1>
|
||
<section id="ou-sont-stockees-les-donnees">
|
||
<h2>Où sont stockées les données<a class="headerlink" href="#ou-sont-stockees-les-donnees" title="Link to this heading"></a></h2>
|
||
<table class="docutils align-default">
|
||
<colgroup>
|
||
<col style="width: 30.0%" />
|
||
<col style="width: 20.0%" />
|
||
<col style="width: 50.0%" />
|
||
</colgroup>
|
||
<thead>
|
||
<tr class="row-odd"><th class="head"><p>Type de donnée</p></th>
|
||
<th class="head"><p>Emplacement</p></th>
|
||
<th class="head"><p>Remarques</p></th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr class="row-even"><td><p>MP4 bruts GoPro</p></td>
|
||
<td><p>z620 <code class="docutils literal notranslate"><span class="pre">/mnt/portablessd</span></code></p></td>
|
||
<td><p>Ne quittent jamais z620. Jamais copiés sur workers.</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p>Frames JPEG</p></td>
|
||
<td><p>Worker <code class="docutils literal notranslate"><span class="pre">/cosma-qc-frames/job_{id}/frame_*.jpg</span></code></p></td>
|
||
<td><p>Conservés pour reprise sur crash. Supprimables après validation du stitch.</p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p>PLY par job</p></td>
|
||
<td><p>Worker <code class="docutils literal notranslate"><span class="pre">/cosma-qc-frames/job_{id}/reconstruction.ply</span></code></p></td>
|
||
<td><p>Entrée du stitch per_auv.</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p>PLY stitch par AUV</p></td>
|
||
<td><p>Worker <code class="docutils literal notranslate"><span class="pre">/cosma-qc-frames/stitch_{N}.ply</span></code></p></td>
|
||
<td><p>Fusion des segments d’un AUV.</p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p>PLY stitch global</p></td>
|
||
<td><p>Worker <code class="docutils literal notranslate"><span class="pre">/cosma-qc-frames/stitch_global.ply</span></code></p></td>
|
||
<td><p>Nuage de points final toute mission.</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p>GLB (export web)</p></td>
|
||
<td><p>Worker <code class="docutils literal notranslate"><span class="pre">/cosma-qc-frames/job_{id}/reconstruction.glb</span></code></p></td>
|
||
<td><p>Généré à la demande. 5M points, ~76 MB.</p></td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</section>
|
||
<section id="budget-disque-observe">
|
||
<h2>Budget disque observé<a class="headerlink" href="#budget-disque-observe" title="Link to this heading"></a></h2>
|
||
<table class="docutils align-default">
|
||
<colgroup>
|
||
<col style="width: 40.0%" />
|
||
<col style="width: 30.0%" />
|
||
<col style="width: 30.0%" />
|
||
</colgroup>
|
||
<thead>
|
||
<tr class="row-odd"><th class="head"><p>Type</p></th>
|
||
<th class="head"><p>Taille typique</p></th>
|
||
<th class="head"><p>Base de calcul</p></th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr class="row-even"><td><p>Frames JPEG par job</p></td>
|
||
<td><p>~11 GB</p></td>
|
||
<td><p>job 45 min à 2 fps, 1920x1080</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p>PLY par job (reconstruction)</p></td>
|
||
<td><p>2 – 5 GB</p></td>
|
||
<td><p>dépend de la densité de la scène</p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p>GLB par job (export web)</p></td>
|
||
<td><p>~76 MB</p></td>
|
||
<td><p>5M points (job_21 observé)</p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p>PLY stitch AUV</p></td>
|
||
<td><p>variable</p></td>
|
||
<td><p>somme des PLY segments</p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p>PLY global</p></td>
|
||
<td><p>variable</p></td>
|
||
<td><p>somme de tous les AUV</p></td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
<p>Pour un AUV avec 4 jobs de 45 min chacun :</p>
|
||
<ul class="simple">
|
||
<li><p>Frames : 4 x 11 GB = <strong>~44 GB</strong> (supprimables après validation)</p></li>
|
||
<li><p>PLY jobs : 4 x 3.5 GB = <strong>~14 GB</strong></p></li>
|
||
<li><p>PLY stitch AUV : <strong>~6-10 GB</strong></p></li>
|
||
</ul>
|
||
</section>
|
||
<section id="politique-de-nettoyage">
|
||
<h2>Politique de nettoyage<a class="headerlink" href="#politique-de-nettoyage" title="Link to this heading"></a></h2>
|
||
<section id="frames-jpeg">
|
||
<h3>Frames JPEG<a class="headerlink" href="#frames-jpeg" title="Link to this heading"></a></h3>
|
||
<p>Les frames sont conservées uniquement pour permettre la reprise sur crash.
|
||
Une fois le stitch per_auv validé visuellement, <strong>les frames peuvent être supprimées</strong>.</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="c1"># Supprimer les frames d'un job (conserver le PLY !)</span>
|
||
rm<span class="w"> </span>-rf<span class="w"> </span>/cosma-qc-frames/job_ID/frame_*.jpg
|
||
rm<span class="w"> </span>-f<span class="w"> </span>/cosma-qc-frames/job_ID/.video_*.done
|
||
|
||
<span class="c1"># Vérifier que le PLY est intact avant suppression</span>
|
||
ls<span class="w"> </span>-lh<span class="w"> </span>/cosma-qc-frames/job_ID/reconstruction.ply
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="ply-intermediaires">
|
||
<h3>PLY intermédiaires<a class="headerlink" href="#ply-intermediaires" title="Link to this heading"></a></h3>
|
||
<p>Les PLY par job peuvent être supprimés après que le stitch per_auv est validé
|
||
et sauvegardé hors-ligne.</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="c1"># Conserver uniquement le stitch, supprimer les PLY jobs</span>
|
||
rm<span class="w"> </span>/cosma-qc-frames/job_ID/reconstruction.ply
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="verification-espace-disque">
|
||
<h3>Vérification espace disque<a class="headerlink" href="#verification-espace-disque" title="Link to this heading"></a></h3>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="c1"># Espace total workers</span>
|
||
ssh<span class="w"> </span>floppyrj45@192.168.0.84<span class="w"> </span><span class="s2">"df -h /cosma-qc-frames"</span>
|
||
ssh<span class="w"> </span>floppyrj45@192.168.0.87<span class="w"> </span><span class="s2">"df -h /cosma-qc-frames"</span>
|
||
|
||
<span class="c1"># Taille par job</span>
|
||
du<span class="w"> </span>-sh<span class="w"> </span>/cosma-qc-frames/job_*/
|
||
|
||
<span class="c1"># Top consommateurs</span>
|
||
du<span class="w"> </span>-sh<span class="w"> </span>/cosma-qc-frames/*<span class="w"> </span><span class="p">|</span><span class="w"> </span>sort<span class="w"> </span>-rh<span class="w"> </span><span class="p">|</span><span class="w"> </span>head<span class="w"> </span>-20
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
<section id="export-glb">
|
||
<h2>Export GLB<a class="headerlink" href="#export-glb" title="Link to this heading"></a></h2>
|
||
<p>Le GLB est une version allégée du nuage de points pour visualisation web.</p>
|
||
<p>Génération via l’API dashboard :</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>curl<span class="w"> </span>-X<span class="w"> </span>POST<span class="w"> </span>http://192.168.0.82:3849/jobs/ID/export_glb
|
||
</pre></div>
|
||
</div>
|
||
<p>Génération manuelle sur le worker :</p>
|
||
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span><span class="w"> </span><span class="nn">trimesh</span><span class="o">,</span><span class="w"> </span><span class="nn">numpy</span><span class="w"> </span><span class="k">as</span><span class="w"> </span><span class="nn">np</span>
|
||
<span class="n">pc</span> <span class="o">=</span> <span class="n">trimesh</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="s1">'/cosma-qc-frames/job_ID/reconstruction.ply'</span><span class="p">)</span>
|
||
<span class="n">idx</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">choice</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">pc</span><span class="o">.</span><span class="n">vertices</span><span class="p">),</span> <span class="mi">5_000_000</span><span class="p">,</span> <span class="n">replace</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||
<span class="n">sub</span> <span class="o">=</span> <span class="n">trimesh</span><span class="o">.</span><span class="n">PointCloud</span><span class="p">(</span><span class="n">pc</span><span class="o">.</span><span class="n">vertices</span><span class="p">[</span><span class="n">idx</span><span class="p">],</span> <span class="n">colors</span><span class="o">=</span><span class="n">pc</span><span class="o">.</span><span class="n">colors</span><span class="p">[</span><span class="n">idx</span><span class="p">])</span>
|
||
<span class="n">sub</span><span class="o">.</span><span class="n">export</span><span class="p">(</span><span class="s1">'/cosma-qc-frames/job_ID/reconstruction.glb'</span><span class="p">)</span>
|
||
</pre></div>
|
||
</div>
|
||
<p>Téléchargement :</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span><span class="c1"># Lancer le serveur HTTP sur le worker</span>
|
||
ssh<span class="w"> </span>floppyrj45@192.168.0.84<span class="w"> </span><span class="se">\</span>
|
||
<span class="w"> </span><span class="s2">"python3 -m http.server 8300 --directory /cosma-qc-frames"</span>
|
||
|
||
<span class="c1"># Télécharger depuis PC</span>
|
||
wget<span class="w"> </span>http://192.168.0.84:8300/job_ID/reconstruction.glb
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
<section id="reprise-sur-crash-marqueurs-done">
|
||
<h2>Reprise sur crash — marqueurs .done<a class="headerlink" href="#reprise-sur-crash-marqueurs-done" title="Link to this heading"></a></h2>
|
||
<p>Chaque MP4 extrait avec succès génère un fichier marqueur :</p>
|
||
<div class="highlight-text notranslate"><div class="highlight"><pre><span></span>/cosma-qc-frames/job_ID/.video_0.done
|
||
/cosma-qc-frames/job_ID/.video_1.done
|
||
...
|
||
</pre></div>
|
||
</div>
|
||
<p>En cas de crash, la reprise saute automatiquement les vidéos déjà traitées.</p>
|
||
<p>Pour forcer une ré-extraction complète :</p>
|
||
<div class="highlight-bash notranslate"><div class="highlight"><pre><span></span>rm<span class="w"> </span>/cosma-qc-frames/job_ID/.video_*.done
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
|
||
|
||
</div>
|
||
</div>
|
||
<footer><div class="rst-footer-buttons" role="navigation" aria-label="Footer">
|
||
<a href="infrastructure.html" class="btn btn-neutral float-left" title="Infrastructure" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
|
||
<a href="usage.html" class="btn btn-neutral float-right" title="Utilisation" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
|
||
</div>
|
||
|
||
<hr/>
|
||
|
||
<div role="contentinfo">
|
||
<p>© Copyright .</p>
|
||
</div>
|
||
|
||
Built with <a href="https://www.sphinx-doc.org/">Sphinx</a> using a
|
||
<a href="https://github.com/readthedocs/sphinx_rtd_theme">theme</a>
|
||
provided by <a href="https://readthedocs.org">Read the Docs</a>.
|
||
|
||
|
||
</footer>
|
||
</div>
|
||
</div>
|
||
</section>
|
||
</div>
|
||
<script>
|
||
jQuery(function () {
|
||
SphinxRtdTheme.Navigation.enable(true);
|
||
});
|
||
</script>
|
||
|
||
</body>
|
||
</html> |