diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..19538bd --- /dev/null +++ b/.env.example @@ -0,0 +1,9 @@ +NATS_URL= # empty = stdout fallback +MCAP_DIR=/data/mcap +POLL_INTERVAL_S=30 +LOG_LEVEL=INFO +# thresholds (optional, defaults in code) +BATTERY_LOW_V=13.5 +USBL_SNR_LOW=5.0 +USBL_DIST_SPIKE_M=50 +WATCHDOG_IMU_S=2.0 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e3bcf0a --- /dev/null +++ b/.gitignore @@ -0,0 +1,20 @@ +__pycache__/ +*.py[cod] +*$py.class +*.egg-info/ +.eggs/ +build/ +dist/ +.venv/ +venv/ +env/ +.env +.pytest_cache/ +.coverage +htmlcov/ +.mypy_cache/ +.ruff_cache/ +*.mcap +!tests/fixtures/*.mcap +.idea/ +.vscode/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e3df0a4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,21 @@ +FROM python:3.11-slim + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 + +WORKDIR /app + +COPY pyproject.toml README.md ./ +COPY src ./src + +RUN pip install --upgrade pip && pip install . + +ENV MCAP_DIR=/data/mcap \ + POLL_INTERVAL_S=30 \ + LOG_LEVEL=INFO + +VOLUME ["/data/mcap"] + +ENTRYPOINT ["cosma-log-analyzer"] +CMD ["serve"] diff --git a/README.md b/README.md index 93c295f..4ebfa4e 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,149 @@ # cosma-log-analyzer -Détecteur anomalies logs AUV COSMA (règles déterministes + NATS) \ No newline at end of file +Deterministic anomaly detection service for COSMA AUV logs. Ingests MCAP +files produced by the AUV/USV pipeline, evaluates a set of rules against +IMU / USBL / battery topics, and publishes each detection as a JSON event +on NATS (or stdout in dev). + +## Context + +COSMA (Flag) operates an AUV that streams telemetry to a surface USV. All +telemetry is persisted as MCAP (ROS2-native container). This service is +livrable #3: the first-pass observability layer before any statistical or +ML detection is added. + +``` + ┌──────┐ MCAP ┌──────┐ MCAP ┌───────────────────┐ NATS ┌────────────────┐ + │ AUV │────────▶│ USV │────────▶│ cosma-log-analyzer│──────────▶│ cosma-monitor │ + └──────┘ └──────┘ │ (this repo) │ events │ UI │ + └───────────────────┘ └────────────────┘ +``` + +## Rules (v0) + +| Rule | Threshold (default) | Severity | Topic | +|-----------------------|-------------------------------------------|----------|----------------------------------| +| `imu_outliers` | rolling 10 s window, \|z\| > 3 | warn | `/mavros/imu/data` | +| `watchdog_imu` | gap > 2 s between two IMU msgs | critical | `/mavros/imu/data` | +| `usbl_snr_low` | SNR < 5 dB for 3 consecutive samples | warn | `/usbl_reading/usbl_solution` | +| `usbl_distance_spike` | \|Δdistance\| > 50 m in less than 1 s | warn | `/usbl_reading/usbl_solution` | +| `battery_low` | voltage < 13.5 V for more than 5 s | critical | `/mavros/battery` | + +All thresholds are tunable via environment variables (`BATTERY_LOW_V`, +`USBL_SNR_LOW`, `USBL_DIST_SPIKE_M`, `WATCHDOG_IMU_S`) or rule +constructor arguments. + +## NATS subject + +``` +cosma.auv.{subject}.anomaly.{rule} +# ex: cosma.auv.AUV206.anomaly.battery_low +``` + +If `NATS_URL` is empty, events are written as JSON Lines to stdout — +useful in dev and CI. + +## Example anomaly payload + +```json +{ + "rule": "battery_low", + "severity": "critical", + "timestamp": 1700000055.0, + "subject": "AUV206", + "topic": "/mavros/battery", + "value": 13.26, + "context": { + "min_voltage_v": 13.5, + "min_duration_s": 5.0, + "run_start_ts": 1700000051.0, + "below_duration_s": 9.0 + } +} +``` + +## Install + +Python 3.11+ recommended. Works on 3.10. + +```bash +pip install -e .[dev] +``` + +## CLI + +```bash +# One-shot on a single MCAP file +cosma-log-analyzer ingest path/to/log.mcap --subject AUV206 + +# Dry-run: force stdout even if NATS_URL is set +cosma-log-analyzer ingest path/to/log.mcap --dry-run + +# Service mode: watch a directory for new MCAP files +cosma-log-analyzer serve --mcap-dir /data/mcap +``` + +## Docker + +```bash +docker compose up --build +# drop MCAP files into ./data/mcap and watch NATS on :4222 +``` + +## systemd + +```bash +sudo cp systemd/cosma-log-analyzer.service /etc/systemd/system/ +sudo systemctl enable --now cosma-log-analyzer +journalctl -u cosma-log-analyzer -f +``` + +## Tests + +```bash +pytest -v # 32 tests, runs the e2e against a fake MCAP +pytest --cov --cov-report=term # coverage (rules/ > 95%) +``` + +The fake MCAP generator (`tests/fixtures/generate_fake_mcap.py`) produces +a synthetic 60 s trace with one instance of each rule's trigger +condition — the e2e test asserts we detect exactly those. + +## Adding a rule + +1. Subclass `Rule` in `src/cosma_log_analyzer/rules/.py`: + + ```python + class MyRule(Rule): + name = "my_rule" + topic = "/my/topic" + severity = "warn" + + def detect(self, df: pd.DataFrame) -> list[Anomaly]: + ... + ``` + +2. Register it in `rules/__init__.py::all_rules()`. +3. Add a test in `tests/test_rules.py`. + +## Roadmap v1 + +- Rolling-stats rules (heading drift, GPS dropout correlated with USBL). +- Time alignment between MCAP IMU and CSV USV nav. +- ML anomaly layer (Isolation Forest) once we have > 50 h of nominal + dive datasets to train against. +- Backpressure + JetStream persistence for the NATS publisher. + +## Layout + +``` +src/cosma_log_analyzer/ # package code + rules/ # one file per rule + main.py # Click CLI: `ingest` + `serve` + ingest.py # MCAP + CSV readers -> pandas + bus.py # NATS publisher + stdout fallback + models.py # Anomaly dataclass +tests/ # pytest suite + fake MCAP fixture +examples/run_on_fake.sh # end-to-end demo +systemd/ # unit file for on-prem deployment +``` diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..133ac1a --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,21 @@ +services: + nats: + image: nats:2.10-alpine + command: ["-js", "-m", "8222"] + ports: + - "4222:4222" + - "8222:8222" + restart: unless-stopped + + analyzer: + build: . + depends_on: + - nats + environment: + NATS_URL: nats://nats:4222 + MCAP_DIR: /data/mcap + POLL_INTERVAL_S: 30 + LOG_LEVEL: INFO + volumes: + - ./data/mcap:/data/mcap + restart: unless-stopped diff --git a/examples/run_on_fake.sh b/examples/run_on_fake.sh new file mode 100755 index 0000000..61cc06d --- /dev/null +++ b/examples/run_on_fake.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# End-to-end demo: generate a synthetic MCAP, run the analyzer on it, +# pipe stdout anomalies to the console (NATS fallback mode). +set -euo pipefail + +ROOT=$(cd "$(dirname "$0")/.." && pwd) +cd "$ROOT" + +OUT=$(mktemp -d)/demo.mcap +python3 tests/fixtures/generate_fake_mcap.py "$OUT" + +echo "---- Running analyzer on $OUT ----" +NATS_URL="" cosma-log-analyzer ingest "$OUT" --subject AUV206 --dry-run diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..4e5057d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,44 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "cosma-log-analyzer" +version = "0.1.0" +description = "Deterministic anomaly detector for COSMA AUV logs (MCAP/CSV -> NATS)" +readme = "README.md" +requires-python = ">=3.10" +license = { text = "Proprietary" } +authors = [{ name = "Flag / COSMA" }] +dependencies = [ + "mcap>=1.2", + "pandas>=2.1", + "numpy>=1.26", + "nats-py>=2.6", + "python-dotenv>=1.0", + "click>=8.1", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.4", + "pytest-cov>=4.1", +] + +[project.scripts] +cosma-log-analyzer = "cosma_log_analyzer.main:cli" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-ra" + +[tool.coverage.run] +source = ["src/cosma_log_analyzer"] +branch = true + +[tool.coverage.report] +show_missing = true +skip_covered = false diff --git a/src/cosma_log_analyzer/__init__.py b/src/cosma_log_analyzer/__init__.py new file mode 100644 index 0000000..3dc1f76 --- /dev/null +++ b/src/cosma_log_analyzer/__init__.py @@ -0,0 +1 @@ +__version__ = "0.1.0" diff --git a/systemd/cosma-log-analyzer.service b/systemd/cosma-log-analyzer.service new file mode 100644 index 0000000..810cb7d --- /dev/null +++ b/systemd/cosma-log-analyzer.service @@ -0,0 +1,19 @@ +[Unit] +Description=COSMA log analyzer (MCAP -> NATS anomaly events) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=cosma +Group=cosma +WorkingDirectory=/opt/cosma-log-analyzer +EnvironmentFile=/etc/cosma-log-analyzer.env +ExecStart=/opt/cosma-log-analyzer/.venv/bin/cosma-log-analyzer serve +Restart=on-failure +RestartSec=5 +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target