istrain-public: from-scratch build of a passive RF train detector

Public community release. The complete working system — DSP + decoders (scripts/), web dashboard + API (dashboard/), container stack (docker/), config templates (config/) — plus APOCALYPSE-EDITION.md, the full build guide for a human or a coding agent. GPLv3 (PyEOT dependency). Scrubbed of all secrets, internal IPs, hostnames, and location detail. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-24 23:19:16 -04:00
commit 559caead36
41 changed files with 6740 additions and 0 deletions
@@ -0,0 +1,26 @@
+# istrain airband — rtl_airband v5.2.0 (pinned = the VM's proven version) on the rtl-sdr-blog
+# librtlsdr fork (required: dongle 1002 is an RTL-SDR Blog V4 — stock librtlsdr can't tune it).
+# DETACH_KERNEL_DRIVER=ON so the library kicks dvb_usb_rtl28xxu off the dongle itself — second
+# line of defense behind the host-side module blacklist (the "Allocating zero-copy" hang class).
+FROM debian:bookworm-slim AS build
+RUN apt-get update && apt-get install -y --no-install-recommends \
+      git ca-certificates build-essential cmake pkg-config \
+      libusb-1.0-0-dev libconfig++-dev libmp3lame-dev libshout3-dev zlib1g-dev libfftw3-dev \
+    && rm -rf /var/lib/apt/lists/*
+RUN git clone --depth 1 https://github.com/rtlsdrblog/rtl-sdr-blog /src/rtl-sdr-blog \
+    && cmake -S /src/rtl-sdr-blog -B /src/rtl-sdr-blog/build \
+         -DDETACH_KERNEL_DRIVER=ON -DINSTALL_UDEV_RULES=OFF \
+    && make -C /src/rtl-sdr-blog/build -j"$(nproc)" install && ldconfig
+RUN git clone --depth 1 --branch v5.2.0 https://github.com/rtl-airband/RTLSDR-Airband /src/airband \
+    && cd /src/airband \
+    && cmake -B build -DNFM=TRUE -DRTLSDR=TRUE \
+    && make -C build -j"$(nproc)" && make -C build install
+
+FROM debian:bookworm-slim
+RUN apt-get update && apt-get install -y --no-install-recommends \
+      libusb-1.0-0 libconfig++9v5 libmp3lame0 libshout3 libfftw3-single3 tini \
+    && rm -rf /var/lib/apt/lists/*
+COPY --from=build /usr/local /usr/local
+RUN ldconfig
+# -F foreground (systemd/docker style), -e errors to stderr; conf bind-mounted by compose.
+ENTRYPOINT ["tini", "--", "rtl_airband", "-F", "-e", "-c", "/etc/rtl_airband/istrain.conf"]
@@ -0,0 +1,143 @@
+# istrain — THE MILL stack (parallel stand-up started 2026-07-18; MILL-MIGRATION.md is the tracker).
+# Deployed by mill/deploy.sh into the Dockge stack dir (stacks/istrain) — Dockge owns the lifecycle,
+# the repo owns the truth. Web runs from day one on dual-pushed data (istrain-push-mill on the VM).
+#
+# airband + worker sit behind the "radio" profile until dongle day:
+#   docker compose --profile radio up -d      ← run this when 1002 is plugged into the Mill
+# (plain `up -d` starts web only; `--profile radio build` pre-builds the radio images now.)
+#
+# ONE CONF HOME: the capture dir mounts at /tmp/airband inside airband+worker, so the VM's canonical
+# airband/istrain.conf and transcribe-worker defaults run VERBATIM — no Mill fork of the channel plan.
+# Clips land in ./data/captures = web's CAPTURES_DIR, so the captures panel reads Mill-captured clips
+# the same way it reads VM-pushed ones. (Unpruned local growth: a DATA.md question for dongle day.)
+services:
+  web:
+    build: { context: ., dockerfile: web.Dockerfile }
+    image: istrain-web:mill
+    container_name: istrain-web
+    restart: unless-stopped
+    ports:
+      - "3456:3456"
+    volumes:
+      - ./app:/app:ro
+      - ./data:/data
+    environment:
+      - BIND=0.0.0.0
+      - PORT=3456
+      # NO PUSH_KEY (security pass 2026-07-24): the Mill AUTHORS its own data (push retired 07-18),
+      # so /api/push + /api/push-clip must stay DISABLED — an unset key makes both return 403. Setting
+      # a key here would re-open a public, key-in-repo data-overwrite path behind the Cloudflare name.
+      - EOT_LOG=/data/eot.jsonl
+      - BOT_LOG=/data/bot.jsonl
+      - MID_LOG=/data/midtrain.jsonl
+      - HEARD_LOG=/data/heard.jsonl
+      - LEVEL_LOG=/data/level.json
+      - SUPERVISOR_SNAPSHOT=/data/supervisor.json
+      - HITS_LOG=/data/hits.json
+      - CAPTURES_DIR=/data/captures
+      - TRANSCRIPTS_LOG=/data/transcripts.jsonl
+      # NO CTL_FORWARD (2026-07-18): the old target was the VM dash, whose buttons now control
+      # disabled units. Control is parked (403s gracefully; supervisor rows carry can_ctl=false via
+      # supervisor-probe.py) until a docker control relay exists — MILL-MIGRATION.md open item.
+
+  airband:
+    build: { context: ., dockerfile: airband.Dockerfile }
+    image: istrain-airband:mill
+    container_name: istrain-airband
+    restart: unless-stopped
+    profiles: ["radio"]
+    # whole-bus bind + cgroup rule (189 = usb) so the dongle survives replug/re-enumeration without
+    # a container restart; librtlsdr is built with DETACH_KERNEL_DRIVER=ON as the second line of
+    # defense behind the host dvb blacklist. rtl_airband exits nonzero with no dongle — the restart
+    # policy does the waiting (the 2026-07-18 boot-race lesson, baked in from day one).
+    volumes:
+      - /dev/bus/usb:/dev/bus/usb
+      - ./conf/istrain.conf:/etc/rtl_airband/istrain.conf:ro
+      - ./data/captures:/tmp/airband
+    device_cgroup_rules:
+      - "c 189:* rwm"
+    environment:
+      # the conf's localtime=true honors TZ; without it clip filenames stamp UTC (caught 2026-07-18,
+      # first Mill clips named _1454xx against 10:54 local — breaks eyeball-parity with VM-era names)
+      - TZ=America/New_York
+
+  worker:
+    build: { context: ., dockerfile: worker.Dockerfile }
+    image: istrain-worker:mill
+    container_name: istrain-worker
+    restart: unless-stopped
+    profiles: ["radio"]
+    volumes:
+      - ./app:/app:ro
+      - ./data:/data
+      - ./data/captures:/tmp/airband
+    environment:
+      # whisper is the istrain-whisper stack on this same box; host tailnet IP reaches it (:9000)
+      - MILL_ASR=http://MILL-IP:9000
+      - TRANSCRIPTS_LOG=/data/transcripts.jsonl
+      - TZ=America/New_York
+    command: ["python3", "/app/scripts/transcribe-worker.py"]
+
+  trim:
+    image: istrain-worker:mill
+    container_name: istrain-trim
+    restart: unless-stopped
+    volumes:
+      - ./data/captures:/tmp/airband
+    environment:
+      # same policy as the VM era: voice audio >24h pruned AFTER heard.jsonl ingest (the script
+      # aborts if the dash is unreachable — an unlogged clip can never be deleted); transcripts +
+      # detections are the durable record. Keeps ./data/captures bounded on flash.
+      - DASH_URL=http://web:3456
+      - CAP_KEEP_HOURS=24
+      - TZ=America/New_York
+    command: ["sh", "-c", "while :; do python3 /app/scripts/trim-captures.py; sleep 3600; done"]
+    depends_on: [web]
+
+  # ---- the 1001 set (profile radio1001) — start ONLY once 1001 is plugged into the Mill: ---------
+  #   docker compose --profile radio --profile radio1001 up -d
+  # Separate profile from 1002's so a compose cycle can never point rtl_tcp at the wrong dongle
+  # while only one radio has moved. The chain writes under ~/istrain (hardcoded in iq_channelize),
+  # so ./data mounts at /root/istrain — the SAME files web serves; push-seeded history continues
+  # in place. ⚠ istrain-push-mill on the VM must be DISABLED before first start (its 60s full-file
+  # replace would fight the appends — the PUSH_SKIP lesson, now for every log).
+  scanhop:
+    build: { context: ., dockerfile: scanhop.Dockerfile }
+    image: istrain-scanhop:mill
+    container_name: istrain-scanhop
+    restart: unless-stopped
+    profiles: ["radio1001"]
+    volumes:
+      - /dev/bus/usb:/dev/bus/usb
+      - ./app:/app:ro
+      - ./data:/root/istrain
+    device_cgroup_rules:
+      - "c 189:* rwm"
+    environment:
+      - HOP_DWELL_452=28
+      - HOP_DWELL_457=28
+      - TZ=America/New_York
+
+  eot-decode:
+    image: istrain-worker:mill
+    container_name: istrain-eot-decode
+    restart: unless-stopped
+    profiles: ["radio1001"]
+    volumes:
+      - ./app:/app:ro
+      - ./data:/root/istrain
+    environment:
+      - TZ=America/New_York
+    command: ["python3", "/app/scripts/eot-decode.py"]
+
+  bot-recover:
+    image: istrain-worker:mill
+    container_name: istrain-bot-recover
+    restart: unless-stopped
+    profiles: ["radio1001"]
+    volumes:
+      - ./app:/app:ro
+      - ./data:/root/istrain
+    environment:
+      - TZ=America/New_York
+    command: ["python3", "/app/scripts/bot-recover.py"]
@@ -0,0 +1,8 @@
+# istrain scan-hop — 1001 retune-in-place: iq_hop.py (python+numpy, from the worker base) driving
+# rtl_tcp (lifted from the airband image's blog-fork build — same librtlsdr for both radios).
+# ⚠ BUILD ORDER: istrain-airband:mill must exist first (deploy.sh builds airband before this).
+FROM istrain-worker:mill
+COPY --from=istrain-airband:mill /usr/local/bin/rtl_tcp /usr/local/bin/rtl_tcp
+COPY --from=istrain-airband:mill /usr/local/lib /usr/local/lib
+RUN ldconfig
+CMD ["python3", "/app/scripts/iq_hop.py"]
@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+"""istrain — Mill supervisor probe. Runs on the TrueNAS HOST via cron (every minute, root):
+docker state -> data/supervisor.json in the dashboard's snapshot shape, so the supervisor card
+tells Docker-era truth (it snapshotted the VM's systemd units until 2026-07-18). Read-only:
+can_ctl=false on every row — site control buttons are parked until a docker control relay exists
+(MILL-MIGRATION.md open item). Atomic write; stdlib only (host python3, no pip)."""
+import json, os, subprocess, tempfile, time
+
+STACK = "/mnt/.ix-apps/app_mounts/dockge/stacks/istrain"
+OUT = os.path.join(STACK, "data", "supervisor.json")
+
+# (container, name, job) — mirrors the VM card's rows; names kept identical where the role carried over.
+ROLES = [
+    ("istrain-web",         "Dashboard",    "serves this dashboard + data API (Mill :3456, the Cloudflare origin)"),
+    ("istrain-scanhop",     "UHF · 1001",  "dongle 1001: 452 ⇄ 457 retune-in-place hop → BOT/EOT/MID logs + clips + live meter"),
+    ("istrain-airband",     "Voice · 1002", "dongle 1002: rail voice, 5 NS channels → mp3 clips + heard log"),
+    ("istrain-worker",      "Transcribe",   "voice clips → whisper on this box (:9000) → sidecars + transcripts log"),
+    ("istrain-eot-decode",  "EOT decode",   "eot_clips → unit/telemetry; enriches eot.jsonl (never culls)"),
+    ("istrain-bot-recover", "BOT recover",  "bot_clips → clean head-end frame; enriches bot.jsonl (never culls)"),
+    ("istrain-trim",        "Trim",         "hourly voice-clip prune (>24h) — heard.jsonl ingest-gated, detections kept"),
+    ("istrain-whisper",     "Whisper ASR",  "faster-whisper small.en (istrain-whisper stack, :9000)"),
+]
+
+DOTS = {"running": ("active", "listening"), "restarting": ("failed", "stalled"),
+        "paused": ("inactive", "held"), "exited": ("inactive", "offline")}
+
+def main():
+    out = subprocess.run(["docker", "ps", "-a", "--format", "{{.Names}}\t{{.State}}\t{{.Status}}"],
+                         capture_output=True, text=True).stdout
+    have = {}
+    for ln in out.splitlines():
+        p = ln.split("\t")
+        if len(p) >= 3:
+            have[p[0]] = (p[1], p[2])
+    units = []
+    for cid, name, job in ROLES:
+        state, status = have.get(cid, ("absent", "no container"))
+        st, dot = DOTS.get(state, ("inactive", "offline"))
+        units.append({"id": cid, "name": name, "kind": "service", "job": job,
+                      "dot": dot, "state": st, "detail": status, "since": "", "can_ctl": False})
+    snap = {"units": units, "ts": time.time(), "controls": False}
+    fd, tmp = tempfile.mkstemp(dir=os.path.dirname(OUT))
+    with os.fdopen(fd, "w") as f:
+        json.dump(snap, f)
+    os.replace(tmp, OUT)
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,6 @@
+# istrain web — Mill flavor. Same animal as nas/Dockerfile (the old-NAS image): stdlib-only serve.py,
+# app bind-mounted at /app so dashboard updates are a re-deploy + restart, never a rebuild.
+FROM python:3.12-slim
+WORKDIR /app
+EXPOSE 3456
+CMD ["python3", "dashboard/serve.py"]
@@ -0,0 +1,8 @@
+# istrain worker — transcribe-worker (ffmpeg comms-filter + POST to local whisper) and the
+# dsp-base image: numpy is here so `iq_hop.py --selftest` / `iq_channelize.py --selftest` can run
+# in-container (the acceptance rail that pre-proves the 1001 port before that dongle ever moves).
+FROM python:3.12-slim
+RUN apt-get update && apt-get install -y --no-install-recommends ffmpeg \
+    && rm -rf /var/lib/apt/lists/* \
+    && pip install --no-cache-dir numpy
+CMD ["python3", "/app/scripts/transcribe-worker.py"]