From 559caead36bedc4c4f397a8e79eb37253e909d4d Mon Sep 17 00:00:00 2001 From: istrain Date: Fri, 24 Jul 2026 23:19:16 -0400 Subject: [PATCH] istrain-public: from-scratch build of a passive RF train detector MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Public community release. The complete working system — DSP + decoders (scripts/), web dashboard + API (dashboard/), container stack (docker/), config templates (config/) — plus APOCALYPSE-EDITION.md, the full build guide for a human or a coding agent. GPLv3 (PyEOT dependency). Scrubbed of all secrets, internal IPs, hostnames, and location detail. Co-Authored-By: Claude Fable 5 --- .gitignore | 6 + APOCALYPSE-EDITION.md | 372 ++++++++ ATTRIBUTION.md | 28 + LICENSE | 674 +++++++++++++ README.md | 66 ++ config/blacklist-rtl-sdr.conf | 7 + config/istrain.conf.example | 70 ++ dashboard/checklist.md | 41 + dashboard/index.html | 300 ++++++ dashboard/main.js | 96 ++ dashboard/panels/strip.js | 127 +++ dashboard/radios.json | 23 + dashboard/serve.py | 1521 ++++++++++++++++++++++++++++++ dashboard/style.css | 240 +++++ docker/airband.Dockerfile | 26 + docker/compose.example.yaml | 143 +++ docker/scanhop.Dockerfile | 8 + docker/supervisor-probe.py | 48 + docker/web.Dockerfile | 6 + docker/worker.Dockerfile | 8 + scripts/bot-recover.py | 138 +++ scripts/correlate.py | 136 +++ scripts/envelope.py | 112 +++ scripts/eot-decode.py | 118 +++ scripts/eot/PyEOT/LICENSE | 674 +++++++++++++ scripts/eot/PyEOT/README.md | 20 + scripts/eot/PyEOT/demo3eot.wav | Bin 0 -> 123984 bytes scripts/eot/PyEOT/eot_decoder.py | 62 ++ scripts/eot/PyEOT/helpers.py | 78 ++ scripts/eot/PyEOT/pyeot.py | 69 ++ scripts/eot/README.md | 74 ++ scripts/eot/eot_scan.py | 169 ++++ scripts/eot/hot_explore.py | 72 ++ scripts/iq_channelize.py | 314 ++++++ scripts/iq_hop.py | 244 +++++ scripts/iq_retune.py | 145 +++ scripts/probe-band.py | 125 +++ scripts/scan-hop.py | 93 ++ scripts/transcribe-worker.py | 97 ++ scripts/trim-captures.py | 35 + scripts/vumon.py | 155 +++ 41 files changed, 6740 insertions(+) create mode 100644 .gitignore create mode 100644 APOCALYPSE-EDITION.md create mode 100644 ATTRIBUTION.md create mode 100644 LICENSE create mode 100644 README.md create mode 100644 config/blacklist-rtl-sdr.conf create mode 100644 config/istrain.conf.example create mode 100644 dashboard/checklist.md create mode 100644 dashboard/index.html create mode 100644 dashboard/main.js create mode 100644 dashboard/panels/strip.js create mode 100644 dashboard/radios.json create mode 100644 dashboard/serve.py create mode 100644 dashboard/style.css create mode 100644 docker/airband.Dockerfile create mode 100644 docker/compose.example.yaml create mode 100644 docker/scanhop.Dockerfile create mode 100644 docker/supervisor-probe.py create mode 100644 docker/web.Dockerfile create mode 100644 docker/worker.Dockerfile create mode 100644 scripts/bot-recover.py create mode 100644 scripts/correlate.py create mode 100644 scripts/envelope.py create mode 100644 scripts/eot-decode.py create mode 100644 scripts/eot/PyEOT/LICENSE create mode 100644 scripts/eot/PyEOT/README.md create mode 100644 scripts/eot/PyEOT/demo3eot.wav create mode 100644 scripts/eot/PyEOT/eot_decoder.py create mode 100644 scripts/eot/PyEOT/helpers.py create mode 100644 scripts/eot/PyEOT/pyeot.py create mode 100644 scripts/eot/README.md create mode 100644 scripts/eot/eot_scan.py create mode 100644 scripts/eot/hot_explore.py create mode 100644 scripts/iq_channelize.py create mode 100644 scripts/iq_hop.py create mode 100644 scripts/iq_retune.py create mode 100755 scripts/probe-band.py create mode 100644 scripts/scan-hop.py create mode 100644 scripts/transcribe-worker.py create mode 100644 scripts/trim-captures.py create mode 100755 scripts/vumon.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a89b757 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +__pycache__/ +*.pyc +data/ +compose.yaml +config/istrain.conf +dashboard/VERSION diff --git a/APOCALYPSE-EDITION.md b/APOCALYPSE-EDITION.md new file mode 100644 index 0000000..c9c2f1b --- /dev/null +++ b/APOCALYPSE-EDITION.md @@ -0,0 +1,372 @@ +# istrain — APOCALYPSE EDITION (public build guide) + +*The complete from-scratch rebuild. This guide assumes you can install Linux and are comfortable +in a terminal, but it spells out every railroad-, SDR-, and container-specific step — because +those are the parts nobody tells you. If a step feels obvious to you, skip it; it's here for the +person behind you who needs the hint.* + +*You are building a **passive** RF receiver. It only listens. No transmit, no license required in +the US to receive, no railroad cooperation needed — the trains announce themselves on public +frequencies, and you just have to be set up to hear them.* + +> **Two ways to use this repo:** +> - **As a human:** read top to bottom, run the commands. +> - **As an agent** (point Claude Code / an LLM coding agent at this repo): the whole working tree +> is here — `scripts/` (the DSP + decoders), `dashboard/` (the web UI + API server), +> `docker/` (the container definitions), `config/` (templates). Every script has a header +> comment explaining what it does and how it's operated. The design rules that keep it robust +> are in §11. Start by reading this file, then `dashboard/serve.py` and `scripts/iq_hop.py`. + +--- + +## 0. What you're building, in one minute + +A train is a chain of radio transmitters. istrain listens passively to three of them and fuses +the result into one question: **is a train blocking my crossing right now?** + +1. **Rail VHF voice (160–162 MHz, narrowband FM, unencrypted).** Crews, dispatchers, and trackside + **defect detectors** speaking plain English: *"Norfolk Southern detector, milepost 148.9, no + defects, total axles 460, speed 52, detector out."* A detector hit on your road channel means a + train just passed that milepost — the most direct "train coming" signal there is, in words. +2. **End-of-Train telemetry (457.9375 MHz, 1200-baud FFSK).** The flashing box on the last car + (EOT / FRED / marker) transmits its unit ID, **brake-pipe pressure**, and a **motion flag** + every few seconds. Decode it and the air tells you: passing, stopped, brakes cut, or leaving. +3. **Head-of-Train (452.9375 MHz).** The locomotive's half of the same conversation. Decode the + EOT unit it's addressing and you can **join head to tail** — proof of a whole train, plus + warning from the front. + +Bonus ears: **±12.5 kHz off the EOT/HOT centers** (distributed-power / mid-train repeaters live +there) and **NOAA weather radio (162.4–162.55 MHz)** as an always-on, known-good test signal. + +**The two facts that make this work at ANY North American crossing:** +- **457.9375 / 452.9375 MHz is continent-wide.** Every EOT and HOT in North America uses this + pair. You change nothing when you move. +- **The voice channels are public.** The AAR (Association of American Railroads) channel plan is + published per railroad and per subdivision. Only your voice channel(s) change with location. + +The synthesis: any one signal can be too weak to decode, but a real train lights several bands in +the same time window. Co-firing bands = a train, even on a modest antenna. + +--- + +## 1. Hardware — the shopping list + +| Item | What we run | Notes | +|---|---|---| +| SDR #1 — voice | **[RTL-SDR Blog V4](https://www.rtl-sdr.com/buy-rtl-sdr-dvb-t-dongles/)** | The V4's front-end filtering genuinely helps at VHF. This is the ears for 160–162 MHz. | +| SDR #2 — telemetry | Any RTL-SDR (R820T/R828D-class) | Runs the 452⇄457 hop. Generic sticks drift; the decoder compensates (§8). A TCXO stick is nicer, not required. | +| Antennas | ¼-wave verticals to start | **18.3 in (46.5 cm)** whip for 161 MHz; **6.4 in (16.3 cm)** for 457 MHz. Length is physics — get it right and a desk whip works day one. Upgrade: a tuned railroad-band base antenna (DPD TrainTenna, KB9VBR J-Pole) mounted high and far from your computers. Height beats gain; distance from noisy electronics beats both. | +| USB | A **powered** hub | Two dongles on one host; a powered hub prevents brownout resets. | +| Host | Any always-on 64-bit Linux box that runs Docker | Ours is a TrueNAS SCALE machine. A NUC, a Raspberry Pi 4/5, a Pi-like SBC, or an old desktop all work. 2+ cores, 2+ GB RAM. The whole rig is containers. | +| A second computer | Your laptop | For setup, watching waterfalls, and reading the dashboard. | + +**Starting-from-nothing cost floor:** two dongles + whips ≈ US$70–100. Everything else is software +in this repo. + +--- + +## 2. Operating system & base software + +Any modern 64-bit Linux works. We'll give commands for **Debian/Ubuntu-family** (apt); translate +to your package manager as needed. Three viable host shapes: + +- **A regular Linux box (recommended for a first build)** — Debian 12, Ubuntu 22.04+, Linux Mint, + Raspberry Pi OS (64-bit). You control everything; nothing fights you. This guide's default. +- **An appliance NAS OS (what we ended on)** — TrueNAS SCALE, Unraid, etc., running Docker/Dockge. + More convenient long-term, but the OS resists you (see the §4 DVB note — appliance `/etc` can + reset on update). Do your *first* build on a regular box, move to the appliance once it works. +- **Bare metal vs. VM:** bare metal is simplest. A VM works but USB passthrough adds a failure + mode and caps your sample rate — we ran in VirtualBox for a month and it was fine at 1.024 MS/s, + but a Pi or NUC on bare metal is less fuss. + +Install the base tools: + +```bash +sudo apt update +sudo apt install -y git rtl-sdr librtlsdr-dev build-essential cmake pkg-config \ + libusb-1.0-0-dev ffmpeg python3 python3-numpy docker.io docker-compose-plugin +``` + +- `rtl-sdr` gives you `rtl_test`, `rtl_eeprom`, `rtl_fm`, `rtl_sdr` — the osmocom tools that drive + the dongles. +- `ffmpeg` is the audio plumbing (and the comms band-pass filter in front of transcription). +- `python3` + `numpy` run every DSP and decode script here — **no other Python packages are + required** for the core rig (the scripts are deliberately stdlib + numpy only). +- Docker runs the whole stack. Optionally add **[Dockge](https://github.com/louislam/dockge)** — a + light web UI for compose stacks; it's what we use, but plain `docker compose` is identical. + +Clone this repo onto the host: + +```bash +git clone istrain && cd istrain +``` + +--- + +## 3. Know your crossing — 30 minutes of desk research + +This is the only location-specific work. Do it once. + +1. **Identify the railroad and subdivision** at your crossing. Read the signage at the crossing + itself (the operating railroad's name is on the signal equipment and the blue emergency-notify + sign — that sign also lists a **DOT crossing number** and a phone number). Cross-check on + [OpenRailwayMap](https://www.openrailwaymap.org/). +2. **Find the road (dispatch) channel.** Use two of these three and confirm they agree: + - [RadioReference — US Railroads DB](https://www.radioreference.com/db/aid/7625) + - [Railroad-Frequencies.com](https://www.railroad-frequencies.com/) + - your region's railfan forum / site. + You want the **AAR channel for the subdivision your crossing sits on**. Note its frequency and + AAR channel number. Grab the **neighboring subdivisions' channels too** — pin 4–6 channels in a + cluster; RTLSDR-Airband demodulates them all at once inside a single ~1 MHz window if they're + within ~500 kHz of each other (most road channels cluster in 160.2–161.6 MHz). +3. **Find your defect detectors.** Railfan detector logs and forums list detector mileposts per + subdivision. The detectors within ~10 miles either side of your crossing are your early-warning + tripwires — write down their milepost numbers so the transcripts make sense. +4. **Note your NOAA weather frequency** (162.400–162.550, whichever is strongest near you). This is + your receive-chain sanity signal forever — if you can hear NOAA, your antenna→dongle→software + path works. + +Write it all down. See `config/istrain.conf.example` — it's our worked example (a Norfolk Southern +subdivision in central Ohio); replace the five channel frequencies with yours. + +--- + +## 4. Free the dongles from the TV driver (the step that bites everyone) + +The Linux kernel grabs RTL-SDR dongles as **DVB television tuners** the instant they're plugged in, +and then no SDR software can open them. Blacklist the DVB driver on the **host** (this repo ships +the file at `config/blacklist-rtl-sdr.conf`): + +```bash +sudo cp config/blacklist-rtl-sdr.conf /etc/modprobe.d/blacklist-rtl-sdr.conf +sudo modprobe -r dvb_usb_rtl28xxu 2>/dev/null # unload it now +lsmod | grep dvb # should print NOTHING +``` + +Reboot to make it stick. **The symptom of losing this fight later:** rtl_airband hangs at +"Allocating zero-copy buffers," or `rtl_test` says the device is busy or open failed. It is +*always* the DVB driver reloading — re-run the check above. + +> ⚠ **On appliance OSes (TrueNAS/Unraid/etc.), `/etc` can be reset by a system update.** Re-verify +> this blacklist after every OS update. This is a permanent standing chore, not a one-time step. + +**Serialize your dongles** so every service can pick the right one by name (index order changes on +reboot; serials don't). With only ONE dongle plugged in at a time: + +```bash +rtl_eeprom -s 1002 # plug in dongle A alone, give it serial 1002 (your voice radio) +# unplug, plug in dongle B alone: +rtl_eeprom -s 1001 # dongle B = 1001 (your telemetry radio) +rtl_test # with both in: lists both by serial; confirms they enumerate +``` + +(We use `1001`/`1002`; any string works — just match it in the configs.) + +--- + +## 5. The stack — what runs + +Everything is one Docker Compose stack (`docker/compose.example.yaml` + the Dockerfiles beside it): + +``` +▣ airband RTLSDR-Airband (rtl-sdr-blog fork, V4-capable) on dongle 1002 + → demodulates your 4–6 pinned voice channels at once, squelch-gated + → one mp3 per squelch opening + a line in heard.jsonl +▣ scanhop rtl_tcp + scripts/iq_hop.py on dongle 1001 + → retunes in place: 452.9375 (10 s) ⇄ 457.9375 (10 s), forever + → scripts/iq_channelize.py splits each dwell into center ±12.5 kHz sub-channels + → burst → bot.jsonl / eot.jsonl / midtrain.jsonl + a raw .s16 clip + → level.json written continuously (the live meter + "is capture alive" signal) +▣ eot-decode scripts/eot-decode.py → scripts/eot/eot_scan.py : two-pass FFSK decode of EOT clips + → unit ID · brake psi · motion flag, enriched back into eot.jsonl +▣ bot-recover scripts/bot-recover.py : head-end frames → addressed-unit → head/tail join +▣ worker scripts/transcribe-worker.py : voice clips >5 KB → ffmpeg comms band-pass + → a Whisper server (§7) → transcript sidecars; real speech → transcripts.jsonl +▣ trim scripts/trim-captures.py : prunes voice audio >24 h (the jsonl records are kept forever) +▣ web dashboard/serve.py : stdlib Python, zero frameworks — the dashboard + all /api/* endpoints +``` + +Copy the compose template and edit the two host-specific things — your stack directory path and +your dongle serials (already `1001`/`1002` if you followed §4): + +```bash +cp docker/compose.example.yaml docker/compose.yaml +cp config/istrain.conf.example config/istrain.conf # then edit YOUR channels into it (§3) +``` + +--- + +## 6. Bring-up, step by step + +1. **Selftest the DSP before any radio.** These run the whole signal chain on synthetic data — no + dongle needed. Green here means the math works on your machine; anything failing after this is + radio/USB, not code: + ```bash + python3 scripts/iq_hop.py --selftest + python3 scripts/iq_channelize.py --selftest + ``` +2. **Prove the receive chain with NOAA first.** Before you wait for a train, confirm the whole + antenna→dongle→software path works on a signal that's always there. Point a dongle at your NOAA + frequency: + ```bash + rtl_fm -d 0 -f 162.550M -M fm -s 12k -g 40 - | aplay -r 12k -f S16_LE + ``` + Weather robot voice = your chain works end to end. Silence = fix the antenna/gain/frequency + before going further. (This is the single most useful debugging move; NOAA is your oscilloscope.) +3. **Edit `config/istrain.conf`** — replace the example channels with your voice frequencies from + §3. Keep the shape: one `channels` entry per frequency, `squelch_snr_threshold` starting near + the example's value (tune per §8), mp3 outputs into the captures dir, and your dongle serial in + the device stanza. +4. **Build and start the stack** (from `docker/`): + ```bash + docker compose --profile radio --profile radio1001 build # airband image builds first; + # scanhop copies rtl_tcp out of it + docker compose up -d + docker compose ps # all containers Up + ``` +5. **Verify capture is alive by DATA, not by "is the process running."** Watch `data/level.json` — + it should be rewritten every second or two (that's the hop breathing). Watch `data/captures/` + for mp3s. This data-freshness habit is the whole health model (§11). +6. **Open the dashboard:** `http://:3456`. The live meter should bounce; the correlate + heatmap fills in 15-minute bins as bands fire. +7. **Wait for a train.** The first EOT decode is unmistakable: a unit ID, a brake pressure, and a + motion flag from a machine a mile away, pulled out of static by a US$30 dongle. That's the + moment the project becomes real. + +--- + +## 7. Ears (optional, and the best part): transcription + +Any Whisper-family ASR server works. We run **[faster-whisper](https://github.com/SYSTRAN/faster-whisper) +(model `small.en`) with VAD enabled** as its own small container/service, and +`scripts/transcribe-worker.py` posts each filtered voice clip to it (set `WHISPER_URL` / +`MILL_ASR` env to point at your server; the template default is `http://whisper:9000`). + +Two rules learned the hard way: +- **Band-pass the audio first.** The worker runs an ffmpeg ~300–3400 Hz comms filter before ASR — + Whisper hallucinates confident nonsense on raw squelch noise. +- **VAD on, always.** Without voice-activity detection, an hour of static becomes an hour of + imaginary dispatcher chatter. Only clips with *detected speech* earn a line in + `transcripts.jsonl` — the project's durable memory of everything the railroad said out loud + near your crossing. + +--- + +## 8. Getting the truth out of cheap hardware + +- **Generic dongles drift off frequency.** Our EOT decoder (`scripts/eot/eot_scan.py`) is + **two-pass**: an exact fast path, then an adaptive pass with fuzzy sync + tone tracking (±800 Hz) + that recovers the off-frequency, short-burst emitters an exact decoder throws away as noise. + Roughly half our real decodes come from the adaptive pass. If you write your own decoder, plan + for drift from the start. +- **Squelch sits just above YOUR noise floor, not a textbook number.** Watch a quiet hour's + squelch-opening rate. If you're capturing hundreds of empty clips, you're squelching on your own + house's RF hash — we measured **~⅔ of desk-whip squelch openings were never radio at all.** Raise + `squelch_snr_threshold` until the empties stop; lower it if you're missing weak transmissions. +- **Gain does not fix weak.** More gain amplifies the same noise. An A/B at gain 25 vs 40 vs auto on + a buried signal sounds identical. The levers that actually work, in order: **antenna height, + distance from electronics, a tuned antenna, better coax.** +- **Beware standing carriers.** We chased a "mystery mid-train transmitter" for days; it was a + **fixed wayside EOT signal booster** (an FCC-licensed repeater) parked on 457.925 that never + moves. Card it as furniture and subtract its floor from your correlation logic, or it fakes a band + forever. +- **Trust the decoder, not quick statistics.** Every shortcut we tried to characterize undecodable + bursts by spectral/envelope stats failed its own control test (a measure that couldn't tell + modulated from unmodulated on *known-good* clips proves nothing). The decoder is the only + instrument that doesn't lie. + +--- + +## 9. Reading a train (the payoff) + +From decoded EOT frames, per train, the **brake-pipe pressure curve is the story silence can't +tell:** + +| Air curve | Meaning | +|---|---| +| ~85–90 psi + motion flag set | **Passing** through | +| Drops to a held ~55–65, motion 0 | **Dwelling** — brakes set, possibly blocking the crossing | +| Bleeds to 0 and holds, head still polling | **Cut & standing** — crew cut the crossing; the road may be OPEN even though a train is "there" | +| Recharges toward ~88, motion 0→1 | **Departing** — the roll-out is your event time | + +**Never read a bleed-to-zero as a departure** — that mistake reports a blocked crossing as clear. +This taxonomy came from ~50 ground-truthed passages; yours may differ by railroad and operating +pattern, so watch and record before you trust it. + +--- + +## 10. Security — before you point this at the internet + +The dashboard (`dashboard/serve.py`) is safe to run on your LAN as-is. Before exposing it publicly +(a reverse proxy, a Cloudflare tunnel, a port-forward), know these: + +- **Run it read-only when public.** Set the `SUPERVISOR_SNAPSHOT` env (any path) to mark the node + as a **presentation node** — that disables clip deletion and refuses control POSTs. Leave it + unset only on a private box you alone reach. +- **Leave `PUSH_KEY` unset unless you actually use the VM→server push path.** Unset = the + `/api/push` and `/api/push-clip` write endpoints are disabled entirely. If you set a key, treat + it as a real secret (env only, never committed) — it can overwrite the data your site displays. +- **The control endpoints are speed-bumped, not authenticated.** They require an `X-Ops` header + (so blind scanners bounce), but that's a bump, not a password. Real control should stay on your + LAN/tailnet, never bare on the public name. Put auth at your proxy if you want remote control. +- **`/api/errors/report` is intentionally public** (browser error beacon) — it's rate-limited and + length-capped server-side; that's the only endpoint that accepts unauthenticated writes, and it + only appends bounded strings to a log. +- **Never expose the raw container port to the internet.** Front it with a proxy that terminates + TLS and, ideally, gates writes. + +There are **no credentials, keys, or personal data in this repository.** The frequencies are +public; the code is stdlib. Keep it that way in your fork — put secrets in env, not in files. + +--- + +## 11. The design rules that keep it robust (read if you're modifying it) + +These were bought with outages and false readings. Keep them: + +- **Every capture entrypoint exits nonzero when its dongle is missing.** The container restart + policy (`restart: unless-stopped`) does the waiting — so a replug or a cold reboot self-heals + with no human. (We lost a morning to a process that exited *zero* on a missing dongle and sat + there looking alive.) +- **No startup ordering between containers.** Each retries until its own dependencies appear — + impossible to deadlock on boot order. +- **Decoders never delete clips.** Capture, decode, and cleanup are separate jobs; a decoder bug + can't eat your evidence. +- **Health is read from data freshness, never from "is the process up."** `level.json`'s timestamp + age, the last capture's age per band, the transcript trickle — a fresh timestamp is the only + health signal a radio can't fake. +- **Keep one append-only, numbered observations log.** Every "mystery signal" we resolved, we + resolved by writing down what we actually *measured* — and the two findings we later retracted + taught more than most that stood. Number the entries. Record the retractions. Future-you (or the + next forker) is who you're writing for. + +--- + +## 12. What's in this repo + +``` +APOCALYPSE-EDITION.md ← you are here +README.md ← the front door + pointers +LICENSE ← GPLv3 +ATTRIBUTION.md ← third-party code (PyEOT) + the shoulders this stands on +scripts/ ← the signal engine + decoders (stdlib + numpy only) + iq_hop.py ← retune-in-place 452⇄457 hop (the mission radio) + iq_channelize.py ← splits a dwell into center + ±12.5 kHz sub-channels + eot-decode.py, eot/ ← two-pass FFSK EOT decoder (eot_scan.py + vendored PyEOT for BCH) + bot-recover.py ← head-end frame recovery + head/tail join + transcribe-worker.py ← voice clip → comms filter → Whisper → transcript + vumon.py ← the live level meter + correlate.py ← the cross-band "why" engine behind the heatmap + trim-captures.py ← the 24 h voice-audio janitor + scan-hop.py, iq_retune.py, envelope.py, probe-band.py ← the hop wrapper + analysis helpers +dashboard/ ← serve.py (the API + static server) + the web UI +docker/ ← compose template + Dockerfiles (web/airband/scanhop/worker) +config/ ← istrain.conf.example (channels), blacklist-rtl-sdr.conf +``` + +--- + +*Built at a Norfolk Southern crossing in central Ohio, 2026, on RTL-SDR Blog V4 hardware, by a +human and his AI pair-programmer. The live instance runs at +[istrain.jhestyr.net](https://istrain.jhestyr.net). This repository is the whole thing — read it, +run it, fork it, and if you build one, tell us what you heard.* diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md new file mode 100644 index 0000000..3d95ee1 --- /dev/null +++ b/ATTRIBUTION.md @@ -0,0 +1,28 @@ +# Attribution & licensing + +**istrain-public is licensed GPLv3** (see [`LICENSE`](LICENSE)). This choice follows the +strongest dependency: the EOT decoder validates frames against **PyEOT**, which is GPLv3. + +## Third-party code vendored here + +- **`scripts/eot/PyEOT/`** — [PyEOT by Eric Reuter](https://github.com/ereuter/PyEOT), the + open-source End-of-Train / FFSK decoder (packet parser + BCH check) and its demo recording. + **GPLv3**, its `LICENSE` kept intact in that directory. Our `scripts/eot/eot_scan.py` calls + into it at runtime for frame-sync and validity checking; the two-pass adaptive front-end + (drift-tolerant tone tracking + fuzzy sync) is our addition. + +## The shoulders this stands on + +The tools and communities credited in the dashboard's **With Thanks** panel are the real +foundation — RTL-SDR Blog and the osmocom rtl-sdr tools, RTLSDR-Airband, faster-whisper / +OpenAI Whisper, FFmpeg, NumPy, Docker + Dockge, and the railfan frequency communities +(RadioReference, Railroad-Frequencies.com, the regional railfan sites) that publish the +channel plans that make any of this findable. See `dashboard/index.html` for the full list +with links. + +## Note for the person who forked this + +If you'd rather license your own build permissively (MIT/BSD), you can — **but not while +PyEOT is vendored and imported.** Swap in your own FFSK frame parser + BCH check (the spec is +public — 1200-baud FFSK, the SigidWiki EOTD page documents it), drop `scripts/eot/PyEOT/`, and +the GPL obligation goes with it. Everything else here is original work. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f288702 --- /dev/null +++ b/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/README.md b/README.md new file mode 100644 index 0000000..292217f --- /dev/null +++ b/README.md @@ -0,0 +1,66 @@ +# istrain — a passive RF train detector you can build + +**Is a train blocking the crossing?** istrain answers that by listening — passively, on public +railroad radio frequencies — to the transmitters every train carries: the crew/dispatch voice and +trackside **defect detectors** on 160–162 MHz, and the **End-of-Train** and **Head-of-Train** +telemetry on 457.9375 / 452.9375 MHz. It decodes the End-of-Train brake-pressure and motion data, +transcribes the voice, and fuses it all into one live verdict. + +**Live instance:** [istrain.jhestyr.net](https://istrain.jhestyr.net) + +It's built on a couple of ~US$30 RTL-SDR dongles, some wire, and an always-on Linux box running +Docker. No transmitting, no license needed to receive in the US, no railroad cooperation — the +trains announce themselves; you just get set up to hear them. It works at **any North American +crossing** because the EOT/HOT frequencies are continent-wide and the voice channel plans are +public — you change one config file for your location. + +This repository is the **complete, working system** — the DSP, the decoders, the web dashboard, and +the container definitions — shared so anyone (or any coding agent) can build one from scratch and +learn from it. + +--- + +## → Start here: [`APOCALYPSE-EDITION.md`](APOCALYPSE-EDITION.md) + +The full from-scratch build guide: hardware shopping list, operating-system setup, freeing the +dongles from the TV driver, researching *your* crossing's channels, bringing up the container +stack step by step, transcription, and the hard-won tuning lessons. It's written to be read +straight through by a human **or** handed to an LLM coding agent pointed at this repo. + +## What's here + +| Path | What it is | +|---|---| +| [`APOCALYPSE-EDITION.md`](APOCALYPSE-EDITION.md) | the complete build guide (read this first) | +| `scripts/` | the signal engine + decoders — stdlib Python + NumPy only | +| `scripts/iq_hop.py`, `iq_channelize.py` | the retune-in-place 452⇄457 hop and the sub-channel splitter (the mission radio) | +| `scripts/eot/` | the two-pass FFSK End-of-Train decoder (drift-tolerant; validates via vendored [PyEOT](https://github.com/ereuter/PyEOT)) | +| `scripts/bot-recover.py` | Head-of-Train frame recovery + head/tail join | +| `scripts/transcribe-worker.py` | voice clip → comms filter → Whisper → transcript | +| `dashboard/` | `serve.py` (the API + static server, stdlib, no framework) + the web UI | +| `docker/` | Compose template + Dockerfiles (web / airband / scanhop / worker) | +| `config/` | `istrain.conf.example` (your channels go here) + the DVB-blacklist file | + +## The idea in three transmitters + +1. **Voice (160–162 MHz):** dispatchers, crews, and defect detectors that read out milepost, axle + count, and speed in plain English. The most direct "a train just passed here" signal. +2. **End-of-Train (457.9375 MHz):** the last car's telemetry box — unit ID, **brake-pipe pressure**, + **motion flag** — a 1200-baud FFSK burst every few seconds. The brake-pressure curve tells you + passing vs. dwelling vs. cut-and-standing vs. departing. +3. **Head-of-Train (452.9375 MHz):** the locomotive's half; decode it and join head to tail for a + confirmed complete train. + +Any one can be too weak to read, but a real train lights several bands at once — so istrain +correlates across all of them. + +## License & credits + +**GPLv3** — see [`LICENSE`](LICENSE). The EOT decoder validates frames against +[PyEOT](https://github.com/ereuter/PyEOT) by Eric Reuter (GPLv3, vendored under `scripts/eot/`); +full attribution and the licensing note for forkers are in [`ATTRIBUTION.md`](ATTRIBUTION.md). The +communities and tools that made this possible — RTL-SDR Blog, osmocom rtl-sdr, RTLSDR-Airband, +faster-whisper / OpenAI Whisper, FFmpeg, NumPy, Docker + Dockge, and the railfan frequency +databases — are credited in the dashboard's **With Thanks** panel. + +*A community project. If you build one, we'd love to hear what you heard.* diff --git a/config/blacklist-rtl-sdr.conf b/config/blacklist-rtl-sdr.conf new file mode 100644 index 0000000..7030943 --- /dev/null +++ b/config/blacklist-rtl-sdr.conf @@ -0,0 +1,7 @@ +# istrain — keep the kernel DVB-T driver off the RTL-SDR dongles so SDR apps (rtl_airband, rtl_fm) own +# them via libusb. Without this, dvb_usb_rtl28xxu claims a dongle at boot and rtl_airband hangs on +# "Allocating zero-copy buffers" (rtl_fm auto-detaches, rtl_airband does not). These dongles are SDR-only. +# +# Deploy (sudo): sudo cp blacklist-rtl-sdr.conf /etc/modprobe.d/ && sudo modprobe -r dvb_usb_rtl28xxu +# (the modprobe -r frees it for THIS session without a reboot; the file keeps it off after reboot) +blacklist dvb_usb_rtl28xxu diff --git a/config/istrain.conf.example b/config/istrain.conf.example new file mode 100644 index 0000000..80e8067 --- /dev/null +++ b/config/istrain.conf.example @@ -0,0 +1,70 @@ +# RTLSDR-Airband — istrain: an example voice-channel cluster (replace with YOUR subdivision — see APOCALYPSE-EDITION.md §3) +# +# Run in the foreground with live per-channel waterfalls: +# rtl_airband -f -e -c config/istrain.conf +# (-f = foreground + textual waterfalls · -e = errors to stderr · Ctrl-C to stop) +# +# One dongle (1002, Blog V4, TCXO). Five rail-voice channels demodulated SIMULTANEOUSLY (no +# scanning/camping — unlike the rtl_fm scan). The span 160.86–161.19 = 330 kHz fits inside a +# 1.024 MS/s window, which stays UNDER the USB sample-loss ceiling that bit the full 2 MS/s scan. +# Each channel records to its own file in /tmp/airband, and ONLY when squelch opens (continuous=false) +# — so the files stay small and hold real audio, not hiss. +# +# Primary (example) = 160.980 (AAR ch58), the road channel + the CJ 144.7 / 157.8 detectors. See CHANNELS.md. +# Tune squelch_snr_threshold up if it records noise, down if it misses weak transmissions. + +fft_size = 1024; +localtime = true; + +devices: +({ + type = "rtlsdr"; + serial = "1002"; + gain = 40; + centerfreq = 161.025; + sample_rate = 1.024; + correction = 0; + channels: + ( + { + freq = 160.980; + label = "NS Dayton Rd 160.980 ch58 *MISSION*"; + modulation = "nfm"; + bandwidth = 12500; + squelch_snr_threshold = 4.0; + outputs: ( { type = "file"; directory = "/tmp/airband"; filename_template = "ns_dayton_road_160980"; continuous = false; split_on_transmission = true; } ); + }, + { + freq = 160.920; + label = "NS Buckeye Yard 160.920 ch54"; + modulation = "nfm"; + bandwidth = 12500; + squelch_snr_threshold = 4.0; + outputs: ( { type = "file"; directory = "/tmp/airband"; filename_template = "ns_buckeye_yard_160920"; continuous = false; split_on_transmission = true; } ); + }, + { + freq = 161.190; + label = "NS Col+Sandusky 161.190 ch72"; + modulation = "nfm"; + bandwidth = 12500; + squelch_snr_threshold = 4.0; + outputs: ( { type = "file"; directory = "/tmp/airband"; filename_template = "ns_col_sandusky_161190"; continuous = false; split_on_transmission = true; } ); + }, + { + freq = 160.860; + label = "CSX Scottslawn 160.860 ch50"; + modulation = "nfm"; + bandwidth = 12500; + squelch_snr_threshold = 4.0; + outputs: ( { type = "file"; directory = "/tmp/airband"; filename_template = "csx_scottslawn_160860"; continuous = false; split_on_transmission = true; } ); + }, + { + freq = 161.085; + label = "NS Col area 161.085 ch65"; + modulation = "nfm"; + bandwidth = 12500; + squelch_snr_threshold = 4.0; + outputs: ( { type = "file"; directory = "/tmp/airband"; filename_template = "ns_col_area_161085"; continuous = false; split_on_transmission = true; } ); + } + ); +}); diff --git a/dashboard/checklist.md b/dashboard/checklist.md new file mode 100644 index 0000000..daf2ae1 --- /dev/null +++ b/dashboard/checklist.md @@ -0,0 +1,41 @@ +# iSTRAIN — build stages + +**Primary goal:** detect a train blocking *your* crossing, from passive RF alone. +**Secondary:** capture and learn — radios, trains, hardware, software. Not a toy. +This file renders in the dashboard's checklist panel; edit it to track your own build. +Full instructions: **APOCALYPSE-EDITION.md**. + +## The plan · two radios, two bands +- **Radio A (a Blog V4 is ideal) → rail VHF voice 160–162 MHz** — RTLSDR-Airband demodulates your + pinned road/detector channels at once (your subdivision's AAR channel is the mission). +- **Radio B (any RTL-SDR) → EOT-system UHF hop** — 452.9375 (head/BOT) ⇄ 457.9375 (EOT/tail), + retune-in-place; ±12.5 kHz DPU neighbors watched inside each dwell. + +## Stage 0 · Host & dongles +- [ ] Linux host with Docker; base tools installed (rtl-sdr, ffmpeg, python3-numpy) +- [ ] DVB driver blacklisted on the host (`config/blacklist-rtl-sdr.conf`); `lsmod | grep dvb` empty +- [ ] Dongles serialized (`rtl_eeprom -s …`) so services pick them by name +- [ ] NOAA weather RX proves the antenna→dongle→software chain end to end + +## Stage 1 · Your crossing +- [ ] Railroad + subdivision identified; road channel + neighbors found (cross-checked on 2 sources) +- [ ] Nearby defect-detector mileposts noted (your early-warning tripwires) +- [ ] `config/istrain.conf` edited with YOUR voice channels + +## Stage 2 · Bring-up +- [ ] DSP selftests pass (`iq_hop.py --selftest`, `iq_channelize.py --selftest`) +- [ ] Stack builds and runs (`docker compose --profile radio --profile radio1001 up -d`) +- [ ] `level.json` refreshing; clips landing; dashboard live at `:3456` +- [ ] First EOT decode on a real train (unit ID + brake psi + motion) 🎉 + +## Stage 3 · Ears & polish +- [ ] Whisper transcription wired (comms band-pass first, VAD on — see APOCALYPSE §7) +- [ ] Squelch tuned to your noise floor (not a textbook number — APOCALYPSE §8) +- [ ] Standing carriers (fixed wayside boosters) identified and carded, not chased + +## Stage 4 · If you expose it publicly +- [ ] Read APOCALYPSE §10 (security): presentation-node read-only mode, no PUSH_KEY, proxy auth +- [ ] Antenna moved high and away from your computers — the #1 range lever + +## The habit that makes it work +- [ ] Keep one append-only, numbered observations log. Record findings AND retractions. diff --git a/dashboard/index.html b/dashboard/index.html new file mode 100644 index 0000000..7c16035 --- /dev/null +++ b/dashboard/index.html @@ -0,0 +1,300 @@ + + + + + +iSTRAIN — railroad VHF watch + + + +
+

🚆 iSTRAIN Lassiter Creek · — · railroad VHF 160–162 MHz

+
+
+ + +
+
+
—
unique
+
—
total hits
+
+
+
+ + +
+ +
+ +
+

Dispatch / Yard rail-voice captures — the "AIS" for trains

+
+ Airband is watching 5 channels on 1002. Captured transmissions (squelch-gated clips → + whisper transcripts) will list here newest-first: channel, time, and the milepost / axle-count / + speed read-outs from detector call-outs. ★ 160.980 NS Dayton District is the mission channel.
+ Live clips appear in the Captures panel (one per transmission, playable); + transcription (whisper) still pending.
+ scan status: service istrain-airband — live status & controls in the Supervisor panel below +
+
+ + + + +
+ + + + +
+ + +
+ + +
+ + +
+ + +
+ + +
+

Defect Detectors reference — your subdivision (example)

+
+ Wayside detectors near your crossing (example: MP 148.99), all on your mission channel (example 160.980). The + nearest talks on every train — its call-out is the unambiguous "train through" event.
+ • CJ 146.4 ★ — nearest, ~2.6 mi east (New Rome OH). HBD-DED · talks every train + (Talk-On-Defect-Only: No) → routine call-out on westbounds · record
+ • CJ 144.7 — ~4.3 mi east, high-car (over-height) — defect-only
+ • CJ 157.8 — ~8.8 mi west, warns eastbounds — type TBD
+ Source: defectdetector.net — static records (location/type/freq/audio), pullable per detector by id. One-time import, not a live poller. +
+
+ + +
+

Community Feeds live audio — when up

+
+ • Broadcastify — South Central Ohio Railroads + — the one Franklin County rail feed; monitors our 160.980 (independently validates the channel). Often offline, ~0 listeners.
+ Their rooftop hears more than our desk whip — "they catch it, we don't" = range, not broken. Someday: we become the local contributor. +
+
+ + +
+ + +
+

Signal Flow how it fits together — the air ▸ the dongles ▸ meaning

+
+iSTRAIN — SIGNAL FLOW        the air ▸ the dongles ▸ meaning
+mission: is a train blocking YOUR crossing? (set your channels in config/istrain.conf)
+legend:  ★ TCXO (precise)   [nnnn] = dongle serial   ─▶ data path
+updated 2026-06-28: voice live (1002); 1001 WEIGHTED hop 452/457 (clock-step-proof); EOT decoded LIVE (ID+brake+motion); HOT head-end ADDRESS cracked → head+tail join; MID standing carrier ID'd = 457.925 EOT booster, carded. ATCS dropped.
+updated 2026-07-05: EOT decode is TWO-PASS — off-frequency emitters (±800 Hz, ~100 ms bursts) now decode
+  via adaptive tones + fuzzy sync (O-11) → dwelling trains identified w/ brake psi: held ~60 = stopped
+  solid · bled to 0 + head polling = CUT & standing (crossing possibly open). Booster floor subtracted
+  from contacts & correlate (O-9).
+updated 2026-07-18: THE WHOLE RIG MOVED TO THE MILL — both dongles + capture + decode + transcription
+  + this site now run as one Docker (Dockge) stack on a TrueNAS box; the old VM runs zero radio
+  services; data is born where it's served. Voice transcription is LIVE (not planned — see below).
+────────────────────────────────────────────────────────────────────
+
+PRIMARY ░░ RAIL VHF VOICE — 160–162 MHz · AAR · NBFM, unencrypted ░░
+ └ defect detectors ("MP 123, no defects, 45 axles, 60 mph") + dispatcher
+    ─▶ RTLSDR-Airband (multi-ch) ─▶ ffmpeg comms-filter ─▶ faster-whisper (small.en + VAD,
+       LIVE on the Mill since 07-10) ─▶ transcripts.jsonl — "a train is moving through," in words
+    ▸ the most direct, gettable "is MY crossing blocked" signal
+    ▸ proven 2026-06-16: same band as NOAA 162.550 — RX confirmed here
+    ▸ CONFIRMED channel (2026-06-18): 160.980 (ch58) NS Dayton District, MP 148.99;
+      detectors CJ 144.7 / 157.8 announce here. Plan + sources → CHANNELS.md
+    ▸ first intelligible speech 2026-07-10, the night the tuned antenna went in:
+      detector CJ at MP 146.4, transcribed (O-22)
+    stick: [1002 ★] Blog V4 — Airband demods the 5 pinned channels at once (160.86–161.19 @ 1.024 MS/s)
+    antenna: tuned railroad-band vertical (upgraded 07-10 — ≈2.4× the desk whip's real content;
+             office placement, rooftop mast = the standing next lever)
+
+SECONDARY ░░ EOT-SYSTEM UHF — 1001 WEIGHTED HOP (452 ⇄ 457) · 1200-baud FFSK ░░
+ └ a train is a chain of RF emitters front-to-back; 1001 hops to watch both ends:
+    ▸ 457.9375  EOT/FRED — the REAR unit; DECODED live: ID · brake · MOTION (rolling/stopped) ─▶ eot.jsonl + clips
+                decode is TWO-PASS (07-05): exact fast path + adaptive/fuzzy escalation for the
+                off-frequency short-burst emitters that used to read "strong but undecodable" (O-11)
+    ▸ 452.9375  BOT / head-end (HOT) — the FRONT unit; frame RECOVERED + addressed-unit DECODED ─▶ bot.jsonl
+    ▸ hop: iq_hop.py — retune-in-place, BOT 10s / EOT 10s (rebalanced 07-07: the head = the join + the EARLY WARNING; was 7/14 EOT-favored before the HOT decoder existed)
+    ▸ EOT decode LIVE 2026-06-27 — real trains identified; brake-release + motion=1 = departed (crossing cleared)
+    ▸ HOT address CRACKED 2026-06-28 from our own air (invert · sync 0110101010 · bit-49) → head↔tail JOIN:
+      a HOT frame addressing a card's own EOT = confirmed complete consist. Command/type + BCH still open.
+    stick: [1001] generic — drifts, ppm-calibrate for a clean FSK decode; 450 band clean (no filtering)
+    antenna: ~6" vertical — ¼λ ≈ 6.4" @ 457, well-matched
+    ▸ MID / DPU — ±12.5k off both centers (452.925/.950, 457.925/.950), watched in-hop → midtrain.jsonl
+ found ░ a STANDING carrier on 457.925 = an FCC railroad EOT SIGNAL-BOOSTER (fixed wayside repeater, not a
+         train); carded as a STANDING SIGNAL. A booster here ⇒ marginal head↔tail RF — likely why our range is short.
+
+SYNTHESIS ░░ the mission is DETECT a train, not decode one signal ░░
+ └ a train lights several bands at once — co-firing in the same 15-min slot = a candidate,
+    even when each is too weak to decode:  160.980 voice + 452 BOT + 457 EOT (+ ±12.5k DPU,
+    booster-floor subtracted so the fixed carrier can't fake a band — O-9)
+    ▸ correlate panel = week heatmap (one row/band, aligned bins) — read DOWN a column
+    ▸ BOT/mid/EOT strips below = the RECENT ZOOM (1-min bins, last 6 h) — correlate's magnifier
+    ▸ live meter = bouncing dBFS bar for whichever band the hop is on (level.json)
+
+MEANING ░░ the AIR CURVE is the state — psi tells the story silence can't (O-10 · O-11) ░░
+ └ from decoded EOT frames, per train:
+    psi ~85–90 + moving            = PASSING through
+    psi drops to a HELD ~55–65 + motion 0 = DWELLING (brakes set — possibly blocking)
+    psi bleeds to 0 AND HOLDS + head still polling = CUT & STANDING — crews cut the crossing
+       (eyes-confirmed 07-05: ~11 h split, crossing open; never read the bleed as a departure)
+    psi RECHARGES toward ~88 + motion 0→1 = DEPARTED (the roll-out is the event time)
+
+sanity ░ NOAA 162.550 (Columbus) — the proven known-good RX test on this band
+         (the per-radio toggle is parked since the Mill move; control relay TBD)
+config ░ radio→task is swappable, not hardcoded — reassign per band
+dropped ░ ATCS (900 MHz): LOS-limited + FCC-retired Sep 2025 → encrypted PTC;
+          our 900 "carriers" were a dongle spur. Why → radio/sdr/atcs/tuning-log.md
+────────────────────────────────────────────────────────────────────
+why voice first: unencrypted, easy at 160 MHz, and it says a train is
+moving through your crossing in plain words — the direct commute signal.
+
+
+ + +
+

System Map the istrain dashboard & the services behind it

+
+iSTRAIN — SYSTEM MAP        the dashboard & the services behind it
+how the pieces talk · keep current as it changes
+legend:  ─▶ request / data    ◂ produced by    ▣ = a container in the Dockge stack `istrain`
+updated 2026-07-18 (the Mill cutover): ONE BOX — everything below is a container on the Mill
+  (a TrueNAS SCALE machine running Dockge); both dongles on its USB; every log AUTHORED where
+  it's served (the VM-era push pipeline is retired); istrain.jhestyr.net serves this stack.
+────────────────────────────────────────────────────────────────────
+
+BROWSER ▸ istrain.jhestyr.net   (Cloudflare ─▶ the Mill's ▣ web container)
+ │
+ └─▶ ▣ web    serve.py — stdlib Python, no framework
+       │      (keeps a jsonl PARSE CACHE keyed on file mtime/size + a 4 s TTL on the card
+       │       compute — polls are ~free; the logs only change every ~20-40 s)
+       ├ static  index.html · main.js · style.css · panels/strip.js (TWO grids: aligned week/15-min
+       │         for correlate · RECENT1M = 6h/1-min zoom for the band strips)
+       │         panels/{trains,live,supervisor,correlate,bot,midtrain,eot,checklist}
+       ├ GET  /api/version    ◂ dashboard/VERSION (stamped by mill/deploy.sh)  ─▶ header build stamp
+       ├ GET  /api/trains     ◂ eot.jsonl + bot.jsonl (cards: tail+head ✓join, ⚓parked, ◉standing,
+       │                        air-curve states: passing/dwelling/CUT&standing/departed) ─▶ Trains
+       ├ GET  /api/istrain    ◂ strength-gated isTRAIN? verdict (+ dwelling-join fallback) ─▶ Banner
+       ├ GET  /api/level      ◂ level.json   (live dBFS)                 ─▶ Live meter (2×/s)
+       ├ GET  /api/heard      ◂ heard.jsonl  (voice detections)          ─▶ Correlate (voice row)
+       ├ GET  /api/eot?hours=6      ◂ eot.jsonl      ─▶ EOT zoom strip   (no ?hours = week ─▶ Correlate)
+       ├ GET  /api/bot?hours=6      ◂ bot.jsonl      ─▶ BOT zoom strip   (ditto)
+       ├ GET  /api/midtrain?hours=6 ◂ midtrain.jsonl ─▶ Mid-train zoom strip (±12.5k DPU + booster)
+       ├ GET  /api/correlate  ◂ scripts/correlate.py (recent-window why; booster-floor aware) ─▶ Correlate
+       ├ GET  /api/supervisor ◂ docker state via supervisor-probe (host cron, 1/min) + DATA-FLOW
+       │                        liveness: hop = level.json freshness · voice = last-capture age
+       └ POST /api/noaa · /api/supervisor/{ctl,reset}  (parked since the move — buttons decline
+                                                        gracefully; a docker control relay is TBD)
+
+DONGLES ░ USB by serial, both on the Mill through a hub · dvb driver blacklisted on the host
+ ├ 1001 ▸ UHF hop — 452(BOT) ⇄ 457(EOT)    └ 1002 ★ ▸ rail voice 160–162 / NOAA
+
+CAPTURE ░ voice + the UHF hop run at once — separate dongles, separate containers
+ ├ ▣ airband  rtl_airband v5.2.0 (rtl-sdr-blog fork, V4-capable) -c istrain.conf
+ │            ─▶ 5 NFM channels on 1002 ─▶ captures/*.mp3 ─▶ heard.jsonl
+ └ ▣ scanhop  rtl_tcp + iq_hop.py: retune-in-place hop on 1001 (BOT 10s ⇄ EOT 10s, monotonic dwell — step-proof)
+              ├ 452 ─▶ bot.jsonl + bot_clips/*.s16  (presence + clip)
+              ├ 457 ─▶ eot.jsonl + eot_clips/*.s16  (presence + clip)
+              └ both ─▶ level.json                  (live meter + the hop's liveness signal)
+
+DECODE ░ containers · no radio · CPU on saved clips · NEVER cull
+ ├ ▣ eot-decode   ─▶ eot-decode.py → eot_scan (TWO-PASS: exact + adaptive/fuzzy for off-freq
+ │                   emitters, O-11) → unit·brake·motion (+dq provenance) → enrich eot.jsonl
+ ├ ▣ bot-recover  ─▶ bot-recover.py: BOT clips → recover HOT frame + DECODE addressed unit → enrich bot.jsonl
+ └ ▣ transcribe-worker ─▶ voice clips >5 KB ─▶ ffmpeg comms-filter ─▶ the whisper stack
+       (faster-whisper small.en + VAD, a sibling Dockge stack, localhost) ─▶ sidecar .txt per clip;
+       real speech only ─▶ transcripts.jsonl (durable)
+
+HOUSEKEEPING ░ ▣ trim — 24 h voice-audio prune (ingest-gated; heard.jsonl KEPT)
+              ░ supervisor-probe — host cron, 1/min: docker truth into the snapshot
+DATA ░ born on the Mill's ZFS datasets — jsonl logs = keep forever · clips accumulate
+       (keep-everything standing policy) · the VM-era corpus is archived alongside
+
+build vs operate ░ the hands build the code; the radios are operated by services —
+  never fired ad hoc. deploy = mill/deploy.sh: rsync ─▶ build ─▶ up ─▶ smoke, idempotent.
+history ░ the VM-capture / NAS-web era (2026-06-15 → 07-18) is preserved in MIGRATION.md;
+  the VM units stay on disk, disabled — the standing rollback.
+────────────────────────────────────────────────────────────────────
+
+
+
+ + +
+
📡 WITH THANKS
+

Railroad frequencies, the tools that pull signal out of the air, and the hands that built it.

+

Frequency & data communities
+ • Columbus Railfan Online (Silver Rails / TrainWeb) — scanner frequencies
+ • RadioReference — NS Columbus Terminal & the US Railroads index
+ • Railroad-Frequencies.com — NS & CSX Ohio assignments +

+

Tools & projects
+ • RTL-SDR Blog — the Blog V4 hardware and the community behind it
+ • Osmocom rtl-sdr — rtl_test / rtl_eeprom, the tools that drive the dongles
+ • RTLSDR-Airband — the multi-channel voice watch (we run the rtl-sdr-blog fork, V4-capable)
+ • faster-whisper (SYSTRAN) + OpenAI Whisper — the transcription engine that lets istrain hear (small.en + VAD)
+ • whisper.cpp — lit the transcription path first, in the VM era
+ • FFmpeg — the comms-filter in front of the ears, and every clip's plumbing
+ • AIS-catcher — AIS decode, our known-good validation signal
+ • PyEOT (Eric Reuter) — the open-source EOT/FFSK decoder (packet parser + BCH check) that proved we can read End-of-Train telemetry; ships the demo recording we validated against
+ • SigidWiki — EOTD & rtl-sdr.com EOT guides — the 1200-baud FFSK spec and the decode trail that pointed the way
+ • NumPy — the signal-crunching behind the EOT demodulator
+ • ATCS Monitor & Rail Watch Monitor — the 900 MHz dispatcher's board, and the community that mapped ATCS (evaluated & dropped — kept for provenance)
+ • RadioReference ATCS DB & SigidWiki — the published channel plans that told us what we'd found
+ • Docker + Dockge (louislam) — the runtime every service lives in since the Mill move
+ • Tailscale — the quiet plumbing between the boxes
+ • Cloudflare — the front door +

+

The machine
+ • TrueNAS SCALE on the Mill — the one box the whole rig runs on (since 2026-07-18): dongles, capture, decode, ears, and this site
+ • Oracle VM VirtualBox + Linux Mint (LMDE) — the dev seat, and the box the rig grew up inside (2026-06-15 → 07-18) +

+

Foundation
+ • built on a reusable radio-capture engine and iterated with an AI pair-programmer (Claude, Anthropic)
+ • a community project — the credits above are the shoulders it stands on; if your tool, frequency, or hand helped and isn’t here yet, it will be. +

+

A living list
+ istrain is a community project — this list is intentionally unfinished and grows as the work brings in + more shoulders to stand on. If your tool, frequency, or hand helped and isn't here yet, it will be. +

+
+ + +
+ + +
+ + +
+ +
+ live — one box: the Mill captures, decodes & presents · + istrain.jhestyr.net · an open community project +
+ + + + diff --git a/dashboard/main.js b/dashboard/main.js new file mode 100644 index 0000000..aea1829 --- /dev/null +++ b/dashboard/main.js @@ -0,0 +1,96 @@ +// main.js — istrain dashboard. Frame chrome (hit counter, refresh / hard-reset) + mounts live panels. +// Panels follow the engine's panels//mount(el) contract (see the radio engine). Static placeholder +// panels (dispatch, detect) stay in index.html until built. +// +// Refresh policy: slow background polls + change-aware renders so the page idles at ~0% CPU (no +// compositor stutter). Manual "refresh" re-pulls now; "hard reset" kills captures + restarts all helpers (/api/supervisor/reset). + +import { mountSupervisor } from '/panels/supervisor/supervisor.js'; +import { mountChecklist } from '/panels/checklist/checklist.js'; +// captures RESTORED 2026-07-10 — the new antenna arrived + whisper transcripts went live; the public +// node gets voiced clips pushed. activity stays SHELVED (voice activity = a correlate-graph signal). +import { mountCaptures } from '/panels/captures/captures.js'; +import { mountEot } from '/panels/eot/eot.js'; +import { mountCorrelate } from '/panels/correlate/correlate.js'; +import { mountBot } from '/panels/bot/bot.js'; +import { mountMidtrain } from '/panels/midtrain/midtrain.js'; +import { mountLive } from '/panels/live/live.js'; +import { mountTrains } from '/panels/trains/trains.js'; +import { mountBanner } from '/panels/banner/banner.js'; +import { mountErrors } from '/panels/errors/errors.js'; + +// --- client-side error reporter → the Errors card (rate-limited + escaped server/panel side) --- +function reportErr(msg, src, line) { + try { + fetch('/api/errors/report', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ msg: String(msg).slice(0, 500), src: String(src || '').slice(0, 200), line: line || 0 }), + }).catch(() => {}); + } catch { /* the reporter must never itself throw */ } +} +window.addEventListener('error', e => reportErr(e.message, e.filename, e.lineno)); +window.addEventListener('unhandledrejection', e => reportErr('unhandledrejection: ' + ((e.reason && e.reason.message) || e.reason), location.pathname, 0)); + +// --- hit counter (change-aware; shows "—" until the server exists) --- +let lastHits = null; +async function loadHits() { + try { + const r = await fetch('/api/hits', { cache: 'no-store' }); + if (!r.ok) return; + const { unique, total } = await r.json(); + const key = unique + '/' + total; + if (key === lastHits) return; // no change → no DOM write + lastHits = key; + if (unique != null) document.getElementById('hits-unique').textContent = unique.toLocaleString(); + if (total != null) document.getElementById('hits-total').textContent = total.toLocaleString(); + } catch { /* no server yet — leave the dashes */ } +} + +// --- build stamp: the commit this dashboard serves (VM: git; NAS: VERSION stamped by fetch-app.sh) --- +async function loadVersion() { + try { + const r = await fetch('/api/version', { cache: 'no-store' }); + if (!r.ok) return; + const { commit } = await r.json(); + if (commit) document.getElementById('ver').textContent = commit; + } catch { /* leave the dash */ } +} + +// --- unique visitor: a persistent client token (robust behind the tunnel, where the real IP collapses) --- +function recordVisit() { + let vid = localStorage.getItem('istrain_vid'); + if (!vid) { + vid = (window.crypto && crypto.randomUUID) ? crypto.randomUUID() + : (Date.now().toString(16) + '-' + Math.floor(Math.random() * 0xffffffff).toString(16)); + localStorage.setItem('istrain_vid', vid); + } + fetch('/api/hit?vid=' + encodeURIComponent(vid), { cache: 'no-store' }).catch(() => {}).finally(loadHits); +} + +// --- live panels --- +const supervisor = mountSupervisor(document.getElementById('supervisor')); +const checklist = mountChecklist(document.getElementById('checklist')); +const banner = mountBanner(document.getElementById('istrain-banner')); +const trains = mountTrains(document.getElementById('trains')); +const eot = mountEot(document.getElementById('eot')); +const correlate = mountCorrelate(document.getElementById('correlate')); +const bot = mountBot(document.getElementById('bot')); +const midtrain = mountMidtrain(document.getElementById('midtrain')); +const live = mountLive(document.getElementById('live')); +const captures = mountCaptures(document.getElementById('captures')); +const errors = mountErrors(document.getElementById('errors')); + +// --- controls --- +document.getElementById('btn-refresh').onclick = () => { loadHits(); banner.refresh(); supervisor.refresh(); checklist.refresh(); trains.refresh(); eot.refresh(); correlate.refresh(); bot.refresh(); midtrain.refresh(); errors.refresh(); captures.refresh(); }; +document.getElementById('btn-reset').onclick = async () => { + if (!confirm('Hard reset — kill the captures and restart all istrain helpers? Frees the dongles; device-level DVB resets still need cmd-palette.')) return; + const b = document.getElementById('btn-reset'), t = b.textContent; + b.disabled = true; b.textContent = 'resetting…'; + try { await fetch('/api/supervisor/reset', { method: 'POST', headers: { 'X-Ops': '1' } }); } catch {} + setTimeout(() => { b.disabled = false; b.textContent = t; supervisor.refresh(); eot.refresh(); correlate.refresh(); }, 3500); +}; + +// initial pull + a slow safety poll (manual refresh drives the rest) +recordVisit(); // beacon once per load (mints/sends the visitor token), then shows counts +loadVersion(); // header build stamp — once per load +setInterval(loadHits, 600000); // 10 min diff --git a/dashboard/panels/strip.js b/dashboard/panels/strip.js new file mode 100644 index 0000000..429267d --- /dev/null +++ b/dashboard/panels/strip.js @@ -0,0 +1,127 @@ +// strip.js — shared "week of 15-minute bins" strip, used by EVERY band panel (EOT/BOT/mid-train/voice and +// the correlation synthesis). One source of truth for binning so their columns LINE UP: bins are keyed to +// absolute wall-clock 15-min slots (floor(ts/900)), not relative-to-now — so the same 15-min slot lands in +// the same column on every panel. Shared canvas draw + one hover-tooltip format (date + time range, ET). + +export const BUCKET = 900, WEEK = 7 * 86400, NB = Math.round(WEEK / BUCKET); // 672 — a week of 15-min bins +export const TZ = 'America/New_York'; +// two grids (2026-07-05): the shared WEEK grid keeps every panel's columns aligned (correlate's job — +// the overview); RECENT1M is the zoom the band strips use so the most recent stretch of correlate can +// be inspected at 1-min resolution (360 bins ≈ same canvas cost as the week's 672). +export const WEEK15 = { bucket: BUCKET, nb: NB, note: 'week · 15-min bins' }; +export const RECENT1M = { bucket: 60, nb: 360, note: 'last 6h · 1-min bins' }; + +// absolute slot alignment — rightmost bin (nb-1) is the current slot +export function binIndex(ts, now, s = WEEK15) { + if (!ts || ts <= 0) return -1; + const idx = s.nb - 1 - (Math.floor(now / s.bucket) - Math.floor(ts / s.bucket)); + return (idx < 0 || idx > s.nb - 1) ? -1 : idx; +} +export function fillBins(events, now, getTs = e => e.ts, s = WEEK15) { + const b = new Int16Array(s.nb); + for (const e of events) { const i = binIndex(getTs(e), now, s); if (i >= 0) b[i]++; } + return b; +} +// wall-clock window [start,end) covered by bin idx — for tooltips +export function binTimes(idx, now, s = WEEK15) { + const start = (Math.floor(now / s.bucket) - (s.nb - 1 - idx)) * s.bucket; + return { start, end: start + s.bucket }; +} + +export function drawStrip(canvas, buckets, rgb, gmaxOverride) { + const w = Math.max(1, canvas.clientWidth | 0), h = (canvas.clientHeight | 0) || 32; + canvas.width = w; canvas.height = h; + const ctx = canvas.getContext('2d'); + ctx.clearRect(0, 0, w, h); + const gmax = gmaxOverride || Math.max(1, ...buckets); + const N = buckets.length; // works for either grid + for (let x = 0; x < w; x++) { + const b0 = Math.floor(x / w * N), b1 = Math.max(b0 + 1, Math.floor((x + 1) / w * N)); + let v = 0; + for (let b = b0; b < b1 && b < N; b++) if (buckets[b] > v) v = buckets[b]; + if (!v) continue; + const t = v / gmax, bh = Math.max(1, Math.round(t * h)); + ctx.fillStyle = `rgba(${rgb},${(0.4 + 0.6 * t).toFixed(3)})`; + ctx.fillRect(x, h - bh, 1, bh); + } +} + +const fmtStart = t => new Date(t * 1000).toLocaleString([], { month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit', timeZone: TZ }); +const fmtEnd = t => new Date(t * 1000).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit', timeZone: TZ }); + +// Shared hover tooltip (reuses .act-tip). resolve(ev) -> {canvas, buckets, now, unit} | null. +// Default-to-LEFT of cursor (busy bins sit at the right edge, where a right popup gets clipped). +export function attachTip(el, bodyEl, resolve) { + const tip = document.createElement('div'); + tip.className = 'act-tip'; tip.style.display = 'none'; + el.appendChild(tip); + bodyEl.addEventListener('mousemove', ev => { + const r = resolve(ev); + if (!r) { tip.style.display = 'none'; return; } + const s = r.scale || WEEK15; + const rect = r.canvas.getBoundingClientRect(); + let idx = Math.floor((ev.clientX - rect.left) / Math.max(1, rect.width) * s.nb); + idx = Math.max(0, Math.min(s.nb - 1, idx)); + const { start, end } = binTimes(idx, r.now, s); + const label = r.label ? r.label(idx) : `${r.buckets[idx]} ${r.unit || 'tx'}`; + tip.textContent = `${label} · ${fmtStart(start)}–${fmtEnd(end)} ET`; + tip.style.display = 'block'; + const gap = 12; + let lx = ev.clientX - gap - tip.offsetWidth; + if (lx < 4) lx = ev.clientX + gap; + tip.style.left = lx + 'px'; + tip.style.top = Math.max(4, ev.clientY - 30) + 'px'; + }); + bodyEl.addEventListener('mouseleave', () => { tip.style.display = 'none'; }); +} + +// Factory for a single-band PRESENCE panel (EOT/BOT/mid-train). opts: +// {id, title, tag, freqTag, rgb, endpoint, nearSec, nearLabel, emptyHtml, scale?} +// scale defaults to the shared week grid; pass RECENT1M for the 1-min zoom (endpoint should then +// carry ?hours=6 so the server only ships the window being drawn). +export function mountBandStrip(el, o) { + const scale = o.scale || WEEK15; + el.innerHTML = + `

${o.title} ${o.tag}

`; + const tagEl = el.querySelector(`#${o.id}-tag`); + const bodyEl = el.querySelector(`#${o.id}-body`); + let lastKey = null, lastBuckets = null, lastNow = 0; + const fmtT = e => new Date(e * 1000).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit', timeZone: TZ }); + + const paint = () => { + const cv = bodyEl.querySelector('canvas.eot-strip'); + if (cv && lastBuckets) drawStrip(cv, lastBuckets, o.rgb); + }; + attachTip(el, bodyEl, ev => { + const cv = ev.target.closest && ev.target.closest('canvas.eot-strip'); + return (cv && lastBuckets) ? { canvas: cv, buckets: lastBuckets, now: lastNow, unit: 'bursts', scale } : null; + }); + + async function refresh() { + let d = { bursts: [], total: 0 }; + try { const r = await fetch(o.endpoint, { cache: 'no-store' }); if (r.ok) d = await r.json(); } + catch { return; } + const bursts = d.bursts || [], now = Date.now() / 1000, w = bodyEl.clientWidth | 0; + const newest = bursts.length ? bursts[0].ts : 0; + const key = (d.total || 0) + '|' + newest + '|' + w + '|' + Math.floor(now / scale.bucket); + if (key === lastKey) return; + lastKey = key; lastNow = now; + lastBuckets = fillBins(bursts, now, e => e.ts, scale); + tagEl.textContent = d.total ? `${o.freqTag} · ${d.total} bursts · ${scale.note}` : o.tag; + if (!bursts.length) { bodyEl.innerHTML = `
${o.emptyHtml}
`; return; } + const last = newest, hr = bursts.filter(b => now - b.ts <= 3600).length, near = last && (now - last <= o.nearSec); + bodyEl.innerHTML = + `
` + + `${near ? o.nearLabel : 'quiet'}` + + `last ${fmtT(last)} ET` + + `${hr}× last hr` + + `
`; + requestAnimationFrame(paint); + } + + refresh(); + setInterval(refresh, 30000); + let rt; + window.addEventListener('resize', () => { clearTimeout(rt); rt = setTimeout(() => { lastKey = null; refresh(); }, 250); }); + return { refresh }; +} diff --git a/dashboard/radios.json b/dashboard/radios.json new file mode 100644 index 0000000..ae3e707 --- /dev/null +++ b/dashboard/radios.json @@ -0,0 +1,23 @@ +{ + "generated": "2026-06-18T15:36:59+00:00", + "radios": [ + { + "serial": "1001", + "product": "RTL2838UHIDIR", + "manufacturer": "Realtek", + "role": "EOT/BOT \u00b7 452/457", + "driver": null, + "blocked_by_dvb": false, + "present": true + }, + { + "serial": "1002", + "product": "Blog V4", + "manufacturer": "RTLSDRBlog", + "role": "rail voice \u00b7 160", + "driver": null, + "blocked_by_dvb": false, + "present": true + } + ] +} diff --git a/dashboard/serve.py b/dashboard/serve.py new file mode 100644 index 0000000..f5547ca --- /dev/null +++ b/dashboard/serve.py @@ -0,0 +1,1521 @@ +#!/usr/bin/env python3 +"""istrain dashboard server (LOCAL preview). + +Serves the static dashboard AND a tiny control endpoint for the per-radio NOAA sanity toggle. +Replaces `python3 -m http.server` so a browser button can start/stop live audio per dongle. + + /api/noaa GET -> {"playing": bool, "serial": , "freq", "channel"} + POST ?serial=X -> toggle dongle X on NOAA: if X is playing, stop it; otherwise stop + whatever's playing and tune X. One dongle plays at a time. + + BUILD vs OPERATE (radio-runtime rule): the maintainer builds this server; the USER OPERATES the radio by + clicking a NOAA button in the browser. The server spawns rtl_fm ONLY in response to a POST driven + by that click. the code author must NOT curl/POST this endpoint to start audio — that's operating. GET is + read-only (no radio op) and safe to hit. + + It's a *temporary hijack* to confirm a radio is responding: while NOAA plays, that dongle is busy; + stopping frees it so it can go back to its assigned task. NOAA voice = signal; plain hiss = + receiver works but weak/no signal (squelch is off so the test always makes sound). +""" +import json, os, glob, re, signal, subprocess, atexit, shutil, time, threading, hashlib +from urllib.parse import urlparse, parse_qs +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer + +HERE = os.path.dirname(os.path.abspath(__file__)) +PORT = int(os.environ.get("PORT", "8157")) +BIND = os.environ.get("BIND", "127.0.0.1") + +# NOAA sanity-toggle config (env-overridable; later this comes from the swappable roles file). +NOAA_FREQ = os.environ.get("NOAA_FREQ", "162.550M") # Columbus KIG76 +NOAA_CHANNEL = os.environ.get("NOAA_CHANNEL", "162.550 · Columbus KIG76") +NOAA_GAIN = os.environ.get("NOAA_GAIN", "auto") # "auto" or a dB number +NOAA_DEFAULT = os.environ.get("NOAA_SERIAL", "1001") # used if POST omits ?serial + +# Airband captures: list + serve the per-transmission MP3 clips. Reads the dir passively — it does +# NOT care how Airband was started (run by hand, or the istrain-airband service). See airband/istrain.conf. +CAPTURES_DIR = os.environ.get("CAPTURES_DIR", "/tmp/airband") +# Persistent "heard" log — one line per transmission ever detected (channel + time + size). Survives +# clip pruning AND reboot, so it's the durable substrate for spotting train patterns. NOT in /tmp. +HEARD_LOG = os.path.expanduser(os.environ.get("HEARD_LOG", "~/istrain/heard.jsonl")) +EOT_LOG = os.path.expanduser(os.environ.get("EOT_LOG", "~/istrain/eot.jsonl")) +BOT_LOG = os.path.expanduser(os.environ.get("BOT_LOG", "~/istrain/bot.jsonl")) # 452 head-end (when the hop runs) +MID_LOG = os.path.expanduser(os.environ.get("MID_LOG", "~/istrain/midtrain.jsonl")) # DPU 452/457 ±12.5k (when IQ sub-channel ID feeds it; NOT auto from bot/eot) +LEVEL_LOG = os.path.expanduser(os.environ.get("LEVEL_LOG", "~/istrain/level.json")) # vumon's live instantaneous level (meter) +# Supervisor snapshot: on the NAS (presentation node) /api/supervisor serves a snapshot PUSHED from the +# VM (the NAS has no systemd/dongles). Set SUPERVISOR_SNAPSHOT on the NAS; empty on the VM = serve live. +SUPERVISOR_SNAPSHOT = os.path.expanduser(os.environ.get("SUPERVISOR_SNAPSHOT", "")) +# Push receiver (NAS): the VM POSTs its data files here over Tailscale (bypasses Cloudflare). PUSH_KEY +# gates it — if EMPTY, /api/push is DISABLED (so the VM's own serve.py, which has no key, can't be pushed +# to). Each whitelisted name maps to the exact path its read endpoint reads, so a push = "replace that file". +PUSH_KEY = os.environ.get("PUSH_KEY", "") +SITE_WINDOW_S = int(os.environ.get("SITE_WINDOW_DAYS", "3")) * 86400 # website shows the last N days only; the raw logs keep EVERYTHING (for the deeper offline analysis) +# Errors card (2026-07-06): ERRORS_LOG = this node's own events (ctl actions, failures, client JS +# reports). ERRORS_PUSHED = the VM's pushed copy (NAS only) — defaults NEXT TO the supervisor snapshot +# so the existing compose needs no new env. /api/errors merges both. +ERRORS_LOG = os.path.expanduser(os.environ.get("ERRORS_LOG", "~/istrain/errors.jsonl")) +ERRORS_PUSHED = os.environ.get("ERRORS_PUSHED", "") or ( + os.path.join(os.path.dirname(SUPERVISOR_SNAPSHOT), "errors-vm.jsonl") if SUPERVISOR_SNAPSHOT else "") +# Voice transcripts (2026-07-10): real-speech lines from the Mill's whisper (transcribe-worker). +TRANSCRIPTS_LOG = os.path.expanduser(os.environ.get("TRANSCRIPTS_LOG", "~/istrain/transcripts.jsonl")) +_PUSH_TARGETS = { + "eot": EOT_LOG, "bot": BOT_LOG, "midtrain": MID_LOG, "heard": HEARD_LOG, + "level": LEVEL_LOG, "supervisor": SUPERVISOR_SNAPSHOT or os.path.expanduser("~/istrain/supervisor.json"), + "errors": ERRORS_PUSHED or os.path.expanduser("~/istrain/errors-vm.jsonl"), + "transcripts": TRANSCRIPTS_LOG, +} +# Hit counter: total page loads + unique visitors. Counted on a real page load (GET /), never on the +# API polls. Behind Cloudflare the visitor IP is in CF-Connecting-IP; IPs are stored HASHED (dedup + +# privacy). Persist to /data on the NAS (set HITS_LOG) so counts survive container restarts. +HITS_LOG = os.path.expanduser(os.environ.get("HITS_LOG", "~/istrain/hits.json")) +_hits_lock = threading.Lock() +_CH_LABELS = { + "ns_dayton_road_160980": "NS Dayton Rd · 160.980 ★", + "ns_buckeye_yard_160920": "NS Buckeye Yard · 160.920", + "ns_col_sandusky_161190": "NS Col+Sandusky · 161.190", + "csx_scottslawn_160860": "CSX Scottslawn · 160.860", + "ns_col_area_161085": "NS Col area · 161.085", +} + +_proc = None # the running rtl_fm|aplay pipeline (Popen), or None +_serial = None # which dongle serial it's tuned to, or None + +# ── jsonl parse cache ──────────────────────────────────────────────────────────────────────────── +# The dashboard polls every 5 s but the logs only change every ~20-40 s (append) or on a decode +# enrich (atomic rewrite) — yet every poll was re-parsing ~2 MB / ~40k lines of jsonl (profiled at +# ~100 ms per /api/trains call, 2026-07-05). Cache the parsed rows keyed on (mtime_ns, size); any +# append or rewrite changes the key. Callers get the SAME list object on a hit — treat it as +# READ-ONLY: filter/sort into new lists, never .sort()/.append() the returned one. +_jsonl_cache = {} +_jsonl_lock = threading.Lock() + + +def _read_jsonl(path): + try: + st = os.stat(path) + except OSError: + return [] + key = (st.st_mtime_ns, st.st_size) + with _jsonl_lock: + hit = _jsonl_cache.get(path) + if hit and hit[0] == key: + return hit[1] + rows = [] + try: + with open(path) as f: + for line in f: + line = line.strip() + if not line: continue + try: rows.append(json.loads(line)) + except Exception: pass + except OSError: + pass + with _jsonl_lock: + _jsonl_cache[path] = (key, rows) + return rows + + +def _dongle_state(serial): + """(present, dvb_claimed) for a dongle by serial — passive sysfs read, never opens it.""" + for d in glob.glob("/sys/bus/usb/devices/*"): + try: + if open(os.path.join(d, "idVendor")).read().strip() != "0bda": continue + if open(os.path.join(d, "idProduct")).read().strip() != "2838": continue + if open(os.path.join(d, "serial")).read().strip() != serial: continue + except OSError: + continue + for intf in glob.glob(d + ":*"): + link = os.path.join(intf, "driver") + if os.path.islink(link) and os.path.basename(os.readlink(link)) == "dvb_usb_rtl28xxu": + return True, True + return True, False + return False, False + + +def _playing(): + global _proc, _serial + if _proc is not None and _proc.poll() is not None: + _proc, _serial = None, None # it exited on its own (dongle busy, killed, etc.) + return _proc is not None + + +def _stop(): + """Kill the whole pipeline group (rtl_fm AND aplay), freeing the dongle.""" + global _proc, _serial + if _proc is not None: + try: + os.killpg(os.getpgid(_proc.pid), signal.SIGTERM) + except (ProcessLookupError, PermissionError): + pass + try: + _proc.wait(timeout=3) + except subprocess.TimeoutExpired: + try: os.killpg(os.getpgid(_proc.pid), signal.SIGKILL) + except ProcessLookupError: pass + _proc, _serial = None, None + + +def _start(serial): + """Tune NOAA on `serial` and play it. Returns None on success or an error string.""" + global _proc, _serial + if not shutil.which("rtl_fm"): + return "rtl_fm not found — install the 'rtl-sdr' package." + present, dvb = _dongle_state(serial) + if not present: + return f"dongle {serial} not present — check the radios panel." + if dvb: + return f"dongle {serial} is DVB-claimed — free it: sudo modprobe -r rtl2832_sdr dvb_usb_rtl28xxu" + # squelch off (-l 0) so the test ALWAYS makes sound: NOAA voice = signal, hiss = receiver works. + gain = "" if NOAA_GAIN == "auto" else f"-g {NOAA_GAIN} " + cmd = (f"rtl_fm -d {serial} -f {NOAA_FREQ} -M fm -s 200000 -r 48000 -E deemp -l 0 {gain}- " + f"| aplay -r 48000 -f S16_LE -t raw -c 1") + # own session so we can kill the pair as a group; detached from the request handler thread. + _proc = subprocess.Popen(cmd, shell=True, start_new_session=True, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + _serial = serial + return None + + +def _state_json(error=None): + d = {"playing": _playing(), "serial": _serial if _playing() else None, + "freq": NOAA_FREQ, "channel": NOAA_CHANNEL} + if error: d["error"] = error + return d + + +def _toggle(serial): + """One-at-a-time: same dongle -> stop; different (or idle) -> stop any, start this one.""" + if _playing() and _serial == serial: + _stop() + return None + _stop() + return _start(serial) + + +def _capture_label(fname): + """Friendly channel label from a recording filename (