admin/istrain-public
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

istrain-public: from-scratch build of a passive RF train detector

Browse Source 
Public community release. The complete working system — DSP + decoders (scripts/),
web dashboard + API (dashboard/), container stack (docker/), config templates (config/) —
plus APOCALYPSE-EDITION.md, the full build guide for a human or a coding agent. GPLv3
(PyEOT dependency). Scrubbed of all secrets, internal IPs, hostnames, and location detail.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
istrain
2026-07-24 23:19:16 -04:00
commit 559caead36
41 changed files with 6740 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/bot-recover.py
+138
View File
@@ -0,0 +1,138 @@
#!/usr/bin/env python3
# bot-recover.py — capture-what-we-can for the BOT/HOT (452.9375) head-end band. We can't *decode* HOT
# yet (the field map / sync word / BCH are unpublished — see ../../EOT-HOT-PROTOCOL.md), but we CAN
# RECOVER the clean 64-bit frame: majority-vote the 3x-repeated block out of a strong BOT clip. This step
# does that for each settled BOT clip and ENRICHES the matching bot.jsonl record with the recovered block
# (undecoded), so the card's head section can show "head-end frame captured" — and the decode just drops
# into the same record later.
#
# Mirrors eot-decode.py exactly: decode-only mindset, NEVER culls (no clip moved or deleted — every BOT
# clip is kept for the ongoing RE), a processed-set so each clip is done once, atomic log write that never
# loses a capture append. Touches no radio — pure CPU on saved clips. CLI: bot-recover.py [--once]
import os, sys, glob, time, json
HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, os.path.join(HERE, "eot"))
import hot_explore as H # best_bits / preamble_end / vote — the proven recovery
CLIPS = os.path.expanduser(os.environ.get("BOT_CLIPS", "~/istrain/bot_clips"))
LOG = os.path.expanduser(os.environ.get("BOT_LOG", "~/istrain/bot.jsonl"))
STATE = os.path.join(CLIPS, ".recovered.json")
MIN_AGE = int(os.environ.get("RECOVER_MIN_AGE", "60"))
INTERVAL = int(os.environ.get("RECOVER_INTERVAL", "30"))
MIN_Q = float(os.environ.get("RECOVER_MIN_Q", "0.90")) # only keep a genuinely clean frame (3-way agreement)
def _hot_address(block, sync="0110101010", off=49, ln=17):
"""CRACKED field — the addressed EOT unit. HOT comes through with opposite polarity to EOT, so invert,
align to the sync, and read 17 bits LSB-first. Validated 2026-06-28: 22/22 recovered frames give a
plausible address and 9 match units decoded independently on 457 (chance ~1e-8). See EOT-HOT-PROTOCOL.md.
The command/type + BCH fields are still open — this is the address only."""
ib = block.translate(str.maketrans("01", "10"))
j = (ib + ib).find(sync)
if j < 0:
return None
frame = (ib + ib)[j:j + len(block)]
bits = (frame + frame)[off:off + ln]
if len(bits) < ln:
return None
return int(bits[::-1], 2)
def _hot_recover(clip):
"""Clip -> {hot_block, hot_q, hot_period, hot_addr} for a clean recovered HOT frame, or None."""
mx, bs = H.best_bits(clip)
body = bs[H.preamble_end(bs):]
if len(body) < 3 * 60:
return None
r = H.vote(body)
if not r:
return None
p, off, block, errs, agree, maxa = r
q = agree / maxa
if q < MIN_Q:
return None
# reject a preamble-dominated lock: the voter can latch the alternating 0101… preamble (trivially
# high agreement). A real HOT data block (30 data + 33 BCH) isn't ~all transitions.
trans = sum(1 for i in range(1, len(block)) if block[i] != block[i-1])
if trans / max(1, len(block) - 1) > 0.85:
return None
rec = {"hot_block": block, "hot_q": round(q, 2), "hot_period": p}
addr = _hot_address(block)
if addr is not None:
rec["hot_addr"] = addr
return rec
def _load_state():
try: return set(json.load(open(STATE)))
except Exception: return set()
def _save_state(s):
tmp = STATE + ".tmp"
with open(tmp, "w") as f: json.dump(sorted(s), f)
os.replace(tmp, STATE)
def _enrich(recovered):
"""Merge {sec: {hot_block,...}} into bot.jsonl by integer-ts match — atomic, never drops an append."""
if not recovered or not os.path.exists(LOG):
return
size = os.path.getsize(LOG); out = []
def _take(fh):
for line in fh:
line = line.strip()
if not line: continue
try: r = json.loads(line)
except Exception: continue
s = int(r.get("ts", 0))
if s in recovered: r.update(recovered[s])
out.append(r)
with open(LOG) as f: _take(f)
try:
with open(LOG) as f: f.seek(size); _take(f)
except Exception: pass
tmp = LOG + ".tmp"
with open(tmp, "w") as f:
for r in out: f.write(json.dumps(r) + "\n")
os.replace(tmp, LOG)
def one_pass():
if not os.path.isdir(CLIPS):
return (0, 0)
state = _load_state(); now = time.time()
recovered = {}; tried = ok = 0; newly = []
for c in sorted(glob.glob(os.path.join(CLIPS, "bot_*.s16"))):
fn = os.path.basename(c)
if fn in state: continue
if now - os.path.getmtime(c) < MIN_AGE: continue
try: r = _hot_recover(c)
except Exception as e: sys.stderr.write("[bot-recover] err %s: %s\n" % (fn, e)); continue
tried += 1; newly.append(fn)
if r:
ok += 1
try: sec = int(time.mktime(time.strptime(fn[4:-4], "%Y%m%d_%H%M%S")))
except Exception: sec = None
if sec is not None: recovered[sec] = r
sys.stderr.write("[bot-recover] ✓ %s frame q=%s block=%s…\n" % (fn, r["hot_q"], r["hot_block"][:16]))
if recovered: _enrich(recovered)
if newly: state.update(newly); _save_state(state)
return (tried, ok)
def main():
if "--once" in sys.argv[1:]:
t, o = one_pass(); print("[bot-recover] one pass: tried %d, frames recovered %d" % (t, o)); return
sys.stderr.write("[bot-recover] HOT frame-recovery watcher up — %s every %ds (no cull)\n" % (CLIPS, INTERVAL)); sys.stderr.flush()
while True:
try:
t, o = one_pass()
if t: sys.stderr.write("[bot-recover] pass: tried %d, recovered %d\n" % (t, o)); sys.stderr.flush()
except Exception as e:
sys.stderr.write("[bot-recover] pass err: %s\n" % e); sys.stderr.flush()
time.sleep(INTERVAL)
if __name__ == "__main__":
main()
scripts/correlate.py
+136
View File
@@ -0,0 +1,136 @@
#!/usr/bin/env python3
# correlate.py — multi-band "train activity" correlation engine. READ-ONLY, non-destructive.
#
# The mission is to *detect a train at the crossing*, not to *decode* any one signal. A train lights
# several bands at once; this marries the capture logs we already keep and reports CO-OCCURRING
# activity as an EXPLAINED candidate — it shows the evidence and lets a human infer. It is deliberately
# NOT a latched verdict: it only ever reflects a sliding time window, so when activity ages out it
# self-resets to "quiet" (can't get stuck in a false "blocked forever" state).
#
# Bands (today): 160.980 road/detector VOICE (1002 -> heard.jsonl) · 457-cluster (1001 -> eot.jsonl) ·
# 452-cluster head-end / BOT (-> bot.jsonl) on the hop.
#
# Mid-train DPU (2026-06-24): the DPU command channels sit ±12.5 kHz off the HOT/EOT centers
# (452.925/.950 and 457.925/.950) — INSIDE each dwell's 200 kHz capture. So a DPU burst already trips
# the 452/457 leg and is counted here; it just isn't *distinguished* from HOT/EOT (that needs IQ
# sub-channel ID — see eot/README "DPU decode plan"). Net: a DPU-emitting train already lights these
# legs; we don't need a separate capture for detection, only for identification/decode.
#
# Touches no radio. CLI: correlate.py [--window MIN] [--json] API: correlate(window_min=20) -> dict
import os, json, time, sys
HEARD = os.path.expanduser(os.environ.get("HEARD_LOG", "~/istrain/heard.jsonl"))
EOT = os.path.expanduser(os.environ.get("EOT_LOG", "~/istrain/eot.jsonl"))
BOT = os.path.expanduser(os.environ.get("BOT_LOG", "~/istrain/bot.jsonl"))
MID = os.path.expanduser(os.environ.get("MID_LOG", "~/istrain/midtrain.jsonl"))
WINDOW_MIN = int(os.environ.get("CORRELATE_WINDOW_MIN", "20"))
def _load(path):
rows = []
if not os.path.exists(path):
return rows
try:
for line in open(path):
line = line.strip()
if not line:
continue
try: rows.append(json.loads(line))
except Exception: continue
except OSError:
pass
return rows
def _ch(r):
return str(r.get("ch") or r.get("channel") or r.get("freq") or r.get("label") or "")
def correlate(window_min=WINDOW_MIN, now=None):
now = now if now is not None else time.time()
lo = now - window_min * 60
heard = [r for r in _load(HEARD) if r.get("ts", 0) >= lo]
road = [r for r in heard if "160.98" in _ch(r)] # the NS road / detector channel
eot = [r for r in _load(EOT) if r.get("ts", 0) >= lo] # 457 EOT-band bursts
bot = [r for r in _load(BOT) if r.get("ts", 0) >= lo] # 452 head-end bursts (when present)
# MID (±12.5 kHz DPU watch) is booster-aware (O-9/O-11): the 457.925 wayside booster paints a flat
# continuous carrier for hours, which would make this band read "active" on no train at all. With
# enough rows to characterize, only excursions >= 6 dB over the median (= the booster floor) count;
# the flat carrier itself is reported but never correlates.
mid_all = [r for r in _load(MID) if r.get("ts", 0) >= lo and r.get("peak") is not None]
booster = False
if len(mid_all) >= 30:
pks = sorted(r["peak"] for r in mid_all)
floor = pks[len(pks) // 2]
mid = [r for r in mid_all if r["peak"] >= floor + 6.0]
booster = True
else:
mid = mid_all
bands = []
def add(key, label, evs, extra=None):
last = max((e.get("ts", 0) for e in evs), default=0)
b = {"key": key, "label": label, "count": len(evs),
"last": last, "last_ago": (round(now - last) if last else None)}
if extra: b.update(extra)
bands.append(b)
add("voice", "160.980 road/detector voice", road)
add("eot", "457-cluster (EOT + mid-train DPU)", eot)
if bot:
add("bot", "452-cluster (head-end + mid-train DPU)", bot)
if mid_all:
add("mid", "±12.5k DPU watch" + (" (booster-floor subtracted)" if booster else ""), mid)
# flat event list for the timeline view (panel renders this)
events = []
for e in road: events.append({"band": "voice", "ts": e.get("ts", 0)})
for e in eot: events.append({"band": "eot", "ts": e.get("ts", 0), "dur": e.get("dur"), "peak": e.get("peak")})
for e in bot: events.append({"band": "bot", "ts": e.get("ts", 0)})
for e in mid: events.append({"band": "mid", "ts": e.get("ts", 0), "peak": e.get("peak")})
events.sort(key=lambda x: x["ts"])
active = [b for b in bands if b["count"] > 0]
reasons = []
if len(active) == 0:
state = "quiet"
reasons.append("No activity on any watched band in the last %d min." % window_min)
elif len(active) == 1:
b = active[0]
state = "single-band"
reasons.append("Only %s active (%d hit%s) — one band alone can be RFI/noise, NOT confirmed as a train."
% (b["label"], b["count"], "" if b["count"] == 1 else "s"))
else:
state = "candidate"
reasons.append("%d bands co-firing in the last %d min — consistent with train activity:"
% (len(active), window_min))
for b in active:
reasons.append(" · %s — %d hit%s, last %ss ago"
% (b["label"], b["count"], "" if b["count"] == 1 else "s", b["last_ago"]))
if booster:
reasons.append("Booster carrier active on the MID band (flat ~median floor) — subtracted; "
"only excursions ≥6 dB over it count as MID activity.")
return {"now": now, "window_min": window_min, "state": state,
"bands": bands, "events": events, "reasons": reasons}
def _pretty(d):
badge = {"quiet": "· QUIET", "single-band": "~ SINGLE-BAND", "candidate": "▸ CANDIDATE"}.get(d["state"], d["state"])
print("=== train-activity correlation [%s window] ===" % (str(d["window_min"]) + "m"))
print("state: %s" % badge)
for b in d["bands"]:
ago = ("%ss ago" % b["last_ago"]) if b["last_ago"] is not None else "—"
print(" %-30s %4d hit(s) last: %s" % (b["label"], b["count"], ago))
print("why:")
for r in d["reasons"]:
print(" " + r)
if __name__ == "__main__":
win = WINDOW_MIN; as_json = False
a = sys.argv[1:]
while a:
t = a.pop(0)
if t == "--window": win = int(a.pop(0))
elif t == "--json": as_json = True
out = correlate(window_min=win)
if as_json:
print(json.dumps(out))
else:
_pretty(out)
scripts/envelope.py
+112
View File
@@ -0,0 +1,112 @@
#!/usr/bin/env python3
# envelope.py — OFFLINE, read-only. Reconstructs a train's signal-strength ENVELOPE from decoded EOT
# frames and classifies the passage SHAPE: passed-through / stopped-blocking / departed / set-out / distant.
#
# WHY THIS IS NEWLY POSSIBLE (O-18): the +10 dB antenna gave the peak-dB trajectory real dynamic range.
# Whip-era passages all sat in the flat -40..-47 mud (O-10) with no clean rise/peak/fall, so envelope
# shape was unreadable and direction/pass-through detection impossible. Now the curve exists.
#
# Touches NO radio and NO service — pure log analysis. Read-first, test-gated, bake before it drives anything.
# usage: envelope.py [eot.jsonl] [--unit N] [--since TS]
import json, sys, time
LOG = os.path.expanduser("~/istrain/eot.jsonl")
SWAP = 1783433400 # antenna epoch (O-15); envelopes only meaningful after this
SESSION_GAP = 1200 # >20 min quiet OR unit change = new session (sessionize per O-8)
PLATEAU_DB = 6.0 # "receded" if last smoothed peak is >= this far below the max
FLAT_DB = 5.0 # dynamic range below this = flat/distant (no real envelope)
def load(path, since=SWAP, unit=None):
rows = []
for line in open(path):
try: r = json.loads(line)
except Exception: continue
if r.get("ts", 0) < since: continue
if "unit" not in r or r.get("peak") is None: continue
if unit and r["unit"] != unit: continue
rows.append(r)
rows.sort(key=lambda r: r["ts"])
return rows
def sessionize(rows):
out, cur = [], []
for r in rows:
if cur and (r["ts"] - cur[-1]["ts"] > SESSION_GAP or r["unit"] != cur[-1]["unit"]):
out.append(cur); cur = []
cur.append(r)
if cur: out.append(cur)
return out
def smooth(vals, k=3):
# rolling max over k samples — frames are sparse (~30-60s); rolling-max tracks the crest, not the fades
out = []
for i in range(len(vals)):
out.append(max(vals[max(0, i - k + 1):i + 1]))
return out
def classify(s, now):
t0 = s[0]["ts"]
t = [(r["ts"] - t0) / 60 for r in s] # minutes from first frame
pk = [r["peak"] for r in s]
sp = smooth(pk)
psi = [r.get("pressure") for r in s]
mo = [r.get("motion") for r in s]
imax = max(range(len(sp)), key=lambda i: sp[i])
peak_db, t_close = sp[imax], t[imax]
rise = (sp[imax] - sp[0]) / (t[imax] - t[0]) if t[imax] > t[0] else 0.0 # dB/min approach slope (speed proxy)
dyn = max(sp) - min(sp)
receded = (sp[imax] - sp[-1]) >= PLATEAU_DB # signal fell back off after the crest
psi_vals = [p for p in psi if p is not None]
psi_end = next((p for p in reversed(psi) if p is not None), None)
psi_start = next((p for p in psi if p is not None), None)
psi_min = min(psi_vals) if psi_vals else None # the braked TROUGH, not the arrival pressure
mo_end = next((m for m in reversed(mo) if m is not None), None)
# departure = a session that STOPPED (braked trough / motion-0 seen) then RECHARGES to ~88 + is rolling
# (O-10 signature). Anchor on the trough, NOT psi_start — the first frame is the arrival, still charged.
had_stop = any(m == 0 for m in mo if m is not None) or (psi_min is not None and psi_min <= 80)
recharged = (had_stop and psi_end is not None and psi_end >= 84 and mo_end == 1)
dwell_min = (now - s[-1]["ts"]) / 60
if psi_end is not None and psi_end < 10 and not recharged:
verdict = f"SET-OUT / air dumped (psi bled to {psi_end}, held) — car left; crossing state unknown from EOT"
elif recharged:
verdict = f"DEPARTED — brakes re-aired (psi trough {psi_min}->{psi_end}) + rolling (motion->1); pulled out"
elif mo_end == 0 and psi_end is not None and psi_end < 84 and not receded:
verdict = (f"STOPPED / BLOCKING — arrived then held (psi {psi_start}->{psi_end}, motion->0), "
f"plateau not receded; last frame {dwell_min:.0f}m ago = still present")
elif receded and dyn >= FLAT_DB and (mo_end == 1 or (psi_end and psi_end >= 84)):
verdict = (f"PASSED THROUGH — closest approach {peak_db:.0f} dB at +{t_close:.0f}m, "
f"then receded {sp[imax]-sp[-1]:.0f} dB; rolling. approach slope {rise:+.1f} dB/min (speed proxy)")
elif dyn < FLAT_DB and peak_db < -38:
verdict = f"DISTANT / weak — flat envelope (dyn {dyn:.0f} dB, peak {peak_db:.0f}); not near the crossing"
else:
verdict = (f"IN PROGRESS / inconclusive — peak {peak_db:.0f} at +{t_close:.0f}m, dyn {dyn:.0f} dB, "
f"psi {psi_start}->{psi_end}, motion->{mo_end}, last {dwell_min:.0f}m ago")
return dict(unit=s[0]["unit"], n=len(s), span=t[-1], peak=peak_db, t_close=t_close,
dyn=dyn, rise=rise, psi=(psi_start, psi_end), mo_end=mo_end, verdict=verdict)
def main():
args = sys.argv[1:]
path = LOG
unit = None; since = SWAP
if args and not args[0].startswith("--"): path = args.pop(0)
if "--unit" in args: unit = int(args[args.index("--unit") + 1])
if "--since" in args: since = float(args[args.index("--since") + 1])
now = time.time()
sess = sessionize(load(path, since, unit))
print(f"envelope: {len(sess)} session(s) since ts {since:.0f} (antenna epoch = {SWAP})\n")
for s in sess:
if len(s) < 2: # a lone frame has no envelope
print(f" unit {s[0]['unit']:>6} n=1 (singleton — no envelope)"); continue
c = classify(s, now)
print(f" unit {c['unit']:>6} n={c['n']:>3} span={c['span']:>4.0f}m peak={c['peak']:>5.0f}dB dyn={c['dyn']:>3.0f}dB")
print(f" -> {c['verdict']}")
if __name__ == "__main__":
main()
scripts/eot-decode.py
+118
View File
@@ -0,0 +1,118 @@
#!/usr/bin/env python3
# eot-decode.py — DECODE-ONLY service for the EOT band. Watches ~/istrain/eot_clips for finished clips,
# decodes each (clip -> unit/telemetry via eot_scan) and ENRICHES the matching record in eot.jsonl.
#
# It does NOT cull. No clip is ever deleted or moved — the BOT reverse-engineering and the "more examples
# to train on" both need every clip kept. This is the half of the old reaper that mattered, split out:
# the reaper bundled decode WITH culling, so pausing the cull (which we want off) also paused identification.
# Decode now stands alone; culling can stay off until storage actually forces it.
#
# Marking done WITHOUT moving/deleting: a tiny processed-set at eot_clips/.decoded.json (each clip decoded
# once). Safe log write: read + enrich matched records, recapture any tail the capture appended since the
# read, then atomic temp-rename — the capture only ever appends, so we never lose a presence line.
#
# Operated as the istrain-eot-decode --user service (on/off + hard-reset via the dashboard supervisor).
# Touches NO radio — pure CPU on saved clips — so it is safe to start/stop/restart at any time.
# CLI: eot-decode.py [--once]
import os, sys, glob, time, json
HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
from reaper import _eot_decode, _clip_epoch # reuse the PROVEN decode adapter + clip->epoch
CLIPS = os.path.expanduser(os.environ.get("EOT_CLIPS", "~/istrain/eot_clips"))
LOG = os.path.expanduser(os.environ.get("EOT_LOG", "~/istrain/eot.jsonl"))
STATE = os.path.join(CLIPS, ".decoded.json")
MIN_AGE = int(os.environ.get("DECODE_MIN_AGE", "60")) # never touch a clip <60s old (maybe still mid-write)
INTERVAL = int(os.environ.get("DECODE_INTERVAL", "30")) # seconds between passes
def _load_state():
try: return set(json.load(open(STATE)))
except Exception: return set()
def _save_state(s):
tmp = STATE + ".tmp"
with open(tmp, "w") as f: json.dump(sorted(s), f)
os.replace(tmp, STATE)
def _enrich(decodes):
"""Merge {sec: decoded-fields} into eot.jsonl by integer-ts match — atomically, without losing any
line the capture appended while we worked (re-read past the byte offset we started at, then rename)."""
if not decodes or not os.path.exists(LOG):
return
size = os.path.getsize(LOG)
out = []
def _take(fh):
for line in fh:
line = line.strip()
if not line: continue
try: r = json.loads(line)
except Exception: continue
s = int(r.get("ts", 0))
if s in decodes: r.update(decodes[s])
out.append(r)
with open(LOG) as f:
_take(f)
try: # recapture appends since our read (capture only appends)
with open(LOG) as f:
f.seek(size); _take(f)
except Exception:
pass
tmp = LOG + ".tmp"
with open(tmp, "w") as f:
for r in out: f.write(json.dumps(r) + "\n")
os.replace(tmp, LOG)
def one_pass():
"""Decode every not-yet-processed, settled clip; enrich the log; mark them done. Returns (tried, ok)."""
if not os.path.isdir(CLIPS):
return (0, 0)
state = _load_state()
now = time.time()
decodes = {}; tried = ok = 0; newly = []
for c in sorted(glob.glob(os.path.join(CLIPS, "eot_*.s16"))): # top-level only — confirmed/ is left alone
fn = os.path.basename(c)
if fn in state:
continue
if now - os.path.getmtime(c) < MIN_AGE: # too fresh — try it next pass, don't mark yet
continue
try:
fields = _eot_decode(c)
except Exception as e:
sys.stderr.write("[eot-decode] err %s: %s\n" % (fn, e)); continue
tried += 1; newly.append(fn)
if fields:
ok += 1
sec = _clip_epoch(c, "eot_")
if sec is not None: decodes[sec] = fields
sys.stderr.write("[eot-decode] ✅ %s unit=%s pressure=%s motion=%s\n"
% (fn, fields.get("unit"), fields.get("pressure"), fields.get("motion")))
if decodes:
_enrich(decodes)
if newly:
state.update(newly); _save_state(state)
return (tried, ok)
def main():
if "--once" in sys.argv[1:]:
t, o = one_pass(); print("[eot-decode] one pass: tried %d, decoded %d" % (t, o)); return
sys.stderr.write("[eot-decode] decode-only watcher up — %s every %ds (no cull, no move, no delete)\n"
% (CLIPS, INTERVAL)); sys.stderr.flush()
while True:
try:
t, o = one_pass()
if t: sys.stderr.write("[eot-decode] pass: tried %d, decoded %d\n" % (t, o)); sys.stderr.flush()
except Exception as e:
sys.stderr.write("[eot-decode] pass err: %s\n" % e); sys.stderr.flush()
time.sleep(INTERVAL)
if __name__ == "__main__":
main()
scripts/eot/PyEOT/LICENSE
+674
View File
@@ -0,0 +1,674 @@
GNU GENERAL PUBLIC LICENSE
Version 3, 29 June 2007
Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
Preamble
The GNU General Public License is a free, copyleft license for
software and other kinds of works.
The licenses for most software and other practical works are designed
to take away your freedom to share and change the works. By contrast,
the GNU General Public License is intended to guarantee your freedom to
share and change all versions of a program--to make sure it remains free
software for all its users. We, the Free Software Foundation, use the
GNU General Public License for most of our software; it applies also to
any other work released this way by its authors. You can apply it to
your programs, too.
When we speak of free software, we are referring to freedom, not
price. Our General Public Licenses are designed to make sure that you
have the freedom to distribute copies of free software (and charge for
them if you wish), that you receive source code or can get it if you
want it, that you can change the software or use pieces of it in new
free programs, and that you know you can do these things.
To protect your rights, we need to prevent others from denying you
these rights or asking you to surrender the rights. Therefore, you have
certain responsibilities if you distribute copies of the software, or if
you modify it: responsibilities to respect the freedom of others.
For example, if you distribute copies of such a program, whether
gratis or for a fee, you must pass on to the recipients the same
freedoms that you received. You must make sure that they, too, receive
or can get the source code. And you must show them these terms so they
know their rights.
Developers that use the GNU GPL protect your rights with two steps:
(1) assert copyright on the software, and (2) offer you this License
giving you legal permission to copy, distribute and/or modify it.
For the developers' and authors' protection, the GPL clearly explains
that there is no warranty for this free software. For both users' and
authors' sake, the GPL requires that modified versions be marked as
changed, so that their problems will not be attributed erroneously to
authors of previous versions.
Some devices are designed to deny users access to install or run
modified versions of the software inside them, although the manufacturer
can do so. This is fundamentally incompatible with the aim of
protecting users' freedom to change the software. The systematic
pattern of such abuse occurs in the area of products for individuals to
use, which is precisely where it is most unacceptable. Therefore, we
have designed this version of the GPL to prohibit the practice for those
products. If such problems arise substantially in other domains, we
stand ready to extend this provision to those domains in future versions
of the GPL, as needed to protect the freedom of users.
Finally, every program is threatened constantly by software patents.
States should not allow patents to restrict development and use of
software on general-purpose computers, but in those that do, we wish to
avoid the special danger that patents applied to a free program could
make it effectively proprietary. To prevent this, the GPL assures that
patents cannot be used to render the program non-free.
The precise terms and conditions for copying, distribution and
modification follow.
TERMS AND CONDITIONS
0. Definitions.
"This License" refers to version 3 of the GNU General Public License.
"Copyright" also means copyright-like laws that apply to other kinds of
works, such as semiconductor masks.
"The Program" refers to any copyrightable work licensed under this
License. Each licensee is addressed as "you". "Licensees" and
"recipients" may be individuals or organizations.
To "modify" a work means to copy from or adapt all or part of the work
in a fashion requiring copyright permission, other than the making of an
exact copy. The resulting work is called a "modified version" of the
earlier work or a work "based on" the earlier work.
A "covered work" means either the unmodified Program or a work based
on the Program.
To "propagate" a work means to do anything with it that, without
permission, would make you directly or secondarily liable for
infringement under applicable copyright law, except executing it on a
computer or modifying a private copy. Propagation includes copying,
distribution (with or without modification), making available to the
public, and in some countries other activities as well.
To "convey" a work means any kind of propagation that enables other
parties to make or receive copies. Mere interaction with a user through
a computer network, with no transfer of a copy, is not conveying.
An interactive user interface displays "Appropriate Legal Notices"
to the extent that it includes a convenient and prominently visible
feature that (1) displays an appropriate copyright notice, and (2)
tells the user that there is no warranty for the work (except to the
extent that warranties are provided), that licensees may convey the
work under this License, and how to view a copy of this License. If
the interface presents a list of user commands or options, such as a
menu, a prominent item in the list meets this criterion.
1. Source Code.
The "source code" for a work means the preferred form of the work
for making modifications to it. "Object code" means any non-source
form of a work.
A "Standard Interface" means an interface that either is an official
standard defined by a recognized standards body, or, in the case of
interfaces specified for a particular programming language, one that
is widely used among developers working in that language.
The "System Libraries" of an executable work include anything, other
than the work as a whole, that (a) is included in the normal form of
packaging a Major Component, but which is not part of that Major
Component, and (b) serves only to enable use of the work with that
Major Component, or to implement a Standard Interface for which an
implementation is available to the public in source code form. A
"Major Component", in this context, means a major essential component
(kernel, window system, and so on) of the specific operating system
(if any) on which the executable work runs, or a compiler used to
produce the work, or an object code interpreter used to run it.
The "Corresponding Source" for a work in object code form means all
the source code needed to generate, install, and (for an executable
work) run the object code and to modify the work, including scripts to
control those activities. However, it does not include the work's
System Libraries, or general-purpose tools or generally available free
programs which are used unmodified in performing those activities but
which are not part of the work. For example, Corresponding Source
includes interface definition files associated with source files for
the work, and the source code for shared libraries and dynamically
linked subprograms that the work is specifically designed to require,
such as by intimate data communication or control flow between those
subprograms and other parts of the work.
The Corresponding Source need not include anything that users
can regenerate automatically from other parts of the Corresponding
Source.
The Corresponding Source for a work in source code form is that
same work.
2. Basic Permissions.
All rights granted under this License are granted for the term of
copyright on the Program, and are irrevocable provided the stated
conditions are met. This License explicitly affirms your unlimited
permission to run the unmodified Program. The output from running a
covered work is covered by this License only if the output, given its
content, constitutes a covered work. This License acknowledges your
rights of fair use or other equivalent, as provided by copyright law.
You may make, run and propagate covered works that you do not
convey, without conditions so long as your license otherwise remains
in force. You may convey covered works to others for the sole purpose
of having them make modifications exclusively for you, or provide you
with facilities for running those works, provided that you comply with
the terms of this License in conveying all material for which you do
not control copyright. Those thus making or running the covered works
for you must do so exclusively on your behalf, under your direction
and control, on terms that prohibit them from making any copies of
your copyrighted material outside their relationship with you.
Conveying under any other circumstances is permitted solely under
the conditions stated below. Sublicensing is not allowed; section 10
makes it unnecessary.
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
No covered work shall be deemed part of an effective technological
measure under any applicable law fulfilling obligations under article
11 of the WIPO copyright treaty adopted on 20 December 1996, or
similar laws prohibiting or restricting circumvention of such
measures.
When you convey a covered work, you waive any legal power to forbid
circumvention of technological measures to the extent such circumvention
is effected by exercising rights under this License with respect to
the covered work, and you disclaim any intention to limit operation or
modification of the work as a means of enforcing, against the work's
users, your or third parties' legal rights to forbid circumvention of
technological measures.
4. Conveying Verbatim Copies.
You may convey verbatim copies of the Program's source code as you
receive it, in any medium, provided that you conspicuously and
appropriately publish on each copy an appropriate copyright notice;
keep intact all notices stating that this License and any
non-permissive terms added in accord with section 7 apply to the code;
keep intact all notices of the absence of any warranty; and give all
recipients a copy of this License along with the Program.
You may charge any price or no price for each copy that you convey,
and you may offer support or warranty protection for a fee.
5. Conveying Modified Source Versions.
You may convey a work based on the Program, or the modifications to
produce it from the Program, in the form of source code under the
terms of section 4, provided that you also meet all of these conditions:
a) The work must carry prominent notices stating that you modified
it, and giving a relevant date.
b) The work must carry prominent notices stating that it is
released under this License and any conditions added under section
7. This requirement modifies the requirement in section 4 to
"keep intact all notices".
c) You must license the entire work, as a whole, under this
License to anyone who comes into possession of a copy. This
License will therefore apply, along with any applicable section 7
additional terms, to the whole of the work, and all its parts,
regardless of how they are packaged. This License gives no
permission to license the work in any other way, but it does not
invalidate such permission if you have separately received it.
d) If the work has interactive user interfaces, each must display
Appropriate Legal Notices; however, if the Program has interactive
interfaces that do not display Appropriate Legal Notices, your
work need not make them do so.
A compilation of a covered work with other separate and independent
works, which are not by their nature extensions of the covered work,
and which are not combined with it such as to form a larger program,
in or on a volume of a storage or distribution medium, is called an
"aggregate" if the compilation and its resulting copyright are not
used to limit the access or legal rights of the compilation's users
beyond what the individual works permit. Inclusion of a covered work
in an aggregate does not cause this License to apply to the other
parts of the aggregate.
6. Conveying Non-Source Forms.
You may convey a covered work in object code form under the terms
of sections 4 and 5, provided that you also convey the
machine-readable Corresponding Source under the terms of this License,
in one of these ways:
a) Convey the object code in, or embodied in, a physical product
(including a physical distribution medium), accompanied by the
Corresponding Source fixed on a durable physical medium
customarily used for software interchange.
b) Convey the object code in, or embodied in, a physical product
(including a physical distribution medium), accompanied by a
written offer, valid for at least three years and valid for as
long as you offer spare parts or customer support for that product
model, to give anyone who possesses the object code either (1) a
copy of the Corresponding Source for all the software in the
product that is covered by this License, on a durable physical
medium customarily used for software interchange, for a price no
more than your reasonable cost of physically performing this
conveying of source, or (2) access to copy the
Corresponding Source from a network server at no charge.
c) Convey individual copies of the object code with a copy of the
written offer to provide the Corresponding Source. This
alternative is allowed only occasionally and noncommercially, and
only if you received the object code with such an offer, in accord
with subsection 6b.
d) Convey the object code by offering access from a designated
place (gratis or for a charge), and offer equivalent access to the
Corresponding Source in the same way through the same place at no
further charge. You need not require recipients to copy the
Corresponding Source along with the object code. If the place to
copy the object code is a network server, the Corresponding Source
may be on a different server (operated by you or a third party)
that supports equivalent copying facilities, provided you maintain
clear directions next to the object code saying where to find the
Corresponding Source. Regardless of what server hosts the
Corresponding Source, you remain obligated to ensure that it is
available for as long as needed to satisfy these requirements.
e) Convey the object code using peer-to-peer transmission, provided
you inform other peers where the object code and Corresponding
Source of the work are being offered to the general public at no
charge under subsection 6d.
A separable portion of the object code, whose source code is excluded
from the Corresponding Source as a System Library, need not be
included in conveying the object code work.
A "User Product" is either (1) a "consumer product", which means any
tangible personal property which is normally used for personal, family,
or household purposes, or (2) anything designed or sold for incorporation
into a dwelling. In determining whether a product is a consumer product,
doubtful cases shall be resolved in favor of coverage. For a particular
product received by a particular user, "normally used" refers to a
typical or common use of that class of product, regardless of the status
of the particular user or of the way in which the particular user
actually uses, or expects or is expected to use, the product. A product
is a consumer product regardless of whether the product has substantial
commercial, industrial or non-consumer uses, unless such uses represent
the only significant mode of use of the product.
"Installation Information" for a User Product means any methods,
procedures, authorization keys, or other information required to install
and execute modified versions of a covered work in that User Product from
a modified version of its Corresponding Source. The information must
suffice to ensure that the continued functioning of the modified object
code is in no case prevented or interfered with solely because
modification has been made.
If you convey an object code work under this section in, or with, or
specifically for use in, a User Product, and the conveying occurs as
part of a transaction in which the right of possession and use of the
User Product is transferred to the recipient in perpetuity or for a
fixed term (regardless of how the transaction is characterized), the
Corresponding Source conveyed under this section must be accompanied
by the Installation Information. But this requirement does not apply
if neither you nor any third party retains the ability to install
modified object code on the User Product (for example, the work has
been installed in ROM).
The requirement to provide Installation Information does not include a
requirement to continue to provide support service, warranty, or updates
for a work that has been modified or installed by the recipient, or for
the User Product in which it has been modified or installed. Access to a
network may be denied when the modification itself materially and
adversely affects the operation of the network or violates the rules and
protocols for communication across the network.
Corresponding Source conveyed, and Installation Information provided,
in accord with this section must be in a format that is publicly
documented (and with an implementation available to the public in
source code form), and must require no special password or key for
unpacking, reading or copying.
7. Additional Terms.
"Additional permissions" are terms that supplement the terms of this
License by making exceptions from one or more of its conditions.
Additional permissions that are applicable to the entire Program shall
be treated as though they were included in this License, to the extent
that they are valid under applicable law. If additional permissions
apply only to part of the Program, that part may be used separately
under those permissions, but the entire Program remains governed by
this License without regard to the additional permissions.
When you convey a copy of a covered work, you may at your option
remove any additional permissions from that copy, or from any part of
it. (Additional permissions may be written to require their own
removal in certain cases when you modify the work.) You may place
additional permissions on material, added by you to a covered work,
for which you have or can give appropriate copyright permission.
Notwithstanding any other provision of this License, for material you
add to a covered work, you may (if authorized by the copyright holders of
that material) supplement the terms of this License with terms:
a) Disclaiming warranty or limiting liability differently from the
terms of sections 15 and 16 of this License; or
b) Requiring preservation of specified reasonable legal notices or
author attributions in that material or in the Appropriate Legal
Notices displayed by works containing it; or
c) Prohibiting misrepresentation of the origin of that material, or
requiring that modified versions of such material be marked in
reasonable ways as different from the original version; or
d) Limiting the use for publicity purposes of names of licensors or
authors of the material; or
e) Declining to grant rights under trademark law for use of some
trade names, trademarks, or service marks; or
f) Requiring indemnification of licensors and authors of that
material by anyone who conveys the material (or modified versions of
it) with contractual assumptions of liability to the recipient, for
any liability that these contractual assumptions directly impose on
those licensors and authors.
All other non-permissive additional terms are considered "further
restrictions" within the meaning of section 10. If the Program as you
received it, or any part of it, contains a notice stating that it is
governed by this License along with a term that is a further
restriction, you may remove that term. If a license document contains
a further restriction but permits relicensing or conveying under this
License, you may add to a covered work material governed by the terms
of that license document, provided that the further restriction does
not survive such relicensing or conveying.
If you add terms to a covered work in accord with this section, you
must place, in the relevant source files, a statement of the
additional terms that apply to those files, or a notice indicating
where to find the applicable terms.
Additional terms, permissive or non-permissive, may be stated in the
form of a separately written license, or stated as exceptions;
the above requirements apply either way.
8. Termination.
You may not propagate or modify a covered work except as expressly
provided under this License. Any attempt otherwise to propagate or
modify it is void, and will automatically terminate your rights under
this License (including any patent licenses granted under the third
paragraph of section 11).
However, if you cease all violation of this License, then your
license from a particular copyright holder is reinstated (a)
provisionally, unless and until the copyright holder explicitly and
finally terminates your license, and (b) permanently, if the copyright
holder fails to notify you of the violation by some reasonable means
prior to 60 days after the cessation.
Moreover, your license from a particular copyright holder is
reinstated permanently if the copyright holder notifies you of the
violation by some reasonable means, this is the first time you have
received notice of violation of this License (for any work) from that
copyright holder, and you cure the violation prior to 30 days after
your receipt of the notice.
Termination of your rights under this section does not terminate the
licenses of parties who have received copies or rights from you under
this License. If your rights have been terminated and not permanently
reinstated, you do not qualify to receive new licenses for the same
material under section 10.
9. Acceptance Not Required for Having Copies.
You are not required to accept this License in order to receive or
run a copy of the Program. Ancillary propagation of a covered work
occurring solely as a consequence of using peer-to-peer transmission
to receive a copy likewise does not require acceptance. However,
nothing other than this License grants you permission to propagate or
modify any covered work. These actions infringe copyright if you do
not accept this License. Therefore, by modifying or propagating a
covered work, you indicate your acceptance of this License to do so.
10. Automatic Licensing of Downstream Recipients.
Each time you convey a covered work, the recipient automatically
receives a license from the original licensors, to run, modify and
propagate that work, subject to this License. You are not responsible
for enforcing compliance by third parties with this License.
An "entity transaction" is a transaction transferring control of an
organization, or substantially all assets of one, or subdividing an
organization, or merging organizations. If propagation of a covered
work results from an entity transaction, each party to that
transaction who receives a copy of the work also receives whatever
licenses to the work the party's predecessor in interest had or could
give under the previous paragraph, plus a right to possession of the
Corresponding Source of the work from the predecessor in interest, if
the predecessor has it or can get it with reasonable efforts.
You may not impose any further restrictions on the exercise of the
rights granted or affirmed under this License. For example, you may
not impose a license fee, royalty, or other charge for exercise of
rights granted under this License, and you may not initiate litigation
(including a cross-claim or counterclaim in a lawsuit) alleging that
any patent claim is infringed by making, using, selling, offering for
sale, or importing the Program or any portion of it.
11. Patents.
A "contributor" is a copyright holder who authorizes use under this
License of the Program or a work on which the Program is based. The
work thus licensed is called the contributor's "contributor version".
A contributor's "essential patent claims" are all patent claims
owned or controlled by the contributor, whether already acquired or
hereafter acquired, that would be infringed by some manner, permitted
by this License, of making, using, or selling its contributor version,
but do not include claims that would be infringed only as a
consequence of further modification of the contributor version. For
purposes of this definition, "control" includes the right to grant
patent sublicenses in a manner consistent with the requirements of
this License.
Each contributor grants you a non-exclusive, worldwide, royalty-free
patent license under the contributor's essential patent claims, to
make, use, sell, offer for sale, import and otherwise run, modify and
propagate the contents of its contributor version.
In the following three paragraphs, a "patent license" is any express
agreement or commitment, however denominated, not to enforce a patent
(such as an express permission to practice a patent or covenant not to
sue for patent infringement). To "grant" such a patent license to a
party means to make such an agreement or commitment not to enforce a
patent against the party.
If you convey a covered work, knowingly relying on a patent license,
and the Corresponding Source of the work is not available for anyone
to copy, free of charge and under the terms of this License, through a
publicly available network server or other readily accessible means,
then you must either (1) cause the Corresponding Source to be so
available, or (2) arrange to deprive yourself of the benefit of the
patent license for this particular work, or (3) arrange, in a manner
consistent with the requirements of this License, to extend the patent
license to downstream recipients. "Knowingly relying" means you have
actual knowledge that, but for the patent license, your conveying the
covered work in a country, or your recipient's use of the covered work
in a country, would infringe one or more identifiable patents in that
country that you have reason to believe are valid.
If, pursuant to or in connection with a single transaction or
arrangement, you convey, or propagate by procuring conveyance of, a
covered work, and grant a patent license to some of the parties
receiving the covered work authorizing them to use, propagate, modify
or convey a specific copy of the covered work, then the patent license
you grant is automatically extended to all recipients of the covered
work and works based on it.
A patent license is "discriminatory" if it does not include within
the scope of its coverage, prohibits the exercise of, or is
conditioned on the non-exercise of one or more of the rights that are
specifically granted under this License. You may not convey a covered
work if you are a party to an arrangement with a third party that is
in the business of distributing software, under which you make payment
to the third party based on the extent of your activity of conveying
the work, and under which the third party grants, to any of the
parties who would receive the covered work from you, a discriminatory
patent license (a) in connection with copies of the covered work
conveyed by you (or copies made from those copies), or (b) primarily
for and in connection with specific products or compilations that
contain the covered work, unless you entered into that arrangement,
or that patent license was granted, prior to 28 March 2007.
Nothing in this License shall be construed as excluding or limiting
any implied license or other defenses to infringement that may
otherwise be available to you under applicable patent law.
12. No Surrender of Others' Freedom.
If conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License. If you cannot convey a
covered work so as to satisfy simultaneously your obligations under this
License and any other pertinent obligations, then as a consequence you may
not convey it at all. For example, if you agree to terms that obligate you
to collect a royalty for further conveying from those to whom you convey
the Program, the only way you could satisfy both those terms and this
License would be to refrain entirely from conveying the Program.
13. Use with the GNU Affero General Public License.
Notwithstanding any other provision of this License, you have
permission to link or combine any covered work with a work licensed
under version 3 of the GNU Affero General Public License into a single
combined work, and to convey the resulting work. The terms of this
License will continue to apply to the part which is the covered work,
but the special requirements of the GNU Affero General Public License,
section 13, concerning interaction through a network will apply to the
combination as such.
14. Revised Versions of this License.
The Free Software Foundation may publish revised and/or new versions of
the GNU General Public License from time to time. Such new versions will
be similar in spirit to the present version, but may differ in detail to
address new problems or concerns.
Each version is given a distinguishing version number. If the
Program specifies that a certain numbered version of the GNU General
Public License "or any later version" applies to it, you have the
option of following the terms and conditions either of that numbered
version or of any later version published by the Free Software
Foundation. If the Program does not specify a version number of the
GNU General Public License, you may choose any version ever published
by the Free Software Foundation.
If the Program specifies that a proxy can decide which future
versions of the GNU General Public License can be used, that proxy's
public statement of acceptance of a version permanently authorizes you
to choose that version for the Program.
Later license versions may give you additional or different
permissions. However, no additional obligations are imposed on any
author or copyright holder as a result of your choosing to follow a
later version.
15. Disclaimer of Warranty.
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
16. Limitation of Liability.
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
SUCH DAMAGES.
17. Interpretation of Sections 15 and 16.
If the disclaimer of warranty and limitation of liability provided
above cannot be given local legal effect according to their terms,
reviewing courts shall apply local law that most closely approximates
an absolute waiver of all civil liability in connection with the
Program, unless a warranty or assumption of liability accompanies a
copy of the Program in return for a fee.
END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Programs
If you develop a new program, and you want it to be of the greatest
possible use to the public, the best way to achieve this is to make it
free software which everyone can redistribute and change under these terms.
To do so, attach the following notices to the program. It is safest
to attach them to the start of each source file to most effectively
state the exclusion of warranty; and each file should have at least
the "copyright" line and a pointer to where the full notice is found.
<one line to give the program's name and a brief idea of what it does.>
Copyright (C) <year> <name of author>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
Also add information on how to contact you by electronic and paper mail.
If the program does terminal interaction, make it output a short
notice like this when it starts in an interactive mode:
<program> Copyright (C) <year> <name of author>
This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
This is free software, and you are welcome to redistribute it
under certain conditions; type `show c' for details.
The hypothetical commands `show w' and `show c' should show the appropriate
parts of the General Public License. Of course, your program's commands
might be different; for a GUI interface, you would use an "about box".
You should also get your employer (if you work as a programmer) or school,
if any, to sign a "copyright disclaimer" for the program, if necessary.
For more information on this, and how to apply and follow the GNU GPL, see
<http://www.gnu.org/licenses/>.
The GNU General Public License does not permit incorporating your program
into proprietary programs. If your program is a subroutine library, you
may consider it more useful to permit linking proprietary applications with
the library. If this is what you want to do, use the GNU Lesser General
Public License instead of this License. But first, please read
<http://www.gnu.org/philosophy/why-not-lgpl.html>.
scripts/eot/PyEOT/README.md
+20
View File
@@ -0,0 +1,20 @@
# PyEOT
GNU Radio/Python-based decoder for EOT packets
This combination of a GNU Radio Companion flowgraph (EOT.grc) and accompanying pyeot.py will receive
and decode packets from the End-of-Train device. The GRC flowgraph should be run before running the pyeot script.
Note that pyeot.py MUST be run in Python 3.x.
ZeroMQ PUB/SUB sockets are used to transfer the bitstream from GRC to the script. It is set to localhost, but also works
over an internet connection. Requires installation of zmq package.
Receive frequnecy should be set 457.9375 MHz.
Note that this software is receive-only, and will not generate packets. It is intended only for passive monitoring.
This software does not decode packets from the Head-of-Train device.
This is a POC. No attempt is made to catch or handle errors. If the GRC flowchart crashes or the TCP connection is interrupted, the receiver script will not know about it, and will not automatically reconnect.
Also included are slides from my talk at DEFCON 26, and a WAV file with some packets to play with.
Not much if any testing has been done with this version.
scripts/eot/PyEOT/demo3eot.wav
BIN
View File
Binary file not shown.
scripts/eot/PyEOT/eot_decoder.py
+62
View File
@@ -0,0 +1,62 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
PyEOT End-of-Train Device Decoder
Copyright (c) 2018 Eric Reuter
This source file is subject of the GNU general public license.
history: 2018-08-09 Initial Version
purpose: Class to parse EOT packet and generate BCH checkbits
for verification.
Requires helpers.py
"""
import helpers
class EOT_decode():
def __init__(self, buffer):
self.packet = buffer[0:74]
self.frame_sync = self.packet[0:11]
self.data_block = self.packet[11:56]
self.batt_cond = (self.packet[13:15][::-1])
self.message_type = self.packet[15:18]
self.unit_addr = int((self.packet[18:35][::-1]), 2)
self.pressure = int((self.packet[35:42][::-1]), 2)
self.batt_charge = \
("{}%".format(int(int((self.packet[42:49][::-1]), 2) / 127 * 100)))
self.spare = self.packet[49]
self.valve_ckt_stat = self.packet[50]
self.conf_ind = self.packet[51]
self.turbine = self.packet[52]
self.motion = self.packet[53]
self.mkr_batt = self.packet[54]
self.mkr_light = self.packet[55]
self.checkbitsRx = self.packet[56:74]
self.batt_cond_dict = {"11": "OK",
"10": "Low",
"01": "Very Low",
"00": "Not Monitored"}
self.batt_cond_text = self.batt_cond_dict[self.batt_cond]
if (self.message_type == "111"):
if (self.conf_ind == "0"):
self.arm_status = "Arming"
else:
self.arm_status = "Armed"
else:
self.arm_status = "Normal"
self.generator = '1111001101000001111' # BCH generator polynomial
self.cipher_key = '101011011101110000' # XOR cipher key
self.data_block = helpers.reverse(self.data_block)
self.checkbits = helpers.checkbits(self.data_block, self.generator)
self.checkbits_cipher = helpers.xor(self.checkbits, self.cipher_key)
self.valid = (self.checkbits_cipher == self.checkbitsRx) # a match?
def get_packet(self):
return ''.join(self.packet)
scripts/eot/PyEOT/helpers.py
+78
View File
@@ -0,0 +1,78 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
PyEOT End-of-Train Device Decoder
Copyright (c) 2018 Eric Reuter
This source file is subject of the GNU general public license.
history: 2018-08-09 Initial Version
purpose: Misc. functions used by eot_decoder.py
Function XOR() and mod2div() adapted from:
https://www.geeksforgeeks.org/cyclic-redundancy-check-python/
"""
# XOR two strings of bytes representing binary symbols
def xor(a, b):
result = []
for i in range(len(b)):
if a[i] == b[i]:
result.append('0')
else:
result.append('1')
return ''.join(result)
# Reverse string
def reverse(data):
return ''.join(data[::-1])
# Perform modulo-2 division on two strings of binary symbols
def mod2div(dividend, divisor):
# Number of bits to be XORed at a time.
pick = len(divisor)
# Slicing the dividend to appropriate
# length for particular step
tmp = dividend[0:pick]
while pick < len(dividend):
if tmp[0] == '1':
# replace the dividend by the result
# of XOR and pull 1 bit down
tmp = xor(divisor[1:], tmp[1:]) + dividend[pick]
else: # If leftmost bit is '0'
# If the leftmost bit of the dividend (or the
# part used in each step) is 0, the step cannot
# use the regular divisor; we need to use an
# all-0s divisor.
tmp = xor(('0'*pick)[1:], tmp[1:]) + dividend[pick]
# increment pick to move further
pick += 1
# For the last n bits, we have to carry it out
# normally as increased value of pick will cause
# Index Out of Bounds.
if tmp[0] == '1':
tmp = xor(divisor[1:], tmp[1:])
else:
tmp = xor(('0'*pick)[1:], tmp[1:])
remainder = tmp
return remainder
# Calculate BCH checkbits
def checkbits(data, key):
appended_data = data + '0'*(len(key)-1) # Appends n-1 zeros at end of data
remainder = mod2div(appended_data, key)
return ''.join(remainder)
scripts/eot/PyEOT/pyeot.py
+69
View File
@@ -0,0 +1,69 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
PyEOT End-of-Train Device Decoder
Copyright (c) 2018 Eric Reuter
This source file is subject of the GNU general public license
history: 2018-08-09 Initial Version
purpose: Receives demodulated FFSK bitstream from GNU Radio, indentifes
potential packets, and passes them to decoder classes for
parsing and verification. Finally human-readable data are printed
to stdout.
Requires eot_decoder.py and helpers.py
"""
import datetime
import collections
from eot_decoder import EOT_decode
import zmq
# Socket to talk to server
context = zmq.Context()
sock = context.socket(zmq.SUB)
# create fixed length queue
queue = collections.deque(maxlen=256)
def printEOT(EOT):
localtime = str(datetime.datetime.now().
strftime('%Y-%m-%d %H:%M:%S.%f'))[:-3]
print("")
print("EOT {}".format(localtime))
# print(EOT.get_packet())
print("---------------------")
print("Unit Address: {}".format(EOT.unit_addr))
print("Pressure: {} psig".format(EOT.pressure))
print("Motion: {}".format(EOT.motion))
print("Marker Light: {}".format(EOT.mkr_light))
print("Turbine: {}".format(EOT.turbine))
print("Battery Cond: {}".format(EOT.batt_cond_text))
print("Battery Charge: {}".format(EOT.batt_charge))
print("Arm Status: {}".format(EOT.arm_status))
def main():
# Connect to GNU Radio and subscribe to stream
sock.connect("tcp://localhost:5555")
sock.setsockopt(zmq.SUBSCRIBE, b'')
while True:
newData = sock.recv() # get whatever data are available
for byte in newData:
queue.append(str(byte)) # append each new symbol to deque
buffer = '' # clear buffer
for bit in queue: # move deque contents into buffer
buffer += bit
if (buffer.find('10101011100010010') == 0): # look for frame sync
EOT = EOT_decode(buffer[6:]) # first 6 bits are bit sync
if (EOT.valid):
printEOT(EOT)
main()
scripts/eot/README.md
+74
View File
@@ -0,0 +1,74 @@
# istrain — EOT decode & reap
Stage A captures EOT-band bursts (457.9375 MHz, dongle 1001) with `vumon --clips`; Stage B decodes them here.
## What's here
- **`eot_scan.py`** — the decoder, TWO passes (2026-07-05): (1) fast/exact — whole file, fixed tone
pairs, exact sync (proven against `PyEOT/demo3eot.wav`, 3 valid packets); (2) adaptive/fuzzy
escalation for off-frequency emitters (OBSERVATIONS **O-11**) — burst located by FM-quieting, tone
pair built on the burst's spectral centroid, sync matched with ≤2 bit errors (BCH still gates).
Packets carry `dq: exact|fuzzy`; the card layer corroborates fuzzy singletons before trusting them.
`scan_file(path)` returns majority-vote-ordered packets (`[]` = not EOT). numpy required.
- **`../reaper.py`** — the scheduled janitor, now **generic/multi-band** (lives in `scripts/`, not here). EOT is wired as a band; BOT/mid are ready slots. See below.
- **`PyEOT/`** — vendored parser + BCH + demo recording by **Eric Reuter**, GPL (see `PyEOT/LICENSE`).
Source: <https://github.com/ereuter/PyEOT>. We use its packet parser and the reference wav.
## Decode a file by hand
```
python3 eot_scan.py PyEOT/demo3eot.wav # self-test → 3 packets (unit/pressure/motion…)
python3 eot_scan.py ~/istrain/eot_clips/eot_*.s16 # our captured bursts
```
## The reaper (scheduled cleanup) — `../reaper.py`
Generic janitor, split out of EOT (2026-06-23) so it can serve BOT/mid later. Per wired band it decodes each
finished clip (EOT → `~/istrain/eot_clips/`):
- **real** → move the recording to `<band>_clips/confirmed/`, enrich its detection in the band's log
- **noise** → delete the clip **and** remove its detection hit from the log (panel only shows real trains)
- **oversized (>32 MB)** → quarantine to `<band>_clips/oversized/`, never load (OOM guard)
Only scans clips older than 60s (never one mid-capture). `--dry-run` previews; `--band NAME` runs one band.
BOT/mid are commented placeholders in `reaper.py` (no decoder yet → nothing to reap). The EOT *decoder*
(`eot_scan.py`) stays separate, so it can be paused/iterated independently of the cleanup.
## Deploy the schedule (systemd --user, every 15 min)
```
cp ../../systemd/istrain-eot-reap.{service,timer} ~/.config/systemd/user/
systemctl --user daemon-reload
systemctl --user enable --now istrain-eot-reap.timer
systemctl --user list-timers istrain-eot-reap.timer # confirm it's scheduled
```
## DPU decode plan (2026-06-24) — the plan-for-a-plan
Mid-train DPU rides UHF 452/457 (NOT ~900 — see `radio/research/railroad-eot-atcs.md` 06-24 correction),
1200-baud FFSK, same modulation family as EOT. Channel = lead loco road-number last digit:
452.925 (3/7) · 452.950 (2/6) · 457.925 (1/5/9) · 457.950 (0/4/8). So most of the chain is **already built**.
**We can't write this yet — we have no DPU sample.** This is the staged plan to execute once a live burst
is in hand (the rooftop antenna is the gate, same as EOT). Stages 1–2 are shared with EOT; only stage 3 is new:
1. **Isolate the sub-channel (the one genuinely-new RF step).** ✅ **BUILT 2026-06-24** —
`../iq_channelize.py`: reads `rtl_sdr` CU8 IQ, NCO-mixes + 151-tap FIR-LPFs + decimates each sub-channel
(center + ±12.5 kHz), presence = in-channel power over an adaptive floor, emits a discriminated s16 clip
per burst. **Routing + isolation proven on synthetic IQ** (`--selftest`); FM-demod audio can't do this
(±12.5 kHz becomes a DC bias, not a separable tone). Opt-in via `HOP_IQ=1` on scan-hop (FM path stays the
fallback). This same channelizer is what lets detection *label* a burst DPU-vs-HOT — distinct-ID and decode
are one job. **Unvalidated against a real DPU burst** until the antenna's up — same gate as live EOT.
2. **FFSK demod → bits.** REUSE `eot_scan.py`'s tone discriminator + 1200-baud clock recovery unchanged —
DPU is the same Bell-202-style FFSK as EOT.
3. **DPU frame parse (the new code).** DPU packets are a **different frame format** than EOT (control vs
telemetry — why railfans run SoftDPU separate from SoftEOT). Need to source the DPU packet layout
(SoftDPU / open references) and write a parser + its error-check. PyEOT's BCH is EOT-specific; verify
whether DPU uses the same FEC before reusing it.
**Validation:** no DPU reference recording ships with PyEOT (its `demo3eot.wav` is EOT only). First real
captured DPU burst becomes our reference; until then, stage 3 is unverifiable — don't trust it blind.
**For the mission, none of this is required** — detection already counts DPU via the 452/457 legs
(`../correlate.py`). Decode is the "what is the lead unit telling the mid-consist" luxury, not the commute signal.
## Status (2026-06-23)
Decoder proven on the reference. **No real EOT received yet** on the desk 6" whip — a range limit, not a
code one; EOT waits on the rooftop **DPD EOTD vertical** (tuned 452+457, the chosen feed for 1001 — `../../ANTENNA-UPGRADE.md`). Reaper split to the generic
`../reaper.py` (2026-06-23) and is currently **PAUSED** (timer stopped) to preserve a capture flurry for
study — resume with `systemctl --user start istrain-eot-reap.timer`. (Promote-candidate: `eot_scan.py` is
generic enough for the radio engine; living here in istrain until it earns the move.)
scripts/eot/eot_scan.py
+169
View File
@@ -0,0 +1,169 @@
#!/usr/bin/env python3
# EOT scan/decode (numpy). FSK audio (rtl_fm .s16 @ 48k, or a wav) -> vectorized tone-energy discriminator
# -> 1200-baud clock recovery -> PyEOT frame-sync + parser + BCH check. Pure analysis — does not touch a radio.
#
# TWO passes (2026-07-05, the O-11 crack):
# 1. FAST/EXACT — the proven original: whole file, fixed tone pairs, exact sync. Catches on-frequency
# EOTs (proven against PyEOT's demo3eot.wav, 3 packets, and weeks of live passages).
# 2. ADAPTIVE/FUZZY escalation — for the off-frequency emitters the fast path is deaf to. The 07-05
# ground-truthed dwelling train (unit 76029, CUT at the crossing) transmitted ~600-800 Hz HIGH of
# our bin: its audio tones sat ~(2000,2400), nowhere near the fixed pairs, in short ~100 ms bursts
# drowned by clip noise. Fix: find the burst by FM-quieting (carrier = LOW audio rms), estimate its
# tone hump by spectral centroid, build tone pairs around it, and allow <=2 bit errors in the sync
# word (frames still gate on the PyEOT/BCH validity check, so a fuzzy sync never invents a packet).
# Validated on 07-04/07-05 dwell clips: 0/357 -> 26/357 morning decodes; both dwell flavors read
# (psi ~62 held = solid stopped train vs psi 0 held = cut/air-dumped). See OBSERVATIONS O-11.
#
# Packets carry dq: "exact" | "fuzzy" (decode-quality provenance — a lone fuzzy decode can be a BCH
# collision; corroboration [>=2 records or a head-addr match] is enforced at the card layer, not here).
# Consensus: the returned list is majority-vote ordered, so callers taking pk[0] get the agreed packet.
# CLI: eot_scan.py <file.wav|file.s16> ...
# API: scan_file(path) -> [ {unit,pressure,motion,marker,turbine,batt,charge,arm,dq}, ... ] (empty = not EOT)
import sys, os, wave
from collections import Counter
import numpy as np
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "PyEOT"))
from eot_decoder import EOT_decode
SYNC = '10101011100010010'; BAUD = 1200
_INV = str.maketrans('01', '10')
_SYNC_A = np.array([int(c) for c in SYNC], np.int8)
TONE_PAIRS = [(1200, 1800), (1300, 1700), (1100, 1900)]
FUZZY_MAX_ERR = 2 # sync-word bit errors tolerated on the escalation path (BCH still gates)
def load(path):
if path.lower().endswith('.wav'):
w = wave.open(path, 'rb'); sw = w.getsampwidth(); fr = w.getframerate(); raw = w.readframes(w.getnframes()); w.close()
if sw == 1: return (np.frombuffer(raw, np.uint8).astype(np.float64) - 128) / 128.0, fr
return np.frombuffer(raw, np.int16).astype(np.float64) / 32768.0, fr
return np.frombuffer(open(path, 'rb').read(), np.int16).astype(np.float64) / 32768.0, 48000
def discriminator(x, fr, flo, fhi, W):
n = np.arange(len(x))
def energy(f):
c = np.concatenate(([0j], np.cumsum(x * np.exp(-2j * np.pi * f * n / fr))))
return np.abs(c[W:] - c[:-W]) ** 2
return energy(fhi) - energy(flo)
def bitstring(d, fr, thresh=None):
t = np.median(d) if thresh is None else thresh
sl = (d > t).astype(np.int8)
spb = fr / BAUD; out = []; phase = spb / 2.0; prev = int(sl[0])
for i in range(1, len(sl)):
b = int(sl[i])
if b != prev: phase = spb / 2.0; prev = b
else:
phase -= 1.0
if phase <= 0: out.append(b); phase += spb
return ''.join('1' if b else '0' for b in out)
def _decode_at(bb, i, seen, out, dq):
for off in range(0, 13):
seg = bb[i + off:i + off + 80]
if len(seg) < 74: break
try:
e = EOT_decode(seg)
if e.valid:
key = (e.unit_addr, e.pressure, int(e.motion))
if key not in seen:
seen.add(key)
out.append(dict(unit=e.unit_addr, pressure=e.pressure, motion=int(e.motion),
marker=int(e.mkr_light), turbine=int(e.turbine),
batt=e.batt_cond_text, charge=e.batt_charge, arm=e.arm_status, dq=dq))
except Exception: pass
def _packets(bs, fuzzy=False, seen=None, out=None):
seen = set() if seen is None else seen
out = [] if out is None else out
dq = "fuzzy" if fuzzy else "exact"
for bb in (bs, bs.translate(_INV)):
if fuzzy and len(bb) >= len(SYNC):
a = np.array([int(c) for c in bb], np.int8)
win = np.lib.stride_tricks.sliding_window_view(a, len(SYNC))
for i in np.where((win != _SYNC_A).sum(axis=1) <= FUZZY_MAX_ERR)[0]:
_decode_at(bb, int(i), seen, out, dq)
else:
i = bb.find(SYNC)
while i != -1:
_decode_at(bb, i, seen, out, dq)
i = bb.find(SYNC, i + 1)
return out
def _quiet_segs(x, fr, thresh):
"""Carrier segments found by FM-quieting: demodulated NOISE is loud, a keyed carrier is quiet.
Returns [(a,b)] sample ranges (>=60 ms quiet runs, padded), so the decoder works the burst alone
instead of drowning a ~100 ms frame in a 0.7 s clip's noise (the clock recovery killer)."""
win = int(fr * 0.010)
n = len(x) // win
if n < 8: return []
r = np.array([x[i*win:(i+1)*win].std() for i in range(n)])
med = np.median(r)
if med <= 0: return []
quiet = r < med * thresh
segs = []; cur = 0; start = 0
for i, v in enumerate(quiet):
if v:
if cur == 0: start = i
cur += 1
else:
if cur >= 6: segs.append((start, start + cur))
cur = 0
if cur >= 6: segs.append((start, start + cur))
return [(max(0, (a - 2) * win), (b + 2) * win) for a, b in segs]
def _hump_center(seg, fr):
"""Spectral centroid of the burst's tone hump (600-3500 Hz) — where this emitter's FSK audio
actually sits. An off-frequency carrier shifts the whole hump (O-11: +600-800 Hz)."""
w = np.hanning(len(seg))
F = np.abs(np.fft.rfft(seg * w)) ** 2
f = np.fft.rfftfreq(len(seg), 1 / fr)
m = (f > 600) & (f < 3500)
P = F[m]
return float((f[m] * P).sum() / P.sum()) if P.sum() > 0 else None
def _consensus(pk):
"""Majority-vote order so pk[0] is the agreed packet (fuzzy sync can add a stray BCH collision)."""
if len(pk) <= 1: return pk
votes = Counter((d["unit"], d["pressure"], d["motion"]) for d in pk)
return sorted(pk, key=lambda d: -votes[(d["unit"], d["pressure"], d["motion"])])
def scan_file(path):
x, fr = load(path); W = max(8, int(round(fr / BAUD)))
# pass 1 — FAST/EXACT (the proven original, unchanged behavior)
for flo, fhi in TONE_PAIRS:
bs = bitstring(discriminator(x, fr, flo, fhi, W), fr)
if SYNC in bs or SYNC in bs.translate(_INV):
pk = _packets(bs)
if pk: return _consensus(pk)
# pass 2 — ADAPTIVE/FUZZY escalation (off-frequency / short-burst emitters)
for thresh in (0.65, 0.8):
for a, b in _quiet_segs(x, fr, thresh):
seg = x[a:b]
if len(seg) < int(fr * 0.05): # <50 ms can't hold a frame
continue
c = _hump_center(seg, fr)
cands = []
if c:
for cc in (c, c - 150, c + 150):
for half in (200, 250, 300, 350):
if cc - half > 300: cands.append((cc - half, cc + half))
cands += TONE_PAIRS + [(2000, 2400)]
for flo, fhi in cands:
d = discriminator(seg, fr, flo, fhi, W)
for thr in (None, d.mean()): # median + mean slicers (skewed short bursts)
pk = _packets(bitstring(d, fr, thr), fuzzy=True)
if pk: return _consensus(pk)
return []
def run(p):
label = os.path.basename(p)
try: pk = scan_file(p)
except Exception as e: print(f"[{label}] err: {e}"); return 0
for d in pk:
print(f" ✅ [{label}] unit={d['unit']} pressure={d['pressure']}psig motion={d['motion']} "
f"marker={d['marker']} turbine={d['turbine']} batt={d['batt']}/{d['charge']} arm={d['arm']} ({d['dq']})")
if not pk: print(f"[{label}] no valid EOT")
return len(pk)
if __name__ == '__main__':
t = sum(run(p) for p in sys.argv[1:]); print(f"\n=== {t} valid EOT packet(s) ===")
scripts/eot/hot_explore.py
+72
View File
@@ -0,0 +1,72 @@
#!/usr/bin/env python3
# hot_explore.py — Head-of-Train (HOT, 452.9375) reverse-engineering explorer. STANDALONE / offline.
#
# STATUS (2026-06-26): structure cracked, fields NOT decoded. There is no public HOT bit-map (see
# ../../EOT-HOT-PROTOCOL.md) — this is pure RE off our own bot_clips/*.s16. What works so far:
# - reuse the proven EOT FFSK front-end (eot_scan: 1200-baud, mark 1200 / space 1800 Hz)
# - a strong HOT burst shows a ~440-bit alternating PREAMBLE (spec: 456) then a 64-bit REPEAT PERIOD
# (spec: a 64-bit block sent 3x). Majority-voting the 3 internal copies -> a CLEAN 64-bit block.
# - artifact test: different sessions give DIFFERENT clean blocks (real frames, not a stuck carrier).
# NEXT (open): pin the 24-bit frame-sync word, locate the 17-bit addressed-EOT-ID in the 30-bit payload
# (validate against a known EOT unit heard concurrently), recover the HOT BCH(33) polynomial.
#
# Usage: python3 hot_explore.py ~/istrain/bot_clips/bot_*.s16 (defaults to a strong-clip sample)
import sys, os, glob, numpy as np
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # eot_scan lives next door
import eot_scan as E
def best_bits(path):
"""Demod FFSK to raw bits; pick the tone pair giving the longest alternating preamble (= bit-sync)."""
x, fr = E.load(path); W = max(8, int(round(fr / E.BAUD)))
best = None
for flo, fhi in E.TONE_PAIRS:
bs = E.bitstring(E.discriminator(x, fr, flo, fhi, W), fr)
run = mx = 0
for i in range(1, len(bs)):
if bs[i] != bs[i-1]: run += 1; mx = max(mx, run)
else: run = 0
if best is None or mx > best[0]: best = (mx, bs)
return best
def preamble_end(bs):
run = mx = end = 0
for i in range(1, len(bs)):
if bs[i] != bs[i-1]:
run += 1
if run > mx: mx = run; end = i + 1
else: run = 0
return end
def vote(bits):
"""Find (period, offset) maximizing 3-way agreement among 3 consecutive blocks, then majority-vote."""
a = np.array([int(c) for c in bits]); best = None
for p in range(60, 69):
for off in range(0, min(len(a) - 3*p, 30)) if len(a) >= 3*p else []:
b1, b2, b3 = a[off:off+p], a[off+p:off+2*p], a[off+2*p:off+3*p]
agree = np.sum(b1 == b2) + np.sum(b1 == b3) + np.sum(b2 == b3)
if best is None or agree > best[0]: best = (agree, p, off, (b1, b2, b3))
if best is None: return None
agree, p, off, (b1, b2, b3) = best
cons = ((b1.astype(int) + b2 + b3) >= 2).astype(int)
errs = [int(np.sum(b != cons)) for b in (b1, b2, b3)]
return p, off, ''.join(map(str, cons.tolist())), errs, agree, 3 * p
def explore(path):
mx, bs = best_bits(path)
body = bs[preamble_end(bs):]
if len(body) < 3 * 60:
print(f"{os.path.basename(path):30s} preamble={mx:3d}b (body too short: {len(body)}b — likely not a frame)")
return None
r = vote(body)
if not r: return None
p, off, cons, errs, agree, maxa = r
print(f"{os.path.basename(path):30s} preamble={mx:3d}b period={p} off={off} "
f"3way={agree}/{maxa}={agree/maxa:.0%} vote-err={errs} block={cons}")
return cons
if __name__ == '__main__':
paths = sys.argv[1:] or sorted(glob.glob(os.path.expanduser("~/istrain/bot_clips/bot_20260625_225*.s16")))[:4]
blocks = [b for b in (explore(p) for p in paths) if b]
if len(blocks) > 1:
print("\nartifact test — distinct consensus blocks (same everywhere = artifact; varying = real):",
len(set(blocks)))
scripts/iq_channelize.py
+314
View File
@@ -0,0 +1,314 @@
#!/usr/bin/env python3
# iq_channelize.py — split ONE RTL-SDR IQ dwell into several sub-channels by frequency offset, detect
# presence per sub-channel, and (on a burst) save a decode-ready clip. This is the IQ swap that lets dongle
# 1001 see mid-train DPU distinctly: DPU rides ±12.5 kHz off the HOT/EOT centers (452.925/.950, 457.925/.950),
# which FM-demod audio blurs into one unrecoverable channel. Raw IQ keeps the whole slice; we digitally
# down-convert each sub-channel and judge it on its own.
#
# 1001 stays a 2-position hop (452.9375 / 457.9375 — 5 MHz apart, wider than the tuner can grab at once).
# This script only changes what each dwell EXTRACTS:
# 452 dwell -> BOT (center) + MID (-12.5k, +12.5k)
# 457 dwell -> EOT (center) + MID (-12.5k, +12.5k)
# So MID is fed from BOTH dwells (a train uses one DPU channel, picked by its lead loco's road number).
#
# PRESENCE = in-channel carrier power above an adaptive noise floor (NOT FM-audio envelope) — the correct,
# squelch-polarity-proof presence metric, and what makes clean channel isolation testable. We STILL emit a
# discriminated s16 audio clip per burst so eot_scan.py / listening keep working unchanged.
#
# Reads CU8 IQ on stdin: rtl_sdr -d 1001 -f 452.9375M -s 240000 -g 40 - | iq_channelize.py --dwell 452
# RUNTIME helper — built by the maintainer, you run it. numpy only.
# No radio needed to prove the DSP: python3 iq_channelize.py --selftest
#
# Detection mirrors vumon.py's adaptive floor+margin (promote-candidate: share one detector module later).
import os, sys, time, json, math
import numpy as np
FS_IN = int(os.environ.get("IQ_FS", "240000")) # rtl_sdr sample rate (Hz)
DECIM = int(os.environ.get("IQ_DECIM", "5")) # 240k -> 48k channel rate
LP_CUT = float(os.environ.get("IQ_LPCUT", "6000")) # per-channel low-pass cutoff (Hz); rejects the ±12.5k neighbor
NTAPS = int(os.environ.get("IQ_TAPS", "151")) # FIR length; ~53 dB stopband -> clean sub-channel isolation
MARGIN = float(os.environ.get("IQ_MARGIN", "6")) # fire this many dB above the rolling floor
# Dial 2 (2026-07-10): only SAVE mid .s16 audio for bursts this many dB above the floor. The booster
# floor is characterized furniture; its audio adds nothing. 0 = save all (the pre-dial behavior).
MID_CLIP_MIN_OVER = float(os.environ.get("MID_CLIP_MIN_OVER", "6"))
FLOOR_A = 0.02 # noise-floor EMA rate (updated only while quiet)
CHUNK = int(os.environ.get("IQ_CHUNK", "24000")) # IQ samples per read (~0.1s @ 240k); multiple of DECIM
FS_AUD = FS_IN // DECIM
HOME = os.path.expanduser("~")
IST = os.path.join(HOME, "istrain")
LEVEL = os.path.join(IST, "level.json") # live meter the dashboard polls (vumon-compatible shape)
_last_level = 0.0
def write_level(now, db, active, label, peak, floor):
"""Throttled (~5 Hz) atomic write of the instantaneous level — same shape vumon writes, so the
dashboard meter works unchanged. In IQ mode we report the HOTTEST sub-channel (label = bot/eot/mid)."""
global _last_level
if now - _last_level < 0.2:
return
_last_level = now
try:
with open(LEVEL + ".tmp", "w") as f:
json.dump({"ts": round(now, 2), "db": round(db, 1), "active": bool(active),
"label": label, "peak": (round(peak, 1) if (active and peak is not None) else None),
"floor": (round(floor, 1) if floor is not None else None)}, f)
os.replace(LEVEL + ".tmp", LEVEL)
except OSError:
pass
def design_lpf(cut, fs, ntaps):
"""Windowed-sinc low-pass FIR (Hamming). Linear phase, ~53 dB stopband at 151 taps."""
n = np.arange(ntaps) - (ntaps - 1) / 2.0
fc = cut / fs # normalized cutoff (cycles/sample)
h = 2 * fc * np.sinc(2 * fc * n)
h *= np.hamming(ntaps)
return (h / h.sum()).astype(np.float64)
class SubChannel:
"""One frequency-offset sub-channel: NCO mix -> LPF -> decimate -> power-presence + discriminated clip."""
def __init__(self, offset_hz, label, jsonl, clips_dir, sub=None):
self.offset = float(offset_hz)
self.label = label
self.sub = sub # sub-channel freq tag (e.g. "457925") — de-collides + labels mid clips
self.jsonl = jsonl
self.clips = clips_dir
if clips_dir:
os.makedirs(clips_dir, exist_ok=True)
self.h = design_lpf(LP_CUT, FS_IN, NTAPS)
# NCO: a fixed per-chunk exponential * a carried start-phase scalar (unit magnitude, renormalized)
self._base = np.exp(-2j * math.pi * self.offset * np.arange(CHUNK) / FS_IN)
self._step = complex(np.exp(-2j * math.pi * self.offset * CHUNK / FS_IN))
self._ph = 1 + 0j
self._hist = np.zeros(NTAPS - 1, dtype=np.complex128) # filter state (overlap)
self._lastc = 0 + 0j # FM discriminator state (prev decimated sample)
# detector state
self.floor = None
self.active = False
self.since = 0.0
self.peak = -999.0
self.last_db = -999.0
self.last_on = False
self._clipbuf = [] # discriminated audio (int16) for the live burst
self._preroll = [] # ~0.5s pre-roll of audio chunks
def _mix_filter_decimate(self, iq):
"""iq: complex chunk @ FS_IN -> decimated complex @ FS_AUD for this sub-channel."""
mixed = iq * self._base * self._ph
self._ph *= self._step
if abs(self._ph) > 1e-6:
self._ph /= abs(self._ph) # kill slow magnitude drift
x = np.concatenate((self._hist, mixed))
y = np.convolve(x, self.h, "valid") # len == len(mixed)
self._hist = mixed[-(NTAPS - 1):]
return y[::DECIM]
def _discriminate(self, yc):
"""Polar FM discriminator (same core as rtl_fm -M fm) -> int16 audio for clips."""
prev = np.empty_like(yc)
prev[0] = self._lastc
prev[1:] = yc[:-1]
self._lastc = yc[-1]
ang = np.angle(yc * np.conj(prev)) # [-pi, pi]
return np.clip(ang / math.pi * 32767, -32768, 32767).astype(np.int16)
def process(self, iq, now):
yc = self._mix_filter_decimate(iq)
pw = float(np.mean(np.abs(yc) ** 2)) + 1e-12
db = 10.0 * math.log10(pw)
audio = self._discriminate(yc)
if self.floor is None:
self.floor = db
on = db > self.floor + MARGIN
if not on:
self.floor += FLOOR_A * (db - self.floor) # track floor ONLY while quiet
self.last_db = db; self.last_on = on # for the live meter (hottest-channel pick)
if on and not self.active:
self.active = True; self.since = now; self.peak = db
if self.clips:
self._clipbuf = list(self._preroll); self._clipbuf.append(audio)
elif on:
self.peak = max(self.peak, db)
if self.clips:
self._clipbuf.append(audio)
elif self.active:
self.active = False
self._emit(now)
elif self.clips:
self._preroll.append(audio)
if len(self._preroll) > 5: # ~0.5s pre-roll
self._preroll.pop(0)
return db, on
def _emit(self, now):
# height above the tracked noise/booster floor = how "train-shaped" this burst is (0 ≈ floor).
over = round(self.peak - self.floor, 1) if self.floor is not None else None
rec = {"ts": round(self.since, 2), "dur": round(now - self.since, 2),
"peak": round(self.peak, 1), "over": over}
if self.sub: # which ±12.5k sub-channel (457925 = the booster)
rec["sub"] = self.sub
if self.jsonl: # the durable ROW always writes — the record is cheap
try:
with open(self.jsonl, "a") as f:
f.write(json.dumps(rec) + "\n")
except OSError:
pass
# Dial 2: don't SAVE floor-level mid AUDIO (the booster). Only gates the mid label's .s16 write;
# eot/bot always save. Reversible via MID_CLIP_MIN_OVER=0. The row above still logged the burst.
if self.label == "mid" and over is not None and over < MID_CLIP_MIN_OVER:
self._clipbuf = []
return
if self.clips and self._clipbuf:
try:
# sub tag in the filename: de-collides the two mid sub-channels (they could share a
# start-second and overwrite each other) AND sorts clips by frequency. "mid457925_…".
prefix = self.label + (self.sub or "")
fn = os.path.join(self.clips, "%s_%s.s16" % (prefix,
time.strftime("%Y%m%d_%H%M%S", time.localtime(self.since))))
np.concatenate(self._clipbuf).tofile(fn)
except OSError:
pass
self._clipbuf = []
return rec
def dwell_channels(dwell):
"""Map a dwell (452|457) to its sub-channels. MID writes midtrain.jsonl from either dwell.
Each mid sub-channel is tagged with its absolute frequency in kHz (dwell center ±12.5), so the
fixed 457.925 booster carrier separates in the log + filenames from any real DPU frequency."""
if dwell == "452":
center = ("bot", os.path.join(IST, "bot.jsonl"), os.path.join(IST, "bot_clips"))
cen_khz = 452937.5 # AAR 452.9375 MHz (head-end / BOT)
elif dwell == "457":
center = ("eot", os.path.join(IST, "eot.jsonl"), os.path.join(IST, "eot_clips"))
cen_khz = 457937.5 # AAR 457.9375 MHz (EOT); −12.5 = 457.925 booster
else:
sys.exit("iq_channelize: --dwell must be 452 or 457")
midlog = os.path.join(IST, "midtrain.jsonl")
midclip = os.path.join(IST, "mid_clips")
sub = lambda off: str(int(cen_khz + off / 1000.0)) # e.g. 457937.5 + (-12.5) = 457925
return [
SubChannel(0, center[0], center[1], center[2]),
SubChannel(-12500, "mid", midlog, midclip, sub=sub(-12500)),
SubChannel(+12500, "mid", midlog, midclip, sub=sub(+12500)),
]
def run_stream(channels):
"""Read CU8 IQ from stdin, drive each sub-channel. CU8 = interleaved uint8 I,Q centered at 127.5."""
nbytes = CHUNK * 2
fd = sys.stdin.buffer
sys.stderr.write("[iq_channelize] %d ch, fs=%d decim=%d -> %d Hz, lpf=%.0f, %d taps | LEVEL=%s\n"
% (len(channels), FS_IN, DECIM, FS_AUD, LP_CUT, NTAPS, LEVEL)); sys.stderr.flush()
while True:
raw = fd.read(nbytes)
if not raw or len(raw) < nbytes:
break
u = np.frombuffer(raw, dtype=np.uint8).astype(np.float64)
iq = (u[0::2] - 127.5) + 1j * (u[1::2] - 127.5)
iq /= 127.5
now = time.time()
for ch in channels:
ch.process(iq, now)
hot = max(channels, key=lambda c: c.last_db) # meter shows the hottest sub-channel
write_level(now, hot.last_db, any(c.active for c in channels),
hot.label, (hot.peak if hot.active else None), hot.floor)
# ---- synthetic self-test: prove routing + isolation without a radio ----------------------------------
def _fsk_burst(t, f_off, on_lo, on_hi, amp=0.6):
"""A crude FFSK-ish burst at carrier offset f_off, active only in [on_lo, on_hi]."""
base = 2 * math.pi * f_off * t
sub = 1200.0 * np.sign(np.sin(2 * math.pi * 90 * t)) # toy ±1200 Hz audio FSK
phase = base + np.cumsum(2 * math.pi * sub / FS_IN)
mask = ((t >= on_lo) & (t < on_hi)).astype(np.float64)
return amp * mask * np.exp(1j * phase)
def selftest():
dur = 1.2
n = int(FS_IN * dur)
t = np.arange(n) / FS_IN
rng = np.random.default_rng(7)
noise = 0.05 * (rng.standard_normal(n) + 1j * rng.standard_normal(n))
sig = noise.copy()
sig += _fsk_burst(t, 0.0, 0.10, 0.25) # CENTER (bot/eot) -> 0.10..0.25
sig += _fsk_burst(t, +12500.0, 0.45, 0.60) # MID (+12.5k) -> 0.45..0.60
sig += _fsk_burst(t, -12500.0, 0.80, 0.95) # MID (-12.5k) -> 0.80..0.95
# build channels with NO file output (capture detections in memory)
chans = [SubChannel(0, "center", None, None),
SubChannel(-12500, "mid_lo", None, None),
SubChannel(+12500, "mid_hi", None, None)]
hits = {c.label: [] for c in chans}
orig = {c.label: c._emit for c in chans}
def make_cap(c):
def cap(now):
hits[c.label].append((round(c.since, 2), round(now - c.since, 2), round(c.peak, 1)))
return orig[c.label](now)
return cap
for c in chans:
c._emit = make_cap(c)
# convert absolute sample stream to CU8 and feed chunk by chunk (using a fake monotonic clock)
u = np.empty(2 * n, dtype=np.float64)
u[0::2] = np.real(sig) * 127.5 + 127.5
u[1::2] = np.imag(sig) * 127.5 + 127.5
cu8 = np.clip(u, 0, 255).astype(np.uint8).tobytes()
nbytes = CHUNK * 2
pos = 0; k = 0
while pos + nbytes <= len(cu8):
raw = cu8[pos:pos + nbytes]; pos += nbytes
arr = np.frombuffer(raw, dtype=np.uint8).astype(np.float64)
iq = ((arr[0::2] - 127.5) + 1j * (arr[1::2] - 127.5)) / 127.5
now = k * CHUNK / FS_IN # synthetic clock (seconds)
for c in chans:
c.process(iq, now)
k += 1
for c in chans: # flush any still-active burst
if c.active:
c._emit((k * CHUNK) / FS_IN)
def near(hitlist, lo, hi):
return any(lo - 0.15 <= h[0] <= hi + 0.05 for h in hitlist)
ok = True
def check(name, cond):
nonlocal ok
print((" PASS " if cond else " FAIL ") + name)
ok = ok and cond
print("self-test — synthetic bursts: center@0.10-0.25, mid+12.5k@0.45-0.60, mid-12.5k@0.80-0.95")
print("detections:")
for c in chans:
print(" %-7s %s" % (c.label, hits[c.label]))
check("center detects its burst", near(hits["center"], 0.10, 0.25))
check("mid_hi (+12.5k) detects its burst", near(hits["mid_hi"], 0.45, 0.60))
check("mid_lo (-12.5k) detects its burst", near(hits["mid_lo"], 0.80, 0.95))
check("center ISOLATED from the two mid bursts", not near(hits["center"], 0.45, 0.60) and not near(hits["center"], 0.80, 0.95))
check("mid_hi ISOLATED from center + mid_lo", not near(hits["mid_hi"], 0.10, 0.25) and not near(hits["mid_hi"], 0.80, 0.95))
check("mid_lo ISOLATED from center + mid_hi", not near(hits["mid_lo"], 0.10, 0.25) and not near(hits["mid_lo"], 0.45, 0.60))
print("RESULT:", "PASS — routing + isolation hold" if ok else "FAIL")
return 0 if ok else 1
def main():
a = sys.argv[1:]
if "--selftest" in a:
sys.exit(selftest())
dwell = None
while a:
t = a.pop(0)
if t == "--dwell": dwell = a.pop(0)
else: sys.exit("iq_channelize: unknown arg %r" % t)
if not dwell:
sys.exit("usage: rtl_sdr ... - | iq_channelize.py --dwell {452|457} (or --selftest)")
os.makedirs(IST, exist_ok=True)
run_stream(dwell_channels(dwell))
if __name__ == "__main__":
main()
scripts/iq_hop.py
+244
View File
@@ -0,0 +1,244 @@
#!/usr/bin/env python3
# iq_hop.py — RETUNE-IN-PLACE 452/457 hop on dongle 1001, single never-closed USB handle.
#
# WHY THIS EXISTS. The old scan-hop tears down + respawns the capture (rtl_fm/rtl_sdr -d 1001) every
# dwell. That reopen — rtlsdr_open() + USB claim/reset on the fragile Realtek 1001 — is what wedged the
# dongle two nights running (06-24, 06-25): "one good read per open, stream stalls after ~70 reopens."
# The retune itself was never the problem; the open/close USB transaction is. So here we open ONCE and
# never close: rtl_tcp holds 1001 open forever and streams CU8 IQ on a local socket; we flip 452<->457 by
# sending its SET_FREQUENCY command (a tuner I2C write on the already-claimed handle) — zero reopen.
#
# WHAT IT REUSES. The detection DSP is iq_channelize.py UNCHANGED (imported): per-dwell SubChannel set
# (center bot/eot + the two +/-12.5k mid-train DPU sub-channels), adaptive floor+margin power presence,
# FM-discriminated s16 clips for eot_scan, and the vumon-shaped level.json. Same outputs, same logs
# (bot.jsonl / eot.jsonl / midtrain.jsonl) — the dashboard and decoder don't know anything changed.
#
# WHAT IT ADDS (the only new logic): own rtl_tcp's lifecycle, send the retune command, DRAIN a short
# settle window after each retune (stale in-flight IQ from the old frequency is still buffered), route
# live IQ into the active dwell's channel set, and FLUSH any burst still open at the dwell boundary so it
# can't bleed across the ~30s the band is unwatched.
#
# Prove the routing + boundary-flush with NO radio: python3 iq_hop.py --selftest
# (the DSP's channel isolation is already proven by iq_channelize.py --selftest)
#
# RUNTIME helper — built by the maintainer, operated as the istrain-scan-hop service. numpy only.
# (a generic "one open, retune-in-place, time-division channelizer").
import os, sys, time, signal, socket, struct, select, subprocess
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import iq_channelize as iqc # the DSP: SubChannel, dwell_channels, FS_IN, CHUNK, write_level
# --- knobs (mirror scan-hop's names so operating it feels the same) ----------------------------------
DWELL_452 = int(os.environ.get("HOP_DWELL_452", "7")) # BOT/head — detection only (no decoder yet); lighter
DWELL_457 = int(os.environ.get("HOP_DWELL_457", "14")) # EOT/tail — the decodable ID + the departure bit; weight it
SETTLE = float(os.environ.get("HOP_SETTLE", "0.4")) # post-retune drain: discard stale in-flight IQ
GAIN_DB = float(os.environ.get("HOP_GAIN", "40")) # manual tuner gain (dB); rtl_tcp wants tenths
FS = iqc.FS_IN # 240000 — MUST match the DSP's expected rate
PORT = int(os.environ.get("HOP_TCP_PORT", "12001")) # local rtl_tcp port (mnemonic: 1001)
DEV = os.environ.get("HOP_DEV", "1001") # dongle serial/index
NBYTES = iqc.CHUNK * 2 # CU8 bytes per processing chunk (I,Q uint8)
# (dwell tag, band center in Hz) — center == the bot/eot sub-channel; mid rides +/-12.5k off it.
DWELLS = [("452", 452_937_500, DWELL_452), ("457", 457_937_500, DWELL_457)]
# rtl_tcp control protocol: 1-byte command + big-endian uint32 param (struct command{u8;u32;}__packed__).
SET_FREQUENCY, SET_SAMPLERATE, SET_GAIN_MODE, SET_GAIN, SET_AGC_MODE = 0x01, 0x02, 0x03, 0x04, 0x08
def send_cmd(sock, cmd, param):
sock.sendall(struct.pack(">BI", cmd, param & 0xFFFFFFFF))
def recv_exact(sock, n):
"""Block until exactly n bytes are read; None if the stream ends (rtl_tcp died)."""
buf = bytearray()
while len(buf) < n:
b = sock.recv(n - len(buf))
if not b:
return None
buf += b
return bytes(buf)
def drain(sock, secs):
"""Read and DISCARD everything arriving for `secs` — clears stale old-frequency IQ that was already
in flight (USB transfer + socket buffers) when we retuned, so it can't pollute the new dwell."""
end = time.monotonic() + secs # O-6 FIX: drain length on the MONOTONIC clock — a wall-clock
while True: # step (NTP slew/jump) must not collapse the drain to instant
left = end - time.monotonic()
if left <= 0:
return True
r, _, _ = select.select([sock], [], [], left)
if r and not sock.recv(65536):
return False # rtl_tcp closed under us
def run_dwell(sock, chans, secs):
"""Feed live IQ into this band's SubChannel set for `secs`, then flush any open burst. Returns False
if the stream died. State (adaptive floor, etc.) persists across visits to the same band by design."""
end = time.monotonic() + secs # O-6 FIX: dwell LENGTH off the monotonic clock (step-proof).
while time.monotonic() < end: # the per-sample `now = time.time()` below stays wall-clock —
# those are real-world data stamps for bot/eot/mid logs.
raw = recv_exact(sock, NBYTES)
if raw is None:
return False
u = iqc.np.frombuffer(raw, dtype=iqc.np.uint8).astype(iqc.np.float64)
iq = ((u[0::2] - 127.5) + 1j * (u[1::2] - 127.5)) / 127.5
now = time.time()
for ch in chans:
ch.process(iq, now)
hot = max(chans, key=lambda c: c.last_db) # meter = hottest sub-channel (vumon-shaped)
iqc.write_level(now, hot.last_db, any(c.active for c in chans),
hot.label, (hot.peak if hot.active else None), hot.floor)
now = time.time()
for ch in chans: # flush: don't let a burst span the off-period
if ch.active:
ch.active = False
ch._emit(now)
return True
def connect(retries=40, delay=0.5):
"""rtl_tcp enumeration is racy on this VBox passthrough (airband needed 20 retries at boot) — keep
trying to connect, then read+discard rtl_tcp's 12-byte 'RTL0' magic header."""
for i in range(retries):
try:
s = socket.create_connection(("127.0.0.1", PORT), timeout=2)
hdr = recv_exact(s, 12)
if not hdr or hdr[:4] != b"RTL0":
s.close()
raise OSError("bad/no rtl_tcp header")
sys.stderr.write("[iq_hop] connected to rtl_tcp (tuner header ok)\n"); sys.stderr.flush()
return s
except OSError:
if i == retries - 1:
raise
time.sleep(delay)
def main():
if "--selftest" in sys.argv[1:]:
sys.exit(selftest())
os.makedirs(iqc.IST, exist_ok=True)
# one open, held forever: rtl_tcp claims 1001 and streams; we never reopen, only retune over the socket.
rtl = subprocess.Popen(
["rtl_tcp", "-d", DEV, "-a", "127.0.0.1", "-p", str(PORT), "-s", str(FS)],
preexec_fn=os.setsid)
sys.stderr.write("[iq_hop] launched rtl_tcp (dev=%s fs=%d port=%d) — single open, retune-in-place\n"
% (DEV, FS, PORT)); sys.stderr.flush()
def shutdown(code=0):
try:
os.killpg(os.getpgid(rtl.pid), signal.SIGTERM)
try: rtl.wait(timeout=3)
except Exception:
# rtl_tcp can wedge in USB teardown and shrug off SIGTERM — every stop then stalls
# the full TimeoutStopSec until systemd SIGKILLs (observed 07-11 and 07-18). Finish it.
os.killpg(os.getpgid(rtl.pid), signal.SIGKILL)
except Exception: pass
sys.exit(code)
signal.signal(signal.SIGTERM, lambda *_: shutdown(0)) # systemctl stop = clean exit, no restart
signal.signal(signal.SIGINT, lambda *_: shutdown(0))
try:
sock = connect()
send_cmd(sock, SET_SAMPLERATE, FS)
send_cmd(sock, SET_GAIN_MODE, 1) # manual gain (no AGC surprises)
send_cmd(sock, SET_GAIN, int(round(GAIN_DB * 10))) # rtl_tcp gain is tenths of a dB
send_cmd(sock, SET_AGC_MODE, 0)
chans = {tag: iqc.dwell_channels(tag) for tag, *_ in DWELLS} # persistent per-band detector sets
sys.stderr.write("[iq_hop] 452/457 retune-in-place — BOT %ss / EOT %ss (weighted), %.1fs settle, gain %.0fdB\n"
% (DWELL_452, DWELL_457, SETTLE, GAIN_DB)); sys.stderr.flush()
while True:
for tag, center, dwell in DWELLS:
send_cmd(sock, SET_FREQUENCY, center) # retune on the SAME open handle — no reopen
if not drain(sock, SETTLE) or not run_dwell(sock, chans[tag], dwell):
sys.stderr.write("[iq_hop] rtl_tcp stream ended — exiting 1 for service restart\n")
shutdown(1)
except Exception as e:
# Failure MUST exit nonzero or Restart=on-failure never fires and the rig stays down until a
# hand starts it (2026-07-18: boot raced USB enumeration, connect() timed out, the old
# `finally: shutdown()` swallowed the OSError into exit 0 — 1001 dead all morning).
sys.stderr.write("[iq_hop] fatal: %s — exiting 1 so systemd retries\n" % e)
shutdown(1)
# ---- synthetic self-test: prove dwell-routing + boundary-flush WITHOUT a radio ----------------------
def selftest():
"""Two fake dwells fed synthetic CU8 (reusing iq_channelize's burst generator). Asserts: a center
burst during the 452 dwell lands in the 452/center channel and NOT the 457 one (and vice-versa), and
a burst still active when the dwell ends gets flushed (emitted), not lost. DSP isolation itself is
covered by iq_channelize.py --selftest; here we only exercise iq_hop's routing/flush wrapper."""
np = iqc.np
def fresh_set():
cs = [iqc.SubChannel(0, "center", None, None),
iqc.SubChannel(-12500, "mid_lo", None, None),
iqc.SubChannel(+12500, "mid_hi", None, None)]
hits = {c.label: [] for c in cs}
for c in cs:
def cap(now, _c=c):
hits[_c.label].append((round(_c.since, 2), round(now - _c.since, 2)))
c._emit = cap # capture emits in memory (no files)
return cs, hits
def feed(chans, sig, t0):
"""Push a complex baseband signal through `chans` as CU8 chunks on a synthetic clock from t0."""
u = np.empty(2 * len(sig))
u[0::2] = np.real(sig) * 127.5 + 127.5
u[1::2] = np.imag(sig) * 127.5 + 127.5
cu8 = np.clip(u, 0, 255).astype(np.uint8).tobytes()
pos = 0; k = 0
while pos + NBYTES <= len(cu8):
raw = cu8[pos:pos + NBYTES]; pos += NBYTES
arr = np.frombuffer(raw, np.uint8).astype(np.float64)
iq = ((arr[0::2] - 127.5) + 1j * (arr[1::2] - 127.5)) / 127.5
now = t0 + k * iqc.CHUNK / FS
for c in chans:
c.process(iq, now)
k += 1
return t0 + k * iqc.CHUNK / FS # end-of-dwell clock
dur = 1.2; n = int(FS * dur); t = np.arange(n) / FS
rng = np.random.default_rng(7)
noise = lambda: 0.05 * (rng.standard_normal(n) + 1j * rng.standard_normal(n))
set452, h452 = fresh_set()
set457, h457 = fresh_set()
# 452 dwell: a center burst 0.10-0.25 → must land in 452/center, never touch the 457 set.
s452 = noise() + iqc._fsk_burst(t, 0.0, 0.10, 0.25)
end452 = feed(set452, s452, 0.0)
for c in set452: # boundary flush
if c.active: c.active = False; c._emit(end452)
# 457 dwell: a center burst that runs to the END (0.9→past 1.2) → must be FLUSHED at the boundary.
s457 = noise() + iqc._fsk_burst(t, 0.0, 0.90, 1.5)
end457 = feed(set457, s457, end452)
for c in set457:
if c.active: c.active = False; c._emit(end457)
def near(hitlist, lo, hi):
return any(lo - 0.15 <= h[0] <= hi + 0.05 for h in hitlist)
ok = True
def check(name, cond):
nonlocal ok; print((" PASS " if cond else " FAIL ") + name); ok = ok and cond
print("self-test — 452 dwell: center burst 0.10-0.25 ; 457 dwell: center burst 0.90-end (flush case)")
print("452 detections:", {k: v for k, v in h452.items()})
print("457 detections:", {k: v for k, v in h457.items()})
check("452/center caught its burst", near(h452["center"], 0.10, 0.25))
check("452/mid channels stayed quiet", not h452["mid_lo"] and not h452["mid_hi"])
check("457/center caught its burst", near(h457["center"], end452 + 0.90, end452 + 0.90))
# the 457 burst runs to the end of the dwell and never goes quiet, so the ONLY thing that can emit it
# is the boundary flush: emitted-and-running-to-the-edge (since + dur ~= end457) proves flush worked.
flushed = bool(h457["center"]) and abs((h457["center"][0][0] + h457["center"][0][1]) - end457) < 0.15
check("boundary burst was FLUSHED, not lost", flushed)
print("RESULT:", "PASS — routing + boundary-flush hold" if ok else "FAIL")
return 0 if ok else 1
if __name__ == "__main__":
main()
scripts/iq_retune.py
+145
View File
@@ -0,0 +1,145 @@
#!/usr/bin/env python3
# iq_retune.py — the NO-CHURN hop. Opens dongle 1001 ONCE via librtlsdr and retunes in place
# (rtlsdr_set_center_freq) to step 452.9375 <-> 457.9375, instead of killing+reopening rtl_sdr every
# dwell. Tonight's wedge (1001 dying after a reopen, esp. through the VirtualBox USB passthrough) was
# caused by the close/reopen; retuning a live device never closes it, so it should sidestep the wedge.
#
# Reuses the PROVEN channelizer from iq_channelize.py (SubChannel: NCO mix -> FIR-LPF -> decimate ->
# in-channel-power presence -> discriminated clip), so each dwell still splits into:
# 452 -> BOT (center) + MID (±12.5k) ; 457 -> EOT (center) + MID (±12.5k)
# and feeds bot/eot/midtrain logs + the live meter exactly like iq_channelize does.
#
# RUNTIME helper — built by the maintainer, operated as a service. numpy only (DSP) + librtlsdr (ctypes).
# Bench smoke-test (no device touch): python3 iq_retune.py --check
# Live (gated; borrows 1001 from the FM hop): python3 iq_retune.py
#
# STATUS 2026-06-24: built, DSP reused from iq_channelize (which passed --selftest). The retune loop /
# ctypes device path is UNTESTED against hardware — the live run is the test of whether retune-in-place
# avoids the wedge. Do not trust it until it's baked on the device.
import os, sys, time, ctypes
import numpy as np
# reuse the proven DSP + meter writer (same dir; underscore name is importable, unlike scan-hop.py)
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from iq_channelize import dwell_channels, write_level, CHUNK, FS_IN # noqa: E402
DWELL_SEC = float(os.environ.get("RETUNE_DWELL", "28"))
SERIAL = os.environ.get("RETUNE_SERIAL", "1001")
GAIN_T = int(os.environ.get("RETUNE_GAIN_TENTHS", "400")) # tenths of dB (~40 dB), 0 => auto
CENTERS = (("452", 452_937_500), ("457", 457_937_500)) # dwell tag -> center Hz
SETTLE_BUFS = 2 # discard N buffers right after a retune
_LIB = "librtlsdr.so.0"
def _load_lib():
rtl = ctypes.CDLL(_LIB)
v = ctypes.c_void_p
rtl.rtlsdr_open.argtypes = [ctypes.POINTER(v), ctypes.c_uint32]
rtl.rtlsdr_open.restype = ctypes.c_int
rtl.rtlsdr_close.argtypes = [v]
rtl.rtlsdr_set_sample_rate.argtypes = [v, ctypes.c_uint32]
rtl.rtlsdr_set_center_freq.argtypes = [v, ctypes.c_uint32]
rtl.rtlsdr_set_tuner_gain_mode.argtypes = [v, ctypes.c_int]
rtl.rtlsdr_set_tuner_gain.argtypes = [v, ctypes.c_int]
rtl.rtlsdr_reset_buffer.argtypes = [v]
rtl.rtlsdr_read_sync.argtypes = [v, ctypes.c_void_p, ctypes.c_int, ctypes.POINTER(ctypes.c_int)]
rtl.rtlsdr_read_sync.restype = ctypes.c_int
# serial -> index, with graceful fallback if the symbol is absent
try:
rtl.rtlsdr_get_index_by_serial.argtypes = [ctypes.c_char_p]
rtl.rtlsdr_get_index_by_serial.restype = ctypes.c_int
rtl.rtlsdr_get_device_count.restype = ctypes.c_int
except AttributeError:
pass
return rtl
def _open_device(rtl):
idx = -1
try:
idx = rtl.rtlsdr_get_index_by_serial(SERIAL.encode())
except Exception:
idx = -1
if idx < 0:
idx = 1 # 1001 enumerated as index 1 alongside 1002; fallback only
sys.stderr.write("[iq_retune] serial %s not matched, falling back to index %d\n" % (SERIAL, idx))
dev = ctypes.c_void_p()
if rtl.rtlsdr_open(ctypes.byref(dev), idx) != 0:
raise RuntimeError("rtlsdr_open failed for index %d" % idx)
rtl.rtlsdr_set_sample_rate(dev, FS_IN)
if GAIN_T > 0:
rtl.rtlsdr_set_tuner_gain_mode(dev, 1)
rtl.rtlsdr_set_tuner_gain(dev, GAIN_T)
else:
rtl.rtlsdr_set_tuner_gain_mode(dev, 0)
return dev
def _to_complex(buf, n):
u = np.frombuffer(bytes(buf[:n]), dtype=np.uint8).astype(np.float64)
return ((u[0::2] - 127.5) + 1j * (u[1::2] - 127.5)) / 127.5
def run():
rtl = _load_lib()
dev = _open_device(rtl)
sets = {tag: dwell_channels(tag) for tag, _ in CENTERS} # build channelizers once; reuse across retunes
nbytes = CHUNK * 2
buf = (ctypes.c_ubyte * nbytes)()
nread = ctypes.c_int()
sys.stderr.write("[iq_retune] open OK; retune-hop %s, dwell %.0fs, fs %d — device stays open\n"
% ("/".join(t for t, _ in CENTERS), DWELL_SEC, FS_IN)); sys.stderr.flush()
try:
while True:
for tag, center in CENTERS:
rtl.rtlsdr_set_center_freq(dev, center) # IN-PLACE retune — no close/reopen
rtl.rtlsdr_reset_buffer(dev)
chans = sets[tag]
discard = SETTLE_BUFS
t_end = time.time() + DWELL_SEC
sys.stderr.write("[iq_retune] tune %s (%d Hz) for %.0fs\n" % (tag, center, DWELL_SEC))
sys.stderr.flush()
while time.time() < t_end:
if rtl.rtlsdr_read_sync(dev, buf, nbytes, ctypes.byref(nread)) < 0:
break
if nread.value < nbytes:
continue
if discard > 0: # drop the post-retune settling buffers
discard -= 1
continue
iq = _to_complex(buf, nread.value)
now = time.time()
for ch in chans:
ch.process(iq, now)
hot = max(chans, key=lambda c: c.last_db)
write_level(now, hot.last_db, any(c.active for c in chans),
hot.label, (hot.peak if hot.active else None), hot.floor)
finally:
rtl.rtlsdr_close(dev)
def check():
"""Bench smoke-test: lib loads, symbols resolve, DSP imports — no device opened."""
ok = True
try:
rtl = _load_lib()
for sym in ("rtlsdr_open", "rtlsdr_set_center_freq", "rtlsdr_read_sync", "rtlsdr_reset_buffer"):
present = hasattr(rtl, sym)
print((" PASS " if present else " FAIL ") + "librtlsdr." + sym)
ok = ok and present
except OSError as e:
print(" FAIL load librtlsdr:", e); ok = False
try:
s = dwell_channels("452")
print(" PASS dwell_channels('452') -> %d sub-channels (%s)" % (len(s), ",".join(c.label for c in s)))
except Exception as e:
print(" FAIL channelizer import:", e); ok = False
print("RESULT:", "PASS — bench wiring OK (device path still needs a live bake)" if ok else "FAIL")
return 0 if ok else 1
if __name__ == "__main__":
if "--check" in sys.argv[1:]:
sys.exit(check())
run()
scripts/probe-band.py
Executable
+125
View File
@@ -0,0 +1,125 @@
#!/usr/bin/env python3
"""probe-band — console first-light: tune a dongle to a band, snapshot the spectrum, print
what's there. The "can we hear anything?" tool, before we build decoders or dashboard panels.
a RADIO RUNTIME, not the code author. This OPERATES a radio (tunes + samples). Run it yourself; the maintainer builds it.
a generic band probe is engine-level (every radio project
wants "what's on this band, and does my receiver even work"). Only the presets below are profile.
Recommended order — validate the KNOWN-GOOD first, then the target:
1. noaa — 162.400-162.550 MHz · NOAA Weather, always-on. If we see THIS, the receiver +
antenna + chain all work. On 1002 (the voice stick, 18" whip — it's 162 MHz).
2. eot — 457.5-458.5 MHz · EOT/FRED telemetry (457.9375) on 1001 (~6" whip, matched). Sparse
— may be quiet with no train near. Seeing the noise floor cleanly still proves the path.
Usage:
./probe-band.py noaa # known-good sanity check first
./probe-band.py eot
SERIAL=1002 LO=162.4M HI=162.55M GAIN=40 SECONDS=8 ./probe-band.py # fully manual
GAIN env: "auto" or a dB number (try 30-49 for weak signals). Take notes per run.
"""
import os, sys, glob, shutil, subprocess, tempfile, statistics
PRESETS = {
# name : (serial, lo, hi, note)
"noaa": ("1002", "162.400M", "162.550M", "NOAA Weather — always-on KNOWN-GOOD sanity check (voice stick)"),
"eot": ("1001", "457.500M", "458.500M", "EOT/FRED 457.9375 — sparse, may be quiet"),
}
preset = sys.argv[1] if len(sys.argv) > 1 and not sys.argv[1].startswith("-") else None
p = PRESETS.get(preset, (None, None, None, "manual"))
SERIAL = os.environ.get("SERIAL", p[0] or "1001")
LO = os.environ.get("LO", p[1] or "162.400M")
HI = os.environ.get("HI", p[2] or "162.550M")
BIN = os.environ.get("BIN", "3k")
SECONDS = os.environ.get("SECONDS", "8")
GAIN = os.environ.get("GAIN", "auto")
TOPN = int(os.environ.get("TOPN", "12"))
NOTE = p[3]
def die(msg, code=1):
print(f"probe-band: {msg}", file=sys.stderr); sys.exit(code)
def preflight():
if not shutil.which("rtl_power"):
die("rtl_power not found — install the 'rtl-sdr' package.")
for d in glob.glob("/sys/bus/usb/devices/*"):
try:
if open(os.path.join(d, "idVendor")).read().strip() != "0bda": continue
if open(os.path.join(d, "idProduct")).read().strip() != "2838": continue
if open(os.path.join(d, "serial")).read().strip() != SERIAL: continue
except OSError:
continue
for intf in glob.glob(d + ":*"):
link = os.path.join(intf, "driver")
if os.path.islink(link) and os.path.basename(os.readlink(link)) == "dvb_usb_rtl28xxu":
die(f"dongle {SERIAL} is DVB-claimed — free it: "
f"sudo modprobe -r rtl2832_sdr dvb_usb_rtl28xxu")
return
die(f"dongle serial {SERIAL} not present (check the radios panel / `rtl_test`).")
def run_sweep(csv_path):
cmd = ["rtl_power", "-d", SERIAL, "-f", f"{LO}:{HI}:{BIN}", "-i", SECONDS, "-1", csv_path]
if GAIN != "auto":
cmd[3:3] = ["-g", GAIN]
print(f"# probing {LO}-{HI} @ {BIN}, {SECONDS}s, dongle :{SERIAL} (gain={GAIN})", file=sys.stderr)
print(f"# {NOTE}", file=sys.stderr)
try:
subprocess.run(cmd, check=True)
except subprocess.CalledProcessError as e:
die(f"rtl_power exited {e.returncode} — dongle free? gain valid? band in range (24-1766 MHz)?")
def parse_bins(csv_path):
bins = []
with open(csv_path) as f:
for line in f:
parts = [p.strip() for p in line.split(",")]
if len(parts) < 7: continue
try:
lo, step = float(parts[2]), float(parts[4])
vals = [float(v) for v in parts[6:] if v]
except ValueError:
continue
for i, dbm in enumerate(vals):
bins.append(((lo + (i + 0.5) * step) / 1e6, dbm))
return bins
def main():
preflight()
with tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as tf:
csv_path = tf.name
try:
run_sweep(csv_path)
bins = parse_bins(csv_path)
finally:
try: os.unlink(csv_path)
except OSError: pass
if not bins:
die("no spectrum parsed from rtl_power.")
floor = statistics.median(d for _, d in bins)
peak_f, peak_d = max(bins, key=lambda b: b[1])
top = sorted(bins, key=lambda b: b[1], reverse=True)[:TOPN]
span = peak_d - floor
verdict = ("SIGNAL — clear carrier above the floor" if span > 10 else
"maybe — slight rise, tune gain / dwell longer" if span > 5 else
"just noise — nothing above the floor here")
print(f"\nProbe {LO}-{HI} · dongle :{SERIAL} · {NOTE}")
print(f"noise floor ~{floor:.1f} dBm · peak {peak_d:.1f} dBm @ {peak_f:.4f} MHz "
f"(+{span:.1f} dB) · {verdict}\n")
print(f" {'freq (MHz)':>11} {'dBm':>7} {'above floor':>11}")
print(" " + "-" * 38)
for f_mhz, dbm in top:
print(f" {f_mhz:>11.4f} {dbm:>7.1f} {dbm - floor:>+10.1f}")
print("\n next: paste this back and we'll read it + pick the next gain. (write each run down)\n")
if __name__ == "__main__":
main()
scripts/scan-hop.py
+93
View File
@@ -0,0 +1,93 @@
#!/usr/bin/env python3
# scan-hop.py — time-division scanner on dongle 1001. Alternates 452.9375 (head-end / BOT) and 457.9375
# (EOT), ~28s each (+~1.5s settle between), forever. 452 and 457 are 5 MHz apart — too wide for one tuner
# to grab at once — so we hop. Each 200k dwell ALSO spans its mid-train DPU neighbors (±12.5 kHz: 452.925/
# .950 and 457.925/.950) — they're inside the capture, so a DPU burst trips this dwell too (counted as
# bot/eot presence; distinguishing DPU needs IQ sub-channel ID — see eot/README "DPU decode plan"). Each band's bursts go to its OWN log (band-tagged) so the working EOT path is
# untouched and nothing cross-contaminates: 452 → bot.jsonl (presence only, no decoder yet), 457 → eot.jsonl
# + eot_clips (as before). Presence detection via vumon; no decode here.
#
# This REPLACES the fixed-457 EOT watch while it runs (1001 can only be one freq at a time). Accepts the
# coverage tradeoff: ~46% dwell per band. Operate it as a service (istrain-scan-hop) — see README.
#
# IQ MODE (opt-in, HOP_IQ=1): instead of rtl_fm|vumon (one blurred FM channel), each dwell runs
# rtl_sdr | iq_channelize.py, which splits the dwell into center (bot/eot) + the two ±12.5 kHz DPU
# sub-channels (mid) and writes bot/eot/midtrain logs itself. Same 2-position hop; the FM path below
# stays the default fallback. Validate the DSP with `iq_channelize.py --selftest` before flipping.
import os, sys, signal, subprocess, time
HOME = os.path.expanduser("~")
IST = os.path.join(HOME, "istrain")
VUMON = os.path.join(os.path.dirname(os.path.abspath(__file__)), "vumon.py")
IQSCRIPT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "iq_channelize.py")
IQ_MODE = os.environ.get("HOP_IQ", "") not in ("", "0", "no", "false")
IQ_FS = os.environ.get("IQ_FS", "240000") # rtl_sdr sample rate for IQ mode
LEVEL = os.path.join(IST, "level.json") # live meter: whichever band is active writes here
DWELL = int(os.environ.get("HOP_DWELL", "28")) # seconds on each band
SETTLE = float(os.environ.get("HOP_SETTLE", "1.5")) # let the tuner/USB release before the next grab
GAIN = os.environ.get("HOP_GAIN", "40")
THRESH = os.environ.get("HOP_THRESH", "-25")
MARGIN = os.environ.get("HOP_MARGIN", "6") # adaptive: fire dB above the rolling noise floor (beats fixed thresh in the noise)
# (freq, jsonl, clips_dir|None, label). BOT is presence-only for now; EOT keeps clips for the (paused) reaper.
BANDS = [
("452.9375M", os.path.join(IST, "bot.jsonl"), os.path.join(IST, "bot_clips"), "bot"),
("457.9375M", os.path.join(IST, "eot.jsonl"), os.path.join(IST, "eot_clips"), "eot"),
]
def _hz(freq):
"""'452.9375M' -> '452937500' for rtl_sdr -f (which wants plain Hz)."""
return str(int(float(freq.rstrip("Mm")) * 1e6)) if freq.lower().endswith("m") else freq
def segment(freq, jsonl, clips, label):
if IQ_MODE:
dwell = {"bot": "452", "eot": "457"}[label] # iq_channelize writes bot/eot + midtrain itself
cmd = ("rtl_sdr -d 1001 -f %s -s %s -g %s - | python3 %s --dwell %s") % (
_hz(freq), IQ_FS, GAIN, IQSCRIPT, dwell)
sys.stderr.write("[scan-hop] IQ %s for %ss → %s + mid\n" % (freq, DWELL, label))
sys.stderr.flush()
p = subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid)
try:
time.sleep(DWELL)
finally:
try: os.killpg(os.getpgid(p.pid), signal.SIGTERM)
except Exception: pass
try: p.wait(timeout=5)
except Exception:
try: os.killpg(os.getpgid(p.pid), signal.SIGKILL)
except Exception: pass
time.sleep(SETTLE)
return
clip_arg = ("--clips %s --prefix %s_" % (clips, label)) if clips else ""
cmd = ("rtl_fm -d 1001 -f %s -M fm -s 200k -r 48k -g %s -l 0 - | "
"python3 %s --thresh %s --margin %s --jsonl %s --level %s --label %s %s") % (
freq, GAIN, VUMON, THRESH, MARGIN, jsonl, LEVEL, label, clip_arg)
sys.stderr.write("[scan-hop] %s for %ss → %s\n" % (freq, DWELL, os.path.basename(jsonl)))
sys.stderr.flush()
p = subprocess.Popen(cmd, shell=True, preexec_fn=os.setsid) # own process group so we can kill the whole pipe
try:
time.sleep(DWELL)
finally:
try: os.killpg(os.getpgid(p.pid), signal.SIGTERM)
except Exception: pass
try: p.wait(timeout=5)
except Exception:
try: os.killpg(os.getpgid(p.pid), signal.SIGKILL)
except Exception: pass
time.sleep(SETTLE) # device release before the next rtl_fm grabs 1001
def main():
for _, _, clips, _ in BANDS:
if clips:
os.makedirs(clips, exist_ok=True)
sys.stderr.write("[scan-hop] 452/457 time-division on 1001 — %ss each, %ss settle\n" % (DWELL, SETTLE))
sys.stderr.flush()
while True:
for freq, jsonl, clips, label in BANDS:
segment(freq, jsonl, clips, label)
if __name__ == "__main__":
main()
scripts/transcribe-worker.py
+97
View File
@@ -0,0 +1,97 @@
#!/usr/bin/env python3
"""istrain — voice transcription worker (VM side).
Watches the airband capture dir for new voice clips and sends each to the Mill's
whisper service (istrain-whisper stack, Dockge on the Mill). CPU lives on the Mill;
this worker only enhances (ffmpeg comms filter) and ships. Results land as:
- a sidecar <clip>.txt next to the mp3 (same convention as radio/transcribe)
- a durable line in ~/istrain/transcripts.jsonl (clips in /tmp are ephemeral;
the transcript record survives reboots — keep-everything policy)
Lineage: projects/radio/transcribe/transcribe-worker.sh (marine post) — whisper.cpp
run locally there; here the model runs on the Mill and VAD replaces the prompt-echo
fixes (radio/transcribe/NOTES.md — VAD was the named "robust upgrade").
"""
import json, os, subprocess, sys, time, urllib.request, urllib.error, uuid
WATCH = os.environ.get("CLIPS_DIR", "/tmp/airband")
MILL = os.environ.get("MILL_ASR", "http://whisper:9000")
ASR = MILL + "/asr?task=transcribe&language=en&vad_filter=true&output=txt"
# env-overridable for the Mill container (writes /data/transcripts.jsonl); VM default unchanged.
OUT_LOG = os.path.expanduser(os.environ.get("TRANSCRIPTS_LOG", "~/istrain/transcripts.jsonl"))
MIN_BYTES = 5000 # below this it's a squelch blip (marine rule)
SETTLE_S = 5 # still being written if newer than this
POLL_S = 10
NO_SPEECH = "(no speech)"
def enhance(src: str, dst: str) -> bool:
"""Comms-audio cleanup: bandpass to voice, normalize. 16k mono wav."""
r = subprocess.run(
["ffmpeg", "-hide_banner", "-loglevel", "error", "-y", "-i", src,
"-af", "highpass=f=250,lowpass=f=3200,speechnorm=e=12:r=0.0001:l=1",
"-ar", "16000", "-ac", "1", dst],
capture_output=True)
return r.returncode == 0 and os.path.exists(dst)
def transcribe(wav: str) -> str | None:
"""POST the clip to the Mill. Returns text ('' if no speech), None on error."""
boundary = uuid.uuid4().hex
with open(wav, "rb") as f:
payload = f.read()
body = (f"--{boundary}\r\nContent-Disposition: form-data; "
f'name="audio_file"; filename="{os.path.basename(wav)}"\r\n'
f"Content-Type: audio/wav\r\n\r\n").encode() + payload + f"\r\n--{boundary}--\r\n".encode()
req = urllib.request.Request(ASR, data=body, method="POST",
headers={"Content-Type": f"multipart/form-data; boundary={boundary}"})
try:
with urllib.request.urlopen(req, timeout=240) as resp:
text = resp.read().decode().strip()
except (urllib.error.URLError, TimeoutError) as e:
print(f"ASR error: {e}", flush=True)
return None
# whisper renders non-speech as runs of "..." — normalize to empty
cleaned = " ".join(t for t in text.split() if set(t) - set(".")).strip()
return cleaned
def main():
print(f"transcribe-worker: watching {WATCH}, ASR={MILL}", flush=True)
while True:
try:
# newest first: fresh clips (what the live panel + push want) never wait behind backlog
names = sorted(os.listdir(WATCH),
key=lambda n: os.path.getmtime(os.path.join(WATCH, n))
if os.path.exists(os.path.join(WATCH, n)) else 0, reverse=True)
except FileNotFoundError:
time.sleep(POLL_S); continue
now = time.time()
for name in names:
if not name.endswith(".mp3"):
continue
mp3 = os.path.join(WATCH, name)
txt = mp3[:-4] + ".txt"
if os.path.exists(txt):
continue
try:
st = os.stat(mp3)
except FileNotFoundError:
continue
if st.st_size < MIN_BYTES or now - st.st_mtime < SETTLE_S:
continue
wav = f"/tmp/transcribe-{os.getpid()}.wav"
ok = enhance(mp3, wav)
text = transcribe(wav if ok else mp3)
if os.path.exists(wav):
os.unlink(wav)
if text is None: # service unreachable — retry next poll, don't mark
break
with open(txt, "w") as f:
f.write(text if text else NO_SPEECH)
if text: # only real speech goes in the durable log
rec = {"ts": st.st_mtime, "file": name, "size": st.st_size, "text": text}
with open(OUT_LOG, "a") as f:
f.write(json.dumps(rec) + "\n")
print(f"TRANSCRIBED {name}: {text[:100]}", flush=True)
time.sleep(POLL_S)
if __name__ == "__main__":
main()
scripts/trim-captures.py
+35
View File
@@ -0,0 +1,35 @@
#!/usr/bin/env python3
# trim-captures — prune old VOICE capture audio from /tmp/airband, keeping the durable detection log.
# Safe by design: heard.jsonl (the detections Activity reads — channel + time) is SEPARATE and never
# touched; this only deletes bulky .mp3 audio AFTER it's logged. It forces an ingest first (GET /api/heard,
# which appends any new clip to heard.jsonl) and ABORTS if the dashboard is unreachable — so a not-yet-logged
# clip can never be deleted. --dry-run to preview.
import os, glob, time, sys, urllib.request
CAP = os.environ.get("AIRBAND_DIR", "/tmp/airband")
KEEP_HOURS = float(os.environ.get("CAP_KEEP_HOURS", "24"))
DASH = os.environ.get("DASH_URL", "http://127.0.0.1:8157")
DRY = "--dry-run" in sys.argv
def main():
try: # log every current clip to heard.jsonl first
urllib.request.urlopen(DASH + "/api/heard", timeout=10).read()
except Exception as e:
print(f"[trim] dashboard unreachable ({e}) — NOT trimming (clips may be unlogged)"); return
cutoff = time.time() - KEEP_HOURS * 3600
mp3s = glob.glob(os.path.join(CAP, "*.mp3"))
old = [m for m in mp3s if os.path.getmtime(m) < cutoff]
freed = 0
for m in old:
freed += os.path.getsize(m)
if not DRY:
try: os.remove(m)
except OSError: pass
# whisper sidecar (2026-07-10) rides with its clip — no orphan .txt buildup
try: os.remove(m[:-4] + ".txt")
except OSError: pass
print(f"[trim{' DRY' if DRY else ''}] {len(mp3s)} clips on disk, deleted {len(old)} older than "
f"{KEEP_HOURS:.0f}h, freed {freed//1024} KB; heard.jsonl untouched")
if __name__ == "__main__":
main()
scripts/vumon.py
Executable
+155
View File
@@ -0,0 +1,155 @@
#!/usr/bin/env python3
"""vumon — live audio activity meter for a raw S16LE mono stream (the radio listen pipe).
The VISUAL the audio-only listen lacks. Sits at the end of an rtl_fm | tee | aplay pipeline
and shows:
• a level BAR — flat when the band's quiet, jumps when a transmission breaks through
• a SPINNER while squelched / no data, so you can see it's alive (never looks frozen)
• a TIMESTAMPED line when activity starts and stops (+ duration, peak) — so you get a log
of WHEN something happened even if you stepped away from the console
RUNTIME helper. Reads a pipe; does NOT touch the dongle (so it's just a display, not radio
operation). built by the maintainer; you run it. a generic
"is this audio stream active right now" meter, useful to any listen pipeline.
Wire it into the pipe (3-way: file + speakers + meter; needs bash for the >() ):
rtl_fm -d 1002 ... - | tee /tmp/cap.s16 | tee >(aplay -r 12000 -f S16_LE -t raw -c 1) \
| python3 scripts/vumon.py --log /tmp/istrain_activity.log
watch-only (no audio): rtl_fm ... - | tee /tmp/cap.s16 | python3 scripts/vumon.py
Options:
--thresh DB activity threshold in dBFS (default -42; raise toward -25 if hiss trips it)
--log FILE append "started/ended" events here (timestamps persist)
--jsonl FILE append one JSON line per burst ({ts,dur,peak}) — structured log for a dashboard
--clips DIR on each burst, save its raw S16 audio (+~0.5s pre-roll) to DIR/eot_<ts>.s16 — for decode
"""
import sys, math, select, time, array, json, os, collections
THRESH = -42.0
LOGF = None
JSONL = None
CLIPS = None
LEVEL = None # --level FILE: write instantaneous level JSON here, for a live dashboard meter
LABEL = "" # --label NAME: band tag stamped into the level file (e.g. "eot" / "bot")
MARGIN = None # --margin DB: ADAPTIVE — fire when db exceeds the rolling noise floor by MARGIN (vs fixed --thresh)
FLOOR_A = 0.02 # noise-floor EMA rate (per chunk; only updated while quiet) — ~2s time constant @ 48k
MAXCLIP = 480000 # cap a saved clip at ~5s @ 48k s16 — EOT bursts are <1s; longer is noise, don't hoard it
PREFIX = "eot_" # --prefix NAME: clip filename prefix per band (eot_ / bot_) so each band's clips stay distinct
_a = sys.argv[1:]
while _a:
o = _a.pop(0)
if o == "--thresh": THRESH = float(_a.pop(0))
elif o == "--log": LOGF = _a.pop(0)
elif o == "--jsonl": JSONL = _a.pop(0)
elif o == "--clips": CLIPS = _a.pop(0)
elif o == "--prefix": PREFIX = _a.pop(0)
elif o == "--level": LEVEL = _a.pop(0)
elif o == "--label": LABEL = _a.pop(0)
elif o == "--margin": MARGIN = float(_a.pop(0))
else: sys.exit(f"vumon: unknown arg {o!r}")
if CLIPS:
os.makedirs(CLIPS, exist_ok=True)
CHUNK = 4096 # bytes (2048 samples @ 2B mono) — ~170ms @ 12k, ~43ms @ 48k
BARW = 24
SPIN = "|/-\\"
err = sys.stderr
def dbfs(buf):
a = array.array('h')
a.frombytes(buf[: len(buf) - (len(buf) % 2)])
if not a: return -99.0
s = sum(v * v for v in a)
rms = math.sqrt(s / len(a))
return 20 * math.log10(rms / 32768.0) if rms > 0 else -99.0
def bar(db):
frac = max(0.0, min(1.0, (db + 60) / 60)) # map -60..0 dBFS -> 0..1
n = int(frac * BARW)
return "█" * n + "▁" * (BARW - n)
def event(msg):
err.write("\r" + " " * 64 + "\r" + msg + "\n"); err.flush()
if LOGF:
with open(LOGF, "a") as f:
f.write(time.strftime("%Y-%m-%d %H:%M:%S ") + msg + "\n")
_last_level = 0.0
def write_level(db, active, peak, floor=None):
"""Throttled atomic write of the instantaneous level — the dashboard polls this for a live meter."""
global _last_level
now = time.time()
if now - _last_level < 0.2: # ~5 Hz is plenty for a live bar; keeps disk churn negligible
return
_last_level = now
try:
with open(LEVEL + ".tmp", "w") as f:
json.dump({"ts": round(now, 2), "db": round(db, 1), "active": bool(active),
"label": LABEL, "peak": (round(peak, 1) if active else None),
"floor": (round(floor, 1) if floor is not None else None)}, f)
os.replace(LEVEL + ".tmp", LEVEL)
except OSError:
pass
fd = sys.stdin.buffer
active = False; since = 0.0; peak = -99.0; spin = 0; last = time.time()
floor = None # rolling noise-floor estimate (adaptive mode)
clipbuf = None; preroll = collections.deque(maxlen=12) # ~0.5 s pre-roll for --clips
_mode = (f"adaptive floor+{MARGIN:.0f}dB" if MARGIN is not None else f"threshold {THRESH:.0f} dBFS")
err.write(f"vumon: watching ({_mode}"
+ (f", logging -> {LOGF}" if LOGF else "") + ") Ctrl-C to stop\n")
try:
while True:
r, _, _ = select.select([fd], [], [], 0.4)
if r:
buf = fd.read1(CHUNK)
if not buf: break
db = dbfs(buf); last = time.time()
if MARGIN is not None: # adaptive: fire on excursions above the rolling floor
if floor is None: floor = db
on = db > floor + MARGIN
if not on: floor += FLOOR_A * (db - floor) # track the floor ONLY while quiet
else:
on = db >= THRESH
if on and not active:
active = True; since = last; peak = db
event(f"● {time.strftime('%H:%M:%S')} ACTIVE {db:6.1f} dBFS")
if CLIPS:
clipbuf = bytearray(b"".join(preroll)); clipbuf += buf
elif on:
peak = max(peak, db)
if CLIPS and clipbuf is not None and len(clipbuf) < MAXCLIP: clipbuf += buf
elif active:
active = False
event(f" └ {time.strftime('%H:%M:%S')} ended ({last - since:4.1f}s, "
f"peak {peak:6.1f} dBFS)")
if JSONL:
try:
with open(JSONL, "a") as jf:
jf.write(json.dumps({"ts": round(since, 2),
"dur": round(last - since, 2),
"peak": round(peak, 1)}) + "\n")
except OSError:
pass
if CLIPS and clipbuf is not None:
clipbuf += buf
try:
fn = os.path.join(CLIPS, PREFIX + time.strftime("%Y%m%d_%H%M%S.s16",
time.localtime(since)))
with open(fn, "wb") as cf: cf.write(clipbuf)
except OSError:
pass
clipbuf = None
elif CLIPS:
preroll.append(buf)
if LEVEL: write_level(db, active, peak, floor)
tag = "\033[1m▶ SIGNAL\033[0m" if active else " listening"
err.write(f"\r[{bar(db)}] {db:6.1f} dBFS {tag} "); err.flush()
else:
spin = (spin + 1) % 4
err.write(f"\r[{'▁' * BARW}] {SPIN[spin]} waiting… "
f"(quiet {time.time() - last:4.1f}s) "); err.flush()
except KeyboardInterrupt:
pass
finally:
err.write("\n")