istrain
559caead36
Public community release. The complete working system — DSP + decoders (scripts/), web dashboard + API (dashboard/), container stack (docker/), config templates (config/) — plus APOCALYPSE-EDITION.md, the full build guide for a human or a coding agent. GPLv3 (PyEOT dependency). Scrubbed of all secrets, internal IPs, hostnames, and location detail. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
119 lines
4.8 KiB
Python
119 lines
4.8 KiB
Python
#!/usr/bin/env python3
|
|
# eot-decode.py — DECODE-ONLY service for the EOT band. Watches ~/istrain/eot_clips for finished clips,
|
|
# decodes each (clip -> unit/telemetry via eot_scan) and ENRICHES the matching record in eot.jsonl.
|
|
#
|
|
# It does NOT cull. No clip is ever deleted or moved — the BOT reverse-engineering and the "more examples
|
|
# to train on" both need every clip kept. This is the half of the old reaper that mattered, split out:
|
|
# the reaper bundled decode WITH culling, so pausing the cull (which we want off) also paused identification.
|
|
# Decode now stands alone; culling can stay off until storage actually forces it.
|
|
#
|
|
# Marking done WITHOUT moving/deleting: a tiny processed-set at eot_clips/.decoded.json (each clip decoded
|
|
# once). Safe log write: read + enrich matched records, recapture any tail the capture appended since the
|
|
# read, then atomic temp-rename — the capture only ever appends, so we never lose a presence line.
|
|
#
|
|
# Operated as the istrain-eot-decode --user service (on/off + hard-reset via the dashboard supervisor).
|
|
# Touches NO radio — pure CPU on saved clips — so it is safe to start/stop/restart at any time.
|
|
# CLI: eot-decode.py [--once]
|
|
import os, sys, glob, time, json
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
sys.path.insert(0, HERE)
|
|
from reaper import _eot_decode, _clip_epoch # reuse the PROVEN decode adapter + clip->epoch
|
|
|
|
CLIPS = os.path.expanduser(os.environ.get("EOT_CLIPS", "~/istrain/eot_clips"))
|
|
LOG = os.path.expanduser(os.environ.get("EOT_LOG", "~/istrain/eot.jsonl"))
|
|
STATE = os.path.join(CLIPS, ".decoded.json")
|
|
MIN_AGE = int(os.environ.get("DECODE_MIN_AGE", "60")) # never touch a clip <60s old (maybe still mid-write)
|
|
INTERVAL = int(os.environ.get("DECODE_INTERVAL", "30")) # seconds between passes
|
|
|
|
|
|
def _load_state():
|
|
try: return set(json.load(open(STATE)))
|
|
except Exception: return set()
|
|
|
|
|
|
def _save_state(s):
|
|
tmp = STATE + ".tmp"
|
|
with open(tmp, "w") as f: json.dump(sorted(s), f)
|
|
os.replace(tmp, STATE)
|
|
|
|
|
|
def _enrich(decodes):
|
|
"""Merge {sec: decoded-fields} into eot.jsonl by integer-ts match — atomically, without losing any
|
|
line the capture appended while we worked (re-read past the byte offset we started at, then rename)."""
|
|
if not decodes or not os.path.exists(LOG):
|
|
return
|
|
size = os.path.getsize(LOG)
|
|
out = []
|
|
|
|
def _take(fh):
|
|
for line in fh:
|
|
line = line.strip()
|
|
if not line: continue
|
|
try: r = json.loads(line)
|
|
except Exception: continue
|
|
s = int(r.get("ts", 0))
|
|
if s in decodes: r.update(decodes[s])
|
|
out.append(r)
|
|
|
|
with open(LOG) as f:
|
|
_take(f)
|
|
try: # recapture appends since our read (capture only appends)
|
|
with open(LOG) as f:
|
|
f.seek(size); _take(f)
|
|
except Exception:
|
|
pass
|
|
tmp = LOG + ".tmp"
|
|
with open(tmp, "w") as f:
|
|
for r in out: f.write(json.dumps(r) + "\n")
|
|
os.replace(tmp, LOG)
|
|
|
|
|
|
def one_pass():
|
|
"""Decode every not-yet-processed, settled clip; enrich the log; mark them done. Returns (tried, ok)."""
|
|
if not os.path.isdir(CLIPS):
|
|
return (0, 0)
|
|
state = _load_state()
|
|
now = time.time()
|
|
decodes = {}; tried = ok = 0; newly = []
|
|
for c in sorted(glob.glob(os.path.join(CLIPS, "eot_*.s16"))): # top-level only — confirmed/ is left alone
|
|
fn = os.path.basename(c)
|
|
if fn in state:
|
|
continue
|
|
if now - os.path.getmtime(c) < MIN_AGE: # too fresh — try it next pass, don't mark yet
|
|
continue
|
|
try:
|
|
fields = _eot_decode(c)
|
|
except Exception as e:
|
|
sys.stderr.write("[eot-decode] err %s: %s\n" % (fn, e)); continue
|
|
tried += 1; newly.append(fn)
|
|
if fields:
|
|
ok += 1
|
|
sec = _clip_epoch(c, "eot_")
|
|
if sec is not None: decodes[sec] = fields
|
|
sys.stderr.write("[eot-decode] ✅ %s unit=%s pressure=%s motion=%s\n"
|
|
% (fn, fields.get("unit"), fields.get("pressure"), fields.get("motion")))
|
|
if decodes:
|
|
_enrich(decodes)
|
|
if newly:
|
|
state.update(newly); _save_state(state)
|
|
return (tried, ok)
|
|
|
|
|
|
def main():
|
|
if "--once" in sys.argv[1:]:
|
|
t, o = one_pass(); print("[eot-decode] one pass: tried %d, decoded %d" % (t, o)); return
|
|
sys.stderr.write("[eot-decode] decode-only watcher up — %s every %ds (no cull, no move, no delete)\n"
|
|
% (CLIPS, INTERVAL)); sys.stderr.flush()
|
|
while True:
|
|
try:
|
|
t, o = one_pass()
|
|
if t: sys.stderr.write("[eot-decode] pass: tried %d, decoded %d\n" % (t, o)); sys.stderr.flush()
|
|
except Exception as e:
|
|
sys.stderr.write("[eot-decode] pass err: %s\n" % e); sys.stderr.flush()
|
|
time.sleep(INTERVAL)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|