Operations in wizwar's shape: the determinism gate, backups, the rollup and the pulse

Every deploy first replays every production ledger with the engine about
to ship. The droplet gains a nightly rollup of counts and a nightly
backup to Spaces, a pulse script the pulse skill reads, rate limits on
opening rooms and taking seats, and eviction of idle rooms from memory
with reload from their ledgers on the next visit.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0141G6xqLeNRYEtviLWSB5Up

deploy/pulse.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+# The operations pulse, printed ON the droplet: the duel server's health,
+# errors in its journal, the last week of the rollup, the backup's last word,
+# and the box's vitals. Read by the waving-hands-pulse skill.
+set -u
+echo "as of $(date -u '+%F %H:%M') UTC on $(hostname); up $(uptime -p | sed 's/up //')"
+echo "--- duel server"
+systemctl is-active waving-hands >/dev/null && echo "waving-hands: active since $(systemctl show waving-hands -p ActiveEnterTimestamp --value)" || echo "waving-hands: NOT ACTIVE"
+systemctl is-active caddy >/dev/null && echo "caddy: active" || echo "caddy: NOT ACTIVE"
+echo "restarts in 7 days: $(journalctl -u waving-hands --since '7 days ago' -o cat | grep -c 'Started waving-hands')"
+ERR=$(journalctl -u waving-hands --since '7 days ago' -p err -o cat | grep -vc '^$')
+echo "journal errors in 7 days: $ERR"
+journalctl -u waving-hands --since '7 days ago' -o cat | grep -iE "error|unhandled|exception" | grep -v "Rate" | tail -3 | sed 's/^/  /'
+echo "--- rollup, last 7 days (people / requests / rooms opened / turns)"
+if [ -s /var/lib/waving-hands/rollup.jsonl ]; then
+  tail -7 /var/lib/waving-hands/rollup.jsonl | python3 -c '
+import json, sys
+for l in sys.stdin:
+    r = json.loads(l); rm = r["rooms"]
+    print("  %s  %3d people  %4d requests  %2d rooms (%d started, %d with turns)  %3d turns  bots %d addr  refs %s" % (
+        r["day"], r["humanAddresses"], r["human"], rm["created"], rm["started"], rm["withTurns"], r["turns"], r["botAddresses"],
+        ",".join(sorted(r["referrers"])) or "-"))'
+else
+  echo "  (no rollup yet)"
+fi
+echo "--- ledgers"
+echo "rooms on disk: $(ls /var/lib/waving-hands/rooms/*.jsonl 2>/dev/null | wc -l); touched in 24h: $(find /var/lib/waving-hands/rooms -name '*.jsonl' -mmin -1440 2>/dev/null | wc -l); size $(du -sh /var/lib/waving-hands/rooms 2>/dev/null | cut -f1)"
+echo "--- backup"
+if [ -f /var/log/waving-hands/backup.log ]; then
+  grep -E "backup (start|done)|skipped|error|ERROR" /var/log/waving-hands/backup.log | tail -2 | sed 's/^/  /'
+else
+  echo "  never run"
+fi
+echo "--- box"
+echo "disk: $(df -h / | awk 'NR==2{print $5" used of "$2}'); memory: $(free -m | awk 'NR==2{print $7" MB available of "$2}'); load: $(cut -d' ' -f1-3 /proc/loadavg)"
+echo "caddy access log: $(du -sh /var/lib/caddy/access.log 2>/dev/null | cut -f1) live, $(ls /var/lib/caddy/access*.log* 2>/dev/null | wc -l) files"