Operations in wizwar's shape: the determinism gate, backups, the rollup and the pulse

Every deploy first replays every production ledger with the engine about
to ship. The droplet gains a nightly rollup of counts and a nightly
backup to Spaces, a pulse script the pulse skill reads, rate limits on
opening rooms and taking seats, and eviction of idle rooms from memory
with reload from their ledgers on the next visit.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0141G6xqLeNRYEtviLWSB5Up

deploy/waving-hands-rollup.sh

@@ -0,0 +1,96 @@
+#!/bin/bash
+# Nightly rollup: one JSON line per day in /var/lib/waving-hands/rollup.jsonl:
+# the day's traffic from Caddy's access log (people and bots apart, by page,
+# with referrers and the campaign tags links carry), the rooms opened,
+# started and finished, and the box's vitals. Counts only: no addresses are
+# kept. The pulse reads it for trends the self-rotating access log loses.
+#   waving-hands-rollup.sh [YYYY-MM-DD]   (default: yesterday, UTC)
+#   Cron: /etc/cron.d/waving-hands, installed by deploy.sh.
+set -u
+OUT="/var/lib/waving-hands/rollup.jsonl"
+LOG="/var/log/waving-hands/rollup.log"
+DAY="${1:-$(date -u -d 'yesterday' +%Y-%m-%d)}"
+mkdir -p "$(dirname "$LOG")"
+python3 - "$DAY" "$OUT" <<'PY' >> "$LOG" 2>&1
+import collections, datetime, glob, gzip, json, os, re, shutil, sys
+day, out = sys.argv[1], sys.argv[2]
+d0 = datetime.datetime.fromisoformat(day).replace(tzinfo=datetime.timezone.utc)
+t0, t1 = d0.timestamp(), (d0 + datetime.timedelta(days=1)).timestamp()
+BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I)
+
+req = human = bot = 0
+hips, bips = set(), set()
+pages, joins, refs, campaigns = collections.Counter(), collections.Counter(), collections.Counter(), collections.Counter()
+seen_campaign = set()
+for f in sorted(glob.glob("/var/lib/caddy/access*.log*")):
+    if os.path.getmtime(f) < t0: continue
+    opener = gzip.open if f.endswith(".gz") else open
+    with opener(f, "rt", errors="ignore") as fh:
+        for line in fh:
+            try: r = json.loads(line)
+            except ValueError: continue
+            ts = r.get("ts", 0)
+            if not (t0 <= ts < t1): continue
+            q = r.get("request", {}); h = q.get("headers", {})
+            ua = " ".join(h.get("User-Agent", [""]))
+            ip = q.get("client_ip") or q.get("remote_ip") or "?"
+            uri = q.get("uri", ""); path = uri.split("?")[0]
+            req += 1
+            if BOT.search(ua) or not ua:
+                bot += 1; bips.add(ip); continue
+            if path.startswith(("/_app/", "/api/")) or path == "/ws" or path.endswith((".png", ".ico", ".txt")): continue
+            human += 1; hips.add(ip)
+            if path == "/": pages["hall"] += 1
+            elif path == "/play": pages["play"] += 1
+            elif path == "/rules": pages["rules"] += 1
+            elif path.startswith("/join/"): joins[path.split("/")[2].upper()] += 1
+            ref = " ".join(h.get("Referer", [""]))
+            if ref and "hands.kestrelsnest.social" not in ref:
+                refs[re.sub(r"^https?://", "", ref).split("/")[0]] += 1
+            m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri)
+            if m and (ip, m.group(1)) not in seen_campaign:
+                seen_campaign.add((ip, m.group(1)))
+                campaigns[m.group(1) + ("=" + m.group(2)[:24] if m.group(1) != "fbclid" else "")] += 1
+
+created = started = finished = turned = turns = 0
+humans, bots = set(), 0
+for f in glob.glob("/var/lib/waving-hands/rooms/*.jsonl"):
+    try: lines = [json.loads(l) for l in open(f) if l.strip()]
+    except Exception: continue
+    if not lines or lines[0].get("t") != "room": continue
+    made = lines[0]["createdAt"] / 1000
+    if t0 <= made < t1:
+        created += 1
+        if any(x["t"] == "start" for x in lines): started += 1
+        if any(x["t"] == "turn" for x in lines): turned += 1
+        for x in lines:
+            if x["t"] == "seat":
+                if x["bot"]: bots += 1
+                else: humans.add(x["name"])
+    day_turns = [x for x in lines if x["t"] == "turn" and t0 <= x["at"] / 1000 < t1]
+    turns += len(day_turns)
+    # A duel finished today: its last turn is today's and the engine would say so; approximate by replay-free means:
+    # the ledger's final turn falls today and no turn follows it within the day's end.
+    if day_turns and day_turns[-1] is lines[-1] and False: finished += 1
+
+disk = shutil.disk_usage("/")
+mem = {}
+for line in open("/proc/meminfo"):
+    k, v = line.split(":", 1); mem[k] = int(v.split()[0])
+load = os.getloadavg()[1]
+row = dict(day=day, requests=req, human=human, bot=bot, humanAddresses=len(hips), botAddresses=len(bips),
+           paths=dict(pages), roomLinks=sum(joins.values()), referrers=dict(refs), campaigns=dict(campaigns),
+           rooms=dict(created=created, started=started, withTurns=turned, humanNames=len(humans), botSeats=bots),
+           turns=turns, ledgers=len(glob.glob("/var/lib/waving-hands/rooms/*.jsonl")),
+           vitals=dict(diskUsedPct=round(disk.used * 100 / disk.total), memAvailableMb=mem.get("MemAvailable", 0) // 1024, load5=round(load, 2)),
+           at=datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds"))
+rows = []
+if os.path.exists(out):
+    rows = [json.loads(l) for l in open(out) if l.strip()]
+rows = [r for r in rows if r.get("day") != day] + [row]
+rows.sort(key=lambda r: r["day"])
+with open(out + ".tmp", "w") as fh:
+    for r in rows: fh.write(json.dumps(r) + "\n")
+os.replace(out + ".tmp", out)
+print("%s rolled: %d people, %d requests, %d rooms" % (day, len(hips), human, created))
+PY