The nightly rollup keeps what the access log forgets; two doors get a limit

Three sysadmin items ahead of the public announcement's arrivals. deploy/wizwar-rollup.sh writes one JSON line per UTC day to /var/lib/wizwar/rollup.jsonl at 00:10: yesterday's traffic from Caddy's access log (requests, human vs bot addresses as counts only, socket connects, path mix, external referrers), the table's growth (new rooms, human seats and names, lifetime game totals, reports and replies), and the box's vitals (service memory and peak, disk, load, protocol errors, service starts, ledger size). Re-rolling a day replaces its line. It checks in to a Sentry cron monitor of its own, whose URL deploy.sh derives from the backup monitor's on first install, alongside the cron entry. The pulse gains a Trends section that reads the last week of it. Caddy keeps 30 log files instead of 5 as the raw backing. Per-address limits on the two doors anyone may use unseated: 12 new rooms and 6 reports per address per hour, in a sliding window keyed by the address Caddy forwards. The per-connection cap stays; it reset on reconnect, which is what a script would do. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Jm2auWk6RP71CjaAb4FMoG
2026-09-03 11:30:44 -04:00
parent 3f909b485b
commit ca108f83a6
7 changed files with 205 additions and 6 deletions
@@ -0,0 +1,117 @@
+#!/bin/bash
+# Nightly rollup: one JSON line per day in /var/lib/wizwar/rollup.jsonl —
+# yesterday's traffic (from Caddy's access log), the table's growth (from
+# the room ledgers, stats.json and feedback.jsonl), and the box's vitals.
+# Counts only: no addresses are kept. The pulse reads it for trends the
+# self-rotating access log cannot keep.
+#   Cron: 10 0 * * * (UTC), see /etc/cron.d/wizwar-rollup (deploy.sh installs both).
+#   Sentry Crons check-in: /root/.wizwar-sentry-cron-rollup holds the URL
+#   (derived from the backup monitor's on first deploy; absent = no check-ins).
+set -u
+OUT="/var/lib/wizwar/rollup.jsonl"
+LOG="/var/log/wizwar/rollup.log"
+DAY="${1:-$(date -u -d 'yesterday' +%Y-%m-%d)}"
+CRON_URL=$(cat /root/.wizwar-sentry-cron-rollup 2>/dev/null || true)
+checkin() { [ -n "$CRON_URL" ] && curl -sf -o /dev/null "$CRON_URL?status=$1" || true; }
+
+checkin in_progress
+if python3 - "$DAY" "$OUT" <<'PY' >> "$LOG" 2>&1
+import json, sys, os, glob, gzip, datetime, collections, subprocess, re
+day, out = sys.argv[1], sys.argv[2]
+d0 = datetime.datetime.fromisoformat(day).replace(tzinfo=datetime.timezone.utc)
+t0, t1 = d0.timestamp(), (d0 + datetime.timedelta(days=1)).timestamp()
+BOT = ("bot", "crawl", "spider", "facebookexternalhit", "slack", "discord", "twitter", "preview", "curl", "python", "go-http", "wget", "headless")
+
+# --- traffic: every access log file touched since the day began ---
+req = 0; human = 0; bot = 0; hips = set(); bips = set()
+paths = collections.Counter(); status = collections.Counter(); refs = collections.Counter(); ws = 0
+for f in sorted(glob.glob("/var/lib/caddy/access*.log*")):
+    if os.path.getmtime(f) < t0: continue
+    opener = gzip.open if f.endswith(".gz") else open
+    with opener(f, "rt", errors="ignore") as fh:
+        for line in fh:
+            try: r = json.loads(line)
+            except ValueError: continue
+            ts = r.get("ts", 0)
+            if not (t0 <= ts < t1): continue
+            q = r.get("request", {}); h = q.get("headers", {})
+            ua = " ".join(h.get("User-Agent", [""])).lower()
+            ip = q.get("client_ip") or q.get("remote_ip") or "?"
+            isbot = any(k in ua for k in BOT)
+            req += 1
+            if isbot: bot += 1; bips.add(ip)
+            else: human += 1; hips.add(ip)
+            uri = q.get("uri", "").split("?")[0]
+            if uri == "/ws": ws += 1
+            key = ("/clips" if uri.startswith("/clips") else "/watch" if uri.startswith("/watch")
+                   else "(assets)" if uri.startswith("/assets") or re.search(r"\.(png|jpg|svg|ico|css|js|webmanifest|mp4)$", uri)
+                   else uri if uri in ("/", "/ws", "/robots.txt") else "(other)")
+            paths[key] += 1
+            status[str(r.get("status", 0))[0] + "xx"] += 1
+            ref = " ".join(h.get("Referer", [""]))
+            m = re.match(r"https?://([^/]+)", ref)
+            host = m.group(1) if m else ""
+            ours = host.endswith("kestrelsnest.social") or "104.236.96.198" in host
+            if host and not ours and not isbot: refs[host] += 1
+
+# --- the table ---
+new_rooms = 0; seats = collections.Counter()
+for f in glob.glob("/var/lib/wizwar/rooms/*.jsonl"):
+    try:
+        with open(f) as fh:
+            meta = json.loads(fh.readline())
+            ca = meta.get("createdAt")
+            if not ca: continue
+            ct = datetime.datetime.fromisoformat(ca.replace("Z", "+00:00")).timestamp()
+            if not (t0 <= ct < t1): continue
+            new_rooms += 1
+            for line in fh:
+                try: r = json.loads(line)
+                except ValueError: continue
+                if r.get("kind") == "join" and not r.get("bot"): seats[r["name"]] += 1
+    except OSError: continue
+stats = {}
+try: stats = json.load(open("/var/lib/wizwar/stats.json"))
+except Exception: pass
+reports = 0; replies = 0
+try:
+    for line in open("/var/lib/wizwar/feedback.jsonl"):
+        try: r = json.loads(line)
+        except ValueError: continue
+        at = datetime.datetime.fromisoformat(r["at"].replace("Z", "+00:00")).timestamp()
+        if t0 <= at < t1: reports += ("happened" in r); replies += ("reportId" in r)
+except OSError: pass
+
+# --- vitals (now, at rollup time) ---
+def sh(cmd):
+    try: return subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=20).stdout.strip()
+    except Exception: return ""
+mem = sh("systemctl show wizwar -p MemoryCurrent --value"); peak = sh("systemctl show wizwar -p MemoryPeak --value")
+disk = sh("df --output=pcent / | tail -1").strip().rstrip("%")
+load = sh("cut -d' ' -f1 /proc/loadavg")
+errs = sh("journalctl -u wizwar --since '%s' --until '%s' --no-pager 2>/dev/null | grep -ci 'unhandled protocol error'" % (day, (d0 + datetime.timedelta(days=1)).date()))
+starts = sh("journalctl -u wizwar --since '%s' --until '%s' --no-pager 2>/dev/null | grep -c 'Started wizwar'" % (day, (d0 + datetime.timedelta(days=1)).date()))
+ledger = sum(os.path.getsize(f) for f in glob.glob("/var/lib/wizwar/rooms/*.jsonl"))
+row = {
+    "day": day, "requests": req, "human": human, "bot": bot,
+    "humanAddresses": len(hips), "botAddresses": len(bips), "socketConnects": ws,
+    "paths": dict(paths.most_common()), "status": dict(status), "referrers": dict(refs.most_common(10)),
+    "newRooms": new_rooms, "humanSeats": sum(seats.values()), "humanNames": sorted(seats),
+    "gamesFinishedTotal": stats.get("gamesFinished"), "gamesCreatedTotal": stats.get("gamesCreated"),
+    "reports": reports, "replies": replies,
+    "vitals": {"memMB": round(int(mem) / 1048576) if mem.isdigit() else None,
+               "memPeakMB": round(int(peak) / 1048576) if peak.isdigit() else None,
+               "diskPct": int(disk) if disk.isdigit() else None, "load1": float(load) if load else None,
+               "protocolErrors": int(errs) if errs.isdigit() else None, "serviceStarts": int(starts) if starts.isdigit() else None,
+               "ledgerMB": round(ledger / 1048576, 1)},
+}
+# One line per day: a rerun for the same day replaces its line.
+rows = []
+if os.path.exists(out):
+    rows = [l for l in open(out).read().splitlines() if l.strip() and json.loads(l).get("day") != day]
+rows.append(json.dumps(row, separators=(",", ":")))
+tmp = out + ".tmp"
+open(tmp, "w").write("\n".join(rows) + "\n"); os.replace(tmp, out)
+print("%s rollup %s: %d requests, %d human addresses, %d new rooms, %d human seats" % (datetime.datetime.now(datetime.timezone.utc).isoformat(), day, req, len(hips), new_rooms, sum(seats.values())))
+PY
+then checkin ok; else checkin error; fi