The nightly rollup keeps what the access log forgets; two doors get a limit

Three sysadmin items ahead of the public announcement's arrivals.

deploy/wizwar-rollup.sh writes one JSON line per UTC day to
/var/lib/wizwar/rollup.jsonl at 00:10: yesterday's traffic from Caddy's
access log (requests, human vs bot addresses as counts only, socket
connects, path mix, external referrers), the table's growth (new
rooms, human seats and names, lifetime game totals, reports and
replies), and the box's vitals (service memory and peak, disk, load,
protocol errors, service starts, ledger size). Re-rolling a day
replaces its line. It checks in to a Sentry cron monitor of its own,
whose URL deploy.sh derives from the backup monitor's on first
install, alongside the cron entry. The pulse gains a Trends section
that reads the last week of it. Caddy keeps 30 log files instead of 5
as the raw backing.

Per-address limits on the two doors anyone may use unseated: 12 new
rooms and 6 reports per address per hour, in a sliding window keyed by
the address Caddy forwards. The per-connection cap stays; it reset on
reconnect, which is what a script would do.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Jm2auWk6RP71CjaAb4FMoG

.claude/skills/wizwar-pulse/SKILL.md

@@ -30,7 +30,18 @@ quiet, say so in one line before the details.
      connector reads fine but 403s on create operations
 3. **Backups**: `tail -3 /var/log/wizwar/backup.log` (or its rotation)
    for the last run, plus the cron monitor above for missed runs
-4. **The table itself**:
+4. **Trends** (`/var/lib/wizwar/rollup.jsonl`, one JSON line per UTC day
+   written at 00:10 by /usr/local/bin/wizwar-rollup.sh; counts only, no
+   addresses): the last 7 rows give requests, human vs bot addresses,
+   socket connects, path mix (/, /clips, /watch, /ws), external
+   referrers (reddit.com, boardgamegeek.com after the 2026-09-03
+   announcement), new rooms, human seats and names, reports/replies,
+   and vitals (memMB, memPeakMB, diskPct, load1, protocolErrors,
+   serviceStarts, ledgerMB). Report day-over-day direction, not raw
+   rows. Its Sentry cron monitor is slug `wizwar-rollup` (a missed
+   check-in there means the rollup itself is broken). Re-roll a day by
+   hand: `wizwar-rollup.sh YYYY-MM-DD`.
+5. **The table itself**:
    - Room count and growth: `ls /var/lib/wizwar/rooms/*.jsonl | wc -l`
    - New human names this week: room ledgers' join lines minus bots
      (Automaton*) and known names, on rooms created in the last 7 days
@@ -42,8 +53,9 @@ quiet, say so in one line before the details.
 
 ## Report
 
-Five sections, tight: **Verdict** (one line), **Box**, **Errors &
-monitors**, **Backups**, **The table** (games, new faces, feedback).
+Six sections, tight: **Verdict** (one line), **Box**, **Errors &
+monitors**, **Backups**, **Trends** (the week's direction from the
+rollup), **The table** (games, new faces, feedback).
 Numbers with their thresholds, not raw dumps. End with any recommended
 action, or "nothing needs you."
 
@@ -57,7 +69,11 @@ action, or "nothing needs you."
 - Unattended-upgrades reboots the box at 09:30 UTC when a kernel patch
   requires it; a reboot there is maintenance, not an outage.
 - Caddy access logs live at /var/lib/caddy/access.log (self-rotating,
-  10MiB × 5); the systemd sandbox denies /var/log/caddy.
+  10MiB × 30 since 2026-09-03); the systemd sandbox denies /var/log/caddy.
+- Per-address limits (2026-09-03): 12 new rooms and 6 reports per
+  address per hour, in packages/server/src/ratelimit.ts. A player who
+  hits one sees "try again in an hour"; a pulse showing many refused
+  creates from one address is a script, not a friend.
 - Restore drill (2026-09-01): a Spaces snapshot booted 56/56 rooms
   clean. The path: `rclone copy spaces:kestrel-wizwar-backups/wizwar/
   snapshots/<date> ...` on the droplet, then run a local server with

deploy/deploy.sh

@@ -15,6 +15,14 @@ ssh "root@$HOST" '
   # deploy, or the repo copy and the live copy drift apart (they did:
   # the Sentry check-ins shipped in the repo and never reached the box).
   install -m 755 /opt/wizwar/deploy/wizwar-backup.sh /usr/local/bin/wizwar-backup.sh
+  # The nightly rollup and its cron entry ride along the same way; its
+  # Sentry check-in URL is the backup monitor'"'"'s with its own slug.
+  install -m 755 /opt/wizwar/deploy/wizwar-rollup.sh /usr/local/bin/wizwar-rollup.sh
+  install -m 644 /opt/wizwar/deploy/wizwar-rollup.cron /etc/cron.d/wizwar-rollup
+  if [ -f /root/.wizwar-sentry-cron ] && [ ! -f /root/.wizwar-sentry-cron-rollup ]; then
+    sed "s#/cron/new-monitor/#/cron/wizwar-rollup/#" /root/.wizwar-sentry-cron > /root/.wizwar-sentry-cron-rollup
+    chmod 600 /root/.wizwar-sentry-cron-rollup
+  fi
   systemctl daemon-reload
   systemctl enable --now wizwar
   systemctl restart wizwar

deploy/setup-droplet.sh

@@ -25,7 +25,9 @@ mkdir -p /opt/wizwar /var/lib/wizwar/rooms
 chown -R wizwar:wizwar /opt/wizwar /var/lib/wizwar
 
 # Caddy vhost: auto-TLS, security headers, proxy to the game.
-printf '%s\n\nheader {\n\tStrict-Transport-Security "max-age=31536000"\n\tX-Content-Type-Options "nosniff"\n\tX-Frame-Options "DENY"\n\tReferrer-Policy "no-referrer"\n}\nreverse_proxy localhost:8787\n' "$HOST" > /etc/caddy/Caddyfile
+# Access log kept ~30 days (10MiB x 30): the nightly rollup keeps the
+# counts forever, the raw lines back it for a month.
+printf '%s\n\nheader {\n\tStrict-Transport-Security "max-age=31536000"\n\tX-Content-Type-Options "nosniff"\n\tX-Frame-Options "DENY"\n\tReferrer-Policy "no-referrer"\n}\nlog {\n\toutput file /var/lib/caddy/access.log {\n\t\troll_size 10MiB\n\t\troll_keep 30\n\t}\n}\nreverse_proxy localhost:8787\n' "$HOST" > /etc/caddy/Caddyfile
 systemctl reload caddy
 
 # Firewall: ssh + web only. The game server binds loopback and is reached

deploy/wizwar-rollup.cron

@@ -0,0 +1,2 @@
+# Nightly rollup of yesterday's traffic, table, and vitals (UTC). Installed by deploy.sh.
+10 0 * * * root /usr/local/bin/wizwar-rollup.sh

deploy/wizwar-rollup.sh

@@ -0,0 +1,117 @@
+#!/bin/bash
+# Nightly rollup: one JSON line per day in /var/lib/wizwar/rollup.jsonl —
+# yesterday's traffic (from Caddy's access log), the table's growth (from
+# the room ledgers, stats.json and feedback.jsonl), and the box's vitals.
+# Counts only: no addresses are kept. The pulse reads it for trends the
+# self-rotating access log cannot keep.
+#   Cron: 10 0 * * * (UTC), see /etc/cron.d/wizwar-rollup (deploy.sh installs both).
+#   Sentry Crons check-in: /root/.wizwar-sentry-cron-rollup holds the URL
+#   (derived from the backup monitor's on first deploy; absent = no check-ins).
+set -u
+OUT="/var/lib/wizwar/rollup.jsonl"
+LOG="/var/log/wizwar/rollup.log"
+DAY="${1:-$(date -u -d 'yesterday' +%Y-%m-%d)}"
+CRON_URL=$(cat /root/.wizwar-sentry-cron-rollup 2>/dev/null || true)
+checkin() { [ -n "$CRON_URL" ] && curl -sf -o /dev/null "$CRON_URL?status=$1" || true; }
+
+checkin in_progress
+if python3 - "$DAY" "$OUT" <<'PY' >> "$LOG" 2>&1
+import json, sys, os, glob, gzip, datetime, collections, subprocess, re
+day, out = sys.argv[1], sys.argv[2]
+d0 = datetime.datetime.fromisoformat(day).replace(tzinfo=datetime.timezone.utc)
+t0, t1 = d0.timestamp(), (d0 + datetime.timedelta(days=1)).timestamp()
+BOT = ("bot", "crawl", "spider", "facebookexternalhit", "slack", "discord", "twitter", "preview", "curl", "python", "go-http", "wget", "headless")
+
+# --- traffic: every access log file touched since the day began ---
+req = 0; human = 0; bot = 0; hips = set(); bips = set()
+paths = collections.Counter(); status = collections.Counter(); refs = collections.Counter(); ws = 0
+for f in sorted(glob.glob("/var/lib/caddy/access*.log*")):
+    if os.path.getmtime(f) < t0: continue
+    opener = gzip.open if f.endswith(".gz") else open
+    with opener(f, "rt", errors="ignore") as fh:
+        for line in fh:
+            try: r = json.loads(line)
+            except ValueError: continue
+            ts = r.get("ts", 0)
+            if not (t0 <= ts < t1): continue
+            q = r.get("request", {}); h = q.get("headers", {})
+            ua = " ".join(h.get("User-Agent", [""])).lower()
+            ip = q.get("client_ip") or q.get("remote_ip") or "?"
+            isbot = any(k in ua for k in BOT)
+            req += 1
+            if isbot: bot += 1; bips.add(ip)
+            else: human += 1; hips.add(ip)
+            uri = q.get("uri", "").split("?")[0]
+            if uri == "/ws": ws += 1
+            key = ("/clips" if uri.startswith("/clips") else "/watch" if uri.startswith("/watch")
+                   else "(assets)" if uri.startswith("/assets") or re.search(r"\.(png|jpg|svg|ico|css|js|webmanifest|mp4)$", uri)
+                   else uri if uri in ("/", "/ws", "/robots.txt") else "(other)")
+            paths[key] += 1
+            status[str(r.get("status", 0))[0] + "xx"] += 1
+            ref = " ".join(h.get("Referer", [""]))
+            m = re.match(r"https?://([^/]+)", ref)
+            host = m.group(1) if m else ""
+            ours = host.endswith("kestrelsnest.social") or "104.236.96.198" in host
+            if host and not ours and not isbot: refs[host] += 1
+
+# --- the table ---
+new_rooms = 0; seats = collections.Counter()
+for f in glob.glob("/var/lib/wizwar/rooms/*.jsonl"):
+    try:
+        with open(f) as fh:
+            meta = json.loads(fh.readline())
+            ca = meta.get("createdAt")
+            if not ca: continue
+            ct = datetime.datetime.fromisoformat(ca.replace("Z", "+00:00")).timestamp()
+            if not (t0 <= ct < t1): continue
+            new_rooms += 1
+            for line in fh:
+                try: r = json.loads(line)
+                except ValueError: continue
+                if r.get("kind") == "join" and not r.get("bot"): seats[r["name"]] += 1
+    except OSError: continue
+stats = {}
+try: stats = json.load(open("/var/lib/wizwar/stats.json"))
+except Exception: pass
+reports = 0; replies = 0
+try:
+    for line in open("/var/lib/wizwar/feedback.jsonl"):
+        try: r = json.loads(line)
+        except ValueError: continue
+        at = datetime.datetime.fromisoformat(r["at"].replace("Z", "+00:00")).timestamp()
+        if t0 <= at < t1: reports += ("happened" in r); replies += ("reportId" in r)
+except OSError: pass
+
+# --- vitals (now, at rollup time) ---
+def sh(cmd):
+    try: return subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=20).stdout.strip()
+    except Exception: return ""
+mem = sh("systemctl show wizwar -p MemoryCurrent --value"); peak = sh("systemctl show wizwar -p MemoryPeak --value")
+disk = sh("df --output=pcent / | tail -1").strip().rstrip("%")
+load = sh("cut -d' ' -f1 /proc/loadavg")
+errs = sh("journalctl -u wizwar --since '%s' --until '%s' --no-pager 2>/dev/null | grep -ci 'unhandled protocol error'" % (day, (d0 + datetime.timedelta(days=1)).date()))
+starts = sh("journalctl -u wizwar --since '%s' --until '%s' --no-pager 2>/dev/null | grep -c 'Started wizwar'" % (day, (d0 + datetime.timedelta(days=1)).date()))
+ledger = sum(os.path.getsize(f) for f in glob.glob("/var/lib/wizwar/rooms/*.jsonl"))
+row = {
+    "day": day, "requests": req, "human": human, "bot": bot,
+    "humanAddresses": len(hips), "botAddresses": len(bips), "socketConnects": ws,
+    "paths": dict(paths.most_common()), "status": dict(status), "referrers": dict(refs.most_common(10)),
+    "newRooms": new_rooms, "humanSeats": sum(seats.values()), "humanNames": sorted(seats),
+    "gamesFinishedTotal": stats.get("gamesFinished"), "gamesCreatedTotal": stats.get("gamesCreated"),
+    "reports": reports, "replies": replies,
+    "vitals": {"memMB": round(int(mem) / 1048576) if mem.isdigit() else None,
+               "memPeakMB": round(int(peak) / 1048576) if peak.isdigit() else None,
+               "diskPct": int(disk) if disk.isdigit() else None, "load1": float(load) if load else None,
+               "protocolErrors": int(errs) if errs.isdigit() else None, "serviceStarts": int(starts) if starts.isdigit() else None,
+               "ledgerMB": round(ledger / 1048576, 1)},
+}
+# One line per day: a rerun for the same day replaces its line.
+rows = []
+if os.path.exists(out):
+    rows = [l for l in open(out).read().splitlines() if l.strip() and json.loads(l).get("day") != day]
+rows.append(json.dumps(row, separators=(",", ":")))
+tmp = out + ".tmp"
+open(tmp, "w").write("\n".join(rows) + "\n"); os.replace(tmp, out)
+print("%s rollup %s: %d requests, %d human addresses, %d new rooms, %d human seats" % (datetime.datetime.now(datetime.timezone.utc).isoformat(), day, req, len(hips), new_rooms, sum(seats.values())))
+PY
+then checkin ok; else checkin error; fi

packages/server/src/index.ts

@@ -67,6 +67,7 @@ import {
 import { engagementStats, recordHotseat } from "./stats";
 import { appendFeedback, readFeedback, readClips, clipAssetPath } from "./store";
 import { clipsIndexHtml, clipPageHtml } from "./clips";
+import { SlidingLimit, clientAddress } from "./ratelimit";
 import { getShare, loadShares, mintShare } from "./shares";
 import { renderSharePng } from "./ogimage";
 import { BOT_LINES, type BanterTrigger } from "./banter";
@@ -81,6 +82,10 @@ if (process.env.SENTRY_DSN) {
 const MAX_SOCKETS = 300;          // concurrent connections
 const MAX_ROOMS = 5000;           // total rooms on the server
 const MAX_ROOMS_PER_CONN = 10;    // rooms one connection may create
+// Per-address limits, held across reconnects: a table of friends never
+// nears them; a script filling the vault or the reports desk does.
+const roomsPerAddress = new SlidingLimit(12, 60 * 60 * 1000);
+const reportsPerAddress = new SlidingLimit(6, 60 * 60 * 1000);
 const MAX_COMMAND_BYTES = 16384;  // serialized game command
 const MAX_MYGAMES_SEATS = 50;     // seats checked per myGames request
 const CATCHUP_COOLDOWN_MS = 3000; // full-game replays are CPU-heavy
@@ -409,6 +414,8 @@ interface Session {
   spectator: boolean;
   /** The raw seat token this connection authenticated with (memory only). */
   token: string | null;
+  /** Where the connection came from, for the per-address limits. */
+  address: string;
   claimFails: number;
   // Token bucket: refills at 15 msg/s up to a burst of 30.
   bucket: number;
@@ -579,7 +586,7 @@ function broadcastRoomState(room: Room): void {
   }
 }
 
-wss.on("connection", (socket) => {
+wss.on("connection", (socket, req) => {
   if (sessions.size >= MAX_SOCKETS) {
     send(socket, { type: "error", message: "the tavern is packed — try again shortly" });
     socket.close();
@@ -587,6 +594,7 @@ wss.on("connection", (socket) => {
   }
   const session: Session = {
     socket, playerId: null, roomId: null, spectator: false, token: null, claimFails: 0,
+    address: clientAddress(req.headers, req.socket.remoteAddress),
     bucket: 30, lastRefill: Date.now(), overLimitStrikes: 0,
     roomsCreated: 0, lastCatchUpAt: 0, hotseatReports: 0,
   };
@@ -626,6 +634,9 @@ wss.on("connection", (socket) => {
           if (session.roomsCreated >= MAX_ROOMS_PER_CONN || roomCount() >= MAX_ROOMS) {
             return send(socket, { type: "error", message: "no new rooms right now — try again later" });
           }
+          if (!roomsPerAddress.allow(session.address)) {
+            return send(socket, { type: "error", message: "that's a lot of new tables from one place — try again in an hour" });
+          }
           session.roomsCreated++;
           leaveGallery(session);
           const { room, token } = createRoom(name);
@@ -868,6 +879,9 @@ wss.on("connection", (socket) => {
             String(raw ?? "").replace(/[\u0000-\u0009\u000b-\u001f\u007f]/g, " ").trim().slice(0, 2000);
           const happened = clean(msg.happened);
           if (!happened) return send(socket, { type: "error", message: "say what happened" });
+          if (!reportsPerAddress.allow(session.address)) {
+            return send(socket, { type: "error", message: "the desk has plenty from you for now — more in an hour" });
+          }
           appendFeedback({
             id: randomBytes(4).toString("hex"),
             at: new Date().toISOString(),

packages/server/src/ratelimit.ts

@@ -0,0 +1,40 @@
+// A per-key sliding-window limiter for the two doors anyone may walk
+// through unseated: creating rooms and filing reports. Per-connection
+// caps reset on reconnect; these are keyed by client address and hold
+// for the window. Memory is bounded by pruning keys whose windows have
+// emptied.
+
+export class SlidingLimit {
+  private hits = new Map<string, number[]>();
+  private lastSweep = 0;
+
+  constructor(private readonly max: number, private readonly windowMs: number) {}
+
+  /** Record a hit for `key` if it is under the limit; false if it is not. */
+  allow(key: string, now = Date.now()): boolean {
+    this.sweep(now);
+    const cutoff = now - this.windowMs;
+    const times = (this.hits.get(key) ?? []).filter((t) => t > cutoff);
+    if (times.length >= this.max) { this.hits.set(key, times); return false; }
+    times.push(now);
+    this.hits.set(key, times);
+    return true;
+  }
+
+  private sweep(now: number): void {
+    if (now - this.lastSweep < this.windowMs) return;
+    this.lastSweep = now;
+    const cutoff = now - this.windowMs;
+    for (const [k, times] of this.hits) {
+      if (!times.some((t) => t > cutoff)) this.hits.delete(k);
+    }
+  }
+}
+
+/** The client's address as Caddy reports it — the first hop in
+ * X-Forwarded-For — falling back to the socket's own peer. */
+export function clientAddress(headers: Record<string, string | string[] | undefined>, remote: string | undefined): string {
+  const fwd = headers["x-forwarded-for"];
+  const first = (Array.isArray(fwd) ? fwd[0] : fwd)?.split(",")[0]?.trim();
+  return first || remote || "unknown";
+}