diff --git a/.claude/skills/wizwar-pulse/SKILL.md b/.claude/skills/wizwar-pulse/SKILL.md index 3132e14..968b332 100644 --- a/.claude/skills/wizwar-pulse/SKILL.md +++ b/.claude/skills/wizwar-pulse/SKILL.md @@ -30,7 +30,18 @@ quiet, say so in one line before the details. connector reads fine but 403s on create operations 3. **Backups**: `tail -3 /var/log/wizwar/backup.log` (or its rotation) for the last run, plus the cron monitor above for missed runs -4. **The table itself**: +4. **Trends** (`/var/lib/wizwar/rollup.jsonl`, one JSON line per UTC day + written at 00:10 by /usr/local/bin/wizwar-rollup.sh; counts only, no + addresses): the last 7 rows give requests, human vs bot addresses, + socket connects, path mix (/, /clips, /watch, /ws), external + referrers (reddit.com, boardgamegeek.com after the 2026-09-03 + announcement), new rooms, human seats and names, reports/replies, + and vitals (memMB, memPeakMB, diskPct, load1, protocolErrors, + serviceStarts, ledgerMB). Report day-over-day direction, not raw + rows. Its Sentry cron monitor is slug `wizwar-rollup` (a missed + check-in there means the rollup itself is broken). Re-roll a day by + hand: `wizwar-rollup.sh YYYY-MM-DD`. +5. **The table itself**: - Room count and growth: `ls /var/lib/wizwar/rooms/*.jsonl | wc -l` - New human names this week: room ledgers' join lines minus bots (Automaton*) and known names, on rooms created in the last 7 days @@ -42,8 +53,9 @@ quiet, say so in one line before the details. ## Report -Five sections, tight: **Verdict** (one line), **Box**, **Errors & -monitors**, **Backups**, **The table** (games, new faces, feedback). +Six sections, tight: **Verdict** (one line), **Box**, **Errors & +monitors**, **Backups**, **Trends** (the week's direction from the +rollup), **The table** (games, new faces, feedback). Numbers with their thresholds, not raw dumps. End with any recommended action, or "nothing needs you." @@ -57,7 +69,11 @@ action, or "nothing needs you." - Unattended-upgrades reboots the box at 09:30 UTC when a kernel patch requires it; a reboot there is maintenance, not an outage. - Caddy access logs live at /var/lib/caddy/access.log (self-rotating, - 10MiB × 5); the systemd sandbox denies /var/log/caddy. + 10MiB × 30 since 2026-09-03); the systemd sandbox denies /var/log/caddy. +- Per-address limits (2026-09-03): 12 new rooms and 6 reports per + address per hour, in packages/server/src/ratelimit.ts. A player who + hits one sees "try again in an hour"; a pulse showing many refused + creates from one address is a script, not a friend. - Restore drill (2026-09-01): a Spaces snapshot booted 56/56 rooms clean. The path: `rclone copy spaces:kestrel-wizwar-backups/wizwar/ snapshots/ ...` on the droplet, then run a local server with diff --git a/deploy/deploy.sh b/deploy/deploy.sh index 9c3920c..2aa0d2a 100755 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -15,6 +15,14 @@ ssh "root@$HOST" ' # deploy, or the repo copy and the live copy drift apart (they did: # the Sentry check-ins shipped in the repo and never reached the box). install -m 755 /opt/wizwar/deploy/wizwar-backup.sh /usr/local/bin/wizwar-backup.sh + # The nightly rollup and its cron entry ride along the same way; its + # Sentry check-in URL is the backup monitor'"'"'s with its own slug. + install -m 755 /opt/wizwar/deploy/wizwar-rollup.sh /usr/local/bin/wizwar-rollup.sh + install -m 644 /opt/wizwar/deploy/wizwar-rollup.cron /etc/cron.d/wizwar-rollup + if [ -f /root/.wizwar-sentry-cron ] && [ ! -f /root/.wizwar-sentry-cron-rollup ]; then + sed "s#/cron/new-monitor/#/cron/wizwar-rollup/#" /root/.wizwar-sentry-cron > /root/.wizwar-sentry-cron-rollup + chmod 600 /root/.wizwar-sentry-cron-rollup + fi systemctl daemon-reload systemctl enable --now wizwar systemctl restart wizwar diff --git a/deploy/setup-droplet.sh b/deploy/setup-droplet.sh index 7608d31..e70907a 100755 --- a/deploy/setup-droplet.sh +++ b/deploy/setup-droplet.sh @@ -25,7 +25,9 @@ mkdir -p /opt/wizwar /var/lib/wizwar/rooms chown -R wizwar:wizwar /opt/wizwar /var/lib/wizwar # Caddy vhost: auto-TLS, security headers, proxy to the game. -printf '%s\n\nheader {\n\tStrict-Transport-Security "max-age=31536000"\n\tX-Content-Type-Options "nosniff"\n\tX-Frame-Options "DENY"\n\tReferrer-Policy "no-referrer"\n}\nreverse_proxy localhost:8787\n' "$HOST" > /etc/caddy/Caddyfile +# Access log kept ~30 days (10MiB x 30): the nightly rollup keeps the +# counts forever, the raw lines back it for a month. +printf '%s\n\nheader {\n\tStrict-Transport-Security "max-age=31536000"\n\tX-Content-Type-Options "nosniff"\n\tX-Frame-Options "DENY"\n\tReferrer-Policy "no-referrer"\n}\nlog {\n\toutput file /var/lib/caddy/access.log {\n\t\troll_size 10MiB\n\t\troll_keep 30\n\t}\n}\nreverse_proxy localhost:8787\n' "$HOST" > /etc/caddy/Caddyfile systemctl reload caddy # Firewall: ssh + web only. The game server binds loopback and is reached diff --git a/deploy/wizwar-rollup.cron b/deploy/wizwar-rollup.cron new file mode 100644 index 0000000..2ef4fcd --- /dev/null +++ b/deploy/wizwar-rollup.cron @@ -0,0 +1,2 @@ +# Nightly rollup of yesterday's traffic, table, and vitals (UTC). Installed by deploy.sh. +10 0 * * * root /usr/local/bin/wizwar-rollup.sh diff --git a/deploy/wizwar-rollup.sh b/deploy/wizwar-rollup.sh new file mode 100755 index 0000000..48e004c --- /dev/null +++ b/deploy/wizwar-rollup.sh @@ -0,0 +1,117 @@ +#!/bin/bash +# Nightly rollup: one JSON line per day in /var/lib/wizwar/rollup.jsonl — +# yesterday's traffic (from Caddy's access log), the table's growth (from +# the room ledgers, stats.json and feedback.jsonl), and the box's vitals. +# Counts only: no addresses are kept. The pulse reads it for trends the +# self-rotating access log cannot keep. +# Cron: 10 0 * * * (UTC), see /etc/cron.d/wizwar-rollup (deploy.sh installs both). +# Sentry Crons check-in: /root/.wizwar-sentry-cron-rollup holds the URL +# (derived from the backup monitor's on first deploy; absent = no check-ins). +set -u +OUT="/var/lib/wizwar/rollup.jsonl" +LOG="/var/log/wizwar/rollup.log" +DAY="${1:-$(date -u -d 'yesterday' +%Y-%m-%d)}" +CRON_URL=$(cat /root/.wizwar-sentry-cron-rollup 2>/dev/null || true) +checkin() { [ -n "$CRON_URL" ] && curl -sf -o /dev/null "$CRON_URL?status=$1" || true; } + +checkin in_progress +if python3 - "$DAY" "$OUT" <<'PY' >> "$LOG" 2>&1 +import json, sys, os, glob, gzip, datetime, collections, subprocess, re +day, out = sys.argv[1], sys.argv[2] +d0 = datetime.datetime.fromisoformat(day).replace(tzinfo=datetime.timezone.utc) +t0, t1 = d0.timestamp(), (d0 + datetime.timedelta(days=1)).timestamp() +BOT = ("bot", "crawl", "spider", "facebookexternalhit", "slack", "discord", "twitter", "preview", "curl", "python", "go-http", "wget", "headless") + +# --- traffic: every access log file touched since the day began --- +req = 0; human = 0; bot = 0; hips = set(); bips = set() +paths = collections.Counter(); status = collections.Counter(); refs = collections.Counter(); ws = 0 +for f in sorted(glob.glob("/var/lib/caddy/access*.log*")): + if os.path.getmtime(f) < t0: continue + opener = gzip.open if f.endswith(".gz") else open + with opener(f, "rt", errors="ignore") as fh: + for line in fh: + try: r = json.loads(line) + except ValueError: continue + ts = r.get("ts", 0) + if not (t0 <= ts < t1): continue + q = r.get("request", {}); h = q.get("headers", {}) + ua = " ".join(h.get("User-Agent", [""])).lower() + ip = q.get("client_ip") or q.get("remote_ip") or "?" + isbot = any(k in ua for k in BOT) + req += 1 + if isbot: bot += 1; bips.add(ip) + else: human += 1; hips.add(ip) + uri = q.get("uri", "").split("?")[0] + if uri == "/ws": ws += 1 + key = ("/clips" if uri.startswith("/clips") else "/watch" if uri.startswith("/watch") + else "(assets)" if uri.startswith("/assets") or re.search(r"\.(png|jpg|svg|ico|css|js|webmanifest|mp4)$", uri) + else uri if uri in ("/", "/ws", "/robots.txt") else "(other)") + paths[key] += 1 + status[str(r.get("status", 0))[0] + "xx"] += 1 + ref = " ".join(h.get("Referer", [""])) + m = re.match(r"https?://([^/]+)", ref) + host = m.group(1) if m else "" + ours = host.endswith("kestrelsnest.social") or "104.236.96.198" in host + if host and not ours and not isbot: refs[host] += 1 + +# --- the table --- +new_rooms = 0; seats = collections.Counter() +for f in glob.glob("/var/lib/wizwar/rooms/*.jsonl"): + try: + with open(f) as fh: + meta = json.loads(fh.readline()) + ca = meta.get("createdAt") + if not ca: continue + ct = datetime.datetime.fromisoformat(ca.replace("Z", "+00:00")).timestamp() + if not (t0 <= ct < t1): continue + new_rooms += 1 + for line in fh: + try: r = json.loads(line) + except ValueError: continue + if r.get("kind") == "join" and not r.get("bot"): seats[r["name"]] += 1 + except OSError: continue +stats = {} +try: stats = json.load(open("/var/lib/wizwar/stats.json")) +except Exception: pass +reports = 0; replies = 0 +try: + for line in open("/var/lib/wizwar/feedback.jsonl"): + try: r = json.loads(line) + except ValueError: continue + at = datetime.datetime.fromisoformat(r["at"].replace("Z", "+00:00")).timestamp() + if t0 <= at < t1: reports += ("happened" in r); replies += ("reportId" in r) +except OSError: pass + +# --- vitals (now, at rollup time) --- +def sh(cmd): + try: return subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=20).stdout.strip() + except Exception: return "" +mem = sh("systemctl show wizwar -p MemoryCurrent --value"); peak = sh("systemctl show wizwar -p MemoryPeak --value") +disk = sh("df --output=pcent / | tail -1").strip().rstrip("%") +load = sh("cut -d' ' -f1 /proc/loadavg") +errs = sh("journalctl -u wizwar --since '%s' --until '%s' --no-pager 2>/dev/null | grep -ci 'unhandled protocol error'" % (day, (d0 + datetime.timedelta(days=1)).date())) +starts = sh("journalctl -u wizwar --since '%s' --until '%s' --no-pager 2>/dev/null | grep -c 'Started wizwar'" % (day, (d0 + datetime.timedelta(days=1)).date())) +ledger = sum(os.path.getsize(f) for f in glob.glob("/var/lib/wizwar/rooms/*.jsonl")) +row = { + "day": day, "requests": req, "human": human, "bot": bot, + "humanAddresses": len(hips), "botAddresses": len(bips), "socketConnects": ws, + "paths": dict(paths.most_common()), "status": dict(status), "referrers": dict(refs.most_common(10)), + "newRooms": new_rooms, "humanSeats": sum(seats.values()), "humanNames": sorted(seats), + "gamesFinishedTotal": stats.get("gamesFinished"), "gamesCreatedTotal": stats.get("gamesCreated"), + "reports": reports, "replies": replies, + "vitals": {"memMB": round(int(mem) / 1048576) if mem.isdigit() else None, + "memPeakMB": round(int(peak) / 1048576) if peak.isdigit() else None, + "diskPct": int(disk) if disk.isdigit() else None, "load1": float(load) if load else None, + "protocolErrors": int(errs) if errs.isdigit() else None, "serviceStarts": int(starts) if starts.isdigit() else None, + "ledgerMB": round(ledger / 1048576, 1)}, +} +# One line per day: a rerun for the same day replaces its line. +rows = [] +if os.path.exists(out): + rows = [l for l in open(out).read().splitlines() if l.strip() and json.loads(l).get("day") != day] +rows.append(json.dumps(row, separators=(",", ":"))) +tmp = out + ".tmp" +open(tmp, "w").write("\n".join(rows) + "\n"); os.replace(tmp, out) +print("%s rollup %s: %d requests, %d human addresses, %d new rooms, %d human seats" % (datetime.datetime.now(datetime.timezone.utc).isoformat(), day, req, len(hips), new_rooms, sum(seats.values()))) +PY +then checkin ok; else checkin error; fi diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index e25b942..8e0c0dd 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -67,6 +67,7 @@ import { import { engagementStats, recordHotseat } from "./stats"; import { appendFeedback, readFeedback, readClips, clipAssetPath } from "./store"; import { clipsIndexHtml, clipPageHtml } from "./clips"; +import { SlidingLimit, clientAddress } from "./ratelimit"; import { getShare, loadShares, mintShare } from "./shares"; import { renderSharePng } from "./ogimage"; import { BOT_LINES, type BanterTrigger } from "./banter"; @@ -81,6 +82,10 @@ if (process.env.SENTRY_DSN) { const MAX_SOCKETS = 300; // concurrent connections const MAX_ROOMS = 5000; // total rooms on the server const MAX_ROOMS_PER_CONN = 10; // rooms one connection may create +// Per-address limits, held across reconnects: a table of friends never +// nears them; a script filling the vault or the reports desk does. +const roomsPerAddress = new SlidingLimit(12, 60 * 60 * 1000); +const reportsPerAddress = new SlidingLimit(6, 60 * 60 * 1000); const MAX_COMMAND_BYTES = 16384; // serialized game command const MAX_MYGAMES_SEATS = 50; // seats checked per myGames request const CATCHUP_COOLDOWN_MS = 3000; // full-game replays are CPU-heavy @@ -409,6 +414,8 @@ interface Session { spectator: boolean; /** The raw seat token this connection authenticated with (memory only). */ token: string | null; + /** Where the connection came from, for the per-address limits. */ + address: string; claimFails: number; // Token bucket: refills at 15 msg/s up to a burst of 30. bucket: number; @@ -579,7 +586,7 @@ function broadcastRoomState(room: Room): void { } } -wss.on("connection", (socket) => { +wss.on("connection", (socket, req) => { if (sessions.size >= MAX_SOCKETS) { send(socket, { type: "error", message: "the tavern is packed — try again shortly" }); socket.close(); @@ -587,6 +594,7 @@ wss.on("connection", (socket) => { } const session: Session = { socket, playerId: null, roomId: null, spectator: false, token: null, claimFails: 0, + address: clientAddress(req.headers, req.socket.remoteAddress), bucket: 30, lastRefill: Date.now(), overLimitStrikes: 0, roomsCreated: 0, lastCatchUpAt: 0, hotseatReports: 0, }; @@ -626,6 +634,9 @@ wss.on("connection", (socket) => { if (session.roomsCreated >= MAX_ROOMS_PER_CONN || roomCount() >= MAX_ROOMS) { return send(socket, { type: "error", message: "no new rooms right now — try again later" }); } + if (!roomsPerAddress.allow(session.address)) { + return send(socket, { type: "error", message: "that's a lot of new tables from one place — try again in an hour" }); + } session.roomsCreated++; leaveGallery(session); const { room, token } = createRoom(name); @@ -868,6 +879,9 @@ wss.on("connection", (socket) => { String(raw ?? "").replace(/[\u0000-\u0009\u000b-\u001f\u007f]/g, " ").trim().slice(0, 2000); const happened = clean(msg.happened); if (!happened) return send(socket, { type: "error", message: "say what happened" }); + if (!reportsPerAddress.allow(session.address)) { + return send(socket, { type: "error", message: "the desk has plenty from you for now — more in an hour" }); + } appendFeedback({ id: randomBytes(4).toString("hex"), at: new Date().toISOString(), diff --git a/packages/server/src/ratelimit.ts b/packages/server/src/ratelimit.ts new file mode 100644 index 0000000..d48001f --- /dev/null +++ b/packages/server/src/ratelimit.ts @@ -0,0 +1,40 @@ +// A per-key sliding-window limiter for the two doors anyone may walk +// through unseated: creating rooms and filing reports. Per-connection +// caps reset on reconnect; these are keyed by client address and hold +// for the window. Memory is bounded by pruning keys whose windows have +// emptied. + +export class SlidingLimit { + private hits = new Map(); + private lastSweep = 0; + + constructor(private readonly max: number, private readonly windowMs: number) {} + + /** Record a hit for `key` if it is under the limit; false if it is not. */ + allow(key: string, now = Date.now()): boolean { + this.sweep(now); + const cutoff = now - this.windowMs; + const times = (this.hits.get(key) ?? []).filter((t) => t > cutoff); + if (times.length >= this.max) { this.hits.set(key, times); return false; } + times.push(now); + this.hits.set(key, times); + return true; + } + + private sweep(now: number): void { + if (now - this.lastSweep < this.windowMs) return; + this.lastSweep = now; + const cutoff = now - this.windowMs; + for (const [k, times] of this.hits) { + if (!times.some((t) => t > cutoff)) this.hits.delete(k); + } + } +} + +/** The client's address as Caddy reports it — the first hop in + * X-Forwarded-For — falling back to the socket's own peer. */ +export function clientAddress(headers: Record, remote: string | undefined): string { + const fwd = headers["x-forwarded-for"]; + const first = (Array.isArray(fwd) ? fwd[0] : fwd)?.split(",")[0]?.trim(); + return first || remote || "unknown"; +}