Begin Wyrm from the game kit

deploy/Caddyfile.tmpl

@@ -0,0 +1,49 @@
+__HOST__ {
+	# The hall-wide notice (hall-announce on the host): one file, served by every game's site.
+	handle_path /hall/* {
+		root * /var/lib/hall
+		header Cache-Control "no-store"
+		file_server
+	}
+
+	root * /opt/wyrm/build
+	encode gzip zstd
+
+	header {
+		Strict-Transport-Security "max-age=31536000"
+		X-Content-Type-Options "nosniff"
+		X-Frame-Options "DENY"
+		Referrer-Policy "no-referrer"
+	}
+
+	log {
+		output file /var/lib/caddy/wyrm-access.log {
+			roll_size 10MiB
+			roll_keep 30
+		}
+	}
+
+	# The game server answers the API and the websocket; a deploy restarts it for
+	# a few seconds, and the proxy holds requests that land in that gap.
+	@game path /api/* /ws
+	handle @game {
+		reverse_proxy localhost:8791 {
+			lb_try_duration 30s
+			lb_try_interval 250ms
+		}
+	}
+
+	# Everything else is the static site. SvelteKit writes each route as
+	# <route>.html, so /rules is tried as /rules.html before falling back to the
+	# app shell, which serves the rooms. Hashed assets under _app/immutable are
+	# cached for a year; every other response is revalidated so a deploy shows
+	# up on the next load.
+	handle {
+		@immutable path /_app/immutable/*
+		header @immutable Cache-Control "public, max-age=31536000, immutable"
+		@mutable not path /_app/immutable/*
+		header @mutable Cache-Control "no-cache"
+		try_files {path} {path}.html /index.html
+		file_server
+	}
+}

deploy/README.md

@@ -0,0 +1,109 @@
+# Deploying Wyrm
+
+Every game is a room on a small Node process that keeps it as an
+append-only ledger of moves in /var/lib/wyrm/rooms. Production is the
+shared Kestrel's Hall host: Caddy terminates TLS with automatic
+certificates, serves the static build from /opt/wyrm/build, and proxies
+/api and /ws to the game server on port 8791, which runs as the
+`wyrm` user under systemd from /opt/wyrm/app.
+
+## The shared host
+
+The games of Kestrel's Hall share one droplet, `hall` (nyc3, s-1vcpu-1gb,
+$6 a month, tag `kestrels-hall`, IP 159.203.115.252). Each game keeps its
+own Caddy site file in /etc/caddy/sites/<slug>.caddy, its own port,
+service user, /opt/<slug> and /var/lib/<slug> trees, access log
+(/var/lib/caddy/<slug>-access.log), cron file and ops scripts;
+/etc/caddy/Caddyfile only says `import sites/*.caddy`. A game server is one
+small Node process capped at 300 MB by its unit file, and a room is a few
+kilobytes of ledger, so the host carries several. Each game needs a port
+of its own on the host: wizwar 8787, waving-hands 8788, hnefatafl 8789,
+dip 8790.
+
+## Current production
+
+- Host: __IP__
+- URLs: https://wyrm.kestrelsnest.social (A record at Hover, where kestrelsnest.social's
+  DNS lives) and https://wyrm.__IP__.sslip.io (always works, zero DNS).
+- Everyday deploy: `deploy/deploy.sh __IP__`
+- Players' reports: `deploy/pull-reports.sh` mirrors /var/lib/wyrm/feedback.jsonl
+  and the screenshots to ~/Desktop/wyrm-reports with a digest;
+  `deploy/report-reply.sh __IP__ <id> <status> "text"` answers one.
+
+## Adding this game to the host
+
+1. `scp deploy/setup-droplet.sh deploy/Caddyfile.tmpl root@<ip>:/root/ && ssh root@<ip> \
+    "bash /root/setup-droplet.sh 'wyrm.kestrelsnest.social, wyrm.<ip>.sslip.io'"`
+   It installs Caddy and Node if the host lacks them, writes this game's
+   site file, and touches nothing of another game's. Point the A record at
+   the host first, or leave the real name out until it is; running it again
+   with the new line is how a name is added later.
+2. `deploy/deploy.sh <ip>`
+
+A fresh host, when one is needed: `doctl compute droplet create hall --region nyc3 \
+--size s-1vcpu-1gb --image ubuntu-24-04-x64 --ssh-keys <your-key-ids> \
+--tag-name kestrels-hall --wait`, then step 1.
+
+## Operations scripts on the droplet
+
+`deploy.sh` installs these to /usr/local/bin and the `traffic.py` they share
+to /usr/local/lib/wyrm on every deploy, so the live copies are the repo
+copies. The cron file goes to /etc/cron.d/wyrm only on a box of the game's
+own: on the hall's shared host, `hall-nightly.sh` (in the kit's `gamehall/ops`)
+runs every game's rollup and backup from /etc/cron.d/gamehall.
+
+- `wyrm-visitors.sh [day]`: who is here now and who came that day.
+- `wyrm-reseat.sh ROOM [SEAT]`: who sits at a table; with a seat, a new
+  token for a player locked out of it, as a link to send them.
+- `wyrm-say.sh ROOM "text"`: a word from the keeper in a table's talk,
+  signed as the keeper, through the same door; it is heard whether or not
+  the gallery may talk, so a call that came too late can be answered.
+- `wyrm-pulse.sh`: the weekly health check (service, errors, rollup
+  trend, backup, box).
+- `wyrm-rollup.sh [day]`: one JSON line per day of counts, run
+  nightly at 00:12 UTC into /var/lib/wyrm/rollup.jsonl.
+- `wyrm-backup.sh`: nightly at 07:23 UTC, mirrors /var/lib/wyrm
+  to the `kestrel-wizwar-backups` Space under `wyrm/` (a current copy
+  less the rollup, and dated snapshots kept 90 days). It needs rclone with the Spaces
+  credentials in /root/.config/rclone/rclone.conf, copied by hand from the
+  wizwar droplet; until then it logs "skipped".
+
+Errors from the game server go to Sentry, project `wyrm` in the
+locallygrownnet organisation; the DSN is in the unit file. On the shared host
+the hall's nightly job checks in with Sentry Crons once for every game's
+rollup and once for every backup (monitors `hall-rollup` and `hall-backup` in
+the `gamehall` project), and each game's script reports by its exit status.
+A game on its own box checks in for itself when /root/.wyrm-sentry-cron-rollup
+and /root/.wyrm-sentry-cron hold check-in URLs (the ingest URL with the
+project's cron path and public key); absent, it sends nothing.
+
+To have every new issue, regression and reappearance posted to Slack the way
+wizwar's are, run `deploy/sentry-slack-alert.sh [#channel]` once from your own
+shell with `SENTRY_TOKEN` (an org auth token with alerts:write) and
+`SLACK_CHANNEL_ID` (the channel's Slack ID) set; the token never leaves the
+shell.
+
+Before every deploy, `deploy/verify-ledgers.sh <ip>` fetches every production
+ledger and replays it with the local engine, comparing each room with what the
+server shows. A ledger the new engine refuses or replays differently stops the
+deploy: the server would rewrite that game on restart.
+
+## Everyday deploys
+
+    deploy/deploy.sh <ip>
+
+Runs the type-checks, the tests and the build locally, rsyncs `build/` to
+the droplet keeping the previous week's hashed assets, rsyncs the server and
+engine sources, installs dependencies, and restarts the game server, then
+waits up to twenty seconds for it to answer HTTP and fails with the tail of
+its journal if it does not. A room is replayed from its ledger when the
+server comes back, which takes a few seconds; Caddy holds requests that
+land in the gap.
+
+## Operations
+
+- Logs: `ssh root@<ip> journalctl -u wyrm -f` for the game server,
+  `journalctl -u caddy -f` and /var/lib/caddy/wyrm-access.log for the web side.
+- Restart: `ssh root@<ip> systemctl restart wyrm`
+- Rooms: `/var/lib/wyrm/rooms/<CODE>.jsonl`, one ledger per game; the
+  nightly backup keeps them in the Space.

deploy/deploy.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+# Build locally, push the static site and the game server to the droplet,
+# and restart the server. Usage: deploy/deploy.sh <droplet-ip-or-host>
+set -euo pipefail
+HOST="${1:?usage: deploy.sh <droplet-ip-or-host>}"
+
+# The family list is the kit's; a game deploys with the kit's latest copy when the kit is beside it.
+KIT_FAMILY="$(dirname "$0")/../../game-kit/family.json"
+[ -f "$KIT_FAMILY" ] && cp "$KIT_FAMILY" static/family.json
+
+npm run check
+npm run check:server
+npm test
+npm run build
+
+# Every production ledger must replay identically with the engine about to ship.
+deploy/verify-ledgers.sh "$HOST"
+
+# Old hashed chunks stay a week: a page loaded before this deploy can still
+# fetch the module it was built against instead of failing mid-game.
+rsync -az --delete --filter='P _app/immutable/*' \
+  build/ "root@$HOST:/opt/wyrm/build/"
+
+# The server runs from source with tsx and carries its own small manifest;
+# it needs only its code and the engine. Ledgers live outside this tree.
+rsync -az --delete --exclude='/server/node_modules' \
+  --include='/server/***' --include='/src/' --include='/src/lib/' --include='/src/lib/game/***' --include='/src/lib/net/' --include='/src/lib/net/view.ts' \
+  --include='/deploy/' --include='/deploy/wyrm.service' --include='/deploy/wyrm.cron' --include='/deploy/*.sh' --include='/deploy/traffic.py' \
+  --exclude='*' \
+  ./ "root@$HOST:/opt/wyrm/app/"
+
+ssh "root@$HOST" '
+  find /opt/wyrm/build/_app/immutable -type f -mtime +7 -delete
+  chown -R root:caddy /opt/wyrm/build
+  chmod -R g+rX /opt/wyrm/build
+  cd /opt/wyrm/app/server && npm install --no-audit --no-fund
+  chown -R wyrm:wyrm /opt/wyrm/app
+  cp /opt/wyrm/app/deploy/wyrm.service /etc/systemd/system/wyrm.service
+  # The ops scripts run from /usr/local/bin and share traffic.py: install them
+  # on every deploy so the live copies are always the repo copies.
+  install -m 755 /opt/wyrm/app/deploy/wyrm-visitors.sh /usr/local/bin/wyrm-visitors.sh
+  install -m 755 /opt/wyrm/app/deploy/wyrm-pulse.sh /usr/local/bin/wyrm-pulse.sh
+  install -m 755 /opt/wyrm/app/deploy/wyrm-rollup.sh /usr/local/bin/wyrm-rollup.sh
+  install -m 755 /opt/wyrm/app/deploy/wyrm-backup.sh /usr/local/bin/wyrm-backup.sh
+  install -m 755 /opt/wyrm/app/deploy/wyrm-reseat.sh /usr/local/bin/wyrm-reseat.sh
+  install -m 755 /opt/wyrm/app/deploy/wyrm-say.sh /usr/local/bin/wyrm-say.sh
+  # The key to the keeper door: made once, readable by root (the reseat script) and the game user (the server) alone.
+  if [ ! -s /etc/wyrm/keeper.key ]; then
+    install -d -m 750 -o root -g wyrm /etc/wyrm
+    (umask 037; openssl rand -hex 32 > /etc/wyrm/keeper.key)
+    chown root:wyrm /etc/wyrm/keeper.key
+  fi
+  install -m 644 -D /opt/wyrm/app/deploy/traffic.py /usr/local/lib/wyrm/traffic.py
+  # On a host the hall shares, its hall-nightly.sh runs the rollup and backup
+  # of every game and checks in with Sentry once for all; a game on a box of
+  # its own schedules its own.
+  if [ -f /etc/cron.d/gamehall ]; then rm -f /etc/cron.d/wyrm
+  else install -m 644 /opt/wyrm/app/deploy/wyrm.cron /etc/cron.d/wyrm; fi
+  mkdir -p /var/log/wyrm
+  systemctl daemon-reload
+  systemctl enable --now wyrm
+  systemctl restart wyrm
+  systemctl reload caddy
+  # "active" means only that the process started: wait for it to answer HTTP,
+  # so a server that dies loading its code fails the deploy instead of looping.
+  for i in $(seq 20); do
+    if curl -s -o /dev/null http://127.0.0.1:8791/api/rooms/ZZZZ; then
+      systemctl --no-pager -l status wyrm | head -3
+      exit 0
+    fi
+    sleep 1
+  done
+  echo "the game server is not answering:" >&2
+  journalctl -u wyrm -n 25 --no-pager >&2
+  exit 1
+'
+echo "deployed."

deploy/pull-reports.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# Mirror the player reports, the keeper's replies and the players' screenshots
+# to a local folder, and write a digest beside them for reading. Run from the
+# repository root:
+#   deploy/pull-reports.sh [host] [folder]   (default ~/Desktop/wyrm-reports)
+set -euo pipefail
+HOST="${1:-__IP__}"
+OUT="${2:-$HOME/Desktop/wyrm-reports}"
+mkdir -p "$OUT/images"
+scp -q "root@$HOST:/var/lib/wyrm/feedback.jsonl" "$OUT/feedback.jsonl" 2>/dev/null || : > "$OUT/feedback.jsonl"
+rsync -aq "root@$HOST:/var/lib/wyrm/feedback-images/" "$OUT/images/" 2>/dev/null || true
+npx tsx deploy/report-digest.ts "$OUT"

deploy/replay-ledgers.ts

@@ -0,0 +1,79 @@
+// Replays every ledger in a directory with THIS checkout's engine and, when
+// a server address is given, compares the result with what that server is
+// showing for the room: the round reached and the outcome. A ledger the
+// engine cannot replay, or replays differently, is reported and fails the
+// run. Used by deploy/verify-ledgers.sh.
+//   tsx deploy/replay-ledgers.ts <ledger-dir> [https://host]
+
+import { readdirSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { game } from '../src/lib/game';
+import { cleanOptions, type SeatId } from '../src/lib/game/spec';
+
+interface Seat {
+	id: SeatId;
+	name: string;
+	bot: boolean;
+}
+
+const [dir, host] = process.argv.slice(2);
+if (!dir) {
+	console.error('usage: replay-ledgers.ts <ledger-dir> [https://host]');
+	process.exit(2);
+}
+
+async function main(): Promise<void> {
+	const files = readdirSync(dir).filter((f) => f.endsWith('.jsonl'));
+	let failures = 0;
+	for (const file of files) {
+		const code = file.slice(0, -'.jsonl'.length);
+		const lines = readFileSync(join(dir, file), 'utf8')
+			.split('\n')
+			.filter((l) => l.trim())
+			.map((l) => JSON.parse(l));
+		const seats: Seat[] = [];
+		let state: ReturnType<typeof game.create> | null = null;
+		let turns = 0;
+		// The host's table options ride the room line, read exactly as the server reads them.
+		const options = cleanOptions(game, lines[0]?.t === 'room' ? lines[0].options : undefined);
+		try {
+			for (const line of lines) {
+				if (line.t === 'seat') seats.push(line);
+				else if (line.t === 'unseat') {
+					const i = seats.findIndex((s) => s.id === line.id);
+					if (i >= 0) seats.splice(i, 1);
+				}
+				else if (line.t === 'start') state = game.create(Object.fromEntries(seats.map((s) => [s.id, s.name])), line.seed, line.rules, options, line.rankings);
+				else if (line.t === 'turn' && state) {
+					state = game.resolve(state, line.inputs);
+					turns += 1;
+				}
+			}
+		} catch (e) {
+			console.log(`${code}: REFUSED at turn ${turns + 1}: ${e instanceof Error ? e.message : String(e)}`);
+			failures += 1;
+			continue;
+		}
+		const outcome = state ? game.over(state) : null;
+		let verdict = state ? `${turns} turns, ${outcome ? 'over' : 'in play'}` : 'not started';
+		if (host && state) {
+			// The gallery's view carries the round and the outcome, which is all the comparison needs.
+			const res = await fetch(`${host}/api/rooms/${code}`);
+			if (!res.ok) {
+				verdict += `, server ${res.status}`;
+			} else {
+				const theirs = (await res.json()) as { turn: number | null; over: { winner: SeatId | null } | null };
+				const same = theirs.turn === game.turn(state) && (theirs.over?.winner ?? null) === (outcome?.winner ?? null);
+				if (!same) {
+					failures += 1;
+					verdict += `, DIFFERS from the server (server round ${theirs.turn}, local round ${game.turn(state)})`;
+				} else verdict += ', matches the server';
+			}
+		}
+		console.log(`${code}: ${verdict}`);
+	}
+	console.log(`${files.length} ledger${files.length === 1 ? '' : 's'}, ${failures} problem${failures === 1 ? '' : 's'}`);
+	process.exit(failures ? 1 : 0);
+}
+
+void main();

deploy/report-digest.ts

@@ -0,0 +1,32 @@
+// The reports digest: every report with its exchange, newest first, written
+// as reports.md beside a mirrored feedback.jsonl. Used by deploy/pull-reports.sh.
+//   tsx deploy/report-digest.ts <folder>
+
+import { writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { Reports } from '../server/src/reports';
+
+const dir = process.argv[2];
+if (!dir) {
+	console.error('usage: report-digest.ts <folder>');
+	process.exit(2);
+}
+
+const when = (iso: string) => iso.slice(0, 16).replace('T', ' ');
+const reports = new Reports(dir).read().reverse();
+const lines = ['# Wyrm reports', '', `${reports.length} reports; newest first. Pictures are in \`images/\`.`, ''];
+for (const r of reports) {
+	lines.push(`## ${when(r.at)} — ${r.player} in ${r.roomId} (turn ${r.turn ?? '?'}, seq ${r.seq}) — id ${r.id}`, '');
+	lines.push(`**What happened:** ${r.happened.trim()}`);
+	if (r.expected.trim()) lines.push(`**What they expected:** ${r.expected.trim()}`);
+	if (r.image) lines.push(`**Picture:** ![${r.image}](images/${r.image})`);
+	for (const line of r.thread) {
+		const who = line.from === 'player' ? `${r.player} answers` : `**${line.status}**`;
+		lines.push(`> ${who} (${when(line.at)}): ${line.text.trim()}`);
+	}
+	const last = r.thread[r.thread.length - 1];
+	if (!last || last.from === 'player') lines.push('> _awaiting the keeper_');
+	lines.push('');
+}
+writeFileSync(join(dir, 'reports.md'), lines.join('\n'));
+console.log(`${reports.length} reports, ${reports.filter((r) => r.image).length} with pictures -> ${join(dir, 'reports.md')}`);

deploy/report-reply.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+# Answer a player's report; the reply appears under it in their hall.
+#   deploy/report-reply.sh <host> <reportId> <resolved|by-design|open> <text...>
+set -euo pipefail
+HOST="${1:?usage: report-reply.sh <host> <reportId> <status> <text...>}"
+REPORT_ID="${2:?usage: report-reply.sh <host> <reportId> <status> <text...>}"
+STATUS="${3:?usage: report-reply.sh <host> <reportId> <status> <text...>}"
+shift 3
+TEXT="$*"
+case "$STATUS" in resolved|by-design|open) ;; *) echo "status must be resolved, by-design, or open" >&2; exit 1;; esac
+[[ "$REPORT_ID" =~ ^[0-9a-f]{8}$ ]] || { echo "a report id is eight hex characters" >&2; exit 1; }
+LINE=$(REPORT_ID="$REPORT_ID" STATUS="$STATUS" TEXT="$TEXT" python3 -c '
+import json, os, datetime
+print(json.dumps({"reportId": os.environ["REPORT_ID"], "at": datetime.datetime.now(datetime.timezone.utc).isoformat().replace("+00:00", "Z"), "status": os.environ["STATUS"], "text": os.environ["TEXT"]}))')
+printf '%s\n' "$LINE" | ssh "root@$HOST" 'cat >> /var/lib/wyrm/feedback.jsonl && chown wyrm:wyrm /var/lib/wyrm/feedback.jsonl && tail -1 /var/lib/wyrm/feedback.jsonl'

deploy/sentry-slack-alert.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+# Route every Wyrm issue to a Slack channel: new issues (any level or
+# priority), regressions, and reappearances. Creates one Sentry workflow
+# via the alerts API. The token never leaves your shell:
+#   SENTRY_TOKEN=sntryu_... SLACK_CHANNEL_ID=C... deploy/sentry-slack-alert.sh [#channel]
+# SLACK_CHANNEL_ID is the channel's Slack ID; the name defaults to
+# #wyrm-notifications and is shown in Sentry beside the ID.
+# The token needs the alerts:write scope (an org auth token from
+# https://locallygrownnet.sentry.io/settings/auth-tokens/), and Slack must
+# already be connected to the org under Settings → Integrations.
+set -euo pipefail
+: "${SENTRY_TOKEN:?SENTRY_TOKEN=sntryu_... is required (alerts:write)}"
+ORG="${SENTRY_ORG:-locallygrownnet}"
+PROJECT="${SENTRY_PROJECT:-wyrm}"
+CHANNEL="${1:-#wyrm-notifications}"
+API="https://us.sentry.io/api/0/organizations/$ORG"
+AUTH="Authorization: Bearer $SENTRY_TOKEN"
+
+# The Slack action takes the channel's ID as well as its name; the
+# project's error detector is looked up by project id; a workflow is bound
+# to its detectors by PUT after creation.
+SLACK_ID="${SENTRY_SLACK_INTEGRATION:-345334}"
+: "${SLACK_CHANNEL_ID:?SLACK_CHANNEL_ID=C... is required (the Slack ID of the channel)}"
+CHANNEL_ID="$SLACK_CHANNEL_ID"
+PROJECT_ID="${SENTRY_PROJECT_ID:-__SENTRY_PROJECT_ID__}"
+echo "Slack integration $SLACK_ID; project $PROJECT_ID; channel $CHANNEL ($CHANNEL_ID)"
+
+DETECTOR_ID=$(curl -s "$API/detectors/?project=$PROJECT_ID" -H "$AUTH" | python3 -c '
+import json, sys
+rows = json.load(sys.stdin)
+for d in (rows if isinstance(rows, list) else []):
+    if str(d.get("projectId")) == sys.argv[1] and d.get("type") == "error": print(d["id"]); break' "$PROJECT_ID")
+[ -n "$DETECTOR_ID" ] || { echo "no error detector for project $PROJECT_ID" >&2; exit 1; }
+
+PAYLOAD=$(CHANNEL="$CHANNEL" CHANNEL_ID="$CHANNEL_ID" SLACK_ID="$SLACK_ID" DETECTOR_ID="$DETECTOR_ID" python3 -c '
+import json, os
+print(json.dumps({
+  "name": "Wyrm → Slack",
+  "enabled": True,
+  "environment": None,
+  "config": {"frequency": 0},
+  "triggers": {
+    "logicType": "any-short",
+    "conditions": [
+      {"type": "first_seen_event", "comparison": True, "conditionResult": True},
+      {"type": "regression_event", "comparison": True, "conditionResult": True},
+      {"type": "reappeared_event", "comparison": True, "conditionResult": True},
+    ],
+    "actions": [],
+  },
+  "actionFilters": [{
+    "logicType": "all",
+    "conditions": [],
+    "actions": [{
+      "type": "slack",
+      "integrationId": int(os.environ["SLACK_ID"]),
+      "data": {},
+      "config": {"targetType": "specific", "targetIdentifier": os.environ["CHANNEL_ID"], "targetDisplay": os.environ["CHANNEL"]},
+      "status": "active",
+    }],
+  }],
+  "detectorIds": [int(os.environ["DETECTOR_ID"])],
+}))')
+
+echo "creating the workflow…"
+RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "$API/workflows/" -H "$AUTH" -H "Content-Type: application/json" -d "$PAYLOAD")
+CODE=$(printf '%s' "$RESPONSE" | tail -1)
+BODY=$(printf '%s' "$RESPONSE" | sed '$d')
+if [ "$CODE" != "201" ]; then
+  echo "Sentry answered $CODE:" >&2; echo "$BODY" >&2; exit 1
+fi
+WORKFLOW_ID=$(printf '%s' "$BODY" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("id",""))')
+echo "created workflow $WORKFLOW_ID"
+# Binding to the detector is done by PUT, the whole workflow resent.
+curl -s -o /dev/null -w "bound to detector $DETECTOR_ID: %{http_code}\n" -X PUT "$API/workflows/$WORKFLOW_ID/" -H "$AUTH" -H "Content-Type: application/json" -d "$PAYLOAD"
+echo "done: https://$ORG.sentry.io/monitors/alerts/$WORKFLOW_ID/"

deploy/setup-droplet.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+# Prepare a host for Wyrm. The host may be shared with other games of
+# Kestrel's Hall: each keeps its own Caddy site file in /etc/caddy/sites, its
+# own port, service user, data directory and access log, and this script
+# touches nothing that belongs to another game. Copy it and Caddyfile.tmpl
+# to the host and run ON it as root; safe to run again.
+#   bash setup-droplet.sh 'wyrm.kestrelsnest.social, wyrm.<host-ip>.sslip.io'
+# The argument is this game's Caddy site address line: one name, or several
+# separated by commas. Every name must already resolve to this host.
+set -euo pipefail
+HOST="${1:?usage: setup-droplet.sh <site address line>}"
+
+apt-get update -q
+apt-get install -qy curl rsync
+
+# Caddy (auto-HTTPS), once per host.
+if ! command -v caddy >/dev/null; then
+  apt-get install -qy debian-keyring debian-archive-keyring apt-transport-https
+  curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' \
+    | gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg
+  curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' \
+    | tee /etc/apt/sources.list.d/caddy-stable.list
+  apt-get update -q && apt-get install -qy caddy
+fi
+
+# Node 22, once per host; every game runs its server on it.
+if ! command -v node >/dev/null || [[ "$(node -v)" != v22* ]]; then
+  curl -fsSL https://deb.nodesource.com/setup_22.x | bash -
+  apt-get install -qy nodejs
+fi
+
+# The host's Caddyfile only gathers the games' site files. A Caddyfile that
+# is neither Caddy's stock one nor this is a single-game host from before the
+# shared layout: stop rather than overwrite it.
+ROOT_CADDY='# Kestrel'"'"'s Hall: every game on this host brings its own site file.
+import sites/*.caddy'
+mkdir -p /etc/caddy/sites
+if ! grep -q '^import sites/\*.caddy' /etc/caddy/Caddyfile 2>/dev/null; then
+  if [ -s /etc/caddy/Caddyfile ] && ! grep -q 'root \* /usr/share/caddy' /etc/caddy/Caddyfile; then
+    echo "/etc/caddy/Caddyfile holds another site; move it into /etc/caddy/sites/ first" >&2
+    exit 1
+  fi
+  printf '%s\n' "$ROOT_CADDY" > /etc/caddy/Caddyfile
+fi
+
+# This game: its user, its directories, its site.
+# The service user shares the slug. A slug that is already a system group
+# (Ubuntu ships "dip", "games", "audio" and others) cannot also be a user:
+# stop and say so, and choose another user name in this script and the unit.
+if ! id -u wyrm &>/dev/null; then
+  if getent group wyrm >/dev/null; then
+    echo "a system group is already called wyrm; pick another service user name" >&2
+    exit 1
+  fi
+  useradd -r -m -d /opt/wyrm-home wyrm
+fi
+mkdir -p /opt/wyrm/build /opt/wyrm/app /var/lib/wyrm/rooms /var/log/wyrm
+chown -R root:caddy /opt/wyrm/build
+chmod -R g+rX /opt/wyrm/build
+chown -R wyrm:wyrm /var/lib/wyrm
+sed "s|__HOST__|$HOST|" "$(dirname "$0")/Caddyfile.tmpl" > /etc/caddy/sites/wyrm.caddy
+
+caddy validate --config /etc/caddy/Caddyfile
+# Validation as root touches the access logs; Caddy runs as caddy and must own them.
+chown -R caddy:caddy /var/lib/caddy
+systemctl reload-or-restart caddy
+
+# Firewall: ssh + web only.
+ufw allow OpenSSH
+ufw allow 80/tcp
+ufw allow 443/tcp
+ufw --force enable
+
+echo "host ready for wyrm: now run deploy/deploy.sh <ip> from your machine"

deploy/traffic.py

@@ -0,0 +1,170 @@
+"""Caddy's access log for one day, people and bots apart. The visitors
+digest and the nightly rollup both count through here, so they agree."""
+
+import datetime
+import glob
+import gzip
+import json
+import os
+import re
+from collections import Counter, defaultdict
+
+BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|whatsapp|telegram|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I)
+# Link previews: the app that fetched the page to show it in a conversation. The people
+# who click such a link arrive with no referrer, so this is the best sign of where it went.
+UNFURL = [
+    ("Slack", re.compile(r"slack", re.I)),
+    ("Discord", re.compile(r"discord", re.I)),
+    ("iMessage", re.compile(r"facebookexternalhit.*twitterbot|applebot.*preview", re.I)),
+    ("Facebook", re.compile(r"facebookexternalhit|facebot|meta-externalagent", re.I)),
+    ("WhatsApp", re.compile(r"whatsapp", re.I)),
+    ("Telegram", re.compile(r"telegram", re.I)),
+    ("Reddit", re.compile(r"redditbot", re.I)),
+    ("LinkedIn", re.compile(r"linkedinbot", re.I)),
+    ("Mastodon", re.compile(r"mastodon", re.I)),
+    ("Bluesky", re.compile(r"bluesky|cardyb", re.I)),
+    ("X", re.compile(r"twitterbot", re.I)),
+]
+# Paths nobody asks a game for but a vulnerability scanner: its address is a bot's all day.
+# Matched against whole path segments, never the site's own hashed assets under /_app/.
+SCANNER = re.compile(r"\.php$|(^|/)(\.env|\.git|\.aws|wp-admin|wp-login|wp-content|wp-includes|wordpress|phpmyadmin|cgi-bin|vendor|actuator|boaform|owa|admin)(/|$)", re.I)
+PAGES = {"/": "hall", "/rules": "rules", "/guide": "guide"}
+SITE = "wyrm.kestrelsnest.social"
+# Each game on a shared host keeps its own access log.
+LOGS = "/var/lib/caddy/wyrm-access*.log*"
+# The site's own addresses, for telling a referrer from a move between its pages: its domain, and its sslip.io name.
+OWN = re.compile(re.escape(SITE) + r"|\.sslip\.io\b", re.I)
+# Requests from one address more than this far apart are separate visits.
+VISIT_GAP = 30 * 60
+
+
+def unfurled_by(ua):
+    """The app a link preview came from, or None."""
+    for name, pattern in UNFURL:
+        if pattern.search(ua):
+            return name
+    return None
+
+
+def parse(r):
+    """The parts of one log line the counts read."""
+    q = r.get("request", {})
+    h = q.get("headers", {})
+    uri = q.get("uri", "")
+    return dict(
+        ts=r.get("ts", 0), status=r.get("status", 0), method=q.get("method"), uri=uri, path=uri.split("?")[0],
+        ip=q.get("client_ip") or q.get("remote_ip") or "?",
+        ua=" ".join(h.get("User-Agent", [""])), referer=" ".join(h.get("Referer", [""])),
+    )
+
+
+def visit_starts(times):
+    """When each visit began: requests more than VISIT_GAP apart are separate visits."""
+    starts, last = [], None
+    for ts in sorted(times):
+        if last is None or ts - last > VISIT_GAP:
+            starts.append(ts)
+        last = ts
+    return starts
+
+
+def scan(day, logs=LOGS):
+    """Counts for one UTC day (YYYY-MM-DD): every request, page requests by
+    people, the addresses of people and of bots, requests by page, room links
+    opened, rooms opened from the site itself, where links were shared,
+    referrers, and campaign tags (each counted once per address).
+
+    A request is a bot's when its browser says so. An address is a bot's for
+    the whole day when it asks for a scanner's paths, or comes back three
+    visits or more without ever opening or joining a game: a monitor,
+    not a person. A monitor that takes a new address every visit gives itself
+    away by keeping time: one browser, page loads at a steady interval, four
+    visits or more, never a game."""
+    d0 = datetime.datetime.fromisoformat(day).replace(tzinfo=datetime.timezone.utc)
+    t0, t1 = d0.timestamp(), (d0 + datetime.timedelta(days=1)).timestamp()
+    requests = []
+    for f in sorted(glob.glob(logs)):
+        if os.path.getmtime(f) < t0:
+            continue
+        opener = gzip.open if f.endswith(".gz") else open
+        try:
+            with opener(f, "rt", errors="ignore") as fh:
+                for line in fh:
+                    try:
+                        r = json.loads(line)
+                    except ValueError:
+                        continue
+                    if t0 <= r.get("ts", 0) < t1:
+                        requests.append(parse(r))
+        except OSError:
+            continue
+
+    # First pass: what each address did all day under a browser's name, to find scanners and monitors.
+    seen = defaultdict(lambda: dict(times=[], played=False, scanner=False))
+    for r in requests:
+        if BOT.search(r["ua"]) or not r["ua"]:
+            continue
+        path = r["path"]
+        a = seen[r["ip"]]
+        a["times"].append(r["ts"])
+        if not path.startswith("/_app/") and SCANNER.search(path):
+            a["scanner"] = True
+        if path.startswith(("/join/", "/api/rooms")) or path == "/ws":
+            a["played"] = True
+
+    def keeps_time(times):
+        starts = visit_starts(times)
+        if len(starts) < 4:
+            return False
+        gaps = [b - a for a, b in zip(starts, starts[1:])]
+        typical = sorted(gaps)[len(gaps) // 2]
+        return all(0.75 * typical <= g <= 1.25 * typical for g in gaps)
+
+    scanners = {ip for ip, a in seen.items() if a["scanner"]}
+    monitors = {ip for ip, a in seen.items() if not (a["scanner"] or a["played"]) and len(visit_starts(a["times"])) >= 3}
+    by_browser = defaultdict(list)
+    for r in requests:
+        if r["ua"] and not BOT.search(r["ua"]) and r["path"] in PAGES and not seen[r["ip"]]["played"]:
+            by_browser[r["ua"]].append((r["ts"], r["ip"]))
+    for hits in by_browser.values():
+        if keeps_time([ts for ts, _ in hits]):
+            monitors |= {ip for _, ip in hits}
+    posing = scanners | monitors
+
+    t = dict(
+        requests=0, human=0, bot=0, addresses=set(), botAddresses=set(), pages=Counter(), joins=Counter(),
+        opened=0, shared=Counter(), scanners=len(scanners), monitors=len(monitors), referrers=Counter(), campaigns=Counter(),
+    )
+    seen_campaign = set()
+    seen_share = set()
+    for r in requests:
+        ip, ua, uri, path = r["ip"], r["ua"], r["uri"], r["path"]
+        t["requests"] += 1
+        if ip in posing or BOT.search(ua) or not ua:
+            t["bot"] += 1
+            t["botAddresses"].add(ip)
+            service = unfurled_by(ua)
+            # One share per app and page: a preview is often fetched twice.
+            if service and (service, path) not in seen_share:
+                seen_share.add((service, path))
+                t["shared"][service] += 1
+            continue
+        # A room opened from the site itself never passes through /join: count the API's answer.
+        if r["method"] == "POST" and path == "/api/rooms" and 200 <= r["status"] < 300:
+            t["opened"] += 1
+        if path.startswith(("/_app/", "/api/")) or path == "/ws" or path.endswith((".png", ".ico", ".txt")):
+            continue
+        t["human"] += 1
+        t["addresses"].add(ip)
+        if path in PAGES:
+            t["pages"][PAGES[path]] += 1
+        elif path.startswith("/join/"):
+            t["joins"][path.split("/")[2].upper()] += 1
+        ref = r["referer"]
+        if ref and not OWN.search(ref):
+            t["referrers"][re.sub(r"^https?://", "", ref).split("/")[0]] += 1
+        m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri)
+        if m and (ip, m.group(1)) not in seen_campaign:
+            seen_campaign.add((ip, m.group(1)))
+            t["campaigns"][m.group(1) + ("=" + m.group(2)[:24] if m.group(1) != "fbclid" else "")] += 1
+    return t

deploy/verify-ledgers.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# The determinism gate: fetch every production ledger and replay it with the
+# LOCAL engine, comparing each room with what the server shows. A ledger the
+# new engine replays differently would rewrite a game in progress when the
+# server restarts, so this runs before every deploy.
+#   deploy/verify-ledgers.sh <droplet-ip-or-host> [https://site]
+set -euo pipefail
+HOST="${1:?usage: verify-ledgers.sh <droplet-ip-or-host> [https://site]}"
+# SITE overrides the public name: the sslip.io address, or a name not yet in DNS.
+SITE="${2:-${SITE:-https://wyrm.kestrelsnest.social}}"
+DIR="$(mktemp -d)"
+trap 'rm -rf "$DIR"' EXIT
+rsync -az "root@$HOST:/var/lib/wyrm/rooms/" "$DIR/"
+npx tsx deploy/replay-ledgers.ts "$DIR" "$SITE"

deploy/wyrm-backup.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# Nightly ledger backup to DigitalOcean Spaces, one prefix per game in the shared bucket.
+#   current/    - mirror of /var/lib/wyrm, less the rollup (rebuilt from logs)
+#   snapshots/  - one dated copy per day, pruned after 90 days
+# Ledgers are append-only JSONL; the game server never needs stopping.
+# Sentry Crons check-in: /root/.wyrm-sentry-cron holds the check-in
+# URL (absent file = no check-ins). SCHEDULE must match /etc/cron.d/wyrm.
+# On a host the hall shares, hall-nightly.sh runs this and checks in once
+# for every game, so the file is absent and the exit status is the word.
+set -u
+SCHEDULE="23 7 * * *"
+BUCKET="kestrel-wizwar-backups" # the hall's shared Space, named for its first tenant
+PREFIX="wyrm"
+SRC="/var/lib/wyrm"
+LOG="/var/log/wyrm/backup.log"
+STAMP=$(date +%Y-%m-%d)
+mkdir -p "$(dirname "$LOG")"
+CRON_URL=$(cat /root/.wyrm-sentry-cron 2>/dev/null || true)
+# One check-in, when the run ends, carrying its duration: a start and a
+# finish sent seconds apart can reach Sentry in either order, and a start
+# that lands second is never closed. A run that never ends sends nothing,
+# and Sentry marks the night missed.
+START=$SECONDS
+checkin() {
+  [ -n "$CRON_URL" ] && curl -sf -o /dev/null -X POST -H "Content-Type: application/json" \
+    -d "{\"status\":\"$1\",\"duration\":$((SECONDS - START)),\"monitor_config\":{\"schedule\":{\"type\":\"crontab\",\"value\":\"$SCHEDULE\"},\"checkin_margin\":30,\"max_runtime\":10,\"timezone\":\"UTC\"}}" \
+    "$CRON_URL" || true
+}
+
+if [ ! -s /root/.config/rclone/rclone.conf ]; then
+  echo "$(date -Is) skipped: rclone is not configured" >> "$LOG"
+  exit 1
+fi
+
+status=0
+{
+  echo "=== $(date -Is) backup start"
+  if rclone sync "$SRC" "spaces:$BUCKET/$PREFIX/current" --exclude 'rollup.jsonl' 2>&1 &&
+     rclone copy "$SRC" "spaces:$BUCKET/$PREFIX/snapshots/$STAMP" 2>&1; then
+    checkin ok
+  else
+    checkin error
+    status=1
+  fi
+  CUTOFF=$(date -d "90 days ago" +%Y-%m-%d)
+  rclone lsf "spaces:$BUCKET/$PREFIX/snapshots/" --dirs-only 2>/dev/null | \
+  while read -r d; do
+    day="${d%/}"
+    if [[ "$day" < "$CUTOFF" ]]; then
+      echo "pruning snapshot $day"
+      rclone purge "spaces:$BUCKET/$PREFIX/snapshots/$day" 2>&1
+    fi
+  done
+  echo "=== $(date -Is) backup done"
+} >> "$LOG" 2>&1
+exit $status

deploy/wyrm-pulse.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# The operations pulse, printed ON the droplet: the game server's health,
+# errors in its journal, the last week of the rollup, the backup's last word,
+# and the box's vitals. Read by the wyrm-pulse skill.
+set -u
+echo "as of $(date -u '+%F %H:%M') UTC on $(hostname); up $(uptime -p | sed 's/up //')"
+echo "--- game server"
+systemctl is-active wyrm >/dev/null && echo "wyrm: active since $(systemctl show wyrm -p ActiveEnterTimestamp --value)" || echo "wyrm: NOT ACTIVE"
+systemctl is-active caddy >/dev/null && echo "caddy: active" || echo "caddy: NOT ACTIVE"
+echo "restarts in 7 days: $(journalctl -u wyrm --since '7 days ago' -o cat | grep -c 'Started wyrm')"
+ERR=$(journalctl -u wyrm --since '7 days ago' -p err -o cat | grep -vc '^$')
+echo "journal errors in 7 days: $ERR"
+journalctl -u wyrm --since '7 days ago' -o cat | grep -iE "error|unhandled|exception" | tail -3 | sed 's/^/  /'
+echo "--- rollup, last 7 days (people / requests / rooms opened / turns)"
+if [ -s /var/lib/wyrm/rollup.jsonl ]; then
+  tail -7 /var/lib/wyrm/rollup.jsonl | python3 -c '
+import json, sys
+for l in sys.stdin:
+    r = json.loads(l); rm = r["rooms"]
+    print("  %s  %3d people  %4d requests  %2d rooms (%d started, %d with turns)  %3d turns  bots %d addr  refs %s" % (
+        r["day"], r["humanAddresses"], r["human"], rm["created"], rm["started"], rm["withTurns"], r["turns"], r["botAddresses"],
+        ",".join(sorted(r["referrers"])) or "-"))'
+else
+  echo "  (no rollup yet)"
+fi
+echo "--- ledgers"
+echo "rooms on disk: $(ls /var/lib/wyrm/rooms/*.jsonl 2>/dev/null | wc -l); touched in 24h: $(find /var/lib/wyrm/rooms -name '*.jsonl' -mmin -1440 2>/dev/null | wc -l); size $(du -sh /var/lib/wyrm/rooms 2>/dev/null | cut -f1)"
+echo "--- backup"
+if [ -f /var/log/wyrm/backup.log ]; then
+  grep -E "backup (start|done)|skipped|error|ERROR" /var/log/wyrm/backup.log | tail -2 | sed 's/^/  /'
+else
+  echo "  never run"
+fi
+echo "--- box"
+echo "disk: $(df -h / | awk 'NR==2{print $5" used of "$2}'); memory: $(free -m | awk 'NR==2{print $7" MB available of "$2}'); load: $(cut -d' ' -f1-3 /proc/loadavg)"
+echo "caddy access log: $(du -sh /var/lib/caddy/wyrm-access.log 2>/dev/null | cut -f1) live, $(ls /var/lib/caddy/wyrm-access*.log* 2>/dev/null | wc -l) files"

deploy/wyrm-reseat.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# A way back for a player whose browser lost their seat: the seat gets a new
+# token, the old one stops working, and the link printed carries the new one.
+# Whether the one asking holds that seat is the keeper's call; send the link
+# only to them. Runs ON the droplet, through the game server's keeper door.
+#   wyrm-reseat.sh ROOM          who sits at the table, and when each person last moved or spoke
+#   wyrm-reseat.sh ROOM SEAT     re-seat SEAT and print the link
+set -euo pipefail
+ROOM="${1:?usage: wyrm-reseat.sh ROOM [SEAT]}"
+DOOR="http://127.0.0.1:8791/keeper/rooms/${ROOM^^}"
+[ -s /etc/wyrm/keeper.key ] || { echo "No keeper's key at /etc/wyrm/keeper.key; a deploy makes it." >&2; exit 1; }
+# The key goes as a header read from a file descriptor, never on a command line another user could see.
+KEY="@/dev/fd/3"
+exec 3< <(printf 'x-keeper-key: %s\n' "$(cat /etc/wyrm/keeper.key)")
+if [ $# -lt 2 ]; then
+  curl -s -H "$KEY" "$DOOR" | python3 -c '
+import datetime, json, sys
+r = json.load(sys.stdin)
+if "error" in r: sys.exit(r["error"])
+print("room %s, %s" % (r["roomId"], "in play" if r["started"] else "not begun"))
+for s in r["seats"]:
+    when = "bot" if s["bot"] else ("a bot stands in; " if s.get("standIn") else "") + ("never moved or spoke" if s["lastActed"] is None else "last acted " + datetime.datetime.fromtimestamp(s["lastActed"] / 1000, datetime.timezone.utc).strftime("%Y-%m-%d %H:%M UTC"))
+    print("  %-3s %-24s %s" % (s["id"], s["name"], when))
+'
+else
+  curl -s -X POST -H "$KEY" -H 'content-type: application/json' -d "{\"seat\":\"$2\"}" "$DOOR/reseat" | python3 -c '
+import json, sys
+r = json.load(sys.stdin)
+if "error" in r: sys.exit(r["error"])
+print("%s has a new token; their old one opens nothing now. Send them this link:" % r["name"])
+print(r["link"])
+'
+fi

deploy/wyrm-rollup.sh

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+# Nightly rollup: one JSON line per day in /var/lib/wyrm/rollup.jsonl:
+# the day's traffic from Caddy's access log (people and bots apart, by page,
+# with referrers and the campaign tags links carry), the rooms opened,
+# started and finished, and the box's vitals. Counts only: no addresses are
+# kept. The pulse reads it for trends the self-rotating access log loses.
+#   wyrm-rollup.sh [YYYY-MM-DD]   (default: yesterday, UTC)
+#   Cron: /etc/cron.d/wyrm, installed by deploy.sh; SCHEDULE below
+#   must match it, because the Sentry monitor is told the same shape.
+#   Sentry Crons check-in: /root/.wyrm-sentry-cron-rollup holds the
+#   check-in URL (absent = no check-ins). On a host the hall shares,
+#   hall-nightly.sh runs this and checks in once for every game, so the
+#   file is absent and the exit status is the word.
+set -u
+OUT="/var/lib/wyrm/rollup.jsonl"
+LOG="/var/log/wyrm/rollup.log"
+DAY="${1:-$(date -u -d 'yesterday' +%Y-%m-%d)}"
+SCHEDULE="12 0 * * *"
+CRON_URL=$(cat /root/.wyrm-sentry-cron-rollup 2>/dev/null || true)
+# One check-in, when the run ends, carrying its duration and the schedule
+# (Sentry keeps a monitor only once told its shape). A start and a finish
+# sent seconds apart can reach Sentry in either order, and a start that lands
+# second is never closed. A run that never ends sends nothing, and Sentry
+# marks the night missed.
+START=$SECONDS
+checkin() {
+  [ -n "$CRON_URL" ] && curl -sf -o /dev/null -X POST -H "Content-Type: application/json" \
+    -d "{\"status\":\"$1\",\"duration\":$((SECONDS - START)),\"monitor_config\":{\"schedule\":{\"type\":\"crontab\",\"value\":\"$SCHEDULE\"},\"checkin_margin\":30,\"max_runtime\":10,\"timezone\":\"UTC\"}}" \
+    "$CRON_URL" || true
+}
+mkdir -p "$(dirname "$LOG")"
+if python3 - "$DAY" "$OUT" <<'PY' >> "$LOG" 2>&1
+import datetime, glob, json, os, shutil, sys
+sys.path.insert(0, "/usr/local/lib/wyrm")
+import traffic
+day, out = sys.argv[1], sys.argv[2]
+d0 = datetime.datetime.fromisoformat(day).replace(tzinfo=datetime.timezone.utc)
+t0, t1 = d0.timestamp(), (d0 + datetime.timedelta(days=1)).timestamp()
+t = traffic.scan(day)
+
+created = started = finished = turned = turns = 0
+humans, bots = set(), 0
+for f in glob.glob("/var/lib/wyrm/rooms/*.jsonl"):
+    try: lines = [json.loads(l) for l in open(f) if l.strip()]
+    except Exception: continue
+    if not lines or lines[0].get("t") != "room": continue
+    made = lines[0]["createdAt"] / 1000
+    if t0 <= made < t1:
+        created += 1
+        if any(x["t"] == "start" for x in lines): started += 1
+        if any(x["t"] == "turn" for x in lines): turned += 1
+        for x in lines:
+            if x["t"] == "seat":
+                if x["bot"]: bots += 1
+                else: humans.add(x["name"])
+            elif x["t"] == "unseat": bots -= 1
+    turns += sum(1 for x in lines if x["t"] == "turn" and t0 <= x["at"] / 1000 < t1)
+    # The over line is written with the turn that ends the game, so a finish counts on its own day.
+    if any(x["t"] == "over" and t0 <= x["at"] / 1000 < t1 for x in lines): finished += 1
+
+disk = shutil.disk_usage("/")
+mem = {}
+for line in open("/proc/meminfo"):
+    k, v = line.split(":", 1); mem[k] = int(v.split()[0])
+load = os.getloadavg()[1]
+row = dict(day=day, requests=t["requests"], human=t["human"], bot=t["bot"], humanAddresses=len(t["addresses"]), botAddresses=len(t["botAddresses"]),
+           paths=dict(t["pages"]), roomLinks=sum(t["joins"].values()), roomsOpened=t["opened"], sharedOn=dict(t["shared"]),
+           scanners=t["scanners"], monitors=t["monitors"], referrers=dict(t["referrers"]), campaigns=dict(t["campaigns"]),
+           rooms=dict(created=created, started=started, withTurns=turned, finished=finished, humanNames=len(humans), botSeats=bots),
+           turns=turns, ledgers=len(glob.glob("/var/lib/wyrm/rooms/*.jsonl")),
+           vitals=dict(diskUsedPct=round(disk.used * 100 / disk.total), memAvailableMb=mem.get("MemAvailable", 0) // 1024, load5=round(load, 2)),
+           at=datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds"))
+rows = []
+if os.path.exists(out):
+    rows = [json.loads(l) for l in open(out) if l.strip()]
+rows = [r for r in rows if r.get("day") != day] + [row]
+rows.sort(key=lambda r: r["day"])
+with open(out + ".tmp", "w") as fh:
+    for r in rows: fh.write(json.dumps(r) + "\n")
+os.replace(out + ".tmp", out)
+print("%s rolled: %d people, %d requests, %d rooms" % (day, len(t["addresses"]), t["human"], created))
+PY
+then checkin ok; else checkin error; exit 1; fi

deploy/wyrm-say.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# A word from the keeper in a table's talk, signed as the keeper, whether or
+# not the host lets the gallery talk: the answer to a call that came too late,
+# or anything else the table should hear from the keeper. Runs ON the droplet,
+# through the game server's keeper door.
+#   wyrm-say.sh ROOM "text"
+set -euo pipefail
+ROOM="${1:?usage: wyrm-say.sh ROOM \"text\"}"
+TEXT="${2:?usage: wyrm-say.sh ROOM \"text\"}"
+DOOR="http://127.0.0.1:8791/keeper/rooms/${ROOM^^}"
+[ -s /etc/wyrm/keeper.key ] || { echo "No keeper's key at /etc/wyrm/keeper.key; a deploy makes it." >&2; exit 1; }
+# The key goes as a header read from a file descriptor, never on a command line another user could see.
+KEY="@/dev/fd/3"
+exec 3< <(printf 'x-keeper-key: %s\n' "$(cat /etc/wyrm/keeper.key)")
+BODY=$(TEXT="$TEXT" python3 -c 'import json, os; print(json.dumps({"text": os.environ["TEXT"]}))')
+CODE=$(curl -s -o /tmp/wyrm-say.$$ -w '%{http_code}' -X POST -H "$KEY" -H 'content-type: application/json' -d "$BODY" "$DOOR/say")
+if [ "$CODE" = 204 ]; then echo "said at ${ROOM^^}"; else cat /tmp/wyrm-say.$$ >&2; echo >&2; rm -f /tmp/wyrm-say.$$; exit 1; fi
+rm -f /tmp/wyrm-say.$$

deploy/wyrm-visitors.sh

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+# The visitors digest: who is at the table right now and who came today,
+# the rooms opened and how far each got, with today's traffic so far.
+# Runs ON the droplet:  wyrm-visitors.sh [YYYY-MM-DD]   (default: today, UTC)
+# Every game, bot games included, is a room with a ledger, so all show in full.
+set -u
+DAY="${1:-$(date -u +%F)}"
+python3 - "$DAY" <<'PY'
+import datetime, glob, json, subprocess, sys
+sys.path.insert(0, "/usr/local/lib/wyrm")
+import traffic
+
+day = sys.argv[1]
+now = datetime.datetime.now(datetime.timezone.utc)
+print("as of %s UTC, day %s" % (now.strftime("%H:%M"), day))
+
+def sh(cmd):
+    try:
+        return subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=20).stdout.strip()
+    except Exception:
+        return "?"
+
+print("live sockets: %s | rooms touched in the last hour: %s" % (
+    sh("ss -tn state established '( sport = :8791 )' | tail -n +2 | wc -l"),
+    sh("find /var/lib/wyrm/rooms -name '*.jsonl' -mmin -60 2>/dev/null | wc -l")))
+
+# --- traffic: Caddy's access log, people and bots apart ---
+t = traffic.scan(day)
+print("traffic: %d addresses of people, %d page requests (%d bot addresses, of them %d scanners, %d monitors); hall %d, rules %d, guide %d; rooms opened here %d, room links %s; shared on %s; referrers %s; campaigns %s" % (
+    len(t["addresses"]), t["human"], len(t["botAddresses"]), t["scanners"], t["monitors"], t["pages"]["hall"], t["pages"]["rules"], t["pages"]["guide"],
+    t["opened"], dict(t["joins"]) or "none", dict(t["shared"]) or "none", dict(t["referrers"]) or "none", dict(t["campaigns"]) or "none"))
+
+# --- rooms: every ledger, today's in detail ---
+first_seen = {}
+rooms = []
+for f in glob.glob("/var/lib/wyrm/rooms/*.jsonl"):
+    try:
+        lines = [json.loads(l) for l in open(f) if l.strip()]
+        meta = lines[0]
+        assert meta["t"] == "room"
+    except Exception:
+        continue
+    created = datetime.datetime.fromtimestamp(meta["createdAt"] / 1000, datetime.timezone.utc)
+    humans, bots_seated, started, turns, last, over, talk = [], [], False, 0, created, None, 0
+    names, seat_changes = {}, []
+    for x in lines[1:]:
+        if x["t"] == "seat":
+            (bots_seated if x["bot"] else humans).append(x["name"])
+            names[x["id"]] = x["name"]
+        elif x["t"] == "unseat":
+            if x["id"] in names: bots_seated.remove(names.pop(x["id"]))
+        elif x["t"] == "start":
+            started = True
+        elif x["t"] == "turn":
+            turns += 1
+            last = datetime.datetime.fromtimestamp(x["at"] / 1000, datetime.timezone.utc)
+        elif x["t"] == "chat":
+            talk += 1
+        elif x["t"] == "standIn":
+            seat_changes.append("%s %s's seat to a bot" % (datetime.datetime.fromtimestamp(x["at"] / 1000, datetime.timezone.utc).strftime("%H:%M"), names.get(x["id"], x["id"])))
+        elif x["t"] == "reseat":
+            seat_changes.append("%s %s re-seated" % (datetime.datetime.fromtimestamp(x["at"] / 1000, datetime.timezone.utc).strftime("%H:%M"), names.get(x["id"], x["id"])))
+        elif x["t"] == "over":
+            over = "a draw" if x["winner"] is None else "%s won" % names.get(x["winner"], x["winner"])
+    for h in humans:
+        if h not in first_seen or created < first_seen[h]:
+            first_seen[h] = created
+    rooms.append(dict(id=meta["id"], size=meta["seats"], created=created, humans=humans, bots=bots_seated, started=started, turns=turns, last=last, over=over, talk=talk, seat_changes=seat_changes))
+
+today = sorted([r for r in rooms if r["created"].strftime("%F") == day], key=lambda r: r["created"])
+names = sorted({h for r in today for h in r["humans"]})
+new = [n for n in names if first_seen[n].strftime("%F") == day]
+print("players today: %s | NEW today: %s" % (", ".join(names) or "none", ", ".join(new) or "none"))
+print("rooms today: %d (of %d ever)" % (len(today), len(rooms)))
+for r in today:
+    idle_h = (now - r["last"]).total_seconds() / 3600
+    state = "lobby only" if not r["started"] else ("started, no turns" if not r["turns"] else (r["over"] or ("abandoned" if idle_h > 12 else "in play")))
+    span = ("%d min" % round((r["last"] - r["created"]).total_seconds() / 60)) if r["turns"] else ""
+    seated = "+".join(r["humans"]) + ((" vs %d bot%s" % (len(r["bots"]), "" if len(r["bots"]) == 1 else "s")) if r["bots"] else "")
+    empty = r["size"] - len(r["humans"]) - len(r["bots"])
+    print("  %s %s  %s%s  %s  %d turns  %s  last turn %s%s" % (
+        r["created"].strftime("%H:%M"), r["id"], seated, (" (%d seat%s empty)" % (empty, "" if empty == 1 else "s")) if empty else "",
+        state, r["turns"], span, r["last"].strftime("%H:%M") if r["turns"] else "never",
+        ("  %d line%s of talk" % (r["talk"], "" if r["talk"] == 1 else "s")) if r["talk"] else ""))
+    for change in r["seat_changes"]:
+        print("        %s" % change)
+PY

deploy/wyrm.cron

@@ -0,0 +1,3 @@
+# Wyrm: nightly rollup of yesterday's counts, then the ledger backup. UTC.
+12 0 * * * root /usr/local/bin/wyrm-rollup.sh
+23 7 * * * root /usr/local/bin/wyrm-backup.sh

deploy/wyrm.service

@@ -0,0 +1,36 @@
+[Unit]
+Description=Wyrm game server
+After=network.target
+
+[Service]
+Type=simple
+User=wyrm
+WorkingDirectory=/opt/wyrm/app/server
+Environment=PORT=8791
+Environment=KEEPER=Kestrel
+Environment=KEEPER_TZ=America/New_York
+# Caddy terminates TLS; the plaintext port must not face the internet.
+Environment=HOST=127.0.0.1
+Environment=DATA_DIR=/var/lib/wyrm/rooms
+# Empty means no error reporting.
+Environment=SENTRY_DSN=__SENTRY_DSN__
+ExecStart=/opt/wyrm/app/server/node_modules/.bin/tsx src/index.ts
+Restart=always
+RestartSec=3
+
+# Sandbox: the process reads /opt/wyrm and writes only its data dir.
+NoNewPrivileges=yes
+PrivateTmp=yes
+ProtectSystem=strict
+ProtectHome=yes
+ReadWritePaths=/var/lib/wyrm
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectControlGroups=yes
+RestrictSUIDSGID=yes
+# A runaway process gets killed and restarted before it can take the box down.
+MemoryMax=300M
+LimitNOFILE=4096
+
+[Install]
+WantedBy=multi-user.target