Sentry: errors from the duel server, and cron check-ins for the rollup and backup

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0141G6xqLeNRYEtviLWSB5Up

deploy/README.md

@@ -60,6 +60,12 @@ fetches the certificate on first request.
   credentials in /root/.config/rclone/rclone.conf, copied by hand from the
   wizwar droplet; until then it logs "skipped".
 
+Errors from the duel server go to Sentry, project `waving-hands` in the
+locallygrownnet organisation; the DSN is in the unit file. The nightly rollup
+and backup check in with Sentry Crons when /root/.waving-hands-sentry-cron-rollup
+and /root/.waving-hands-sentry-cron hold their check-in URLs (the ingest
+URL with the project's cron path and public key), so a missed night is noticed.
+
 Before every deploy, `deploy/verify-ledgers.sh <ip>` fetches every production
 ledger and replays it with the local engine, comparing each room with what the
 server shows. A ledger the new engine refuses or replays differently stops the

deploy/waving-hands-backup.sh

@@ -4,8 +4,9 @@
 #   snapshots/  - one dated copy per day, pruned after 90 days
 # Ledgers are append-only JSONL; the duel server never needs stopping.
 # Sentry Crons check-in: /root/.waving-hands-sentry-cron holds the check-in
-# URL (absent file = no check-ins).
+# URL (absent file = no check-ins). SCHEDULE must match /etc/cron.d/waving-hands.
 set -u
+SCHEDULE="23 7 * * *"
 BUCKET="kestrel-wizwar-backups"
 PREFIX="waving-hands"
 SRC="/var/lib/waving-hands"
@@ -13,7 +14,11 @@ LOG="/var/log/waving-hands/backup.log"
 STAMP=$(date +%Y-%m-%d)
 mkdir -p "$(dirname "$LOG")"
 CRON_URL=$(cat /root/.waving-hands-sentry-cron 2>/dev/null || true)
-checkin() { [ -n "$CRON_URL" ] && curl -sf -o /dev/null "$CRON_URL?status=$1" || true; }
+checkin() {
+  [ -n "$CRON_URL" ] && curl -sf -o /dev/null -X POST -H "Content-Type: application/json" \
+    -d "{\"status\":\"$1\",\"monitor_config\":{\"schedule\":{\"type\":\"crontab\",\"value\":\"$SCHEDULE\"},\"checkin_margin\":30,\"max_runtime\":10,\"timezone\":\"UTC\"}}" \
+    "$CRON_URL" || true
+}
 
 if [ ! -s /root/.config/rclone/rclone.conf ]; then
   echo "$(date -Is) skipped: rclone is not configured" >> "$LOG"

deploy/waving-hands-rollup.sh

@@ -5,13 +5,25 @@
 # started and finished, and the box's vitals. Counts only: no addresses are
 # kept. The pulse reads it for trends the self-rotating access log loses.
 #   waving-hands-rollup.sh [YYYY-MM-DD]   (default: yesterday, UTC)
-#   Cron: /etc/cron.d/waving-hands, installed by deploy.sh.
+#   Cron: /etc/cron.d/waving-hands, installed by deploy.sh; SCHEDULE below
+#   must match it, because the Sentry monitor is told the same shape.
+#   Sentry Crons check-in: /root/.waving-hands-sentry-cron-rollup holds the
+#   check-in URL (absent = no check-ins).
 set -u
 OUT="/var/lib/waving-hands/rollup.jsonl"
 LOG="/var/log/waving-hands/rollup.log"
 DAY="${1:-$(date -u -d 'yesterday' +%Y-%m-%d)}"
+SCHEDULE="12 0 * * *"
+CRON_URL=$(cat /root/.waving-hands-sentry-cron-rollup 2>/dev/null || true)
+# The check-in carries the schedule: Sentry keeps a monitor only once told its shape.
+checkin() {
+  [ -n "$CRON_URL" ] && curl -sf -o /dev/null -X POST -H "Content-Type: application/json" \
+    -d "{\"status\":\"$1\",\"monitor_config\":{\"schedule\":{\"type\":\"crontab\",\"value\":\"$SCHEDULE\"},\"checkin_margin\":30,\"max_runtime\":10,\"timezone\":\"UTC\"}}" \
+    "$CRON_URL" || true
+}
 mkdir -p "$(dirname "$LOG")"
-python3 - "$DAY" "$OUT" <<'PY' >> "$LOG" 2>&1
+checkin in_progress
+if python3 - "$DAY" "$OUT" <<'PY' >> "$LOG" 2>&1
 import collections, datetime, glob, gzip, json, os, re, shutil, sys
 day, out = sys.argv[1], sys.argv[2]
 d0 = datetime.datetime.fromisoformat(day).replace(tzinfo=datetime.timezone.utc)
@@ -94,3 +106,4 @@ with open(out + ".tmp", "w") as fh:
 os.replace(out + ".tmp", out)
 print("%s rolled: %d people, %d requests, %d rooms" % (day, len(hips), human, created))
 PY
+then checkin ok; else checkin error; fi

deploy/waving-hands.service

@@ -10,6 +10,7 @@ Environment=PORT=8788
 # Caddy terminates TLS; the plaintext port must not face the internet.
 Environment=HOST=127.0.0.1
 Environment=WH_DATA_DIR=/var/lib/waving-hands/rooms
+Environment=SENTRY_DSN=https://e11c1f168789d9d7706bfc5eea4f373e@o4509525984149504.ingest.us.sentry.io/4512133232787456
 ExecStart=/opt/waving-hands/app/server/node_modules/.bin/tsx src/index.ts
 Restart=always
 RestartSec=3

server/package-lock.json

@@ -6,6 +6,7 @@
 		"": {
 			"name": "waving-hands-server",
 			"dependencies": {
+				"@sentry/node": "^10.75.2",
 				"tsx": "^4.23.15",
 				"ws": "^8.21.3"
 			}
@@ -426,6 +427,259 @@
 				"node": ">=18"
 			}
 		},
+		"node_modules/@opentelemetry/api": {
+			"version": "1.9.1",
+			"resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz",
+			"integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==",
+			"license": "Apache-2.0",
+			"engines": {
+				"node": ">=8.0.0"
+			}
+		},
+		"node_modules/@opentelemetry/api-logs": {
+			"version": "0.220.0",
+			"resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.220.0.tgz",
+			"integrity": "sha512-CmVa4ImJ+ynfrPMNaAXHET6Bhb44SwzmfyVJFq9ni2jgXJR/l7C6gfVFddNmHP+ZOkP9cf4f9DBe68qVLTHc9w==",
+			"license": "Apache-2.0",
+			"dependencies": {
+				"@opentelemetry/api": "^1.3.0"
+			},
+			"engines": {
+				"node": ">=8.0.0"
+			}
+		},
+		"node_modules/@opentelemetry/core": {
+			"version": "2.11.0",
+			"resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.11.0.tgz",
+			"integrity": "sha512-7YP44XH0tV6+Mb54x2YGf84i7yi+31MBZlE8JwvozkxyTvXbSp10X7cI7YE49ChJ3shMJoBmCJF3+1QFBJctGA==",
+			"license": "Apache-2.0",
+			"dependencies": {
+				"@opentelemetry/semantic-conventions": "^1.29.0"
+			},
+			"engines": {
+				"node": "^18.19.0 || >=20.6.0"
+			},
+			"peerDependencies": {
+				"@opentelemetry/api": ">=1.0.0 <1.10.0"
+			}
+		},
+		"node_modules/@opentelemetry/instrumentation": {
+			"version": "0.220.0",
+			"resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.220.0.tgz",
+			"integrity": "sha512-xQx3E2WxP1mDvKzxLxX+CTCtNLa560YJZ3087qYHerl2YmiKpv7AH+dAy7vmx+eVrZ5BwhfWUAVoKOoxCNHcpw==",
+			"license": "Apache-2.0",
+			"dependencies": {
+				"@opentelemetry/api-logs": "0.220.0",
+				"import-in-the-middle": "^3.0.0",
+				"require-in-the-middle": "^8.0.0"
+			},
+			"engines": {
+				"node": "^18.19.0 || >=20.6.0"
+			},
+			"peerDependencies": {
+				"@opentelemetry/api": "^1.3.0"
+			}
+		},
+		"node_modules/@opentelemetry/resources": {
+			"version": "2.11.0",
+			"resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.11.0.tgz",
+			"integrity": "sha512-Ie7+8q8MDF4FAEQCKVMTx3ReUvxiIAgIiiW3c9JdmP8+HMcDy20puT+AHjexnExgnbvBxjQ9fjkFDWrikJ2jQA==",
+			"license": "Apache-2.0",
+			"dependencies": {
+				"@opentelemetry/core": "2.11.0",
+				"@opentelemetry/semantic-conventions": "^1.29.0"
+			},
+			"engines": {
+				"node": "^18.19.0 || >=20.6.0"
+			},
+			"peerDependencies": {
+				"@opentelemetry/api": ">=1.3.0 <1.10.0"
+			}
+		},
+		"node_modules/@opentelemetry/sdk-trace": {
+			"version": "2.11.0",
+			"resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace/-/sdk-trace-2.11.0.tgz",
+			"integrity": "sha512-fFnTqGm8/G73GQVnxYi7LXa1ZVYEUvgL6XI1LpvV0bPC7WQ/ZGgKxCSl8FnlZBKto9JHHEFTO6s6CUpvvtwFrA==",
+			"license": "Apache-2.0",
+			"dependencies": {
+				"@opentelemetry/core": "2.11.0",
+				"@opentelemetry/resources": "2.11.0",
+				"@opentelemetry/semantic-conventions": "^1.29.0"
+			},
+			"engines": {
+				"node": "^18.19.0 || >=20.6.0"
+			},
+			"peerDependencies": {
+				"@opentelemetry/api": ">=1.3.0 <1.10.0"
+			}
+		},
+		"node_modules/@opentelemetry/sdk-trace-base": {
+			"version": "2.11.0",
+			"resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.11.0.tgz",
+			"integrity": "sha512-H19x/TX/LZdqiYOjM7fqtSxwlplC5pgelavqbQdHbhdq0q/AI/TGkM2dfGuuynTXmJPeF2HoZVoPDu+TGoW78A==",
+			"license": "Apache-2.0",
+			"dependencies": {
+				"@opentelemetry/core": "2.11.0",
+				"@opentelemetry/resources": "2.11.0",
+				"@opentelemetry/sdk-trace": "2.11.0",
+				"@opentelemetry/semantic-conventions": "^1.29.0"
+			},
+			"engines": {
+				"node": "^18.19.0 || >=20.6.0"
+			},
+			"peerDependencies": {
+				"@opentelemetry/api": ">=1.3.0 <1.10.0"
+			}
+		},
+		"node_modules/@opentelemetry/semantic-conventions": {
+			"version": "1.43.0",
+			"resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.43.0.tgz",
+			"integrity": "sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg==",
+			"license": "Apache-2.0",
+			"engines": {
+				"node": ">=14"
+			}
+		},
+		"node_modules/@sentry/conventions": {
+			"version": "0.16.0",
+			"resolved": "https://registry.npmjs.org/@sentry/conventions/-/conventions-0.16.0.tgz",
+			"integrity": "sha512-fO9PLmHdVURcSPUpWCItWAtgKiMwGdJHbovoSEyLplX5sxs2ugvI4CBPTrkkgqhObnZOD0CnWBKDzSVQYBKEyQ==",
+			"license": "MIT",
+			"engines": {
+				"node": ">=14"
+			}
+		},
+		"node_modules/@sentry/core": {
+			"version": "10.75.2",
+			"resolved": "https://registry.npmjs.org/@sentry/core/-/core-10.75.2.tgz",
+			"integrity": "sha512-KXzNf2c430cJIOMv84+MKpkXgave1DBNMNNwn073olVD773NME5IhDv/9VQyiFM2wIU+6bm4FEBVEVSuzqgjJw==",
+			"license": "MIT",
+			"dependencies": {
+				"@sentry/conventions": "^0.16.0"
+			},
+			"engines": {
+				"node": ">=18"
+			}
+		},
+		"node_modules/@sentry/node": {
+			"version": "10.75.2",
+			"resolved": "https://registry.npmjs.org/@sentry/node/-/node-10.75.2.tgz",
+			"integrity": "sha512-ffdx1i3Fx9Q3Ofdaone2I+HWOj9+Wcu268xgV+Bn8WJYABm8BZIBNvS+PORKBPesWILhQCThWOs+E/8vcV1HuQ==",
+			"license": "MIT",
+			"dependencies": {
+				"@opentelemetry/api": "^1.9.1",
+				"@opentelemetry/instrumentation": "^0.220.0",
+				"@opentelemetry/sdk-trace-base": "^2.9.0",
+				"@sentry/conventions": "^0.16.0",
+				"@sentry/core": "10.75.2",
+				"@sentry/node-core": "10.75.2",
+				"@sentry/opentelemetry": "10.75.2",
+				"@sentry/server-utils": "10.75.2",
+				"import-in-the-middle": "^3.0.0"
+			},
+			"engines": {
+				"node": ">=18"
+			}
+		},
+		"node_modules/@sentry/node-core": {
+			"version": "10.75.2",
+			"resolved": "https://registry.npmjs.org/@sentry/node-core/-/node-core-10.75.2.tgz",
+			"integrity": "sha512-6sFhrvWVLAPRTOYkwrOOLwgsWINe6Y75oX9xKj11zZcKd2e1K2Y3bcx3uzDU06+cxfVbUQUBulsCr/TD7yx/vw==",
+			"license": "MIT",
+			"dependencies": {
+				"@sentry/conventions": "^0.16.0",
+				"@sentry/core": "10.75.2",
+				"@sentry/opentelemetry": "10.75.2",
+				"import-in-the-middle": "^3.0.0"
+			},
+			"engines": {
+				"node": ">=18"
+			},
+			"peerDependencies": {
+				"@opentelemetry/api": "^1.9.0",
+				"@opentelemetry/core": "^1.30.1 || ^2.1.0",
+				"@opentelemetry/exporter-trace-otlp-http": ">=0.57.0 <1",
+				"@opentelemetry/instrumentation": ">=0.57.1 <1",
+				"@opentelemetry/sdk-trace-base": "^1.30.1 || ^2.1.0"
+			},
+			"peerDependenciesMeta": {
+				"@opentelemetry/api": {
+					"optional": true
+				},
+				"@opentelemetry/core": {
+					"optional": true
+				},
+				"@opentelemetry/exporter-trace-otlp-http": {
+					"optional": true
+				},
+				"@opentelemetry/instrumentation": {
+					"optional": true
+				},
+				"@opentelemetry/sdk-trace-base": {
+					"optional": true
+				}
+			}
+		},
+		"node_modules/@sentry/opentelemetry": {
+			"version": "10.75.2",
+			"resolved": "https://registry.npmjs.org/@sentry/opentelemetry/-/opentelemetry-10.75.2.tgz",
+			"integrity": "sha512-qoJIdrXiFPPYs7AmdkktB5O4vhHYKzk6EulpQlvROe+ZZeG4/S0yeJApAbg3ThDob/2zEAxYEPEvaHv2epipgA==",
+			"license": "MIT",
+			"dependencies": {
+				"@sentry/conventions": "^0.16.0",
+				"@sentry/core": "10.75.2"
+			},
+			"engines": {
+				"node": ">=18"
+			},
+			"peerDependencies": {
+				"@opentelemetry/api": "^1.9.0",
+				"@opentelemetry/core": "^1.30.1 || ^2.1.0",
+				"@opentelemetry/sdk-trace-base": "^1.30.1 || ^2.1.0"
+			}
+		},
+		"node_modules/@sentry/server-utils": {
+			"version": "10.75.2",
+			"resolved": "https://registry.npmjs.org/@sentry/server-utils/-/server-utils-10.75.2.tgz",
+			"integrity": "sha512-ep8Ntj/vsLHbql1jUMFiRYrleuQ3dENb8oD4Nkjv7hou2j0UKLct1eLbx+TckeVOhYdWmFeO7WJWjRio/a4TOg==",
+			"license": "MIT",
+			"dependencies": {
+				"@sentry/conventions": "^0.16.0",
+				"@sentry/core": "10.75.2"
+			},
+			"engines": {
+				"node": ">=18"
+			}
+		},
+		"node_modules/cjs-module-lexer": {
+			"version": "2.2.1",
+			"resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-2.2.1.tgz",
+			"integrity": "sha512-Ca8swihM+/4yKecYHY52kgJd300hi2lADU/a1RxNTRe+RJ9jvqQlESpbz9DnG9mowez8qwXHB8qYdIUw9e+F5Q==",
+			"license": "MIT"
+		},
+		"node_modules/debug": {
+			"version": "4.4.3",
+			"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz",
+			"integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==",
+			"license": "MIT",
+			"dependencies": {
+				"ms": "^2.1.3"
+			},
+			"engines": {
+				"node": ">=6.0"
+			},
+			"peerDependenciesMeta": {
+				"supports-color": {
+					"optional": true
+				}
+			}
+		},
+		"node_modules/es-module-lexer": {
+			"version": "3.0.2",
+			"resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-3.0.2.tgz",
+			"integrity": "sha512-BuIB67FngDSyQ/dpQNOZybwdEBDUGJQvOqwWr4ha/ufYiqzuEwPkKO2zLhRAgay28tStRIHUeWmszZAJo3GCOg==",
+			"license": "MIT"
+		},
 		"node_modules/esbuild": {
 			"version": "0.28.2",
 			"resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz",
@@ -481,6 +735,45 @@
 				"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
 			}
 		},
+		"node_modules/import-in-the-middle": {
+			"version": "3.5.1",
+			"resolved": "https://registry.npmjs.org/import-in-the-middle/-/import-in-the-middle-3.5.1.tgz",
+			"integrity": "sha512-mPKuL8bPQzecui2KK6Gb+M8JvJoHnhS1FeYGa22QopBmlevF5F0FE6ued/B5EgHDeIoMTONIpDWWFKUPOG0DBQ==",
+			"license": "Apache-2.0",
+			"dependencies": {
+				"cjs-module-lexer": "^2.2.0",
+				"es-module-lexer": "^3.0.2",
+				"module-details-from-path": "^1.0.4"
+			},
+			"engines": {
+				"node": ">=18"
+			}
+		},
+		"node_modules/module-details-from-path": {
+			"version": "1.0.4",
+			"resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.4.tgz",
+			"integrity": "sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==",
+			"license": "MIT"
+		},
+		"node_modules/ms": {
+			"version": "2.1.3",
+			"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+			"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
+			"license": "MIT"
+		},
+		"node_modules/require-in-the-middle": {
+			"version": "8.0.1",
+			"resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-8.0.1.tgz",
+			"integrity": "sha512-QT7FVMXfWOYFbeRBF6nu+I6tr2Tf3u0q8RIEjNob/heKY/nh7drD/k7eeMFmSQgnTtCzLDcCu/XEnpW2wk4xCQ==",
+			"license": "MIT",
+			"dependencies": {
+				"debug": "^4.3.5",
+				"module-details-from-path": "^1.0.3"
+			},
+			"engines": {
+				"node": ">=9.3.0 || >=8.10.0 <9.0.0"
+			}
+		},
 		"node_modules/tsx": {
 			"version": "4.23.15",
 			"resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.15.tgz",

server/package.json

@@ -7,6 +7,7 @@
 		"start": "tsx src/index.ts"
 	},
 	"dependencies": {
+		"@sentry/node": "^10.75.2",
 		"tsx": "^4.23.15",
 		"ws": "^8.21.3"
 	}

server/src/index.ts

@@ -10,6 +10,7 @@
 //
 // Caddy serves the static site and proxies /api and /ws here.
 
+import * as Sentry from '@sentry/node';
 import { createServer, type IncomingMessage, type ServerResponse } from 'node:http';
 import { WebSocketServer, WebSocket } from 'ws';
 import { RateLimit } from './ratelimit';
@@ -21,6 +22,15 @@ const HOST = process.env.HOST ?? '127.0.0.1';
 const DATA_DIR = process.env.WH_DATA_DIR ?? '../data/rooms';
 const BODY_LIMIT = 16 * 1024;
 
+// Errors go to Sentry when a DSN is set; the SDK drops them otherwise.
+if (process.env.SENTRY_DSN) {
+	Sentry.init({ dsn: process.env.SENTRY_DSN, environment: 'production', tracesSampleRate: 0 });
+}
+process.on('unhandledRejection', (reason) => {
+	console.error(reason);
+	Sentry.captureException(reason);
+});
+
 const rooms = new Rooms(new Store(DATA_DIR));
 /** Opening rooms and taking seats are open to anyone; a script gets a few dozen an hour, not thousands. */
 const doors = new RateLimit(40, 60 * 60 * 1000);
@@ -112,6 +122,7 @@ const server = createServer((req, res) => {
 	handle(req, res).catch((err: unknown) => {
 		if (err instanceof RoomError) return send(res, err.status, { error: err.message });
 		console.error(err);
+		Sentry.captureException(err, { extra: { url: req.url, method: req.method } });
 		send(res, 500, { error: 'Something went wrong on the server.' });
 	});
 });