From b25abd4d69237d0a89e3858073a65c8b98df5410 Mon Sep 17 00:00:00 2001 From: Eric Wagoner Date: Tue, 22 Sep 2026 22:09:33 -0400 Subject: [PATCH] Sentry: errors from the duel server, and cron check-ins for the rollup and backup Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_0141G6xqLeNRYEtviLWSB5Up --- deploy/README.md | 6 + deploy/waving-hands-backup.sh | 9 +- deploy/waving-hands-rollup.sh | 17 +- deploy/waving-hands.service | 1 + server/package-lock.json | 293 ++++++++++++++++++++++++++++++++++ server/package.json | 1 + server/src/index.ts | 11 ++ 7 files changed, 334 insertions(+), 4 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index a0bb2d4..f202d75 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -60,6 +60,12 @@ fetches the certificate on first request. credentials in /root/.config/rclone/rclone.conf, copied by hand from the wizwar droplet; until then it logs "skipped". +Errors from the duel server go to Sentry, project `waving-hands` in the +locallygrownnet organisation; the DSN is in the unit file. The nightly rollup +and backup check in with Sentry Crons when /root/.waving-hands-sentry-cron-rollup +and /root/.waving-hands-sentry-cron hold their check-in URLs (the ingest +URL with the project's cron path and public key), so a missed night is noticed. + Before every deploy, `deploy/verify-ledgers.sh ` fetches every production ledger and replays it with the local engine, comparing each room with what the server shows. A ledger the new engine refuses or replays differently stops the diff --git a/deploy/waving-hands-backup.sh b/deploy/waving-hands-backup.sh index 95b4566..fe9dbe4 100755 --- a/deploy/waving-hands-backup.sh +++ b/deploy/waving-hands-backup.sh @@ -4,8 +4,9 @@ # snapshots/ - one dated copy per day, pruned after 90 days # Ledgers are append-only JSONL; the duel server never needs stopping. # Sentry Crons check-in: /root/.waving-hands-sentry-cron holds the check-in -# URL (absent file = no check-ins). +# URL (absent file = no check-ins). SCHEDULE must match /etc/cron.d/waving-hands. set -u +SCHEDULE="23 7 * * *" BUCKET="kestrel-wizwar-backups" PREFIX="waving-hands" SRC="/var/lib/waving-hands" @@ -13,7 +14,11 @@ LOG="/var/log/waving-hands/backup.log" STAMP=$(date +%Y-%m-%d) mkdir -p "$(dirname "$LOG")" CRON_URL=$(cat /root/.waving-hands-sentry-cron 2>/dev/null || true) -checkin() { [ -n "$CRON_URL" ] && curl -sf -o /dev/null "$CRON_URL?status=$1" || true; } +checkin() { + [ -n "$CRON_URL" ] && curl -sf -o /dev/null -X POST -H "Content-Type: application/json" \ + -d "{\"status\":\"$1\",\"monitor_config\":{\"schedule\":{\"type\":\"crontab\",\"value\":\"$SCHEDULE\"},\"checkin_margin\":30,\"max_runtime\":10,\"timezone\":\"UTC\"}}" \ + "$CRON_URL" || true +} if [ ! -s /root/.config/rclone/rclone.conf ]; then echo "$(date -Is) skipped: rclone is not configured" >> "$LOG" diff --git a/deploy/waving-hands-rollup.sh b/deploy/waving-hands-rollup.sh index a0893a0..31e9035 100755 --- a/deploy/waving-hands-rollup.sh +++ b/deploy/waving-hands-rollup.sh @@ -5,13 +5,25 @@ # started and finished, and the box's vitals. Counts only: no addresses are # kept. The pulse reads it for trends the self-rotating access log loses. # waving-hands-rollup.sh [YYYY-MM-DD] (default: yesterday, UTC) -# Cron: /etc/cron.d/waving-hands, installed by deploy.sh. +# Cron: /etc/cron.d/waving-hands, installed by deploy.sh; SCHEDULE below +# must match it, because the Sentry monitor is told the same shape. +# Sentry Crons check-in: /root/.waving-hands-sentry-cron-rollup holds the +# check-in URL (absent = no check-ins). set -u OUT="/var/lib/waving-hands/rollup.jsonl" LOG="/var/log/waving-hands/rollup.log" DAY="${1:-$(date -u -d 'yesterday' +%Y-%m-%d)}" +SCHEDULE="12 0 * * *" +CRON_URL=$(cat /root/.waving-hands-sentry-cron-rollup 2>/dev/null || true) +# The check-in carries the schedule: Sentry keeps a monitor only once told its shape. +checkin() { + [ -n "$CRON_URL" ] && curl -sf -o /dev/null -X POST -H "Content-Type: application/json" \ + -d "{\"status\":\"$1\",\"monitor_config\":{\"schedule\":{\"type\":\"crontab\",\"value\":\"$SCHEDULE\"},\"checkin_margin\":30,\"max_runtime\":10,\"timezone\":\"UTC\"}}" \ + "$CRON_URL" || true +} mkdir -p "$(dirname "$LOG")" -python3 - "$DAY" "$OUT" <<'PY' >> "$LOG" 2>&1 +checkin in_progress +if python3 - "$DAY" "$OUT" <<'PY' >> "$LOG" 2>&1 import collections, datetime, glob, gzip, json, os, re, shutil, sys day, out = sys.argv[1], sys.argv[2] d0 = datetime.datetime.fromisoformat(day).replace(tzinfo=datetime.timezone.utc) @@ -94,3 +106,4 @@ with open(out + ".tmp", "w") as fh: os.replace(out + ".tmp", out) print("%s rolled: %d people, %d requests, %d rooms" % (day, len(hips), human, created)) PY +then checkin ok; else checkin error; fi diff --git a/deploy/waving-hands.service b/deploy/waving-hands.service index 9fd1cbe..81efab7 100644 --- a/deploy/waving-hands.service +++ b/deploy/waving-hands.service @@ -10,6 +10,7 @@ Environment=PORT=8788 # Caddy terminates TLS; the plaintext port must not face the internet. Environment=HOST=127.0.0.1 Environment=WH_DATA_DIR=/var/lib/waving-hands/rooms +Environment=SENTRY_DSN=https://e11c1f168789d9d7706bfc5eea4f373e@o4509525984149504.ingest.us.sentry.io/4512133232787456 ExecStart=/opt/waving-hands/app/server/node_modules/.bin/tsx src/index.ts Restart=always RestartSec=3 diff --git a/server/package-lock.json b/server/package-lock.json index 0a085a2..760f843 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -6,6 +6,7 @@ "": { "name": "waving-hands-server", "dependencies": { + "@sentry/node": "^10.75.2", "tsx": "^4.23.15", "ws": "^8.21.3" } @@ -426,6 +427,259 @@ "node": ">=18" } }, + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/api-logs": { + "version": "0.220.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.220.0.tgz", + "integrity": "sha512-CmVa4ImJ+ynfrPMNaAXHET6Bhb44SwzmfyVJFq9ni2jgXJR/l7C6gfVFddNmHP+ZOkP9cf4f9DBe68qVLTHc9w==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.3.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/core": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.11.0.tgz", + "integrity": "sha512-7YP44XH0tV6+Mb54x2YGf84i7yi+31MBZlE8JwvozkxyTvXbSp10X7cI7YE49ChJ3shMJoBmCJF3+1QFBJctGA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/instrumentation": { + "version": "0.220.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.220.0.tgz", + "integrity": "sha512-xQx3E2WxP1mDvKzxLxX+CTCtNLa560YJZ3087qYHerl2YmiKpv7AH+dAy7vmx+eVrZ5BwhfWUAVoKOoxCNHcpw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.220.0", + "import-in-the-middle": "^3.0.0", + "require-in-the-middle": "^8.0.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/resources": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.11.0.tgz", + "integrity": "sha512-Ie7+8q8MDF4FAEQCKVMTx3ReUvxiIAgIiiW3c9JdmP8+HMcDy20puT+AHjexnExgnbvBxjQ9fjkFDWrikJ2jQA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.11.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace/-/sdk-trace-2.11.0.tgz", + "integrity": "sha512-fFnTqGm8/G73GQVnxYi7LXa1ZVYEUvgL6XI1LpvV0bPC7WQ/ZGgKxCSl8FnlZBKto9JHHEFTO6s6CUpvvtwFrA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.11.0", + "@opentelemetry/resources": "2.11.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-base": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.11.0.tgz", + "integrity": "sha512-H19x/TX/LZdqiYOjM7fqtSxwlplC5pgelavqbQdHbhdq0q/AI/TGkM2dfGuuynTXmJPeF2HoZVoPDu+TGoW78A==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.11.0", + "@opentelemetry/resources": "2.11.0", + "@opentelemetry/sdk-trace": "2.11.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/semantic-conventions": { + "version": "1.43.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.43.0.tgz", + "integrity": "sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/@sentry/conventions": { + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/@sentry/conventions/-/conventions-0.16.0.tgz", + "integrity": "sha512-fO9PLmHdVURcSPUpWCItWAtgKiMwGdJHbovoSEyLplX5sxs2ugvI4CBPTrkkgqhObnZOD0CnWBKDzSVQYBKEyQ==", + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/@sentry/core": { + "version": "10.75.2", + "resolved": "https://registry.npmjs.org/@sentry/core/-/core-10.75.2.tgz", + "integrity": "sha512-KXzNf2c430cJIOMv84+MKpkXgave1DBNMNNwn073olVD773NME5IhDv/9VQyiFM2wIU+6bm4FEBVEVSuzqgjJw==", + "license": "MIT", + "dependencies": { + "@sentry/conventions": "^0.16.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@sentry/node": { + "version": "10.75.2", + "resolved": "https://registry.npmjs.org/@sentry/node/-/node-10.75.2.tgz", + "integrity": "sha512-ffdx1i3Fx9Q3Ofdaone2I+HWOj9+Wcu268xgV+Bn8WJYABm8BZIBNvS+PORKBPesWILhQCThWOs+E/8vcV1HuQ==", + "license": "MIT", + "dependencies": { + "@opentelemetry/api": "^1.9.1", + "@opentelemetry/instrumentation": "^0.220.0", + "@opentelemetry/sdk-trace-base": "^2.9.0", + "@sentry/conventions": "^0.16.0", + "@sentry/core": "10.75.2", + "@sentry/node-core": "10.75.2", + "@sentry/opentelemetry": "10.75.2", + "@sentry/server-utils": "10.75.2", + "import-in-the-middle": "^3.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@sentry/node-core": { + "version": "10.75.2", + "resolved": "https://registry.npmjs.org/@sentry/node-core/-/node-core-10.75.2.tgz", + "integrity": "sha512-6sFhrvWVLAPRTOYkwrOOLwgsWINe6Y75oX9xKj11zZcKd2e1K2Y3bcx3uzDU06+cxfVbUQUBulsCr/TD7yx/vw==", + "license": "MIT", + "dependencies": { + "@sentry/conventions": "^0.16.0", + "@sentry/core": "10.75.2", + "@sentry/opentelemetry": "10.75.2", + "import-in-the-middle": "^3.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/core": "^1.30.1 || ^2.1.0", + "@opentelemetry/exporter-trace-otlp-http": ">=0.57.0 <1", + "@opentelemetry/instrumentation": ">=0.57.1 <1", + "@opentelemetry/sdk-trace-base": "^1.30.1 || ^2.1.0" + }, + "peerDependenciesMeta": { + "@opentelemetry/api": { + "optional": true + }, + "@opentelemetry/core": { + "optional": true + }, + "@opentelemetry/exporter-trace-otlp-http": { + "optional": true + }, + "@opentelemetry/instrumentation": { + "optional": true + }, + "@opentelemetry/sdk-trace-base": { + "optional": true + } + } + }, + "node_modules/@sentry/opentelemetry": { + "version": "10.75.2", + "resolved": "https://registry.npmjs.org/@sentry/opentelemetry/-/opentelemetry-10.75.2.tgz", + "integrity": "sha512-qoJIdrXiFPPYs7AmdkktB5O4vhHYKzk6EulpQlvROe+ZZeG4/S0yeJApAbg3ThDob/2zEAxYEPEvaHv2epipgA==", + "license": "MIT", + "dependencies": { + "@sentry/conventions": "^0.16.0", + "@sentry/core": "10.75.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/core": "^1.30.1 || ^2.1.0", + "@opentelemetry/sdk-trace-base": "^1.30.1 || ^2.1.0" + } + }, + "node_modules/@sentry/server-utils": { + "version": "10.75.2", + "resolved": "https://registry.npmjs.org/@sentry/server-utils/-/server-utils-10.75.2.tgz", + "integrity": "sha512-ep8Ntj/vsLHbql1jUMFiRYrleuQ3dENb8oD4Nkjv7hou2j0UKLct1eLbx+TckeVOhYdWmFeO7WJWjRio/a4TOg==", + "license": "MIT", + "dependencies": { + "@sentry/conventions": "^0.16.0", + "@sentry/core": "10.75.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/cjs-module-lexer": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-2.2.1.tgz", + "integrity": "sha512-Ca8swihM+/4yKecYHY52kgJd300hi2lADU/a1RxNTRe+RJ9jvqQlESpbz9DnG9mowez8qwXHB8qYdIUw9e+F5Q==", + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/es-module-lexer": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-3.0.2.tgz", + "integrity": "sha512-BuIB67FngDSyQ/dpQNOZybwdEBDUGJQvOqwWr4ha/ufYiqzuEwPkKO2zLhRAgay28tStRIHUeWmszZAJo3GCOg==", + "license": "MIT" + }, "node_modules/esbuild": { "version": "0.28.2", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", @@ -481,6 +735,45 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/import-in-the-middle": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/import-in-the-middle/-/import-in-the-middle-3.5.1.tgz", + "integrity": "sha512-mPKuL8bPQzecui2KK6Gb+M8JvJoHnhS1FeYGa22QopBmlevF5F0FE6ued/B5EgHDeIoMTONIpDWWFKUPOG0DBQ==", + "license": "Apache-2.0", + "dependencies": { + "cjs-module-lexer": "^2.2.0", + "es-module-lexer": "^3.0.2", + "module-details-from-path": "^1.0.4" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/module-details-from-path": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.4.tgz", + "integrity": "sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==", + "license": "MIT" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/require-in-the-middle": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-8.0.1.tgz", + "integrity": "sha512-QT7FVMXfWOYFbeRBF6nu+I6tr2Tf3u0q8RIEjNob/heKY/nh7drD/k7eeMFmSQgnTtCzLDcCu/XEnpW2wk4xCQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.3.5", + "module-details-from-path": "^1.0.3" + }, + "engines": { + "node": ">=9.3.0 || >=8.10.0 <9.0.0" + } + }, "node_modules/tsx": { "version": "4.23.15", "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.15.tgz", diff --git a/server/package.json b/server/package.json index a6a0d19..29e53af 100644 --- a/server/package.json +++ b/server/package.json @@ -7,6 +7,7 @@ "start": "tsx src/index.ts" }, "dependencies": { + "@sentry/node": "^10.75.2", "tsx": "^4.23.15", "ws": "^8.21.3" } diff --git a/server/src/index.ts b/server/src/index.ts index 4638209..b01fbfb 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -10,6 +10,7 @@ // // Caddy serves the static site and proxies /api and /ws here. +import * as Sentry from '@sentry/node'; import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'; import { WebSocketServer, WebSocket } from 'ws'; import { RateLimit } from './ratelimit'; @@ -21,6 +22,15 @@ const HOST = process.env.HOST ?? '127.0.0.1'; const DATA_DIR = process.env.WH_DATA_DIR ?? '../data/rooms'; const BODY_LIMIT = 16 * 1024; +// Errors go to Sentry when a DSN is set; the SDK drops them otherwise. +if (process.env.SENTRY_DSN) { + Sentry.init({ dsn: process.env.SENTRY_DSN, environment: 'production', tracesSampleRate: 0 }); +} +process.on('unhandledRejection', (reason) => { + console.error(reason); + Sentry.captureException(reason); +}); + const rooms = new Rooms(new Store(DATA_DIR)); /** Opening rooms and taking seats are open to anyone; a script gets a few dozen an hour, not thousands. */ const doors = new RateLimit(40, 60 * 60 * 1000); @@ -112,6 +122,7 @@ const server = createServer((req, res) => { handle(req, res).catch((err: unknown) => { if (err instanceof RoomError) return send(res, err.status, { error: err.message }); console.error(err); + Sentry.captureException(err, { extra: { url: req.url, method: req.method } }); send(res, 500, { error: 'Something went wrong on the server.' }); }); });