From 60435fef84590f43b37dc714f5afb6adaec8ac48 Mon Sep 17 00:00:00 2001 From: Eric Wagoner Date: Mon, 12 Oct 2026 10:07:05 -0400 Subject: [PATCH] The hall keeps the night on the shared host: one job runs every game's rollup and backup and checks in with Sentry once for each Eight cron monitors, two per game, watched eight scripts that run on the same box at the same two minutes. hall-nightly.sh now runs the hall's own rollup and every game's rollup at 00:12, and every game's backup at 07:23, finding the games by their scripts in /usr/local/bin, and checks in once for each job to the gamehall project's hall-rollup and hall-backup monitors. The template's rollup and backup say by their exit status whether they managed, the skipped backup included, and its deploy leaves the game's own cron file off a host the hall schedules. Co-Authored-By: Claude Fable 5.1 --- .claude/skills/gamehall-visitors/SKILL.md | 2 +- CONVENTIONS.md | 5 ++- README.md | 8 ++++ gamehall/deploy.sh | 2 + gamehall/ops/gamehall-rollup.sh | 2 +- gamehall/ops/gamehall.cron | 6 ++- gamehall/ops/hall-nightly.sh | 52 +++++++++++++++++++++++ template/deploy/README.md | 19 ++++++--- template/deploy/__SLUG__-backup.sh | 7 ++- template/deploy/__SLUG__-rollup.sh | 6 ++- template/deploy/deploy.sh | 6 ++- 11 files changed, 98 insertions(+), 17 deletions(-) create mode 100755 gamehall/ops/hall-nightly.sh diff --git a/.claude/skills/gamehall-visitors/SKILL.md b/.claude/skills/gamehall-visitors/SKILL.md index 349aaa6..8823b50 100644 --- a/.claude/skills/gamehall-visitors/SKILL.md +++ b/.claude/skills/gamehall-visitors/SKILL.md @@ -23,7 +23,7 @@ name (the sslip test address is left out) and prints: own digest (/wizwar-visitors) lists gamehall.kestrelsnest.social among its referrers. - **last 7 days** from `/var/lib/gamehall/rollup.jsonl`, written nightly - at 00:05 UTC by `gamehall-rollup.sh` for the day before (counts only, + at 00:12 UTC by `gamehall-rollup.sh` (run by `hall-nightly.sh`) for the day before (counts only, no addresses). ## Read it diff --git a/CONVENTIONS.md b/CONVENTIONS.md index aec8699..44c3a41 100644 --- a/CONVENTIONS.md +++ b/CONVENTIONS.md @@ -198,8 +198,9 @@ Every game gets the same tools, installed by `deploy.sh` on every deploy: the rollup's last week, the backup's last word, the box's vitals. - `-rollup.sh` nightly at 00:12 UTC: one JSON line a day of counts, no addresses. `-backup.sh` nightly at 07:23 UTC: rclone to the - shared Space, mirror plus dated snapshots pruned after 90 days. Both check - in with Sentry Crons so a missed night is noticed. + shared Space, mirror plus dated snapshots pruned after 90 days. On the + shared host the hall's `hall-nightly.sh` runs both for every game and + checks in with Sentry Crons once for all, so a missed night is noticed. - The reports desk (skill `-reports`): `pull-reports.sh` mirrors the desk to the Desktop with a digest; `report-reply.sh "text"` answers with `resolved`, `by-design` or `open`. A fix goes live diff --git a/README.md b/README.md index 9123ee5..fa302fb 100644 --- a/README.md +++ b/README.md @@ -115,3 +115,11 @@ nightly rollup line in /var/lib/gamehall/rollup.jsonl. The skill `gamehall-visitors` reads them. A new game on the shared host is added to `GAMES` in `gamehall/ops/traffic.py` so the digest counts the people the hall sends it. + +The hall also keeps the night on the shared host: `hall-nightly.sh rollup` +at 00:12 UTC and `hall-nightly.sh backup` at 07:23 UTC (`gamehall.cron`) +run the hall's own rollup and every game's rollup and backup found in +/usr/local/bin, and check in with Sentry Crons once for each job, monitors +`hall-rollup` and `hall-backup` in the `gamehall` project, from the URLs in +/root/.hall-sentry-cron-rollup and /root/.hall-sentry-cron-backup. A game's +own cron file and check-ins are for a box of its own. diff --git a/gamehall/deploy.sh b/gamehall/deploy.sh index b0b800f..ceb1ea7 100755 --- a/gamehall/deploy.sh +++ b/gamehall/deploy.sh @@ -21,6 +21,8 @@ ssh "root@$HOST" ' # The counting tools: installed on every publish so the live copies are the repo copies. install -m 755 /tmp/gamehall-ops/gamehall-visitors.sh /usr/local/bin/gamehall-visitors.sh install -m 755 /tmp/gamehall-ops/gamehall-rollup.sh /usr/local/bin/gamehall-rollup.sh + # The nightly job that runs the rollup and backup of every game here and checks in once for all. + install -m 755 /tmp/gamehall-ops/hall-nightly.sh /usr/local/bin/hall-nightly.sh install -m 644 -D /tmp/gamehall-ops/traffic.py /usr/local/lib/gamehall/traffic.py install -m 644 /tmp/gamehall-ops/gamehall.cron /etc/cron.d/gamehall # The hall-wide notice every game serves at /hall/announcement.json. diff --git a/gamehall/ops/gamehall-rollup.sh b/gamehall/ops/gamehall-rollup.sh index 08f43b9..fef0c2f 100755 --- a/gamehall/ops/gamehall-rollup.sh +++ b/gamehall/ops/gamehall-rollup.sh @@ -4,7 +4,7 @@ # and the people sent on to each game here). Counts only: no addresses are # kept, so the trend outlives the self-rotating access log. # gamehall-rollup.sh [YYYY-MM-DD] (default: yesterday, UTC) -# Cron: /etc/cron.d/gamehall, installed by gamehall/deploy.sh. +# Run nightly by hall-nightly.sh rollup (/etc/cron.d/gamehall). set -u OUT="/var/lib/gamehall/rollup.jsonl" LOG="/var/log/gamehall/rollup.log" diff --git a/gamehall/ops/gamehall.cron b/gamehall/ops/gamehall.cron index e1dbe2e..eaec068 100644 --- a/gamehall/ops/gamehall.cron +++ b/gamehall/ops/gamehall.cron @@ -1,2 +1,4 @@ -# Kestrel's Hall: nightly rollup of yesterday's counts. UTC. -5 0 * * * root /usr/local/bin/gamehall-rollup.sh +# Kestrel's Hall: the nightly work of every game on this host, and the hall's own count. UTC. +# The times are also SCHEDULE in hall-nightly.sh, which tells Sentry to expect them; change both. +12 0 * * * root /usr/local/bin/hall-nightly.sh rollup +23 7 * * * root /usr/local/bin/hall-nightly.sh backup diff --git a/gamehall/ops/hall-nightly.sh b/gamehall/ops/hall-nightly.sh new file mode 100755 index 0000000..9f84171 --- /dev/null +++ b/gamehall/ops/hall-nightly.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# The hall's nightly work for every game on this host, with one Sentry Crons +# check-in for the lot: the rollups just after midnight, the backups at +# breakfast. Each game's own script does its work and says by its exit +# status whether it managed; the games are found by their scripts in +# /usr/local/bin, so a game deployed here is covered without being listed. +# hall-nightly.sh rollup|backup +# Cron: /etc/cron.d/gamehall, installed by gamehall/deploy.sh. SCHEDULE +# below must match it, because Sentry is told the same shape. +# Sentry Crons check-in: /root/.hall-sentry-cron- holds the check-in +# URL (absent = no check-ins). +set -u +JOB="${1:-}" +case "$JOB" in + rollup) SCHEDULE="12 0 * * *" ;; + backup) SCHEDULE="23 7 * * *" ;; + *) echo "usage: hall-nightly.sh rollup|backup" >&2; exit 2 ;; +esac +LOG="/var/log/gamehall/nightly.log" +CRON_URL=$(cat "/root/.hall-sentry-cron-$JOB" 2>/dev/null || true) +# One check-in, when the run ends, carrying its duration and the schedule +# (Sentry keeps a monitor only once told its shape). A start and a finish +# sent seconds apart can reach Sentry in either order, and a start that lands +# second is never closed. A run that never ends sends nothing, and Sentry +# marks the night missed. +START=$SECONDS +checkin() { + [ -n "$CRON_URL" ] && curl -sf -o /dev/null -X POST -H "Content-Type: application/json" \ + -d "{\"status\":\"$1\",\"duration\":$((SECONDS - START)),\"monitor_config\":{\"schedule\":{\"type\":\"crontab\",\"value\":\"$SCHEDULE\"},\"checkin_margin\":30,\"max_runtime\":30,\"timezone\":\"UTC\"}}" \ + "$CRON_URL" || true +} +mkdir -p "$(dirname "$LOG")" + +# The hall's own count first, then each game's script, in name order. +scripts=() +[ "$JOB" = rollup ] && scripts+=(/usr/local/bin/gamehall-rollup.sh) +for s in /usr/local/bin/*-"$JOB".sh; do + [ "$s" = /usr/local/bin/gamehall-rollup.sh ] || scripts+=("$s") +done +failed=() +for s in "${scripts[@]}"; do + name=$(basename "$s" "-$JOB.sh") + "$s" || failed+=("$name") +done +if [ ${#failed[@]} -eq 0 ]; then + echo "$(date -Is) $JOB: ${#scripts[@]} done" >> "$LOG" + checkin ok +else + echo "$(date -Is) $JOB: ${#scripts[@]} run, failed: ${failed[*]}" >> "$LOG" + checkin error + exit 1 +fi diff --git a/template/deploy/README.md b/template/deploy/README.md index 1d2bab3..995ff8c 100644 --- a/template/deploy/README.md +++ b/template/deploy/README.md @@ -46,9 +46,11 @@ A fresh host, when one is needed: `doctl compute droplet create hall --region ny ## Operations scripts on the droplet -`deploy.sh` installs these to /usr/local/bin, the `traffic.py` they share to -/usr/local/lib/__SLUG__, and the cron file to /etc/cron.d/__SLUG__ on every -deploy, so the live copies are the repo copies: +`deploy.sh` installs these to /usr/local/bin and the `traffic.py` they share +to /usr/local/lib/__SLUG__ on every deploy, so the live copies are the repo +copies. The cron file goes to /etc/cron.d/__SLUG__ only on a box of the game's +own: on the hall's shared host, `hall-nightly.sh` (in the kit's `gamehall/ops`) +runs every game's rollup and backup from /etc/cron.d/gamehall. - `__SLUG__-visitors.sh [day]`: who is here now and who came that day. - `__SLUG__-pulse.sh`: the weekly health check (service, errors, rollup @@ -62,10 +64,13 @@ deploy, so the live copies are the repo copies: wizwar droplet; until then it logs "skipped". Errors from the game server go to Sentry, project `__SLUG__` in the -locallygrownnet organisation; the DSN is in the unit file. The nightly rollup -and backup check in with Sentry Crons when /root/.__SLUG__-sentry-cron-rollup -and /root/.__SLUG__-sentry-cron hold their check-in URLs (the ingest -URL with the project's cron path and public key), so a missed night is noticed. +locallygrownnet organisation; the DSN is in the unit file. On the shared host +the hall's nightly job checks in with Sentry Crons once for every game's +rollup and once for every backup (monitors `hall-rollup` and `hall-backup` in +the `gamehall` project), and each game's script reports by its exit status. +A game on its own box checks in for itself when /root/.__SLUG__-sentry-cron-rollup +and /root/.__SLUG__-sentry-cron hold check-in URLs (the ingest URL with the +project's cron path and public key); absent, it sends nothing. To have every new issue, regression and reappearance posted to Slack the way wizwar's are, run `deploy/sentry-slack-alert.sh [#channel]` once from your own diff --git a/template/deploy/__SLUG__-backup.sh b/template/deploy/__SLUG__-backup.sh index a69257b..2f0514b 100755 --- a/template/deploy/__SLUG__-backup.sh +++ b/template/deploy/__SLUG__-backup.sh @@ -5,6 +5,8 @@ # Ledgers are append-only JSONL; the game server never needs stopping. # Sentry Crons check-in: /root/.__SLUG__-sentry-cron holds the check-in # URL (absent file = no check-ins). SCHEDULE must match /etc/cron.d/__SLUG__. +# On a host the hall shares, hall-nightly.sh runs this and checks in once +# for every game, so the file is absent and the exit status is the word. set -u SCHEDULE="23 7 * * *" BUCKET="kestrel-wizwar-backups" # the hall's shared Space, named for its first tenant @@ -27,9 +29,10 @@ checkin() { if [ ! -s /root/.config/rclone/rclone.conf ]; then echo "$(date -Is) skipped: rclone is not configured" >> "$LOG" - exit 0 + exit 1 fi +status=0 { echo "=== $(date -Is) backup start" if rclone sync "$SRC" "spaces:$BUCKET/$PREFIX/current" --exclude 'rollup.jsonl' 2>&1 && @@ -37,6 +40,7 @@ fi checkin ok else checkin error + status=1 fi CUTOFF=$(date -d "90 days ago" +%Y-%m-%d) rclone lsf "spaces:$BUCKET/$PREFIX/snapshots/" --dirs-only 2>/dev/null | \ @@ -49,3 +53,4 @@ fi done echo "=== $(date -Is) backup done" } >> "$LOG" 2>&1 +exit $status diff --git a/template/deploy/__SLUG__-rollup.sh b/template/deploy/__SLUG__-rollup.sh index 6e476a0..cc2acc3 100755 --- a/template/deploy/__SLUG__-rollup.sh +++ b/template/deploy/__SLUG__-rollup.sh @@ -8,7 +8,9 @@ # Cron: /etc/cron.d/__SLUG__, installed by deploy.sh; SCHEDULE below # must match it, because the Sentry monitor is told the same shape. # Sentry Crons check-in: /root/.__SLUG__-sentry-cron-rollup holds the -# check-in URL (absent = no check-ins). +# check-in URL (absent = no check-ins). On a host the hall shares, +# hall-nightly.sh runs this and checks in once for every game, so the +# file is absent and the exit status is the word. set -u OUT="/var/lib/__SLUG__/rollup.jsonl" LOG="/var/log/__SLUG__/rollup.log" @@ -78,4 +80,4 @@ with open(out + ".tmp", "w") as fh: os.replace(out + ".tmp", out) print("%s rolled: %d people, %d requests, %d rooms" % (day, len(t["addresses"]), t["human"], created)) PY -then checkin ok; else checkin error; fi +then checkin ok; else checkin error; exit 1; fi diff --git a/template/deploy/deploy.sh b/template/deploy/deploy.sh index bb754e1..248c382 100755 --- a/template/deploy/deploy.sh +++ b/template/deploy/deploy.sh @@ -43,7 +43,11 @@ ssh "root@$HOST" ' install -m 755 /opt/__SLUG__/app/deploy/__SLUG__-rollup.sh /usr/local/bin/__SLUG__-rollup.sh install -m 755 /opt/__SLUG__/app/deploy/__SLUG__-backup.sh /usr/local/bin/__SLUG__-backup.sh install -m 644 -D /opt/__SLUG__/app/deploy/traffic.py /usr/local/lib/__SLUG__/traffic.py - install -m 644 /opt/__SLUG__/app/deploy/__SLUG__.cron /etc/cron.d/__SLUG__ + # On a host the hall shares, its hall-nightly.sh runs the rollup and backup + # of every game and checks in with Sentry once for all; a game on a box of + # its own schedules its own. + if [ -f /etc/cron.d/gamehall ]; then rm -f /etc/cron.d/__SLUG__ + else install -m 644 /opt/__SLUG__/app/deploy/__SLUG__.cron /etc/cron.d/__SLUG__; fi mkdir -p /var/log/__SLUG__ systemctl daemon-reload systemctl enable --now __SLUG__