diff --git a/.claude/skills/gamehall-visitors/SKILL.md b/.claude/skills/gamehall-visitors/SKILL.md index 349aaa6..8823b50 100644 --- a/.claude/skills/gamehall-visitors/SKILL.md +++ b/.claude/skills/gamehall-visitors/SKILL.md @@ -23,7 +23,7 @@ name (the sslip test address is left out) and prints: own digest (/wizwar-visitors) lists gamehall.kestrelsnest.social among its referrers. - **last 7 days** from `/var/lib/gamehall/rollup.jsonl`, written nightly - at 00:05 UTC by `gamehall-rollup.sh` for the day before (counts only, + at 00:12 UTC by `gamehall-rollup.sh` (run by `hall-nightly.sh`) for the day before (counts only, no addresses). ## Read it diff --git a/CONVENTIONS.md b/CONVENTIONS.md index aec8699..44c3a41 100644 --- a/CONVENTIONS.md +++ b/CONVENTIONS.md @@ -198,8 +198,9 @@ Every game gets the same tools, installed by `deploy.sh` on every deploy: the rollup's last week, the backup's last word, the box's vitals. - `-rollup.sh` nightly at 00:12 UTC: one JSON line a day of counts, no addresses. `-backup.sh` nightly at 07:23 UTC: rclone to the - shared Space, mirror plus dated snapshots pruned after 90 days. Both check - in with Sentry Crons so a missed night is noticed. + shared Space, mirror plus dated snapshots pruned after 90 days. On the + shared host the hall's `hall-nightly.sh` runs both for every game and + checks in with Sentry Crons once for all, so a missed night is noticed. - The reports desk (skill `-reports`): `pull-reports.sh` mirrors the desk to the Desktop with a digest; `report-reply.sh "text"` answers with `resolved`, `by-design` or `open`. A fix goes live diff --git a/README.md b/README.md index 9123ee5..fa302fb 100644 --- a/README.md +++ b/README.md @@ -115,3 +115,11 @@ nightly rollup line in /var/lib/gamehall/rollup.jsonl. The skill `gamehall-visitors` reads them. A new game on the shared host is added to `GAMES` in `gamehall/ops/traffic.py` so the digest counts the people the hall sends it. + +The hall also keeps the night on the shared host: `hall-nightly.sh rollup` +at 00:12 UTC and `hall-nightly.sh backup` at 07:23 UTC (`gamehall.cron`) +run the hall's own rollup and every game's rollup and backup found in +/usr/local/bin, and check in with Sentry Crons once for each job, monitors +`hall-rollup` and `hall-backup` in the `gamehall` project, from the URLs in +/root/.hall-sentry-cron-rollup and /root/.hall-sentry-cron-backup. A game's +own cron file and check-ins are for a box of its own. diff --git a/gamehall/deploy.sh b/gamehall/deploy.sh index b0b800f..ceb1ea7 100755 --- a/gamehall/deploy.sh +++ b/gamehall/deploy.sh @@ -21,6 +21,8 @@ ssh "root@$HOST" ' # The counting tools: installed on every publish so the live copies are the repo copies. install -m 755 /tmp/gamehall-ops/gamehall-visitors.sh /usr/local/bin/gamehall-visitors.sh install -m 755 /tmp/gamehall-ops/gamehall-rollup.sh /usr/local/bin/gamehall-rollup.sh + # The nightly job that runs the rollup and backup of every game here and checks in once for all. + install -m 755 /tmp/gamehall-ops/hall-nightly.sh /usr/local/bin/hall-nightly.sh install -m 644 -D /tmp/gamehall-ops/traffic.py /usr/local/lib/gamehall/traffic.py install -m 644 /tmp/gamehall-ops/gamehall.cron /etc/cron.d/gamehall # The hall-wide notice every game serves at /hall/announcement.json. diff --git a/gamehall/ops/gamehall-rollup.sh b/gamehall/ops/gamehall-rollup.sh index 08f43b9..fef0c2f 100755 --- a/gamehall/ops/gamehall-rollup.sh +++ b/gamehall/ops/gamehall-rollup.sh @@ -4,7 +4,7 @@ # and the people sent on to each game here). Counts only: no addresses are # kept, so the trend outlives the self-rotating access log. # gamehall-rollup.sh [YYYY-MM-DD] (default: yesterday, UTC) -# Cron: /etc/cron.d/gamehall, installed by gamehall/deploy.sh. +# Run nightly by hall-nightly.sh rollup (/etc/cron.d/gamehall). set -u OUT="/var/lib/gamehall/rollup.jsonl" LOG="/var/log/gamehall/rollup.log" diff --git a/gamehall/ops/gamehall.cron b/gamehall/ops/gamehall.cron index e1dbe2e..eaec068 100644 --- a/gamehall/ops/gamehall.cron +++ b/gamehall/ops/gamehall.cron @@ -1,2 +1,4 @@ -# Kestrel's Hall: nightly rollup of yesterday's counts. UTC. -5 0 * * * root /usr/local/bin/gamehall-rollup.sh +# Kestrel's Hall: the nightly work of every game on this host, and the hall's own count. UTC. +# The times are also SCHEDULE in hall-nightly.sh, which tells Sentry to expect them; change both. +12 0 * * * root /usr/local/bin/hall-nightly.sh rollup +23 7 * * * root /usr/local/bin/hall-nightly.sh backup diff --git a/gamehall/ops/hall-nightly.sh b/gamehall/ops/hall-nightly.sh new file mode 100755 index 0000000..9f84171 --- /dev/null +++ b/gamehall/ops/hall-nightly.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# The hall's nightly work for every game on this host, with one Sentry Crons +# check-in for the lot: the rollups just after midnight, the backups at +# breakfast. Each game's own script does its work and says by its exit +# status whether it managed; the games are found by their scripts in +# /usr/local/bin, so a game deployed here is covered without being listed. +# hall-nightly.sh rollup|backup +# Cron: /etc/cron.d/gamehall, installed by gamehall/deploy.sh. SCHEDULE +# below must match it, because Sentry is told the same shape. +# Sentry Crons check-in: /root/.hall-sentry-cron- holds the check-in +# URL (absent = no check-ins). +set -u +JOB="${1:-}" +case "$JOB" in + rollup) SCHEDULE="12 0 * * *" ;; + backup) SCHEDULE="23 7 * * *" ;; + *) echo "usage: hall-nightly.sh rollup|backup" >&2; exit 2 ;; +esac +LOG="/var/log/gamehall/nightly.log" +CRON_URL=$(cat "/root/.hall-sentry-cron-$JOB" 2>/dev/null || true) +# One check-in, when the run ends, carrying its duration and the schedule +# (Sentry keeps a monitor only once told its shape). A start and a finish +# sent seconds apart can reach Sentry in either order, and a start that lands +# second is never closed. A run that never ends sends nothing, and Sentry +# marks the night missed. +START=$SECONDS +checkin() { + [ -n "$CRON_URL" ] && curl -sf -o /dev/null -X POST -H "Content-Type: application/json" \ + -d "{\"status\":\"$1\",\"duration\":$((SECONDS - START)),\"monitor_config\":{\"schedule\":{\"type\":\"crontab\",\"value\":\"$SCHEDULE\"},\"checkin_margin\":30,\"max_runtime\":30,\"timezone\":\"UTC\"}}" \ + "$CRON_URL" || true +} +mkdir -p "$(dirname "$LOG")" + +# The hall's own count first, then each game's script, in name order. +scripts=() +[ "$JOB" = rollup ] && scripts+=(/usr/local/bin/gamehall-rollup.sh) +for s in /usr/local/bin/*-"$JOB".sh; do + [ "$s" = /usr/local/bin/gamehall-rollup.sh ] || scripts+=("$s") +done +failed=() +for s in "${scripts[@]}"; do + name=$(basename "$s" "-$JOB.sh") + "$s" || failed+=("$name") +done +if [ ${#failed[@]} -eq 0 ]; then + echo "$(date -Is) $JOB: ${#scripts[@]} done" >> "$LOG" + checkin ok +else + echo "$(date -Is) $JOB: ${#scripts[@]} run, failed: ${failed[*]}" >> "$LOG" + checkin error + exit 1 +fi diff --git a/template/deploy/README.md b/template/deploy/README.md index 1d2bab3..995ff8c 100644 --- a/template/deploy/README.md +++ b/template/deploy/README.md @@ -46,9 +46,11 @@ A fresh host, when one is needed: `doctl compute droplet create hall --region ny ## Operations scripts on the droplet -`deploy.sh` installs these to /usr/local/bin, the `traffic.py` they share to -/usr/local/lib/__SLUG__, and the cron file to /etc/cron.d/__SLUG__ on every -deploy, so the live copies are the repo copies: +`deploy.sh` installs these to /usr/local/bin and the `traffic.py` they share +to /usr/local/lib/__SLUG__ on every deploy, so the live copies are the repo +copies. The cron file goes to /etc/cron.d/__SLUG__ only on a box of the game's +own: on the hall's shared host, `hall-nightly.sh` (in the kit's `gamehall/ops`) +runs every game's rollup and backup from /etc/cron.d/gamehall. - `__SLUG__-visitors.sh [day]`: who is here now and who came that day. - `__SLUG__-pulse.sh`: the weekly health check (service, errors, rollup @@ -62,10 +64,13 @@ deploy, so the live copies are the repo copies: wizwar droplet; until then it logs "skipped". Errors from the game server go to Sentry, project `__SLUG__` in the -locallygrownnet organisation; the DSN is in the unit file. The nightly rollup -and backup check in with Sentry Crons when /root/.__SLUG__-sentry-cron-rollup -and /root/.__SLUG__-sentry-cron hold their check-in URLs (the ingest -URL with the project's cron path and public key), so a missed night is noticed. +locallygrownnet organisation; the DSN is in the unit file. On the shared host +the hall's nightly job checks in with Sentry Crons once for every game's +rollup and once for every backup (monitors `hall-rollup` and `hall-backup` in +the `gamehall` project), and each game's script reports by its exit status. +A game on its own box checks in for itself when /root/.__SLUG__-sentry-cron-rollup +and /root/.__SLUG__-sentry-cron hold check-in URLs (the ingest URL with the +project's cron path and public key); absent, it sends nothing. To have every new issue, regression and reappearance posted to Slack the way wizwar's are, run `deploy/sentry-slack-alert.sh [#channel]` once from your own diff --git a/template/deploy/__SLUG__-backup.sh b/template/deploy/__SLUG__-backup.sh index a69257b..2f0514b 100755 --- a/template/deploy/__SLUG__-backup.sh +++ b/template/deploy/__SLUG__-backup.sh @@ -5,6 +5,8 @@ # Ledgers are append-only JSONL; the game server never needs stopping. # Sentry Crons check-in: /root/.__SLUG__-sentry-cron holds the check-in # URL (absent file = no check-ins). SCHEDULE must match /etc/cron.d/__SLUG__. +# On a host the hall shares, hall-nightly.sh runs this and checks in once +# for every game, so the file is absent and the exit status is the word. set -u SCHEDULE="23 7 * * *" BUCKET="kestrel-wizwar-backups" # the hall's shared Space, named for its first tenant @@ -27,9 +29,10 @@ checkin() { if [ ! -s /root/.config/rclone/rclone.conf ]; then echo "$(date -Is) skipped: rclone is not configured" >> "$LOG" - exit 0 + exit 1 fi +status=0 { echo "=== $(date -Is) backup start" if rclone sync "$SRC" "spaces:$BUCKET/$PREFIX/current" --exclude 'rollup.jsonl' 2>&1 && @@ -37,6 +40,7 @@ fi checkin ok else checkin error + status=1 fi CUTOFF=$(date -d "90 days ago" +%Y-%m-%d) rclone lsf "spaces:$BUCKET/$PREFIX/snapshots/" --dirs-only 2>/dev/null | \ @@ -49,3 +53,4 @@ fi done echo "=== $(date -Is) backup done" } >> "$LOG" 2>&1 +exit $status diff --git a/template/deploy/__SLUG__-rollup.sh b/template/deploy/__SLUG__-rollup.sh index 6e476a0..cc2acc3 100755 --- a/template/deploy/__SLUG__-rollup.sh +++ b/template/deploy/__SLUG__-rollup.sh @@ -8,7 +8,9 @@ # Cron: /etc/cron.d/__SLUG__, installed by deploy.sh; SCHEDULE below # must match it, because the Sentry monitor is told the same shape. # Sentry Crons check-in: /root/.__SLUG__-sentry-cron-rollup holds the -# check-in URL (absent = no check-ins). +# check-in URL (absent = no check-ins). On a host the hall shares, +# hall-nightly.sh runs this and checks in once for every game, so the +# file is absent and the exit status is the word. set -u OUT="/var/lib/__SLUG__/rollup.jsonl" LOG="/var/log/__SLUG__/rollup.log" @@ -78,4 +80,4 @@ with open(out + ".tmp", "w") as fh: os.replace(out + ".tmp", out) print("%s rolled: %d people, %d requests, %d rooms" % (day, len(t["addresses"]), t["human"], created)) PY -then checkin ok; else checkin error; fi +then checkin ok; else checkin error; exit 1; fi diff --git a/template/deploy/deploy.sh b/template/deploy/deploy.sh index bb754e1..248c382 100755 --- a/template/deploy/deploy.sh +++ b/template/deploy/deploy.sh @@ -43,7 +43,11 @@ ssh "root@$HOST" ' install -m 755 /opt/__SLUG__/app/deploy/__SLUG__-rollup.sh /usr/local/bin/__SLUG__-rollup.sh install -m 755 /opt/__SLUG__/app/deploy/__SLUG__-backup.sh /usr/local/bin/__SLUG__-backup.sh install -m 644 -D /opt/__SLUG__/app/deploy/traffic.py /usr/local/lib/__SLUG__/traffic.py - install -m 644 /opt/__SLUG__/app/deploy/__SLUG__.cron /etc/cron.d/__SLUG__ + # On a host the hall shares, its hall-nightly.sh runs the rollup and backup + # of every game and checks in with Sentry once for all; a game on a box of + # its own schedules its own. + if [ -f /etc/cron.d/gamehall ]; then rm -f /etc/cron.d/__SLUG__ + else install -m 644 /opt/__SLUG__/app/deploy/__SLUG__.cron /etc/cron.d/__SLUG__; fi mkdir -p /var/log/__SLUG__ systemctl daemon-reload systemctl enable --now __SLUG__