From f0d7237c2bde6520179826d94b8cb02f518df707 Mon Sep 17 00:00:00 2001 From: Eric Wagoner Date: Sat, 10 Oct 2026 17:44:43 -0400 Subject: [PATCH] The rollup and the backup check in once, when they end, with the run's duration A start and a finish sent seconds apart can reach Sentry in either order, and a start that lands second is never closed, so a fast rollup was reported as a timeout on nights it succeeded. Hnefatafl's monitor did so nine nights in fifteen. One check-in at the end cannot race; a run that hangs sends nothing and is marked missed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GwFKMuQnPAEHJ5yA1q4orh --- template/deploy/__SLUG__-backup.sh | 8 ++++++-- template/deploy/__SLUG__-rollup.sh | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/template/deploy/__SLUG__-backup.sh b/template/deploy/__SLUG__-backup.sh index f0b6d34..a69257b 100755 --- a/template/deploy/__SLUG__-backup.sh +++ b/template/deploy/__SLUG__-backup.sh @@ -14,9 +14,14 @@ LOG="/var/log/__SLUG__/backup.log" STAMP=$(date +%Y-%m-%d) mkdir -p "$(dirname "$LOG")" CRON_URL=$(cat /root/.__SLUG__-sentry-cron 2>/dev/null || true) +# One check-in, when the run ends, carrying its duration: a start and a +# finish sent seconds apart can reach Sentry in either order, and a start +# that lands second is never closed. A run that never ends sends nothing, +# and Sentry marks the night missed. +START=$SECONDS checkin() { [ -n "$CRON_URL" ] && curl -sf -o /dev/null -X POST -H "Content-Type: application/json" \ - -d "{\"status\":\"$1\",\"monitor_config\":{\"schedule\":{\"type\":\"crontab\",\"value\":\"$SCHEDULE\"},\"checkin_margin\":30,\"max_runtime\":10,\"timezone\":\"UTC\"}}" \ + -d "{\"status\":\"$1\",\"duration\":$((SECONDS - START)),\"monitor_config\":{\"schedule\":{\"type\":\"crontab\",\"value\":\"$SCHEDULE\"},\"checkin_margin\":30,\"max_runtime\":10,\"timezone\":\"UTC\"}}" \ "$CRON_URL" || true } @@ -27,7 +32,6 @@ fi { echo "=== $(date -Is) backup start" - checkin in_progress if rclone sync "$SRC" "spaces:$BUCKET/$PREFIX/current" --exclude 'rollup.jsonl' 2>&1 && rclone copy "$SRC" "spaces:$BUCKET/$PREFIX/snapshots/$STAMP" 2>&1; then checkin ok diff --git a/template/deploy/__SLUG__-rollup.sh b/template/deploy/__SLUG__-rollup.sh index 123c601..685bd9b 100755 --- a/template/deploy/__SLUG__-rollup.sh +++ b/template/deploy/__SLUG__-rollup.sh @@ -16,13 +16,17 @@ DAY="${1:-$(date -u -d 'yesterday' +%Y-%m-%d)}" SCHEDULE="12 0 * * *" CRON_URL=$(cat /root/.__SLUG__-sentry-cron-rollup 2>/dev/null || true) # The check-in carries the schedule: Sentry keeps a monitor only once told its shape. +# One check-in, when the run ends, carrying its duration: a start and a +# finish sent seconds apart can reach Sentry in either order, and a start +# that lands second is never closed. A run that never ends sends nothing, +# and Sentry marks the night missed. +START=$SECONDS checkin() { [ -n "$CRON_URL" ] && curl -sf -o /dev/null -X POST -H "Content-Type: application/json" \ - -d "{\"status\":\"$1\",\"monitor_config\":{\"schedule\":{\"type\":\"crontab\",\"value\":\"$SCHEDULE\"},\"checkin_margin\":30,\"max_runtime\":10,\"timezone\":\"UTC\"}}" \ + -d "{\"status\":\"$1\",\"duration\":$((SECONDS - START)),\"monitor_config\":{\"schedule\":{\"type\":\"crontab\",\"value\":\"$SCHEDULE\"},\"checkin_margin\":30,\"max_runtime\":10,\"timezone\":\"UTC\"}}" \ "$CRON_URL" || true } mkdir -p "$(dirname "$LOG")" -checkin in_progress if python3 - "$DAY" "$OUT" <<'PY' >> "$LOG" 2>&1 import datetime, glob, json, os, shutil, sys sys.path.insert(0, "/usr/local/lib/__SLUG__")