Eric WagonerandClaude Fable 5.1
60435fef84
Eight cron monitors, two per game, watched eight scripts that run on the same box at the same two minutes. hall-nightly.sh now runs the hall's own rollup and every game's rollup at 00:12, and every game's backup at 07:23, finding the games by their scripts in /usr/local/bin, and checks in once for each job to the gamehall project's hall-rollup and hall-backup monitors. The template's rollup and backup say by their exit status whether they managed, the skipped backup included, and its deploy leaves the game's own cron file off a host the hall schedules. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
84 lines
4.5 KiB
Bash
Executable File
84 lines
4.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Nightly rollup: one JSON line per day in /var/lib/__SLUG__/rollup.jsonl:
|
|
# the day's traffic from Caddy's access log (people and bots apart, by page,
|
|
# with referrers and the campaign tags links carry), the rooms opened,
|
|
# started and finished, and the box's vitals. Counts only: no addresses are
|
|
# kept. The pulse reads it for trends the self-rotating access log loses.
|
|
# __SLUG__-rollup.sh [YYYY-MM-DD] (default: yesterday, UTC)
|
|
# Cron: /etc/cron.d/__SLUG__, installed by deploy.sh; SCHEDULE below
|
|
# must match it, because the Sentry monitor is told the same shape.
|
|
# Sentry Crons check-in: /root/.__SLUG__-sentry-cron-rollup holds the
|
|
# check-in URL (absent = no check-ins). On a host the hall shares,
|
|
# hall-nightly.sh runs this and checks in once for every game, so the
|
|
# file is absent and the exit status is the word.
|
|
set -u
|
|
OUT="/var/lib/__SLUG__/rollup.jsonl"
|
|
LOG="/var/log/__SLUG__/rollup.log"
|
|
DAY="${1:-$(date -u -d 'yesterday' +%Y-%m-%d)}"
|
|
SCHEDULE="12 0 * * *"
|
|
CRON_URL=$(cat /root/.__SLUG__-sentry-cron-rollup 2>/dev/null || true)
|
|
# One check-in, when the run ends, carrying its duration and the schedule
|
|
# (Sentry keeps a monitor only once told its shape). A start and a finish
|
|
# sent seconds apart can reach Sentry in either order, and a start that lands
|
|
# second is never closed. A run that never ends sends nothing, and Sentry
|
|
# marks the night missed.
|
|
START=$SECONDS
|
|
checkin() {
|
|
[ -n "$CRON_URL" ] && curl -sf -o /dev/null -X POST -H "Content-Type: application/json" \
|
|
-d "{\"status\":\"$1\",\"duration\":$((SECONDS - START)),\"monitor_config\":{\"schedule\":{\"type\":\"crontab\",\"value\":\"$SCHEDULE\"},\"checkin_margin\":30,\"max_runtime\":10,\"timezone\":\"UTC\"}}" \
|
|
"$CRON_URL" || true
|
|
}
|
|
mkdir -p "$(dirname "$LOG")"
|
|
if python3 - "$DAY" "$OUT" <<'PY' >> "$LOG" 2>&1
|
|
import datetime, glob, json, os, shutil, sys
|
|
sys.path.insert(0, "/usr/local/lib/__SLUG__")
|
|
import traffic
|
|
day, out = sys.argv[1], sys.argv[2]
|
|
d0 = datetime.datetime.fromisoformat(day).replace(tzinfo=datetime.timezone.utc)
|
|
t0, t1 = d0.timestamp(), (d0 + datetime.timedelta(days=1)).timestamp()
|
|
t = traffic.scan(day)
|
|
|
|
created = started = finished = turned = turns = 0
|
|
humans, bots = set(), 0
|
|
for f in glob.glob("/var/lib/__SLUG__/rooms/*.jsonl"):
|
|
try: lines = [json.loads(l) for l in open(f) if l.strip()]
|
|
except Exception: continue
|
|
if not lines or lines[0].get("t") != "room": continue
|
|
made = lines[0]["createdAt"] / 1000
|
|
if t0 <= made < t1:
|
|
created += 1
|
|
if any(x["t"] == "start" for x in lines): started += 1
|
|
if any(x["t"] == "turn" for x in lines): turned += 1
|
|
for x in lines:
|
|
if x["t"] == "seat":
|
|
if x["bot"]: bots += 1
|
|
else: humans.add(x["name"])
|
|
elif x["t"] == "unseat": bots -= 1
|
|
turns += sum(1 for x in lines if x["t"] == "turn" and t0 <= x["at"] / 1000 < t1)
|
|
# The over line is written with the turn that ends the game, so a finish counts on its own day.
|
|
if any(x["t"] == "over" and t0 <= x["at"] / 1000 < t1 for x in lines): finished += 1
|
|
|
|
disk = shutil.disk_usage("/")
|
|
mem = {}
|
|
for line in open("/proc/meminfo"):
|
|
k, v = line.split(":", 1); mem[k] = int(v.split()[0])
|
|
load = os.getloadavg()[1]
|
|
row = dict(day=day, requests=t["requests"], human=t["human"], bot=t["bot"], humanAddresses=len(t["addresses"]), botAddresses=len(t["botAddresses"]),
|
|
paths=dict(t["pages"]), roomLinks=sum(t["joins"].values()), roomsOpened=t["opened"], sharedOn=dict(t["shared"]),
|
|
scanners=t["scanners"], monitors=t["monitors"], referrers=dict(t["referrers"]), campaigns=dict(t["campaigns"]),
|
|
rooms=dict(created=created, started=started, withTurns=turned, finished=finished, humanNames=len(humans), botSeats=bots),
|
|
turns=turns, ledgers=len(glob.glob("/var/lib/__SLUG__/rooms/*.jsonl")),
|
|
vitals=dict(diskUsedPct=round(disk.used * 100 / disk.total), memAvailableMb=mem.get("MemAvailable", 0) // 1024, load5=round(load, 2)),
|
|
at=datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds"))
|
|
rows = []
|
|
if os.path.exists(out):
|
|
rows = [json.loads(l) for l in open(out) if l.strip()]
|
|
rows = [r for r in rows if r.get("day") != day] + [row]
|
|
rows.sort(key=lambda r: r["day"])
|
|
with open(out + ".tmp", "w") as fh:
|
|
for r in rows: fh.write(json.dumps(r) + "\n")
|
|
os.replace(out + ".tmp", out)
|
|
print("%s rolled: %d people, %d requests, %d rooms" % (day, len(t["addresses"]), t["human"], created))
|
|
PY
|
|
then checkin ok; else checkin error; exit 1; fi
|