The nightly rollup keeps what the access log forgets; two doors get a limit
Three sysadmin items ahead of the public announcement's arrivals. deploy/wizwar-rollup.sh writes one JSON line per UTC day to /var/lib/wizwar/rollup.jsonl at 00:10: yesterday's traffic from Caddy's access log (requests, human vs bot addresses as counts only, socket connects, path mix, external referrers), the table's growth (new rooms, human seats and names, lifetime game totals, reports and replies), and the box's vitals (service memory and peak, disk, load, protocol errors, service starts, ledger size). Re-rolling a day replaces its line. It checks in to a Sentry cron monitor of its own, whose URL deploy.sh derives from the backup monitor's on first install, alongside the cron entry. The pulse gains a Trends section that reads the last week of it. Caddy keeps 30 log files instead of 5 as the raw backing. Per-address limits on the two doors anyone may use unseated: 12 new rooms and 6 reports per address per hour, in a sliding window keyed by the address Caddy forwards. The per-connection cap stays; it reset on reconnect, which is what a script would do. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Jm2auWk6RP71CjaAb4FMoG
This commit is contained in:
Eric Wagoner
co-authored by
Claude Fable 5.1
Claude Fable 5.1
parent
3f909b485b
commit
ca108f83a6
@@ -15,6 +15,14 @@ ssh "root@$HOST" '
|
||||
# deploy, or the repo copy and the live copy drift apart (they did:
|
||||
# the Sentry check-ins shipped in the repo and never reached the box).
|
||||
install -m 755 /opt/wizwar/deploy/wizwar-backup.sh /usr/local/bin/wizwar-backup.sh
|
||||
# The nightly rollup and its cron entry ride along the same way; its
|
||||
# Sentry check-in URL is the backup monitor'"'"'s with its own slug.
|
||||
install -m 755 /opt/wizwar/deploy/wizwar-rollup.sh /usr/local/bin/wizwar-rollup.sh
|
||||
install -m 644 /opt/wizwar/deploy/wizwar-rollup.cron /etc/cron.d/wizwar-rollup
|
||||
if [ -f /root/.wizwar-sentry-cron ] && [ ! -f /root/.wizwar-sentry-cron-rollup ]; then
|
||||
sed "s#/cron/new-monitor/#/cron/wizwar-rollup/#" /root/.wizwar-sentry-cron > /root/.wizwar-sentry-cron-rollup
|
||||
chmod 600 /root/.wizwar-sentry-cron-rollup
|
||||
fi
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now wizwar
|
||||
systemctl restart wizwar
|
||||
|
||||
@@ -25,7 +25,9 @@ mkdir -p /opt/wizwar /var/lib/wizwar/rooms
|
||||
chown -R wizwar:wizwar /opt/wizwar /var/lib/wizwar
|
||||
|
||||
# Caddy vhost: auto-TLS, security headers, proxy to the game.
|
||||
printf '%s\n\nheader {\n\tStrict-Transport-Security "max-age=31536000"\n\tX-Content-Type-Options "nosniff"\n\tX-Frame-Options "DENY"\n\tReferrer-Policy "no-referrer"\n}\nreverse_proxy localhost:8787\n' "$HOST" > /etc/caddy/Caddyfile
|
||||
# Access log kept ~30 days (10MiB x 30): the nightly rollup keeps the
|
||||
# counts forever, the raw lines back it for a month.
|
||||
printf '%s\n\nheader {\n\tStrict-Transport-Security "max-age=31536000"\n\tX-Content-Type-Options "nosniff"\n\tX-Frame-Options "DENY"\n\tReferrer-Policy "no-referrer"\n}\nlog {\n\toutput file /var/lib/caddy/access.log {\n\t\troll_size 10MiB\n\t\troll_keep 30\n\t}\n}\nreverse_proxy localhost:8787\n' "$HOST" > /etc/caddy/Caddyfile
|
||||
systemctl reload caddy
|
||||
|
||||
# Firewall: ssh + web only. The game server binds loopback and is reached
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# Nightly rollup of yesterday's traffic, table, and vitals (UTC). Installed by deploy.sh.
|
||||
10 0 * * * root /usr/local/bin/wizwar-rollup.sh
|
||||
Executable
+117
@@ -0,0 +1,117 @@
|
||||
#!/bin/bash
|
||||
# Nightly rollup: one JSON line per day in /var/lib/wizwar/rollup.jsonl —
|
||||
# yesterday's traffic (from Caddy's access log), the table's growth (from
|
||||
# the room ledgers, stats.json and feedback.jsonl), and the box's vitals.
|
||||
# Counts only: no addresses are kept. The pulse reads it for trends the
|
||||
# self-rotating access log cannot keep.
|
||||
# Cron: 10 0 * * * (UTC), see /etc/cron.d/wizwar-rollup (deploy.sh installs both).
|
||||
# Sentry Crons check-in: /root/.wizwar-sentry-cron-rollup holds the URL
|
||||
# (derived from the backup monitor's on first deploy; absent = no check-ins).
|
||||
set -u
|
||||
OUT="/var/lib/wizwar/rollup.jsonl"
|
||||
LOG="/var/log/wizwar/rollup.log"
|
||||
DAY="${1:-$(date -u -d 'yesterday' +%Y-%m-%d)}"
|
||||
CRON_URL=$(cat /root/.wizwar-sentry-cron-rollup 2>/dev/null || true)
|
||||
checkin() { [ -n "$CRON_URL" ] && curl -sf -o /dev/null "$CRON_URL?status=$1" || true; }
|
||||
|
||||
checkin in_progress
|
||||
if python3 - "$DAY" "$OUT" <<'PY' >> "$LOG" 2>&1
|
||||
import json, sys, os, glob, gzip, datetime, collections, subprocess, re
|
||||
day, out = sys.argv[1], sys.argv[2]
|
||||
d0 = datetime.datetime.fromisoformat(day).replace(tzinfo=datetime.timezone.utc)
|
||||
t0, t1 = d0.timestamp(), (d0 + datetime.timedelta(days=1)).timestamp()
|
||||
BOT = ("bot", "crawl", "spider", "facebookexternalhit", "slack", "discord", "twitter", "preview", "curl", "python", "go-http", "wget", "headless")
|
||||
|
||||
# --- traffic: every access log file touched since the day began ---
|
||||
req = 0; human = 0; bot = 0; hips = set(); bips = set()
|
||||
paths = collections.Counter(); status = collections.Counter(); refs = collections.Counter(); ws = 0
|
||||
for f in sorted(glob.glob("/var/lib/caddy/access*.log*")):
|
||||
if os.path.getmtime(f) < t0: continue
|
||||
opener = gzip.open if f.endswith(".gz") else open
|
||||
with opener(f, "rt", errors="ignore") as fh:
|
||||
for line in fh:
|
||||
try: r = json.loads(line)
|
||||
except ValueError: continue
|
||||
ts = r.get("ts", 0)
|
||||
if not (t0 <= ts < t1): continue
|
||||
q = r.get("request", {}); h = q.get("headers", {})
|
||||
ua = " ".join(h.get("User-Agent", [""])).lower()
|
||||
ip = q.get("client_ip") or q.get("remote_ip") or "?"
|
||||
isbot = any(k in ua for k in BOT)
|
||||
req += 1
|
||||
if isbot: bot += 1; bips.add(ip)
|
||||
else: human += 1; hips.add(ip)
|
||||
uri = q.get("uri", "").split("?")[0]
|
||||
if uri == "/ws": ws += 1
|
||||
key = ("/clips" if uri.startswith("/clips") else "/watch" if uri.startswith("/watch")
|
||||
else "(assets)" if uri.startswith("/assets") or re.search(r"\.(png|jpg|svg|ico|css|js|webmanifest|mp4)$", uri)
|
||||
else uri if uri in ("/", "/ws", "/robots.txt") else "(other)")
|
||||
paths[key] += 1
|
||||
status[str(r.get("status", 0))[0] + "xx"] += 1
|
||||
ref = " ".join(h.get("Referer", [""]))
|
||||
m = re.match(r"https?://([^/]+)", ref)
|
||||
host = m.group(1) if m else ""
|
||||
ours = host.endswith("kestrelsnest.social") or "104.236.96.198" in host
|
||||
if host and not ours and not isbot: refs[host] += 1
|
||||
|
||||
# --- the table ---
|
||||
new_rooms = 0; seats = collections.Counter()
|
||||
for f in glob.glob("/var/lib/wizwar/rooms/*.jsonl"):
|
||||
try:
|
||||
with open(f) as fh:
|
||||
meta = json.loads(fh.readline())
|
||||
ca = meta.get("createdAt")
|
||||
if not ca: continue
|
||||
ct = datetime.datetime.fromisoformat(ca.replace("Z", "+00:00")).timestamp()
|
||||
if not (t0 <= ct < t1): continue
|
||||
new_rooms += 1
|
||||
for line in fh:
|
||||
try: r = json.loads(line)
|
||||
except ValueError: continue
|
||||
if r.get("kind") == "join" and not r.get("bot"): seats[r["name"]] += 1
|
||||
except OSError: continue
|
||||
stats = {}
|
||||
try: stats = json.load(open("/var/lib/wizwar/stats.json"))
|
||||
except Exception: pass
|
||||
reports = 0; replies = 0
|
||||
try:
|
||||
for line in open("/var/lib/wizwar/feedback.jsonl"):
|
||||
try: r = json.loads(line)
|
||||
except ValueError: continue
|
||||
at = datetime.datetime.fromisoformat(r["at"].replace("Z", "+00:00")).timestamp()
|
||||
if t0 <= at < t1: reports += ("happened" in r); replies += ("reportId" in r)
|
||||
except OSError: pass
|
||||
|
||||
# --- vitals (now, at rollup time) ---
|
||||
def sh(cmd):
|
||||
try: return subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=20).stdout.strip()
|
||||
except Exception: return ""
|
||||
mem = sh("systemctl show wizwar -p MemoryCurrent --value"); peak = sh("systemctl show wizwar -p MemoryPeak --value")
|
||||
disk = sh("df --output=pcent / | tail -1").strip().rstrip("%")
|
||||
load = sh("cut -d' ' -f1 /proc/loadavg")
|
||||
errs = sh("journalctl -u wizwar --since '%s' --until '%s' --no-pager 2>/dev/null | grep -ci 'unhandled protocol error'" % (day, (d0 + datetime.timedelta(days=1)).date()))
|
||||
starts = sh("journalctl -u wizwar --since '%s' --until '%s' --no-pager 2>/dev/null | grep -c 'Started wizwar'" % (day, (d0 + datetime.timedelta(days=1)).date()))
|
||||
ledger = sum(os.path.getsize(f) for f in glob.glob("/var/lib/wizwar/rooms/*.jsonl"))
|
||||
row = {
|
||||
"day": day, "requests": req, "human": human, "bot": bot,
|
||||
"humanAddresses": len(hips), "botAddresses": len(bips), "socketConnects": ws,
|
||||
"paths": dict(paths.most_common()), "status": dict(status), "referrers": dict(refs.most_common(10)),
|
||||
"newRooms": new_rooms, "humanSeats": sum(seats.values()), "humanNames": sorted(seats),
|
||||
"gamesFinishedTotal": stats.get("gamesFinished"), "gamesCreatedTotal": stats.get("gamesCreated"),
|
||||
"reports": reports, "replies": replies,
|
||||
"vitals": {"memMB": round(int(mem) / 1048576) if mem.isdigit() else None,
|
||||
"memPeakMB": round(int(peak) / 1048576) if peak.isdigit() else None,
|
||||
"diskPct": int(disk) if disk.isdigit() else None, "load1": float(load) if load else None,
|
||||
"protocolErrors": int(errs) if errs.isdigit() else None, "serviceStarts": int(starts) if starts.isdigit() else None,
|
||||
"ledgerMB": round(ledger / 1048576, 1)},
|
||||
}
|
||||
# One line per day: a rerun for the same day replaces its line.
|
||||
rows = []
|
||||
if os.path.exists(out):
|
||||
rows = [l for l in open(out).read().splitlines() if l.strip() and json.loads(l).get("day") != day]
|
||||
rows.append(json.dumps(row, separators=(",", ":")))
|
||||
tmp = out + ".tmp"
|
||||
open(tmp, "w").write("\n".join(rows) + "\n"); os.replace(tmp, out)
|
||||
print("%s rollup %s: %d requests, %d human addresses, %d new rooms, %d human seats" % (datetime.datetime.now(datetime.timezone.utc).isoformat(), day, req, len(hips), new_rooms, sum(seats.values())))
|
||||
PY
|
||||
then checkin ok; else checkin error; fi
|
||||
Reference in New Issue
Block a user