Eric WagonerandClaude Fable 5.1
7af9de4a9a
A bare ping to a slug Sentry has never been told about is accepted and dropped, so the rollup monitor never existed and a missed night would have alarmed nobody. The check-in now carries the monitor's shape and upserts it. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Jm2auWk6RP71CjaAb4FMoG
139 lines
7.7 KiB
Bash
Executable File
139 lines
7.7 KiB
Bash
Executable File
#!/bin/bash
|
|
# Nightly rollup: one JSON line per day in /var/lib/wizwar/rollup.jsonl —
|
|
# yesterday's traffic (from Caddy's access log), the table's growth (from
|
|
# the room ledgers, stats.json and feedback.jsonl), and the box's vitals.
|
|
# Counts only: no addresses are kept. The pulse reads it for trends the
|
|
# self-rotating access log cannot keep. Referrers are what browsers send
|
|
# (Reddit and BGG send none); campaigns are the ?ref=<name> tags on links
|
|
# Eric posts, one count per address per day, plus Facebook's fbclid.
|
|
# Cron: 10 0 * * * (UTC), see /etc/cron.d/wizwar-rollup (deploy.sh installs both).
|
|
# Sentry Crons check-in: /root/.wizwar-sentry-cron-rollup holds the URL
|
|
# (absent = no check-ins).
|
|
set -u
|
|
OUT="/var/lib/wizwar/rollup.jsonl"
|
|
LOG="/var/log/wizwar/rollup.log"
|
|
DAY="${1:-$(date -u -d 'yesterday' +%Y-%m-%d)}"
|
|
CRON_URL=$(cat /root/.wizwar-sentry-cron-rollup 2>/dev/null || true)
|
|
# The check-in carries the schedule: Sentry only keeps a monitor it has
|
|
# been told the shape of, and a bare ping to an unknown slug is accepted
|
|
# and dropped without a word.
|
|
checkin() {
|
|
[ -n "$CRON_URL" ] && curl -sf -o /dev/null -X POST -H "Content-Type: application/json" \
|
|
-d "{\"status\":\"$1\",\"monitor_config\":{\"schedule\":{\"type\":\"crontab\",\"value\":\"10 0 * * *\"},\"checkin_margin\":30,\"max_runtime\":10,\"timezone\":\"UTC\"}}" \
|
|
"$CRON_URL" || true
|
|
}
|
|
|
|
checkin in_progress
|
|
if python3 - "$DAY" "$OUT" <<'PY' >> "$LOG" 2>&1
|
|
import json, sys, os, glob, gzip, datetime, collections, subprocess, re
|
|
day, out = sys.argv[1], sys.argv[2]
|
|
d0 = datetime.datetime.fromisoformat(day).replace(tzinfo=datetime.timezone.utc)
|
|
t0, t1 = d0.timestamp(), (d0 + datetime.timedelta(days=1)).timestamp()
|
|
BOT = ("bot", "crawl", "spider", "facebookexternalhit", "slack", "discord", "twitter", "preview", "curl", "python", "go-http", "wget", "headless")
|
|
|
|
# --- traffic: every access log file touched since the day began ---
|
|
req = 0; human = 0; bot = 0; hips = set(); bips = set()
|
|
paths = collections.Counter(); status = collections.Counter(); refs = collections.Counter(); ws = 0
|
|
# Where a visit came from, when the link itself says so: ?ref=<name> on a
|
|
# link Eric posted, a Facebook click's fbclid, or a utm_source.
|
|
campaigns = collections.Counter(); seen_campaign = set()
|
|
for f in sorted(glob.glob("/var/lib/caddy/access*.log*")):
|
|
if os.path.getmtime(f) < t0: continue
|
|
opener = gzip.open if f.endswith(".gz") else open
|
|
with opener(f, "rt", errors="ignore") as fh:
|
|
for line in fh:
|
|
try: r = json.loads(line)
|
|
except ValueError: continue
|
|
ts = r.get("ts", 0)
|
|
if not (t0 <= ts < t1): continue
|
|
q = r.get("request", {}); h = q.get("headers", {})
|
|
ua = " ".join(h.get("User-Agent", [""])).lower()
|
|
ip = q.get("client_ip") or q.get("remote_ip") or "?"
|
|
isbot = any(k in ua for k in BOT)
|
|
req += 1
|
|
if isbot: bot += 1; bips.add(ip)
|
|
else: human += 1; hips.add(ip)
|
|
full = q.get("uri", ""); uri = full.split("?")[0]
|
|
if uri == "/ws": ws += 1
|
|
if not isbot and "?" in full and ip not in seen_campaign:
|
|
qs = dict(kv.split("=", 1) if "=" in kv else (kv, "") for kv in full.split("?", 1)[1].split("&"))
|
|
tag = qs.get("ref") or qs.get("utm_source") or ("facebook" if "fbclid" in qs else None)
|
|
if tag and re.fullmatch(r"[a-z0-9_-]{1,24}", tag):
|
|
campaigns[tag] += 1; seen_campaign.add(ip)
|
|
key = ("/clips" if uri.startswith("/clips") else "/watch" if uri.startswith("/watch")
|
|
else "(assets)" if uri.startswith("/assets") or re.search(r"\.(png|jpg|svg|ico|css|js|webmanifest|mp4)$", uri)
|
|
else uri if uri in ("/", "/ws", "/robots.txt") else "(other)")
|
|
if not isbot: paths[key] += 1 # the path mix is what people looked at
|
|
status[str(r.get("status", 0))[0] + "xx"] += 1
|
|
ref = " ".join(h.get("Referer", [""]))
|
|
m = re.match(r"https?://([^/]+)", ref)
|
|
host = m.group(1) if m else ""
|
|
ours = host.endswith("kestrelsnest.social") or "104.236.96.198" in host
|
|
if host and not ours and not isbot: refs[host] += 1
|
|
|
|
# --- the table ---
|
|
new_rooms = 0; seats = collections.Counter()
|
|
for f in glob.glob("/var/lib/wizwar/rooms/*.jsonl"):
|
|
try:
|
|
with open(f) as fh:
|
|
meta = json.loads(fh.readline())
|
|
ca = meta.get("createdAt")
|
|
if not ca: continue
|
|
ct = datetime.datetime.fromisoformat(ca.replace("Z", "+00:00")).timestamp()
|
|
if not (t0 <= ct < t1): continue
|
|
new_rooms += 1
|
|
# The host takes the first seat by creating; only others join.
|
|
if meta.get("hostId"): seats[meta["hostId"]] += 1
|
|
for line in fh:
|
|
try: r = json.loads(line)
|
|
except ValueError: continue
|
|
if r.get("kind") == "join" and not r.get("bot"): seats[r["name"]] += 1
|
|
except OSError: continue
|
|
stats = {}
|
|
try: stats = json.load(open("/var/lib/wizwar/stats.json"))
|
|
except Exception: pass
|
|
reports = 0; replies = 0
|
|
try:
|
|
for line in open("/var/lib/wizwar/feedback.jsonl"):
|
|
try: r = json.loads(line)
|
|
except ValueError: continue
|
|
at = datetime.datetime.fromisoformat(r["at"].replace("Z", "+00:00")).timestamp()
|
|
if t0 <= at < t1: reports += ("happened" in r); replies += ("reportId" in r)
|
|
except OSError: pass
|
|
|
|
# --- vitals (now, at rollup time) ---
|
|
def sh(cmd):
|
|
try: return subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=20).stdout.strip()
|
|
except Exception: return ""
|
|
mem = sh("systemctl show wizwar -p MemoryCurrent --value"); peak = sh("systemctl show wizwar -p MemoryPeak --value")
|
|
disk = sh("df --output=pcent / | tail -1").strip().rstrip("%")
|
|
load = sh("cut -d' ' -f1 /proc/loadavg")
|
|
journal = sh("journalctl -u wizwar --since '%s' --until '%s' --no-pager 2>/dev/null" % (day, (d0 + datetime.timedelta(days=1)).date()))
|
|
errs = str(len(re.findall(r"(?i)unhandled protocol error", journal)))
|
|
starts = str(journal.count("Started wizwar"))
|
|
ledger = sum(os.path.getsize(f) for f in glob.glob("/var/lib/wizwar/rooms/*.jsonl"))
|
|
row = {
|
|
"day": day, "requests": req, "human": human, "bot": bot,
|
|
"humanAddresses": len(hips), "botAddresses": len(bips), "socketConnects": ws,
|
|
"paths": dict(paths.most_common()), "status": dict(status), "referrers": dict(refs.most_common(10)),
|
|
"campaigns": dict(campaigns.most_common(10)),
|
|
"newRooms": new_rooms, "humanSeats": sum(seats.values()), "humanNames": sorted(seats),
|
|
"gamesFinishedTotal": stats.get("gamesFinished"), "gamesCreatedTotal": stats.get("gamesCreated"),
|
|
"reports": reports, "replies": replies,
|
|
"vitals": {"memMB": round(int(mem) / 1048576) if mem.isdigit() else None,
|
|
"memPeakMB": round(int(peak) / 1048576) if peak.isdigit() else None,
|
|
"diskPct": int(disk) if disk.isdigit() else None, "load1": float(load) if load else None,
|
|
"protocolErrors": int(errs) if errs.isdigit() else None, "serviceStarts": int(starts) if starts.isdigit() else None,
|
|
"ledgerMB": round(ledger / 1048576, 1)},
|
|
}
|
|
# One line per day: a rerun for the same day replaces its line.
|
|
rows = []
|
|
if os.path.exists(out):
|
|
rows = [l for l in open(out).read().splitlines() if l.strip() and json.loads(l).get("day") != day]
|
|
rows.append(json.dumps(row, separators=(",", ":")))
|
|
tmp = out + ".tmp"
|
|
open(tmp, "w").write("\n".join(rows) + "\n"); os.replace(tmp, out)
|
|
print("%s rollup %s: %d requests, %d human addresses, %d new rooms, %d human seats" % (datetime.datetime.now(datetime.timezone.utc).isoformat(), day, req, len(hips), new_rooms, sum(seats.values())))
|
|
PY
|
|
then checkin ok; else checkin error; fi
|