wizwar6e/deploy/wizwar-rollup.sh

#!/bin/bash
# Nightly rollup: one JSON line per day in /var/lib/wizwar/rollup.jsonl —
# yesterday's traffic (from Caddy's access log), the table's growth (from
# the room ledgers, stats.json and feedback.jsonl), and the box's vitals.
# Counts only: no addresses are kept. The pulse reads it for trends the
# self-rotating access log cannot keep.
#   Cron: 10 0 * * * (UTC), see /etc/cron.d/wizwar-rollup (deploy.sh installs both).
#   Sentry Crons check-in: /root/.wizwar-sentry-cron-rollup holds the URL
#   (absent = no check-ins).
set -u
OUT="/var/lib/wizwar/rollup.jsonl"
LOG="/var/log/wizwar/rollup.log"
DAY="${1:-$(date -u -d 'yesterday' +%Y-%m-%d)}"
CRON_URL=$(cat /root/.wizwar-sentry-cron-rollup 2>/dev/null || true)
checkin() { [ -n "$CRON_URL" ] && curl -sf -o /dev/null "$CRON_URL?status=$1" || true; }

checkin in_progress
if python3 - "$DAY" "$OUT" <<'PY' >> "$LOG" 2>&1
import json, sys, os, glob, gzip, datetime, collections, subprocess, re
day, out = sys.argv[1], sys.argv[2]
d0 = datetime.datetime.fromisoformat(day).replace(tzinfo=datetime.timezone.utc)
t0, t1 = d0.timestamp(), (d0 + datetime.timedelta(days=1)).timestamp()
BOT = ("bot", "crawl", "spider", "facebookexternalhit", "slack", "discord", "twitter", "preview", "curl", "python", "go-http", "wget", "headless")

# --- traffic: every access log file touched since the day began ---
req = 0; human = 0; bot = 0; hips = set(); bips = set()
paths = collections.Counter(); status = collections.Counter(); refs = collections.Counter(); ws = 0
for f in sorted(glob.glob("/var/lib/caddy/access*.log*")):
    if os.path.getmtime(f) < t0: continue
    opener = gzip.open if f.endswith(".gz") else open
    with opener(f, "rt", errors="ignore") as fh:
        for line in fh:
            try: r = json.loads(line)
            except ValueError: continue
            ts = r.get("ts", 0)
            if not (t0 <= ts < t1): continue
            q = r.get("request", {}); h = q.get("headers", {})
            ua = " ".join(h.get("User-Agent", [""])).lower()
            ip = q.get("client_ip") or q.get("remote_ip") or "?"
            isbot = any(k in ua for k in BOT)
            req += 1
            if isbot: bot += 1; bips.add(ip)
            else: human += 1; hips.add(ip)
            uri = q.get("uri", "").split("?")[0]
            if uri == "/ws": ws += 1
            key = ("/clips" if uri.startswith("/clips") else "/watch" if uri.startswith("/watch")
                   else "(assets)" if uri.startswith("/assets") or re.search(r"\.(png|jpg|svg|ico|css|js|webmanifest|mp4)$", uri)
                   else uri if uri in ("/", "/ws", "/robots.txt") else "(other)")
            paths[key] += 1
            status[str(r.get("status", 0))[0] + "xx"] += 1
            ref = " ".join(h.get("Referer", [""]))
            m = re.match(r"https?://([^/]+)", ref)
            host = m.group(1) if m else ""
            ours = host.endswith("kestrelsnest.social") or "104.236.96.198" in host
            if host and not ours and not isbot: refs[host] += 1

# --- the table ---
new_rooms = 0; seats = collections.Counter()
for f in glob.glob("/var/lib/wizwar/rooms/*.jsonl"):
    try:
        with open(f) as fh:
            meta = json.loads(fh.readline())
            ca = meta.get("createdAt")
            if not ca: continue
            ct = datetime.datetime.fromisoformat(ca.replace("Z", "+00:00")).timestamp()
            if not (t0 <= ct < t1): continue
            new_rooms += 1
            for line in fh:
                try: r = json.loads(line)
                except ValueError: continue
                if r.get("kind") == "join" and not r.get("bot"): seats[r["name"]] += 1
    except OSError: continue
stats = {}
try: stats = json.load(open("/var/lib/wizwar/stats.json"))
except Exception: pass
reports = 0; replies = 0
try:
    for line in open("/var/lib/wizwar/feedback.jsonl"):
        try: r = json.loads(line)
        except ValueError: continue
        at = datetime.datetime.fromisoformat(r["at"].replace("Z", "+00:00")).timestamp()
        if t0 <= at < t1: reports += ("happened" in r); replies += ("reportId" in r)
except OSError: pass

# --- vitals (now, at rollup time) ---
def sh(cmd):
    try: return subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=20).stdout.strip()
    except Exception: return ""
mem = sh("systemctl show wizwar -p MemoryCurrent --value"); peak = sh("systemctl show wizwar -p MemoryPeak --value")
disk = sh("df --output=pcent / | tail -1").strip().rstrip("%")
load = sh("cut -d' ' -f1 /proc/loadavg")
journal = sh("journalctl -u wizwar --since '%s' --until '%s' --no-pager 2>/dev/null" % (day, (d0 + datetime.timedelta(days=1)).date()))
errs = str(len(re.findall(r"(?i)unhandled protocol error", journal)))
starts = str(journal.count("Started wizwar"))
ledger = sum(os.path.getsize(f) for f in glob.glob("/var/lib/wizwar/rooms/*.jsonl"))
row = {
    "day": day, "requests": req, "human": human, "bot": bot,
    "humanAddresses": len(hips), "botAddresses": len(bips), "socketConnects": ws,
    "paths": dict(paths.most_common()), "status": dict(status), "referrers": dict(refs.most_common(10)),
    "newRooms": new_rooms, "humanSeats": sum(seats.values()), "humanNames": sorted(seats),
    "gamesFinishedTotal": stats.get("gamesFinished"), "gamesCreatedTotal": stats.get("gamesCreated"),
    "reports": reports, "replies": replies,
    "vitals": {"memMB": round(int(mem) / 1048576) if mem.isdigit() else None,
               "memPeakMB": round(int(peak) / 1048576) if peak.isdigit() else None,
               "diskPct": int(disk) if disk.isdigit() else None, "load1": float(load) if load else None,
               "protocolErrors": int(errs) if errs.isdigit() else None, "serviceStarts": int(starts) if starts.isdigit() else None,
               "ledgerMB": round(ledger / 1048576, 1)},
}
# One line per day: a rerun for the same day replaces its line.
rows = []
if os.path.exists(out):
    rows = [l for l in open(out).read().splitlines() if l.strip() and json.loads(l).get("day") != day]
rows.append(json.dumps(row, separators=(",", ":")))
tmp = out + ".tmp"
open(tmp, "w").write("\n".join(rows) + "\n"); os.replace(tmp, out)
print("%s rollup %s: %d requests, %d human addresses, %d new rooms, %d human seats" % (datetime.datetime.now(datetime.timezone.utc).isoformat(), day, req, len(hips), new_rooms, sum(seats.values())))
PY
then checkin ok; else checkin error; fi