waving-hands/deploy/waving-hands-rollup.sh

#!/bin/bash
# Nightly rollup: one JSON line per day in /var/lib/waving-hands/rollup.jsonl:
# the day's traffic from Caddy's access log (people and bots apart, by page,
# with referrers and the campaign tags links carry), the rooms opened,
# started and finished, and the box's vitals. Counts only: no addresses are
# kept. The pulse reads it for trends the self-rotating access log loses.
#   waving-hands-rollup.sh [YYYY-MM-DD]   (default: yesterday, UTC)
#   Cron: /etc/cron.d/waving-hands, installed by deploy.sh; SCHEDULE below
#   must match it, because the Sentry monitor is told the same shape.
#   Sentry Crons check-in: /root/.waving-hands-sentry-cron-rollup holds the
#   check-in URL (absent = no check-ins).
set -u
OUT="/var/lib/waving-hands/rollup.jsonl"
LOG="/var/log/waving-hands/rollup.log"
DAY="${1:-$(date -u -d 'yesterday' +%Y-%m-%d)}"
SCHEDULE="12 0 * * *"
CRON_URL=$(cat /root/.waving-hands-sentry-cron-rollup 2>/dev/null || true)
# The check-in carries the schedule: Sentry keeps a monitor only once told its shape.
checkin() {
  [ -n "$CRON_URL" ] && curl -sf -o /dev/null -X POST -H "Content-Type: application/json" \
    -d "{\"status\":\"$1\",\"monitor_config\":{\"schedule\":{\"type\":\"crontab\",\"value\":\"$SCHEDULE\"},\"checkin_margin\":30,\"max_runtime\":10,\"timezone\":\"UTC\"}}" \
    "$CRON_URL" || true
}
mkdir -p "$(dirname "$LOG")"
checkin in_progress
if python3 - "$DAY" "$OUT" <<'PY' >> "$LOG" 2>&1
import collections, datetime, glob, gzip, json, os, re, shutil, sys
day, out = sys.argv[1], sys.argv[2]
d0 = datetime.datetime.fromisoformat(day).replace(tzinfo=datetime.timezone.utc)
t0, t1 = d0.timestamp(), (d0 + datetime.timedelta(days=1)).timestamp()
BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I)

req = human = bot = 0
hips, bips = set(), set()
pages, joins, refs, campaigns = collections.Counter(), collections.Counter(), collections.Counter(), collections.Counter()
seen_campaign = set()
for f in sorted(glob.glob("/var/lib/caddy/access*.log*")):
    if os.path.getmtime(f) < t0: continue
    opener = gzip.open if f.endswith(".gz") else open
    with opener(f, "rt", errors="ignore") as fh:
        for line in fh:
            try: r = json.loads(line)
            except ValueError: continue
            ts = r.get("ts", 0)
            if not (t0 <= ts < t1): continue
            q = r.get("request", {}); h = q.get("headers", {})
            ua = " ".join(h.get("User-Agent", [""]))
            ip = q.get("client_ip") or q.get("remote_ip") or "?"
            uri = q.get("uri", ""); path = uri.split("?")[0]
            req += 1
            if BOT.search(ua) or not ua:
                bot += 1; bips.add(ip); continue
            if path.startswith(("/_app/", "/api/")) or path == "/ws" or path.endswith((".png", ".ico", ".txt")): continue
            human += 1; hips.add(ip)
            if path == "/": pages["hall"] += 1
            elif path == "/play": pages["play"] += 1
            elif path == "/rules": pages["rules"] += 1
            elif path.startswith("/join/"): joins[path.split("/")[2].upper()] += 1
            ref = " ".join(h.get("Referer", [""]))
            if ref and "hands.kestrelsnest.social" not in ref:
                refs[re.sub(r"^https?://", "", ref).split("/")[0]] += 1
            m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri)
            if m and (ip, m.group(1)) not in seen_campaign:
                seen_campaign.add((ip, m.group(1)))
                campaigns[m.group(1) + ("=" + m.group(2)[:24] if m.group(1) != "fbclid" else "")] += 1

created = started = finished = turned = turns = 0
humans, bots = set(), 0
for f in glob.glob("/var/lib/waving-hands/rooms/*.jsonl"):
    try: lines = [json.loads(l) for l in open(f) if l.strip()]
    except Exception: continue
    if not lines or lines[0].get("t") != "room": continue
    made = lines[0]["createdAt"] / 1000
    if t0 <= made < t1:
        created += 1
        if any(x["t"] == "start" for x in lines): started += 1
        if any(x["t"] == "turn" for x in lines): turned += 1
        for x in lines:
            if x["t"] == "seat":
                if x["bot"]: bots += 1
                else: humans.add(x["name"])
    day_turns = [x for x in lines if x["t"] == "turn" and t0 <= x["at"] / 1000 < t1]
    turns += len(day_turns)
    # A duel finished today: its last turn is today's and the engine would say so; approximate by replay-free means:
    # the ledger's final turn falls today and no turn follows it within the day's end.
    if day_turns and day_turns[-1] is lines[-1] and False: finished += 1

disk = shutil.disk_usage("/")
mem = {}
for line in open("/proc/meminfo"):
    k, v = line.split(":", 1); mem[k] = int(v.split()[0])
load = os.getloadavg()[1]
row = dict(day=day, requests=req, human=human, bot=bot, humanAddresses=len(hips), botAddresses=len(bips),
           paths=dict(pages), roomLinks=sum(joins.values()), referrers=dict(refs), campaigns=dict(campaigns),
           rooms=dict(created=created, started=started, withTurns=turned, humanNames=len(humans), botSeats=bots),
           turns=turns, ledgers=len(glob.glob("/var/lib/waving-hands/rooms/*.jsonl")),
           vitals=dict(diskUsedPct=round(disk.used * 100 / disk.total), memAvailableMb=mem.get("MemAvailable", 0) // 1024, load5=round(load, 2)),
           at=datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds"))
rows = []
if os.path.exists(out):
    rows = [json.loads(l) for l in open(out) if l.strip()]
rows = [r for r in rows if r.get("day") != day] + [row]
rows.sort(key=lambda r: r["day"])
with open(out + ".tmp", "w") as fh:
    for r in rows: fh.write(json.dumps(r) + "\n")
os.replace(out + ".tmp", out)
print("%s rolled: %d people, %d requests, %d rooms" % (day, len(hips), human, created))
PY
then checkin ok; else checkin error; fi