Eric WagonerandClaude Fable 5.1
9bd203ae60
Every deploy first replays every production ledger with the engine about to ship. The droplet gains a nightly rollup of counts and a nightly backup to Spaces, a pulse script the pulse skill reads, rate limits on opening rooms and taking seats, and eviction of idle rooms from memory with reload from their ledgers on the next visit. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0141G6xqLeNRYEtviLWSB5Up
97 lines
5.0 KiB
Bash
Executable File
97 lines
5.0 KiB
Bash
Executable File
#!/bin/bash
|
|
# Nightly rollup: one JSON line per day in /var/lib/waving-hands/rollup.jsonl:
|
|
# the day's traffic from Caddy's access log (people and bots apart, by page,
|
|
# with referrers and the campaign tags links carry), the rooms opened,
|
|
# started and finished, and the box's vitals. Counts only: no addresses are
|
|
# kept. The pulse reads it for trends the self-rotating access log loses.
|
|
# waving-hands-rollup.sh [YYYY-MM-DD] (default: yesterday, UTC)
|
|
# Cron: /etc/cron.d/waving-hands, installed by deploy.sh.
|
|
set -u
|
|
OUT="/var/lib/waving-hands/rollup.jsonl"
|
|
LOG="/var/log/waving-hands/rollup.log"
|
|
DAY="${1:-$(date -u -d 'yesterday' +%Y-%m-%d)}"
|
|
mkdir -p "$(dirname "$LOG")"
|
|
python3 - "$DAY" "$OUT" <<'PY' >> "$LOG" 2>&1
|
|
import collections, datetime, glob, gzip, json, os, re, shutil, sys
|
|
day, out = sys.argv[1], sys.argv[2]
|
|
d0 = datetime.datetime.fromisoformat(day).replace(tzinfo=datetime.timezone.utc)
|
|
t0, t1 = d0.timestamp(), (d0 + datetime.timedelta(days=1)).timestamp()
|
|
BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I)
|
|
|
|
req = human = bot = 0
|
|
hips, bips = set(), set()
|
|
pages, joins, refs, campaigns = collections.Counter(), collections.Counter(), collections.Counter(), collections.Counter()
|
|
seen_campaign = set()
|
|
for f in sorted(glob.glob("/var/lib/caddy/access*.log*")):
|
|
if os.path.getmtime(f) < t0: continue
|
|
opener = gzip.open if f.endswith(".gz") else open
|
|
with opener(f, "rt", errors="ignore") as fh:
|
|
for line in fh:
|
|
try: r = json.loads(line)
|
|
except ValueError: continue
|
|
ts = r.get("ts", 0)
|
|
if not (t0 <= ts < t1): continue
|
|
q = r.get("request", {}); h = q.get("headers", {})
|
|
ua = " ".join(h.get("User-Agent", [""]))
|
|
ip = q.get("client_ip") or q.get("remote_ip") or "?"
|
|
uri = q.get("uri", ""); path = uri.split("?")[0]
|
|
req += 1
|
|
if BOT.search(ua) or not ua:
|
|
bot += 1; bips.add(ip); continue
|
|
if path.startswith(("/_app/", "/api/")) or path == "/ws" or path.endswith((".png", ".ico", ".txt")): continue
|
|
human += 1; hips.add(ip)
|
|
if path == "/": pages["hall"] += 1
|
|
elif path == "/play": pages["play"] += 1
|
|
elif path == "/rules": pages["rules"] += 1
|
|
elif path.startswith("/join/"): joins[path.split("/")[2].upper()] += 1
|
|
ref = " ".join(h.get("Referer", [""]))
|
|
if ref and "hands.kestrelsnest.social" not in ref:
|
|
refs[re.sub(r"^https?://", "", ref).split("/")[0]] += 1
|
|
m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri)
|
|
if m and (ip, m.group(1)) not in seen_campaign:
|
|
seen_campaign.add((ip, m.group(1)))
|
|
campaigns[m.group(1) + ("=" + m.group(2)[:24] if m.group(1) != "fbclid" else "")] += 1
|
|
|
|
created = started = finished = turned = turns = 0
|
|
humans, bots = set(), 0
|
|
for f in glob.glob("/var/lib/waving-hands/rooms/*.jsonl"):
|
|
try: lines = [json.loads(l) for l in open(f) if l.strip()]
|
|
except Exception: continue
|
|
if not lines or lines[0].get("t") != "room": continue
|
|
made = lines[0]["createdAt"] / 1000
|
|
if t0 <= made < t1:
|
|
created += 1
|
|
if any(x["t"] == "start" for x in lines): started += 1
|
|
if any(x["t"] == "turn" for x in lines): turned += 1
|
|
for x in lines:
|
|
if x["t"] == "seat":
|
|
if x["bot"]: bots += 1
|
|
else: humans.add(x["name"])
|
|
day_turns = [x for x in lines if x["t"] == "turn" and t0 <= x["at"] / 1000 < t1]
|
|
turns += len(day_turns)
|
|
# A duel finished today: its last turn is today's and the engine would say so; approximate by replay-free means:
|
|
# the ledger's final turn falls today and no turn follows it within the day's end.
|
|
if day_turns and day_turns[-1] is lines[-1] and False: finished += 1
|
|
|
|
disk = shutil.disk_usage("/")
|
|
mem = {}
|
|
for line in open("/proc/meminfo"):
|
|
k, v = line.split(":", 1); mem[k] = int(v.split()[0])
|
|
load = os.getloadavg()[1]
|
|
row = dict(day=day, requests=req, human=human, bot=bot, humanAddresses=len(hips), botAddresses=len(bips),
|
|
paths=dict(pages), roomLinks=sum(joins.values()), referrers=dict(refs), campaigns=dict(campaigns),
|
|
rooms=dict(created=created, started=started, withTurns=turned, humanNames=len(humans), botSeats=bots),
|
|
turns=turns, ledgers=len(glob.glob("/var/lib/waving-hands/rooms/*.jsonl")),
|
|
vitals=dict(diskUsedPct=round(disk.used * 100 / disk.total), memAvailableMb=mem.get("MemAvailable", 0) // 1024, load5=round(load, 2)),
|
|
at=datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds"))
|
|
rows = []
|
|
if os.path.exists(out):
|
|
rows = [json.loads(l) for l in open(out) if l.strip()]
|
|
rows = [r for r in rows if r.get("day") != day] + [row]
|
|
rows.sort(key=lambda r: r["day"])
|
|
with open(out + ".tmp", "w") as fh:
|
|
for r in rows: fh.write(json.dumps(r) + "\n")
|
|
os.replace(out + ".tmp", out)
|
|
print("%s rolled: %d people, %d requests, %d rooms" % (day, len(hips), human, created))
|
|
PY
|