From 1dbf5763e105c7188f202fae55aa5449a0c1d56f Mon Sep 17 00:00:00 2001 From: Eric Wagoner Date: Sat, 26 Sep 2026 12:41:55 -0400 Subject: [PATCH] The hall counts its visitors: a digest, a nightly rollup, and the games it sends people to gamehall/ops holds the traffic counter, the visitors digest and the nightly rollup, installed by gamehall/deploy.sh; the gamehall-visitors skill reads them. The hall now sends its origin as referrer, so a game can tell who came from it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Jm2auWk6RP71CjaAb4FMoG --- .claude/skills/gamehall-visitors/SKILL.md | 38 ++++++++++ README.md | 7 ++ gamehall/deploy.sh | 8 ++ gamehall/gamehall.caddy | 3 +- gamehall/ops/gamehall-rollup.sh | 29 +++++++ gamehall/ops/gamehall-visitors.sh | 34 +++++++++ gamehall/ops/gamehall.cron | 2 + gamehall/ops/traffic.py | 92 +++++++++++++++++++++++ 8 files changed, 212 insertions(+), 1 deletion(-) create mode 100644 .claude/skills/gamehall-visitors/SKILL.md create mode 100755 gamehall/ops/gamehall-rollup.sh create mode 100755 gamehall/ops/gamehall-visitors.sh create mode 100644 gamehall/ops/gamehall.cron create mode 100644 gamehall/ops/traffic.py diff --git a/.claude/skills/gamehall-visitors/SKILL.md b/.claude/skills/gamehall-visitors/SKILL.md new file mode 100644 index 0000000..349aaa6 --- /dev/null +++ b/.claude/skills/gamehall-visitors/SKILL.md @@ -0,0 +1,38 @@ +--- +name: gamehall-visitors +description: Who is visiting Kestrel's Hall, the landing page at gamehall.kestrelsnest.social - the last hour, today, where they came from, how many went on into each game, and the week's trend. Use when Eric asks about hall traffic or visitors. +--- + +# Kestrel's Hall visitors + +One command on the shared host prints the digest: + + ssh root@159.203.115.252 /usr/local/bin/gamehall-visitors.sh [YYYY-MM-DD] + +Default is today, UTC. It reads Caddy's access log for the hall's own +name (the sslip test address is left out) and prints: + +- **last hour** (today only): people and page views, the "right now". +- **today**: people (distinct addresses), page views, bot addresses. +- **referrers** and **campaigns**: the sites that sent people, and the + tags links carry (`?ref=`, utm_source, Facebook's fbclid). Reddit and + some apps send no referrer. +- **went on to**: people each game on this host (Dip, Waving Hands, + Hnefatafl) received with the hall as referrer. The hall sends its + origin as referrer, nothing more. Wiz-War is on its own droplet; its + own digest (/wizwar-visitors) lists gamehall.kestrelsnest.social among + its referrers. +- **last 7 days** from `/var/lib/gamehall/rollup.jsonl`, written nightly + at 00:05 UTC by `gamehall-rollup.sh` for the day before (counts only, + no addresses). + +## Read it + +Lead with whether anyone is here now and today's people, then where they +came from, then how many went on into a game (the hall's job is to send +people to tables). Give the week's direction from the rollup in one +sentence. Eric's own devices count as people; a handful on a quiet day +may be him. + +The source is `gamehall/ops/` in game-kit; `gamehall/deploy.sh` installs +it on every publish. diff --git a/README.md b/README.md index 943aba4..89fb287 100644 --- a/README.md +++ b/README.md @@ -83,3 +83,10 @@ in `gamehall/site/` with a card per game, written by hand. It is served by Caddy on the shared host beside the games (`gamehall/gamehall.caddy`) and published with `gamehall/deploy.sh 159.203.115.252`. A new game is not in the family until it has its card there too. + +Its visitors are counted from Caddy's log by `gamehall/ops/` (installed by +the same deploy): `gamehall-visitors.sh` on the host for today, and a +nightly rollup line in /var/lib/gamehall/rollup.jsonl. The skill +`gamehall-visitors` reads them. A new game on the shared host is added to +`GAMES` in `gamehall/ops/traffic.py` so the digest counts the people the +hall sends it. diff --git a/gamehall/deploy.sh b/gamehall/deploy.sh index 5e84b71..04fe789 100755 --- a/gamehall/deploy.sh +++ b/gamehall/deploy.sh @@ -7,6 +7,7 @@ HERE="$(cd "$(dirname "$0")" && pwd)" rsync -az --delete "$HERE/site/" "root@$HOST:/opt/gamehall/" scp -q "$HERE/gamehall.caddy" "root@$HOST:/etc/caddy/sites/gamehall.caddy" +rsync -az "$HERE/ops/" "root@$HOST:/tmp/gamehall-ops/" ssh "root@$HOST" ' set -e chown -R root:caddy /opt/gamehall @@ -17,5 +18,12 @@ ssh "root@$HOST" ' caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile >/dev/null 2>&1 chown caddy:caddy /var/lib/caddy/gamehall-access.log systemctl reload caddy + # The counting tools: installed on every publish so the live copies are the repo copies. + install -m 755 /tmp/gamehall-ops/gamehall-visitors.sh /usr/local/bin/gamehall-visitors.sh + install -m 755 /tmp/gamehall-ops/gamehall-rollup.sh /usr/local/bin/gamehall-rollup.sh + install -m 644 -D /tmp/gamehall-ops/traffic.py /usr/local/lib/gamehall/traffic.py + install -m 644 /tmp/gamehall-ops/gamehall.cron /etc/cron.d/gamehall + mkdir -p /var/lib/gamehall /var/log/gamehall + rm -r /tmp/gamehall-ops ' echo "published." diff --git a/gamehall/gamehall.caddy b/gamehall/gamehall.caddy index f6dc74a..eb8d1e3 100644 --- a/gamehall/gamehall.caddy +++ b/gamehall/gamehall.caddy @@ -7,7 +7,8 @@ gamehall.kestrelsnest.social, gamehall.159.203.115.252.sslip.io { Strict-Transport-Security "max-age=31536000" X-Content-Type-Options "nosniff" X-Frame-Options "DENY" - Referrer-Policy "no-referrer" + # A card followed into a game says it came from the hall, and nothing more. + Referrer-Policy "strict-origin" # Revalidated on every load, so a deploy shows up at once. Cache-Control "no-cache" } diff --git a/gamehall/ops/gamehall-rollup.sh b/gamehall/ops/gamehall-rollup.sh new file mode 100755 index 0000000..8a8adfe --- /dev/null +++ b/gamehall/ops/gamehall-rollup.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Nightly rollup: one JSON line per day in /var/lib/gamehall/rollup.jsonl, +# the hall's counts for that day (people, views, bots, referrers, campaigns, +# and the people sent on to each game here). Counts only: no addresses are +# kept, so the trend outlives the self-rotating access log. +# gamehall-rollup.sh [YYYY-MM-DD] (default: yesterday, UTC) +# Cron: /etc/cron.d/gamehall, installed by gamehall/deploy.sh. +set -u +OUT="/var/lib/gamehall/rollup.jsonl" +LOG="/var/log/gamehall/rollup.log" +DAY="${1:-$(date -u -d 'yesterday' +%Y-%m-%d)}" +mkdir -p "$(dirname "$OUT")" "$(dirname "$LOG")" +python3 - "$DAY" "$OUT" <<'PY' >> "$LOG" 2>&1 +import datetime, json, os, sys +sys.path.insert(0, "/usr/local/lib/gamehall") +import traffic +day, out = sys.argv[1], sys.argv[2] +t = traffic.scan(day) +row = dict(day=day, people=len(t["addresses"]), views=t["views"], botAddresses=len(t["botAddresses"]), + referrers=dict(t["referrers"]), campaigns=dict(t["campaigns"]), sent=t["sent"], + at=datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds")) +rows = [json.loads(l) for l in open(out) if l.strip()] if os.path.exists(out) else [] +rows = sorted([r for r in rows if r.get("day") != day] + [row], key=lambda r: r["day"]) +with open(out + ".tmp", "w") as fh: + for r in rows: + fh.write(json.dumps(r) + "\n") +os.replace(out + ".tmp", out) +print("%s rolled: %d people, %d views" % (day, row["people"], row["views"])) +PY diff --git a/gamehall/ops/gamehall-visitors.sh b/gamehall/ops/gamehall-visitors.sh new file mode 100755 index 0000000..7735361 --- /dev/null +++ b/gamehall/ops/gamehall-visitors.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# The hall's visitors: who came in the last hour and today, where from, how +# many went on into each game here, and the last week from the rollup. +# Runs ON the shared host: gamehall-visitors.sh [YYYY-MM-DD] (default: today, UTC) +set -u +DAY="${1:-$(date -u +%F)}" +python3 - "$DAY" <<'PY' +import datetime, json, os, sys +sys.path.insert(0, "/usr/local/lib/gamehall") +import traffic + +day = sys.argv[1] +now = datetime.datetime.now(datetime.timezone.utc) +print("as of %s UTC, day %s" % (now.strftime("%H:%M"), day)) + +if day == now.strftime("%F"): + h = traffic.scan(day, since=now.timestamp() - 3600) + print("last hour: %d people, %d page views" % (len(h["addresses"]), h["views"])) + +t = traffic.scan(day) +print("today: %d people, %d page views, %d bot addresses" % (len(t["addresses"]), t["views"], len(t["botAddresses"]))) +print("referrers: %s" % (dict(t["referrers"].most_common()) or "none")) +print("campaigns: %s" % (dict(t["campaigns"].most_common()) or "none")) +print("went on to: %s (Wiz-War is on its own host; its digest shows the hall as a referrer)" % (", ".join("%s %d" % kv for kv in t["sent"].items()))) + +out = "/var/lib/gamehall/rollup.jsonl" +if os.path.exists(out): + rows = [json.loads(l) for l in open(out) if l.strip()][-7:] + if rows: + print("last %d days:" % len(rows)) + for r in rows: + sent = ", ".join("%s %d" % kv for kv in r.get("sent", {}).items()) + print(" %s %3d people %3d views refs %s on to %s" % (r["day"], r["people"], r["views"], r.get("referrers") or "-", sent or "-")) +PY diff --git a/gamehall/ops/gamehall.cron b/gamehall/ops/gamehall.cron new file mode 100644 index 0000000..e1dbe2e --- /dev/null +++ b/gamehall/ops/gamehall.cron @@ -0,0 +1,2 @@ +# Kestrel's Hall: nightly rollup of yesterday's counts. UTC. +5 0 * * * root /usr/local/bin/gamehall-rollup.sh diff --git a/gamehall/ops/traffic.py b/gamehall/ops/traffic.py new file mode 100644 index 0000000..50df817 --- /dev/null +++ b/gamehall/ops/traffic.py @@ -0,0 +1,92 @@ +"""Caddy's access logs for one UTC day: the hall's visitors, people and +bots apart, and the people each game on this host received from the hall. +The digest and the nightly rollup both count through here, so they agree.""" + +import datetime +import glob +import gzip +import json +import os +import re +from collections import Counter + +BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I) +SITE = "gamehall.kestrelsnest.social" +LOGS = "/var/lib/caddy/gamehall-access*.log*" +# The games that share this host, by the name the hall links to. A visitor +# who follows a card arrives with the hall as referrer (the hall sends its +# origin and nothing more). Wiz-War keeps its own host and its own count. +GAMES = {"dip": "dip.kestrelsnest.social", "waving-hands": "hands.kestrelsnest.social", "hnefatafl": "tafl.kestrelsnest.social"} +ASSET = (".png", ".jpg", ".svg", ".ico", ".txt", ".webmanifest") + + +def _lines(pattern, t0): + for f in sorted(glob.glob(pattern)): + if os.path.getmtime(f) < t0: + continue + opener = gzip.open if f.endswith(".gz") else open + try: + with opener(f, "rt", errors="ignore") as fh: + for line in fh: + try: + yield json.loads(line) + except ValueError: + continue + except OSError: + continue + + +def _bounds(day): + d0 = datetime.datetime.fromisoformat(day).replace(tzinfo=datetime.timezone.utc) + return d0.timestamp(), (d0 + datetime.timedelta(days=1)).timestamp() + + +def scan(day, since=None): + """Counts for one UTC day (YYYY-MM-DD), or from `since` (a timestamp) + to the end of it: page views and addresses of people and of bots, + referrers, campaign tags (once per address), and the people each game + here received from the hall.""" + t0, t1 = _bounds(day) + if since is not None: + t0 = max(t0, since) + t = dict(views=0, addresses=set(), botAddresses=set(), referrers=Counter(), campaigns=Counter(), sent={}) + seen_campaign = set() + for r in _lines(LOGS, t0): + if not (t0 <= r.get("ts", 0) < t1): + continue + q = r.get("request", {}) + if q.get("host", "").split(":")[0] != SITE: + continue + h = q.get("headers", {}) + ua = " ".join(h.get("User-Agent", [""])) + ip = q.get("client_ip") or q.get("remote_ip") or "?" + uri = q.get("uri", "") + path = uri.split("?")[0] + if BOT.search(ua) or not ua: + t["botAddresses"].add(ip) + continue + if path not in ("/", "/index.html"): + continue + t["views"] += 1 + t["addresses"].add(ip) + ref = " ".join(h.get("Referer", [""])) + if ref and SITE not in ref: + t["referrers"][re.sub(r"^https?://", "", ref).split("/")[0]] += 1 + m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri) + if m and (ip, m.group(1)) not in seen_campaign: + seen_campaign.add((ip, m.group(1))) + t["campaigns"][m.group(1) + ("=" + m.group(2)[:24] if m.group(1) != "fbclid" else "")] += 1 + for slug, host in GAMES.items(): + people = set() + for r in _lines("/var/lib/caddy/%s-access*.log*" % slug, t0): + if not (t0 <= r.get("ts", 0) < t1): + continue + q = r.get("request", {}) + h = q.get("headers", {}) + if SITE not in " ".join(h.get("Referer", [""])) or BOT.search(" ".join(h.get("User-Agent", [""]))): + continue + if q.get("uri", "").split("?")[0].endswith(ASSET): + continue + people.add(q.get("client_ip") or q.get("remote_ip") or "?") + t["sent"][slug] = len(people) + return t