The hall counts its visitors: a digest, a nightly rollup, and the games it sends people to
gamehall/ops holds the traffic counter, the visitors digest and the nightly rollup, installed by gamehall/deploy.sh; the gamehall-visitors skill reads them. The hall now sends its origin as referrer, so a game can tell who came from it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Jm2auWk6RP71CjaAb4FMoG
This commit is contained in:
Eric Wagoner
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
8ab1e0b794
commit
1dbf5763e1
Executable
+29
@@ -0,0 +1,29 @@
|
||||
#!/usr/bin/env bash
|
||||
# Nightly rollup: one JSON line per day in /var/lib/gamehall/rollup.jsonl,
|
||||
# the hall's counts for that day (people, views, bots, referrers, campaigns,
|
||||
# and the people sent on to each game here). Counts only: no addresses are
|
||||
# kept, so the trend outlives the self-rotating access log.
|
||||
# gamehall-rollup.sh [YYYY-MM-DD] (default: yesterday, UTC)
|
||||
# Cron: /etc/cron.d/gamehall, installed by gamehall/deploy.sh.
|
||||
set -u
|
||||
OUT="/var/lib/gamehall/rollup.jsonl"
|
||||
LOG="/var/log/gamehall/rollup.log"
|
||||
DAY="${1:-$(date -u -d 'yesterday' +%Y-%m-%d)}"
|
||||
mkdir -p "$(dirname "$OUT")" "$(dirname "$LOG")"
|
||||
python3 - "$DAY" "$OUT" <<'PY' >> "$LOG" 2>&1
|
||||
import datetime, json, os, sys
|
||||
sys.path.insert(0, "/usr/local/lib/gamehall")
|
||||
import traffic
|
||||
day, out = sys.argv[1], sys.argv[2]
|
||||
t = traffic.scan(day)
|
||||
row = dict(day=day, people=len(t["addresses"]), views=t["views"], botAddresses=len(t["botAddresses"]),
|
||||
referrers=dict(t["referrers"]), campaigns=dict(t["campaigns"]), sent=t["sent"],
|
||||
at=datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds"))
|
||||
rows = [json.loads(l) for l in open(out) if l.strip()] if os.path.exists(out) else []
|
||||
rows = sorted([r for r in rows if r.get("day") != day] + [row], key=lambda r: r["day"])
|
||||
with open(out + ".tmp", "w") as fh:
|
||||
for r in rows:
|
||||
fh.write(json.dumps(r) + "\n")
|
||||
os.replace(out + ".tmp", out)
|
||||
print("%s rolled: %d people, %d views" % (day, row["people"], row["views"]))
|
||||
PY
|
||||
Executable
+34
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env bash
|
||||
# The hall's visitors: who came in the last hour and today, where from, how
|
||||
# many went on into each game here, and the last week from the rollup.
|
||||
# Runs ON the shared host: gamehall-visitors.sh [YYYY-MM-DD] (default: today, UTC)
|
||||
set -u
|
||||
DAY="${1:-$(date -u +%F)}"
|
||||
python3 - "$DAY" <<'PY'
|
||||
import datetime, json, os, sys
|
||||
sys.path.insert(0, "/usr/local/lib/gamehall")
|
||||
import traffic
|
||||
|
||||
day = sys.argv[1]
|
||||
now = datetime.datetime.now(datetime.timezone.utc)
|
||||
print("as of %s UTC, day %s" % (now.strftime("%H:%M"), day))
|
||||
|
||||
if day == now.strftime("%F"):
|
||||
h = traffic.scan(day, since=now.timestamp() - 3600)
|
||||
print("last hour: %d people, %d page views" % (len(h["addresses"]), h["views"]))
|
||||
|
||||
t = traffic.scan(day)
|
||||
print("today: %d people, %d page views, %d bot addresses" % (len(t["addresses"]), t["views"], len(t["botAddresses"])))
|
||||
print("referrers: %s" % (dict(t["referrers"].most_common()) or "none"))
|
||||
print("campaigns: %s" % (dict(t["campaigns"].most_common()) or "none"))
|
||||
print("went on to: %s (Wiz-War is on its own host; its digest shows the hall as a referrer)" % (", ".join("%s %d" % kv for kv in t["sent"].items())))
|
||||
|
||||
out = "/var/lib/gamehall/rollup.jsonl"
|
||||
if os.path.exists(out):
|
||||
rows = [json.loads(l) for l in open(out) if l.strip()][-7:]
|
||||
if rows:
|
||||
print("last %d days:" % len(rows))
|
||||
for r in rows:
|
||||
sent = ", ".join("%s %d" % kv for kv in r.get("sent", {}).items())
|
||||
print(" %s %3d people %3d views refs %s on to %s" % (r["day"], r["people"], r["views"], r.get("referrers") or "-", sent or "-"))
|
||||
PY
|
||||
@@ -0,0 +1,2 @@
|
||||
# Kestrel's Hall: nightly rollup of yesterday's counts. UTC.
|
||||
5 0 * * * root /usr/local/bin/gamehall-rollup.sh
|
||||
@@ -0,0 +1,92 @@
|
||||
"""Caddy's access logs for one UTC day: the hall's visitors, people and
|
||||
bots apart, and the people each game on this host received from the hall.
|
||||
The digest and the nightly rollup both count through here, so they agree."""
|
||||
|
||||
import datetime
|
||||
import glob
|
||||
import gzip
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from collections import Counter
|
||||
|
||||
BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I)
|
||||
SITE = "gamehall.kestrelsnest.social"
|
||||
LOGS = "/var/lib/caddy/gamehall-access*.log*"
|
||||
# The games that share this host, by the name the hall links to. A visitor
|
||||
# who follows a card arrives with the hall as referrer (the hall sends its
|
||||
# origin and nothing more). Wiz-War keeps its own host and its own count.
|
||||
GAMES = {"dip": "dip.kestrelsnest.social", "waving-hands": "hands.kestrelsnest.social", "hnefatafl": "tafl.kestrelsnest.social"}
|
||||
ASSET = (".png", ".jpg", ".svg", ".ico", ".txt", ".webmanifest")
|
||||
|
||||
|
||||
def _lines(pattern, t0):
|
||||
for f in sorted(glob.glob(pattern)):
|
||||
if os.path.getmtime(f) < t0:
|
||||
continue
|
||||
opener = gzip.open if f.endswith(".gz") else open
|
||||
try:
|
||||
with opener(f, "rt", errors="ignore") as fh:
|
||||
for line in fh:
|
||||
try:
|
||||
yield json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
|
||||
def _bounds(day):
|
||||
d0 = datetime.datetime.fromisoformat(day).replace(tzinfo=datetime.timezone.utc)
|
||||
return d0.timestamp(), (d0 + datetime.timedelta(days=1)).timestamp()
|
||||
|
||||
|
||||
def scan(day, since=None):
|
||||
"""Counts for one UTC day (YYYY-MM-DD), or from `since` (a timestamp)
|
||||
to the end of it: page views and addresses of people and of bots,
|
||||
referrers, campaign tags (once per address), and the people each game
|
||||
here received from the hall."""
|
||||
t0, t1 = _bounds(day)
|
||||
if since is not None:
|
||||
t0 = max(t0, since)
|
||||
t = dict(views=0, addresses=set(), botAddresses=set(), referrers=Counter(), campaigns=Counter(), sent={})
|
||||
seen_campaign = set()
|
||||
for r in _lines(LOGS, t0):
|
||||
if not (t0 <= r.get("ts", 0) < t1):
|
||||
continue
|
||||
q = r.get("request", {})
|
||||
if q.get("host", "").split(":")[0] != SITE:
|
||||
continue
|
||||
h = q.get("headers", {})
|
||||
ua = " ".join(h.get("User-Agent", [""]))
|
||||
ip = q.get("client_ip") or q.get("remote_ip") or "?"
|
||||
uri = q.get("uri", "")
|
||||
path = uri.split("?")[0]
|
||||
if BOT.search(ua) or not ua:
|
||||
t["botAddresses"].add(ip)
|
||||
continue
|
||||
if path not in ("/", "/index.html"):
|
||||
continue
|
||||
t["views"] += 1
|
||||
t["addresses"].add(ip)
|
||||
ref = " ".join(h.get("Referer", [""]))
|
||||
if ref and SITE not in ref:
|
||||
t["referrers"][re.sub(r"^https?://", "", ref).split("/")[0]] += 1
|
||||
m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri)
|
||||
if m and (ip, m.group(1)) not in seen_campaign:
|
||||
seen_campaign.add((ip, m.group(1)))
|
||||
t["campaigns"][m.group(1) + ("=" + m.group(2)[:24] if m.group(1) != "fbclid" else "")] += 1
|
||||
for slug, host in GAMES.items():
|
||||
people = set()
|
||||
for r in _lines("/var/lib/caddy/%s-access*.log*" % slug, t0):
|
||||
if not (t0 <= r.get("ts", 0) < t1):
|
||||
continue
|
||||
q = r.get("request", {})
|
||||
h = q.get("headers", {})
|
||||
if SITE not in " ".join(h.get("Referer", [""])) or BOT.search(" ".join(h.get("User-Agent", [""]))):
|
||||
continue
|
||||
if q.get("uri", "").split("?")[0].endswith(ASSET):
|
||||
continue
|
||||
people.add(q.get("client_ip") or q.get("remote_ip") or "?")
|
||||
t["sent"][slug] = len(people)
|
||||
return t
|
||||
Reference in New Issue
Block a user