Eric WagonerandClaude Opus 5.5
1dbf5763e1
gamehall/ops holds the traffic counter, the visitors digest and the nightly rollup, installed by gamehall/deploy.sh; the gamehall-visitors skill reads them. The hall now sends its origin as referrer, so a game can tell who came from it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Jm2auWk6RP71CjaAb4FMoG
93 lines
3.7 KiB
Python
93 lines
3.7 KiB
Python
"""Caddy's access logs for one UTC day: the hall's visitors, people and
|
|
bots apart, and the people each game on this host received from the hall.
|
|
The digest and the nightly rollup both count through here, so they agree."""
|
|
|
|
import datetime
|
|
import glob
|
|
import gzip
|
|
import json
|
|
import os
|
|
import re
|
|
from collections import Counter
|
|
|
|
BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I)
|
|
SITE = "gamehall.kestrelsnest.social"
|
|
LOGS = "/var/lib/caddy/gamehall-access*.log*"
|
|
# The games that share this host, by the name the hall links to. A visitor
|
|
# who follows a card arrives with the hall as referrer (the hall sends its
|
|
# origin and nothing more). Wiz-War keeps its own host and its own count.
|
|
GAMES = {"dip": "dip.kestrelsnest.social", "waving-hands": "hands.kestrelsnest.social", "hnefatafl": "tafl.kestrelsnest.social"}
|
|
ASSET = (".png", ".jpg", ".svg", ".ico", ".txt", ".webmanifest")
|
|
|
|
|
|
def _lines(pattern, t0):
|
|
for f in sorted(glob.glob(pattern)):
|
|
if os.path.getmtime(f) < t0:
|
|
continue
|
|
opener = gzip.open if f.endswith(".gz") else open
|
|
try:
|
|
with opener(f, "rt", errors="ignore") as fh:
|
|
for line in fh:
|
|
try:
|
|
yield json.loads(line)
|
|
except ValueError:
|
|
continue
|
|
except OSError:
|
|
continue
|
|
|
|
|
|
def _bounds(day):
|
|
d0 = datetime.datetime.fromisoformat(day).replace(tzinfo=datetime.timezone.utc)
|
|
return d0.timestamp(), (d0 + datetime.timedelta(days=1)).timestamp()
|
|
|
|
|
|
def scan(day, since=None):
|
|
"""Counts for one UTC day (YYYY-MM-DD), or from `since` (a timestamp)
|
|
to the end of it: page views and addresses of people and of bots,
|
|
referrers, campaign tags (once per address), and the people each game
|
|
here received from the hall."""
|
|
t0, t1 = _bounds(day)
|
|
if since is not None:
|
|
t0 = max(t0, since)
|
|
t = dict(views=0, addresses=set(), botAddresses=set(), referrers=Counter(), campaigns=Counter(), sent={})
|
|
seen_campaign = set()
|
|
for r in _lines(LOGS, t0):
|
|
if not (t0 <= r.get("ts", 0) < t1):
|
|
continue
|
|
q = r.get("request", {})
|
|
if q.get("host", "").split(":")[0] != SITE:
|
|
continue
|
|
h = q.get("headers", {})
|
|
ua = " ".join(h.get("User-Agent", [""]))
|
|
ip = q.get("client_ip") or q.get("remote_ip") or "?"
|
|
uri = q.get("uri", "")
|
|
path = uri.split("?")[0]
|
|
if BOT.search(ua) or not ua:
|
|
t["botAddresses"].add(ip)
|
|
continue
|
|
if path not in ("/", "/index.html"):
|
|
continue
|
|
t["views"] += 1
|
|
t["addresses"].add(ip)
|
|
ref = " ".join(h.get("Referer", [""]))
|
|
if ref and SITE not in ref:
|
|
t["referrers"][re.sub(r"^https?://", "", ref).split("/")[0]] += 1
|
|
m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri)
|
|
if m and (ip, m.group(1)) not in seen_campaign:
|
|
seen_campaign.add((ip, m.group(1)))
|
|
t["campaigns"][m.group(1) + ("=" + m.group(2)[:24] if m.group(1) != "fbclid" else "")] += 1
|
|
for slug, host in GAMES.items():
|
|
people = set()
|
|
for r in _lines("/var/lib/caddy/%s-access*.log*" % slug, t0):
|
|
if not (t0 <= r.get("ts", 0) < t1):
|
|
continue
|
|
q = r.get("request", {})
|
|
h = q.get("headers", {})
|
|
if SITE not in " ".join(h.get("Referer", [""])) or BOT.search(" ".join(h.get("User-Agent", [""]))):
|
|
continue
|
|
if q.get("uri", "").split("?")[0].endswith(ASSET):
|
|
continue
|
|
people.add(q.get("client_ip") or q.get("remote_ip") or "?")
|
|
t["sent"][slug] = len(people)
|
|
return t
|