Eric WagonerandClaude Opus 5.5
010f99aa9c
The new kit setup, its own Caddy site file and access log, and the host's address everywhere. The old droplet forwards to hall until the A record moves. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Jm2auWk6RP71CjaAb4FMoG
69 lines
3.2 KiB
Python
69 lines
3.2 KiB
Python
"""Caddy's access log for one day, people and bots apart. The visitors
|
|
digest and the nightly rollup both count through here, so they agree."""
|
|
|
|
import datetime
|
|
import glob
|
|
import gzip
|
|
import json
|
|
import os
|
|
import re
|
|
from collections import Counter
|
|
|
|
BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I)
|
|
PAGES = {"/": "hall", "/rules": "rules", "/guide": "guide"}
|
|
SITE = "tafl.kestrelsnest.social"
|
|
# Each game on a shared host keeps its own access log.
|
|
LOGS = "/var/lib/caddy/hnefatafl-access*.log*"
|
|
|
|
|
|
def scan(day, logs=LOGS):
|
|
"""Counts for one UTC day (YYYY-MM-DD): every request, page requests by
|
|
people, the addresses of people and of bots, requests by page, room links
|
|
opened, referrers, and campaign tags (each counted once per address)."""
|
|
d0 = datetime.datetime.fromisoformat(day).replace(tzinfo=datetime.timezone.utc)
|
|
t0, t1 = d0.timestamp(), (d0 + datetime.timedelta(days=1)).timestamp()
|
|
t = dict(requests=0, human=0, bot=0, addresses=set(), botAddresses=set(), pages=Counter(), joins=Counter(), referrers=Counter(), campaigns=Counter())
|
|
seen_campaign = set()
|
|
for f in sorted(glob.glob(logs)):
|
|
if os.path.getmtime(f) < t0:
|
|
continue
|
|
opener = gzip.open if f.endswith(".gz") else open
|
|
try:
|
|
with opener(f, "rt", errors="ignore") as fh:
|
|
for line in fh:
|
|
try:
|
|
r = json.loads(line)
|
|
except ValueError:
|
|
continue
|
|
if not (t0 <= r.get("ts", 0) < t1):
|
|
continue
|
|
q = r.get("request", {})
|
|
h = q.get("headers", {})
|
|
ua = " ".join(h.get("User-Agent", [""]))
|
|
ip = q.get("client_ip") or q.get("remote_ip") or "?"
|
|
uri = q.get("uri", "")
|
|
path = uri.split("?")[0]
|
|
t["requests"] += 1
|
|
if BOT.search(ua) or not ua:
|
|
t["bot"] += 1
|
|
t["botAddresses"].add(ip)
|
|
continue
|
|
if path.startswith(("/_app/", "/api/")) or path == "/ws" or path.endswith((".png", ".ico", ".txt")):
|
|
continue
|
|
t["human"] += 1
|
|
t["addresses"].add(ip)
|
|
if path in PAGES:
|
|
t["pages"][PAGES[path]] += 1
|
|
elif path.startswith("/join/"):
|
|
t["joins"][path.split("/")[2].upper()] += 1
|
|
ref = " ".join(h.get("Referer", [""]))
|
|
if ref and SITE not in ref:
|
|
t["referrers"][re.sub(r"^https?://", "", ref).split("/")[0]] += 1
|
|
m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri)
|
|
if m and (ip, m.group(1)) not in seen_campaign:
|
|
seen_campaign.add((ip, m.group(1)))
|
|
t["campaigns"][m.group(1) + ("=" + m.group(2)[:24] if m.group(1) != "fbclid" else "")] += 1
|
|
except OSError:
|
|
continue
|
|
return t
|