From ae904aa88caac58366d9b92f13290516bc7682da Mon Sep 17 00:00:00 2001 From: Eric Wagoner Date: Sun, 27 Sep 2026 09:59:36 -0400 Subject: [PATCH] Traffic counts what matters: rooms opened from the site, where links were shared, scanners and monitors as bots - A room opened with Play now never loads /join: the API's answer to POST /api/rooms is counted as rooms opened here. - Link previews (Slack, Discord, Facebook, iMessage, WhatsApp and the rest) are reported as where the link was shared, since the people who click them arrive with no referrer. WhatsApp's fetcher is a bot now. - An address that asks for a scanner's paths, or keeps coming back without opening a game, is a bot's all day; so is a browser that loads pages at a steady interval from a new address each time, never playing. A declared bot is still judged request by request. - The site's own addresses, sslip.io included, are not referrers. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Jm2auWk6RP71CjaAb4FMoG --- gamehall/ops/gamehall-rollup.sh | 1 + gamehall/ops/gamehall-visitors.sh | 3 +- gamehall/ops/traffic.py | 123 ++++++++++++++----- template/deploy/__SLUG__-rollup.sh | 3 +- template/deploy/__SLUG__-visitors.sh | 6 +- template/deploy/traffic.py | 169 +++++++++++++++++++++------ 6 files changed, 236 insertions(+), 69 deletions(-) diff --git a/gamehall/ops/gamehall-rollup.sh b/gamehall/ops/gamehall-rollup.sh index 8a8adfe..08f43b9 100755 --- a/gamehall/ops/gamehall-rollup.sh +++ b/gamehall/ops/gamehall-rollup.sh @@ -17,6 +17,7 @@ import traffic day, out = sys.argv[1], sys.argv[2] t = traffic.scan(day) row = dict(day=day, people=len(t["addresses"]), views=t["views"], botAddresses=len(t["botAddresses"]), + scanners=t["scanners"], monitors=t["monitors"], sharedOn=dict(t["shared"]), referrers=dict(t["referrers"]), campaigns=dict(t["campaigns"]), sent=t["sent"], at=datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds")) rows = [json.loads(l) for l in open(out) if l.strip()] if os.path.exists(out) else [] diff --git a/gamehall/ops/gamehall-visitors.sh b/gamehall/ops/gamehall-visitors.sh index 7735361..7b60081 100755 --- a/gamehall/ops/gamehall-visitors.sh +++ b/gamehall/ops/gamehall-visitors.sh @@ -18,7 +18,8 @@ if day == now.strftime("%F"): print("last hour: %d people, %d page views" % (len(h["addresses"]), h["views"])) t = traffic.scan(day) -print("today: %d people, %d page views, %d bot addresses" % (len(t["addresses"]), t["views"], len(t["botAddresses"]))) +print("today: %d people, %d page views, %d bot addresses (of them %d scanners, %d monitors)" % (len(t["addresses"]), t["views"], len(t["botAddresses"]), t["scanners"], t["monitors"])) +print("shared on: %s" % (dict(t["shared"].most_common()) or "none")) print("referrers: %s" % (dict(t["referrers"].most_common()) or "none")) print("campaigns: %s" % (dict(t["campaigns"].most_common()) or "none")) print("went on to: %s (Wiz-War is on its own host; its digest shows the hall as a referrer)" % (", ".join("%s %d" % kv for kv in t["sent"].items()))) diff --git a/gamehall/ops/traffic.py b/gamehall/ops/traffic.py index 50df817..199f7aa 100644 --- a/gamehall/ops/traffic.py +++ b/gamehall/ops/traffic.py @@ -10,7 +10,27 @@ import os import re from collections import Counter -BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I) +# The bot, link-preview and scanner patterns are kept in step with template/deploy/traffic.py. +BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|whatsapp|telegram|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I) +# Link previews: the app that fetched the page to show it in a conversation. The people +# who click such a link arrive with no referrer, so this is the best sign of where it went. +UNFURL = [ + ("Slack", re.compile(r"slack", re.I)), + ("Discord", re.compile(r"discord", re.I)), + ("iMessage", re.compile(r"facebookexternalhit.*twitterbot|applebot.*preview", re.I)), + ("Facebook", re.compile(r"facebookexternalhit|facebot|meta-externalagent", re.I)), + ("WhatsApp", re.compile(r"whatsapp", re.I)), + ("Telegram", re.compile(r"telegram", re.I)), + ("Reddit", re.compile(r"redditbot", re.I)), + ("LinkedIn", re.compile(r"linkedinbot", re.I)), + ("Mastodon", re.compile(r"mastodon", re.I)), + ("Bluesky", re.compile(r"bluesky|cardyb", re.I)), + ("X", re.compile(r"twitterbot", re.I)), +] +# Paths nobody asks the hall for but a vulnerability scanner: its address is a bot's all day. +SCANNER = re.compile(r"\.php$|(^|/)(\.env|\.git|\.aws|wp-admin|wp-login|wp-content|wp-includes|wordpress|phpmyadmin|cgi-bin|vendor|actuator|boaform|owa|admin)(/|$)", re.I) +# Requests from one address more than this far apart are separate visits. +VISIT_GAP = 30 * 60 SITE = "gamehall.kestrelsnest.social" LOGS = "/var/lib/caddy/gamehall-access*.log*" # The games that share this host, by the name the hall links to. A visitor @@ -20,6 +40,23 @@ GAMES = {"dip": "dip.kestrelsnest.social", "waving-hands": "hands.kestrelsnest.s ASSET = (".png", ".jpg", ".svg", ".ico", ".txt", ".webmanifest") +def unfurled_by(ua): + """The app a link preview came from, or None.""" + for name, pattern in UNFURL: + if pattern.search(ua): + return name + return None + + +def _visits(times): + n, last = 0, None + for ts in sorted(times): + if last is None or ts - last > VISIT_GAP: + n += 1 + last = ts + return n + + def _lines(pattern, t0): for f in sorted(glob.glob(pattern)): if os.path.getmtime(f) < t0: @@ -43,39 +80,18 @@ def _bounds(day): def scan(day, since=None): """Counts for one UTC day (YYYY-MM-DD), or from `since` (a timestamp) - to the end of it: page views and addresses of people and of bots, - referrers, campaign tags (once per address), and the people each game - here received from the hall.""" + to the end of it: page views and addresses of people and of bots, where + links to the hall were shared, referrers, campaign tags (once per + address), and the people each game here received from the hall. + + A request is a bot's when its browser says so. An address is a bot's for + the day when it asks for a scanner's paths, or loads the hall three + visits or more without ever following a card to a game: a monitor.""" t0, t1 = _bounds(day) if since is not None: t0 = max(t0, since) - t = dict(views=0, addresses=set(), botAddresses=set(), referrers=Counter(), campaigns=Counter(), sent={}) - seen_campaign = set() - for r in _lines(LOGS, t0): - if not (t0 <= r.get("ts", 0) < t1): - continue - q = r.get("request", {}) - if q.get("host", "").split(":")[0] != SITE: - continue - h = q.get("headers", {}) - ua = " ".join(h.get("User-Agent", [""])) - ip = q.get("client_ip") or q.get("remote_ip") or "?" - uri = q.get("uri", "") - path = uri.split("?")[0] - if BOT.search(ua) or not ua: - t["botAddresses"].add(ip) - continue - if path not in ("/", "/index.html"): - continue - t["views"] += 1 - t["addresses"].add(ip) - ref = " ".join(h.get("Referer", [""])) - if ref and SITE not in ref: - t["referrers"][re.sub(r"^https?://", "", ref).split("/")[0]] += 1 - m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri) - if m and (ip, m.group(1)) not in seen_campaign: - seen_campaign.add((ip, m.group(1))) - t["campaigns"][m.group(1) + ("=" + m.group(2)[:24] if m.group(1) != "fbclid" else "")] += 1 + t = dict(views=0, addresses=set(), botAddresses=set(), shared=Counter(), scanners=0, monitors=0, referrers=Counter(), campaigns=Counter(), sent={}) + followed = set() for slug, host in GAMES.items(): people = set() for r in _lines("/var/lib/caddy/%s-access*.log*" % slug, t0): @@ -89,4 +105,49 @@ def scan(day, since=None): continue people.add(q.get("client_ip") or q.get("remote_ip") or "?") t["sent"][slug] = len(people) + followed |= people + + hall = [r for r in _lines(LOGS, t0) if t0 <= r.get("ts", 0) < t1 and r.get("request", {}).get("host", "").split(":")[0] == SITE] + times, scanning = {}, set() + for r in hall: + q = r.get("request", {}) + ua = " ".join(q.get("headers", {}).get("User-Agent", [""])) + if BOT.search(ua) or not ua: + continue + ip = q.get("client_ip") or q.get("remote_ip") or "?" + times.setdefault(ip, []).append(r.get("ts", 0)) + if SCANNER.search(q.get("uri", "").split("?")[0]): + scanning.add(ip) + monitors = {ip for ip, ts in times.items() if ip not in scanning and ip not in followed and _visits(ts) >= 3} + posing = scanning | monitors + t["scanners"], t["monitors"] = len(scanning), len(monitors) + + seen_campaign = set() + seen_share = set() + for r in hall: + q = r.get("request", {}) + h = q.get("headers", {}) + ua = " ".join(h.get("User-Agent", [""])) + ip = q.get("client_ip") or q.get("remote_ip") or "?" + uri = q.get("uri", "") + path = uri.split("?")[0] + if ip in posing or BOT.search(ua) or not ua: + t["botAddresses"].add(ip) + service = unfurled_by(ua) + # One share per app and page: a preview is often fetched twice. + if service and (service, path) not in seen_share: + seen_share.add((service, path)) + t["shared"][service] += 1 + continue + if path not in ("/", "/index.html"): + continue + t["views"] += 1 + t["addresses"].add(ip) + ref = " ".join(h.get("Referer", [""])) + if ref and SITE not in ref: + t["referrers"][re.sub(r"^https?://", "", ref).split("/")[0]] += 1 + m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri) + if m and (ip, m.group(1)) not in seen_campaign: + seen_campaign.add((ip, m.group(1))) + t["campaigns"][m.group(1) + ("=" + m.group(2)[:24] if m.group(1) != "fbclid" else "")] += 1 return t diff --git a/template/deploy/__SLUG__-rollup.sh b/template/deploy/__SLUG__-rollup.sh index a028525..123c601 100755 --- a/template/deploy/__SLUG__-rollup.sh +++ b/template/deploy/__SLUG__-rollup.sh @@ -58,7 +58,8 @@ for line in open("/proc/meminfo"): k, v = line.split(":", 1); mem[k] = int(v.split()[0]) load = os.getloadavg()[1] row = dict(day=day, requests=t["requests"], human=t["human"], bot=t["bot"], humanAddresses=len(t["addresses"]), botAddresses=len(t["botAddresses"]), - paths=dict(t["pages"]), roomLinks=sum(t["joins"].values()), referrers=dict(t["referrers"]), campaigns=dict(t["campaigns"]), + paths=dict(t["pages"]), roomLinks=sum(t["joins"].values()), roomsOpened=t["opened"], sharedOn=dict(t["shared"]), + scanners=t["scanners"], monitors=t["monitors"], referrers=dict(t["referrers"]), campaigns=dict(t["campaigns"]), rooms=dict(created=created, started=started, withTurns=turned, finished=finished, humanNames=len(humans), botSeats=bots), turns=turns, ledgers=len(glob.glob("/var/lib/__SLUG__/rooms/*.jsonl")), vitals=dict(diskUsedPct=round(disk.used * 100 / disk.total), memAvailableMb=mem.get("MemAvailable", 0) // 1024, load5=round(load, 2)), diff --git a/template/deploy/__SLUG__-visitors.sh b/template/deploy/__SLUG__-visitors.sh index 6861099..957a755 100755 --- a/template/deploy/__SLUG__-visitors.sh +++ b/template/deploy/__SLUG__-visitors.sh @@ -26,9 +26,9 @@ print("live sockets: %s | rooms touched in the last hour: %s" % ( # --- traffic: Caddy's access log, people and bots apart --- t = traffic.scan(day) -print("traffic: %d addresses of people, %d page requests (%d bot addresses); hall %d, rules %d, guide %d, room links %s; referrers %s; campaigns %s" % ( - len(t["addresses"]), t["human"], len(t["botAddresses"]), t["pages"]["hall"], t["pages"]["rules"], t["pages"]["guide"], - dict(t["joins"]) or "none", dict(t["referrers"]) or "none", dict(t["campaigns"]) or "none")) +print("traffic: %d addresses of people, %d page requests (%d bot addresses, of them %d scanners, %d monitors); hall %d, rules %d, guide %d; rooms opened here %d, room links %s; shared on %s; referrers %s; campaigns %s" % ( + len(t["addresses"]), t["human"], len(t["botAddresses"]), t["scanners"], t["monitors"], t["pages"]["hall"], t["pages"]["rules"], t["pages"]["guide"], + t["opened"], dict(t["joins"]) or "none", dict(t["shared"]) or "none", dict(t["referrers"]) or "none", dict(t["campaigns"]) or "none")) # --- rooms: every ledger, today's in detail --- first_seen = {} diff --git a/template/deploy/traffic.py b/template/deploy/traffic.py index f4eb073..2e288c0 100644 --- a/template/deploy/traffic.py +++ b/template/deploy/traffic.py @@ -7,23 +7,60 @@ import gzip import json import os import re -from collections import Counter +from collections import Counter, defaultdict -BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I) +BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|whatsapp|telegram|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I) +# Link previews: the app that fetched the page to show it in a conversation. The people +# who click such a link arrive with no referrer, so this is the best sign of where it went. +UNFURL = [ + ("Slack", re.compile(r"slack", re.I)), + ("Discord", re.compile(r"discord", re.I)), + ("iMessage", re.compile(r"facebookexternalhit.*twitterbot|applebot.*preview", re.I)), + ("Facebook", re.compile(r"facebookexternalhit|facebot|meta-externalagent", re.I)), + ("WhatsApp", re.compile(r"whatsapp", re.I)), + ("Telegram", re.compile(r"telegram", re.I)), + ("Reddit", re.compile(r"redditbot", re.I)), + ("LinkedIn", re.compile(r"linkedinbot", re.I)), + ("Mastodon", re.compile(r"mastodon", re.I)), + ("Bluesky", re.compile(r"bluesky|cardyb", re.I)), + ("X", re.compile(r"twitterbot", re.I)), +] +# Paths nobody asks a game for but a vulnerability scanner: its address is a bot's all day. +# Matched against whole path segments, never the site's own hashed assets under /_app/. +SCANNER = re.compile(r"\.php$|(^|/)(\.env|\.git|\.aws|wp-admin|wp-login|wp-content|wp-includes|wordpress|phpmyadmin|cgi-bin|vendor|actuator|boaform|owa|admin)(/|$)", re.I) PAGES = {"/": "hall", "/rules": "rules", "/guide": "guide"} SITE = "__DOMAIN__" # Each game on a shared host keeps its own access log. LOGS = "/var/lib/caddy/__SLUG__-access*.log*" +# The site's own addresses, for telling a referrer from a move between its pages: its domain, and its sslip.io name. +OWN = re.compile(re.escape(SITE) + r"|\.sslip\.io\b", re.I) +# Requests from one address more than this far apart are separate visits. +VISIT_GAP = 30 * 60 + + +def unfurled_by(ua): + """The app a link preview came from, or None.""" + for name, pattern in UNFURL: + if pattern.search(ua): + return name + return None def scan(day, logs=LOGS): """Counts for one UTC day (YYYY-MM-DD): every request, page requests by people, the addresses of people and of bots, requests by page, room links - opened, referrers, and campaign tags (each counted once per address).""" + opened, rooms opened from the site itself, where links were shared, + referrers, and campaign tags (each counted once per address). + + A request is a bot's when its browser says so. An address is a bot's for + the whole day when it asks for a scanner's paths, or comes back for pages + three visits or more without ever opening or joining a game: a monitor, + not a person. A monitor that takes a new address every visit gives itself + away by keeping time: one browser, page loads at a steady interval, four + visits or more, never a game.""" d0 = datetime.datetime.fromisoformat(day).replace(tzinfo=datetime.timezone.utc) t0, t1 = d0.timestamp(), (d0 + datetime.timedelta(days=1)).timestamp() - t = dict(requests=0, human=0, bot=0, addresses=set(), botAddresses=set(), pages=Counter(), joins=Counter(), referrers=Counter(), campaigns=Counter()) - seen_campaign = set() + requests = [] for f in sorted(glob.glob(logs)): if os.path.getmtime(f) < t0: continue @@ -35,34 +72,100 @@ def scan(day, logs=LOGS): r = json.loads(line) except ValueError: continue - if not (t0 <= r.get("ts", 0) < t1): - continue - q = r.get("request", {}) - h = q.get("headers", {}) - ua = " ".join(h.get("User-Agent", [""])) - ip = q.get("client_ip") or q.get("remote_ip") or "?" - uri = q.get("uri", "") - path = uri.split("?")[0] - t["requests"] += 1 - if BOT.search(ua) or not ua: - t["bot"] += 1 - t["botAddresses"].add(ip) - continue - if path.startswith(("/_app/", "/api/")) or path == "/ws" or path.endswith((".png", ".ico", ".txt")): - continue - t["human"] += 1 - t["addresses"].add(ip) - if path in PAGES: - t["pages"][PAGES[path]] += 1 - elif path.startswith("/join/"): - t["joins"][path.split("/")[2].upper()] += 1 - ref = " ".join(h.get("Referer", [""])) - if ref and SITE not in ref: - t["referrers"][re.sub(r"^https?://", "", ref).split("/")[0]] += 1 - m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri) - if m and (ip, m.group(1)) not in seen_campaign: - seen_campaign.add((ip, m.group(1))) - t["campaigns"][m.group(1) + ("=" + m.group(2)[:24] if m.group(1) != "fbclid" else "")] += 1 + if t0 <= r.get("ts", 0) < t1: + requests.append(r) except OSError: continue + + # First pass: what each address did all day under a browser's name, to find scanners and monitors. + seen = defaultdict(lambda: dict(times=[], played=False, scanner=False)) + for r in requests: + q = r.get("request", {}) + ip = q.get("client_ip") or q.get("remote_ip") or "?" + ua = " ".join(q.get("headers", {}).get("User-Agent", [""])) + if BOT.search(ua) or not ua: + continue + path = q.get("uri", "").split("?")[0] + a = seen[ip] + a["times"].append(r.get("ts", 0)) + if not path.startswith("/_app/") and SCANNER.search(path): + a["scanner"] = True + if path.startswith(("/join/", "/api/rooms")) or path == "/ws": + a["played"] = True + + def visits(times): + n, last = 0, None + for ts in sorted(times): + if last is None or ts - last > VISIT_GAP: + n += 1 + last = ts + return n + + def keeps_time(times): + starts, last = [], None + for ts in sorted(times): + if last is None or ts - last > VISIT_GAP: + starts.append(ts) + last = ts + gaps = [b - a for a, b in zip(starts, starts[1:])] + if len(starts) < 4: + return False + typical = sorted(gaps)[len(gaps) // 2] + return all(0.75 * typical <= g <= 1.25 * typical for g in gaps) + + scanners = {ip for ip, a in seen.items() if a["scanner"]} + monitors = {ip for ip, a in seen.items() if not (a["scanner"] or a["played"]) and visits(a["times"]) >= 3} + by_browser = defaultdict(list) + for r in requests: + q = r.get("request", {}) + ip = q.get("client_ip") or q.get("remote_ip") or "?" + ua = " ".join(q.get("headers", {}).get("User-Agent", [""])) + if ua and not BOT.search(ua) and q.get("uri", "").split("?")[0] in PAGES and not seen[ip]["played"]: + by_browser[ua].append((r.get("ts", 0), ip)) + for hits in by_browser.values(): + if keeps_time([ts for ts, _ in hits]): + monitors |= {ip for _, ip in hits} + posing = scanners | monitors + + t = dict( + requests=0, human=0, bot=0, addresses=set(), botAddresses=set(), pages=Counter(), joins=Counter(), + opened=0, shared=Counter(), scanners=len(scanners), monitors=len(monitors), referrers=Counter(), campaigns=Counter(), + ) + seen_campaign = set() + seen_share = set() + for r in requests: + q = r.get("request", {}) + h = q.get("headers", {}) + ua = " ".join(h.get("User-Agent", [""])) + ip = q.get("client_ip") or q.get("remote_ip") or "?" + uri = q.get("uri", "") + path = uri.split("?")[0] + t["requests"] += 1 + if ip in posing or BOT.search(ua) or not ua: + t["bot"] += 1 + t["botAddresses"].add(ip) + service = unfurled_by(ua) + # One share per app and page: a preview is often fetched twice. + if service and (service, path) not in seen_share: + seen_share.add((service, path)) + t["shared"][service] += 1 + continue + # A room opened from the site itself never passes through /join: count the API's answer. + if q.get("method") == "POST" and path == "/api/rooms" and 200 <= r.get("status", 0) < 300: + t["opened"] += 1 + if path.startswith(("/_app/", "/api/")) or path == "/ws" or path.endswith((".png", ".ico", ".txt")): + continue + t["human"] += 1 + t["addresses"].add(ip) + if path in PAGES: + t["pages"][PAGES[path]] += 1 + elif path.startswith("/join/"): + t["joins"][path.split("/")[2].upper()] += 1 + ref = " ".join(h.get("Referer", [""])) + if ref and not OWN.search(ref): + t["referrers"][re.sub(r"^https?://", "", ref).split("/")[0]] += 1 + m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri) + if m and (ip, m.group(1)) not in seen_campaign: + seen_campaign.add((ip, m.group(1))) + t["campaigns"][m.group(1) + ("=" + m.group(2)[:24] if m.group(1) != "fbclid" else "")] += 1 return t