Traffic counts what matters: rooms opened from the site, where links were shared, scanners and monitors as bots

- A room opened with Play now never loads /join: the API's answer to POST /api/rooms is counted as rooms opened here. - Link previews (Slack, Discord, Facebook, iMessage, WhatsApp and the rest) are reported as where the link was shared, since the people who click them arrive with no referrer. WhatsApp's fetcher is a bot now. - An address that asks for a scanner's paths, or keeps coming back without opening a game, is a bot's all day; so is a browser that loads pages at a steady interval from a new address each time, never playing. A declared bot is still judged request by request. - The site's own addresses, sslip.io included, are not referrers. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Jm2auWk6RP71CjaAb4FMoG
2026-09-27 09:59:36 -04:00
parent 5f11bec96d
commit ae904aa88c
6 changed files with 236 additions and 69 deletions
@@ -10,7 +10,27 @@ import os
 import re
 from collections import Counter

-BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I)
+# The bot, link-preview and scanner patterns are kept in step with template/deploy/traffic.py.
+BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|whatsapp|telegram|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I)
+# Link previews: the app that fetched the page to show it in a conversation. The people
+# who click such a link arrive with no referrer, so this is the best sign of where it went.
+UNFURL = [
+    ("Slack", re.compile(r"slack", re.I)),
+    ("Discord", re.compile(r"discord", re.I)),
+    ("iMessage", re.compile(r"facebookexternalhit.*twitterbot|applebot.*preview", re.I)),
+    ("Facebook", re.compile(r"facebookexternalhit|facebot|meta-externalagent", re.I)),
+    ("WhatsApp", re.compile(r"whatsapp", re.I)),
+    ("Telegram", re.compile(r"telegram", re.I)),
+    ("Reddit", re.compile(r"redditbot", re.I)),
+    ("LinkedIn", re.compile(r"linkedinbot", re.I)),
+    ("Mastodon", re.compile(r"mastodon", re.I)),
+    ("Bluesky", re.compile(r"bluesky|cardyb", re.I)),
+    ("X", re.compile(r"twitterbot", re.I)),
+]
+# Paths nobody asks the hall for but a vulnerability scanner: its address is a bot's all day.
+SCANNER = re.compile(r"\.php$|(^|/)(\.env|\.git|\.aws|wp-admin|wp-login|wp-content|wp-includes|wordpress|phpmyadmin|cgi-bin|vendor|actuator|boaform|owa|admin)(/|$)", re.I)
+# Requests from one address more than this far apart are separate visits.
+VISIT_GAP = 30 * 60
 SITE = "gamehall.kestrelsnest.social"
 LOGS = "/var/lib/caddy/gamehall-access*.log*"
 # The games that share this host, by the name the hall links to. A visitor
@@ -20,6 +40,23 @@ GAMES = {"dip": "dip.kestrelsnest.social", "waving-hands": "hands.kestrelsnest.s
 ASSET = (".png", ".jpg", ".svg", ".ico", ".txt", ".webmanifest")


+def unfurled_by(ua):
+    """The app a link preview came from, or None."""
+    for name, pattern in UNFURL:
+        if pattern.search(ua):
+            return name
+    return None
+
+
+def _visits(times):
+    n, last = 0, None
+    for ts in sorted(times):
+        if last is None or ts - last > VISIT_GAP:
+            n += 1
+        last = ts
+    return n
+
+
 def _lines(pattern, t0):
    for f in sorted(glob.glob(pattern)):
        if os.path.getmtime(f) < t0:
@@ -43,39 +80,18 @@ def _bounds(day):

 def scan(day, since=None):
    """Counts for one UTC day (YYYY-MM-DD), or from `since` (a timestamp)
-    to the end of it: page views and addresses of people and of bots,
-    referrers, campaign tags (once per address), and the people each game
-    here received from the hall."""
+    to the end of it: page views and addresses of people and of bots, where
+    links to the hall were shared, referrers, campaign tags (once per
+    address), and the people each game here received from the hall.
+
+    A request is a bot's when its browser says so. An address is a bot's for
+    the day when it asks for a scanner's paths, or loads the hall three
+    visits or more without ever following a card to a game: a monitor."""
    t0, t1 = _bounds(day)
    if since is not None:
        t0 = max(t0, since)
-    t = dict(views=0, addresses=set(), botAddresses=set(), referrers=Counter(), campaigns=Counter(), sent={})
-    seen_campaign = set()
-    for r in _lines(LOGS, t0):
-        if not (t0 <= r.get("ts", 0) < t1):
-            continue
-        q = r.get("request", {})
-        if q.get("host", "").split(":")[0] != SITE:
-            continue
-        h = q.get("headers", {})
-        ua = " ".join(h.get("User-Agent", [""]))
-        ip = q.get("client_ip") or q.get("remote_ip") or "?"
-        uri = q.get("uri", "")
-        path = uri.split("?")[0]
-        if BOT.search(ua) or not ua:
-            t["botAddresses"].add(ip)
-            continue
-        if path not in ("/", "/index.html"):
-            continue
-        t["views"] += 1
-        t["addresses"].add(ip)
-        ref = " ".join(h.get("Referer", [""]))
-        if ref and SITE not in ref:
-            t["referrers"][re.sub(r"^https?://", "", ref).split("/")[0]] += 1
-        m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri)
-        if m and (ip, m.group(1)) not in seen_campaign:
-            seen_campaign.add((ip, m.group(1)))
-            t["campaigns"][m.group(1) + ("=" + m.group(2)[:24] if m.group(1) != "fbclid" else "")] += 1
+    t = dict(views=0, addresses=set(), botAddresses=set(), shared=Counter(), scanners=0, monitors=0, referrers=Counter(), campaigns=Counter(), sent={})
+    followed = set()
    for slug, host in GAMES.items():
        people = set()
        for r in _lines("/var/lib/caddy/%s-access*.log*" % slug, t0):
@@ -89,4 +105,49 @@ def scan(day, since=None):
                continue
            people.add(q.get("client_ip") or q.get("remote_ip") or "?")
        t["sent"][slug] = len(people)
+        followed |= people
+
+    hall = [r for r in _lines(LOGS, t0) if t0 <= r.get("ts", 0) < t1 and r.get("request", {}).get("host", "").split(":")[0] == SITE]
+    times, scanning = {}, set()
+    for r in hall:
+        q = r.get("request", {})
+        ua = " ".join(q.get("headers", {}).get("User-Agent", [""]))
+        if BOT.search(ua) or not ua:
+            continue
+        ip = q.get("client_ip") or q.get("remote_ip") or "?"
+        times.setdefault(ip, []).append(r.get("ts", 0))
+        if SCANNER.search(q.get("uri", "").split("?")[0]):
+            scanning.add(ip)
+    monitors = {ip for ip, ts in times.items() if ip not in scanning and ip not in followed and _visits(ts) >= 3}
+    posing = scanning | monitors
+    t["scanners"], t["monitors"] = len(scanning), len(monitors)
+
+    seen_campaign = set()
+    seen_share = set()
+    for r in hall:
+        q = r.get("request", {})
+        h = q.get("headers", {})
+        ua = " ".join(h.get("User-Agent", [""]))
+        ip = q.get("client_ip") or q.get("remote_ip") or "?"
+        uri = q.get("uri", "")
+        path = uri.split("?")[0]
+        if ip in posing or BOT.search(ua) or not ua:
+            t["botAddresses"].add(ip)
+            service = unfurled_by(ua)
+            # One share per app and page: a preview is often fetched twice.
+            if service and (service, path) not in seen_share:
+                seen_share.add((service, path))
+                t["shared"][service] += 1
+            continue
+        if path not in ("/", "/index.html"):
+            continue
+        t["views"] += 1
+        t["addresses"].add(ip)
+        ref = " ".join(h.get("Referer", [""]))
+        if ref and SITE not in ref:
+            t["referrers"][re.sub(r"^https?://", "", ref).split("/")[0]] += 1
+        m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri)
+        if m and (ip, m.group(1)) not in seen_campaign:
+            seen_campaign.add((ip, m.group(1)))
+            t["campaigns"][m.group(1) + ("=" + m.group(2)[:24] if m.group(1) != "fbclid" else "")] += 1
    return t