Traffic counts what matters: rooms opened from the site, where links were shared, scanners and monitors as bots

- A room opened with Play now never loads /join: the API's answer to
  POST /api/rooms is counted as rooms opened here.
- Link previews (Slack, Discord, Facebook, iMessage, WhatsApp and the
  rest) are reported as where the link was shared, since the people who
  click them arrive with no referrer. WhatsApp's fetcher is a bot now.
- An address that asks for a scanner's paths, or keeps coming back
  without opening a game, is a bot's all day; so is a browser that loads
  pages at a steady interval from a new address each time, never
  playing. A declared bot is still judged request by request.
- The site's own addresses, sslip.io included, are not referrers.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Jm2auWk6RP71CjaAb4FMoG

gamehall/ops/gamehall-rollup.sh

@@ -17,6 +17,7 @@ import traffic
 day, out = sys.argv[1], sys.argv[2]
 t = traffic.scan(day)
 row = dict(day=day, people=len(t["addresses"]), views=t["views"], botAddresses=len(t["botAddresses"]),
+           scanners=t["scanners"], monitors=t["monitors"], sharedOn=dict(t["shared"]),
            referrers=dict(t["referrers"]), campaigns=dict(t["campaigns"]), sent=t["sent"],
            at=datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds"))
 rows = [json.loads(l) for l in open(out) if l.strip()] if os.path.exists(out) else []

gamehall/ops/gamehall-visitors.sh

@@ -18,7 +18,8 @@ if day == now.strftime("%F"):
     print("last hour: %d people, %d page views" % (len(h["addresses"]), h["views"]))
 
 t = traffic.scan(day)
-print("today: %d people, %d page views, %d bot addresses" % (len(t["addresses"]), t["views"], len(t["botAddresses"])))
+print("today: %d people, %d page views, %d bot addresses (of them %d scanners, %d monitors)" % (len(t["addresses"]), t["views"], len(t["botAddresses"]), t["scanners"], t["monitors"]))
+print("shared on: %s" % (dict(t["shared"].most_common()) or "none"))
 print("referrers: %s" % (dict(t["referrers"].most_common()) or "none"))
 print("campaigns: %s" % (dict(t["campaigns"].most_common()) or "none"))
 print("went on to: %s (Wiz-War is on its own host; its digest shows the hall as a referrer)" % (", ".join("%s %d" % kv for kv in t["sent"].items())))

gamehall/ops/traffic.py

@@ -10,7 +10,27 @@ import os
 import re
 from collections import Counter
 
-BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I)
+# The bot, link-preview and scanner patterns are kept in step with template/deploy/traffic.py.
+BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|whatsapp|telegram|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I)
+# Link previews: the app that fetched the page to show it in a conversation. The people
+# who click such a link arrive with no referrer, so this is the best sign of where it went.
+UNFURL = [
+    ("Slack", re.compile(r"slack", re.I)),
+    ("Discord", re.compile(r"discord", re.I)),
+    ("iMessage", re.compile(r"facebookexternalhit.*twitterbot|applebot.*preview", re.I)),
+    ("Facebook", re.compile(r"facebookexternalhit|facebot|meta-externalagent", re.I)),
+    ("WhatsApp", re.compile(r"whatsapp", re.I)),
+    ("Telegram", re.compile(r"telegram", re.I)),
+    ("Reddit", re.compile(r"redditbot", re.I)),
+    ("LinkedIn", re.compile(r"linkedinbot", re.I)),
+    ("Mastodon", re.compile(r"mastodon", re.I)),
+    ("Bluesky", re.compile(r"bluesky|cardyb", re.I)),
+    ("X", re.compile(r"twitterbot", re.I)),
+]
+# Paths nobody asks the hall for but a vulnerability scanner: its address is a bot's all day.
+SCANNER = re.compile(r"\.php$|(^|/)(\.env|\.git|\.aws|wp-admin|wp-login|wp-content|wp-includes|wordpress|phpmyadmin|cgi-bin|vendor|actuator|boaform|owa|admin)(/|$)", re.I)
+# Requests from one address more than this far apart are separate visits.
+VISIT_GAP = 30 * 60
 SITE = "gamehall.kestrelsnest.social"
 LOGS = "/var/lib/caddy/gamehall-access*.log*"
 # The games that share this host, by the name the hall links to. A visitor
@@ -20,6 +40,23 @@ GAMES = {"dip": "dip.kestrelsnest.social", "waving-hands": "hands.kestrelsnest.s
 ASSET = (".png", ".jpg", ".svg", ".ico", ".txt", ".webmanifest")
 
 
+def unfurled_by(ua):
+    """The app a link preview came from, or None."""
+    for name, pattern in UNFURL:
+        if pattern.search(ua):
+            return name
+    return None
+
+
+def _visits(times):
+    n, last = 0, None
+    for ts in sorted(times):
+        if last is None or ts - last > VISIT_GAP:
+            n += 1
+        last = ts
+    return n
+
+
 def _lines(pattern, t0):
     for f in sorted(glob.glob(pattern)):
         if os.path.getmtime(f) < t0:
@@ -43,39 +80,18 @@ def _bounds(day):
 
 def scan(day, since=None):
     """Counts for one UTC day (YYYY-MM-DD), or from `since` (a timestamp)
-    to the end of it: page views and addresses of people and of bots,
+    to the end of it: page views and addresses of people and of bots, where
-    referrers, campaign tags (once per address), and the people each game
+    links to the hall were shared, referrers, campaign tags (once per
-    here received from the hall."""
+    address), and the people each game here received from the hall.
+
+    A request is a bot's when its browser says so. An address is a bot's for
+    the day when it asks for a scanner's paths, or loads the hall three
+    visits or more without ever following a card to a game: a monitor."""
     t0, t1 = _bounds(day)
     if since is not None:
         t0 = max(t0, since)
-    t = dict(views=0, addresses=set(), botAddresses=set(), referrers=Counter(), campaigns=Counter(), sent={})
+    t = dict(views=0, addresses=set(), botAddresses=set(), shared=Counter(), scanners=0, monitors=0, referrers=Counter(), campaigns=Counter(), sent={})
-    seen_campaign = set()
+    followed = set()
-    for r in _lines(LOGS, t0):
-        if not (t0 <= r.get("ts", 0) < t1):
-            continue
-        q = r.get("request", {})
-        if q.get("host", "").split(":")[0] != SITE:
-            continue
-        h = q.get("headers", {})
-        ua = " ".join(h.get("User-Agent", [""]))
-        ip = q.get("client_ip") or q.get("remote_ip") or "?"
-        uri = q.get("uri", "")
-        path = uri.split("?")[0]
-        if BOT.search(ua) or not ua:
-            t["botAddresses"].add(ip)
-            continue
-        if path not in ("/", "/index.html"):
-            continue
-        t["views"] += 1
-        t["addresses"].add(ip)
-        ref = " ".join(h.get("Referer", [""]))
-        if ref and SITE not in ref:
-            t["referrers"][re.sub(r"^https?://", "", ref).split("/")[0]] += 1
-        m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri)
-        if m and (ip, m.group(1)) not in seen_campaign:
-            seen_campaign.add((ip, m.group(1)))
-            t["campaigns"][m.group(1) + ("=" + m.group(2)[:24] if m.group(1) != "fbclid" else "")] += 1
     for slug, host in GAMES.items():
         people = set()
         for r in _lines("/var/lib/caddy/%s-access*.log*" % slug, t0):
@@ -89,4 +105,49 @@ def scan(day, since=None):
                 continue
             people.add(q.get("client_ip") or q.get("remote_ip") or "?")
         t["sent"][slug] = len(people)
+        followed |= people
+
+    hall = [r for r in _lines(LOGS, t0) if t0 <= r.get("ts", 0) < t1 and r.get("request", {}).get("host", "").split(":")[0] == SITE]
+    times, scanning = {}, set()
+    for r in hall:
+        q = r.get("request", {})
+        ua = " ".join(q.get("headers", {}).get("User-Agent", [""]))
+        if BOT.search(ua) or not ua:
+            continue
+        ip = q.get("client_ip") or q.get("remote_ip") or "?"
+        times.setdefault(ip, []).append(r.get("ts", 0))
+        if SCANNER.search(q.get("uri", "").split("?")[0]):
+            scanning.add(ip)
+    monitors = {ip for ip, ts in times.items() if ip not in scanning and ip not in followed and _visits(ts) >= 3}
+    posing = scanning | monitors
+    t["scanners"], t["monitors"] = len(scanning), len(monitors)
+
+    seen_campaign = set()
+    seen_share = set()
+    for r in hall:
+        q = r.get("request", {})
+        h = q.get("headers", {})
+        ua = " ".join(h.get("User-Agent", [""]))
+        ip = q.get("client_ip") or q.get("remote_ip") or "?"
+        uri = q.get("uri", "")
+        path = uri.split("?")[0]
+        if ip in posing or BOT.search(ua) or not ua:
+            t["botAddresses"].add(ip)
+            service = unfurled_by(ua)
+            # One share per app and page: a preview is often fetched twice.
+            if service and (service, path) not in seen_share:
+                seen_share.add((service, path))
+                t["shared"][service] += 1
+            continue
+        if path not in ("/", "/index.html"):
+            continue
+        t["views"] += 1
+        t["addresses"].add(ip)
+        ref = " ".join(h.get("Referer", [""]))
+        if ref and SITE not in ref:
+            t["referrers"][re.sub(r"^https?://", "", ref).split("/")[0]] += 1
+        m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri)
+        if m and (ip, m.group(1)) not in seen_campaign:
+            seen_campaign.add((ip, m.group(1)))
+            t["campaigns"][m.group(1) + ("=" + m.group(2)[:24] if m.group(1) != "fbclid" else "")] += 1
     return t

template/deploy/__SLUG__-rollup.sh

@@ -58,7 +58,8 @@ for line in open("/proc/meminfo"):
     k, v = line.split(":", 1); mem[k] = int(v.split()[0])
 load = os.getloadavg()[1]
 row = dict(day=day, requests=t["requests"], human=t["human"], bot=t["bot"], humanAddresses=len(t["addresses"]), botAddresses=len(t["botAddresses"]),
-           paths=dict(t["pages"]), roomLinks=sum(t["joins"].values()), referrers=dict(t["referrers"]), campaigns=dict(t["campaigns"]),
+           paths=dict(t["pages"]), roomLinks=sum(t["joins"].values()), roomsOpened=t["opened"], sharedOn=dict(t["shared"]),
+           scanners=t["scanners"], monitors=t["monitors"], referrers=dict(t["referrers"]), campaigns=dict(t["campaigns"]),
            rooms=dict(created=created, started=started, withTurns=turned, finished=finished, humanNames=len(humans), botSeats=bots),
            turns=turns, ledgers=len(glob.glob("/var/lib/__SLUG__/rooms/*.jsonl")),
            vitals=dict(diskUsedPct=round(disk.used * 100 / disk.total), memAvailableMb=mem.get("MemAvailable", 0) // 1024, load5=round(load, 2)),

template/deploy/__SLUG__-visitors.sh

@@ -26,9 +26,9 @@ print("live sockets: %s | rooms touched in the last hour: %s" % (
 
 # --- traffic: Caddy's access log, people and bots apart ---
 t = traffic.scan(day)
-print("traffic: %d addresses of people, %d page requests (%d bot addresses); hall %d, rules %d, guide %d, room links %s; referrers %s; campaigns %s" % (
+print("traffic: %d addresses of people, %d page requests (%d bot addresses, of them %d scanners, %d monitors); hall %d, rules %d, guide %d; rooms opened here %d, room links %s; shared on %s; referrers %s; campaigns %s" % (
-    len(t["addresses"]), t["human"], len(t["botAddresses"]), t["pages"]["hall"], t["pages"]["rules"], t["pages"]["guide"],
+    len(t["addresses"]), t["human"], len(t["botAddresses"]), t["scanners"], t["monitors"], t["pages"]["hall"], t["pages"]["rules"], t["pages"]["guide"],
-    dict(t["joins"]) or "none", dict(t["referrers"]) or "none", dict(t["campaigns"]) or "none"))
+    t["opened"], dict(t["joins"]) or "none", dict(t["shared"]) or "none", dict(t["referrers"]) or "none", dict(t["campaigns"]) or "none"))
 
 # --- rooms: every ledger, today's in detail ---
 first_seen = {}

template/deploy/traffic.py

@@ -7,23 +7,60 @@ import gzip
 import json
 import os
 import re
-from collections import Counter
+from collections import Counter, defaultdict
 
-BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I)
+BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|whatsapp|telegram|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I)
+# Link previews: the app that fetched the page to show it in a conversation. The people
+# who click such a link arrive with no referrer, so this is the best sign of where it went.
+UNFURL = [
+    ("Slack", re.compile(r"slack", re.I)),
+    ("Discord", re.compile(r"discord", re.I)),
+    ("iMessage", re.compile(r"facebookexternalhit.*twitterbot|applebot.*preview", re.I)),
+    ("Facebook", re.compile(r"facebookexternalhit|facebot|meta-externalagent", re.I)),
+    ("WhatsApp", re.compile(r"whatsapp", re.I)),
+    ("Telegram", re.compile(r"telegram", re.I)),
+    ("Reddit", re.compile(r"redditbot", re.I)),
+    ("LinkedIn", re.compile(r"linkedinbot", re.I)),
+    ("Mastodon", re.compile(r"mastodon", re.I)),
+    ("Bluesky", re.compile(r"bluesky|cardyb", re.I)),
+    ("X", re.compile(r"twitterbot", re.I)),
+]
+# Paths nobody asks a game for but a vulnerability scanner: its address is a bot's all day.
+# Matched against whole path segments, never the site's own hashed assets under /_app/.
+SCANNER = re.compile(r"\.php$|(^|/)(\.env|\.git|\.aws|wp-admin|wp-login|wp-content|wp-includes|wordpress|phpmyadmin|cgi-bin|vendor|actuator|boaform|owa|admin)(/|$)", re.I)
 PAGES = {"/": "hall", "/rules": "rules", "/guide": "guide"}
 SITE = "__DOMAIN__"
 # Each game on a shared host keeps its own access log.
 LOGS = "/var/lib/caddy/__SLUG__-access*.log*"
+# The site's own addresses, for telling a referrer from a move between its pages: its domain, and its sslip.io name.
+OWN = re.compile(re.escape(SITE) + r"|\.sslip\.io\b", re.I)
+# Requests from one address more than this far apart are separate visits.
+VISIT_GAP = 30 * 60
+
+
+def unfurled_by(ua):
+    """The app a link preview came from, or None."""
+    for name, pattern in UNFURL:
+        if pattern.search(ua):
+            return name
+    return None
 
 
 def scan(day, logs=LOGS):
     """Counts for one UTC day (YYYY-MM-DD): every request, page requests by
     people, the addresses of people and of bots, requests by page, room links
-    opened, referrers, and campaign tags (each counted once per address)."""
+    opened, rooms opened from the site itself, where links were shared,
+    referrers, and campaign tags (each counted once per address).
+
+    A request is a bot's when its browser says so. An address is a bot's for
+    the whole day when it asks for a scanner's paths, or comes back for pages
+    three visits or more without ever opening or joining a game: a monitor,
+    not a person. A monitor that takes a new address every visit gives itself
+    away by keeping time: one browser, page loads at a steady interval, four
+    visits or more, never a game."""
     d0 = datetime.datetime.fromisoformat(day).replace(tzinfo=datetime.timezone.utc)
     t0, t1 = d0.timestamp(), (d0 + datetime.timedelta(days=1)).timestamp()
-    t = dict(requests=0, human=0, bot=0, addresses=set(), botAddresses=set(), pages=Counter(), joins=Counter(), referrers=Counter(), campaigns=Counter())
+    requests = []
-    seen_campaign = set()
     for f in sorted(glob.glob(logs)):
         if os.path.getmtime(f) < t0:
             continue
@@ -35,8 +72,68 @@ def scan(day, logs=LOGS):
                         r = json.loads(line)
                     except ValueError:
                         continue
-                    if not (t0 <= r.get("ts", 0) < t1):
+                    if t0 <= r.get("ts", 0) < t1:
+                        requests.append(r)
+        except OSError:
             continue
+
+    # First pass: what each address did all day under a browser's name, to find scanners and monitors.
+    seen = defaultdict(lambda: dict(times=[], played=False, scanner=False))
+    for r in requests:
+        q = r.get("request", {})
+        ip = q.get("client_ip") or q.get("remote_ip") or "?"
+        ua = " ".join(q.get("headers", {}).get("User-Agent", [""]))
+        if BOT.search(ua) or not ua:
+            continue
+        path = q.get("uri", "").split("?")[0]
+        a = seen[ip]
+        a["times"].append(r.get("ts", 0))
+        if not path.startswith("/_app/") and SCANNER.search(path):
+            a["scanner"] = True
+        if path.startswith(("/join/", "/api/rooms")) or path == "/ws":
+            a["played"] = True
+
+    def visits(times):
+        n, last = 0, None
+        for ts in sorted(times):
+            if last is None or ts - last > VISIT_GAP:
+                n += 1
+            last = ts
+        return n
+
+    def keeps_time(times):
+        starts, last = [], None
+        for ts in sorted(times):
+            if last is None or ts - last > VISIT_GAP:
+                starts.append(ts)
+            last = ts
+        gaps = [b - a for a, b in zip(starts, starts[1:])]
+        if len(starts) < 4:
+            return False
+        typical = sorted(gaps)[len(gaps) // 2]
+        return all(0.75 * typical <= g <= 1.25 * typical for g in gaps)
+
+    scanners = {ip for ip, a in seen.items() if a["scanner"]}
+    monitors = {ip for ip, a in seen.items() if not (a["scanner"] or a["played"]) and visits(a["times"]) >= 3}
+    by_browser = defaultdict(list)
+    for r in requests:
+        q = r.get("request", {})
+        ip = q.get("client_ip") or q.get("remote_ip") or "?"
+        ua = " ".join(q.get("headers", {}).get("User-Agent", [""]))
+        if ua and not BOT.search(ua) and q.get("uri", "").split("?")[0] in PAGES and not seen[ip]["played"]:
+            by_browser[ua].append((r.get("ts", 0), ip))
+    for hits in by_browser.values():
+        if keeps_time([ts for ts, _ in hits]):
+            monitors |= {ip for _, ip in hits}
+    posing = scanners | monitors
+
+    t = dict(
+        requests=0, human=0, bot=0, addresses=set(), botAddresses=set(), pages=Counter(), joins=Counter(),
+        opened=0, shared=Counter(), scanners=len(scanners), monitors=len(monitors), referrers=Counter(), campaigns=Counter(),
+    )
+    seen_campaign = set()
+    seen_share = set()
+    for r in requests:
         q = r.get("request", {})
         h = q.get("headers", {})
         ua = " ".join(h.get("User-Agent", [""]))
@@ -44,10 +141,18 @@ def scan(day, logs=LOGS):
         uri = q.get("uri", "")
         path = uri.split("?")[0]
         t["requests"] += 1
-                    if BOT.search(ua) or not ua:
+        if ip in posing or BOT.search(ua) or not ua:
             t["bot"] += 1
             t["botAddresses"].add(ip)
+            service = unfurled_by(ua)
+            # One share per app and page: a preview is often fetched twice.
+            if service and (service, path) not in seen_share:
+                seen_share.add((service, path))
+                t["shared"][service] += 1
             continue
+        # A room opened from the site itself never passes through /join: count the API's answer.
+        if q.get("method") == "POST" and path == "/api/rooms" and 200 <= r.get("status", 0) < 300:
+            t["opened"] += 1
         if path.startswith(("/_app/", "/api/")) or path == "/ws" or path.endswith((".png", ".ico", ".txt")):
             continue
         t["human"] += 1
@@ -57,12 +162,10 @@ def scan(day, logs=LOGS):
         elif path.startswith("/join/"):
             t["joins"][path.split("/")[2].upper()] += 1
         ref = " ".join(h.get("Referer", [""]))
-                    if ref and SITE not in ref:
+        if ref and not OWN.search(ref):
             t["referrers"][re.sub(r"^https?://", "", ref).split("/")[0]] += 1
         m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri)
         if m and (ip, m.group(1)) not in seen_campaign:
             seen_campaign.add((ip, m.group(1)))
             t["campaigns"][m.group(1) + ("=" + m.group(2)[:24] if m.group(1) != "fbclid" else "")] += 1
-        except OSError:
-            continue
     return t