Traffic counts what matters: rooms opened from the site, where links were shared, scanners and monitors as bots

- A room opened with Play now never loads /join: the API's answer to
  POST /api/rooms is counted as rooms opened here.
- Link previews (Slack, Discord, Facebook, iMessage, WhatsApp and the
  rest) are reported as where the link was shared, since the people who
  click them arrive with no referrer. WhatsApp's fetcher is a bot now.
- An address that asks for a scanner's paths, or keeps coming back
  without opening a game, is a bot's all day; so is a browser that loads
  pages at a steady interval from a new address each time, never
  playing. A declared bot is still judged request by request.
- The site's own addresses, sslip.io included, are not referrers.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Jm2auWk6RP71CjaAb4FMoG

deploy/deploy.sh

@@ -25,7 +25,7 @@ rsync -az --delete --filter='P _app/immutable/*' \
 # it needs only its code and the engine. Ledgers live outside this tree.
 rsync -az --delete --exclude='/server/node_modules' \
   --include='/server/***' --include='/src/' --include='/src/lib/' --include='/src/lib/game/***' \
-  --include='/deploy/' --include='/deploy/waving-hands.service' --include='/deploy/waving-hands.cron' --include='/deploy/*.sh' --include='/deploy/hash-tokens.ts' \
+  --include='/deploy/' --include='/deploy/waving-hands.service' --include='/deploy/waving-hands.cron' --include='/deploy/*.sh' --include='/deploy/hash-tokens.ts' --include='/deploy/traffic.py' \
   --exclude='*' \
   ./ "root@$HOST:/opt/waving-hands/app/"
 
@@ -39,6 +39,7 @@ ssh "root@$HOST" '
   # Cron runs the rollup and the backup from /usr/local/bin: install them on
   # every deploy so the live copies are always the repo copies.
   install -m 755 /opt/waving-hands/app/deploy/visitors.sh /usr/local/bin/waving-hands-visitors.sh
+  install -m 644 -D /opt/waving-hands/app/deploy/traffic.py /usr/local/lib/waving-hands/traffic.py
   install -m 755 /opt/waving-hands/app/deploy/pulse.sh /usr/local/bin/waving-hands-pulse.sh
   install -m 755 /opt/waving-hands/app/deploy/waving-hands-rollup.sh /usr/local/bin/waving-hands-rollup.sh
   install -m 755 /opt/waving-hands/app/deploy/waving-hands-backup.sh /usr/local/bin/waving-hands-backup.sh

deploy/traffic.py

@@ -0,0 +1,171 @@
+"""Caddy's access log for one day, people and bots apart. The visitors
+digest and the nightly rollup both count through here, so they agree."""
+
+import datetime
+import glob
+import gzip
+import json
+import os
+import re
+from collections import Counter, defaultdict
+
+BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|whatsapp|telegram|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I)
+# Link previews: the app that fetched the page to show it in a conversation. The people
+# who click such a link arrive with no referrer, so this is the best sign of where it went.
+UNFURL = [
+    ("Slack", re.compile(r"slack", re.I)),
+    ("Discord", re.compile(r"discord", re.I)),
+    ("iMessage", re.compile(r"facebookexternalhit.*twitterbot|applebot.*preview", re.I)),
+    ("Facebook", re.compile(r"facebookexternalhit|facebot|meta-externalagent", re.I)),
+    ("WhatsApp", re.compile(r"whatsapp", re.I)),
+    ("Telegram", re.compile(r"telegram", re.I)),
+    ("Reddit", re.compile(r"redditbot", re.I)),
+    ("LinkedIn", re.compile(r"linkedinbot", re.I)),
+    ("Mastodon", re.compile(r"mastodon", re.I)),
+    ("Bluesky", re.compile(r"bluesky|cardyb", re.I)),
+    ("X", re.compile(r"twitterbot", re.I)),
+]
+# Paths nobody asks a game for but a vulnerability scanner: its address is a bot's all day.
+# Matched against whole path segments, never the site's own hashed assets under /_app/.
+SCANNER = re.compile(r"\.php$|(^|/)(\.env|\.git|\.aws|wp-admin|wp-login|wp-content|wp-includes|wordpress|phpmyadmin|cgi-bin|vendor|actuator|boaform|owa|admin)(/|$)", re.I)
+PAGES = {"/": "hall", "/play": "play", "/rules": "rules"}
+SITE = "hands.kestrelsnest.social"
+# Each game on a shared host keeps its own access log.
+LOGS = "/var/lib/caddy/waving-hands-access*.log*"
+# The site's own addresses, for telling a referrer from a move between its pages: its domain, and its sslip.io name.
+OWN = re.compile(re.escape(SITE) + r"|\.sslip\.io\b", re.I)
+# Requests from one address more than this far apart are separate visits.
+VISIT_GAP = 30 * 60
+
+
+def unfurled_by(ua):
+    """The app a link preview came from, or None."""
+    for name, pattern in UNFURL:
+        if pattern.search(ua):
+            return name
+    return None
+
+
+def scan(day, logs=LOGS):
+    """Counts for one UTC day (YYYY-MM-DD): every request, page requests by
+    people, the addresses of people and of bots, requests by page, room links
+    opened, rooms opened from the site itself, where links were shared,
+    referrers, and campaign tags (each counted once per address).
+
+    A request is a bot's when its browser says so. An address is a bot's for
+    the whole day when it asks for a scanner's paths, or comes back for pages
+    three visits or more without ever opening or joining a game: a monitor,
+    not a person. A monitor that takes a new address every visit gives itself
+    away by keeping time: one browser, page loads at a steady interval, four
+    visits or more, never a game."""
+    d0 = datetime.datetime.fromisoformat(day).replace(tzinfo=datetime.timezone.utc)
+    t0, t1 = d0.timestamp(), (d0 + datetime.timedelta(days=1)).timestamp()
+    requests = []
+    for f in sorted(glob.glob(logs)):
+        if os.path.getmtime(f) < t0:
+            continue
+        opener = gzip.open if f.endswith(".gz") else open
+        try:
+            with opener(f, "rt", errors="ignore") as fh:
+                for line in fh:
+                    try:
+                        r = json.loads(line)
+                    except ValueError:
+                        continue
+                    if t0 <= r.get("ts", 0) < t1:
+                        requests.append(r)
+        except OSError:
+            continue
+
+    # First pass: what each address did all day under a browser's name, to find scanners and monitors.
+    seen = defaultdict(lambda: dict(times=[], played=False, scanner=False))
+    for r in requests:
+        q = r.get("request", {})
+        ip = q.get("client_ip") or q.get("remote_ip") or "?"
+        ua = " ".join(q.get("headers", {}).get("User-Agent", [""]))
+        if BOT.search(ua) or not ua:
+            continue
+        path = q.get("uri", "").split("?")[0]
+        a = seen[ip]
+        a["times"].append(r.get("ts", 0))
+        if not path.startswith("/_app/") and SCANNER.search(path):
+            a["scanner"] = True
+        if path.startswith(("/join/", "/api/rooms")) or path == "/ws":
+            a["played"] = True
+
+    def visits(times):
+        n, last = 0, None
+        for ts in sorted(times):
+            if last is None or ts - last > VISIT_GAP:
+                n += 1
+            last = ts
+        return n
+
+    def keeps_time(times):
+        starts, last = [], None
+        for ts in sorted(times):
+            if last is None or ts - last > VISIT_GAP:
+                starts.append(ts)
+            last = ts
+        gaps = [b - a for a, b in zip(starts, starts[1:])]
+        if len(starts) < 4:
+            return False
+        typical = sorted(gaps)[len(gaps) // 2]
+        return all(0.75 * typical <= g <= 1.25 * typical for g in gaps)
+
+    scanners = {ip for ip, a in seen.items() if a["scanner"]}
+    monitors = {ip for ip, a in seen.items() if not (a["scanner"] or a["played"]) and visits(a["times"]) >= 3}
+    by_browser = defaultdict(list)
+    for r in requests:
+        q = r.get("request", {})
+        ip = q.get("client_ip") or q.get("remote_ip") or "?"
+        ua = " ".join(q.get("headers", {}).get("User-Agent", [""]))
+        if ua and not BOT.search(ua) and q.get("uri", "").split("?")[0] in PAGES and not seen[ip]["played"]:
+            by_browser[ua].append((r.get("ts", 0), ip))
+    for hits in by_browser.values():
+        if keeps_time([ts for ts, _ in hits]):
+            monitors |= {ip for _, ip in hits}
+    posing = scanners | monitors
+
+    t = dict(
+        requests=0, human=0, bot=0, addresses=set(), botAddresses=set(), pages=Counter(), joins=Counter(),
+        opened=0, shared=Counter(), scanners=len(scanners), monitors=len(monitors), referrers=Counter(), campaigns=Counter(),
+    )
+    seen_campaign = set()
+    seen_share = set()
+    for r in requests:
+        q = r.get("request", {})
+        h = q.get("headers", {})
+        ua = " ".join(h.get("User-Agent", [""]))
+        ip = q.get("client_ip") or q.get("remote_ip") or "?"
+        uri = q.get("uri", "")
+        path = uri.split("?")[0]
+        t["requests"] += 1
+        if ip in posing or BOT.search(ua) or not ua:
+            t["bot"] += 1
+            t["botAddresses"].add(ip)
+            service = unfurled_by(ua)
+            # One share per app and page: a preview is often fetched twice.
+            if service and (service, path) not in seen_share:
+                seen_share.add((service, path))
+                t["shared"][service] += 1
+            continue
+        # A room opened from the site itself never passes through /join: count the API's answer.
+        if q.get("method") == "POST" and path == "/api/rooms" and 200 <= r.get("status", 0) < 300:
+            t["opened"] += 1
+        if path.startswith(("/_app/", "/api/")) or path == "/ws" or path.endswith((".png", ".ico", ".txt")):
+            continue
+        t["human"] += 1
+        t["addresses"].add(ip)
+        if path in PAGES:
+            t["pages"][PAGES[path]] += 1
+        elif path.startswith("/join/"):
+            t["joins"][path.split("/")[2].upper()] += 1
+        ref = " ".join(h.get("Referer", [""]))
+        if ref and not OWN.search(ref):
+            t["referrers"][re.sub(r"^https?://", "", ref).split("/")[0]] += 1
+        m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri)
+        if m and (ip, m.group(1)) not in seen_campaign:
+            seen_campaign.add((ip, m.group(1)))
+            t["campaigns"][m.group(1) + ("=" + m.group(2)[:24] if m.group(1) != "fbclid" else "")] += 1
+    return t

deploy/visitors.sh

@@ -25,50 +25,12 @@ print("live sockets: %s | rooms touched in the last hour: %s" % (
     sh("find /var/lib/waving-hands/rooms -name '*.jsonl' -mmin -60 2>/dev/null | wc -l")))
 
 # --- traffic: Caddy's access log, people and bots apart ---
-BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|preview|fetch|curl|python|go-http|headless|scan|monitor|uptime", re.I)
-PAGES = {"/": "hall", "/play": "play", "/rules": "rules"}
-addresses, bots = set(), set()
-pages, joins, referrers, campaigns, agents = Counter(), Counter(), Counter(), Counter(), Counter()
-requests = 0
-for path in sorted(glob.glob("/var/lib/caddy/waving-hands-access*.log*")):
-    try:
-        with open(path) as f:
-            for line in f:
-                try:
-                    x = json.loads(line)
-                except Exception:
-                    continue
-                ts = datetime.datetime.fromtimestamp(x["ts"], datetime.timezone.utc)
-                if ts.strftime("%F") != day:
-                    continue
-                req = x["request"]
-                ua = " ".join(req.get("headers", {}).get("User-Agent", [""]))
-                ip = req.get("remote_ip") or req.get("client_ip", "?")
-                uri = req.get("uri", "")
-                path_only = uri.split("?")[0]
-                if BOT.search(ua) or not ua:
-                    bots.add(ip)
-                    continue
-                if path_only.startswith("/_app/") or path_only.startswith("/api/") or path_only == "/ws" or path_only.endswith((".png", ".ico", ".txt")):
-                    continue
-                requests += 1
-                addresses.add(ip)
-                if path_only in PAGES:
-                    pages[PAGES[path_only]] += 1
-                elif path_only.startswith("/join/"):
-                    joins[path_only.split("/")[2].upper()] += 1
-                ref = " ".join(req.get("headers", {}).get("Referer", [""]))
-                if ref and "hands.kestrelsnest.social" not in ref:
-                    referrers[re.sub(r"^https?://", "", ref).split("/")[0]] += 1
-                m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri)
-                if m:
-                    campaigns[m.group(1) + "=" + m.group(2)[:24]] += 1
-                agents[re.sub(r"\(.*?\)", "", ua)[:40].strip()] += 1
-    except OSError:
-        continue
-print("traffic: %d addresses of people, %d page requests (%d bot addresses); hall %d, play %d, rules %d, room links %s; referrers %s; campaigns %s" % (
-    len(addresses), requests, len(bots), pages["hall"], pages["play"], pages["rules"],
-    dict(joins) or "none", dict(referrers) or "none", dict(campaigns) or "none"))
+sys.path.insert(0, "/usr/local/lib/waving-hands")
+import traffic
+t = traffic.scan(day)
+print("traffic: %d addresses of people, %d page requests (%d bot addresses, of them %d scanners, %d monitors); hall %d, play %d, rules %d; rooms opened here %d, room links %s; shared on %s; referrers %s; campaigns %s" % (
+    len(t["addresses"]), t["human"], len(t["botAddresses"]), t["scanners"], t["monitors"], t["pages"]["hall"], t["pages"]["play"], t["pages"]["rules"],
+    t["opened"], dict(t["joins"]) or "none", dict(t["shared"]) or "none", dict(t["referrers"]) or "none", dict(t["campaigns"]) or "none"))
 
 # --- rooms: every ledger, today's in detail ---
 first_seen = {}

deploy/waving-hands-rollup.sh

@@ -28,41 +28,9 @@ import collections, datetime, glob, gzip, json, os, re, shutil, sys
 day, out = sys.argv[1], sys.argv[2]
 d0 = datetime.datetime.fromisoformat(day).replace(tzinfo=datetime.timezone.utc)
 t0, t1 = d0.timestamp(), (d0 + datetime.timedelta(days=1)).timestamp()
-BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I)
-
-req = human = bot = 0
-hips, bips = set(), set()
-pages, joins, refs, campaigns = collections.Counter(), collections.Counter(), collections.Counter(), collections.Counter()
-seen_campaign = set()
-for f in sorted(glob.glob("/var/lib/caddy/waving-hands-access*.log*")):
-    if os.path.getmtime(f) < t0: continue
-    opener = gzip.open if f.endswith(".gz") else open
-    with opener(f, "rt", errors="ignore") as fh:
-        for line in fh:
-            try: r = json.loads(line)
-            except ValueError: continue
-            ts = r.get("ts", 0)
-            if not (t0 <= ts < t1): continue
-            q = r.get("request", {}); h = q.get("headers", {})
-            ua = " ".join(h.get("User-Agent", [""]))
-            ip = q.get("client_ip") or q.get("remote_ip") or "?"
-            uri = q.get("uri", ""); path = uri.split("?")[0]
-            req += 1
-            if BOT.search(ua) or not ua:
-                bot += 1; bips.add(ip); continue
-            if path.startswith(("/_app/", "/api/")) or path == "/ws" or path.endswith((".png", ".ico", ".txt")): continue
-            human += 1; hips.add(ip)
-            if path == "/": pages["hall"] += 1
-            elif path == "/play": pages["play"] += 1
-            elif path == "/rules": pages["rules"] += 1
-            elif path.startswith("/join/"): joins[path.split("/")[2].upper()] += 1
-            ref = " ".join(h.get("Referer", [""]))
-            if ref and "hands.kestrelsnest.social" not in ref:
-                refs[re.sub(r"^https?://", "", ref).split("/")[0]] += 1
-            m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri)
-            if m and (ip, m.group(1)) not in seen_campaign:
-                seen_campaign.add((ip, m.group(1)))
-                campaigns[m.group(1) + ("=" + m.group(2)[:24] if m.group(1) != "fbclid" else "")] += 1
+sys.path.insert(0, "/usr/local/lib/waving-hands")
+import traffic
+t = traffic.scan(day)
 
 created = started = finished = turned = turns = 0
 humans, bots = set(), 0
@@ -80,20 +48,19 @@ for f in glob.glob("/var/lib/waving-hands/rooms/*.jsonl"):
                 if x["bot"]: bots += 1
                 else: humans.add(x["name"])
             elif x["t"] == "unseat": bots -= 1
-    day_turns = [x for x in lines if x["t"] == "turn" and t0 <= x["at"] / 1000 < t1]
-    turns += len(day_turns)
-    # A duel finished today: its last turn is today's and the engine would say so; approximate by replay-free means:
-    # the ledger's final turn falls today and no turn follows it within the day's end.
-    if day_turns and day_turns[-1] is lines[-1] and False: finished += 1
+    turns += sum(1 for x in lines if x["t"] == "turn" and t0 <= x["at"] / 1000 < t1)
+    # The over line is written with the turn that ends the game, so a finish counts on its own day.
+    if any(x["t"] == "over" and t0 <= x["at"] / 1000 < t1 for x in lines): finished += 1
 
 disk = shutil.disk_usage("/")
 mem = {}
 for line in open("/proc/meminfo"):
     k, v = line.split(":", 1); mem[k] = int(v.split()[0])
 load = os.getloadavg()[1]
-row = dict(day=day, requests=req, human=human, bot=bot, humanAddresses=len(hips), botAddresses=len(bips),
-           paths=dict(pages), roomLinks=sum(joins.values()), referrers=dict(refs), campaigns=dict(campaigns),
-           rooms=dict(created=created, started=started, withTurns=turned, humanNames=len(humans), botSeats=bots),
+row = dict(day=day, requests=t["requests"], human=t["human"], bot=t["bot"], humanAddresses=len(t["addresses"]), botAddresses=len(t["botAddresses"]),
+           paths=dict(t["pages"]), roomLinks=sum(t["joins"].values()), roomsOpened=t["opened"], sharedOn=dict(t["shared"]),
+           scanners=t["scanners"], monitors=t["monitors"], referrers=dict(t["referrers"]), campaigns=dict(t["campaigns"]),
+           rooms=dict(created=created, started=started, withTurns=turned, finished=finished, humanNames=len(humans), botSeats=bots),
            turns=turns, ledgers=len(glob.glob("/var/lib/waving-hands/rooms/*.jsonl")),
            vitals=dict(diskUsedPct=round(disk.used * 100 / disk.total), memAvailableMb=mem.get("MemAvailable", 0) // 1024, load5=round(load, 2)),
            at=datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds"))