Traffic counts what matters: rooms opened from the site, where links were shared, scanners and monitors as bots
- A room opened with Play now never loads /join: the API's answer to POST /api/rooms is counted as rooms opened here. - Link previews (Slack, Discord, Facebook, iMessage, WhatsApp and the rest) are reported as where the link was shared, since the people who click them arrive with no referrer. WhatsApp's fetcher is a bot now. - An address that asks for a scanner's paths, or keeps coming back without opening a game, is a bot's all day; so is a browser that loads pages at a steady interval from a new address each time, never playing. A declared bot is still judged request by request. - The site's own addresses, sslip.io included, are not referrers. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Jm2auWk6RP71CjaAb4FMoG
This commit is contained in:
Eric Wagoner
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
e3fcb19a22
commit
7f19ec3ea9
@@ -58,7 +58,8 @@ for line in open("/proc/meminfo"):
|
||||
k, v = line.split(":", 1); mem[k] = int(v.split()[0])
|
||||
load = os.getloadavg()[1]
|
||||
row = dict(day=day, requests=t["requests"], human=t["human"], bot=t["bot"], humanAddresses=len(t["addresses"]), botAddresses=len(t["botAddresses"]),
|
||||
paths=dict(t["pages"]), roomLinks=sum(t["joins"].values()), referrers=dict(t["referrers"]), campaigns=dict(t["campaigns"]),
|
||||
paths=dict(t["pages"]), roomLinks=sum(t["joins"].values()), roomsOpened=t["opened"], sharedOn=dict(t["shared"]),
|
||||
scanners=t["scanners"], monitors=t["monitors"], referrers=dict(t["referrers"]), campaigns=dict(t["campaigns"]),
|
||||
rooms=dict(created=created, started=started, withTurns=turned, finished=finished, humanNames=len(humans), botSeats=bots),
|
||||
turns=turns, ledgers=len(glob.glob("/var/lib/hnefatafl/rooms/*.jsonl")),
|
||||
vitals=dict(diskUsedPct=round(disk.used * 100 / disk.total), memAvailableMb=mem.get("MemAvailable", 0) // 1024, load5=round(load, 2)),
|
||||
|
||||
@@ -26,9 +26,9 @@ print("live sockets: %s | rooms touched in the last hour: %s" % (
|
||||
|
||||
# --- traffic: Caddy's access log, people and bots apart ---
|
||||
t = traffic.scan(day)
|
||||
print("traffic: %d addresses of people, %d page requests (%d bot addresses); hall %d, rules %d, guide %d, room links %s; referrers %s; campaigns %s" % (
|
||||
len(t["addresses"]), t["human"], len(t["botAddresses"]), t["pages"]["hall"], t["pages"]["rules"], t["pages"]["guide"],
|
||||
dict(t["joins"]) or "none", dict(t["referrers"]) or "none", dict(t["campaigns"]) or "none"))
|
||||
print("traffic: %d addresses of people, %d page requests (%d bot addresses, of them %d scanners, %d monitors); hall %d, rules %d, guide %d; rooms opened here %d, room links %s; shared on %s; referrers %s; campaigns %s" % (
|
||||
len(t["addresses"]), t["human"], len(t["botAddresses"]), t["scanners"], t["monitors"], t["pages"]["hall"], t["pages"]["rules"], t["pages"]["guide"],
|
||||
t["opened"], dict(t["joins"]) or "none", dict(t["shared"]) or "none", dict(t["referrers"]) or "none", dict(t["campaigns"]) or "none"))
|
||||
|
||||
# --- rooms: every ledger, today's in detail ---
|
||||
first_seen = {}
|
||||
|
||||
+113
-10
@@ -7,23 +7,60 @@ import gzip
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from collections import Counter
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I)
|
||||
BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|whatsapp|telegram|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I)
|
||||
# Link previews: the app that fetched the page to show it in a conversation. The people
|
||||
# who click such a link arrive with no referrer, so this is the best sign of where it went.
|
||||
UNFURL = [
|
||||
("Slack", re.compile(r"slack", re.I)),
|
||||
("Discord", re.compile(r"discord", re.I)),
|
||||
("iMessage", re.compile(r"facebookexternalhit.*twitterbot|applebot.*preview", re.I)),
|
||||
("Facebook", re.compile(r"facebookexternalhit|facebot|meta-externalagent", re.I)),
|
||||
("WhatsApp", re.compile(r"whatsapp", re.I)),
|
||||
("Telegram", re.compile(r"telegram", re.I)),
|
||||
("Reddit", re.compile(r"redditbot", re.I)),
|
||||
("LinkedIn", re.compile(r"linkedinbot", re.I)),
|
||||
("Mastodon", re.compile(r"mastodon", re.I)),
|
||||
("Bluesky", re.compile(r"bluesky|cardyb", re.I)),
|
||||
("X", re.compile(r"twitterbot", re.I)),
|
||||
]
|
||||
# Paths nobody asks a game for but a vulnerability scanner: its address is a bot's all day.
|
||||
# Matched against whole path segments, never the site's own hashed assets under /_app/.
|
||||
SCANNER = re.compile(r"\.php$|(^|/)(\.env|\.git|\.aws|wp-admin|wp-login|wp-content|wp-includes|wordpress|phpmyadmin|cgi-bin|vendor|actuator|boaform|owa|admin)(/|$)", re.I)
|
||||
PAGES = {"/": "hall", "/rules": "rules", "/guide": "guide"}
|
||||
SITE = "tafl.kestrelsnest.social"
|
||||
# Each game on a shared host keeps its own access log.
|
||||
LOGS = "/var/lib/caddy/hnefatafl-access*.log*"
|
||||
# The site's own addresses, for telling a referrer from a move between its pages: its domain, and its sslip.io name.
|
||||
OWN = re.compile(re.escape(SITE) + r"|\.sslip\.io\b", re.I)
|
||||
# Requests from one address more than this far apart are separate visits.
|
||||
VISIT_GAP = 30 * 60
|
||||
|
||||
|
||||
def unfurled_by(ua):
|
||||
"""The app a link preview came from, or None."""
|
||||
for name, pattern in UNFURL:
|
||||
if pattern.search(ua):
|
||||
return name
|
||||
return None
|
||||
|
||||
|
||||
def scan(day, logs=LOGS):
|
||||
"""Counts for one UTC day (YYYY-MM-DD): every request, page requests by
|
||||
people, the addresses of people and of bots, requests by page, room links
|
||||
opened, referrers, and campaign tags (each counted once per address)."""
|
||||
opened, rooms opened from the site itself, where links were shared,
|
||||
referrers, and campaign tags (each counted once per address).
|
||||
|
||||
A request is a bot's when its browser says so. An address is a bot's for
|
||||
the whole day when it asks for a scanner's paths, or comes back for pages
|
||||
three visits or more without ever opening or joining a game: a monitor,
|
||||
not a person. A monitor that takes a new address every visit gives itself
|
||||
away by keeping time: one browser, page loads at a steady interval, four
|
||||
visits or more, never a game."""
|
||||
d0 = datetime.datetime.fromisoformat(day).replace(tzinfo=datetime.timezone.utc)
|
||||
t0, t1 = d0.timestamp(), (d0 + datetime.timedelta(days=1)).timestamp()
|
||||
t = dict(requests=0, human=0, bot=0, addresses=set(), botAddresses=set(), pages=Counter(), joins=Counter(), referrers=Counter(), campaigns=Counter())
|
||||
seen_campaign = set()
|
||||
requests = []
|
||||
for f in sorted(glob.glob(logs)):
|
||||
if os.path.getmtime(f) < t0:
|
||||
continue
|
||||
@@ -35,8 +72,68 @@ def scan(day, logs=LOGS):
|
||||
r = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
if not (t0 <= r.get("ts", 0) < t1):
|
||||
if t0 <= r.get("ts", 0) < t1:
|
||||
requests.append(r)
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
# First pass: what each address did all day under a browser's name, to find scanners and monitors.
|
||||
seen = defaultdict(lambda: dict(times=[], played=False, scanner=False))
|
||||
for r in requests:
|
||||
q = r.get("request", {})
|
||||
ip = q.get("client_ip") or q.get("remote_ip") or "?"
|
||||
ua = " ".join(q.get("headers", {}).get("User-Agent", [""]))
|
||||
if BOT.search(ua) or not ua:
|
||||
continue
|
||||
path = q.get("uri", "").split("?")[0]
|
||||
a = seen[ip]
|
||||
a["times"].append(r.get("ts", 0))
|
||||
if not path.startswith("/_app/") and SCANNER.search(path):
|
||||
a["scanner"] = True
|
||||
if path.startswith(("/join/", "/api/rooms")) or path == "/ws":
|
||||
a["played"] = True
|
||||
|
||||
def visits(times):
|
||||
n, last = 0, None
|
||||
for ts in sorted(times):
|
||||
if last is None or ts - last > VISIT_GAP:
|
||||
n += 1
|
||||
last = ts
|
||||
return n
|
||||
|
||||
def keeps_time(times):
|
||||
starts, last = [], None
|
||||
for ts in sorted(times):
|
||||
if last is None or ts - last > VISIT_GAP:
|
||||
starts.append(ts)
|
||||
last = ts
|
||||
gaps = [b - a for a, b in zip(starts, starts[1:])]
|
||||
if len(starts) < 4:
|
||||
return False
|
||||
typical = sorted(gaps)[len(gaps) // 2]
|
||||
return all(0.75 * typical <= g <= 1.25 * typical for g in gaps)
|
||||
|
||||
scanners = {ip for ip, a in seen.items() if a["scanner"]}
|
||||
monitors = {ip for ip, a in seen.items() if not (a["scanner"] or a["played"]) and visits(a["times"]) >= 3}
|
||||
by_browser = defaultdict(list)
|
||||
for r in requests:
|
||||
q = r.get("request", {})
|
||||
ip = q.get("client_ip") or q.get("remote_ip") or "?"
|
||||
ua = " ".join(q.get("headers", {}).get("User-Agent", [""]))
|
||||
if ua and not BOT.search(ua) and q.get("uri", "").split("?")[0] in PAGES and not seen[ip]["played"]:
|
||||
by_browser[ua].append((r.get("ts", 0), ip))
|
||||
for hits in by_browser.values():
|
||||
if keeps_time([ts for ts, _ in hits]):
|
||||
monitors |= {ip for _, ip in hits}
|
||||
posing = scanners | monitors
|
||||
|
||||
t = dict(
|
||||
requests=0, human=0, bot=0, addresses=set(), botAddresses=set(), pages=Counter(), joins=Counter(),
|
||||
opened=0, shared=Counter(), scanners=len(scanners), monitors=len(monitors), referrers=Counter(), campaigns=Counter(),
|
||||
)
|
||||
seen_campaign = set()
|
||||
seen_share = set()
|
||||
for r in requests:
|
||||
q = r.get("request", {})
|
||||
h = q.get("headers", {})
|
||||
ua = " ".join(h.get("User-Agent", [""]))
|
||||
@@ -44,10 +141,18 @@ def scan(day, logs=LOGS):
|
||||
uri = q.get("uri", "")
|
||||
path = uri.split("?")[0]
|
||||
t["requests"] += 1
|
||||
if BOT.search(ua) or not ua:
|
||||
if ip in posing or BOT.search(ua) or not ua:
|
||||
t["bot"] += 1
|
||||
t["botAddresses"].add(ip)
|
||||
service = unfurled_by(ua)
|
||||
# One share per app and page: a preview is often fetched twice.
|
||||
if service and (service, path) not in seen_share:
|
||||
seen_share.add((service, path))
|
||||
t["shared"][service] += 1
|
||||
continue
|
||||
# A room opened from the site itself never passes through /join: count the API's answer.
|
||||
if q.get("method") == "POST" and path == "/api/rooms" and 200 <= r.get("status", 0) < 300:
|
||||
t["opened"] += 1
|
||||
if path.startswith(("/_app/", "/api/")) or path == "/ws" or path.endswith((".png", ".ico", ".txt")):
|
||||
continue
|
||||
t["human"] += 1
|
||||
@@ -57,12 +162,10 @@ def scan(day, logs=LOGS):
|
||||
elif path.startswith("/join/"):
|
||||
t["joins"][path.split("/")[2].upper()] += 1
|
||||
ref = " ".join(h.get("Referer", [""]))
|
||||
if ref and SITE not in ref:
|
||||
if ref and not OWN.search(ref):
|
||||
t["referrers"][re.sub(r"^https?://", "", ref).split("/")[0]] += 1
|
||||
m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri)
|
||||
if m and (ip, m.group(1)) not in seen_campaign:
|
||||
seen_campaign.add((ip, m.group(1)))
|
||||
t["campaigns"][m.group(1) + ("=" + m.group(2)[:24] if m.group(1) != "fbclid" else "")] += 1
|
||||
except OSError:
|
||||
continue
|
||||
return t
|
||||
|
||||
@@ -25,6 +25,14 @@
|
||||
"pitch": "The Viking board game. One king, one throne, four corners, and a ring of attackers closing in.",
|
||||
"players": "2",
|
||||
"origin": "the Norse tafl game, reconstructed from Linnaeus's 1732 notes; Copenhagen rules or Tablut"
|
||||
},
|
||||
{
|
||||
"slug": "dip",
|
||||
"name": "Dip",
|
||||
"url": "https://dip.kestrelsnest.social",
|
||||
"pitch": "Seven powers. One map. Nobody has to keep their word.",
|
||||
"players": "2 to 7",
|
||||
"origin": "Allan B. Calhamer's Diplomacy, 1959; an unofficial rebuild"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user