diff --git a/gamehall/ops/gamehall-rollup.sh b/gamehall/ops/gamehall-rollup.sh index 8a8adfe..08f43b9 100755 --- a/gamehall/ops/gamehall-rollup.sh +++ b/gamehall/ops/gamehall-rollup.sh @@ -17,6 +17,7 @@ import traffic day, out = sys.argv[1], sys.argv[2] t = traffic.scan(day) row = dict(day=day, people=len(t["addresses"]), views=t["views"], botAddresses=len(t["botAddresses"]), + scanners=t["scanners"], monitors=t["monitors"], sharedOn=dict(t["shared"]), referrers=dict(t["referrers"]), campaigns=dict(t["campaigns"]), sent=t["sent"], at=datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds")) rows = [json.loads(l) for l in open(out) if l.strip()] if os.path.exists(out) else [] diff --git a/gamehall/ops/gamehall-visitors.sh b/gamehall/ops/gamehall-visitors.sh index 7735361..7b60081 100755 --- a/gamehall/ops/gamehall-visitors.sh +++ b/gamehall/ops/gamehall-visitors.sh @@ -18,7 +18,8 @@ if day == now.strftime("%F"): print("last hour: %d people, %d page views" % (len(h["addresses"]), h["views"])) t = traffic.scan(day) -print("today: %d people, %d page views, %d bot addresses" % (len(t["addresses"]), t["views"], len(t["botAddresses"]))) +print("today: %d people, %d page views, %d bot addresses (of them %d scanners, %d monitors)" % (len(t["addresses"]), t["views"], len(t["botAddresses"]), t["scanners"], t["monitors"])) +print("shared on: %s" % (dict(t["shared"].most_common()) or "none")) print("referrers: %s" % (dict(t["referrers"].most_common()) or "none")) print("campaigns: %s" % (dict(t["campaigns"].most_common()) or "none")) print("went on to: %s (Wiz-War is on its own host; its digest shows the hall as a referrer)" % (", ".join("%s %d" % kv for kv in t["sent"].items()))) diff --git a/gamehall/ops/traffic.py b/gamehall/ops/traffic.py index 50df817..199f7aa 100644 --- a/gamehall/ops/traffic.py +++ b/gamehall/ops/traffic.py @@ -10,7 +10,27 @@ import os import re from collections import Counter -BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I) +# The bot, link-preview and scanner patterns are kept in step with template/deploy/traffic.py. +BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|whatsapp|telegram|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I) +# Link previews: the app that fetched the page to show it in a conversation. The people +# who click such a link arrive with no referrer, so this is the best sign of where it went. +UNFURL = [ + ("Slack", re.compile(r"slack", re.I)), + ("Discord", re.compile(r"discord", re.I)), + ("iMessage", re.compile(r"facebookexternalhit.*twitterbot|applebot.*preview", re.I)), + ("Facebook", re.compile(r"facebookexternalhit|facebot|meta-externalagent", re.I)), + ("WhatsApp", re.compile(r"whatsapp", re.I)), + ("Telegram", re.compile(r"telegram", re.I)), + ("Reddit", re.compile(r"redditbot", re.I)), + ("LinkedIn", re.compile(r"linkedinbot", re.I)), + ("Mastodon", re.compile(r"mastodon", re.I)), + ("Bluesky", re.compile(r"bluesky|cardyb", re.I)), + ("X", re.compile(r"twitterbot", re.I)), +] +# Paths nobody asks the hall for but a vulnerability scanner: its address is a bot's all day. +SCANNER = re.compile(r"\.php$|(^|/)(\.env|\.git|\.aws|wp-admin|wp-login|wp-content|wp-includes|wordpress|phpmyadmin|cgi-bin|vendor|actuator|boaform|owa|admin)(/|$)", re.I) +# Requests from one address more than this far apart are separate visits. +VISIT_GAP = 30 * 60 SITE = "gamehall.kestrelsnest.social" LOGS = "/var/lib/caddy/gamehall-access*.log*" # The games that share this host, by the name the hall links to. A visitor @@ -20,6 +40,23 @@ GAMES = {"dip": "dip.kestrelsnest.social", "waving-hands": "hands.kestrelsnest.s ASSET = (".png", ".jpg", ".svg", ".ico", ".txt", ".webmanifest") +def unfurled_by(ua): + """The app a link preview came from, or None.""" + for name, pattern in UNFURL: + if pattern.search(ua): + return name + return None + + +def _visits(times): + n, last = 0, None + for ts in sorted(times): + if last is None or ts - last > VISIT_GAP: + n += 1 + last = ts + return n + + def _lines(pattern, t0): for f in sorted(glob.glob(pattern)): if os.path.getmtime(f) < t0: @@ -43,39 +80,18 @@ def _bounds(day): def scan(day, since=None): """Counts for one UTC day (YYYY-MM-DD), or from `since` (a timestamp) - to the end of it: page views and addresses of people and of bots, - referrers, campaign tags (once per address), and the people each game - here received from the hall.""" + to the end of it: page views and addresses of people and of bots, where + links to the hall were shared, referrers, campaign tags (once per + address), and the people each game here received from the hall. + + A request is a bot's when its browser says so. An address is a bot's for + the day when it asks for a scanner's paths, or loads the hall three + visits or more without ever following a card to a game: a monitor.""" t0, t1 = _bounds(day) if since is not None: t0 = max(t0, since) - t = dict(views=0, addresses=set(), botAddresses=set(), referrers=Counter(), campaigns=Counter(), sent={}) - seen_campaign = set() - for r in _lines(LOGS, t0): - if not (t0 <= r.get("ts", 0) < t1): - continue - q = r.get("request", {}) - if q.get("host", "").split(":")[0] != SITE: - continue - h = q.get("headers", {}) - ua = " ".join(h.get("User-Agent", [""])) - ip = q.get("client_ip") or q.get("remote_ip") or "?" - uri = q.get("uri", "") - path = uri.split("?")[0] - if BOT.search(ua) or not ua: - t["botAddresses"].add(ip) - continue - if path not in ("/", "/index.html"): - continue - t["views"] += 1 - t["addresses"].add(ip) - ref = " ".join(h.get("Referer", [""])) - if ref and SITE not in ref: - t["referrers"][re.sub(r"^https?://", "", ref).split("/")[0]] += 1 - m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri) - if m and (ip, m.group(1)) not in seen_campaign: - seen_campaign.add((ip, m.group(1))) - t["campaigns"][m.group(1) + ("=" + m.group(2)[:24] if m.group(1) != "fbclid" else "")] += 1 + t = dict(views=0, addresses=set(), botAddresses=set(), shared=Counter(), scanners=0, monitors=0, referrers=Counter(), campaigns=Counter(), sent={}) + followed = set() for slug, host in GAMES.items(): people = set() for r in _lines("/var/lib/caddy/%s-access*.log*" % slug, t0): @@ -89,4 +105,49 @@ def scan(day, since=None): continue people.add(q.get("client_ip") or q.get("remote_ip") or "?") t["sent"][slug] = len(people) + followed |= people + + hall = [r for r in _lines(LOGS, t0) if t0 <= r.get("ts", 0) < t1 and r.get("request", {}).get("host", "").split(":")[0] == SITE] + times, scanning = {}, set() + for r in hall: + q = r.get("request", {}) + ua = " ".join(q.get("headers", {}).get("User-Agent", [""])) + if BOT.search(ua) or not ua: + continue + ip = q.get("client_ip") or q.get("remote_ip") or "?" + times.setdefault(ip, []).append(r.get("ts", 0)) + if SCANNER.search(q.get("uri", "").split("?")[0]): + scanning.add(ip) + monitors = {ip for ip, ts in times.items() if ip not in scanning and ip not in followed and _visits(ts) >= 3} + posing = scanning | monitors + t["scanners"], t["monitors"] = len(scanning), len(monitors) + + seen_campaign = set() + seen_share = set() + for r in hall: + q = r.get("request", {}) + h = q.get("headers", {}) + ua = " ".join(h.get("User-Agent", [""])) + ip = q.get("client_ip") or q.get("remote_ip") or "?" + uri = q.get("uri", "") + path = uri.split("?")[0] + if ip in posing or BOT.search(ua) or not ua: + t["botAddresses"].add(ip) + service = unfurled_by(ua) + # One share per app and page: a preview is often fetched twice. + if service and (service, path) not in seen_share: + seen_share.add((service, path)) + t["shared"][service] += 1 + continue + if path not in ("/", "/index.html"): + continue + t["views"] += 1 + t["addresses"].add(ip) + ref = " ".join(h.get("Referer", [""])) + if ref and SITE not in ref: + t["referrers"][re.sub(r"^https?://", "", ref).split("/")[0]] += 1 + m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri) + if m and (ip, m.group(1)) not in seen_campaign: + seen_campaign.add((ip, m.group(1))) + t["campaigns"][m.group(1) + ("=" + m.group(2)[:24] if m.group(1) != "fbclid" else "")] += 1 return t diff --git a/template/deploy/__SLUG__-rollup.sh b/template/deploy/__SLUG__-rollup.sh index a028525..123c601 100755 --- a/template/deploy/__SLUG__-rollup.sh +++ b/template/deploy/__SLUG__-rollup.sh @@ -58,7 +58,8 @@ for line in open("/proc/meminfo"): k, v = line.split(":", 1); mem[k] = int(v.split()[0]) load = os.getloadavg()[1] row = dict(day=day, requests=t["requests"], human=t["human"], bot=t["bot"], humanAddresses=len(t["addresses"]), botAddresses=len(t["botAddresses"]), - paths=dict(t["pages"]), roomLinks=sum(t["joins"].values()), referrers=dict(t["referrers"]), campaigns=dict(t["campaigns"]), + paths=dict(t["pages"]), roomLinks=sum(t["joins"].values()), roomsOpened=t["opened"], sharedOn=dict(t["shared"]), + scanners=t["scanners"], monitors=t["monitors"], referrers=dict(t["referrers"]), campaigns=dict(t["campaigns"]), rooms=dict(created=created, started=started, withTurns=turned, finished=finished, humanNames=len(humans), botSeats=bots), turns=turns, ledgers=len(glob.glob("/var/lib/__SLUG__/rooms/*.jsonl")), vitals=dict(diskUsedPct=round(disk.used * 100 / disk.total), memAvailableMb=mem.get("MemAvailable", 0) // 1024, load5=round(load, 2)), diff --git a/template/deploy/__SLUG__-visitors.sh b/template/deploy/__SLUG__-visitors.sh index 6861099..957a755 100755 --- a/template/deploy/__SLUG__-visitors.sh +++ b/template/deploy/__SLUG__-visitors.sh @@ -26,9 +26,9 @@ print("live sockets: %s | rooms touched in the last hour: %s" % ( # --- traffic: Caddy's access log, people and bots apart --- t = traffic.scan(day) -print("traffic: %d addresses of people, %d page requests (%d bot addresses); hall %d, rules %d, guide %d, room links %s; referrers %s; campaigns %s" % ( - len(t["addresses"]), t["human"], len(t["botAddresses"]), t["pages"]["hall"], t["pages"]["rules"], t["pages"]["guide"], - dict(t["joins"]) or "none", dict(t["referrers"]) or "none", dict(t["campaigns"]) or "none")) +print("traffic: %d addresses of people, %d page requests (%d bot addresses, of them %d scanners, %d monitors); hall %d, rules %d, guide %d; rooms opened here %d, room links %s; shared on %s; referrers %s; campaigns %s" % ( + len(t["addresses"]), t["human"], len(t["botAddresses"]), t["scanners"], t["monitors"], t["pages"]["hall"], t["pages"]["rules"], t["pages"]["guide"], + t["opened"], dict(t["joins"]) or "none", dict(t["shared"]) or "none", dict(t["referrers"]) or "none", dict(t["campaigns"]) or "none")) # --- rooms: every ledger, today's in detail --- first_seen = {} diff --git a/template/deploy/traffic.py b/template/deploy/traffic.py index f4eb073..2e288c0 100644 --- a/template/deploy/traffic.py +++ b/template/deploy/traffic.py @@ -7,23 +7,60 @@ import gzip import json import os import re -from collections import Counter +from collections import Counter, defaultdict -BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I) +BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|whatsapp|telegram|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I) +# Link previews: the app that fetched the page to show it in a conversation. The people +# who click such a link arrive with no referrer, so this is the best sign of where it went. +UNFURL = [ + ("Slack", re.compile(r"slack", re.I)), + ("Discord", re.compile(r"discord", re.I)), + ("iMessage", re.compile(r"facebookexternalhit.*twitterbot|applebot.*preview", re.I)), + ("Facebook", re.compile(r"facebookexternalhit|facebot|meta-externalagent", re.I)), + ("WhatsApp", re.compile(r"whatsapp", re.I)), + ("Telegram", re.compile(r"telegram", re.I)), + ("Reddit", re.compile(r"redditbot", re.I)), + ("LinkedIn", re.compile(r"linkedinbot", re.I)), + ("Mastodon", re.compile(r"mastodon", re.I)), + ("Bluesky", re.compile(r"bluesky|cardyb", re.I)), + ("X", re.compile(r"twitterbot", re.I)), +] +# Paths nobody asks a game for but a vulnerability scanner: its address is a bot's all day. +# Matched against whole path segments, never the site's own hashed assets under /_app/. +SCANNER = re.compile(r"\.php$|(^|/)(\.env|\.git|\.aws|wp-admin|wp-login|wp-content|wp-includes|wordpress|phpmyadmin|cgi-bin|vendor|actuator|boaform|owa|admin)(/|$)", re.I) PAGES = {"/": "hall", "/rules": "rules", "/guide": "guide"} SITE = "__DOMAIN__" # Each game on a shared host keeps its own access log. LOGS = "/var/lib/caddy/__SLUG__-access*.log*" +# The site's own addresses, for telling a referrer from a move between its pages: its domain, and its sslip.io name. +OWN = re.compile(re.escape(SITE) + r"|\.sslip\.io\b", re.I) +# Requests from one address more than this far apart are separate visits. +VISIT_GAP = 30 * 60 + + +def unfurled_by(ua): + """The app a link preview came from, or None.""" + for name, pattern in UNFURL: + if pattern.search(ua): + return name + return None def scan(day, logs=LOGS): """Counts for one UTC day (YYYY-MM-DD): every request, page requests by people, the addresses of people and of bots, requests by page, room links - opened, referrers, and campaign tags (each counted once per address).""" + opened, rooms opened from the site itself, where links were shared, + referrers, and campaign tags (each counted once per address). + + A request is a bot's when its browser says so. An address is a bot's for + the whole day when it asks for a scanner's paths, or comes back for pages + three visits or more without ever opening or joining a game: a monitor, + not a person. A monitor that takes a new address every visit gives itself + away by keeping time: one browser, page loads at a steady interval, four + visits or more, never a game.""" d0 = datetime.datetime.fromisoformat(day).replace(tzinfo=datetime.timezone.utc) t0, t1 = d0.timestamp(), (d0 + datetime.timedelta(days=1)).timestamp() - t = dict(requests=0, human=0, bot=0, addresses=set(), botAddresses=set(), pages=Counter(), joins=Counter(), referrers=Counter(), campaigns=Counter()) - seen_campaign = set() + requests = [] for f in sorted(glob.glob(logs)): if os.path.getmtime(f) < t0: continue @@ -35,34 +72,100 @@ def scan(day, logs=LOGS): r = json.loads(line) except ValueError: continue - if not (t0 <= r.get("ts", 0) < t1): - continue - q = r.get("request", {}) - h = q.get("headers", {}) - ua = " ".join(h.get("User-Agent", [""])) - ip = q.get("client_ip") or q.get("remote_ip") or "?" - uri = q.get("uri", "") - path = uri.split("?")[0] - t["requests"] += 1 - if BOT.search(ua) or not ua: - t["bot"] += 1 - t["botAddresses"].add(ip) - continue - if path.startswith(("/_app/", "/api/")) or path == "/ws" or path.endswith((".png", ".ico", ".txt")): - continue - t["human"] += 1 - t["addresses"].add(ip) - if path in PAGES: - t["pages"][PAGES[path]] += 1 - elif path.startswith("/join/"): - t["joins"][path.split("/")[2].upper()] += 1 - ref = " ".join(h.get("Referer", [""])) - if ref and SITE not in ref: - t["referrers"][re.sub(r"^https?://", "", ref).split("/")[0]] += 1 - m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri) - if m and (ip, m.group(1)) not in seen_campaign: - seen_campaign.add((ip, m.group(1))) - t["campaigns"][m.group(1) + ("=" + m.group(2)[:24] if m.group(1) != "fbclid" else "")] += 1 + if t0 <= r.get("ts", 0) < t1: + requests.append(r) except OSError: continue + + # First pass: what each address did all day under a browser's name, to find scanners and monitors. + seen = defaultdict(lambda: dict(times=[], played=False, scanner=False)) + for r in requests: + q = r.get("request", {}) + ip = q.get("client_ip") or q.get("remote_ip") or "?" + ua = " ".join(q.get("headers", {}).get("User-Agent", [""])) + if BOT.search(ua) or not ua: + continue + path = q.get("uri", "").split("?")[0] + a = seen[ip] + a["times"].append(r.get("ts", 0)) + if not path.startswith("/_app/") and SCANNER.search(path): + a["scanner"] = True + if path.startswith(("/join/", "/api/rooms")) or path == "/ws": + a["played"] = True + + def visits(times): + n, last = 0, None + for ts in sorted(times): + if last is None or ts - last > VISIT_GAP: + n += 1 + last = ts + return n + + def keeps_time(times): + starts, last = [], None + for ts in sorted(times): + if last is None or ts - last > VISIT_GAP: + starts.append(ts) + last = ts + gaps = [b - a for a, b in zip(starts, starts[1:])] + if len(starts) < 4: + return False + typical = sorted(gaps)[len(gaps) // 2] + return all(0.75 * typical <= g <= 1.25 * typical for g in gaps) + + scanners = {ip for ip, a in seen.items() if a["scanner"]} + monitors = {ip for ip, a in seen.items() if not (a["scanner"] or a["played"]) and visits(a["times"]) >= 3} + by_browser = defaultdict(list) + for r in requests: + q = r.get("request", {}) + ip = q.get("client_ip") or q.get("remote_ip") or "?" + ua = " ".join(q.get("headers", {}).get("User-Agent", [""])) + if ua and not BOT.search(ua) and q.get("uri", "").split("?")[0] in PAGES and not seen[ip]["played"]: + by_browser[ua].append((r.get("ts", 0), ip)) + for hits in by_browser.values(): + if keeps_time([ts for ts, _ in hits]): + monitors |= {ip for _, ip in hits} + posing = scanners | monitors + + t = dict( + requests=0, human=0, bot=0, addresses=set(), botAddresses=set(), pages=Counter(), joins=Counter(), + opened=0, shared=Counter(), scanners=len(scanners), monitors=len(monitors), referrers=Counter(), campaigns=Counter(), + ) + seen_campaign = set() + seen_share = set() + for r in requests: + q = r.get("request", {}) + h = q.get("headers", {}) + ua = " ".join(h.get("User-Agent", [""])) + ip = q.get("client_ip") or q.get("remote_ip") or "?" + uri = q.get("uri", "") + path = uri.split("?")[0] + t["requests"] += 1 + if ip in posing or BOT.search(ua) or not ua: + t["bot"] += 1 + t["botAddresses"].add(ip) + service = unfurled_by(ua) + # One share per app and page: a preview is often fetched twice. + if service and (service, path) not in seen_share: + seen_share.add((service, path)) + t["shared"][service] += 1 + continue + # A room opened from the site itself never passes through /join: count the API's answer. + if q.get("method") == "POST" and path == "/api/rooms" and 200 <= r.get("status", 0) < 300: + t["opened"] += 1 + if path.startswith(("/_app/", "/api/")) or path == "/ws" or path.endswith((".png", ".ico", ".txt")): + continue + t["human"] += 1 + t["addresses"].add(ip) + if path in PAGES: + t["pages"][PAGES[path]] += 1 + elif path.startswith("/join/"): + t["joins"][path.split("/")[2].upper()] += 1 + ref = " ".join(h.get("Referer", [""])) + if ref and not OWN.search(ref): + t["referrers"][re.sub(r"^https?://", "", ref).split("/")[0]] += 1 + m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri) + if m and (ip, m.group(1)) not in seen_campaign: + seen_campaign.add((ip, m.group(1))) + t["campaigns"][m.group(1) + ("=" + m.group(2)[:24] if m.group(1) != "fbclid" else "")] += 1 return t