"""Caddy's access logs for one UTC day: the hall's visitors, people and bots apart, and the people each game on this host received from the hall. The digest and the nightly rollup both count through here, so they agree.""" import datetime import glob import gzip import json import os import re from collections import Counter # The bot, link-preview and scanner patterns are kept in step with template/deploy/traffic.py. BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|whatsapp|telegram|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I) # Link previews: the app that fetched the page to show it in a conversation. The people # who click such a link arrive with no referrer, so this is the best sign of where it went. UNFURL = [ ("Slack", re.compile(r"slack", re.I)), ("Discord", re.compile(r"discord", re.I)), ("iMessage", re.compile(r"facebookexternalhit.*twitterbot|applebot.*preview", re.I)), ("Facebook", re.compile(r"facebookexternalhit|facebot|meta-externalagent", re.I)), ("WhatsApp", re.compile(r"whatsapp", re.I)), ("Telegram", re.compile(r"telegram", re.I)), ("Reddit", re.compile(r"redditbot", re.I)), ("LinkedIn", re.compile(r"linkedinbot", re.I)), ("Mastodon", re.compile(r"mastodon", re.I)), ("Bluesky", re.compile(r"bluesky|cardyb", re.I)), ("X", re.compile(r"twitterbot", re.I)), ] # Paths nobody asks the hall for but a vulnerability scanner: its address is a bot's all day. SCANNER = re.compile(r"\.php$|(^|/)(\.env|\.git|\.aws|wp-admin|wp-login|wp-content|wp-includes|wordpress|phpmyadmin|cgi-bin|vendor|actuator|boaform|owa|admin)(/|$)", re.I) # Requests from one address more than this far apart are separate visits. VISIT_GAP = 30 * 60 SITE = "gamehall.kestrelsnest.social" LOGS = "/var/lib/caddy/gamehall-access*.log*" # The games that share this host, by the name the hall links to. A visitor # who follows a card arrives with the hall as referrer (the hall sends its # origin and nothing more). Wiz-War keeps its own host and its own count. GAMES = {"dip": "dip.kestrelsnest.social", "waving-hands": "hands.kestrelsnest.social", "hnefatafl": "tafl.kestrelsnest.social"} ASSET = (".png", ".jpg", ".svg", ".ico", ".txt", ".webmanifest") def unfurled_by(ua): """The app a link preview came from, or None.""" for name, pattern in UNFURL: if pattern.search(ua): return name return None def _visits(times): n, last = 0, None for ts in sorted(times): if last is None or ts - last > VISIT_GAP: n += 1 last = ts return n def _lines(pattern, t0): for f in sorted(glob.glob(pattern)): if os.path.getmtime(f) < t0: continue opener = gzip.open if f.endswith(".gz") else open try: with opener(f, "rt", errors="ignore") as fh: for line in fh: try: yield json.loads(line) except ValueError: continue except OSError: continue def _bounds(day): d0 = datetime.datetime.fromisoformat(day).replace(tzinfo=datetime.timezone.utc) return d0.timestamp(), (d0 + datetime.timedelta(days=1)).timestamp() def scan(day, since=None): """Counts for one UTC day (YYYY-MM-DD), or from `since` (a timestamp) to the end of it: page views and addresses of people and of bots, where links to the hall were shared, referrers, campaign tags (once per address), and the people each game here received from the hall. A request is a bot's when its browser says so. An address is a bot's for the day when it asks for a scanner's paths, or loads the hall three visits or more without ever following a card to a game: a monitor.""" t0, t1 = _bounds(day) if since is not None: t0 = max(t0, since) t = dict(views=0, addresses=set(), botAddresses=set(), shared=Counter(), scanners=0, monitors=0, referrers=Counter(), campaigns=Counter(), sent={}) followed = set() for slug, host in GAMES.items(): people = set() for r in _lines("/var/lib/caddy/%s-access*.log*" % slug, t0): if not (t0 <= r.get("ts", 0) < t1): continue q = r.get("request", {}) h = q.get("headers", {}) if SITE not in " ".join(h.get("Referer", [""])) or BOT.search(" ".join(h.get("User-Agent", [""]))): continue if q.get("uri", "").split("?")[0].endswith(ASSET): continue people.add(q.get("client_ip") or q.get("remote_ip") or "?") t["sent"][slug] = len(people) followed |= people hall = [r for r in _lines(LOGS, t0) if t0 <= r.get("ts", 0) < t1 and r.get("request", {}).get("host", "").split(":")[0] == SITE] times, scanning = {}, set() for r in hall: q = r.get("request", {}) ua = " ".join(q.get("headers", {}).get("User-Agent", [""])) if BOT.search(ua) or not ua: continue ip = q.get("client_ip") or q.get("remote_ip") or "?" times.setdefault(ip, []).append(r.get("ts", 0)) if SCANNER.search(q.get("uri", "").split("?")[0]): scanning.add(ip) monitors = {ip for ip, ts in times.items() if ip not in scanning and ip not in followed and _visits(ts) >= 3} posing = scanning | monitors t["scanners"], t["monitors"] = len(scanning), len(monitors) seen_campaign = set() seen_share = set() for r in hall: q = r.get("request", {}) h = q.get("headers", {}) ua = " ".join(h.get("User-Agent", [""])) ip = q.get("client_ip") or q.get("remote_ip") or "?" uri = q.get("uri", "") path = uri.split("?")[0] if ip in posing or BOT.search(ua) or not ua: t["botAddresses"].add(ip) service = unfurled_by(ua) # One share per app and page: a preview is often fetched twice. if service and (service, path) not in seen_share: seen_share.add((service, path)) t["shared"][service] += 1 continue if path not in ("/", "/index.html"): continue t["views"] += 1 t["addresses"].add(ip) ref = " ".join(h.get("Referer", [""])) if ref and SITE not in ref: t["referrers"][re.sub(r"^https?://", "", ref).split("/")[0]] += 1 m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri) if m and (ip, m.group(1)) not in seen_campaign: seen_campaign.add((ip, m.group(1))) t["campaigns"][m.group(1) + ("=" + m.group(2)[:24] if m.group(1) != "fbclid" else "")] += 1 return t