game-kit/gamehall/ops/traffic.py

"""Caddy's access logs for one UTC day: the hall's visitors, people and
bots apart, and the people each game on this host received from the hall.
The digest and the nightly rollup both count through here, so they agree."""

import datetime
import glob
import gzip
import json
import os
import re
from collections import Counter

# The bot, link-preview and scanner patterns are kept in step with template/deploy/traffic.py.
BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|whatsapp|telegram|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I)
# Link previews: the app that fetched the page to show it in a conversation. The people
# who click such a link arrive with no referrer, so this is the best sign of where it went.
UNFURL = [
    ("Slack", re.compile(r"slack", re.I)),
    ("Discord", re.compile(r"discord", re.I)),
    ("iMessage", re.compile(r"facebookexternalhit.*twitterbot|applebot.*preview", re.I)),
    ("Facebook", re.compile(r"facebookexternalhit|facebot|meta-externalagent", re.I)),
    ("WhatsApp", re.compile(r"whatsapp", re.I)),
    ("Telegram", re.compile(r"telegram", re.I)),
    ("Reddit", re.compile(r"redditbot", re.I)),
    ("LinkedIn", re.compile(r"linkedinbot", re.I)),
    ("Mastodon", re.compile(r"mastodon", re.I)),
    ("Bluesky", re.compile(r"bluesky|cardyb", re.I)),
    ("X", re.compile(r"twitterbot", re.I)),
]
# Paths nobody asks the hall for but a vulnerability scanner: its address is a bot's all day.
SCANNER = re.compile(r"\.php$|(^|/)(\.env|\.git|\.aws|wp-admin|wp-login|wp-content|wp-includes|wordpress|phpmyadmin|cgi-bin|vendor|actuator|boaform|owa|admin)(/|$)", re.I)
# Requests from one address more than this far apart are separate visits.
VISIT_GAP = 30 * 60
SITE = "gamehall.kestrelsnest.social"
LOGS = "/var/lib/caddy/gamehall-access*.log*"
# The games that share this host, by the name the hall links to. A visitor
# who follows a card arrives with the hall as referrer (the hall sends its
# origin and nothing more). Wiz-War keeps its own host and its own count.
GAMES = {"dip": "dip.kestrelsnest.social", "waving-hands": "hands.kestrelsnest.social", "hnefatafl": "tafl.kestrelsnest.social"}
ASSET = (".png", ".jpg", ".svg", ".ico", ".txt", ".webmanifest")


def unfurled_by(ua):
    """The app a link preview came from, or None."""
    for name, pattern in UNFURL:
        if pattern.search(ua):
            return name
    return None


def _visits(times):
    n, last = 0, None
    for ts in sorted(times):
        if last is None or ts - last > VISIT_GAP:
            n += 1
        last = ts
    return n


def _lines(pattern, t0):
    for f in sorted(glob.glob(pattern)):
        if os.path.getmtime(f) < t0:
            continue
        opener = gzip.open if f.endswith(".gz") else open
        try:
            with opener(f, "rt", errors="ignore") as fh:
                for line in fh:
                    try:
                        yield json.loads(line)
                    except ValueError:
                        continue
        except OSError:
            continue


def _bounds(day):
    d0 = datetime.datetime.fromisoformat(day).replace(tzinfo=datetime.timezone.utc)
    return d0.timestamp(), (d0 + datetime.timedelta(days=1)).timestamp()


def scan(day, since=None):
    """Counts for one UTC day (YYYY-MM-DD), or from `since` (a timestamp)
    to the end of it: page views and addresses of people and of bots, where
    links to the hall were shared, referrers, campaign tags (once per
    address), and the people each game here received from the hall.

    A request is a bot's when its browser says so. An address is a bot's for
    the day when it asks for a scanner's paths, or loads the hall three
    visits or more without ever following a card to a game: a monitor."""
    t0, t1 = _bounds(day)
    if since is not None:
        t0 = max(t0, since)
    t = dict(views=0, addresses=set(), botAddresses=set(), shared=Counter(), scanners=0, monitors=0, referrers=Counter(), campaigns=Counter(), sent={})
    followed = set()
    for slug, host in GAMES.items():
        people = set()
        for r in _lines("/var/lib/caddy/%s-access*.log*" % slug, t0):
            if not (t0 <= r.get("ts", 0) < t1):
                continue
            q = r.get("request", {})
            h = q.get("headers", {})
            if SITE not in " ".join(h.get("Referer", [""])) or BOT.search(" ".join(h.get("User-Agent", [""]))):
                continue
            if q.get("uri", "").split("?")[0].endswith(ASSET):
                continue
            people.add(q.get("client_ip") or q.get("remote_ip") or "?")
        t["sent"][slug] = len(people)
        followed |= people

    hall = [r for r in _lines(LOGS, t0) if t0 <= r.get("ts", 0) < t1 and r.get("request", {}).get("host", "").split(":")[0] == SITE]
    times, scanning = {}, set()
    for r in hall:
        q = r.get("request", {})
        ua = " ".join(q.get("headers", {}).get("User-Agent", [""]))
        if BOT.search(ua) or not ua:
            continue
        ip = q.get("client_ip") or q.get("remote_ip") or "?"
        times.setdefault(ip, []).append(r.get("ts", 0))
        if SCANNER.search(q.get("uri", "").split("?")[0]):
            scanning.add(ip)
    monitors = {ip for ip, ts in times.items() if ip not in scanning and ip not in followed and _visits(ts) >= 3}
    posing = scanning | monitors
    t["scanners"], t["monitors"] = len(scanning), len(monitors)

    seen_campaign = set()
    seen_share = set()
    for r in hall:
        q = r.get("request", {})
        h = q.get("headers", {})
        ua = " ".join(h.get("User-Agent", [""]))
        ip = q.get("client_ip") or q.get("remote_ip") or "?"
        uri = q.get("uri", "")
        path = uri.split("?")[0]
        if ip in posing or BOT.search(ua) or not ua:
            t["botAddresses"].add(ip)
            service = unfurled_by(ua)
            # One share per app and page: a preview is often fetched twice.
            if service and (service, path) not in seen_share:
                seen_share.add((service, path))
                t["shared"][service] += 1
            continue
        if path not in ("/", "/index.html"):
            continue
        t["views"] += 1
        t["addresses"].add(ip)
        ref = " ".join(h.get("Referer", [""]))
        if ref and SITE not in ref:
            t["referrers"][re.sub(r"^https?://", "", ref).split("/")[0]] += 1
        m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri)
        if m and (ip, m.group(1)) not in seen_campaign:
            seen_campaign.add((ip, m.group(1)))
            t["campaigns"][m.group(1) + ("=" + m.group(2)[:24] if m.group(1) != "fbclid" else "")] += 1
    return t