"""Caddy's access logs for one UTC day: the hall's visitors, people and
bots apart, and the people each game on this host received from the hall.
The digest and the nightly rollup both count through here, so they agree."""

import datetime
import glob
import gzip
import json
import os
import re
from collections import Counter

BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I)
SITE = "gamehall.kestrelsnest.social"
LOGS = "/var/lib/caddy/gamehall-access*.log*"
# The games that share this host, by the name the hall links to. A visitor
# who follows a card arrives with the hall as referrer (the hall sends its
# origin and nothing more). Wiz-War keeps its own host and its own count.
GAMES = {"dip": "dip.kestrelsnest.social", "waving-hands": "hands.kestrelsnest.social", "hnefatafl": "tafl.kestrelsnest.social"}
ASSET = (".png", ".jpg", ".svg", ".ico", ".txt", ".webmanifest")


def _lines(pattern, t0):
    for f in sorted(glob.glob(pattern)):
        if os.path.getmtime(f) < t0:
            continue
        opener = gzip.open if f.endswith(".gz") else open
        try:
            with opener(f, "rt", errors="ignore") as fh:
                for line in fh:
                    try:
                        yield json.loads(line)
                    except ValueError:
                        continue
        except OSError:
            continue


def _bounds(day):
    d0 = datetime.datetime.fromisoformat(day).replace(tzinfo=datetime.timezone.utc)
    return d0.timestamp(), (d0 + datetime.timedelta(days=1)).timestamp()


def scan(day, since=None):
    """Counts for one UTC day (YYYY-MM-DD), or from `since` (a timestamp)
    to the end of it: page views and addresses of people and of bots,
    referrers, campaign tags (once per address), and the people each game
    here received from the hall."""
    t0, t1 = _bounds(day)
    if since is not None:
        t0 = max(t0, since)
    t = dict(views=0, addresses=set(), botAddresses=set(), referrers=Counter(), campaigns=Counter(), sent={})
    seen_campaign = set()
    for r in _lines(LOGS, t0):
        if not (t0 <= r.get("ts", 0) < t1):
            continue
        q = r.get("request", {})
        if q.get("host", "").split(":")[0] != SITE:
            continue
        h = q.get("headers", {})
        ua = " ".join(h.get("User-Agent", [""]))
        ip = q.get("client_ip") or q.get("remote_ip") or "?"
        uri = q.get("uri", "")
        path = uri.split("?")[0]
        if BOT.search(ua) or not ua:
            t["botAddresses"].add(ip)
            continue
        if path not in ("/", "/index.html"):
            continue
        t["views"] += 1
        t["addresses"].add(ip)
        ref = " ".join(h.get("Referer", [""]))
        if ref and SITE not in ref:
            t["referrers"][re.sub(r"^https?://", "", ref).split("/")[0]] += 1
        m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri)
        if m and (ip, m.group(1)) not in seen_campaign:
            seen_campaign.add((ip, m.group(1)))
            t["campaigns"][m.group(1) + ("=" + m.group(2)[:24] if m.group(1) != "fbclid" else "")] += 1
    for slug, host in GAMES.items():
        people = set()
        for r in _lines("/var/lib/caddy/%s-access*.log*" % slug, t0):
            if not (t0 <= r.get("ts", 0) < t1):
                continue
            q = r.get("request", {})
            h = q.get("headers", {})
            if SITE not in " ".join(h.get("Referer", [""])) or BOT.search(" ".join(h.get("User-Agent", [""]))):
                continue
            if q.get("uri", "").split("?")[0].endswith(ASSET):
                continue
            people.add(q.get("client_ip") or q.get("remote_ip") or "?")
        t["sent"][slug] = len(people)
    return t