"""Caddy's access log for one day, people and bots apart. The visitors
digest and the nightly rollup both count through here, so they agree."""

import datetime
import glob
import gzip
import json
import os
import re
from collections import Counter

BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I)
PAGES = {"/": "hall", "/rules": "rules", "/guide": "guide"}
SITE = "tafl.kestrelsnest.social"
LOGS = "/var/lib/caddy/access*.log*"


def scan(day, logs=LOGS):
    """Counts for one UTC day (YYYY-MM-DD): every request, page requests by
    people, the addresses of people and of bots, requests by page, room links
    opened, referrers, and campaign tags (each counted once per address)."""
    d0 = datetime.datetime.fromisoformat(day).replace(tzinfo=datetime.timezone.utc)
    t0, t1 = d0.timestamp(), (d0 + datetime.timedelta(days=1)).timestamp()
    t = dict(requests=0, human=0, bot=0, addresses=set(), botAddresses=set(), pages=Counter(), joins=Counter(), referrers=Counter(), campaigns=Counter())
    seen_campaign = set()
    for f in sorted(glob.glob(logs)):
        if os.path.getmtime(f) < t0:
            continue
        opener = gzip.open if f.endswith(".gz") else open
        try:
            with opener(f, "rt", errors="ignore") as fh:
                for line in fh:
                    try:
                        r = json.loads(line)
                    except ValueError:
                        continue
                    if not (t0 <= r.get("ts", 0) < t1):
                        continue
                    q = r.get("request", {})
                    h = q.get("headers", {})
                    ua = " ".join(h.get("User-Agent", [""]))
                    ip = q.get("client_ip") or q.get("remote_ip") or "?"
                    uri = q.get("uri", "")
                    path = uri.split("?")[0]
                    t["requests"] += 1
                    if BOT.search(ua) or not ua:
                        t["bot"] += 1
                        t["botAddresses"].add(ip)
                        continue
                    if path.startswith(("/_app/", "/api/")) or path == "/ws" or path.endswith((".png", ".ico", ".txt")):
                        continue
                    t["human"] += 1
                    t["addresses"].add(ip)
                    if path in PAGES:
                        t["pages"][PAGES[path]] += 1
                    elif path.startswith("/join/"):
                        t["joins"][path.split("/")[2].upper()] += 1
                    ref = " ".join(h.get("Referer", [""]))
                    if ref and SITE not in ref:
                        t["referrers"][re.sub(r"^https?://", "", ref).split("/")[0]] += 1
                    m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri)
                    if m and (ip, m.group(1)) not in seen_campaign:
                        seen_campaign.add((ip, m.group(1)))
                        t["campaigns"][m.group(1) + ("=" + m.group(2)[:24] if m.group(1) != "fbclid" else "")] += 1
        except OSError:
            continue
    return t