diff --git a/deploy/deploy.sh b/deploy/deploy.sh index dcfe53d..a16db81 100755 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -25,7 +25,7 @@ rsync -az --delete --filter='P _app/immutable/*' \ # it needs only its code and the engine. Ledgers live outside this tree. rsync -az --delete --exclude='/server/node_modules' \ --include='/server/***' --include='/src/' --include='/src/lib/' --include='/src/lib/game/***' \ - --include='/deploy/' --include='/deploy/waving-hands.service' --include='/deploy/waving-hands.cron' --include='/deploy/*.sh' --include='/deploy/hash-tokens.ts' \ + --include='/deploy/' --include='/deploy/waving-hands.service' --include='/deploy/waving-hands.cron' --include='/deploy/*.sh' --include='/deploy/hash-tokens.ts' --include='/deploy/traffic.py' \ --exclude='*' \ ./ "root@$HOST:/opt/waving-hands/app/" @@ -39,6 +39,7 @@ ssh "root@$HOST" ' # Cron runs the rollup and the backup from /usr/local/bin: install them on # every deploy so the live copies are always the repo copies. install -m 755 /opt/waving-hands/app/deploy/visitors.sh /usr/local/bin/waving-hands-visitors.sh + install -m 644 -D /opt/waving-hands/app/deploy/traffic.py /usr/local/lib/waving-hands/traffic.py install -m 755 /opt/waving-hands/app/deploy/pulse.sh /usr/local/bin/waving-hands-pulse.sh install -m 755 /opt/waving-hands/app/deploy/waving-hands-rollup.sh /usr/local/bin/waving-hands-rollup.sh install -m 755 /opt/waving-hands/app/deploy/waving-hands-backup.sh /usr/local/bin/waving-hands-backup.sh diff --git a/deploy/traffic.py b/deploy/traffic.py new file mode 100644 index 0000000..2d9c917 --- /dev/null +++ b/deploy/traffic.py @@ -0,0 +1,171 @@ +"""Caddy's access log for one day, people and bots apart. The visitors +digest and the nightly rollup both count through here, so they agree.""" + +import datetime +import glob +import gzip +import json +import os +import re +from collections import Counter, defaultdict + +BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|whatsapp|telegram|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I) +# Link previews: the app that fetched the page to show it in a conversation. The people +# who click such a link arrive with no referrer, so this is the best sign of where it went. +UNFURL = [ + ("Slack", re.compile(r"slack", re.I)), + ("Discord", re.compile(r"discord", re.I)), + ("iMessage", re.compile(r"facebookexternalhit.*twitterbot|applebot.*preview", re.I)), + ("Facebook", re.compile(r"facebookexternalhit|facebot|meta-externalagent", re.I)), + ("WhatsApp", re.compile(r"whatsapp", re.I)), + ("Telegram", re.compile(r"telegram", re.I)), + ("Reddit", re.compile(r"redditbot", re.I)), + ("LinkedIn", re.compile(r"linkedinbot", re.I)), + ("Mastodon", re.compile(r"mastodon", re.I)), + ("Bluesky", re.compile(r"bluesky|cardyb", re.I)), + ("X", re.compile(r"twitterbot", re.I)), +] +# Paths nobody asks a game for but a vulnerability scanner: its address is a bot's all day. +# Matched against whole path segments, never the site's own hashed assets under /_app/. +SCANNER = re.compile(r"\.php$|(^|/)(\.env|\.git|\.aws|wp-admin|wp-login|wp-content|wp-includes|wordpress|phpmyadmin|cgi-bin|vendor|actuator|boaform|owa|admin)(/|$)", re.I) +PAGES = {"/": "hall", "/play": "play", "/rules": "rules"} +SITE = "hands.kestrelsnest.social" +# Each game on a shared host keeps its own access log. +LOGS = "/var/lib/caddy/waving-hands-access*.log*" +# The site's own addresses, for telling a referrer from a move between its pages: its domain, and its sslip.io name. +OWN = re.compile(re.escape(SITE) + r"|\.sslip\.io\b", re.I) +# Requests from one address more than this far apart are separate visits. +VISIT_GAP = 30 * 60 + + +def unfurled_by(ua): + """The app a link preview came from, or None.""" + for name, pattern in UNFURL: + if pattern.search(ua): + return name + return None + + +def scan(day, logs=LOGS): + """Counts for one UTC day (YYYY-MM-DD): every request, page requests by + people, the addresses of people and of bots, requests by page, room links + opened, rooms opened from the site itself, where links were shared, + referrers, and campaign tags (each counted once per address). + + A request is a bot's when its browser says so. An address is a bot's for + the whole day when it asks for a scanner's paths, or comes back for pages + three visits or more without ever opening or joining a game: a monitor, + not a person. A monitor that takes a new address every visit gives itself + away by keeping time: one browser, page loads at a steady interval, four + visits or more, never a game.""" + d0 = datetime.datetime.fromisoformat(day).replace(tzinfo=datetime.timezone.utc) + t0, t1 = d0.timestamp(), (d0 + datetime.timedelta(days=1)).timestamp() + requests = [] + for f in sorted(glob.glob(logs)): + if os.path.getmtime(f) < t0: + continue + opener = gzip.open if f.endswith(".gz") else open + try: + with opener(f, "rt", errors="ignore") as fh: + for line in fh: + try: + r = json.loads(line) + except ValueError: + continue + if t0 <= r.get("ts", 0) < t1: + requests.append(r) + except OSError: + continue + + # First pass: what each address did all day under a browser's name, to find scanners and monitors. + seen = defaultdict(lambda: dict(times=[], played=False, scanner=False)) + for r in requests: + q = r.get("request", {}) + ip = q.get("client_ip") or q.get("remote_ip") or "?" + ua = " ".join(q.get("headers", {}).get("User-Agent", [""])) + if BOT.search(ua) or not ua: + continue + path = q.get("uri", "").split("?")[0] + a = seen[ip] + a["times"].append(r.get("ts", 0)) + if not path.startswith("/_app/") and SCANNER.search(path): + a["scanner"] = True + if path.startswith(("/join/", "/api/rooms")) or path == "/ws": + a["played"] = True + + def visits(times): + n, last = 0, None + for ts in sorted(times): + if last is None or ts - last > VISIT_GAP: + n += 1 + last = ts + return n + + def keeps_time(times): + starts, last = [], None + for ts in sorted(times): + if last is None or ts - last > VISIT_GAP: + starts.append(ts) + last = ts + gaps = [b - a for a, b in zip(starts, starts[1:])] + if len(starts) < 4: + return False + typical = sorted(gaps)[len(gaps) // 2] + return all(0.75 * typical <= g <= 1.25 * typical for g in gaps) + + scanners = {ip for ip, a in seen.items() if a["scanner"]} + monitors = {ip for ip, a in seen.items() if not (a["scanner"] or a["played"]) and visits(a["times"]) >= 3} + by_browser = defaultdict(list) + for r in requests: + q = r.get("request", {}) + ip = q.get("client_ip") or q.get("remote_ip") or "?" + ua = " ".join(q.get("headers", {}).get("User-Agent", [""])) + if ua and not BOT.search(ua) and q.get("uri", "").split("?")[0] in PAGES and not seen[ip]["played"]: + by_browser[ua].append((r.get("ts", 0), ip)) + for hits in by_browser.values(): + if keeps_time([ts for ts, _ in hits]): + monitors |= {ip for _, ip in hits} + posing = scanners | monitors + + t = dict( + requests=0, human=0, bot=0, addresses=set(), botAddresses=set(), pages=Counter(), joins=Counter(), + opened=0, shared=Counter(), scanners=len(scanners), monitors=len(monitors), referrers=Counter(), campaigns=Counter(), + ) + seen_campaign = set() + seen_share = set() + for r in requests: + q = r.get("request", {}) + h = q.get("headers", {}) + ua = " ".join(h.get("User-Agent", [""])) + ip = q.get("client_ip") or q.get("remote_ip") or "?" + uri = q.get("uri", "") + path = uri.split("?")[0] + t["requests"] += 1 + if ip in posing or BOT.search(ua) or not ua: + t["bot"] += 1 + t["botAddresses"].add(ip) + service = unfurled_by(ua) + # One share per app and page: a preview is often fetched twice. + if service and (service, path) not in seen_share: + seen_share.add((service, path)) + t["shared"][service] += 1 + continue + # A room opened from the site itself never passes through /join: count the API's answer. + if q.get("method") == "POST" and path == "/api/rooms" and 200 <= r.get("status", 0) < 300: + t["opened"] += 1 + if path.startswith(("/_app/", "/api/")) or path == "/ws" or path.endswith((".png", ".ico", ".txt")): + continue + t["human"] += 1 + t["addresses"].add(ip) + if path in PAGES: + t["pages"][PAGES[path]] += 1 + elif path.startswith("/join/"): + t["joins"][path.split("/")[2].upper()] += 1 + ref = " ".join(h.get("Referer", [""])) + if ref and not OWN.search(ref): + t["referrers"][re.sub(r"^https?://", "", ref).split("/")[0]] += 1 + m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri) + if m and (ip, m.group(1)) not in seen_campaign: + seen_campaign.add((ip, m.group(1))) + t["campaigns"][m.group(1) + ("=" + m.group(2)[:24] if m.group(1) != "fbclid" else "")] += 1 + return t diff --git a/deploy/visitors.sh b/deploy/visitors.sh index 573bfea..7e5aff6 100755 --- a/deploy/visitors.sh +++ b/deploy/visitors.sh @@ -25,50 +25,12 @@ print("live sockets: %s | rooms touched in the last hour: %s" % ( sh("find /var/lib/waving-hands/rooms -name '*.jsonl' -mmin -60 2>/dev/null | wc -l"))) # --- traffic: Caddy's access log, people and bots apart --- -BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|preview|fetch|curl|python|go-http|headless|scan|monitor|uptime", re.I) -PAGES = {"/": "hall", "/play": "play", "/rules": "rules"} -addresses, bots = set(), set() -pages, joins, referrers, campaigns, agents = Counter(), Counter(), Counter(), Counter(), Counter() -requests = 0 -for path in sorted(glob.glob("/var/lib/caddy/waving-hands-access*.log*")): - try: - with open(path) as f: - for line in f: - try: - x = json.loads(line) - except Exception: - continue - ts = datetime.datetime.fromtimestamp(x["ts"], datetime.timezone.utc) - if ts.strftime("%F") != day: - continue - req = x["request"] - ua = " ".join(req.get("headers", {}).get("User-Agent", [""])) - ip = req.get("remote_ip") or req.get("client_ip", "?") - uri = req.get("uri", "") - path_only = uri.split("?")[0] - if BOT.search(ua) or not ua: - bots.add(ip) - continue - if path_only.startswith("/_app/") or path_only.startswith("/api/") or path_only == "/ws" or path_only.endswith((".png", ".ico", ".txt")): - continue - requests += 1 - addresses.add(ip) - if path_only in PAGES: - pages[PAGES[path_only]] += 1 - elif path_only.startswith("/join/"): - joins[path_only.split("/")[2].upper()] += 1 - ref = " ".join(req.get("headers", {}).get("Referer", [""])) - if ref and "hands.kestrelsnest.social" not in ref: - referrers[re.sub(r"^https?://", "", ref).split("/")[0]] += 1 - m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri) - if m: - campaigns[m.group(1) + "=" + m.group(2)[:24]] += 1 - agents[re.sub(r"\(.*?\)", "", ua)[:40].strip()] += 1 - except OSError: - continue -print("traffic: %d addresses of people, %d page requests (%d bot addresses); hall %d, play %d, rules %d, room links %s; referrers %s; campaigns %s" % ( - len(addresses), requests, len(bots), pages["hall"], pages["play"], pages["rules"], - dict(joins) or "none", dict(referrers) or "none", dict(campaigns) or "none")) +sys.path.insert(0, "/usr/local/lib/waving-hands") +import traffic +t = traffic.scan(day) +print("traffic: %d addresses of people, %d page requests (%d bot addresses, of them %d scanners, %d monitors); hall %d, play %d, rules %d; rooms opened here %d, room links %s; shared on %s; referrers %s; campaigns %s" % ( + len(t["addresses"]), t["human"], len(t["botAddresses"]), t["scanners"], t["monitors"], t["pages"]["hall"], t["pages"]["play"], t["pages"]["rules"], + t["opened"], dict(t["joins"]) or "none", dict(t["shared"]) or "none", dict(t["referrers"]) or "none", dict(t["campaigns"]) or "none")) # --- rooms: every ledger, today's in detail --- first_seen = {} diff --git a/deploy/waving-hands-rollup.sh b/deploy/waving-hands-rollup.sh index 7d1ad76..c035d57 100755 --- a/deploy/waving-hands-rollup.sh +++ b/deploy/waving-hands-rollup.sh @@ -28,41 +28,9 @@ import collections, datetime, glob, gzip, json, os, re, shutil, sys day, out = sys.argv[1], sys.argv[2] d0 = datetime.datetime.fromisoformat(day).replace(tzinfo=datetime.timezone.utc) t0, t1 = d0.timestamp(), (d0 + datetime.timedelta(days=1)).timestamp() -BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I) - -req = human = bot = 0 -hips, bips = set(), set() -pages, joins, refs, campaigns = collections.Counter(), collections.Counter(), collections.Counter(), collections.Counter() -seen_campaign = set() -for f in sorted(glob.glob("/var/lib/caddy/waving-hands-access*.log*")): - if os.path.getmtime(f) < t0: continue - opener = gzip.open if f.endswith(".gz") else open - with opener(f, "rt", errors="ignore") as fh: - for line in fh: - try: r = json.loads(line) - except ValueError: continue - ts = r.get("ts", 0) - if not (t0 <= ts < t1): continue - q = r.get("request", {}); h = q.get("headers", {}) - ua = " ".join(h.get("User-Agent", [""])) - ip = q.get("client_ip") or q.get("remote_ip") or "?" - uri = q.get("uri", ""); path = uri.split("?")[0] - req += 1 - if BOT.search(ua) or not ua: - bot += 1; bips.add(ip); continue - if path.startswith(("/_app/", "/api/")) or path == "/ws" or path.endswith((".png", ".ico", ".txt")): continue - human += 1; hips.add(ip) - if path == "/": pages["hall"] += 1 - elif path == "/play": pages["play"] += 1 - elif path == "/rules": pages["rules"] += 1 - elif path.startswith("/join/"): joins[path.split("/")[2].upper()] += 1 - ref = " ".join(h.get("Referer", [""])) - if ref and "hands.kestrelsnest.social" not in ref: - refs[re.sub(r"^https?://", "", ref).split("/")[0]] += 1 - m = re.search(r"[?&](ref|utm_source|fbclid)=([^&]*)", uri) - if m and (ip, m.group(1)) not in seen_campaign: - seen_campaign.add((ip, m.group(1))) - campaigns[m.group(1) + ("=" + m.group(2)[:24] if m.group(1) != "fbclid" else "")] += 1 +sys.path.insert(0, "/usr/local/lib/waving-hands") +import traffic +t = traffic.scan(day) created = started = finished = turned = turns = 0 humans, bots = set(), 0 @@ -80,20 +48,19 @@ for f in glob.glob("/var/lib/waving-hands/rooms/*.jsonl"): if x["bot"]: bots += 1 else: humans.add(x["name"]) elif x["t"] == "unseat": bots -= 1 - day_turns = [x for x in lines if x["t"] == "turn" and t0 <= x["at"] / 1000 < t1] - turns += len(day_turns) - # A duel finished today: its last turn is today's and the engine would say so; approximate by replay-free means: - # the ledger's final turn falls today and no turn follows it within the day's end. - if day_turns and day_turns[-1] is lines[-1] and False: finished += 1 + turns += sum(1 for x in lines if x["t"] == "turn" and t0 <= x["at"] / 1000 < t1) + # The over line is written with the turn that ends the game, so a finish counts on its own day. + if any(x["t"] == "over" and t0 <= x["at"] / 1000 < t1 for x in lines): finished += 1 disk = shutil.disk_usage("/") mem = {} for line in open("/proc/meminfo"): k, v = line.split(":", 1); mem[k] = int(v.split()[0]) load = os.getloadavg()[1] -row = dict(day=day, requests=req, human=human, bot=bot, humanAddresses=len(hips), botAddresses=len(bips), - paths=dict(pages), roomLinks=sum(joins.values()), referrers=dict(refs), campaigns=dict(campaigns), - rooms=dict(created=created, started=started, withTurns=turned, humanNames=len(humans), botSeats=bots), +row = dict(day=day, requests=t["requests"], human=t["human"], bot=t["bot"], humanAddresses=len(t["addresses"]), botAddresses=len(t["botAddresses"]), + paths=dict(t["pages"]), roomLinks=sum(t["joins"].values()), roomsOpened=t["opened"], sharedOn=dict(t["shared"]), + scanners=t["scanners"], monitors=t["monitors"], referrers=dict(t["referrers"]), campaigns=dict(t["campaigns"]), + rooms=dict(created=created, started=started, withTurns=turned, finished=finished, humanNames=len(humans), botSeats=bots), turns=turns, ledgers=len(glob.glob("/var/lib/waving-hands/rooms/*.jsonl")), vitals=dict(diskUsedPct=round(disk.used * 100 / disk.total), memAvailableMb=mem.get("MemAvailable", 0) // 1024, load5=round(load, 2)), at=datetime.datetime.now(datetime.timezone.utc).isoformat(timespec="seconds")) diff --git a/static/family.json b/static/family.json index 48572da..83cbeac 100644 --- a/static/family.json +++ b/static/family.json @@ -25,6 +25,14 @@ "pitch": "The Viking board game. One king, one throne, four corners, and a ring of attackers closing in.", "players": "2", "origin": "the Norse tafl game, reconstructed from Linnaeus's 1732 notes; Copenhagen rules or Tablut" + }, + { + "slug": "dip", + "name": "Dip", + "url": "https://dip.kestrelsnest.social", + "pitch": "Seven powers. One map. Nobody has to keep their word.", + "players": "2 to 7", + "origin": "Allan B. Calhamer's Diplomacy, 1959; an unofficial rebuild" } ] }