From ee7b1d68ecd5d3f9bd64c7ebd8acb7b6648e280f Mon Sep 17 00:00:00 2001 From: Eric Wagoner Date: Fri, 6 Nov 2026 20:02:52 -0500 Subject: [PATCH] From Hnefatafl: a link counts as shared when a preview app fetches a page, not when Meta's crawlers or robots.txt checks pass by Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GwFKMuQnPAEHJ5yA1q4orh --- template/deploy/traffic.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/template/deploy/traffic.py b/template/deploy/traffic.py index aa1daa3..9fb46cd 100644 --- a/template/deploy/traffic.py +++ b/template/deploy/traffic.py @@ -12,11 +12,13 @@ from collections import Counter, defaultdict BOT = re.compile(r"bot|crawl|spider|slurp|facebookexternalhit|slack|discord|twitter|whatsapp|telegram|preview|fetch|curl|python|go-http|wget|headless|scan|monitor|uptime", re.I) # Link previews: the app that fetched the page to show it in a conversation. The people # who click such a link arrive with no referrer, so this is the best sign of where it went. +# Meta's crawlers (meta-externalagent, meta-webindexer) name facebook.com too but share nothing. UNFURL = [ ("Slack", re.compile(r"slack", re.I)), ("Discord", re.compile(r"discord", re.I)), + ("Google Messages", re.compile(r"googlemessages", re.I)), ("iMessage", re.compile(r"facebookexternalhit.*twitterbot|applebot.*preview", re.I)), - ("Facebook", re.compile(r"facebookexternalhit|facebot|meta-externalagent", re.I)), + ("Facebook", re.compile(r"facebookexternalhit|facebot", re.I)), ("WhatsApp", re.compile(r"whatsapp", re.I)), ("Telegram", re.compile(r"telegram", re.I)), ("Reddit", re.compile(r"redditbot", re.I)), @@ -144,8 +146,10 @@ def scan(day, logs=LOGS): t["bot"] += 1 t["botAddresses"].add(ip) service = unfurled_by(ua) - # One share per app and page: a preview is often fetched twice. - if service and (service, path) not in seen_share: + # One share per app and page: a preview is often fetched twice. Only a + # page counts: an app that already holds a preview fetches its card + # image again to show it, and robots.txt and scripts are not shared. + if service and (path in PAGES or path.startswith("/join/")) and (service, path) not in seen_share: seen_share.add((service, path)) t["shared"][service] += 1 continue