#!/bin/bash
# The operations pulse, printed ON the droplet: the duel server's health,
# errors in its journal, the last week of the rollup, the backup's last word,
# and the box's vitals. Read by the waving-hands-pulse skill.
set -u
echo "as of $(date -u '+%F %H:%M') UTC on $(hostname); up $(uptime -p | sed 's/up //')"
echo "--- duel server"
systemctl is-active waving-hands >/dev/null && echo "waving-hands: active since $(systemctl show waving-hands -p ActiveEnterTimestamp --value)" || echo "waving-hands: NOT ACTIVE"
systemctl is-active caddy >/dev/null && echo "caddy: active" || echo "caddy: NOT ACTIVE"
echo "restarts in 7 days: $(journalctl -u waving-hands --since '7 days ago' -o cat | grep -c 'Started waving-hands')"
ERR=$(journalctl -u waving-hands --since '7 days ago' -p err -o cat | grep -vc '^$')
echo "journal errors in 7 days: $ERR"
journalctl -u waving-hands --since '7 days ago' -o cat | grep -iE "error|unhandled|exception" | grep -v "Rate" | tail -3 | sed 's/^/  /'
echo "--- rollup, last 7 days (people / requests / rooms opened / turns)"
if [ -s /var/lib/waving-hands/rollup.jsonl ]; then
  tail -7 /var/lib/waving-hands/rollup.jsonl | python3 -c '
import json, sys
for l in sys.stdin:
    r = json.loads(l); rm = r["rooms"]
    print("  %s  %3d people  %4d requests  %2d rooms (%d started, %d with turns)  %3d turns  bots %d addr  refs %s" % (
        r["day"], r["humanAddresses"], r["human"], rm["created"], rm["started"], rm["withTurns"], r["turns"], r["botAddresses"],
        ",".join(sorted(r["referrers"])) or "-"))'
else
  echo "  (no rollup yet)"
fi
echo "--- ledgers"
echo "rooms on disk: $(ls /var/lib/waving-hands/rooms/*.jsonl 2>/dev/null | wc -l); touched in 24h: $(find /var/lib/waving-hands/rooms -name '*.jsonl' -mmin -1440 2>/dev/null | wc -l); size $(du -sh /var/lib/waving-hands/rooms 2>/dev/null | cut -f1)"
echo "--- backup"
if [ -f /var/log/waving-hands/backup.log ]; then
  grep -E "backup (start|done)|skipped|error|ERROR" /var/log/waving-hands/backup.log | tail -2 | sed 's/^/  /'
else
  echo "  never run"
fi
echo "--- box"
echo "disk: $(df -h / | awk 'NR==2{print $5" used of "$2}'); memory: $(free -m | awk 'NR==2{print $7" MB available of "$2}'); load: $(cut -d' ' -f1-3 /proc/loadavg)"
echo "caddy access log: $(du -sh /var/lib/caddy/access.log 2>/dev/null | cut -f1) live, $(ls /var/lib/caddy/access*.log* 2>/dev/null | wc -l) files"