#!/usr/bin/env bash
# susipere.de server report - free, read-only, local-only.
#
# What this does: inspects THIS machine (memory headroom, disk, exposed
# ports vs firewall, nginx vhosts pointing at dead backends, TLS cert
# expiry and renewal wiring, failed/flapping services) and prints a plain
# report to your terminal. Nothing leaves this box: no network calls, no
# upload, no phone-home. Safe to read before you run it:
#   https://susipere.de/tools/server-report.sh
#
# Usage:
#   curl -fsSL https://susipere.de/tools/server-report.sh | bash
#   sudo bash server-report.sh     # a few checks (firewall, certs) need root
set -uo pipefail

VERSION="1.1.0 (2026-09-26)"
FLAGS=0
CHECKS=0

if [ -t 1 ]; then
  BOLD="$(tput bold 2>/dev/null || true)"; DIM="$(tput dim 2>/dev/null || true)"
  RED="$(tput setaf 1 2>/dev/null || true)"; YEL="$(tput setaf 3 2>/dev/null || true)"
  GRN="$(tput setaf 2 2>/dev/null || true)"; RST="$(tput sgr0 2>/dev/null || true)"
else
  BOLD=""; DIM=""; RED=""; YEL=""; GRN=""; RST=""
fi

section() { printf '\n%s== %s ==%s\n' "$BOLD" "$1" "$RST"; }
ok()      { printf '  %sOK%s   %s\n' "$GRN" "$RST" "$1"; }
warn()    { printf '  %sFLAG%s %s\n' "$YEL" "$RST" "$1"; FLAGS=$((FLAGS+1)); }
crit()    { printf '  %sFLAG%s %s\n' "$RED" "$RST" "$1"; FLAGS=$((FLAGS+1)); }
info()    { printf '  %s--%s   %s\n' "$DIM" "$RST" "$1"; }
have()    { command -v "$1" >/dev/null 2>&1; }

IS_ROOT=0
[ "$(id -u)" -eq 0 ] && IS_ROOT=1

printf '%ssusipere.de server report%s  (v%s)\n' "$BOLD" "$RST" "$VERSION"
printf 'Read-only. Local only. Nothing sent anywhere. Run again any time.\n'

# ---------------------------------------------------------------- overview
section "System"
CHECKS=$((CHECKS+1))
info "Host: $(hostname 2>/dev/null || echo unknown)"
if [ -r /etc/os-release ]; then
  # shellcheck disable=SC1091
  . /etc/os-release
  info "OS: ${PRETTY_NAME:-unknown}"
fi
info "Kernel: $(uname -r 2>/dev/null)"
info "Uptime: $(uptime -p 2>/dev/null || uptime 2>/dev/null)"
CPUS=$(nproc 2>/dev/null || echo '?')
info "CPUs: $CPUS"
if [ -r /proc/loadavg ]; then
  info "Load average (1/5/15m): $(awk '{print $1, $2, $3}' /proc/loadavg)"
fi
[ "$IS_ROOT" -eq 0 ] && info "Not running as root - firewall and TLS cert checks below will be skipped. Re-run with sudo for those."

# ---------------------------------------------------------------- memory
section "Memory & swap headroom"
CHECKS=$((CHECKS+1))
if have free; then
  read -r _ MEM_TOTAL MEM_USED MEM_FREE _ MEM_AVAIL < <(free -m | awk '/^Mem:/{print $1,$2,$3,$4,$6,$7}')
  read -r _ SWAP_TOTAL SWAP_USED SWAP_FREE < <(free -m | awk '/^Swap:/{print $1,$2,$3,$4}')
  info "RAM: ${MEM_USED}MiB used / ${MEM_TOTAL}MiB total (${MEM_AVAIL}MiB available)"
  if [ "${SWAP_TOTAL:-0}" -gt 0 ]; then
    SWAP_FREE_PCT=$(( SWAP_FREE * 100 / SWAP_TOTAL ))
    info "Swap: ${SWAP_USED}MiB used / ${SWAP_TOTAL}MiB total (${SWAP_FREE_PCT}% free)"
    if [ "$SWAP_FREE_PCT" -le 5 ]; then
      crit "Swap is almost exhausted (${SWAP_FREE_PCT}% free). The next memory spike has nowhere to go; the kernel OOM-killer picks the casualty, not you."
    elif [ "$SWAP_FREE_PCT" -le 20 ]; then
      warn "Swap headroom is getting thin (${SWAP_FREE_PCT}% free)."
    else
      ok "Swap headroom looks fine (${SWAP_FREE_PCT}% free)."
    fi
  else
    info "No swap configured."
  fi
fi
if [ -r /proc/meminfo ]; then
  COMMIT_LIMIT_KB=$(awk '/^CommitLimit:/{print $2}' /proc/meminfo)
  COMMITTED_KB=$(awk '/^Committed_AS:/{print $2}' /proc/meminfo)
  if [ -n "${COMMIT_LIMIT_KB:-}" ] && [ -n "${COMMITTED_KB:-}" ] && [ "$COMMIT_LIMIT_KB" -gt 0 ]; then
    COMMIT_PCT=$(( COMMITTED_KB * 100 / COMMIT_LIMIT_KB ))
    info "Committed memory: ${COMMIT_PCT}% of the kernel's commit limit ($(( COMMITTED_KB/1024 ))MiB / $(( COMMIT_LIMIT_KB/1024 ))MiB)"
    if [ "$COMMIT_PCT" -ge 150 ]; then
      warn "Processes have collectively promised far more memory than the box can ever back (${COMMIT_PCT}%). It can run fine for months on optimistic overcommit right up until it doesn't."
    fi
  fi
fi

# ---------------------------------------------------------------- disk
section "Disk"
CHECKS=$((CHECKS+1))
if have df; then
  while read -r SRC SIZE USED AVAIL PCT MNT; do
    [ "$SRC" = "Filesystem" ] && continue
    case "$MNT" in /snap/*|/boot/efi) continue ;; esac
    PCTNUM=${PCT%%%}
    LINE="$MNT: ${USED}/${SIZE} used (${PCT}), ${AVAIL} free"
    if [ "$PCTNUM" -ge 90 ]; then
      crit "$LINE"
    elif [ "$PCTNUM" -ge 80 ]; then
      warn "$LINE"
    else
      ok "$LINE"
    fi
  done < <(df -h -x tmpfs -x devtmpfs -x squashfs 2>/dev/null)
fi

# ---------------------------------------------------------------- ports
section "Network exposure"
CHECKS=$((CHECKS+1))
FW_ACTIVE=""
if [ "$IS_ROOT" -eq 1 ]; then
  if have ufw && ufw status 2>/dev/null | grep -qi '^Status: active'; then
    FW_ACTIVE="ufw"
  elif have nft && nft list ruleset 2>/dev/null | grep -qE 'chain (input|INPUT)'; then
    FW_ACTIVE="nftables"
  elif have iptables && [ "$(iptables -S INPUT 2>/dev/null | grep -c '^-A INPUT')" -gt 0 ]; then
    FW_ACTIVE="iptables"
  fi
  if [ -n "$FW_ACTIVE" ]; then
    ok "Host firewall active ($FW_ACTIVE)."
  else
    warn "No active host firewall detected (checked ufw/nftables/iptables). Anything listening on 0.0.0.0 below is reachable from the internet with nothing in front of it except whatever that service does itself."
  fi
else
  info "Firewall status needs root - skipped. Re-run with sudo to check."
fi

if have ss; then
  PUBLIC_PORTS=$(ss -tlnH 2>/dev/null | awk '{print $4}' | grep -vE '^(127\.0\.0\.1|\[::1\]|::1)' | sed -E 's/.*[:.]([0-9]+)$/\1/' | sort -un)
  # Ports that are ordinarily meant to be world-reachable: ssh, web, dns,
  # mail (smtp/submission/smtps, imap/imaps, pop3/pop3s). Anything else
  # listening on a public interface is worth a human's attention, and
  # doubly so with no host firewall in front of it.
  EXPECTED_PORTS=" 22 53 80 443 25 465 587 110 995 143 993 "
  if [ -n "$PUBLIC_PORTS" ]; then
    info "TCP ports listening on a public interface (not localhost-only):"
    UNUSUAL_PORTS=""
    for p in $PUBLIC_PORTS; do
      PROC=""
      if [ "$IS_ROOT" -eq 1 ]; then
        PROC=$(ss -tlnHp 2>/dev/null | awk -v port=":$p" '$4 ~ port"$" {print $NF}' | head -1)
      fi
      TAG=""
      case "$EXPECTED_PORTS" in *" $p "*) TAG=" (expected: ssh/web/dns/mail)" ;; *) UNUSUAL_PORTS="$UNUSUAL_PORTS $p" ;; esac
      printf '       %s%s%s%s\n' "$p" "${PROC:+  }" "${PROC:-}" "$TAG"
    done
    if [ -z "$FW_ACTIVE" ] && [ "$IS_ROOT" -eq 1 ]; then
      warn "No host firewall, so every port above is reachable straight from the internet - each service is its own gatekeeper. That is fine if you meant it."
    fi
    for p in $UNUSUAL_PORTS; do
      if [ "$IS_ROOT" -eq 1 ] && [ -z "$FW_ACTIVE" ]; then
        crit "Port $p is public, outside the usual ssh/web/dns/mail set, and nothing is blocking it - worth confirming it is meant to be reachable directly (e.g. a backend that should only be reached through a reverse proxy)."
      else
        warn "Port $p is public, outside the usual ssh/web/dns/mail set - worth a second look."
      fi
    done
  else
    ok "No TCP ports listening on a public interface (all bound to localhost)."
  fi
  LOOPBACK_COUNT=$(ss -tlnH 2>/dev/null | awk '{print $4}' | grep -cE '^(127\.0\.0\.1|\[::1\]|::1)')
  [ "${LOOPBACK_COUNT:-0}" -gt 0 ] && info "$LOOPBACK_COUNT TCP port(s) also listening on loopback only (127.0.0.1/::1) - not reachable from outside this machine."
else
  info "'ss' not available - skipping port scan."
fi

# ------------------------------------------------------ nginx vs reality
# The signature check: nginx can return a perfectly good HTTP status (even
# a 401 from a basic-auth layer) for a vhost whose backend is not running
# at all. "Not a 5xx" is not the same as "alive". This only ever reads
# config files and checks local listening sockets - no requests are made.
section "Reverse proxy vs reality"
CHECKS=$((CHECKS+1))
NGINX_DIRS=""
for d in /etc/nginx/sites-enabled /etc/nginx/conf.d; do
  [ -d "$d" ] && NGINX_DIRS="$NGINX_DIRS $d"
done
if have nginx && [ -n "$NGINX_DIRS" ]; then
  if [ -r /etc/nginx/sites-enabled ] || [ "$IS_ROOT" -eq 1 ]; then
    LISTENING_PORTS=" $(ss -tlnH 2>/dev/null | awk '{print $4}' | sed -E 's/.*[:.]([0-9]+)$/\1/' | sort -un | tr '\n' ' ') "
    # -L: sites-enabled is normally a directory of symlinks into
    # sites-available: without following them, "-type f" matches nothing.
    CONF_FILES=$(find -L $NGINX_DIRS -maxdepth 1 -type f 2>/dev/null | sort -u | head -200)
    FOUND_BACKEND=0
    DEAD_BACKENDS=0
    if [ -n "$CONF_FILES" ]; then
      while IFS= read -r f; do
        [ -r "$f" ] || continue
        NAME=$(basename "$f")
        # proxy_pass http(s)://host:port  -- loopback/localhost backends only;
        # a remote upstream is out of scope for a purely local check.
        while IFS= read -r backend; do
          [ -n "$backend" ] || continue
          FOUND_BACKEND=$((FOUND_BACKEND+1))
          host=${backend%%:*}
          port=${backend##*:}
          case "$host" in
            127.0.0.1|localhost|::1) ;;
            *) continue ;;
          esac
          case "$LISTENING_PORTS" in
            *" $port "*) : ;; # listening - fine
            *)
              DEAD_BACKENDS=$((DEAD_BACKENDS+1))
              crit "$NAME proxies to $host:$port, but nothing is listening on $port. nginx can still answer for this vhost (redirect, auth challenge, static error page) while the thing behind it is completely dead."
              ;;
          esac
        done < <(grep -ohE 'proxy_pass[[:space:]]+https?://[A-Za-z0-9.:-]+' "$f" 2>/dev/null | awk '{print $2}' | sed -E 's#https?://##' | sort -u)
      done <<< "$CONF_FILES"
    fi
    if [ "$FOUND_BACKEND" -eq 0 ]; then
      info "No local (127.0.0.1/localhost) proxy_pass backends found in nginx config - nothing to cross-check."
    elif [ "$DEAD_BACKENDS" -eq 0 ]; then
      ok "Checked $FOUND_BACKEND local nginx backend(s); all have something listening."
    fi
  else
    info "nginx config needs root to read - skipped. Re-run with sudo."
  fi
else
  info "nginx not found or not configured - skipping reverse-proxy check."
fi

# ---------------------------------------------------------------- TLS
section "TLS certificates"
CHECKS=$((CHECKS+1))
CERT_DIRS="/etc/letsencrypt/live"
if [ "$IS_ROOT" -eq 1 ] && [ -d "$CERT_DIRS" ] && have openssl; then
  FOUND_ANY=0
  for d in "$CERT_DIRS"/*/; do
    [ -e "$d/cert.pem" ] || continue
    FOUND_ANY=1
    NAME=$(basename "$d")
    END_DATE=$(openssl x509 -enddate -noout -in "$d/cert.pem" 2>/dev/null | cut -d= -f2)
    if [ -n "$END_DATE" ]; then
      END_EPOCH=$(date -d "$END_DATE" +%s 2>/dev/null || echo 0)
      NOW_EPOCH=$(date +%s)
      DAYS_LEFT=$(( (END_EPOCH - NOW_EPOCH) / 86400 ))
      if [ "$END_EPOCH" -eq 0 ]; then
        info "$NAME: could not parse expiry"
      elif [ "$DAYS_LEFT" -lt 0 ]; then
        crit "$NAME: certificate EXPIRED $(( -DAYS_LEFT )) days ago"
      elif [ "$DAYS_LEFT" -le 14 ]; then
        crit "$NAME: certificate expires in $DAYS_LEFT days"
      elif [ "$DAYS_LEFT" -le 30 ]; then
        warn "$NAME: certificate expires in $DAYS_LEFT days"
      else
        ok "$NAME: certificate valid for $DAYS_LEFT more days"
      fi
    fi
  done
  if [ "$FOUND_ANY" -eq 0 ]; then
    info "No Let's Encrypt certificates found under $CERT_DIRS."
  elif have systemctl; then
    # Installed is not the same as renewing. certbot ships a systemd timer;
    # some setups still rely on a cron job instead - check both before
    # concluding renewal isn't wired up.
    RENEWAL_WIRED=0
    if systemctl is-enabled certbot.timer >/dev/null 2>&1 && systemctl is-active certbot.timer >/dev/null 2>&1; then
      RENEWAL_WIRED=1
      ok "certbot.timer is enabled and active - renewal is wired up, not just installed."
    elif grep -qrl 'certbot' /etc/cron.d /etc/cron.daily /etc/cron.weekly 2>/dev/null; then
      RENEWAL_WIRED=1
      ok "Found a cron job referencing certbot - renewal appears wired up (not verified to run successfully)."
    fi
    [ "$RENEWAL_WIRED" -eq 0 ] && warn "Certificates exist but no active certbot.timer or cron job was found. They may only renew if someone remembers to run certbot by hand."
  fi
elif [ "$IS_ROOT" -eq 0 ]; then
  info "Certificate expiry needs root to read $CERT_DIRS - skipped. Re-run with sudo."
else
  info "No certificate directory or openssl found - skipping."
fi

# ---------------------------------------------------------------- systemd
section "Service health"
CHECKS=$((CHECKS+1))
if have systemctl; then
  FAILED=$(systemctl --failed --no-legend --plain 2>/dev/null)
  if [ -z "$FAILED" ]; then
    ok "No failed systemd units."
  else
    while IFS= read -r line; do
      [ -n "$line" ] && warn "systemd unit failed: $(printf '%s' "$line" | awk '{print $1}')"
    done <<< "$FAILED"
  fi

  # Flapping: a unit that keeps restarting is not "failed" (systemd may have
  # given up retrying and left it "active"), so --failed alone misses it.
  # Bounded to the first 300 loaded service units so this stays fast on a
  # box with a lot of units.
  UNITS=$(systemctl list-units --type=service --no-legend --plain --state=running,exited,failed 2>/dev/null | awk '{print $1}' | head -300)
  if [ -n "$UNITS" ]; then
    # shellcheck disable=SC2086
    RESTART_INFO=$(systemctl show $UNITS -p Id -p NRestarts 2>/dev/null)
    FLAPPING=0
    CUR_ID=""
    while IFS= read -r line; do
      case "$line" in
        Id=*) CUR_ID=${line#Id=} ;;
        NRestarts=*)
          n=${line#NRestarts=}
          if [ "${n:-0}" -ge 3 ] 2>/dev/null; then
            warn "$CUR_ID has restarted $n times since boot - flapping, worth a look even though it is not currently 'failed'."
            FLAPPING=$((FLAPPING+1))
          fi
          ;;
      esac
    done <<< "$RESTART_INFO"
    [ "$FLAPPING" -eq 0 ] && ok "No flapping units found among $(printf '%s\n' "$UNITS" | wc -l) checked (restart count >= 3 would flag)."
  fi
else
  info "systemd not present - skipping."
fi

# ---------------------------------------------------------------- summary
section "Summary"
if [ "$FLAGS" -eq 0 ]; then
  printf '  %sNo flags raised across %d checks.%s\n' "$GRN" "$CHECKS" "$RST"
else
  printf '  %s%d flag(s) raised across %d checks.%s Scroll up for detail.\n' "$BOLD" "$FLAGS" "$CHECKS" "$RST"
fi
printf '\nThis is a free, local, point-in-time read. It does not fix anything and sends nothing anywhere.\n'
printf 'Want the things a script cannot see? EUR 750 - https://susipere.de/#kontakt (placeholder until the offer page ships)\n'