#!/usr/bin/env bash
# Monitoring daemon - runs via cron, checks all critical metrics
# Install: add to cron as */5 * * * * /opt/server-resilience/bin/monitoring-daemon

set -uo pipefail

STATE_DIR="/opt/server-resilience/etc/state"
LOG_FILE="/opt/server-resilience/logs/monitor-$(date +%Y%m%d).log"
ALERT_SCRIPT="/opt/server-resilience/bin/send-alert"
PREFLIGHT="/opt/server-resilience/bin/preflight-check"

mkdir -p "$STATE_DIR"

log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >> "$LOG_FILE"; }

alert_once() {
    local key="$1" level="$2" msg="$3"
    local state_file="$STATE_DIR/alert_${key// /_}"
    local last_alert=0
    [[ -f "$state_file" ]] && last_alert=$(cat "$state_file" 2>/dev/null || echo 0)
    local now=$(date +%s)
    local cooldown=1800  # 30 minutes between repeat WARN/INFO alerts

    # CRITICAL/DOWN still alert, but a *repeating* critical respects its own
    # cooldown (2h) so a permanently-stuck condition (expired staging cert, a
    # load spike that never clears) can't email hundreds of times a day. The
    # event is still recorded every cycle in the daily digest by alert-spool,
    # so nothing is lost — only the duplicate immediate emails are suppressed.
    local crit_cooldown=7200  # 2 hours between repeat CRITICAL/DOWN alerts
    local eff_cooldown=$cooldown
    [[ "$level" == "CRITICAL" || "$level" == "DOWN" ]] && eff_cooldown=$crit_cooldown

    if [[ $((now - last_alert)) -gt $eff_cooldown ]]; then
        "$ALERT_SCRIPT" "$level" "$msg" 2>/dev/null || true
        echo "$now" > "$state_file"
        log "ALERT SENT [$level]: $msg"
    fi
}

clear_alert() {
    local key="$1"
    local state_file="$STATE_DIR/alert_${key// /_}"
    if [[ -f "$state_file" ]]; then
        rm -f "$state_file"
        log "Alert cleared: $key"
    fi
}

# ─── 1. APACHE DOWN CHECK ─────────────────────────────────────────────────────
if ! systemctl is-active httpd &>/dev/null; then
    log "Apache is DOWN"
    alert_once "apache_down" "DOWN" "Apache (httpd) is not running on $(hostname)"

    # Auto-recovery attempt: run preflight and restart if clean
    if "$PREFLIGHT" &>/dev/null; then
        log "Preflight passed - attempting auto-recovery restart"
        if systemctl start httpd &>/dev/null && systemctl is-active httpd &>/dev/null; then
            log "Auto-recovery: Apache restarted successfully"
            "$ALERT_SCRIPT" "RECOVERED" "Apache auto-recovered after being found down" 2>/dev/null || true
            clear_alert "apache_down"
        else
            log "Auto-recovery FAILED"
            alert_once "apache_down_hard" "CRITICAL" "Apache DOWN and auto-recovery failed on $(hostname). Manual intervention required."
        fi
    else
        log "Preflight failed - not auto-restarting (config issue)"
        alert_once "apache_preflight" "CRITICAL" "Apache DOWN and preflight check also failing on $(hostname). Config broken."
    fi
else
    clear_alert "apache_down"
    clear_alert "apache_down_hard"
    clear_alert "apache_preflight"
fi

# ─── 2. MEMORY PRESSURE ───────────────────────────────────────────────────────
MEM_AVAIL_MB=$(free -m | awk '/^Mem/{print $7}')
MEM_TOTAL_MB=$(free -m | awk '/^Mem/{print $2}')
MEM_AVAIL_PCT=$((MEM_AVAIL_MB * 100 / MEM_TOTAL_MB))

SWAP_USED_MB=$(free -m | awk '/^Swap/{print $3}')
SWAP_TOTAL_MB=$(free -m | awk '/^Swap/{print $2}')

if [[ $MEM_AVAIL_PCT -lt 5 ]]; then
    alert_once "mem_critical" "CRITICAL" "RAM critically low: only ${MEM_AVAIL_MB}MB available (${MEM_AVAIL_PCT}%) on $(hostname)"
elif [[ $MEM_AVAIL_PCT -lt 15 ]]; then
    alert_once "mem_warn" "WARN" "RAM low: ${MEM_AVAIL_MB}MB available (${MEM_AVAIL_PCT}%) on $(hostname)"
else
    clear_alert "mem_critical"
    clear_alert "mem_warn"
fi

if [[ $SWAP_TOTAL_MB -gt 0 ]]; then
    SWAP_PCT=$((SWAP_USED_MB * 100 / SWAP_TOTAL_MB))
    if [[ $SWAP_PCT -gt 95 ]]; then
        alert_once "swap_critical" "CRITICAL" "SWAP exhausted: ${SWAP_USED_MB}/${SWAP_TOTAL_MB}MB (${SWAP_PCT}%) on $(hostname). OOM risk."
    elif [[ $SWAP_PCT -gt 70 ]]; then
        alert_once "swap_warn" "WARN" "SWAP heavy usage: ${SWAP_USED_MB}/${SWAP_TOTAL_MB}MB (${SWAP_PCT}%) on $(hostname)"
    else
        clear_alert "swap_critical"
        clear_alert "swap_warn"
    fi
fi

# ─── 3. DISK SPACE ────────────────────────────────────────────────────────────
while IFS= read -r line; do
    PCT=$(echo "$line" | awk '{print $5}' | tr -d '%')
    MOUNT=$(echo "$line" | awk '{print $6}')
    KEY="disk_${MOUNT//\//_}"
    if [[ $PCT -gt 95 ]]; then
        alert_once "$KEY" "CRITICAL" "Disk ${PCT}% full on $MOUNT ($(hostname))"
    elif [[ $PCT -gt 85 ]]; then
        alert_once "$KEY" "WARN" "Disk ${PCT}% full on $MOUNT ($(hostname))"
    else
        clear_alert "$KEY"
    fi
done < <(df -h 2>/dev/null | awk 'NR>1 && /^\/dev/' | grep -v "tmpfs\|devtmpfs")

# ─── 4. HIGH CPU ──────────────────────────────────────────────────────────────
LOAD1=$(uptime | awk -F'load average:' '{print $2}' | cut -d, -f1 | xargs)
NCPU=$(nproc 2>/dev/null || echo 1)
LOAD_PCT=$(echo "$LOAD1 $NCPU" | awk '{printf "%d", ($1/$2)*100}')
if [[ $LOAD_PCT -gt 200 ]]; then
    TOP_PROC=$(ps aux --sort=-%cpu 2>/dev/null | awk 'NR==2{printf "%s (%.1f%% CPU)", $11, $3}')
    alert_once "cpu_critical" "CRITICAL" "CPU load ${LOAD_PCT}% on $(hostname). Load: $LOAD1 / ${NCPU} CPUs. Top: $TOP_PROC"
elif [[ $LOAD_PCT -gt 120 ]]; then
    alert_once "cpu_warn" "WARN" "CPU load high: ${LOAD_PCT}% on $(hostname) (load: $LOAD1)"
else
    clear_alert "cpu_critical"
    clear_alert "cpu_warn"
fi

# ─── 5. SSL EXPIRY CHECK (runs less frequently) ───────────────────────────────
# Only full SSL scan every 6 hours (check via timestamp)
SSL_CHECK_STAMP="$STATE_DIR/ssl_last_check"
NOW=$(date +%s)
LAST_SSL_CHECK=$(cat "$SSL_CHECK_STAMP" 2>/dev/null || echo 0)
if [[ $((NOW - LAST_SSL_CHECK)) -gt 21600 ]]; then
    echo "$NOW" > "$SSL_CHECK_STAMP"
    TODAY_TS=$NOW
    CRITICAL_TS=$((NOW + 7*86400))
    WARN_TS=$((NOW + 21*86400))
    # Domains we never alert on for cert expiry: staging/dev vhosts and any
    # domain whose DNS does not resolve to this server. AutoSSL cannot renew a
    # cert for a name that points elsewhere, so alerting on it forever is noise,
    # not signal. Maintained in /etc/server-resilience/ssl-ignore (one regex/line).
    SSL_IGNORE_FILE="/etc/server-resilience/ssl-ignore"
    is_ssl_ignored() {
        local d="$1"
        # Built-in: anything under a staging./dev./test. label
        [[ "$d" =~ (^|\.)(staging|dev|test)\. ]] && return 0
        [[ -f "$SSL_IGNORE_FILE" ]] || return 1
        grep -qiE -f "$SSL_IGNORE_FILE" <<<"$d" 2>/dev/null
    }

    for dir in /var/cpanel/ssl/apache_tls/*/; do
        domain=$(basename "$dir")
        combined="$dir/combined"
        key_file="$dir/key"
        [[ ! -f "$combined" || ! -f "$key_file" ]] && continue

        # Skip excluded (staging/dev/off-server) domains entirely.
        is_ssl_ignored "$domain" && { log "SSL check: skipping excluded domain $domain"; continue; }

        # Check mismatch on active vhosts
        grep -q "apache_tls/$domain/" /etc/apache2/conf/httpd.conf /etc/apache2/conf.d/*.conf 2>/dev/null || continue

        cert_md5=$(openssl x509 -noout -modulus -in "$combined" 2>/dev/null | openssl md5 | awk '{print $2}')
        key_md5=$(openssl rsa -noout -modulus -in "$key_file" 2>/dev/null | openssl md5 | awk '{print $2}')
        if [[ -n "$cert_md5" && "$cert_md5" != "$key_md5" ]]; then
            alert_once "ssl_mismatch_$domain" "CRITICAL" "SSL cert/key MISMATCH on active vhost: $domain ($(hostname)). Apache WILL fail to restart."
        fi

        expiry_str=$(openssl x509 -noout -enddate -in "$combined" 2>/dev/null | cut -d= -f2)
        expiry_ts=$(date -d "$expiry_str" +%s 2>/dev/null || echo 0)
        [[ $expiry_ts -eq 0 ]] && continue
        if [[ $expiry_ts -lt $TODAY_TS ]]; then
            alert_once "ssl_expired_$domain" "CRITICAL" "SSL cert EXPIRED: $domain on $(hostname)"
        elif [[ $expiry_ts -lt $CRITICAL_TS ]]; then
            days=$(( (expiry_ts - TODAY_TS) / 86400 ))
            alert_once "ssl_expiring_$domain" "WARN" "SSL cert expiring in ${days}d: $domain on $(hostname)"
        fi
    done
    log "SSL expiry check completed"
fi

log "Monitoring cycle complete. Apache=$(systemctl is-active httpd 2>/dev/null). RAM=${MEM_AVAIL_PCT}% avail. Load=${LOAD1}"