#!/bin/bash

################################################################################
# Intelligent Website Error Analyzer
################################################################################
# Purpose: Find REAL website issues affecting REAL users
# Filters out: Bots, plugin warnings, deprecation notices, minor issues
# Focuses on: Critical errors that break sites for actual visitors
################################################################################

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
source "$SCRIPT_DIR/lib/common-functions.sh"
source "$SCRIPT_DIR/lib/system-detect.sh"
source "$SCRIPT_DIR/lib/user-manager.sh"
source "$SCRIPT_DIR/lib/ip-reputation.sh"

# Configuration - Use system-detected paths
APACHE_ERROR_LOG="/var/log/apache2/error_log"  # Will be auto-detected
DOMLOGS_DIR="${SYS_LOG_DIR}"  # From system-detect.sh
MODSEC_AUDIT_LOG="/usr/local/apache/logs/modsec_audit.log"  # Will be auto-detected
HOURS_TO_ANALYZE=24
FILTER_USER=""
FILTER_DOMAIN=""

# Multi-panel support
CONTROL_PANEL="${SYS_CONTROL_PANEL}"

print_banner "Intelligent Website Error Analyzer"

echo ""
echo "Finding REAL issues affecting REAL users..."
echo ""
echo "What we filter out:"
echo "  ✗ Bot/scanner activity"
echo "  ✗ Plugin deprecation warnings"
echo "  ✗ PHP notices (non-critical)"
echo "  ✗ Random 404s for missing favicons/assets"
echo "  ✗ Security probe attempts"
echo ""
echo "What we focus on:"
echo "  ✓ 500 errors users actually see"
echo "  ✓ Fatal PHP errors breaking functionality"
echo "  ✓ Database connection failures"
echo "  ✓ Permission issues blocking access"
echo "  ✓ Memory exhaustion affecting performance"
echo ""

# Ask for filtering scope
echo -e "${CYAN}Analysis Scope:${NC}"
echo "  1) All users/domains (default)"
echo "  2) Specific cPanel user"
echo "  3) Specific domain"
echo "  0) Cancel and return to menu"
echo ""
read -p "Select option [1]: " scope_choice
scope_choice=${scope_choice:-1}

case $scope_choice in
    0)
        echo ""
        echo "Analysis cancelled."
        echo ""
        exit 0
        ;;
    2)
        # Select specific user
        select_user_interactive "Select cPanel user to analyze"
        if [ -n "$SELECTED_USER" ]; then
            FILTER_USER="$SELECTED_USER"
            echo "→ Filtering for user: $FILTER_USER"
        else
            echo ""
            echo "No user selected. Analysis cancelled."
            echo ""
            exit 0
        fi
        ;;
    3)
        # Enter specific domain
        echo ""
        read -p "Enter domain name (e.g., example.com) or 0 to cancel: " FILTER_DOMAIN
        if [ "$FILTER_DOMAIN" = "0" ] || [ -z "$FILTER_DOMAIN" ]; then
            echo ""
            echo "Analysis cancelled."
            echo ""
            exit 0
        fi
        echo "→ Filtering for domain: $FILTER_DOMAIN"
        ;;
    *)
        echo "→ Analyzing all users/domains"
        ;;
esac

echo ""

# Ask for time range
echo -e "${CYAN}How far back should we analyze?${NC}"
echo "  1) Last 1 hour"
echo "  2) Last 6 hours"
echo "  3) Last 24 hours (default)"
echo "  4) Last 7 days"
echo "  5) Last 30 days"
echo "  0) Cancel and return to menu"
echo ""
read -p "Select option [3]: " time_choice
time_choice=${time_choice:-3}

case $time_choice in
    0)
        echo ""
        echo "Analysis cancelled."
        echo ""
        exit 0
        ;;
    1) HOURS_TO_ANALYZE=1 ;;
    2) HOURS_TO_ANALYZE=6 ;;
    3) HOURS_TO_ANALYZE=24 ;;
    4) HOURS_TO_ANALYZE=168 ;;
    5) HOURS_TO_ANALYZE=720 ;;
    *) HOURS_TO_ANALYZE=24 ;;
esac

echo ""
echo "→ Analyzing last $HOURS_TO_ANALYZE hours..."
echo ""

# Temporary files
TEMP_DIR="/tmp/website-error-analysis-$$"
mkdir -p "$TEMP_DIR"
trap "rm -rf $TEMP_DIR" EXIT

CRITICAL_ERRORS="$TEMP_DIR/critical.txt"
USER_IMPACT_ERRORS="$TEMP_DIR/user_impact.txt"
LOG_FILES_LIST="$TEMP_DIR/log_files.txt"

# Discover all log file locations
echo "→ Discovering log file locations..."
> "$LOG_FILES_LIST"

# Apache main error log (only if not filtering by user)
if [ -z "$FILTER_USER" ] && [ -z "$FILTER_DOMAIN" ]; then
    for log in /var/log/apache2/error_log /usr/local/apache/logs/error_log /var/log/httpd/error_log; do
        [ -f "$log" ] && echo "$log|apache_main" >> "$LOG_FILES_LIST"
    done
fi

# Per-domain PHP error logs - Multi-panel support
if [ -n "$FILTER_USER" ]; then
    # Specific user
    find "/home/$FILTER_USER" -name "error_log" -type f 2>/dev/null | while read -r log; do
        echo "$log|php_$FILTER_USER" >> "$LOG_FILES_LIST"
    done
elif [ -n "$FILTER_DOMAIN" ]; then
    # Try to find domain's user using multi-panel method
    user=""
    case "$CONTROL_PANEL" in
        cpanel)
            user=$(grep "^${FILTER_DOMAIN}:" /etc/userdatadomains 2>/dev/null | cut -d: -f2 | awk -F'==' '{print $1}' | head -1)
            ;;
        interworx)
            user=$(grep -l "ServerName ${FILTER_DOMAIN}" /etc/httpd/conf.d/vhost_*.conf 2>/dev/null | head -1 | \
                   xargs grep "SuexecUserGroup" 2>/dev/null | awk '{print $2}')
            ;;
        plesk)
            # Plesk domain-to-user lookup would require DB query
            user=$(plesk bin subscription --info "$FILTER_DOMAIN" 2>/dev/null | grep "Owner" | awk '{print $2}')
            ;;
    esac

    if [ -n "$user" ]; then
        find "/home/$user" -name "error_log" -type f 2>/dev/null | while read -r log; do
            echo "$log|php_$FILTER_DOMAIN" >> "$LOG_FILES_LIST"
        done
    fi
else
    # All users - Search based on control panel structure
    case "$CONTROL_PANEL" in
        cpanel)
            find /home/*/public_html -name "error_log" -type f 2>/dev/null | while read -r log; do
                username=$(echo "$log" | grep -oE '/home/[^/]+' | sed 's|/home/||')
                echo "$log|php_$username" >> "$LOG_FILES_LIST"
            done
            ;;
        interworx)
            find /home/*/*/html -name "error_log" -type f 2>/dev/null | while read -r log; do
                username=$(echo "$log" | grep -oE '/home/[^/]+' | sed 's|/home/||')
                echo "$log|php_$username" >> "$LOG_FILES_LIST"
            done
            ;;
        plesk)
            find /var/www/vhosts/*/httpdocs -name "error_log" -type f 2>/dev/null | while read -r log; do
                domain=$(echo "$log" | grep -oE '/vhosts/[^/]+' | sed 's|/vhosts/||')
                echo "$log|php_$domain" >> "$LOG_FILES_LIST"
            done
            ;;
        *)
            # Standalone - try common locations
            find /var/www/html -name "error_log" -type f 2>/dev/null | while read -r log; do
                echo "$log|php_standalone" >> "$LOG_FILES_LIST"
            done
            ;;
    esac
fi

# Per-domain Apache logs - Multi-panel support
case "$CONTROL_PANEL" in
    cpanel)
        # cPanel: Centralized domlogs directory
        if [ -d "$DOMLOGS_DIR" ]; then
            if [ -n "$FILTER_DOMAIN" ]; then
                # Specific domain
                for log in "$DOMLOGS_DIR/$FILTER_DOMAIN" "$DOMLOGS_DIR/$FILTER_DOMAIN-"*; do
                    [ -f "$log" ] && echo "$log|domlog_$FILTER_DOMAIN" >> "$LOG_FILES_LIST"
                done
            elif [ -n "$FILTER_USER" ]; then
                # Specific user - use get_user_domains from user-manager.sh
                local user_domains=$(get_user_domains "$FILTER_USER" 2>/dev/null)
                if [ -n "$user_domains" ]; then
                    while IFS= read -r domain; do
                        for log in "$DOMLOGS_DIR/$domain" "$DOMLOGS_DIR/$domain-"*; do
                            [ -f "$log" ] && echo "$log|domlog_$domain" >> "$LOG_FILES_LIST"
                        done
                    done <<< "$user_domains"
                fi
            else
                # All domains
                for log in "$DOMLOGS_DIR"/*; do
                    [ -f "$log" ] && ! [[ "$log" =~ (bytes_log|offset|error_log|ftpxferlog)$ ]] && \
                        echo "$log|domlog_$(basename "$log")" >> "$LOG_FILES_LIST"
                done
            fi
        fi
        ;;

    interworx)
        # InterWorx: Per-domain logs in user home directories
        if [ -n "$FILTER_DOMAIN" ]; then
            # Specific domain - find its user
            local user=$(grep -l "ServerName ${FILTER_DOMAIN}" /etc/httpd/conf.d/vhost_*.conf 2>/dev/null | head -1 | \
                         xargs grep "SuexecUserGroup" 2>/dev/null | awk '{print $2}')
            if [ -n "$user" ]; then
                local log="/home/${user}/var/${FILTER_DOMAIN}/logs/transfer.log"
                [ -f "$log" ] && echo "$log|domlog_$FILTER_DOMAIN" >> "$LOG_FILES_LIST"
            fi
        elif [ -n "$FILTER_USER" ]; then
            # Specific user - get their domains
            local user_domains=$(get_user_domains "$FILTER_USER" 2>/dev/null)
            if [ -n "$user_domains" ]; then
                while IFS= read -r domain; do
                    local log="/home/${FILTER_USER}/var/${domain}/logs/transfer.log"
                    [ -f "$log" ] && echo "$log|domlog_$domain" >> "$LOG_FILES_LIST"
                done <<< "$user_domains"
            fi
        else
            # All domains - find all transfer.log files (InterWorx uses 'transfer.log' not 'access_log')
            find /home/*/var/*/logs -type f -name "transfer.log" 2>/dev/null | while read -r log; do
                local domain=$(echo "$log" | grep -oE '/var/[^/]+' | sed 's|/var/||')
                echo "$log|domlog_$domain" >> "$LOG_FILES_LIST"
            done
        fi
        ;;

    plesk)
        # Plesk: System vhosts logs
        if [ -n "$FILTER_DOMAIN" ]; then
            # Specific domain
            for log in /var/www/vhosts/system/"$FILTER_DOMAIN"/logs/access_log \
                       /var/www/vhosts/system/"$FILTER_DOMAIN"/logs/access_ssl_log; do
                [ -f "$log" ] && echo "$log|domlog_$FILTER_DOMAIN" >> "$LOG_FILES_LIST"
            done
        elif [ -n "$FILTER_USER" ]; then
            # Specific user - get their domains
            local user_domains=$(get_user_domains "$FILTER_USER" 2>/dev/null)
            if [ -n "$user_domains" ]; then
                while IFS= read -r domain; do
                    for log in /var/www/vhosts/system/"$domain"/logs/access_log \
                               /var/www/vhosts/system/"$domain"/logs/access_ssl_log; do
                        [ -f "$log" ] && echo "$log|domlog_$domain" >> "$LOG_FILES_LIST"
                    done
                done <<< "$user_domains"
            fi
        else
            # All domains
            find /var/www/vhosts/system/*/logs -type f \( -name "access_log" -o -name "access_ssl_log" \) 2>/dev/null | \
            while read -r log; do
                local domain=$(echo "$log" | grep -oE '/system/[^/]+' | sed 's|/system/||')
                echo "$log|domlog_$domain" >> "$LOG_FILES_LIST"
            done
        fi
        ;;

    *)
        # Standalone Apache - try common locations
        if [ -f "/var/log/httpd/access_log" ]; then
            echo "/var/log/httpd/access_log|domlog_standalone" >> "$LOG_FILES_LIST"
        fi
        if [ -f "/var/log/apache2/access.log" ]; then
            echo "/var/log/apache2/access.log|domlog_standalone" >> "$LOG_FILES_LIST"
        fi
        ;;
esac

# ModSecurity audit log (only if not filtering or matches filter)
if [ -f "$MODSEC_AUDIT_LOG" ]; then
    if [ -z "$FILTER_USER" ] && [ -z "$FILTER_DOMAIN" ]; then
        echo "$MODSEC_AUDIT_LOG|modsec_audit" >> "$LOG_FILES_LIST"
    elif [ -n "$FILTER_DOMAIN" ]; then
        # Will filter ModSec entries during processing
        echo "$MODSEC_AUDIT_LOG|modsec_audit" >> "$LOG_FILES_LIST"
    fi
fi

log_count=$(wc -l < "$LOG_FILES_LIST" 2>/dev/null || echo "0")
echo "  Found $log_count log files to analyze"
if [ -n "$FILTER_USER" ]; then
    echo "  Scope: User '$FILTER_USER'"
elif [ -n "$FILTER_DOMAIN" ]; then
    echo "  Scope: Domain '$FILTER_DOMAIN'"
fi
echo ""

################################################################################
# Intelligent Filtering
################################################################################

is_noise() {
    local line="$1"
    local line_lower="${line,,}"  # Convert to lowercase once

    # Bot/Scanner patterns
    [[ "$line_lower" =~ (bot|crawler|spider|scanner|nikto|nmap|masscan|sqlmap|nessus|acunetix|burp|shodan|censys|zgrab|nuclei|semrush|ahrefs|mj12) ]] && return 0

    # Known scanner IPs
    [[ "$line" =~ (45\.148\.|185\.220\.|89\.248\.165\.) ]] && return 0

    # WordPress plugin deprecation warnings (not user-facing)
    [[ "$line" =~ (PHP\ Deprecated|Deprecated.*plugin|call_user_func_array.*deprecated) ]] && return 0

    # Non-critical PHP notices (unless critical indicators present)
    if [[ "$line" =~ PHP\ Notice.*Undefined\ (variable|index|offset) ]] && ! [[ "$line_lower" =~ (fatal|error\ 500|white\ screen) ]]; then
        return 0
    fi

    # Missing favicon/common assets (not real errors)
    [[ "$line_lower" =~ (favicon\.ico|apple-touch-icon|robots\.txt|sitemap\.xml.*404) ]] && return 0

    # Security probes
    if [[ "$line_lower" =~ (wp-admin|wp-login|phpmyadmin|\.env|\.git|config\.php|xmlrpc|eval-stdin|shell\.php|c99\.php|r57\.php|adminer\.php) ]] && \
       [[ "$line" =~ (404|403|not\ found) ]]; then
        return 0
    fi

    # WordPress auto-updates and cron (normal operations)
    [[ "$line_lower" =~ (wp-cron\.php|doing_cron|auto.*update) ]] && return 0

    # Common plugin update checks (not errors)
    [[ "$line_lower" =~ (update-check|version-check|api\.wordpress\.org) ]] && return 0

    return 1
}

is_critical_user_facing() {
    local line="$1"
    local line_lower="${line,,}"

    # 500 Internal Server Error (users see white page)
    [[ "$line" =~ (\ 500\ |Internal\ Server\ Error) ]] && return 0

    # PHP Fatal Errors (breaks functionality)
    [[ "$line" =~ (PHP\ Fatal\ error|Fatal\ error:.*in\ /) ]] && return 0

    # PHP Parse Errors (site completely broken)
    [[ "$line" =~ (PHP\ Parse\ error|syntax\ error) ]] && return 0

    # Database connection failures (site down)
    [[ "$line" =~ (Error\ establishing.*database|Can\'t\ connect.*MySQL|Access\ denied\ for\ user.*database|Too\ many\ connections|MySQL\ server\ has\ gone\ away) ]] && return 0

    # Memory exhaustion (white screen/incomplete pages)
    [[ "$line" =~ (Allowed\ memory\ size.*exhausted|Out\ of\ memory|Fatal.*memory) ]] && return 0

    # Segmentation fault (complete crash)
    [[ "$line" =~ (Segmentation\ fault|signal\ 11) ]] && return 0

    # Permission denied on critical files
    if [[ "$line" =~ Permission\ denied ]] && [[ "$line" =~ (index\.php|wp-config\.php|\.htaccess|config\.php) ]]; then
        return 0
    fi

    # File not found for MAIN page (404 on homepage/index)
    [[ "$line" =~ (File\ does\ not\ exist.*index\.(php|html)|\ 404\ .*\"\ \"GET\ /) ]] && return 0

    return 1
}

extract_useful_info() {
    local line="$1"
    local domain="unknown"
    local file_path=""
    local error_msg

    # Extract domain using bash regex (faster than grep|sed pipeline)
    if [[ "$line" =~ \[vhost\ ([^:]+) ]]; then
        domain="${BASH_REMATCH[1]}"
    elif [[ "$line" =~ ([a-zA-Z0-9.-]+\.(com|net|org|io|co|uk|us|dev)) ]]; then
        domain="${BASH_REMATCH[1]}"
    elif [[ "$line" =~ /home/([^/]+) ]]; then
        domain="${BASH_REMATCH[1]}"
    fi

    # Extract file path if PHP error
    if [[ "$line" =~ in\ (/[^ ]+\.php) ]]; then
        file_path="${BASH_REMATCH[1]}"
    fi

    # Extract error message (clean up ModSec noise, timestamps, etc.)
    # Use single sed command instead of pipeline
    error_msg=$(echo "$line" | sed -E 's/^\[.*\] //; s/\[client [^]]*\] //; s/\[unique_id "[^"]*"\]//g; s/\[pid [^]]*\]//g; s/\[tid [^]]*\]//g' | cut -c1-150)

    # Skip if error message is empty or just whitespace
    error_msg="${error_msg#"${error_msg%%[![:space:]]*}"}"  # ltrim
    error_msg="${error_msg%"${error_msg##*[![:space:]]}"}"  # rtrim
    [ -z "$error_msg" ] && return 1

    # Correlate to root cause
    local root_cause=$(correlate_root_cause "$line" "$error_msg" "$domain")

    echo "$domain|$file_path|$error_msg|$root_cause"
}

correlate_root_cause() {
    local line="$1"
    local error_msg="$2"
    local domain="$3"
    local cause="UNKNOWN"
    local line_lower="${line,,}"
    local error_lower="${error_msg,,}"

    # .htaccess issues
    if [[ "$line_lower" =~ (\.htaccess|invalid\ command|rewriterule|rewritecond) ]]; then
        cause="HTACCESS"

    # ModSecurity blocks
    elif [[ "$line_lower" =~ (modsecurity|mod_security|waf) ]]; then
        cause="MODSECURITY"

    # PHP memory limit
    elif [[ "$error_lower" =~ (memory.*exhausted|allowed\ memory\ size) ]]; then
        # Get current memory limit for this domain/user
        if [ -n "$domain" ] && [ "$domain" != "unknown" ]; then
            user=$(grep -l "DNS.*$domain" /var/cpanel/users/* 2>/dev/null | head -1 | xargs basename 2>/dev/null)
            if [ -n "$user" ]; then
                mem_limit=$(grep -h "memory_limit" /home/$user/public_html/.user.ini /home/$user/.php.ini 2>/dev/null | tail -1 | cut -d'=' -f2 | tr -d ' ')
                [ -n "$mem_limit" ] && cause="PHP_MEMORY:$mem_limit" || cause="PHP_MEMORY:default"
            else
                cause="PHP_MEMORY"
            fi
        else
            cause="PHP_MEMORY"
        fi

    # PHP max execution time
    elif [[ "$error_lower" =~ (max_execution_time|maximum\ execution\ time) ]]; then
        cause="PHP_TIMEOUT"

    # PHP upload/post size limits
    elif [[ "$error_lower" =~ (upload_max_filesize|post_max_size) ]]; then
        cause="PHP_UPLOAD_LIMIT"

    # File permissions
    elif [[ "$error_lower" =~ (permission\ denied|failed\ to\ open\ stream.*permission) ]]; then
        cause="FILE_PERMISSIONS"

    # Missing PHP modules/extensions
    elif [[ "$error_lower" =~ (undefined\ function|call\ to\ undefined\ function|class\ .*\ not\ found) ]]; then
        # Try to identify which module
        if [[ "$error_lower" =~ (imagecreate|imagefilter|gd_) ]]; then
            cause="MISSING_PHP_GD"
        elif [[ "$error_msg" =~ (curl_|CURL) ]]; then
            cause="MISSING_PHP_CURL"
        elif [[ "$error_msg" =~ (json_|JSON) ]]; then
            cause="MISSING_PHP_JSON"
        elif [[ "$error_lower" =~ (mysqli|mysql_connect) ]]; then
            cause="MISSING_PHP_MYSQLI"
        else
            cause="MISSING_PHP_MODULE"
        fi

    # Database issues
    elif [[ "$error_lower" =~ (database|mysql|mysqli) ]]; then
        if [[ "$error_lower" =~ (too\ many\ connections|max_connections) ]]; then
            cause="DB_MAX_CONNECTIONS"
        elif [[ "$error_lower" =~ (access\ denied|authentication) ]]; then
            cause="DB_AUTH_FAILED"
        elif [[ "$error_lower" =~ (gone\ away|lost\ connection) ]]; then
            cause="DB_TIMEOUT"
        else
            cause="DB_ERROR"
        fi

    # Apache configuration
    elif [[ "$error_lower" =~ (invalid\ uri|request\ exceeded\ the\ limit|limitrequestline) ]]; then
        cause="APACHE_CONFIG"

    # PHP parse/syntax errors (code issues)
    elif [[ "$error_lower" =~ (parse\ error|syntax\ error|unexpected) ]]; then
        cause="PHP_SYNTAX_ERROR"

    # File not found (may indicate bad symlinks or missing files)
    elif [[ "$error_lower" =~ (no\ such\ file|failed\ to\ open\ stream) ]]; then
        cause="MISSING_FILE"

    # Generic PHP fatal error
    elif [[ "$error_lower" =~ fatal\ error ]]; then
        cause="PHP_FATAL_ERROR"

    # 500 errors without specific cause
    elif [[ "$error_msg" =~ (500|Internal\ Server) ]]; then
        cause="SERVER_ERROR_500"
    fi

    echo "$cause"
}

################################################################################
# Main Analysis
################################################################################

echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " SCANNING FOR USER-IMPACTING ERRORS"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""

total_lines=0
filtered_out=0
critical_found=0

# Analyze all discovered log files
while IFS='|' read -r log_path log_type; do
    echo "  Analyzing: $(basename "$log_path")..."

    # Determine log type
    is_access_log=false
    is_modsec_log=false
    [[ "$log_type" == domlog_* ]] && ! [[ "$log_path" =~ error ]] && is_access_log=true
    [[ "$log_type" == modsec_* ]] && is_modsec_log=true

    if $is_modsec_log; then
        # ModSecurity audit log - parse for blocks/denials
        while IFS= read -r line; do
            ((total_lines++))

            # Skip if not a security event
            echo "$line" | grep -qiE "ModSecurity.*Action.*Intercepted|status: (403|406|500)" || continue

            # Extract domain/host if filtering
            if [ -n "$FILTER_DOMAIN" ]; then
                echo "$line" | grep -q "$FILTER_DOMAIN" || continue
            fi

            # Extract useful info
            if echo "$line" | grep -qiE "SQL Injection|XSS|Command Injection|Path Traversal"; then
                ((critical_found++))
                attack_type=$(echo "$line" | grep -oiE "SQL Injection|XSS|Command Injection|Path Traversal" | head -1)
                uri=$(echo "$line" | grep -oE "uri: [^ ]+" | sed 's/uri: //' || echo "")
                domain=$(echo "$line" | grep -oE "hostname: [^ ]+" | sed 's/hostname: //' || echo "unknown")

                # Extract rule ID if available
                rule_id=$(echo "$line" | grep -oE "id \"[0-9]+\"" | grep -oE "[0-9]+" || echo "")
                root_cause="MODSECURITY"
                [ -n "$rule_id" ] && root_cause="MODSECURITY:rule_$rule_id"

                echo "$domain||ModSecurity blocked: $attack_type $uri|$root_cause" >> "$CRITICAL_ERRORS"
            fi

        done < <(tail -n 5000 "$log_path" 2>/dev/null)

    elif $is_access_log; then
        # Access log - look for 5xx status codes
        # Use time-based filtering if possible, otherwise last 50k lines
        cutoff_time=$(date -d "$HOURS_TO_ANALYZE hours ago" +%s 2>/dev/null || echo "0")

        while IFS= read -r line; do
            ((total_lines++))

            # Skip if bot/scanner
            if is_noise "$line"; then
                ((filtered_out++))
                # Track bot/scanner IPs
                read -r ip _ _ _ _ _ _ _ <<< "$line"
                if [[ "$ip" =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
                    flag_ip_attack "$ip" "BOT" 0 "Bot/scanner filtered in error analysis" >/dev/null 2>&1 &
                fi
                continue
            fi

            # Time filtering (Apache format: [DD/Mon/YYYY:HH:MM:SS +ZONE])
            if [ "$cutoff_time" != "0" ]; then
                if [[ "$line" =~ \[([0-9]{2}/[A-Z][a-z]{2}/[0-9]{4}:[0-9]{2}:[0-9]{2}:[0-9]{2}) ]]; then
                    log_date="${BASH_REMATCH[1]}"
                    log_time=$(date -d "${log_date/:/ }" +%s 2>/dev/null || echo "0")
                    [ "$log_time" != "0" ] && [ "$log_time" -lt "$cutoff_time" ] && continue
                fi
            fi

            # Extract status code and URL using bash regex and read
            if [[ "$line" =~ '"'[[:space:]](5[0-9]{2})[[:space:]] ]]; then
                status="${BASH_REMATCH[1]}"

                # Parse Apache log format: IP - - [timestamp] "METHOD URL PROTOCOL" STATUS SIZE
                read -r ip _ _ timestamp _ request status_check _ <<< "$line"

                # Extract URL from request (format: "GET /path HTTP/1.1")
                if [[ "$request" =~ '"'[A-Z]+[[:space:]]([^[:space:]]+) ]]; then
                    url="${BASH_REMATCH[1]:0:80}"
                else
                    url="/"
                fi

                # Extract timestamp
                if [[ "$line" =~ \[([^]]+)\] ]]; then
                    timestamp="${BASH_REMATCH[1]}"
                else
                    timestamp=""
                fi

                # Get domain from log filename
                domain="${log_path##*/}"  # basename
                domain="${domain%%-*}"    # remove everything after first dash

                # Apply domain filter if set
                if [ -n "$FILTER_DOMAIN" ] && [ "$domain" != "$FILTER_DOMAIN" ]; then
                    continue
                fi

                # Determine root cause from status code
                case $status in
                    500) root_cause="SERVER_ERROR_500" ;;
                    502) root_cause="APACHE_CONFIG:bad_gateway" ;;
                    503) root_cause="APACHE_CONFIG:service_unavailable" ;;
                    504) root_cause="APACHE_CONFIG:gateway_timeout" ;;
                    *) root_cause="SERVER_ERROR_5XX" ;;
                esac

                ((critical_found++))
                echo "$domain||[$timestamp] HTTP $status - $url (from $ip)|$root_cause" >> "$CRITICAL_ERRORS"
            fi

        done < <(tail -n 50000 "$log_path" 2>/dev/null)
    else
        # Error log - look for critical errors
        # Use time-based filtering if possible, otherwise last 50k lines
        cutoff_time=$(date -d "$HOURS_TO_ANALYZE hours ago" +%s 2>/dev/null || echo "0")

        while IFS= read -r line; do
            ((total_lines++))

            # Skip noise
            if is_noise "$line"; then
                ((filtered_out++))
                continue
            fi

            # Time filtering (Apache/PHP error log format: [Day Mon DD HH:MM:SS YYYY])
            if [ "$cutoff_time" != "0" ]; then
                if [[ "$line" =~ \[([A-Z][a-z]{2}\ [A-Z][a-z]{2}\ [0-9]{2}\ [0-9]{2}:[0-9]{2}:[0-9]{2}\ [0-9]{4})\] ]]; then
                    log_date="${BASH_REMATCH[1]}"
                    log_time=$(date -d "$log_date" +%s 2>/dev/null || echo "0")
                    [ "$log_time" != "0" ] && [ "$log_time" -lt "$cutoff_time" ] && continue
                fi
            fi

            # Apply user/domain filter if set
            if [ -n "$FILTER_USER" ]; then
                [[ "$line" =~ /home/$FILTER_USER ]] || continue
            fi
            if [ -n "$FILTER_DOMAIN" ]; then
                [[ "$line" =~ $FILTER_DOMAIN ]] || continue
            fi

            # Check if it's critical and user-facing
            if is_critical_user_facing "$line"; then
                ((critical_found++))
                extract_useful_info "$line" >> "$CRITICAL_ERRORS"
            fi

        done < <(tail -n 50000 "$log_path" 2>/dev/null)
    fi

done < "$LOG_FILES_LIST"

echo ""
echo "Scanned: $total_lines log entries"
echo "Filtered: $filtered_out noise entries (bots, warnings, etc.)"
echo "Found: $critical_found critical user-facing errors"
echo ""

################################################################################
# Generate Report
################################################################################

if [ ! -f "$CRITICAL_ERRORS" ] || [ ! -s "$CRITICAL_ERRORS" ]; then
    echo ""
    echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
    echo -e "${GREEN} ✓ NO CRITICAL USER-FACING ERRORS FOUND!${NC}"
    echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
    echo ""
    echo "Your websites appear to be functioning normally for real users."
    echo "All errors found were from bots, scanners, or non-critical warnings."
    echo ""
    exit 0
fi

echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " ⚠️  CRITICAL ERRORS AFFECTING REAL USERS"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""

# Group by domain and count
echo -e "${RED}${BOLD}AFFECTED WEBSITES:${NC}"
echo ""

cat "$CRITICAL_ERRORS" | cut -d'|' -f1 | sort | uniq -c | sort -rn | head -10 | while read -r count domain; do
    printf "  ${RED}●${NC} %-40s ${BOLD}%4d errors${NC}\n" "$domain" "$count"
done

echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " ERROR DETAILS (Top Issues)"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""

# Group identical errors and count them
awk -F'|' '{
    key = $1 "|" $3 "|" $4  # domain|error_msg|root_cause
    file[$1"|"$3"|"$4] = $2  # Store file path
    count[key]++
}
END {
    for (key in count) {
        split(key, parts, "|")
        domain = parts[1]
        error = parts[2]
        root_cause = parts[3]
        file_path = file[key]

        # Skip empty errors
        if (length(error) > 0) {
            print count[key] "|" domain "|" file_path "|" error "|" root_cause
        }
    }
}' "$CRITICAL_ERRORS" | sort -t'|' -k1 -rn | head -20 | while IFS='|' read -r count domain file_path error_msg root_cause; do

    # Color code by frequency
    if [ "$count" -ge 10 ]; then
        color="${RED}"
        priority="[HIGH FREQUENCY]"
    elif [ "$count" -ge 5 ]; then
        color="${YELLOW}"
        priority="[MEDIUM]"
    else
        color="${INFO_COLOR}"
        priority=""
    fi

    echo -e "${color}● Occurred $count times${NC} $priority"
    [ -n "$domain" ] && [ "$domain" != "unknown" ] && echo "  Domain: $domain"
    [ -n "$file_path" ] && echo "  File: $file_path"
    echo "  Error: $error_msg"

    # Display root cause with color coding and actionable fix
    if [ -n "$root_cause" ] && [ "$root_cause" != "UNKNOWN" ]; then
        echo -ne "  ${CYAN}Root Cause: ${BOLD}"

        case "$root_cause" in
            HTACCESS)
                echo -e "${YELLOW}⚙️  .htaccess Configuration${NC}"
                echo "  → Check .htaccess syntax in domain root"
                echo "  → Look for invalid RewriteRule or directives"
                ;;
            MODSECURITY*)
                rule=$(echo "$root_cause" | cut -d':' -f2)
                echo -e "${YELLOW}🛡️  ModSecurity WAF Block${NC}"
                [ -n "$rule" ] && echo "  → Rule ID: $rule"
                echo "  → Check if this is a false positive"
                echo "  → Review: /usr/local/apache/logs/modsec_audit.log"
                ;;
            PHP_MEMORY*)
                limit=$(echo "$root_cause" | cut -d':' -f2)
                echo -e "${RED}💾 PHP Memory Exhausted${NC}"
                [ -n "$limit" ] && echo "  → Current limit: $limit"
                echo "  → Increase memory_limit in .user.ini or php.ini"
                echo "  → Recommended: 256M minimum, 512M for WooCommerce"
                ;;
            PHP_TIMEOUT)
                echo -e "${YELLOW}⏱️  PHP Execution Timeout${NC}"
                echo "  → Increase max_execution_time in php.ini"
                echo "  → Recommended: 300 for imports/backups"
                ;;
            PHP_UPLOAD_LIMIT)
                echo -e "${YELLOW}📤 PHP Upload Size Limit${NC}"
                echo "  → Increase upload_max_filesize and post_max_size"
                echo "  → Match both values (e.g., 64M)"
                ;;
            FILE_PERMISSIONS)
                echo -e "${RED}🔒 File Permission Denied${NC}"
                echo "  → Check file ownership and permissions"
                echo "  → Files should be 644, directories 755"
                echo "  → Owner should match cPanel user"
                ;;
            MISSING_PHP_GD)
                echo -e "${RED}📦 Missing PHP GD Extension${NC}"
                echo "  → Install: yum install ea-phpXX-php-gd"
                echo "  → Required for image processing"
                ;;
            MISSING_PHP_CURL)
                echo -e "${RED}📦 Missing PHP cURL Extension${NC}"
                echo "  → Install: yum install ea-phpXX-php-curl"
                ;;
            MISSING_PHP_*)
                module=$(echo "$root_cause" | sed 's/MISSING_PHP_//' | tr '[:upper:]' '[:lower:]')
                echo -e "${RED}📦 Missing PHP Extension: $module${NC}"
                echo "  → Install: yum install ea-phpXX-php-$module"
                ;;
            DB_MAX_CONNECTIONS)
                echo -e "${RED}🗄️  Database Max Connections Reached${NC}"
                echo "  → Check: mysql -e 'SHOW VARIABLES LIKE \"max_connections\"'"
                echo "  → Increase max_connections in my.cnf"
                echo "  → Or optimize slow queries reducing connection time"
                ;;
            DB_AUTH_FAILED)
                echo -e "${RED}🗄️  Database Authentication Failed${NC}"
                echo "  → Verify credentials in wp-config.php / config files"
                echo "  → Check database user permissions"
                ;;
            DB_TIMEOUT)
                echo -e "${YELLOW}🗄️  Database Connection Timeout${NC}"
                echo "  → MySQL server may be overloaded"
                echo "  → Check slow query log"
                ;;
            DB_ERROR)
                echo -e "${RED}🗄️  Database Error${NC}"
                echo "  → Check MySQL error log for details"
                ;;
            APACHE_CONFIG*)
                issue=$(echo "$root_cause" | cut -d':' -f2)
                echo -e "${YELLOW}⚙️  Apache Configuration${NC}"
                [ -n "$issue" ] && echo "  → Issue: $issue"
                echo "  → Check Apache config and vhost settings"
                ;;
            PHP_SYNTAX_ERROR)
                echo -e "${RED}⚠️  PHP Syntax Error in Code${NC}"
                echo "  → Review recent code changes"
                echo "  → Check for missing semicolons, brackets, quotes"
                ;;
            MISSING_FILE)
                echo -e "${YELLOW}📄 File Not Found${NC}"
                echo "  → File may have been deleted or moved"
                echo "  → Check for broken symlinks"
                ;;
            PHP_FATAL_ERROR)
                echo -e "${RED}⚠️  PHP Fatal Error${NC}"
                echo "  → Review PHP error logs for details"
                echo "  → May be compatibility issue or code bug"
                ;;
            SERVER_ERROR_500)
                echo -e "${RED}🔥 Generic 500 Internal Server Error${NC}"
                echo "  → Check PHP/Apache error logs for specifics"
                ;;
            *)
                echo -e "${NC}$root_cause"
                ;;
        esac
    fi

    echo ""
done

################################################################################
# Root Cause Summary
################################################################################

echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " 📊 ROOT CAUSE BREAKDOWN"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""

# Count errors by root cause
cut -d'|' -f4 "$CRITICAL_ERRORS" | grep -v "^$" | sort | uniq -c | sort -rn | while read -r count cause; do
    # Clean up cause display
    clean_cause=$(echo "$cause" | sed 's/:.*//; s/_/ /g')

    # Color code by severity
    case "$cause" in
        PHP_MEMORY*|DB_MAX_CONNECTIONS|PHP_FATAL_ERROR|SERVER_ERROR_500)
            color="${RED}"; icon="🔥" ;;
        HTACCESS|MODSECURITY*|PHP_TIMEOUT|DB_*)
            color="${YELLOW}"; icon="⚠️ " ;;
        MISSING_PHP*|FILE_PERMISSIONS)
            color="${YELLOW}"; icon="📦" ;;
        *)
            color="${INFO_COLOR}"; icon="•" ;;
    esac

    printf "  ${color}${icon} %-35s %4d errors${NC}\n" "$clean_cause" "$count"
done

echo ""

################################################################################
# Intelligent Recommendations
################################################################################

echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " 🔧 RECOMMENDED ACTIONS"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""

# Detect issue types and provide specific recommendations
if grep -qiE "500|Internal Server" "$CRITICAL_ERRORS"; then
    echo -e "${RED}● 500 Internal Server Errors detected${NC}"
    echo "  Priority: URGENT - Users see broken pages"
    echo ""
    echo "  Actions:"
    echo "    1. Check PHP error logs for affected domains:"
    for domain in $(grep -iE "500|Internal Server" "$CRITICAL_ERRORS" | cut -d'|' -f1 | sort -u | head -3); do
        echo "       tail -100 /home/$domain/public_html/error_log"
    done
    echo ""
    echo "    2. Common causes:"
    echo "       - PHP syntax errors (check recent file changes)"
    echo "       - .htaccess syntax errors"
    echo "       - Incorrect file permissions"
    echo "       - Missing PHP modules"
    echo ""
fi

if grep -qiE "Fatal error|Parse error" "$CRITICAL_ERRORS"; then
    echo -e "${RED}● PHP Fatal/Parse Errors detected${NC}"
    echo "  Priority: URGENT - Site functionality broken"
    echo ""
    echo "  Actions:"
    echo "    1. Review recent plugin/theme updates"
    echo "    2. Check PHP version compatibility"
    echo "    3. Look for syntax errors in custom code"
    echo "    4. Temporarily disable recently added plugins"
    echo ""
fi

if grep -qiE "database|MySQL|Connection" "$CRITICAL_ERRORS"; then
    echo -e "${RED}● Database Connection Errors detected${NC}"
    echo "  Priority: CRITICAL - Site may be completely down"
    echo ""
    echo "  Actions:"
    echo "    1. Check MySQL service: systemctl status mysql"
    echo "    2. Verify database credentials in config files"
    echo "    3. Check max_connections: mysql -e 'SHOW VARIABLES LIKE \"max_connections\"'"
    echo "    4. Review current connections: mysql -e 'SHOW PROCESSLIST'"
    echo ""
fi

if grep -qiE "memory.*exhausted|Out of memory" "$CRITICAL_ERRORS"; then
    echo -e "${RED}● Memory Exhaustion detected${NC}"
    echo "  Priority: HIGH - Pages loading slowly or incompletely"
    echo ""
    echo "  Current PHP memory limit: $(php -r 'echo ini_get("memory_limit");' 2>/dev/null)"
    echo ""
    echo "  Actions:"
    echo "    1. Increase memory_limit in php.ini or .user.ini"
    echo "    2. Recommended: 256M minimum, 512M for WooCommerce"
    echo "    3. Check for memory leaks in plugins"
    echo "    4. Optimize database queries"
    echo ""
fi

if grep -qiE "ModSecurity" "$CRITICAL_ERRORS"; then
    echo -e "${YELLOW}● ModSecurity Blocks detected${NC}"
    echo "  Priority: MEDIUM - Security rules blocking requests"
    echo ""
    echo "  Actions:"
    echo "    1. Review ModSecurity audit log for false positives"
    echo "    2. Check if legitimate functionality is being blocked"
    echo "    3. Consider whitelisting specific rules if needed"
    echo "    4. Review blocked attack types to understand threats"
    echo "    5. ModSecurity log: $MODSEC_AUDIT_LOG"
    echo ""
fi

echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""
echo "Full analysis saved to: $CRITICAL_ERRORS"
echo ""
echo "Next steps:"
echo "  1. Address critical errors first (500s, database, fatal errors)"
echo "  2. Check error_log files for affected domains"
echo "  3. Test affected pages after fixes"
echo "  4. Re-run this tool to verify issues are resolved"
echo ""

press_enter