Files
Linux-Server-Management-Too…/modules/website/website-error-analyzer.sh
T
cschantz 296113db99 Fix SOURCE command safety issues (HIGH priority)
Added existence checks and error handling for all source commands
to prevent silent failures when dependencies are missing.

Library files (use 'return' for error):
- reference-db.sh: Added checks for 3 dependencies
- mysql-analyzer.sh: Added checks for 3 dependencies
- domain-discovery.sh: Added checks for 2 dependencies
- system-detect.sh: Added check for common-functions.sh
- plesk-helpers.sh: Added check for common-functions.sh
- user-manager.sh: Added checks for 2 dependencies

Executable scripts (use 'exit' for error):
- wordpress-cron-manager.sh: Added checks for 2 dependencies
- website-error-analyzer.sh: Added checks for 4 dependencies

Pattern: [ -f "file" ] && source "file" || { echo "ERROR" >&2; return/exit 1; }

This ensures scripts fail fast with clear error messages when
required dependencies are missing, rather than continuing with
undefined functions.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
2026-01-02 17:26:21 -05:00

1002 lines
41 KiB
Bash
Executable File

#!/bin/bash
################################################################################
# Intelligent Website Error Analyzer
################################################################################
# Purpose: Find REAL website issues affecting REAL users
# Filters out: Bots, plugin warnings, deprecation notices, minor issues
# Focuses on: Critical errors that break sites for actual visitors
################################################################################
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
[ -f "$SCRIPT_DIR/lib/common-functions.sh" ] && source "$SCRIPT_DIR/lib/common-functions.sh" || { echo "ERROR: common-functions.sh not found" >&2; exit 1; }
[ -f "$SCRIPT_DIR/lib/system-detect.sh" ] && source "$SCRIPT_DIR/lib/system-detect.sh" || { echo "ERROR: system-detect.sh not found" >&2; exit 1; }
[ -f "$SCRIPT_DIR/lib/user-manager.sh" ] && source "$SCRIPT_DIR/lib/user-manager.sh" || { echo "ERROR: user-manager.sh not found" >&2; exit 1; }
[ -f "$SCRIPT_DIR/lib/ip-reputation.sh" ] && source "$SCRIPT_DIR/lib/ip-reputation.sh" || { echo "ERROR: ip-reputation.sh not found" >&2; exit 1; }
# Configuration - Use system-detected paths
APACHE_ERROR_LOG="/var/log/apache2/error_log" # Will be auto-detected
DOMLOGS_DIR="${SYS_LOG_DIR}" # From system-detect.sh
MODSEC_AUDIT_LOG="/usr/local/apache/logs/modsec_audit.log" # Will be auto-detected
HOURS_TO_ANALYZE=24
FILTER_USER=""
FILTER_DOMAIN=""
# Multi-panel support
CONTROL_PANEL="${SYS_CONTROL_PANEL}"
print_banner "Intelligent Website Error Analyzer"
echo ""
echo "Finding REAL issues affecting REAL users..."
echo ""
echo "What we filter out:"
echo " ✗ Bot/scanner activity"
echo " ✗ Plugin deprecation warnings"
echo " ✗ PHP notices (non-critical)"
echo " ✗ Random 404s for missing favicons/assets"
echo " ✗ Security probe attempts"
echo ""
echo "What we focus on:"
echo " ✓ 500 errors users actually see"
echo " ✓ Fatal PHP errors breaking functionality"
echo " ✓ Database connection failures"
echo " ✓ Permission issues blocking access"
echo " ✓ Memory exhaustion affecting performance"
echo ""
# Ask for filtering scope
echo -e "${CYAN}Analysis Scope:${NC}"
echo " 1) All users/domains (default)"
echo " 2) Specific cPanel user"
echo " 3) Specific domain"
echo " 0) Cancel and return to menu"
echo ""
read -p "Select option [1]: " scope_choice
scope_choice=${scope_choice:-1}
case $scope_choice in
0)
echo ""
echo "Analysis cancelled."
echo ""
exit 0
;;
2)
# Select specific user
select_user_interactive "Select cPanel user to analyze"
if [ -n "$SELECTED_USER" ]; then
FILTER_USER="$SELECTED_USER"
echo "→ Filtering for user: $FILTER_USER"
else
echo ""
echo "No user selected. Analysis cancelled."
echo ""
exit 0
fi
;;
3)
# Enter specific domain
echo ""
read -p "Enter domain name (e.g., example.com) or 0 to cancel: " FILTER_DOMAIN
if [ "$FILTER_DOMAIN" = "0" ] || [ -z "$FILTER_DOMAIN" ]; then
echo ""
echo "Analysis cancelled."
echo ""
exit 0
fi
echo "→ Filtering for domain: $FILTER_DOMAIN"
;;
*)
echo "→ Analyzing all users/domains"
;;
esac
echo ""
# Ask for time range
echo -e "${CYAN}How far back should we analyze?${NC}"
echo " 1) Last 1 hour"
echo " 2) Last 6 hours"
echo " 3) Last 24 hours (default)"
echo " 4) Last 7 days"
echo " 5) Last 30 days"
echo " 0) Cancel and return to menu"
echo ""
read -p "Select option [3]: " time_choice
time_choice=${time_choice:-3}
case $time_choice in
0)
echo ""
echo "Analysis cancelled."
echo ""
exit 0
;;
1) HOURS_TO_ANALYZE=1 ;;
2) HOURS_TO_ANALYZE=6 ;;
3) HOURS_TO_ANALYZE=24 ;;
4) HOURS_TO_ANALYZE=168 ;;
5) HOURS_TO_ANALYZE=720 ;;
*) HOURS_TO_ANALYZE=24 ;;
esac
echo ""
echo "→ Analyzing last $HOURS_TO_ANALYZE hours..."
echo ""
# Temporary files
TEMP_DIR="/tmp/website-error-analysis-$$"
mkdir -p "$TEMP_DIR"
trap '[ -n "$TEMP_DIR" ] && rm -rf "$TEMP_DIR"' EXIT
CRITICAL_ERRORS="$TEMP_DIR/critical.txt"
USER_IMPACT_ERRORS="$TEMP_DIR/user_impact.txt"
LOG_FILES_LIST="$TEMP_DIR/log_files.txt"
# Discover all log file locations
echo "→ Discovering log file locations..."
> "$LOG_FILES_LIST"
# Apache main error log (only if not filtering by user)
if [ -z "$FILTER_USER" ] && [ -z "$FILTER_DOMAIN" ]; then
for log in /var/log/apache2/error_log /usr/local/apache/logs/error_log /var/log/httpd/error_log; do
[ -f "$log" ] && echo "$log|apache_main" >> "$LOG_FILES_LIST"
done
fi
# Per-domain PHP error logs - Multi-panel support
if [ -n "$FILTER_USER" ]; then
# Specific user
find "/home/$FILTER_USER" -name "error_log" -type f 2>/dev/null | while read -r log; do
echo "$log|php_$FILTER_USER" >> "$LOG_FILES_LIST"
done
elif [ -n "$FILTER_DOMAIN" ]; then
# Try to find domain's user using multi-panel method
user=""
case "$CONTROL_PANEL" in
cpanel)
user=$(grep "^${FILTER_DOMAIN}:" /etc/userdatadomains 2>/dev/null | cut -d: -f2 | awk -F'==' '{print $1}' | head -1)
;;
interworx)
user=$(grep -l "ServerName ${FILTER_DOMAIN}" /etc/httpd/conf.d/vhost_*.conf 2>/dev/null | head -1 | \
xargs grep "SuexecUserGroup" 2>/dev/null | awk '{print $2}')
;;
plesk)
# Plesk domain-to-user lookup would require DB query
user=$(plesk bin subscription --info "$FILTER_DOMAIN" 2>/dev/null | grep "Owner" | awk '{print $2}')
;;
esac
if [ -n "$user" ]; then
find "/home/$user" -name "error_log" -type f 2>/dev/null | while read -r log; do
echo "$log|php_$FILTER_DOMAIN" >> "$LOG_FILES_LIST"
done
fi
else
# All users - Search based on control panel structure
case "$CONTROL_PANEL" in
cpanel)
find /home/*/public_html -name "error_log" -type f 2>/dev/null | while read -r log; do
username=$(echo "$log" | grep -oE '/home/[^/]+' | sed 's|/home/||')
echo "$log|php_$username" >> "$LOG_FILES_LIST"
done
;;
interworx)
find /home/*/*/html -name "error_log" -type f 2>/dev/null | while read -r log; do
username=$(echo "$log" | grep -oE '/home/[^/]+' | sed 's|/home/||')
echo "$log|php_$username" >> "$LOG_FILES_LIST"
done
;;
plesk)
find /var/www/vhosts/*/httpdocs -name "error_log" -type f 2>/dev/null | while read -r log; do
domain=$(echo "$log" | grep -oE '/vhosts/[^/]+' | sed 's|/vhosts/||')
echo "$log|php_$domain" >> "$LOG_FILES_LIST"
done
;;
*)
# Standalone - try common locations
find /var/www/html -name "error_log" -type f 2>/dev/null | while read -r log; do
echo "$log|php_standalone" >> "$LOG_FILES_LIST"
done
;;
esac
fi
# Per-domain Apache logs - Multi-panel support
case "$CONTROL_PANEL" in
cpanel)
# cPanel: Centralized domlogs directory
if [ -d "$DOMLOGS_DIR" ]; then
if [ -n "$FILTER_DOMAIN" ]; then
# Specific domain
for log in "$DOMLOGS_DIR/$FILTER_DOMAIN" "$DOMLOGS_DIR/$FILTER_DOMAIN-"*; do
[ -f "$log" ] && echo "$log|domlog_$FILTER_DOMAIN" >> "$LOG_FILES_LIST"
done
elif [ -n "$FILTER_USER" ]; then
# Specific user - use get_user_domains from user-manager.sh
user_domains=$(get_user_domains "$FILTER_USER" 2>/dev/null)
if [ -n "$user_domains" ]; then
while IFS= read -r domain; do
for log in "$DOMLOGS_DIR/$domain" "$DOMLOGS_DIR/$domain-"*; do
[ -f "$log" ] && echo "$log|domlog_$domain" >> "$LOG_FILES_LIST"
done
done <<< "$user_domains"
fi
else
# All domains
for log in "$DOMLOGS_DIR"/*; do
[ -f "$log" ] && ! [[ "$log" =~ (bytes_log|offset|error_log|ftpxferlog)$ ]] && \
echo "$log|domlog_$(basename "$log")" >> "$LOG_FILES_LIST"
done
fi
fi
;;
interworx)
# InterWorx: Per-domain logs in user home directories
if [ -n "$FILTER_DOMAIN" ]; then
# Specific domain - find its user
user=$(grep -l "ServerName ${FILTER_DOMAIN}" /etc/httpd/conf.d/vhost_*.conf 2>/dev/null | head -1 | \
xargs grep "SuexecUserGroup" 2>/dev/null | awk '{print $2}')
if [ -n "$user" ]; then
log="/home/${user}/var/${FILTER_DOMAIN}/logs/transfer.log"
[ -f "$log" ] && echo "$log|domlog_$FILTER_DOMAIN" >> "$LOG_FILES_LIST"
fi
elif [ -n "$FILTER_USER" ]; then
# Specific user - get their domains
user_domains=$(get_user_domains "$FILTER_USER" 2>/dev/null)
if [ -n "$user_domains" ]; then
while IFS= read -r domain; do
log="/home/${FILTER_USER}/var/${domain}/logs/transfer.log"
[ -f "$log" ] && echo "$log|domlog_$domain" >> "$LOG_FILES_LIST"
done <<< "$user_domains"
fi
else
# All domains - find all transfer.log files (InterWorx uses 'transfer.log' not 'access_log')
find /home/*/var/*/logs -type f -name "transfer.log" 2>/dev/null | while read -r log; do
domain=$(echo "$log" | grep -oE '/var/[^/]+' | sed 's|/var/||')
echo "$log|domlog_$domain" >> "$LOG_FILES_LIST"
done
fi
;;
plesk)
# Plesk: System vhosts logs
if [ -n "$FILTER_DOMAIN" ]; then
# Specific domain
for log in /var/www/vhosts/system/"$FILTER_DOMAIN"/logs/access_log \
/var/www/vhosts/system/"$FILTER_DOMAIN"/logs/access_ssl_log; do
[ -f "$log" ] && echo "$log|domlog_$FILTER_DOMAIN" >> "$LOG_FILES_LIST"
done
elif [ -n "$FILTER_USER" ]; then
# Specific user - get their domains
user_domains=$(get_user_domains "$FILTER_USER" 2>/dev/null)
if [ -n "$user_domains" ]; then
while IFS= read -r domain; do
for log in /var/www/vhosts/system/"$domain"/logs/access_log \
/var/www/vhosts/system/"$domain"/logs/access_ssl_log; do
[ -f "$log" ] && echo "$log|domlog_$domain" >> "$LOG_FILES_LIST"
done
done <<< "$user_domains"
fi
else
# All domains
find /var/www/vhosts/system/*/logs -type f \( -name "access_log" -o -name "access_ssl_log" \) 2>/dev/null | \
while read -r log; do
domain=$(echo "$log" | grep -oE '/system/[^/]+' | sed 's|/system/||')
echo "$log|domlog_$domain" >> "$LOG_FILES_LIST"
done
fi
;;
*)
# Standalone Apache - try common locations
if [ -f "/var/log/httpd/access_log" ]; then
echo "/var/log/httpd/access_log|domlog_standalone" >> "$LOG_FILES_LIST"
fi
if [ -f "/var/log/apache2/access.log" ]; then
echo "/var/log/apache2/access.log|domlog_standalone" >> "$LOG_FILES_LIST"
fi
;;
esac
# ModSecurity audit log (only if not filtering or matches filter)
if [ -f "$MODSEC_AUDIT_LOG" ]; then
if [ -z "$FILTER_USER" ] && [ -z "$FILTER_DOMAIN" ]; then
echo "$MODSEC_AUDIT_LOG|modsec_audit" >> "$LOG_FILES_LIST"
elif [ -n "$FILTER_DOMAIN" ]; then
# Will filter ModSec entries during processing
echo "$MODSEC_AUDIT_LOG|modsec_audit" >> "$LOG_FILES_LIST"
fi
fi
log_count=$(wc -l < "$LOG_FILES_LIST" 2>/dev/null || echo "0")
echo " Found $log_count log files to analyze"
if [ -n "$FILTER_USER" ]; then
echo " Scope: User '$FILTER_USER'"
elif [ -n "$FILTER_DOMAIN" ]; then
echo " Scope: Domain '$FILTER_DOMAIN'"
fi
echo ""
################################################################################
# Intelligent Filtering
################################################################################
is_noise() {
local line="$1"
local line_lower="${line,,}" # Convert to lowercase once
# Bot/Scanner patterns
[[ "$line_lower" =~ (bot|crawler|spider|scanner|nikto|nmap|masscan|sqlmap|nessus|acunetix|burp|shodan|censys|zgrab|nuclei|semrush|ahrefs|mj12) ]] && return 0
# Known scanner IPs
[[ "$line" =~ (45\.148\.|185\.220\.|89\.248\.165\.) ]] && return 0
# WordPress plugin deprecation warnings (not user-facing)
[[ "$line" =~ (PHP\ Deprecated|Deprecated.*plugin|call_user_func_array.*deprecated) ]] && return 0
# Non-critical PHP notices (unless critical indicators present)
if [[ "$line" =~ PHP\ Notice.*Undefined\ (variable|index|offset) ]] && ! [[ "$line_lower" =~ (fatal|error\ 500|white\ screen) ]]; then
return 0
fi
# Missing favicon/common assets (not real errors)
[[ "$line_lower" =~ (favicon\.ico|apple-touch-icon|robots\.txt|sitemap\.xml.*404) ]] && return 0
# Security probes
if [[ "$line_lower" =~ (wp-admin|wp-login|phpmyadmin|\.env|\.git|config\.php|xmlrpc|eval-stdin|shell\.php|c99\.php|r57\.php|adminer\.php) ]] && \
[[ "$line" =~ (404|403|not\ found) ]]; then
return 0
fi
# WordPress auto-updates and cron (normal operations)
[[ "$line_lower" =~ (wp-cron\.php|doing_cron|auto.*update) ]] && return 0
# Common plugin update checks (not errors)
[[ "$line_lower" =~ (update-check|version-check|api\.wordpress\.org) ]] && return 0
return 1
}
is_critical_user_facing() {
local line="$1"
local line_lower="${line,,}"
# 500 Internal Server Error (users see white page)
[[ "$line" =~ (\ 500\ |Internal\ Server\ Error) ]] && return 0
# PHP Fatal Errors (breaks functionality)
[[ "$line" =~ (PHP\ Fatal\ error|Fatal\ error:.*in\ /) ]] && return 0
# PHP Parse Errors (site completely broken)
[[ "$line" =~ (PHP\ Parse\ error|syntax\ error) ]] && return 0
# Database connection failures (site down)
[[ "$line" =~ (Error\ establishing.*database|Can\'t\ connect.*MySQL|Access\ denied\ for\ user.*database|Too\ many\ connections|MySQL\ server\ has\ gone\ away) ]] && return 0
# Memory exhaustion (white screen/incomplete pages)
[[ "$line" =~ (Allowed\ memory\ size.*exhausted|Out\ of\ memory|Fatal.*memory) ]] && return 0
# Segmentation fault (complete crash)
[[ "$line" =~ (Segmentation\ fault|signal\ 11) ]] && return 0
# Permission denied on critical files
if [[ "$line" =~ Permission\ denied ]] && [[ "$line" =~ (index\.php|wp-config\.php|\.htaccess|config\.php) ]]; then
return 0
fi
# File not found for MAIN page (404 on homepage/index)
[[ "$line" =~ (File\ does\ not\ exist.*index\.(php|html)|\ 404\ .*\"\ \"GET\ /) ]] && return 0
return 1
}
extract_useful_info() {
local line="$1"
local domain="unknown"
local file_path=""
local error_msg
# Extract domain using bash regex (faster than grep|sed pipeline)
if [[ "$line" =~ \[vhost\ ([^:]+) ]]; then
domain="${BASH_REMATCH[1]}"
elif [[ "$line" =~ ([a-zA-Z0-9.-]+\.(com|net|org|io|co|uk|us|dev)) ]]; then
domain="${BASH_REMATCH[1]}"
elif [[ "$line" =~ /home/([^/]+) ]]; then
domain="${BASH_REMATCH[1]}"
fi
# Extract file path if PHP error
if [[ "$line" =~ in\ (/[^ ]+\.php) ]]; then
file_path="${BASH_REMATCH[1]}"
fi
# Extract error message (clean up ModSec noise, timestamps, etc.)
# Use single sed command instead of pipeline
error_msg=$(echo "$line" | sed -E 's/^\[.*\] //; s/\[client [^]]*\] //; s/\[unique_id "[^"]*"\]//g; s/\[pid [^]]*\]//g; s/\[tid [^]]*\]//g' | cut -c1-150)
# Skip if error message is empty or just whitespace
error_msg="${error_msg#"${error_msg%%[![:space:]]*}"}" # ltrim
error_msg="${error_msg%"${error_msg##*[![:space:]]}"}" # rtrim
[ -z "$error_msg" ] && return 1
# Correlate to root cause
local root_cause=$(correlate_root_cause "$line" "$error_msg" "$domain")
echo "$domain|$file_path|$error_msg|$root_cause"
}
correlate_root_cause() {
local line="$1"
local error_msg="$2"
local domain="$3"
local cause="UNKNOWN"
local line_lower="${line,,}"
local error_lower="${error_msg,,}"
# .htaccess issues
if [[ "$line_lower" =~ (\.htaccess|invalid\ command|rewriterule|rewritecond) ]]; then
cause="HTACCESS"
# ModSecurity blocks
elif [[ "$line_lower" =~ (modsecurity|mod_security|waf) ]]; then
cause="MODSECURITY"
# PHP memory limit
elif [[ "$error_lower" =~ (memory.*exhausted|allowed\ memory\ size) ]]; then
# Get current memory limit for this domain/user
if [ -n "$domain" ] && [ "$domain" != "unknown" ]; then
user=$(grep -l "DNS.*$domain" /var/cpanel/users/* 2>/dev/null | head -1 | xargs basename 2>/dev/null)
if [ -n "$user" ]; then
mem_limit=$(grep -h "memory_limit" /home/$user/public_html/.user.ini /home/$user/.php.ini 2>/dev/null | tail -1 | cut -d'=' -f2 | tr -d ' ')
[ -n "$mem_limit" ] && cause="PHP_MEMORY:$mem_limit" || cause="PHP_MEMORY:default"
else
cause="PHP_MEMORY"
fi
else
cause="PHP_MEMORY"
fi
# PHP max execution time
elif [[ "$error_lower" =~ (max_execution_time|maximum\ execution\ time) ]]; then
cause="PHP_TIMEOUT"
# PHP upload/post size limits
elif [[ "$error_lower" =~ (upload_max_filesize|post_max_size) ]]; then
cause="PHP_UPLOAD_LIMIT"
# File permissions
elif [[ "$error_lower" =~ (permission\ denied|failed\ to\ open\ stream.*permission) ]]; then
cause="FILE_PERMISSIONS"
# Missing PHP modules/extensions
elif [[ "$error_lower" =~ (undefined\ function|call\ to\ undefined\ function|class\ .*\ not\ found) ]]; then
# Try to identify which module
if [[ "$error_lower" =~ (imagecreate|imagefilter|gd_) ]]; then
cause="MISSING_PHP_GD"
elif [[ "$error_msg" =~ (curl_|CURL) ]]; then
cause="MISSING_PHP_CURL"
elif [[ "$error_msg" =~ (json_|JSON) ]]; then
cause="MISSING_PHP_JSON"
elif [[ "$error_lower" =~ (mysqli|mysql_connect) ]]; then
cause="MISSING_PHP_MYSQLI"
else
cause="MISSING_PHP_MODULE"
fi
# Database issues
elif [[ "$error_lower" =~ (database|mysql|mysqli) ]]; then
if [[ "$error_lower" =~ (too\ many\ connections|max_connections) ]]; then
cause="DB_MAX_CONNECTIONS"
elif [[ "$error_lower" =~ (access\ denied|authentication) ]]; then
cause="DB_AUTH_FAILED"
elif [[ "$error_lower" =~ (gone\ away|lost\ connection) ]]; then
cause="DB_TIMEOUT"
else
cause="DB_ERROR"
fi
# Apache configuration
elif [[ "$error_lower" =~ (invalid\ uri|request\ exceeded\ the\ limit|limitrequestline) ]]; then
cause="APACHE_CONFIG"
# PHP parse/syntax errors (code issues)
elif [[ "$error_lower" =~ (parse\ error|syntax\ error|unexpected) ]]; then
cause="PHP_SYNTAX_ERROR"
# File not found (may indicate bad symlinks or missing files)
elif [[ "$error_lower" =~ (no\ such\ file|failed\ to\ open\ stream) ]]; then
cause="MISSING_FILE"
# Generic PHP fatal error
elif [[ "$error_lower" =~ fatal\ error ]]; then
cause="PHP_FATAL_ERROR"
# 500 errors without specific cause
elif [[ "$error_msg" =~ (500|Internal\ Server) ]]; then
cause="SERVER_ERROR_500"
fi
echo "$cause"
}
################################################################################
# Main Analysis
################################################################################
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " SCANNING FOR USER-IMPACTING ERRORS"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""
total_lines=0
filtered_out=0
critical_found=0
# Analyze all discovered log files
while IFS='|' read -r log_path log_type; do
echo " Analyzing: $(basename "$log_path")..."
# Determine log type
is_access_log=false
is_modsec_log=false
[[ "$log_type" == domlog_* ]] && ! [[ "$log_path" =~ error ]] && is_access_log=true
[[ "$log_type" == modsec_* ]] && is_modsec_log=true
if $is_modsec_log; then
# ModSecurity audit log - parse for blocks/denials
while IFS= read -r line; do
((total_lines++))
# Skip if not a security event
echo "$line" | grep -qiE "ModSecurity.*Action.*Intercepted|status: (403|406|500)" || continue
# Extract domain/host if filtering
if [ -n "$FILTER_DOMAIN" ]; then
echo "$line" | grep -q "$FILTER_DOMAIN" || continue
fi
# Extract useful info
if echo "$line" | grep -qiE "SQL Injection|XSS|Command Injection|Path Traversal"; then
((critical_found++))
attack_type=$(echo "$line" | grep -oiE "SQL Injection|XSS|Command Injection|Path Traversal" | head -1)
uri=$(echo "$line" | grep -oE "uri: [^ ]+" | sed 's/uri: //' || echo "")
domain=$(echo "$line" | grep -oE "hostname: [^ ]+" | sed 's/hostname: //' || echo "unknown")
# Extract rule ID if available
rule_id=$(echo "$line" | grep -oE "id \"[0-9]+\"" | grep -oE "[0-9]+" || echo "")
root_cause="MODSECURITY"
[ -n "$rule_id" ] && root_cause="MODSECURITY:rule_$rule_id"
echo "$domain||ModSecurity blocked: $attack_type $uri|$root_cause" >> "$CRITICAL_ERRORS"
fi
done < <(tail -n 5000 "$log_path" 2>/dev/null)
elif $is_access_log; then
# Access log - look for 5xx status codes
# Use time-based filtering if possible, otherwise last 50k lines
cutoff_time=$(date -d "$HOURS_TO_ANALYZE hours ago" +%s 2>/dev/null || echo "0")
while IFS= read -r line; do
((total_lines++))
# Skip if bot/scanner
if is_noise "$line"; then
((filtered_out++))
# Track bot/scanner IPs
read -r ip _ _ _ _ _ _ _ <<< "$line"
if [[ "$ip" =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
flag_ip_attack "$ip" "BOT" 0 "Bot/scanner filtered in error analysis" >/dev/null 2>&1 &
fi
continue
fi
# Time filtering (Apache format: [DD/Mon/YYYY:HH:MM:SS +ZONE])
if [ "$cutoff_time" != "0" ]; then
if [[ "$line" =~ \[([0-9]{2}/[A-Z][a-z]{2}/[0-9]{4}:[0-9]{2}:[0-9]{2}:[0-9]{2}) ]]; then
log_date="${BASH_REMATCH[1]}"
log_time=$(date -d "${log_date/:/ }" +%s 2>/dev/null || echo "0")
[ "$log_time" != "0" ] && [ "$log_time" -lt "$cutoff_time" ] && continue
fi
fi
# Extract status code and URL using bash regex and read
if [[ "$line" =~ '"'[[:space:]](5[0-9]{2})[[:space:]] ]]; then
status="${BASH_REMATCH[1]}"
# Parse Apache log format: IP - - [timestamp] "METHOD URL PROTOCOL" STATUS SIZE
read -r ip _ _ timestamp _ request status_check _ <<< "$line"
# Extract URL from request (format: "GET /path HTTP/1.1")
if [[ "$request" =~ '"'[A-Z]+[[:space:]]([^[:space:]]+) ]]; then
url="${BASH_REMATCH[1]:0:80}"
else
url="/"
fi
# Extract timestamp
if [[ "$line" =~ \[([^]]+)\] ]]; then
timestamp="${BASH_REMATCH[1]}"
else
timestamp=""
fi
# Get domain from log filename
domain="${log_path##*/}" # basename
domain="${domain%%-*}" # remove everything after first dash
# Apply domain filter if set
if [ -n "$FILTER_DOMAIN" ] && [ "$domain" != "$FILTER_DOMAIN" ]; then
continue
fi
# Determine root cause from status code
case $status in
500) root_cause="SERVER_ERROR_500" ;;
502) root_cause="APACHE_CONFIG:bad_gateway" ;;
503) root_cause="APACHE_CONFIG:service_unavailable" ;;
504) root_cause="APACHE_CONFIG:gateway_timeout" ;;
*) root_cause="SERVER_ERROR_5XX" ;;
esac
((critical_found++))
echo "$domain||[$timestamp] HTTP $status - $url (from $ip)|$root_cause" >> "$CRITICAL_ERRORS"
fi
done < <(tail -n 50000 "$log_path" 2>/dev/null)
else
# Error log - look for critical errors
# Use time-based filtering if possible, otherwise last 50k lines
cutoff_time=$(date -d "$HOURS_TO_ANALYZE hours ago" +%s 2>/dev/null || echo "0")
while IFS= read -r line; do
((total_lines++))
# Skip noise
if is_noise "$line"; then
((filtered_out++))
continue
fi
# Time filtering (Apache/PHP error log format: [Day Mon DD HH:MM:SS YYYY])
if [ "$cutoff_time" != "0" ]; then
if [[ "$line" =~ \[([A-Z][a-z]{2}\ [A-Z][a-z]{2}\ [0-9]{2}\ [0-9]{2}:[0-9]{2}:[0-9]{2}\ [0-9]{4})\] ]]; then
log_date="${BASH_REMATCH[1]}"
log_time=$(date -d "$log_date" +%s 2>/dev/null || echo "0")
[ "$log_time" != "0" ] && [ "$log_time" -lt "$cutoff_time" ] && continue
fi
fi
# Apply user/domain filter if set
if [ -n "$FILTER_USER" ]; then
[[ "$line" =~ /home/$FILTER_USER ]] || continue
fi
if [ -n "$FILTER_DOMAIN" ]; then
[[ "$line" =~ $FILTER_DOMAIN ]] || continue
fi
# Check if it's critical and user-facing
if is_critical_user_facing "$line"; then
((critical_found++))
extract_useful_info "$line" >> "$CRITICAL_ERRORS"
fi
done < <(tail -n 50000 "$log_path" 2>/dev/null)
fi
done < "$LOG_FILES_LIST"
echo ""
echo "Scanned: $total_lines log entries"
echo "Filtered: $filtered_out noise entries (bots, warnings, etc.)"
echo "Found: $critical_found critical user-facing errors"
echo ""
################################################################################
# Generate Report
################################################################################
if [ ! -f "$CRITICAL_ERRORS" ] || [ ! -s "$CRITICAL_ERRORS" ]; then
echo ""
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo -e "${GREEN} ✓ NO CRITICAL USER-FACING ERRORS FOUND!${NC}"
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo ""
echo "Your websites appear to be functioning normally for real users."
echo "All errors found were from bots, scanners, or non-critical warnings."
echo ""
exit 0
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " ⚠️ CRITICAL ERRORS AFFECTING REAL USERS"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""
# Group by domain and count
echo -e "${RED}${BOLD}AFFECTED WEBSITES:${NC}"
echo ""
cat "$CRITICAL_ERRORS" | cut -d'|' -f1 | sort | uniq -c | sort -rn | head -10 | while read -r count domain; do
printf " ${RED}${NC} %-40s ${BOLD}%4d errors${NC}\n" "$domain" "$count"
done
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " ERROR DETAILS (Top Issues)"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""
# Group identical errors and count them
awk -F'|' '{
key = $1 "|" $3 "|" $4 # domain|error_msg|root_cause
file[$1"|"$3"|"$4] = $2 # Store file path
count[key]++
}
END {
for (key in count) {
split(key, parts, "|")
domain = parts[1]
error = parts[2]
root_cause = parts[3]
file_path = file[key]
# Skip empty errors
if (length(error) > 0) {
print count[key] "|" domain "|" file_path "|" error "|" root_cause
}
}
}' "$CRITICAL_ERRORS" | sort -t'|' -k1 -rn | head -20 | while IFS='|' read -r count domain file_path error_msg root_cause; do
# Color code by frequency
if [ "$count" -ge 10 ]; then
color="${RED}"
priority="[HIGH FREQUENCY]"
elif [ "$count" -ge 5 ]; then
color="${YELLOW}"
priority="[MEDIUM]"
else
color="${INFO_COLOR}"
priority=""
fi
echo -e "${color}● Occurred $count times${NC} $priority"
[ -n "$domain" ] && [ "$domain" != "unknown" ] && echo " Domain: $domain"
[ -n "$file_path" ] && echo " File: $file_path"
echo " Error: $error_msg"
# Display root cause with color coding and actionable fix
if [ -n "$root_cause" ] && [ "$root_cause" != "UNKNOWN" ]; then
echo -ne " ${CYAN}Root Cause: ${BOLD}"
case "$root_cause" in
HTACCESS)
echo -e "${YELLOW}⚙️ .htaccess Configuration${NC}"
echo " → Check .htaccess syntax in domain root"
echo " → Look for invalid RewriteRule or directives"
;;
MODSECURITY*)
rule=$(echo "$root_cause" | cut -d':' -f2)
echo -e "${YELLOW}🛡️ ModSecurity WAF Block${NC}"
[ -n "$rule" ] && echo " → Rule ID: $rule"
echo " → Check if this is a false positive"
echo " → Review: /usr/local/apache/logs/modsec_audit.log"
;;
PHP_MEMORY*)
limit=$(echo "$root_cause" | cut -d':' -f2)
echo -e "${RED}💾 PHP Memory Exhausted${NC}"
[ -n "$limit" ] && echo " → Current limit: $limit"
echo " → Increase memory_limit in .user.ini or php.ini"
echo " → Recommended: 256M minimum, 512M for WooCommerce"
;;
PHP_TIMEOUT)
echo -e "${YELLOW}⏱️ PHP Execution Timeout${NC}"
echo " → Increase max_execution_time in php.ini"
echo " → Recommended: 300 for imports/backups"
;;
PHP_UPLOAD_LIMIT)
echo -e "${YELLOW}📤 PHP Upload Size Limit${NC}"
echo " → Increase upload_max_filesize and post_max_size"
echo " → Match both values (e.g., 64M)"
;;
FILE_PERMISSIONS)
echo -e "${RED}🔒 File Permission Denied${NC}"
echo " → Check file ownership and permissions"
echo " → Files should be 644, directories 755"
echo " → Owner should match cPanel user"
;;
MISSING_PHP_GD)
echo -e "${RED}📦 Missing PHP GD Extension${NC}"
echo " → Install: yum install ea-phpXX-php-gd"
echo " → Required for image processing"
;;
MISSING_PHP_CURL)
echo -e "${RED}📦 Missing PHP cURL Extension${NC}"
echo " → Install: yum install ea-phpXX-php-curl"
;;
MISSING_PHP_*)
module=$(echo "$root_cause" | sed 's/MISSING_PHP_//' | tr '[:upper:]' '[:lower:]')
echo -e "${RED}📦 Missing PHP Extension: $module${NC}"
echo " → Install: yum install ea-phpXX-php-$module"
;;
DB_MAX_CONNECTIONS)
echo -e "${RED}🗄️ Database Max Connections Reached${NC}"
echo " → Check: mysql -e 'SHOW VARIABLES LIKE \"max_connections\"'"
echo " → Increase max_connections in my.cnf"
echo " → Or optimize slow queries reducing connection time"
;;
DB_AUTH_FAILED)
echo -e "${RED}🗄️ Database Authentication Failed${NC}"
echo " → Verify credentials in wp-config.php / config files"
echo " → Check database user permissions"
;;
DB_TIMEOUT)
echo -e "${YELLOW}🗄️ Database Connection Timeout${NC}"
echo " → MySQL server may be overloaded"
echo " → Check slow query log"
;;
DB_ERROR)
echo -e "${RED}🗄️ Database Error${NC}"
echo " → Check MySQL error log for details"
;;
APACHE_CONFIG*)
issue=$(echo "$root_cause" | cut -d':' -f2)
echo -e "${YELLOW}⚙️ Apache Configuration${NC}"
[ -n "$issue" ] && echo " → Issue: $issue"
echo " → Check Apache config and vhost settings"
;;
PHP_SYNTAX_ERROR)
echo -e "${RED}⚠️ PHP Syntax Error in Code${NC}"
echo " → Review recent code changes"
echo " → Check for missing semicolons, brackets, quotes"
;;
MISSING_FILE)
echo -e "${YELLOW}📄 File Not Found${NC}"
echo " → File may have been deleted or moved"
echo " → Check for broken symlinks"
;;
PHP_FATAL_ERROR)
echo -e "${RED}⚠️ PHP Fatal Error${NC}"
echo " → Review PHP error logs for details"
echo " → May be compatibility issue or code bug"
;;
SERVER_ERROR_500)
echo -e "${RED}🔥 Generic 500 Internal Server Error${NC}"
echo " → Check PHP/Apache error logs for specifics"
;;
*)
echo -e "${NC}$root_cause"
;;
esac
fi
echo ""
done
################################################################################
# Root Cause Summary
################################################################################
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " 📊 ROOT CAUSE BREAKDOWN"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""
# Count errors by root cause
cut -d'|' -f4 "$CRITICAL_ERRORS" | grep -v "^$" | sort | uniq -c | sort -rn | while read -r count cause; do
# Clean up cause display
clean_cause=$(echo "$cause" | sed 's/:.*//; s/_/ /g')
# Color code by severity
case "$cause" in
PHP_MEMORY*|DB_MAX_CONNECTIONS|PHP_FATAL_ERROR|SERVER_ERROR_500)
color="${RED}"; icon="🔥" ;;
HTACCESS|MODSECURITY*|PHP_TIMEOUT|DB_*)
color="${YELLOW}"; icon="⚠️ " ;;
MISSING_PHP*|FILE_PERMISSIONS)
color="${YELLOW}"; icon="📦" ;;
*)
color="${INFO_COLOR}"; icon="•" ;;
esac
printf " ${color}${icon} %-35s %4d errors${NC}\n" "$clean_cause" "$count"
done
echo ""
################################################################################
# Intelligent Recommendations
################################################################################
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " 🔧 RECOMMENDED ACTIONS"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""
# Detect issue types and provide specific recommendations
if grep -qiE "500|Internal Server" "$CRITICAL_ERRORS"; then
echo -e "${RED}● 500 Internal Server Errors detected${NC}"
echo " Priority: URGENT - Users see broken pages"
echo ""
echo " Actions:"
echo " 1. Check PHP error logs for affected domains:"
for domain in $(grep -iE "500|Internal Server" "$CRITICAL_ERRORS" | cut -d'|' -f1 | sort -u | head -3); do
echo " tail -100 /home/$domain/public_html/error_log"
done
echo ""
echo " 2. Common causes:"
echo " - PHP syntax errors (check recent file changes)"
echo " - .htaccess syntax errors"
echo " - Incorrect file permissions"
echo " - Missing PHP modules"
echo ""
fi
if grep -qiE "Fatal error|Parse error" "$CRITICAL_ERRORS"; then
echo -e "${RED}● PHP Fatal/Parse Errors detected${NC}"
echo " Priority: URGENT - Site functionality broken"
echo ""
echo " Actions:"
echo " 1. Review recent plugin/theme updates"
echo " 2. Check PHP version compatibility"
echo " 3. Look for syntax errors in custom code"
echo " 4. Temporarily disable recently added plugins"
echo ""
fi
if grep -qiE "database|MySQL|Connection" "$CRITICAL_ERRORS"; then
echo -e "${RED}● Database Connection Errors detected${NC}"
echo " Priority: CRITICAL - Site may be completely down"
echo ""
echo " Actions:"
echo " 1. Check MySQL service: systemctl status mysql"
echo " 2. Verify database credentials in config files"
echo " 3. Check max_connections: mysql -e 'SHOW VARIABLES LIKE \"max_connections\"'"
echo " 4. Review current connections: mysql -e 'SHOW PROCESSLIST'"
echo ""
fi
if grep -qiE "memory.*exhausted|Out of memory" "$CRITICAL_ERRORS"; then
echo -e "${RED}● Memory Exhaustion detected${NC}"
echo " Priority: HIGH - Pages loading slowly or incompletely"
echo ""
echo " Current PHP memory limit: $(php -r 'echo ini_get("memory_limit");' 2>/dev/null)"
echo ""
echo " Actions:"
echo " 1. Increase memory_limit in php.ini or .user.ini"
echo " 2. Recommended: 256M minimum, 512M for WooCommerce"
echo " 3. Check for memory leaks in plugins"
echo " 4. Optimize database queries"
echo ""
fi
if grep -qiE "ModSecurity" "$CRITICAL_ERRORS"; then
echo -e "${YELLOW}● ModSecurity Blocks detected${NC}"
echo " Priority: MEDIUM - Security rules blocking requests"
echo ""
echo " Actions:"
echo " 1. Review ModSecurity audit log for false positives"
echo " 2. Check if legitimate functionality is being blocked"
echo " 3. Consider whitelisting specific rules if needed"
echo " 4. Review blocked attack types to understand threats"
echo " 5. ModSecurity log: $MODSEC_AUDIT_LOG"
echo ""
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""
echo "Full analysis saved to: $CRITICAL_ERRORS"
echo ""
echo "Next steps:"
echo " 1. Address critical errors first (500s, database, fatal errors)"
echo " 2. Check error_log files for affected domains"
echo " 3. Test affected pages after fixes"
echo " 4. Re-run this tool to verify issues are resolved"
echo ""
press_enter