cschantz
d7febf68d3
NEW SCRIPT: modules/website/500-error-tracker.sh - FAST-ONLY 500 error detection (no menus, no options) - Scans access logs for 500 errors - Maps domains to cPanel usernames - Automatically diagnoses root causes by checking error_log files - Shows actual PHP errors causing the 500s ROOT CAUSE DETECTION: - PHP Memory Exhausted (shows current limit) - PHP Fatal Errors - PHP Syntax Errors - Missing PHP Functions/Extensions - Database Connection Failures - .htaccess Issues - Shows ACTUAL error examples, not just suggestions FIXES: - Fixed awk error in website-error-analyzer.sh: • Changed "next" in END block to "if (length > 0)" • "next" cannot be used in END block in awk - Added option 2 in Website Management menu - Renumbered all WordPress tools (3-16) DIFFERENCE FROM FULL ANALYZER: Full Analyzer: All errors, filters, time ranges, user choices Fast Tracker: ONLY 500s, auto-diagnosis, shows WHY not suggestions Use Fast Tracker when you need to quickly find which domains are getting 500 errors and the exact PHP errors causing them.
860 lines
34 KiB
Bash
Executable File
860 lines
34 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
################################################################################
|
|
# Intelligent Website Error Analyzer
|
|
################################################################################
|
|
# Purpose: Find REAL website issues affecting REAL users
|
|
# Filters out: Bots, plugin warnings, deprecation notices, minor issues
|
|
# Focuses on: Critical errors that break sites for actual visitors
|
|
################################################################################
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
source "$SCRIPT_DIR/lib/common-functions.sh"
|
|
source "$SCRIPT_DIR/lib/system-detect.sh"
|
|
source "$SCRIPT_DIR/lib/user-manager.sh"
|
|
|
|
# Configuration
|
|
APACHE_ERROR_LOG="/var/log/apache2/error_log"
|
|
DOMLOGS_DIR="/var/log/apache2/domlogs"
|
|
MODSEC_AUDIT_LOG="/usr/local/apache/logs/modsec_audit.log"
|
|
HOURS_TO_ANALYZE=24
|
|
FILTER_USER=""
|
|
FILTER_DOMAIN=""
|
|
|
|
print_banner "Intelligent Website Error Analyzer"
|
|
|
|
echo ""
|
|
echo "Finding REAL issues affecting REAL users..."
|
|
echo ""
|
|
echo "What we filter out:"
|
|
echo " ✗ Bot/scanner activity"
|
|
echo " ✗ Plugin deprecation warnings"
|
|
echo " ✗ PHP notices (non-critical)"
|
|
echo " ✗ Random 404s for missing favicons/assets"
|
|
echo " ✗ Security probe attempts"
|
|
echo ""
|
|
echo "What we focus on:"
|
|
echo " ✓ 500 errors users actually see"
|
|
echo " ✓ Fatal PHP errors breaking functionality"
|
|
echo " ✓ Database connection failures"
|
|
echo " ✓ Permission issues blocking access"
|
|
echo " ✓ Memory exhaustion affecting performance"
|
|
echo ""
|
|
|
|
# Ask for filtering scope
|
|
echo -e "${CYAN}Analysis Scope:${NC}"
|
|
echo " 1) All users/domains (default)"
|
|
echo " 2) Specific cPanel user"
|
|
echo " 3) Specific domain"
|
|
echo ""
|
|
read -p "Select option [1]: " scope_choice
|
|
scope_choice=${scope_choice:-1}
|
|
|
|
case $scope_choice in
|
|
2)
|
|
# Select specific user
|
|
select_user_interactive "Select cPanel user to analyze"
|
|
if [ -n "$SELECTED_USER" ]; then
|
|
FILTER_USER="$SELECTED_USER"
|
|
echo "→ Filtering for user: $FILTER_USER"
|
|
fi
|
|
;;
|
|
3)
|
|
# Enter specific domain
|
|
echo ""
|
|
read -p "Enter domain name (e.g., example.com): " FILTER_DOMAIN
|
|
if [ -n "$FILTER_DOMAIN" ]; then
|
|
echo "→ Filtering for domain: $FILTER_DOMAIN"
|
|
fi
|
|
;;
|
|
*)
|
|
echo "→ Analyzing all users/domains"
|
|
;;
|
|
esac
|
|
|
|
echo ""
|
|
|
|
# Ask for time range
|
|
echo -e "${CYAN}How far back should we analyze?${NC}"
|
|
echo " 1) Last 1 hour"
|
|
echo " 2) Last 6 hours"
|
|
echo " 3) Last 24 hours (default)"
|
|
echo " 4) Last 7 days"
|
|
echo " 5) Last 30 days"
|
|
echo ""
|
|
read -p "Select option [3]: " time_choice
|
|
time_choice=${time_choice:-3}
|
|
|
|
case $time_choice in
|
|
1) HOURS_TO_ANALYZE=1 ;;
|
|
2) HOURS_TO_ANALYZE=6 ;;
|
|
3) HOURS_TO_ANALYZE=24 ;;
|
|
4) HOURS_TO_ANALYZE=168 ;;
|
|
5) HOURS_TO_ANALYZE=720 ;;
|
|
*) HOURS_TO_ANALYZE=24 ;;
|
|
esac
|
|
|
|
echo ""
|
|
echo "→ Analyzing last $HOURS_TO_ANALYZE hours..."
|
|
echo ""
|
|
|
|
# Temporary files
|
|
TEMP_DIR="/tmp/website-error-analysis-$$"
|
|
mkdir -p "$TEMP_DIR"
|
|
trap "rm -rf $TEMP_DIR" EXIT
|
|
|
|
CRITICAL_ERRORS="$TEMP_DIR/critical.txt"
|
|
USER_IMPACT_ERRORS="$TEMP_DIR/user_impact.txt"
|
|
LOG_FILES_LIST="$TEMP_DIR/log_files.txt"
|
|
|
|
# Discover all log file locations
|
|
echo "→ Discovering log file locations..."
|
|
> "$LOG_FILES_LIST"
|
|
|
|
# Apache main error log (only if not filtering by user)
|
|
if [ -z "$FILTER_USER" ] && [ -z "$FILTER_DOMAIN" ]; then
|
|
for log in /var/log/apache2/error_log /usr/local/apache/logs/error_log /var/log/httpd/error_log; do
|
|
[ -f "$log" ] && echo "$log|apache_main" >> "$LOG_FILES_LIST"
|
|
done
|
|
fi
|
|
|
|
# Per-domain PHP error logs in public_html
|
|
if [ -n "$FILTER_USER" ]; then
|
|
# Specific user
|
|
find "/home/$FILTER_USER" -name "error_log" -type f 2>/dev/null | while read -r log; do
|
|
echo "$log|php_$FILTER_USER" >> "$LOG_FILES_LIST"
|
|
done
|
|
elif [ -n "$FILTER_DOMAIN" ]; then
|
|
# Try to find domain's user and error logs
|
|
user=$(grep -l "DNS.*$FILTER_DOMAIN" /var/cpanel/users/* 2>/dev/null | head -1 | xargs basename 2>/dev/null)
|
|
if [ -n "$user" ]; then
|
|
find "/home/$user" -name "error_log" -type f 2>/dev/null | while read -r log; do
|
|
echo "$log|php_$FILTER_DOMAIN" >> "$LOG_FILES_LIST"
|
|
done
|
|
fi
|
|
else
|
|
# All users
|
|
find /home/*/public_html -name "error_log" -type f 2>/dev/null | while read -r log; do
|
|
username=$(echo "$log" | grep -oE '/home/[^/]+' | sed 's|/home/||')
|
|
echo "$log|php_$username" >> "$LOG_FILES_LIST"
|
|
done
|
|
fi
|
|
|
|
# Per-domain Apache logs in domlogs
|
|
if [ -d "$DOMLOGS_DIR" ]; then
|
|
if [ -n "$FILTER_DOMAIN" ]; then
|
|
# Specific domain
|
|
for log in "$DOMLOGS_DIR/$FILTER_DOMAIN" "$DOMLOGS_DIR/$FILTER_DOMAIN-"*; do
|
|
[ -f "$log" ] && echo "$log|domlog_$FILTER_DOMAIN" >> "$LOG_FILES_LIST"
|
|
done
|
|
elif [ -n "$FILTER_USER" ]; then
|
|
# Specific user - find their domains
|
|
if [ -f "/var/cpanel/users/$FILTER_USER" ]; then
|
|
grep "^DNS" "/var/cpanel/users/$FILTER_USER" | awk '{print $2}' | while read -r domain; do
|
|
for log in "$DOMLOGS_DIR/$domain" "$DOMLOGS_DIR/$domain-"*; do
|
|
[ -f "$log" ] && echo "$log|domlog_$domain" >> "$LOG_FILES_LIST"
|
|
done
|
|
done
|
|
fi
|
|
else
|
|
# All domains
|
|
for log in "$DOMLOGS_DIR"/*; do
|
|
[ -f "$log" ] && ! [[ "$log" =~ (bytes_log|offset|error_log|ftpxferlog)$ ]] && \
|
|
echo "$log|domlog_$(basename "$log")" >> "$LOG_FILES_LIST"
|
|
done
|
|
fi
|
|
fi
|
|
|
|
# ModSecurity audit log (only if not filtering or matches filter)
|
|
if [ -f "$MODSEC_AUDIT_LOG" ]; then
|
|
if [ -z "$FILTER_USER" ] && [ -z "$FILTER_DOMAIN" ]; then
|
|
echo "$MODSEC_AUDIT_LOG|modsec_audit" >> "$LOG_FILES_LIST"
|
|
elif [ -n "$FILTER_DOMAIN" ]; then
|
|
# Will filter ModSec entries during processing
|
|
echo "$MODSEC_AUDIT_LOG|modsec_audit" >> "$LOG_FILES_LIST"
|
|
fi
|
|
fi
|
|
|
|
log_count=$(wc -l < "$LOG_FILES_LIST" 2>/dev/null || echo "0")
|
|
echo " Found $log_count log files to analyze"
|
|
if [ -n "$FILTER_USER" ]; then
|
|
echo " Scope: User '$FILTER_USER'"
|
|
elif [ -n "$FILTER_DOMAIN" ]; then
|
|
echo " Scope: Domain '$FILTER_DOMAIN'"
|
|
fi
|
|
echo ""
|
|
|
|
################################################################################
|
|
# Intelligent Filtering
|
|
################################################################################
|
|
|
|
is_noise() {
|
|
local line="$1"
|
|
local line_lower="${line,,}" # Convert to lowercase once
|
|
|
|
# Bot/Scanner patterns
|
|
[[ "$line_lower" =~ (bot|crawler|spider|scanner|nikto|nmap|masscan|sqlmap|nessus|acunetix|burp|shodan|censys|zgrab|nuclei|semrush|ahrefs|mj12) ]] && return 0
|
|
|
|
# Known scanner IPs
|
|
[[ "$line" =~ (45\.148\.|185\.220\.|89\.248\.165\.) ]] && return 0
|
|
|
|
# WordPress plugin deprecation warnings (not user-facing)
|
|
[[ "$line" =~ (PHP\ Deprecated|Deprecated.*plugin|call_user_func_array.*deprecated) ]] && return 0
|
|
|
|
# Non-critical PHP notices (unless critical indicators present)
|
|
if [[ "$line" =~ PHP\ Notice.*Undefined\ (variable|index|offset) ]] && ! [[ "$line_lower" =~ (fatal|error\ 500|white\ screen) ]]; then
|
|
return 0
|
|
fi
|
|
|
|
# Missing favicon/common assets (not real errors)
|
|
[[ "$line_lower" =~ (favicon\.ico|apple-touch-icon|robots\.txt|sitemap\.xml.*404) ]] && return 0
|
|
|
|
# Security probes
|
|
if [[ "$line_lower" =~ (wp-admin|wp-login|phpmyadmin|\.env|\.git|config\.php|xmlrpc|eval-stdin|shell\.php|c99\.php|r57\.php|adminer\.php) ]] && \
|
|
[[ "$line" =~ (404|403|not\ found) ]]; then
|
|
return 0
|
|
fi
|
|
|
|
# WordPress auto-updates and cron (normal operations)
|
|
[[ "$line_lower" =~ (wp-cron\.php|doing_cron|auto.*update) ]] && return 0
|
|
|
|
# Common plugin update checks (not errors)
|
|
[[ "$line_lower" =~ (update-check|version-check|api\.wordpress\.org) ]] && return 0
|
|
|
|
return 1
|
|
}
|
|
|
|
is_critical_user_facing() {
|
|
local line="$1"
|
|
local line_lower="${line,,}"
|
|
|
|
# 500 Internal Server Error (users see white page)
|
|
[[ "$line" =~ (\ 500\ |Internal\ Server\ Error) ]] && return 0
|
|
|
|
# PHP Fatal Errors (breaks functionality)
|
|
[[ "$line" =~ (PHP\ Fatal\ error|Fatal\ error:.*in\ /) ]] && return 0
|
|
|
|
# PHP Parse Errors (site completely broken)
|
|
[[ "$line" =~ (PHP\ Parse\ error|syntax\ error) ]] && return 0
|
|
|
|
# Database connection failures (site down)
|
|
[[ "$line" =~ (Error\ establishing.*database|Can\'t\ connect.*MySQL|Access\ denied\ for\ user.*database|Too\ many\ connections|MySQL\ server\ has\ gone\ away) ]] && return 0
|
|
|
|
# Memory exhaustion (white screen/incomplete pages)
|
|
[[ "$line" =~ (Allowed\ memory\ size.*exhausted|Out\ of\ memory|Fatal.*memory) ]] && return 0
|
|
|
|
# Segmentation fault (complete crash)
|
|
[[ "$line" =~ (Segmentation\ fault|signal\ 11) ]] && return 0
|
|
|
|
# Permission denied on critical files
|
|
if [[ "$line" =~ Permission\ denied ]] && [[ "$line" =~ (index\.php|wp-config\.php|\.htaccess|config\.php) ]]; then
|
|
return 0
|
|
fi
|
|
|
|
# File not found for MAIN page (404 on homepage/index)
|
|
[[ "$line" =~ (File\ does\ not\ exist.*index\.(php|html)|\ 404\ .*\"\ \"GET\ /) ]] && return 0
|
|
|
|
return 1
|
|
}
|
|
|
|
extract_useful_info() {
|
|
local line="$1"
|
|
local domain="unknown"
|
|
local file_path=""
|
|
local error_msg
|
|
|
|
# Extract domain using bash regex (faster than grep|sed pipeline)
|
|
if [[ "$line" =~ \[vhost\ ([^:]+) ]]; then
|
|
domain="${BASH_REMATCH[1]}"
|
|
elif [[ "$line" =~ ([a-zA-Z0-9.-]+\.(com|net|org|io|co|uk|us|dev)) ]]; then
|
|
domain="${BASH_REMATCH[1]}"
|
|
elif [[ "$line" =~ /home/([^/]+) ]]; then
|
|
domain="${BASH_REMATCH[1]}"
|
|
fi
|
|
|
|
# Extract file path if PHP error
|
|
if [[ "$line" =~ in\ (/[^ ]+\.php) ]]; then
|
|
file_path="${BASH_REMATCH[1]}"
|
|
fi
|
|
|
|
# Extract error message (clean up ModSec noise, timestamps, etc.)
|
|
# Use single sed command instead of pipeline
|
|
error_msg=$(echo "$line" | sed -E 's/^\[.*\] //; s/\[client [^]]*\] //; s/\[unique_id "[^"]*"\]//g; s/\[pid [^]]*\]//g; s/\[tid [^]]*\]//g' | cut -c1-150)
|
|
|
|
# Skip if error message is empty or just whitespace
|
|
error_msg="${error_msg#"${error_msg%%[![:space:]]*}"}" # ltrim
|
|
error_msg="${error_msg%"${error_msg##*[![:space:]]}"}" # rtrim
|
|
[ -z "$error_msg" ] && return 1
|
|
|
|
# Correlate to root cause
|
|
local root_cause=$(correlate_root_cause "$line" "$error_msg" "$domain")
|
|
|
|
echo "$domain|$file_path|$error_msg|$root_cause"
|
|
}
|
|
|
|
correlate_root_cause() {
|
|
local line="$1"
|
|
local error_msg="$2"
|
|
local domain="$3"
|
|
local cause="UNKNOWN"
|
|
local line_lower="${line,,}"
|
|
local error_lower="${error_msg,,}"
|
|
|
|
# .htaccess issues
|
|
if [[ "$line_lower" =~ (\.htaccess|invalid\ command|rewriterule|rewritecond) ]]; then
|
|
cause="HTACCESS"
|
|
|
|
# ModSecurity blocks
|
|
elif [[ "$line_lower" =~ (modsecurity|mod_security|waf) ]]; then
|
|
cause="MODSECURITY"
|
|
|
|
# PHP memory limit
|
|
elif [[ "$error_lower" =~ (memory.*exhausted|allowed\ memory\ size) ]]; then
|
|
# Get current memory limit for this domain/user
|
|
if [ -n "$domain" ] && [ "$domain" != "unknown" ]; then
|
|
user=$(grep -l "DNS.*$domain" /var/cpanel/users/* 2>/dev/null | head -1 | xargs basename 2>/dev/null)
|
|
if [ -n "$user" ]; then
|
|
mem_limit=$(grep -h "memory_limit" /home/$user/public_html/.user.ini /home/$user/.php.ini 2>/dev/null | tail -1 | cut -d'=' -f2 | tr -d ' ')
|
|
[ -n "$mem_limit" ] && cause="PHP_MEMORY:$mem_limit" || cause="PHP_MEMORY:default"
|
|
else
|
|
cause="PHP_MEMORY"
|
|
fi
|
|
else
|
|
cause="PHP_MEMORY"
|
|
fi
|
|
|
|
# PHP max execution time
|
|
elif [[ "$error_lower" =~ (max_execution_time|maximum\ execution\ time) ]]; then
|
|
cause="PHP_TIMEOUT"
|
|
|
|
# PHP upload/post size limits
|
|
elif [[ "$error_lower" =~ (upload_max_filesize|post_max_size) ]]; then
|
|
cause="PHP_UPLOAD_LIMIT"
|
|
|
|
# File permissions
|
|
elif [[ "$error_lower" =~ (permission\ denied|failed\ to\ open\ stream.*permission) ]]; then
|
|
cause="FILE_PERMISSIONS"
|
|
|
|
# Missing PHP modules/extensions
|
|
elif [[ "$error_lower" =~ (undefined\ function|call\ to\ undefined\ function|class\ .*\ not\ found) ]]; then
|
|
# Try to identify which module
|
|
if [[ "$error_lower" =~ (imagecreate|imagefilter|gd_) ]]; then
|
|
cause="MISSING_PHP_GD"
|
|
elif [[ "$error_msg" =~ (curl_|CURL) ]]; then
|
|
cause="MISSING_PHP_CURL"
|
|
elif [[ "$error_msg" =~ (json_|JSON) ]]; then
|
|
cause="MISSING_PHP_JSON"
|
|
elif [[ "$error_lower" =~ (mysqli|mysql_connect) ]]; then
|
|
cause="MISSING_PHP_MYSQLI"
|
|
else
|
|
cause="MISSING_PHP_MODULE"
|
|
fi
|
|
|
|
# Database issues
|
|
elif [[ "$error_lower" =~ (database|mysql|mysqli) ]]; then
|
|
if [[ "$error_lower" =~ (too\ many\ connections|max_connections) ]]; then
|
|
cause="DB_MAX_CONNECTIONS"
|
|
elif [[ "$error_lower" =~ (access\ denied|authentication) ]]; then
|
|
cause="DB_AUTH_FAILED"
|
|
elif [[ "$error_lower" =~ (gone\ away|lost\ connection) ]]; then
|
|
cause="DB_TIMEOUT"
|
|
else
|
|
cause="DB_ERROR"
|
|
fi
|
|
|
|
# Apache configuration
|
|
elif [[ "$error_lower" =~ (invalid\ uri|request\ exceeded\ the\ limit|limitrequestline) ]]; then
|
|
cause="APACHE_CONFIG"
|
|
|
|
# PHP parse/syntax errors (code issues)
|
|
elif [[ "$error_lower" =~ (parse\ error|syntax\ error|unexpected) ]]; then
|
|
cause="PHP_SYNTAX_ERROR"
|
|
|
|
# File not found (may indicate bad symlinks or missing files)
|
|
elif [[ "$error_lower" =~ (no\ such\ file|failed\ to\ open\ stream) ]]; then
|
|
cause="MISSING_FILE"
|
|
|
|
# Generic PHP fatal error
|
|
elif [[ "$error_lower" =~ fatal\ error ]]; then
|
|
cause="PHP_FATAL_ERROR"
|
|
|
|
# 500 errors without specific cause
|
|
elif [[ "$error_msg" =~ (500|Internal\ Server) ]]; then
|
|
cause="SERVER_ERROR_500"
|
|
fi
|
|
|
|
echo "$cause"
|
|
}
|
|
|
|
################################################################################
|
|
# Main Analysis
|
|
################################################################################
|
|
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo " SCANNING FOR USER-IMPACTING ERRORS"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo ""
|
|
|
|
total_lines=0
|
|
filtered_out=0
|
|
critical_found=0
|
|
|
|
# Analyze all discovered log files
|
|
while IFS='|' read -r log_path log_type; do
|
|
echo " Analyzing: $(basename "$log_path")..."
|
|
|
|
# Determine log type
|
|
is_access_log=false
|
|
is_modsec_log=false
|
|
[[ "$log_type" == domlog_* ]] && ! [[ "$log_path" =~ error ]] && is_access_log=true
|
|
[[ "$log_type" == modsec_* ]] && is_modsec_log=true
|
|
|
|
if $is_modsec_log; then
|
|
# ModSecurity audit log - parse for blocks/denials
|
|
while IFS= read -r line; do
|
|
((total_lines++))
|
|
|
|
# Skip if not a security event
|
|
echo "$line" | grep -qiE "ModSecurity.*Action.*Intercepted|status: (403|406|500)" || continue
|
|
|
|
# Extract domain/host if filtering
|
|
if [ -n "$FILTER_DOMAIN" ]; then
|
|
echo "$line" | grep -q "$FILTER_DOMAIN" || continue
|
|
fi
|
|
|
|
# Extract useful info
|
|
if echo "$line" | grep -qiE "SQL Injection|XSS|Command Injection|Path Traversal"; then
|
|
((critical_found++))
|
|
attack_type=$(echo "$line" | grep -oiE "SQL Injection|XSS|Command Injection|Path Traversal" | head -1)
|
|
uri=$(echo "$line" | grep -oE "uri: [^ ]+" | sed 's/uri: //' || echo "")
|
|
domain=$(echo "$line" | grep -oE "hostname: [^ ]+" | sed 's/hostname: //' || echo "unknown")
|
|
|
|
# Extract rule ID if available
|
|
rule_id=$(echo "$line" | grep -oE "id \"[0-9]+\"" | grep -oE "[0-9]+" || echo "")
|
|
root_cause="MODSECURITY"
|
|
[ -n "$rule_id" ] && root_cause="MODSECURITY:rule_$rule_id"
|
|
|
|
echo "$domain||ModSecurity blocked: $attack_type $uri|$root_cause" >> "$CRITICAL_ERRORS"
|
|
fi
|
|
|
|
done < <(tail -n 5000 "$log_path" 2>/dev/null)
|
|
|
|
elif $is_access_log; then
|
|
# Access log - look for 5xx status codes
|
|
# Use time-based filtering if possible, otherwise last 50k lines
|
|
cutoff_time=$(date -d "$HOURS_TO_ANALYZE hours ago" +%s 2>/dev/null || echo "0")
|
|
|
|
while IFS= read -r line; do
|
|
((total_lines++))
|
|
|
|
# Skip if bot/scanner
|
|
if is_noise "$line"; then
|
|
((filtered_out++))
|
|
continue
|
|
fi
|
|
|
|
# Time filtering (Apache format: [DD/Mon/YYYY:HH:MM:SS +ZONE])
|
|
if [ "$cutoff_time" != "0" ]; then
|
|
if [[ "$line" =~ \[([0-9]{2}/[A-Z][a-z]{2}/[0-9]{4}:[0-9]{2}:[0-9]{2}:[0-9]{2}) ]]; then
|
|
log_date="${BASH_REMATCH[1]}"
|
|
log_time=$(date -d "${log_date/:/ }" +%s 2>/dev/null || echo "0")
|
|
[ "$log_time" != "0" ] && [ "$log_time" -lt "$cutoff_time" ] && continue
|
|
fi
|
|
fi
|
|
|
|
# Extract status code and URL using bash regex and read
|
|
if [[ "$line" =~ '"'[[:space:]](5[0-9]{2})[[:space:]] ]]; then
|
|
status="${BASH_REMATCH[1]}"
|
|
|
|
# Parse Apache log format: IP - - [timestamp] "METHOD URL PROTOCOL" STATUS SIZE
|
|
read -r ip _ _ timestamp _ request status_check _ <<< "$line"
|
|
|
|
# Extract URL from request (format: "GET /path HTTP/1.1")
|
|
if [[ "$request" =~ '"'[A-Z]+[[:space:]]([^[:space:]]+) ]]; then
|
|
url="${BASH_REMATCH[1]:0:80}"
|
|
else
|
|
url="/"
|
|
fi
|
|
|
|
# Extract timestamp
|
|
if [[ "$line" =~ \[([^]]+)\] ]]; then
|
|
timestamp="${BASH_REMATCH[1]}"
|
|
else
|
|
timestamp=""
|
|
fi
|
|
|
|
# Get domain from log filename
|
|
domain="${log_path##*/}" # basename
|
|
domain="${domain%%-*}" # remove everything after first dash
|
|
|
|
# Apply domain filter if set
|
|
if [ -n "$FILTER_DOMAIN" ] && [ "$domain" != "$FILTER_DOMAIN" ]; then
|
|
continue
|
|
fi
|
|
|
|
# Determine root cause from status code
|
|
case $status in
|
|
500) root_cause="SERVER_ERROR_500" ;;
|
|
502) root_cause="APACHE_CONFIG:bad_gateway" ;;
|
|
503) root_cause="APACHE_CONFIG:service_unavailable" ;;
|
|
504) root_cause="APACHE_CONFIG:gateway_timeout" ;;
|
|
*) root_cause="SERVER_ERROR_5XX" ;;
|
|
esac
|
|
|
|
((critical_found++))
|
|
echo "$domain||[$timestamp] HTTP $status - $url (from $ip)|$root_cause" >> "$CRITICAL_ERRORS"
|
|
fi
|
|
|
|
done < <(tail -n 50000 "$log_path" 2>/dev/null)
|
|
else
|
|
# Error log - look for critical errors
|
|
# Use time-based filtering if possible, otherwise last 50k lines
|
|
cutoff_time=$(date -d "$HOURS_TO_ANALYZE hours ago" +%s 2>/dev/null || echo "0")
|
|
|
|
while IFS= read -r line; do
|
|
((total_lines++))
|
|
|
|
# Skip noise
|
|
if is_noise "$line"; then
|
|
((filtered_out++))
|
|
continue
|
|
fi
|
|
|
|
# Time filtering (Apache/PHP error log format: [Day Mon DD HH:MM:SS YYYY])
|
|
if [ "$cutoff_time" != "0" ]; then
|
|
if [[ "$line" =~ \[([A-Z][a-z]{2}\ [A-Z][a-z]{2}\ [0-9]{2}\ [0-9]{2}:[0-9]{2}:[0-9]{2}\ [0-9]{4})\] ]]; then
|
|
log_date="${BASH_REMATCH[1]}"
|
|
log_time=$(date -d "$log_date" +%s 2>/dev/null || echo "0")
|
|
[ "$log_time" != "0" ] && [ "$log_time" -lt "$cutoff_time" ] && continue
|
|
fi
|
|
fi
|
|
|
|
# Apply user/domain filter if set
|
|
if [ -n "$FILTER_USER" ]; then
|
|
[[ "$line" =~ /home/$FILTER_USER ]] || continue
|
|
fi
|
|
if [ -n "$FILTER_DOMAIN" ]; then
|
|
[[ "$line" =~ $FILTER_DOMAIN ]] || continue
|
|
fi
|
|
|
|
# Check if it's critical and user-facing
|
|
if is_critical_user_facing "$line"; then
|
|
((critical_found++))
|
|
extract_useful_info "$line" >> "$CRITICAL_ERRORS"
|
|
fi
|
|
|
|
done < <(tail -n 50000 "$log_path" 2>/dev/null)
|
|
fi
|
|
|
|
done < "$LOG_FILES_LIST"
|
|
|
|
echo ""
|
|
echo "Scanned: $total_lines log entries"
|
|
echo "Filtered: $filtered_out noise entries (bots, warnings, etc.)"
|
|
echo "Found: $critical_found critical user-facing errors"
|
|
echo ""
|
|
|
|
################################################################################
|
|
# Generate Report
|
|
################################################################################
|
|
|
|
if [ ! -f "$CRITICAL_ERRORS" ] || [ ! -s "$CRITICAL_ERRORS" ]; then
|
|
echo ""
|
|
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
|
echo -e "${GREEN} ✓ NO CRITICAL USER-FACING ERRORS FOUND!${NC}"
|
|
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
|
echo ""
|
|
echo "Your websites appear to be functioning normally for real users."
|
|
echo "All errors found were from bots, scanners, or non-critical warnings."
|
|
echo ""
|
|
exit 0
|
|
fi
|
|
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo " ⚠️ CRITICAL ERRORS AFFECTING REAL USERS"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo ""
|
|
|
|
# Group by domain and count
|
|
echo -e "${RED}${BOLD}AFFECTED WEBSITES:${NC}"
|
|
echo ""
|
|
|
|
cat "$CRITICAL_ERRORS" | cut -d'|' -f1 | sort | uniq -c | sort -rn | head -10 | while read -r count domain; do
|
|
printf " ${RED}●${NC} %-40s ${BOLD}%4d errors${NC}\n" "$domain" "$count"
|
|
done
|
|
|
|
echo ""
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo " ERROR DETAILS (Top Issues)"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo ""
|
|
|
|
# Group identical errors and count them
|
|
awk -F'|' '{
|
|
key = $1 "|" $3 "|" $4 # domain|error_msg|root_cause
|
|
file[$1"|"$3"|"$4] = $2 # Store file path
|
|
count[key]++
|
|
}
|
|
END {
|
|
for (key in count) {
|
|
split(key, parts, "|")
|
|
domain = parts[1]
|
|
error = parts[2]
|
|
root_cause = parts[3]
|
|
file_path = file[key]
|
|
|
|
# Skip empty errors
|
|
if (length(error) > 0) {
|
|
print count[key] "|" domain "|" file_path "|" error "|" root_cause
|
|
}
|
|
}
|
|
}' "$CRITICAL_ERRORS" | sort -t'|' -k1 -rn | head -20 | while IFS='|' read -r count domain file_path error_msg root_cause; do
|
|
|
|
# Color code by frequency
|
|
if [ "$count" -ge 10 ]; then
|
|
color="${RED}"
|
|
priority="[HIGH FREQUENCY]"
|
|
elif [ "$count" -ge 5 ]; then
|
|
color="${YELLOW}"
|
|
priority="[MEDIUM]"
|
|
else
|
|
color="${INFO_COLOR}"
|
|
priority=""
|
|
fi
|
|
|
|
echo -e "${color}● Occurred $count times${NC} $priority"
|
|
[ -n "$domain" ] && [ "$domain" != "unknown" ] && echo " Domain: $domain"
|
|
[ -n "$file_path" ] && echo " File: $file_path"
|
|
echo " Error: $error_msg"
|
|
|
|
# Display root cause with color coding and actionable fix
|
|
if [ -n "$root_cause" ] && [ "$root_cause" != "UNKNOWN" ]; then
|
|
echo -ne " ${CYAN}Root Cause: ${BOLD}"
|
|
|
|
case "$root_cause" in
|
|
HTACCESS)
|
|
echo -e "${YELLOW}⚙️ .htaccess Configuration${NC}"
|
|
echo " → Check .htaccess syntax in domain root"
|
|
echo " → Look for invalid RewriteRule or directives"
|
|
;;
|
|
MODSECURITY*)
|
|
rule=$(echo "$root_cause" | cut -d':' -f2)
|
|
echo -e "${YELLOW}🛡️ ModSecurity WAF Block${NC}"
|
|
[ -n "$rule" ] && echo " → Rule ID: $rule"
|
|
echo " → Check if this is a false positive"
|
|
echo " → Review: /usr/local/apache/logs/modsec_audit.log"
|
|
;;
|
|
PHP_MEMORY*)
|
|
limit=$(echo "$root_cause" | cut -d':' -f2)
|
|
echo -e "${RED}💾 PHP Memory Exhausted${NC}"
|
|
[ -n "$limit" ] && echo " → Current limit: $limit"
|
|
echo " → Increase memory_limit in .user.ini or php.ini"
|
|
echo " → Recommended: 256M minimum, 512M for WooCommerce"
|
|
;;
|
|
PHP_TIMEOUT)
|
|
echo -e "${YELLOW}⏱️ PHP Execution Timeout${NC}"
|
|
echo " → Increase max_execution_time in php.ini"
|
|
echo " → Recommended: 300 for imports/backups"
|
|
;;
|
|
PHP_UPLOAD_LIMIT)
|
|
echo -e "${YELLOW}📤 PHP Upload Size Limit${NC}"
|
|
echo " → Increase upload_max_filesize and post_max_size"
|
|
echo " → Match both values (e.g., 64M)"
|
|
;;
|
|
FILE_PERMISSIONS)
|
|
echo -e "${RED}🔒 File Permission Denied${NC}"
|
|
echo " → Check file ownership and permissions"
|
|
echo " → Files should be 644, directories 755"
|
|
echo " → Owner should match cPanel user"
|
|
;;
|
|
MISSING_PHP_GD)
|
|
echo -e "${RED}📦 Missing PHP GD Extension${NC}"
|
|
echo " → Install: yum install ea-phpXX-php-gd"
|
|
echo " → Required for image processing"
|
|
;;
|
|
MISSING_PHP_CURL)
|
|
echo -e "${RED}📦 Missing PHP cURL Extension${NC}"
|
|
echo " → Install: yum install ea-phpXX-php-curl"
|
|
;;
|
|
MISSING_PHP_*)
|
|
module=$(echo "$root_cause" | sed 's/MISSING_PHP_//' | tr '[:upper:]' '[:lower:]')
|
|
echo -e "${RED}📦 Missing PHP Extension: $module${NC}"
|
|
echo " → Install: yum install ea-phpXX-php-$module"
|
|
;;
|
|
DB_MAX_CONNECTIONS)
|
|
echo -e "${RED}🗄️ Database Max Connections Reached${NC}"
|
|
echo " → Check: mysql -e 'SHOW VARIABLES LIKE \"max_connections\"'"
|
|
echo " → Increase max_connections in my.cnf"
|
|
echo " → Or optimize slow queries reducing connection time"
|
|
;;
|
|
DB_AUTH_FAILED)
|
|
echo -e "${RED}🗄️ Database Authentication Failed${NC}"
|
|
echo " → Verify credentials in wp-config.php / config files"
|
|
echo " → Check database user permissions"
|
|
;;
|
|
DB_TIMEOUT)
|
|
echo -e "${YELLOW}🗄️ Database Connection Timeout${NC}"
|
|
echo " → MySQL server may be overloaded"
|
|
echo " → Check slow query log"
|
|
;;
|
|
DB_ERROR)
|
|
echo -e "${RED}🗄️ Database Error${NC}"
|
|
echo " → Check MySQL error log for details"
|
|
;;
|
|
APACHE_CONFIG*)
|
|
issue=$(echo "$root_cause" | cut -d':' -f2)
|
|
echo -e "${YELLOW}⚙️ Apache Configuration${NC}"
|
|
[ -n "$issue" ] && echo " → Issue: $issue"
|
|
echo " → Check Apache config and vhost settings"
|
|
;;
|
|
PHP_SYNTAX_ERROR)
|
|
echo -e "${RED}⚠️ PHP Syntax Error in Code${NC}"
|
|
echo " → Review recent code changes"
|
|
echo " → Check for missing semicolons, brackets, quotes"
|
|
;;
|
|
MISSING_FILE)
|
|
echo -e "${YELLOW}📄 File Not Found${NC}"
|
|
echo " → File may have been deleted or moved"
|
|
echo " → Check for broken symlinks"
|
|
;;
|
|
PHP_FATAL_ERROR)
|
|
echo -e "${RED}⚠️ PHP Fatal Error${NC}"
|
|
echo " → Review PHP error logs for details"
|
|
echo " → May be compatibility issue or code bug"
|
|
;;
|
|
SERVER_ERROR_500)
|
|
echo -e "${RED}🔥 Generic 500 Internal Server Error${NC}"
|
|
echo " → Check PHP/Apache error logs for specifics"
|
|
;;
|
|
*)
|
|
echo -e "${NC}$root_cause"
|
|
;;
|
|
esac
|
|
fi
|
|
|
|
echo ""
|
|
done
|
|
|
|
################################################################################
|
|
# Root Cause Summary
|
|
################################################################################
|
|
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo " 📊 ROOT CAUSE BREAKDOWN"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo ""
|
|
|
|
# Count errors by root cause
|
|
cut -d'|' -f4 "$CRITICAL_ERRORS" | grep -v "^$" | sort | uniq -c | sort -rn | while read -r count cause; do
|
|
# Clean up cause display
|
|
clean_cause=$(echo "$cause" | sed 's/:.*//; s/_/ /g')
|
|
|
|
# Color code by severity
|
|
case "$cause" in
|
|
PHP_MEMORY*|DB_MAX_CONNECTIONS|PHP_FATAL_ERROR|SERVER_ERROR_500)
|
|
color="${RED}"; icon="🔥" ;;
|
|
HTACCESS|MODSECURITY*|PHP_TIMEOUT|DB_*)
|
|
color="${YELLOW}"; icon="⚠️ " ;;
|
|
MISSING_PHP*|FILE_PERMISSIONS)
|
|
color="${YELLOW}"; icon="📦" ;;
|
|
*)
|
|
color="${INFO_COLOR}"; icon="•" ;;
|
|
esac
|
|
|
|
printf " ${color}${icon} %-35s %4d errors${NC}\n" "$clean_cause" "$count"
|
|
done
|
|
|
|
echo ""
|
|
|
|
################################################################################
|
|
# Intelligent Recommendations
|
|
################################################################################
|
|
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo " 🔧 RECOMMENDED ACTIONS"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo ""
|
|
|
|
# Detect issue types and provide specific recommendations
|
|
if grep -qiE "500|Internal Server" "$CRITICAL_ERRORS"; then
|
|
echo -e "${RED}● 500 Internal Server Errors detected${NC}"
|
|
echo " Priority: URGENT - Users see broken pages"
|
|
echo ""
|
|
echo " Actions:"
|
|
echo " 1. Check PHP error logs for affected domains:"
|
|
for domain in $(grep -iE "500|Internal Server" "$CRITICAL_ERRORS" | cut -d'|' -f1 | sort -u | head -3); do
|
|
echo " tail -100 /home/$domain/public_html/error_log"
|
|
done
|
|
echo ""
|
|
echo " 2. Common causes:"
|
|
echo " - PHP syntax errors (check recent file changes)"
|
|
echo " - .htaccess syntax errors"
|
|
echo " - Incorrect file permissions"
|
|
echo " - Missing PHP modules"
|
|
echo ""
|
|
fi
|
|
|
|
if grep -qiE "Fatal error|Parse error" "$CRITICAL_ERRORS"; then
|
|
echo -e "${RED}● PHP Fatal/Parse Errors detected${NC}"
|
|
echo " Priority: URGENT - Site functionality broken"
|
|
echo ""
|
|
echo " Actions:"
|
|
echo " 1. Review recent plugin/theme updates"
|
|
echo " 2. Check PHP version compatibility"
|
|
echo " 3. Look for syntax errors in custom code"
|
|
echo " 4. Temporarily disable recently added plugins"
|
|
echo ""
|
|
fi
|
|
|
|
if grep -qiE "database|MySQL|Connection" "$CRITICAL_ERRORS"; then
|
|
echo -e "${RED}● Database Connection Errors detected${NC}"
|
|
echo " Priority: CRITICAL - Site may be completely down"
|
|
echo ""
|
|
echo " Actions:"
|
|
echo " 1. Check MySQL service: systemctl status mysql"
|
|
echo " 2. Verify database credentials in config files"
|
|
echo " 3. Check max_connections: mysql -e 'SHOW VARIABLES LIKE \"max_connections\"'"
|
|
echo " 4. Review current connections: mysql -e 'SHOW PROCESSLIST'"
|
|
echo ""
|
|
fi
|
|
|
|
if grep -qiE "memory.*exhausted|Out of memory" "$CRITICAL_ERRORS"; then
|
|
echo -e "${RED}● Memory Exhaustion detected${NC}"
|
|
echo " Priority: HIGH - Pages loading slowly or incompletely"
|
|
echo ""
|
|
echo " Current PHP memory limit: $(php -r 'echo ini_get("memory_limit");' 2>/dev/null)"
|
|
echo ""
|
|
echo " Actions:"
|
|
echo " 1. Increase memory_limit in php.ini or .user.ini"
|
|
echo " 2. Recommended: 256M minimum, 512M for WooCommerce"
|
|
echo " 3. Check for memory leaks in plugins"
|
|
echo " 4. Optimize database queries"
|
|
echo ""
|
|
fi
|
|
|
|
if grep -qiE "ModSecurity" "$CRITICAL_ERRORS"; then
|
|
echo -e "${YELLOW}● ModSecurity Blocks detected${NC}"
|
|
echo " Priority: MEDIUM - Security rules blocking requests"
|
|
echo ""
|
|
echo " Actions:"
|
|
echo " 1. Review ModSecurity audit log for false positives"
|
|
echo " 2. Check if legitimate functionality is being blocked"
|
|
echo " 3. Consider whitelisting specific rules if needed"
|
|
echo " 4. Review blocked attack types to understand threats"
|
|
echo " 5. ModSecurity log: $MODSEC_AUDIT_LOG"
|
|
echo ""
|
|
fi
|
|
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo ""
|
|
echo "Full analysis saved to: $CRITICAL_ERRORS"
|
|
echo ""
|
|
echo "Next steps:"
|
|
echo " 1. Address critical errors first (500s, database, fatal errors)"
|
|
echo " 2. Check error_log files for affected domains"
|
|
echo " 3. Test affected pages after fixes"
|
|
echo " 4. Re-run this tool to verify issues are resolved"
|
|
echo ""
|
|
|
|
press_enter
|