cschantz
a71946d9c3
- Increased line scanning from 5k/10k to 50k lines (covers more data) - Added actual time-based filtering using log timestamps - Now respects the user's time range selection (1h, 6h, 24h, 7d, 30d) - Filters access logs by Apache timestamp format - Filters error logs by PHP/Apache error timestamp format - Shows timestamp with each 500 error for correlation - Better catches intermittent 500 errors for real users Example: If you select "Last 24 hours", it now actually filters logs to only show errors from the last 24 hours, not just the last N lines which could be 5 minutes on a busy server.
865 lines
34 KiB
Bash
Executable File
865 lines
34 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
################################################################################
|
|
# Intelligent Website Error Analyzer
|
|
################################################################################
|
|
# Purpose: Find REAL website issues affecting REAL users
|
|
# Filters out: Bots, plugin warnings, deprecation notices, minor issues
|
|
# Focuses on: Critical errors that break sites for actual visitors
|
|
################################################################################
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
source "$SCRIPT_DIR/lib/common-functions.sh"
|
|
source "$SCRIPT_DIR/lib/system-detect.sh"
|
|
source "$SCRIPT_DIR/lib/user-manager.sh"
|
|
|
|
# Configuration
|
|
APACHE_ERROR_LOG="/var/log/apache2/error_log"
|
|
DOMLOGS_DIR="/var/log/apache2/domlogs"
|
|
MODSEC_AUDIT_LOG="/usr/local/apache/logs/modsec_audit.log"
|
|
HOURS_TO_ANALYZE=24
|
|
FILTER_USER=""
|
|
FILTER_DOMAIN=""
|
|
|
|
print_banner "Intelligent Website Error Analyzer"
|
|
|
|
echo ""
|
|
echo "Finding REAL issues affecting REAL users..."
|
|
echo ""
|
|
echo "What we filter out:"
|
|
echo " ✗ Bot/scanner activity"
|
|
echo " ✗ Plugin deprecation warnings"
|
|
echo " ✗ PHP notices (non-critical)"
|
|
echo " ✗ Random 404s for missing favicons/assets"
|
|
echo " ✗ Security probe attempts"
|
|
echo ""
|
|
echo "What we focus on:"
|
|
echo " ✓ 500 errors users actually see"
|
|
echo " ✓ Fatal PHP errors breaking functionality"
|
|
echo " ✓ Database connection failures"
|
|
echo " ✓ Permission issues blocking access"
|
|
echo " ✓ Memory exhaustion affecting performance"
|
|
echo ""
|
|
|
|
# Ask for filtering scope
|
|
echo -e "${CYAN}Analysis Scope:${NC}"
|
|
echo " 1) All users/domains (default)"
|
|
echo " 2) Specific cPanel user"
|
|
echo " 3) Specific domain"
|
|
echo ""
|
|
read -p "Select option [1]: " scope_choice
|
|
scope_choice=${scope_choice:-1}
|
|
|
|
case $scope_choice in
|
|
2)
|
|
# Select specific user
|
|
select_user_interactive "Select cPanel user to analyze"
|
|
if [ -n "$SELECTED_USER" ]; then
|
|
FILTER_USER="$SELECTED_USER"
|
|
echo "→ Filtering for user: $FILTER_USER"
|
|
fi
|
|
;;
|
|
3)
|
|
# Enter specific domain
|
|
echo ""
|
|
read -p "Enter domain name (e.g., example.com): " FILTER_DOMAIN
|
|
if [ -n "$FILTER_DOMAIN" ]; then
|
|
echo "→ Filtering for domain: $FILTER_DOMAIN"
|
|
fi
|
|
;;
|
|
*)
|
|
echo "→ Analyzing all users/domains"
|
|
;;
|
|
esac
|
|
|
|
echo ""
|
|
|
|
# Ask for time range
|
|
echo -e "${CYAN}How far back should we analyze?${NC}"
|
|
echo " 1) Last 1 hour"
|
|
echo " 2) Last 6 hours"
|
|
echo " 3) Last 24 hours (default)"
|
|
echo " 4) Last 7 days"
|
|
echo " 5) Last 30 days"
|
|
echo ""
|
|
read -p "Select option [3]: " time_choice
|
|
time_choice=${time_choice:-3}
|
|
|
|
case $time_choice in
|
|
1) HOURS_TO_ANALYZE=1 ;;
|
|
2) HOURS_TO_ANALYZE=6 ;;
|
|
3) HOURS_TO_ANALYZE=24 ;;
|
|
4) HOURS_TO_ANALYZE=168 ;;
|
|
5) HOURS_TO_ANALYZE=720 ;;
|
|
*) HOURS_TO_ANALYZE=24 ;;
|
|
esac
|
|
|
|
echo ""
|
|
echo "→ Analyzing last $HOURS_TO_ANALYZE hours..."
|
|
echo ""
|
|
|
|
# Temporary files
|
|
TEMP_DIR="/tmp/website-error-analysis-$$"
|
|
mkdir -p "$TEMP_DIR"
|
|
trap "rm -rf $TEMP_DIR" EXIT
|
|
|
|
CRITICAL_ERRORS="$TEMP_DIR/critical.txt"
|
|
USER_IMPACT_ERRORS="$TEMP_DIR/user_impact.txt"
|
|
LOG_FILES_LIST="$TEMP_DIR/log_files.txt"
|
|
|
|
# Discover all log file locations
|
|
echo "→ Discovering log file locations..."
|
|
> "$LOG_FILES_LIST"
|
|
|
|
# Apache main error log (only if not filtering by user)
|
|
if [ -z "$FILTER_USER" ] && [ -z "$FILTER_DOMAIN" ]; then
|
|
for log in /var/log/apache2/error_log /usr/local/apache/logs/error_log /var/log/httpd/error_log; do
|
|
[ -f "$log" ] && echo "$log|apache_main" >> "$LOG_FILES_LIST"
|
|
done
|
|
fi
|
|
|
|
# Per-domain PHP error logs in public_html
|
|
if [ -n "$FILTER_USER" ]; then
|
|
# Specific user
|
|
find "/home/$FILTER_USER" -name "error_log" -type f 2>/dev/null | while read -r log; do
|
|
echo "$log|php_$FILTER_USER" >> "$LOG_FILES_LIST"
|
|
done
|
|
elif [ -n "$FILTER_DOMAIN" ]; then
|
|
# Try to find domain's user and error logs
|
|
user=$(grep -l "DNS.*$FILTER_DOMAIN" /var/cpanel/users/* 2>/dev/null | head -1 | xargs basename 2>/dev/null)
|
|
if [ -n "$user" ]; then
|
|
find "/home/$user" -name "error_log" -type f 2>/dev/null | while read -r log; do
|
|
echo "$log|php_$FILTER_DOMAIN" >> "$LOG_FILES_LIST"
|
|
done
|
|
fi
|
|
else
|
|
# All users
|
|
find /home/*/public_html -name "error_log" -type f 2>/dev/null | while read -r log; do
|
|
username=$(echo "$log" | grep -oE '/home/[^/]+' | sed 's|/home/||')
|
|
echo "$log|php_$username" >> "$LOG_FILES_LIST"
|
|
done
|
|
fi
|
|
|
|
# Per-domain Apache logs in domlogs
|
|
if [ -d "$DOMLOGS_DIR" ]; then
|
|
if [ -n "$FILTER_DOMAIN" ]; then
|
|
# Specific domain
|
|
for log in "$DOMLOGS_DIR/$FILTER_DOMAIN" "$DOMLOGS_DIR/$FILTER_DOMAIN-"*; do
|
|
[ -f "$log" ] && echo "$log|domlog_$FILTER_DOMAIN" >> "$LOG_FILES_LIST"
|
|
done
|
|
elif [ -n "$FILTER_USER" ]; then
|
|
# Specific user - find their domains
|
|
if [ -f "/var/cpanel/users/$FILTER_USER" ]; then
|
|
grep "^DNS" "/var/cpanel/users/$FILTER_USER" | awk '{print $2}' | while read -r domain; do
|
|
for log in "$DOMLOGS_DIR/$domain" "$DOMLOGS_DIR/$domain-"*; do
|
|
[ -f "$log" ] && echo "$log|domlog_$domain" >> "$LOG_FILES_LIST"
|
|
done
|
|
done
|
|
fi
|
|
else
|
|
# All domains
|
|
for log in "$DOMLOGS_DIR"/*; do
|
|
[ -f "$log" ] && ! [[ "$log" =~ (bytes_log|offset|error_log|ftpxferlog)$ ]] && \
|
|
echo "$log|domlog_$(basename "$log")" >> "$LOG_FILES_LIST"
|
|
done
|
|
fi
|
|
fi
|
|
|
|
# ModSecurity audit log (only if not filtering or matches filter)
|
|
if [ -f "$MODSEC_AUDIT_LOG" ]; then
|
|
if [ -z "$FILTER_USER" ] && [ -z "$FILTER_DOMAIN" ]; then
|
|
echo "$MODSEC_AUDIT_LOG|modsec_audit" >> "$LOG_FILES_LIST"
|
|
elif [ -n "$FILTER_DOMAIN" ]; then
|
|
# Will filter ModSec entries during processing
|
|
echo "$MODSEC_AUDIT_LOG|modsec_audit" >> "$LOG_FILES_LIST"
|
|
fi
|
|
fi
|
|
|
|
log_count=$(wc -l < "$LOG_FILES_LIST" 2>/dev/null || echo "0")
|
|
echo " Found $log_count log files to analyze"
|
|
if [ -n "$FILTER_USER" ]; then
|
|
echo " Scope: User '$FILTER_USER'"
|
|
elif [ -n "$FILTER_DOMAIN" ]; then
|
|
echo " Scope: Domain '$FILTER_DOMAIN'"
|
|
fi
|
|
echo ""
|
|
|
|
################################################################################
|
|
# Intelligent Filtering
|
|
################################################################################
|
|
|
|
is_noise() {
|
|
local line="$1"
|
|
|
|
# Bot/Scanner patterns
|
|
if echo "$line" | grep -qiE "bot|crawler|spider|scanner|nikto|nmap|masscan|sqlmap|nessus|acunetix|burp|shodan|censys|zgrab|nuclei|semrush|ahrefs|mj12"; then
|
|
return 0
|
|
fi
|
|
|
|
# Known scanner IPs
|
|
if echo "$line" | grep -qE "45\.148\.|185\.220\.|89\.248\.165\."; then
|
|
return 0
|
|
fi
|
|
|
|
# WordPress plugin deprecation warnings (not user-facing)
|
|
if echo "$line" | grep -qiE "PHP Deprecated|Deprecated.*plugin|call_user_func_array.*deprecated"; then
|
|
return 0
|
|
fi
|
|
|
|
# Non-critical PHP notices
|
|
if echo "$line" | grep -qiE "PHP Notice.*Undefined (variable|index|offset)" && \
|
|
! echo "$line" | grep -qiE "fatal|error 500|white screen"; then
|
|
return 0
|
|
fi
|
|
|
|
# Missing favicon/common assets (not real errors)
|
|
if echo "$line" | grep -qiE "favicon\.ico|apple-touch-icon|robots\.txt|sitemap\.xml.*404"; then
|
|
return 0
|
|
fi
|
|
|
|
# Security probes
|
|
if echo "$line" | grep -qiE "wp-admin|wp-login|phpMyAdmin|phpmyadmin|\.env|\.git|config\.php|xmlrpc|eval-stdin|shell\.php|c99\.php|r57\.php|adminer\.php" && \
|
|
echo "$line" | grep -qiE "404|403|not found"; then
|
|
return 0
|
|
fi
|
|
|
|
# WordPress auto-updates and cron (normal operations)
|
|
if echo "$line" | grep -qiE "wp-cron\.php|doing_cron|auto.*update"; then
|
|
return 0
|
|
fi
|
|
|
|
# Common plugin update checks (not errors)
|
|
if echo "$line" | grep -qiE "update-check|version-check|api\.wordpress\.org"; then
|
|
return 0
|
|
fi
|
|
|
|
return 1
|
|
}
|
|
|
|
is_critical_user_facing() {
|
|
local line="$1"
|
|
|
|
# 500 Internal Server Error (users see white page)
|
|
if echo "$line" | grep -qiE " 500 |Internal Server Error"; then
|
|
return 0
|
|
fi
|
|
|
|
# PHP Fatal Errors (breaks functionality)
|
|
if echo "$line" | grep -qiE "PHP Fatal error|Fatal error:.*in /"; then
|
|
return 0
|
|
fi
|
|
|
|
# PHP Parse Errors (site completely broken)
|
|
if echo "$line" | grep -qiE "PHP Parse error|syntax error"; then
|
|
return 0
|
|
fi
|
|
|
|
# Database connection failures (site down)
|
|
if echo "$line" | grep -qiE "Error establishing.*database|Can't connect.*MySQL|Access denied for user.*database|Too many connections|MySQL server has gone away"; then
|
|
return 0
|
|
fi
|
|
|
|
# Memory exhaustion (white screen/incomplete pages)
|
|
if echo "$line" | grep -qiE "Allowed memory size.*exhausted|Out of memory|Fatal.*memory"; then
|
|
return 0
|
|
fi
|
|
|
|
# Segmentation fault (complete crash)
|
|
if echo "$line" | grep -qiE "Segmentation fault|signal 11"; then
|
|
return 0
|
|
fi
|
|
|
|
# Permission denied on critical files
|
|
if echo "$line" | grep -qiE "Permission denied" && \
|
|
echo "$line" | grep -qE "index\.php|wp-config\.php|\.htaccess|config\.php"; then
|
|
return 0
|
|
fi
|
|
|
|
# File not found for MAIN page (404 on homepage/index)
|
|
if echo "$line" | grep -qiE "File does not exist.*index\.(php|html)" || \
|
|
echo "$line" | grep -qE " 404 .*\" \"GET / "; then
|
|
return 0
|
|
fi
|
|
|
|
return 1
|
|
}
|
|
|
|
extract_useful_info() {
|
|
local line="$1"
|
|
|
|
# Extract domain
|
|
domain=$(echo "$line" | grep -oE '\[vhost [^:]+' | sed 's/\[vhost //' || \
|
|
echo "$line" | grep -oE '[a-zA-Z0-9.-]+\.(com|net|org|io|co|uk|us|dev)' | head -1 || \
|
|
echo "$line" | grep -oE '/home/[^/]+' | sed 's|/home/||' || echo "unknown")
|
|
|
|
# Extract file path if PHP error
|
|
file_path=$(echo "$line" | grep -oE "in /[^ ]+\.php" | sed 's/in //' || echo "")
|
|
|
|
# Extract error message (clean up ModSec noise, timestamps, etc.)
|
|
error_msg=$(echo "$line" | \
|
|
sed 's/^\[.*\] //' | \
|
|
sed 's/\[client [^]]*\] //' | \
|
|
sed 's/\[unique_id "[^"]*"\]//g' | \
|
|
sed 's/\[pid [^]]*\]//g' | \
|
|
sed 's/\[tid [^]]*\]//g' | \
|
|
grep -v "^$" | \
|
|
cut -c1-150)
|
|
|
|
# Skip if error message is empty or just whitespace
|
|
if [ -z "$(echo "$error_msg" | tr -d '[:space:]')" ]; then
|
|
return 1
|
|
fi
|
|
|
|
# Correlate to root cause
|
|
root_cause=$(correlate_root_cause "$line" "$error_msg" "$domain")
|
|
|
|
echo "$domain|$file_path|$error_msg|$root_cause"
|
|
}
|
|
|
|
correlate_root_cause() {
|
|
local line="$1"
|
|
local error_msg="$2"
|
|
local domain="$3"
|
|
local cause="UNKNOWN"
|
|
|
|
# .htaccess issues
|
|
if echo "$line" | grep -qiE "\.htaccess|Invalid command|RewriteRule|RewriteCond"; then
|
|
cause="HTACCESS"
|
|
|
|
# ModSecurity blocks
|
|
elif echo "$line" | grep -qiE "ModSecurity|mod_security|WAF"; then
|
|
cause="MODSECURITY"
|
|
|
|
# PHP memory limit
|
|
elif echo "$error_msg" | grep -qiE "memory.*exhausted|Allowed memory size"; then
|
|
# Get current memory limit for this domain/user
|
|
if [ -n "$domain" ] && [ "$domain" != "unknown" ]; then
|
|
user=$(grep -l "DNS.*$domain" /var/cpanel/users/* 2>/dev/null | head -1 | xargs basename 2>/dev/null)
|
|
if [ -n "$user" ]; then
|
|
mem_limit=$(grep -h "memory_limit" /home/$user/public_html/.user.ini /home/$user/.php.ini 2>/dev/null | tail -1 | cut -d'=' -f2 | tr -d ' ')
|
|
[ -n "$mem_limit" ] && cause="PHP_MEMORY:$mem_limit" || cause="PHP_MEMORY:default"
|
|
else
|
|
cause="PHP_MEMORY"
|
|
fi
|
|
else
|
|
cause="PHP_MEMORY"
|
|
fi
|
|
|
|
# PHP max execution time
|
|
elif echo "$error_msg" | grep -qiE "max_execution_time|Maximum execution time"; then
|
|
cause="PHP_TIMEOUT"
|
|
|
|
# PHP upload/post size limits
|
|
elif echo "$error_msg" | grep -qiE "upload_max_filesize|post_max_size"; then
|
|
cause="PHP_UPLOAD_LIMIT"
|
|
|
|
# File permissions
|
|
elif echo "$error_msg" | grep -qiE "Permission denied|Failed to open stream.*permission"; then
|
|
cause="FILE_PERMISSIONS"
|
|
|
|
# Missing PHP modules/extensions
|
|
elif echo "$error_msg" | grep -qiE "undefined function|Call to undefined function|Class .* not found"; then
|
|
# Try to identify which module
|
|
if echo "$error_msg" | grep -qiE "imagecreate|imagefilter|gd_"; then
|
|
cause="MISSING_PHP_GD"
|
|
elif echo "$error_msg" | grep -qiE "curl_|CURL"; then
|
|
cause="MISSING_PHP_CURL"
|
|
elif echo "$error_msg" | grep -qiE "json_|JSON"; then
|
|
cause="MISSING_PHP_JSON"
|
|
elif echo "$error_msg" | grep -qiE "mysqli|mysql_connect"; then
|
|
cause="MISSING_PHP_MYSQLI"
|
|
else
|
|
cause="MISSING_PHP_MODULE"
|
|
fi
|
|
|
|
# Database issues
|
|
elif echo "$error_msg" | grep -qiE "database|MySQL|mysqli"; then
|
|
if echo "$error_msg" | grep -qiE "Too many connections|max_connections"; then
|
|
cause="DB_MAX_CONNECTIONS"
|
|
elif echo "$error_msg" | grep -qiE "Access denied|authentication"; then
|
|
cause="DB_AUTH_FAILED"
|
|
elif echo "$error_msg" | grep -qiE "gone away|Lost connection"; then
|
|
cause="DB_TIMEOUT"
|
|
else
|
|
cause="DB_ERROR"
|
|
fi
|
|
|
|
# Apache configuration
|
|
elif echo "$error_msg" | grep -qiE "Invalid URI|Request exceeded the limit|LimitRequestLine"; then
|
|
cause="APACHE_CONFIG"
|
|
|
|
# PHP parse/syntax errors (code issues)
|
|
elif echo "$error_msg" | grep -qiE "Parse error|syntax error|unexpected"; then
|
|
cause="PHP_SYNTAX_ERROR"
|
|
|
|
# File not found (may indicate bad symlinks or missing files)
|
|
elif echo "$error_msg" | grep -qiE "No such file|failed to open stream"; then
|
|
cause="MISSING_FILE"
|
|
|
|
# Generic PHP fatal error
|
|
elif echo "$error_msg" | grep -qiE "Fatal error"; then
|
|
cause="PHP_FATAL_ERROR"
|
|
|
|
# 500 errors without specific cause
|
|
elif echo "$error_msg" | grep -qiE "500|Internal Server"; then
|
|
cause="SERVER_ERROR_500"
|
|
fi
|
|
|
|
echo "$cause"
|
|
}
|
|
|
|
################################################################################
|
|
# Main Analysis
|
|
################################################################################
|
|
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo " SCANNING FOR USER-IMPACTING ERRORS"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo ""
|
|
|
|
total_lines=0
|
|
filtered_out=0
|
|
critical_found=0
|
|
|
|
# Analyze all discovered log files
|
|
while IFS='|' read -r log_path log_type; do
|
|
echo " Analyzing: $(basename "$log_path")..."
|
|
|
|
# Determine log type
|
|
is_access_log=false
|
|
is_modsec_log=false
|
|
[[ "$log_type" == domlog_* ]] && ! [[ "$log_path" =~ error ]] && is_access_log=true
|
|
[[ "$log_type" == modsec_* ]] && is_modsec_log=true
|
|
|
|
if $is_modsec_log; then
|
|
# ModSecurity audit log - parse for blocks/denials
|
|
while IFS= read -r line; do
|
|
((total_lines++))
|
|
|
|
# Skip if not a security event
|
|
echo "$line" | grep -qiE "ModSecurity.*Action.*Intercepted|status: (403|406|500)" || continue
|
|
|
|
# Extract domain/host if filtering
|
|
if [ -n "$FILTER_DOMAIN" ]; then
|
|
echo "$line" | grep -q "$FILTER_DOMAIN" || continue
|
|
fi
|
|
|
|
# Extract useful info
|
|
if echo "$line" | grep -qiE "SQL Injection|XSS|Command Injection|Path Traversal"; then
|
|
((critical_found++))
|
|
attack_type=$(echo "$line" | grep -oiE "SQL Injection|XSS|Command Injection|Path Traversal" | head -1)
|
|
uri=$(echo "$line" | grep -oE "uri: [^ ]+" | sed 's/uri: //' || echo "")
|
|
domain=$(echo "$line" | grep -oE "hostname: [^ ]+" | sed 's/hostname: //' || echo "unknown")
|
|
|
|
# Extract rule ID if available
|
|
rule_id=$(echo "$line" | grep -oE "id \"[0-9]+\"" | grep -oE "[0-9]+" || echo "")
|
|
root_cause="MODSECURITY"
|
|
[ -n "$rule_id" ] && root_cause="MODSECURITY:rule_$rule_id"
|
|
|
|
echo "$domain||ModSecurity blocked: $attack_type $uri|$root_cause" >> "$CRITICAL_ERRORS"
|
|
fi
|
|
|
|
done < <(tail -n 5000 "$log_path" 2>/dev/null)
|
|
|
|
elif $is_access_log; then
|
|
# Access log - look for 5xx status codes
|
|
# Use time-based filtering if possible, otherwise last 50k lines
|
|
cutoff_time=$(date -d "$HOURS_TO_ANALYZE hours ago" +%s 2>/dev/null || echo "0")
|
|
|
|
while IFS= read -r line; do
|
|
((total_lines++))
|
|
|
|
# Skip if bot/scanner
|
|
if is_noise "$line"; then
|
|
((filtered_out++))
|
|
continue
|
|
fi
|
|
|
|
# Time filtering (Apache format: [DD/Mon/YYYY:HH:MM:SS +ZONE])
|
|
if [ "$cutoff_time" != "0" ]; then
|
|
log_date=$(echo "$line" | grep -oE '\[[0-9]{2}/[A-Z][a-z]{2}/[0-9]{4}:[0-9]{2}:[0-9]{2}:[0-9]{2}' | tr -d '[')
|
|
if [ -n "$log_date" ]; then
|
|
log_time=$(date -d "$(echo "$log_date" | sed 's/:/ /')" +%s 2>/dev/null || echo "0")
|
|
[ "$log_time" != "0" ] && [ "$log_time" -lt "$cutoff_time" ] && continue
|
|
fi
|
|
fi
|
|
|
|
# Extract status code and URL
|
|
if echo "$line" | grep -qE '" 5[0-9]{2} '; then
|
|
status=$(echo "$line" | grep -oE '" 5[0-9]{2} ' | tr -d '" ')
|
|
url=$(echo "$line" | awk '{print $7}' | cut -c1-80)
|
|
ip=$(echo "$line" | awk '{print $1}')
|
|
domain=$(basename "$log_path" | sed 's/-.*//')
|
|
timestamp=$(echo "$line" | grep -oE '\[[^]]+\]' | head -1 | tr -d '[]')
|
|
|
|
# Apply domain filter if set
|
|
if [ -n "$FILTER_DOMAIN" ] && [ "$domain" != "$FILTER_DOMAIN" ]; then
|
|
continue
|
|
fi
|
|
|
|
# Determine root cause from status code
|
|
case $status in
|
|
500) root_cause="SERVER_ERROR_500" ;;
|
|
502) root_cause="APACHE_CONFIG:bad_gateway" ;;
|
|
503) root_cause="APACHE_CONFIG:service_unavailable" ;;
|
|
504) root_cause="APACHE_CONFIG:gateway_timeout" ;;
|
|
*) root_cause="SERVER_ERROR_5XX" ;;
|
|
esac
|
|
|
|
((critical_found++))
|
|
echo "$domain||[$timestamp] HTTP $status - $url (from $ip)|$root_cause" >> "$CRITICAL_ERRORS"
|
|
fi
|
|
|
|
done < <(tail -n 50000 "$log_path" 2>/dev/null)
|
|
else
|
|
# Error log - look for critical errors
|
|
# Use time-based filtering if possible, otherwise last 50k lines
|
|
cutoff_time=$(date -d "$HOURS_TO_ANALYZE hours ago" +%s 2>/dev/null || echo "0")
|
|
|
|
while IFS= read -r line; do
|
|
((total_lines++))
|
|
|
|
# Skip noise
|
|
if is_noise "$line"; then
|
|
((filtered_out++))
|
|
continue
|
|
fi
|
|
|
|
# Time filtering (Apache/PHP error log format: [Day Mon DD HH:MM:SS YYYY])
|
|
if [ "$cutoff_time" != "0" ]; then
|
|
log_date=$(echo "$line" | grep -oE '\[[A-Z][a-z]{2} [A-Z][a-z]{2} [0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} [0-9]{4}\]' | tr -d '[]')
|
|
if [ -n "$log_date" ]; then
|
|
log_time=$(date -d "$log_date" +%s 2>/dev/null || echo "0")
|
|
[ "$log_time" != "0" ] && [ "$log_time" -lt "$cutoff_time" ] && continue
|
|
fi
|
|
fi
|
|
|
|
# Apply user/domain filter if set
|
|
if [ -n "$FILTER_USER" ]; then
|
|
echo "$line" | grep -q "/home/$FILTER_USER" || continue
|
|
fi
|
|
if [ -n "$FILTER_DOMAIN" ]; then
|
|
echo "$line" | grep -q "$FILTER_DOMAIN" || continue
|
|
fi
|
|
|
|
# Check if it's critical and user-facing
|
|
if is_critical_user_facing "$line"; then
|
|
((critical_found++))
|
|
extract_useful_info "$line" >> "$CRITICAL_ERRORS"
|
|
fi
|
|
|
|
done < <(tail -n 50000 "$log_path" 2>/dev/null)
|
|
fi
|
|
|
|
done < "$LOG_FILES_LIST"
|
|
|
|
echo ""
|
|
echo "Scanned: $total_lines log entries"
|
|
echo "Filtered: $filtered_out noise entries (bots, warnings, etc.)"
|
|
echo "Found: $critical_found critical user-facing errors"
|
|
echo ""
|
|
|
|
################################################################################
|
|
# Generate Report
|
|
################################################################################
|
|
|
|
if [ ! -f "$CRITICAL_ERRORS" ] || [ ! -s "$CRITICAL_ERRORS" ]; then
|
|
echo ""
|
|
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
|
echo -e "${GREEN} ✓ NO CRITICAL USER-FACING ERRORS FOUND!${NC}"
|
|
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
|
echo ""
|
|
echo "Your websites appear to be functioning normally for real users."
|
|
echo "All errors found were from bots, scanners, or non-critical warnings."
|
|
echo ""
|
|
exit 0
|
|
fi
|
|
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo " ⚠️ CRITICAL ERRORS AFFECTING REAL USERS"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo ""
|
|
|
|
# Group by domain and count
|
|
echo -e "${RED}${BOLD}AFFECTED WEBSITES:${NC}"
|
|
echo ""
|
|
|
|
cat "$CRITICAL_ERRORS" | cut -d'|' -f1 | sort | uniq -c | sort -rn | head -10 | while read -r count domain; do
|
|
printf " ${RED}●${NC} %-40s ${BOLD}%4d errors${NC}\n" "$domain" "$count"
|
|
done
|
|
|
|
echo ""
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo " ERROR DETAILS (Top Issues)"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo ""
|
|
|
|
# Group identical errors and count them
|
|
awk -F'|' '{
|
|
key = $1 "|" $3 "|" $4 # domain|error_msg|root_cause
|
|
file[$1"|"$3"|"$4] = $2 # Store file path
|
|
count[key]++
|
|
}
|
|
END {
|
|
for (key in count) {
|
|
split(key, parts, "|")
|
|
domain = parts[1]
|
|
error = parts[2]
|
|
root_cause = parts[3]
|
|
file_path = file[key]
|
|
|
|
# Skip empty errors
|
|
if (length(error) == 0) next
|
|
|
|
print count[key] "|" domain "|" file_path "|" error "|" root_cause
|
|
}
|
|
}' "$CRITICAL_ERRORS" | sort -t'|' -k1 -rn | head -20 | while IFS='|' read -r count domain file_path error_msg root_cause; do
|
|
|
|
# Color code by frequency
|
|
if [ "$count" -ge 10 ]; then
|
|
color="${RED}"
|
|
priority="[HIGH FREQUENCY]"
|
|
elif [ "$count" -ge 5 ]; then
|
|
color="${YELLOW}"
|
|
priority="[MEDIUM]"
|
|
else
|
|
color="${INFO_COLOR}"
|
|
priority=""
|
|
fi
|
|
|
|
echo -e "${color}● Occurred $count times${NC} $priority"
|
|
[ -n "$domain" ] && [ "$domain" != "unknown" ] && echo " Domain: $domain"
|
|
[ -n "$file_path" ] && echo " File: $file_path"
|
|
echo " Error: $error_msg"
|
|
|
|
# Display root cause with color coding and actionable fix
|
|
if [ -n "$root_cause" ] && [ "$root_cause" != "UNKNOWN" ]; then
|
|
echo -ne " ${CYAN}Root Cause: ${BOLD}"
|
|
|
|
case "$root_cause" in
|
|
HTACCESS)
|
|
echo -e "${YELLOW}⚙️ .htaccess Configuration${NC}"
|
|
echo " → Check .htaccess syntax in domain root"
|
|
echo " → Look for invalid RewriteRule or directives"
|
|
;;
|
|
MODSECURITY*)
|
|
rule=$(echo "$root_cause" | cut -d':' -f2)
|
|
echo -e "${YELLOW}🛡️ ModSecurity WAF Block${NC}"
|
|
[ -n "$rule" ] && echo " → Rule ID: $rule"
|
|
echo " → Check if this is a false positive"
|
|
echo " → Review: /usr/local/apache/logs/modsec_audit.log"
|
|
;;
|
|
PHP_MEMORY*)
|
|
limit=$(echo "$root_cause" | cut -d':' -f2)
|
|
echo -e "${RED}💾 PHP Memory Exhausted${NC}"
|
|
[ -n "$limit" ] && echo " → Current limit: $limit"
|
|
echo " → Increase memory_limit in .user.ini or php.ini"
|
|
echo " → Recommended: 256M minimum, 512M for WooCommerce"
|
|
;;
|
|
PHP_TIMEOUT)
|
|
echo -e "${YELLOW}⏱️ PHP Execution Timeout${NC}"
|
|
echo " → Increase max_execution_time in php.ini"
|
|
echo " → Recommended: 300 for imports/backups"
|
|
;;
|
|
PHP_UPLOAD_LIMIT)
|
|
echo -e "${YELLOW}📤 PHP Upload Size Limit${NC}"
|
|
echo " → Increase upload_max_filesize and post_max_size"
|
|
echo " → Match both values (e.g., 64M)"
|
|
;;
|
|
FILE_PERMISSIONS)
|
|
echo -e "${RED}🔒 File Permission Denied${NC}"
|
|
echo " → Check file ownership and permissions"
|
|
echo " → Files should be 644, directories 755"
|
|
echo " → Owner should match cPanel user"
|
|
;;
|
|
MISSING_PHP_GD)
|
|
echo -e "${RED}📦 Missing PHP GD Extension${NC}"
|
|
echo " → Install: yum install ea-phpXX-php-gd"
|
|
echo " → Required for image processing"
|
|
;;
|
|
MISSING_PHP_CURL)
|
|
echo -e "${RED}📦 Missing PHP cURL Extension${NC}"
|
|
echo " → Install: yum install ea-phpXX-php-curl"
|
|
;;
|
|
MISSING_PHP_*)
|
|
module=$(echo "$root_cause" | sed 's/MISSING_PHP_//' | tr '[:upper:]' '[:lower:]')
|
|
echo -e "${RED}📦 Missing PHP Extension: $module${NC}"
|
|
echo " → Install: yum install ea-phpXX-php-$module"
|
|
;;
|
|
DB_MAX_CONNECTIONS)
|
|
echo -e "${RED}🗄️ Database Max Connections Reached${NC}"
|
|
echo " → Check: mysql -e 'SHOW VARIABLES LIKE \"max_connections\"'"
|
|
echo " → Increase max_connections in my.cnf"
|
|
echo " → Or optimize slow queries reducing connection time"
|
|
;;
|
|
DB_AUTH_FAILED)
|
|
echo -e "${RED}🗄️ Database Authentication Failed${NC}"
|
|
echo " → Verify credentials in wp-config.php / config files"
|
|
echo " → Check database user permissions"
|
|
;;
|
|
DB_TIMEOUT)
|
|
echo -e "${YELLOW}🗄️ Database Connection Timeout${NC}"
|
|
echo " → MySQL server may be overloaded"
|
|
echo " → Check slow query log"
|
|
;;
|
|
DB_ERROR)
|
|
echo -e "${RED}🗄️ Database Error${NC}"
|
|
echo " → Check MySQL error log for details"
|
|
;;
|
|
APACHE_CONFIG*)
|
|
issue=$(echo "$root_cause" | cut -d':' -f2)
|
|
echo -e "${YELLOW}⚙️ Apache Configuration${NC}"
|
|
[ -n "$issue" ] && echo " → Issue: $issue"
|
|
echo " → Check Apache config and vhost settings"
|
|
;;
|
|
PHP_SYNTAX_ERROR)
|
|
echo -e "${RED}⚠️ PHP Syntax Error in Code${NC}"
|
|
echo " → Review recent code changes"
|
|
echo " → Check for missing semicolons, brackets, quotes"
|
|
;;
|
|
MISSING_FILE)
|
|
echo -e "${YELLOW}📄 File Not Found${NC}"
|
|
echo " → File may have been deleted or moved"
|
|
echo " → Check for broken symlinks"
|
|
;;
|
|
PHP_FATAL_ERROR)
|
|
echo -e "${RED}⚠️ PHP Fatal Error${NC}"
|
|
echo " → Review PHP error logs for details"
|
|
echo " → May be compatibility issue or code bug"
|
|
;;
|
|
SERVER_ERROR_500)
|
|
echo -e "${RED}🔥 Generic 500 Internal Server Error${NC}"
|
|
echo " → Check PHP/Apache error logs for specifics"
|
|
;;
|
|
*)
|
|
echo -e "${NC}$root_cause"
|
|
;;
|
|
esac
|
|
fi
|
|
|
|
echo ""
|
|
done
|
|
|
|
################################################################################
|
|
# Root Cause Summary
|
|
################################################################################
|
|
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo " 📊 ROOT CAUSE BREAKDOWN"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo ""
|
|
|
|
# Count errors by root cause
|
|
cut -d'|' -f4 "$CRITICAL_ERRORS" | grep -v "^$" | sort | uniq -c | sort -rn | while read -r count cause; do
|
|
# Clean up cause display
|
|
clean_cause=$(echo "$cause" | sed 's/:.*//; s/_/ /g')
|
|
|
|
# Color code by severity
|
|
case "$cause" in
|
|
PHP_MEMORY*|DB_MAX_CONNECTIONS|PHP_FATAL_ERROR|SERVER_ERROR_500)
|
|
color="${RED}"; icon="🔥" ;;
|
|
HTACCESS|MODSECURITY*|PHP_TIMEOUT|DB_*)
|
|
color="${YELLOW}"; icon="⚠️ " ;;
|
|
MISSING_PHP*|FILE_PERMISSIONS)
|
|
color="${YELLOW}"; icon="📦" ;;
|
|
*)
|
|
color="${INFO_COLOR}"; icon="•" ;;
|
|
esac
|
|
|
|
printf " ${color}${icon} %-35s %4d errors${NC}\n" "$clean_cause" "$count"
|
|
done
|
|
|
|
echo ""
|
|
|
|
################################################################################
|
|
# Intelligent Recommendations
|
|
################################################################################
|
|
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo " 🔧 RECOMMENDED ACTIONS"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo ""
|
|
|
|
# Detect issue types and provide specific recommendations
|
|
if grep -qiE "500|Internal Server" "$CRITICAL_ERRORS"; then
|
|
echo -e "${RED}● 500 Internal Server Errors detected${NC}"
|
|
echo " Priority: URGENT - Users see broken pages"
|
|
echo ""
|
|
echo " Actions:"
|
|
echo " 1. Check PHP error logs for affected domains:"
|
|
for domain in $(grep -iE "500|Internal Server" "$CRITICAL_ERRORS" | cut -d'|' -f1 | sort -u | head -3); do
|
|
echo " tail -100 /home/$domain/public_html/error_log"
|
|
done
|
|
echo ""
|
|
echo " 2. Common causes:"
|
|
echo " - PHP syntax errors (check recent file changes)"
|
|
echo " - .htaccess syntax errors"
|
|
echo " - Incorrect file permissions"
|
|
echo " - Missing PHP modules"
|
|
echo ""
|
|
fi
|
|
|
|
if grep -qiE "Fatal error|Parse error" "$CRITICAL_ERRORS"; then
|
|
echo -e "${RED}● PHP Fatal/Parse Errors detected${NC}"
|
|
echo " Priority: URGENT - Site functionality broken"
|
|
echo ""
|
|
echo " Actions:"
|
|
echo " 1. Review recent plugin/theme updates"
|
|
echo " 2. Check PHP version compatibility"
|
|
echo " 3. Look for syntax errors in custom code"
|
|
echo " 4. Temporarily disable recently added plugins"
|
|
echo ""
|
|
fi
|
|
|
|
if grep -qiE "database|MySQL|Connection" "$CRITICAL_ERRORS"; then
|
|
echo -e "${RED}● Database Connection Errors detected${NC}"
|
|
echo " Priority: CRITICAL - Site may be completely down"
|
|
echo ""
|
|
echo " Actions:"
|
|
echo " 1. Check MySQL service: systemctl status mysql"
|
|
echo " 2. Verify database credentials in config files"
|
|
echo " 3. Check max_connections: mysql -e 'SHOW VARIABLES LIKE \"max_connections\"'"
|
|
echo " 4. Review current connections: mysql -e 'SHOW PROCESSLIST'"
|
|
echo ""
|
|
fi
|
|
|
|
if grep -qiE "memory.*exhausted|Out of memory" "$CRITICAL_ERRORS"; then
|
|
echo -e "${RED}● Memory Exhaustion detected${NC}"
|
|
echo " Priority: HIGH - Pages loading slowly or incompletely"
|
|
echo ""
|
|
echo " Current PHP memory limit: $(php -r 'echo ini_get("memory_limit");' 2>/dev/null)"
|
|
echo ""
|
|
echo " Actions:"
|
|
echo " 1. Increase memory_limit in php.ini or .user.ini"
|
|
echo " 2. Recommended: 256M minimum, 512M for WooCommerce"
|
|
echo " 3. Check for memory leaks in plugins"
|
|
echo " 4. Optimize database queries"
|
|
echo ""
|
|
fi
|
|
|
|
if grep -qiE "ModSecurity" "$CRITICAL_ERRORS"; then
|
|
echo -e "${YELLOW}● ModSecurity Blocks detected${NC}"
|
|
echo " Priority: MEDIUM - Security rules blocking requests"
|
|
echo ""
|
|
echo " Actions:"
|
|
echo " 1. Review ModSecurity audit log for false positives"
|
|
echo " 2. Check if legitimate functionality is being blocked"
|
|
echo " 3. Consider whitelisting specific rules if needed"
|
|
echo " 4. Review blocked attack types to understand threats"
|
|
echo " 5. ModSecurity log: $MODSEC_AUDIT_LOG"
|
|
echo ""
|
|
fi
|
|
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo ""
|
|
echo "Full analysis saved to: $CRITICAL_ERRORS"
|
|
echo ""
|
|
echo "Next steps:"
|
|
echo " 1. Address critical errors first (500s, database, fatal errors)"
|
|
echo " 2. Check error_log files for affected domains"
|
|
echo " 3. Test affected pages after fixes"
|
|
echo " 4. Re-run this tool to verify issues are resolved"
|
|
echo ""
|
|
|
|
press_enter
|