cschantz
85a17d7b4c
FILTERED LOG FILES: - proxy (Apache reverse proxy logs) - localhost (local connections) - default (default vhost) - cpanel, webmail, whm (cPanel services) - cpcalendars, cpcontacts, webdisk (cPanel apps) These are cPanel system services, not actual customer domains. They were showing as 'unknown' user and cluttering results. Now only tracks actual customer domain 500 errors.
301 lines
12 KiB
Bash
Executable File
301 lines
12 KiB
Bash
Executable File
#!/bin/bash
|
||
|
||
################################################################################
|
||
# Fast 500 Error Tracker
|
||
################################################################################
|
||
# Purpose: ONLY track 500 errors - find them fast and show WHY
|
||
# No menus, no options - just find 500s and diagnose the root cause
|
||
################################################################################
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||
source "$SCRIPT_DIR/lib/common-functions.sh"
|
||
|
||
print_banner "Fast 500 Error Tracker"
|
||
|
||
echo ""
|
||
echo "Scanning for 500 errors and diagnosing root causes..."
|
||
echo ""
|
||
|
||
# Ask for time range
|
||
echo -e "${CYAN}How far back to scan?${NC}"
|
||
echo " 1) Last 24 hours (default)"
|
||
echo " 2) Last 7 days"
|
||
echo " 3) Last 30 days"
|
||
echo ""
|
||
read -p "Select option [1]: " time_choice
|
||
time_choice=${time_choice:-1}
|
||
|
||
case $time_choice in
|
||
1) HOURS_TO_SCAN=24 ;;
|
||
2) HOURS_TO_SCAN=168 ;;
|
||
3) HOURS_TO_SCAN=720 ;;
|
||
*) HOURS_TO_SCAN=24 ;;
|
||
esac
|
||
|
||
echo ""
|
||
echo "→ Scanning last $HOURS_TO_SCAN hours of access logs..."
|
||
echo ""
|
||
|
||
# Temporary files
|
||
TEMP_DIR="/tmp/500-tracker-$$"
|
||
mkdir -p "$TEMP_DIR"
|
||
trap "rm -rf $TEMP_DIR" EXIT
|
||
|
||
ERRORS_500="$TEMP_DIR/errors_500.txt"
|
||
ERROR_DETAILS="$TEMP_DIR/error_details.txt"
|
||
|
||
# Configuration
|
||
DOMLOGS_DIR="/var/log/apache2/domlogs"
|
||
|
||
# Get cutoff time
|
||
cutoff_time=$(date -d "$HOURS_TO_SCAN hours ago" +%s 2>/dev/null || echo "0")
|
||
|
||
declare -A domain_count
|
||
declare -A domain_user
|
||
total_500s=0
|
||
|
||
# Scan all domain access logs for 500 errors
|
||
for log in "$DOMLOGS_DIR"/*; do
|
||
[ -f "$log" ] || continue
|
||
[[ "$log" =~ (bytes_log|offset|error_log|ftpxferlog|-ssl_log)$ ]] && continue
|
||
|
||
domain="${log##*/}"
|
||
domain="${domain%%-*}"
|
||
|
||
# Skip non-domain system logs (proxy, localhost, etc.)
|
||
[[ "$domain" =~ ^(proxy|localhost|default|cpanel|webmail|whm|cpcalendars|cpcontacts|webdisk)$ ]] && continue
|
||
|
||
# Find cPanel user for this domain
|
||
user=$(grep -l "DNS.*$domain" /var/cpanel/users/* 2>/dev/null | head -1 | xargs basename 2>/dev/null)
|
||
[ -z "$user" ] && user="unknown"
|
||
|
||
domain_user["$domain"]="$user"
|
||
|
||
# Scan for 500 errors in this log
|
||
while IFS= read -r line; do
|
||
# Check for 500 status code
|
||
if [[ "$line" =~ '"'[[:space:]](5[0-9]{2})[[:space:]] ]]; then
|
||
status="${BASH_REMATCH[1]}"
|
||
|
||
# Time filtering
|
||
if [ "$cutoff_time" != "0" ]; then
|
||
if [[ "$line" =~ \[([0-9]{2}/[A-Z][a-z]{2}/[0-9]{4}:[0-9]{2}:[0-9]{2}:[0-9]{2}) ]]; then
|
||
log_date="${BASH_REMATCH[1]}"
|
||
log_time=$(date -d "${log_date/:/ }" +%s 2>/dev/null || echo "0")
|
||
[ "$log_time" != "0" ] && [ "$log_time" -lt "$cutoff_time" ] && continue
|
||
fi
|
||
fi
|
||
|
||
# Parse log line
|
||
read -r ip _ _ timestamp _ request _ _ <<< "$line"
|
||
|
||
# Extract URL
|
||
if [[ "$request" =~ '"'[A-Z]+[[:space:]]([^[:space:]]+) ]]; then
|
||
url="${BASH_REMATCH[1]:0:80}"
|
||
else
|
||
url="/"
|
||
fi
|
||
|
||
# Extract timestamp
|
||
if [[ "$line" =~ \[([^]]+)\] ]]; then
|
||
timestamp="${BASH_REMATCH[1]}"
|
||
else
|
||
timestamp="unknown"
|
||
fi
|
||
|
||
((domain_count["$domain"]++))
|
||
((total_500s++))
|
||
|
||
# Save for analysis
|
||
echo "$domain|$user|$status|$url|$timestamp|$ip" >> "$ERRORS_500"
|
||
fi
|
||
done < <(tail -n 100000 "$log" 2>/dev/null)
|
||
done
|
||
|
||
if [ "$total_500s" -eq 0 ]; then
|
||
echo ""
|
||
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||
echo -e "${GREEN} ✓ NO 500 ERRORS FOUND IN LAST $HOURS_TO_SCAN HOURS!${NC}"
|
||
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||
echo ""
|
||
exit 0
|
||
fi
|
||
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
echo " 🔥 500 ERRORS DETECTED: $total_500s errors"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
echo ""
|
||
|
||
echo -e "${RED}${BOLD}TOP AFFECTED DOMAINS:${NC}"
|
||
echo ""
|
||
|
||
# Sort and display top domains
|
||
for domain in "${!domain_count[@]}"; do
|
||
echo "${domain_count[$domain]} $domain ${domain_user[$domain]}"
|
||
done | sort -rn | head -10 | while read count domain user; do
|
||
printf " ${RED}●${NC} %-40s ${BOLD}%4d errors${NC} (user: %s)\n" "$domain" "$count" "$user"
|
||
done
|
||
|
||
echo ""
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
echo " 🔍 DIAGNOSING ROOT CAUSES"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
echo ""
|
||
|
||
# For each affected domain, check the actual PHP error logs
|
||
declare -A diagnosed_causes
|
||
declare -A cause_examples
|
||
|
||
while IFS='|' read -r domain user status url timestamp ip; do
|
||
[ -z "$domain" ] && continue
|
||
|
||
# Find PHP error log - check multiple locations
|
||
error_log=""
|
||
if [ "$user" != "unknown" ]; then
|
||
# Try multiple possible locations
|
||
for potential_log in \
|
||
"/home/$user/public_html/error_log" \
|
||
"/home/$user/logs/error_log" \
|
||
"/home/$user/error_log" \
|
||
"/var/log/apache2/domlogs/$domain-error_log" \
|
||
"/usr/local/apache/domlogs/$domain"; do
|
||
|
||
if [ -f "$potential_log" ]; then
|
||
error_log="$potential_log"
|
||
break
|
||
fi
|
||
done
|
||
fi
|
||
|
||
# Check if error log exists and has recent errors
|
||
if [ -n "$error_log" ] && [ -f "$error_log" ]; then
|
||
# Look for errors around the same time as the 500
|
||
recent_error=$(tail -500 "$error_log" | grep -E "Fatal error|Parse error|syntax error|memory.*exhausted|database|MySQL|Permission denied" | tail -1)
|
||
|
||
if [ -n "$recent_error" ]; then
|
||
# Determine cause
|
||
cause="UNKNOWN"
|
||
if [[ "$recent_error" =~ (memory.*exhausted|Allowed memory size) ]]; then
|
||
cause="PHP_MEMORY_EXHAUSTED"
|
||
cause_examples["$cause"]="$domain: $recent_error"
|
||
elif [[ "$recent_error" =~ (Fatal error|PHP Fatal error) ]]; then
|
||
if [[ "$recent_error" =~ (undefined function|Call to undefined function) ]]; then
|
||
cause="MISSING_PHP_FUNCTION"
|
||
cause_examples["$cause"]="$domain: $recent_error"
|
||
else
|
||
cause="PHP_FATAL_ERROR"
|
||
cause_examples["$cause"]="$domain: $recent_error"
|
||
fi
|
||
elif [[ "$recent_error" =~ (Parse error|syntax error) ]]; then
|
||
cause="PHP_SYNTAX_ERROR"
|
||
cause_examples["$cause"]="$domain: $recent_error"
|
||
elif [[ "$recent_error" =~ (database|MySQL|Can\'t connect) ]]; then
|
||
cause="DATABASE_CONNECTION"
|
||
cause_examples["$cause"]="$domain: $recent_error"
|
||
fi
|
||
|
||
((diagnosed_causes["$cause"]++))
|
||
else
|
||
((diagnosed_causes["NO_PHP_ERROR_LOGGED"]++))
|
||
fi
|
||
else
|
||
# No error log found - likely .htaccess issue
|
||
# Check if .htaccess exists and might have issues
|
||
if [ "$user" != "unknown" ] && [ -f "/home/$user/public_html/.htaccess" ]; then
|
||
# Check for common .htaccess syntax errors
|
||
htaccess_check=$(grep -E "RewriteEngine|RewriteRule|RewriteCond|php_value|php_flag" "/home/$user/public_html/.htaccess" 2>/dev/null)
|
||
if [ -n "$htaccess_check" ]; then
|
||
((diagnosed_causes["HTACCESS_LIKELY"]++))
|
||
cause_examples["HTACCESS_LIKELY"]="$domain (user: $user) - has .htaccess with rewrite rules, check syntax"
|
||
else
|
||
((diagnosed_causes["NO_ERROR_LOG_FILE"]++))
|
||
cause_examples["NO_ERROR_LOG_FILE"]="$domain (user: $user) - checked multiple locations, no error_log found"
|
||
fi
|
||
else
|
||
((diagnosed_causes["NO_ERROR_LOG_FILE"]++))
|
||
cause_examples["NO_ERROR_LOG_FILE"]="$domain (user: $user) - checked multiple locations, no error_log found"
|
||
fi
|
||
fi
|
||
done < "$ERRORS_500"
|
||
|
||
# Display diagnosed causes
|
||
echo -e "${CYAN}${BOLD}ROOT CAUSES IDENTIFIED:${NC}"
|
||
echo ""
|
||
|
||
for cause in "${!diagnosed_causes[@]}"; do
|
||
count="${diagnosed_causes[$cause]}"
|
||
echo "$count|$cause"
|
||
done | sort -rn | while IFS='|' read count cause; do
|
||
clean_cause=$(echo "$cause" | tr '_' ' ')
|
||
|
||
case "$cause" in
|
||
PHP_MEMORY_EXHAUSTED)
|
||
echo -e "${RED}🔥 $clean_cause${NC} - $count occurrences"
|
||
echo " ${YELLOW}Fix:${NC} Increase memory_limit in /home/USER/public_html/.user.ini"
|
||
echo " ${YELLOW}Set:${NC} memory_limit = 512M"
|
||
;;
|
||
PHP_FATAL_ERROR)
|
||
echo -e "${RED}⚠️ $clean_cause${NC} - $count occurrences"
|
||
echo " ${YELLOW}Fix:${NC} Review recent code/plugin changes"
|
||
;;
|
||
PHP_SYNTAX_ERROR)
|
||
echo -e "${RED}📝 $clean_cause${NC} - $count occurrences"
|
||
echo " ${YELLOW}Fix:${NC} Check for PHP syntax errors in recent file edits"
|
||
;;
|
||
MISSING_PHP_FUNCTION)
|
||
echo -e "${RED}📦 $clean_cause${NC} - $count occurrences"
|
||
echo " ${YELLOW}Fix:${NC} Install missing PHP extension (check error for which one)"
|
||
;;
|
||
DATABASE_CONNECTION)
|
||
echo -e "${RED}🗄️ $clean_cause${NC} - $count occurrences"
|
||
echo " ${YELLOW}Fix:${NC} Check database credentials or MySQL service"
|
||
;;
|
||
HTACCESS_LIKELY)
|
||
echo -e "${RED}⚙️ .HTACCESS SYNTAX ERROR (LIKELY)${NC} - $count occurrences"
|
||
echo " ${YELLOW}Fix:${NC} Check .htaccess file for syntax errors"
|
||
echo " ${YELLOW}Test:${NC} Temporarily rename .htaccess to .htaccess.bak and test"
|
||
echo " ${YELLOW}Common issues:${NC} Invalid RewriteRule, bad php_value directives"
|
||
;;
|
||
NO_ERROR_LOG_FILE)
|
||
echo -e "${YELLOW}📄 $clean_cause${NC} - $count occurrences"
|
||
echo " ${YELLOW}Note:${NC} Checked multiple error_log locations - none found"
|
||
echo " ${YELLOW}Check:${NC} May need to enable PHP error logging"
|
||
;;
|
||
NO_PHP_ERROR_LOGGED)
|
||
echo -e "${YELLOW}❓ $clean_cause${NC} - $count occurrences"
|
||
echo " ${YELLOW}Note:${NC} 500 error but no PHP error in log - likely .htaccess or Apache config"
|
||
;;
|
||
*)
|
||
echo -e "${INFO_COLOR}• $clean_cause${NC} - $count occurrences"
|
||
;;
|
||
esac
|
||
|
||
# Show example if we have one
|
||
if [ -n "${cause_examples[$cause]}" ]; then
|
||
example="${cause_examples[$cause]}"
|
||
echo " ${DIM}Example: ${example:0:120}...${NC}"
|
||
fi
|
||
echo ""
|
||
done
|
||
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
echo " 📋 DETAILED 500 ERROR LIST"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
echo ""
|
||
|
||
# Show most frequent URLs getting 500s
|
||
echo "Most Frequent 500 Error URLs:"
|
||
echo ""
|
||
cut -d'|' -f1,4 "$ERRORS_500" | sort | uniq -c | sort -rn | head -15 | while read count domain_url; do
|
||
domain=$(echo "$domain_url" | cut -d'|' -f1)
|
||
url=$(echo "$domain_url" | cut -d'|' -f2)
|
||
user="${domain_user[$domain]}"
|
||
printf " ${RED}%4d×${NC} %s%s ${DIM}(user: %s)${NC}\n" "$count" "$domain" "$url" "$user"
|
||
done
|
||
|
||
echo ""
|
||
echo "Full error list saved to: $ERRORS_500"
|
||
echo ""
|
||
|
||
press_enter
|