From 6a64951d9aaf15341315f17fd04400478c8883f5 Mon Sep 17 00:00:00 2001
From: cschantz <admin@server.local>
Date: Mon, 3 Nov 2025 19:24:54 -0500
Subject: [PATCH] Improve 500 error detection with time-based filtering

- Increased line scanning from 5k/10k to 50k lines (covers more data)
- Added actual time-based filtering using log timestamps
- Now respects the user's time range selection (1h, 6h, 24h, 7d, 30d)
- Filters access logs by Apache timestamp format
- Filters error logs by PHP/Apache error timestamp format
- Shows timestamp with each 500 error for correlation
- Better catches intermittent 500 errors for real users

Example: If you select "Last 24 hours", it now actually filters
logs to only show errors from the last 24 hours, not just the
last N lines which could be 5 minutes on a busy server.
---
 modules/website/website-error-analyzer.sh | 31 ++++++++++++++++++++---
 1 file changed, 28 insertions(+), 3 deletions(-)

diff --git a/modules/website/website-error-analyzer.sh b/modules/website/website-error-analyzer.sh
index fbc3df2..d64a142 100755
--- a/modules/website/website-error-analyzer.sh
+++ b/modules/website/website-error-analyzer.sh
@@ -463,6 +463,9 @@ while IFS='|' read -r log_path log_type; do
 
     elif $is_access_log; then
         # Access log - look for 5xx status codes
+        # Use time-based filtering if possible, otherwise last 50k lines
+        cutoff_time=$(date -d "$HOURS_TO_ANALYZE hours ago" +%s 2>/dev/null || echo "0")
+
         while IFS= read -r line; do
             ((total_lines++))
 
@@ -472,12 +475,22 @@ while IFS='|' read -r log_path log_type; do
                 continue
             fi
 
+            # Time filtering (Apache format: [DD/Mon/YYYY:HH:MM:SS +ZONE])
+            if [ "$cutoff_time" != "0" ]; then
+                log_date=$(echo "$line" | grep -oE '\[[0-9]{2}/[A-Z][a-z]{2}/[0-9]{4}:[0-9]{2}:[0-9]{2}:[0-9]{2}' | tr -d '[')
+                if [ -n "$log_date" ]; then
+                    log_time=$(date -d "$(echo "$log_date" | sed 's/:/ /')" +%s 2>/dev/null || echo "0")
+                    [ "$log_time" != "0" ] && [ "$log_time" -lt "$cutoff_time" ] && continue
+                fi
+            fi
+
             # Extract status code and URL
             if echo "$line" | grep -qE '" 5[0-9]{2} '; then
                 status=$(echo "$line" | grep -oE '" 5[0-9]{2} ' | tr -d '" ')
                 url=$(echo "$line" | awk '{print $7}' | cut -c1-80)
                 ip=$(echo "$line" | awk '{print $1}')
                 domain=$(basename "$log_path" | sed 's/-.*//')
+                timestamp=$(echo "$line" | grep -oE '\[[^]]+\]' | head -1 | tr -d '[]')
 
                 # Apply domain filter if set
                 if [ -n "$FILTER_DOMAIN" ] && [ "$domain" != "$FILTER_DOMAIN" ]; then
@@ -494,12 +507,15 @@ while IFS='|' read -r log_path log_type; do
                 esac
 
                 ((critical_found++))
-                echo "$domain||HTTP $status - $url (from $ip)|$root_cause" >> "$CRITICAL_ERRORS"
+                echo "$domain||[$timestamp] HTTP $status - $url (from $ip)|$root_cause" >> "$CRITICAL_ERRORS"
             fi
 
-        done < <(tail -n 5000 "$log_path" 2>/dev/null)
+        done < <(tail -n 50000 "$log_path" 2>/dev/null)
     else
         # Error log - look for critical errors
+        # Use time-based filtering if possible, otherwise last 50k lines
+        cutoff_time=$(date -d "$HOURS_TO_ANALYZE hours ago" +%s 2>/dev/null || echo "0")
+
         while IFS= read -r line; do
             ((total_lines++))
 
@@ -509,6 +525,15 @@ while IFS='|' read -r log_path log_type; do
                 continue
             fi
 
+            # Time filtering (Apache/PHP error log format: [Day Mon DD HH:MM:SS YYYY])
+            if [ "$cutoff_time" != "0" ]; then
+                log_date=$(echo "$line" | grep -oE '\[[A-Z][a-z]{2} [A-Z][a-z]{2} [0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} [0-9]{4}\]' | tr -d '[]')
+                if [ -n "$log_date" ]; then
+                    log_time=$(date -d "$log_date" +%s 2>/dev/null || echo "0")
+                    [ "$log_time" != "0" ] && [ "$log_time" -lt "$cutoff_time" ] && continue
+                fi
+            fi
+
             # Apply user/domain filter if set
             if [ -n "$FILTER_USER" ]; then
                 echo "$line" | grep -q "/home/$FILTER_USER" || continue
@@ -523,7 +548,7 @@ while IFS='|' read -r log_path log_type; do
                 extract_useful_info "$line" >> "$CRITICAL_ERRORS"
             fi
 
-        done < <(tail -n 10000 "$log_path" 2>/dev/null)
+        done < <(tail -n 50000 "$log_path" 2>/dev/null)
     fi
 
 done < "$LOG_FILES_LIST"