Add progress indicator to bot analyzer log parsing
The bot analyzer was silently processing thousands of log files with no progress feedback, appearing to stall on large servers. Changes: • Added progress counter showing every 50 log files parsed • Displays current domain being processed • Shows format: "Parsed 150 log files... (current: domain.com)" • Clears progress line when complete to avoid clutter • Interval set to 50 files (adjustable via progress_interval variable) Example output: Parsing logs from: /var/log/apache2/domlogs Parsed 50 log files... (current: example.com) Parsed 100 log files... (current: another.com) Logs parsed successfully (125432 entries) This gives real-time feedback on servers with 1000+ log files without overwhelming the output.
This commit is contained in:
@@ -352,6 +352,9 @@ parse_logs() {
|
|||||||
|
|
||||||
# Parse all domain logs (excluding -bytes_log, .offset, and error_log files)
|
# Parse all domain logs (excluding -bytes_log, .offset, and error_log files)
|
||||||
# cPanel creates files like: domain.com, domain.com-ssl_log
|
# cPanel creates files like: domain.com, domain.com-ssl_log
|
||||||
|
local file_count=0
|
||||||
|
local progress_interval=50
|
||||||
|
echo ""
|
||||||
find "$LOG_DIR" -type f ! -name "*-bytes_log" ! -name "*.offset" ! -name "*error_log" "${find_opts[@]}" 2>/dev/null | while read -r logfile; do
|
find "$LOG_DIR" -type f ! -name "*-bytes_log" ! -name "*.offset" ! -name "*error_log" "${find_opts[@]}" 2>/dev/null | while read -r logfile; do
|
||||||
# Skip empty files
|
# Skip empty files
|
||||||
[ -s "$logfile" ] || continue
|
[ -s "$logfile" ] || continue
|
||||||
@@ -365,6 +368,12 @@ parse_logs() {
|
|||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Show progress every N files
|
||||||
|
file_count=$((file_count + 1))
|
||||||
|
if [ $((file_count % progress_interval)) -eq 0 ]; then
|
||||||
|
echo -ne "\r Parsed $file_count log files... (current: $domain)"
|
||||||
|
fi
|
||||||
|
|
||||||
# Parse Apache Combined Log Format with error handling
|
# Parse Apache Combined Log Format with error handling
|
||||||
# Format: IP - - [timestamp] "METHOD URL PROTOCOL" STATUS SIZE "REFERRER" "USER-AGENT"
|
# Format: IP - - [timestamp] "METHOD URL PROTOCOL" STATUS SIZE "REFERRER" "USER-AGENT"
|
||||||
awk -v domain="$domain" '
|
awk -v domain="$domain" '
|
||||||
@@ -411,6 +420,9 @@ parse_logs() {
|
|||||||
}' "$logfile" >> "$TEMP_DIR/parsed_logs.txt" 2>/dev/null
|
}' "$logfile" >> "$TEMP_DIR/parsed_logs.txt" 2>/dev/null
|
||||||
done
|
done
|
||||||
|
|
||||||
|
# Clear the progress line
|
||||||
|
echo -ne "\r\033[K"
|
||||||
|
|
||||||
if [ ! -s "$TEMP_DIR/parsed_logs.txt" ]; then
|
if [ ! -s "$TEMP_DIR/parsed_logs.txt" ]; then
|
||||||
print_alert "No log entries were parsed. Check log format or permissions."
|
print_alert "No log entries were parsed. Check log format or permissions."
|
||||||
return 1
|
return 1
|
||||||
|
|||||||
Reference in New Issue
Block a user