Add progress indicator to bot analyzer log parsing
The bot analyzer was silently processing thousands of log files with no progress feedback, appearing to stall on large servers. Changes: • Added progress counter showing every 50 log files parsed • Displays current domain being processed • Shows format: "Parsed 150 log files... (current: domain.com)" • Clears progress line when complete to avoid clutter • Interval set to 50 files (adjustable via progress_interval variable) Example output: Parsing logs from: /var/log/apache2/domlogs Parsed 50 log files... (current: example.com) Parsed 100 log files... (current: another.com) Logs parsed successfully (125432 entries) This gives real-time feedback on servers with 1000+ log files without overwhelming the output.
This commit is contained in:
@@ -352,6 +352,9 @@ parse_logs() {
|
||||
|
||||
# Parse all domain logs (excluding -bytes_log, .offset, and error_log files)
|
||||
# cPanel creates files like: domain.com, domain.com-ssl_log
|
||||
local file_count=0
|
||||
local progress_interval=50
|
||||
echo ""
|
||||
find "$LOG_DIR" -type f ! -name "*-bytes_log" ! -name "*.offset" ! -name "*error_log" "${find_opts[@]}" 2>/dev/null | while read -r logfile; do
|
||||
# Skip empty files
|
||||
[ -s "$logfile" ] || continue
|
||||
@@ -365,6 +368,12 @@ parse_logs() {
|
||||
fi
|
||||
fi
|
||||
|
||||
# Show progress every N files
|
||||
file_count=$((file_count + 1))
|
||||
if [ $((file_count % progress_interval)) -eq 0 ]; then
|
||||
echo -ne "\r Parsed $file_count log files... (current: $domain)"
|
||||
fi
|
||||
|
||||
# Parse Apache Combined Log Format with error handling
|
||||
# Format: IP - - [timestamp] "METHOD URL PROTOCOL" STATUS SIZE "REFERRER" "USER-AGENT"
|
||||
awk -v domain="$domain" '
|
||||
@@ -411,6 +420,9 @@ parse_logs() {
|
||||
}' "$logfile" >> "$TEMP_DIR/parsed_logs.txt" 2>/dev/null
|
||||
done
|
||||
|
||||
# Clear the progress line
|
||||
echo -ne "\r\033[K"
|
||||
|
||||
if [ ! -s "$TEMP_DIR/parsed_logs.txt" ]; then
|
||||
print_alert "No log entries were parsed. Check log format or permissions."
|
||||
return 1
|
||||
|
||||
Reference in New Issue
Block a user