Fix: Use mapfile for IP request counting to prevent read hangs
ISSUE: while IFS='|' read loop on 3000+ line files was causing hangs
SOLUTION: Replaced with mapfile -t which reads entire file at once
Extraction using parameter expansion: ${line%%|*} for first field
Result: Script now progresses past threat score calculation phase
This commit is contained in:
@@ -1656,23 +1656,37 @@ calculate_threat_scores() {
|
||||
|
||||
# Pre-load server IPs for fast exclusion checking (avoids grep in loop)
|
||||
declare -A server_ips_array
|
||||
echo " [DEBUG] Pre-loading server IPs..." >&2
|
||||
if [ -f "$TEMP_DIR/server_ips.txt" ]; then
|
||||
while read -r ip; do
|
||||
[ -n "$ip" ] && server_ips_array["$ip"]=1
|
||||
done < "$TEMP_DIR/server_ips.txt"
|
||||
fi
|
||||
echo " [DEBUG] Loaded $(( ${#server_ips_array[@]} )) server IPs" >&2
|
||||
|
||||
# Pre-count requests per IP (MUCH faster than grepping for each IP)
|
||||
declare -A ip_request_counts
|
||||
while IFS='|' read -r ip rest; do
|
||||
echo " [DEBUG] Counting requests per IP from: $TEMP_DIR/parsed_logs.txt" >&2
|
||||
echo " [DEBUG] File exists: $([ -f "$TEMP_DIR/parsed_logs.txt" ] && echo "yes" || echo "no")" >&2
|
||||
echo " [DEBUG] File size: $(wc -c < "$TEMP_DIR/parsed_logs.txt" 2>/dev/null || echo "error") bytes" >&2
|
||||
|
||||
# Use mapfile for faster counting instead of while-read
|
||||
mapfile -t parsed_lines < "$TEMP_DIR/parsed_logs.txt"
|
||||
echo " [DEBUG] Read $(( ${#parsed_lines[@]} )) lines" >&2
|
||||
|
||||
for line in "${parsed_lines[@]}"; do
|
||||
ip="${line%%|*}"
|
||||
((ip_request_counts["$ip"]++))
|
||||
done < "$TEMP_DIR/parsed_logs.txt"
|
||||
done
|
||||
echo " [DEBUG] Found $(( ${#ip_request_counts[@]} )) unique IPs" >&2
|
||||
|
||||
# Build hash tables from threat files for O(1) lookups
|
||||
# OPTIMIZATION: Use awk instead of echo|awk|cut in loops (10x faster)
|
||||
declare -A threat_ips_sqli threat_ips_xss threat_ips_path threat_ips_rce threat_ips_login
|
||||
declare -A threat_ips_suspicious threat_ips_ddos threat_admin_count threat_404_count
|
||||
|
||||
echo " [DEBUG] Loading threat files..." >&2
|
||||
|
||||
# Parse each threat file and build hash tables (using mapfile to avoid subshells)
|
||||
if [ -f "$TEMP_DIR/sqli_attempts.txt" ]; then
|
||||
mapfile -t sqli_ips < <(awk '{print $2}' "$TEMP_DIR/sqli_attempts.txt" | cut -d'|' -f1)
|
||||
|
||||
Reference in New Issue
Block a user