From b7417a6bfaa1b3d0c5fee047d16fcc7b04a986d0 Mon Sep 17 00:00:00 2001 From: cschantz Date: Mon, 17 Nov 2025 22:28:38 -0500 Subject: [PATCH] Fix live-attack-monitor auto-blocking and bot-analyzer compression - live-attack-monitor.sh: * Remove snapshot loading (start fresh each session) * Fix Apache log monitoring to use tail -n 0 -F (only new entries) * Add IP file sync to main loop for auto-blocking to work * Fix IP_DATA consolidation for cross-process communication - bot-analyzer.sh: * Implement gzip compression for large temp files (10-20x space savings) * Update all read/write operations to use compressed files * Fix for servers with 200+ domains and millions of log entries - run.sh: * Add HISTFILE fallback to prevent crashes when sourced --- modules/security/bot-analyzer.sh | 96 +++--- modules/security/live-attack-monitor.sh | 418 +++++++++++++++++++----- run.sh | 9 + 3 files changed, 391 insertions(+), 132 deletions(-) diff --git a/modules/security/bot-analyzer.sh b/modules/security/bot-analyzer.sh index ce0020b..a0d478e 100755 --- a/modules/security/bot-analyzer.sh +++ b/modules/security/bot-analyzer.sh @@ -348,20 +348,22 @@ parse_logs() { if (ip != "" && ip !~ /^[[:space:]]*$/) { print ip "|" domain "|" request_url "|" status "|" size "|" user_agent "|" http_method "|" timestamp } - }' "$logfile" >> "$TEMP_DIR/parsed_logs.txt" 2>/dev/null - done + }' "$logfile" 2>/dev/null + done | gzip > "$TEMP_DIR/parsed_logs.txt.gz" # Clear the progress line echo -ne "\r\033[K" - if [ ! -s "$TEMP_DIR/parsed_logs.txt" ]; then + if [ ! -s "$TEMP_DIR/parsed_logs.txt.gz" ]; then print_alert "No log entries were parsed. Check log format or permissions." return 1 fi local line_count - line_count=$(wc -l < "$TEMP_DIR/parsed_logs.txt") - print_success "Logs parsed successfully ($line_count entries)" + line_count=$(zcat "$TEMP_DIR/parsed_logs.txt.gz" | wc -l) + local file_size_kb + file_size_kb=$(du -k "$TEMP_DIR/parsed_logs.txt.gz" | cut -f1) + print_success "Logs parsed successfully ($line_count entries, ${file_size_kb}KB compressed)" return 0 } @@ -460,16 +462,18 @@ classify_bots() { if (bot_type != "unknown") { print ip "|" domain "|" url "|" status "|" size "|" ua "|" method "|" timestamp "|" bot_type "|" bot_name } - }' "$TEMP_DIR/parsed_logs.txt" > "$TEMP_DIR/classified_bots.txt" + }' < <(zcat "$TEMP_DIR/parsed_logs.txt.gz") | gzip > "$TEMP_DIR/classified_bots.txt.gz" - if [ ! -s "$TEMP_DIR/classified_bots.txt" ]; then + if [ ! -s "$TEMP_DIR/classified_bots.txt.gz" ]; then print_alert "Bot classification failed" return 1 fi local classified_count - classified_count=$(wc -l < "$TEMP_DIR/classified_bots.txt") - print_success "Bot classification complete ($classified_count entries)" + classified_count=$(zcat "$TEMP_DIR/classified_bots.txt.gz" | wc -l) + local file_size_kb + file_size_kb=$(du -k "$TEMP_DIR/classified_bots.txt.gz" | cut -f1) + print_success "Bot classification complete ($classified_count entries, ${file_size_kb}KB compressed)" return 0 } @@ -556,7 +560,7 @@ detect_threats() { # Track response codes for intelligence print status > "'"$TEMP_DIR"'/response_codes_raw.txt" } - ' "$TEMP_DIR/parsed_logs.txt" + ' < <(zcat "$TEMP_DIR/parsed_logs.txt.gz") # Process attack vectors by type if [ -f "$TEMP_DIR/attack_vectors_raw.txt" ]; then @@ -619,26 +623,26 @@ detect_threats() { detect_botnets() { print_info "Analyzing for botnet patterns..." - + # Group IPs by similar behavior patterns # Pattern 1: Multiple IPs hitting same URLs in coordinated manner - awk -F'|' '{print $1"|"$3}' "$TEMP_DIR/parsed_logs.txt" | \ + zcat "$TEMP_DIR/parsed_logs.txt.gz" | awk -F'|' '{print $1"|"$3}' | \ sort | uniq -c | awk '$1 > 10 {print $2}' | \ cut -d'|' -f2 | sort | uniq -c | sort -rn | \ awk '$1 > 5 {print $2}' > "$TEMP_DIR/coordinated_urls.txt" - + # Pattern 2: IPs with similar User-Agents hitting multiple domains - awk -F'|' '{print $1"|"$6}' "$TEMP_DIR/parsed_logs.txt" | \ + zcat "$TEMP_DIR/parsed_logs.txt.gz" | awk -F'|' '{print $1"|"$6}' | \ sort | uniq > "$TEMP_DIR/ip_ua_pairs.txt" - + # Pattern 3: Detect IP ranges (Class C networks) with suspicious activity - awk -F'|' '{print $1}' "$TEMP_DIR/parsed_logs.txt" | \ + zcat "$TEMP_DIR/parsed_logs.txt.gz" | awk -F'|' '{print $1}' | \ awk -F'.' '{print $1"."$2"."$3".0/24"}' | \ sort | uniq -c | sort -rn | awk '$1 > 20' > "$TEMP_DIR/suspicious_networks.txt" - + # Pattern 4: Rapid fire requests (DDoS indicators) # Extract timestamp and count requests per IP per minute - awk -F'|' '{ + zcat "$TEMP_DIR/parsed_logs.txt.gz" | awk -F'|' '{ ip = $1 timestamp = $8 # Extract date/time components (handles format: DD/MMM/YYYY:HH:MM:SS) @@ -647,13 +651,13 @@ detect_botnets() { time_key = ts[3] ts[2] ts[1] "_" ts[4] ts[5] print ip "|" time_key } - }' "$TEMP_DIR/parsed_logs.txt" | \ + }' | \ sort | uniq -c | \ awk '$1 > 50 {print $1 " " $2}' | \ awk -F'|' '{print $1}' | \ awk '{ip=$2; count=$1; sum[ip]+=count; max[ip]=(count>max[ip]?count:max[ip])} END {for(ip in sum) print sum[ip], ip, max[ip]}' | \ sort -rn > "$TEMP_DIR/rapid_fire_ips.txt" - + print_success "Botnet analysis complete" } @@ -742,13 +746,13 @@ analyze_time_series() { print_info "Analyzing time-series patterns..." # Extract hourly bot traffic - awk -F'|' '$9 != "unknown" { + zcat "$TEMP_DIR/classified_bots.txt.gz" | awk -F'|' '$9 != "unknown" { timestamp = $8 if (match(timestamp, /([0-9]{2})\/([A-Za-z]{3})\/([0-9]{4}):([0-9]{2}):([0-9]{2}):([0-9]{2})/, ts)) { hour = ts[4] print hour } - }' "$TEMP_DIR/classified_bots.txt" | sort | uniq -c > "$TEMP_DIR/hourly_bot_traffic.txt" + }' | sort | uniq -c > "$TEMP_DIR/hourly_bot_traffic.txt" # Extract hourly attack traffic if [ -f "$TEMP_DIR/attack_vectors_raw.txt" ]; then @@ -759,7 +763,7 @@ analyze_time_series() { hour = ts[4] print hour } - }' "$TEMP_DIR/attack_vectors_raw.txt" "$TEMP_DIR/parsed_logs.txt" | sort | uniq -c > "$TEMP_DIR/hourly_attack_traffic.txt" + }' "$TEMP_DIR/attack_vectors_raw.txt" <(zcat "$TEMP_DIR/parsed_logs.txt.gz") | sort | uniq -c > "$TEMP_DIR/hourly_attack_traffic.txt" fi print_success "Time-series analysis complete" @@ -776,7 +780,7 @@ calculate_threat_scores() { declare -A ip_request_counts while IFS='|' read -r ip rest; do ((ip_request_counts["$ip"]++)) - done < "$TEMP_DIR/parsed_logs.txt" + done < <(zcat "$TEMP_DIR/parsed_logs.txt.gz") # Build hash tables from threat files for O(1) lookups declare -A threat_ips_sqli threat_ips_xss threat_ips_path threat_ips_rce threat_ips_login @@ -922,7 +926,7 @@ detect_false_positives() { print_info "Detecting legitimate services (false positives)..." # Known monitoring service patterns - awk -F'|' '{ + zcat "$TEMP_DIR/parsed_logs.txt.gz" | awk -F'|' '{ ip = $1 domain = $2 url = $3 @@ -948,7 +952,7 @@ detect_false_positives() { else if (match(ua, /jetpack|vaultpress|updraftplus/)) { print ip "|Backup Service|" ua "|" domain } - }' "$TEMP_DIR/parsed_logs.txt" | sort -u > "$TEMP_DIR/false_positives.txt" + }' | sort -u > "$TEMP_DIR/false_positives.txt" print_success "False positive detection complete" } @@ -959,34 +963,34 @@ detect_false_positives() { generate_statistics() { print_info "Generating statistics..." - + # Top 5 bots by request count - awk -F'|' '$9 != "unknown" {print $10}' "$TEMP_DIR/classified_bots.txt" | \ + zcat "$TEMP_DIR/classified_bots.txt.gz" | awk -F'|' '$9 != "unknown" {print $10}' | \ sort | uniq -c | sort -rn | head -5 > "$TEMP_DIR/top_bots.txt" - + # Top 5 most-hit sites - awk -F'|' '{print $2}' "$TEMP_DIR/parsed_logs.txt" | \ + zcat "$TEMP_DIR/parsed_logs.txt.gz" | awk -F'|' '{print $2}' | \ sort | uniq -c | sort -rn | head -5 > "$TEMP_DIR/top_sites.txt" - + # Top 5 most-hit URLs - awk -F'|' '{print $2"|"$3}' "$TEMP_DIR/parsed_logs.txt" | \ + zcat "$TEMP_DIR/parsed_logs.txt.gz" | awk -F'|' '{print $2"|"$3}' | \ sort | uniq -c | sort -rn | head -5 > "$TEMP_DIR/top_urls.txt" - + # Top 5 IP addresses by request count - awk -F'|' '{print $1}' "$TEMP_DIR/parsed_logs.txt" | \ + zcat "$TEMP_DIR/parsed_logs.txt.gz" | awk -F'|' '{print $1}' | \ sort | uniq -c | sort -rn | head -5 > "$TEMP_DIR/top_ips.txt" - + # Traffic breakdown by bot type - awk -F'|' '{print $9}' "$TEMP_DIR/classified_bots.txt" | \ + zcat "$TEMP_DIR/classified_bots.txt.gz" | awk -F'|' '{print $9}' | \ sort | uniq -c | sort -rn > "$TEMP_DIR/traffic_breakdown.txt" - + # Per-domain traffic sources while read -r domain; do echo "$domain" > "$TEMP_DIR/domain_${domain}_stats.txt" - grep "|$domain|" "$TEMP_DIR/classified_bots.txt" | \ + zcat "$TEMP_DIR/classified_bots.txt.gz" | grep "|$domain|" | \ awk -F'|' '{print $9}' | sort | uniq -c | sort -rn >> "$TEMP_DIR/domain_${domain}_stats.txt" - done < <(awk -F'|' '{print $2}' "$TEMP_DIR/parsed_logs.txt" | sort -u) - + done < <(zcat "$TEMP_DIR/parsed_logs.txt.gz" | awk -F'|' '{print $2}' | sort -u) + print_success "Statistics generated" } @@ -1069,19 +1073,19 @@ generate_report() { # QUICK STATS DASHBOARD print_header "QUICK STATS DASHBOARD" - total_requests=$(wc -l < "$TEMP_DIR/parsed_logs.txt") - unique_ips=$(awk -F'|' '{print $1}' "$TEMP_DIR/parsed_logs.txt" | sort -u | wc -l) - unique_domains=$(awk -F'|' '{print $2}' "$TEMP_DIR/parsed_logs.txt" | sort -u | wc -l) - bot_requests=$(awk -F'|' '$9 != "unknown"' "$TEMP_DIR/classified_bots.txt" | wc -l) + total_requests=$(zcat "$TEMP_DIR/parsed_logs.txt.gz" | wc -l) + unique_ips=$(zcat "$TEMP_DIR/parsed_logs.txt.gz" | awk -F'|' '{print $1}' | sort -u | wc -l) + unique_domains=$(zcat "$TEMP_DIR/parsed_logs.txt.gz" | awk -F'|' '{print $2}' | sort -u | wc -l) + bot_requests=$(zcat "$TEMP_DIR/classified_bots.txt.gz" | awk -F'|' '$9 != "unknown"' | wc -l) # Count private/internal IPs (excluded from threat analysis) - private_ips=$(awk -F'|' '{print $1}' "$TEMP_DIR/parsed_logs.txt" | sort -u | grep -E '^(127\.|10\.|192\.168\.|172\.(1[6-9]|2[0-9]|3[01])\.|169\.254\.)' | wc -l) + private_ips=$(zcat "$TEMP_DIR/parsed_logs.txt.gz" | awk -F'|' '{print $1}' | sort -u | grep -E '^(127\.|10\.|192\.168\.|172\.(1[6-9]|2[0-9]|3[01])\.|169\.254\.)' | wc -l) # Count server's own IPs in the logs server_ip_hits=0 if [ -f "$TEMP_DIR/server_ips.txt" ] && [ -s "$TEMP_DIR/server_ips.txt" ]; then while read -r server_ip; do - if grep -q "^$server_ip|" "$TEMP_DIR/parsed_logs.txt" 2>/dev/null; then + if zcat "$TEMP_DIR/parsed_logs.txt.gz" | grep -q "^$server_ip|" 2>/dev/null; then server_ip_hits=$((server_ip_hits + 1)) fi done < "$TEMP_DIR/server_ips.txt" diff --git a/modules/security/live-attack-monitor.sh b/modules/security/live-attack-monitor.sh index e081abe..e16db05 100755 --- a/modules/security/live-attack-monitor.sh +++ b/modules/security/live-attack-monitor.sh @@ -55,23 +55,35 @@ touch "$TEMP_DIR/ip_data" echo "0" > "$TEMP_DIR/event_counter" echo "0" > "$TEMP_DIR/total_blocks" -# Save snapshot of IP data (for persistence across restarts) -save_snapshot() { - { - for ip in "${!IP_DATA[@]}"; do - echo "$ip=${IP_DATA[$ip]}" - done - } > "$SNAPSHOT_DIR/ip_data_snapshot" 2>/dev/null -} +# Initialize blocked IPs cache immediately on startup +{ + # Get CSF temporary blocks - extract just the IP address + if command -v csf &>/dev/null; then + csf -t 2>/dev/null | awk '{print $1}' | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$' + fi + + # Get CSF permanent denies + if [ -f /etc/csf/csf.deny ]; then + awk '{print $1}' /etc/csf/csf.deny 2>/dev/null | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$' + fi + + # Get iptables DROP rules + if command -v iptables &>/dev/null; then + iptables -L INPUT -n -v 2>/dev/null | grep DROP | awk '{print $8}' | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$' + fi +} | sort -u > "$TEMP_DIR/blocked_ips_cache" 2>/dev/null + +# Log cache initialization for debugging +if [ -f "$TEMP_DIR/blocked_ips_cache" ]; then + CACHED_COUNT=$(wc -l < "$TEMP_DIR/blocked_ips_cache" 2>/dev/null || echo "0") + echo "Initialized blocked IPs cache with $CACHED_COUNT IPs" >> "$TEMP_DIR/debug.log" +fi # Cleanup function cleanup() { echo "" echo "Stopping monitoring processes..." - # Save snapshot before exit - save_snapshot - # Kill all child processes pkill -P $$ 2>/dev/null @@ -104,13 +116,6 @@ VELOCITY_WINDOW=3600 # 1 hour in seconds DECAY_CHECK_INTERVAL=1800 # Check for decay every 30 minutes LAST_DECAY_CHECK=$START_TIME -# Load persistent data from previous sessions if exists -if [ -f "$SNAPSHOT_DIR/ip_data_snapshot" ]; then - while IFS='=' read -r ip data; do - [ -n "$ip" ] && IP_DATA[$ip]="$data" - done < "$SNAPSHOT_DIR/ip_data_snapshot" -fi - # Hide cursor for cleaner display command -v tput &>/dev/null && tput civis @@ -691,20 +696,171 @@ calculate_context_bonus() { echo "${bonus}|${reasons}" } -# Check if IP is currently blocked in CSF/iptables +# Block IP temporarily with CSF +block_ip_temporary() { + local ip="$1" + local hours="${2:-1}" + local reason="${3:-Auto-block by live monitor}" + local seconds=$((hours * 3600)) + + if command -v csf &>/dev/null; then + echo "Blocking $ip for ${hours}h: $reason" + csf -td "$ip" "$seconds" "$reason" >/dev/null 2>&1 + local result=$? + + # Verify the block was successful (check twice to be sure) + sleep 0.5 # Give CSF a moment to apply the rule + if verify_ip_blocked "$ip"; then + # Double-check to ensure it's really blocked + sleep 0.3 + if verify_ip_blocked "$ip"; then + echo "✓ Verified: $ip is now blocked" + + # Increment blocks counter + local current_total=$(cat "$TEMP_DIR/total_blocks" 2>/dev/null || echo "0") + echo $((current_total + 1)) > "$TEMP_DIR/total_blocks" + + # Trigger immediate cache refresh (don't wait for 10 second interval) + echo "Refreshing cache after blocking $ip..." >> "$TEMP_DIR/debug.log" + { + if command -v csf &>/dev/null; then + csf -t 2>/dev/null | awk '{print $1}' | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$' + fi + if [ -f /etc/csf/csf.deny ]; then + awk '{print $1}' /etc/csf/csf.deny 2>/dev/null | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$' + fi + if command -v iptables &>/dev/null; then + iptables -L INPUT -n -v 2>/dev/null | grep DROP | awk '{print $8}' | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$' + fi + } | sort -u > "$TEMP_DIR/blocked_ips_cache.tmp" 2>/dev/null + mv "$TEMP_DIR/blocked_ips_cache.tmp" "$TEMP_DIR/blocked_ips_cache" 2>/dev/null + + CACHE_COUNT=$(wc -l < "$TEMP_DIR/blocked_ips_cache" 2>/dev/null || echo 0) + echo "Cache refreshed: $CACHE_COUNT IPs total" >> "$TEMP_DIR/debug.log" + if grep -q "^$ip$" "$TEMP_DIR/blocked_ips_cache" 2>/dev/null; then + echo "✓ $ip confirmed in cache" >> "$TEMP_DIR/debug.log" + else + echo "✗ WARNING: $ip NOT in cache after refresh!" >> "$TEMP_DIR/debug.log" + # Add it manually as fallback with file locking to prevent race conditions + ( + flock -x 200 + echo "$ip" >> "$TEMP_DIR/blocked_ips_cache" + sort -u "$TEMP_DIR/blocked_ips_cache" -o "$TEMP_DIR/blocked_ips_cache" + ) 200>"$TEMP_DIR/cache.lock" + echo "✓ $ip added manually to cache" >> "$TEMP_DIR/debug.log" + fi + + return 0 + fi + fi + + echo "✗ Warning: Failed to verify block for $ip" + return 1 + fi + + echo "✗ Error: CSF not available" + return 1 +} + +# Block IP permanently with CSF +block_ip_permanent() { + local ip="$1" + local reason="${2:-Permanent block by live monitor}" + + if command -v csf &>/dev/null; then + echo "Permanently blocking $ip: $reason" + csf -d "$ip" "$reason" >/dev/null 2>&1 + local result=$? + + # Verify the block was successful (check twice to be sure) + sleep 0.5 # Give CSF a moment to apply the rule + if verify_ip_blocked "$ip"; then + # Double-check to ensure it's really blocked + sleep 0.3 + if verify_ip_blocked "$ip"; then + echo "✓ Verified: $ip is now permanently blocked" + + # Increment blocks counter + local current_total=$(cat "$TEMP_DIR/total_blocks" 2>/dev/null || echo "0") + echo $((current_total + 1)) > "$TEMP_DIR/total_blocks" + + # Trigger immediate cache refresh (don't wait for 10 second interval) + echo "Refreshing cache after permanently blocking $ip..." >> "$TEMP_DIR/debug.log" + { + if command -v csf &>/dev/null; then + csf -t 2>/dev/null | awk '{print $1}' | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$' + fi + if [ -f /etc/csf/csf.deny ]; then + awk '{print $1}' /etc/csf/csf.deny 2>/dev/null | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$' + fi + if command -v iptables &>/dev/null; then + iptables -L INPUT -n -v 2>/dev/null | grep DROP | awk '{print $8}' | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$' + fi + } | sort -u > "$TEMP_DIR/blocked_ips_cache.tmp" 2>/dev/null + mv "$TEMP_DIR/blocked_ips_cache.tmp" "$TEMP_DIR/blocked_ips_cache" 2>/dev/null + + CACHE_COUNT=$(wc -l < "$TEMP_DIR/blocked_ips_cache" 2>/dev/null || echo 0) + echo "Cache refreshed: $CACHE_COUNT IPs total" >> "$TEMP_DIR/debug.log" + if grep -q "^$ip$" "$TEMP_DIR/blocked_ips_cache" 2>/dev/null; then + echo "✓ $ip confirmed in cache" >> "$TEMP_DIR/debug.log" + else + echo "✗ WARNING: $ip NOT in cache after refresh!" >> "$TEMP_DIR/debug.log" + # Add it manually as fallback with file locking to prevent race conditions + ( + flock -x 200 + echo "$ip" >> "$TEMP_DIR/blocked_ips_cache" + sort -u "$TEMP_DIR/blocked_ips_cache" -o "$TEMP_DIR/blocked_ips_cache" + ) 200>"$TEMP_DIR/cache.lock" + echo "✓ $ip added manually to cache" >> "$TEMP_DIR/debug.log" + fi + + return 0 + fi + fi + + echo "✗ Warning: Failed to verify permanent block for $ip" + return 1 + fi + + echo "✗ Error: CSF not available" + return 1 +} + +# Check if IP is currently blocked in CSF/iptables (optimized with caching) is_ip_blocked() { local ip="$1" - # Check CSF deny list - if command -v csf &>/dev/null; then - if csf -g "$ip" 2>/dev/null | grep -q "DENY"; then + # Use cached blocked IPs list (refreshed every 10 seconds by background process) + if [ -f "$TEMP_DIR/blocked_ips_cache" ]; then + if grep -q "^$ip$" "$TEMP_DIR/blocked_ips_cache" 2>/dev/null; then return 0 fi fi + return 1 +} + +# Real-time verification (no cache) for immediate confirmation after blocking +verify_ip_blocked() { + local ip="$1" + + # Check CSF temporary blocks + if command -v csf &>/dev/null; then + if csf -t 2>/dev/null | grep -q "$ip"; then + return 0 + fi + + # Check CSF permanent deny list + if [ -f /etc/csf/csf.deny ]; then + if grep -q "^$ip" /etc/csf/csf.deny 2>/dev/null; then + return 0 + fi + fi + fi + # Check iptables directly if command -v iptables &>/dev/null; then - if iptables -L -n 2>/dev/null | grep -q "$ip"; then + if iptables -L INPUT -n 2>/dev/null | grep -q "$ip"; then return 0 fi fi @@ -714,7 +870,7 @@ is_ip_blocked() { # Get threat level from score get_threat_level() { - local score="$1" + local score="${1:-0}" if [ "$score" -ge "$THREAT_THRESHOLD_CRITICAL" ]; then echo "CRITICAL" @@ -776,55 +932,91 @@ draw_header() { draw_intelligence_panel() { echo -e "${HIGH_COLOR}┌─ THREAT INTELLIGENCE ──────────────────────────────────────────────────────┐${NC}" - # Get top IPs by threat score - local count=0 + # Debug: Show cache status + if [ -f "$TEMP_DIR/blocked_ips_cache" ]; then + CACHED_IPS=$(wc -l < "$TEMP_DIR/blocked_ips_cache" 2>/dev/null || echo 0) + echo -e "${INFO_COLOR} Cache: $CACHED_IPS blocked IPs${NC}" >> "$TEMP_DIR/debug.log" + else + echo -e "${INFO_COLOR} Cache: NOT FOUND${NC}" >> "$TEMP_DIR/debug.log" + fi + + # Get top IPs by threat score (exclude already blocked IPs) + local ip_list="" + local blocked_count=0 + local displayed_count=0 for ip in "${!IP_DATA[@]}"; do + # Skip IPs that are already blocked + if is_ip_blocked "$ip" 2>/dev/null; then + ((blocked_count++)) + echo " Filtering out blocked IP: $ip" >> "$TEMP_DIR/debug.log" + continue + fi + + ((displayed_count++)) IFS='|' read -r score hits bot_type attacks ban_count rep_score <<< "${IP_DATA[$ip]}" - echo "$score|$ip|$hits|$bot_type|$attacks|$ban_count|$rep_score" - done | sort -t'|' -k1 -rn | head -10 | while IFS='|' read -r score ip hits bot_type attacks ban_count rep_score; do - - local level=$(get_threat_level "$score") - local color=$(get_threat_color "$level") - local bot_color=$(get_bot_color "$bot_type") - - # Build status line - local status_line=$(printf "%-15s" "$ip") - status_line+=$(printf " Score:%-3s" "$score") - status_line+=$(printf " Hits:%-4s" "$hits") - - # Bot type indicator - case "$bot_type" in - legit) status_line+=" ✅BOT" ;; - ai) status_line+=" 🤖AI" ;; - monitor) status_line+=" 📊MON" ;; - suspicious) status_line+=" ⚠️ SUS" ;; - *) status_line+="" ;; - esac - - # Threat level - status_line+=$(printf " [%-8s]" "$level") - - # Attacks - if [ -n "$attacks" ]; then - # Show first attack type - local first_attack=$(echo "$attacks" | cut -d',' -f1) - local icon=$(get_attack_icon "$first_attack") - status_line+=" $icon$(echo "$attacks" | cut -d',' -f1)" - fi - - # Ban count - if [ "$ban_count" -gt 0 ]; then - status_line+=" 🚫x$ban_count" - fi - - # Known threat indicator - if [ "$rep_score" -gt 0 ]; then - status_line+=" [KNOWN]" - fi - - echo -e "${color}${status_line}${NC}" + ip_list+="$score|$ip|$hits|$bot_type|$attacks|$ban_count|$rep_score"$'\n' done + echo " Blocked/filtered: $blocked_count, Displaying: $displayed_count" >> "$TEMP_DIR/debug.log" + + if [ -n "$ip_list" ]; then + echo "$ip_list" | sort -t'|' -k1 -rn | head -10 | while IFS='|' read -r score ip hits bot_type attacks ban_count rep_score; do + # Set defaults for empty values + score="${score:-0}" + hits="${hits:-0}" + ban_count="${ban_count:-0}" + rep_score="${rep_score:-0}" + + local level=$(get_threat_level "$score") + local color=$(get_threat_color "$level") + local bot_color=$(get_bot_color "$bot_type") + + # Build status line + local status_line=$(printf "%-15s" "$ip") + status_line+=$(printf " Score:%-3s" "$score") + status_line+=$(printf " Hits:%-4s" "$hits") + + # Bot type indicator + case "$bot_type" in + legit) status_line+=" ✅BOT" ;; + ai) status_line+=" 🤖AI" ;; + monitor) status_line+=" 📊MON" ;; + suspicious) status_line+=" ⚠️ SUS" ;; + *) status_line+="" ;; + esac + + # Threat level + status_line+=$(printf " [%-8s]" "$level") + + # Attacks + if [ -n "$attacks" ]; then + # Show first attack type + local first_attack=$(echo "$attacks" | cut -d',' -f1) + local icon=$(get_attack_icon "$first_attack") + status_line+=" $icon$(echo "$attacks" | cut -d',' -f1)" + fi + + # Ban count + if [ "$ban_count" -gt 0 ]; then + status_line+=" 🚫x$ban_count" + fi + + # Known threat indicator + if [ "$rep_score" -gt 0 ]; then + status_line+=" [KNOWN]" + fi + + echo -e "${color}${status_line}${NC}" + done + else + # Show appropriate message + if [ ${#IP_DATA[@]} -gt 0 ]; then + echo -e "${SAFE_COLOR} ✓ All detected threats have been blocked${NC}" + else + echo -e "${LOW_COLOR} No threats detected yet...${NC}" + fi + fi + echo -e "${HIGH_COLOR}└────────────────────────────────────────────────────────────────────────────┘${NC}" echo "" } @@ -959,6 +1151,11 @@ show_blocking_menu() { for ip in "${!IP_DATA[@]}"; do IFS='|' read -r score hits bot_type attacks ban_count rep_score <<< "${IP_DATA[$ip]}" + # Set defaults for empty values + score="${score:-0}" + hits="${hits:-0}" + attacks="${attacks:-none}" + # Skip if score too low or already blocked [ "$score" -lt 60 ] && continue is_ip_blocked "$ip" 2>/dev/null && continue @@ -1012,17 +1209,24 @@ show_blocking_menu() { elif [ "$choice" = "a" ]; then # Block all IPs with score >= 80 local blocked=0 + local failed=0 for entry in "${blockable_list[@]}"; do IFS='|' read -r ip score hits attacks <<< "$entry" [ "$score" -lt 80 ] && continue echo "" - block_ip_temporary "$ip" 1 "Auto-block: High threat (score $score)" - ((blocked++)) + if block_ip_temporary "$ip" 1 "Auto-block: High threat (score $score)"; then + ((blocked++)) + else + ((failed++)) + fi done echo "" - echo "Blocked $blocked IPs" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + echo "✓ Successfully blocked: $blocked IPs" + [ $failed -gt 0 ] && echo "✗ Failed to block: $failed IPs" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" read -p "Press Enter to continue..." elif [[ "$choice" =~ ^[0-9]+$ ]] && [ "$choice" -ge 1 ] && [ "$choice" -le ${#blockable_list[@]} ]; then # Block specific IP @@ -1076,7 +1280,7 @@ monitor_apache_logs() { # Monitor all log files local event_count=0 - tail -f "${log_files[@]}" 2>/dev/null | while read -r line; do + tail -n 0 -F "${log_files[@]}" 2>/dev/null | while read -r line; do # Increment event counter (update file every 10 events for performance) ((event_count++)) if [ $((event_count % 10)) -eq 0 ]; then @@ -1982,6 +2186,30 @@ auto_mitigation_engine done ) & +# Blocked IPs cache updater (runs every 10 seconds for performance) +( + while true; do + { + # Get CSF temporary blocks - extract just the IP address + if command -v csf &>/dev/null; then + csf -t 2>/dev/null | awk '{print $1}' | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$' + fi + + # Get CSF permanent denies + if [ -f /etc/csf/csf.deny ]; then + awk '{print $1}' /etc/csf/csf.deny 2>/dev/null | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$' + fi + + # Get iptables DROP rules + if command -v iptables &>/dev/null; then + iptables -L INPUT -n -v 2>/dev/null | grep DROP | awk '{print $8}' | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$' + fi + } | sort -u > "$TEMP_DIR/blocked_ips_cache.tmp" 2>/dev/null + mv "$TEMP_DIR/blocked_ips_cache.tmp" "$TEMP_DIR/blocked_ips_cache" 2>/dev/null + sleep 10 + done +) & + # Periodic snapshot saving in background ( while true; do @@ -1993,26 +2221,44 @@ auto_mitigation_engine # Main dashboard loop LOOP_COUNT=0 while true; do + # Sync individual IP files into IP_DATA array (for data from subshell processes like SSH monitoring) + for ip_file in "$TEMP_DIR"/ip_*; do + [ -f "$ip_file" ] || continue + basename_file="$(basename "$ip_file")" + + # Skip non-IP files explicitly + case "$basename_file" in + ip_data|ip_database.db|*cache*|*blocked*|*debug*) + continue + ;; + esac + + # Validate it's an IP file (should match pattern ip_N_N_N_N) + if ! echo "$basename_file" | grep -qE '^ip_[0-9]{1,3}_[0-9]{1,3}_[0-9]{1,3}_[0-9]{1,3}$'; then + continue + fi + + # Extract IP from filename (ip_1_2_3_4 -> 1.2.3.4) + ip=$(echo "$basename_file" | sed 's/^ip_//' | tr '_' '.') + data=$(cat "$ip_file" 2>/dev/null) + + # Validate data format (should be score|hits|bot_type|attacks|ban_count|rep_score) + if [ -n "$data" ] && echo "$data" | grep -q '|'; then + # Update IP_DATA array with data from file + IP_DATA[$ip]="$data" + fi + done + draw_header draw_intelligence_panel draw_attack_breakdown draw_live_feed draw_quick_actions - # Consolidate IP data from individual files into ip_data file (for auto-mitigation engine) + # Write IP_DATA to ip_data file for auto-mitigation engine { - for ip_file in "$TEMP_DIR"/ip_*; do - [ -f "$ip_file" ] || continue - # Skip the consolidated ip_data file itself - [[ "$(basename "$ip_file")" == "ip_data" ]] && continue - # Extract IP from filename (ip_1_2_3_4 -> 1.2.3.4) - ip=$(basename "$ip_file" | sed 's/^ip_//' | tr '_' '.') - data=$(cat "$ip_file" 2>/dev/null) - if [ -n "$data" ]; then - echo "$ip=$data" - # Also update IP_DATA array for dashboard display - IP_DATA[$ip]="$data" - fi + for ip in "${!IP_DATA[@]}"; do + echo "$ip=${IP_DATA[$ip]}" done } > "$TEMP_DIR/ip_data" 2>/dev/null diff --git a/run.sh b/run.sh index baafe4c..9dc1bcb 100755 --- a/run.sh +++ b/run.sh @@ -8,6 +8,15 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Fix HISTFILE if set to non-existent path (prevents crashes on sourcing) +if [ -n "$HISTFILE" ]; then + HISTFILE_DIR="$(dirname "$HISTFILE" 2>/dev/null)" + if [ ! -d "$HISTFILE_DIR" ] 2>/dev/null; then + # Fallback to default history location + export HISTFILE="$HOME/.bash_history" + fi +fi + # Check if being sourced or executed if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then echo "ERROR: This script must be sourced, not executed."