Fix live-attack-monitor auto-blocking and bot-analyzer compression

- live-attack-monitor.sh:
  * Remove snapshot loading (start fresh each session)
  * Fix Apache log monitoring to use tail -n 0 -F (only new entries)
  * Add IP file sync to main loop for auto-blocking to work
  * Fix IP_DATA consolidation for cross-process communication

- bot-analyzer.sh:
  * Implement gzip compression for large temp files (10-20x space savings)
  * Update all read/write operations to use compressed files
  * Fix for servers with 200+ domains and millions of log entries

- run.sh:
  * Add HISTFILE fallback to prevent crashes when sourced

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
cschantz
2025-11-17 22:28:38 -05:00
parent f0d53322e6
commit 2d30cc0aea
3 changed files with 391 additions and 132 deletions
+332 -86
View File
@@ -55,23 +55,35 @@ touch "$TEMP_DIR/ip_data"
echo "0" > "$TEMP_DIR/event_counter"
echo "0" > "$TEMP_DIR/total_blocks"
# Save snapshot of IP data (for persistence across restarts)
save_snapshot() {
{
for ip in "${!IP_DATA[@]}"; do
echo "$ip=${IP_DATA[$ip]}"
done
} > "$SNAPSHOT_DIR/ip_data_snapshot" 2>/dev/null
}
# Initialize blocked IPs cache immediately on startup
{
# Get CSF temporary blocks - extract just the IP address
if command -v csf &>/dev/null; then
csf -t 2>/dev/null | awk '{print $1}' | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$'
fi
# Get CSF permanent denies
if [ -f /etc/csf/csf.deny ]; then
awk '{print $1}' /etc/csf/csf.deny 2>/dev/null | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$'
fi
# Get iptables DROP rules
if command -v iptables &>/dev/null; then
iptables -L INPUT -n -v 2>/dev/null | grep DROP | awk '{print $8}' | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$'
fi
} | sort -u > "$TEMP_DIR/blocked_ips_cache" 2>/dev/null
# Log cache initialization for debugging
if [ -f "$TEMP_DIR/blocked_ips_cache" ]; then
CACHED_COUNT=$(wc -l < "$TEMP_DIR/blocked_ips_cache" 2>/dev/null || echo "0")
echo "Initialized blocked IPs cache with $CACHED_COUNT IPs" >> "$TEMP_DIR/debug.log"
fi
# Cleanup function
cleanup() {
echo ""
echo "Stopping monitoring processes..."
# Save snapshot before exit
save_snapshot
# Kill all child processes
pkill -P $$ 2>/dev/null
@@ -104,13 +116,6 @@ VELOCITY_WINDOW=3600 # 1 hour in seconds
DECAY_CHECK_INTERVAL=1800 # Check for decay every 30 minutes
LAST_DECAY_CHECK=$START_TIME
# Load persistent data from previous sessions if exists
if [ -f "$SNAPSHOT_DIR/ip_data_snapshot" ]; then
while IFS='=' read -r ip data; do
[ -n "$ip" ] && IP_DATA[$ip]="$data"
done < "$SNAPSHOT_DIR/ip_data_snapshot"
fi
# Hide cursor for cleaner display
command -v tput &>/dev/null && tput civis
@@ -691,20 +696,171 @@ calculate_context_bonus() {
echo "${bonus}|${reasons}"
}
# Check if IP is currently blocked in CSF/iptables
# Block IP temporarily with CSF
block_ip_temporary() {
local ip="$1"
local hours="${2:-1}"
local reason="${3:-Auto-block by live monitor}"
local seconds=$((hours * 3600))
if command -v csf &>/dev/null; then
echo "Blocking $ip for ${hours}h: $reason"
csf -td "$ip" "$seconds" "$reason" >/dev/null 2>&1
local result=$?
# Verify the block was successful (check twice to be sure)
sleep 0.5 # Give CSF a moment to apply the rule
if verify_ip_blocked "$ip"; then
# Double-check to ensure it's really blocked
sleep 0.3
if verify_ip_blocked "$ip"; then
echo "✓ Verified: $ip is now blocked"
# Increment blocks counter
local current_total=$(cat "$TEMP_DIR/total_blocks" 2>/dev/null || echo "0")
echo $((current_total + 1)) > "$TEMP_DIR/total_blocks"
# Trigger immediate cache refresh (don't wait for 10 second interval)
echo "Refreshing cache after blocking $ip..." >> "$TEMP_DIR/debug.log"
{
if command -v csf &>/dev/null; then
csf -t 2>/dev/null | awk '{print $1}' | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$'
fi
if [ -f /etc/csf/csf.deny ]; then
awk '{print $1}' /etc/csf/csf.deny 2>/dev/null | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$'
fi
if command -v iptables &>/dev/null; then
iptables -L INPUT -n -v 2>/dev/null | grep DROP | awk '{print $8}' | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$'
fi
} | sort -u > "$TEMP_DIR/blocked_ips_cache.tmp" 2>/dev/null
mv "$TEMP_DIR/blocked_ips_cache.tmp" "$TEMP_DIR/blocked_ips_cache" 2>/dev/null
CACHE_COUNT=$(wc -l < "$TEMP_DIR/blocked_ips_cache" 2>/dev/null || echo 0)
echo "Cache refreshed: $CACHE_COUNT IPs total" >> "$TEMP_DIR/debug.log"
if grep -q "^$ip$" "$TEMP_DIR/blocked_ips_cache" 2>/dev/null; then
echo "$ip confirmed in cache" >> "$TEMP_DIR/debug.log"
else
echo "✗ WARNING: $ip NOT in cache after refresh!" >> "$TEMP_DIR/debug.log"
# Add it manually as fallback with file locking to prevent race conditions
(
flock -x 200
echo "$ip" >> "$TEMP_DIR/blocked_ips_cache"
sort -u "$TEMP_DIR/blocked_ips_cache" -o "$TEMP_DIR/blocked_ips_cache"
) 200>"$TEMP_DIR/cache.lock"
echo "$ip added manually to cache" >> "$TEMP_DIR/debug.log"
fi
return 0
fi
fi
echo "✗ Warning: Failed to verify block for $ip"
return 1
fi
echo "✗ Error: CSF not available"
return 1
}
# Block IP permanently with CSF
block_ip_permanent() {
local ip="$1"
local reason="${2:-Permanent block by live monitor}"
if command -v csf &>/dev/null; then
echo "Permanently blocking $ip: $reason"
csf -d "$ip" "$reason" >/dev/null 2>&1
local result=$?
# Verify the block was successful (check twice to be sure)
sleep 0.5 # Give CSF a moment to apply the rule
if verify_ip_blocked "$ip"; then
# Double-check to ensure it's really blocked
sleep 0.3
if verify_ip_blocked "$ip"; then
echo "✓ Verified: $ip is now permanently blocked"
# Increment blocks counter
local current_total=$(cat "$TEMP_DIR/total_blocks" 2>/dev/null || echo "0")
echo $((current_total + 1)) > "$TEMP_DIR/total_blocks"
# Trigger immediate cache refresh (don't wait for 10 second interval)
echo "Refreshing cache after permanently blocking $ip..." >> "$TEMP_DIR/debug.log"
{
if command -v csf &>/dev/null; then
csf -t 2>/dev/null | awk '{print $1}' | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$'
fi
if [ -f /etc/csf/csf.deny ]; then
awk '{print $1}' /etc/csf/csf.deny 2>/dev/null | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$'
fi
if command -v iptables &>/dev/null; then
iptables -L INPUT -n -v 2>/dev/null | grep DROP | awk '{print $8}' | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$'
fi
} | sort -u > "$TEMP_DIR/blocked_ips_cache.tmp" 2>/dev/null
mv "$TEMP_DIR/blocked_ips_cache.tmp" "$TEMP_DIR/blocked_ips_cache" 2>/dev/null
CACHE_COUNT=$(wc -l < "$TEMP_DIR/blocked_ips_cache" 2>/dev/null || echo 0)
echo "Cache refreshed: $CACHE_COUNT IPs total" >> "$TEMP_DIR/debug.log"
if grep -q "^$ip$" "$TEMP_DIR/blocked_ips_cache" 2>/dev/null; then
echo "$ip confirmed in cache" >> "$TEMP_DIR/debug.log"
else
echo "✗ WARNING: $ip NOT in cache after refresh!" >> "$TEMP_DIR/debug.log"
# Add it manually as fallback with file locking to prevent race conditions
(
flock -x 200
echo "$ip" >> "$TEMP_DIR/blocked_ips_cache"
sort -u "$TEMP_DIR/blocked_ips_cache" -o "$TEMP_DIR/blocked_ips_cache"
) 200>"$TEMP_DIR/cache.lock"
echo "$ip added manually to cache" >> "$TEMP_DIR/debug.log"
fi
return 0
fi
fi
echo "✗ Warning: Failed to verify permanent block for $ip"
return 1
fi
echo "✗ Error: CSF not available"
return 1
}
# Check if IP is currently blocked in CSF/iptables (optimized with caching)
is_ip_blocked() {
local ip="$1"
# Check CSF deny list
if command -v csf &>/dev/null; then
if csf -g "$ip" 2>/dev/null | grep -q "DENY"; then
# Use cached blocked IPs list (refreshed every 10 seconds by background process)
if [ -f "$TEMP_DIR/blocked_ips_cache" ]; then
if grep -q "^$ip$" "$TEMP_DIR/blocked_ips_cache" 2>/dev/null; then
return 0
fi
fi
return 1
}
# Real-time verification (no cache) for immediate confirmation after blocking
verify_ip_blocked() {
local ip="$1"
# Check CSF temporary blocks
if command -v csf &>/dev/null; then
if csf -t 2>/dev/null | grep -q "$ip"; then
return 0
fi
# Check CSF permanent deny list
if [ -f /etc/csf/csf.deny ]; then
if grep -q "^$ip" /etc/csf/csf.deny 2>/dev/null; then
return 0
fi
fi
fi
# Check iptables directly
if command -v iptables &>/dev/null; then
if iptables -L -n 2>/dev/null | grep -q "$ip"; then
if iptables -L INPUT -n 2>/dev/null | grep -q "$ip"; then
return 0
fi
fi
@@ -714,7 +870,7 @@ is_ip_blocked() {
# Get threat level from score
get_threat_level() {
local score="$1"
local score="${1:-0}"
if [ "$score" -ge "$THREAT_THRESHOLD_CRITICAL" ]; then
echo "CRITICAL"
@@ -776,55 +932,91 @@ draw_header() {
draw_intelligence_panel() {
echo -e "${HIGH_COLOR}┌─ THREAT INTELLIGENCE ──────────────────────────────────────────────────────┐${NC}"
# Get top IPs by threat score
local count=0
# Debug: Show cache status
if [ -f "$TEMP_DIR/blocked_ips_cache" ]; then
CACHED_IPS=$(wc -l < "$TEMP_DIR/blocked_ips_cache" 2>/dev/null || echo 0)
echo -e "${INFO_COLOR} Cache: $CACHED_IPS blocked IPs${NC}" >> "$TEMP_DIR/debug.log"
else
echo -e "${INFO_COLOR} Cache: NOT FOUND${NC}" >> "$TEMP_DIR/debug.log"
fi
# Get top IPs by threat score (exclude already blocked IPs)
local ip_list=""
local blocked_count=0
local displayed_count=0
for ip in "${!IP_DATA[@]}"; do
# Skip IPs that are already blocked
if is_ip_blocked "$ip" 2>/dev/null; then
((blocked_count++))
echo " Filtering out blocked IP: $ip" >> "$TEMP_DIR/debug.log"
continue
fi
((displayed_count++))
IFS='|' read -r score hits bot_type attacks ban_count rep_score <<< "${IP_DATA[$ip]}"
echo "$score|$ip|$hits|$bot_type|$attacks|$ban_count|$rep_score"
done | sort -t'|' -k1 -rn | head -10 | while IFS='|' read -r score ip hits bot_type attacks ban_count rep_score; do
local level=$(get_threat_level "$score")
local color=$(get_threat_color "$level")
local bot_color=$(get_bot_color "$bot_type")
# Build status line
local status_line=$(printf "%-15s" "$ip")
status_line+=$(printf " Score:%-3s" "$score")
status_line+=$(printf " Hits:%-4s" "$hits")
# Bot type indicator
case "$bot_type" in
legit) status_line+=" ✅BOT" ;;
ai) status_line+=" 🤖AI" ;;
monitor) status_line+=" 📊MON" ;;
suspicious) status_line+=" ⚠️ SUS" ;;
*) status_line+="" ;;
esac
# Threat level
status_line+=$(printf " [%-8s]" "$level")
# Attacks
if [ -n "$attacks" ]; then
# Show first attack type
local first_attack=$(echo "$attacks" | cut -d',' -f1)
local icon=$(get_attack_icon "$first_attack")
status_line+=" $icon$(echo "$attacks" | cut -d',' -f1)"
fi
# Ban count
if [ "$ban_count" -gt 0 ]; then
status_line+=" 🚫x$ban_count"
fi
# Known threat indicator
if [ "$rep_score" -gt 0 ]; then
status_line+=" [KNOWN]"
fi
echo -e "${color}${status_line}${NC}"
ip_list+="$score|$ip|$hits|$bot_type|$attacks|$ban_count|$rep_score"$'\n'
done
echo " Blocked/filtered: $blocked_count, Displaying: $displayed_count" >> "$TEMP_DIR/debug.log"
if [ -n "$ip_list" ]; then
echo "$ip_list" | sort -t'|' -k1 -rn | head -10 | while IFS='|' read -r score ip hits bot_type attacks ban_count rep_score; do
# Set defaults for empty values
score="${score:-0}"
hits="${hits:-0}"
ban_count="${ban_count:-0}"
rep_score="${rep_score:-0}"
local level=$(get_threat_level "$score")
local color=$(get_threat_color "$level")
local bot_color=$(get_bot_color "$bot_type")
# Build status line
local status_line=$(printf "%-15s" "$ip")
status_line+=$(printf " Score:%-3s" "$score")
status_line+=$(printf " Hits:%-4s" "$hits")
# Bot type indicator
case "$bot_type" in
legit) status_line+=" ✅BOT" ;;
ai) status_line+=" 🤖AI" ;;
monitor) status_line+=" 📊MON" ;;
suspicious) status_line+=" ⚠️ SUS" ;;
*) status_line+="" ;;
esac
# Threat level
status_line+=$(printf " [%-8s]" "$level")
# Attacks
if [ -n "$attacks" ]; then
# Show first attack type
local first_attack=$(echo "$attacks" | cut -d',' -f1)
local icon=$(get_attack_icon "$first_attack")
status_line+=" $icon$(echo "$attacks" | cut -d',' -f1)"
fi
# Ban count
if [ "$ban_count" -gt 0 ]; then
status_line+=" 🚫x$ban_count"
fi
# Known threat indicator
if [ "$rep_score" -gt 0 ]; then
status_line+=" [KNOWN]"
fi
echo -e "${color}${status_line}${NC}"
done
else
# Show appropriate message
if [ ${#IP_DATA[@]} -gt 0 ]; then
echo -e "${SAFE_COLOR} ✓ All detected threats have been blocked${NC}"
else
echo -e "${LOW_COLOR} No threats detected yet...${NC}"
fi
fi
echo -e "${HIGH_COLOR}└────────────────────────────────────────────────────────────────────────────┘${NC}"
echo ""
}
@@ -959,6 +1151,11 @@ show_blocking_menu() {
for ip in "${!IP_DATA[@]}"; do
IFS='|' read -r score hits bot_type attacks ban_count rep_score <<< "${IP_DATA[$ip]}"
# Set defaults for empty values
score="${score:-0}"
hits="${hits:-0}"
attacks="${attacks:-none}"
# Skip if score too low or already blocked
[ "$score" -lt 60 ] && continue
is_ip_blocked "$ip" 2>/dev/null && continue
@@ -1012,17 +1209,24 @@ show_blocking_menu() {
elif [ "$choice" = "a" ]; then
# Block all IPs with score >= 80
local blocked=0
local failed=0
for entry in "${blockable_list[@]}"; do
IFS='|' read -r ip score hits attacks <<< "$entry"
[ "$score" -lt 80 ] && continue
echo ""
block_ip_temporary "$ip" 1 "Auto-block: High threat (score $score)"
((blocked++))
if block_ip_temporary "$ip" 1 "Auto-block: High threat (score $score)"; then
((blocked++))
else
((failed++))
fi
done
echo ""
echo "Blocked $blocked IPs"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "✓ Successfully blocked: $blocked IPs"
[ $failed -gt 0 ] && echo "✗ Failed to block: $failed IPs"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
read -p "Press Enter to continue..."
elif [[ "$choice" =~ ^[0-9]+$ ]] && [ "$choice" -ge 1 ] && [ "$choice" -le ${#blockable_list[@]} ]; then
# Block specific IP
@@ -1076,7 +1280,7 @@ monitor_apache_logs() {
# Monitor all log files
local event_count=0
tail -f "${log_files[@]}" 2>/dev/null | while read -r line; do
tail -n 0 -F "${log_files[@]}" 2>/dev/null | while read -r line; do
# Increment event counter (update file every 10 events for performance)
((event_count++))
if [ $((event_count % 10)) -eq 0 ]; then
@@ -1982,6 +2186,30 @@ auto_mitigation_engine
done
) &
# Blocked IPs cache updater (runs every 10 seconds for performance)
(
while true; do
{
# Get CSF temporary blocks - extract just the IP address
if command -v csf &>/dev/null; then
csf -t 2>/dev/null | awk '{print $1}' | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$'
fi
# Get CSF permanent denies
if [ -f /etc/csf/csf.deny ]; then
awk '{print $1}' /etc/csf/csf.deny 2>/dev/null | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$'
fi
# Get iptables DROP rules
if command -v iptables &>/dev/null; then
iptables -L INPUT -n -v 2>/dev/null | grep DROP | awk '{print $8}' | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$'
fi
} | sort -u > "$TEMP_DIR/blocked_ips_cache.tmp" 2>/dev/null
mv "$TEMP_DIR/blocked_ips_cache.tmp" "$TEMP_DIR/blocked_ips_cache" 2>/dev/null
sleep 10
done
) &
# Periodic snapshot saving in background
(
while true; do
@@ -1993,26 +2221,44 @@ auto_mitigation_engine
# Main dashboard loop
LOOP_COUNT=0
while true; do
# Sync individual IP files into IP_DATA array (for data from subshell processes like SSH monitoring)
for ip_file in "$TEMP_DIR"/ip_*; do
[ -f "$ip_file" ] || continue
basename_file="$(basename "$ip_file")"
# Skip non-IP files explicitly
case "$basename_file" in
ip_data|ip_database.db|*cache*|*blocked*|*debug*)
continue
;;
esac
# Validate it's an IP file (should match pattern ip_N_N_N_N)
if ! echo "$basename_file" | grep -qE '^ip_[0-9]{1,3}_[0-9]{1,3}_[0-9]{1,3}_[0-9]{1,3}$'; then
continue
fi
# Extract IP from filename (ip_1_2_3_4 -> 1.2.3.4)
ip=$(echo "$basename_file" | sed 's/^ip_//' | tr '_' '.')
data=$(cat "$ip_file" 2>/dev/null)
# Validate data format (should be score|hits|bot_type|attacks|ban_count|rep_score)
if [ -n "$data" ] && echo "$data" | grep -q '|'; then
# Update IP_DATA array with data from file
IP_DATA[$ip]="$data"
fi
done
draw_header
draw_intelligence_panel
draw_attack_breakdown
draw_live_feed
draw_quick_actions
# Consolidate IP data from individual files into ip_data file (for auto-mitigation engine)
# Write IP_DATA to ip_data file for auto-mitigation engine
{
for ip_file in "$TEMP_DIR"/ip_*; do
[ -f "$ip_file" ] || continue
# Skip the consolidated ip_data file itself
[[ "$(basename "$ip_file")" == "ip_data" ]] && continue
# Extract IP from filename (ip_1_2_3_4 -> 1.2.3.4)
ip=$(basename "$ip_file" | sed 's/^ip_//' | tr '_' '.')
data=$(cat "$ip_file" 2>/dev/null)
if [ -n "$data" ]; then
echo "$ip=$data"
# Also update IP_DATA array for dashboard display
IP_DATA[$ip]="$data"
fi
for ip in "${!IP_DATA[@]}"; do
echo "$ip=${IP_DATA[$ip]}"
done
} > "$TEMP_DIR/ip_data" 2>/dev/null