Fix live-attack-monitor auto-blocking and bot-analyzer compression
- live-attack-monitor.sh: * Remove snapshot loading (start fresh each session) * Fix Apache log monitoring to use tail -n 0 -F (only new entries) * Add IP file sync to main loop for auto-blocking to work * Fix IP_DATA consolidation for cross-process communication - bot-analyzer.sh: * Implement gzip compression for large temp files (10-20x space savings) * Update all read/write operations to use compressed files * Fix for servers with 200+ domains and millions of log entries - run.sh: * Add HISTFILE fallback to prevent crashes when sourced
This commit is contained in:
cschantz
@@ -348,20 +348,22 @@ parse_logs() {
|
||||
if (ip != "" && ip !~ /^[[:space:]]*$/) {
|
||||
print ip "|" domain "|" request_url "|" status "|" size "|" user_agent "|" http_method "|" timestamp
|
||||
}
|
||||
}' "$logfile" >> "$TEMP_DIR/parsed_logs.txt" 2>/dev/null
|
||||
done
|
||||
}' "$logfile" 2>/dev/null
|
||||
done | gzip > "$TEMP_DIR/parsed_logs.txt.gz"
|
||||
|
||||
# Clear the progress line
|
||||
echo -ne "\r\033[K"
|
||||
|
||||
if [ ! -s "$TEMP_DIR/parsed_logs.txt" ]; then
|
||||
if [ ! -s "$TEMP_DIR/parsed_logs.txt.gz" ]; then
|
||||
print_alert "No log entries were parsed. Check log format or permissions."
|
||||
return 1
|
||||
fi
|
||||
|
||||
local line_count
|
||||
line_count=$(wc -l < "$TEMP_DIR/parsed_logs.txt")
|
||||
print_success "Logs parsed successfully ($line_count entries)"
|
||||
line_count=$(zcat "$TEMP_DIR/parsed_logs.txt.gz" | wc -l)
|
||||
local file_size_kb
|
||||
file_size_kb=$(du -k "$TEMP_DIR/parsed_logs.txt.gz" | cut -f1)
|
||||
print_success "Logs parsed successfully ($line_count entries, ${file_size_kb}KB compressed)"
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -460,16 +462,18 @@ classify_bots() {
|
||||
if (bot_type != "unknown") {
|
||||
print ip "|" domain "|" url "|" status "|" size "|" ua "|" method "|" timestamp "|" bot_type "|" bot_name
|
||||
}
|
||||
}' "$TEMP_DIR/parsed_logs.txt" > "$TEMP_DIR/classified_bots.txt"
|
||||
}' < <(zcat "$TEMP_DIR/parsed_logs.txt.gz") | gzip > "$TEMP_DIR/classified_bots.txt.gz"
|
||||
|
||||
if [ ! -s "$TEMP_DIR/classified_bots.txt" ]; then
|
||||
if [ ! -s "$TEMP_DIR/classified_bots.txt.gz" ]; then
|
||||
print_alert "Bot classification failed"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local classified_count
|
||||
classified_count=$(wc -l < "$TEMP_DIR/classified_bots.txt")
|
||||
print_success "Bot classification complete ($classified_count entries)"
|
||||
classified_count=$(zcat "$TEMP_DIR/classified_bots.txt.gz" | wc -l)
|
||||
local file_size_kb
|
||||
file_size_kb=$(du -k "$TEMP_DIR/classified_bots.txt.gz" | cut -f1)
|
||||
print_success "Bot classification complete ($classified_count entries, ${file_size_kb}KB compressed)"
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -556,7 +560,7 @@ detect_threats() {
|
||||
# Track response codes for intelligence
|
||||
print status > "'"$TEMP_DIR"'/response_codes_raw.txt"
|
||||
}
|
||||
' "$TEMP_DIR/parsed_logs.txt"
|
||||
' < <(zcat "$TEMP_DIR/parsed_logs.txt.gz")
|
||||
|
||||
# Process attack vectors by type
|
||||
if [ -f "$TEMP_DIR/attack_vectors_raw.txt" ]; then
|
||||
@@ -622,23 +626,23 @@ detect_botnets() {
|
||||
|
||||
# Group IPs by similar behavior patterns
|
||||
# Pattern 1: Multiple IPs hitting same URLs in coordinated manner
|
||||
awk -F'|' '{print $1"|"$3}' "$TEMP_DIR/parsed_logs.txt" | \
|
||||
zcat "$TEMP_DIR/parsed_logs.txt.gz" | awk -F'|' '{print $1"|"$3}' | \
|
||||
sort | uniq -c | awk '$1 > 10 {print $2}' | \
|
||||
cut -d'|' -f2 | sort | uniq -c | sort -rn | \
|
||||
awk '$1 > 5 {print $2}' > "$TEMP_DIR/coordinated_urls.txt"
|
||||
|
||||
# Pattern 2: IPs with similar User-Agents hitting multiple domains
|
||||
awk -F'|' '{print $1"|"$6}' "$TEMP_DIR/parsed_logs.txt" | \
|
||||
zcat "$TEMP_DIR/parsed_logs.txt.gz" | awk -F'|' '{print $1"|"$6}' | \
|
||||
sort | uniq > "$TEMP_DIR/ip_ua_pairs.txt"
|
||||
|
||||
# Pattern 3: Detect IP ranges (Class C networks) with suspicious activity
|
||||
awk -F'|' '{print $1}' "$TEMP_DIR/parsed_logs.txt" | \
|
||||
zcat "$TEMP_DIR/parsed_logs.txt.gz" | awk -F'|' '{print $1}' | \
|
||||
awk -F'.' '{print $1"."$2"."$3".0/24"}' | \
|
||||
sort | uniq -c | sort -rn | awk '$1 > 20' > "$TEMP_DIR/suspicious_networks.txt"
|
||||
|
||||
# Pattern 4: Rapid fire requests (DDoS indicators)
|
||||
# Extract timestamp and count requests per IP per minute
|
||||
awk -F'|' '{
|
||||
zcat "$TEMP_DIR/parsed_logs.txt.gz" | awk -F'|' '{
|
||||
ip = $1
|
||||
timestamp = $8
|
||||
# Extract date/time components (handles format: DD/MMM/YYYY:HH:MM:SS)
|
||||
@@ -647,7 +651,7 @@ detect_botnets() {
|
||||
time_key = ts[3] ts[2] ts[1] "_" ts[4] ts[5]
|
||||
print ip "|" time_key
|
||||
}
|
||||
}' "$TEMP_DIR/parsed_logs.txt" | \
|
||||
}' | \
|
||||
sort | uniq -c | \
|
||||
awk '$1 > 50 {print $1 " " $2}' | \
|
||||
awk -F'|' '{print $1}' | \
|
||||
@@ -742,13 +746,13 @@ analyze_time_series() {
|
||||
print_info "Analyzing time-series patterns..."
|
||||
|
||||
# Extract hourly bot traffic
|
||||
awk -F'|' '$9 != "unknown" {
|
||||
zcat "$TEMP_DIR/classified_bots.txt.gz" | awk -F'|' '$9 != "unknown" {
|
||||
timestamp = $8
|
||||
if (match(timestamp, /([0-9]{2})\/([A-Za-z]{3})\/([0-9]{4}):([0-9]{2}):([0-9]{2}):([0-9]{2})/, ts)) {
|
||||
hour = ts[4]
|
||||
print hour
|
||||
}
|
||||
}' "$TEMP_DIR/classified_bots.txt" | sort | uniq -c > "$TEMP_DIR/hourly_bot_traffic.txt"
|
||||
}' | sort | uniq -c > "$TEMP_DIR/hourly_bot_traffic.txt"
|
||||
|
||||
# Extract hourly attack traffic
|
||||
if [ -f "$TEMP_DIR/attack_vectors_raw.txt" ]; then
|
||||
@@ -759,7 +763,7 @@ analyze_time_series() {
|
||||
hour = ts[4]
|
||||
print hour
|
||||
}
|
||||
}' "$TEMP_DIR/attack_vectors_raw.txt" "$TEMP_DIR/parsed_logs.txt" | sort | uniq -c > "$TEMP_DIR/hourly_attack_traffic.txt"
|
||||
}' "$TEMP_DIR/attack_vectors_raw.txt" <(zcat "$TEMP_DIR/parsed_logs.txt.gz") | sort | uniq -c > "$TEMP_DIR/hourly_attack_traffic.txt"
|
||||
fi
|
||||
|
||||
print_success "Time-series analysis complete"
|
||||
@@ -776,7 +780,7 @@ calculate_threat_scores() {
|
||||
declare -A ip_request_counts
|
||||
while IFS='|' read -r ip rest; do
|
||||
((ip_request_counts["$ip"]++))
|
||||
done < "$TEMP_DIR/parsed_logs.txt"
|
||||
done < <(zcat "$TEMP_DIR/parsed_logs.txt.gz")
|
||||
|
||||
# Build hash tables from threat files for O(1) lookups
|
||||
declare -A threat_ips_sqli threat_ips_xss threat_ips_path threat_ips_rce threat_ips_login
|
||||
@@ -922,7 +926,7 @@ detect_false_positives() {
|
||||
print_info "Detecting legitimate services (false positives)..."
|
||||
|
||||
# Known monitoring service patterns
|
||||
awk -F'|' '{
|
||||
zcat "$TEMP_DIR/parsed_logs.txt.gz" | awk -F'|' '{
|
||||
ip = $1
|
||||
domain = $2
|
||||
url = $3
|
||||
@@ -948,7 +952,7 @@ detect_false_positives() {
|
||||
else if (match(ua, /jetpack|vaultpress|updraftplus/)) {
|
||||
print ip "|Backup Service|" ua "|" domain
|
||||
}
|
||||
}' "$TEMP_DIR/parsed_logs.txt" | sort -u > "$TEMP_DIR/false_positives.txt"
|
||||
}' | sort -u > "$TEMP_DIR/false_positives.txt"
|
||||
|
||||
print_success "False positive detection complete"
|
||||
}
|
||||
@@ -961,31 +965,31 @@ generate_statistics() {
|
||||
print_info "Generating statistics..."
|
||||
|
||||
# Top 5 bots by request count
|
||||
awk -F'|' '$9 != "unknown" {print $10}' "$TEMP_DIR/classified_bots.txt" | \
|
||||
zcat "$TEMP_DIR/classified_bots.txt.gz" | awk -F'|' '$9 != "unknown" {print $10}' | \
|
||||
sort | uniq -c | sort -rn | head -5 > "$TEMP_DIR/top_bots.txt"
|
||||
|
||||
# Top 5 most-hit sites
|
||||
awk -F'|' '{print $2}' "$TEMP_DIR/parsed_logs.txt" | \
|
||||
zcat "$TEMP_DIR/parsed_logs.txt.gz" | awk -F'|' '{print $2}' | \
|
||||
sort | uniq -c | sort -rn | head -5 > "$TEMP_DIR/top_sites.txt"
|
||||
|
||||
# Top 5 most-hit URLs
|
||||
awk -F'|' '{print $2"|"$3}' "$TEMP_DIR/parsed_logs.txt" | \
|
||||
zcat "$TEMP_DIR/parsed_logs.txt.gz" | awk -F'|' '{print $2"|"$3}' | \
|
||||
sort | uniq -c | sort -rn | head -5 > "$TEMP_DIR/top_urls.txt"
|
||||
|
||||
# Top 5 IP addresses by request count
|
||||
awk -F'|' '{print $1}' "$TEMP_DIR/parsed_logs.txt" | \
|
||||
zcat "$TEMP_DIR/parsed_logs.txt.gz" | awk -F'|' '{print $1}' | \
|
||||
sort | uniq -c | sort -rn | head -5 > "$TEMP_DIR/top_ips.txt"
|
||||
|
||||
# Traffic breakdown by bot type
|
||||
awk -F'|' '{print $9}' "$TEMP_DIR/classified_bots.txt" | \
|
||||
zcat "$TEMP_DIR/classified_bots.txt.gz" | awk -F'|' '{print $9}' | \
|
||||
sort | uniq -c | sort -rn > "$TEMP_DIR/traffic_breakdown.txt"
|
||||
|
||||
# Per-domain traffic sources
|
||||
while read -r domain; do
|
||||
echo "$domain" > "$TEMP_DIR/domain_${domain}_stats.txt"
|
||||
grep "|$domain|" "$TEMP_DIR/classified_bots.txt" | \
|
||||
zcat "$TEMP_DIR/classified_bots.txt.gz" | grep "|$domain|" | \
|
||||
awk -F'|' '{print $9}' | sort | uniq -c | sort -rn >> "$TEMP_DIR/domain_${domain}_stats.txt"
|
||||
done < <(awk -F'|' '{print $2}' "$TEMP_DIR/parsed_logs.txt" | sort -u)
|
||||
done < <(zcat "$TEMP_DIR/parsed_logs.txt.gz" | awk -F'|' '{print $2}' | sort -u)
|
||||
|
||||
print_success "Statistics generated"
|
||||
}
|
||||
@@ -1069,19 +1073,19 @@ generate_report() {
|
||||
# QUICK STATS DASHBOARD
|
||||
print_header "QUICK STATS DASHBOARD"
|
||||
|
||||
total_requests=$(wc -l < "$TEMP_DIR/parsed_logs.txt")
|
||||
unique_ips=$(awk -F'|' '{print $1}' "$TEMP_DIR/parsed_logs.txt" | sort -u | wc -l)
|
||||
unique_domains=$(awk -F'|' '{print $2}' "$TEMP_DIR/parsed_logs.txt" | sort -u | wc -l)
|
||||
bot_requests=$(awk -F'|' '$9 != "unknown"' "$TEMP_DIR/classified_bots.txt" | wc -l)
|
||||
total_requests=$(zcat "$TEMP_DIR/parsed_logs.txt.gz" | wc -l)
|
||||
unique_ips=$(zcat "$TEMP_DIR/parsed_logs.txt.gz" | awk -F'|' '{print $1}' | sort -u | wc -l)
|
||||
unique_domains=$(zcat "$TEMP_DIR/parsed_logs.txt.gz" | awk -F'|' '{print $2}' | sort -u | wc -l)
|
||||
bot_requests=$(zcat "$TEMP_DIR/classified_bots.txt.gz" | awk -F'|' '$9 != "unknown"' | wc -l)
|
||||
|
||||
# Count private/internal IPs (excluded from threat analysis)
|
||||
private_ips=$(awk -F'|' '{print $1}' "$TEMP_DIR/parsed_logs.txt" | sort -u | grep -E '^(127\.|10\.|192\.168\.|172\.(1[6-9]|2[0-9]|3[01])\.|169\.254\.)' | wc -l)
|
||||
private_ips=$(zcat "$TEMP_DIR/parsed_logs.txt.gz" | awk -F'|' '{print $1}' | sort -u | grep -E '^(127\.|10\.|192\.168\.|172\.(1[6-9]|2[0-9]|3[01])\.|169\.254\.)' | wc -l)
|
||||
|
||||
# Count server's own IPs in the logs
|
||||
server_ip_hits=0
|
||||
if [ -f "$TEMP_DIR/server_ips.txt" ] && [ -s "$TEMP_DIR/server_ips.txt" ]; then
|
||||
while read -r server_ip; do
|
||||
if grep -q "^$server_ip|" "$TEMP_DIR/parsed_logs.txt" 2>/dev/null; then
|
||||
if zcat "$TEMP_DIR/parsed_logs.txt.gz" | grep -q "^$server_ip|" 2>/dev/null; then
|
||||
server_ip_hits=$((server_ip_hits + 1))
|
||||
fi
|
||||
done < "$TEMP_DIR/server_ips.txt"
|
||||
|
||||
@@ -55,23 +55,35 @@ touch "$TEMP_DIR/ip_data"
|
||||
echo "0" > "$TEMP_DIR/event_counter"
|
||||
echo "0" > "$TEMP_DIR/total_blocks"
|
||||
|
||||
# Save snapshot of IP data (for persistence across restarts)
|
||||
save_snapshot() {
|
||||
{
|
||||
for ip in "${!IP_DATA[@]}"; do
|
||||
echo "$ip=${IP_DATA[$ip]}"
|
||||
done
|
||||
} > "$SNAPSHOT_DIR/ip_data_snapshot" 2>/dev/null
|
||||
}
|
||||
# Initialize blocked IPs cache immediately on startup
|
||||
{
|
||||
# Get CSF temporary blocks - extract just the IP address
|
||||
if command -v csf &>/dev/null; then
|
||||
csf -t 2>/dev/null | awk '{print $1}' | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$'
|
||||
fi
|
||||
|
||||
# Get CSF permanent denies
|
||||
if [ -f /etc/csf/csf.deny ]; then
|
||||
awk '{print $1}' /etc/csf/csf.deny 2>/dev/null | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$'
|
||||
fi
|
||||
|
||||
# Get iptables DROP rules
|
||||
if command -v iptables &>/dev/null; then
|
||||
iptables -L INPUT -n -v 2>/dev/null | grep DROP | awk '{print $8}' | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$'
|
||||
fi
|
||||
} | sort -u > "$TEMP_DIR/blocked_ips_cache" 2>/dev/null
|
||||
|
||||
# Log cache initialization for debugging
|
||||
if [ -f "$TEMP_DIR/blocked_ips_cache" ]; then
|
||||
CACHED_COUNT=$(wc -l < "$TEMP_DIR/blocked_ips_cache" 2>/dev/null || echo "0")
|
||||
echo "Initialized blocked IPs cache with $CACHED_COUNT IPs" >> "$TEMP_DIR/debug.log"
|
||||
fi
|
||||
|
||||
# Cleanup function
|
||||
cleanup() {
|
||||
echo ""
|
||||
echo "Stopping monitoring processes..."
|
||||
|
||||
# Save snapshot before exit
|
||||
save_snapshot
|
||||
|
||||
# Kill all child processes
|
||||
pkill -P $$ 2>/dev/null
|
||||
|
||||
@@ -104,13 +116,6 @@ VELOCITY_WINDOW=3600 # 1 hour in seconds
|
||||
DECAY_CHECK_INTERVAL=1800 # Check for decay every 30 minutes
|
||||
LAST_DECAY_CHECK=$START_TIME
|
||||
|
||||
# Load persistent data from previous sessions if exists
|
||||
if [ -f "$SNAPSHOT_DIR/ip_data_snapshot" ]; then
|
||||
while IFS='=' read -r ip data; do
|
||||
[ -n "$ip" ] && IP_DATA[$ip]="$data"
|
||||
done < "$SNAPSHOT_DIR/ip_data_snapshot"
|
||||
fi
|
||||
|
||||
# Hide cursor for cleaner display
|
||||
command -v tput &>/dev/null && tput civis
|
||||
|
||||
@@ -691,20 +696,171 @@ calculate_context_bonus() {
|
||||
echo "${bonus}|${reasons}"
|
||||
}
|
||||
|
||||
# Check if IP is currently blocked in CSF/iptables
|
||||
# Block IP temporarily with CSF
|
||||
block_ip_temporary() {
|
||||
local ip="$1"
|
||||
local hours="${2:-1}"
|
||||
local reason="${3:-Auto-block by live monitor}"
|
||||
local seconds=$((hours * 3600))
|
||||
|
||||
if command -v csf &>/dev/null; then
|
||||
echo "Blocking $ip for ${hours}h: $reason"
|
||||
csf -td "$ip" "$seconds" "$reason" >/dev/null 2>&1
|
||||
local result=$?
|
||||
|
||||
# Verify the block was successful (check twice to be sure)
|
||||
sleep 0.5 # Give CSF a moment to apply the rule
|
||||
if verify_ip_blocked "$ip"; then
|
||||
# Double-check to ensure it's really blocked
|
||||
sleep 0.3
|
||||
if verify_ip_blocked "$ip"; then
|
||||
echo "✓ Verified: $ip is now blocked"
|
||||
|
||||
# Increment blocks counter
|
||||
local current_total=$(cat "$TEMP_DIR/total_blocks" 2>/dev/null || echo "0")
|
||||
echo $((current_total + 1)) > "$TEMP_DIR/total_blocks"
|
||||
|
||||
# Trigger immediate cache refresh (don't wait for 10 second interval)
|
||||
echo "Refreshing cache after blocking $ip..." >> "$TEMP_DIR/debug.log"
|
||||
{
|
||||
if command -v csf &>/dev/null; then
|
||||
csf -t 2>/dev/null | awk '{print $1}' | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$'
|
||||
fi
|
||||
if [ -f /etc/csf/csf.deny ]; then
|
||||
awk '{print $1}' /etc/csf/csf.deny 2>/dev/null | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$'
|
||||
fi
|
||||
if command -v iptables &>/dev/null; then
|
||||
iptables -L INPUT -n -v 2>/dev/null | grep DROP | awk '{print $8}' | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$'
|
||||
fi
|
||||
} | sort -u > "$TEMP_DIR/blocked_ips_cache.tmp" 2>/dev/null
|
||||
mv "$TEMP_DIR/blocked_ips_cache.tmp" "$TEMP_DIR/blocked_ips_cache" 2>/dev/null
|
||||
|
||||
CACHE_COUNT=$(wc -l < "$TEMP_DIR/blocked_ips_cache" 2>/dev/null || echo 0)
|
||||
echo "Cache refreshed: $CACHE_COUNT IPs total" >> "$TEMP_DIR/debug.log"
|
||||
if grep -q "^$ip$" "$TEMP_DIR/blocked_ips_cache" 2>/dev/null; then
|
||||
echo "✓ $ip confirmed in cache" >> "$TEMP_DIR/debug.log"
|
||||
else
|
||||
echo "✗ WARNING: $ip NOT in cache after refresh!" >> "$TEMP_DIR/debug.log"
|
||||
# Add it manually as fallback with file locking to prevent race conditions
|
||||
(
|
||||
flock -x 200
|
||||
echo "$ip" >> "$TEMP_DIR/blocked_ips_cache"
|
||||
sort -u "$TEMP_DIR/blocked_ips_cache" -o "$TEMP_DIR/blocked_ips_cache"
|
||||
) 200>"$TEMP_DIR/cache.lock"
|
||||
echo "✓ $ip added manually to cache" >> "$TEMP_DIR/debug.log"
|
||||
fi
|
||||
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "✗ Warning: Failed to verify block for $ip"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "✗ Error: CSF not available"
|
||||
return 1
|
||||
}
|
||||
|
||||
# Block IP permanently with CSF
|
||||
block_ip_permanent() {
|
||||
local ip="$1"
|
||||
local reason="${2:-Permanent block by live monitor}"
|
||||
|
||||
if command -v csf &>/dev/null; then
|
||||
echo "Permanently blocking $ip: $reason"
|
||||
csf -d "$ip" "$reason" >/dev/null 2>&1
|
||||
local result=$?
|
||||
|
||||
# Verify the block was successful (check twice to be sure)
|
||||
sleep 0.5 # Give CSF a moment to apply the rule
|
||||
if verify_ip_blocked "$ip"; then
|
||||
# Double-check to ensure it's really blocked
|
||||
sleep 0.3
|
||||
if verify_ip_blocked "$ip"; then
|
||||
echo "✓ Verified: $ip is now permanently blocked"
|
||||
|
||||
# Increment blocks counter
|
||||
local current_total=$(cat "$TEMP_DIR/total_blocks" 2>/dev/null || echo "0")
|
||||
echo $((current_total + 1)) > "$TEMP_DIR/total_blocks"
|
||||
|
||||
# Trigger immediate cache refresh (don't wait for 10 second interval)
|
||||
echo "Refreshing cache after permanently blocking $ip..." >> "$TEMP_DIR/debug.log"
|
||||
{
|
||||
if command -v csf &>/dev/null; then
|
||||
csf -t 2>/dev/null | awk '{print $1}' | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$'
|
||||
fi
|
||||
if [ -f /etc/csf/csf.deny ]; then
|
||||
awk '{print $1}' /etc/csf/csf.deny 2>/dev/null | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$'
|
||||
fi
|
||||
if command -v iptables &>/dev/null; then
|
||||
iptables -L INPUT -n -v 2>/dev/null | grep DROP | awk '{print $8}' | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$'
|
||||
fi
|
||||
} | sort -u > "$TEMP_DIR/blocked_ips_cache.tmp" 2>/dev/null
|
||||
mv "$TEMP_DIR/blocked_ips_cache.tmp" "$TEMP_DIR/blocked_ips_cache" 2>/dev/null
|
||||
|
||||
CACHE_COUNT=$(wc -l < "$TEMP_DIR/blocked_ips_cache" 2>/dev/null || echo 0)
|
||||
echo "Cache refreshed: $CACHE_COUNT IPs total" >> "$TEMP_DIR/debug.log"
|
||||
if grep -q "^$ip$" "$TEMP_DIR/blocked_ips_cache" 2>/dev/null; then
|
||||
echo "✓ $ip confirmed in cache" >> "$TEMP_DIR/debug.log"
|
||||
else
|
||||
echo "✗ WARNING: $ip NOT in cache after refresh!" >> "$TEMP_DIR/debug.log"
|
||||
# Add it manually as fallback with file locking to prevent race conditions
|
||||
(
|
||||
flock -x 200
|
||||
echo "$ip" >> "$TEMP_DIR/blocked_ips_cache"
|
||||
sort -u "$TEMP_DIR/blocked_ips_cache" -o "$TEMP_DIR/blocked_ips_cache"
|
||||
) 200>"$TEMP_DIR/cache.lock"
|
||||
echo "✓ $ip added manually to cache" >> "$TEMP_DIR/debug.log"
|
||||
fi
|
||||
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "✗ Warning: Failed to verify permanent block for $ip"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "✗ Error: CSF not available"
|
||||
return 1
|
||||
}
|
||||
|
||||
# Check if IP is currently blocked in CSF/iptables (optimized with caching)
|
||||
is_ip_blocked() {
|
||||
local ip="$1"
|
||||
|
||||
# Check CSF deny list
|
||||
if command -v csf &>/dev/null; then
|
||||
if csf -g "$ip" 2>/dev/null | grep -q "DENY"; then
|
||||
# Use cached blocked IPs list (refreshed every 10 seconds by background process)
|
||||
if [ -f "$TEMP_DIR/blocked_ips_cache" ]; then
|
||||
if grep -q "^$ip$" "$TEMP_DIR/blocked_ips_cache" 2>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
# Real-time verification (no cache) for immediate confirmation after blocking
|
||||
verify_ip_blocked() {
|
||||
local ip="$1"
|
||||
|
||||
# Check CSF temporary blocks
|
||||
if command -v csf &>/dev/null; then
|
||||
if csf -t 2>/dev/null | grep -q "$ip"; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Check CSF permanent deny list
|
||||
if [ -f /etc/csf/csf.deny ]; then
|
||||
if grep -q "^$ip" /etc/csf/csf.deny 2>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check iptables directly
|
||||
if command -v iptables &>/dev/null; then
|
||||
if iptables -L -n 2>/dev/null | grep -q "$ip"; then
|
||||
if iptables -L INPUT -n 2>/dev/null | grep -q "$ip"; then
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
@@ -714,7 +870,7 @@ is_ip_blocked() {
|
||||
|
||||
# Get threat level from score
|
||||
get_threat_level() {
|
||||
local score="$1"
|
||||
local score="${1:-0}"
|
||||
|
||||
if [ "$score" -ge "$THREAT_THRESHOLD_CRITICAL" ]; then
|
||||
echo "CRITICAL"
|
||||
@@ -776,12 +932,40 @@ draw_header() {
|
||||
draw_intelligence_panel() {
|
||||
echo -e "${HIGH_COLOR}┌─ THREAT INTELLIGENCE ──────────────────────────────────────────────────────┐${NC}"
|
||||
|
||||
# Get top IPs by threat score
|
||||
local count=0
|
||||
# Debug: Show cache status
|
||||
if [ -f "$TEMP_DIR/blocked_ips_cache" ]; then
|
||||
CACHED_IPS=$(wc -l < "$TEMP_DIR/blocked_ips_cache" 2>/dev/null || echo 0)
|
||||
echo -e "${INFO_COLOR} Cache: $CACHED_IPS blocked IPs${NC}" >> "$TEMP_DIR/debug.log"
|
||||
else
|
||||
echo -e "${INFO_COLOR} Cache: NOT FOUND${NC}" >> "$TEMP_DIR/debug.log"
|
||||
fi
|
||||
|
||||
# Get top IPs by threat score (exclude already blocked IPs)
|
||||
local ip_list=""
|
||||
local blocked_count=0
|
||||
local displayed_count=0
|
||||
for ip in "${!IP_DATA[@]}"; do
|
||||
# Skip IPs that are already blocked
|
||||
if is_ip_blocked "$ip" 2>/dev/null; then
|
||||
((blocked_count++))
|
||||
echo " Filtering out blocked IP: $ip" >> "$TEMP_DIR/debug.log"
|
||||
continue
|
||||
fi
|
||||
|
||||
((displayed_count++))
|
||||
IFS='|' read -r score hits bot_type attacks ban_count rep_score <<< "${IP_DATA[$ip]}"
|
||||
echo "$score|$ip|$hits|$bot_type|$attacks|$ban_count|$rep_score"
|
||||
done | sort -t'|' -k1 -rn | head -10 | while IFS='|' read -r score ip hits bot_type attacks ban_count rep_score; do
|
||||
ip_list+="$score|$ip|$hits|$bot_type|$attacks|$ban_count|$rep_score"$'\n'
|
||||
done
|
||||
|
||||
echo " Blocked/filtered: $blocked_count, Displaying: $displayed_count" >> "$TEMP_DIR/debug.log"
|
||||
|
||||
if [ -n "$ip_list" ]; then
|
||||
echo "$ip_list" | sort -t'|' -k1 -rn | head -10 | while IFS='|' read -r score ip hits bot_type attacks ban_count rep_score; do
|
||||
# Set defaults for empty values
|
||||
score="${score:-0}"
|
||||
hits="${hits:-0}"
|
||||
ban_count="${ban_count:-0}"
|
||||
rep_score="${rep_score:-0}"
|
||||
|
||||
local level=$(get_threat_level "$score")
|
||||
local color=$(get_threat_color "$level")
|
||||
@@ -824,6 +1008,14 @@ draw_intelligence_panel() {
|
||||
|
||||
echo -e "${color}${status_line}${NC}"
|
||||
done
|
||||
else
|
||||
# Show appropriate message
|
||||
if [ ${#IP_DATA[@]} -gt 0 ]; then
|
||||
echo -e "${SAFE_COLOR} ✓ All detected threats have been blocked${NC}"
|
||||
else
|
||||
echo -e "${LOW_COLOR} No threats detected yet...${NC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo -e "${HIGH_COLOR}└────────────────────────────────────────────────────────────────────────────┘${NC}"
|
||||
echo ""
|
||||
@@ -959,6 +1151,11 @@ show_blocking_menu() {
|
||||
for ip in "${!IP_DATA[@]}"; do
|
||||
IFS='|' read -r score hits bot_type attacks ban_count rep_score <<< "${IP_DATA[$ip]}"
|
||||
|
||||
# Set defaults for empty values
|
||||
score="${score:-0}"
|
||||
hits="${hits:-0}"
|
||||
attacks="${attacks:-none}"
|
||||
|
||||
# Skip if score too low or already blocked
|
||||
[ "$score" -lt 60 ] && continue
|
||||
is_ip_blocked "$ip" 2>/dev/null && continue
|
||||
@@ -1012,17 +1209,24 @@ show_blocking_menu() {
|
||||
elif [ "$choice" = "a" ]; then
|
||||
# Block all IPs with score >= 80
|
||||
local blocked=0
|
||||
local failed=0
|
||||
for entry in "${blockable_list[@]}"; do
|
||||
IFS='|' read -r ip score hits attacks <<< "$entry"
|
||||
[ "$score" -lt 80 ] && continue
|
||||
|
||||
echo ""
|
||||
block_ip_temporary "$ip" 1 "Auto-block: High threat (score $score)"
|
||||
if block_ip_temporary "$ip" 1 "Auto-block: High threat (score $score)"; then
|
||||
((blocked++))
|
||||
else
|
||||
((failed++))
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Blocked $blocked IPs"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "✓ Successfully blocked: $blocked IPs"
|
||||
[ $failed -gt 0 ] && echo "✗ Failed to block: $failed IPs"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
read -p "Press Enter to continue..."
|
||||
elif [[ "$choice" =~ ^[0-9]+$ ]] && [ "$choice" -ge 1 ] && [ "$choice" -le ${#blockable_list[@]} ]; then
|
||||
# Block specific IP
|
||||
@@ -1076,7 +1280,7 @@ monitor_apache_logs() {
|
||||
|
||||
# Monitor all log files
|
||||
local event_count=0
|
||||
tail -f "${log_files[@]}" 2>/dev/null | while read -r line; do
|
||||
tail -n 0 -F "${log_files[@]}" 2>/dev/null | while read -r line; do
|
||||
# Increment event counter (update file every 10 events for performance)
|
||||
((event_count++))
|
||||
if [ $((event_count % 10)) -eq 0 ]; then
|
||||
@@ -1982,6 +2186,30 @@ auto_mitigation_engine
|
||||
done
|
||||
) &
|
||||
|
||||
# Blocked IPs cache updater (runs every 10 seconds for performance)
|
||||
(
|
||||
while true; do
|
||||
{
|
||||
# Get CSF temporary blocks - extract just the IP address
|
||||
if command -v csf &>/dev/null; then
|
||||
csf -t 2>/dev/null | awk '{print $1}' | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$'
|
||||
fi
|
||||
|
||||
# Get CSF permanent denies
|
||||
if [ -f /etc/csf/csf.deny ]; then
|
||||
awk '{print $1}' /etc/csf/csf.deny 2>/dev/null | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$'
|
||||
fi
|
||||
|
||||
# Get iptables DROP rules
|
||||
if command -v iptables &>/dev/null; then
|
||||
iptables -L INPUT -n -v 2>/dev/null | grep DROP | awk '{print $8}' | grep -E '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$'
|
||||
fi
|
||||
} | sort -u > "$TEMP_DIR/blocked_ips_cache.tmp" 2>/dev/null
|
||||
mv "$TEMP_DIR/blocked_ips_cache.tmp" "$TEMP_DIR/blocked_ips_cache" 2>/dev/null
|
||||
sleep 10
|
||||
done
|
||||
) &
|
||||
|
||||
# Periodic snapshot saving in background
|
||||
(
|
||||
while true; do
|
||||
@@ -1993,26 +2221,44 @@ auto_mitigation_engine
|
||||
# Main dashboard loop
|
||||
LOOP_COUNT=0
|
||||
while true; do
|
||||
# Sync individual IP files into IP_DATA array (for data from subshell processes like SSH monitoring)
|
||||
for ip_file in "$TEMP_DIR"/ip_*; do
|
||||
[ -f "$ip_file" ] || continue
|
||||
basename_file="$(basename "$ip_file")"
|
||||
|
||||
# Skip non-IP files explicitly
|
||||
case "$basename_file" in
|
||||
ip_data|ip_database.db|*cache*|*blocked*|*debug*)
|
||||
continue
|
||||
;;
|
||||
esac
|
||||
|
||||
# Validate it's an IP file (should match pattern ip_N_N_N_N)
|
||||
if ! echo "$basename_file" | grep -qE '^ip_[0-9]{1,3}_[0-9]{1,3}_[0-9]{1,3}_[0-9]{1,3}$'; then
|
||||
continue
|
||||
fi
|
||||
|
||||
# Extract IP from filename (ip_1_2_3_4 -> 1.2.3.4)
|
||||
ip=$(echo "$basename_file" | sed 's/^ip_//' | tr '_' '.')
|
||||
data=$(cat "$ip_file" 2>/dev/null)
|
||||
|
||||
# Validate data format (should be score|hits|bot_type|attacks|ban_count|rep_score)
|
||||
if [ -n "$data" ] && echo "$data" | grep -q '|'; then
|
||||
# Update IP_DATA array with data from file
|
||||
IP_DATA[$ip]="$data"
|
||||
fi
|
||||
done
|
||||
|
||||
draw_header
|
||||
draw_intelligence_panel
|
||||
draw_attack_breakdown
|
||||
draw_live_feed
|
||||
draw_quick_actions
|
||||
|
||||
# Consolidate IP data from individual files into ip_data file (for auto-mitigation engine)
|
||||
# Write IP_DATA to ip_data file for auto-mitigation engine
|
||||
{
|
||||
for ip_file in "$TEMP_DIR"/ip_*; do
|
||||
[ -f "$ip_file" ] || continue
|
||||
# Skip the consolidated ip_data file itself
|
||||
[[ "$(basename "$ip_file")" == "ip_data" ]] && continue
|
||||
# Extract IP from filename (ip_1_2_3_4 -> 1.2.3.4)
|
||||
ip=$(basename "$ip_file" | sed 's/^ip_//' | tr '_' '.')
|
||||
data=$(cat "$ip_file" 2>/dev/null)
|
||||
if [ -n "$data" ]; then
|
||||
echo "$ip=$data"
|
||||
# Also update IP_DATA array for dashboard display
|
||||
IP_DATA[$ip]="$data"
|
||||
fi
|
||||
for ip in "${!IP_DATA[@]}"; do
|
||||
echo "$ip=${IP_DATA[$ip]}"
|
||||
done
|
||||
} > "$TEMP_DIR/ip_data" 2>/dev/null
|
||||
|
||||
|
||||
@@ -8,6 +8,15 @@
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
# Fix HISTFILE if set to non-existent path (prevents crashes on sourcing)
|
||||
if [ -n "$HISTFILE" ]; then
|
||||
HISTFILE_DIR="$(dirname "$HISTFILE" 2>/dev/null)"
|
||||
if [ ! -d "$HISTFILE_DIR" ] 2>/dev/null; then
|
||||
# Fallback to default history location
|
||||
export HISTFILE="$HOME/.bash_history"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check if being sourced or executed
|
||||
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
|
||||
echo "ERROR: This script must be sourced, not executed."
|
||||
|
||||
Reference in New Issue
Block a user