From 1e2b9946e8d00807a62e51aa97611b805a98c2a9 Mon Sep 17 00:00:00 2001 From: cschantz Date: Fri, 14 Nov 2025 16:17:59 -0500 Subject: [PATCH] Add comprehensive threat intelligence and behavioral analysis Created new threat intelligence library with extensive monitoring capabilities: Threat Intelligence Integration: - AbuseIPDB API integration with caching (24hr TTL) - Geolocation detection via geoiplookup/whois - High-risk country identification - ISP and country-based risk scoring Smart Whitelisting: - Automatic detection of legitimate services (Google, Cloudflare, Microsoft, Akamai) - CDN IP range recognition - Configurable whitelist management Behavioral Analysis: - Request timing pattern analysis (human vs bot detection) - Attack pattern learning and recording - Pattern matching for repeat attackers Performance Monitoring: - Server load tracking integration - Stress detection for adaptive mitigation - CPU and load average monitoring Incident Response: - Automated incident report generation - Comprehensive threat intelligence summaries - Attack history tracking - Recommended action suggestions Multi-Server Coordination: - Shared threat data logging - Cross-server attack correlation preparation Live Monitor Integration: - Auto-enrichment on first IP encounter - AbuseIPDB confidence scoring boost (30pts for 75%+, 15pts for 50%+) - High-risk country detection adds 5pts - Attack pattern recording for learning - New keyboard commands: i) Threat intelligence lookup with incident reports p) Performance impact monitor All features use existing system tools only (no new services installed) --- lib/threat-intelligence.sh | 466 ++++++++++++++++++++++++ modules/security/live-attack-monitor.sh | 130 +++++++ 2 files changed, 596 insertions(+) create mode 100644 lib/threat-intelligence.sh diff --git a/lib/threat-intelligence.sh b/lib/threat-intelligence.sh new file mode 100644 index 0000000..3781f62 --- /dev/null +++ b/lib/threat-intelligence.sh @@ -0,0 +1,466 @@ +#!/bin/bash + +################################################################################ +# Threat Intelligence Library +################################################################################ +# Purpose: External threat intelligence integration using existing tools +# Features: IP reputation lookups, geolocation, whitelist management +# No new services - uses only existing APIs and tools +################################################################################ + +# Cache directory for threat intelligence +THREAT_CACHE_DIR="/var/lib/server-toolkit/threat-cache" +mkdir -p "$THREAT_CACHE_DIR" 2>/dev/null + +# Cache TTL (24 hours) +CACHE_TTL=86400 + +################################################################################ +# AbuseIPDB Integration (Free API - 1000 requests/day) +################################################################################ + +# Check if IP is in AbuseIPDB +# Returns: confidence_score|total_reports|country|isp +check_abuseipdb() { + local ip="$1" + local cache_file="$THREAT_CACHE_DIR/abuseipdb_${ip//\./_}" + + # Check cache first + if [ -f "$cache_file" ]; then + local cache_age=$(($(date +%s) - $(stat -c %Y "$cache_file" 2>/dev/null || echo 0))) + if [ "$cache_age" -lt "$CACHE_TTL" ]; then + cat "$cache_file" + return 0 + fi + fi + + # Check if API key exists + local api_key_file="/root/.abuseipdb_api_key" + if [ ! -f "$api_key_file" ]; then + echo "0|0|Unknown|Unknown" + return 1 + fi + + local api_key=$(cat "$api_key_file") + + # Query AbuseIPDB API + local response=$(curl -s -G https://api.abuseipdb.com/api/v2/check \ + --data-urlencode "ipAddress=$ip" \ + -d maxAgeInDays=90 \ + -H "Key: $api_key" \ + -H "Accept: application/json" 2>/dev/null) + + if [ -n "$response" ]; then + local confidence=$(echo "$response" | grep -oP '"abuseConfidenceScore":\K[0-9]+' | head -1) + local reports=$(echo "$response" | grep -oP '"totalReports":\K[0-9]+' | head -1) + local country=$(echo "$response" | grep -oP '"countryCode":"\K[^"]+' | head -1) + local isp=$(echo "$response" | grep -oP '"isp":"\K[^"]+' | head -1) + + local result="${confidence:-0}|${reports:-0}|${country:-Unknown}|${isp:-Unknown}" + echo "$result" | tee "$cache_file" + return 0 + fi + + echo "0|0|Unknown|Unknown" + return 1 +} + +################################################################################ +# Geolocation Detection (Using existing geoiplookup or geoip-bin) +################################################################################ + +# Get country code for IP +get_country_code() { + local ip="$1" + local cache_file="$THREAT_CACHE_DIR/geo_${ip//\./_}" + + # Check cache + if [ -f "$cache_file" ]; then + local cache_age=$(($(date +%s) - $(stat -c %Y "$cache_file" 2>/dev/null || echo 0))) + if [ "$cache_age" -lt "$CACHE_TTL" ]; then + cat "$cache_file" + return 0 + fi + fi + + # Try geoiplookup (if installed) + if command -v geoiplookup &>/dev/null; then + local country=$(geoiplookup "$ip" 2>/dev/null | head -1 | grep -oP 'GeoIP Country Edition: \K[A-Z]{2}') + if [ -n "$country" ]; then + echo "$country" | tee "$cache_file" + return 0 + fi + fi + + # Try geoip-bin (alternative) + if command -v geoiplookup6 &>/dev/null; then + local country=$(geoiplookup6 "$ip" 2>/dev/null | head -1 | grep -oP 'GeoIP Country Edition: \K[A-Z]{2}') + if [ -n "$country" ]; then + echo "$country" | tee "$cache_file" + return 0 + fi + fi + + # Fallback to whois (slower, not cached as aggressively) + if command -v whois &>/dev/null; then + local country=$(whois "$ip" 2>/dev/null | grep -i "^country:" | head -1 | awk '{print $2}' | tr '[:lower:]' '[:upper:]') + if [ -n "$country" ] && [ ${#country} -eq 2 ]; then + echo "$country" | tee "$cache_file" + return 0 + fi + fi + + echo "XX" + return 1 +} + +# Check if country is high-risk +is_high_risk_country() { + local country="$1" + + # High-risk countries (commonly seen in attacks) + local high_risk="CN RU UA BY KP IR VN TH ID BR" + + if echo "$high_risk" | grep -qw "$country"; then + return 0 + fi + + return 1 +} + +################################################################################ +# Smart Whitelisting +################################################################################ + +# Check if IP should be whitelisted (legitimate services) +is_whitelisted_service() { + local ip="$1" + local whitelist_file="/var/lib/server-toolkit/whitelist_ips.txt" + + # Check static whitelist + if [ -f "$whitelist_file" ]; then + if grep -q "^$ip$" "$whitelist_file"; then + return 0 + fi + fi + + # Check if IP belongs to known legitimate networks + # Google IPs (8.8.0.0/16, 66.249.64.0/19, etc.) + if [[ "$ip" =~ ^8\.8\. ]] || [[ "$ip" =~ ^66\.249\. ]] || [[ "$ip" =~ ^66\.102\. ]]; then + return 0 + fi + + # Cloudflare IPs (173.245.48.0/20, 103.21.244.0/22, etc.) + if [[ "$ip" =~ ^173\.245\.(4[8-9]|5[0-9]|6[0-3])\. ]] || [[ "$ip" =~ ^103\.21\.24[4-7]\. ]]; then + return 0 + fi + + # Microsoft/Bing IPs (40.0.0.0/8, 65.52.0.0/14, etc.) + if [[ "$ip" =~ ^40\. ]] || [[ "$ip" =~ ^65\.5[2-5]\. ]]; then + return 0 + fi + + # Common CDN/monitoring services + # Akamai: 23.0.0.0/8 + if [[ "$ip" =~ ^23\. ]]; then + return 0 + fi + + return 1 +} + +# Add IP to whitelist +add_to_whitelist() { + local ip="$1" + local reason="$2" + local whitelist_file="/var/lib/server-toolkit/whitelist_ips.txt" + + if ! grep -q "^$ip$" "$whitelist_file" 2>/dev/null; then + echo "$ip # $reason" >> "$whitelist_file" + fi +} + +################################################################################ +# Behavioral Analysis +################################################################################ + +# Analyze request timing pattern +# Returns: human|bot|suspicious +analyze_timing_pattern() { + local ip="$1" + local timing_file="$THREAT_CACHE_DIR/timing_${ip//\./_}" + + # Record timestamp + echo "$(date +%s)" >> "$timing_file" + + # Keep only last 100 requests + tail -100 "$timing_file" > "${timing_file}.tmp" 2>/dev/null + mv "${timing_file}.tmp" "$timing_file" 2>/dev/null + + # Analyze if we have enough data + local request_count=$(wc -l < "$timing_file" 2>/dev/null || echo 0) + if [ "$request_count" -lt 10 ]; then + echo "unknown" + return + fi + + # Calculate average time between requests + local timestamps=$(cat "$timing_file") + local total_gap=0 + local gap_count=0 + local prev_ts="" + + while IFS= read -r ts; do + if [ -n "$prev_ts" ]; then + local gap=$((ts - prev_ts)) + total_gap=$((total_gap + gap)) + gap_count=$((gap_count + 1)) + fi + prev_ts="$ts" + done <<< "$timestamps" + + if [ "$gap_count" -gt 0 ]; then + local avg_gap=$((total_gap / gap_count)) + + # Bot patterns: < 2 seconds between requests consistently + if [ "$avg_gap" -lt 2 ]; then + echo "bot" + return + fi + + # Human patterns: 5-30 seconds between requests with variation + if [ "$avg_gap" -ge 5 ] && [ "$avg_gap" -le 30 ]; then + echo "human" + return + fi + + # Suspicious: Too fast but not consistent bot pattern + echo "suspicious" + return + fi + + echo "unknown" +} + +################################################################################ +# Attack Pattern Learning +################################################################################ + +# Record attack pattern for learning +record_attack_pattern() { + local ip="$1" + local attack_type="$2" + local uri="$3" + local user_agent="$4" + + local pattern_file="/var/lib/server-toolkit/attack-patterns/patterns.log" + mkdir -p "$(dirname "$pattern_file")" 2>/dev/null + + # Format: timestamp|ip|attack_type|uri|user_agent + echo "$(date +%s)|$ip|$attack_type|$uri|$user_agent" >> "$pattern_file" + + # Keep only last 10000 patterns (prevent unbounded growth) + tail -10000 "$pattern_file" > "${pattern_file}.tmp" 2>/dev/null + mv "${pattern_file}.tmp" "$pattern_file" 2>/dev/null +} + +# Check if attack matches known pattern +matches_known_pattern() { + local attack_type="$1" + local uri="$2" + + local pattern_file="/var/lib/server-toolkit/attack-patterns/patterns.log" + + if [ ! -f "$pattern_file" ]; then + return 1 + fi + + # Check if this attack type + similar URI has been seen before + local similar_count=$(grep "|$attack_type|" "$pattern_file" | grep -c "$uri" || echo 0) + + if [ "$similar_count" -ge 3 ]; then + return 0 # Known pattern + fi + + return 1 # New pattern +} + +################################################################################ +# Performance Impact Monitoring +################################################################################ + +# Get current server load +get_server_load() { + # Returns: load_1min|load_5min|load_15min|cpu_count + local load=$(uptime | awk -F'load average:' '{print $2}' | sed 's/,//g' | xargs) + local cpu_count=$(nproc) + + echo "${load}|${cpu_count}" +} + +# Check if server is under stress +is_server_stressed() { + local load_data=$(get_server_load) + IFS='|' read -r load1 load5 load15 cpu_count <<< "$load_data" + + # Remove any extra spaces + load1=$(echo "$load1" | awk '{print $1}') + + # Convert to integer (multiply by 100 to handle decimals) + local load_int=$(echo "$load1 * 100" | bc 2>/dev/null | cut -d. -f1) + local threshold=$((cpu_count * 80)) # 80% of CPU count + + if [ "$load_int" -gt "$threshold" ]; then + return 0 # Server is stressed + fi + + return 1 +} + +################################################################################ +# Incident Report Generation +################################################################################ + +# Generate incident report for an IP +generate_incident_report() { + local ip="$1" + local report_file="/var/lib/server-toolkit/incident-reports/report_${ip//\./_}_$(date +%Y%m%d_%H%M%S).txt" + + mkdir -p "$(dirname "$report_file")" 2>/dev/null + + { + echo "═══════════════════════════════════════════════════════════════" + echo "SECURITY INCIDENT REPORT" + echo "═══════════════════════════════════════════════════════════════" + echo "" + echo "Generated: $(date '+%Y-%m-%d %H:%M:%S %Z')" + echo "IP Address: $ip" + echo "" + + echo "─────────────────────────────────────────────────────────────" + echo "THREAT INTELLIGENCE" + echo "─────────────────────────────────────────────────────────────" + + # AbuseIPDB data + local abuse_data=$(check_abuseipdb "$ip") + IFS='|' read -r confidence reports country isp <<< "$abuse_data" + echo "AbuseIPDB Confidence: ${confidence}%" + echo "Total Reports: $reports" + echo "Country: $country" + echo "ISP: $isp" + echo "" + + # Geolocation + local geo=$(get_country_code "$ip") + echo "Geolocation: $geo" + if is_high_risk_country "$geo"; then + echo "Risk Level: HIGH (Known attack source country)" + else + echo "Risk Level: MEDIUM" + fi + echo "" + + echo "─────────────────────────────────────────────────────────────" + echo "ATTACK HISTORY" + echo "─────────────────────────────────────────────────────────────" + + # Get attacks from pattern log + local pattern_file="/var/lib/server-toolkit/attack-patterns/patterns.log" + if [ -f "$pattern_file" ]; then + echo "Recent attacks from this IP:" + grep "|$ip|" "$pattern_file" | tail -20 | while IFS='|' read -r ts ip_addr attack_type uri ua; do + echo " [$(date -d @$ts '+%Y-%m-%d %H:%M:%S')] $attack_type - $uri" + done + echo "" + fi + + echo "─────────────────────────────────────────────────────────────" + echo "RECOMMENDED ACTIONS" + echo "─────────────────────────────────────────────────────────────" + + if [ "$confidence" -ge 75 ]; then + echo "• IMMEDIATE BLOCK - High confidence malicious IP" + echo " Command: csf -d $ip \"AbuseIPDB: ${confidence}% confidence\"" + elif [ "$reports" -ge 10 ]; then + echo "• TEMPORARY BLOCK - Multiple abuse reports" + echo " Command: csf -td $ip 86400 \"Multiple abuse reports\"" + else + echo "• MONITOR - Watch for continued activity" + fi + + echo "" + echo "═══════════════════════════════════════════════════════════════" + echo "END OF REPORT" + echo "═══════════════════════════════════════════════════════════════" + + } > "$report_file" + + echo "$report_file" +} + +################################################################################ +# Multi-Server Coordination (for environments with multiple servers) +################################################################################ + +# Share threat data with other servers (if configured) +share_threat_data() { + local ip="$1" + local attack_type="$2" + local score="$3" + + local coordination_file="/var/lib/server-toolkit/shared-threats.log" + + # Log for potential sharing + echo "$(date +%s)|$(hostname)|$ip|$attack_type|$score" >> "$coordination_file" + + # Keep only last 1000 entries + tail -1000 "$coordination_file" > "${coordination_file}.tmp" 2>/dev/null + mv "${coordination_file}.tmp" "$coordination_file" 2>/dev/null +} + +# Check if IP is flagged by other servers +check_shared_threats() { + local ip="$1" + local coordination_file="/var/lib/server-toolkit/shared-threats.log" + + if [ -f "$coordination_file" ]; then + local count=$(grep "|$ip|" "$coordination_file" | wc -l) + echo "$count" + else + echo "0" + fi +} + +################################################################################ +# Threat Intelligence Summary +################################################################################ + +# Get comprehensive threat intelligence for an IP +get_threat_intelligence() { + local ip="$1" + + local abuse_data=$(check_abuseipdb "$ip" 2>/dev/null || echo "0|0|Unknown|Unknown") + local geo=$(get_country_code "$ip" 2>/dev/null || echo "XX") + local timing=$(analyze_timing_pattern "$ip" 2>/dev/null || echo "unknown") + local whitelisted="no" + is_whitelisted_service "$ip" && whitelisted="yes" + + # Format: abuse_confidence|abuse_reports|country|isp|timing_pattern|whitelisted + echo "${abuse_data}|${geo}|${timing}|${whitelisted}" +} + +# Export functions for use in other scripts +export -f check_abuseipdb +export -f get_country_code +export -f is_high_risk_country +export -f is_whitelisted_service +export -f add_to_whitelist +export -f analyze_timing_pattern +export -f record_attack_pattern +export -f matches_known_pattern +export -f get_server_load +export -f is_server_stressed +export -f generate_incident_report +export -f share_threat_data +export -f check_shared_threats +export -f get_threat_intelligence diff --git a/modules/security/live-attack-monitor.sh b/modules/security/live-attack-monitor.sh index 492e20c..5be8db5 100755 --- a/modules/security/live-attack-monitor.sh +++ b/modules/security/live-attack-monitor.sh @@ -21,6 +21,7 @@ source "$SCRIPT_DIR/lib/system-detect.sh" source "$SCRIPT_DIR/lib/ip-reputation.sh" source "$SCRIPT_DIR/lib/bot-signatures.sh" source "$SCRIPT_DIR/lib/attack-patterns.sh" +source "$SCRIPT_DIR/lib/threat-intelligence.sh" # Require root if [ "$EUID" -ne 0 ]; then @@ -158,11 +159,61 @@ update_ip_intelligence() { # Increment hits hits=$((hits + 1)) + # Enrich with threat intelligence on first encounter (hits == 1) + if [ $hits -eq 1 ]; then + # Check if whitelisted first + if is_whitelisted_service "$ip" 2>/dev/null; then + score=0 + bot_type="legit" + else + # Get threat intelligence (in background to avoid slowing down) + ( + local threat_intel=$(get_threat_intelligence "$ip" 2>/dev/null) + IFS='|' read -r abuse_conf abuse_rpts country isp geo timing whitelisted <<< "$threat_intel" + + # Store enrichment data for later use + local enrich_file="$TEMP_DIR/threat_enrich_${ip//\./_}" + echo "$threat_intel" > "$enrich_file" + + # Boost score based on AbuseIPDB confidence + if [ "${abuse_conf:-0}" -ge 75 ]; then + # High confidence malicious - add 30 points + local current_data="${IP_DATA[$ip]}" + IFS='|' read -r old_score old_hits old_bot old_attacks old_ban old_rep <<< "$current_data" + local new_score=$((old_score + 30)) + [ $new_score -gt 100 ] && new_score=100 + IP_DATA[$ip]="$new_score|$old_hits|$old_bot|$old_attacks|$old_ban|$old_rep" + elif [ "${abuse_conf:-0}" -ge 50 ]; then + # Medium confidence - add 15 points + local current_data="${IP_DATA[$ip]}" + IFS='|' read -r old_score old_hits old_bot old_attacks old_ban old_rep <<< "$current_data" + local new_score=$((old_score + 15)) + [ $new_score -gt 100 ] && new_score=100 + IP_DATA[$ip]="$new_score|$old_hits|$old_bot|$old_attacks|$old_ban|$old_rep" + fi + + # High-risk country adds 5 points + if is_high_risk_country "${geo:-XX}" 2>/dev/null; then + local current_data="${IP_DATA[$ip]}" + IFS='|' read -r old_score old_hits old_bot old_attacks old_ban old_rep <<< "$current_data" + local new_score=$((old_score + 5)) + [ $new_score -gt 100 ] && new_score=100 + IP_DATA[$ip]="$new_score|$old_hits|$old_bot|$old_attacks|$old_ban|$old_rep" + fi + ) & + fi + fi + # Classify bot if unknown if [ "$bot_type" = "unknown" ] && [ -n "$user_agent" ]; then bot_type=$(classify_bot_type "$user_agent") fi + # Record attack pattern for learning + if [ -n "$url" ]; then + record_attack_pattern "$ip" "${attacks:-unknown}" "$url" "${user_agent:-unknown}" 2>/dev/null & + fi + # Detect attacks in URL local new_attacks=$(detect_all_attacks "$url" "$method") @@ -1254,6 +1305,78 @@ while true; do "$SCRIPT_DIR/modules/security/optimize-ct-limit.sh" read -p "Press Enter to return to monitor..." ;; + i|I) + # Show threat intelligence for specific IP + clear + print_banner "Threat Intelligence Lookup" + echo "" + read -p "Enter IP address: " lookup_ip + if [ -n "$lookup_ip" ]; then + echo "" + echo "Querying threat intelligence for $lookup_ip..." + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + local threat_intel=$(get_threat_intelligence "$lookup_ip") + IFS='|' read -r abuse_conf abuse_rpts country isp geo timing whitelisted <<< "$threat_intel" + echo "" + echo "${BOLD}Threat Intelligence:${NC}" + echo " AbuseIPDB Confidence: ${abuse_conf}%" + echo " Total Abuse Reports: $abuse_rpts" + echo " Country: ${geo:-$country}" + echo " ISP: $isp" + echo " Timing Pattern: $timing" + echo " Whitelisted: $whitelisted" + echo "" + if is_high_risk_country "${geo:-XX}"; then + echo -e "${HIGH_COLOR} ⚠️ HIGH RISK COUNTRY${NC}" + fi + if [ "${abuse_conf:-0}" -ge 75 ]; then + echo -e "${CRITICAL_COLOR} 🚨 HIGH CONFIDENCE MALICIOUS${NC}" + elif [ "${abuse_conf:-0}" -ge 50 ]; then + echo -e "${HIGH_COLOR} ⚠️ MEDIUM CONFIDENCE THREAT${NC}" + fi + echo "" + read -p "Generate full incident report? (y/n): " gen_report + if [[ "$gen_report" =~ ^[Yy]$ ]]; then + local report_file=$(generate_incident_report "$lookup_ip") + echo "" + echo "Report generated: $report_file" + echo "" + echo "View report? (y/n): " + read -n 1 view_report + if [[ "$view_report" =~ ^[Yy]$ ]]; then + less "$report_file" + fi + fi + fi + echo "" + read -p "Press Enter to return to monitor..." + ;; + p|P) + # Show performance impact + clear + print_banner "Server Performance Monitor" + echo "" + local load_data=$(get_server_load) + IFS='|' read -r load1 load5 load15 cpu_count <<< "$load_data" + echo "${BOLD}Current Load:${NC}" + echo " 1 min: $load1" + echo " 5 min: $load5" + echo " 15 min: $load15" + echo " CPU cores: $cpu_count" + echo "" + if is_server_stressed; then + echo -e "${CRITICAL_COLOR} 🔥 SERVER UNDER STRESS${NC}" + echo "" + echo " Recommended Actions:" + echo " • Enable aggressive auto-blocking (higher threshold)" + echo " • Reduce CT_LIMIT temporarily" + echo " • Block high-volume attack IPs immediately" + else + echo -e "${SAFE_COLOR} ✓ Server load normal${NC}" + fi + echo "" + read -p "Press Enter to return to monitor..." + ;; q|Q) cleanup ;; @@ -1275,6 +1398,8 @@ while true; do echo "Available Commands:" echo " ${BOLD}b${NC} - Open IP blocking menu (batch or individual)" echo " ${BOLD}c${NC} - Run CT_LIMIT optimizer (analyze traffic & recommend limit)" + echo " ${BOLD}i${NC} - Threat intelligence lookup (AbuseIPDB, geo, incident reports)" + echo " ${BOLD}p${NC} - Show performance impact monitor (server load)" echo " ${BOLD}s${NC} - Show IP reputation database statistics" echo " ${BOLD}r${NC} - Force refresh display" echo " ${BOLD}h${NC} - Show this help screen" @@ -1284,8 +1409,13 @@ while true; do echo " • Real-time bot classification (legit/AI/monitor/suspicious)" echo " • Attack vector detection (SQL, XSS, RCE, etc.)" echo " • Threat scoring (0-100 scale)" + echo " • Threat intelligence integration (AbuseIPDB, geolocation)" + echo " • Attack pattern learning & behavioral analysis" + echo " • Automated incident report generation" + echo " • Smart whitelisting (CDNs, search engines)" echo " • IP reputation DB integration" echo " • CSF/iptables temporary bans (1 hour default)" + echo " • Auto-mitigation at critical threshold (score ≥80)" echo " • Memory protection (max ${MAX_TRACKED_IPS} IPs tracked)" echo " • Auto-save every 5 minutes + on exit" echo ""