Add comprehensive threat intelligence and behavioral analysis

Created new threat intelligence library with extensive monitoring capabilities: Threat Intelligence Integration: - AbuseIPDB API integration with caching (24hr TTL) - Geolocation detection via geoiplookup/whois - High-risk country identification - ISP and country-based risk scoring Smart Whitelisting: - Automatic detection of legitimate services (Google, Cloudflare, Microsoft, Akamai) - CDN IP range recognition - Configurable whitelist management Behavioral Analysis: - Request timing pattern analysis (human vs bot detection) - Attack pattern learning and recording - Pattern matching for repeat attackers Performance Monitoring: - Server load tracking integration - Stress detection for adaptive mitigation - CPU and load average monitoring Incident Response: - Automated incident report generation - Comprehensive threat intelligence summaries - Attack history tracking - Recommended action suggestions Multi-Server Coordination: - Shared threat data logging - Cross-server attack correlation preparation Live Monitor Integration: - Auto-enrichment on first IP encounter - AbuseIPDB confidence scoring boost (30pts for 75%+, 15pts for 50%+) - High-risk country detection adds 5pts - Attack pattern recording for learning - New keyboard commands: i) Threat intelligence lookup with incident reports p) Performance impact monitor All features use existing system tools only (no new services installed)
2025-11-14 16:17:59 -05:00
parent e179c4c213
commit 3e97dd86d9
2 changed files with 596 additions and 0 deletions
@@ -0,0 +1,466 @@
+#!/bin/bash
+
+################################################################################
+# Threat Intelligence Library
+################################################################################
+# Purpose: External threat intelligence integration using existing tools
+# Features: IP reputation lookups, geolocation, whitelist management
+# No new services - uses only existing APIs and tools
+################################################################################
+
+# Cache directory for threat intelligence
+THREAT_CACHE_DIR="/var/lib/server-toolkit/threat-cache"
+mkdir -p "$THREAT_CACHE_DIR" 2>/dev/null
+
+# Cache TTL (24 hours)
+CACHE_TTL=86400
+
+################################################################################
+# AbuseIPDB Integration (Free API - 1000 requests/day)
+################################################################################
+
+# Check if IP is in AbuseIPDB
+# Returns: confidence_score|total_reports|country|isp
+check_abuseipdb() {
+    local ip="$1"
+    local cache_file="$THREAT_CACHE_DIR/abuseipdb_${ip//\./_}"
+
+    # Check cache first
+    if [ -f "$cache_file" ]; then
+        local cache_age=$(($(date +%s) - $(stat -c %Y "$cache_file" 2>/dev/null || echo 0)))
+        if [ "$cache_age" -lt "$CACHE_TTL" ]; then
+            cat "$cache_file"
+            return 0
+        fi
+    fi
+
+    # Check if API key exists
+    local api_key_file="/root/.abuseipdb_api_key"
+    if [ ! -f "$api_key_file" ]; then
+        echo "0|0|Unknown|Unknown"
+        return 1
+    fi
+
+    local api_key=$(cat "$api_key_file")
+
+    # Query AbuseIPDB API
+    local response=$(curl -s -G https://api.abuseipdb.com/api/v2/check \
+        --data-urlencode "ipAddress=$ip" \
+        -d maxAgeInDays=90 \
+        -H "Key: $api_key" \
+        -H "Accept: application/json" 2>/dev/null)
+
+    if [ -n "$response" ]; then
+        local confidence=$(echo "$response" | grep -oP '"abuseConfidenceScore":\K[0-9]+' | head -1)
+        local reports=$(echo "$response" | grep -oP '"totalReports":\K[0-9]+' | head -1)
+        local country=$(echo "$response" | grep -oP '"countryCode":"\K[^"]+' | head -1)
+        local isp=$(echo "$response" | grep -oP '"isp":"\K[^"]+' | head -1)
+
+        local result="${confidence:-0}|${reports:-0}|${country:-Unknown}|${isp:-Unknown}"
+        echo "$result" | tee "$cache_file"
+        return 0
+    fi
+
+    echo "0|0|Unknown|Unknown"
+    return 1
+}
+
+################################################################################
+# Geolocation Detection (Using existing geoiplookup or geoip-bin)
+################################################################################
+
+# Get country code for IP
+get_country_code() {
+    local ip="$1"
+    local cache_file="$THREAT_CACHE_DIR/geo_${ip//\./_}"
+
+    # Check cache
+    if [ -f "$cache_file" ]; then
+        local cache_age=$(($(date +%s) - $(stat -c %Y "$cache_file" 2>/dev/null || echo 0)))
+        if [ "$cache_age" -lt "$CACHE_TTL" ]; then
+            cat "$cache_file"
+            return 0
+        fi
+    fi
+
+    # Try geoiplookup (if installed)
+    if command -v geoiplookup &>/dev/null; then
+        local country=$(geoiplookup "$ip" 2>/dev/null | head -1 | grep -oP 'GeoIP Country Edition: \K[A-Z]{2}')
+        if [ -n "$country" ]; then
+            echo "$country" | tee "$cache_file"
+            return 0
+        fi
+    fi
+
+    # Try geoip-bin (alternative)
+    if command -v geoiplookup6 &>/dev/null; then
+        local country=$(geoiplookup6 "$ip" 2>/dev/null | head -1 | grep -oP 'GeoIP Country Edition: \K[A-Z]{2}')
+        if [ -n "$country" ]; then
+            echo "$country" | tee "$cache_file"
+            return 0
+        fi
+    fi
+
+    # Fallback to whois (slower, not cached as aggressively)
+    if command -v whois &>/dev/null; then
+        local country=$(whois "$ip" 2>/dev/null | grep -i "^country:" | head -1 | awk '{print $2}' | tr '[:lower:]' '[:upper:]')
+        if [ -n "$country" ] && [ ${#country} -eq 2 ]; then
+            echo "$country" | tee "$cache_file"
+            return 0
+        fi
+    fi
+
+    echo "XX"
+    return 1
+}
+
+# Check if country is high-risk
+is_high_risk_country() {
+    local country="$1"
+
+    # High-risk countries (commonly seen in attacks)
+    local high_risk="CN RU UA BY KP IR VN TH ID BR"
+
+    if echo "$high_risk" | grep -qw "$country"; then
+        return 0
+    fi
+
+    return 1
+}
+
+################################################################################
+# Smart Whitelisting
+################################################################################
+
+# Check if IP should be whitelisted (legitimate services)
+is_whitelisted_service() {
+    local ip="$1"
+    local whitelist_file="/var/lib/server-toolkit/whitelist_ips.txt"
+
+    # Check static whitelist
+    if [ -f "$whitelist_file" ]; then
+        if grep -q "^$ip$" "$whitelist_file"; then
+            return 0
+        fi
+    fi
+
+    # Check if IP belongs to known legitimate networks
+    # Google IPs (8.8.0.0/16, 66.249.64.0/19, etc.)
+    if [[ "$ip" =~ ^8\.8\. ]] || [[ "$ip" =~ ^66\.249\. ]] || [[ "$ip" =~ ^66\.102\. ]]; then
+        return 0
+    fi
+
+    # Cloudflare IPs (173.245.48.0/20, 103.21.244.0/22, etc.)
+    if [[ "$ip" =~ ^173\.245\.(4[8-9]|5[0-9]|6[0-3])\. ]] || [[ "$ip" =~ ^103\.21\.24[4-7]\. ]]; then
+        return 0
+    fi
+
+    # Microsoft/Bing IPs (40.0.0.0/8, 65.52.0.0/14, etc.)
+    if [[ "$ip" =~ ^40\. ]] || [[ "$ip" =~ ^65\.5[2-5]\. ]]; then
+        return 0
+    fi
+
+    # Common CDN/monitoring services
+    # Akamai: 23.0.0.0/8
+    if [[ "$ip" =~ ^23\. ]]; then
+        return 0
+    fi
+
+    return 1
+}
+
+# Add IP to whitelist
+add_to_whitelist() {
+    local ip="$1"
+    local reason="$2"
+    local whitelist_file="/var/lib/server-toolkit/whitelist_ips.txt"
+
+    if ! grep -q "^$ip$" "$whitelist_file" 2>/dev/null; then
+        echo "$ip  # $reason" >> "$whitelist_file"
+    fi
+}
+
+################################################################################
+# Behavioral Analysis
+################################################################################
+
+# Analyze request timing pattern
+# Returns: human|bot|suspicious
+analyze_timing_pattern() {
+    local ip="$1"
+    local timing_file="$THREAT_CACHE_DIR/timing_${ip//\./_}"
+
+    # Record timestamp
+    echo "$(date +%s)" >> "$timing_file"
+
+    # Keep only last 100 requests
+    tail -100 "$timing_file" > "${timing_file}.tmp" 2>/dev/null
+    mv "${timing_file}.tmp" "$timing_file" 2>/dev/null
+
+    # Analyze if we have enough data
+    local request_count=$(wc -l < "$timing_file" 2>/dev/null || echo 0)
+    if [ "$request_count" -lt 10 ]; then
+        echo "unknown"
+        return
+    fi
+
+    # Calculate average time between requests
+    local timestamps=$(cat "$timing_file")
+    local total_gap=0
+    local gap_count=0
+    local prev_ts=""
+
+    while IFS= read -r ts; do
+        if [ -n "$prev_ts" ]; then
+            local gap=$((ts - prev_ts))
+            total_gap=$((total_gap + gap))
+            gap_count=$((gap_count + 1))
+        fi
+        prev_ts="$ts"
+    done <<< "$timestamps"
+
+    if [ "$gap_count" -gt 0 ]; then
+        local avg_gap=$((total_gap / gap_count))
+
+        # Bot patterns: < 2 seconds between requests consistently
+        if [ "$avg_gap" -lt 2 ]; then
+            echo "bot"
+            return
+        fi
+
+        # Human patterns: 5-30 seconds between requests with variation
+        if [ "$avg_gap" -ge 5 ] && [ "$avg_gap" -le 30 ]; then
+            echo "human"
+            return
+        fi
+
+        # Suspicious: Too fast but not consistent bot pattern
+        echo "suspicious"
+        return
+    fi
+
+    echo "unknown"
+}
+
+################################################################################
+# Attack Pattern Learning
+################################################################################
+
+# Record attack pattern for learning
+record_attack_pattern() {
+    local ip="$1"
+    local attack_type="$2"
+    local uri="$3"
+    local user_agent="$4"
+
+    local pattern_file="/var/lib/server-toolkit/attack-patterns/patterns.log"
+    mkdir -p "$(dirname "$pattern_file")" 2>/dev/null
+
+    # Format: timestamp|ip|attack_type|uri|user_agent
+    echo "$(date +%s)|$ip|$attack_type|$uri|$user_agent" >> "$pattern_file"
+
+    # Keep only last 10000 patterns (prevent unbounded growth)
+    tail -10000 "$pattern_file" > "${pattern_file}.tmp" 2>/dev/null
+    mv "${pattern_file}.tmp" "$pattern_file" 2>/dev/null
+}
+
+# Check if attack matches known pattern
+matches_known_pattern() {
+    local attack_type="$1"
+    local uri="$2"
+
+    local pattern_file="/var/lib/server-toolkit/attack-patterns/patterns.log"
+
+    if [ ! -f "$pattern_file" ]; then
+        return 1
+    fi
+
+    # Check if this attack type + similar URI has been seen before
+    local similar_count=$(grep "|$attack_type|" "$pattern_file" | grep -c "$uri" || echo 0)
+
+    if [ "$similar_count" -ge 3 ]; then
+        return 0  # Known pattern
+    fi
+
+    return 1  # New pattern
+}
+
+################################################################################
+# Performance Impact Monitoring
+################################################################################
+
+# Get current server load
+get_server_load() {
+    # Returns: load_1min|load_5min|load_15min|cpu_count
+    local load=$(uptime | awk -F'load average:' '{print $2}' | sed 's/,//g' | xargs)
+    local cpu_count=$(nproc)
+
+    echo "${load}|${cpu_count}"
+}
+
+# Check if server is under stress
+is_server_stressed() {
+    local load_data=$(get_server_load)
+    IFS='|' read -r load1 load5 load15 cpu_count <<< "$load_data"
+
+    # Remove any extra spaces
+    load1=$(echo "$load1" | awk '{print $1}')
+
+    # Convert to integer (multiply by 100 to handle decimals)
+    local load_int=$(echo "$load1 * 100" | bc 2>/dev/null | cut -d. -f1)
+    local threshold=$((cpu_count * 80))  # 80% of CPU count
+
+    if [ "$load_int" -gt "$threshold" ]; then
+        return 0  # Server is stressed
+    fi
+
+    return 1
+}
+
+################################################################################
+# Incident Report Generation
+################################################################################
+
+# Generate incident report for an IP
+generate_incident_report() {
+    local ip="$1"
+    local report_file="/var/lib/server-toolkit/incident-reports/report_${ip//\./_}_$(date +%Y%m%d_%H%M%S).txt"
+
+    mkdir -p "$(dirname "$report_file")" 2>/dev/null
+
+    {
+        echo "═══════════════════════════════════════════════════════════════"
+        echo "SECURITY INCIDENT REPORT"
+        echo "═══════════════════════════════════════════════════════════════"
+        echo ""
+        echo "Generated: $(date '+%Y-%m-%d %H:%M:%S %Z')"
+        echo "IP Address: $ip"
+        echo ""
+
+        echo "─────────────────────────────────────────────────────────────"
+        echo "THREAT INTELLIGENCE"
+        echo "─────────────────────────────────────────────────────────────"
+
+        # AbuseIPDB data
+        local abuse_data=$(check_abuseipdb "$ip")
+        IFS='|' read -r confidence reports country isp <<< "$abuse_data"
+        echo "AbuseIPDB Confidence: ${confidence}%"
+        echo "Total Reports: $reports"
+        echo "Country: $country"
+        echo "ISP: $isp"
+        echo ""
+
+        # Geolocation
+        local geo=$(get_country_code "$ip")
+        echo "Geolocation: $geo"
+        if is_high_risk_country "$geo"; then
+            echo "Risk Level: HIGH (Known attack source country)"
+        else
+            echo "Risk Level: MEDIUM"
+        fi
+        echo ""
+
+        echo "─────────────────────────────────────────────────────────────"
+        echo "ATTACK HISTORY"
+        echo "─────────────────────────────────────────────────────────────"
+
+        # Get attacks from pattern log
+        local pattern_file="/var/lib/server-toolkit/attack-patterns/patterns.log"
+        if [ -f "$pattern_file" ]; then
+            echo "Recent attacks from this IP:"
+            grep "|$ip|" "$pattern_file" | tail -20 | while IFS='|' read -r ts ip_addr attack_type uri ua; do
+                echo "  [$(date -d @$ts '+%Y-%m-%d %H:%M:%S')] $attack_type - $uri"
+            done
+            echo ""
+        fi
+
+        echo "─────────────────────────────────────────────────────────────"
+        echo "RECOMMENDED ACTIONS"
+        echo "─────────────────────────────────────────────────────────────"
+
+        if [ "$confidence" -ge 75 ]; then
+            echo "• IMMEDIATE BLOCK - High confidence malicious IP"
+            echo "  Command: csf -d $ip \"AbuseIPDB: ${confidence}% confidence\""
+        elif [ "$reports" -ge 10 ]; then
+            echo "• TEMPORARY BLOCK - Multiple abuse reports"
+            echo "  Command: csf -td $ip 86400 \"Multiple abuse reports\""
+        else
+            echo "• MONITOR - Watch for continued activity"
+        fi
+
+        echo ""
+        echo "═══════════════════════════════════════════════════════════════"
+        echo "END OF REPORT"
+        echo "═══════════════════════════════════════════════════════════════"
+
+    } > "$report_file"
+
+    echo "$report_file"
+}
+
+################################################################################
+# Multi-Server Coordination (for environments with multiple servers)
+################################################################################
+
+# Share threat data with other servers (if configured)
+share_threat_data() {
+    local ip="$1"
+    local attack_type="$2"
+    local score="$3"
+
+    local coordination_file="/var/lib/server-toolkit/shared-threats.log"
+
+    # Log for potential sharing
+    echo "$(date +%s)|$(hostname)|$ip|$attack_type|$score" >> "$coordination_file"
+
+    # Keep only last 1000 entries
+    tail -1000 "$coordination_file" > "${coordination_file}.tmp" 2>/dev/null
+    mv "${coordination_file}.tmp" "$coordination_file" 2>/dev/null
+}
+
+# Check if IP is flagged by other servers
+check_shared_threats() {
+    local ip="$1"
+    local coordination_file="/var/lib/server-toolkit/shared-threats.log"
+
+    if [ -f "$coordination_file" ]; then
+        local count=$(grep "|$ip|" "$coordination_file" | wc -l)
+        echo "$count"
+    else
+        echo "0"
+    fi
+}
+
+################################################################################
+# Threat Intelligence Summary
+################################################################################
+
+# Get comprehensive threat intelligence for an IP
+get_threat_intelligence() {
+    local ip="$1"
+
+    local abuse_data=$(check_abuseipdb "$ip" 2>/dev/null || echo "0|0|Unknown|Unknown")
+    local geo=$(get_country_code "$ip" 2>/dev/null || echo "XX")
+    local timing=$(analyze_timing_pattern "$ip" 2>/dev/null || echo "unknown")
+    local whitelisted="no"
+    is_whitelisted_service "$ip" && whitelisted="yes"
+
+    # Format: abuse_confidence|abuse_reports|country|isp|timing_pattern|whitelisted
+    echo "${abuse_data}|${geo}|${timing}|${whitelisted}"
+}
+
+# Export functions for use in other scripts
+export -f check_abuseipdb
+export -f get_country_code
+export -f is_high_risk_country
+export -f is_whitelisted_service
+export -f add_to_whitelist
+export -f analyze_timing_pattern
+export -f record_attack_pattern
+export -f matches_known_pattern
+export -f get_server_load
+export -f is_server_stressed
+export -f generate_incident_report
+export -f share_threat_data
+export -f check_shared_threats
+export -f get_threat_intelligence