cschantz
31306a520f
lib/threat-intelligence.sh: - Add --max-time 10 to AbuseIPDB API curl call (line 47) tools/update-attack-signatures.sh: - Add --timeout=60 to ET Open rules download wget (line 68) tools/toolkit-qa-check.sh: - Improve NET-TIMEOUT detection to exclude false positives: * Skip comment lines * Skip echo/string statements * Skip variable assignments with pipes * Only flag actual network calls without timeouts This reduces false positive NET-TIMEOUT detections from 10 to 2. Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
467 lines
16 KiB
Bash
467 lines
16 KiB
Bash
#!/bin/bash
|
|
|
|
################################################################################
|
|
# Threat Intelligence Library
|
|
################################################################################
|
|
# Purpose: External threat intelligence integration using existing tools
|
|
# Features: IP reputation lookups, geolocation, whitelist management
|
|
# No new services - uses only existing APIs and tools
|
|
################################################################################
|
|
|
|
# Cache directory for threat intelligence
|
|
THREAT_CACHE_DIR="/tmp/server-toolkit-threat-cache"
|
|
mkdir -p "$THREAT_CACHE_DIR" 2>/dev/null
|
|
|
|
# Cache TTL (24 hours)
|
|
CACHE_TTL=86400
|
|
|
|
################################################################################
|
|
# AbuseIPDB Integration (Free API - 1000 requests/day)
|
|
################################################################################
|
|
|
|
# Check if IP is in AbuseIPDB
|
|
# Returns: confidence_score|total_reports|country|isp
|
|
check_abuseipdb() {
|
|
local ip="$1"
|
|
local cache_file="$THREAT_CACHE_DIR/abuseipdb_${ip//\./_}"
|
|
|
|
# Check cache first
|
|
if [ -f "$cache_file" ]; then
|
|
local cache_age=$(($(date +%s) - $(stat -c %Y "$cache_file" 2>/dev/null || echo 0)))
|
|
if [ "$cache_age" -lt "$CACHE_TTL" ]; then
|
|
cat "$cache_file"
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
# Check if API key exists
|
|
local api_key_file="/root/.abuseipdb_api_key"
|
|
if [ ! -f "$api_key_file" ]; then
|
|
echo "0|0|Unknown|Unknown"
|
|
return 1
|
|
fi
|
|
|
|
local api_key=$(cat "$api_key_file")
|
|
|
|
# Query AbuseIPDB API
|
|
local response=$(curl -s -G --max-time 10 https://api.abuseipdb.com/api/v2/check \
|
|
--data-urlencode "ipAddress=$ip" \
|
|
-d maxAgeInDays=90 \
|
|
-H "Key: $api_key" \
|
|
-H "Accept: application/json" 2>/dev/null)
|
|
|
|
if [ -n "$response" ]; then
|
|
local confidence=$(echo "$response" | grep -oP '"abuseConfidenceScore":\K[0-9]+' 2>/dev/null | head -1)
|
|
local reports=$(echo "$response" | grep -oP '"totalReports":\K[0-9]+' 2>/dev/null | head -1)
|
|
local country=$(echo "$response" | grep -oP '"countryCode":"\K[^"]+' 2>/dev/null | head -1)
|
|
local isp=$(echo "$response" | grep -oP '"isp":"\K[^"]+' 2>/dev/null | head -1)
|
|
|
|
local result="${confidence:-0}|${reports:-0}|${country:-Unknown}|${isp:-Unknown}"
|
|
echo "$result" | tee "$cache_file"
|
|
return 0
|
|
fi
|
|
|
|
echo "0|0|Unknown|Unknown"
|
|
return 1
|
|
}
|
|
|
|
################################################################################
|
|
# Geolocation Detection (Using existing geoiplookup or geoip-bin)
|
|
################################################################################
|
|
|
|
# Get country code for IP
|
|
get_country_code() {
|
|
local ip="$1"
|
|
local cache_file="$THREAT_CACHE_DIR/geo_${ip//\./_}"
|
|
|
|
# Check cache
|
|
if [ -f "$cache_file" ]; then
|
|
local cache_age=$(($(date +%s) - $(stat -c %Y "$cache_file" 2>/dev/null || echo 0)))
|
|
if [ "$cache_age" -lt "$CACHE_TTL" ]; then
|
|
cat "$cache_file"
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
# Try geoiplookup (if installed)
|
|
if command -v geoiplookup &>/dev/null; then
|
|
local country=$(geoiplookup "$ip" 2>/dev/null | head -1 | grep -oP 'GeoIP Country Edition: \K[A-Z]{2}')
|
|
if [ -n "$country" ]; then
|
|
echo "$country" | tee "$cache_file"
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
# Try geoip-bin (alternative)
|
|
if command -v geoiplookup6 &>/dev/null; then
|
|
local country=$(geoiplookup6 "$ip" 2>/dev/null | head -1 | grep -oP 'GeoIP Country Edition: \K[A-Z]{2}')
|
|
if [ -n "$country" ]; then
|
|
echo "$country" | tee "$cache_file"
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
# Fallback to whois (slower, not cached as aggressively)
|
|
if command -v whois &>/dev/null; then
|
|
local country=$(whois "$ip" 2>/dev/null | grep -i "^country:" | head -1 | awk '{print $2}' | tr '[:lower:]' '[:upper:]')
|
|
if [ -n "$country" ] && [ ${#country} -eq 2 ]; then
|
|
echo "$country" | tee "$cache_file"
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
echo "XX"
|
|
return 1
|
|
}
|
|
|
|
# Check if country is high-risk
|
|
is_high_risk_country() {
|
|
local country="$1"
|
|
|
|
# High-risk countries (commonly seen in attacks)
|
|
local high_risk="CN RU UA BY KP IR VN TH ID BR"
|
|
|
|
if echo "$high_risk" | grep -qw "$country"; then
|
|
return 0
|
|
fi
|
|
|
|
return 1
|
|
}
|
|
|
|
################################################################################
|
|
# Smart Whitelisting
|
|
################################################################################
|
|
|
|
# Check if IP should be whitelisted (legitimate services)
|
|
is_whitelisted_service() {
|
|
local ip="$1"
|
|
local whitelist_file="/tmp/server-toolkit-whitelist_ips.txt"
|
|
|
|
# Check static whitelist
|
|
if [ -f "$whitelist_file" ]; then
|
|
if grep -q "^$ip$" -- "$whitelist_file"; then
|
|
return 0
|
|
fi
|
|
fi
|
|
|
|
# Check if IP belongs to known legitimate networks
|
|
# Google IPs (8.8.0.0/16, 66.249.64.0/19, etc.)
|
|
if [[ "$ip" =~ ^8\.8\. ]] || [[ "$ip" =~ ^66\.249\. ]] || [[ "$ip" =~ ^66\.102\. ]]; then
|
|
return 0
|
|
fi
|
|
|
|
# Cloudflare IPs (173.245.48.0/20, 103.21.244.0/22, etc.)
|
|
if [[ "$ip" =~ ^173\.245\.(4[8-9]|5[0-9]|6[0-3])\. ]] || [[ "$ip" =~ ^103\.21\.24[4-7]\. ]]; then
|
|
return 0
|
|
fi
|
|
|
|
# Microsoft/Bing IPs (40.0.0.0/8, 65.52.0.0/14, etc.)
|
|
if [[ "$ip" =~ ^40\. ]] || [[ "$ip" =~ ^65\.5[2-5]\. ]]; then
|
|
return 0
|
|
fi
|
|
|
|
# Common CDN/monitoring services
|
|
# Akamai: 23.0.0.0/8
|
|
if [[ "$ip" =~ ^23\. ]]; then
|
|
return 0
|
|
fi
|
|
|
|
return 1
|
|
}
|
|
|
|
# Add IP to whitelist
|
|
add_to_whitelist() {
|
|
local ip="$1"
|
|
local reason="$2"
|
|
local whitelist_file="/tmp/server-toolkit-whitelist_ips.txt"
|
|
|
|
if ! grep -q "^$ip$" -- "$whitelist_file" 2>/dev/null; then
|
|
echo "$ip # $reason" >> "$whitelist_file"
|
|
fi
|
|
}
|
|
|
|
################################################################################
|
|
# Behavioral Analysis
|
|
################################################################################
|
|
|
|
# Analyze request timing pattern
|
|
# Returns: human|bot|suspicious
|
|
analyze_timing_pattern() {
|
|
local ip="$1"
|
|
local timing_file="$THREAT_CACHE_DIR/timing_${ip//\./_}"
|
|
|
|
# Record timestamp
|
|
echo "$(date +%s)" >> "$timing_file"
|
|
|
|
# Keep only last 100 requests
|
|
tail -100 "$timing_file" > "${timing_file}.tmp" 2>/dev/null
|
|
mv "${timing_file}.tmp" "$timing_file" 2>/dev/null
|
|
|
|
# Analyze if we have enough data
|
|
local request_count=$(wc -l < "$timing_file" 2>/dev/null || echo 0)
|
|
if [ "$request_count" -lt 10 ]; then
|
|
echo "unknown"
|
|
return
|
|
fi
|
|
|
|
# Calculate average time between requests
|
|
local timestamps=$(cat "$timing_file")
|
|
local total_gap=0
|
|
local gap_count=0
|
|
local prev_ts=""
|
|
|
|
while IFS= read -r ts; do
|
|
if [ -n "$prev_ts" ]; then
|
|
local gap=$((ts - prev_ts))
|
|
total_gap=$((total_gap + gap))
|
|
gap_count=$((gap_count + 1))
|
|
fi
|
|
prev_ts="$ts"
|
|
done <<< "$timestamps"
|
|
|
|
if [ "$gap_count" -gt 0 ]; then
|
|
local avg_gap=$((total_gap / gap_count))
|
|
|
|
# Bot patterns: < 2 seconds between requests consistently
|
|
if [ "$avg_gap" -lt 2 ]; then
|
|
echo "bot"
|
|
return
|
|
fi
|
|
|
|
# Human patterns: 5-30 seconds between requests with variation
|
|
if [ "$avg_gap" -ge 5 ] && [ "$avg_gap" -le 30 ]; then
|
|
echo "human"
|
|
return
|
|
fi
|
|
|
|
# Suspicious: Too fast but not consistent bot pattern
|
|
echo "suspicious"
|
|
return
|
|
fi
|
|
|
|
echo "unknown"
|
|
}
|
|
|
|
################################################################################
|
|
# Attack Pattern Learning
|
|
################################################################################
|
|
|
|
# Record attack pattern for learning
|
|
record_attack_pattern() {
|
|
local ip="$1"
|
|
local attack_type="$2"
|
|
local uri="$3"
|
|
local user_agent="$4"
|
|
|
|
local pattern_file="/tmp/server-toolkit-attack-patterns.log"
|
|
mkdir -p "$(dirname "$pattern_file")" 2>/dev/null
|
|
|
|
# Format: timestamp|ip|attack_type|uri|user_agent
|
|
echo "$(date +%s)|$ip|$attack_type|$uri|$user_agent" >> "$pattern_file"
|
|
|
|
# Keep only last 10000 patterns (prevent unbounded growth)
|
|
tail -10000 "$pattern_file" > "${pattern_file}.tmp" 2>/dev/null
|
|
mv "${pattern_file}.tmp" "$pattern_file" 2>/dev/null
|
|
}
|
|
|
|
# Check if attack matches known pattern
|
|
matches_known_pattern() {
|
|
local attack_type="$1"
|
|
local uri="$2"
|
|
|
|
local pattern_file="/tmp/server-toolkit-attack-patterns.log"
|
|
|
|
if [ ! -f "$pattern_file" ]; then
|
|
return 1
|
|
fi
|
|
|
|
# Check if this attack type + similar URI has been seen before
|
|
local similar_count=$(grep "|$attack_type|" -- "$pattern_file" | grep -c "$uri" || echo 0)
|
|
|
|
if [ "$similar_count" -ge 3 ]; then
|
|
return 0 # Known pattern
|
|
fi
|
|
|
|
return 1 # New pattern
|
|
}
|
|
|
|
################################################################################
|
|
# Performance Impact Monitoring
|
|
################################################################################
|
|
|
|
# Get current server load
|
|
get_server_load() {
|
|
# Returns: load_1min|load_5min|load_15min|cpu_count
|
|
local load=$(uptime | awk -F'load average:' '{print $2}' | sed 's/,//g' | xargs)
|
|
local cpu_count=$(nproc)
|
|
|
|
echo "${load}|${cpu_count}"
|
|
}
|
|
|
|
# Check if server is under stress
|
|
is_server_stressed() {
|
|
local load_data=$(get_server_load)
|
|
IFS='|' read -r load1 load5 load15 cpu_count <<< "$load_data"
|
|
|
|
# Remove any extra spaces
|
|
load1=$(echo "$load1" | awk '{print $1}')
|
|
|
|
# Convert to integer (multiply by 100 to handle decimals)
|
|
local load_int=$(awk "BEGIN {printf \"%.0f\", $load1 * 100}" 2>/dev/null)
|
|
local threshold=$((cpu_count * 80)) # 80% of CPU count
|
|
|
|
if [ "$load_int" -gt "$threshold" ]; then
|
|
return 0 # Server is stressed
|
|
fi
|
|
|
|
return 1
|
|
}
|
|
|
|
################################################################################
|
|
# Incident Report Generation
|
|
################################################################################
|
|
|
|
# Generate incident report for an IP
|
|
generate_incident_report() {
|
|
local ip="$1"
|
|
local report_file="/tmp/server-toolkit-incident-report_${ip//\./_}_$(date +%Y%m%d_%H%M%S).txt"
|
|
|
|
mkdir -p "$(dirname "$report_file")" 2>/dev/null
|
|
|
|
{
|
|
echo "═══════════════════════════════════════════════════════════════"
|
|
echo "SECURITY INCIDENT REPORT"
|
|
echo "═══════════════════════════════════════════════════════════════"
|
|
echo ""
|
|
echo "Generated: $(date '+%Y-%m-%d %H:%M:%S %Z')"
|
|
echo "IP Address: $ip"
|
|
echo ""
|
|
|
|
echo "─────────────────────────────────────────────────────────────"
|
|
echo "THREAT INTELLIGENCE"
|
|
echo "─────────────────────────────────────────────────────────────"
|
|
|
|
# AbuseIPDB data
|
|
local abuse_data=$(check_abuseipdb "$ip")
|
|
IFS='|' read -r confidence reports country isp <<< "$abuse_data"
|
|
echo "AbuseIPDB Confidence: ${confidence}%"
|
|
echo "Total Reports: $reports"
|
|
echo "Country: $country"
|
|
echo "ISP: $isp"
|
|
echo ""
|
|
|
|
# Geolocation
|
|
local geo=$(get_country_code "$ip")
|
|
echo "Geolocation: $geo"
|
|
if is_high_risk_country "$geo"; then
|
|
echo "Risk Level: HIGH (Known attack source country)"
|
|
else
|
|
echo "Risk Level: MEDIUM"
|
|
fi
|
|
echo ""
|
|
|
|
echo "─────────────────────────────────────────────────────────────"
|
|
echo "ATTACK HISTORY"
|
|
echo "─────────────────────────────────────────────────────────────"
|
|
|
|
# Get attacks from pattern log
|
|
local pattern_file="/tmp/server-toolkit-attack-patterns.log"
|
|
if [ -f "$pattern_file" ]; then
|
|
echo "Recent attacks from this IP:"
|
|
grep "|$ip|" -- "$pattern_file" | tail -20 | while IFS='|' read -r ts ip_addr attack_type uri ua; do
|
|
echo " [$(date -d @$ts '+%Y-%m-%d %H:%M:%S')] $attack_type - $uri"
|
|
done
|
|
echo ""
|
|
fi
|
|
|
|
echo "─────────────────────────────────────────────────────────────"
|
|
echo "RECOMMENDED ACTIONS"
|
|
echo "─────────────────────────────────────────────────────────────"
|
|
|
|
if [ "$confidence" -ge 75 ]; then
|
|
echo "• IMMEDIATE BLOCK - High confidence malicious IP"
|
|
echo " Command: csf -d $ip \"AbuseIPDB: ${confidence}% confidence\""
|
|
elif [ "$reports" -ge 10 ]; then
|
|
echo "• TEMPORARY BLOCK - Multiple abuse reports"
|
|
echo " Command: csf -td $ip 86400 \"Multiple abuse reports\""
|
|
else
|
|
echo "• MONITOR - Watch for continued activity"
|
|
fi
|
|
|
|
echo ""
|
|
echo "═══════════════════════════════════════════════════════════════"
|
|
echo "END OF REPORT"
|
|
echo "═══════════════════════════════════════════════════════════════"
|
|
|
|
} > "$report_file"
|
|
|
|
echo "$report_file"
|
|
}
|
|
|
|
################################################################################
|
|
# Multi-Server Coordination (for environments with multiple servers)
|
|
################################################################################
|
|
|
|
# Share threat data with other servers (if configured)
|
|
share_threat_data() {
|
|
local ip="$1"
|
|
local attack_type="$2"
|
|
local score="$3"
|
|
|
|
local coordination_file="/tmp/server-toolkit-shared-threats.log"
|
|
|
|
# Log for potential sharing
|
|
echo "$(date +%s)|$(hostname)|$ip|$attack_type|$score" >> "$coordination_file"
|
|
|
|
# Keep only last 1000 entries
|
|
tail -1000 "$coordination_file" > "${coordination_file}.tmp" 2>/dev/null
|
|
mv "${coordination_file}.tmp" "$coordination_file" 2>/dev/null
|
|
}
|
|
|
|
# Check if IP is flagged by other servers
|
|
check_shared_threats() {
|
|
local ip="$1"
|
|
local coordination_file="/tmp/server-toolkit-shared-threats.log"
|
|
|
|
if [ -f "$coordination_file" ]; then
|
|
local count=$(grep "|$ip|" -- "$coordination_file" | wc -l)
|
|
echo "$count"
|
|
else
|
|
echo "0"
|
|
fi
|
|
}
|
|
|
|
################################################################################
|
|
# Threat Intelligence Summary
|
|
################################################################################
|
|
|
|
# Get comprehensive threat intelligence for an IP
|
|
get_threat_intelligence() {
|
|
local ip="$1"
|
|
|
|
local abuse_data=$(check_abuseipdb "$ip" 2>/dev/null || echo "0|0|Unknown|Unknown")
|
|
local geo=$(get_country_code "$ip" 2>/dev/null || echo "XX")
|
|
local timing=$(analyze_timing_pattern "$ip" 2>/dev/null || echo "unknown")
|
|
local whitelisted="no"
|
|
is_whitelisted_service "$ip" && whitelisted="yes"
|
|
|
|
# Format: abuse_confidence|abuse_reports|country|isp|timing_pattern|whitelisted
|
|
echo "${abuse_data}|${geo}|${timing}|${whitelisted}"
|
|
}
|
|
|
|
# Export functions for use in other scripts
|
|
export -f check_abuseipdb
|
|
export -f get_country_code
|
|
export -f is_high_risk_country
|
|
export -f is_whitelisted_service
|
|
export -f add_to_whitelist
|
|
export -f analyze_timing_pattern
|
|
export -f record_attack_pattern
|
|
export -f matches_known_pattern
|
|
export -f get_server_load
|
|
export -f is_server_stressed
|
|
export -f generate_incident_report
|
|
export -f share_threat_data
|
|
export -f check_shared_threats
|
|
export -f get_threat_intelligence
|