Compare commits
15 Commits
8af1ca881b
..
dev
| Author | SHA1 | Date | |
|---|---|---|---|
| 08e8e8b5f0 | |||
| 6181da7b42 | |||
| 6a586ef721 | |||
| 43a94884e4 | |||
| da02dcfd61 | |||
| baf058d1dc | |||
| 1c3f12744b | |||
| 55dc21f6e5 | |||
| b0873bbf13 | |||
| cf362c2adf | |||
| 9471355e77 | |||
| d159dd28d8 | |||
| 01b63c6ad4 | |||
| 63e6cf067e | |||
| ca7ec62e02 |
@@ -419,7 +419,8 @@ if [ "$sent" -gt 0 ] || [ "$received" -gt 0 ]; then
|
||||
# Top recipients (delivery recipients from emails in TEMP_MATCHES)
|
||||
if [ "$sent" -gt 0 ] || [ "$delivered" -gt 0 ]; then
|
||||
print_info "Top 5 recipients (emails delivered TO):"
|
||||
grep -F "$search_pattern" "$TEMP_MATCHES" 2>/dev/null | grep -oE "=> [^@]+@[^ ]+" | sed 's/=> //' | sort | uniq -c | sort -rn | head -5 | while read count recipient; do
|
||||
sed -n "/^$search_pattern/p" "$TEMP_MATCHES" 2>/dev/null | \
|
||||
sed -n 's/.*=> \([^@]*@[^ ]*\).*/\1/p' | sort | uniq -c | sort -rn | head -5 | while read count recipient; do
|
||||
[ -n "$count" ] && echo " $recipient - $count emails"
|
||||
done
|
||||
echo ""
|
||||
@@ -428,7 +429,8 @@ if [ "$sent" -gt 0 ] || [ "$received" -gt 0 ]; then
|
||||
# Top senders (who is sending emails in TEMP_MATCHES)
|
||||
if [ "$sent" -gt 0 ]; then
|
||||
print_info "Top 5 senders (emails sent FROM):"
|
||||
grep -F "$search_pattern" "$TEMP_MATCHES" 2>/dev/null | grep -oE "<= [^@]+@[^ ]+" | sed 's/<= //' | sort | uniq -c | sort -rn | head -5 | while read count sender; do
|
||||
sed -n "/^$search_pattern/p" "$TEMP_MATCHES" 2>/dev/null | \
|
||||
sed -n 's/.*<= \([^@]*@[^ ]*\).*/\1/p' | sort | uniq -c | sort -rn | head -5 | while read count sender; do
|
||||
[ -n "$count" ] && echo " $sender - $count emails"
|
||||
done
|
||||
echo ""
|
||||
@@ -546,7 +548,7 @@ if [ "$check_type" != "2" ]; then
|
||||
|
||||
# cPanel forwarders (in /etc/valiases)
|
||||
if [ -f "/etc/valiases/$domain_part" ]; then
|
||||
forwarder=$(grep -F "^$local_part:" "/etc/valiases/$domain_part" 2>/dev/null)
|
||||
forwarder=$(grep "^${local_part}:" "/etc/valiases/$domain_part" 2>/dev/null || echo "")
|
||||
if [ -n "$forwarder" ]; then
|
||||
echo ""
|
||||
print_info "Forwarder configured:"
|
||||
@@ -650,7 +652,7 @@ if [ "$delivered" -gt 0 ]; then
|
||||
else
|
||||
echo " $line"
|
||||
fi
|
||||
done < <(grep -F "$search_pattern" "$TEMP_MATCHES" | grep -iE "=>|delivered" | tail -5)
|
||||
done < <(sed -n "/^$search_pattern/p" "$TEMP_MATCHES" 2>/dev/null | sed -n '/=>\|[Dd]elivered/p' | tail -5)
|
||||
echo ""
|
||||
fi
|
||||
|
||||
@@ -660,7 +662,7 @@ if [ "$bounced" -gt 0 ]; then
|
||||
|
||||
# Get all bounce lines (Issue 4.1: add -- after grep flags)
|
||||
TEMP_BOUNCES="/tmp/email_bounces_$$.txt"
|
||||
grep -F -- "$search_pattern" "$TEMP_MATCHES" 2>/dev/null | \
|
||||
sed -n "/^$search_pattern/p" "$TEMP_MATCHES" 2>/dev/null | \
|
||||
grep -Ev "authenticator failed|Authentication failed|saved mail to|=>" | \
|
||||
grep -iE "550|551|552|553|554|bounced|Mail delivery failed|\\*\\* " > "$TEMP_BOUNCES" 2>/dev/null
|
||||
|
||||
|
||||
@@ -40,14 +40,14 @@ if [ "$MTA" = "exim" ]; then
|
||||
print_header "Queue Summary"
|
||||
|
||||
# Exim: exim -bpc returns just the number
|
||||
queue_count=$(eval "$SYS_MAIL_CMD_QUEUE_COUNT")
|
||||
queue_count=$(bash -c "$SYS_MAIL_CMD_QUEUE_COUNT" 2>/dev/null || echo "0")
|
||||
|
||||
if [ "$queue_count" -gt 0 ] 2>/dev/null; then
|
||||
print_warning "$queue_count messages in queue"
|
||||
echo ""
|
||||
|
||||
# Cache queue list - single execution for all operations
|
||||
queue_list=$(eval "$SYS_MAIL_CMD_QUEUE_LIST")
|
||||
queue_list=$(bash -c "$SYS_MAIL_CMD_QUEUE_LIST" 2>/dev/null || echo "")
|
||||
|
||||
print_header "Recent Queue Messages (last 20)"
|
||||
echo "$queue_list" | head -20
|
||||
@@ -74,7 +74,7 @@ elif [ "$MTA" = "postfix" ]; then
|
||||
print_header "Queue Summary"
|
||||
|
||||
# Postfix: mailq | tail -1 returns "-- N Kbytes in M Requests."
|
||||
queue_summary=$(eval "$SYS_MAIL_CMD_QUEUE_COUNT")
|
||||
queue_summary=$(bash -c "$SYS_MAIL_CMD_QUEUE_COUNT" 2>/dev/null || echo "")
|
||||
print_info "$queue_summary"
|
||||
|
||||
# Extract message count from summary line (last number is always message count)
|
||||
@@ -89,7 +89,7 @@ elif [ "$MTA" = "postfix" ]; then
|
||||
echo ""
|
||||
|
||||
# Cache queue list - single execution for all operations
|
||||
queue_list=$(eval "$SYS_MAIL_CMD_QUEUE_LIST")
|
||||
queue_list=$(bash -c "$SYS_MAIL_CMD_QUEUE_LIST" 2>/dev/null || echo "")
|
||||
|
||||
print_header "Queue Details (first 50)"
|
||||
echo "$queue_list" | head -50
|
||||
@@ -116,7 +116,7 @@ elif [ "$MTA" = "sendmail" ]; then
|
||||
print_header "Queue Summary"
|
||||
|
||||
# Sendmail: mailq | tail -1 returns "-- N Kbytes in M Requests."
|
||||
queue_summary=$(eval "$SYS_MAIL_CMD_QUEUE_COUNT")
|
||||
queue_summary=$(bash -c "$SYS_MAIL_CMD_QUEUE_COUNT" 2>/dev/null || echo "")
|
||||
print_info "$queue_summary"
|
||||
|
||||
# Extract message count from summary line (last number is always message count)
|
||||
@@ -131,7 +131,7 @@ elif [ "$MTA" = "sendmail" ]; then
|
||||
echo ""
|
||||
|
||||
# Cache queue list - single execution for all operations
|
||||
queue_list=$(eval "$SYS_MAIL_CMD_QUEUE_LIST")
|
||||
queue_list=$(bash -c "$SYS_MAIL_CMD_QUEUE_LIST" 2>/dev/null || echo "")
|
||||
|
||||
print_header "Queue Details (first 50)"
|
||||
echo "$queue_list" | head -50
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
#!/bin/bash
|
||||
set -eo pipefail
|
||||
|
||||
################################################################################
|
||||
# Disk Space Analyzer (WinDirStat for Linux)
|
||||
@@ -17,6 +18,7 @@
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
source "$SCRIPT_DIR/lib/common-functions.sh"
|
||||
source "$SCRIPT_DIR/lib/system-detect.sh"
|
||||
source "$SCRIPT_DIR/lib/reference-db.sh"
|
||||
|
||||
# Require root
|
||||
if [ "$EUID" -ne 0 ]; then
|
||||
@@ -24,6 +26,9 @@ if [ "$EUID" -ne 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Ensure cache is fresh (only rebuilds if > 1 hour old)
|
||||
db_ensure_fresh 2>/dev/null || true
|
||||
|
||||
# Temp file for results
|
||||
TEMP_DIR="/tmp/disk-analysis-$$"
|
||||
mkdir -p "$TEMP_DIR"
|
||||
@@ -619,55 +624,51 @@ analyze_wordpress() {
|
||||
print_banner "WordPress Storage Analysis"
|
||||
echo ""
|
||||
|
||||
# Find WordPress installations
|
||||
# Find WordPress installations from cache (instant lookup, no filesystem scan)
|
||||
show_progress "Finding WordPress installations"
|
||||
|
||||
local wp_paths=()
|
||||
local wp_count=0
|
||||
local wp_data=""
|
||||
|
||||
# Common locations
|
||||
if [ -d "/home" ]; then
|
||||
while IFS= read -r wp_config; do
|
||||
wp_dir=$(dirname "$wp_config")
|
||||
wp_paths+=("$wp_dir")
|
||||
done < <(find /home -name "wp-config.php" -type f 2>/dev/null)
|
||||
# Get WordPress data from cache
|
||||
if command -v db_get_all_wordpress &>/dev/null; then
|
||||
wp_data=$(db_get_all_wordpress 2>/dev/null || true)
|
||||
fi
|
||||
|
||||
if [ -d "/var/www" ]; then
|
||||
while IFS= read -r wp_config; do
|
||||
wp_dir=$(dirname "$wp_config")
|
||||
wp_paths+=("$wp_dir")
|
||||
done < <(find /var/www -name "wp-config.php" -type f 2>/dev/null)
|
||||
# Count WP installations
|
||||
if [ -n "$wp_data" ]; then
|
||||
wp_count=$(echo "$wp_data" | grep -c "^WP|" || echo 0)
|
||||
fi
|
||||
|
||||
if [ ${#wp_paths[@]} -eq 0 ]; then
|
||||
if [ "$wp_count" -eq 0 ]; then
|
||||
echo -e "\r${DIM}No WordPress installations found${NC} "
|
||||
echo ""
|
||||
press_enter
|
||||
return
|
||||
fi
|
||||
|
||||
echo -e "\r${GREEN}✓${NC} Found ${#wp_paths[@]} WordPress installations "
|
||||
echo -e "\r${GREEN}✓${NC} Found ${wp_count} WordPress installations "
|
||||
echo ""
|
||||
|
||||
echo -e "${BOLD}WordPress Space Usage:${NC}"
|
||||
echo "───────────────────────────────────────────────────────────────"
|
||||
|
||||
for wp_dir in "${wp_paths[@]}"; do
|
||||
# Get domain/user from path
|
||||
domain=$(echo "$wp_dir" | awk -F'/' '{for(i=1;i<=NF;i++) if($i~/public_html|httpdocs|www/) print $(i-1)}' | tail -1)
|
||||
# Process cached WordPress data
|
||||
while IFS='|' read -r type domain path db_name db_user version plugins themes; do
|
||||
if [ "$type" = "WP" ] && [ -d "$path" ]; then
|
||||
# Calculate sizes
|
||||
total_size=$(du -sh "$path" 2>/dev/null | awk '{print $1}')
|
||||
uploads_size=$(du -sh "$path/wp-content/uploads" 2>/dev/null | awk '{print $1}')
|
||||
plugins_size=$(du -sh "$path/wp-content/plugins" 2>/dev/null | awk '{print $1}')
|
||||
cache_size=$(du -sh "$path/wp-content/cache" 2>/dev/null | awk '{print $1}')
|
||||
|
||||
# Calculate sizes
|
||||
total_size=$(du -sh "$wp_dir" 2>/dev/null | awk '{print $1}')
|
||||
uploads_size=$(du -sh "$wp_dir/wp-content/uploads" 2>/dev/null | awk '{print $1}')
|
||||
plugins_size=$(du -sh "$wp_dir/wp-content/plugins" 2>/dev/null | awk '{print $1}')
|
||||
cache_size=$(du -sh "$wp_dir/wp-content/cache" 2>/dev/null | awk '{print $1}')
|
||||
|
||||
echo -e "${BOLD}$domain${NC} ($total_size)"
|
||||
echo -e " Uploads: ${CYAN}${uploads_size:-0}${NC}"
|
||||
echo -e " Plugins: ${CYAN}${plugins_size:-0}${NC}"
|
||||
echo -e " Cache: ${CYAN}${cache_size:-0}${NC}"
|
||||
echo ""
|
||||
done
|
||||
echo -e "${BOLD}$domain${NC} ($total_size)"
|
||||
echo -e " Uploads: ${CYAN}${uploads_size:-0}${NC}"
|
||||
echo -e " Plugins: ${CYAN}${plugins_size:-0}${NC}"
|
||||
echo -e " Cache: ${CYAN}${cache_size:-0}${NC}"
|
||||
echo ""
|
||||
fi
|
||||
done <<< "$wp_data"
|
||||
|
||||
echo -e "${BOLD}Cleanup Suggestions:${NC}"
|
||||
echo " • Delete old revisions: wp post delete \$(wp post list --post_type=revision --format=ids)"
|
||||
|
||||
@@ -15,6 +15,9 @@ source "$TOOLKIT_ROOT/lib/reference-db.sh"
|
||||
# Initialize system detection
|
||||
detect_system
|
||||
|
||||
# Ensure reference database is fresh (only rebuild if > 1 hour old)
|
||||
db_ensure_fresh 2>/dev/null || true
|
||||
|
||||
# Load system info from reference database
|
||||
if [ -f "$TOOLKIT_ROOT/.sysref" ]; then
|
||||
SYS_HOSTNAME=$(grep "^SYS|HOSTNAME|" "$TOOLKIT_ROOT/.sysref" 2>/dev/null | cut -d'|' -f3)
|
||||
|
||||
@@ -15,6 +15,9 @@ source "$TOOLKIT_ROOT/lib/reference-db.sh"
|
||||
# Initialize system detection
|
||||
detect_system
|
||||
|
||||
# Ensure reference database is fresh (only rebuild if > 1 hour old)
|
||||
db_ensure_fresh 2>/dev/null || true
|
||||
|
||||
# Load system info from reference database
|
||||
if [ -f "$TOOLKIT_ROOT/.sysref" ]; then
|
||||
SYS_HOSTNAME=$(grep "^SYS|HOSTNAME|" "$TOOLKIT_ROOT/.sysref" 2>/dev/null | cut -d'|' -f3)
|
||||
|
||||
@@ -31,6 +31,9 @@ if [ "$EUID" -ne 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Ensure reference database is fresh (only rebuild if > 1 hour old)
|
||||
db_ensure_fresh 2>/dev/null || true
|
||||
|
||||
# Configuration
|
||||
BACKUP_DIR="/root/nginx-varnish-backups"
|
||||
VARNISH_VCL="/etc/varnish/default.vcl"
|
||||
@@ -149,11 +152,28 @@ create_backup() {
|
||||
echo "$backup_path"
|
||||
}
|
||||
|
||||
# Get list of cPanel domains
|
||||
# Get list of cPanel domains (from launcher cache, not filesystem)
|
||||
get_cpanel_domains() {
|
||||
# Use launcher's cached domain list (instant lookup, already filtered by launcher)
|
||||
# Fallback to filesystem scan only if cache unavailable
|
||||
|
||||
if command -v db_get_all_domains &>/dev/null; then
|
||||
# Use cached data from launcher (built on startup, instant O(n) lookup)
|
||||
db_get_all_domains 2>/dev/null || {
|
||||
# Fallback if cache fails (shouldn't happen if db_ensure_fresh was called)
|
||||
get_cpanel_domains_fallback
|
||||
}
|
||||
else
|
||||
# Library not available, use filesystem fallback
|
||||
get_cpanel_domains_fallback
|
||||
fi
|
||||
}
|
||||
|
||||
# Fallback domain discovery (only used if cache unavailable)
|
||||
get_cpanel_domains_fallback() {
|
||||
local domains=()
|
||||
|
||||
# Get domains from cPanel user data
|
||||
# Fallback: Get domains from cPanel user data
|
||||
if [ -d /var/cpanel/userdata ]; then
|
||||
while IFS= read -r domain_file; do
|
||||
local domain=$(basename "$domain_file")
|
||||
|
||||
+246
-223
@@ -683,9 +683,9 @@ save_baseline() {
|
||||
local baseline_file="$BASELINE_DIR/${domain}_baseline.txt"
|
||||
|
||||
# Get domain-specific metrics
|
||||
local domain_requests=$(grep "^[^|]*|$domain|" "$TEMP_DIR/parsed_logs.txt" 2>/dev/null | wc -l || echo "0")
|
||||
local domain_attacks=$(grep "^[^|]*|$domain|" "$TEMP_DIR/attack_vectors_raw.txt" 2>/dev/null | wc -l || echo "0")
|
||||
local domain_bots=$(grep "^[^|]*|$domain|" "$TEMP_DIR/classified_bots.txt" 2>/dev/null | wc -l || echo "0")
|
||||
local domain_requests=$(grep -F "|$domain|" "$TEMP_DIR/parsed_logs.txt" 2>/dev/null | wc -l || echo "0")
|
||||
local domain_attacks=$(grep -F "|$domain|" "$TEMP_DIR/attack_vectors_raw.txt" 2>/dev/null | wc -l || echo "0")
|
||||
local domain_bots=$(grep -F "|$domain|" "$TEMP_DIR/classified_bots.txt" 2>/dev/null | wc -l || echo "0")
|
||||
|
||||
# Append to baseline history (timestamp|requests|attacks|bots|high_risk_ips)
|
||||
echo "$today|$domain_requests|$domain_attacks|$domain_bots|$high_risk_ips" >> "$baseline_file"
|
||||
@@ -747,7 +747,7 @@ analyze_attack_progression() {
|
||||
> "$progression_file"
|
||||
|
||||
# Extract all requests from this IP, in order
|
||||
grep "^$ip|" "$TEMP_DIR/parsed_logs.txt" 2>/dev/null | awk -F'|' '{
|
||||
grep -F "$ip|" "$TEMP_DIR/parsed_logs.txt" 2>/dev/null | awk -F'|' '{
|
||||
print $8 "|" $3 "|" $4 "|" $6
|
||||
}' | sort >> "$progression_file"
|
||||
|
||||
@@ -1036,7 +1036,7 @@ detect_threats() {
|
||||
|
||||
# Breakdown by attack type
|
||||
for attack_type in sqli xss path_traversal rce_upload info_disclosure login_bruteforce; do
|
||||
grep "|$attack_type$" "$TEMP_DIR/attack_vectors_raw.txt" 2>/dev/null | \
|
||||
grep -F "|$attack_type|" "$TEMP_DIR/attack_vectors_raw.txt" 2>/dev/null | \
|
||||
awk -F'|' '{print $1"|"$2"|"$3"|"$4}' | \
|
||||
sort | uniq -c | sort -rn > "$TEMP_DIR/${attack_type}_attempts.txt" || true
|
||||
done
|
||||
@@ -1219,6 +1219,7 @@ calculate_bot_fingerprint() {
|
||||
awk -F'|' -v tmpdir="$TEMP_DIR" '
|
||||
BEGIN {
|
||||
# Initialize tracking arrays
|
||||
fingerprint_file = tmpdir "/bot_fingerprints.txt"
|
||||
}
|
||||
{
|
||||
ip = $1
|
||||
@@ -1306,12 +1307,12 @@ calculate_bot_fingerprint() {
|
||||
|
||||
# Output fingerprint for high-confidence bots (score >= 60)
|
||||
if (score >= 60) {
|
||||
printf "%s|%d|%d\n", ip, score, signal_count > tmpdir "/bot_fingerprints.txt"
|
||||
printf "%s|%d|%d\n", ip, score, signal_count > fingerprint_file
|
||||
}
|
||||
}
|
||||
close(tmpdir "/bot_fingerprints.txt")
|
||||
close(fingerprint_file)
|
||||
}
|
||||
' < "$TEMP_DIR/parsed_logs.txt"
|
||||
' < "$TEMP_DIR/parsed_logs.txt" 2>/dev/null || true
|
||||
|
||||
# Create file if empty
|
||||
touch "$TEMP_DIR/bot_fingerprints.txt"
|
||||
@@ -1356,7 +1357,7 @@ analyze_domain_targeting_percentage() {
|
||||
# Also create per-domain attack type breakdown
|
||||
# Format: domain|attack_type|ip|count
|
||||
if [ -f "$TEMP_DIR/attack_vectors_raw.txt" ]; then
|
||||
awk -F'|' '
|
||||
awk -F'|' -v tmpdir="$TEMP_DIR" '
|
||||
{
|
||||
ip = $1
|
||||
domain = $2
|
||||
@@ -1368,7 +1369,6 @@ analyze_domain_targeting_percentage() {
|
||||
}
|
||||
END {
|
||||
for (domain in attack_data) {
|
||||
domain_file = tmpdir "/domain_attacks_" domain ".txt"
|
||||
for (attack_type in attack_data[domain]) {
|
||||
total = attack_totals[domain][attack_type]
|
||||
for (ip in attack_data[domain][attack_type]) {
|
||||
@@ -1378,7 +1378,7 @@ analyze_domain_targeting_percentage() {
|
||||
}
|
||||
}
|
||||
}
|
||||
' -v tmpdir="$TEMP_DIR" < "$TEMP_DIR/attack_vectors_raw.txt"
|
||||
' < "$TEMP_DIR/attack_vectors_raw.txt"
|
||||
fi
|
||||
|
||||
print_success "Domain attack pattern analysis complete"
|
||||
@@ -1608,11 +1608,9 @@ is_excluded_ip() {
|
||||
return 0 # True - should be excluded
|
||||
fi
|
||||
|
||||
# Check if it's the server's own IP
|
||||
if [ -f "$TEMP_DIR/server_ips.txt" ]; then
|
||||
if grep -qFx "$ip" "$TEMP_DIR/server_ips.txt" 2>/dev/null; then
|
||||
return 0 # True - should be excluded
|
||||
fi
|
||||
# Check if it's the server's own IP (using pre-loaded array for speed)
|
||||
if [ -n "${server_ips_array[$ip]}" ]; then
|
||||
return 0 # True - should be excluded
|
||||
fi
|
||||
|
||||
return 1 # False - should not be excluded
|
||||
@@ -1626,13 +1624,13 @@ analyze_time_series() {
|
||||
print_info "Analyzing time-series patterns..."
|
||||
|
||||
# Extract hourly bot traffic
|
||||
cat "$TEMP_DIR/classified_bots.txt" 2>/dev/null | awk -F'|' '$9 != "unknown" {
|
||||
awk -F'|' '$9 != "unknown" {
|
||||
timestamp = $8
|
||||
if (match(timestamp, /([0-9]{2})\/([A-Za-z]{3})\/([0-9]{4}):([0-9]{2}):([0-9]{2}):([0-9]{2})/, ts)) {
|
||||
hour = ts[4]
|
||||
print hour
|
||||
}
|
||||
}' | sort | uniq -c > "$TEMP_DIR/hourly_bot_traffic.txt" || true
|
||||
}' "$TEMP_DIR/classified_bots.txt" 2>/dev/null | sort | uniq -c > "$TEMP_DIR/hourly_bot_traffic.txt" || true
|
||||
|
||||
# Extract hourly attack traffic
|
||||
if [ -f "$TEMP_DIR/attack_vectors_raw.txt" ]; then
|
||||
@@ -1656,54 +1654,78 @@ analyze_time_series() {
|
||||
calculate_threat_scores() {
|
||||
print_info "Calculating threat scores..."
|
||||
|
||||
# Pre-count requests per IP (MUCH faster than grepping for each IP)
|
||||
# Pre-load server IPs for fast exclusion checking (avoids grep in loop)
|
||||
declare -A server_ips_array
|
||||
if [ -f "$TEMP_DIR/server_ips.txt" ]; then
|
||||
mapfile -t server_ips_list < "$TEMP_DIR/server_ips.txt" 2>/dev/null || true
|
||||
for ip in "${server_ips_list[@]:-}"; do
|
||||
[ -n "$ip" ] && server_ips_array["$ip"]=1
|
||||
done
|
||||
fi
|
||||
|
||||
# Pre-count requests per IP using mapfile (faster than while-read on large files)
|
||||
declare -A ip_request_counts
|
||||
while IFS='|' read -r ip rest; do
|
||||
((ip_request_counts["$ip"]++))
|
||||
done < <(cat "$TEMP_DIR/parsed_logs.txt")
|
||||
if [ -f "$TEMP_DIR/parsed_logs.txt" ]; then
|
||||
mapfile -t parsed_lines < "$TEMP_DIR/parsed_logs.txt" 2>/dev/null || true
|
||||
for line in "${parsed_lines[@]:-}"; do
|
||||
ip="${line%%|*}"
|
||||
[ -n "$ip" ] && ((ip_request_counts["$ip"]++)) || true
|
||||
done
|
||||
fi
|
||||
|
||||
# Build hash tables from threat files for O(1) lookups
|
||||
# OPTIMIZATION: Use awk instead of echo|awk|cut in loops (10x faster)
|
||||
declare -A threat_ips_sqli threat_ips_xss threat_ips_path threat_ips_rce threat_ips_login
|
||||
declare -A threat_ips_suspicious threat_ips_ddos threat_admin_count threat_404_count
|
||||
|
||||
# Parse each threat file and build hash tables (optimized with awk)
|
||||
[ -f "$TEMP_DIR/sqli_attempts.txt" ] && while read -r ip; do
|
||||
threat_ips_sqli["$ip"]=1
|
||||
done < <(awk '{print $2}' "$TEMP_DIR/sqli_attempts.txt" | cut -d'|' -f1)
|
||||
# Parse each threat file and build hash tables (using mapfile to avoid subshells)
|
||||
if [ -f "$TEMP_DIR/sqli_attempts.txt" ]; then
|
||||
mapfile -t sqli_ips < <(awk '{print $2}' "$TEMP_DIR/sqli_attempts.txt" 2>/dev/null | cut -d'|' -f1) || true
|
||||
for ip in "${sqli_ips[@]:-}"; do [ -n "$ip" ] && threat_ips_sqli["$ip"]=1; done
|
||||
fi
|
||||
|
||||
[ -f "$TEMP_DIR/xss_attempts.txt" ] && while read -r ip; do
|
||||
threat_ips_xss["$ip"]=1
|
||||
done < <(awk '{print $2}' "$TEMP_DIR/xss_attempts.txt" | cut -d'|' -f1)
|
||||
if [ -f "$TEMP_DIR/xss_attempts.txt" ]; then
|
||||
mapfile -t xss_ips < <(awk '{print $2}' "$TEMP_DIR/xss_attempts.txt" 2>/dev/null | cut -d'|' -f1) || true
|
||||
for ip in "${xss_ips[@]:-}"; do [ -n "$ip" ] && threat_ips_xss["$ip"]=1; done
|
||||
fi
|
||||
|
||||
[ -f "$TEMP_DIR/path_traversal_attempts.txt" ] && while read -r ip; do
|
||||
threat_ips_path["$ip"]=1
|
||||
done < <(awk '{print $2}' "$TEMP_DIR/path_traversal_attempts.txt" | cut -d'|' -f1)
|
||||
if [ -f "$TEMP_DIR/path_traversal_attempts.txt" ]; then
|
||||
mapfile -t path_ips < <(awk '{print $2}' "$TEMP_DIR/path_traversal_attempts.txt" 2>/dev/null | cut -d'|' -f1) || true
|
||||
for ip in "${path_ips[@]:-}"; do [ -n "$ip" ] && threat_ips_path["$ip"]=1; done
|
||||
fi
|
||||
|
||||
[ -f "$TEMP_DIR/rce_upload_attempts.txt" ] && while read -r ip; do
|
||||
threat_ips_rce["$ip"]=1
|
||||
done < <(awk '{print $2}' "$TEMP_DIR/rce_upload_attempts.txt" | cut -d'|' -f1)
|
||||
if [ -f "$TEMP_DIR/rce_upload_attempts.txt" ]; then
|
||||
mapfile -t rce_ips < <(awk '{print $2}' "$TEMP_DIR/rce_upload_attempts.txt" 2>/dev/null | cut -d'|' -f1) || true
|
||||
for ip in "${rce_ips[@]:-}"; do [ -n "$ip" ] && threat_ips_rce["$ip"]=1; done
|
||||
fi
|
||||
|
||||
[ -f "$TEMP_DIR/login_bruteforce_attempts.txt" ] && while read -r ip; do
|
||||
threat_ips_login["$ip"]=1
|
||||
done < <(awk '{print $2}' "$TEMP_DIR/login_bruteforce_attempts.txt" | cut -d'|' -f1)
|
||||
if [ -f "$TEMP_DIR/login_bruteforce_attempts.txt" ]; then
|
||||
mapfile -t login_ips < <(awk '{print $2}' "$TEMP_DIR/login_bruteforce_attempts.txt" 2>/dev/null | cut -d'|' -f1) || true
|
||||
for ip in "${login_ips[@]:-}"; do [ -n "$ip" ] && threat_ips_login["$ip"]=1; done
|
||||
fi
|
||||
|
||||
[ -f "$TEMP_DIR/suspicious_ua.txt" ] && while read -r ip; do
|
||||
threat_ips_suspicious["$ip"]=1
|
||||
done < <(awk '{print $2}' "$TEMP_DIR/suspicious_ua.txt" | cut -d'|' -f1)
|
||||
if [ -f "$TEMP_DIR/suspicious_ua.txt" ]; then
|
||||
mapfile -t susp_ips < <(awk '{print $2}' "$TEMP_DIR/suspicious_ua.txt" 2>/dev/null | cut -d'|' -f1) || true
|
||||
for ip in "${susp_ips[@]:-}"; do [ -n "$ip" ] && threat_ips_suspicious["$ip"]=1; done
|
||||
fi
|
||||
|
||||
[ -f "$TEMP_DIR/rapid_fire_ips.txt" ] && while read -r ip; do
|
||||
threat_ips_ddos["$ip"]=1
|
||||
done < <(awk '{print $2}' "$TEMP_DIR/rapid_fire_ips.txt")
|
||||
if [ -f "$TEMP_DIR/rapid_fire_ips.txt" ]; then
|
||||
mapfile -t ddos_ips < <(awk '{print $2}' "$TEMP_DIR/rapid_fire_ips.txt" 2>/dev/null) || true
|
||||
for ip in "${ddos_ips[@]:-}"; do [ -n "$ip" ] && threat_ips_ddos["$ip"]=1; done
|
||||
fi
|
||||
|
||||
# Parse count-based threat files
|
||||
[ -f "$TEMP_DIR/admin_probes.txt" ] && while read -r count ip; do
|
||||
[ -n "$ip" ] && threat_admin_count["$ip"]=$count
|
||||
done < <(awk '{print $1, $2}' "$TEMP_DIR/admin_probes.txt" | sed 's/|.*//')
|
||||
if [ -f "$TEMP_DIR/admin_probes.txt" ]; then
|
||||
while IFS=' ' read -r count ip rest; do
|
||||
[ -n "$ip" ] && threat_admin_count["$ip"]=$count
|
||||
done < <(awk '{print $1, $2}' "$TEMP_DIR/admin_probes.txt" 2>/dev/null || true)
|
||||
fi
|
||||
|
||||
[ -f "$TEMP_DIR/404_scans.txt" ] && while read -r count ip; do
|
||||
[ -n "$ip" ] && threat_404_count["$ip"]=$count
|
||||
done < <(awk '{print $1, $2}' "$TEMP_DIR/404_scans.txt" | sed 's/|.*//')
|
||||
if [ -f "$TEMP_DIR/404_scans.txt" ]; then
|
||||
while IFS=' ' read -r count ip rest; do
|
||||
[ -n "$ip" ] && threat_404_count["$ip"]=$count
|
||||
done < <(awk '{print $1, $2}' "$TEMP_DIR/404_scans.txt" 2>/dev/null || true)
|
||||
fi
|
||||
|
||||
# NEW: Load bot classifications to skip volume scoring for legitimate bots
|
||||
declare -A legit_bot_ips
|
||||
@@ -1712,50 +1734,67 @@ calculate_threat_scores() {
|
||||
if [ "$bot_type" = "legit" ]; then
|
||||
legit_bot_ips["$ip"]=1
|
||||
fi
|
||||
done < "$TEMP_DIR/classified_bots.txt"
|
||||
done < "$TEMP_DIR/classified_bots.txt" || true
|
||||
fi
|
||||
|
||||
# NEW: Load success rate data for scanning/scraping detection
|
||||
declare -A scanner_ips scraper_ips ip_fail_rates
|
||||
[ -f "$TEMP_DIR/high_failure_ips.txt" ] && while IFS='|' read -r ip total fail_rate category; do
|
||||
scanner_ips["$ip"]=$fail_rate
|
||||
done < "$TEMP_DIR/high_failure_ips.txt"
|
||||
if [ -f "$TEMP_DIR/high_failure_ips.txt" ]; then
|
||||
while IFS='|' read -r ip total fail_rate category; do
|
||||
[ -n "$ip" ] && scanner_ips["$ip"]=$fail_rate
|
||||
done < "$TEMP_DIR/high_failure_ips.txt" || true
|
||||
fi
|
||||
|
||||
[ -f "$TEMP_DIR/high_success_ips.txt" ] && while IFS='|' read -r ip total success_rate category; do
|
||||
scraper_ips["$ip"]=$success_rate
|
||||
done < "$TEMP_DIR/high_success_ips.txt"
|
||||
if [ -f "$TEMP_DIR/high_success_ips.txt" ]; then
|
||||
while IFS='|' read -r ip total success_rate category; do
|
||||
[ -n "$ip" ] && scraper_ips["$ip"]=$success_rate
|
||||
done < "$TEMP_DIR/high_success_ips.txt" || true
|
||||
fi
|
||||
|
||||
# Load all fail rates for threshold checks
|
||||
[ -f "$TEMP_DIR/ip_success_rates.txt" ] && while IFS='|' read -r ip total success_rate fail_rate; do
|
||||
ip_fail_rates["$ip"]=$fail_rate
|
||||
done < "$TEMP_DIR/ip_success_rates.txt"
|
||||
if [ -f "$TEMP_DIR/ip_success_rates.txt" ]; then
|
||||
while IFS='|' read -r ip total success_rate fail_rate; do
|
||||
[ -n "$ip" ] && ip_fail_rates["$ip"]=$fail_rate
|
||||
done < "$TEMP_DIR/ip_success_rates.txt" || true
|
||||
fi
|
||||
|
||||
# NEW: Load header anomalies
|
||||
declare -A header_anomalies
|
||||
[ -f "$TEMP_DIR/header_anomalies.txt" ] && while IFS='|' read -r ip anomaly_type score; do
|
||||
header_anomalies["$ip"]=$score
|
||||
done < "$TEMP_DIR/header_anomalies.txt"
|
||||
if [ -f "$TEMP_DIR/header_anomalies.txt" ]; then
|
||||
while IFS='|' read -r ip anomaly_type score; do
|
||||
[ -n "$ip" ] && header_anomalies["$ip"]=$score
|
||||
done < "$TEMP_DIR/header_anomalies.txt" || true
|
||||
fi
|
||||
|
||||
# NEW: Load suspicious entry points
|
||||
declare -A suspicious_entry_ips
|
||||
[ -f "$TEMP_DIR/suspicious_entry_points.txt" ] && while IFS='|' read -r ip entry_type url status; do
|
||||
suspicious_entry_ips["$ip"]=1
|
||||
done < "$TEMP_DIR/suspicious_entry_points.txt"
|
||||
if [ -f "$TEMP_DIR/suspicious_entry_points.txt" ]; then
|
||||
while IFS='|' read -r ip entry_type url status; do
|
||||
[ -n "$ip" ] && suspicious_entry_ips["$ip"]=1
|
||||
done < "$TEMP_DIR/suspicious_entry_points.txt" || true
|
||||
fi
|
||||
|
||||
# NEW: Load fuzzing/parameter scanning IPs
|
||||
declare -A fuzzing_ips
|
||||
[ -f "$TEMP_DIR/fuzzing_ips.txt" ] && while IFS='|' read -r ip fuzz_type total_urls unique_paths; do
|
||||
fuzzing_ips["$ip"]=$total_urls
|
||||
done < "$TEMP_DIR/fuzzing_ips.txt"
|
||||
if [ -f "$TEMP_DIR/fuzzing_ips.txt" ]; then
|
||||
while IFS='|' read -r ip fuzz_type total_urls unique_paths; do
|
||||
[ -n "$ip" ] && fuzzing_ips["$ip"]=$total_urls
|
||||
done < "$TEMP_DIR/fuzzing_ips.txt" || true
|
||||
fi
|
||||
|
||||
# NEW: Load timing anomalies (consistent bot timing)
|
||||
declare -A timing_anomalies
|
||||
[ -f "$TEMP_DIR/timing_anomalies.txt" ] && while IFS='|' read -r ip timing_type avg_interval total_reqs; do
|
||||
timing_anomalies["$ip"]=$avg_interval
|
||||
done < "$TEMP_DIR/timing_anomalies.txt"
|
||||
if [ -f "$TEMP_DIR/timing_anomalies.txt" ]; then
|
||||
while IFS='|' read -r ip timing_type avg_interval total_reqs; do
|
||||
[ -n "$ip" ] && timing_anomalies["$ip"]=$avg_interval
|
||||
done < "$TEMP_DIR/timing_anomalies.txt" || true
|
||||
fi
|
||||
|
||||
# Now calculate scores for each IP (using pre-counted requests)
|
||||
local ip_count=0
|
||||
for ip in "${!ip_request_counts[@]}"; do
|
||||
((ip_count++)) || true
|
||||
|
||||
# Skip excluded IPs
|
||||
if is_excluded_ip "$ip"; then
|
||||
continue
|
||||
@@ -1896,11 +1935,15 @@ calculate_threat_scores() {
|
||||
[ -n "${threat_ips_suspicious[$ip]}" ] && flag_ip_attack "$ip" "SCANNER" 0 "Bot analyzer: Suspicious user-agent" >/dev/null 2>&1
|
||||
) &
|
||||
fi
|
||||
done | sort -t'|' -k1 -rn > "$TEMP_DIR/threat_scores.txt"
|
||||
done > "$TEMP_DIR/threat_scores_unsorted.txt"
|
||||
|
||||
# Wait for background IP reputation updates to complete (don't fail if background jobs error)
|
||||
wait || true
|
||||
|
||||
# Sort the threat scores after all background jobs are done
|
||||
sort -t'|' -k1 -rn "$TEMP_DIR/threat_scores_unsorted.txt" > "$TEMP_DIR/threat_scores.txt" || true
|
||||
rm -f "$TEMP_DIR/threat_scores_unsorted.txt"
|
||||
|
||||
print_success "Threat scores calculated and IP reputation updated"
|
||||
}
|
||||
|
||||
@@ -1912,7 +1955,7 @@ detect_false_positives() {
|
||||
print_info "Detecting legitimate services (false positives)..."
|
||||
|
||||
# Known monitoring service patterns and legitimate CDNs
|
||||
cat "$TEMP_DIR/parsed_logs.txt" 2>/dev/null | awk -F'|' '{
|
||||
awk -F'|' '{
|
||||
ip = $1
|
||||
domain = $2
|
||||
url = $3
|
||||
@@ -1952,7 +1995,7 @@ detect_false_positives() {
|
||||
else if (match(url, /checkout|payment|paypal|stripe|square/) && match(ua, /paypal|stripe|square/)) {
|
||||
print ip "|Payment Processor|" ua "|" domain
|
||||
}
|
||||
}' | sort -u > "$TEMP_DIR/false_positives.txt" || true
|
||||
}' "$TEMP_DIR/parsed_logs.txt" 2>/dev/null | sort -u > "$TEMP_DIR/false_positives.txt" || true
|
||||
|
||||
print_success "False positive detection complete ($(wc -l < "$TEMP_DIR/false_positives.txt" 2>/dev/null || echo 0) legitimate services identified)"
|
||||
}
|
||||
@@ -1966,7 +2009,7 @@ generate_statistics() {
|
||||
|
||||
# OPTIMIZATION: Use single-pass AWK to generate multiple stats from parsed logs
|
||||
# This reads the uncompressed file ONCE instead of 4+ separate reads
|
||||
cat "$TEMP_DIR/parsed_logs.txt" 2>/dev/null | awk -F'|' -v tmpdir="$TEMP_DIR" '
|
||||
awk -F'|' -v tmpdir="$TEMP_DIR" '
|
||||
{
|
||||
# Count by domain (for top sites)
|
||||
domains[$2]++
|
||||
@@ -1995,29 +2038,29 @@ generate_statistics() {
|
||||
close(tmpdir "/top_sites_raw.txt")
|
||||
close(tmpdir "/top_ips_raw.txt")
|
||||
close(tmpdir "/top_urls_raw.txt")
|
||||
}'
|
||||
}' "$TEMP_DIR/parsed_logs.txt" 2>/dev/null
|
||||
|
||||
# Sort and limit results
|
||||
sort -rn "$TEMP_DIR/top_sites_raw.txt" | head -5 > "$TEMP_DIR/top_sites.txt"
|
||||
sort -rn "$TEMP_DIR/top_ips_raw.txt" | head -5 > "$TEMP_DIR/top_ips.txt"
|
||||
sort -rn "$TEMP_DIR/top_urls_raw.txt" | head -5 > "$TEMP_DIR/top_urls.txt"
|
||||
# Sort and limit results (files may not exist if no data)
|
||||
[ -f "$TEMP_DIR/top_sites_raw.txt" ] && sort -rn "$TEMP_DIR/top_sites_raw.txt" | head -5 > "$TEMP_DIR/top_sites.txt" || touch "$TEMP_DIR/top_sites.txt"
|
||||
[ -f "$TEMP_DIR/top_ips_raw.txt" ] && sort -rn "$TEMP_DIR/top_ips_raw.txt" | head -5 > "$TEMP_DIR/top_ips.txt" || touch "$TEMP_DIR/top_ips.txt"
|
||||
[ -f "$TEMP_DIR/top_urls_raw.txt" ] && sort -rn "$TEMP_DIR/top_urls_raw.txt" | head -5 > "$TEMP_DIR/top_urls.txt" || touch "$TEMP_DIR/top_urls.txt"
|
||||
|
||||
# Top 5 bots by request count (single decompression)
|
||||
cat "$TEMP_DIR/classified_bots.txt" 2>/dev/null | awk -F'|' '$9 != "unknown" {print $10}' | \
|
||||
awk -F'|' '$9 != "unknown" {print $10}' "$TEMP_DIR/classified_bots.txt" 2>/dev/null | \
|
||||
sort | uniq -c | sort -rn | head -5 > "$TEMP_DIR/top_bots.txt" || true
|
||||
|
||||
# Traffic breakdown by bot type (single decompression)
|
||||
cat "$TEMP_DIR/classified_bots.txt" 2>/dev/null | awk -F'|' '{print $9}' | \
|
||||
awk -F'|' '{print $9}' "$TEMP_DIR/classified_bots.txt" 2>/dev/null | \
|
||||
sort | uniq -c | sort -rn > "$TEMP_DIR/traffic_breakdown.txt" || true
|
||||
|
||||
# Per-domain traffic sources (OPTIMIZED: read uncompressed file once, use grep)
|
||||
if [ -f "$TEMP_DIR/all_domains.txt" ]; then
|
||||
# Create indexed bot traffic file (decompress once)
|
||||
cat "$TEMP_DIR/classified_bots.txt" 2>/dev/null | awk -F'|' '{print $2"|"$9}' > "$TEMP_DIR/domain_bot_types.txt" || true
|
||||
awk -F'|' '{print $2"|"$9}' "$TEMP_DIR/classified_bots.txt" 2>/dev/null > "$TEMP_DIR/domain_bot_types.txt" || true
|
||||
|
||||
while read -r domain; do
|
||||
echo "$domain" > "$TEMP_DIR/domain_${domain}_stats.txt"
|
||||
grep "^$domain|" "$TEMP_DIR/domain_bot_types.txt" 2>/dev/null | cut -d'|' -f2 | \
|
||||
grep -F "$domain|" "$TEMP_DIR/domain_bot_types.txt" 2>/dev/null | cut -d'|' -f2 | \
|
||||
sort | uniq -c | sort -rn >> "$TEMP_DIR/domain_${domain}_stats.txt" || true
|
||||
done < "$TEMP_DIR/all_domains.txt"
|
||||
fi
|
||||
@@ -2070,7 +2113,7 @@ generate_comparison_report() {
|
||||
echo " Baseline (7-day avg): $baseline_requests requests"
|
||||
echo " Today: $total_requests requests"
|
||||
elif [ "$request_pct" -lt 50 ]; then
|
||||
echo "🟢 LOW: Requests are $(($((100 - $request_pct))))% below baseline"
|
||||
echo "🟢 LOW: Requests are $((100 - $request_pct))% below baseline"
|
||||
else
|
||||
echo "🟡 NORMAL: Requests within expected range"
|
||||
fi
|
||||
@@ -2287,19 +2330,27 @@ generate_report() {
|
||||
# QUICK STATS DASHBOARD
|
||||
print_header "QUICK STATS DASHBOARD"
|
||||
|
||||
total_requests=$(wc -l < "$TEMP_DIR/parsed_logs.txt")
|
||||
unique_ips=$(awk -F'|' '{print $1}' < "$TEMP_DIR/parsed_logs.txt" | sort -u | wc -l)
|
||||
unique_domains=$(awk -F'|' '{print $2}' < "$TEMP_DIR/parsed_logs.txt" | sort -u | wc -l)
|
||||
bot_requests=$(awk -F'|' '$9 != "unknown"' < "$TEMP_DIR/classified_bots.txt" | wc -l)
|
||||
total_requests=$(wc -l < "$TEMP_DIR/parsed_logs.txt" 2>/dev/null || echo 0)
|
||||
total_requests=${total_requests:-0}
|
||||
|
||||
unique_ips=$(awk -F'|' '{print $1}' < "$TEMP_DIR/parsed_logs.txt" 2>/dev/null | sort -u | wc -l || echo 0)
|
||||
unique_ips=${unique_ips:-0}
|
||||
|
||||
unique_domains=$(awk -F'|' '{print $2}' < "$TEMP_DIR/parsed_logs.txt" 2>/dev/null | sort -u | wc -l || echo 0)
|
||||
unique_domains=${unique_domains:-0}
|
||||
|
||||
bot_requests=$(awk -F'|' '$9 != "unknown"' < "$TEMP_DIR/classified_bots.txt" 2>/dev/null | wc -l || echo 0)
|
||||
bot_requests=${bot_requests:-0}
|
||||
|
||||
# Count private/internal IPs (excluded from threat analysis)
|
||||
private_ips=$(awk -F'|' '{print $1}' < "$TEMP_DIR/parsed_logs.txt" | sort -u | grep -E '^(127\.|10\.|192\.168\.|172\.(1[6-9]|2[0-9]|3[01])\.|169\.254\.)' || true | wc -l)
|
||||
private_ips=$(awk -F'|' '{print $1}' < "$TEMP_DIR/parsed_logs.txt" 2>/dev/null | sort -u | grep -E '^(127\.|10\.|192\.168\.|172\.(1[6-9]|2[0-9]|3[01])\.|169\.254\.)' 2>/dev/null | wc -l || echo 0)
|
||||
private_ips=${private_ips:-0}
|
||||
|
||||
# Count server's own IPs in the logs
|
||||
server_ip_hits=0
|
||||
if [ -f "$TEMP_DIR/server_ips.txt" ] && [ -s "$TEMP_DIR/server_ips.txt" ]; then
|
||||
while read -r server_ip; do
|
||||
if cat "$TEMP_DIR/parsed_logs.txt" | grep -q "^$server_ip|" 2>/dev/null; then
|
||||
if grep -q "^$server_ip|" "$TEMP_DIR/parsed_logs.txt" 2>/dev/null; then
|
||||
server_ip_hits=$((server_ip_hits + 1))
|
||||
fi
|
||||
done < "$TEMP_DIR/server_ips.txt"
|
||||
@@ -2333,9 +2384,9 @@ generate_report() {
|
||||
# Traffic breakdown
|
||||
echo "Traffic Breakdown:"
|
||||
while read -r line; do
|
||||
count=$(echo "$line" | awk '{print $1}')
|
||||
type=$(echo "$line" | awk '{print $2}')
|
||||
pct=$(awk "BEGIN {printf \"%.1f\", ($count/$total_requests)*100}")
|
||||
count=$(echo "$line" | awk '{print $1}' || echo "0")
|
||||
type=$(echo "$line" | awk '{print $2}' || echo "unknown")
|
||||
pct=$(awk "BEGIN {printf \"%.1f\", (${count:-0}/${total_requests:-1})*100}" 2>/dev/null || echo "0.0")
|
||||
|
||||
case $type in
|
||||
legit) echo " Legitimate Bots: $(printf "%'7d" $count) ($pct%)" ;;
|
||||
@@ -2352,6 +2403,7 @@ generate_report() {
|
||||
echo ""
|
||||
echo "Bot Traffic Timeline (hourly):"
|
||||
max_bot_traffic=$(awk '{print $1}' "$TEMP_DIR/hourly_bot_traffic.txt" | sort -rn | head -1)
|
||||
max_bot_traffic=${max_bot_traffic:-1} # Prevent division by zero
|
||||
while read -r line; do
|
||||
count=$(echo "$line" | awk '{print $1}')
|
||||
hour=$(echo "$line" | awk '{print $2}')
|
||||
@@ -2378,9 +2430,9 @@ generate_report() {
|
||||
echo ""
|
||||
echo "Response Code Analysis:"
|
||||
while read -r line; do
|
||||
count=$(echo "$line" | awk '{print $1}')
|
||||
code=$(echo "$line" | awk '{print $2}')
|
||||
pct=$(awk "BEGIN {printf \"%.1f\", ($count/$total_requests)*100}")
|
||||
count=$(echo "$line" | awk '{print $1}' || echo "0")
|
||||
code=$(echo "$line" | awk '{print $2}' || echo "000")
|
||||
pct=$(awk "BEGIN {printf \"%.1f\", (${count:-0}/${total_requests:-1})*100}" 2>/dev/null || echo "0.0")
|
||||
|
||||
case $code in
|
||||
200) echo " 200 (Success): $(printf "%'7d" $count) ($pct%) Bots are getting data" ;;
|
||||
@@ -2398,11 +2450,19 @@ generate_report() {
|
||||
if [ -s "$TEMP_DIR/false_positives.txt" ]; then
|
||||
echo ""
|
||||
echo "Whitelist Recommendations (Legitimate Services):"
|
||||
# Pre-build IP count cache to avoid repeated grep on large file
|
||||
declare -A ip_counts_cache
|
||||
if [ -f "$TEMP_DIR/parsed_logs.txt" ]; then
|
||||
while IFS='|' read -r ip rest; do
|
||||
[ -n "$ip" ] && ((ip_counts_cache["$ip"]++)) || true
|
||||
done < "$TEMP_DIR/parsed_logs.txt" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
while read -r line; do
|
||||
ip=$(echo "$line" | cut -d'|' -f1)
|
||||
service=$(echo "$line" | cut -d'|' -f2)
|
||||
domain=$(echo "$line" | cut -d'|' -f4)
|
||||
req_count=$(cat "$TEMP_DIR/parsed_logs.txt" 2>/dev/null | grep -c "^$ip|" || echo 0)
|
||||
req_count=${ip_counts_cache["$ip"]:-0}
|
||||
echo " $ip - $req_count requests - Identified as: $service"
|
||||
echo " → Domain: $domain"
|
||||
echo " → Action: VERIFY OWNERSHIP then whitelist"
|
||||
@@ -2411,30 +2471,32 @@ generate_report() {
|
||||
|
||||
# NEW: HIGH-CONFIDENCE BOT FINGERPRINTS
|
||||
if [ -s "$TEMP_DIR/bot_fingerprints.txt" ]; then
|
||||
echo ""
|
||||
print_header "HIGH-CONFIDENCE BOT FINGERPRINTS (Multi-signal analysis - reduced false positives)"
|
||||
echo "These IPs show MULTIPLE bot indicators combined (not just single signal):"
|
||||
echo ""
|
||||
(
|
||||
echo ""
|
||||
print_header "HIGH-CONFIDENCE BOT FINGERPRINTS (Multi-signal analysis - reduced false positives)"
|
||||
echo "These IPs show MULTIPLE bot indicators combined (not just single signal):"
|
||||
echo ""
|
||||
|
||||
awk -F'|' '
|
||||
NR <= 15 {
|
||||
ip = $1
|
||||
score = $2
|
||||
signals = $3
|
||||
awk -F'|' '
|
||||
NR <= 15 {
|
||||
ip = $1
|
||||
score = $2
|
||||
signals = $3
|
||||
|
||||
# Risk level based on score
|
||||
if (score >= 80) risk = "CRITICAL"
|
||||
else if (score >= 70) risk = "HIGH"
|
||||
else if (score >= 60) risk = "MEDIUM"
|
||||
else risk = "LOW"
|
||||
# Risk level based on score
|
||||
if (score >= 80) risk = "CRITICAL"
|
||||
else if (score >= 70) risk = "HIGH"
|
||||
else if (score >= 60) risk = "MEDIUM"
|
||||
else risk = "LOW"
|
||||
|
||||
printf " %s - Score: %2d/100 - Risk: %s - Signals: %d\n", ip, score, risk, signals
|
||||
}' "$TEMP_DIR/bot_fingerprints.txt"
|
||||
printf " %s - Score: %2d/100 - Risk: %s - Signals: %d\n", ip, score, risk, signals
|
||||
}' "$TEMP_DIR/bot_fingerprints.txt" || true
|
||||
|
||||
total=$(wc -l < "$TEMP_DIR/bot_fingerprints.txt" 2>/dev/null || echo "0")
|
||||
echo ""
|
||||
echo " Total high-confidence bots detected: $total IPs"
|
||||
echo ""
|
||||
total=$(wc -l < "$TEMP_DIR/bot_fingerprints.txt" 2>/dev/null || echo "0")
|
||||
echo ""
|
||||
echo " Total high-confidence bots detected: $total IPs"
|
||||
echo ""
|
||||
) || true
|
||||
else
|
||||
echo ""
|
||||
echo " No high-confidence bot fingerprints detected (requires multiple signals)"
|
||||
@@ -2452,44 +2514,24 @@ generate_report() {
|
||||
echo ""
|
||||
|
||||
# Show top attacked domains with attack details
|
||||
awk -F'|' 'NR <= 10 {print $1}' "$TEMP_DIR/domain_targeting.txt" | while read -r domain; do
|
||||
domain_attack_count=$(grep "^[^|]*|${domain}|" "$TEMP_DIR/attack_vectors_raw.txt" 2>/dev/null | wc -l || echo "0")
|
||||
# Limit to top 5 domains for performance with large datasets
|
||||
awk -F'|' 'NR <= 5 {print $1}' "$TEMP_DIR/domain_targeting.txt" 2>/dev/null | {
|
||||
while read -r domain; do
|
||||
[ -z "$domain" ] && continue
|
||||
|
||||
if [ "$domain_attack_count" -gt 0 ]; then
|
||||
echo " Domain: $domain ($domain_attack_count attack attempts)"
|
||||
# Use grep with strict error handling for large file searches
|
||||
domain_attack_count=0
|
||||
if [ -f "$TEMP_DIR/attack_vectors_raw.txt" ]; then
|
||||
domain_attack_count=$(grep -F "|${domain}|" "$TEMP_DIR/attack_vectors_raw.txt" 2>/dev/null | wc -l) || domain_attack_count=0
|
||||
fi
|
||||
domain_attack_count=${domain_attack_count:-0}
|
||||
|
||||
# Get all attacks on this domain, group by type
|
||||
awk -F'|' -v dom="$domain" '
|
||||
$2 == dom {
|
||||
ip = $1
|
||||
attack_type = $5
|
||||
|
||||
# Validate IP format
|
||||
if (match(ip, /^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$/)) {
|
||||
attack_data[attack_type][ip]++
|
||||
attack_totals[attack_type]++
|
||||
subnet_hits[attack_type][substr(ip, 1, index(ip, ".", index(ip, ".")+1)-1)]++
|
||||
}
|
||||
}
|
||||
END {
|
||||
for (attack_type in attack_totals) {
|
||||
printf " └─ %s: %d attempts\n", attack_type, attack_totals[attack_type]
|
||||
|
||||
# Show top 3 IPs for this attack type
|
||||
attack_count = 0
|
||||
for (ip in attack_data[attack_type]) {
|
||||
if (attack_count >= 3) break
|
||||
count = attack_data[attack_type][ip]
|
||||
split(ip, parts, ".")
|
||||
subnet = parts[1] "." parts[2] "." parts[3] ".0/24"
|
||||
printf " ├─ %s (%d reqs) [subnet: %s]\n", ip, count, subnet
|
||||
attack_count++
|
||||
}
|
||||
}
|
||||
}' "$TEMP_DIR/attack_vectors_raw.txt"
|
||||
echo ""
|
||||
fi
|
||||
done
|
||||
if [ "$domain_attack_count" -gt 0 ] 2>/dev/null; then
|
||||
echo " Domain: $domain ($domain_attack_count attack attempts)"
|
||||
echo ""
|
||||
fi
|
||||
done
|
||||
} || true
|
||||
else
|
||||
echo ""
|
||||
echo " No domain attack data available (all domains may be healthy)"
|
||||
@@ -2497,34 +2539,11 @@ generate_report() {
|
||||
fi
|
||||
|
||||
# NEW: TOP URLs BEING ATTACKED
|
||||
if [ -f "$TEMP_DIR/domain_targeting.txt" ]; then
|
||||
if [ -s "$TEMP_DIR/domain_targeting.txt" ]; then
|
||||
echo ""
|
||||
print_header "TOP TARGETED URLs (What files/endpoints are bots hitting?)"
|
||||
echo ""
|
||||
|
||||
# Show top URLs for top 3 most-attacked domains
|
||||
urls_shown=0
|
||||
awk -F'|' 'NR <= 3 {print $1}' "$TEMP_DIR/domain_targeting.txt" | while read -r domain; do
|
||||
local domain_file="$TEMP_DIR/domain_urls_${domain}.txt"
|
||||
if [ -f "$domain_file" ] && [ -s "$domain_file" ]; then
|
||||
echo " Domain: $domain"
|
||||
awk -F'|' '{
|
||||
url = $1
|
||||
count = $2
|
||||
printf " %3d requests → %s\n", count, url
|
||||
}' "$domain_file" # Show all URLs, not just top 5
|
||||
echo ""
|
||||
fi
|
||||
done
|
||||
|
||||
# Check if no URL data was shown
|
||||
if [ "$urls_shown" -eq 0 ]; then
|
||||
echo " No URL targeting data available"
|
||||
echo ""
|
||||
fi
|
||||
else
|
||||
echo ""
|
||||
echo " No domain targeting data available"
|
||||
echo " (Targeted URL data not available in summary - see log files for details)"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
@@ -2584,19 +2603,23 @@ generate_report() {
|
||||
echo ""
|
||||
|
||||
echo "2. Top Aggressive Bots:"
|
||||
counter=1
|
||||
while read -r line && [ "${counter:-0}" -le 5 ]; do
|
||||
count=$(echo "$line" | awk 'BEGIN {count=0} {print $1}')
|
||||
bot=$(echo "$line" | awk 'BEGIN {f=""} {$1=""; print $0}' | xargs)
|
||||
|
||||
action="Allow"
|
||||
if echo "$bot" | grep -qiE "ahrefs|semrush|dotbot|blex|megaindex"; then
|
||||
action="Consider blocking (aggressive)"
|
||||
fi
|
||||
|
||||
echo " [$counter] $bot - $count requests - Action: $action"
|
||||
counter=$((counter + 1))
|
||||
done < "$TEMP_DIR/top_bots.txt"
|
||||
if [ -s "$TEMP_DIR/top_bots.txt" ]; then
|
||||
counter=1
|
||||
while read -r line && [ "${counter:-0}" -le 5 ]; do
|
||||
count=$(echo "$line" | awk '{print $1}' 2>/dev/null || echo "0")
|
||||
bot=$(echo "$line" | awk '{$1=""; print $0}' 2>/dev/null | xargs || echo "$line")
|
||||
|
||||
action="Allow"
|
||||
if echo "$bot" | grep -qiE "ahrefs|semrush|dotbot|blex|megaindex" 2>/dev/null; then
|
||||
action="Consider blocking (aggressive)"
|
||||
fi
|
||||
|
||||
echo " [$counter] $bot - $count requests - Action: $action"
|
||||
counter=$((counter + 1))
|
||||
done < "$TEMP_DIR/top_bots.txt"
|
||||
else
|
||||
echo " No bot data available"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
echo "3. Admin Endpoint Probing:"
|
||||
@@ -2633,7 +2656,7 @@ generate_report() {
|
||||
# Calculate total bot bandwidth
|
||||
total_bot_bandwidth=0
|
||||
if [ -f "$TEMP_DIR/classified_bots.txt.gz" ]; then
|
||||
total_bot_bandwidth=$(cat "$TEMP_DIR/classified_bots.txt" | awk -F'|' '$9 != "unknown" && $5 ~ /^[0-9]+$/ {sum += $5} END {print sum}')
|
||||
total_bot_bandwidth=$(awk -F'|' '$9 != "unknown" && $5 ~ /^[0-9]+$/ {sum += $5} END {print sum}' "$TEMP_DIR/classified_bots.txt")
|
||||
fi
|
||||
|
||||
if [ -n "$total_bot_bandwidth" ] && [ "$total_bot_bandwidth" -gt 0 ]; then
|
||||
@@ -2642,7 +2665,7 @@ generate_report() {
|
||||
# Estimate cost at $0.09/GB (typical CDN pricing)
|
||||
estimated_cost=$(awk "BEGIN {printf \"%.2f\", ($total_bot_bandwidth/1073741824) * 0.09}")
|
||||
|
||||
total_bandwidth=$(cat "$TEMP_DIR/parsed_logs.txt" | awk -F'|' '$5 ~ /^[0-9]+$/ {sum += $5} END {print sum}')
|
||||
total_bandwidth=$(awk -F'|' '$5 ~ /^[0-9]+$/ {sum += $5} END {print sum}' "$TEMP_DIR/parsed_logs.txt")
|
||||
bot_pct=$(awk "BEGIN {printf \"%.1f\", ($total_bot_bandwidth/$total_bandwidth)*100}")
|
||||
|
||||
echo ""
|
||||
@@ -2654,13 +2677,13 @@ generate_report() {
|
||||
echo " Top bandwidth consumers:"
|
||||
|
||||
head -3 "$TEMP_DIR/large_transfers.txt" | while read -r line; do
|
||||
count=$(echo "$line" | awk '{print $1}')
|
||||
ip=$(echo "$line" | awk '{print $2}' | cut -d'|' -f1)
|
||||
domain=$(echo "$line" | awk '{print $2}' | cut -d'|' -f2)
|
||||
url=$(echo "$line" | awk '{print $2}' | cut -d'|' -f3)
|
||||
size=$(echo "$line" | awk '{print $2}' | cut -d'|' -f4)
|
||||
size_mb=$(awk "BEGIN {printf \"%.1f\", $size/1048576}")
|
||||
total_ip_mb=$(awk "BEGIN {printf \"%.0f\", $size * $count / 1048576}")
|
||||
count=$(echo "$line" | awk '{print $1}' || echo "0")
|
||||
ip=$(echo "$line" | awk '{print $2}' 2>/dev/null | cut -d'|' -f1 || echo "unknown")
|
||||
domain=$(echo "$line" | awk '{print $2}' 2>/dev/null | cut -d'|' -f2 || echo "unknown")
|
||||
url=$(echo "$line" | awk '{print $2}' 2>/dev/null | cut -d'|' -f3 || echo "unknown")
|
||||
size=$(echo "$line" | awk '{print $2}' 2>/dev/null | cut -d'|' -f4 || echo "0")
|
||||
size_mb=$(awk "BEGIN {printf \"%.1f\", ${size:-0}/1048576}" 2>/dev/null || echo "0.0")
|
||||
total_ip_mb=$(awk "BEGIN {printf \"%.0f\", ${size:-0} * ${count:-0} / 1048576}" 2>/dev/null || echo "0")
|
||||
printf " %s transfers from %s - %.1f MB avg (%s MB total) - %s%s\n" "$count" "$ip" "$size_mb" "$total_ip_mb" "$domain" "$url"
|
||||
done
|
||||
echo " Action: Verify if scraping, consider serving WebP/optimized images"
|
||||
@@ -2673,17 +2696,17 @@ generate_report() {
|
||||
|
||||
counter=1
|
||||
while read -r line && [ "${counter:-0}" -le 5 ]; do
|
||||
count=$(echo "$line" | awk '{print $1}')
|
||||
domain=$(echo "$line" | awk '{print $2}')
|
||||
|
||||
count=$(echo "$line" | awk '{print $1}' || echo "0")
|
||||
domain=$(echo "$line" | awk '{print $2}' || echo "unknown")
|
||||
|
||||
echo "[$counter] $domain - $count requests"
|
||||
|
||||
|
||||
# Show traffic breakdown for this domain
|
||||
if [ -f "$TEMP_DIR/domain_${domain}_stats.txt" ]; then
|
||||
tail -n +2 "$TEMP_DIR/domain_${domain}_stats.txt" | while read -r stat_line; do
|
||||
stat_count=$(echo "$stat_line" | awk '{print $1}')
|
||||
stat_type=$(echo "$stat_line" | awk '{print $2}')
|
||||
pct=$(awk "BEGIN {printf \"%.1f\", ($stat_count/$count)*100}")
|
||||
stat_count=$(echo "$stat_line" | awk '{print $1}' || echo "0")
|
||||
stat_type=$(echo "$stat_line" | awk '{print $2}' || echo "unknown")
|
||||
pct=$(awk "BEGIN {printf \"%.1f\", (${stat_count:-0}/${count:-1})*100}" 2>/dev/null || echo "0.0")
|
||||
|
||||
case $stat_type in
|
||||
suspicious) echo -e " ${YELLOW}Suspicious: $stat_count ($pct%)${NC}" ;;
|
||||
@@ -3364,15 +3387,15 @@ generate_recommendations() {
|
||||
attack_scope="single_domain"
|
||||
primary_target=$(head -1 "$TEMP_DIR/domain_high_risk_ips.txt" 2>/dev/null | cut -d'|' -f1)
|
||||
# Calculate what % of high-risk IPs are targeting this domain
|
||||
local domain_risk_count=$(head -1 "$TEMP_DIR/domain_high_risk_ips.txt" 2>/dev/null | cut -d'|' -f2)
|
||||
if [ "${total_high_risk_ips:-0}" -gt 0 ]; then
|
||||
primary_target_percentage=$(awk "BEGIN {printf \"%.0f\", ($domain_risk_count / $total_high_risk_ips) * 100}")
|
||||
local domain_risk_count=$(head -1 "$TEMP_DIR/domain_high_risk_ips.txt" 2>/dev/null | cut -d'|' -f2 || echo "0")
|
||||
if [ "${total_high_risk_ips:-0}" -gt 0 ] && [ "${domain_risk_count:-0}" -gt 0 ]; then
|
||||
primary_target_percentage=$(awk "BEGIN {printf \"%.0f\", (${domain_risk_count:-0} / ${total_high_risk_ips:-0}) * 100}")
|
||||
fi
|
||||
elif [ "${affected_domains:-0}" -gt 1 ] && [ "${total_domains:-0}" -gt 1 ]; then
|
||||
# Check if one domain is getting most of the traffic
|
||||
local top_domain_count=$(head -1 "$TEMP_DIR/domain_threats_sorted.txt" 2>/dev/null | cut -d'|' -f5)
|
||||
local top_domain_count=$(head -1 "$TEMP_DIR/domain_threats_sorted.txt" 2>/dev/null | cut -d'|' -f5 || echo "0")
|
||||
if [ "${top_domain_count:-0}" -gt 0 ] && [ "${total_high_risk_ips:-0}" -gt 0 ]; then
|
||||
local top_percentage=$(awk "BEGIN {printf \"%.0f\", ($top_domain_count / $total_high_risk_ips) * 100}")
|
||||
local top_percentage=$(awk "BEGIN {printf \"%.0f\", (${top_domain_count:-0} / ${total_high_risk_ips:-0}) * 100}")
|
||||
if [ "$top_percentage" -ge 75 ]; then
|
||||
attack_scope="primary_target"
|
||||
primary_target=$(head -1 "$TEMP_DIR/domain_threats_sorted.txt" 2>/dev/null | cut -d'|' -f1)
|
||||
@@ -3675,7 +3698,7 @@ show_detailed_recommendations() {
|
||||
local target_domain=$(echo "$action_title" | grep -oP 'to \K[^ ]+' 2>/dev/null || echo "")
|
||||
echo "Target Domain: $target_domain"
|
||||
if [ -s "$TEMP_DIR/domain_threats_sorted.txt" ]; then
|
||||
grep "^$target_domain|" "$TEMP_DIR/domain_threats_sorted.txt" 2>/dev/null | while IFS='|' read -r domain total_req bot_req bot_pct high_risk attacks ips; do
|
||||
grep -F "$target_domain|" "$TEMP_DIR/domain_threats_sorted.txt" 2>/dev/null | while IFS='|' read -r domain total_req bot_req bot_pct high_risk attacks ips; do
|
||||
echo " • Total Requests: $total_req"
|
||||
echo " • Bot Requests: $bot_req ($bot_pct%)"
|
||||
echo " • High-Risk IPs: $high_risk"
|
||||
@@ -4003,7 +4026,7 @@ execute_ip_blocking_specific() {
|
||||
local fail_count=0
|
||||
|
||||
for ip in "${ips_to_block[@]}"; do
|
||||
local score=$(grep "|$ip|" "$TEMP_DIR/threat_scores.txt" 2>/dev/null | cut -d'|' -f1 || echo "unknown")
|
||||
local score=$(grep -F "|$ip|" "$TEMP_DIR/threat_scores.txt" 2>/dev/null | cut -d'|' -f1 || echo "unknown")
|
||||
|
||||
if csf -td "$ip" "$duration" "Bot threat score: $score/100 - Auto-blocked by toolkit" >/dev/null 2>&1; then
|
||||
echo -e " ${GREEN}${NC} Blocked $ip for $duration_text (score: $score/100)"
|
||||
@@ -4153,7 +4176,7 @@ execute_htaccess_domain_blocking() {
|
||||
# Find document root for this domain using reference database
|
||||
local doc_root=""
|
||||
if [ -s "$SCRIPT_DIR/.sysref" ]; then
|
||||
doc_root=$(grep "^DOMAIN|$target_domain|" "$SCRIPT_DIR/.sysref" 2>/dev/null | head -1 | cut -d'|' -f4 || echo "")
|
||||
doc_root=$(grep -F "DOMAIN|$target_domain|" "$SCRIPT_DIR/.sysref" 2>/dev/null | head -1 | cut -d'|' -f4 || echo "")
|
||||
fi
|
||||
|
||||
if [ -z "$doc_root" ]; then
|
||||
@@ -4197,10 +4220,10 @@ execute_htaccess_domain_blocking() {
|
||||
print_info "Adding bot blocking rules..."
|
||||
|
||||
# Get high-risk IPs for this domain
|
||||
local block_ips=$(cat "$TEMP_DIR/parsed_logs.txt" 2>/dev/null | grep "^[^|]*|$target_domain|" 2>/dev/null || true | cut -d'|' -f1 | sort -u | while read ip; do
|
||||
local block_ips=$(cat "$TEMP_DIR/parsed_logs.txt" 2>/dev/null | grep -F "|$target_domain|" 2>/dev/null || true | cut -d'|' -f1 | sort -u | while read ip; do
|
||||
# Check if this IP has high threat score
|
||||
if grep -q "|$ip$" "$TEMP_DIR/threat_scores.txt" 2>/dev/null; then
|
||||
local score=$(grep "|$ip$" "$TEMP_DIR/threat_scores.txt" 2>/dev/null | cut -d'|' -f1 || echo "0")
|
||||
if grep -q -F "|$ip|" "$TEMP_DIR/threat_scores.txt" 2>/dev/null; then
|
||||
local score=$(grep -F "|$ip|" "$TEMP_DIR/threat_scores.txt" 2>/dev/null | cut -d'|' -f1 || echo "0")
|
||||
if [ "${score:-0}" -ge 70 ]; then
|
||||
echo "$ip"
|
||||
fi
|
||||
@@ -4520,7 +4543,7 @@ offer_csf_blocking() {
|
||||
count=$((count + 1))
|
||||
local ip="${high_risk_ips[$i]}"
|
||||
local score="${ip_scores[$i]}"
|
||||
local requests=$(grep "^$ip|" "$TEMP_DIR/bot_ips.txt" 2>/dev/null | cut -d'|' -f2 || echo "0")
|
||||
local requests=$(grep -F "$ip|" "$TEMP_DIR/bot_ips.txt" 2>/dev/null | cut -d'|' -f2 || echo "0")
|
||||
|
||||
# Color code by severity
|
||||
if [ "$score" -ge 90 ]; then
|
||||
@@ -4586,7 +4609,7 @@ apply_csf_blocks() {
|
||||
|
||||
for ip in "${ips[@]}"; do
|
||||
# Get threat score for comment
|
||||
local score=$(grep "|$ip$" "$TEMP_DIR/threat_scores.txt" 2>/dev/null | cut -d'|' -f1 || echo "unknown")
|
||||
local score=$(grep -F "|$ip|" "$TEMP_DIR/threat_scores.txt" 2>/dev/null | cut -d'|' -f1 || echo "unknown")
|
||||
|
||||
# Use csf -td for temporary deny
|
||||
if csf -td "$ip" "$duration" "Bot threat score: $score/100 - Auto-blocked by toolkit" >/dev/null 2>&1; then
|
||||
@@ -4639,7 +4662,7 @@ apply_csf_permanent_blocks() {
|
||||
local fail_count=0
|
||||
|
||||
for ip in "${ips[@]}"; do
|
||||
local score=$(grep "|$ip$" "$TEMP_DIR/threat_scores.txt" 2>/dev/null | cut -d'|' -f1 || echo "unknown")
|
||||
local score=$(grep -F "|$ip|" "$TEMP_DIR/threat_scores.txt" 2>/dev/null | cut -d'|' -f1 || echo "unknown")
|
||||
|
||||
# Use csf -d for permanent deny
|
||||
if csf -d "$ip" "Bot threat score: $score/100 - Permanently blocked by toolkit" >/dev/null 2>&1; then
|
||||
|
||||
@@ -2156,7 +2156,7 @@ for scanner in "${available_scanners[@]}"; do
|
||||
# Extract scan results from event log (more reliable than parsing output)
|
||||
# Maldet logs to /usr/local/maldetect/logs/event_log
|
||||
# Use dynamic path search for portability across all platforms (FIXED Issue 2: comprehensive path discovery)
|
||||
local event_log=""
|
||||
event_log=""
|
||||
|
||||
# Search standard locations in order of likelihood
|
||||
for search_path in \
|
||||
@@ -2556,7 +2556,7 @@ STANDALONE_EOF
|
||||
fi
|
||||
|
||||
# Inject MALDET_ONLY flag for Maldet-dedicated scans
|
||||
local maldet_flag="${MALDET_ONLY:-0}"
|
||||
maldet_flag="${MALDET_ONLY:-0}"
|
||||
if ! sed -i "s|PLACEHOLDER_MALDET_ONLY|$maldet_flag|" "$session_dir/scan.sh"; then
|
||||
echo -e "${RED}ERROR: Failed to inject MALDET_ONLY flag${NC}"
|
||||
return 1
|
||||
|
||||
@@ -826,11 +826,8 @@ main() {
|
||||
echo ""
|
||||
fi
|
||||
|
||||
# Check if sysref database exists, build if needed
|
||||
if [ ! -f "$SYSREF_DB" ] || [ ! -s "$SYSREF_DB" ]; then
|
||||
print_status "Building system reference database (first run)..."
|
||||
build_reference_database >/dev/null 2>&1
|
||||
fi
|
||||
# Ensure reference database is fresh (only rebuild if > 1 hour old)
|
||||
db_ensure_fresh >/dev/null 2>&1
|
||||
|
||||
# Run analysis
|
||||
check_server_resources
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
#!/bin/bash
|
||||
set -eo pipefail
|
||||
|
||||
#
|
||||
# Suspicious Login Monitor - Integrated Security Analysis & Compromise Detection
|
||||
@@ -11,6 +12,9 @@
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
TOOLKIT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
|
||||
# Source reference-db for cache support (avoid redundant /etc/passwd parsing)
|
||||
source "$TOOLKIT_ROOT/lib/reference-db.sh" 2>/dev/null || true
|
||||
|
||||
# Configuration
|
||||
SUSPICIOUS_LOGIN_AUTO_BLOCK="${SUSPICIOUS_LOGIN_AUTO_BLOCK:-yes}"
|
||||
SUSPICIOUS_LOGIN_AUTO_SCAN="${SUSPICIOUS_LOGIN_AUTO_SCAN:-yes}"
|
||||
@@ -1673,7 +1677,7 @@ check_maintenance_mode() {
|
||||
fi
|
||||
|
||||
if [ -n "$indicators" ]; then
|
||||
echo "maintenance-mode:$(echo $indicators | sed 's/ $//')"
|
||||
echo "maintenance-mode:$(sed 's/ $//' <<< "$indicators")"
|
||||
return 0
|
||||
fi
|
||||
|
||||
@@ -1823,6 +1827,10 @@ check_recent_password_changes() {
|
||||
fi
|
||||
|
||||
# Check for locked accounts that were recently unlocked
|
||||
# OPTIMIZATION: Read /etc/passwd ONCE, build nologin list, then check against it
|
||||
# (avoiding redundant grep for each user in the loop)
|
||||
local nologin_users=$(awk -F: '/\/sbin\/nologin|\/bin\/false/ {print $1}' /etc/passwd 2>/dev/null | tr '\n' '|')
|
||||
|
||||
local recently_unlocked=$(awk -F: -v cutoff=$(( $(date +%s) / 86400 - 7 )) '
|
||||
# Field 2 starts with ! or !! = locked
|
||||
# If field 3 (last change) is recent and field 2 does NOT start with !, might have been unlocked
|
||||
@@ -1830,8 +1838,8 @@ check_recent_password_changes() {
|
||||
print $1
|
||||
}
|
||||
' /etc/shadow 2>/dev/null | while read user; do
|
||||
# Check if account was previously locked (this is imperfect without history)
|
||||
if grep "^$user:" /etc/passwd | grep -q "/sbin/nologin\|/bin/false"; then
|
||||
# Check if account has nologin shell (from pre-built list)
|
||||
if [[ "|$nologin_users" =~ \|$user\| ]]; then
|
||||
echo "$user"
|
||||
fi
|
||||
done)
|
||||
@@ -2947,6 +2955,11 @@ main() {
|
||||
echo -e "${CYAN}Starting Suspicious Login Monitor...${NC}"
|
||||
echo ""
|
||||
|
||||
# Ensure cache is fresh (only rebuilds if > 1 hour old)
|
||||
if command -v db_ensure_fresh &>/dev/null; then
|
||||
db_ensure_fresh 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# Detect panel
|
||||
local panel=$(detect_panel)
|
||||
echo "Detected panel: $panel"
|
||||
|
||||
@@ -1977,18 +1977,18 @@ calculate_performance_score() {
|
||||
|
||||
# Calculate score (100 - issues)
|
||||
local score=$((100 - (critical_count * 10) - (warning_count * 2)))
|
||||
[ $score -lt 0 ] && score=0
|
||||
[ $score -gt 100 ] && score=100
|
||||
[ "$score" -lt 0 ] && score=0
|
||||
[ "$score" -gt 100 ] && score=100
|
||||
|
||||
# Determine grade
|
||||
local grade
|
||||
if [ $score -ge 90 ]; then
|
||||
if [ "$score" -ge 90 ]; then
|
||||
grade="A - EXCELLENT"
|
||||
elif [ $score -ge 80 ]; then
|
||||
elif [ "$score" -ge 80 ]; then
|
||||
grade="B - GOOD"
|
||||
elif [ $score -ge 70 ]; then
|
||||
elif [ "$score" -ge 70 ]; then
|
||||
grade="C - FAIR"
|
||||
elif [ $score -ge 60 ]; then
|
||||
elif [ "$score" -ge 60 ]; then
|
||||
grade="D - POOR"
|
||||
else
|
||||
grade="F - CRITICAL"
|
||||
|
||||
@@ -10,14 +10,22 @@ set -o pipefail
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
CLEANUP_FLAG="/tmp/.cleanup_requested"
|
||||
|
||||
# Save original history setting to restore even if interrupted
|
||||
HISTORY_SETTING=$(set +o | grep history)
|
||||
# Save original history state to restore even if interrupted
|
||||
HISTORY_STATE="off"
|
||||
if set -o | grep -q "^set +o history" 2>/dev/null; then
|
||||
HISTORY_STATE="on"
|
||||
fi
|
||||
RESTORE_HISTORY=false
|
||||
|
||||
# Cleanup function: restore history even on error/interrupt
|
||||
cleanup_on_exit() {
|
||||
if [ "$RESTORE_HISTORY" = true ]; then
|
||||
eval "$HISTORY_SETTING" 2>/dev/null || true
|
||||
if [ "$RESTORE_HISTORY" = true ] && [ -n "$HISTORY_STATE" ]; then
|
||||
set +H # Disable history expansion temporarily
|
||||
if [ "$HISTORY_STATE" = "on" ]; then
|
||||
set -o history
|
||||
else
|
||||
set +o history
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -59,7 +67,11 @@ source "$SCRIPT_DIR/launcher.sh"
|
||||
LAUNCHER_EXIT=$?
|
||||
|
||||
# Re-enable history (trap will also do this)
|
||||
eval "$HISTORY_SETTING" 2>/dev/null || true
|
||||
if [ "$HISTORY_STATE" = "on" ]; then
|
||||
set -o history 2>/dev/null || true
|
||||
else
|
||||
set +o history 2>/dev/null || true
|
||||
fi
|
||||
RESTORE_HISTORY=false
|
||||
|
||||
# Handle cleanup request (if user selected "Clean and remove traces")
|
||||
|
||||
Reference in New Issue
Block a user