Add Bot Blocker - Apache User-Agent blocking manager
Features: - Enable/disable bot blocking with one click - Blocks security scanners (nikto, sqlmap, nmap, etc.) - Blocks aggressive SEO bots (AhrefsBot, SemrushBot, etc.) - Blocks AI crawlers (GPTBot, Claude-Web, ChatGPT-User, etc.) - Blocks generic scrapers (Go-http-client, etc.) - Automatic backups before changes - Apache syntax validation before applying - Safe restart with rollback on failure - View current configuration - Manage backups and restore Configuration: - File: /etc/apache2/conf.d/includes/pre_main_global.conf - Blocks 24+ malicious bot user-agents - Returns HTTP 403 Forbidden to blocked bots - Zero impact on legitimate traffic Integrated into Security Menu (option 16)
This commit is contained in:
cschantz
Executable
+497
@@ -0,0 +1,497 @@
|
||||
#!/bin/bash
|
||||
################################################################################
|
||||
# Bot Blocker - Apache User-Agent Blocking Manager
|
||||
################################################################################
|
||||
# Blocks malicious bots, scrapers, and AI crawlers at Apache level
|
||||
# Safe implementation with backup, validation, and rollback
|
||||
################################################################################
|
||||
|
||||
# Source common functions
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "$SCRIPT_DIR/../../lib/common-functions.sh" 2>/dev/null || {
|
||||
echo "[ERROR] Cannot load common-functions.sh"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Detect control panel and system (optional)
|
||||
if type -t detect_system &>/dev/null; then
|
||||
detect_system
|
||||
fi
|
||||
|
||||
# Configuration
|
||||
APACHE_CONF="/etc/apache2/conf.d/includes/pre_main_global.conf"
|
||||
BACKUP_DIR="/root/.bot-blocker-backups"
|
||||
MARKER_START="# BEGIN Bot Blocker - Managed by Server Toolkit"
|
||||
MARKER_END="# END Bot Blocker"
|
||||
|
||||
# Default bot list (security scanners, aggressive SEO bots, AI crawlers)
|
||||
DEFAULT_BOT_LIST="nikto|nmap|masscan|sqlmap|havij|acunetix|nessus|burp|metasploit|AhrefsBot|SemrushBot|MJ12bot|DotBot|Meta-ExternalAgent|Go-http-client|ChatGPT-User|GPTBot|Google-Extended|anthropic-ai|Claude-Web|CCBot|Omgilibot|FacebookBot|Bytespider|PerplexityBot"
|
||||
|
||||
################################################################################
|
||||
# Helper Functions
|
||||
################################################################################
|
||||
|
||||
check_apache_installed() {
|
||||
if ! command -v httpd &>/dev/null; then
|
||||
print_error "Apache is not installed"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
check_apache_config_exists() {
|
||||
local conf_dir=$(dirname "$APACHE_CONF")
|
||||
|
||||
if [ ! -d "$conf_dir" ]; then
|
||||
print_warning "Apache config directory doesn't exist: $conf_dir"
|
||||
echo ""
|
||||
read -p "Create directory? (yes/no): " create_dir
|
||||
if [ "$create_dir" = "yes" ]; then
|
||||
mkdir -p "$conf_dir"
|
||||
print_success "Created directory: $conf_dir"
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ! -f "$APACHE_CONF" ]; then
|
||||
print_info "Config file doesn't exist, will be created: $APACHE_CONF"
|
||||
touch "$APACHE_CONF"
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
is_bot_blocking_enabled() {
|
||||
if [ ! -f "$APACHE_CONF" ]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
grep -q "$MARKER_START" "$APACHE_CONF" 2>/dev/null
|
||||
}
|
||||
|
||||
create_backup() {
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
|
||||
local backup_file="$BACKUP_DIR/pre_main_global.conf.$(date +%Y%m%d_%H%M%S)"
|
||||
|
||||
if [ -f "$APACHE_CONF" ]; then
|
||||
cp "$APACHE_CONF" "$backup_file"
|
||||
print_success "Backup created: $backup_file"
|
||||
echo "$backup_file"
|
||||
else
|
||||
# No existing file to backup
|
||||
echo ""
|
||||
fi
|
||||
}
|
||||
|
||||
validate_apache_syntax() {
|
||||
print_info "Validating Apache configuration..."
|
||||
|
||||
if httpd -t 2>&1 | grep -q "Syntax OK"; then
|
||||
print_success "Apache configuration syntax is valid"
|
||||
return 0
|
||||
else
|
||||
print_error "Apache configuration has syntax errors:"
|
||||
echo ""
|
||||
httpd -t 2>&1 | grep -v "Syntax OK"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
restart_apache() {
|
||||
print_info "Restarting Apache..."
|
||||
|
||||
if systemctl restart httpd; then
|
||||
print_success "Apache restarted successfully"
|
||||
return 0
|
||||
else
|
||||
print_error "Failed to restart Apache"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
restore_backup() {
|
||||
local backup_file="$1"
|
||||
|
||||
if [ -f "$backup_file" ]; then
|
||||
cp "$backup_file" "$APACHE_CONF"
|
||||
print_success "Configuration restored from backup"
|
||||
restart_apache
|
||||
else
|
||||
print_error "Backup file not found: $backup_file"
|
||||
fi
|
||||
}
|
||||
|
||||
################################################################################
|
||||
# Core Functions
|
||||
################################################################################
|
||||
|
||||
enable_bot_blocking() {
|
||||
print_banner "Enable Bot Blocking"
|
||||
|
||||
# Check prerequisites
|
||||
if ! check_apache_installed; then
|
||||
press_enter
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! check_apache_config_exists; then
|
||||
press_enter
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Check if already enabled
|
||||
if is_bot_blocking_enabled; then
|
||||
print_warning "Bot blocking is already enabled"
|
||||
echo ""
|
||||
read -p "Re-apply configuration? (yes/no): " reapply
|
||||
if [ "$reapply" != "yes" ]; then
|
||||
return 0
|
||||
fi
|
||||
disable_bot_blocking_silent
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Bot Blocking Configuration:"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo ""
|
||||
echo "This will block the following types of bots:"
|
||||
echo ""
|
||||
echo " 🔴 Security Scanners:"
|
||||
echo " nikto, nmap, masscan, sqlmap, havij, acunetix, nessus, burp, metasploit"
|
||||
echo ""
|
||||
echo " 🟡 Aggressive SEO Bots:"
|
||||
echo " AhrefsBot, SemrushBot, MJ12bot, DotBot"
|
||||
echo ""
|
||||
echo " 🟣 AI Crawlers:"
|
||||
echo " ChatGPT-User, GPTBot, Google-Extended, Claude-Web, anthropic-ai"
|
||||
echo ""
|
||||
echo " 🔵 Generic Scrapers:"
|
||||
echo " Go-http-client, Meta-ExternalAgent, CCBot, Omgilibot, FacebookBot,"
|
||||
echo " Bytespider, PerplexityBot"
|
||||
echo ""
|
||||
echo "Configuration file: $APACHE_CONF"
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo ""
|
||||
read -p "Proceed with enabling bot blocking? (yes/no): " confirm
|
||||
|
||||
if [ "$confirm" != "yes" ]; then
|
||||
echo "Cancelled."
|
||||
press_enter
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Create backup
|
||||
echo ""
|
||||
local backup_file=$(create_backup)
|
||||
|
||||
# Add bot blocking rules
|
||||
echo ""
|
||||
print_info "Adding bot blocking rules..."
|
||||
|
||||
# Build the configuration with variables expanded
|
||||
local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
|
||||
cat >> "$APACHE_CONF" <<EOF
|
||||
|
||||
# BEGIN Bot Blocker - Managed by Server Toolkit
|
||||
# Blocks malicious bots, scrapers, and AI crawlers
|
||||
# Last updated: ${timestamp}
|
||||
<IfModule mod_rewrite.c>
|
||||
RewriteEngine On
|
||||
RewriteCond %{HTTP_USER_AGENT} "(${DEFAULT_BOT_LIST})" [NC]
|
||||
RewriteRule ^ - [F,L]
|
||||
</IfModule>
|
||||
# END Bot Blocker
|
||||
|
||||
EOF
|
||||
|
||||
print_success "Bot blocking rules added"
|
||||
|
||||
# Validate syntax
|
||||
echo ""
|
||||
if ! validate_apache_syntax; then
|
||||
print_error "Configuration validation failed!"
|
||||
echo ""
|
||||
read -p "Rollback to previous configuration? (yes/no): " rollback
|
||||
if [ "$rollback" = "yes" ] && [ -n "$backup_file" ]; then
|
||||
restore_backup "$backup_file"
|
||||
fi
|
||||
press_enter
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Restart Apache
|
||||
echo ""
|
||||
if restart_apache; then
|
||||
echo ""
|
||||
print_success "═══════════════════════════════════════════════════════"
|
||||
print_success "Bot blocking enabled successfully!"
|
||||
print_success "═══════════════════════════════════════════════════════"
|
||||
echo ""
|
||||
echo "What happens now:"
|
||||
echo " ✓ Blocked bots receive HTTP 403 Forbidden"
|
||||
echo " ✓ Legitimate traffic is unaffected"
|
||||
echo " ✓ Configuration survives Apache restarts"
|
||||
echo ""
|
||||
echo "To disable: Use option 2 from the menu"
|
||||
echo ""
|
||||
else
|
||||
print_error "Failed to restart Apache"
|
||||
if [ -n "$backup_file" ]; then
|
||||
echo ""
|
||||
read -p "Rollback to previous configuration? (yes/no): " rollback
|
||||
if [ "$rollback" = "yes" ]; then
|
||||
restore_backup "$backup_file"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
press_enter
|
||||
}
|
||||
|
||||
disable_bot_blocking_silent() {
|
||||
# Silent version for internal use (re-apply scenario)
|
||||
if [ -f "$APACHE_CONF" ]; then
|
||||
sed -i "/$MARKER_START/,/$MARKER_END/d" "$APACHE_CONF"
|
||||
fi
|
||||
}
|
||||
|
||||
disable_bot_blocking() {
|
||||
print_banner "Disable Bot Blocking"
|
||||
|
||||
# Check if enabled
|
||||
if ! is_bot_blocking_enabled; then
|
||||
print_warning "Bot blocking is not currently enabled"
|
||||
press_enter
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "This will remove bot blocking rules from:"
|
||||
echo " $APACHE_CONF"
|
||||
echo ""
|
||||
read -p "Proceed with disabling bot blocking? (yes/no): " confirm
|
||||
|
||||
if [ "$confirm" != "yes" ]; then
|
||||
echo "Cancelled."
|
||||
press_enter
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Create backup
|
||||
echo ""
|
||||
local backup_file=$(create_backup)
|
||||
|
||||
# Remove bot blocking rules
|
||||
echo ""
|
||||
print_info "Removing bot blocking rules..."
|
||||
|
||||
sed -i "/$MARKER_START/,/$MARKER_END/d" "$APACHE_CONF"
|
||||
|
||||
print_success "Bot blocking rules removed"
|
||||
|
||||
# Validate syntax
|
||||
echo ""
|
||||
if ! validate_apache_syntax; then
|
||||
print_error "Configuration validation failed!"
|
||||
echo ""
|
||||
read -p "Rollback to previous configuration? (yes/no): " rollback
|
||||
if [ "$rollback" = "yes" ] && [ -n "$backup_file" ]; then
|
||||
restore_backup "$backup_file"
|
||||
fi
|
||||
press_enter
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Restart Apache
|
||||
echo ""
|
||||
if restart_apache; then
|
||||
echo ""
|
||||
print_success "Bot blocking disabled successfully"
|
||||
echo ""
|
||||
else
|
||||
print_error "Failed to restart Apache"
|
||||
if [ -n "$backup_file" ]; then
|
||||
echo ""
|
||||
read -p "Rollback to previous configuration? (yes/no): " rollback
|
||||
if [ "$rollback" = "yes" ]; then
|
||||
restore_backup "$backup_file"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
press_enter
|
||||
}
|
||||
|
||||
view_configuration() {
|
||||
print_banner "Current Bot Blocking Configuration"
|
||||
|
||||
if ! is_bot_blocking_enabled; then
|
||||
print_warning "Bot blocking is not currently enabled"
|
||||
echo ""
|
||||
echo "Use option 1 to enable bot blocking"
|
||||
press_enter
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Status: ${GREEN}Enabled${NC}"
|
||||
echo "Config file: $APACHE_CONF"
|
||||
echo ""
|
||||
echo "Current rules:"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
# Extract and display bot blocking section
|
||||
sed -n "/$MARKER_START/,/$MARKER_END/p" "$APACHE_CONF"
|
||||
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo ""
|
||||
|
||||
# Parse and display bot list
|
||||
local bot_list=$(sed -n "/$MARKER_START/,/$MARKER_END/p" "$APACHE_CONF" | grep "RewriteCond.*HTTP_USER_AGENT" | sed 's/.*"(\(.*\))".*/\1/')
|
||||
|
||||
if [ -n "$bot_list" ]; then
|
||||
echo "Blocked User-Agents:"
|
||||
echo ""
|
||||
echo "$bot_list" | tr '|' '\n' | nl | column -t
|
||||
echo ""
|
||||
fi
|
||||
|
||||
press_enter
|
||||
}
|
||||
|
||||
test_configuration() {
|
||||
print_banner "Test Apache Configuration"
|
||||
|
||||
echo ""
|
||||
print_info "Running Apache syntax check..."
|
||||
echo ""
|
||||
|
||||
if validate_apache_syntax; then
|
||||
echo ""
|
||||
print_success "Configuration is valid and ready to apply"
|
||||
else
|
||||
echo ""
|
||||
print_error "Configuration has errors - fix before applying"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
press_enter
|
||||
}
|
||||
|
||||
manage_backups() {
|
||||
print_banner "Backup Management"
|
||||
|
||||
if [ ! -d "$BACKUP_DIR" ] || [ -z "$(ls -A "$BACKUP_DIR" 2>/dev/null)" ]; then
|
||||
print_warning "No backups found"
|
||||
echo ""
|
||||
echo "Backups are created automatically when you enable/disable bot blocking"
|
||||
press_enter
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Available backups:"
|
||||
echo ""
|
||||
|
||||
local count=1
|
||||
while IFS= read -r backup; do
|
||||
local size=$(du -h "$backup" | cut -f1)
|
||||
local date=$(basename "$backup" | sed 's/pre_main_global.conf.//')
|
||||
echo " $count) $date ($size)"
|
||||
count=$((count + 1))
|
||||
done < <(ls -t "$BACKUP_DIR"/*.conf.* 2>/dev/null)
|
||||
|
||||
echo ""
|
||||
echo " 0) Back to menu"
|
||||
echo ""
|
||||
read -p "Select backup to restore (or 0 to cancel): " selection
|
||||
|
||||
if [ "$selection" = "0" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local backup_file=$(ls -t "$BACKUP_DIR"/*.conf.* 2>/dev/null | sed -n "${selection}p")
|
||||
|
||||
if [ -z "$backup_file" ]; then
|
||||
print_error "Invalid selection"
|
||||
press_enter
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Selected backup: $(basename "$backup_file")"
|
||||
echo ""
|
||||
read -p "Restore this backup? (yes/no): " confirm
|
||||
|
||||
if [ "$confirm" = "yes" ]; then
|
||||
restore_backup "$backup_file"
|
||||
echo ""
|
||||
print_success "Backup restored successfully"
|
||||
fi
|
||||
|
||||
press_enter
|
||||
}
|
||||
|
||||
################################################################################
|
||||
# Main Menu
|
||||
################################################################################
|
||||
|
||||
show_menu() {
|
||||
clear
|
||||
print_banner "Bot Blocker - Apache User-Agent Management"
|
||||
|
||||
# Show status
|
||||
if is_bot_blocking_enabled; then
|
||||
echo -e "Status: ${GREEN}●${NC} Enabled"
|
||||
else
|
||||
echo -e "Status: ${RED}○${NC} Disabled"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo -e "${BOLD}Configuration:${NC}"
|
||||
echo ""
|
||||
echo " 1) Enable Bot Blocking - Block malicious bots and scrapers"
|
||||
echo " 2) Disable Bot Blocking - Remove blocking rules"
|
||||
echo " 3) View Configuration - Show current rules"
|
||||
echo ""
|
||||
echo -e "${BOLD}Maintenance:${NC}"
|
||||
echo ""
|
||||
echo " 4) Test Configuration - Validate Apache syntax"
|
||||
echo " 5) Manage Backups - View and restore backups"
|
||||
echo ""
|
||||
echo " 0) Back to Security Menu"
|
||||
echo ""
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo -n "Select option: "
|
||||
}
|
||||
|
||||
main() {
|
||||
while true; do
|
||||
show_menu
|
||||
read -r choice
|
||||
|
||||
case $choice in
|
||||
1) enable_bot_blocking ;;
|
||||
2) disable_bot_blocking ;;
|
||||
3) view_configuration ;;
|
||||
4) test_configuration ;;
|
||||
5) manage_backups ;;
|
||||
0)
|
||||
clear
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo ""
|
||||
print_error "Invalid option"
|
||||
sleep 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
# Run main menu
|
||||
main
|
||||
Reference in New Issue
Block a user