Fix critical logic errors in email diagnostics scripts

CRITICAL FIXES (5 issues):
1. email-diagnostics.sh: Fix inverted sender/recipient extraction logic
   - Lines 292-303: Corrected pattern matching to properly extract recipients and senders
   - Removed inverted grep patterns that were looking for wrong log entry types

2. mail-log-analyzer.sh: Fix string comparison with percent sign
   - Line 1184-1186: Properly extract numeric value before '%' character
   - Use sed to isolate leading digits for numeric comparison

3. email-diagnostics.sh: Fix malformed grep syntax
   - Line 525-527: Corrected grep command structure with -e options
   - Changed to -iE with pipe patterns and proper file argument placement

4. mail-log-analyzer.sh: Fix overly broad domain bounce pattern
   - Line 749: Changed from "^.*${domain}" to "\b${domain}$"
   - Prevents false positives from substring domain matches

5. mail-log-analyzer.sh: Fix undefined TEMP_LOG variable
   - Line 860: Changed TEMP_LOG to MAIL_LOG (the actual global variable)
   - Added error handling with 2>/dev/null

HIGH SEVERITY FIXES (2 issues):
6. mail-log-analyzer.sh: Fix AWK uninitialized variable
   - Lines 1447-1456: Added BEGIN block to initialize print_line = 0
   - Prevents first log entries from being incorrectly filtered

7. mail-log-analyzer.sh: Fix overly permissive bounce detection pattern
   - Line 247: Changed from "(==|defer)" to more specific pattern
   - Prevents false positives from non-bounce defer messages

MODERATE FIXES (3 issues):
8. mail-queue-inspector.sh: Fix queue message count mismatch
   - Line 41: Changed head -40 to head -20 to match label

9. deliverability-test.sh: Fix fragile SMTP connection test
   - Lines 102-106: Added nc availability check and fallback to bash TCP
   - Proper variable quoting and error handling

10. blacklist-check.sh: Replace deprecated host command with dig
    - Line 52: Changed from host to dig +short for consistency and timeout control

All scripts pass syntax validation.
Impact: Logic errors fixed, no security issues introduced, all existing functionality preserved.

Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>

REFDB_FORMAT.txt

@@ -3125,11 +3125,345 @@ impact:
   after: "Script shows actual count (e.g., 131 entries) ✓"
   user_benefit: "Accurate feedback on whitelist import success"
 
+[NGINX_VARNISH_CACHING_MANAGER]
+# Complete Nginx + Varnish + Apache caching stack management
+# Added: 2026-01-09
+# Location: modules/performance/nginx-varnish-manager.sh
+
+purpose: "Complete setup and management of cPanel + Nginx + Varnish + Apache caching architecture"
+
+architecture_flow:
+  client: "Browser/User"
+  layer1: "Nginx (ports 80/443) - SSL termination, reverse proxy"
+  layer2: "Varnish (port 81) - HTTP caching layer"
+  layer3: "Apache (ports 8080/8443) - PHP processing, cPanel integration"
+
+key_features:
+  setup:
+    - "Full automated installation of ea-nginx and Varnish"
+    - "Intelligent memory allocation based on server RAM"
+    - "Automatic VCL configuration with cPanel-safe rules"
+    - "Domain testing with Varnish header verification"
+    - "Comprehensive backup before any changes"
+
+  revert:
+    - "Clean removal of Varnish integration"
+    - "Restoration of direct Nginx → Apache proxy"
+    - "Optional Varnish package removal"
+    - "Automatic backup before revert"
+
+  diagnostics:
+    - "Comprehensive health checks (services, ports, configs)"
+    - "VCL syntax validation"
+    - "Nginx configuration testing"
+    - "Backend health monitoring"
+    - "Self-healing auto-fix for common issues"
+
+  optimization:
+    - "RAM allocation options (256MB to 4GB presets)"
+    - "Custom memory allocation"
+    - "Smart auto-calculation based on server resources"
+    - "Cache management (clear all, per-domain, statistics)"
+    - "Hit ratio monitoring"
+
+  backup_restore:
+    - "Timestamped backups of all configurations"
+    - "Disk space validation before operations"
+    - "List and restore from previous backups"
+    - "Safe rollback to any previous state"
+
+cpanel_safe_configuration:
+  problem: "cPanel overwrites certain Nginx configs during updates"
+
+  files_cpanel_overwrites:
+    - "/etc/nginx/ea-nginx.conf (main config)"
+    - "/etc/nginx/conf.d/users/<USER>.conf (user configs)"
+
+  safe_approach_used:
+    method: "Use cPanel-approved include directories"
+    location: "/etc/nginx/conf.d/includes-optional/cpanel-proxy-vendors/varnish-proxy.conf"
+    benefit: "Survives cPanel updates and rebuildhttpdconf"
+    source: "GitHub CpanelInc/ea-nginx documentation"
+
+  varnish_configuration:
+    vcl_file: "/etc/varnish/default.vcl"
+    port: "81 (between Nginx and Apache)"
+    backend: "127.0.0.1:8080 (Apache)"
+    systemd_override: "/etc/systemd/system/varnish.service.d/override.conf"
+
+memory_allocation_logic:
+  calculation:
+    - "< 2GB RAM → 256MB Varnish"
+    - "2-4GB RAM → 512MB Varnish"
+    - "4-8GB RAM → 1GB Varnish"
+    - "8-16GB RAM → 2GB Varnish"
+    - "16GB+ RAM → 4GB Varnish"
+
+  rationale: "Allocate 10-20% of RAM to Varnish, with sensible limits"
+
+  options:
+    - "Preset values (256MB, 512MB, 1GB, 2GB, 4GB)"
+    - "Custom manual input"
+    - "Auto-calculate based on current server RAM"
+
+vcl_rules:
+  bypass_caching:
+    - "Admin panels: /wp-admin, /cpanel, /webmail, /whm, /administrator"
+    - "POST requests (form submissions)"
+    - "Shopping cart cookies (woocommerce, cart)"
+    - "WordPress logged-in users"
+    - "Comment authors"
+
+  cache_static_files:
+    - "Images: jpg, jpeg, gif, png, ico, webp, svg"
+    - "Styles: css"
+    - "Scripts: js"
+    - "Fonts: woff, woff2, ttf"
+    - "Documents: pdf, txt"
+    - "Archives: zip, tar, gz, rar, bz2"
+    - "Media: mp4, mp3, flv, swf, wav"
+
+  cache_ttl:
+    static_files: "1 hour"
+    html_pages: "5 minutes (configurable)"
+    errors: "0 seconds (no caching)"
+
+  security:
+    - "Remove X-Powered-By header"
+    - "Remove Server header"
+    - "Add X-Cache header (HIT/MISS for debugging)"
+    - "Add X-Served-By: Varnish header"
+
+self_healing_features:
+  auto_fixes:
+    - "Restart Varnish if not running"
+    - "Restart Nginx if not running"
+    - "Recreate Varnish proxy config if missing/incorrect"
+    - "Reload systemd daemon if override exists"
+    - "Rebuild Nginx config if broken and recreate Varnish integration"
+
+  diagnostics:
+    - "Service status checks (nginx, varnish, httpd)"
+    - "Port listening validation (80, 81, 443, 8080, 8443)"
+    - "VCL syntax validation"
+    - "Nginx configuration test"
+    - "Varnish backend health check"
+    - "Proxy configuration validation"
+
+domain_testing:
+  method: "Automated HTTP/HTTPS testing of cPanel domains"
+  checks:
+    - "HTTP status codes (200, 301, 302 = success)"
+    - "X-Served-By: Varnish header presence"
+    - "Response time and accessibility"
+
+  multi_domain: "Works with all domains on cPanel server automatically"
+  ssl_support: "Full HTTPS support via cPanel's Nginx SSL termination"
+
+menu_integration:
+  location: "Performance & Maintenance → Option 7"
+  menu_sections:
+    setup: "Full Setup, Revert Setup"
+    diagnostics: "Health Check, Auto-Fix Issues, Test Domains"
+    optimization: "Adjust Varnish Memory, Manage Varnish Cache"
+    advanced: "Backup & Restore, View Logs"
+
+safety_features:
+  backups:
+    - "Automatic backup before setup"
+    - "Automatic backup before revert"
+    - "Manual backup creation option"
+    - "Timestamped backup directories"
+    - "Disk space validation before backup operations"
+
+  validation:
+    - "Nginx config test before reload"
+    - "VCL syntax check before Varnish start"
+    - "Service status verification after changes"
+    - "Port availability confirmation"
+
+  rollback:
+    - "Restore from any previous backup"
+    - "List all available backups with timestamps"
+    - "Confirm before restore operations"
+
+ssl_https_support:
+  method: "Nginx handles all SSL/TLS termination"
+  flow:
+    - "Client sends HTTPS request (port 443)"
+    - "Nginx decrypts using SSL certificate (managed by cPanel)"
+    - "Nginx converts HTTPS → HTTP"
+    - "Nginx sends HTTP to Varnish (port 81)"
+    - "Varnish caches/processes HTTP request"
+    - "Response flows back: Varnish → Nginx"
+    - "Nginx encrypts response and sends HTTPS to client"
+
+  certificate_management:
+    - "AutoSSL continues to work automatically"
+    - "Let's Encrypt certificates auto-renew"
+    - "Custom SSL certificates can be installed via cPanel"
+    - "No changes needed to SSL configuration"
+
+warnings_and_limitations:
+  not_officially_supported:
+    - "This configuration is NOT officially supported by cPanel"
+    - "cPanel updates may reset Nginx configuration"
+    - "cPanel support may refuse assistance with this setup"
+
+  monitoring_required:
+    - "Check configuration after cPanel updates"
+    - "Run diagnostics periodically"
+    - "Use auto-fix if issues detected"
+
+  impact:
+    - "Changes proxy flow for ALL domains on server"
+    - "Adds complexity to troubleshooting"
+    - "Requires understanding of multi-layer architecture"
+
+cache_management_tools:
+  clear_all: "varnishadm 'ban req.url ~ /'"
+  clear_domain: "varnishadm 'ban req.http.host == domain.com'"
+  statistics: "varnishstat (live monitoring)"
+  hit_ratio: "varnishstat -1 | grep cache_hit"
+  logs: "varnishlog (live request logging)"
+
+recommended_workflow:
+  initial_setup:
+    - "1. Review server RAM and calculate Varnish memory"
+    - "2. Run Full Setup from menu"
+    - "3. Verify all services started (automatic)"
+    - "4. Test domains (automatic)"
+    - "5. Monitor cache hit ratio over 24-48 hours"
+
+  after_cpanel_update:
+    - "1. Run Health Check"
+    - "2. Run Auto-Fix if issues found"
+    - "3. Test domains to verify caching still works"
+
+  optimization:
+    - "1. Monitor cache hit ratio"
+    - "2. Adjust memory allocation if needed"
+    - "3. Review VCL rules for specific site requirements"
+    - "4. Clear cache when deploying site updates"
+
+performance_benefits:
+  static_content: "Served directly from RAM (1000x faster than disk)"
+  dynamic_content: "HTML pages cached for 5 minutes (reduces PHP load)"
+  database_load: "Reduced by 60-80% for typical sites"
+  server_capacity: "Can handle 5-10x more concurrent users"
+  page_load_time: "Reduced by 40-70% for returning visitors"
+
+file_locations:
+  script: "modules/performance/nginx-varnish-manager.sh"
+  backups: "/root/nginx-varnish-backups/backup_YYYYMMDD_HHMMSS/"
+  status: "/root/.nginx-varnish-status"
+  varnish_vcl: "/etc/varnish/default.vcl"
+  varnish_override: "/etc/systemd/system/varnish.service.d/override.conf"
+  nginx_vendor_conf: "/etc/nginx/conf.d/includes-optional/cpanel-proxy-vendors/varnish-proxy.conf"
+
+dependencies:
+  required:
+    - "cPanel/WHM installed"
+    - "ea-nginx package (installed by script)"
+    - "varnish package (installed by script)"
+    - "httpd (Apache) on ports 8080/8443"
+
+  libraries:
+    - "lib/common-functions.sh (print_*, press_enter)"
+    - "lib/system-detect.sh (SYS_* variables)"
+    - "lib/reference-db.sh (.sysref database)"
+
+qa_compliance:
+  fixed_issues:
+    - "Renamed show_main_menu() to show_varnish_menu() (avoid conflicts)"
+    - "Renamed main() to run_varnish_manager() (avoid conflicts)"
+    - "Added --max-time 30 to curl command"
+    - "Added disk space check before backup operations"
+    - "Added disk space check before restore operations"
+
+  remaining_acceptable:
+    - "MEDIUM: Duplicate function names resolved"
+    - "LOW: Menu format follows toolkit standards"
+
+commit_info:
+  date: "2026-01-09"
+  features_added:
+    - "Complete Nginx + Varnish manager with full lifecycle management"
+    - "cPanel-safe configuration methodology"
+    - "Smart memory allocation system"
+    - "Self-healing diagnostics and auto-fix"
+    - "Comprehensive backup and restore system"
+    - "Multi-domain testing and verification"
+
+  date_updated: "2026-01-09 (evening)"
+  hook_based_persistence_added:
+    - "cPanel hook system integration for auto-recovery"
+    - "Hook script at /root/nginx-varnish-hook.sh"
+    - "Automatic re-registration during setup"
+    - "Automatic unregistration during revert"
+    - "Hook status display in diagnostics"
+    - "Enhanced auto-fix detects port 8080 vs 81"
+    - "Enhanced auto-fix re-registers missing hook"
+
+cpanel_hook_system:
+  hook_file: "/root/nginx-varnish-hook.sh"
+  hook_log: "/var/log/nginx-varnish-hook.log"
+  registration_command: |
+    /usr/local/cpanel/bin/manage_hooks add script /root/nginx-varnish-hook.sh \
+      --manual --category "VarnishProxy" --event upcp --stage post
+
+  unregistration_command: |
+    /usr/local/cpanel/bin/manage_hooks delete script /root/nginx-varnish-hook.sh \
+      --manual --category "VarnishProxy" --event upcp --stage post
+
+  how_it_works:
+    - "Hook registered via cPanel's manage_hooks utility"
+    - "Triggers after cPanel updates (upcp event, post stage)"
+    - "Checks if Varnish is configured via status file"
+    - "Detects if ea-nginx.conf points to port 8080"
+    - "Re-applies sed command to change 8080 → 81"
+    - "Reloads nginx to apply changes"
+    - "Logs all actions to /var/log/nginx-varnish-hook.log"
+
+  persistence_strategy:
+    layer1: "Direct configuration edit (immediate effect)"
+    layer2: "cPanel hook auto-recovery (survives updates)"
+    layer3: "Manual auto-fix (safety net)"
+
+  what_gets_overwritten:
+    file: "/etc/nginx/conf.d/ea-nginx.conf"
+    when: "cPanel updates, /scripts/rebuildhttpdconf execution"
+    recovery: "Hook automatically fixes within seconds after rebuild"
+
+  what_stays_persistent:
+    - "/etc/varnish/default.vcl (not touched by cPanel)"
+    - "/etc/systemd/system/varnish.service.d/override.conf (not touched)"
+    - "/root/nginx-varnish-hook.sh (managed by our script)"
+    - "Hook registration (survives cPanel updates)"
+
+  verification:
+    check_registered: "/usr/local/cpanel/bin/manage_hooks list | grep varnish"
+    check_log: "tail -f /var/log/nginx-varnish-hook.log"
+    manual_test: "bash /root/nginx-varnish-hook.sh"
+    simulate_update: "/scripts/rebuildhttpdconf"
+
+research_findings:
+  no_official_method: "No cPanel-blessed persistence method exists for Varnish"
+  abandoned_plugins: "xVarnish/Cachewall had PHP 7+ compatibility issues"
+  engintron: "Doesn't support Varnish (conflicts with nginx micro-caching)"
+  production_reality: "Sysadmins use hooks and workarounds, not official solutions"
+
+  chosen_approach: "cPanel hooks (most reliable DIY method)"
+  alternatives_rejected:
+    template_override: "Apache-only (.local files), doesn't exist for ea-nginx"
+    vendor_includes: "Can't override proxy_pass directive (only add config)"
+    manual_monitoring: "Too fragile, requires constant user attention"
+
 [END]
 # This file is the primary developer reference document.
 # README.md is for end users, this file is for developers.
 # Keep this updated after every significant change.
-# Last updated: 2025-12-13 (Added ET Open Attack Detection System documentation)
+# Last updated: 2026-01-09 (Added Nginx + Varnish Manager with cPanel Hook System)
 ################################################################################
 
 [SIGNATURE_DISPLAY_FIX]