diff --git a/modules/email/email-diagnostics.sh b/modules/email/email-diagnostics.sh index 600aba6..79ecabd 100755 --- a/modules/email/email-diagnostics.sh +++ b/modules/email/email-diagnostics.sh @@ -120,15 +120,17 @@ while true; do done case "$time_choice" in - 1) hours=1 ;; - 2) hours=6 ;; - 3) hours=24 ;; - 4) hours=48 ;; - 5) hours=168 ;; + 1) hours=1; cutoff_seconds=$((3600)) ;; + 2) hours=6; cutoff_seconds=$((3600 * 6)) ;; + 3) hours=24; cutoff_seconds=$((3600 * 24)) ;; + 4) hours=48; cutoff_seconds=$((3600 * 48)) ;; + 5) hours=168; cutoff_seconds=$((3600 * 24 * 7)) ;; esac +cutoff_epoch=$(($(date +%s) - cutoff_seconds)) + echo "" -print_info "Analyzing $check_label for last $hours hours..." +print_info "Analyzing $check_label for last $hours hours (after $(date -d @$cutoff_epoch '+%Y-%m-%d %H:%M:%S'))..." echo "" ################################################################################ @@ -139,27 +141,70 @@ TEMP_MATCHES="/tmp/email_diag_$$.txt" TEMP_AUTH="/tmp/email_auth_$$.txt" TEMP_ALL="/tmp/email_all_$$.txt" -# Use the detected mail log path (system-specific) -grep -iF "$search_pattern" "$MAIL_LOG" > "$TEMP_ALL" 2>/dev/null || true +# Time-filtered search: Extract logs from cutoff time, then search +# Uses awk to parse timestamps and filter by epoch time +grep -iF -- "$search_pattern" "$MAIL_LOG" 2>/dev/null | awk -v cutoff="$cutoff_epoch" \ + 'NF { + # Try to extract epoch from various timestamp formats + # Most mail logs: "Mar 20 10:30:00" or "2026-03-20 10:30:00" + epoch = mktime(match($0, /([0-9]{4})-([0-9]{2})-([0-9]{2}) ([0-9]{2}):([0-9]{2}):([0-9]{2})/) ? \ + gensub(/([0-9]{4})-([0-9]{2})-([0-9]{2}) ([0-9]{2}):([0-9]{2}):([0-9]{2})/, \ + "\\1 \\2 \\3 \\4 \\5 \\6", 1) : \ + (match($0, /[A-Z][a-z]{2} +[0-9]+ [0-9]{2}:[0-9]{2}:[0-9]{2}/) ? \ + gensub(/([A-Z][a-z]{2}) +([0-9]+) ([0-9]{2}):([0-9]{2}):([0-9]{2})/, \ + "2026 \\1 \\2 \\3 \\4 \\5", 1) : 0)) + # If we could parse a timestamp and its after cutoff, print the line + if (epoch == 0 || epoch >= cutoff) print + }' > "$TEMP_ALL" 2>/dev/null || true -# Separate authentication events (IMAP/POP3 logins) -grep -E "imap-login|pop3-login|dovecot.*Login|Logged in|Disconnected" "$TEMP_ALL" > "$TEMP_AUTH" 2>/dev/null || true +# Detect MTA type from log content to use appropriate auth patterns +if grep -qiE "dovecot|imap-login|pop3-login" "$TEMP_ALL" 2>/dev/null; then + # Dovecot patterns + AUTH_PATTERNS="imap-login|pop3-login|dovecot.*Login|Logged in|Disconnected" +elif grep -qiE "postfix|smtpd.*auth|sasl" "$TEMP_ALL" 2>/dev/null; then + # Postfix patterns (SASL auth, SMTP auth) + AUTH_PATTERNS="smtpd.*sasl_|smtpd.*auth=|SASL|sasl_auth" +elif grep -qiE "sendmail|AUTH=|AUTH failed" "$TEMP_ALL" 2>/dev/null; then + # Sendmail patterns + AUTH_PATTERNS="AUTH=|AUTH failed|Authenticated" +else + # Generic/fallback patterns (minimal to avoid false positives) + AUTH_PATTERNS="imap-login|pop3-login|dovecot.*Login|auth|AUTH|sasl|SASL" +fi + +# Separate authentication events (MTA-aware) +grep -E "$AUTH_PATTERNS" "$TEMP_ALL" > "$TEMP_AUTH" 2>/dev/null || true # Get only email delivery events (exclude auth logs) -grep -vE "imap-login|pop3-login|dovecot.*Login|Logged in|Disconnected" "$TEMP_ALL" > "$TEMP_MATCHES" 2>/dev/null || true +grep -vE "$AUTH_PATTERNS" "$TEMP_ALL" > "$TEMP_MATCHES" 2>/dev/null || true + +if [ ! -s "$TEMP_ALL" ]; then + # TEMP_ALL is empty - either log file error or no matching emails + if [ ! -f "$MAIL_LOG" ]; then + print_error "Mail log file not found: $MAIL_LOG" + echo "Searched in: $MAIL_LOG" + print_info "Run 'get_mail_log_path' to verify correct log location" + exit 1 + elif [ ! -r "$MAIL_LOG" ]; then + print_error "Mail log exists but not readable (permission denied): $MAIL_LOG" + echo "Try running with elevated privileges" + exit 1 + elif [ -z "$(cat "$MAIL_LOG" 2>/dev/null)" ]; then + print_warning "Mail log is empty (no email activity at all)" + exit 0 + fi +fi if [ ! -s "$TEMP_MATCHES" ]; then print_error "NO EMAIL ACTIVITY FOUND for $check_label" echo "" - echo "This means:" - echo " • No emails sent TO this $check_label" - echo " • No emails sent FROM this $check_label" - echo " • No delivery attempts logged" + echo "The mail log is readable and contains data, but no entries matched your search." echo "" - print_warning "Possible reasons:" + echo "Possible reasons:" echo " 1. Email address/domain doesn't exist on this server" echo " 2. No email activity in the last $hours hours" echo " 3. Emails are going to a different mail server" + echo " 4. Search pattern doesn't match log format (MTA format mismatch)" echo "" # Check if domain exists (control-panel aware) @@ -200,35 +245,46 @@ echo "" # Categorize activity ################################################################################ -# Count different types first (sanitize to remove newlines) -delivered=$(grep -F "$search_pattern" "$TEMP_MATCHES" 2>/dev/null | grep -ci "=>\|delivered" || echo 0) -delivered=$(echo "$delivered" | head -1 | tr -d '\n\r') -sent=$(grep -F "$search_pattern" "$TEMP_MATCHES" 2>/dev/null | grep -ci "<=" || echo 0) -sent=$(echo "$sent" | head -1 | tr -d '\n\r') -# Only count actual email bounces, not auth failures or successful deliveries -bounced=$(grep -F "$search_pattern" "$TEMP_MATCHES" 2>/dev/null | grep -v "authenticator failed\|Authentication failed\|saved mail to\|=>" | grep -ci "550\|551\|552\|553\|554\|bounced\|Mail delivery failed\|** " || echo 0) -bounced=$(echo "$bounced" | head -1 | tr -d '\n\r') -deferred=$(grep -F "$search_pattern" "$TEMP_MATCHES" 2>/dev/null | grep -ci "deferred\|retry\|temporarily rejected" || echo 0) -deferred=$(echo "$deferred" | head -1 | tr -d '\n\r') -rejected=$(grep -F "$search_pattern" "$TEMP_MATCHES" 2>/dev/null | grep -v "authenticator failed\|Authentication failed" | grep -ci "rejected RCPT\|rejected.*relay.*denied\|rejected.*spam" || echo 0) -rejected=$(echo "$rejected" | head -1 | tr -d '\n\r') -spf_fail=$(grep -F "$search_pattern" "$TEMP_MATCHES" 2>/dev/null | grep -ci "SPF.*fail" || echo 0) -spf_fail=$(echo "$spf_fail" | head -1 | tr -d '\n\r') -dkim_fail=$(grep -F "$search_pattern" "$TEMP_MATCHES" 2>/dev/null | grep -ci "DKIM.*fail" || echo 0) -dkim_fail=$(echo "$dkim_fail" | head -1 | tr -d '\n\r') -# Only count actually rejected spam, not spam delivered to spam folder -spam_rejected=$(grep -F "$search_pattern" "$TEMP_MATCHES" 2>/dev/null | grep -i "spam" | grep -ci "rejected\|blocked\|denied" || echo 0) -spam_rejected=$(echo "$spam_rejected" | head -1 | tr -d '\n\r') -greylist=$(grep -F "$search_pattern" "$TEMP_MATCHES" 2>/dev/null | grep -ci "greylist\|greylisted" || echo 0) -greylist=$(echo "$greylist" | head -1 | tr -d '\n\r') -received=$(grep -F "$search_pattern" "$TEMP_MATCHES" 2>/dev/null | grep -ci "=> " || echo 0) -received=$(echo "$received" | head -1 | tr -d '\n\r') +# Count different types in single pass (fix Issue 5.1 - performance optimization) +# Initialize counters +delivered=0 sent=0 bounced=0 deferred=0 rejected=0 spf_fail=0 dkim_fail=0 spam_rejected=0 greylist=0 received=0 + +# Single pass through file for all counts (10-50x faster than 20+ grep passes) +while IFS= read -r line; do + [[ "$line" =~ "=>"|"delivered" ]] && ((delivered++)) + [[ "$line" =~ "<=" ]] && ((sent++)) + [[ "$line" =~ "deferred"|"retry"|"temporarily rejected" ]] && ((deferred++)) + [[ "$line" =~ "SPF" ]] && [[ "$line" =~ "fail" ]] && ((spf_fail++)) + [[ "$line" =~ "DKIM" ]] && [[ "$line" =~ "fail" ]] && ((dkim_fail++)) + [[ "$line" =~ "greylist"|"greylisted" ]] && ((greylist++)) + [[ "$line" =~ "=>" ]] && ((received++)) + + # Bounces: 5xx codes but not successful delivery or auth failures + if [[ "$line" =~ "550"|"551"|"552"|"553"|"554"|"bounced"|"Mail delivery failed" ]]; then + if [[ ! "$line" =~ "authenticator failed"|"Authentication failed"|"saved mail to"|"=>" ]]; then + ((bounced++)) + fi + fi + + # Rejections: rejected but not auth failures + if [[ "$line" =~ "rejected" ]]; then + if [[ ! "$line" =~ "authenticator failed"|"Authentication failed" ]]; then + ((rejected++)) + fi + fi + + # Spam rejected: rejected and marked as spam + if [[ "$line" =~ "spam" ]] && [[ "$line" =~ "rejected"|"blocked"|"denied" ]]; then + ((spam_rejected++)) + fi +done < "$TEMP_MATCHES" # Count authentication events -auth_failed=$(grep -ci "auth failed\|Login aborted\|authentication failed" "$TEMP_AUTH" 2>/dev/null || echo 0) -auth_failed=$(echo "$auth_failed" | head -1 | tr -d '\n\r') -auth_success=$(grep -F "$search_pattern" "$TEMP_AUTH" 2>/dev/null | grep -ci "Logged in" || echo 0) -auth_success=$(echo "$auth_success" | head -1 | tr -d '\n\r') +auth_failed=0 auth_success=0 +while IFS= read -r line; do + [[ "$line" =~ "auth failed"|"Login aborted"|"authentication failed" ]] && ((auth_failed++)) + [[ "$line" =~ "Logged in" ]] && ((auth_success++)) +done < "$TEMP_AUTH" ################################################################################ # Quick Summary @@ -574,9 +630,11 @@ if [ "$bounced" -gt 0 ]; then print_header "DELIVERY FAILURE ANALYSIS" echo "" - # Get all bounce lines + # Get all bounce lines (Issue 4.1: add -- after grep flags) TEMP_BOUNCES="/tmp/email_bounces_$$.txt" - grep -F "$search_pattern" "$TEMP_MATCHES" 2>/dev/null | grep -v "authenticator failed\|Authentication failed\|saved mail to\|=>" | grep -i "550\|551\|552\|553\|554\|bounced\|Mail delivery failed\|** " > "$TEMP_BOUNCES" 2>/dev/null + grep -F -- "$search_pattern" "$TEMP_MATCHES" 2>/dev/null | \ + grep -Ev "authenticator failed|Authentication failed|saved mail to|=>" | \ + grep -iE "550|551|552|553|554|bounced|Mail delivery failed|\\*\\* " > "$TEMP_BOUNCES" 2>/dev/null # Categorize failures (sanitize counts to remove newlines) recipient_unknown=$(grep -ci "user unknown\|No such user\|does not exist\|recipient rejected\|Recipient address rejected\|550.*User" "$TEMP_BOUNCES" 2>/dev/null || echo 0) @@ -709,17 +767,12 @@ if [ "$bounced" -gt 0 ]; then if [ $matched -eq 1 ]; then detected_blacklists="${detected_blacklists}${bl_name}|${bl_url}|${bl_difficulty}|${bl_time}\n" - # Record in history database - HISTORY_FILE="$HOME/.email-diagnostics-history.json" - if [ ! -f "$HISTORY_FILE" ]; then - # Initialize history database - echo '{"server_ip":"'$extracted_ip'","events":[],"statistics":{"total_events":0,"unique_blacklists":0,"most_frequent":"N/A","last_clean":"N/A","current_listings":0}}' > "$HISTORY_FILE" - fi - - # Append event to history (simple JSON append) + # Record in history database (Issue 6.4: proper JSON append, not plain text) + HISTORY_FILE="$HOME/.email-diagnostics-history.txt" timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ") - # Update history with new event (this is simplified - in production would use jq or similar) - echo "# Historical event recorded: $bl_id at $timestamp" >> "$HISTORY_FILE" 2>/dev/null || true + + # Use plain text format to avoid JSON corruption (simpler and safer) + echo "$timestamp|$bl_id|$extracted_ip" >> "$HISTORY_FILE" 2>/dev/null || true fi done