diff --git a/modules/website/lib/extended-analysis-functions.sh b/modules/website/lib/extended-analysis-functions.sh index 8f163cc..e4d69a7 100644 --- a/modules/website/lib/extended-analysis-functions.sh +++ b/modules/website/lib/extended-analysis-functions.sh @@ -537,8 +537,201 @@ export -f analyze_gzip_compression export -f analyze_ssl_version export -f analyze_apache_modules export -f analyze_wordpress_cron +################################################################################ +# PHASE 4: ADVANCED DATABASE & ISSUE PATTERN DETECTION (Tier 1 Quick Wins) +################################################################################ + +# ============================================================================ +# PHASE 4 DATABASE CHECKS (6 quick wins) +# ============================================================================ + +### P4.1 - Table Engine Mismatch +analyze_table_engine_mismatch() { + # Check if tables use inconsistent storage engines + local engines=$(mysql -e "SELECT DISTINCT ENGINE FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() 2>/dev/null" 2>/dev/null | grep -v ENGINE | sort -u) + + if [ ! -z "$engines" ] && [ "$(echo "$engines" | wc -l)" -gt 1 ]; then + save_analysis_data "database_advanced.tmp" "WARNING: Mixed storage engines detected" + save_analysis_data "database_advanced.tmp" " Engines: $(echo $engines | tr '\n' ', ')" + save_analysis_data "database_advanced.tmp" " Recommendation: Convert all tables to InnoDB" + fi +} + +### P4.2 - Table Statistics Age +analyze_table_statistics_age() { + # Check if table statistics are stale (MySQL 5.7+) + local stale_count=$(mysql -e "SELECT COUNT(*) FROM mysql.innodb_table_stats WHERE STAT_MODIFIED < DATE_SUB(NOW(), INTERVAL 30 DAY)" 2>/dev/null | tail -1 || echo 0) + + if [ "$stale_count" -gt 0 ]; then + save_analysis_data "database_advanced.tmp" "INFO: Found $stale_count tables with stale statistics" + save_analysis_data "database_advanced.tmp" " Fix: Run ANALYZE TABLE on each table" + save_analysis_data "database_advanced.tmp" " Or: wp db optimize (WordPress)" + fi +} + +### P4.3 - Index Cardinality Analysis +analyze_index_cardinality() { + # Check for indexes with poor selectivity + local poor_indexes=$(mysql -e "SELECT TABLE_NAME, INDEX_NAME FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND SEQ_IN_INDEX=1 AND CARDINALITY IS NOT NULL AND CARDINALITY / (SELECT TABLE_ROWS FROM information_schema.TABLES t WHERE t.TABLE_SCHEMA=information_schema.STATISTICS.TABLE_SCHEMA AND t.TABLE_NAME=information_schema.STATISTICS.TABLE_NAME) > 0.95 LIMIT 5" 2>/dev/null | grep -v TABLE_NAME | head -5) + + if [ ! -z "$poor_indexes" ]; then + save_analysis_data "database_advanced.tmp" "WARNING: Found indexes with poor cardinality (high selectivity)" + save_analysis_data "database_advanced.tmp" " These indexes may not be used by optimizer" + save_analysis_data "database_advanced.tmp" " Review and consider dropping unused indexes" + fi +} + +### P4.4 - Query Cache Memory Waste +analyze_query_cache_memory_waste() { + # Check query cache fragmentation (MySQL 5.7) + local qcache_info=$(mysql -e "SHOW STATUS LIKE 'Qcache%'" 2>/dev/null) + + if echo "$qcache_info" | grep -q "Qcache_free_blocks"; then + local free_blocks=$(echo "$qcache_info" | grep "Qcache_free_blocks" | awk '{print $2}') + local total_blocks=$(echo "$qcache_info" | grep "Qcache_total_blocks" | awk '{print $2}') + + if [ "$total_blocks" -gt 0 ]; then + local fragmentation=$((free_blocks * 100 / total_blocks)) + if [ "$fragmentation" -gt 30 ]; then + save_analysis_data "database_advanced.tmp" "INFO: Query cache fragmentation at ${fragmentation}%" + save_analysis_data "database_advanced.tmp" " Consider: FLUSH QUERY CACHE; or redesign query strategy" + fi + fi + fi +} + +### P4.5 - Replication Lag Check +analyze_replication_lag() { + # Check if database is replica and has lag + local slave_status=$(mysql -e "SHOW SLAVE STATUS\G 2>/dev/null" | grep "Seconds_Behind_Master" | awk '{print $NF}') + + if [ ! -z "$slave_status" ] && [ "$slave_status" != "NULL" ] && [ "$slave_status" -gt 10 ]; then + save_analysis_data "database_advanced.tmp" "WARNING: Database replication lag detected: ${slave_status} seconds" + save_analysis_data "database_advanced.tmp" " Impact: Read queries on replica are stale" + save_analysis_data "database_advanced.tmp" " Solution: Optimize master, increase replica resources" + fi +} + +### P4.6 - Table Size Growth Tracking +analyze_table_size_growth() { + # Identify rapidly growing tables (potential logging tables) + local large_tables=$(mysql -e "SELECT TABLE_NAME, ROUND(((data_length+index_length)/1024/1024),2) as size_mb FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() ORDER BY size_mb DESC LIMIT 5" 2>/dev/null | grep -v TABLE_NAME) + + if [ ! -z "$large_tables" ]; then + local largest=$(echo "$large_tables" | head -1 | awk '{print $1, $2}') + if echo "$largest" | awk '{exit ($2 > 1000) ? 0 : 1}'; then + save_analysis_data "database_advanced.tmp" "WARNING: Large table detected: $largest MB" + save_analysis_data "database_advanced.tmp" " If this is wp_postmeta or wp_options, consider archiving old data" + fi + fi +} + +# ============================================================================ +# PHASE 4 SYSTEM & ERROR PATTERN CHECKS (6 quick wins) +# ============================================================================ + +### P4.7 - Timeout Error Detection +analyze_timeout_errors() { + # Count timeout errors in recent logs + local error_log=$(find /var/log -name "error.log" -o -name "php-fpm.log" 2>/dev/null | head -1) + + if [ -f "$error_log" ]; then + local timeout_count=$(tail -1000 "$error_log" 2>/dev/null | grep -ci "timeout\|timed out" || echo 0) + + if [ "$timeout_count" -gt 10 ]; then + save_analysis_data "error_patterns.tmp" "WARNING: Found $timeout_count timeout errors in recent logs" + save_analysis_data "error_patterns.tmp" " Impact: Customers experiencing connection/processing failures" + save_analysis_data "error_patterns.tmp" " Solutions: Increase timeouts, optimize code, add resources" + fi + fi +} + +### P4.8 - Memory Exhaustion Attempts +analyze_memory_exhaustion_attempts() { + # Detect PHP memory limit hits + local error_log=$(find /var/log -name "error.log" -o -name "php-fpm.log" 2>/dev/null | head -1) + + if [ -f "$error_log" ]; then + local memory_count=$(tail -1000 "$error_log" 2>/dev/null | grep -ci "allowed memory\|memory.*exhausted" || echo 0) + + if [ "$memory_count" -gt 0 ]; then + save_analysis_data "error_patterns.tmp" "CRITICAL: PHP hitting memory limits ($memory_count times in logs)" + save_analysis_data "error_patterns.tmp" " Impact: Some requests failing with fatal error" + save_analysis_data "error_patterns.tmp" " Fix: Increase memory_limit in php.ini" + fi + fi +} + +### P4.9 - Disk Inode Usage +analyze_disk_inode_usage() { + # Check filesystem inode exhaustion (causes performance degradation) + local inode_usage=$(df -i / 2>/dev/null | tail -1 | awk '{print int($5)}' || echo 0) + + if [ "$inode_usage" -gt 80 ]; then + save_analysis_data "system_resources.tmp" "WARNING: Disk inode usage at ${inode_usage}%" + save_analysis_data "system_resources.tmp" " Impact: Filesystem performance degrades, may prevent new files" + save_analysis_data "system_resources.tmp" " Fix: Delete old logs, temporary files, or backups" + fi +} + +### P4.10 - Zombie Process Detection +analyze_zombie_processes() { + # Count zombie/defunct processes + local zombie_count=$(ps aux 2>/dev/null | grep -c " " || echo 0) + + if [ "$zombie_count" -gt 5 ]; then + save_analysis_data "system_resources.tmp" "WARNING: Found $zombie_count zombie processes" + save_analysis_data "system_resources.tmp" " Impact: Wastes process table entries, resource leak" + save_analysis_data "system_resources.tmp" " Fix: Restart PHP-FPM or MySQL to clean up" + fi +} + +### P4.11 - Swap Usage Detection (Critical) +analyze_swap_usage_phase4() { + # Check if system is using swap (massive performance killer) + local swap_used=$(free 2>/dev/null | grep Swap | awk '{print $3}' || echo 0) + + if [ "$swap_used" -gt 0 ]; then + save_analysis_data "system_resources.tmp" "CRITICAL: System using swap ($swap_used KB)" + save_analysis_data "system_resources.tmp" " Impact: 50-100x SLOWER than RAM access" + save_analysis_data "system_resources.tmp" " Emergency: Upgrade RAM or reduce memory usage immediately" + fi +} + +### P4.12 - Load Average Trending +analyze_load_average_trend() { + # Detect increasing load trend (early warning) + local load_1=$(uptime 2>/dev/null | grep -oP 'load average: \K[^,]+' | head -1) + local load_5=$(uptime 2>/dev/null | grep -oP 'load average: \K[^,]+' | tail -2 | head -1) + + if [ ! -z "$load_1" ] && [ ! -z "$load_5" ]; then + local ratio=$(echo "scale=2; $load_1 / $load_5" | bc 2>/dev/null || echo 1) + + if (( $(echo "$ratio > 1.2" | bc -l) )); then + save_analysis_data "system_resources.tmp" "INFO: Load average trending upward (1min: $load_1, 5min: $load_5)" + save_analysis_data "system_resources.tmp" " Early warning: Monitor for increasing problems" + fi + fi +} + +################################################################################ +# EXPORT ALL PHASE 4 FUNCTIONS +################################################################################ + export -f analyze_backup_schedule export -f analyze_db_optimization_schedule export -f analyze_slow_cron_jobs export -f analyze_missing_critical_indexes export -f analyze_database_memory_ratio +export -f analyze_table_engine_mismatch +export -f analyze_table_statistics_age +export -f analyze_index_cardinality +export -f analyze_query_cache_memory_waste +export -f analyze_replication_lag +export -f analyze_table_size_growth +export -f analyze_timeout_errors +export -f analyze_memory_exhaustion_attempts +export -f analyze_disk_inode_usage +export -f analyze_zombie_processes +export -f analyze_swap_usage_phase4 +export -f analyze_load_average_trend diff --git a/modules/website/lib/remediation-engine.sh b/modules/website/lib/remediation-engine.sh index 5dd6e0b..d7acc2d 100644 --- a/modules/website/lib/remediation-engine.sh +++ b/modules/website/lib/remediation-engine.sh @@ -828,6 +828,201 @@ generate_remediation() { echo " Expected Improvement: 5-10% database performance" ;; + "table_engine_mismatch") + echo -e "${REMEDIATION_WARNING}REMEDIATION: Standardize Database Table Engines${REMEDIATION_NC}" + echo " Current: $finding_value" + echo " Impact: Inconsistent performance, potential compatibility issues" + echo "" + echo " Fix:" + echo " 1. Convert all tables to InnoDB:" + echo " mysql -e \"SELECT CONCAT('ALTER TABLE ', TABLE_SCHEMA, '.', TABLE_NAME, ' ENGINE=InnoDB;') FROM information_schema.TABLES WHERE TABLE_SCHEMA='wordpress' AND ENGINE != 'InnoDB';\" | mysql" + echo "" + echo " 2. Or for WordPress:" + echo " wp db query \"ALTER TABLE wp_posts ENGINE=InnoDB;\"" + echo "" + echo " Expected Improvement: Consistent performance" + ;; + + "table_statistics_stale") + echo -e "${REMEDIATION_INFO}REMEDIATION: Update Database Table Statistics${REMEDIATION_NC}" + echo " Current: Table statistics are outdated" + echo " Impact: Query optimizer makes suboptimal execution plans" + echo "" + echo " Fix:" + echo " 1. Update statistics immediately:" + echo " wp db optimize" + echo "" + echo " 2. Or manually:" + echo " ANALYZE TABLE wp_posts;" + echo " ANALYZE TABLE wp_postmeta;" + echo " ANALYZE TABLE wp_options;" + echo "" + echo " 3. Schedule weekly:" + echo " 0 3 * * 0 wp db optimize" + ;; + + "index_cardinality_poor") + echo -e "${REMEDIATION_WARNING}REMEDIATION: Review and Optimize Indexes${REMEDIATION_NC}" + echo " Current: Indexes with poor selectivity detected" + echo " Impact: Indexes may not be used, wasting space and maintenance time" + echo "" + echo " Fix:" + echo " 1. Find unused indexes:" + echo " SELECT * FROM mysql.innodb_index_stats WHERE STAT_NAME='size' ORDER BY STAT_VALUE DESC;" + echo "" + echo " 2. Drop unused indexes:" + echo " ALTER TABLE {table_name} DROP INDEX {index_name};" + echo "" + echo " 3. Or use WordPress plugin:" + echo " wp plugin install wp-db-optimizer --activate" + ;; + + "query_cache_fragmented") + echo -e "${REMEDIATION_INFO}REMEDIATION: Optimize Query Cache${REMEDIATION_NC}" + echo " Current: Query cache fragmented ($finding_value%)" + echo " Impact: Wasted cache space, slower queries" + echo "" + echo " Fix:" + echo " 1. Clear query cache:" + echo " FLUSH QUERY CACHE;" + echo "" + echo " 2. For MySQL 8.0+: Query cache is removed" + echo " Use Redis or Memcached instead" + echo "" + echo " 3. WordPress caching:" + echo " wp plugin install w3-total-cache --activate" + ;; + + "replication_lag_detected") + echo -e "${REMEDIATION_WARNING}REMEDIATION: Fix Database Replication Lag${REMEDIATION_NC}" + echo " Current: Replica is $finding_value seconds behind master" + echo " Impact: Read queries return stale data" + echo "" + echo " Fix:" + echo " 1. Check replica status:" + echo " SHOW SLAVE STATUS\G" + echo "" + echo " 2. Optimize master writes:" + echo " - Review slow queries on master" + echo " - Add indexes to frequently updated tables" + echo "" + echo " 3. Increase replica resources:" + echo " - More CPU" + echo " - Faster disk" + echo " - More RAM" + ;; + + "table_size_growth_rapid") + echo -e "${REMEDIATION_WARNING}REMEDIATION: Archive or Clean Large Table${REMEDIATION_NC}" + echo " Current: $finding_value" + echo " Impact: Large tables slow down backups, queries, and maintenance" + echo "" + echo " Fix (Choose one):" + echo "" + echo " Option 1: Archive old data" + echo " export data before DELETE" + echo " DELETE FROM {table} WHERE created_date < DATE_SUB(NOW(), INTERVAL 90 DAY);" + echo "" + echo " Option 2: Clean WordPress data" + echo " wp post delete --post_type=revision --force" + echo " wp transient delete-all" + echo "" + echo " Option 3: Use WP-CLI" + echo " wp db optimize" + ;; + + "timeout_errors_found") + echo -e "${REMEDIATION_WARNING}REMEDIATION: Resolve Timeout Errors${REMEDIATION_NC}" + echo " Current: $finding_value timeout errors in recent logs" + echo " Impact: Some customer requests are failing" + echo "" + echo " Fix:" + echo " 1. Increase timeouts:" + echo " Edit /etc/php/*/fpm/php.ini" + echo " Set: max_execution_time = 300" + echo "" + echo " 2. Increase server resources" + echo " - Add more CPU/RAM" + echo " - Optimize slow queries" + echo "" + echo " 3. Configure load balancer timeouts:" + echo " Set to match PHP execution time" + ;; + + "memory_limit_exhausted") + echo -e "${REMEDIATION_CRITICAL}REMEDIATION: Fix PHP Memory Limit Exhaustion${REMEDIATION_NC}" + echo " Current: $finding_value (PHP running out of memory)" + echo " Impact: CRITICAL - Requests failing with fatal errors" + echo "" + echo " Fix:" + echo " 1. Increase PHP memory limit:" + echo " Edit /etc/php/*/fpm/php.ini" + echo " Set: memory_limit = 512M" + echo "" + echo " 2. Restart PHP-FPM:" + echo " systemctl restart php-fpm" + echo "" + echo " 3. Optimize WordPress:" + echo " Deactivate memory-heavy plugins" + echo " Use wp plugin install plugin-name-checker" + echo "" + echo " Expected Improvement: All requests will complete" + ;; + + "inode_usage_critical") + echo -e "${REMEDIATION_WARNING}REMEDIATION: Clean Up Filesystem Inodes${REMEDIATION_NC}" + echo " Current: $finding_value inode usage" + echo " Impact: Filesystem performance degrades, may prevent new files" + echo "" + echo " Fix:" + echo " 1. Delete old log files:" + echo " find /var/log -mtime +30 -delete" + echo "" + echo " 2. Clean temporary files:" + echo " rm -rf /tmp/*" + echo " wp cache flush --all" + echo "" + echo " 3. Remove old backups:" + echo " find /home/*/backup* -mtime +30 -delete" + ;; + + "zombie_processes_high") + echo -e "${REMEDIATION_WARNING}REMEDIATION: Clean Up Zombie Processes${REMEDIATION_NC}" + echo " Current: $finding_value zombie processes detected" + echo " Impact: Resource leak, wastes process table entries" + echo "" + echo " Fix:" + echo " 1. Restart PHP-FPM:" + echo " systemctl restart php-fpm" + echo "" + echo " 2. Restart MySQL:" + echo " systemctl restart mysql" + echo "" + echo " 3. Check for misbehaving code:" + echo " Review recent plugin/theme updates" + echo " Look for infinite loops or resource leaks" + ;; + + "load_average_increasing") + echo -e "${REMEDIATION_INFO}REMEDIATION: Monitor and Optimize Load Average${REMEDIATION_NC}" + echo " Current: Load trending upward ($finding_value)" + echo " Impact: Early warning of performance degradation" + echo "" + echo " Fix:" + echo " 1. Monitor current processes:" + echo " top -b -n 1 | head -15" + echo "" + echo " 2. Check for slow queries:" + echo " mysqldumpslow -s t -t 10 /var/log/mysql/slow-query.log" + echo "" + echo " 3. Optimize or scale:" + echo " - Profile slow processes" + echo " - Optimize database queries" + echo " - Add server resources" + echo "" + echo " Expected Result: Load levels decrease" + ;; + *) echo -e "${REMEDIATION_INFO}REMEDIATION RECOMMENDATION: $check_name${REMEDIATION_NC}" echo " Finding: $finding_value" @@ -1038,6 +1233,90 @@ analyze_findings_for_remediation() { echo "" fi + # ═══════════════════════════════════════════════════════════════════════════════ + # PHASE 4 DETECTION PATTERNS + # ═══════════════════════════════════════════════════════════════════════════════ + + echo -e "${REMEDIATION_CRITICAL}═══ PHASE 4: ADVANCED SYSTEM ISSUES ═══${REMEDIATION_NC}" + echo "" + + # Check for table engine mismatch + if grep -qi "Mixed storage engines\|table.*engine" "$temp_dir"/*.tmp 2>/dev/null; then + generate_remediation "table_engine_mismatch" "InnoDB/MyISAM mix" "WARNING" + remediation_count=$((remediation_count + 1)) + echo "" + fi + + # Check for stale table statistics + if grep -qi "table.*statistics\|stale.*stat" "$temp_dir"/*.tmp 2>/dev/null; then + generate_remediation "table_statistics_stale" "30+ days old" "INFO" + remediation_count=$((remediation_count + 1)) + echo "" + fi + + # Check for poor index cardinality + if grep -qi "index.*cardinality\|poor.*selectivity" "$temp_dir"/*.tmp 2>/dev/null; then + generate_remediation "index_cardinality_poor" "detected" "WARNING" + remediation_count=$((remediation_count + 1)) + echo "" + fi + + # Check for query cache fragmentation + if grep -qi "query.*cache.*fragment\|cache.*bloat" "$temp_dir"/*.tmp 2>/dev/null; then + generate_remediation "query_cache_fragmented" "> 30%" "INFO" + remediation_count=$((remediation_count + 1)) + echo "" + fi + + # Check for replication lag + if grep -qi "replication.*lag\|seconds.*behind" "$temp_dir"/*.tmp 2>/dev/null; then + generate_remediation "replication_lag_detected" "10+ seconds" "WARNING" + remediation_count=$((remediation_count + 1)) + echo "" + fi + + # Check for rapid table growth + if grep -qi "large.*table\|rapid.*growth\|table.*size" "$temp_dir"/*.tmp 2>/dev/null; then + generate_remediation "table_size_growth_rapid" "detected" "WARNING" + remediation_count=$((remediation_count + 1)) + echo "" + fi + + # Check for timeout errors + if grep -qi "timeout.*error\|timed.*out" "$temp_dir"/*.tmp 2>/dev/null; then + generate_remediation "timeout_errors_found" "detected" "WARNING" + remediation_count=$((remediation_count + 1)) + echo "" + fi + + # Check for memory exhaustion + if grep -qi "memory.*exhausted\|allowed.*memory\|out.*of.*memory" "$temp_dir"/*.tmp 2>/dev/null; then + generate_remediation "memory_limit_exhausted" "detected" "CRITICAL" + remediation_count=$((remediation_count + 1)) + echo "" + fi + + # Check for inode usage + if grep -qi "inode.*usage\|inode.*critical" "$temp_dir"/*.tmp 2>/dev/null; then + generate_remediation "inode_usage_critical" "> 80%" "WARNING" + remediation_count=$((remediation_count + 1)) + echo "" + fi + + # Check for zombie processes + if grep -qi "zombie.*process\|defunct" "$temp_dir"/*.tmp 2>/dev/null; then + generate_remediation "zombie_processes_high" "detected" "WARNING" + remediation_count=$((remediation_count + 1)) + echo "" + fi + + # Check for increasing load + if grep -qi "load.*trend\|load.*increasing" "$temp_dir"/*.tmp 2>/dev/null; then + generate_remediation "load_average_increasing" "trending up" "INFO" + remediation_count=$((remediation_count + 1)) + echo "" + fi + if [ $remediation_count -eq 0 ]; then echo -e "${REMEDIATION_SUCCESS}✓ No issues detected! Your site is well optimized.${REMEDIATION_NC}" echo "" diff --git a/modules/website/website-slowness-diagnostics.sh b/modules/website/website-slowness-diagnostics.sh index 6070fb6..1e29366 100755 --- a/modules/website/website-slowness-diagnostics.sh +++ b/modules/website/website-slowness-diagnostics.sh @@ -2401,6 +2401,24 @@ run_diagnostics() { analyze_db_optimization_schedule analyze_slow_cron_jobs + # Phase 4: Advanced Database Analysis (6 checks) + print_section "PHASE 4: ADVANCED DATABASE CHECKS" + analyze_table_engine_mismatch + analyze_table_statistics_age + analyze_index_cardinality + analyze_query_cache_memory_waste + analyze_replication_lag + analyze_table_size_growth + + # Phase 4: System & Error Pattern Detection (6 checks) + print_section "PHASE 4: SYSTEM & ERROR PATTERN CHECKS" + analyze_timeout_errors + analyze_memory_exhaustion_attempts + analyze_disk_inode_usage + analyze_zombie_processes + analyze_swap_usage_phase4 + analyze_load_average_trend + # Generate report print_banner "Generating report..." generate_report