Database convert script

REFDB_FORMAT.txt

@@ -2,14 +2,14 @@
 # SERVER TOOLKIT - DEVELOPER CONTEXT DATABASE
 ################################################################################
 # OPTIMIZED FOR: Fast context loading and code navigation
-# LAST UPDATED: 2025-11-20
+# LAST UPDATED: 2025-12-10
-# VERSION: 2.2.0
+# VERSION: 2.3.0
 # FORMAT: Structured key-value with hierarchical sections
 ################################################################################
 
 [META]
-version: 2.1.0
+version: 2.3.0
-updated: 2025-11-12
+updated: 2025-12-10
 status: production_ready
 base_path: /root/server-toolkit
 entry_point: launcher.sh
@@ -1812,9 +1812,262 @@ integration_with_git:
     - "Cache file lists between runs"
     - "Whitelist mechanism for known false positives"
 
+[UPDATE_2025_12_10_MYSQL_RESTORE_TOOL]
+# New feature: MySQL/MariaDB File-Based Restore to SQL
+# Interactive guided workflow for restoring databases from file-based backups
+
+new_module:
+  file: "modules/backup/mysql-restore-to-sql.sh"
+  menu_location: "Backup & Recovery → Option 11"
+  purpose: "Convert restored MySQL/MariaDB data files to usable .sql dumps"
+
+use_case:
+  scenario: "Restore InnoDB databases from file-based backups (Acronis, Guardian, cloud backups)"
+  problem: "Direct file restoration doesn't work with InnoDB due to tablespace requirements"
+  solution: "Start second MySQL instance with restored files, dump to SQL, import to live"
+
+features:
+  - "Detects MySQL version (MySQL 5.7, MySQL 8.0+, MariaDB)"
+  - "Version-specific validation (ib_logfile0/1 vs #innodb_redo)"
+  - "Auto-creates timestamped restore directory (/home/temp/restoreYYYYMMDD/mysql)"
+  - "Lists exact files needed from backup (with emoji visual markers)"
+  - "Validates data structure before proceeding"
+  - "Checks and fixes file ownership (mysql:mysql)"
+  - "Starts second MySQL instance on separate socket"
+  - "InnoDB error log monitoring (checks for corruption/errors on startup)"
+  - "Automatic error log backup (preserves old logs)"
+  - "Supports InnoDB force recovery modes (0-6)"
+  - "Creates verified SQL dumps with completion markers"
+  - "Comprehensive SQL dump validation (7 checks)"
+  - "Database comparison (table count, size verification)"
+  - "SQL syntax validation"
+  - "Provides clear next-step instructions"
+  - "Cancel buttons at every step (0 to cancel)"
+
+workflow:
+  step_1: "Detect live MySQL data directory and version"
+  step_2: "Create/select restore directory with guided file list"
+  step_3: "User restores required files from backup system"
+  step_4: "Validate restored data structure"
+  step_5: "Fix file ownership if needed"
+  step_6: "Select target database"
+  step_7: "Configure optional settings (ticket number, force recovery)"
+  step_8: "Start second MySQL instance"
+  step_9: "Create SQL dump from second instance"
+  step_10: "Provide import instructions"
+
+required_files_mysql_5_7_mariadb:
+  - "ibdata1 (InnoDB system tablespace)"
+  - "ib_logfile0 (redo log)"
+  - "ib_logfile1 (redo log)"
+  - "mysql/ directory (system database)"
+  - "sys/ directory (optional but recommended)"
+  - "<database_name>/ directory (target database)"
+
+required_files_mysql_8_0_0_to_8_0_29:
+  - "ibdata1 (InnoDB system tablespace)"
+  - "ib_logfile0 (redo log - old format)"
+  - "ib_logfile1 (redo log - old format)"
+  - "#innodb_temp/ directory (optional, temp tablespace)"
+  - "ibtmp1 (optional, global temp tablespace)"
+  - "mysql/ directory (system database)"
+  - "sys/ directory (optional but recommended)"
+  - "<database_name>/ directory (target database)"
+  - "NOTE: performance_schema NOT needed (in-memory only)"
+
+required_files_mysql_8_0_30_plus:
+  critical_change_2022: "MySQL 8.0.30 introduced new redo log architecture"
+  - "ibdata1 (InnoDB system tablespace)"
+  - "#innodb_redo/ directory (NEW FORMAT - contains #ib_redo0, #ib_redo1, #ib_redoN files)"
+  - "#innodb_temp/ directory (optional, contains temp_N.ibt files)"
+  - "ibtmp1 (optional, global temp tablespace)"
+  - "mysql/ directory (system database)"
+  - "sys/ directory (optional but recommended)"
+  - "<database_name>/ directory (target database)"
+  - "NOTE: performance_schema NOT needed (in-memory only)"
+  - "NOTE: NO MORE ib_logfile0/ib_logfile1 in 8.0.30+"
+
+required_files_mysql_9_0_plus:
+  - "Same as MySQL 8.0.30+ (uses #innodb_redo directory)"
+
+technical_details:
+  second_instance_command: |
+    mysqld --datadir=$dir --socket=$dir/socket.mysql --pid-file=$dir/mysql.pid
+    --log-error=$dir/mysql.err --skip-grant-tables --skip-networking --user=mysql
+
+  dump_command: |
+    mysqldump -h localhost -S $dir/socket.mysql --single-transaction database_name > output.sql
+
+  mysqld_binary_detection: |
+    Tries /usr/libexec/mysqld first (RHEL/CentOS), falls back to mysqld
+
+  force_recovery_levels:
+    0: "No force recovery (default)"
+    1: "Ignore corrupt pages"
+    2: "Prevent background operations"
+    3: "Prevent transaction rollbacks"
+    4: "Prevent insert buffer merge"
+    5: "Skip log redo"
+    6: "Skip page checksums"
+
+safety_features:
+  - "Second instance runs on separate socket (doesn't affect live MySQL)"
+  - "Critical safety check: Prevents using /var/lib/mysql as restore directory"
+  - "Trap handler: Cleanup on Ctrl+C, interruption, or exit (cleanup_on_exit)"
+  - "Disk space validation: Checks available space before creating dump (2x estimated size)"
+  - "Force recovery warnings: Levels 5-6 require explicit risk acknowledgment"
+  - "All operations validated before execution"
+  - "InnoDB error log parsing (detects corruption during startup)"
+  - "Automatic shutdown if InnoDB errors detected"
+  - "SQL dump integrity validation (7-point check)"
+  - "Database comparison (table count, size ratio verification)"
+  - "Clear error messages with troubleshooting steps"
+  - "Automatic cleanup of second instance on completion"
+  - "Dump verification (checks for 'Dump completed' marker)"
+  - "No eval usage: Uses bash array for mysqld arguments (security)"
+
+validation_checks_innodb_startup:
+  error_patterns_detected:
+    - "InnoDB: Corrupted"
+    - "InnoDB: Database page corruption"
+    - "InnoDB: Unable to open"
+    - "InnoDB: Cannot allocate memory"
+    - "InnoDB: Tablespace.*missing"
+    - "InnoDB: Redo log.*corrupt"
+    - "InnoDB:.*redo log.*incompatible"
+    - "InnoDB: Plugin initialization aborted"
+    - "[ERROR].*InnoDB"
+  action_on_error: "Automatically shuts down second instance and suggests force recovery"
+
+validation_checks_sql_dump:
+  check_1: "File exists and minimum size (>100 bytes)"
+  check_2: "Dump completion marker present ('Dump completed on')"
+  check_3: "Database name found in dump"
+  check_4: "CREATE TABLE statement count"
+  check_5: "INSERT INTO statement count (data verification)"
+  check_6: "SQL syntax spot check (first 100 lines)"
+  check_7: "Database comparison (table count match, size ratio 1-3x)"
+  action_on_failure: "Prompts user to continue or abort if validation issues found"
+
+output:
+  format: "<database_name>_restored_<timestamp>.sql"
+  example: "myuser_wordpress_restored_20251210_143022.sql"
+  with_ticket: "myuser_wordpress_ticket01234567_20251210_143022.sql"
+  location: "Current working directory"
+
+next_steps_provided:
+  1: "Verify dump: grep 'Dump completed on' output.sql"
+  2: "Import to live: mysql database_name < output.sql"
+  3: "Or fresh import: DROP/CREATE then import"
+
+error_handling:
+  second_instance_fails:
+    - "Analyzes error log to diagnose failure type"
+    - "Provides intelligent recovery options based on error"
+    - "Shows targeted guidance (not generic troubleshooting)"
+    - "Offers to display error log immediately"
+
+  intelligent_recovery_system:
+    missing_tablespace_files:
+      diagnosis: "Missing or unopenable .ibd files"
+      option_1: "Restore additional files (grep error log for specific tables)"
+      option_2: "Start fresh (clear directory, restore complete set)"
+      action: "Lists exact grep command to find missing tables"
+
+    redo_log_incompatibility:
+      diagnosis: "Version mismatch or mixed redo log formats"
+      option_1: "Remove redo logs, restore from same backup date"
+      option_2: "Force recovery level 6 (if redo logs lost)"
+      action: "Shows exact rm command for current MySQL version"
+
+    innodb_corruption:
+      diagnosis: "Page corruption detected"
+      progressive_recovery: "Suggests recovery levels based on current level"
+      level_0: "Try force recovery 1 (ignore corrupt pages)"
+      level_1: "Try force recovery 4 (prevent insert buffer)"
+      level_4+: "Try force recovery 6 (skip checksums - last resort)"
+      option_final: "Start fresh from older backup snapshot"
+
+    memory_allocation:
+      diagnosis: "Out of memory"
+      action_1: "Check free memory (free -h)"
+      action_2: "Stop other MySQL instances"
+      action_3: "Re-run script"
+
+  validation_fails:
+    - "Lists missing required files with version-specific paths"
+    - "Shows expected file paths"
+    - "Explains version-specific requirements"
+    - "Offers retry loop (restore files and try again)"
+
+  dump_fails:
+    - "Checks if database exists in second instance"
+    - "Verifies dump completion marker"
+    - "Reports file size of partial dump"
+    - "Prompts to continue or abort on validation errors"
+
+integration:
+  menu_path: "Main Menu (5) → Backup & Recovery (11) → MySQL File Restore"
+  launcher_changes:
+    - "Added option 11 to show_backup_menu()"
+    - "Added case 11 to handle_backup_menu()"
+
+standards_compliance:
+  - "Uses print_* functions from common-functions.sh"
+  - "Cancel buttons at every prompt (0 to cancel)"
+  - "Calls press_enter() before exit"
+  - "Root check at script start"
+  - "No hardcoded paths (version-aware)"
+  - "Proper error handling with exit codes"
+  - "QA check: 0 issues (CRITICAL/HIGH/MEDIUM/LOW)"
+
+testing_status:
+  syntax_validation: "PASS (bash -n)"
+  qa_validation: "PASS (0 issues)"
+  standards_compliance: "PASS (all toolkit standards followed)"
+  live_testing: "PENDING (awaiting real-world file-based restore scenario)"
+
+documentation_references:
+  based_on: "Guardian and cloud backup restoration procedures"
+  kb_article: "InnoDB Second MySQL Instance Restore Process"
+  research_date: "2025-12-10"
+  verified_sources:
+    - "MySQL 8.0 Reference Manual - InnoDB Backup (dev.mysql.com)"
+    - "MySQL 8.4 Reference Manual - Redo Log (dev.mysql.com)"
+    - "MariaDB Documentation - InnoDB Redo Log"
+    - "MySQL Dynamic InnoDB Redo Log blog (MySQL 8.0.30 changes)"
+    - "MySQL Enterprise Backup 8.0 User Guide"
+  critical_findings:
+    - "MySQL 8.0.30+ uses #innodb_redo directory (not ib_logfile anymore)"
+    - "Redo log files named #ib_redoN (N = sequence number)"
+    - "Temporary tablespace in #innodb_temp directory"
+    - "performance_schema is in-memory only, not needed for restore"
+    - "sys schema should be included (optional but recommended)"
+
+known_limitations:
+  - "Requires sufficient disk space for SQL dumps"
+  - "Cannot restore if InnoDB data files are severely corrupted (even with force recovery 6)"
+  - "Does not handle MyISAM-only databases (but will work if MyISAM mixed with InnoDB)"
+  - "Second instance start may fail if port 3306 socket conflicts (but uses custom socket)"
+
+future_enhancements:
+  - "Batch restore (multiple databases at once)"
+  - "Progress indicator for large dump operations"
+  - "Automatic backup of existing live database before import"
+  - "Integration with Acronis API for automated file restore"
+  - "Support for remote MySQL servers (not just localhost)"
+
+commit_info:
+  date: "2025-12-10"
+  files_added:
+    - "modules/backup/mysql-restore-to-sql.sh (540 lines)"
+  files_modified:
+    - "launcher.sh (added menu option 11)"
+  qa_result: "0 issues detected"
+
 [END]
 # This file is the primary developer reference document.
 # README.md is for end users, this file is for developers.
 # Keep this updated after every significant change.
-# Last updated: 2025-12-03 (Created QA checking tool + documented workflow)
+# Last updated: 2025-12-10 (Added MySQL File-Based Restore Tool)
 ################################################################################

launcher.sh

@@ -569,6 +569,10 @@ show_backup_menu() {
     echo ""
     echo -e "  ${YELLOW}9)${NC} 🔷 Acronis Management        → Install, configure, manage backups"
     echo ""
+    echo -e "${BOLD}Database Tools:${NC}"
+    echo ""
+    echo -e "  ${CYAN}11)${NC} 🔄 MySQL File Restore        - Convert restored DB files to .sql"
+    echo ""
     echo -e "${BOLD}Data Management:${NC}"
     echo ""
     echo -e "  ${RED}10)${NC} 🗑️  Cleanup Toolkit Data      - Remove IP reputation & temp files"
@@ -1424,6 +1428,7 @@ handle_backup_menu() {
             8) run_module "backup" "offsite-sync.sh" ;;
             9) handle_acronis_menu ;;
             10) run_module "maintenance" "cleanup-toolkit-data.sh" ;;
+            11) run_module "backup" "mysql-restore-to-sql.sh" ;;
             0) return ;;
             *) echo -e "${RED}Invalid option${NC}"; sleep 1 ;;
         esac