Database convert script
This commit is contained in:
cschantz
+258
-5
@@ -2,14 +2,14 @@
|
||||
# SERVER TOOLKIT - DEVELOPER CONTEXT DATABASE
|
||||
################################################################################
|
||||
# OPTIMIZED FOR: Fast context loading and code navigation
|
||||
# LAST UPDATED: 2025-11-20
|
||||
# VERSION: 2.2.0
|
||||
# LAST UPDATED: 2025-12-10
|
||||
# VERSION: 2.3.0
|
||||
# FORMAT: Structured key-value with hierarchical sections
|
||||
################################################################################
|
||||
|
||||
[META]
|
||||
version: 2.1.0
|
||||
updated: 2025-11-12
|
||||
version: 2.3.0
|
||||
updated: 2025-12-10
|
||||
status: production_ready
|
||||
base_path: /root/server-toolkit
|
||||
entry_point: launcher.sh
|
||||
@@ -1812,9 +1812,262 @@ integration_with_git:
|
||||
- "Cache file lists between runs"
|
||||
- "Whitelist mechanism for known false positives"
|
||||
|
||||
[UPDATE_2025_12_10_MYSQL_RESTORE_TOOL]
|
||||
# New feature: MySQL/MariaDB File-Based Restore to SQL
|
||||
# Interactive guided workflow for restoring databases from file-based backups
|
||||
|
||||
new_module:
|
||||
file: "modules/backup/mysql-restore-to-sql.sh"
|
||||
menu_location: "Backup & Recovery → Option 11"
|
||||
purpose: "Convert restored MySQL/MariaDB data files to usable .sql dumps"
|
||||
|
||||
use_case:
|
||||
scenario: "Restore InnoDB databases from file-based backups (Acronis, Guardian, cloud backups)"
|
||||
problem: "Direct file restoration doesn't work with InnoDB due to tablespace requirements"
|
||||
solution: "Start second MySQL instance with restored files, dump to SQL, import to live"
|
||||
|
||||
features:
|
||||
- "Detects MySQL version (MySQL 5.7, MySQL 8.0+, MariaDB)"
|
||||
- "Version-specific validation (ib_logfile0/1 vs #innodb_redo)"
|
||||
- "Auto-creates timestamped restore directory (/home/temp/restoreYYYYMMDD/mysql)"
|
||||
- "Lists exact files needed from backup (with emoji visual markers)"
|
||||
- "Validates data structure before proceeding"
|
||||
- "Checks and fixes file ownership (mysql:mysql)"
|
||||
- "Starts second MySQL instance on separate socket"
|
||||
- "InnoDB error log monitoring (checks for corruption/errors on startup)"
|
||||
- "Automatic error log backup (preserves old logs)"
|
||||
- "Supports InnoDB force recovery modes (0-6)"
|
||||
- "Creates verified SQL dumps with completion markers"
|
||||
- "Comprehensive SQL dump validation (7 checks)"
|
||||
- "Database comparison (table count, size verification)"
|
||||
- "SQL syntax validation"
|
||||
- "Provides clear next-step instructions"
|
||||
- "Cancel buttons at every step (0 to cancel)"
|
||||
|
||||
workflow:
|
||||
step_1: "Detect live MySQL data directory and version"
|
||||
step_2: "Create/select restore directory with guided file list"
|
||||
step_3: "User restores required files from backup system"
|
||||
step_4: "Validate restored data structure"
|
||||
step_5: "Fix file ownership if needed"
|
||||
step_6: "Select target database"
|
||||
step_7: "Configure optional settings (ticket number, force recovery)"
|
||||
step_8: "Start second MySQL instance"
|
||||
step_9: "Create SQL dump from second instance"
|
||||
step_10: "Provide import instructions"
|
||||
|
||||
required_files_mysql_5_7_mariadb:
|
||||
- "ibdata1 (InnoDB system tablespace)"
|
||||
- "ib_logfile0 (redo log)"
|
||||
- "ib_logfile1 (redo log)"
|
||||
- "mysql/ directory (system database)"
|
||||
- "sys/ directory (optional but recommended)"
|
||||
- "<database_name>/ directory (target database)"
|
||||
|
||||
required_files_mysql_8_0_0_to_8_0_29:
|
||||
- "ibdata1 (InnoDB system tablespace)"
|
||||
- "ib_logfile0 (redo log - old format)"
|
||||
- "ib_logfile1 (redo log - old format)"
|
||||
- "#innodb_temp/ directory (optional, temp tablespace)"
|
||||
- "ibtmp1 (optional, global temp tablespace)"
|
||||
- "mysql/ directory (system database)"
|
||||
- "sys/ directory (optional but recommended)"
|
||||
- "<database_name>/ directory (target database)"
|
||||
- "NOTE: performance_schema NOT needed (in-memory only)"
|
||||
|
||||
required_files_mysql_8_0_30_plus:
|
||||
critical_change_2022: "MySQL 8.0.30 introduced new redo log architecture"
|
||||
- "ibdata1 (InnoDB system tablespace)"
|
||||
- "#innodb_redo/ directory (NEW FORMAT - contains #ib_redo0, #ib_redo1, #ib_redoN files)"
|
||||
- "#innodb_temp/ directory (optional, contains temp_N.ibt files)"
|
||||
- "ibtmp1 (optional, global temp tablespace)"
|
||||
- "mysql/ directory (system database)"
|
||||
- "sys/ directory (optional but recommended)"
|
||||
- "<database_name>/ directory (target database)"
|
||||
- "NOTE: performance_schema NOT needed (in-memory only)"
|
||||
- "NOTE: NO MORE ib_logfile0/ib_logfile1 in 8.0.30+"
|
||||
|
||||
required_files_mysql_9_0_plus:
|
||||
- "Same as MySQL 8.0.30+ (uses #innodb_redo directory)"
|
||||
|
||||
technical_details:
|
||||
second_instance_command: |
|
||||
mysqld --datadir=$dir --socket=$dir/socket.mysql --pid-file=$dir/mysql.pid
|
||||
--log-error=$dir/mysql.err --skip-grant-tables --skip-networking --user=mysql
|
||||
|
||||
dump_command: |
|
||||
mysqldump -h localhost -S $dir/socket.mysql --single-transaction database_name > output.sql
|
||||
|
||||
mysqld_binary_detection: |
|
||||
Tries /usr/libexec/mysqld first (RHEL/CentOS), falls back to mysqld
|
||||
|
||||
force_recovery_levels:
|
||||
0: "No force recovery (default)"
|
||||
1: "Ignore corrupt pages"
|
||||
2: "Prevent background operations"
|
||||
3: "Prevent transaction rollbacks"
|
||||
4: "Prevent insert buffer merge"
|
||||
5: "Skip log redo"
|
||||
6: "Skip page checksums"
|
||||
|
||||
safety_features:
|
||||
- "Second instance runs on separate socket (doesn't affect live MySQL)"
|
||||
- "Critical safety check: Prevents using /var/lib/mysql as restore directory"
|
||||
- "Trap handler: Cleanup on Ctrl+C, interruption, or exit (cleanup_on_exit)"
|
||||
- "Disk space validation: Checks available space before creating dump (2x estimated size)"
|
||||
- "Force recovery warnings: Levels 5-6 require explicit risk acknowledgment"
|
||||
- "All operations validated before execution"
|
||||
- "InnoDB error log parsing (detects corruption during startup)"
|
||||
- "Automatic shutdown if InnoDB errors detected"
|
||||
- "SQL dump integrity validation (7-point check)"
|
||||
- "Database comparison (table count, size ratio verification)"
|
||||
- "Clear error messages with troubleshooting steps"
|
||||
- "Automatic cleanup of second instance on completion"
|
||||
- "Dump verification (checks for 'Dump completed' marker)"
|
||||
- "No eval usage: Uses bash array for mysqld arguments (security)"
|
||||
|
||||
validation_checks_innodb_startup:
|
||||
error_patterns_detected:
|
||||
- "InnoDB: Corrupted"
|
||||
- "InnoDB: Database page corruption"
|
||||
- "InnoDB: Unable to open"
|
||||
- "InnoDB: Cannot allocate memory"
|
||||
- "InnoDB: Tablespace.*missing"
|
||||
- "InnoDB: Redo log.*corrupt"
|
||||
- "InnoDB:.*redo log.*incompatible"
|
||||
- "InnoDB: Plugin initialization aborted"
|
||||
- "[ERROR].*InnoDB"
|
||||
action_on_error: "Automatically shuts down second instance and suggests force recovery"
|
||||
|
||||
validation_checks_sql_dump:
|
||||
check_1: "File exists and minimum size (>100 bytes)"
|
||||
check_2: "Dump completion marker present ('Dump completed on')"
|
||||
check_3: "Database name found in dump"
|
||||
check_4: "CREATE TABLE statement count"
|
||||
check_5: "INSERT INTO statement count (data verification)"
|
||||
check_6: "SQL syntax spot check (first 100 lines)"
|
||||
check_7: "Database comparison (table count match, size ratio 1-3x)"
|
||||
action_on_failure: "Prompts user to continue or abort if validation issues found"
|
||||
|
||||
output:
|
||||
format: "<database_name>_restored_<timestamp>.sql"
|
||||
example: "myuser_wordpress_restored_20251210_143022.sql"
|
||||
with_ticket: "myuser_wordpress_ticket01234567_20251210_143022.sql"
|
||||
location: "Current working directory"
|
||||
|
||||
next_steps_provided:
|
||||
1: "Verify dump: grep 'Dump completed on' output.sql"
|
||||
2: "Import to live: mysql database_name < output.sql"
|
||||
3: "Or fresh import: DROP/CREATE then import"
|
||||
|
||||
error_handling:
|
||||
second_instance_fails:
|
||||
- "Analyzes error log to diagnose failure type"
|
||||
- "Provides intelligent recovery options based on error"
|
||||
- "Shows targeted guidance (not generic troubleshooting)"
|
||||
- "Offers to display error log immediately"
|
||||
|
||||
intelligent_recovery_system:
|
||||
missing_tablespace_files:
|
||||
diagnosis: "Missing or unopenable .ibd files"
|
||||
option_1: "Restore additional files (grep error log for specific tables)"
|
||||
option_2: "Start fresh (clear directory, restore complete set)"
|
||||
action: "Lists exact grep command to find missing tables"
|
||||
|
||||
redo_log_incompatibility:
|
||||
diagnosis: "Version mismatch or mixed redo log formats"
|
||||
option_1: "Remove redo logs, restore from same backup date"
|
||||
option_2: "Force recovery level 6 (if redo logs lost)"
|
||||
action: "Shows exact rm command for current MySQL version"
|
||||
|
||||
innodb_corruption:
|
||||
diagnosis: "Page corruption detected"
|
||||
progressive_recovery: "Suggests recovery levels based on current level"
|
||||
level_0: "Try force recovery 1 (ignore corrupt pages)"
|
||||
level_1: "Try force recovery 4 (prevent insert buffer)"
|
||||
level_4+: "Try force recovery 6 (skip checksums - last resort)"
|
||||
option_final: "Start fresh from older backup snapshot"
|
||||
|
||||
memory_allocation:
|
||||
diagnosis: "Out of memory"
|
||||
action_1: "Check free memory (free -h)"
|
||||
action_2: "Stop other MySQL instances"
|
||||
action_3: "Re-run script"
|
||||
|
||||
validation_fails:
|
||||
- "Lists missing required files with version-specific paths"
|
||||
- "Shows expected file paths"
|
||||
- "Explains version-specific requirements"
|
||||
- "Offers retry loop (restore files and try again)"
|
||||
|
||||
dump_fails:
|
||||
- "Checks if database exists in second instance"
|
||||
- "Verifies dump completion marker"
|
||||
- "Reports file size of partial dump"
|
||||
- "Prompts to continue or abort on validation errors"
|
||||
|
||||
integration:
|
||||
menu_path: "Main Menu (5) → Backup & Recovery (11) → MySQL File Restore"
|
||||
launcher_changes:
|
||||
- "Added option 11 to show_backup_menu()"
|
||||
- "Added case 11 to handle_backup_menu()"
|
||||
|
||||
standards_compliance:
|
||||
- "Uses print_* functions from common-functions.sh"
|
||||
- "Cancel buttons at every prompt (0 to cancel)"
|
||||
- "Calls press_enter() before exit"
|
||||
- "Root check at script start"
|
||||
- "No hardcoded paths (version-aware)"
|
||||
- "Proper error handling with exit codes"
|
||||
- "QA check: 0 issues (CRITICAL/HIGH/MEDIUM/LOW)"
|
||||
|
||||
testing_status:
|
||||
syntax_validation: "PASS (bash -n)"
|
||||
qa_validation: "PASS (0 issues)"
|
||||
standards_compliance: "PASS (all toolkit standards followed)"
|
||||
live_testing: "PENDING (awaiting real-world file-based restore scenario)"
|
||||
|
||||
documentation_references:
|
||||
based_on: "Guardian and cloud backup restoration procedures"
|
||||
kb_article: "InnoDB Second MySQL Instance Restore Process"
|
||||
research_date: "2025-12-10"
|
||||
verified_sources:
|
||||
- "MySQL 8.0 Reference Manual - InnoDB Backup (dev.mysql.com)"
|
||||
- "MySQL 8.4 Reference Manual - Redo Log (dev.mysql.com)"
|
||||
- "MariaDB Documentation - InnoDB Redo Log"
|
||||
- "MySQL Dynamic InnoDB Redo Log blog (MySQL 8.0.30 changes)"
|
||||
- "MySQL Enterprise Backup 8.0 User Guide"
|
||||
critical_findings:
|
||||
- "MySQL 8.0.30+ uses #innodb_redo directory (not ib_logfile anymore)"
|
||||
- "Redo log files named #ib_redoN (N = sequence number)"
|
||||
- "Temporary tablespace in #innodb_temp directory"
|
||||
- "performance_schema is in-memory only, not needed for restore"
|
||||
- "sys schema should be included (optional but recommended)"
|
||||
|
||||
known_limitations:
|
||||
- "Requires sufficient disk space for SQL dumps"
|
||||
- "Cannot restore if InnoDB data files are severely corrupted (even with force recovery 6)"
|
||||
- "Does not handle MyISAM-only databases (but will work if MyISAM mixed with InnoDB)"
|
||||
- "Second instance start may fail if port 3306 socket conflicts (but uses custom socket)"
|
||||
|
||||
future_enhancements:
|
||||
- "Batch restore (multiple databases at once)"
|
||||
- "Progress indicator for large dump operations"
|
||||
- "Automatic backup of existing live database before import"
|
||||
- "Integration with Acronis API for automated file restore"
|
||||
- "Support for remote MySQL servers (not just localhost)"
|
||||
|
||||
commit_info:
|
||||
date: "2025-12-10"
|
||||
files_added:
|
||||
- "modules/backup/mysql-restore-to-sql.sh (540 lines)"
|
||||
files_modified:
|
||||
- "launcher.sh (added menu option 11)"
|
||||
qa_result: "0 issues detected"
|
||||
|
||||
[END]
|
||||
# This file is the primary developer reference document.
|
||||
# README.md is for end users, this file is for developers.
|
||||
# Keep this updated after every significant change.
|
||||
# Last updated: 2025-12-03 (Created QA checking tool + documented workflow)
|
||||
# Last updated: 2025-12-10 (Added MySQL File-Based Restore Tool)
|
||||
################################################################################
|
||||
|
||||
Reference in New Issue
Block a user