diff --git a/tools/qa-functional-tests.sh b/tools/qa-functional-tests.sh new file mode 100755 index 0000000..8421192 --- /dev/null +++ b/tools/qa-functional-tests.sh @@ -0,0 +1,353 @@ +#!/bin/bash + +################################################################################ +# Server Toolkit - Functional Testing Module +################################################################################ +# Purpose: Verify scripts actually work, not just pass static analysis +# Usage: Source this file from toolkit-qa-check.sh or run standalone +################################################################################ + +# Track test results +FUNC_TESTS_PASSED=0 +FUNC_TESTS_FAILED=0 +FUNC_TESTS_SKIPPED=0 + +################################################################################ +# TEST 1: Bash Syntax Validation +################################################################################ +# Verify all scripts have valid bash syntax +test_bash_syntax() { + echo "═══════════════════════════════════════════════════════════════" + echo "FUNCTIONAL TEST 1: Bash Syntax Validation" + echo "═══════════════════════════════════════════════════════════════" + echo "" + + local failed=0 + local passed=0 + + while IFS= read -r script; do + # Run bash -n (syntax check without execution) + if bash -n "$script" 2>/dev/null; then + ((passed++)) + else + ((failed++)) + echo "FAIL|$script|SYNTAX|Script has syntax errors:" + bash -n "$script" 2>&1 | head -5 | sed 's/^/ /' + echo "" + fi + done < <(find "$TOOLKIT_PATH" -name "*.sh" -type f 2>/dev/null) + + echo "Results: $passed passed, $failed failed" + echo "" + + FUNC_TESTS_PASSED=$((FUNC_TESTS_PASSED + passed)) + FUNC_TESTS_FAILED=$((FUNC_TESTS_FAILED + failed)) +} + +################################################################################ +# TEST 2: Function Call Validation +################################################################################ +# Verify all called functions are actually defined +test_function_calls() { + echo "═══════════════════════════════════════════════════════════════" + echo "FUNCTIONAL TEST 2: Function Call Validation" + echo "═══════════════════════════════════════════════════════════════" + echo "" + + local issues=0 + + # Check each script + while IFS= read -r script; do + # Skip test files themselves + [[ "$script" =~ qa.*test ]] && continue + + # Extract function definitions in this script and sourced files + local defined_functions=$(mktemp) + + # Get functions defined in this script + grep -oE "^[[:space:]]*[a-zA-Z_][a-zA-Z0-9_]*\s*\(\)" "$script" 2>/dev/null | \ + sed 's/[[:space:]]*()$//' >> "$defined_functions" + + # Get sourced files and their functions + local sourced_files=$(grep -oE 'source.*\.sh|\..*\.sh' "$script" 2>/dev/null | \ + grep -oE '[a-zA-Z0-9_-]+\.sh') + + for sourced in $sourced_files; do + # Find the actual file + local source_path=$(find "$TOOLKIT_PATH" -name "$sourced" -type f 2>/dev/null | head -1) + if [ -f "$source_path" ]; then + grep -oE "^[[:space:]]*[a-zA-Z_][a-zA-Z0-9_]*\s*\(\)" "$source_path" 2>/dev/null | \ + sed 's/[[:space:]]*()$//' >> "$defined_functions" + fi + done + + # Find function calls in the script + local called_functions=$(mktemp) + grep -oE '[a-zA-Z_][a-zA-Z0-9_]*\s+' "$script" 2>/dev/null | \ + sed 's/[[:space:]]*$//' | \ + sort -u | \ + grep -v "^if$\|^then$\|^else$\|^fi$\|^do$\|^done$\|^case$\|^esac$\|^for$\|^while$\|^local$\|^echo$\|^return$\|^exit$" \ + > "$called_functions" + + # Check each called function + while IFS= read -r func; do + # Skip common bash builtins and external commands + if ! command -v "$func" >/dev/null 2>&1 && \ + ! grep -qx "$func" "$defined_functions" 2>/dev/null; then + # Might be undefined - report only if it looks like a custom function + if [[ "$func" =~ ^(get_|check_|print_|show_|detect_|list_|find_|validate_|extract_|parse_) ]]; then + echo "WARN|$script|UNDEFINED|Function '$func' called but not defined" + ((issues++)) + fi + fi + done < "$called_functions" + + rm -f "$defined_functions" "$called_functions" + + done < <(find "$TOOLKIT_PATH/modules" "$TOOLKIT_PATH/tools" -name "*.sh" -type f 2>/dev/null) + + if [ $issues -eq 0 ]; then + echo "✓ All function calls appear valid" + ((FUNC_TESTS_PASSED++)) + else + echo "✗ Found $issues potential undefined function calls" + ((FUNC_TESTS_FAILED++)) + fi + echo "" +} + +################################################################################ +# TEST 3: Dependency Validation +################################################################################ +# Verify all sourced files actually exist +test_dependencies() { + echo "═══════════════════════════════════════════════════════════════" + echo "FUNCTIONAL TEST 3: Dependency Validation" + echo "═══════════════════════════════════════════════════════════════" + echo "" + + local missing=0 + local found=0 + local skipped_vars=0 + + while IFS= read -r script; do + # Find all source commands + while IFS= read -r source_line; do + # Extract the file being sourced (handle various patterns) + local sourced_file=$(echo "$source_line" | sed 's/.*source\s\+//; s/.*\.\s\+//' | awk '{print $1}' | tr -d '"' | head -1) + + # Skip if it's primarily a variable (we can't validate dynamic paths easily) + if [[ "$sourced_file" =~ ^\$ ]] || [[ "$sourced_file" =~ ^\$\{ ]]; then + ((skipped_vars++)) + continue + fi + + # Try to resolve common variable patterns + sourced_file=$(echo "$sourced_file" | sed \ + -e "s|\$SCRIPT_DIR|$(dirname "$script")|" \ + -e "s|\$TOOLKIT_PATH|$TOOLKIT_PATH|" \ + -e "s|\$_LIB_DIR|$TOOLKIT_PATH/lib|" \ + -e "s|\$LIB_DIR|$TOOLKIT_PATH/lib|" \ + -e "s|\$TOOLKIT_ROOT|$TOOLKIT_PATH|" \ + -e 's/[{}]//g') + + # Skip if still contains variables + [[ "$sourced_file" =~ \$ ]] && { ((skipped_vars++)); continue; } + + # Check if file exists + if [ -f "$sourced_file" ]; then + ((found++)) + elif [ -f "$TOOLKIT_PATH/$sourced_file" ]; then + ((found++)) + elif [ -f "$TOOLKIT_PATH/lib/$sourced_file" ]; then + ((found++)) + else + echo "FAIL|$script|MISSING-DEP|Cannot find: $sourced_file" + ((missing++)) + fi + done < <(grep -n "^\s*source\s\|^\s*\.\s" "$script" 2>/dev/null | grep -v "^#") + + done < <(find "$TOOLKIT_PATH" -name "*.sh" -type f 2>/dev/null) + + echo "Results: $found dependencies found, $missing missing, $skipped_vars dynamic paths" + echo "" + + if [ $missing -eq 0 ]; then + ((FUNC_TESTS_PASSED++)) + else + ((FUNC_TESTS_FAILED++)) + fi +} + +################################################################################ +# TEST 4: Library Function Unit Tests +################################################################################ +# Test key library functions with sample data +test_library_functions() { + echo "═══════════════════════════════════════════════════════════════" + echo "FUNCTIONAL TEST 4: Library Function Unit Tests" + echo "═══════════════════════════════════════════════════════════════" + echo "" + + # Source common libraries + source "$TOOLKIT_PATH/lib/common-functions.sh" 2>/dev/null || { + echo "SKIP|Cannot load common-functions.sh" + ((FUNC_TESTS_SKIPPED++)) + return + } + + local tests_run=0 + local tests_passed=0 + + # Test 1: validate_email function (if it exists) + if type validate_email >/dev/null 2>&1; then + ((tests_run++)) + source "$TOOLKIT_PATH/lib/email-functions.sh" 2>/dev/null + + if validate_email "test@example.com" && \ + ! validate_email "invalid-email" && \ + ! validate_email ""; then + echo "✓ validate_email() works correctly" + ((tests_passed++)) + else + echo "✗ validate_email() failed tests" + fi + fi + + # Test 2: is_valid_ip function + if type is_valid_ip >/dev/null 2>&1; then + ((tests_run++)) + source "$TOOLKIT_PATH/lib/email-functions.sh" 2>/dev/null + + if is_valid_ip "192.168.1.1" && \ + ! is_valid_ip "999.999.999.999" && \ + ! is_valid_ip "not-an-ip"; then + echo "✓ is_valid_ip() works correctly" + ((tests_passed++)) + else + echo "✗ is_valid_ip() failed tests" + fi + fi + + # Test 3: format_size function + if type format_size >/dev/null 2>&1; then + ((tests_run++)) + source "$TOOLKIT_PATH/lib/email-functions.sh" 2>/dev/null + + local result=$(format_size 500) + if [[ "$result" == "500B" ]]; then + echo "✓ format_size() works correctly" + ((tests_passed++)) + else + echo "✗ format_size() returned unexpected: $result" + fi + fi + + echo "" + echo "Results: $tests_passed/$tests_run unit tests passed" + echo "" + + if [ $tests_run -eq 0 ]; then + ((FUNC_TESTS_SKIPPED++)) + elif [ $tests_passed -eq $tests_run ]; then + ((FUNC_TESTS_PASSED++)) + else + ((FUNC_TESTS_FAILED++)) + fi +} + +################################################################################ +# TEST 5: Script Execution Smoke Tests +################################################################################ +# Verify scripts can at least show help/usage without crashing +test_script_execution() { + echo "═══════════════════════════════════════════════════════════════" + echo "FUNCTIONAL TEST 5: Script Execution Smoke Tests" + echo "═══════════════════════════════════════════════════════════════" + echo "" + + local tested=0 + local passed=0 + local failed=0 + + # Test each executable script + while IFS= read -r script; do + ((tested++)) + + # Try to run with --help (many scripts support this) + if timeout 5 bash "$script" --help >/dev/null 2>&1 || \ + timeout 5 bash "$script" -h >/dev/null 2>&1; then + ((passed++)) + else + # Try to at least parse it without execution + if bash -n "$script" 2>/dev/null; then + echo "INFO|$script|NO-HELP|Script has no --help but syntax is valid" + ((passed++)) + else + echo "FAIL|$script|CRASH|Script fails to parse or execute" + ((failed++)) + fi + fi + + done < <(find "$TOOLKIT_PATH/modules" "$TOOLKIT_PATH/tools" -name "*.sh" -type f 2>/dev/null | head -20) + + echo "" + echo "Results: $passed/$tested scripts validated" + echo "" + + if [ $failed -eq 0 ]; then + ((FUNC_TESTS_PASSED++)) + else + ((FUNC_TESTS_FAILED++)) + fi +} + +################################################################################ +# Main Test Runner +################################################################################ +run_functional_tests() { + echo "" + echo "═══════════════════════════════════════════════════════════════" + echo "FUNCTIONAL TESTING SUITE" + echo "═══════════════════════════════════════════════════════════════" + echo "Verifying scripts actually work, not just pass static analysis" + echo "" + + # Run all tests + test_bash_syntax + test_function_calls + test_dependencies + test_library_functions + test_script_execution + + # Summary + echo "═══════════════════════════════════════════════════════════════" + echo "FUNCTIONAL TEST SUMMARY" + echo "═══════════════════════════════════════════════════════════════" + echo "" + echo "Tests Passed: $FUNC_TESTS_PASSED" + echo "Tests Failed: $FUNC_TESTS_FAILED" + echo "Tests Skipped: $FUNC_TESTS_SKIPPED" + echo "" + + local total=$((FUNC_TESTS_PASSED + FUNC_TESTS_FAILED)) + if [ $total -gt 0 ]; then + local pass_rate=$((FUNC_TESTS_PASSED * 100 / total)) + echo "Pass Rate: ${pass_rate}%" + fi + echo "" + + if [ $FUNC_TESTS_FAILED -gt 0 ]; then + echo "⚠ Some functional tests failed - review output above" + return 1 + else + echo "✓ All functional tests passed" + return 0 + fi +} + +# If run standalone +if [ "${BASH_SOURCE[0]}" = "${0}" ]; then + TOOLKIT_PATH="${1:-/root/server-toolkit}" + run_functional_tests +fi