diff --git a/.gitignore b/.gitignore index db448945b784..2fbaea687bb2 100644 --- a/.gitignore +++ b/.gitignore @@ -61,5 +61,3 @@ test/allure-results/ .idea/ cmake-build-debug/ cmake-build-release/ -plugins/incremental_bitmap/build/ -plugins/incremental_bitmap/build_real/ diff --git a/plugins/incremental_bitmap/.github/workflows/ci.yml b/plugins/incremental_bitmap/.github/workflows/ci.yml deleted file mode 100644 index 3904a6dc861a..000000000000 --- a/plugins/incremental_bitmap/.github/workflows/ci.yml +++ /dev/null @@ -1,684 +0,0 @@ -name: CI - Incremental Bitmap Plugin - -on: - workflow_dispatch: - inputs: - run_real_tests: - description: "Run real-environment tests (USE_MOCK=OFF, E2E_TDENGINE_REAL_TESTS=ON)" - type: boolean - default: false - build_type: - description: "CMake build type" - required: false - default: Debug - type: choice - options: [Debug, Release] - push: - branches: [ main, develop ] - paths: - - 'plugins/incremental_bitmap/**' - pull_request: - branches: [ main, develop ] - paths: - - 'plugins/incremental_bitmap/**' - -env: - BUILD_TYPE: Debug - PLUGIN_DIR: plugins/incremental_bitmap - -jobs: - build-and-test: - runs-on: ubuntu-latest - strategy: - matrix: - compiler: [gcc, clang] - build_type: [Debug, Release] - exclude: - - compiler: clang - build_type: Debug # 减少重复构建 - - outputs: - build_type: ${{ matrix.build_type }} - compiler: ${{ matrix.compiler }} - run_real_tests: ${{ steps.setup-env.outputs.run_real_tests }} - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Setup environment - id: setup-env - run: | - echo "PLUGIN_DIR=${{ env.PLUGIN_DIR }}" >> $GITHUB_ENV - echo "BUILD_TYPE=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.build_type || matrix.build_type }}" >> $GITHUB_ENV - echo "CC=${{ matrix.compiler }}" >> $GITHUB_ENV - # Detect whether to run real tests (only when manually dispatched with run_real_tests=true) - if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ github.event.inputs.run_real_tests }}" = "true" ]; then - echo "RUN_REAL_TESTS=true" >> $GITHUB_ENV - echo "run_real_tests=true" >> $GITHUB_OUTPUT - else - echo "RUN_REAL_TESTS=false" >> $GITHUB_ENV - echo "run_real_tests=false" >> $GITHUB_OUTPUT - fi - # Use lighter test scale in CI to avoid long runtime for heavy benchmarks - echo "IB_TEST_SCALE=small" >> $GITHUB_ENV - - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y \ - build-essential \ - cmake \ - git \ - valgrind \ - gdb \ - libpthread-stubs0-dev \ - libroaring-dev \ - clang-tidy \ - cppcheck \ - clang-format \ - wget \ - curl - - - name: Configure build - working-directory: ${{ env.PLUGIN_DIR }} - run: | - mkdir -p build - cd build - EXTRA_FLAGS="" - if [ "${{ env.BUILD_TYPE }}" = "Debug" ]; then - # 在Debug下开启覆盖率编译标志，确保lcov有数据 - EXTRA_FLAGS="-DCMAKE_C_FLAGS=--coverage -DCMAKE_EXE_LINKER_FLAGS=--coverage" - fi - # 设置 TDengine 相关环境变量（如果启用真实测试） - if [ "${{ env.RUN_REAL_TESTS }}" = "true" ]; then - export TDENGINE_HOME=/usr/local - export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH - export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH - fi - - cmake .. \ - -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \ - -DCMAKE_C_COMPILER=${{ matrix.compiler }} \ - -DBUILD_TESTING=ON \ - -DBUILD_TAOSX_PLUGIN=ON \ - -DUSE_MOCK=${{ env.RUN_REAL_TESTS == 'true' && 'OFF' || 'ON' }} \ - -DE2E_TDENGINE_REAL_TESTS=${{ env.RUN_REAL_TESTS == 'true' && 'ON' || 'OFF' }} \ - $EXTRA_FLAGS - - - name: Build plugin - working-directory: ${{ env.PLUGIN_DIR }}/build - run: | - make -j$(nproc) VERBOSE=1 - - # 🔧 修复：将真实环境准备移到测试运行之前 - - name: Install TDengine (only when real tests enabled) - if: env.RUN_REAL_TESTS == 'true' - run: | - echo "=== Installing TDengine for real environment tests ===" - # 安装 TDengine 客户端库 - wget -O - https://www.taosdata.com/assets-download/3.0/TDengine-client-3.0.6.0-Linux-x64.tar.gz | tar -xz - sudo cp TDengine-client-3.0.6.0/lib/* /usr/local/lib/ - sudo ldconfig - echo "export TDENGINE_HOME=/usr/local" >> $GITHUB_ENV - echo "export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV - - - name: Prepare real TDengine environment (only when enabled) - if: env.RUN_REAL_TESTS == 'true' - working-directory: ${{ env.PLUGIN_DIR }} - timeout-minutes: 15 # 增加超时时间 - run: | - echo "=== Preparing TDengine real environment ===" - # 设置环境变量 - export TDENGINE_HOME=/usr/local - export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH - - # 尝试启动 TDengine 服务（如果存在） - if command -v systemctl >/dev/null 2>&1; then - sudo systemctl stop taosd || true - sudo systemctl start taosd || true - sudo systemctl status taosd || true - else - echo "systemctl not available, skipping service management" - fi - - # 运行设置脚本 - if [ -x "./setup_tdengine_test.sh" ]; then - ./setup_tdengine_test.sh || echo "setup_tdengine_test.sh completed with non-zero exit, continuing" - else - echo "setup_tdengine_test.sh not found, skipping explicit DB setup" - fi - - - name: Run tests (with per-test timeout) - working-directory: ${{ env.PLUGIN_DIR }}/build - timeout-minutes: ${{ env.RUN_REAL_TESTS == 'true' && 45 || 20 }} # 真实环境测试需要更长时间 - run: | - # 运行所有测试，添加错误处理 - test_failed=0 - exec_count=0 - for test in test_*; do - if [ -x "$test" ]; then - echo "Running test: $test" - exec_count=$((exec_count+1)) - # 跳过真实环境相关二进制（仅在未启用真实测试时） - if [ "${RUN_REAL_TESTS}" != "true" ]; then - if echo "$test" | grep -Eq "(e2e_tdengine_real|offset_semantics_real|offset_semantics_realtime)"; then - echo "Skipping real-environment test: $test" - continue - fi - # CI默认跳过重型PITR完整流程，避免超时与假阴性 - if echo "$test" | grep -Eq "(^|/)test_pitr_e2e$"; then - echo "Skipping heavy PITR E2E in CI: $test" - continue - fi - fi - # 为单个用例增加超时，真实环境测试需要更长时间 - per_timeout=90 - if [ "${RUN_REAL_TESTS}" = "true" ]; then - per_timeout=180 # 真实环境测试需要更长时间 - fi - if echo "$test" | grep -Eq "(^|/)test_pitr_e2e_simple$"; then - per_timeout=$((per_timeout + 60)) - fi - if ! timeout ${per_timeout}s ./$test; then - echo "Test $test failed, but continuing..." - test_failed=1 - fi - fi - done - if [ $exec_count -eq 0 ]; then - echo "ERROR: No executable test_* binaries found. Failing the job to avoid false green." - exit 1 - fi - - # 运行taosX插件测试 - if [ -x "test_taosx_plugin_interface" ]; then - echo "Running taosX plugin test: test_taosx_plugin_interface" - if ! ./test_taosx_plugin_interface; then - echo "taosX plugin test failed, but continuing..." - test_failed=1 - fi - else - echo "taosX plugin test not found" - fi - - # 如果有测试失败，退出（确保PR能发现失败） - if [ $test_failed -eq 1 ]; then - echo "Some tests failed. Exiting with non-zero code." - exit 1 - fi - - - name: Run tests with Valgrind (Debug builds only, reduced set) - working-directory: ${{ env.PLUGIN_DIR }}/build - if: matrix.build_type == 'Debug' - timeout-minutes: ${{ env.RUN_REAL_TESTS == 'true' && 30 || 20 }} # 真实环境测试需要更长时间 - run: | - # 使用Valgrind运行关键测试（精简子集，严格单测120s限时） - # 核心功能（2项） - for test in test_bitmap_engine_core test_abstraction_layer; do - if [ -x "$test" ]; then - echo "Running core test $test with Valgrind (120s)" - timeout 120 valgrind --tool=memcheck --leak-check=full --show-leak-kinds=all \ - --error-exitcode=1 --track-origins=yes ./$test || echo "Valgrind test $test failed or timed out" - fi - done - - # 可观测性/插件（1项，选其一） - for test in test_taosx_plugin_interface; do - if [ -x "$test" ]; then - echo "Running plugin test $test with Valgrind (120s)" - timeout 120 valgrind --tool=memcheck --leak-check=full --show-leak-kinds=all \ - --error-exitcode=1 --track-origins=yes ./$test || echo "Valgrind test $test failed or timed out" - fi - done - - - - name: Collect test results - if: always() - working-directory: ${{ env.PLUGIN_DIR }}/build - run: | - # 收集测试结果 - echo "=== Test Results ===" - ls -la test_* 2>/dev/null || echo "No test files found" - echo "=== Build Artifacts ===" - ls -la *.so *.a 2>/dev/null || echo "No libraries found" - echo "=== taosX Plugin Artifacts ===" - ls -la *taosx* 2>/dev/null || echo "No taosX plugin artifacts found" - - # 生成测试结果报告 - echo "=== Test Results Summary ===" - echo "# Test Results Report" > test_results_summary.md - echo "## Test Execution Summary" >> test_results_summary.md - echo "- Total test files found: $(ls test_* 2>/dev/null | wc -l)" >> test_results_summary.md - echo "- Executable test files: $(find . -maxdepth 1 -type f -name 'test_*' -executable | wc -l)" >> test_results_summary.md - echo "- Build artifacts: $(ls *.so *.a 2>/dev/null | wc -l)" >> test_results_summary.md - echo "- taosX plugin artifacts: $(ls *taosx* 2>/dev/null | wc -l)" >> test_results_summary.md - echo "" >> test_results_summary.md - echo "## Discovered Executable Tests" >> test_results_summary.md - find . -maxdepth 1 -type f -name 'test_*' -executable | sort | sed 's#^\./#- #g' >> test_results_summary.md - echo "" >> test_results_summary.md - echo "## Notes" >> test_results_summary.md - echo "- Heavy tests (e.g., test_pitr_e2e) are skipped in default CI unless RUN_REAL_TESTS=true." >> test_results_summary.md - echo "- Real-environment tests are only executed on manual dispatch with run_real_tests=true." >> test_results_summary.md - - cat test_results_summary.md - - taosx-plugin-test: - runs-on: ubuntu-latest - needs: build-and-test - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y \ - build-essential \ - cmake \ - git \ - libpthread-stubs0-dev \ - libroaring-dev - - - name: Build taosX plugin - working-directory: ${{ env.PLUGIN_DIR }} - run: | - mkdir -p build - cd build - cmake .. -DBUILD_TAOSX_PLUGIN=ON - make taosx_incremental_bitmap_plugin test_taosx_plugin_interface - - - name: Test taosX plugin interface - working-directory: ${{ env.PLUGIN_DIR }}/build - timeout-minutes: 10 # 添加超时保护 - run: | - echo "=== taosX Plugin Interface Test ===" - if [ -x "test_taosx_plugin_interface" ]; then - echo "Running taosX plugin interface test..." - ./test_taosx_plugin_interface - echo "taosX plugin interface test completed successfully" - else - echo "ERROR: taosX plugin test executable not found" - exit 1 - fi - - - name: Verify taosX plugin library - working-directory: ${{ env.PLUGIN_DIR }}/build - run: | - echo "=== taosX Plugin Library Verification ===" - if [ -f "libtaosx_incremental_bitmap_plugin.so" ]; then - echo "✓ taosX plugin library exists" - file libtaosx_incremental_bitmap_plugin.so - ldd libtaosx_incremental_bitmap_plugin.so || echo "Library dependencies:" - else - echo "✗ taosX plugin library not found" - exit 1 - fi - - - name: Test plugin API compatibility - working-directory: ${{ env.PLUGIN_DIR }}/build - run: | - echo "=== taosX Plugin API Compatibility Test ===" - # 测试插件API的基本功能 - cat > test_api_compatibility.c << 'EOF' - #include - #include - #include - - int main() { - void *handle = dlopen("./libtaosx_incremental_bitmap_plugin.so", RTLD_LAZY); - if (!handle) { - printf("ERROR: Cannot load plugin: %s\n", dlerror()); - return 1; - } - - // 测试基本API函数 - typedef const char* (*get_name_func)(); - typedef const char* (*get_version_func)(); - typedef int (*init_func)(); - typedef int (*shutdown_func)(); - - get_name_func get_name = (get_name_func)dlsym(handle, "taosx_plugin_get_name"); - get_version_func get_version = (get_version_func)dlsym(handle, "taosx_plugin_get_version"); - init_func init = (init_func)dlsym(handle, "taosx_plugin_init"); - shutdown_func shutdown = (shutdown_func)dlsym(handle, "taosx_plugin_shutdown"); - - if (!get_name || !get_version || !init || !shutdown) { - printf("ERROR: Required API functions not found\n"); - dlclose(handle); - return 1; - } - - printf("✓ Plugin name: %s\n", get_name()); - printf("✓ Plugin version: %s\n", get_version()); - - if (init() == 0) { - printf("✓ Plugin initialization successful\n"); - shutdown(); - printf("✓ Plugin shutdown successful\n"); - } else { - printf("ERROR: Plugin initialization failed\n"); - dlclose(handle); - return 1; - } - - dlclose(handle); - printf("✓ All API compatibility tests passed\n"); - return 0; - } - EOF - - gcc -o test_api_compatibility test_api_compatibility.c -ldl - ./test_api_compatibility - - static-analysis: - runs-on: ubuntu-latest - needs: build-and-test - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Install static analysis tools - run: | - sudo apt-get update - sudo apt-get install -y \ - clang-tidy \ - cppcheck \ - clang-format \ - splint - - - name: Run clang-tidy - working-directory: ${{ env.PLUGIN_DIR }} - run: | - # 创建编译数据库 - mkdir -p build - cd build - cmake .. -DCMAKE_EXPORT_COMPILE_COMMANDS=ON - - # 运行clang-tidy - run-clang-tidy -header-filter='.*' -checks='*,-fuchsia-*,-google-*,-zircon-*,-abseil-*,-modernize-use-trailing-return-type' \ - -p build/ > clang-tidy-report.txt 2>&1 || true - - # 显示报告 - cat clang-tidy-report.txt - - - name: Run cppcheck - working-directory: ${{ env.PLUGIN_DIR }} - run: | - cppcheck --enable=all --std=c99 --language=c \ - --suppress=missingIncludeSystem \ - --suppress=unusedFunction \ - --xml --xml-version=2 \ - src/ include/ > cppcheck-report.xml 2>&1 || true - - # 显示报告 - cppcheck --enable=all --std=c99 --language=c \ - --suppress=missingIncludeSystem \ - --suppress=unusedFunction \ - src/ include/ - - - name: Check code formatting - working-directory: ${{ env.PLUGIN_DIR }} - run: | - # 检查代码格式，添加目录存在检查 - if [ -d "src" ] && [ -d "include" ]; then - find src/ include/ -name "*.c" -o -name "*.h" | xargs clang-format --dry-run --Werror - else - echo "Source directories not found, skipping format check" - fi - - - name: Upload static analysis reports - uses: actions/upload-artifact@v3 - if: always() - with: - name: static-analysis-reports - path: | - ${{ env.PLUGIN_DIR }}/build/clang-tidy-report.txt - ${{ env.PLUGIN_DIR }}/cppcheck-report.xml - - code-coverage: - runs-on: ubuntu-latest - needs: build-and-test - if: needs.build-and-test.outputs.build_type == 'Debug' - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Install coverage tools - run: | - sudo apt-get update - sudo apt-get install -y lcov - - - name: Generate coverage report - working-directory: ${{ env.PLUGIN_DIR }}/build - run: | - # 生成覆盖率报告 - lcov --capture --directory . --output-file coverage.info - lcov --remove coverage.info '/usr/*' --output-file coverage.info - lcov --remove coverage.info '*/test/*' --output-file coverage.info - lcov --list coverage.info - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - file: ${{ env.PLUGIN_DIR }}/build/coverage.info - flags: unittests - name: codecov-umbrella - - security-scan: - runs-on: ubuntu-latest - needs: build-and-test - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Run security scan - working-directory: ${{ env.PLUGIN_DIR }} - run: | - # 使用semgrep进行安全扫描，指定版本 - sudo apt-get update - sudo apt-get install -y jq python3-pip - pip install semgrep==1.45.0 - semgrep --config=auto --json --output=semgrep-results.json src/ || true - - # 显示结果 - if [ -f semgrep-results.json ]; then - echo "Security scan results:" - cat semgrep-results.json | jq '.results[] | {rule_id, message, path, start_line}' || echo "No security issues found" - fi - - - name: Upload security scan results - uses: actions/upload-artifact@v3 - if: always() - with: - name: security-scan-results - path: ${{ env.PLUGIN_DIR }}/semgrep-results.json - - documentation: - runs-on: ubuntu-latest - needs: build-and-test - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Check documentation - working-directory: ${{ env.PLUGIN_DIR }} - run: | - # 检查文档完整性 - echo "=== Documentation Check ===" - - # 检查必要文档是否存在 - required_docs=( - "README.md" - "docs/installation_guide.md" - "docs/api_usage_guide.md" - "docs/troubleshooting_guide.md" - "docs/observability_metrics.md" - ) - - missing_docs=0 - for doc in "${required_docs[@]}"; do - if [ -f "$doc" ]; then - echo "✓ $doc exists" - else - echo "✗ $doc missing" - missing_docs=1 - fi - done - - # 检查文档链接 - echo "=== Checking Documentation Links ===" - if [ -d "docs" ]; then - find docs/ -name "*.md" -exec grep -l "\[.*\](" {} \; | while read file; do - echo "Checking links in $file" - grep -o "\[.*\]([^)]*)" "$file" | while read link; do - target=$(echo "$link" | sed 's/\[.*\](\([^)]*\))/\1/') - if [[ "$target" == http* ]]; then - echo " External link: $target" - elif [[ "$target" == \#* ]]; then - echo " Anchor link: $target" - else - if [ -f "$target" ] || [ -f "$(dirname "$file")/$target" ]; then - echo " ✓ Internal link: $target" - else - echo " ✗ Broken internal link: $target in $file" - missing_docs=1 - fi - fi - done - done - fi - - if [ $missing_docs -eq 1 ]; then - echo "Some documentation issues found, but continuing..." - # exit 1 # 取消注释以在文档检查失败时停止 - fi - - - name: Generate documentation coverage report - working-directory: ${{ env.PLUGIN_DIR }} - run: | - echo "=== Documentation Coverage Report ===" > doc_coverage.txt - - # 统计代码文件 - code_files=$(find src/ -name "*.c" -o -name "*.h" 2>/dev/null | wc -l) - echo "Code files: $code_files" >> doc_coverage.txt - - # 统计文档文件 - doc_files=$(find docs/ -name "*.md" 2>/dev/null | wc -l) - echo "Documentation files: $doc_files" >> doc_coverage.txt - - # 统计测试文件 - test_files=$(find test/ -name "*.c" 2>/dev/null | wc -l) - echo "Test files: $test_files" >> doc_coverage.txt - - # 计算文档覆盖率 - total_files=$((code_files + test_files)) - if [ $total_files -gt 0 ]; then - coverage=$((doc_files * 100 / total_files)) - echo "Documentation coverage: ${coverage}%" >> doc_coverage.txt - fi - - cat doc_coverage.txt - - - name: Upload documentation report - uses: actions/upload-artifact@v3 - with: - name: documentation-coverage - path: ${{ env.PLUGIN_DIR }}/doc_coverage.txt - - performance-benchmark: - runs-on: ubuntu-latest - needs: build-and-test - if: needs.build-and-test.outputs.build_type == 'Release' - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Run performance benchmarks - working-directory: ${{ env.PLUGIN_DIR }}/build - timeout-minutes: 15 # 添加超时保护 - run: | - echo "=== Performance Benchmark ===" - - # 运行性能测试（如果有） - if [ -x "test_performance" ]; then - echo "Running performance tests..." - ./test_performance - else - echo "No performance test executable found" - fi - - # 基本性能检查 - echo "=== Basic Performance Check ===" - for test in test_*; do - if [ -x "$test" ]; then - echo "Timing $test..." - time ./$test > /dev/null 2>&1 - fi - done - - build-artifacts: - runs-on: ubuntu-latest - needs: [build-and-test, static-analysis, code-coverage, security-scan, documentation, taosx-plugin-test] - if: always() - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Download all artifacts - uses: actions/download-artifact@v3 - with: - path: artifacts/ - - - name: Create build summary - run: | - echo "=== Build Summary ===" > build_summary.txt - echo "Build completed at: $(date)" >> build_summary.txt - echo "" >> build_summary.txt - - # 检查构建状态 - if [ -d "artifacts" ]; then - echo "Artifacts collected:" >> build_summary.txt - find artifacts/ -type f >> build_summary.txt - fi - - echo "" >> build_summary.txt - echo "=== Job Status ===" >> build_summary.txt - echo "Build and Test: ${{ needs.build-and-test.result }}" >> build_summary.txt - echo "Static Analysis: ${{ needs.static-analysis.result }}" >> build_summary.txt - echo "Code Coverage: ${{ needs.code-coverage.result }}" >> build_summary.txt - echo "Security Scan: ${{ needs.security-scan.result }}" >> build_summary.txt - echo "Documentation: ${{ needs.documentation.result }}" >> build_summary.txt - echo "taosX Plugin Test: ${{ needs.taosx-plugin-test.result }}" >> build_summary.txt - - cat build_summary.txt - - - name: Upload build summary - uses: actions/upload-artifact@v3 - with: - name: build-summary - path: build_summary.txt - - - name: Comment on PR - if: github.event_name == 'pull_request' - uses: actions/github-script@v6 - with: - script: | - const fs = require('fs'); - const summary = fs.readFileSync('build_summary.txt', 'utf8'); - - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: `## CI Build Summary\n\n\`\`\`\n${summary}\n\`\`\`\n\n[View full logs](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})` - }); diff --git a/plugins/incremental_bitmap/.gitignore b/plugins/incremental_bitmap/.gitignore deleted file mode 100644 index 1828d52532ec..000000000000 --- a/plugins/incremental_bitmap/.gitignore +++ /dev/null @@ -1,32 +0,0 @@ -dump_result.txt - -# 构建产物 -build/ -CMakeFiles/ -*.o -*.so -*.a -*.dylib -*.dll -build_real/ - -# CMake 临时文件 -CMakeCache.txt -cmake_install.cmake -Makefile -*.cmake - -# 编译产物 -*.exe -*.out -*.log - -# IDE 文件 -.vscode/ -.idea/ -*.swp -*.swo - -# 系统文件 -.DS_Store -Thumbs.db diff --git a/plugins/incremental_bitmap/CMakeLists.txt b/plugins/incremental_bitmap/CMakeLists.txt deleted file mode 100644 index d61317bd5ca0..000000000000 --- a/plugins/incremental_bitmap/CMakeLists.txt +++ /dev/null @@ -1,338 +0,0 @@ -cmake_minimum_required(VERSION 3.16) -project(incremental_bitmap_plugin) - -set(CMAKE_C_STANDARD 11) -set(CMAKE_CXX_STANDARD 17) - -# 添加编译标志来抑制可执行栈警告 -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wl,-z,noexecstack") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-z,noexecstack") - -# 选项: 强制使用 mock 引擎 -option(USE_MOCK "Force using mock storage engine and disable TMQ" OFF) -if(USE_MOCK) - add_compile_definitions(USE_MOCK=1) -endif() - -# 选项: 构建真实TDengine全链路E2E测试（默认关闭，不会自动运行） -option(E2E_TDENGINE_REAL_TESTS "Build real TDengine end-to-end tests" OFF) - -# 选项: 构建taosX插件接口（默认关闭） -option(BUILD_TAOSX_PLUGIN "Build taosX plugin interface" OFF) - - -# 设置TDengine路径 -set(TDENGINE_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../..) -set(TDENGINE_BUILD_DIR ${TDENGINE_ROOT}/build) - -# 包含目录 -include_directories( - ${CMAKE_CURRENT_SOURCE_DIR}/include - ${TDENGINE_ROOT}/include - ${TDENGINE_BUILD_DIR}/include - ${TDENGINE_ROOT}/deps/roaring/include -) - -# 设置库目录 -link_directories( - ${TDENGINE_BUILD_DIR}/lib - ${TDENGINE_BUILD_DIR}/build/lib -) - -# 源文件 -set(SOURCES - src/bitmap_engine.c - src/event_interceptor.c - src/backup_coordinator.c - src/skiplist.c - src/simple_bitmap.c - src/roaring_bitmap.c - src/ring_buffer.c - src/storage_engine_interface.c - src/tdengine_storage_engine.c - src/incremental_backup_tool.c - src/observability.c -) - -# 创建动态库 -add_library(incremental_bitmap_plugin SHARED ${SOURCES}) - -# 链接库 -if(NOT USE_MOCK) - target_link_libraries(incremental_bitmap_plugin taos pthread roaring) -else() - message(WARNING "Building with USE_MOCK=ON: TDengine client not linked; TMQ backend disabled") - target_link_libraries(incremental_bitmap_plugin pthread roaring) -endif() - -# 设置输出目录 -set_target_properties(incremental_bitmap_plugin PROPERTIES - LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/build - RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/build -) - -# taosX插件接口（可选） -if(BUILD_TAOSX_PLUGIN) - add_subdirectory(taosx_plugin) -endif() - -# 安装规则 -install(TARGETS incremental_bitmap_plugin - LIBRARY DESTINATION /usr/local/taos/plugins/backup -) - -# 测试程序 -if(BUILD_TESTING) - # 环形队列测试 - add_executable(test_ring_buffer test/test_ring_buffer.c src/ring_buffer.c) - target_link_libraries(test_ring_buffer pthread) - - # 位图引擎核心功能测试 - add_executable(test_bitmap_engine_core test/test_bitmap_engine_core.c src/bitmap_engine.c src/skiplist.c src/simple_bitmap.c src/roaring_bitmap.c) - target_link_libraries(test_bitmap_engine_core pthread roaring) - - # 抽象层接口测试 - add_executable(test_abstraction_layer test/test_abstraction_layer.c src/simple_bitmap.c src/roaring_bitmap.c) - target_link_libraries(test_abstraction_layer pthread roaring) - - # RoaringBitmap专用测试 - add_executable(test_roaring_bitmap_specific test/test_roaring_bitmap_specific.c src/simple_bitmap.c src/roaring_bitmap.c) - target_link_libraries(test_roaring_bitmap_specific pthread roaring) - - # 事件拦截器测试 - add_executable(test_event_interceptor - test/test_event_interceptor.c - src/event_interceptor.c - src/bitmap_engine.c - src/ring_buffer.c - src/skiplist.c - src/simple_bitmap.c - src/roaring_bitmap.c - ) - target_link_libraries(test_event_interceptor pthread roaring) - - # 备份协同器测试 - add_executable(test_backup_coordinator - test/test_backup_coordinator.c - src/backup_coordinator.c - src/event_interceptor.c - src/bitmap_engine.c - src/ring_buffer.c - src/skiplist.c - src/simple_bitmap.c - src/roaring_bitmap.c - ) - target_link_libraries(test_backup_coordinator pthread roaring) - - # 状态转换测试 - add_executable(test_state_transitions - test/test_state_transitions.c - src/bitmap_engine.c - src/skiplist.c - src/simple_bitmap.c - src/roaring_bitmap.c - ) - target_link_libraries(test_state_transitions pthread roaring) - - # 跳表测试 - add_executable(test_skiplist - test/test_skiplist.c - src/skiplist.c - ) - target_link_libraries(test_skiplist pthread) - - # 可观测性接口测试 - add_executable(test_observability_interface test/test_observability_interface.c src/observability.c) - target_link_libraries(test_observability_interface) - - # 增强可观测性指标测试 - add_executable(test_observability_enhanced - test/test_observability_enhanced.c - src/observability.c - src/bitmap_engine.c - src/event_interceptor.c - src/ring_buffer.c - src/skiplist.c - src/simple_bitmap.c - src/roaring_bitmap.c - ) - target_link_libraries(test_observability_enhanced pthread roaring) - - # 全面可观测性指标测试 - add_executable(test_observability_comprehensive - test/test_observability_comprehensive.c - src/observability.c - src/bitmap_engine.c - src/event_interceptor.c - src/ring_buffer.c - src/skiplist.c - src/simple_bitmap.c - src/roaring_bitmap.c - ) - target_link_libraries(test_observability_comprehensive pthread roaring) - - # Offset语义验证测试 - add_executable(test_offset_semantics - test/test_offset_semantics.c - ) - target_link_libraries(test_offset_semantics pthread) - - # 真实TDengine Offset语义验证测试（仅在显式开启E2E且NOT USE_MOCK时构建） - if(E2E_TDENGINE_REAL_TESTS AND NOT USE_MOCK) - add_executable(test_offset_semantics_realtime - test/test_offset_semantics_realtime.c - ) - target_link_libraries(test_offset_semantics_realtime pthread taos) - - set(TEST_OFFSET_REAL_SRCS - test/test_offset_semantics_real.c - src/storage_engine_interface.c - src/tdengine_storage_engine.c - src/event_interceptor.c - src/bitmap_engine.c - src/ring_buffer.c - src/skiplist.c - src/simple_bitmap.c - src/roaring_bitmap.c - ) - add_executable(test_offset_semantics_real ${TEST_OFFSET_REAL_SRCS}) - target_link_libraries(test_offset_semantics_real pthread roaring taos) - else() - message(STATUS "跳过构建真实Offset语义测试: 需要 E2E_TDENGINE_REAL_TESTS=ON 且 NOT USE_MOCK") - endif() - - # 故障注入测试 - add_executable(test_fault_injection - test/test_fault_injection.c - ) - target_link_libraries(test_fault_injection pthread) - - # PITR E2E测试（完整版） - add_executable(test_pitr_e2e - test/test_pitr_e2e.c - test/pitr_e2e_test.c - src/bitmap_engine.c - src/backup_coordinator.c - src/event_interceptor.c - src/ring_buffer.c - src/skiplist.c - src/simple_bitmap.c - src/roaring_bitmap.c - ) - target_link_libraries(test_pitr_e2e pthread roaring) - - # PITR E2E测试（简化版） - add_executable(test_pitr_e2e_simple - test/test_pitr_e2e_simple.c - test/pitr_e2e_test.c - src/bitmap_engine.c - src/backup_coordinator.c - src/event_interceptor.c - src/ring_buffer.c - src/skiplist.c - src/simple_bitmap.c - src/roaring_bitmap.c - ) - target_link_libraries(test_pitr_e2e_simple pthread roaring) - - # 最小化一致性验证数据生成器 - add_executable(test_consistency_minimal - test/test_consistency_minimal.c - test/pitr_e2e_test.c - src/bitmap_engine.c - src/backup_coordinator.c - src/event_interceptor.c - src/ring_buffer.c - src/skiplist.c - src/simple_bitmap.c - src/roaring_bitmap.c - ) - target_link_libraries(test_consistency_minimal pthread roaring) - - # 重试机制测试 - # add_executable(test_retry_mechanism - # test/test_retry_mechanism.c - # src/backup_coordinator.c - # src/event_interceptor.c - # src/bitmap_engine.c - # src/ring_buffer.c - # src/skiplist.c - # src/simple_bitmap.c - # src/roaring_bitmap.c - # ) - # target_link_libraries(test_retry_mechanism pthread roaring) - - # TMQ 集成测试 - set(TEST_TMQ_SRCS - test/test_tmq_integration.c - src/storage_engine_interface.c - test/mock_storage_engine.c - src/event_interceptor.c - src/bitmap_engine.c - src/ring_buffer.c - src/skiplist.c - src/simple_bitmap.c - src/roaring_bitmap.c - ) - if(NOT USE_MOCK) - list(APPEND TEST_TMQ_SRCS src/tdengine_storage_engine.c) - endif() - add_executable(test_tmq_integration ${TEST_TMQ_SRCS}) - if(NOT USE_MOCK) - target_link_libraries(test_tmq_integration pthread roaring taos) - else() - message(WARNING "Building tests with USE_MOCK=ON: test_tmq_integration will run in mock mode") - target_link_libraries(test_tmq_integration pthread roaring) - endif() - - # 设置测试输出目录 - set_target_properties(test_ring_buffer test_bitmap_engine_core test_abstraction_layer test_roaring_bitmap_specific test_event_interceptor test_backup_coordinator test_state_transitions test_skiplist test_observability_interface test_observability_enhanced test_observability_comprehensive test_offset_semantics test_fault_injection test_pitr_e2e test_pitr_e2e_simple test_tmq_integration - PROPERTIES - RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/build - ) -endif() - -# 真实 TDengine 全链路 E2E 测试（默认不构建，需显式开启 E2E_TDENGINE_REAL_TESTS） -if(E2E_TDENGINE_REAL_TESTS) - if(USE_MOCK) - message(WARNING "E2E_TDENGINE_REAL_TESTS=ON 但 USE_MOCK=ON；将跳过真实E2E测试目标的构建") - else() - add_executable(test_e2e_tdengine_real - test/test_e2e_tdengine_real.c - test/e2e_consistency.c - test/e2e_perf.c - src/bitmap_engine.c - src/backup_coordinator.c - src/event_interceptor.c - src/ring_buffer.c - src/skiplist.c - src/simple_bitmap.c - src/roaring_bitmap.c - src/storage_engine_interface.c - src/tdengine_storage_engine.c - ) - target_link_libraries(test_e2e_tdengine_real pthread roaring taos) - set_target_properties(test_e2e_tdengine_real PROPERTIES - RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/build - ) - endif() -else() - message(STATUS "E2E_TDENGINE_REAL_TESTS=OFF: 跳过真实TDengine全链路E2E测试目标的构建") -endif() - -# taosdump增量对比测试 -add_executable(test_taosdump_comparison - test/test_taosdump_comparison.c - src/bitmap_engine.c - src/backup_coordinator.c - src/event_interceptor.c - src/ring_buffer.c - src/skiplist.c - src/simple_bitmap.c - src/roaring_bitmap.c -) -target_link_libraries(test_taosdump_comparison pthread roaring taos) -set_target_properties(test_taosdump_comparison PROPERTIES - RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/build -) - diff --git a/plugins/incremental_bitmap/CMakeLists_debug.txt b/plugins/incremental_bitmap/CMakeLists_debug.txt deleted file mode 100644 index e2d715878126..000000000000 --- a/plugins/incremental_bitmap/CMakeLists_debug.txt +++ /dev/null @@ -1,17 +0,0 @@ -# 添加调试测试 -add_executable(test_pitr_e2e_debug - test/test_pitr_e2e_debug.c - test/pitr_e2e_test.c -) - -target_link_libraries(test_pitr_e2e_debug - incremental_bitmap_plugin - ${ROARING_LIBRARIES} - pthread - m -) - -target_include_directories(test_pitr_e2e_debug PRIVATE - include - src -) diff --git a/plugins/incremental_bitmap/docs/TDengine_Logical_Backup_Functional_Specification.md b/plugins/incremental_bitmap/docs/TDengine_Logical_Backup_Functional_Specification.md deleted file mode 100644 index 259f0ce3ed54..000000000000 --- a/plugins/incremental_bitmap/docs/TDengine_Logical_Backup_Functional_Specification.md +++ /dev/null @@ -1,907 +0,0 @@ -# TDengine 逻辑备份和恢复功能规格说明 - -## 文档信息 -- **文档版本**: 1.0 -- **创建日期**: 2025-09-05 -- **最后更新**: 2025-09-05 -- **作者**: 章子渝 - -## 目录 -- [1. 概述](#1-概述) -- [2. 功能需求](#2-功能需求) -- [3. 系统架构](#3-系统架构) -- [4. 核心功能模块](#4-核心功能模块) -- [5. 接口规范](#5-接口规范) -- [6. 数据模型](#6-数据模型) -- [7. 性能要求](#7-性能要求) -- [8. 可靠性要求](#8-可靠性要求) -- [9. 兼容性要求](#9-兼容性要求) -- [10. 扩展性要求](#10-扩展性要求) - -## 1. 概述 - -### 1.1 项目背景 -TDengine Enterprise已支持基于TMQ（Time-Series Message Queue）的增量备份和恢复解决方案，但该方案存在初始备份性能差、耗时长的问题。本项目旨在通过增量位图插件技术，提供高性能的逻辑备份和恢复解决方案。 - -### 1.2 项目目标 -- 解决TMQ增量备份初始性能差、耗时长的问题 -- 基于数据库查询/存储引擎调用/数据文件扫描等方式，对指定时间范围的时序数据进行备份 -- 提供支持高效数据恢复的备份文件 -- 与现有taosdump工具深度集成 - -### 1.3 技术方案 -采用RoaringBitmap压缩算法和高效的事件处理框架，通过位图引擎、事件拦截器、备份协调器等核心组件，实现高性能的增量备份和恢复。 - -## 2. 功能需求 - -### 2.1 核心功能需求 - -#### 2.1.1 增量检测功能 -- **功能描述**: 实时检测数据块的变更状态 -- **输入**: 存储引擎事件流 -- **输出**: 变更块的状态信息（CLEAN/DIRTY/NEW/DELETED） -- **性能要求**: 支持每秒100万+块状态更新 - -#### 2.1.2 时间范围查询功能 -- **功能描述**: 根据时间范围查询需要备份的数据块 -- **输入**: 起始时间戳、结束时间戳 -- **输出**: 指定时间范围内的脏块列表 -- **性能要求**: 毫秒级响应时间 - -#### 2.1.3 WAL偏移量查询功能 -- **功能描述**: 根据WAL偏移量范围查询需要备份的数据块 -- **输入**: 起始WAL偏移量、结束WAL偏移量 -- **输出**: 指定WAL范围内的脏块列表 -- **性能要求**: 毫秒级响应时间 - -#### 2.1.4 备份协调功能 -- **功能描述**: 协调增量备份的整个流程 -- **输入**: 备份配置、时间范围或WAL范围 -- **输出**: 备份脚本、备份统计信息 -- **性能要求**: 支持10万+块批量处理 - -#### 2.1.5 与taosdump集成功能 -- **功能描述**: 生成taosdump兼容的备份脚本 -- **输入**: 数据库配置、时间范围 -- **输出**: 可执行的备份脚本 -- **性能要求**: 脚本生成时间<1秒 - -### 2.2 辅助功能需求 - -#### 2.2.1 可观测性功能 -- **功能描述**: 提供系统运行状态的监控和统计 -- **输入**: 系统运行数据 -- **输出**: 25个关键指标（性能、资源、错误等） -- **性能要求**: 毫秒级指标更新 - -#### 2.2.2 错误处理功能 -- **功能描述**: 处理各种异常情况和错误恢复 -- **输入**: 异常事件 -- **输出**: 错误信息、恢复建议 -- **性能要求**: 自动错误检测和恢复 - -#### 2.2.3 配置管理功能 -- **功能描述**: 管理系统配置参数 -- **输入**: 配置参数 -- **输出**: 配置验证结果 -- **性能要求**: 配置更新实时生效 - -## 3. 系统架构 - -### 3.1 整体架构 - -``` -┌─────────────────────────────────────────────────────────────────┐ -│ TDengine 增量位图插件 │ -├─────────────────────────────────────────────────────────────────┤ -│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ -│ │ 位图引擎 │ │ 事件拦截器 │ │ 备份协调器 │ │ 可观测性系统 │ │ -│ │ Bitmap │ │ Event │ │ Backup │ │ Observability│ │ -│ │ Engine │ │ Interceptor │ │ Coordinator │ │ System │ │ -│ └─────────────┘ └─────────────┘ └─────────────┘ └─────────────┘ │ -├─────────────────────────────────────────────────────────────────┤ -│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ -│ │ RoaringBitmap│ │ 环形缓冲区 │ │ 跳表索引 │ │ 存储引擎接口 │ │ -│ │ Algorithm │ │ Ring Buffer │ │ SkipList │ │ Storage │ │ -│ │ │ │ │ │ Index │ │ Interface │ │ -│ └─────────────┘ └─────────────┘ └─────────────┘ └─────────────┘ │ -├─────────────────────────────────────────────────────────────────┤ -│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ -│ │ taosdump │ │ TDengine │ │ 文件系统 │ │ 网络接口 │ │ -│ │ Integration │ │ Storage │ │ File System │ │ Network │ │ -│ │ │ │ Engine │ │ │ │ Interface │ │ -│ └─────────────┘ └─────────────┘ └─────────────┘ └─────────────┘ │ -└─────────────────────────────────────────────────────────────────┘ -``` - -### 3.2 数据流架构 - -``` -存储引擎事件 → 事件拦截器 → 环形缓冲区 → 回调线程池 → 位图引擎 - ↓ -备份协调器 ← 跳表索引 ← 位图状态管理 ← 事件处理结果 - ↓ -taosdump脚本生成 → 备份执行 → 验证恢复 -``` - -### 3.3 组件交互图 - -``` -┌─────────────┐ ┌─────────────┐ ┌─────────────┐ -│ Storage │───▶│ Event │───▶│ Ring │ -│ Engine │ │ Interceptor │ │ Buffer │ -└─────────────┘ └─────────────┘ └─────────────┘ - │ │ - ▼ ▼ -┌─────────────┐ ┌─────────────┐ ┌─────────────┐ -│ Backup │◀───│ Bitmap │◀───│ Callback │ -│ Coordinator │ │ Engine │ │ Thread Pool │ -└─────────────┘ └─────────────┘ └─────────────┘ - │ │ - ▼ ▼ -┌─────────────┐ ┌─────────────┐ -│ taosdump │ │ Observability│ -│ Integration │ │ System │ -└─────────────┘ └─────────────┘ -``` - -### 3.4 技术架构详细设计 - -#### 核心组件架构 -``` -┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ -│ Storage Engine│───▶│ Event Interceptor│───▶│ Ring Buffer │ -│ Events │ │ │ │ │ -└─────────────────┘ └─────────────────┘ └─────────────────┘ - │ │ - ▼ ▼ - ┌─────────────────┐ ┌─────────────────┐ - │ Callback Thread │ │ Event Statistics│ - │ Pool │ │ │ - └─────────────────┘ └─────────────────┘ - │ - ▼ - ┌─────────────────┐ - │ Bitmap Engine │ - │ │ - └─────────────────┘ - │ - ▼ - ┌─────────────────┐ - │ Backup │ - │ Coordinator │ - └─────────────────┘ -``` - -#### 数据结构设计 - -**SObservabilityMetrics**: 可观测性指标结构 -- 包含25个关键指标 -- 支持实时更新和查询 -- 内存布局优化 - -**SBitmapEngine**: 位图引擎结构 -- RoaringBitmap集成 -- 状态管理 (CLEAN/DIRTY/NEW/DELETED) -- 双重索引 (时间+WAL偏移量) - -**SEventInterceptor**: 事件拦截器结构 -- 环形缓冲区 -- 多线程处理 -- 事件统计 - -**SBackupCoordinator**: 备份协调器结构 -- 增量游标管理 -- 批量数据获取 -- 备份大小估算 - -### 3.5 集成架构设计 - -#### 插件接口设计 - -```c -// 备份插件接口 -typedef struct { - // 插件信息 - const char* (*get_plugin_name)(void); - const char* (*get_plugin_version)(void); - - // 初始化 - int32_t (*init)(const char* config); - void (*destroy)(void); - - // 增量检测 - int32_t (*detect_incremental_blocks)(const char* database, - uint64_t since_timestamp, - SIncrementalBlock** blocks, - uint32_t* block_count); - - // 数据导出 - int32_t (*export_blocks)(const SIncrementalBlock* blocks, - uint32_t block_count, - const char* output_path); - - // 数据恢复 - int32_t (*restore_blocks)(const char* backup_path, - const char* target_database); -} SBackupPluginInterface; -``` - -#### 配置文件格式 - -```json -{ - "plugin": { - "name": "incremental_bitmap", - "version": "1.0.0", - "library_path": "/usr/local/taos/plugins/backup/libincremental_bitmap_plugin.so" - }, - "bitmap_engine": { - "type": "roaring", - "memory_limit_mb": 1024, - "persistence_path": "/var/lib/taos/bitmap_cache" - }, - "event_interceptor": { - "enable": true, - "buffer_size": 10000, - "callback_threads": 4 - }, - "backup": { - "batch_size": 1000, - "compression": true, - "encryption": false, - "retry_count": 3, - "retry_interval_ms": 1000 - } -} -``` - -#### 与TDengine的集成点 - -```c -// 1. 存储引擎事件监听 -// 通过storage_engine_interface监听块变更事件 -SStorageEngineInterface* interface = get_storage_engine_interface("tdengine"); -interface->install_interception(); - -// 2. WAL文件监控 -// 监控WAL文件变化，获取增量信息 -int32_t monitor_wal_changes(const char* wal_path, - WALChangeCallback callback); - -// 3. 数据块访问 -// 通过TDengine API访问数据块 -int32_t read_data_block(uint64_t block_id, - void** data, - uint32_t* size); -``` - -### 3.6 开发进度与质量保证 - -#### 开发阶段完成情况 -- ✅ **第一阶段：可观测性指标** (100%) - 实现完整的可观测性监控体系 -- ✅ **第二阶段：Offset语义验证** (100%) - 验证Offset处理的正确性和一致性 -- ✅ **第三阶段：故障注入测试** (100%) - 验证系统在各种故障场景下的健壮性 -- ✅ **第四阶段：基础文档和CI** (100%) - 建立完整的文档体系和CI流程 -- ✅ **第五阶段：PITR E2E测试** (100%) - 实现完整的PITR端到端测试 - -#### 测试覆盖情况 -- **功能测试**: 100% 核心功能覆盖 -- **边界条件**: 100% 边界场景覆盖 -- **并发安全**: 100% 多线程安全验证 -- **内存管理**: 100% 内存泄漏检查 -- **错误恢复**: 100% 故障处理验证 - -#### 性能指标 -- **位图操作性能**: 每秒可处理100万+块状态更新 -- **事件处理性能**: 每秒可处理10万+事件 -- **备份协调性能**: 毫秒级的增量块查询 -- **内存使用**: 相比传统位图节省90%内存 -- **并发性能**: 默认自适应线程数（min(2×在线核数, 64)），可通过环境变量覆盖 - -## 4. 核心功能模块 - -### 4.1 位图引擎模块 - -#### 4.1.1 功能描述 -位图引擎是系统的核心组件，负责维护数据块的状态信息，使用RoaringBitmap算法实现高效的位图操作。 - -#### 4.1.2 主要功能 -- **状态管理**: 维护CLEAN/DIRTY/NEW/DELETED四种块状态 -- **位图操作**: 支持添加、删除、查询等位图操作 -- **索引管理**: 维护时间和WAL偏移量双重索引 -- **内存管理**: 支持内存限制和持久化策略 - -#### 4.1.3 性能指标 -- 支持10亿级块状态管理 -- 相比传统位图节省90%内存 -- O(1)时间复杂度的状态查询 -- 支持多线程并发（默认自适应线程数，支持环境变量覆盖） - -#### 4.1.4 接口规范 -```c -// 初始化位图引擎 -SBitmapEngine* bitmap_engine_init(void); - -// 销毁位图引擎 -void bitmap_engine_destroy(SBitmapEngine* engine); - -// 标记块状态 -int32_t bitmap_engine_mark_dirty(SBitmapEngine* engine, uint64_t block_id, - uint64_t wal_offset, int64_t timestamp); -int32_t bitmap_engine_mark_new(SBitmapEngine* engine, uint64_t block_id, - uint64_t wal_offset, int64_t timestamp); -int32_t bitmap_engine_mark_deleted(SBitmapEngine* engine, uint64_t block_id, - uint64_t wal_offset, int64_t timestamp); -int32_t bitmap_engine_clear_block(SBitmapEngine* engine, uint64_t block_id); - -// 查询块状态 -int32_t bitmap_engine_get_block_state(SBitmapEngine* engine, uint64_t block_id, - EBlockState* state); - -// 范围查询 -uint32_t bitmap_engine_get_dirty_blocks_by_time(SBitmapEngine* engine, - int64_t start_time, int64_t end_time, - uint64_t* block_ids, uint32_t max_count); -uint32_t bitmap_engine_get_dirty_blocks_by_wal(SBitmapEngine* engine, - uint64_t start_offset, uint64_t end_offset, - uint64_t* block_ids, uint32_t max_count); - -// 统计信息 -void bitmap_engine_get_stats(SBitmapEngine* engine, uint64_t* total_blocks, - uint64_t* dirty_count, uint64_t* new_count, uint64_t* deleted_count); -``` - -### 4.2 事件拦截器模块 - -#### 4.2.1 功能描述 -事件拦截器负责捕获存储引擎的事件，并将事件传递给位图引擎进行处理。 - -#### 4.2.2 主要功能 -- **事件捕获**: 拦截存储引擎的块变更事件 -- **事件缓冲**: 使用环形缓冲区缓存事件 -- **事件分发**: 将事件分发给回调线程池处理 -- **统计监控**: 提供事件处理统计信息 - -#### 4.2.3 性能指标 -- 每秒可处理10万+事件 -- 平均事件处理延迟<1ms -- 环形缓冲区零拷贝设计 -- CPU利用率>90% - -#### 4.2.4 接口规范 -```c -// 初始化事件拦截器 -SEventInterceptor* event_interceptor_init(const SEventInterceptorConfig* config, - struct SBitmapEngine* bitmap_engine); - -// 销毁事件拦截器 -void event_interceptor_destroy(SEventInterceptor* interceptor); - -// 启动/停止事件拦截器 -int32_t event_interceptor_start(SEventInterceptor* interceptor); -int32_t event_interceptor_stop(SEventInterceptor* interceptor); - -// 事件处理 -int32_t event_interceptor_on_block_create(SEventInterceptor* interceptor, - uint64_t block_id, uint64_t wal_offset, int64_t timestamp); -int32_t event_interceptor_on_block_update(SEventInterceptor* interceptor, - uint64_t block_id, uint64_t wal_offset, int64_t timestamp); -int32_t event_interceptor_on_block_flush(SEventInterceptor* interceptor, - uint64_t block_id, uint64_t wal_offset, int64_t timestamp); -int32_t event_interceptor_on_block_delete(SEventInterceptor* interceptor, - uint64_t block_id, uint64_t wal_offset, int64_t timestamp); - -// 统计信息 -void event_interceptor_get_stats(SEventInterceptor* interceptor, - uint64_t* events_processed, uint64_t* events_dropped); -``` - -### 4.3 备份协调器模块 - -#### 4.3.1 功能描述 -备份协调器负责协调整个增量备份流程，包括增量块检测、备份脚本生成、备份执行等。 - -#### 4.3.2 主要功能 -- **增量检测**: 识别需要备份的增量块 -- **游标管理**: 管理时间和WAL偏移量游标 -- **批量处理**: 支持批量块获取和处理 -- **脚本生成**: 生成taosdump兼容的备份脚本 -- **统计监控**: 提供备份统计信息 - -#### 4.3.3 性能指标 -- 毫秒级的增量块查询 -- 支持10万+块批量处理 -- 智能的内存分配和回收 -- 自动的错误检测和恢复 - -#### 4.3.4 接口规范 -```c -// 初始化备份协调器 -SBackupCoordinator* backup_coordinator_init(struct SBitmapEngine* bitmap_engine, - const SBackupConfig* config); - -// 销毁备份协调器 -void backup_coordinator_destroy(SBackupCoordinator* coordinator); - -// 启动/停止备份协调器 -int32_t backup_coordinator_start(SBackupCoordinator* coordinator); -int32_t backup_coordinator_stop(SBackupCoordinator* coordinator); - -// 获取增量块 -uint32_t backup_coordinator_get_dirty_blocks(SBackupCoordinator* coordinator, - uint64_t start_wal, uint64_t end_wal, - uint64_t* block_ids, uint32_t max_count); -uint32_t backup_coordinator_get_dirty_blocks_by_time(SBackupCoordinator* coordinator, - int64_t start_time, int64_t end_time, - uint64_t* block_ids, uint32_t max_count); - -// 获取增量块信息 -uint32_t backup_coordinator_get_incremental_blocks(SBackupCoordinator* coordinator, - uint64_t start_wal, uint64_t end_wal, - SIncrementalBlock* blocks, uint32_t max_count); - -// 估算备份大小 -uint64_t backup_coordinator_estimate_backup_size(SBackupCoordinator* coordinator, - uint64_t start_wal, uint64_t end_wal); - -// 统计信息 -int32_t backup_coordinator_get_stats(SBackupCoordinator* coordinator, SBackupStats* stats); -``` - -### 4.4 可观测性系统模块 - -#### 4.4.1 功能描述 -可观测性系统提供系统运行状态的监控和统计，包括性能指标、资源使用、错误统计等。 - -#### 4.4.2 主要功能 -- **指标收集**: 收集25个关键指标 -- **实时监控**: 毫秒级的指标更新 -- **多格式输出**: 支持JSON、Prometheus等格式 -- **智能告警**: 基于阈值的自动告警 - -#### 4.4.3 性能指标 -- 毫秒级指标更新频率 -- 对主流程影响<1% -- 高效的内存使用和序列化 -- 亚毫秒级指标查询响应 - -#### 4.4.4 指标分类 - -**速率指标 (Rate Metrics)** -- `events_per_second`: 事件处理速率，每秒处理的事件数量 -- `messages_per_second`: 消息消费速率，每秒消费的消息数量 -- `bytes_per_second`: 数据吞吐量，每秒处理的数据字节数 - -**滞后指标 (Lag Metrics)** -- `consumer_lag_ms`: 消费者滞后时间，消息从产生到被处理的时间差 -- `offset_lag`: Offset滞后数量，未处理的消息数量 -- `processing_delay_ms`: 处理延迟，单个事件从接收到处理完成的时间 - -**错误指标 (Error Metrics)** -- `events_dropped`: 丢弃事件数，由于各种原因未能处理的事件数量 -- `messages_dropped`: 丢弃消息数，未能处理的消息数量 -- `parse_errors`: 解析错误数，消息解析失败的数量 - -**重试指标 (Retry Metrics)** -- `connection_retries`: 连接重试次数，TMQ连接失败后的重试次数 -- `subscription_retries`: 订阅重试次数，主题订阅失败后的重试次数 -- `commit_retries`: 提交重试次数，Offset提交失败后的重试次数 - -**队列水位指标 (Queue Watermark Metrics)** -- `ring_buffer_usage`: 环形队列使用率，当前使用量占总容量的百分比 -- `ring_buffer_capacity`: 环形队列容量，缓冲区总容量 -- `event_queue_size`: 事件队列大小，当前队列中的事件数量 - -**内存指标 (Memory Metrics)** -- `memory_usage_bytes`: 总内存使用量，插件占用的总内存 -- `bitmap_memory_bytes`: 位图内存使用量，位图数据结构占用的内存 -- `metadata_memory_bytes`: 元数据内存使用量，元数据占用的内存 - -**时间指标 (Time Metrics)** -- `last_update_time`: 最后更新时间，指标最后更新的时间戳 -- `uptime_seconds`: 运行时间，插件启动后的运行时长 - -#### 4.4.5 接口规范 -```c -// 指标结构定义 -typedef struct { - // 速率指标 - uint64_t events_per_second; // 每秒事件数 - uint64_t messages_per_second; // 每秒消息数 - uint64_t bytes_per_second; // 每秒字节数 - - // 延迟指标 - uint64_t consumer_lag_ms; // 消费者延迟（毫秒） - uint64_t offset_lag; // 偏移量延迟 - uint64_t processing_delay_ms; // 处理延迟（毫秒） - - // 错误指标 - uint64_t dropped_events; // 丢弃的事件数 - uint64_t dropped_messages; // 丢弃的消息数 - uint64_t parse_errors; // 解析错误数 - - // 重试指标 - uint64_t connection_retries; // 连接重试次数 - uint64_t subscription_retries; // 订阅重试次数 - uint64_t commit_retries; // 提交重试次数 - - // 队列水位 - uint32_t ring_buffer_usage; // 环形缓冲区使用率（百分比） - uint32_t ring_buffer_capacity; // 环形缓冲区容量 - uint32_t event_queue_size; // 事件队列大小 - - // 内存使用 - size_t memory_usage_bytes; // 总内存使用（字节） - size_t bitmap_memory_bytes; // 位图内存使用（字节） - size_t metadata_memory_bytes; // 元数据内存使用（字节） - - // 时间信息 - int64_t last_update_time; // 最后更新时间 - int64_t uptime_ms; // 运行时间（毫秒） -} SObservabilityMetrics; - -// 更新指标 -void update_observability_metrics(void); - -// 打印人类可读格式 -void print_observability_metrics(void); - -// 生成JSON格式 -int32_t format_observability_metrics_json(char* buffer, size_t buffer_size); - -// 生成Prometheus格式 -int32_t format_observability_metrics_prometheus(char* buffer, size_t buffer_size); -``` - -## 5. 接口规范 - -### 5.1 C API接口 - -#### 5.1.1 位图引擎接口 -```c -// 错误码定义 -#define ERR_SUCCESS 0 // 成功 -#define ERR_INVALID_PARAM -1 // 无效参数 -#define ERR_INVALID_STATE_TRANS -1001 // 无效状态转换 -#define ERR_BLOCK_NOT_FOUND -1002 // 块未找到 - -// 块状态枚举 -typedef enum { - BLOCK_STATE_CLEAN = 0, // 未修改 - BLOCK_STATE_DIRTY = 1, // 已修改 - BLOCK_STATE_NEW = 2, // 新增 - BLOCK_STATE_DELETED = 3 // 已删除 -} EBlockState; - -// 块元数据 -typedef struct { - uint64_t block_id; // 物理块ID - uint64_t wal_offset; // WAL偏移量 - int64_t timestamp; // 纳秒级时间戳 - EBlockState state; // 块状态 -} SBlockMetadata; -``` - -#### 5.1.2 事件拦截器接口 -```c -// 块事件类型 -typedef enum { - EVENT_BLOCK_CREATE = 0, - EVENT_BLOCK_UPDATE, - EVENT_BLOCK_FLUSH, - EVENT_BLOCK_DELETE, - EVENT_MAX -} EBlockEventType; - -// 块事件结构 -typedef struct { - EBlockEventType event_type; - uint64_t block_id; - uint64_t wal_offset; - int64_t timestamp; - void* user_data; -} SBlockEvent; - -// 事件回调函数类型 -typedef void (*BlockEventCallback)(const SBlockEvent* event, void* user_data); -``` - -#### 5.1.3 备份协调器接口 -```c -// 备份游标类型 -typedef enum { - BACKUP_CURSOR_TYPE_TIME = 0, // 基于时间的游标 - BACKUP_CURSOR_TYPE_WAL = 1, // 基于WAL偏移量的游标 - BACKUP_CURSOR_TYPE_HYBRID = 2 // 混合游标 -} EBackupCursorType; - -// 增量块信息 -typedef struct { - uint64_t block_id; // 块ID - uint64_t wal_offset; // WAL偏移量 - int64_t timestamp; // 时间戳 - uint32_t data_size; // 数据大小 - void* data; // 块数据（可选） -} SIncrementalBlock; -``` - -### 5.2 插件接口 - -#### 5.2.1 taosX插件接口 -```c -// 插件接口结构 -typedef struct { - // 插件初始化 - int32_t (*init)(const char* config); - - // 插件销毁 - void (*destroy)(void); - - // 获取脏块 - uint32_t (*get_dirty_blocks)(uint64_t start_wal, uint64_t end_wal, - uint64_t* block_ids, uint32_t max_count); - - // 获取增量块 - uint32_t (*get_incremental_blocks)(uint64_t start_wal, uint64_t end_wal, - SIncrementalBlock* blocks, uint32_t max_count); - - // 估算备份大小 - uint64_t (*estimate_backup_size)(uint64_t start_wal, uint64_t end_wal); - - // 获取统计信息 - int32_t (*get_stats)(SBackupStats* stats); - - // 重置统计信息 - int32_t (*reset_stats)(void); -} SBackupPluginInterface; -``` - -### 5.3 命令行接口 - -#### 5.3.1 增量备份工具接口 -```bash -# 基本用法 -./incremental_bitmap_tool --detect \ - --host localhost --port 6030 \ - --database test_db \ - --since 1640995200 \ - --output incremental_blocks.json - -# 验证备份 -./incremental_bitmap_tool --verify \ - --backup /backup/ \ - --blocks incremental_blocks.json \ - --report backup_verification_report.json -``` - -## 6. 数据模型 - -### 6.1 块状态模型 - -#### 6.1.1 状态定义 -- **CLEAN**: 块未被修改，不需要备份 -- **DIRTY**: 块已被修改，需要备份 -- **NEW**: 块是新增的，需要备份 -- **DELETED**: 块已被删除，需要记录删除操作 - -#### 6.1.2 状态转换规则 -``` -CLEAN → DIRTY: 块被修改 -CLEAN → NEW: 块被创建 -DIRTY → CLEAN: 块被刷新到磁盘 -NEW → CLEAN: 新块被刷新到磁盘 -任何状态 → DELETED: 块被删除 -DELETED → CLEAN: 删除操作被确认 -``` - -### 6.2 索引模型 - -#### 6.2.1 时间索引 -- **结构**: 跳表（SkipList） -- **键**: 时间戳（纳秒级） -- **值**: 块ID位图 -- **用途**: 支持时间范围查询 - -#### 6.2.2 WAL索引 -- **结构**: 跳表（SkipList） -- **键**: WAL偏移量 -- **值**: 块ID位图 -- **用途**: 支持WAL范围查询 - -#### 6.2.3 元数据映射 -- **结构**: 哈希表 -- **键**: 块ID -- **值**: 块元数据 -- **用途**: O(1)时间复杂度的元数据访问 - -### 6.3 事件模型 - -#### 6.3.1 事件类型 -- **EVENT_BLOCK_CREATE**: 块创建事件 -- **EVENT_BLOCK_UPDATE**: 块更新事件 -- **EVENT_BLOCK_FLUSH**: 块刷新事件 -- **EVENT_BLOCK_DELETE**: 块删除事件 - -#### 6.3.2 事件结构 -```c -typedef struct { - EBlockEventType event_type; // 事件类型 - uint64_t block_id; // 块ID - uint64_t wal_offset; // WAL偏移量 - int64_t timestamp; // 时间戳 - void* user_data; // 用户数据 -} SBlockEvent; -``` - -## 7. 性能要求 - -### 7.1 响应时间要求 - -#### 7.1.1 位图操作性能 -- **添加操作**: 每秒可处理100万+块状态更新 -- **查询操作**: O(1)时间复杂度的状态查询 -- **范围查询**: 毫秒级的时间范围查询响应 - -#### 7.1.2 事件处理性能 -- **事件吞吐量**: 每秒可处理10万+事件 -- **事件延迟**: 平均事件处理延迟<1ms -- **缓冲区效率**: 环形缓冲区零拷贝设计 - -#### 7.1.3 备份协调性能 -- **增量查询**: 毫秒级的增量块查询 -- **批量处理**: 支持10万+块批量处理 -- **脚本生成**: 备份脚本生成时间<1秒 - -### 7.2 吞吐量要求 - -#### 7.2.1 并发处理能力 -- **并发线程**: 默认自适应线程数（min(2×在线核数, 64)），可通过 IB_CALLBACK_THREADS 覆盖 -- **CPU利用率**: >90% -- **内存效率**: 相比传统位图节省90%内存 - -#### 7.2.2 数据规模支持 -- **块数量**: 支持10亿级块状态管理 -- **时间范围**: 支持任意时间范围查询 -- **WAL范围**: 支持任意WAL偏移量范围查询 - -### 7.3 资源使用要求 - -#### 7.3.1 内存使用 -- **位图内存**: 相比传统位图节省90%内存 -- **缓冲区内存**: 可配置的环形缓冲区大小 -- **索引内存**: 高效的跳表索引内存使用 - -#### 7.3.2 磁盘使用 -- **持久化**: 支持位图状态持久化 -- **临时文件**: 可配置的临时文件路径 -- **备份文件**: 支持压缩和加密的备份文件 - -## 8. 可靠性要求 - -### 8.1 错误处理 - -#### 8.1.1 错误检测 -- **参数验证**: 所有API参数必须进行有效性验证 -- **状态检查**: 所有状态转换必须进行合法性检查 -- **资源检查**: 所有资源分配必须进行可用性检查 - -#### 8.1.2 错误恢复 -- **自动重试**: 支持可配置的重试机制 -- **降级处理**: 在资源不足时支持降级处理 -- **优雅关闭**: 支持优雅的系统关闭和资源清理 - -### 8.2 数据一致性 - -#### 8.2.1 状态一致性 -- **原子操作**: 所有状态更新必须是原子操作 -- **事务支持**: 支持多操作的事务性处理 -- **一致性检查**: 定期进行数据一致性检查 - -#### 8.2.2 备份一致性 -- **增量一致性**: 确保增量备份的数据一致性 -- **时间一致性**: 确保时间点恢复的数据一致性 -- **完整性验证**: 提供备份完整性验证功能 - -### 8.3 故障恢复 - -#### 8.3.1 系统故障恢复 -- **内存故障**: 支持内存不足时的优雅处理 -- **磁盘故障**: 支持磁盘空间不足时的处理 -- **网络故障**: 支持网络连接异常时的处理 - -#### 8.3.2 数据故障恢复 -- **位图损坏**: 支持位图数据损坏时的恢复 -- **索引损坏**: 支持索引数据损坏时的重建 -- **备份损坏**: 支持备份文件损坏时的处理 - -## 9. 兼容性要求 - -### 9.1 系统兼容性 - -#### 9.1.1 操作系统兼容性 -- **Linux**: Ubuntu 18.04+, CentOS 7+, RHEL 7+ -- **macOS**: 10.14+ (Mojave) -- **Windows**: Windows 10+ (WSL2推荐) - -#### 9.1.2 硬件兼容性 -- **CPU架构**: x86_64, ARM64 -- **内存要求**: 最小4GB，推荐8GB+ -- **磁盘要求**: 最小10GB可用空间 - -### 9.2 软件兼容性 - -#### 9.2.1 编译器兼容性 -- **GCC**: 7.0+ -- **Clang**: 5.0+ -- **MSVC**: Visual Studio 2017+ - -#### 9.2.2 库兼容性 -- **C库**: glibc 2.17+ 或 musl 1.1.20+ -- **线程库**: pthread -- **构建工具**: CMake 3.10+ - -### 9.3 接口兼容性 - -#### 9.3.1 API兼容性 -- **C API**: 标准C99接口 -- **ABI兼容性**: 保证二进制接口兼容性 -- **版本兼容性**: 支持向后兼容 - -#### 9.3.2 工具兼容性 -- **taosdump**: 与现有taosdump工具完全兼容 -- **TDengine**: 与TDengine存储引擎完全兼容 -- **第三方工具**: 支持与第三方备份工具集成 - -## 10. 扩展性要求 - -### 10.1 功能扩展性 - -#### 10.1.1 插件架构 -- **插件接口**: 提供标准化的插件接口 -- **动态加载**: 支持插件的动态加载和卸载 -- **配置管理**: 支持插件的配置管理 - -#### 10.1.2 算法扩展性 -- **位图算法**: 支持不同的位图算法实现 -- **索引算法**: 支持不同的索引算法实现 -- **压缩算法**: 支持不同的压缩算法实现 - -### 10.2 性能扩展性 - -#### 10.2.1 水平扩展 -- **分布式支持**: 支持分布式部署 -- **负载均衡**: 支持负载均衡和故障转移 -- **数据分片**: 支持数据分片和并行处理 - -#### 10.2.2 垂直扩展 -- **多核支持**: 充分利用多核CPU资源 -- **内存扩展**: 支持大内存配置 -- **存储扩展**: 支持高速存储设备 - -### 10.3 集成扩展性 - -#### 10.3.1 存储引擎集成 -- **接口标准化**: 提供标准化的存储引擎接口 -- **多引擎支持**: 支持多种存储引擎 -- **引擎切换**: 支持存储引擎的动态切换 - -#### 10.3.2 备份工具集成 -- **工具接口**: 提供标准化的备份工具接口 -- **多工具支持**: 支持多种备份工具 -- **工具链集成**: 支持完整的备份工具链 - ---- - -## 总结 - -本功能规格说明文档详细描述了TDengine增量位图插件的功能需求、系统架构、接口规范、数据模型、性能要求、可靠性要求、兼容性要求和扩展性要求。该文档为系统的设计、开发、测试和部署提供了完整的技术规范。 - -关键要点： -1. **高性能**: 支持10亿级块管理，性能提升数十倍 -2. **高可靠**: 完整的错误处理和故障恢复机制 -3. **高兼容**: 与现有TDengine和taosdump工具完全兼容 -4. **高扩展**: 支持插件架构和多种扩展方式 -5. **高可用**: 支持分布式部署和负载均衡 - -通过遵循本规格说明，可以确保系统满足所有功能需求，并具备良好的性能和可靠性。 diff --git a/plugins/incremental_bitmap/docs/TDengine_Logical_Backup_Test_Specification.md b/plugins/incremental_bitmap/docs/TDengine_Logical_Backup_Test_Specification.md deleted file mode 100644 index 70ed49a079eb..000000000000 --- a/plugins/incremental_bitmap/docs/TDengine_Logical_Backup_Test_Specification.md +++ /dev/null @@ -1,1925 +0,0 @@ -# TDengine 逻辑备份和恢复测试规格说明 - -## 文档信息 -- **文档版本**: 1.0 -- **创建日期**: 2025-09-05 -- **最后更新**: 2025-09-05 -- **作者**: 章子渝 - -## 快速开始（评审人员） - -### 非真实环境测试（推荐） -```bash -# 1. 进入插件目录 -cd /home/hp/TDengine/plugins/incremental_bitmap - -# 2. 运行完整测试（推荐） -./run_tests.sh - -# 3. 或运行快速测试 -./quick_test.sh -``` - -> 权限提示：若出现 "Permission denied"，请先执行 -> `chmod +x run_tests.sh quick_test.sh`，或使用 `bash ./run_tests.sh` 运行。 - -**预期结果**: 15个测试全部通过，成功率100% - -### 真实环境测试（可选） -```bash -# 1. 确保TDengine服务运行 -sudo systemctl start taosd - -# 2. 设置TDengine测试环境 -./setup_tdengine_test.sh - -# 3. 运行真实环境测试 -./run_real_tests.sh -``` - -> 权限提示：若出现 "Permission denied"，请先执行 -> `chmod +x setup_tdengine_test.sh run_real_tests.sh`，或使用 `bash ./run_real_tests.sh` 运行。 - -**预期结果**: 4个真实环境测试全部通过，成功率100% - -## 目录 -- [1. 概述](#1-概述) -- [2. 评审人员快速测试指南](#2-评审人员快速测试指南) - - [2.1 非真实环境测试（推荐）](#21-非真实环境测试推荐) - - [2.2 真实环境测试（可选）](#22-真实环境测试可选) - - [2.3 手动测试步骤](#23-手动测试步骤) - - [2.4 测试覆盖范围](#24-测试覆盖范围) - - [2.5 测试结果验证](#25-测试结果验证) - - [2.6 故障排除](#26-故障排除) - - [2.7 测试分类](#27-测试分类) -- [3. 测试策略](#3-测试策略) -- [4. 单元测试](#4-单元测试) -- [5. 集成测试](#5-集成测试) -- [6. 端到端测试](#6-端到端测试) - - [6.1 PITR端到端测试](#61-pitr端到端测试) - - [6.2 PITR端到端测试详细规格](#62-pitr端到端测试详细规格) - - [6.3 taosdump集成测试](#63-taosdump集成测试) -- [7. 性能测试](#7-性能测试) -- [8. 压力测试](#8-压力测试) -- [9. 兼容性测试](#9-兼容性测试) -- [10. 可靠性测试](#10-可靠性测试) -- [11. 测试环境](#11-测试环境) -- [12. 测试工具](#12-测试工具) -- [13. 测试数据](#13-测试数据) - -## 1. 概述 - -### 1.1 测试目标 -本测试规格说明旨在确保TDengine增量位图插件在功能、性能、可靠性和兼容性方面满足所有要求。测试覆盖从单元测试到端到端测试的完整测试体系。 - -### 1.2 测试范围 -- **功能测试**: 验证所有功能模块的正确性 -- **性能测试**: 验证系统性能指标是否达标 -- **压力测试**: 验证系统在高负载下的稳定性 -- **兼容性测试**: 验证与现有系统的兼容性 -- **可靠性测试**: 验证系统的错误处理和恢复能力 - -### 1.3 测试原则 -- **全面性**: 覆盖所有功能模块和代码路径 -- **自动化**: 尽可能实现测试自动化 -- **可重复性**: 测试结果必须可重复 -- **可维护性**: 测试代码易于维护和扩展 - -## 2. 评审人员快速测试指南 - -### 2.1 非真实环境测试（推荐） - -非真实环境测试使用Mock模式，无需真实的TDengine服务，适合快速验证代码逻辑。 - -#### 2.1.1 完整测试（推荐） -```bash -# 进入插件目录 -cd /home/hp/TDengine/plugins/incremental_bitmap - -# 运行完整测试脚本（15个测试） -./run_tests.sh -``` - -#### 2.1.2 快速测试 -```bash -# 进入插件目录 -cd /home/hp/TDengine/plugins/incremental_bitmap - -# 运行快速测试脚本（4个核心测试） -./quick_test.sh -``` - -### 2.2 真实环境测试（可选） - -真实环境测试需要运行中的TDengine服务，验证插件与真实数据库的集成。 - -#### 2.2.1 环境准备 -```bash -# 1. 确保TDengine服务运行 -sudo systemctl start taosd -sudo systemctl status taosd - -# 2. 检查TDengine版本 -/home/hp/TDengine/build/build/bin/taos --version -``` - -#### 2.2.2 设置测试环境 -```bash -# 进入插件目录 -cd /home/hp/TDengine/plugins/incremental_bitmap - -# 设置TDengine测试环境（创建数据库、表、TMQ主题） -./setup_tdengine_test.sh -``` - -**环境设置包括：** -- 创建测试数据库 `test` -- 创建超级表 `meters` 和子表 `d0`, `d1`, `d2` -- 创建TMQ主题 `incremental_backup_topic` -- 插入测试数据 - -#### 2.2.3 运行真实环境测试 -```bash -# 运行真实环境测试（4个测试） -./run_real_tests.sh -``` - -**真实环境测试包括：** -- `test_offset_semantics_realtime` - 实时偏移量语义测试 -- `test_taosdump_comparison` - taosdump对比测试 -- `test_pitr_e2e` - 完整PITR端到端测试 -- `test_e2e_tdengine_real` - 真实TDengine端到端测试 - -#### 2.2.4 清理测试环境（可选） -```bash -# 清理测试数据 -/home/hp/TDengine/build/build/bin/taos -c /home/hp/TDengine/build/test/cfg/taos.cfg -s "DROP DATABASE IF EXISTS test;" -``` - -#### 2.2.5 测试选择指南 -- **推荐使用**: `./run_tests.sh` - 非真实环境完整测试（15个测试） -- **快速验证**: `./quick_test.sh` - 非真实环境快速测试（4个核心测试） -- **真实环境**: `./run_real_tests.sh` - 真实TDengine环境测试（4个测试） -- **环境设置**: `./setup_tdengine_test.sh` - 设置TDengine测试环境 -- **手动测试**: 当需要调试特定问题时使用 - -**测试时间估算：** -- **非真实环境完整测试**: 约5-8分钟（15个测试） -- **非真实环境快速测试**: 约1-2分钟（4个核心测试） -- **真实环境测试**: 约3-5分钟（4个测试） -- **环境设置**: 约30秒 - -### 2.3 手动测试步骤 - -#### 2.3.1 构建项目 -```bash -# 1. 构建项目 -mkdir -p build && cd build -cmake -DUSE_MOCK=ON -DBUILD_TESTING=ON .. && make -j$(nproc) -``` - -#### 2.3.2 运行核心测试 -```bash -# 2. 运行核心测试 -./test_bitmap_engine_core -./test_state_transitions -./test_abstraction_layer -./test_backup_coordinator -``` - -#### 2.3.3 运行集成测试 -```bash -# 3. 运行集成测试 -./test_pitr_e2e_simple -``` - -#### 2.3.4 运行完整测试（可选） -```bash -# 4. 运行完整测试（可选） -./test_pitr_e2e -``` - -### 2.4 测试覆盖范围 - -#### 2.4.1 非真实环境测试（15个） - -**单元测试（13个）：** -- `test_bitmap_engine_core` - 位图引擎核心功能 -- `test_state_transitions` - 状态转换逻辑 -- `test_abstraction_layer` - 抽象层接口 -- `test_backup_coordinator` - 备份协调器 -- `test_event_interceptor` - 事件拦截器 -- `test_ring_buffer` - 环形缓冲区 -- `test_skiplist` - 跳表数据结构 -- `test_roaring_bitmap_specific` - RoaringBitmap特定功能 -- `test_fault_injection` - 故障注入测试 -- `test_offset_semantics` - 偏移量语义测试 -- `test_observability_interface` - 可观测性接口 -- `test_observability_enhanced` - 可观测性增强 -- `test_observability_comprehensive` - 可观测性综合 - -**集成测试（2个）：** -- `test_pitr_e2e_simple` - PITR端到端测试（简化版） -- `test_consistency_minimal` - 一致性测试（最小化） - -#### 2.4.2 真实环境测试（4个，可选） -- `test_offset_semantics_realtime` - 实时偏移量语义测试 -- `test_taosdump_comparison` - taosdump对比测试 -- `test_pitr_e2e` - 完整PITR端到端测试 -- `test_e2e_tdengine_real` - 真实TDengine端到端测试 - -#### 2.4.3 功能覆盖 -- ✅ **核心功能** - 位图引擎、状态转换、抽象层 -- ✅ **事件处理** - 事件拦截器、备份协调器 -- ✅ **数据结构** - 环形缓冲区、跳表、RoaringBitmap -- ✅ **故障处理** - 故障注入、偏移量语义 -- ✅ **可观测性** - 接口测试、增强测试、综合测试 -- ✅ **集成功能** - PITR端到端、一致性测试 - -### 2.5 测试结果验证 - -#### 2.5.1 成功标准 -- ✅ 所有测试显示 "Test passed" -- ✅ 成功率 100% -- ✅ 无段错误 -- ✅ 无内存泄漏（使用Valgrind检查） - -#### 2.5.2 预期输出 -``` -========================================== - PITR E2E Test Suite -========================================== - -Running test: PITR Tester Creation ------------------------------------------- -✓ Test passed: PITR Tester Creation - -Running test: Snapshot Functionality ------------------------------------------- -✓ Test passed: Snapshot Functionality - -... - -========================================== - Test Results -========================================== -Total tests: 15 -Passed: 15 -Failed: 0 -Success rate: 100.0% - -🎉 All tests passed! -``` - -#### 2.5.3 测试数据安全 -- **数据量限制**：< 3MB（默认配置） -- **路径安全**：使用相对路径，避免系统目录 -- **自动检查**：数据量和路径验证已启用 - -### 2.6 故障排除 - -#### 2.6.1 常见问题 -```bash -# 编译失败 -sudo apt-get install libroaring-dev cmake build-essential - -# 测试超时 -timeout 300s ./test_pitr_e2e - -# 内存检查 -valgrind --leak-check=full ./test_bitmap_engine_core -``` - -#### 2.6.2 日志文件 -- 编译日志：`build/cmake.log`, `build/make.log` -- 测试日志：`build/pitr_simple.log`, `build/pitr_full.log` -- 内存日志：`build/valgrind.log` - -## 3. 测试策略 - -### 3.1 测试金字塔 - -``` - ┌─────────────────┐ - │ 端到端测试 │ ← 少量，高价值 - │ (E2E Tests) │ - └─────────────────┘ - ┌─────────────────────┐ - │ 集成测试 │ ← 中等数量 - │ (Integration) │ - └─────────────────────┘ - ┌─────────────────────────┐ - │ 单元测试 │ ← 大量，快速 - │ (Unit Tests) │ - └─────────────────────────┘ -``` - -### 2.7 测试分类 - -#### 2.7.1 按测试类型分类 -- **单元测试**: 测试单个函数或模块 -- **集成测试**: 测试模块间的交互 -- **系统测试**: 测试整个系统的功能 -- **验收测试**: 验证用户需求是否满足 - -#### 2.7.2 按测试环境分类 -- **Mock测试**: 使用模拟环境进行测试 -- **真实环境测试**: 使用真实TDengine环境进行测试 -- **混合测试**: 结合Mock和真实环境进行测试 - -#### 2.7.3 按测试数据分类 -- **小数据量测试**: 使用少量数据进行功能验证 -- **大数据量测试**: 使用大量数据进行性能验证 -- **边界值测试**: 使用边界值进行极限测试 - -## 4. 单元测试 - -### 3.1 位图引擎单元测试 - -#### 3.1.1 测试目标 -验证位图引擎的所有API接口和内部逻辑的正确性。 - -#### 3.1.2 测试用例 - -##### 3.1.2.1 初始化和销毁测试 -```c -// 测试用例: 位图引擎初始化 -void test_bitmap_engine_init() { - SBitmapEngine* engine = bitmap_engine_init(); - assert(engine != NULL); - assert(engine->dirty_blocks != NULL); - assert(engine->new_blocks != NULL); - assert(engine->deleted_blocks != NULL); - bitmap_engine_destroy(engine); -} - -// 测试用例: 位图引擎销毁 -void test_bitmap_engine_destroy() { - SBitmapEngine* engine = bitmap_engine_init(); - assert(engine != NULL); - bitmap_engine_destroy(engine); - // 验证内存已释放 -} -``` - -##### 3.1.2.2 状态管理测试 -```c -// 测试用例: 标记块为脏状态 -void test_mark_dirty() { - SBitmapEngine* engine = bitmap_engine_init(); - uint64_t block_id = 12345; - uint64_t wal_offset = 1000; - int64_t timestamp = 1640995200000000000LL; - - int result = bitmap_engine_mark_dirty(engine, block_id, wal_offset, timestamp); - assert(result == 0); - - EBlockState state; - result = bitmap_engine_get_block_state(engine, block_id, &state); - assert(result == 0); - assert(state == BLOCK_STATE_DIRTY); - - bitmap_engine_destroy(engine); -} - -// 测试用例: 状态转换验证 -void test_state_transitions() { - SBitmapEngine* engine = bitmap_engine_init(); - uint64_t block_id = 12345; - - // CLEAN -> DIRTY - bitmap_engine_mark_dirty(engine, block_id, 1000, 1640995200000000000LL); - EBlockState state; - bitmap_engine_get_block_state(engine, block_id, &state); - assert(state == BLOCK_STATE_DIRTY); - - // DIRTY -> CLEAN - bitmap_engine_clear_block(engine, block_id); - bitmap_engine_get_block_state(engine, block_id, &state); - assert(state == BLOCK_STATE_CLEAN); - - bitmap_engine_destroy(engine); -} -``` - -##### 3.1.2.3 范围查询测试 -```c -// 测试用例: 时间范围查询 -void test_time_range_query() { - SBitmapEngine* engine = bitmap_engine_init(); - - // 添加测试数据 - int64_t base_time = 1640995200000000000LL; // 2022-01-01 00:00:00 - for (int i = 0; i < 100; i++) { - bitmap_engine_mark_dirty(engine, i, i * 1000, base_time + i * 1000000000LL); - } - - // 查询时间范围 - int64_t start_time = base_time + 10 * 1000000000LL; - int64_t end_time = base_time + 20 * 1000000000LL; - uint64_t block_ids[100]; - uint32_t count = bitmap_engine_get_dirty_blocks_by_time( - engine, start_time, end_time, block_ids, 100); - - assert(count == 10); // 应该找到10个块 - - bitmap_engine_destroy(engine); -} -``` - -### 3.2 事件拦截器单元测试 - -#### 3.2.1 测试目标 -验证事件拦截器的事件捕获、缓冲和分发功能。 - -#### 3.2.2 测试用例 - -##### 3.2.2.1 事件处理测试 -```c -// 测试用例: 事件拦截器初始化 -void test_event_interceptor_init() { - SBitmapEngine* engine = bitmap_engine_init(); - SEventInterceptorConfig config = { - .enable_interception = true, - .event_buffer_size = 1000, - .callback_threads = 2, - .callback = NULL, - .callback_user_data = NULL - }; - - SEventInterceptor* interceptor = event_interceptor_init(&config, engine); - assert(interceptor != NULL); - assert(interceptor->config.enable_interception == true); - assert(interceptor->bitmap_engine == engine); - - event_interceptor_destroy(interceptor); - bitmap_engine_destroy(engine); -} - -// 测试用例: 事件处理 -void test_event_processing() { - SBitmapEngine* engine = bitmap_engine_init(); - SEventInterceptorConfig config = { - .enable_interception = true, - .event_buffer_size = 1000, - .callback_threads = 2, - .callback = NULL, - .callback_user_data = NULL - }; - - SEventInterceptor* interceptor = event_interceptor_init(&config, engine); - assert(event_interceptor_start(interceptor) == 0); - - // 触发事件 - uint64_t block_id = 12345; - uint64_t wal_offset = 1000; - int64_t timestamp = 1640995200000000000LL; - - int result = event_interceptor_on_block_create(interceptor, block_id, wal_offset, timestamp); - assert(result == 0); - - // 验证事件被处理 - uint64_t events_processed, events_dropped; - event_interceptor_get_stats(interceptor, &events_processed, &events_dropped); - assert(events_processed > 0); - - event_interceptor_stop(interceptor); - event_interceptor_destroy(interceptor); - bitmap_engine_destroy(engine); -} -``` - -### 3.3 备份协调器单元测试 - -#### 3.3.1 测试目标 -验证备份协调器的增量检测、游标管理和脚本生成功能。 - -#### 3.3.2 测试用例 - -##### 3.3.2.1 增量检测测试 -```c -// 测试用例: 增量块检测 -void test_incremental_detection() { - SBitmapEngine* engine = bitmap_engine_init(); - SBackupConfig backup_config = { - .batch_size = 1000, - .max_retries = 3, - .retry_interval_ms = 1000, - .timeout_ms = 5000, - .enable_compression = true, - .enable_encryption = false, - .backup_path = "/tmp/backup", - .temp_path = "/tmp" - }; - - SBackupCoordinator* coordinator = backup_coordinator_init(engine, &backup_config); - assert(coordinator != NULL); - - // 添加一些脏块 - for (int i = 0; i < 100; i++) { - bitmap_engine_mark_dirty(engine, i, i * 1000, 1640995200000000000LL + i * 1000000000LL); - } - - // 检测增量块 - uint64_t block_ids[100]; - uint32_t count = backup_coordinator_get_dirty_blocks(coordinator, 0, 100000, block_ids, 100); - assert(count == 100); - - backup_coordinator_destroy(coordinator); - bitmap_engine_destroy(engine); -} -``` - -## 5. 集成测试 - -### 4.1 位图引擎与事件拦截器集成测试 - -#### 4.1.1 测试目标 -验证位图引擎与事件拦截器之间的集成工作是否正常。 - -#### 4.1.2 测试用例 - -```c -// 测试用例: 事件到位图状态的转换 -void test_event_to_bitmap_integration() { - SBitmapEngine* engine = bitmap_engine_init(); - - // 设置事件回调 - SEventInterceptorConfig config = { - .enable_interception = true, - .event_buffer_size = 1000, - .callback_threads = 2, - .callback = test_event_callback, - .callback_user_data = engine - }; - - SEventInterceptor* interceptor = event_interceptor_init(&config, engine); - assert(event_interceptor_start(interceptor) == 0); - - // 触发各种事件 - uint64_t block_id = 12345; - uint64_t wal_offset = 1000; - int64_t timestamp = 1640995200000000000LL; - - // CREATE事件 - event_interceptor_on_block_create(interceptor, block_id, wal_offset, timestamp); - EBlockState state; - bitmap_engine_get_block_state(engine, block_id, &state); - assert(state == BLOCK_STATE_NEW); - - // UPDATE事件 - event_interceptor_on_block_update(interceptor, block_id, wal_offset + 100, timestamp + 1000000000LL); - bitmap_engine_get_block_state(engine, block_id, &state); - assert(state == BLOCK_STATE_DIRTY); - - // FLUSH事件 - event_interceptor_on_block_flush(interceptor, block_id, wal_offset + 200, timestamp + 2000000000LL); - bitmap_engine_get_block_state(engine, block_id, &state); - assert(state == BLOCK_STATE_CLEAN); - - event_interceptor_stop(interceptor); - event_interceptor_destroy(interceptor); - bitmap_engine_destroy(engine); -} -``` - -### 4.2 备份协调器与位图引擎集成测试 - -#### 4.2.1 测试目标 -验证备份协调器与位图引擎之间的集成工作是否正常。 - -#### 4.2.2 测试用例 - -```c -// 测试用例: 备份协调器与位图引擎集成 -void test_backup_coordinator_bitmap_integration() { - SBitmapEngine* engine = bitmap_engine_init(); - SBackupConfig backup_config = { - .batch_size = 1000, - .max_retries = 3, - .retry_interval_ms = 1000, - .timeout_ms = 5000, - .enable_compression = true, - .enable_encryption = false, - .backup_path = "/tmp/backup", - .temp_path = "/tmp" - }; - - SBackupCoordinator* coordinator = backup_coordinator_init(engine, &backup_config); - assert(coordinator != NULL); - - // 添加测试数据到位图引擎 - for (int i = 0; i < 1000; i++) { - bitmap_engine_mark_dirty(engine, i, i * 1000, 1640995200000000000LL + i * 1000000000LL); - } - - // 通过备份协调器查询增量块 - uint64_t block_ids[1000]; - uint32_t count = backup_coordinator_get_dirty_blocks(coordinator, 0, 1000000, block_ids, 1000); - assert(count == 1000); - - // 验证查询结果 - for (uint32_t i = 0; i < count; i++) { - EBlockState state; - bitmap_engine_get_block_state(engine, block_ids[i], &state); - assert(state == BLOCK_STATE_DIRTY); - } - - backup_coordinator_destroy(coordinator); - bitmap_engine_destroy(engine); -} -``` - -## 6. 端到端测试 - -### 5.1 PITR端到端测试 - -#### 5.1.1 测试目标 -验证完整的PITR（Point-in-Time Recovery）功能，包括快照创建、恢复点管理和数据一致性验证。 - -#### 5.1.2 测试用例 - -##### 5.1.2.1 基本PITR测试 -```c -// 测试用例: 基本PITR功能 -void test_pitr_basic() { - SPitrTestConfig config = PITR_DEFAULT_CONFIG; - SPitrTester* tester = pitr_tester_create(&config); - assert(tester != NULL); - - // 启动PITR测试 - assert(pitr_tester_start(tester) == 0); - - // 等待测试完成 - sleep(config.test_duration_seconds); - - // 停止测试 - assert(pitr_tester_stop(tester) == 0); - - // 验证测试结果 - SPitrTestStatus status; - pitr_tester_get_status(tester, &status); - assert(status.test_passed == true); - assert(status.snapshots_created > 0); - assert(status.recovery_points_created > 0); - - pitr_tester_destroy(tester); -} -``` - -##### 5.1.2.2 多阶段集成测试 -```c -// 测试用例: 多阶段集成测试 -void test_pitr_multi_stage() { - SPitrTestConfig config = PITR_DEFAULT_CONFIG; - config.test_duration_seconds = 120; // 2分钟测试 - config.recovery_points = 10; // 10个恢复点 - - SPitrTester* tester = pitr_tester_create(&config); - assert(tester != NULL); - - // 重置测试器状态 - pitr_tester_reset(tester); - - // 启动测试 - assert(pitr_tester_start(tester) == 0); - - // 等待测试完成 - sleep(config.test_duration_seconds); - - // 停止测试 - assert(pitr_tester_stop(tester) == 0); - - // 验证多阶段结果 - SPitrTestStatus status; - pitr_tester_get_status(tester, &status); - assert(status.test_passed == true); - assert(status.snapshots_created >= config.recovery_points); - - pitr_tester_destroy(tester); -} -``` - -### 5.2 PITR端到端测试详细规格 - -#### 5.2.1 测试目标 -验证完整的PITR（Point-in-Time Recovery）功能，包括快照创建、恢复点管理和数据一致性验证。 - -#### 5.2.2 测试架构 - -**核心组件** -- **PITR测试器** (`SPitrTester`): 主要的测试执行引擎 -- **快照管理**: 自动创建和管理数据快照 -- **恢复点管理**: 验证时间点恢复功能 -- **数据一致性检查**: 确保数据在不同时间点的一致性 -- **性能基准测试**: 测量关键操作的性能指标 - -**测试流程** -``` -测试数据创建 → 快照生成 → 恢复点验证 → 乱序处理 → 删除一致性 → 边界条件 → 报告生成 -``` - -#### 5.2.3 测试功能详解 - -##### 5.2.3.1 快照+TMQ时间点恢复 - -**功能描述** -- 按配置的时间间隔自动创建数据快照 -- 支持TMQ（TDengine Message Queue）集成 -- 验证快照的完整性和一致性 - -**配置参数** -```c -.snapshot_interval_ms = 5000, // 快照间隔（毫秒） -.recovery_points = 10, // 恢复点数量 -.data_block_count = 10000, // 数据块数量 -``` - -**测试用例** -- 快照创建频率验证 -- 快照文件完整性检查 -- 快照元数据验证 -- 快照时间戳排序验证 - -##### 5.2.3.2 乱序数据处理 - -**功能描述** -- 模拟网络延迟和乱序事件 -- 测试不同乱序比例的处理能力 -- 验证乱序后的数据一致性 - -**测试场景** -```c -double disorder_ratios[] = {0.1, 0.3, 0.5, 0.7, 0.9}; -// 测试10%到90%的乱序比例 -``` - -**验证要点** -- 乱序事件正确重排序 -- 数据完整性保持 -- 性能影响评估 -- 内存使用监控 - -##### 5.2.3.3 边界条件测试 - -**功能描述** -- 测试极端数据量（0, 1, UINT64_MAX） -- 验证时间边界值处理 -- 内存压力测试 - -**边界值** -```c -uint64_t boundary_block_counts[] = {0, 1, 100, 1000000, UINT64_MAX}; -int64_t time_boundaries[] = {0, 1, INT64_MAX}; -``` - -**测试重点** -- 空数据处理 -- 单块数据处理 -- 大内存分配 -- 时间戳边界处理 - -##### 5.2.3.4 删除覆盖一致性验证 - -**功能描述** -- 模拟删除操作 -- 验证删除后的数据一致性 -- 测试删除恢复机制 - -**测试参数** -```c -uint64_t deletion_counts[] = {100, 500, 1000, 5000}; -// 测试不同数量的删除操作 -``` - -**一致性检查** -- 删除操作成功率 -- 数据完整性保持 -- 恢复点正确性 -- 元数据一致性 - -#### 5.2.4 测试用例 - -##### 5.2.4.1 基本PITR测试 -```c -// 测试用例: 基本PITR功能 -void test_pitr_basic() { - SPitrTestConfig config = PITR_DEFAULT_CONFIG; - SPitrTester* tester = pitr_tester_create(&config); - assert(tester != NULL); - - // 启动PITR测试 - assert(pitr_tester_start(tester) == 0); - - // 等待测试完成 - sleep(config.test_duration_seconds); - - // 停止测试 - assert(pitr_tester_stop(tester) == 0); - - // 验证测试结果 - SPitrTestStatus status; - pitr_tester_get_status(tester, &status); - assert(status.test_passed == true); - assert(status.snapshots_created > 0); - assert(status.recovery_points_created > 0); - - pitr_tester_destroy(tester); -} -``` - -##### 5.2.4.2 多阶段集成测试 -```c -// 测试用例: 多阶段集成测试 -void test_pitr_multi_stage() { - SPitrTestConfig config = PITR_DEFAULT_CONFIG; - config.test_duration_seconds = 120; // 2分钟测试 - config.recovery_points = 10; // 10个恢复点 - - SPitrTester* tester = pitr_tester_create(&config); - assert(tester != NULL); - - // 重置测试器状态 - pitr_tester_reset(tester); - - // 启动测试 - assert(pitr_tester_start(tester) == 0); - - // 等待测试完成 - sleep(config.test_duration_seconds); - - // 停止测试 - assert(pitr_tester_stop(tester) == 0); - - // 验证多阶段结果 - SPitrTestStatus status; - pitr_tester_get_status(tester, &status); - assert(status.test_passed == true); - assert(status.snapshots_created >= config.recovery_points); - - pitr_tester_destroy(tester); -} -``` - -### 5.3 taosdump集成测试 - -#### 5.3.1 测试目标 -验证与taosdump工具的集成功能，包括脚本生成、备份执行和验证。 - -#### 5.3.2 测试用例 - -##### 5.3.2.1 taosdump脚本生成测试 -```c -// 测试用例: taosdump脚本生成 -void test_taosdump_script_generation() { - SIncrementalBackupConfig config = { - .source_host = "localhost", - .source_port = 6030, - .database = "test_db", - .backup_path = "/tmp/backup", - .bitmap_cache_path = "/tmp/bitmap_cache", - .since_timestamp = 1640995200LL, - .batch_size = 1000, - .enable_compression = true, - .enable_encryption = false - }; - - SIncrementalBackupTool* tool = incremental_backup_tool_create(&config); - assert(tool != NULL); - - // 生成taosdump脚本 - const char* script_path = "/tmp/test_backup_script.sh"; - int result = incremental_backup_tool_generate_taosdump_script(tool, script_path); - assert(result == 0); - - // 验证脚本文件存在 - struct stat st; - assert(stat(script_path, &st) == 0); - assert(st.st_size > 0); - - // 验证脚本内容 - FILE* file = fopen(script_path, "r"); - assert(file != NULL); - - char line[1024]; - bool found_taosdump = false; - while (fgets(line, sizeof(line), file)) { - if (strstr(line, "taosdump") != NULL) { - found_taosdump = true; - break; - } - } - fclose(file); - assert(found_taosdump == true); - - incremental_backup_tool_destroy(tool); -} -``` - -##### 5.3.2.2 taosdump集成工作流测试 -```c -// 测试用例: taosdump集成工作流 -void test_taosdump_integration_workflow() { - // 创建测试数据 - assert(create_test_data() == 0); - - // 测试位图插件增量检测 - int64_t start_time = get_current_time_ms(); - uint64_t detected_blocks = test_bitmap_incremental_detection(); - int64_t detection_time = get_current_time_ms() - start_time; - - assert(detected_blocks > 0); - assert(detection_time < 1000); // 检测时间应小于1秒 - - // 测试taosdump备份 - start_time = get_current_time_ms(); - uint64_t backup_size = test_taosdump_backup(); - int64_t backup_time = get_current_time_ms() - start_time; - - assert(backup_size > 0); - assert(backup_time < 10000); // 备份时间应小于10秒 - - // 生成协作脚本 - assert(generate_collaboration_script() == 0); - - // 性能对比分析 - printf("位图插件检测: %lu blocks in %ld ms\n", detected_blocks, detection_time); - printf("taosdump备份: %lu bytes in %ld ms\n", backup_size, backup_time); -} -``` - -#### 5.3.3 测试结果分析 - -**测试状态结构** -```c -typedef struct { - uint64_t snapshots_created; // 已创建快照数量 - uint64_t recovery_points_verified; // 已验证恢复点数量 - uint64_t data_consistency_checks; // 数据一致性检查次数 - uint64_t disorder_handled; // 处理的乱序数据数量 - uint64_t deletion_handled; // 处理的删除操作数量 - uint64_t total_test_time_ms; // 总测试时间（毫秒） - bool test_passed; // 测试是否通过 - char error_message[512]; // 错误信息 -} SPitrTestStatus; -``` - -**数据一致性结果** -```c -typedef struct { - uint64_t expected_blocks; // 期望的块数量 - uint64_t actual_blocks; // 实际的块数量 - uint64_t mismatched_blocks; // 不匹配的块数量 - uint64_t missing_blocks; // 缺失的块数量 - uint64_t extra_blocks; // 多余的块数量 - double consistency_percentage; // 一致性百分比 - bool is_consistent; // 是否一致 - char details[512]; // 详细信息 -} SDataConsistencyResult; -``` - -**性能指标** -- **快照创建时间**: 每个快照的创建耗时 -- **恢复时间**: 从快照恢复到指定时间点的耗时 -- **乱序处理吞吐量**: 每秒处理的乱序事件数 -- **删除操作延迟**: 删除操作的响应时间 -- **内存使用峰值**: 测试过程中的最大内存使用量 - -## 7. 性能测试 - -### 6.1 位图引擎性能测试 - -#### 6.1.1 测试目标 -验证位图引擎在高负载下的性能表现。 - -#### 6.1.2 测试用例 - -##### 6.1.2.1 并发写入性能测试 -```c -// 测试用例: 并发写入性能 -void test_concurrent_write_performance() { - SBitmapEngine* engine = bitmap_engine_init(); - const int num_threads = 10; - const int operations_per_thread = 100000; - - pthread_t threads[num_threads]; - ThreadArgs args[num_threads]; - - // 创建线程 - for (int i = 0; i < num_threads; i++) { - args[i].engine = engine; - args[i].thread_id = i; - args[i].operations = operations_per_thread; - pthread_create(&threads[i], NULL, concurrent_write_thread, &args[i]); - } - - // 等待所有线程完成 - for (int i = 0; i < num_threads; i++) { - pthread_join(threads[i], NULL); - } - - // 验证结果 - uint64_t total_blocks, dirty_count, new_count, deleted_count; - bitmap_engine_get_stats(engine, &total_blocks, &dirty_count, &new_count, &deleted_count); - - assert(total_blocks == num_threads * operations_per_thread); - - bitmap_engine_destroy(engine); -} - -// 线程函数 -void* concurrent_write_thread(void* arg) { - ThreadArgs* args = (ThreadArgs*)arg; - SBitmapEngine* engine = args->engine; - - for (int i = 0; i < args->operations; i++) { - uint64_t block_id = args->thread_id * 1000000 + i; - uint64_t wal_offset = i * 1000; - int64_t timestamp = 1640995200000000000LL + i * 1000000LL; - - bitmap_engine_mark_dirty(engine, block_id, wal_offset, timestamp); - } - - return NULL; -} -``` - -##### 6.1.2.2 范围查询性能测试 -```c -// 测试用例: 范围查询性能 -void test_range_query_performance() { - SBitmapEngine* engine = bitmap_engine_init(); - - // 添加测试数据 - const int num_blocks = 1000000; - for (int i = 0; i < num_blocks; i++) { - bitmap_engine_mark_dirty(engine, i, i * 1000, 1640995200000000000LL + i * 1000000LL); - } - - // 测试时间范围查询性能 - int64_t start_time = get_current_time_ms(); - const int num_queries = 1000; - - for (int i = 0; i < num_queries; i++) { - int64_t query_start = 1640995200000000000LL + i * 1000000LL; - int64_t query_end = query_start + 1000000000LL; - uint64_t block_ids[1000]; - - uint32_t count = bitmap_engine_get_dirty_blocks_by_time( - engine, query_start, query_end, block_ids, 1000); - } - - int64_t end_time = get_current_time_ms(); - int64_t total_time = end_time - start_time; - - printf("范围查询性能: %d queries in %ld ms (%.2f ms/query)\n", - num_queries, total_time, (double)total_time / num_queries); - - assert(total_time < 1000); // 总时间应小于1秒 - - bitmap_engine_destroy(engine); -} -``` - -### 6.2 事件拦截器性能测试 - -#### 6.2.1 测试目标 -验证事件拦截器在高事件吞吐量下的性能表现。 - -#### 6.2.2 测试用例 - -##### 6.2.2.1 事件吞吐量测试 -```c -// 测试用例: 事件吞吐量测试 -void test_event_throughput() { - SBitmapEngine* engine = bitmap_engine_init(); - SEventInterceptorConfig config = { - .enable_interception = true, - .event_buffer_size = 100000, - .callback_threads = 8, - .callback = NULL, - .callback_user_data = NULL - }; - - SEventInterceptor* interceptor = event_interceptor_init(&config, engine); - assert(event_interceptor_start(interceptor) == 0); - - // 发送大量事件 - const int num_events = 1000000; - int64_t start_time = get_current_time_ms(); - - for (int i = 0; i < num_events; i++) { - uint64_t block_id = i; - uint64_t wal_offset = i * 1000; - int64_t timestamp = 1640995200000000000LL + i * 1000LL; - - event_interceptor_on_block_update(interceptor, block_id, wal_offset, timestamp); - } - - // 等待所有事件处理完成 - sleep(2); - - int64_t end_time = get_current_time_ms(); - int64_t total_time = end_time - start_time; - - // 验证事件处理统计 - uint64_t events_processed, events_dropped; - event_interceptor_get_stats(interceptor, &events_processed, &events_dropped); - - printf("事件吞吐量: %lu events in %ld ms (%.2f events/ms)\n", - events_processed, total_time, (double)events_processed / total_time); - - assert(events_processed >= num_events * 0.9); // 至少处理90%的事件 - assert(events_dropped < num_events * 0.1); // 丢弃事件应少于10% - - event_interceptor_stop(interceptor); - event_interceptor_destroy(interceptor); - bitmap_engine_destroy(engine); -} -``` - -### 6.3 并发自检测试 - -#### 6.3.1 测试目标 -验证并发自适应与环境变量覆盖是否生效，确保在不同核数机器上默认并发合理且可被覆盖。 - -#### 6.3.2 测试用例 -```bash -# 自动模式（未设置环境变量）应在启动日志打印： -# [并发配置] Detected cores=X, using callback_threads=Y (source=auto) -./build/test_e2e_tdengine_real 2>&1 | sed -n '1,40p' - -# 覆盖模式（设置环境变量）应打印： -# [并发配置] 使用环境变量 IB_CALLBACK_THREADS=32 -IB_CALLBACK_THREADS=32 ./build/test_e2e_tdengine_real 2>&1 | sed -n '1,40p' -``` - -#### 6.3.3 预期结果 -- 自动模式下，Y 应等于 min(2×在线核数, 64)。 -- 覆盖模式下，Y 应等于 32，且覆盖提示可见。 - -## 8. 压力测试 - -### 7.1 内存压力测试 - -#### 7.1.1 测试目标 -验证系统在内存压力下的稳定性和性能。 - -#### 7.1.2 测试用例 - -```c -// 测试用例: 内存压力测试 -void test_memory_pressure() { - SBitmapEngine* engine = bitmap_engine_init(); - - // 添加大量数据直到接近内存限制 - const int max_blocks = 10000000; // 1000万个块 - int64_t start_time = get_current_time_ms(); - - for (int i = 0; i < max_blocks; i++) { - bitmap_engine_mark_dirty(engine, i, i * 1000, 1640995200000000000LL + i * 1000LL); - - // 每100万个块检查一次内存使用 - if (i % 1000000 == 0) { - uint64_t total_blocks, dirty_count, new_count, deleted_count; - bitmap_engine_get_stats(engine, &total_blocks, &dirty_count, &new_count, &deleted_count); - printf("已处理 %lu 个块\n", total_blocks); - } - } - - int64_t end_time = get_current_time_ms(); - printf("内存压力测试: %d blocks in %ld ms\n", max_blocks, end_time - start_time); - - // 验证最终状态 - uint64_t total_blocks, dirty_count, new_count, deleted_count; - bitmap_engine_get_stats(engine, &total_blocks, &dirty_count, &new_count, &deleted_count); - assert(total_blocks == max_blocks); - assert(dirty_count == max_blocks); - - bitmap_engine_destroy(engine); -} -``` - -### 7.2 CPU压力测试 - -#### 7.2.1 测试目标 -验证系统在高CPU负载下的稳定性。 - -#### 7.2.2 测试用例 - -```c -// 测试用例: CPU压力测试 -void test_cpu_pressure() { - SBitmapEngine* engine = bitmap_engine_init(); - SEventInterceptorConfig config = { - .enable_interception = true, - .event_buffer_size = 100000, - .callback_threads = 16, // 使用更多线程 - .callback = NULL, - .callback_user_data = NULL - }; - - SEventInterceptor* interceptor = event_interceptor_init(&config, engine); - assert(event_interceptor_start(interceptor) == 0); - - // 持续发送事件 - const int duration_seconds = 60; - const int events_per_second = 100000; - int64_t start_time = get_current_time_ms(); - - for (int second = 0; second < duration_seconds; second++) { - int64_t second_start = get_current_time_ms(); - - for (int i = 0; i < events_per_second; i++) { - uint64_t block_id = second * events_per_second + i; - uint64_t wal_offset = block_id * 1000; - int64_t timestamp = 1640995200000000000LL + block_id * 1000LL; - - event_interceptor_on_block_update(interceptor, block_id, wal_offset, timestamp); - } - - // 等待到下一秒 - int64_t second_end = get_current_time_ms(); - int64_t sleep_time = 1000 - (second_end - second_start); - if (sleep_time > 0) { - usleep(sleep_time * 1000); - } - } - - int64_t end_time = get_current_time_ms(); - - // 验证处理统计 - uint64_t events_processed, events_dropped; - event_interceptor_get_stats(interceptor, &events_processed, &events_dropped); - - printf("CPU压力测试: %lu events processed, %lu dropped in %ld ms\n", - events_processed, events_dropped, end_time - start_time); - - assert(events_processed > duration_seconds * events_per_second * 0.8); - assert(events_dropped < duration_seconds * events_per_second * 0.2); - - event_interceptor_stop(interceptor); - event_interceptor_destroy(interceptor); - bitmap_engine_destroy(engine); -} -``` - -## 9. 兼容性测试 - -### 8.1 操作系统兼容性测试 - -#### 8.1.1 测试目标 -验证系统在不同操作系统上的兼容性。 - -#### 8.1.2 测试用例 - -```c -// 测试用例: Linux兼容性测试 -void test_linux_compatibility() { - // 测试基本功能 - SBitmapEngine* engine = bitmap_engine_init(); - assert(engine != NULL); - - // 测试位图操作 - bitmap_engine_mark_dirty(engine, 1, 1000, 1640995200000000000LL); - EBlockState state; - assert(bitmap_engine_get_block_state(engine, 1, &state) == 0); - assert(state == BLOCK_STATE_DIRTY); - - bitmap_engine_destroy(engine); -} - -// 测试用例: macOS兼容性测试 -void test_macos_compatibility() { - // 类似的测试逻辑 - // ... -} - -// 测试用例: Windows WSL2兼容性测试 -void test_windows_wsl2_compatibility() { - // 类似的测试逻辑 - // ... -} -``` - -### 8.2 编译器兼容性测试 - -#### 8.2.1 测试目标 -验证系统在不同编译器下的兼容性。 - -#### 8.2.2 测试用例 - -```bash -#!/bin/bash -# 测试用例: 编译器兼容性测试 - -# 测试GCC编译 -echo "Testing GCC compilation..." -gcc -std=c99 -Wall -Wextra -O2 -c test_bitmap_engine.c -if [ $? -eq 0 ]; then - echo "GCC compilation: PASSED" -else - echo "GCC compilation: FAILED" - exit 1 -fi - -# 测试Clang编译 -echo "Testing Clang compilation..." -clang -std=c99 -Wall -Wextra -O2 -c test_bitmap_engine.c -if [ $? -eq 0 ]; then - echo "Clang compilation: PASSED" -else - echo "Clang compilation: FAILED" - exit 1 -fi - -# 测试MSVC编译 (在Windows上) -if command -v cl.exe &> /dev/null; then - echo "Testing MSVC compilation..." - cl.exe /std:c99 /Wall /O2 /c test_bitmap_engine.c - if [ $? -eq 0 ]; then - echo "MSVC compilation: PASSED" - else - echo "MSVC compilation: FAILED" - exit 1 - fi -fi -``` - -## 10. 可靠性测试 - -### 9.1 错误注入测试 - -#### 9.1.1 测试目标 -验证系统在错误条件下的稳定性和恢复能力。 - -#### 9.1.2 测试用例 - -```c -// 测试用例: 内存分配失败测试 -void test_memory_allocation_failure() { - // 模拟内存分配失败 - // 这里需要使用内存分配钩子来模拟失败 - // 验证系统是否能优雅处理内存不足的情况 -} - -// 测试用例: 磁盘空间不足测试 -void test_disk_space_insufficient() { - // 创建临时目录并填满磁盘空间 - // 验证系统是否能检测并处理磁盘空间不足 -} - -// 测试用例: 网络连接失败测试 -void test_network_connection_failure() { - // 模拟网络连接失败 - // 验证系统是否能处理网络异常 -} -``` - -### 9.2 故障恢复测试 - -#### 9.2.1 测试目标 -验证系统在故障发生后的恢复能力。 - -#### 9.2.2 测试用例 - -```c -// 测试用例: 进程崩溃恢复测试 -void test_process_crash_recovery() { - // 启动系统 - SBitmapEngine* engine = bitmap_engine_init(); - // ... 初始化其他组件 - - // 模拟进程崩溃 - // 重启系统 - // 验证数据恢复 -} - -// 测试用例: 数据损坏恢复测试 -void test_data_corruption_recovery() { - // 创建正常数据 - // 模拟数据损坏 - // 验证系统是否能检测并恢复 -} -``` - -## 11. 测试环境 - -### 10.1 硬件环境 - -#### 10.1.1 最低配置 -- **CPU**: 2核心，2.0GHz -- **内存**: 4GB RAM -- **磁盘**: 20GB可用空间 -- **网络**: 100Mbps - -#### 10.1.2 推荐配置 -- **CPU**: 8核心，3.0GHz -- **内存**: 16GB RAM -- **磁盘**: 100GB SSD -- **网络**: 1Gbps - -#### 10.1.3 生产配置 -- **CPU**: 16核心，3.5GHz -- **内存**: 64GB RAM -- **磁盘**: 1TB NVMe SSD -- **网络**: 10Gbps - -### 10.2 软件环境 - -#### 10.2.1 操作系统 -- **Linux**: Ubuntu 20.04 LTS, CentOS 8, RHEL 8 -- **macOS**: 10.15+ (Catalina) -- **Windows**: Windows 10+ with WSL2 - -#### 10.2.2 开发工具 -- **编译器**: GCC 9.0+, Clang 10.0+ -- **构建工具**: CMake 3.16+ -- **调试工具**: GDB, Valgrind -- **测试框架**: CUnit, Google Test - -#### 10.2.3 运行时环境 -- **TDengine**: 3.0.0+ -- **taosdump**: 3.0.0+ -- **依赖库**: pthread, zlib, lz4 - -## 12. 测试工具 - -### 11.1 单元测试工具 - -#### 11.1.1 CUnit测试框架 -```c -#include -#include - -// 测试套件初始化 -int init_suite(void) { - return 0; -} - -// 测试套件清理 -int clean_suite(void) { - return 0; -} - -// 测试用例 -void test_bitmap_engine_init(void) { - SBitmapEngine* engine = bitmap_engine_init(); - CU_ASSERT_PTR_NOT_NULL(engine); - bitmap_engine_destroy(engine); -} - -// 主函数 -int main() { - CU_pSuite pSuite = NULL; - - if (CUE_SUCCESS != CU_initialize_registry()) - return CU_get_error(); - - pSuite = CU_add_suite("Bitmap Engine Suite", init_suite, clean_suite); - if (NULL == pSuite) { - CU_cleanup_registry(); - return CU_get_error(); - } - - if ((NULL == CU_add_test(pSuite, "test bitmap engine init", test_bitmap_engine_init))) { - CU_cleanup_registry(); - return CU_get_error(); - } - - CU_basic_set_mode(CU_BRM_VERBOSE); - CU_basic_run_tests(); - CU_cleanup_registry(); - - return CU_get_error(); -} -``` - -### 11.2 性能测试工具 - -#### 11.2.1 基准测试工具 -```c -// 基准测试工具 -typedef struct { - const char* name; - void (*test_func)(void); - int iterations; -} BenchmarkTest; - -void run_benchmark(const BenchmarkTest* tests, int num_tests) { - for (int i = 0; i < num_tests; i++) { - printf("Running benchmark: %s\n", tests[i].name); - - int64_t start_time = get_current_time_ms(); - for (int j = 0; j < tests[i].iterations; j++) { - tests[i].test_func(); - } - int64_t end_time = get_current_time_ms(); - - double avg_time = (double)(end_time - start_time) / tests[i].iterations; - printf("Average time: %.2f ms\n", avg_time); - } -} -``` - -### 11.3 压力测试工具 - -#### 11.3.1 内存压力测试工具 -```c -// 内存压力测试工具 -void memory_stress_test() { - SBitmapEngine* engine = bitmap_engine_init(); - - // 逐步增加内存使用 - for (int i = 0; i < 10000000; i++) { - bitmap_engine_mark_dirty(engine, i, i * 1000, 1640995200000000000LL + i * 1000LL); - - // 监控内存使用 - if (i % 100000 == 0) { - // 获取内存使用统计 - // 检查是否接近内存限制 - } - } - - bitmap_engine_destroy(engine); -} -``` - -## 13. 测试数据 - -### 12.1 测试数据生成 - -#### 12.1.1 基础测试数据 -```c -// 生成基础测试数据 -void generate_basic_test_data(SBitmapEngine* engine, int num_blocks) { - for (int i = 0; i < num_blocks; i++) { - uint64_t block_id = i; - uint64_t wal_offset = i * 1000; - int64_t timestamp = 1640995200000000000LL + i * 1000000LL; - - bitmap_engine_mark_dirty(engine, block_id, wal_offset, timestamp); - } -} -``` - -#### 12.1.2 复杂测试数据 -```c -// 生成复杂测试数据 -void generate_complex_test_data(SBitmapEngine* engine, int num_blocks) { - // 生成不同状态的块 - for (int i = 0; i < num_blocks; i++) { - uint64_t block_id = i; - uint64_t wal_offset = i * 1000; - int64_t timestamp = 1640995200000000000LL + i * 1000000LL; - - // 随机选择状态 - int state = rand() % 4; - switch (state) { - case 0: - bitmap_engine_mark_dirty(engine, block_id, wal_offset, timestamp); - break; - case 1: - bitmap_engine_mark_new(engine, block_id, wal_offset, timestamp); - break; - case 2: - bitmap_engine_mark_deleted(engine, block_id, wal_offset, timestamp); - break; - case 3: - bitmap_engine_clear_block(engine, block_id); - break; - } - } -} -``` - -#### 12.1.3 PITR测试数据 -```c -// 生成PITR测试数据 -void generate_pitr_test_data(SPitrTester* tester) { - // 创建测试数据缓冲区 - tester->test_data_size = tester->config.data_block_count * 1024; // 每个块1KB - tester->test_data_buffer = malloc(tester->test_data_size); - assert(tester->test_data_buffer != NULL); - - // 填充测试数据 - for (size_t i = 0; i < tester->test_data_size; i++) { - tester->test_data_buffer[i] = (char)(i % 256); - } - - // 创建数据文件 - char data_file[512]; - snprintf(data_file, sizeof(data_file), "%s/test_data.bin", tester->config.test_data_path); - - FILE* file = fopen(data_file, "wb"); - assert(file != NULL); - fwrite(tester->test_data_buffer, 1, tester->test_data_size, file); - fclose(file); - - printf("PITR测试数据已生成: %s (%zu bytes)\n", data_file, tester->test_data_size); -} -``` - -#### 12.1.4 乱序测试数据 -```c -// 生成乱序测试数据 -void generate_disorder_test_data(SPitrTester* tester, double disorder_ratio) { - // 创建事件序列 - SBlockEvent* events = malloc(tester->config.data_block_count * sizeof(SBlockEvent)); - assert(events != NULL); - - // 生成有序事件 - for (int i = 0; i < tester->config.data_block_count; i++) { - events[i].event_type = EVENT_BLOCK_UPDATE; - events[i].block_id = i; - events[i].wal_offset = i * 1000; - events[i].timestamp = 1640995200000000000LL + i * 1000000LL; - } - - // 应用乱序 - int disorder_count = (int)(tester->config.data_block_count * disorder_ratio); - for (int i = 0; i < disorder_count; i++) { - int idx1 = rand() % tester->config.data_block_count; - int idx2 = rand() % tester->config.data_block_count; - - // 交换事件 - SBlockEvent temp = events[idx1]; - events[idx1] = events[idx2]; - events[idx2] = temp; - } - - // 处理乱序事件 - for (int i = 0; i < tester->config.data_block_count; i++) { - event_interceptor_on_block_update(tester->event_interceptor, - events[i].block_id, - events[i].wal_offset, - events[i].timestamp); - } - - free(events); - printf("乱序测试数据已生成: 乱序比例=%.1f%%, 事件数=%d\n", - disorder_ratio * 100, tester->config.data_block_count); -} -``` - -### 12.2 测试数据验证 - -#### 12.2.1 数据一致性验证 -```c -// 验证数据一致性 -bool verify_data_consistency(SBitmapEngine* engine) { - uint64_t total_blocks, dirty_count, new_count, deleted_count; - bitmap_engine_get_stats(engine, &total_blocks, &dirty_count, &new_count, &deleted_count); - - // 验证统计数据的合理性 - if (total_blocks != dirty_count + new_count + deleted_count) { - printf("Data consistency check failed: total=%lu, dirty=%lu, new=%lu, deleted=%lu\n", - total_blocks, dirty_count, new_count, deleted_count); - return false; - } - - return true; -} -``` - -#### 12.2.2 快照一致性验证 -```c -// 验证快照一致性 -bool verify_snapshot_consistency(SPitrTester* tester, const SSnapshotInfo* snapshot) { - // 检查快照文件存在 - char snapshot_file[512]; - snprintf(snapshot_file, sizeof(snapshot_file), "%s/snapshot_%lu.bin", - tester->config.snapshot_path, snapshot->snapshot_id); - - struct stat st; - if (stat(snapshot_file, &st) != 0) { - printf("快照文件不存在: %s\n", snapshot_file); - return false; - } - - // 检查快照大小 - if (st.st_size != tester->test_data_size) { - printf("快照大小不匹配: 期望=%zu, 实际=%ld\n", tester->test_data_size, st.st_size); - return false; - } - - // 检查快照内容 - FILE* file = fopen(snapshot_file, "rb"); - if (!file) { - printf("无法打开快照文件: %s\n", snapshot_file); - return false; - } - - char* snapshot_data = malloc(tester->test_data_size); - size_t read_size = fread(snapshot_data, 1, tester->test_data_size, file); - fclose(file); - - if (read_size != tester->test_data_size) { - printf("快照读取不完整: 期望=%zu, 实际=%zu\n", tester->test_data_size, read_size); - free(snapshot_data); - return false; - } - - // 比较内容 - if (memcmp(snapshot_data, tester->test_data_buffer, tester->test_data_size) != 0) { - printf("快照内容不匹配\n"); - free(snapshot_data); - return false; - } - - free(snapshot_data); - printf("快照一致性验证通过: %s\n", snapshot_file); - return true; -} -``` - -#### 12.2.3 恢复点验证 -```c -// 验证恢复点 -bool verify_recovery_point(SPitrTester* tester, const SRecoveryPoint* recovery_point) { - // 检查恢复点时间戳 - if (recovery_point->timestamp <= 0) { - printf("恢复点时间戳无效: %ld\n", recovery_point->timestamp); - return false; - } - - // 检查恢复点数据 - if (recovery_point->data_size != tester->test_data_size) { - printf("恢复点数据大小不匹配: 期望=%zu, 实际=%lu\n", - tester->test_data_size, recovery_point->data_size); - return false; - } - - // 检查恢复点完整性 - if (recovery_point->checksum != calculate_checksum(tester->test_data_buffer, tester->test_data_size)) { - printf("恢复点校验和不匹配\n"); - return false; - } - - printf("恢复点验证通过: 时间戳=%ld, 大小=%lu\n", - recovery_point->timestamp, recovery_point->data_size); - return true; -} -``` - -### 12.3 测试数据清理 - -#### 12.3.1 清理测试数据 -```c -// 清理测试数据 -void cleanup_test_data(SPitrTester* tester) { - // 清理测试数据缓冲区 - if (tester->test_data_buffer) { - free(tester->test_data_buffer); - tester->test_data_buffer = NULL; - } - - // 清理测试目录 - char cmd[1024]; - snprintf(cmd, sizeof(cmd), "rm -rf %s %s %s", - tester->config.test_data_path, - tester->config.snapshot_path, - tester->config.recovery_path); - system(cmd); - - printf("测试数据已清理\n"); -} -``` - -#### 12.3.2 清理快照数据 -```c -// 清理快照数据 -void cleanup_snapshots(SPitrTester* tester) { - char cmd[512]; - snprintf(cmd, sizeof(cmd), "rm -rf %s/*", tester->config.snapshot_path); - system(cmd); - - // 重置快照计数 - tester->snapshot_count = 0; - - printf("快照数据已清理\n"); -} -``` - ---- - -## 测试验证总结 - -### 可观测性指标测试验证 - -#### 测试覆盖情况 -- **基础功能测试**: 5个测试，100%通过 -- **增强功能测试**: 8个测试，100%通过 -- **全面验证测试**: 12个测试，100%通过 -- **总测试数**: 45个断言，100%通过 - -#### 测试质量评估 -- **函数覆盖率**: 100% - 所有可观测性相关函数都有测试 -- **分支覆盖率**: 95% - 覆盖了主要的条件分支 -- **语句覆盖率**: 98% - 几乎所有的代码语句都有测试 - -#### 核心功能验证 -- **结构体定义**: 完整的SObservabilityMetrics结构体 -- **指标收集**: 25个关键指标的正确收集和更新 -- **格式化输出**: JSON和Prometheus格式输出 -- **集成功能**: 位图引擎、事件拦截器、环形队列集成 - -### Offset语义测试验证 - -#### 真实TDengine环境测试结果 -- **总测试数**: 83个测试 -- **通过测试**: 82个测试 -- **失败测试**: 1个测试（边界条件） -- **通过率**: 98.80% - -#### 性能指标 -- **批量提交性能**: 14,118.31 ops/sec -- **单次提交延迟**: ~1ms -- **并发处理能力**: 支持3个并发线程 - -#### 功能验证 -- **同步/异步提交**: 100%通过 -- **至少一次/至多一次语义**: 100%通过 -- **断点恢复测试**: 100%通过 -- **幂等性验证**: 100%通过 -- **并发提交测试**: 100%通过 -- **错误恢复测试**: 100%通过 - -### 测试执行结果 - -#### 基础测试执行 -```bash -# 基础接口测试 -./build/test_observability_interface -# 结果: 所有5个测试通过 ✅ - -# 增强功能测试 -./build/test_observability_enhanced -# 结果: 所有8个测试通过 ✅ -``` - -#### 全面测试执行 -```bash -# 全面验证测试 -./build/test_observability_comprehensive -# 结果: 所有12个测试通过 ✅ -# 总测试数: 45个断言 -# 通过率: 100% -``` - -#### 真实环境测试执行 -```bash -# 真实TDengine Offset语义测试 -./build/test_offset_semantics_realtime -# 结果: 82/83个测试通过 ✅ -# 通过率: 98.80% -``` - -### 质量保证 - -#### 代码质量 -- **静态分析**: 使用clang-tidy和cppcheck -- **代码格式**: 使用clang-format统一格式 -- **内存检查**: 使用Valgrind检查内存问题 -- **并发安全**: 完整的线程安全测试 - -#### 测试质量 -- **单元测试**: 100% 核心功能覆盖 -- **集成测试**: 真实TDengine环境验证 -- **性能测试**: 基准性能测试 -- **故障测试**: 完整的故障注入测试 - -#### 文档质量 -- **API文档**: 完整的接口说明和示例 -- **安装指南**: 详细的安装和配置说明 -- **故障排查**: 常见问题和解决方案 -- **最佳实践**: 使用建议和性能优化 - -### 测试完成度总结 -- **总体完成度**: 100% ✅ -- **功能覆盖**: 100% ✅ -- **测试质量**: 优秀 ✅ -- **可靠性**: 高 ✅ - -### 质量保证成果 -1. **功能完整性**: 所有核心功能都已实现并测试 -2. **代码质量**: 代码规范，错误处理完善 -3. **测试覆盖**: 全面的测试覆盖，包括边界条件和异常情况 -4. **文档完整**: 详细的技术文档和使用指南 -5. **构建集成**: 完整的构建系统集成 - ---- - -## 总结 - -本测试规格说明文档详细描述了TDengine增量位图插件的完整测试体系，包括单元测试、集成测试、端到端测试、性能测试、压力测试、兼容性测试和可靠性测试。通过遵循本测试规格，可以确保系统在各种条件下都能正常工作，并满足所有功能和性能要求。 - -关键要点： -1. **全面性**: 覆盖所有功能模块和代码路径 -2. **自动化**: 实现测试自动化，提高测试效率 -3. **可重复性**: 确保测试结果的一致性和可重复性 -4. **可维护性**: 测试代码易于维护和扩展 -5. **性能验证**: 确保系统满足性能要求 -6. **可靠性验证**: 确保系统在各种异常条件下都能稳定运行 - -通过执行本测试规格中定义的所有测试用例，可以全面验证TDengine增量位图插件的功能、性能和可靠性，确保其能够满足生产环境的要求。 diff --git a/plugins/incremental_bitmap/docs/TDengine_Logical_Backup_User_Manual.md b/plugins/incremental_bitmap/docs/TDengine_Logical_Backup_User_Manual.md deleted file mode 100644 index 8cb0fdf4f2f0..000000000000 --- a/plugins/incremental_bitmap/docs/TDengine_Logical_Backup_User_Manual.md +++ /dev/null @@ -1,1191 +0,0 @@ -# TDengine 逻辑备份和恢复用户手册 - -## 目录 -- [概述](#概述) -- [安装和配置](#安装和配置) -- [基本使用](#基本使用) -- [高级功能](#高级功能) -- [taosX插件集成](#taosx插件集成) -- [与taosdump集成](#与taosdump集成) -- [性能优化](#性能优化) -- [故障排查](#故障排查) -- [最佳实践](#最佳实践) - -## 概述 - -TDengine增量位图插件是一个高性能的逻辑备份和恢复解决方案，专门设计用于解决TDengine Enterprise基于TMQ的增量备份方案中存在的初始备份性能差、耗时长的问题。 - - -## 安装和配置 - -### 系统要求 -- **操作系统**：Linux (Ubuntu 18.04+, CentOS 7+), macOS 10.14+, Windows 10+ (WSL2) -- **硬件**：x86_64/ARM64, 最小4GB内存，推荐8GB+ -- **软件**：GCC 7.0+, CMake 3.10+, pthread库 - -### 安装步骤 - -```bash -# 1. 克隆TDengine仓库 -git clone https://github.com/taosdata/TDengine.git -cd TDengine - -# 2. 构建插件 -mkdir build && cd build -cmake .. -DBUILD_PLUGINS=ON -DENABLE_TESTS=ON -make -j$(nproc) - -# 3. 安装插件 -sudo make install -``` - -### 配置选项 - -#### CMake配置选项 - -| 选项 | 默认值 | 说明 | -|------|--------|------| -| `BUILD_PLUGINS` | OFF | 是否构建插件 | -| `BUILD_TMQ` | OFF | 是否构建TMQ支持 | -| `CMAKE_BUILD_TYPE` | Debug | 构建类型 (Debug/Release/RelWithDebInfo) | -| `CMAKE_INSTALL_PREFIX` | /usr/local | 安装前缀 | -| `ENABLE_TESTS` | ON | 是否构建测试 | -| `ENABLE_COVERAGE` | OFF | 是否启用代码覆盖率 | -| `ENABLE_SANITIZERS` | OFF | 是否启用地址/线程检查器 | -| `USE_MOCK` | ON | 是否使用Mock环境 | -| `E2E_TDENGINE_REAL_TESTS` | OFF | 是否启用真实TDengine测试 | -| `BUILD_TAOSX_PLUGIN` | OFF | 是否构建taosX插件 | - -#### 构建类型说明 -- **Debug**: 包含调试信息，性能较低 -- **Release**: 优化构建，性能最高 -- **RelWithDebInfo**: 优化构建+调试信息 - -#### 示例配置 - -```bash -# 生产环境配置 -cmake .. \ - -DBUILD_PLUGINS=ON \ - -DBUILD_TMQ=ON \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=/opt/tdengine \ - -DENABLE_TESTS=OFF \ - -DUSE_MOCK=OFF \ - -DE2E_TDENGINE_REAL_TESTS=ON \ - -DBUILD_TAOSX_PLUGIN=ON - -# 开发环境配置 -cmake .. \ - -DBUILD_PLUGINS=ON \ - -DBUILD_TMQ=ON \ - -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_INSTALL_PREFIX=$HOME/.local \ - -DENABLE_TESTS=ON \ - -DENABLE_COVERAGE=ON \ - -DENABLE_SANITIZERS=ON \ - -DUSE_MOCK=ON \ - -DE2E_TDENGINE_REAL_TESTS=OFF \ - -DBUILD_TAOSX_PLUGIN=ON -``` - -### 环境变量 - -#### 构建环境变量 -```bash -# 编译器选择 -export CC=gcc-9 -export CXX=g++-9 - -# 构建并行度 -export CMAKE_BUILD_PARALLEL_LEVEL=4 - -# 安装路径 -export CMAKE_INSTALL_PREFIX=/opt/tdengine - -# 测试环境 -export CTEST_PARALLEL_LEVEL=4 -export CTEST_OUTPUT_ON_FAILURE=1 -``` - -#### 运行时环境变量 -```bash -# 插件路径 -export TDENGINE_PLUGIN_PATH=/usr/local/lib/tdengine/plugins - -# 日志级别 -export TDENGINE_LOG_LEVEL=INFO - -# 配置文件路径 -export TDENGINE_CONFIG_PATH=/etc/tdengine - -# 临时目录 -export TDENGINE_TEMP_PATH=/tmp/tdengine - -# TDengine连接配置 -export TD_CONNECT_IP=localhost -export TD_CONNECT_PORT=6030 -export TD_USERNAME=root -export TD_PASSWORD=taosdata -export TD_DATABASE=test - -# TMQ配置 -export TD_TOPIC_NAME=test_topic -export TD_GROUP_ID=test_group - -# TMQ超时配置（可选） -export TMQ_KEY_CONNECT_TIMEOUT=connect.timeout -export TMQ_KEY_REQUEST_TIMEOUT=request.timeout.ms - -# 并发线程覆盖（可选） -# 若未设置，则默认采用自适应: callback_threads = min(2×在线CPU核数, 64) -export IB_CALLBACK_THREADS=32 -``` - -#### TMQ兼容性配置 - -插件支持TDengine 3.2/3.3系列，通过"多键名回退 + 环境变量覆盖"实现TMQ配置键名的版本兼容。 - -**键名回退策略**: -- **连接超时（connect timeout）** - - 尝试顺序：`connect.timeout` → `td.connect.timeout` → `connection.timeout` - - 可用环境变量：`TMQ_KEY_CONNECT_TIMEOUT` -- **请求超时（request timeout）** - - 尝试顺序：`request.timeout.ms` → `td.request.timeout` → `request.timeout` - - 可用环境变量：`TMQ_KEY_REQUEST_TIMEOUT` -- **数据库名（database）** - - 尝试顺序：`td.connect.database` → `td.connect.db`（失败则跳过，不致命） - -**版本支持矩阵**: -- **3.3.x（社区/企业）**：推荐键`td.connect.*`前缀优先，`request.timeout.ms`支持度依版本而异 -- **3.2.x（社区/企业）**：推荐键`connect.timeout` / `request.timeout.ms`，部分`td.connect.*`可能不可用 - -**最佳实践**: -- 首次联调：开启`REAL_TDENGINE=1`，观察启动日志中TMQ键名是否生效 -- 如出现警告：使用`TMQ_KEY_*`或`TD_*`环境变量覆盖 -- CI中：固定一套环境变量，保证结果稳定 - -## 基本使用 - -### 1. 初始化位图引擎 - -```c -#include "bitmap_engine.h" -#include "event_interceptor.h" -#include "backup_coordinator.h" - -int main() { - // 初始化位图引擎 - SBitmapEngine* engine = bitmap_engine_init(); - if (!engine) { - fprintf(stderr, "Failed to initialize bitmap engine\n"); - return -1; - } - - // 配置事件拦截器 - SEventInterceptorConfig event_config = { - .enable_interception = true, - .event_buffer_size = 10000, - .callback_threads = 4, // 示例值；实际运行时默认采用自适应或由 IB_CALLBACK_THREADS 覆盖 - .callback = NULL, - .callback_user_data = NULL - }; - - SEventInterceptor* interceptor = event_interceptor_init(&event_config, engine); - if (!interceptor) { - fprintf(stderr, "Failed to initialize event interceptor\n"); - bitmap_engine_destroy(engine); - return -1; - } - - // 启动事件处理 - if (event_interceptor_start(interceptor) != 0) { - fprintf(stderr, "Failed to start event interceptor\n"); - event_interceptor_destroy(interceptor); - bitmap_engine_destroy(engine); - return -1; - } - - // 使用位图引擎进行业务操作 - // ... - - // 清理资源 - event_interceptor_stop(interceptor); - event_interceptor_destroy(interceptor); - bitmap_engine_destroy(engine); - - return 0; -} -``` - -### 2. 标记块状态 - -```c -// 标记块为脏状态 -int result = bitmap_engine_mark_dirty(engine, block_id, wal_offset, timestamp); -if (result != 0) { - fprintf(stderr, "Failed to mark block as dirty\n"); -} - -// 标记块为新增状态 -result = bitmap_engine_mark_new(engine, block_id, wal_offset, timestamp); - -// 标记块为删除状态 -result = bitmap_engine_mark_deleted(engine, block_id, wal_offset, timestamp); - -// 清除块状态 -result = bitmap_engine_clear_block(engine, block_id); -``` - -### 3. 查询块状态 - -```c -// 获取块当前状态 -EBlockState state; -int result = bitmap_engine_get_block_state(engine, block_id, &state); -if (result == 0) { - switch (state) { - case BLOCK_STATE_CLEAN: - printf("Block is clean\n"); - break; - case BLOCK_STATE_DIRTY: - printf("Block is dirty\n"); - break; - case BLOCK_STATE_NEW: - printf("Block is new\n"); - break; - case BLOCK_STATE_DELETED: - printf("Block is deleted\n"); - break; - } -} else { - printf("Block not found\n"); -} -``` - -## 高级功能 - -### 1. 时间范围查询 - -```c -// 获取指定时间范围内的脏块 -int64_t start_time = 1640995200000000000LL; // 2022-01-01 00:00:00 -int64_t end_time = 1641081600000000000LL; // 2022-01-02 00:00:00 -uint64_t block_ids[1000]; -uint32_t max_count = 1000; - -uint32_t count = bitmap_engine_get_dirty_blocks_by_time( - engine, start_time, end_time, block_ids, max_count); - -printf("Found %u dirty blocks in time range\n", count); -for (uint32_t i = 0; i < count; i++) { - printf("Block ID: %lu\n", block_ids[i]); -} -``` - -### 2. WAL偏移量查询 - -```c -// 获取指定WAL偏移量范围内的脏块 -uint64_t start_offset = 0; -uint64_t end_offset = 1000000; -uint64_t block_ids[1000]; -uint32_t max_count = 1000; - -uint32_t count = bitmap_engine_get_dirty_blocks_by_wal( - engine, start_offset, end_offset, block_ids, max_count); - -printf("Found %u dirty blocks in WAL range\n", count); -``` - -### 3. 备份协调器使用 - -```c -// 初始化备份协调器 -SBackupConfig backup_config = { - .batch_size = 1000, - .max_retries = 3, - .retry_interval_ms = 1000, - .timeout_ms = 30000, - .enable_compression = true, - .enable_encryption = false, - .backup_path = "/backup", - .temp_path = "/tmp" -}; - -SBackupCoordinator* coordinator = backup_coordinator_init(engine, &backup_config); -if (!coordinator) { - fprintf(stderr, "Failed to initialize backup coordinator\n"); - return -1; -} - -// 启动备份协调器 -if (backup_coordinator_start(coordinator) != 0) { - fprintf(stderr, "Failed to start backup coordinator\n"); - backup_coordinator_destroy(coordinator); - return -1; -} - -// 获取增量块 -SIncrementalBlock blocks[1000]; -uint32_t count = backup_coordinator_get_incremental_blocks( - coordinator, start_wal, end_wal, blocks, 1000); - -// 估算备份大小 -uint64_t size = backup_coordinator_estimate_backup_size( - coordinator, start_wal, end_wal); - -printf("Estimated backup size: %lu bytes\n", size); - -// 获取统计信息 -SBackupStats stats; -backup_coordinator_get_stats(coordinator, &stats); -printf("Total blocks: %lu, Processed: %lu, Failed: %lu\n", - stats.total_blocks, stats.processed_blocks, stats.failed_blocks); - -// 清理资源 -backup_coordinator_stop(coordinator); -backup_coordinator_destroy(coordinator); -``` - -## taosX插件集成 - -### taosX插件接口 - -位图插件提供了标准的taosX插件接口，支持与taosX数据流平台的集成： - -#### 核心功能 -- **标准插件接口**：实现所需的taosX插件API -- **事件处理**：处理来自taosX的块事件 -- **统计信息**：提供运行时统计和监控 -- **生命周期管理**：正确的初始化、配置和关闭 -- **最小依赖**：仅需要标准C库和pthread - -#### API参考 - -**核心函数**: -- `taosx_plugin_get_name()` - 返回插件名称 -- `taosx_plugin_get_version()` - 返回插件版本 -- `taosx_plugin_get_capabilities()` - 返回插件能力 -- `taosx_plugin_init()` - 初始化插件 -- `taosx_plugin_shutdown()` - 关闭插件 -- `taosx_plugin_configure()` - 配置插件 -- `taosx_plugin_start()` - 启动插件 -- `taosx_plugin_stop()` - 停止插件 -- `taosx_plugin_on_block_event()` - 处理块事件 -- `taosx_plugin_get_stats()` - 获取插件统计 - -**数据结构**: -- `TaosX_Config` - 插件配置 -- `TaosX_BlockEvent` - 块事件数据 -- `TaosX_PluginStats` - 插件统计 -- `TaosX_EventType` - 事件类型枚举 - -#### 构建和安装 - -taosX插件接口**默认禁用**。要启用它： - -```bash -# 启用taosX插件接口 -cmake -DBUILD_TAOSX_PLUGIN=ON .. - -# 构建 -make taosx_incremental_bitmap_plugin -``` - -#### 使用示例 - -```c -#include "taosx_plugin_interface.h" - -// 初始化插件 -int rc = taosx_plugin_init(); -if (rc != TAOSX_PLUGIN_OK) { - // 处理错误 -} - -// 配置插件 -TaosX_Config config = {0}; // 根据需要配置 -rc = taosx_plugin_configure(&config); - -// 启动插件 -rc = taosx_plugin_start(); - -// 处理事件 -TaosX_BlockEvent event = { - .block_id = 123, - .wal_offset = 456, - .timestamp_ns = 789, - .event_type = TAOSX_EVENT_BLOCK_CREATE -}; -rc = taosx_plugin_on_block_event(&event); - -// 获取统计信息 -TaosX_PluginStats stats; -rc = taosx_plugin_get_stats(&stats); - -// 停止并关闭 -taosx_plugin_stop(); -taosx_plugin_shutdown(); -``` - -#### 事件类型 - -- `TAOSX_EVENT_BLOCK_CREATE` - 块创建事件 -- `TAOSX_EVENT_BLOCK_UPDATE` - 块更新事件 -- `TAOSX_EVENT_BLOCK_FLUSH` - 块刷盘事件 - -#### 错误处理 - -所有函数返回`TaosX_PluginCode`中定义的错误代码： -- `TAOSX_PLUGIN_OK` - 成功 -- `TAOSX_PLUGIN_ERR_INVALID_ARG` - 无效参数 -- `TAOSX_PLUGIN_ERR_NOT_INITIALIZED` - 插件未初始化 -- `TAOSX_PLUGIN_ERR_ALREADY_RUNNING` - 插件已在运行 -- `TAOSX_PLUGIN_ERR_INTERNAL` - 内部错误 - -#### 开发状态 - -这是一个**技术预览**实现。当前状态： -- ✅ 基础插件接口已实现 -- ✅ 事件处理框架 -- ✅ 统计信息收集 -- ✅ 生命周期管理 -- ✅ 测试套件 -- 🔄 与主位图引擎集成（计划中） -- 🔄 高级事件处理（计划中） - -#### 兼容性 - -- **taosX版本**：兼容taosX 1.0+ -- **TDengine版本**：兼容TDengine 3.0+ -- **平台**：Linux、Windows、macOS -- **架构**：x86_64、ARM64 - -## 与taosdump集成 - -### 集成方案 - -#### 方案一：作为独立工具集成（推荐） - -将位图插件作为独立的增量备份工具，与taosdump配合使用： - -```bash -# 1. 使用位图插件进行增量检测 -./incremental_bitmap_tool --config bitmap_config.json --output incremental_blocks.json - -# 2. 使用taosdump进行数据导出 -taosdump -h localhost -P 6030 -D dbname -o /backup/full/ - -# 3. 使用位图插件进行增量数据导出 -./incremental_bitmap_tool --export --blocks incremental_blocks.json --output /backup/incremental/ -``` - -**优势：** -- 无需修改TDengine核心代码 -- 可以独立开发和维护 -- 支持多种备份策略 - -#### 方案二：扩展taosdump - -在taosdump中添加增量备份功能： - -```bash -# 新增的增量备份命令 -taosdump --incremental --bitmap-plugin /path/to/libincremental_bitmap_plugin.so \ - -h localhost -P 6030 -D dbname -o /backup/incremental/ -``` - -**优势：** -- 统一的备份工具 -- 用户使用简单 - -#### 方案三：创建新的备份工具 - -开发专门的增量备份工具： - -```bash -# 新的增量备份工具 -taosbackup --engine bitmap --config backup_config.json \ - --source localhost:6030 --database dbname \ - --output /backup/incremental/ -``` - -**优势：** -- 专门为增量备份设计 -- 功能完整且灵活 - -### 1. 生成taosdump兼容脚本 - -```c -// 创建增量备份工具 -SIncrementalBackupConfig config = { - .source_host = "localhost", - .source_port = 6030, - .database = "test_db", - .backup_path = "/backup", - .bitmap_cache_path = "/tmp/bitmap_cache", - .since_timestamp = 1640995200LL, // 2022-01-01 00:00:00 - .batch_size = 1000, - .enable_compression = true, - .enable_encryption = false -}; - -SIncrementalBackupTool* tool = incremental_backup_tool_create(&config); -if (!tool) { - fprintf(stderr, "Failed to create incremental backup tool\n"); - return -1; -} - -// 启动工具 -if (incremental_backup_tool_start(tool) != 0) { - fprintf(stderr, "Failed to start incremental backup tool\n"); - incremental_backup_tool_destroy(tool); - return -1; -} - -// 生成taosdump脚本 -if (incremental_backup_tool_generate_taosdump_script(tool, "/tmp/backup_script.sh") != 0) { - fprintf(stderr, "Failed to generate taosdump script\n"); - incremental_backup_tool_destroy(tool); - return -1; -} - -// 执行增量备份 -if (incremental_backup_tool_backup(tool, config.since_timestamp) != 0) { - fprintf(stderr, "Failed to execute incremental backup\n"); - incremental_backup_tool_destroy(tool); - return -1; -} - -// 获取统计信息 -uint64_t total_blocks, processed_blocks, failed_blocks; -incremental_backup_tool_get_stats(tool, &total_blocks, &processed_blocks, &failed_blocks); -printf("Backup completed: %lu total, %lu processed, %lu failed\n", - total_blocks, processed_blocks, failed_blocks); - -// 清理资源 -incremental_backup_tool_destroy(tool); -``` - -### 2. 生成的备份脚本示例 - -```bash -#!/bin/bash - -# TDengine增量备份脚本 - 由位图插件生成 -# 生成时间: Mon Jan 1 00:00:00 2024 - -SOURCE_HOST=localhost -SOURCE_PORT=6030 -DATABASE=test_db -BACKUP_PATH=/backup -SINCE_TIMESTAMP=1640995200 - -echo "步骤1: 检测增量数据块..." -./incremental_bitmap_tool --detect \ - --host $SOURCE_HOST --port $SOURCE_PORT \ - --database $DATABASE \ - --since $SINCE_TIMESTAMP \ - --output incremental_blocks.json - -echo "步骤2: 使用taosdump备份增量数据..." -taosdump -h $SOURCE_HOST -P $SOURCE_PORT \ - -D $DATABASE \ - -S $SINCE_TIMESTAMP \ - -o $BACKUP_PATH/incremental_$(date +%Y%m%d_%H%M%S) - -echo "步骤3: 验证备份完整性..." -./incremental_bitmap_tool --verify \ - --backup $BACKUP_PATH \ - --blocks incremental_blocks.json \ - --report backup_verification_report.json - -echo "增量备份完成!" -``` - -### 3. 执行备份脚本 - -```bash -# 设置执行权限 -chmod +x /tmp/backup_script.sh - -# 执行备份 -/tmp/backup_script.sh - -# 检查备份结果 -ls -la /backup/ -cat backup_verification_report.json -``` - -## 性能优化 - -### 1. 内存优化 - -```c -// 配置合适的事件缓冲区大小 -SEventInterceptorConfig config = { - .event_buffer_size = 50000, // 根据内存情况调整 - .callback_threads = 8, // 示例；实际默认值由自适应策略决定（可用 IB_CALLBACK_THREADS 覆盖） - // ... -}; - -// 配置合适的批处理大小 -SBackupConfig backup_config = { - .batch_size = 5000, // 根据网络和磁盘性能调整 - .timeout_ms = 60000, // 根据数据量调整超时时间 - // ... -}; -``` - -### 2. 并发优化 - -```c -// 使用多线程处理事件 -SEventInterceptorConfig config = { - .callback_threads = 4, // 示例；推荐：不指定时采用自适应（min(2×核数, 64)） - // ... -}; - -// 使用异步处理 -if (event_interceptor_start(interceptor) != 0) { - // 处理启动失败 -} -``` - -### 3. 存储优化 - -```c -// 启用压缩 -SBackupConfig backup_config = { - .enable_compression = true, - .enable_encryption = false, // 根据安全需求决定 - // ... -}; -``` - -## 故障排查 - -### 1. 常见问题 - -#### 1.1 插件加载失败 - -**症状** -```bash -# 错误信息 -Failed to load plugin: libincremental_bitmap.so -Plugin initialization failed -Symbol not found: bitmap_engine_init -``` - -**可能原因** -- 插件文件不存在或权限不足 -- 依赖库缺失 -- 架构不匹配（32位/64位） -- 符号版本不兼容 - -**排查步骤** -```bash -# 1. 检查插件文件 -ls -la /usr/local/lib/tdengine/plugins/libincremental_bitmap.so - -# 2. 检查文件权限 -file /usr/local/lib/tdengine/plugins/libincremental_bitmap.so - -# 3. 检查依赖库 -ldd /usr/local/lib/tdengine/plugins/libincremental_bitmap.so - -# 4. 检查符号 -nm -D /usr/local/lib/tdengine/plugins/libincremental_bitmap.so | grep bitmap_engine_init - -# 5. 检查架构 -uname -m -file /usr/local/lib/tdengine/plugins/libincremental_bitmap.so -``` - -**解决方案** -```bash -# 重新安装插件 -cd /path/to/TDengine/plugins/incremental_bitmap/build -sudo make install - -# 检查依赖库 -sudo apt install -y libpthread-stubs0-dev libroaring-dev - -# 重新构建 -make clean && make -``` - -#### 1.2 内存不足错误 - -**症状** -```bash -# 错误信息 -Out of memory -Failed to allocate memory -Memory allocation failed -``` - -**可能原因** -- 系统内存不足 -- 内存碎片化 -- 内存泄漏 -- 配置的内存限制过低 - -**排查步骤** -```bash -# 1. 检查系统内存 -free -h -cat /proc/meminfo | grep MemAvailable - -# 2. 检查进程内存使用 -ps aux | grep taosd -cat /proc/PID/status | grep VmRSS - -# 3. 检查内存限制 -ulimit -a -cat /proc/PID/limits - -# 4. 检查内存泄漏 -valgrind --leak-check=full --show-leak-kinds=all ./your_program -``` - -**解决方案** -```bash -# 增加交换空间 -sudo fallocate -l 2G /swapfile -sudo chmod 600 /swapfile -sudo mkswap /swapfile -sudo swapon /swapfile - -# 调整内存限制 -ulimit -m unlimited -ulimit -v unlimited - -# 使用内存分配器 -export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.2 -``` - -#### 1.3 线程创建失败 - -**症状** -```bash -# 错误信息 -Failed to create thread -pthread_create failed -Too many open files -``` - -**可能原因** -- 线程数超过系统限制 -- 文件描述符不足 -- 系统资源耗尽 -- 权限不足 - -**排查步骤** -```bash -# 1. 检查线程限制 -cat /proc/sys/kernel/threads-max -ulimit -u - -# 2. 检查文件描述符限制 -ulimit -n -cat /proc/sys/fs/file-max - -# 3. 检查当前线程数 -ps -eLf | wc -l -cat /proc/PID/status | grep Threads - -# 4. 检查系统负载 -top -htop -``` - -**解决方案** -```bash -# 增加线程限制 -echo 32768 > /proc/sys/kernel/threads-max -ulimit -u 32768 - -# 增加文件描述符限制 -echo 65536 > /proc/sys/fs/file-max -ulimit -n 65536 - -# 调整线程池大小 -# 在配置中减少callback_threads数量 -``` - -#### 1.4 网络连接失败 - -**症状** -```bash -# 错误信息 -Connection refused -Connection timeout -Network unreachable -``` - -**可能原因** -- TDengine服务未启动 -- 端口被占用 -- 防火墙阻止 -- 网络配置错误 - -**排查步骤** -```bash -# 1. 检查TDengine服务状态 -sudo systemctl status taosd -ps aux | grep taosd - -# 2. 检查端口监听 -netstat -tlnp | grep 6030 -ss -tlnp | grep 6030 - -# 3. 检查防火墙 -sudo ufw status -sudo iptables -L - -# 4. 测试网络连接 -telnet localhost 6030 -nc -zv localhost 6030 -``` - -**解决方案** -```bash -# 启动TDengine服务 -sudo systemctl start taosd -sudo systemctl enable taosd - -# 配置防火墙 -sudo ufw allow 6030/tcp -sudo iptables -A INPUT -p tcp --dport 6030 -j ACCEPT - -# 检查配置文件 -sudo cat /etc/taos/taos.cfg | grep -E "(serverPort|fqdn)" -``` - -### 2. 常见错误 - -#### 初始化失败 -```c -SBitmapEngine* engine = bitmap_engine_init(); -if (!engine) { - // 检查内存是否足够 - // 检查依赖库是否正确安装 - // 检查权限是否正确 -} -``` - -#### 事件拦截器启动失败 -```c -if (event_interceptor_start(interceptor) != 0) { - // 检查线程资源是否足够 - // 检查存储引擎接口是否正确 - // 检查配置参数是否合理 -} -``` - -#### 备份协调器初始化失败 -```c -SBackupCoordinator* coordinator = backup_coordinator_init(engine, &config); -if (!coordinator) { - // 检查位图引擎是否正常 - // 检查配置参数是否有效 - // 检查路径权限是否正确 -} -``` - -### 2. 调试技巧 - -#### 启用详细日志 -```c -// 设置环境变量启用调试 -setenv("TDENGINE_DEBUG", "1", 1); -setenv("BITMAP_DEBUG", "1", 1); -``` - -#### 使用统计信息 -```c -// 获取位图引擎统计 -uint64_t total_blocks, dirty_count, new_count, deleted_count; -bitmap_engine_get_stats(engine, &total_blocks, &dirty_count, &new_count, &deleted_count); -printf("Stats: total=%lu, dirty=%lu, new=%lu, deleted=%lu\n", - total_blocks, dirty_count, new_count, deleted_count); - -// 获取事件拦截器统计 -uint64_t events_processed, events_dropped; -event_interceptor_get_stats(interceptor, &events_processed, &events_dropped); -printf("Events: processed=%lu, dropped=%lu\n", events_processed, events_dropped); -``` - -#### 使用可观测性指标 -```c -#include "observability.h" - -// 更新指标 -update_observability_metrics(); - -// 打印人类可读格式 -print_observability_metrics(); - -// 生成JSON格式 -char json_buffer[4096]; -if (format_observability_metrics_json(json_buffer, sizeof(json_buffer)) == 0) { - printf("JSON Metrics:\n%s\n", json_buffer); -} -``` - -### 3. 可观测性监控 - -#### 3.1 关键指标监控 - -**速率指标** -- `events_per_second`: 事件处理速率，每秒处理的事件数量 -- `messages_per_second`: 消息消费速率，每秒消费的消息数量 -- `bytes_per_second`: 数据吞吐量，每秒处理的数据字节数 - -**滞后指标** -- `consumer_lag_ms`: 消费者滞后时间，消息从产生到被处理的时间差 -- `offset_lag`: Offset滞后数量，未处理的消息数量 -- `processing_delay_ms`: 处理延迟，单个事件从接收到处理完成的时间 - -**错误指标** -- `events_dropped`: 丢弃事件数，由于各种原因未能处理的事件数量 -- `messages_dropped`: 丢弃消息数，未能处理的消息数量 -- `parse_errors`: 解析错误数，消息解析失败的数量 - -**资源指标** -- `memory_usage_bytes`: 总内存使用量，插件占用的总内存 -- `bitmap_memory_bytes`: 位图内存使用量，位图数据结构占用的内存 -- `metadata_memory_bytes`: 元数据内存使用量，元数据占用的内存 - -#### 3.2 告警配置 - -**告警阈值建议** -```yaml -# Prometheus告警规则示例 -groups: - - name: tdengine_incremental_bitmap - rules: - # 事件处理速率过低 - - alert: LowEventProcessingRate - expr: tdengine_events_per_second < 1000 - for: 5m - labels: - severity: warning - annotations: - summary: "事件处理速率过低" - description: "事件处理速率低于1000事件/秒，持续5分钟" - - # 内存使用过高 - - alert: HighMemoryUsage - expr: tdengine_memory_usage_bytes > 1073741824 - for: 2m - labels: - severity: critical - annotations: - summary: "内存使用过高" - description: "内存使用超过1GB" - - # 队列使用率过高 - - alert: HighQueueUsage - expr: tdengine_ring_buffer_usage > 80 - for: 3m - labels: - severity: warning - annotations: - summary: "队列使用率过高" - description: "环形队列使用率超过80%" -``` - -#### 3.3 性能调优建议 - -**基于指标的调优** -- **事件处理速率低**: 增加回调线程数，优化事件处理逻辑 -- **内存使用过高**: 调整位图引擎内存限制，启用内存压缩 -- **队列使用率高**: 增加环形队列容量，优化事件处理速度 -- **重试次数过多**: 检查网络连接，优化重试策略 - -**配置参数调优** -```c -// 事件拦截器配置优化 -SEventInterceptorConfig config = { - .enable_interception = true, - .event_buffer_size = 50000, // 根据事件产生速率调整 - .callback_threads = 8, // 示例；可通过 IB_CALLBACK_THREADS 显式覆盖 - .callback = NULL, - .callback_user_data = NULL -}; - -// 位图引擎配置优化 -SBitmapEngineConfig bitmap_config = { - .max_memory_mb = 2048, // 根据可用内存调整 - .persistence_enabled = true, // 启用持久化减少内存使用 - .persistence_path = "/tmp/bitmap_cache" -}; -``` - -## 最佳实践 - -### 1. 配置建议 - -```c -// 生产环境配置 -SEventInterceptorConfig event_config = { - .enable_interception = true, - .event_buffer_size = 100000, // 大缓冲区 - .callback_threads = 8, // 示例；默认采用自适应或由 IB_CALLBACK_THREADS 覆盖 - .callback = NULL, - .callback_user_data = NULL -}; - -SBackupConfig backup_config = { - .batch_size = 10000, // 大批处理 - .max_retries = 5, // 多重试 - .retry_interval_ms = 2000, // 长间隔 - .timeout_ms = 120000, // 长超时 - .enable_compression = true, // 启用压缩 - .enable_encryption = true, // 启用加密 - .backup_path = "/secure/backup", - .temp_path = "/tmp" -}; -``` - -### 2. 错误处理 - -```c -// 完整的错误处理示例 -int perform_backup() { - SBitmapEngine* engine = NULL; - SEventInterceptor* interceptor = NULL; - SBackupCoordinator* coordinator = NULL; - - // 初始化位图引擎 - engine = bitmap_engine_init(); - if (!engine) { - fprintf(stderr, "Failed to initialize bitmap engine\n"); - goto cleanup; - } - - // 初始化事件拦截器 - SEventInterceptorConfig event_config = { /* ... */ }; - interceptor = event_interceptor_init(&event_config, engine); - if (!interceptor) { - fprintf(stderr, "Failed to initialize event interceptor\n"); - goto cleanup; - } - - // 启动事件拦截器 - if (event_interceptor_start(interceptor) != 0) { - fprintf(stderr, "Failed to start event interceptor\n"); - goto cleanup; - } - - // 初始化备份协调器 - SBackupConfig backup_config = { /* ... */ }; - coordinator = backup_coordinator_init(engine, &backup_config); - if (!coordinator) { - fprintf(stderr, "Failed to initialize backup coordinator\n"); - goto cleanup; - } - - // 执行备份逻辑 - // ... - - return 0; - -cleanup: - if (coordinator) { - backup_coordinator_destroy(coordinator); - } - if (interceptor) { - event_interceptor_destroy(interceptor); - } - if (engine) { - bitmap_engine_destroy(engine); - } - return -1; -} -``` - -### 3. 资源管理 - -```c -// 使用RAII模式管理资源 -typedef struct { - SBitmapEngine* engine; - SEventInterceptor* interceptor; - SBackupCoordinator* coordinator; -} BackupContext; - -BackupContext* create_backup_context() { - BackupContext* ctx = malloc(sizeof(BackupContext)); - if (!ctx) return NULL; - - memset(ctx, 0, sizeof(BackupContext)); - return ctx; -} - -void destroy_backup_context(BackupContext* ctx) { - if (!ctx) return; - - if (ctx->coordinator) { - backup_coordinator_destroy(ctx->coordinator); - } - if (ctx->interceptor) { - event_interceptor_destroy(ctx->interceptor); - } - if (ctx->engine) { - bitmap_engine_destroy(ctx->engine); - } - - free(ctx); -} -``` - -### 4. 性能监控 - -```c -// 定期监控性能指标 -void monitor_performance(BackupContext* ctx) { - // 获取位图引擎统计 - uint64_t total_blocks, dirty_count, new_count, deleted_count; - bitmap_engine_get_stats(ctx->engine, &total_blocks, &dirty_count, &new_count, &deleted_count); - - // 获取事件拦截器统计 - uint64_t events_processed, events_dropped; - event_interceptor_get_stats(ctx->interceptor, &events_processed, &events_dropped); - - // 获取备份协调器统计 - SBackupStats stats; - backup_coordinator_get_stats(ctx->coordinator, &stats); - - // 记录性能指标 - printf("Performance: blocks=%lu, events=%lu, backup_size=%lu\n", - total_blocks, events_processed, stats.total_size); - - // 检查异常情况 - if (events_dropped > 0) { - printf("Warning: %lu events dropped\n", events_dropped); - } - - if (stats.failed_blocks > 0) { - printf("Warning: %lu blocks failed\n", stats.failed_blocks); - } -} -``` - ---- - -## 总结 - -TDengine增量位图插件提供了一个完整、高性能的逻辑备份和恢复解决方案。通过合理配置和使用，可以显著提升TDengine的备份性能，同时保证数据的一致性和完整性。 - -关键要点： -1. **正确初始化**：按照正确的顺序初始化和配置各个组件 -2. **合理配置**：根据实际环境调整配置参数 -3. **错误处理**：实现完整的错误处理和资源清理 -4. **性能监控**：定期监控性能指标，及时发现和解决问题 -5. **与taosdump集成**：充分利用与taosdump的集成优势 - -通过遵循本手册的指导，您可以有效地使用TDengine增量位图插件来提升备份和恢复的性能和可靠性。 diff --git a/plugins/incremental_bitmap/include/backup_coordinator.h b/plugins/incremental_bitmap/include/backup_coordinator.h deleted file mode 100644 index cec7dac90c79..000000000000 --- a/plugins/incremental_bitmap/include/backup_coordinator.h +++ /dev/null @@ -1,241 +0,0 @@ -#ifndef BACKUP_COORDINATOR_H -#define BACKUP_COORDINATOR_H - -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -// 前向声明 -struct SBitmapEngine; - -// 备份游标类型 -typedef enum { - BACKUP_CURSOR_TYPE_TIME = 0, // 基于时间的游标 - BACKUP_CURSOR_TYPE_WAL = 1, // 基于WAL偏移量的游标 - BACKUP_CURSOR_TYPE_HYBRID = 2 // 混合游标 -} EBackupCursorType; - -// 备份游标 -typedef struct { - EBackupCursorType type; - int64_t time_cursor; // 时间游标（纳秒） - uint64_t wal_cursor; // WAL偏移量游标 - uint64_t block_count; // 已备份块数 - int64_t last_update_time; // 最后更新时间 -} SBackupCursor; - -// 增量块信息 -typedef struct { - uint64_t block_id; // 块ID - uint64_t wal_offset; // WAL偏移量 - int64_t timestamp; // 时间戳 - uint32_t data_size; // 数据大小 - void* data; // 块数据（可选） -} SIncrementalBlock; - -// 备份配置 -typedef struct { - uint32_t batch_size; // 批处理大小 - uint32_t max_retries; // 最大重试次数 - uint32_t retry_interval_ms; // 重试间隔（毫秒） - uint32_t timeout_ms; // 超时时间（毫秒） - bool enable_compression; // 是否启用压缩 - bool enable_encryption; // 是否启用加密 - char* backup_path; // 备份路径 - char* temp_path; // 临时路径 -} SBackupConfig; - -// 备份统计信息 -typedef struct { - uint64_t total_blocks; // 总块数 - uint64_t processed_blocks; // 已处理块数 - uint64_t failed_blocks; // 失败块数 - uint64_t total_size; // 总大小（字节） - int64_t start_time; // 开始时间 - int64_t end_time; // 结束时间 - uint32_t retry_count; // 重试次数 -} SBackupStats; - -// 备份协调器实例 -typedef struct { - struct SBitmapEngine* bitmap_engine; - SBackupCursor cursor; - SBackupConfig config; - SBackupStats stats; - bool is_running; - void* user_data; -} SBackupCoordinator; - -// 备份协调器API - -/** - * 初始化备份协调器 - * @param bitmap_engine 位图引擎实例 - * @param config 备份配置 - * @return 协调器实例，失败返回NULL - */ -SBackupCoordinator* backup_coordinator_init(struct SBitmapEngine* bitmap_engine, - const SBackupConfig* config); - -/** - * 销毁备份协调器 - * @param coordinator 协调器实例 - */ -void backup_coordinator_destroy(SBackupCoordinator* coordinator); - -/** - * 启动备份协调器 - * @param coordinator 协调器实例 - * @return 0成功，非0失败 - */ -int32_t backup_coordinator_start(SBackupCoordinator* coordinator); - -/** - * 停止备份协调器 - * @param coordinator 协调器实例 - * @return 0成功，非0失败 - */ -int32_t backup_coordinator_stop(SBackupCoordinator* coordinator); - -/** - * 获取指定WAL偏移量范围内的脏块 - * @param coordinator 协调器实例 - * @param start_wal 起始WAL偏移量 - * @param end_wal 结束WAL偏移量 - * @param block_ids 块ID数组 - * @param max_count 最大块数 - * @return 实际获取的块数 - */ -uint32_t backup_coordinator_get_dirty_blocks(SBackupCoordinator* coordinator, - uint64_t start_wal, uint64_t end_wal, - uint64_t* block_ids, uint32_t max_count); - -/** - * 获取指定时间范围内的脏块 - * @param coordinator 协调器实例 - * @param start_time 起始时间 - * @param end_time 结束时间 - * @param block_ids 块ID数组 - * @param max_count 最大块数 - * @return 实际获取的块数 - */ -uint32_t backup_coordinator_get_dirty_blocks_by_time(SBackupCoordinator* coordinator, - int64_t start_time, int64_t end_time, - uint64_t* block_ids, uint32_t max_count); - -/** - * 获取增量块信息 - * @param coordinator 协调器实例 - * @param start_wal 起始WAL偏移量 - * @param end_wal 结束WAL偏移量 - * @param blocks 块信息数组 - * @param max_count 最大块数 - * @return 实际获取的块数 - */ -uint32_t backup_coordinator_get_incremental_blocks(SBackupCoordinator* coordinator, - uint64_t start_wal, uint64_t end_wal, - SIncrementalBlock* blocks, uint32_t max_count); - -/** - * 估算备份大小 - * @param coordinator 协调器实例 - * @param start_wal 起始WAL偏移量 - * @param end_wal 结束WAL偏移量 - * @return 估算的备份大小（字节） - */ -uint64_t backup_coordinator_estimate_backup_size(SBackupCoordinator* coordinator, - uint64_t start_wal, uint64_t end_wal); - -/** - * 获取备份统计信息 - * @param coordinator 协调器实例 - * @param stats 统计信息结构 - * @return 0成功，非0失败 - */ -int32_t backup_coordinator_get_stats(SBackupCoordinator* coordinator, SBackupStats* stats); - -/** - * 重置备份统计信息 - * @param coordinator 协调器实例 - * @return 0成功，非0失败 - */ -int32_t backup_coordinator_reset_stats(SBackupCoordinator* coordinator); - -/** - * 获取备份游标 - * @param coordinator 协调器实例 - * @param cursor 游标结构 - * @return 0成功，非0失败 - */ -int32_t backup_coordinator_get_cursor(SBackupCoordinator* coordinator, SBackupCursor* cursor); - -/** - * 设置备份游标 - * @param coordinator 协调器实例 - * @param cursor 游标结构 - * @return 0成功，非0失败 - */ -int32_t backup_coordinator_set_cursor(SBackupCoordinator* coordinator, const SBackupCursor* cursor); - -/** - * 更新备份游标 - * @param coordinator 协调器实例 - * @param block_id 块ID - * @param wal_offset WAL偏移量 - * @param timestamp 时间戳 - * @return 0成功，非0失败 - */ -int32_t backup_coordinator_update_cursor(SBackupCoordinator* coordinator, - uint64_t block_id, uint64_t wal_offset, int64_t timestamp); - -// 插件接口函数（通用备份插件接口） -typedef struct { - // 插件初始化 - int32_t (*init)(const char* config); - - // 插件销毁 - void (*destroy)(void); - - // 获取脏块 - uint32_t (*get_dirty_blocks)(uint64_t start_wal, uint64_t end_wal, - uint64_t* block_ids, uint32_t max_count); - - // 获取增量块 - uint32_t (*get_incremental_blocks)(uint64_t start_wal, uint64_t end_wal, - SIncrementalBlock* blocks, uint32_t max_count); - - // 估算备份大小 - uint64_t (*estimate_backup_size)(uint64_t start_wal, uint64_t end_wal); - - // 获取统计信息 - int32_t (*get_stats)(SBackupStats* stats); - - // 重置统计信息 - int32_t (*reset_stats)(void); -} SBackupPluginInterface; - -// 插件接口函数 -extern SBackupPluginInterface* backup_plugin_get_interface(void); - -// 便捷函数 -uint32_t backup_plugin_get_dirty_blocks(uint64_t start_wal, uint64_t end_wal, - uint64_t* block_ids, uint32_t max_count); - -uint32_t backup_plugin_get_incremental_blocks(uint64_t start_wal, uint64_t end_wal, - SIncrementalBlock* blocks, uint32_t max_count); - -uint64_t backup_plugin_estimate_backup_size(uint64_t start_wal, uint64_t end_wal); - -int32_t backup_plugin_get_stats(SBackupStats* stats); - -int32_t backup_plugin_reset_stats(void); - -#ifdef __cplusplus -} -#endif - -#endif // BACKUP_COORDINATOR_H \ No newline at end of file diff --git a/plugins/incremental_bitmap/include/bitmap_engine.h b/plugins/incremental_bitmap/include/bitmap_engine.h deleted file mode 100644 index a24216972b41..000000000000 --- a/plugins/incremental_bitmap/include/bitmap_engine.h +++ /dev/null @@ -1,268 +0,0 @@ -// API版本号定义 -#define BITMAP_ENGINE_API_VERSION 1 - -/* - * Copyright (c) 2024 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_BITMAP_ENGINE_H -#define TDENGINE_BITMAP_ENGINE_H - -#include -#include -#include -#include "bitmap_interface.h" -#include "skiplist.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// 错误码定义 -#define ERR_SUCCESS 0 // 成功 -#define ERR_INVALID_PARAM -1 // 无效参数 -#define ERR_INVALID_STATE_TRANS -1001 // 无效状态转换 -#define ERR_BLOCK_NOT_FOUND -1002 // 块未找到 - -// 块状态枚举 -typedef enum { - BLOCK_STATE_CLEAN = 0, // 未修改 - BLOCK_STATE_DIRTY = 1, // 已修改 - BLOCK_STATE_NEW = 2, // 新增 - BLOCK_STATE_DELETED = 3 // 已删除 -} EBlockState; - -// 块元数据 -typedef struct { - uint64_t block_id; // 物理块ID - uint64_t wal_offset; // WAL偏移量 - int64_t timestamp; // 纳秒级时间戳 - EBlockState state; // 块状态 -} SBlockMetadata; - -// LRU节点结构 -typedef struct SLruNode { - uint64_t block_id; // 块ID - int64_t last_access; // 最后访问时间 - struct SLruNode* prev; // 前驱节点 - struct SLruNode* next; // 后继节点 -} SLruNode; - -// 块元数据映射节点 -typedef struct SBlockMetadataNode { - uint64_t block_id; - SBlockMetadata metadata; - struct SBlockMetadataNode* next; -} SBlockMetadataNode; - -// 时间索引节点 -typedef struct STimeIndexNode { - int64_t timestamp; - SBitmapInterface* block_ids; - struct STimeIndexNode* next; -} STimeIndexNode; - -// WAL索引节点 -typedef struct SWalIndexNode { - uint64_t wal_offset; - SBitmapInterface* block_ids; - struct SWalIndexNode* next; -} SWalIndexNode; - -// LRU映射节点 -typedef struct SLruMapNode { - uint64_t block_id; - SLruNode* lru_node; - struct SLruMapNode* next; -} SLruMapNode; - - - - - -// 位图引擎实例 -// API版本：BITMAP_ENGINE_API_VERSION -typedef struct SBitmapEngine { - SBitmapInterface* dirty_blocks; // 脏块位图 - SBitmapInterface* new_blocks; // 新块位图 - SBitmapInterface* deleted_blocks; // 删除块位图 - - // 元数据映射（哈希表） - SBlockMetadataNode** metadata_map; - uint32_t metadata_map_size; - uint32_t metadata_count; - - // 时空索引 - STimeIndexNode* time_index_head; - SWalIndexNode* wal_index_head; - - - - // 统计信息 - uint64_t total_blocks; - uint64_t dirty_count; - uint64_t new_count; - uint64_t deleted_count; - - // 线程安全 - pthread_mutex_t mutex; - pthread_rwlock_t rwlock; - - // 跳表索引 - skiplist_t* time_index; - skiplist_t* wal_index; - - -} SBitmapEngine; - - - -// 位图引擎API - -/** - * 初始化位图引擎 - * @return 引擎实例，失败返回NULL - */ -SBitmapEngine* bitmap_engine_init(void); - -/** - * 销毁位图引擎 - * @param engine 引擎实例 - */ -void bitmap_engine_destroy(SBitmapEngine* engine); - -/** - * 标记块为脏状态 - * @param engine 引擎实例 - * @param block_id 块ID - * @param wal_offset WAL偏移量 - * @param timestamp 时间戳 - * @return 0成功，非0失败 - */ -int32_t bitmap_engine_mark_dirty(SBitmapEngine* engine, uint64_t block_id, - uint64_t wal_offset, int64_t timestamp); - -/** - * 标记块为新增状态 - * @param engine 引擎实例 - * @param block_id 块ID - * @param wal_offset WAL偏移量 - * @param timestamp 时间戳 - * @return 0成功，非0失败 - */ -int32_t bitmap_engine_mark_new(SBitmapEngine* engine, uint64_t block_id, - uint64_t wal_offset, int64_t timestamp); - -/** - * 标记块为删除状态 - * @param engine 引擎实例 - * @param block_id 块ID - * @param wal_offset WAL偏移量 - * @param timestamp 时间戳 - * @return 0成功，非0失败 - */ -int32_t bitmap_engine_mark_deleted(SBitmapEngine* engine, uint64_t block_id, - uint64_t wal_offset, int64_t timestamp); - -/** - * 清除块状态 - * @param engine 引擎实例 - * @param block_id 块ID - * @return 0成功，非0失败 - */ -int32_t bitmap_engine_clear_block(SBitmapEngine* engine, uint64_t block_id); - -/** - * 获取时间范围内的脏块 - * @param engine 引擎实例 - * @param start_time 开始时间 - * @param end_time 结束时间 - * @param block_ids 输出块ID数组 - * @param max_count 最大返回数量 - * @return 实际返回的块数量 - */ -uint32_t bitmap_engine_get_dirty_blocks_by_time(SBitmapEngine* engine, - int64_t start_time, int64_t end_time, - uint64_t* block_ids, uint32_t max_count); - -/** - * 获取WAL偏移量范围内的脏块 - * @param engine 引擎实例 - * @param start_offset 开始偏移量 - * @param end_offset 结束偏移量 - * @param block_ids 输出块ID数组 - * @param max_count 最大返回数量 - * @return 实际返回的块数量 - */ -uint32_t bitmap_engine_get_dirty_blocks_by_wal(SBitmapEngine* engine, - uint64_t start_offset, uint64_t end_offset, - uint64_t* block_ids, uint32_t max_count); - -/** - * 获取块元数据 - * @param engine 引擎实例 - * @param block_id 块ID - * @param metadata 输出元数据 - * @return 0成功，非0失败 - */ -int32_t bitmap_engine_get_block_metadata(SBitmapEngine* engine, uint64_t block_id, - SBlockMetadata* metadata); - - - -/** - * 获取统计信息 - * @param engine 引擎实例 - * @param total_blocks 总块数 - * @param dirty_count 脏块数 - * @param new_count 新块数 - * @param deleted_count 删除块数 - */ -void bitmap_engine_get_stats(SBitmapEngine* engine, uint64_t* total_blocks, - uint64_t* dirty_count, uint64_t* new_count, uint64_t* deleted_count); - - - -// 状态转换验证API - -/** - * 验证状态转换是否合法 - * @param current_state 当前状态 - * @param target_state 目标状态 - * @return 0合法，ERR_INVALID_STATE_TRANS非法 - */ -int32_t bitmap_engine_validate_state_transition(EBlockState current_state, EBlockState target_state); - -/** - * 获取状态转换错误信息 - * @param current_state 当前状态 - * @param target_state 目标状态 - * @return 错误信息字符串 - */ -const char* bitmap_engine_get_state_transition_error(EBlockState current_state, EBlockState target_state); - -/** - * 获取块当前状态 - * @param engine 引擎实例 - * @param block_id 块ID - * @param state 输出状态 - * @return 0成功，ERR_BLOCK_NOT_FOUND块未找到 - */ -int32_t bitmap_engine_get_block_state(SBitmapEngine* engine, uint64_t block_id, EBlockState* state); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_BITMAP_ENGINE_H \ No newline at end of file diff --git a/plugins/incremental_bitmap/include/bitmap_interface.h b/plugins/incremental_bitmap/include/bitmap_interface.h deleted file mode 100644 index e432c0103815..000000000000 --- a/plugins/incremental_bitmap/include/bitmap_interface.h +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (c) 2024 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_BITMAP_INTERFACE_H -#define TDENGINE_BITMAP_INTERFACE_H - -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -// 位图抽象接口 -typedef struct SBitmapInterface { - void* bitmap; // 位图实现（具体实现由实现层提供） - - // 基本操作 - void (*add)(void* bitmap, uint64_t value); - void (*remove)(void* bitmap, uint64_t value); - bool (*contains)(void* bitmap, uint64_t value); - uint32_t (*cardinality)(void* bitmap); - void (*clear)(void* bitmap); - - // 集合操作 - void (*union_with)(void* bitmap, const void* other); - void (*intersect_with)(void* bitmap, const void* other); - void (*subtract)(void* bitmap, const void* other); - - // 迭代操作 - uint32_t (*to_array)(void* bitmap, uint64_t* array, uint32_t max_count); - - // 序列化操作 - size_t (*serialized_size)(void* bitmap); - int32_t (*serialize)(void* bitmap, void* buffer, size_t buffer_size); - int32_t (*deserialize)(void** bitmap, const void* buffer, size_t buffer_size); - - // 内存管理 - void (*destroy)(void* bitmap); - size_t (*memory_usage)(void* bitmap); - - // 创建和复制 - void* (*create)(void); - void* (*clone)(const void* bitmap); -} SBitmapInterface; - -// 位图接口实现函数声明 -extern SBitmapInterface* bitmap_interface_create(void); -extern void bitmap_interface_destroy(SBitmapInterface* interface); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_BITMAP_INTERFACE_H \ No newline at end of file diff --git a/plugins/incremental_bitmap/include/e2e_consistency.h b/plugins/incremental_bitmap/include/e2e_consistency.h deleted file mode 100644 index 1d9053990e95..000000000000 --- a/plugins/incremental_bitmap/include/e2e_consistency.h +++ /dev/null @@ -1,39 +0,0 @@ -#ifndef INCREMENTAL_BITMAP_E2E_CONSISTENCY_H -#define INCREMENTAL_BITMAP_E2E_CONSISTENCY_H - -#include -#include - -typedef struct E2EConsistencyConfig { - const char *data_path; - const char *snapshot_path; - const char *recovery_path; -} E2EConsistencyConfig; - -typedef struct E2EConsistencyReport { - uint64_t snapshots_checked; - uint64_t recovery_points_checked; - uint64_t data_blocks_compared; - uint64_t consistency_errors; - uint64_t missing_files; - double validation_time_ms; - char details[512]; -} E2EConsistencyReport; - -int e2e_validate_consistency(const E2EConsistencyConfig *config, - char *error_buffer, - size_t error_buffer_length); - -int e2e_validate_consistency_detailed(const E2EConsistencyConfig *config, - E2EConsistencyReport *report); - -int e2e_compare_snapshot_data(const char *snapshot_file, - const char *reference_file, - uint64_t *blocks_compared, - uint64_t *mismatches); - -void e2e_print_consistency_report(const E2EConsistencyReport *report); - -#endif /* INCREMENTAL_BITMAP_E2E_CONSISTENCY_H */ - - diff --git a/plugins/incremental_bitmap/include/e2e_perf.h b/plugins/incremental_bitmap/include/e2e_perf.h deleted file mode 100644 index 6e9f09726e15..000000000000 --- a/plugins/incremental_bitmap/include/e2e_perf.h +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef INCREMENTAL_BITMAP_E2E_PERF_H -#define INCREMENTAL_BITMAP_E2E_PERF_H - -#include - -typedef struct E2EPerfTimer { - uint64_t start_ms; - uint64_t end_ms; -} E2EPerfTimer; - -void e2e_perf_timer_start(E2EPerfTimer *timer); -void e2e_perf_timer_stop(E2EPerfTimer *timer); -double e2e_perf_elapsed_ms(const E2EPerfTimer *timer); -double e2e_perf_throughput_per_sec(uint64_t items, const E2EPerfTimer *timer); -void e2e_perf_print_summary(const char *label, uint64_t items, const E2EPerfTimer *timer); - -#endif /* INCREMENTAL_BITMAP_E2E_PERF_H */ - - - diff --git a/plugins/incremental_bitmap/include/event_interceptor.h b/plugins/incremental_bitmap/include/event_interceptor.h deleted file mode 100644 index 9806586b81c6..000000000000 --- a/plugins/incremental_bitmap/include/event_interceptor.h +++ /dev/null @@ -1,117 +0,0 @@ -#ifndef EVENT_INTERCEPTOR_H -#define EVENT_INTERCEPTOR_H - -#include -#include -#include -#include "storage_engine_interface.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// 前向声明 -struct SBitmapEngine; -struct SStorageEngineInterface; - -// 块事件类型 -typedef enum { - EVENT_BLOCK_CREATE = 0, - EVENT_BLOCK_UPDATE, - EVENT_BLOCK_FLUSH, - EVENT_BLOCK_DELETE, - EVENT_MAX -} EBlockEventType; - -// 块事件结构 -typedef struct { - EBlockEventType event_type; - uint64_t block_id; - uint64_t wal_offset; - int64_t timestamp; - void* user_data; -} SBlockEvent; - -// 事件回调函数类型 -typedef void (*BlockEventCallback)(const SBlockEvent* event, void* user_data); - -// 事件拦截器配置 -typedef struct { - bool enable_interception; - BlockEventCallback callback; - void* callback_user_data; - uint32_t event_buffer_size; - uint32_t callback_threads; -} SEventInterceptorConfig; - -// 事件拦截器结构 -typedef struct { - SEventInterceptorConfig config; - struct SBitmapEngine* bitmap_engine; - SStorageEngineInterface* storage_interface; - - // 事件缓冲区 - void* event_buffer; // SRingBuffer* - - // 线程管理 - pthread_t* callback_threads; - uint32_t thread_count; - bool stop_threads; - - // 同步机制 - pthread_mutex_t mutex; - pthread_cond_t condition; - - // 统计信息 - uint64_t events_processed; - uint64_t events_dropped; - - // 配置参数 - uint32_t buffer_size; -} SEventInterceptor; - -// 事件拦截器管理函数 -SEventInterceptor* event_interceptor_init(const SEventInterceptorConfig* config, - struct SBitmapEngine* bitmap_engine); - -void event_interceptor_destroy(SEventInterceptor* interceptor); - -int32_t event_interceptor_start(SEventInterceptor* interceptor); - -int32_t event_interceptor_stop(SEventInterceptor* interceptor); - -// 事件处理函数 -int32_t event_interceptor_on_block_create(SEventInterceptor* interceptor, - uint64_t block_id, uint64_t wal_offset, int64_t timestamp); - -int32_t event_interceptor_on_block_update(SEventInterceptor* interceptor, - uint64_t block_id, uint64_t wal_offset, int64_t timestamp); - -int32_t event_interceptor_on_block_flush(SEventInterceptor* interceptor, - uint64_t block_id, uint64_t wal_offset, int64_t timestamp); - -int32_t event_interceptor_on_block_delete(SEventInterceptor* interceptor, - uint64_t block_id, uint64_t wal_offset, int64_t timestamp); - -// 统计信息 -void event_interceptor_get_stats(SEventInterceptor* interceptor, - uint64_t* events_processed, uint64_t* events_dropped); - -// 存储引擎接口管理 -int32_t event_interceptor_set_storage_interface(SEventInterceptor* interceptor, - SStorageEngineInterface* interface); - -int32_t event_interceptor_install_storage_interception(SEventInterceptor* interceptor); - -int32_t event_interceptor_uninstall_storage_interception(SEventInterceptor* interceptor); - -// 测试支持 -int32_t event_interceptor_trigger_test_event(SEventInterceptor* interceptor, - EBlockEventType event_type, - uint64_t block_id, uint64_t wal_offset, int64_t timestamp); - -#ifdef __cplusplus -} -#endif - -#endif // EVENT_INTERCEPTOR_H \ No newline at end of file diff --git a/plugins/incremental_bitmap/include/observability.h b/plugins/incremental_bitmap/include/observability.h deleted file mode 100644 index 5868826c573e..000000000000 --- a/plugins/incremental_bitmap/include/observability.h +++ /dev/null @@ -1,65 +0,0 @@ -#ifndef OBSERVABILITY_H -#define OBSERVABILITY_H - -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -// 可观测性指标结构体 -typedef struct { - // 速率指标 - uint64_t events_per_second; // 事件处理速率 - uint64_t messages_per_second; // 消息消费速率 - uint64_t bytes_per_second; // 数据吞吐量 - - // 滞后指标 - int64_t consumer_lag_ms; // 消费者滞后时间(毫秒) - uint64_t offset_lag; // Offset滞后数量 - int64_t processing_delay_ms; // 处理延迟(毫秒) - - // 丢弃指标 - uint64_t events_dropped; // 丢弃事件数 - uint64_t messages_dropped; // 丢弃消息数 - uint64_t parse_errors; // 解析错误数 - - // 重试指标 - uint64_t connection_retries; // 连接重试次数 - uint64_t subscription_retries; // 订阅重试次数 - uint64_t commit_retries; // 提交重试次数 - - // 队列水位 - uint32_t ring_buffer_usage; // 环形队列使用率(%) - uint32_t ring_buffer_capacity; // 环形队列容量 - uint32_t event_queue_size; // 事件队列大小 - - // 内存指标 - size_t memory_usage_bytes; // 内存使用量(字节) - size_t bitmap_memory_bytes; // 位图内存使用量 - size_t metadata_memory_bytes; // 元数据内存使用量 - - // 时间戳 - int64_t last_update_time; // 最后更新时间 - int64_t uptime_seconds; // 运行时间(秒) -} SObservabilityMetrics; - -// 可观测性指标更新函数 -void update_observability_metrics(SObservabilityMetrics* metrics); - -// 可观测性指标打印函数 -void print_observability_metrics(const SObservabilityMetrics* metrics); - -// 格式化可观测性指标为JSON字符串 -void format_observability_metrics_json(const SObservabilityMetrics* metrics, char* buffer, size_t buffer_size); - -// 格式化可观测性指标为Prometheus格式 -void format_observability_metrics_prometheus(const SObservabilityMetrics* metrics, char* buffer, size_t buffer_size); - -#ifdef __cplusplus -} -#endif - -#endif // OBSERVABILITY_H - diff --git a/plugins/incremental_bitmap/include/pitr_e2e_test.h b/plugins/incremental_bitmap/include/pitr_e2e_test.h deleted file mode 100644 index 563250db17c8..000000000000 --- a/plugins/incremental_bitmap/include/pitr_e2e_test.h +++ /dev/null @@ -1,223 +0,0 @@ -#ifndef PITR_E2E_TEST_H -#define PITR_E2E_TEST_H - -#include -#include -#include "bitmap_engine.h" -#include "backup_coordinator.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// ============================================================================ -// 数据量限制常量 - 必须保证测试数据量在1GB以内 -// ============================================================================ -#define MAX_DATA_SIZE_GB 1 -#define MAX_DATA_SIZE_BYTES (MAX_DATA_SIZE_GB * 1024ULL * 1024ULL * 1024ULL) -#define ESTIMATED_BLOCK_SIZE_BYTES 1024 // 每个数据块约1KB -#define ESTIMATED_SNAPSHOT_OVERHEAD 0.1 // 快照开销10% -#define ESTIMATED_RECOVERY_OVERHEAD 0.2 // 恢复点开销20% - -// PITR测试配置 -typedef struct { - uint32_t snapshot_interval_ms; // 快照间隔（毫秒） - uint32_t recovery_points; // 恢复点数量 - uint64_t data_block_count; // 数据块数量 - uint32_t concurrent_writers; // 并发写入线程数 - uint32_t test_duration_seconds; // 测试持续时间（秒） - bool enable_disorder_test; // 是否启用乱序测试 - bool enable_deletion_test; // 是否启用删除测试 - const char* test_data_path; // 测试数据路径 - const char* snapshot_path; // 快照存储路径 - const char* recovery_path; // 恢复数据路径 -} SPitrTestConfig; - -// PITR测试状态 -typedef struct { - uint64_t snapshots_created; // 已创建快照数量 - uint64_t recovery_points_verified; // 已验证恢复点数量 - uint64_t data_consistency_checks; // 数据一致性检查次数 - uint64_t disorder_handled; // 处理的乱序数据数量 - uint64_t deletion_handled; // 处理的删除操作数量 - uint64_t total_test_time_ms; // 总测试时间（毫秒） - bool test_passed; // 测试是否通过 - char error_message[512]; // 错误信息 -} SPitrTestStatus; - -// 快照信息 -typedef struct { - int64_t timestamp; // 快照时间戳 - uint64_t snapshot_id; // 快照ID - uint64_t block_count; // 块数量 - uint64_t data_size_bytes; // 数据大小（字节） - char metadata_path[256]; // 元数据路径 - char data_path[256]; // 数据路径 - bool is_valid; // 快照是否有效 -} SSnapshotInfo; - -// 恢复点信息 -typedef struct { - int64_t recovery_timestamp; // 恢复时间戳 - uint64_t recovery_id; // 恢复点ID - SSnapshotInfo* base_snapshot; // 基础快照 - uint64_t incremental_blocks; // 增量块数量 - uint64_t expected_block_count; // 期望的块数量 - bool recovery_successful; // 恢复是否成功 -} SRecoveryPoint; - -// 数据一致性检查结果 -typedef struct { - uint64_t expected_blocks; // 期望的块数量 - uint64_t actual_blocks; // 实际的块数量 - uint64_t mismatched_blocks; // 不匹配的块数量 - uint64_t missing_blocks; // 缺失的块数量 - uint64_t extra_blocks; // 多余的块数量 - double consistency_percentage; // 一致性百分比 - bool is_consistent; // 是否一致 - char details[512]; // 详细信息 -} SDataConsistencyResult; - -// 乱序数据处理结果 -typedef struct { - uint64_t total_events; // 总事件数 - uint64_t ordered_events; // 有序事件数 - uint64_t disorder_events; // 乱序事件数 - uint64_t reordered_events; // 重排序事件数 - uint64_t dropped_events; // 丢弃事件数 - double disorder_ratio; // 乱序比例 - bool handling_successful; // 处理是否成功 -} SDisorderHandlingResult; - -// 删除操作处理结果 -typedef struct { - uint64_t total_deletions; // 总删除操作数 - uint64_t successful_deletions; // 成功删除数 - uint64_t failed_deletions; // 失败删除数 - uint64_t recovered_deletions; // 恢复的删除数 - double deletion_success_rate; // 删除成功率 - bool deletion_handling_successful; // 删除处理是否成功 -} SDeletionHandlingResult; - -// PITR测试器 -typedef struct SPitrTester SPitrTester; - -// 创建PITR测试器 -SPitrTester* pitr_tester_create(const SPitrTestConfig* config); - -// 销毁PITR测试器 -void pitr_tester_destroy(SPitrTester* tester); - -// 运行完整的PITR E2E测试 -int pitr_tester_run_full_test(SPitrTester* tester); - -// 运行快照创建测试 -int pitr_tester_run_snapshot_test(SPitrTester* tester); - -// 运行时间点恢复测试 -int pitr_tester_run_recovery_test(SPitrTester* tester); - -// 运行乱序数据处理测试 -int pitr_tester_run_disorder_test(SPitrTester* tester); - -// 运行删除覆盖一致性测试 -int pitr_tester_run_deletion_consistency_test(SPitrTester* tester); - -// 运行边界条件测试 -int pitr_tester_run_boundary_test(SPitrTester* tester); - -// 获取测试状态 -int pitr_tester_get_status(SPitrTester* tester, SPitrTestStatus* status); - -// 获取快照列表 -int pitr_tester_get_snapshots(SPitrTester* tester, SSnapshotInfo* snapshots, uint32_t max_count, uint32_t* actual_count); - -// 获取恢复点列表 -int pitr_tester_get_recovery_points(SPitrTester* tester, SRecoveryPoint* recovery_points, uint32_t max_count, uint32_t* actual_count); - -// 验证数据一致性 -int pitr_tester_verify_consistency(SPitrTester* tester, int64_t timestamp, SDataConsistencyResult* result); - -// 生成测试报告 -int pitr_tester_generate_report(SPitrTester* tester, const char* report_path); - -// 重置测试器状态 -int pitr_tester_reset(SPitrTester* tester); - -// 设置测试配置 -int pitr_tester_set_config(SPitrTester* tester, const SPitrTestConfig* config); - -// 获取测试统计信息 -int pitr_tester_get_statistics(SPitrTester* tester, void* stats); - -// 时间点恢复测试 -int pitr_tester_run_time_point_recovery(SPitrTester* tester, int64_t timestamp); - -// 数据一致性验证 -int pitr_tester_verify_data_consistency(SPitrTester* tester); - -// 边界条件测试 -int pitr_tester_run_boundary_tests(SPitrTester* tester); - -// 性能测试 -int pitr_tester_run_performance_test(SPitrTester* tester); - -// 测试辅助函数 - -// 创建测试数据 -int pitr_create_test_data(const char* data_path, uint64_t block_count, uint32_t concurrent_writers); - - - -// 验证快照完整性 -bool pitr_verify_snapshot_integrity(const SSnapshotInfo* snapshot); - -// 比较两个时间点的数据 -int pitr_compare_data_at_timestamps(const char* data_path, int64_t timestamp1, int64_t timestamp2, SDataConsistencyResult* result); - -// 模拟网络延迟和乱序 -int pitr_simulate_network_disorder(const char* data_path, double disorder_ratio, uint32_t max_delay_ms); - -// 模拟删除操作 -int pitr_simulate_deletions(const char* data_path, uint64_t deletion_count, double success_rate); - -// 性能基准测试 -int pitr_run_performance_benchmark(const char* test_name, void (*test_func)(void*), void* test_data, uint32_t iterations); - -// 内存使用监控 -int pitr_monitor_memory_usage(size_t* peak_memory, size_t* current_memory); - -// 时间测量工具 -typedef struct { - struct timespec start_time; - struct timespec end_time; - char operation_name[64]; -} SPitrTimer; - -// 开始计时 -SPitrTimer* pitr_timer_start(const char* operation_name); - -// 停止计时 -void pitr_timer_stop(SPitrTimer* timer); - -// 获取耗时（毫秒） -double pitr_timer_get_elapsed_ms(SPitrTimer* timer); - -// 打印计时结果 -void pitr_timer_print_result(SPitrTimer* timer); - -// 默认测试配置 -extern const SPitrTestConfig PITR_DEFAULT_CONFIG; - -// 测试结果常量 -#define PITR_TEST_SUCCESS 0 -#define PITR_TEST_FAILED -1 -#define PITR_TEST_INVALID_CONFIG -2 -#define PITR_TEST_TIMEOUT -3 -#define PITR_TEST_MEMORY_ERROR -4 - -#ifdef __cplusplus -} -#endif - -#endif // PITR_E2E_TEST_H diff --git a/plugins/incremental_bitmap/include/ring_buffer.h b/plugins/incremental_bitmap/include/ring_buffer.h deleted file mode 100644 index ec25e168a95f..000000000000 --- a/plugins/incremental_bitmap/include/ring_buffer.h +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright (c) 2024 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_RING_BUFFER_H -#define TDENGINE_RING_BUFFER_H - -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -// 环形队列状态 -typedef enum { - RING_BUFFER_EMPTY = 0, // 空 - RING_BUFFER_PARTIAL = 1, // 部分填充 - RING_BUFFER_FULL = 2 // 满 -} ERingBufferState; - -// 环形队列 -typedef struct { - void** buffer; // 数据缓冲区 - uint32_t capacity; // 容量 - uint32_t size; // 当前大小 - uint32_t head; // 头部索引 - uint32_t tail; // 尾部索引 - - // 线程同步 - pthread_mutex_t mutex; - pthread_cond_t not_empty; - pthread_cond_t not_full; - - // 统计信息 - uint64_t enqueue_count; - uint64_t dequeue_count; - uint64_t overflow_count; -} SRingBuffer; - -// 环形队列API - -/** - * 初始化环形队列 - * @param capacity 队列容量 - * @return 队列实例，失败返回NULL - */ -SRingBuffer* ring_buffer_init(uint32_t capacity); - -/** - * 销毁环形队列 - * @param ring_buffer 队列实例 - */ -void ring_buffer_destroy(SRingBuffer* ring_buffer); - -/** - * 入队（非阻塞） - * @param ring_buffer 队列实例 - * @param item 数据项 - * @return 0成功，非0失败 - */ -int32_t ring_buffer_enqueue(SRingBuffer* ring_buffer, void* item); - -/** - * 入队（阻塞） - * @param ring_buffer 队列实例 - * @param item 数据项 - * @param timeout_ms 超时时间(毫秒)，0表示无限等待 - * @return 0成功，非0失败 - */ -int32_t ring_buffer_enqueue_blocking(SRingBuffer* ring_buffer, void* item, uint32_t timeout_ms); - -/** - * 出队（非阻塞） - * @param ring_buffer 队列实例 - * @param item 输出数据项 - * @return 0成功，非0失败 - */ -int32_t ring_buffer_dequeue(SRingBuffer* ring_buffer, void** item); - -/** - * 出队（阻塞） - * @param ring_buffer 队列实例 - * @param item 输出数据项 - * @param timeout_ms 超时时间(毫秒)，0表示无限等待 - * @return 0成功，非0失败 - */ -int32_t ring_buffer_dequeue_blocking(SRingBuffer* ring_buffer, void** item, uint32_t timeout_ms); - -/** - * 查看队首元素（不移除） - * @param ring_buffer 队列实例 - * @param item 输出数据项 - * @return 0成功，非0失败 - */ -int32_t ring_buffer_peek(SRingBuffer* ring_buffer, void** item); - -/** - * 清空队列 - * @param ring_buffer 队列实例 - * @param free_func 释放函数，NULL表示不释放 - */ -void ring_buffer_clear(SRingBuffer* ring_buffer, void (*free_func)(void*)); - -/** - * 获取队列状态 - * @param ring_buffer 队列实例 - * @return 队列状态 - */ -ERingBufferState ring_buffer_get_state(SRingBuffer* ring_buffer); - -/** - * 获取队列大小 - * @param ring_buffer 队列实例 - * @return 当前大小 - */ -uint32_t ring_buffer_get_size(SRingBuffer* ring_buffer); - -/** - * 获取队列容量 - * @param ring_buffer 队列实例 - * @return 容量 - */ -uint32_t ring_buffer_get_capacity(SRingBuffer* ring_buffer); - -/** - * 检查队列是否为空 - * @param ring_buffer 队列实例 - * @return true为空，false非空 - */ -bool ring_buffer_is_empty(SRingBuffer* ring_buffer); - -/** - * 检查队列是否已满 - * @param ring_buffer 队列实例 - * @return true已满，false未满 - */ -bool ring_buffer_is_full(SRingBuffer* ring_buffer); - -/** - * 获取统计信息 - * @param ring_buffer 队列实例 - * @param enqueue_count 入队次数 - * @param dequeue_count 出队次数 - * @param overflow_count 溢出次数 - */ -void ring_buffer_get_stats(SRingBuffer* ring_buffer, uint64_t* enqueue_count, - uint64_t* dequeue_count, uint64_t* overflow_count); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_RING_BUFFER_H \ No newline at end of file diff --git a/plugins/incremental_bitmap/include/roaring_bitmap.h b/plugins/incremental_bitmap/include/roaring_bitmap.h deleted file mode 100644 index f8795cdc5abb..000000000000 --- a/plugins/incremental_bitmap/include/roaring_bitmap.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2024 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_ROARING_BITMAP_H -#define TDENGINE_ROARING_BITMAP_H - -#include "bitmap_interface.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * 创建基于 RoaringBitmap 的位图接口 - * @return 位图接口实例，失败返回NULL - */ -SBitmapInterface* roaring_bitmap_interface_create(void); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_ROARING_BITMAP_H \ No newline at end of file diff --git a/plugins/incremental_bitmap/include/skiplist.h b/plugins/incremental_bitmap/include/skiplist.h deleted file mode 100644 index ce2710fad16e..000000000000 --- a/plugins/incremental_bitmap/include/skiplist.h +++ /dev/null @@ -1,67 +0,0 @@ -#ifndef BITMAP_SKIPLIST_H -#define BITMAP_SKIPLIST_H - -#include -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -#define SKIPLIST_MAX_LEVEL 32 -#define SKIPLIST_P 0.5 - -// 跳表节点 -typedef struct skiplist_node { - uint64_t key; // 可用于时间戳或WAL偏移量 - void* value; // 使用void*以支持任意类型 - struct skiplist_node* forward[SKIPLIST_MAX_LEVEL]; - struct skiplist_node* backward; // 支持双向遍历 - int level; -} skiplist_node_t; - -// 跳表结构体 -typedef struct skiplist { - skiplist_node_t* header; - skiplist_node_t* tail; - int level; - int size; - pthread_rwlock_t rwlock; // 跳表内部并发保护 - // 内存池 - skiplist_node_t* free_nodes; - int free_count; - // value内存统计函数指针，由外部实现并传入 - uint64_t (*value_mem_usage)(void* value); -} skiplist_t; - -// 跳表API -skiplist_t* skiplist_create(); -void skiplist_destroy(skiplist_t* sl); - -// 插入/查找/删除 -void skiplist_insert(skiplist_t* sl, uint64_t key, void* value); -void* skiplist_find(skiplist_t* sl, uint64_t key); -void skiplist_remove(skiplist_t* sl, uint64_t key); - -// 范围查询（升序/降序） -typedef void (*skiplist_range_cb)(uint64_t key, void* value, void* user_data); -void skiplist_range_query(skiplist_t* sl, uint64_t start, uint64_t end, bool reverse, skiplist_range_cb cb, void* user_data); - -// 内存池复用 -void skiplist_node_pool_clear(skiplist_t* sl); - -// 内存使用统计 -uint64_t skiplist_get_memory_usage(skiplist_t* sl); - -// 线程安全（外部可加锁，也可内部加锁） -void skiplist_rdlock(skiplist_t* sl); -void skiplist_wrlock(skiplist_t* sl); -void skiplist_unlock(skiplist_t* sl); - -#ifdef __cplusplus -} -#endif - -#endif // BITMAP_SKIPLIST_H \ No newline at end of file diff --git a/plugins/incremental_bitmap/include/storage_engine_interface.h b/plugins/incremental_bitmap/include/storage_engine_interface.h deleted file mode 100644 index d20afe01df71..000000000000 --- a/plugins/incremental_bitmap/include/storage_engine_interface.h +++ /dev/null @@ -1,98 +0,0 @@ -#ifndef STORAGE_ENGINE_INTERFACE_H -#define STORAGE_ENGINE_INTERFACE_H - -#include -#include -#include "observability.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// 存储引擎事件类型 -typedef enum { - STORAGE_EVENT_BLOCK_CREATE = 0, - STORAGE_EVENT_BLOCK_UPDATE, - STORAGE_EVENT_BLOCK_FLUSH, - STORAGE_EVENT_BLOCK_DELETE, - STORAGE_EVENT_MAX -} EStorageEventType; - -// 存储引擎事件结构 -typedef struct { - EStorageEventType event_type; - uint64_t block_id; - uint64_t wal_offset; - int64_t timestamp; - void* user_data; -} SStorageEvent; - -// 存储引擎事件回调函数类型 -typedef void (*StorageEventCallback)(const SStorageEvent* event, void* user_data); - -// 存储引擎接口配置 -typedef struct { - bool enable_interception; - StorageEventCallback event_callback; - void* callback_user_data; - uint32_t event_buffer_size; - uint32_t callback_threads; -} SStorageEngineConfig; - -// 存储引擎接口函数指针 -typedef struct { - // 初始化存储引擎接口 - int32_t (*init)(const SStorageEngineConfig* config); - - // 销毁存储引擎接口 - void (*destroy)(void); - - // 安装事件拦截 - int32_t (*install_interception)(void); - - // 卸载事件拦截 - int32_t (*uninstall_interception)(void); - - // 手动触发事件（用于测试） - int32_t (*trigger_event)(const SStorageEvent* event); - - // 获取统计信息 - int32_t (*get_stats)(uint64_t* events_processed, uint64_t* events_dropped); - - // 获取详细统计信息 - int32_t (*get_detailed_stats)(void* engine, void* stats); - - // 获取可观测性指标 - int32_t (*get_observability_metrics)(void* engine, SObservabilityMetrics* metrics); - - // 重置统计信息 - int32_t (*reset_stats)(void* engine); - - // 检查是否支持当前存储引擎 - bool (*is_supported)(void); - - // 获取存储引擎名称 - const char* (*get_engine_name)(void); -} SStorageEngineInterface; - -// 存储引擎接口工厂函数 -typedef SStorageEngineInterface* (*StorageEngineInterfaceFactory)(void); - -// 注册存储引擎接口工厂 -int32_t register_storage_engine_interface(const char* name, StorageEngineInterfaceFactory factory); - -// 获取存储引擎接口 -SStorageEngineInterface* get_storage_engine_interface(const char* name); - -// 获取默认存储引擎接口 -SStorageEngineInterface* get_default_storage_engine_interface(void); - -// 列出所有可用的存储引擎接口 -int32_t list_storage_engine_interfaces(char** names, uint32_t max_count, uint32_t* actual_count); - -#ifdef __cplusplus -} -#endif - -#endif // STORAGE_ENGINE_INTERFACE_H - diff --git a/plugins/incremental_bitmap/include/tdengine_storage_engine.h b/plugins/incremental_bitmap/include/tdengine_storage_engine.h deleted file mode 100644 index d2e0bc9d8b27..000000000000 --- a/plugins/incremental_bitmap/include/tdengine_storage_engine.h +++ /dev/null @@ -1,35 +0,0 @@ -#ifndef TDENGINE_STORAGE_ENGINE_H -#define TDENGINE_STORAGE_ENGINE_H - -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -// TDengine TMQ 存储引擎工厂函数 -struct SStorageEngineInterface* tdengine_storage_engine_create(void); - -// 便捷函数：注册TDengine存储引擎 -int32_t register_tdengine_storage_engine(void); - -// 配置 TMQ 参数的高级接口 -int32_t tdengine_set_tmq_config(const char* server_ip, int32_t server_port, - const char* username, const char* password, - const char* database, const char* topic_name, - const char* group_id); - -// 设置 offset 提交策略 -int32_t tdengine_set_commit_strategy(bool auto_commit, bool at_least_once, int64_t commit_interval_ms); - -// 获取详细的统计信息 -int32_t tdengine_get_detailed_stats(uint64_t* events_processed, uint64_t* events_dropped, - uint64_t* messages_consumed, uint64_t* offset_commits, - int64_t* last_commit_time); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_STORAGE_ENGINE_H diff --git a/plugins/incremental_bitmap/quick_test.sh b/plugins/incremental_bitmap/quick_test.sh deleted file mode 100644 index 7279d357f7bd..000000000000 --- a/plugins/incremental_bitmap/quick_test.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -# 快速测试脚本 - 只运行核心测试 -echo "🚀 快速测试开始" -echo "==================" - -cd /home/hp/TDengine/plugins/incremental_bitmap/build - -echo "1. 测试位图引擎核心..." -if timeout 30s ./test_bitmap_engine_core > /dev/null 2>&1; then - echo " ✅ 通过" -else - echo " ❌ 失败" -fi - -echo "2. 测试状态转换..." -if timeout 30s ./test_state_transitions > /dev/null 2>&1; then - echo " ✅ 通过" -else - echo " ❌ 失败" -fi - -echo "3. 测试抽象层..." -if timeout 30s ./test_abstraction_layer > /dev/null 2>&1; then - echo " ✅ 通过" -else - echo " ❌ 失败" -fi - -echo "4. 测试备份协调器..." -if timeout 30s ./test_backup_coordinator > /dev/null 2>&1; then - echo " ✅ 通过" -else - echo " ❌ 失败" -fi - -echo "==================" -echo "✅ 快速测试完成" diff --git a/plugins/incremental_bitmap/run_real_tests.sh b/plugins/incremental_bitmap/run_real_tests.sh deleted file mode 100644 index ec1c829f16a9..000000000000 --- a/plugins/incremental_bitmap/run_real_tests.sh +++ /dev/null @@ -1,171 +0,0 @@ -#!/bin/bash - -# TDengine 增量位图插件 - 真实环境测试脚本 -# 作者：章子渝 -# 版本：1.0 - -echo "🚀 TDengine 增量位图插件真实环境测试开始" -echo "==========================================" - -# 检查当前目录 -if [ ! -f "CMakeLists.txt" ]; then - echo "❌ 错误：请在插件根目录运行此脚本" - echo " 正确路径：/home/hp/TDengine/plugins/incremental_bitmap" - exit 1 -fi - -# 1. 检查TDengine服务状态 -echo "1️⃣ 检查TDengine服务状态..." -if ! pgrep -f taosd > /dev/null; then - echo "❌ 错误：TDengine服务未运行，请先启动taosd" - echo " 启动命令：sudo systemctl start taosd" - exit 1 -fi -echo " ✅ TDengine服务正在运行" - -# 2. 构建真实环境测试 -echo "" -echo "2️⃣ 构建真实环境测试..." -if [ -d "build" ]; then - echo " 清理旧构建..." - rm -rf build -fi - -mkdir -p build -cd build - -echo " 配置CMake（真实环境）..." -cmake -DUSE_MOCK=OFF -DBUILD_TESTING=ON -DE2E_TDENGINE_REAL_TESTS=ON .. > cmake.log 2>&1 -if [ $? -ne 0 ]; then - echo " ❌ CMake配置失败！" - echo " 错误日志：" - cat cmake.log - exit 1 -fi - -echo " 编译真实环境测试..." -make -j$(nproc) > make.log 2>&1 -if [ $? -ne 0 ]; then - echo " ❌ 编译失败！" - echo " 错误日志：" - cat make.log - exit 1 -fi - -echo " ✅ 构建成功" - -# 3. 运行真实环境测试 -echo "" -echo "3️⃣ 运行真实环境测试..." - -test_results=0 -passed=0 -failed=0 - -# 日志目录 -mkdir -p logs - -# 真实环境测试列表 -real_tests=( - "test_offset_semantics_realtime:实时偏移量语义测试" - "test_taosdump_comparison:taosdump对比测试" - "test_pitr_e2e:完整PITR端到端测试" - "test_e2e_tdengine_real:真实TDengine端到端测试" -) - -for test_info in "${real_tests[@]}"; do - test_name=$(echo "$test_info" | cut -d: -f1) - test_desc=$(echo "$test_info" | cut -d: -f2) - - echo " 📋 运行$test_desc..." - log_file="logs/${test_name}.log" - if timeout 120s ./$test_name > "$log_file" 2>&1; then - echo " ✅ 通过" - ((passed++)) - else - echo " ❌ 失败" - echo " ⤷ 查看日志: $log_file" - ((failed++)) - test_results=1 - fi - - # 额外校验: taosdump 对比输出必须包含匹配关键词 - if [ "$test_name" = "test_taosdump_comparison" ]; then - if ! grep -i -E "matched|一致|equal|no differences|全部通过|通过率[:：]\s*100|失败测试[:：]\s*0" "$log_file" > /dev/null 2>&1; then - echo " ❌ 对比结果关键字校验失败(未发现 matched/一致/全部通过/通过率100/失败0)" - test_results=1 - # 修正统计：将之前计为通过的项改为失败 - if [ $passed -gt 0 ]; then passed=$((passed-1)); fi - failed=$((failed+1)) - fi - fi - - # 额外校验: 在运行完真实E2E后检查产物 - if [ "$test_name" = "test_e2e_tdengine_real" ]; then - echo " 🔍 校验E2E产物..." - # 可能的路径（根据二进制运行目录不同做兼容） - paths=( - "./pitr_snapshots" - "./pitr_recovery" - "../build/pitr_snapshots" - "../build/pitr_recovery" - "./build/pitr_snapshots" - "./build/pitr_recovery" - ) - - artifacts_ok=0 - for d in "${paths[@]}"; do - if [ -d "$d" ]; then - # 是否存在非零大小文件 - if find "$d" -type f -size +0c | head -n 1 | grep -q .; then - artifacts_ok=1 - fi - fi - done - - # 报告文件（如果生成） - reports_ok=1 - if [ -f "/tmp/pitr_test_report.txt" ]; then - if [ ! -s "/tmp/pitr_test_report.txt" ]; then reports_ok=0; fi - fi - if [ -f "/tmp/pitr_detailed_report.txt" ]; then - if [ ! -s "/tmp/pitr_detailed_report.txt" ]; then reports_ok=0; fi - # 关键字段校验 - if ! grep -E "Total:|Passed:|Success Rate" /tmp/pitr_detailed_report.txt > /dev/null 2>&1; then - reports_ok=0 - fi - fi - - if [ $artifacts_ok -ne 1 ] || [ $reports_ok -ne 1 ]; then - echo " ❌ E2E 产物/报告校验失败" - test_results=1 - else - echo " ✅ E2E 产物/报告校验通过" - fi - fi -done - -# 4. 测试结果总结 -echo "" -echo "==========================================" -if [ $test_results -eq 0 ]; then - echo "✅ 所有真实环境测试通过！" - echo " 通过: $passed" - echo " 失败: $failed" - echo " 总计: $((passed + failed))" -else - echo "❌ 部分真实环境测试失败" - echo " 通过: $passed" - echo " 失败: $failed" - echo " 总计: $((passed + failed))" -fi - -echo "" -echo "🔧 故障排除：" -echo " - 检查TDengine服务：systemctl status taosd" -echo " - 检查编译日志：build/cmake.log, build/make.log" -echo " - 检查测试日志：运行单个测试查看详细输出" - -echo "==========================================" - -exit $test_results diff --git a/plugins/incremental_bitmap/run_tests.sh b/plugins/incremental_bitmap/run_tests.sh deleted file mode 100644 index ad16888ba1fd..000000000000 --- a/plugins/incremental_bitmap/run_tests.sh +++ /dev/null @@ -1,209 +0,0 @@ -#!/bin/bash - -# TDengine 增量位图插件 - 修复版测试脚本 -# 作者：章子渝 -# 版本：1.1 - -echo "🚀 TDengine 增量位图插件测试开始" -echo "==================================" - -# 检查当前目录 -if [ ! -f "CMakeLists.txt" ]; then - echo "❌ 错误：请在插件根目录运行此脚本" - echo " 正确路径：/home/hp/TDengine/plugins/incremental_bitmap" - exit 1 -fi - -# 1. 环境检查 -echo "1️⃣ 检查环境..." -if ! command -v cmake &> /dev/null; then - echo "❌ 错误：未找到cmake，请安装：sudo apt-get install cmake" - exit 1 -fi - -if ! command -v make &> /dev/null; then - echo "❌ 错误：未找到make，请安装：sudo apt-get install build-essential" - exit 1 -fi - -echo " ✅ 环境检查通过" - -# 2. 构建项目 -echo "" -echo "2️⃣ 构建项目..." -if [ -d "build" ]; then - echo " 清理旧构建..." - rm -rf build -fi - -mkdir -p build -cd build - -echo " 配置CMake..." -cmake -DUSE_MOCK=ON -DBUILD_TESTING=ON .. > cmake.log 2>&1 -if [ $? -ne 0 ]; then - echo " ❌ CMake配置失败！" - echo " 错误日志：" - cat cmake.log - exit 1 -fi - -echo " 编译代码..." -make -j$(nproc) > make.log 2>&1 -if [ $? -ne 0 ]; then - echo " ❌ 编译失败！" - echo " 错误日志：" - cat make.log - exit 1 -fi - -echo " ✅ 构建成功" - -# 3. 运行核心测试 -echo "" -echo "3️⃣ 运行核心测试..." -echo " 📋 运行单元测试..." - -test_results=0 -passed=0 -failed=0 -mem_issues=0 - -# 实际存在的测试列表 -tests=( - "test_bitmap_engine_core:位图引擎核心测试" - "test_state_transitions:状态转换测试" - "test_abstraction_layer:抽象层测试" - "test_backup_coordinator:备份协调器测试" - "test_event_interceptor:事件拦截器测试" - "test_ring_buffer:环形缓冲区测试" - "test_skiplist:跳表测试" - "test_roaring_bitmap_specific:RoaringBitmap特定测试" - "test_fault_injection:故障注入测试" - "test_offset_semantics:偏移量语义测试" - "test_observability_interface:可观测性接口测试" - "test_observability_enhanced:可观测性增强测试" - "test_observability_comprehensive:可观测性综合测试" -) - -for test_info in "${tests[@]}"; do - test_name=$(echo "$test_info" | cut -d: -f1) - test_desc=$(echo "$test_info" | cut -d: -f2) - - echo " - $test_desc..." - if timeout 30s ./$test_name > /dev/null 2>&1; then - echo " ✅ 通过" - ((passed++)) - else - echo " ❌ 失败" - ((failed++)) - test_results=1 - fi -done - -# 4. 运行集成测试 -echo "" -echo "4️⃣ 运行集成测试..." -echo " 📋 运行PITR端到端测试（简化版）..." -if timeout 60s ./test_pitr_e2e_simple > /dev/null 2>&1; then - echo " ✅ 通过" - ((passed++)) -else - echo " ❌ 失败" - ((failed++)) - test_results=1 -fi - -echo " 📋 运行一致性测试（最小化）..." -if timeout 30s ./test_consistency_minimal > /dev/null 2>&1; then - echo " ✅ 通过" - ((passed++)) -else - echo " ❌ 失败" - ((failed++)) - test_results=1 -fi - -# 5. 运行完整功能测试（可选） -echo "" -echo "5️⃣ 运行完整功能测试（可选）..." -echo " ⚠️ 注意：完整测试需要约10-15分钟" -read -p " 是否运行完整测试？(y/N): " run_full_test - -if [[ $run_full_test =~ ^[Yy]$ ]]; then - echo " 📋 运行完整PITR端到端测试..." - if timeout 300s ./test_pitr_e2e > /dev/null 2>&1; then - echo " ✅ 通过" - ((passed++)) - else - echo " ❌ 失败" - ((failed++)) - test_results=1 - fi -fi - -# 6. 内存检查（可选） -echo "" -echo "6️⃣ 内存检查（可选）..." -read -p " 是否运行内存检查？(y/N): " run_memory_check - -if [[ $run_memory_check =~ ^[Yy]$ ]]; then - echo " 📋 运行Valgrind内存检查..." - declare -a vg_targets=( - "test_bitmap_engine_core" - "test_backup_coordinator" - "test_skiplist" - "test_ring_buffer" - ) - mkdir -p valgrind_logs - vg_failed=() - for t in "${vg_targets[@]}"; do - log_file="valgrind_logs/${t}.log" - echo " ▶ $t..." - if timeout 90s valgrind --leak-check=full --error-exitcode=1 ./$t > "$log_file" 2>&1; then - echo " ✅ 通过 ($t)" - else - echo " ❌ 发现内存问题 ($t)" - echo " ⤷ 日志：$log_file" - mem_issues=1 - vg_failed+=("$t") - fi - done -fi - -# 7. 测试结果总结 -echo "" -echo "==================================" -if [ $test_results -eq 0 ]; then - echo "✅ 所有测试通过！" - echo " 通过: $passed" - echo " 失败: $failed" - echo " 总计: $((passed + failed))" - if [ $mem_issues -eq 1 ]; then - echo " ⚠️ 注意：可选内存检查发现问题，详见 valgrind_logs/*.log" - if [ ${#vg_failed[@]} -gt 0 ]; then - echo " 受影响的目标：${vg_failed[*]}" - fi - fi -else - echo "❌ 部分测试失败，请检查上述错误信息" - echo " 通过: $passed" - echo " 失败: $failed" - echo " 总计: $((passed + failed))" - if [ $mem_issues -eq 1 ]; then - echo " ⚠️ 同时存在内存检查问题，详见 valgrind_logs/*.log" - if [ ${#vg_failed[@]} -gt 0 ]; then - echo " 受影响的目标：${vg_failed[*]}" - fi - fi -fi - -echo "" -echo "🔧 故障排除：" -echo " - 检查编译日志：build/cmake.log, build/make.log" -echo " - 检查测试日志：build/pitr_simple.log, build/pitr_full.log" -echo " - 检查内存日志：build/valgrind.log" - -echo "==================================" - -exit $test_results diff --git a/plugins/incremental_bitmap/scripts/ci_trigger.sh b/plugins/incremental_bitmap/scripts/ci_trigger.sh deleted file mode 100644 index 9b6d3b84e2eb..000000000000 --- a/plugins/incremental_bitmap/scripts/ci_trigger.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash - -# Purpose: Create and push a lightweight branch to trigger CI without running anything locally. -# Usage: ./scripts/ci_trigger.sh [branch-name] -# Example: ./scripts/ci_trigger.sh # creates ci/smoke-YYYYMMDD_HHMMSS -# ./scripts/ci_trigger.sh ci/smoke-try # uses provided branch name - -set -euo pipefail - -if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then - echo "ERROR: Not inside a git repository." >&2 - exit 1 -fi - -repo_root=$(git rev-parse --show-toplevel) -cd "$repo_root" - -timestamp=$(date +%Y%m%d_%H%M%S) -branch_name="${1:-ci/smoke-$timestamp}" - -# Marker file to ensure CI triggers without functional code changes -marker_path="plugins/incremental_bitmap/.ci_trigger" -mkdir -p "$(dirname "$marker_path")" -echo "$timestamp" > "$marker_path" - -echo "Creating and pushing branch: $branch_name" -git checkout -b "$branch_name" -git add -A -git commit -m "ci: trigger smoke run ($timestamp)" - -if git remote -v | grep -q '^origin\s'; then - git push -u origin "$branch_name" - echo "Pushed branch to origin: $branch_name" -else - echo "WARNING: No 'origin' remote configured. Add one and push manually, e.g.:" >&2 - echo " git remote add origin " >&2 - echo " git push -u origin $branch_name" >&2 -fi - -cat </dev/null -else - cmake -S "$PLUGIN_DIR" -B "$BUILD_DIR" -DBUILD_TESTING=ON >/dev/null -fi -cmake --build "$BUILD_DIR" --target test_tmq_integration -j >/dev/null - -# 若真实模式，准备数据库/主题/数据 -DB_NAME="test" -TOPIC_NAME="incremental_backup_topic" -if [[ "$REAL_MODE" == "1" ]]; then - echo "[E2E] 准备 TDengine 数据库与主题..." - taos -s "CREATE DATABASE IF NOT EXISTS ${DB_NAME}; - USE ${DB_NAME}; - CREATE STABLE IF NOT EXISTS meters (ts TIMESTAMP, v INT) TAGS (t INT); - CREATE TABLE IF NOT EXISTS d0 USING meters TAGS (1); - INSERT INTO d0 VALUES (now, 1); - CREATE TOPIC IF NOT EXISTS ${TOPIC_NAME} AS DATABASE ${DB_NAME};" >/dev/null || { - echo "[E2E] taos 初始化失败，请确认 TDengine 已启动并可用" >&2 - exit 1 - } -fi - -# 运行集成测试 -echo "[E2E] 运行 test_tmq_integration ..." -"$BIN_TEST" - -# 若真实模式，执行 taosdump 导出并校验 -if [[ "$REAL_MODE" == "1" ]]; then - OUT_DIR="$BUILD_DIR/e2e_dump_out" - rm -rf "$OUT_DIR" && mkdir -p "$OUT_DIR" - echo "[E2E] 运行 taosdump 导出数据库 ${DB_NAME} 到 $OUT_DIR ..." - taosdump -D "$DB_NAME" -o "$OUT_DIR" >/dev/null - echo "[E2E] 校验导出结果..." - # 检查多种可能的导出文件格式 - AVRO_COUNT=$(find "$OUT_DIR" -type f -name "*.avro*" | wc -l || true) - SQL_COUNT=$(find "$OUT_DIR" -type f -name "*.sql" | wc -l || true) - CSV_COUNT=$(find "$OUT_DIR" -type f -name "*.csv" | wc -l || true) - - TOTAL_FILES=$((AVRO_COUNT + SQL_COUNT + CSV_COUNT)) - if [[ "$TOTAL_FILES" -lt 1 ]]; then - echo "[E2E] 导出校验失败：未发现任何导出文件" >&2 - echo "[E2E] 检查到的文件：AVRO=$AVRO_COUNT, SQL=$SQL_COUNT, CSV=$CSV_COUNT" >&2 - exit 1 - fi - echo "[E2E] 导出校验通过：发现 $TOTAL_FILES 个文件 (AVRO=$AVRO_COUNT, SQL=$SQL_COUNT, CSV=$CSV_COUNT)" -else - echo "[E2E] 跳过 taosdump 导出校验（回退模式）" -fi - -echo "[E2E] 本地联调完成" - - diff --git a/plugins/incremental_bitmap/scripts/local_ci.sh b/plugins/incremental_bitmap/scripts/local_ci.sh deleted file mode 100644 index 9842010e42b4..000000000000 --- a/plugins/incremental_bitmap/scripts/local_ci.sh +++ /dev/null @@ -1,567 +0,0 @@ -#!/bin/bash - -# 本地CI脚本 - 增量位图插件 -# 用于在本地运行CI检查，确保代码质量 - -set -e # 遇到错误立即退出 - -# 颜色定义 -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# 配置 -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_DIR="$(dirname "$SCRIPT_DIR")" -BUILD_DIR="$PROJECT_DIR/build" -SRC_DIR="$PROJECT_DIR/src" -INCLUDE_DIR="$PROJECT_DIR/include" -TEST_DIR="$PROJECT_DIR/test" -DOCS_DIR="$PROJECT_DIR/docs" - -# 检查工具函数 -check_command() { - if ! command -v "$1" &> /dev/null; then - echo -e "${RED}错误: 命令 '$1' 未找到${NC}" - echo "请安装 $1 后重试" - exit 1 - fi -} - -# 打印状态 -print_status() { - echo -e "${BLUE}[INFO]${NC} $1" -} - -print_success() { - echo -e "${GREEN}[SUCCESS]${NC} $1" -} - -print_warning() { - echo -e "${YELLOW}[WARNING]${NC} $1" -} - -print_error() { - echo -e "${RED}[ERROR]${NC} $1" -} - -# 检查必要的工具 -check_tools() { - print_status "检查必要的工具..." - - local tools=("cmake" "make" "gcc" "valgrind" "clang-format") - local missing_tools=() - - for tool in "${tools[@]}"; do - if ! command -v "$tool" &> /dev/null; then - missing_tools+=("$tool") - fi - done - - if [ ${#missing_tools[@]} -gt 0 ]; then - print_warning "以下工具未安装: ${missing_tools[*]}" - echo "建议安装这些工具以获得完整的CI检查" - else - print_success "所有必要工具都已安装" - fi -} - -# 代码格式检查 -check_code_format() { - print_status "检查代码格式..." - - if ! command -v clang-format &> /dev/null; then - print_warning "clang-format 未安装，跳过代码格式检查" - return 0 - fi - - local format_issues=0 - - # 检查C文件格式 - while IFS= read -r -d '' file; do - if ! clang-format --dry-run --Werror "$file" &> /dev/null; then - print_error "代码格式问题: $file" - format_issues=$((format_issues + 1)) - fi - done < <(find "$SRC_DIR" "$INCLUDE_DIR" -name "*.c" -o -name "*.h" -print0) - - if [ $format_issues -eq 0 ]; then - print_success "代码格式检查通过" - else - print_error "发现 $format_issues 个代码格式问题" - return 1 - fi -} - -# 静态代码分析 -run_static_analysis() { - print_status "运行静态代码分析..." - - if ! command -v cppcheck &> /dev/null; then - print_warning "cppcheck 未安装，跳过静态分析" - return 0 - fi - - # 创建输出目录 - mkdir -p "$BUILD_DIR/static_analysis" - - # 运行cppcheck - print_status "运行 cppcheck..." - cppcheck --enable=all --std=c99 --language=c \ - --suppress=missingIncludeSystem \ - --suppress=unusedFunction \ - --xml --xml-version=2 \ - --output-file="$BUILD_DIR/static_analysis/cppcheck-report.xml" \ - "$SRC_DIR" "$INCLUDE_DIR" 2>&1 || true - - # 显示简要报告 - if [ -f "$BUILD_DIR/static_analysis/cppcheck-report.xml" ]; then - local issues=$(grep -c " /dev/null; then - print_warning "valgrind 未安装，跳过内存检查" - return 0 - fi - - cd "$BUILD_DIR" - - local memory_issues=0 - - # 对关键测试运行内存检查 - local critical_tests=("test_bitmap_engine_core" "test_observability_comprehensive") - - for test in "${critical_tests[@]}"; do - if [ -x "$test" ]; then - print_status "使用Valgrind检查: $test" - - if valgrind --tool=memcheck --leak-check=full --show-leak-kinds=all \ - --error-exitcode=1 --track-origins=yes \ - --log-file="valgrind_${test}.log" \ - ./"$test" 2>&1; then - print_success "内存检查通过: $test" - else - print_error "内存检查发现问题: $test" - memory_issues=$((memory_issues + 1)) - - # 显示内存问题摘要 - if [ -f "valgrind_${test}.log" ]; then - echo "内存问题摘要:" - grep -E "(ERROR|WARNING)" "valgrind_${test}.log" | head -10 - fi - fi - fi - done - - if [ $memory_issues -eq 0 ]; then - print_success "内存检查完成，未发现问题" - else - print_error "发现 $memory_issues 个内存问题" - return 1 - fi -} - -# 代码覆盖率检查 -check_coverage() { - print_status "检查代码覆盖率..." - - if ! command -v lcov &> /dev/null; then - print_warning "lcov 未安装，跳过覆盖率检查" - return 0 - fi - - cd "$BUILD_DIR" - - # 生成覆盖率报告 - print_status "生成覆盖率报告..." - lcov --capture --directory . --output-file coverage.info - lcov --remove coverage.info '/usr/*' --output-file coverage.info - lcov --remove coverage.info '*/test/*' --output-file coverage.info - - # 显示覆盖率摘要 - local coverage_summary=$(lcov --summary coverage.info) - echo "$coverage_summary" - - # 提取覆盖率百分比 - local coverage_percent=$(echo "$coverage_summary" | grep "lines" | grep -o "[0-9.]*%" | head -1) - - if [ -n "$coverage_percent" ]; then - print_status "代码覆盖率: $coverage_percent" - - # 检查覆盖率是否达到要求（例如80%） - local coverage_number=$(echo "$coverage_percent" | sed 's/%//') - if (( $(echo "$coverage_number >= 80" | bc -l) )); then - print_success "代码覆盖率达标" - else - print_warning "代码覆盖率较低: $coverage_percent" - fi - fi - - print_success "覆盖率检查完成" -} - -# 文档检查 -check_documentation() { - print_status "检查文档..." - - local missing_docs=() - local required_docs=( - "README.md" - "docs/installation_guide.md" - "docs/api_usage_guide.md" - "docs/troubleshooting_guide.md" - "docs/observability_metrics.md" - ) - - # 检查必要文档是否存在 - for doc in "${required_docs[@]}"; do - if [ ! -f "$PROJECT_DIR/$doc" ]; then - missing_docs+=("$doc") - fi - done - - if [ ${#missing_docs[@]} -gt 0 ]; then - print_error "缺少必要文档: ${missing_docs[*]}" - return 1 - fi - - # 检查文档链接 - print_status "检查文档链接..." - local broken_links=0 - - while IFS= read -r -d '' file; do - if [ -f "$file" ]; then - while IFS= read -r link; do - target=$(echo "$link" | sed 's/\[.*\](\([^)]*\))/\1/') - if [[ "$target" != http* && "$target" != \#* ]]; then - if [ ! -f "$target" ] && [ ! -f "$(dirname "$file")/$target" ]; then - print_warning "断开的链接: $target in $file" - broken_links=$((broken_links + 1)) - fi - fi - done < <(grep -o "\[.*\]([^)]*)" "$file" || true) - fi - done < <(find "$DOCS_DIR" -name "*.md" -print0) - - if [ $broken_links -eq 0 ]; then - print_success "文档链接检查通过" - else - print_warning "发现 $broken_links 个断开的链接" - fi - - print_success "文档检查完成" -} - -# 性能基准测试 -run_performance_benchmark() { - print_status "运行性能基准测试..." - - cd "$BUILD_DIR" - - # 检查是否有性能测试 - if [ -x "test_performance" ]; then - print_status "运行性能测试..." - ./test_performance - else - print_status "没有找到性能测试，运行基本性能检查..." - - # 对主要测试进行时间测量 - local main_tests=("test_bitmap_engine_core" "test_observability_comprehensive") - - for test in "${main_tests[@]}"; do - if [ -x "$test" ]; then - print_status "测量 $test 执行时间..." - time ./"$test" > /dev/null 2>&1 - fi - done - fi - - print_success "性能基准测试完成" -} - -# 生成报告 -generate_report() { - print_status "生成CI报告..." - - local report_file="$BUILD_DIR/ci_report.txt" - local timestamp=$(date '+%Y-%m-%d %H:%M:%S') - - cat > "$report_file" << EOF -增量位图插件 - CI检查报告 -生成时间: $timestamp - -=== 检查项目 === -1. 工具检查: ${TOOLS_CHECK_RESULT:-未执行} -2. 代码格式: ${FORMAT_CHECK_RESULT:-未执行} -3. 静态分析: ${STATIC_ANALYSIS_RESULT:-未执行} -4. 项目构建: ${BUILD_RESULT:-未执行} -5. 测试执行: ${TEST_RESULT:-未执行} -6. 内存检查: ${MEMORY_CHECK_RESULT:-未执行} -7. 代码覆盖率: ${COVERAGE_RESULT:-未执行} -8. 文档检查: ${DOC_CHECK_RESULT:-未执行} -9. 性能测试: ${PERFORMANCE_RESULT:-未执行} - -=== 详细结果 === -$CI_DETAILS - -=== 总结 === -总检查项目: 9 -通过: $PASSED_COUNT -失败: $FAILED_COUNT -警告: $WARNING_COUNT - -EOF - - print_success "CI报告已生成: $report_file" - cat "$report_file" -} - -# 主函数 -main() { - echo "==========================================" - echo " 增量位图插件 - 本地CI检查" - echo "==========================================" - echo - - # 初始化计数器 - PASSED_COUNT=0 - FAILED_COUNT=0 - WARNING_COUNT=0 - CI_DETAILS="" - - # 检查工具 - if check_tools; then - TOOLS_CHECK_RESULT="通过" - PASSED_COUNT=$((PASSED_COUNT + 1)) - else - TOOLS_CHECK_RESULT="失败" - FAILED_COUNT=$((FAILED_COUNT + 1)) - fi - - # 代码格式检查 - if check_code_format; then - FORMAT_CHECK_RESULT="通过" - PASSED_COUNT=$((PASSED_COUNT + 1)) - else - FORMAT_CHECK_RESULT="失败" - FAILED_COUNT=$((FAILED_COUNT + 1)) - fi - - # 静态分析 - if run_static_analysis; then - STATIC_ANALYSIS_RESULT="通过" - PASSED_COUNT=$((PASSED_COUNT + 1)) - else - STATIC_ANALYSIS_RESULT="失败" - FAILED_COUNT=$((FAILED_COUNT + 1)) - fi - - # 构建项目 - if build_project; then - BUILD_RESULT="通过" - PASSED_COUNT=$((PASSED_COUNT + 1)) - else - BUILD_RESULT="失败" - FAILED_COUNT=$((FAILED_COUNT + 1)) - fi - - # 运行测试 - if run_tests; then - TEST_RESULT="通过" - PASSED_COUNT=$((PASSED_COUNT + 1)) - else - TEST_RESULT="失败" - FAILED_COUNT=$((FAILED_COUNT + 1)) - fi - - # 内存检查 - if run_memory_check; then - MEMORY_CHECK_RESULT="通过" - PASSED_COUNT=$((PASSED_COUNT + 1)) - else - MEMORY_CHECK_RESULT="失败" - FAILED_COUNT=$((FAILED_COUNT + 1)) - fi - - # 代码覆盖率 - if check_coverage; then - COVERAGE_RESULT="通过" - PASSED_COUNT=$((PASSED_COUNT + 1)) - else - COVERAGE_RESULT="失败" - FAILED_COUNT=$((FAILED_COUNT + 1)) - fi - - # 文档检查 - if check_documentation; then - DOC_CHECK_RESULT="通过" - PASSED_COUNT=$((PASSED_COUNT + 1)) - else - DOC_CHECK_RESULT="失败" - FAILED_COUNT=$((FAILED_COUNT + 1)) - fi - - # 性能测试 - if run_performance_benchmark; then - PERFORMANCE_RESULT="通过" - PASSED_COUNT=$((PASSED_COUNT + 1)) - else - PERFORMANCE_RESULT="失败" - FAILED_COUNT=$((FAILED_COUNT + 1)) - fi - - echo - echo "==========================================" - echo " CI检查完成" - echo "==========================================" - - # 生成报告 - generate_report - - # 返回结果 - if [ $FAILED_COUNT -eq 0 ]; then - print_success "所有检查通过！" - exit 0 - else - print_error "$FAILED_COUNT 个检查失败" - exit 1 - fi -} - -# 处理命令行参数 -case "${1:-}" in - --help|-h) - echo "用法: $0 [选项]" - echo "选项:" - echo " --help, -h 显示帮助信息" - echo " --format 仅检查代码格式" - echo " --static 仅运行静态分析" - echo " --build 仅构建项目" - echo " --test 仅运行测试" - echo " --memory 仅运行内存检查" - echo " --coverage 仅检查代码覆盖率" - echo " --docs 仅检查文档" - echo " --performance 仅运行性能测试" - echo " --quick 快速检查（跳过耗时项目）" - ;; - --format) - check_code_format - ;; - --static) - run_static_analysis - ;; - --build) - build_project - ;; - --test) - build_project - run_tests - ;; - --memory) - build_project - run_memory_check - ;; - --coverage) - build_project - check_coverage - ;; - --docs) - check_documentation - ;; - --performance) - build_project - run_performance_benchmark - ;; - --quick) - # 快速检查：跳过耗时项目 - check_tools - check_code_format - check_documentation - ;; - "") - # 默认运行所有检查 - main - ;; - *) - print_error "未知选项: $1" - echo "使用 --help 查看帮助信息" - exit 1 - ;; -esac diff --git a/plugins/incremental_bitmap/scripts/prepare_tmq_environment.sh b/plugins/incremental_bitmap/scripts/prepare_tmq_environment.sh deleted file mode 100644 index a13ccb6a54f9..000000000000 --- a/plugins/incremental_bitmap/scripts/prepare_tmq_environment.sh +++ /dev/null @@ -1,279 +0,0 @@ -#!/bin/bash - -# TDengine TMQ环境准备脚本 -# 用于Offset语义验证的真实测试 - -set -e - -# 颜色定义 -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# 配置 -TDENGINE_HOST="127.0.0.1" -TDENGINE_PORT="6030" -TDENGINE_USER="root" -TDENGINE_PASS="taosdata" -DATABASE="test_db" -STABLE="test_stable" -TABLE_PREFIX="test_table" -TOPIC_NAME="test_topic" -GROUP_NAME="test_group" - -# taos 配置文件选项：仅当存在时才添加 -c 选项，避免阻塞或连接失败 -TAOS_C_OPT="" -if [ -f /tmp/taos.cfg ]; then - TAOS_C_OPT="-c /tmp/taos.cfg" -elif [ -f /etc/taos/taos.cfg ]; then - TAOS_C_OPT="-c /etc/taos/taos.cfg" -elif [ -f "$HOME/.taos/taos.cfg" ]; then - TAOS_C_OPT="-c $HOME/.taos/taos.cfg" -fi - -# 日志函数 -log_info() { - echo -e "${BLUE}[INFO]${NC} $1" -} - -log_success() { - echo -e "${GREEN}[SUCCESS]${NC} $1" -} - -log_warning() { - echo -e "${YELLOW}[WARNING]${NC} $1" -} - -log_error() { - echo -e "${RED}[ERROR]${NC} $1" -} - -# 检查TDengine连接 -check_tdengine_connection() { - log_info "检查TDengine连接..." - - if ! taos -h $TDENGINE_HOST -P $TDENGINE_PORT -u $TDENGINE_USER -p $TDENGINE_PASS $TAOS_C_OPT -s "SELECT 1;" > /dev/null 2>&1; then - log_error "无法连接到TDengine，请确保服务正在运行" - log_info "启动命令: sudo systemctl start taosd" - exit 1 - fi - - log_success "TDengine连接成功" -} - -# 创建数据库和表 -create_database_and_tables() { - log_info "创建数据库和表..." - - # 创建数据库 - taos -h $TDENGINE_HOST -P $TDENGINE_PORT -u $TDENGINE_USER -p $TDENGINE_PASS $TAOS_C_OPT -s "CREATE DATABASE IF NOT EXISTS $DATABASE;" - log_success "数据库 $DATABASE 创建成功" - - # 使用数据库 - taos -h $TDENGINE_HOST -P $TDENGINE_PORT -u $TDENGINE_USER -p $TDENGINE_PASS $TAOS_C_OPT -s "USE $DATABASE;" - - # 创建超级表 - taos -h $TDENGINE_HOST -P $TDENGINE_PORT -u $TDENGINE_USER -p $TDENGINE_PASS $TAOS_C_OPT -s " - CREATE STABLE IF NOT EXISTS $STABLE ( - ts TIMESTAMP, - value DOUBLE, - tag1 INT, - tag2 VARCHAR(20) - ) TAGS ( - device_id VARCHAR(20), - location VARCHAR(50) - ); - " - log_success "超级表 $STABLE 创建成功" - - # 创建子表 - for i in {1..5}; do - taos -h $TDENGINE_HOST -P $TDENGINE_PORT -u $TDENGINE_USER -p $TDENGINE_PASS $TAOS_C_OPT -s " - CREATE TABLE IF NOT EXISTS ${TABLE_PREFIX}_$i USING $STABLE TAGS ( - 'device_$i', - 'location_$i' - ); - " - done - log_success "子表创建成功" -} - -# 插入测试数据 -insert_test_data() { - log_info "插入测试数据..." - - # 使用数据库 - taos -h $TDENGINE_HOST -P $TDENGINE_PORT -u $TDENGINE_USER -p $TDENGINE_PASS $TAOS_C_OPT -s "USE $DATABASE;" - - # 为每个子表插入数据 - for i in {1..5}; do - log_info "为表 ${TABLE_PREFIX}_$i 插入数据..." - - # 插入历史数据 - taos -h $TDENGINE_HOST -P $TDENGINE_PORT -u $TDENGINE_USER -p $TDENGINE_PASS $TAOS_C_OPT -s " - INSERT INTO ${TABLE_PREFIX}_$i VALUES - (NOW() - 1d, $i.1, $i, 'tag1_$i', 'device_$i', 'location_$i'), - (NOW() - 12h, $i.2, $i, 'tag2_$i', 'device_$i', 'location_$i'), - (NOW() - 6h, $i.3, $i, 'tag3_$i', 'device_$i', 'location_$i'), - (NOW() - 1h, $i.4, $i, 'tag4_$i', 'device_$i', 'location_$i'), - (NOW() - 30m, $i.5, $i, 'tag5_$i', 'device_$i', 'location_$i'); - " - - # 插入实时数据（持续插入） - taos -h $TDENGINE_HOST -P $TDENGINE_PORT -u $TDENGINE_USER -p $TDENGINE_PASS $TAOS_C_OPT -s " - INSERT INTO ${TABLE_PREFIX}_$i VALUES - (NOW(), $i.6, $i, 'tag6_$i', 'device_$i', 'location_$i'); - " - done - - log_success "测试数据插入完成" -} - -# 创建TMQ Topic -create_tmq_topic() { - log_info "创建TMQ Topic..." - - # 使用数据库 - taos -h $TDENGINE_HOST -P $TDENGINE_PORT -u $TDENGINE_USER -p $TDENGINE_PASS $TAOS_C_OPT -s "USE $DATABASE;" - - # 创建Topic - taos -h $TDENGINE_HOST -P $TDENGINE_PORT -u $TDENGINE_USER -p $TDENGINE_PASS $TAOS_C_OPT -s " - CREATE TOPIC IF NOT EXISTS $TOPIC_NAME AS - SELECT * FROM $STABLE; - " - log_success "TMQ Topic $TOPIC_NAME 创建成功" -} - -# 验证TMQ Topic -verify_tmq_topic() { - log_info "验证TMQ Topic..." - - # 使用数据库 - taos -h $TDENGINE_HOST -P $TDENGINE_PORT -u $TDENGINE_USER -p $TDENGINE_PASS $TAOS_C_OPT -s "USE $DATABASE;" - - # 查看Topic列表 - log_info "Topic列表:" - taos -h $TDENGINE_HOST -P $TDENGINE_PORT -u $TDENGINE_USER -p $TDENGINE_PASS $TAOS_C_OPT -s "SHOW TOPICS;" - - # 查看Topic详情 - log_info "Topic $TOPIC_NAME 详情:" - taos -h $TDENGINE_HOST -P $TDENGINE_PORT -u $TDENGINE_USER -p $TDENGINE_PASS $TAOS_C_OPT -s "DESCRIBE TOPIC $TOPIC_NAME;" -} - -# 启动持续数据插入（后台进程） -start_continuous_data_insertion() { - log_info "启动持续数据插入（后台进程）..." - - # 创建后台插入脚本 - cat > /tmp/continuous_insert.sh << 'EOF' -#!/bin/bash -TDENGINE_HOST="127.0.0.1" -TDENGINE_PORT="6030" -TDENGINE_USER="root" -TDENGINE_PASS="taosdata" -DATABASE="test_db" -STABLE="test_stable" -TABLE_PREFIX="test_table" - -while true; do - for i in {1..5}; do - taos -h $TDENGINE_HOST -P $TDENGINE_PORT -u $TDENGINE_USER -p $TDENGINE_PASS -c /tmp/taos.cfg -s " - USE $DATABASE; - INSERT INTO ${TABLE_PREFIX}_$i VALUES - (NOW(), $i.$(($RANDOM % 100 + 1)), $i, 'tag_$(($RANDOM % 1000))', 'device_$i', 'location_$i'); - " > /dev/null 2>&1 - done - sleep 1 -done -EOF - - chmod +x /tmp/continuous_insert.sh - - # 启动后台进程 - nohup /tmp/continuous_insert.sh > /tmp/continuous_insert.log 2>&1 & - CONTINUOUS_PID=$! - - echo $CONTINUOUS_PID > /tmp/continuous_insert.pid - log_success "持续数据插入已启动 (PID: $CONTINUOUS_PID)" - log_info "日志文件: /tmp/continuous_insert.log" - log_info "停止命令: kill \$(cat /tmp/continuous_insert.pid)" -} - -# 显示环境信息 -show_environment_info() { - log_info "环境信息:" - echo " TDengine主机: $TDENGINE_HOST:$TDENGINE_PORT" - echo " 数据库: $DATABASE" - echo " 超级表: $STABLE" - echo " 子表前缀: $TABLE_PREFIX" - echo " TMQ Topic: $TOPIC_NAME" - echo " 消费者组: $GROUP_NAME" - echo "" - log_info "测试准备完成！现在可以运行Offset语义验证测试了。" - echo "" - log_info "运行测试命令:" - echo " cd /home/hp/TDengine/plugins/incremental_bitmap/build" - echo " ./test_offset_semantics_real" - echo "" - log_info "停止持续数据插入:" - echo " kill \$(cat /tmp/continuous_insert.pid)" -} - -# 清理函数 -cleanup() { - log_info "清理资源..." - - # 停止持续数据插入 - if [ -f /tmp/continuous_insert.pid ]; then - PID=$(cat /tmp/continuous_insert.pid) - if kill -0 $PID 2>/dev/null; then - kill $PID - log_success "已停止持续数据插入进程 (PID: $PID)" - fi - rm -f /tmp/continuous_insert.pid - fi - - # 清理临时文件 - rm -f /tmp/continuous_insert.sh - rm -f /tmp/continuous_insert.log - - log_success "清理完成" -} - -# 主函数 -main() { - log_info "开始准备TDengine TMQ环境..." - - # 设置信号处理 - trap cleanup EXIT INT TERM - - # 检查连接 - check_tdengine_connection - - # 创建数据库和表 - create_database_and_tables - - # 插入测试数据 - insert_test_data - - # 创建TMQ Topic - create_tmq_topic - - # 验证TMQ Topic - verify_tmq_topic - - # 启动持续数据插入 - start_continuous_data_insertion - - # 显示环境信息 - show_environment_info - - log_success "TMQ环境准备完成！" -} - -# 运行主函数 -main "$@" - - diff --git a/plugins/incremental_bitmap/setup_tdengine_test.sh b/plugins/incremental_bitmap/setup_tdengine_test.sh deleted file mode 100644 index 754ed1f10f9d..000000000000 --- a/plugins/incremental_bitmap/setup_tdengine_test.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/bin/bash - -# TDengine 测试环境设置脚本 -# 解决SQL语法问题，正确创建测试环境 - -echo "🔧 设置TDengine测试环境" -echo "========================" - -# 检查TDengine服务 -if ! pgrep -f taosd > /dev/null; then - echo "❌ 错误：TDengine服务未运行" - echo " 请先启动：sudo systemctl start taosd" - exit 1 -fi - -echo "✅ TDengine服务正在运行" - -# 设置TDengine配置路径 -TAOS_CFG="/home/hp/TDengine/build/test/cfg/taos.cfg" -TAOS_BIN="/home/hp/TDengine/build/build/bin/taos" - -if [ ! -f "$TAOS_CFG" ]; then - echo "❌ 错误：找不到TDengine配置文件：$TAOS_CFG" - exit 1 -fi - -if [ ! -f "$TAOS_BIN" ]; then - echo "❌ 错误：找不到TDengine客户端：$TAOS_BIN" - exit 1 -fi - -echo "✅ 找到TDengine配置文件：$TAOS_CFG" -echo "✅ 找到TDengine客户端：$TAOS_BIN" - -# 创建测试数据库和表 -echo "" -echo "📋 创建测试数据库和表..." - -# 使用正确的TDengine语法 -$TAOS_BIN -c "$TAOS_CFG" -s " -CREATE DATABASE IF NOT EXISTS test; -USE test; -CREATE STABLE IF NOT EXISTS meters (ts TIMESTAMP, v INT) TAGS (t INT); -CREATE TABLE IF NOT EXISTS d0 USING meters TAGS (1); -CREATE TABLE IF NOT EXISTS d1 USING meters TAGS (2); -CREATE TABLE IF NOT EXISTS d2 USING meters TAGS (3); -CREATE TOPIC IF NOT EXISTS incremental_backup_topic AS DATABASE test; -" - -if [ $? -eq 0 ]; then - echo "✅ 测试环境创建成功" -else - echo "❌ 测试环境创建失败" - exit 1 -fi - -# 插入测试数据 -echo "" -echo "📊 插入测试数据..." -$TAOS_BIN -c "$TAOS_CFG" -s " -USE test; -INSERT INTO d0 VALUES (now, 1); -INSERT INTO d0 VALUES (now+1s, 2); -INSERT INTO d0 VALUES (now+2s, 3); -INSERT INTO d1 VALUES (now, 10); -INSERT INTO d1 VALUES (now+1s, 20); -INSERT INTO d1 VALUES (now+2s, 30); -INSERT INTO d2 VALUES (now, 100); -INSERT INTO d2 VALUES (now+1s, 200); -INSERT INTO d2 VALUES (now+2s, 300); -" - -if [ $? -eq 0 ]; then - echo "✅ 测试数据插入成功" -else - echo "❌ 测试数据插入失败" - exit 1 -fi - -# 验证数据 -echo "" -echo "🔍 验证测试数据..." -$TAOS_BIN -c "$TAOS_CFG" -s " -USE test; -SELECT COUNT(*) FROM d0; -SELECT COUNT(*) FROM d1; -SELECT COUNT(*) FROM d2; -" - -echo "" -echo "🎉 TDengine测试环境设置完成！" -echo " 数据库：test" -echo " 超级表：meters" -echo " 子表：d0, d1, d2" -echo " TMQ主题：incremental_backup_topic" -echo " 测试数据：已插入" diff --git a/plugins/incremental_bitmap/src/backup_coordinator.c b/plugins/incremental_bitmap/src/backup_coordinator.c deleted file mode 100644 index 83fd5696c067..000000000000 --- a/plugins/incremental_bitmap/src/backup_coordinator.c +++ /dev/null @@ -1,485 +0,0 @@ -#include "backup_coordinator.h" -#include "bitmap_engine.h" -#include -#include -#include -#include -#include -#include - -// 获取当前时间戳（纳秒） -static int64_t get_current_timestamp() { - struct timespec ts; - clock_gettime(CLOCK_REALTIME, &ts); - return (int64_t)ts.tv_sec * 1000000000LL + (int64_t)ts.tv_nsec; -} - -// 深拷贝字符串 -static char* deep_copy_string(const char* src) { - if (src == NULL) { - return NULL; - } - - size_t len = strlen(src); - char* dst = (char*)malloc(len + 1); - if (dst == NULL) { - return NULL; - } - - strcpy(dst, src); - return dst; -} - -SBackupCoordinator* backup_coordinator_init(struct SBitmapEngine* bitmap_engine, - const SBackupConfig* config) { - if (!bitmap_engine || !config) { - return NULL; - } - - SBackupCoordinator* coordinator = (SBackupCoordinator*)malloc(sizeof(SBackupCoordinator)); - if (!coordinator) { - return NULL; - } - - // 初始化基本字段 - coordinator->bitmap_engine = bitmap_engine; - coordinator->is_running = false; - coordinator->user_data = NULL; - - // 复制配置 - coordinator->config.batch_size = config->batch_size; - coordinator->config.max_retries = config->max_retries; - coordinator->config.retry_interval_ms = config->retry_interval_ms; - coordinator->config.timeout_ms = config->timeout_ms; - coordinator->config.enable_compression = config->enable_compression; - coordinator->config.enable_encryption = config->enable_encryption; - coordinator->config.backup_path = deep_copy_string(config->backup_path); - coordinator->config.temp_path = deep_copy_string(config->temp_path); - - // 初始化游标 - coordinator->cursor.type = BACKUP_CURSOR_TYPE_HYBRID; - coordinator->cursor.time_cursor = 0; - coordinator->cursor.wal_cursor = 0; - coordinator->cursor.block_count = 0; - coordinator->cursor.last_update_time = get_current_timestamp(); - - // 初始化统计信息 - coordinator->stats.total_blocks = 0; - coordinator->stats.processed_blocks = 0; - coordinator->stats.failed_blocks = 0; - coordinator->stats.total_size = 0; - coordinator->stats.start_time = 0; - coordinator->stats.end_time = 0; - coordinator->stats.retry_count = 0; - - return coordinator; -} - -void backup_coordinator_destroy(SBackupCoordinator* coordinator) { - if (!coordinator) { - return; - } - - // 释放配置中的字符串 - if (coordinator->config.backup_path) { - free(coordinator->config.backup_path); - } - if (coordinator->config.temp_path) { - free(coordinator->config.temp_path); - } - - free(coordinator); -} - -int32_t backup_coordinator_start(SBackupCoordinator* coordinator) { - if (!coordinator) { - return -1; - } - - if (coordinator->is_running) { - return 0; // 已经运行 - } - - coordinator->is_running = true; - coordinator->stats.start_time = get_current_timestamp(); - - return 0; -} - -int32_t backup_coordinator_stop(SBackupCoordinator* coordinator) { - if (!coordinator) { - return -1; - } - - if (!coordinator->is_running) { - return 0; // 已经停止 - } - - coordinator->is_running = false; - coordinator->stats.end_time = get_current_timestamp(); - - return 0; -} - -uint32_t backup_coordinator_get_dirty_blocks(SBackupCoordinator* coordinator, - uint64_t start_wal, uint64_t end_wal, - uint64_t* block_ids, uint32_t max_count) { - if (!coordinator || !block_ids || max_count == 0) { - return 0; - } - - return bitmap_engine_get_dirty_blocks_by_wal(coordinator->bitmap_engine, - start_wal, end_wal, block_ids, max_count); -} - -uint32_t backup_coordinator_get_dirty_blocks_by_time(SBackupCoordinator* coordinator, - int64_t start_time, int64_t end_time, - uint64_t* block_ids, uint32_t max_count) { - if (!coordinator || !block_ids || max_count == 0) { - return 0; - } - - return bitmap_engine_get_dirty_blocks_by_time(coordinator->bitmap_engine, - start_time, end_time, block_ids, max_count); -} - -uint32_t backup_coordinator_get_incremental_blocks(SBackupCoordinator* coordinator, - uint64_t start_wal, uint64_t end_wal, - SIncrementalBlock* blocks, uint32_t max_count) { - // printf("DEBUG: backup_coordinator_get_incremental_blocks: ENTRY\n"); - - if (!coordinator || !blocks || max_count == 0) { - // printf("DEBUG: backup_coordinator_get_incremental_blocks: Invalid parameters\n"); - return 0; - } - - // printf("DEBUG: backup_coordinator_get_incremental_blocks: start_wal=%lu, end_wal=%lu, max_count=%u\n", - // start_wal, end_wal, max_count); - - // 获取脏块ID - uint64_t* block_ids = (uint64_t*)malloc(sizeof(uint64_t) * max_count); - if (block_ids == NULL) { - // printf("DEBUG: Failed to allocate block_ids\n"); - return 0; - } - - // printf("DEBUG: Allocated block_ids at %p\n", block_ids); - - uint32_t count = bitmap_engine_get_dirty_blocks_by_wal(coordinator->bitmap_engine, - start_wal, end_wal, block_ids, max_count); - - // printf("DEBUG: backup_coordinator_get_incremental_blocks: count=%u, max_count=%u\n", count, max_count); - - if (count == 0) { - // printf("DEBUG: No dirty blocks found, freeing block_ids and returning 0\n"); - free(block_ids); - return 0; - } - - // 填充块信息 - // printf("DEBUG: Starting to fill block information\n"); - uint32_t valid_count = 0; - - for (uint32_t i = 0; i < count && valid_count < max_count; i++) { - // printf("DEBUG: Processing block %u: block_id=%lu\n", i, block_ids[i]); - - // 检查数组边界 - if (valid_count >= max_count) { - // printf("DEBUG: Array boundary check failed: valid_count=%u, max_count=%u\n", valid_count, max_count); - break; - } - - // 检查blocks数组是否有效 - if (blocks == NULL) { - // printf("DEBUG: blocks array is NULL\n"); - break; - } - - SBlockMetadata metadata; - // printf("DEBUG: About to call bitmap_engine_get_block_metadata for block_id=%lu\n", block_ids[i]); - - int32_t result = bitmap_engine_get_block_metadata(coordinator->bitmap_engine, block_ids[i], &metadata); - // printf("DEBUG: bitmap_engine_get_block_metadata returned %d for block_id=%lu\n", result, block_ids[i]); - - if (result == 0) { - // printf("DEBUG: Successfully got metadata for block_id=%lu\n", block_ids[i]); - blocks[valid_count].block_id = block_ids[i]; - blocks[valid_count].wal_offset = metadata.wal_offset; - blocks[valid_count].timestamp = metadata.timestamp; - blocks[valid_count].data_size = 0; // 实际数据大小需要从存储引擎获取 - blocks[valid_count].data = NULL; // 实际数据需要从存储引擎获取 - valid_count++; - } else { - // printf("DEBUG: Failed to get metadata for block_id=%lu, skipping this block\n", block_ids[i]); - // 跳过这个块，不添加到结果中 - continue; - } - // printf("DEBUG: Completed processing block %u, valid_count=%u\n", i, valid_count); - } - // printf("DEBUG: Finished filling block information, valid_count=%u\n", valid_count); - - // printf("DEBUG: About to free block_ids\n"); - free(block_ids); - // printf("DEBUG: Freed block_ids\n"); - - // printf("DEBUG: Returning valid_count=%u\n", valid_count); - return valid_count; -} - -uint64_t backup_coordinator_estimate_backup_size(SBackupCoordinator* coordinator, - uint64_t start_wal, uint64_t end_wal) { - if (!coordinator) { - return 0; - } - - // 获取块数量 - uint64_t temp_block_ids[1]; - uint32_t count = bitmap_engine_get_dirty_blocks_by_wal(coordinator->bitmap_engine, - start_wal, end_wal, temp_block_ids, 1); - - if (count == 0) { - return 0; - } - - // 估算每个块的平均大小（这里使用固定值，实际应该根据数据统计） - const uint64_t avg_block_size = 4096; // 4KB per block - - // 获取总块数 - uint64_t total_blocks = 0; - uint64_t offset = start_wal; - while (offset <= end_wal) { - uint32_t batch_count = bitmap_engine_get_dirty_blocks_by_wal(coordinator->bitmap_engine, - offset, end_wal, temp_block_ids, 1); - if (batch_count == 0) { - break; - } - total_blocks += batch_count; - offset += 1000; // 假设WAL偏移量间隔 - } - - return total_blocks * avg_block_size; -} - -int32_t backup_coordinator_get_stats(SBackupCoordinator* coordinator, SBackupStats* stats) { - if (!coordinator || !stats) { - return -1; - } - - *stats = coordinator->stats; - return 0; -} - -int32_t backup_coordinator_reset_stats(SBackupCoordinator* coordinator) { - if (!coordinator) { - return -1; - } - - coordinator->stats.total_blocks = 0; - coordinator->stats.processed_blocks = 0; - coordinator->stats.failed_blocks = 0; - coordinator->stats.total_size = 0; - coordinator->stats.start_time = 0; - coordinator->stats.end_time = 0; - coordinator->stats.retry_count = 0; - - return 0; -} - -int32_t backup_coordinator_get_cursor(SBackupCoordinator* coordinator, SBackupCursor* cursor) { - if (!coordinator || !cursor) { - return -1; - } - - *cursor = coordinator->cursor; - return 0; -} - -int32_t backup_coordinator_set_cursor(SBackupCoordinator* coordinator, const SBackupCursor* cursor) { - if (!coordinator || !cursor) { - return -1; - } - - coordinator->cursor = *cursor; - return 0; -} - -int32_t backup_coordinator_update_cursor(SBackupCoordinator* coordinator, - uint64_t block_id, uint64_t wal_offset, int64_t timestamp) { - if (!coordinator) { - return -1; - } - - coordinator->cursor.wal_cursor = wal_offset; - coordinator->cursor.time_cursor = timestamp; - coordinator->cursor.block_count++; - coordinator->cursor.last_update_time = get_current_timestamp(); - - return 0; -} - -// 插件接口实现 -static SBackupPluginInterface g_backup_plugin_interface = {0}; -static SBackupCoordinator* g_backup_coordinator = NULL; - -static int32_t plugin_init(const char* config) { - // 创建位图引擎（使用默认配置） - SBitmapEngine* bitmap_engine = bitmap_engine_init(); - if (!bitmap_engine) { - // printf("DEBUG: plugin_init: Failed to create bitmap engine\n"); - return -1; - } - - // 创建备份配置 - SBackupConfig backup_config = { - .batch_size = 100, - .max_retries = 3, - .retry_interval_ms = 1000, - .timeout_ms = 5000, - .enable_compression = true, - .enable_encryption = false, - .backup_path = NULL, - .temp_path = NULL - }; - - // 创建备份协调器 - g_backup_coordinator = backup_coordinator_init(bitmap_engine, &backup_config); - if (!g_backup_coordinator) { - // printf("DEBUG: plugin_init: Failed to create backup coordinator\n"); - bitmap_engine_destroy(bitmap_engine); - return -1; - } - - // printf("DEBUG: plugin_init: Successfully initialized plugin\n"); - return 0; -} - -static void plugin_destroy(void) { - if (g_backup_coordinator) { - // 获取位图引擎引用 - SBitmapEngine* bitmap_engine = g_backup_coordinator->bitmap_engine; - - // 销毁备份协调器 - backup_coordinator_destroy(g_backup_coordinator); - g_backup_coordinator = NULL; - - // 销毁位图引擎 - if (bitmap_engine) { - bitmap_engine_destroy(bitmap_engine); - } - - // printf("DEBUG: plugin_destroy: Successfully destroyed plugin\n"); - } -} - -static uint32_t plugin_get_dirty_blocks(uint64_t start_wal, uint64_t end_wal, - uint64_t* block_ids, uint32_t max_count) { - if (!g_backup_coordinator) { - return 0; - } - - return backup_coordinator_get_dirty_blocks(g_backup_coordinator, start_wal, end_wal, block_ids, max_count); -} - -static uint32_t plugin_get_incremental_blocks(uint64_t start_wal, uint64_t end_wal, - SIncrementalBlock* blocks, uint32_t max_count) { - if (!g_backup_coordinator) { - return 0; - } - - return backup_coordinator_get_incremental_blocks(g_backup_coordinator, start_wal, end_wal, blocks, max_count); -} - -static uint64_t plugin_estimate_backup_size(uint64_t start_wal, uint64_t end_wal) { - if (!g_backup_coordinator) { - return 0; - } - - return backup_coordinator_estimate_backup_size(g_backup_coordinator, start_wal, end_wal); -} - -static int32_t plugin_get_stats(SBackupStats* stats) { - if (!g_backup_coordinator) { - return -1; - } - - return backup_coordinator_get_stats(g_backup_coordinator, stats); -} - -static int32_t plugin_reset_stats(void) { - if (!g_backup_coordinator) { - return -1; - } - - return backup_coordinator_reset_stats(g_backup_coordinator); -} - -// 初始化插件接口 -static void init_plugin_interface(void) { - static bool initialized = false; - if (initialized) { - return; - } - - g_backup_plugin_interface.init = plugin_init; - g_backup_plugin_interface.destroy = plugin_destroy; - g_backup_plugin_interface.get_dirty_blocks = plugin_get_dirty_blocks; - g_backup_plugin_interface.get_incremental_blocks = plugin_get_incremental_blocks; - g_backup_plugin_interface.estimate_backup_size = plugin_estimate_backup_size; - g_backup_plugin_interface.get_stats = plugin_get_stats; - g_backup_plugin_interface.reset_stats = plugin_reset_stats; - - initialized = true; -} - -SBackupPluginInterface* backup_plugin_get_interface(void) { - init_plugin_interface(); - return &g_backup_plugin_interface; -} - -// 便捷函数实现 -uint32_t backup_plugin_get_dirty_blocks(uint64_t start_wal, uint64_t end_wal, - uint64_t* block_ids, uint32_t max_count) { - SBackupPluginInterface* interface = backup_plugin_get_interface(); - if (!interface || !interface->get_dirty_blocks) { - return 0; - } - - return interface->get_dirty_blocks(start_wal, end_wal, block_ids, max_count); -} - -uint32_t backup_plugin_get_incremental_blocks(uint64_t start_wal, uint64_t end_wal, - SIncrementalBlock* blocks, uint32_t max_count) { - SBackupPluginInterface* interface = backup_plugin_get_interface(); - if (!interface || !interface->get_incremental_blocks) { - return 0; - } - - return interface->get_incremental_blocks(start_wal, end_wal, blocks, max_count); -} - -uint64_t backup_plugin_estimate_backup_size(uint64_t start_wal, uint64_t end_wal) { - SBackupPluginInterface* interface = backup_plugin_get_interface(); - if (!interface || !interface->estimate_backup_size) { - return 0; - } - - return interface->estimate_backup_size(start_wal, end_wal); -} - -int32_t backup_plugin_get_stats(SBackupStats* stats) { - SBackupPluginInterface* interface = backup_plugin_get_interface(); - if (!interface || !interface->get_stats) { - return -1; - } - - return interface->get_stats(stats); -} - -int32_t backup_plugin_reset_stats(void) { - SBackupPluginInterface* interface = backup_plugin_get_interface(); - if (!interface || !interface->reset_stats) { - return -1; - } - - return interface->reset_stats(); -} \ No newline at end of file diff --git a/plugins/incremental_bitmap/src/bitmap_engine.c b/plugins/incremental_bitmap/src/bitmap_engine.c deleted file mode 100644 index a017912b79a5..000000000000 --- a/plugins/incremental_bitmap/src/bitmap_engine.c +++ /dev/null @@ -1,798 +0,0 @@ -#include "bitmap_engine.h" -#include "bitmap_interface.h" -#include "skiplist.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include "skiplist.h" -#include -#include - -// 轻量级调试日志开关（默认关闭）。 -// 如需开启，编译时加 -DBITMAP_ENGINE_DEBUG 或在此处改为 1。 -#ifndef BITMAP_ENGINE_DEBUG -#define BITMAP_ENGINE_DEBUG 0 -#endif -#if BITMAP_ENGINE_DEBUG -#define DEBUG_LOG(fmt, ...) printf(fmt, ##__VA_ARGS__) -#else -#define DEBUG_LOG(fmt, ...) do { } while (0) -#endif - - - -// 哈希函数 -static uint32_t hash_block_id(uint64_t block_id, uint32_t map_size) { - return (uint32_t)(block_id % map_size); -} - -// 获取当前时间戳（纳秒） -static int64_t get_current_timestamp() { - struct timespec ts; - clock_gettime(CLOCK_MONOTONIC, &ts); - return (int64_t)ts.tv_sec * 1000000000LL + ts.tv_nsec; -} - -// 跳表范围查询上下文结构体 -typedef struct { - SBitmapEngine* engine_ptr; - SBitmapInterface* result_ptr; -} SRangeQueryCtx; - -// 跳表范围查询回调函数（文件级静态函数，避免GCC嵌套函数的栈上trampoline带来的段错误） -static void bitmap_range_accumulate_cb(uint64_t key, void* bm_ptr, void* user_data) { - (void)key; - SRangeQueryCtx* ctx = (SRangeQueryCtx*)user_data; - if (ctx == NULL || ctx->engine_ptr == NULL || ctx->result_ptr == NULL || bm_ptr == NULL) { - return; - } - SBitmapInterface* bm = (SBitmapInterface*)bm_ptr; - SBitmapInterface* intersection = bitmap_interface_create(); - if (intersection == NULL) { - return; - } - // 复制dirty_blocks到intersection - intersection->union_with(intersection->bitmap, ctx->engine_ptr->dirty_blocks->bitmap); - // 与bm求交集 - intersection->intersect_with(intersection->bitmap, bm->bitmap); - // 与result求并集 - ctx->result_ptr->union_with(ctx->result_ptr->bitmap, intersection->bitmap); - bitmap_interface_destroy(intersection); -} - - -// 深拷贝字符串 -static char* deep_copy_string(const char* src) { - if (src == NULL) { - return NULL; - } - - size_t len = strlen(src); - char* dst = (char*)malloc(len + 1); - if (dst == NULL) { - return NULL; - } - - strcpy(dst, src); - return dst; -} - - -// 查找块元数据 -static SBlockMetadataNode* find_block_metadata(SBitmapEngine* engine, uint64_t block_id) { - if (engine == NULL || engine->metadata_map == NULL) { - DEBUG_LOG("DEBUG: find_block_metadata: engine=%p, metadata_map=%p\n", engine, - engine ? engine->metadata_map : NULL); - return NULL; - } - - uint32_t hash = hash_block_id(block_id, engine->metadata_map_size); - DEBUG_LOG("DEBUG: find_block_metadata: block_id=%lu, hash=%u, map_size=%u\n", - block_id, hash, engine->metadata_map_size); - - SBlockMetadataNode* node = engine->metadata_map[hash]; - DEBUG_LOG("DEBUG: find_block_metadata: node at hash %u = %p\n", hash, node); - - while (node != NULL) { - DEBUG_LOG("DEBUG: find_block_metadata: checking node %p, block_id=%lu\n", node, node->block_id); - if (node->block_id == block_id) { - DEBUG_LOG("DEBUG: find_block_metadata: found node for block_id=%lu\n", block_id); - return node; - } - node = node->next; - } - DEBUG_LOG("DEBUG: find_block_metadata: no node found for block_id=%lu\n", block_id); - return NULL; -} - -// 插入块元数据 -static int32_t insert_block_metadata(SBitmapEngine* engine, const SBlockMetadata* metadata) { - DEBUG_LOG("DEBUG: insert_block_metadata: block_id=%lu\n", metadata->block_id); - - uint32_t hash = hash_block_id(metadata->block_id, engine->metadata_map_size); - DEBUG_LOG("DEBUG: insert_block_metadata: hash=%u, map_size=%u\n", hash, engine->metadata_map_size); - - // 检查是否已存在（直接遍历链表，避免调用find_block_metadata） - SBlockMetadataNode* existing = engine->metadata_map[hash]; - while (existing != NULL) { - if (existing->block_id == metadata->block_id) { - DEBUG_LOG("DEBUG: insert_block_metadata: updating existing node\n"); - existing->metadata = *metadata; - return 0; - } - existing = existing->next; - } - - // 创建新节点 - SBlockMetadataNode* node = (SBlockMetadataNode*)malloc(sizeof(SBlockMetadataNode)); - if (node == NULL) { - DEBUG_LOG("DEBUG: insert_block_metadata: failed to allocate node\n"); - return -1; - } - - DEBUG_LOG("DEBUG: insert_block_metadata: created new node at %p\n", node); - - node->block_id = metadata->block_id; - node->metadata = *metadata; - node->next = engine->metadata_map[hash]; - engine->metadata_map[hash] = node; - engine->metadata_count++; - - DEBUG_LOG("DEBUG: insert_block_metadata: inserted node at hash %u, count=%u\n", hash, engine->metadata_count); - - return 0; -} - -// 删除块元数据 -static int32_t remove_block_metadata(SBitmapEngine* engine, uint64_t block_id) { - uint32_t hash = hash_block_id(block_id, engine->metadata_map_size); - SBlockMetadataNode* node = engine->metadata_map[hash]; - SBlockMetadataNode* prev = NULL; - - while (node != NULL) { - if (node->block_id == block_id) { - if (prev == NULL) { - engine->metadata_map[hash] = node->next; - } else { - prev->next = node->next; - } - free(node); - engine->metadata_count--; - - return 0; - } - prev = node; - node = node->next; - } - - return -1; -} - -// 添加时间索引 -static int32_t add_time_index(SBitmapEngine* engine, int64_t timestamp, uint64_t block_id) { - SBitmapInterface* bm = (SBitmapInterface*)skiplist_find(engine->time_index, timestamp); - if (!bm) { - bm = bitmap_interface_create(); - skiplist_insert(engine->time_index, timestamp, bm); - } - bm->add(bm->bitmap, block_id); - return 0; -} - -// 添加WAL索引 -static int32_t add_wal_index(SBitmapEngine* engine, uint64_t wal_offset, uint64_t block_id) { - DEBUG_LOG("DEBUG: add_wal_index: wal_offset=%lu, block_id=%lu\n", wal_offset, block_id); - - if (engine->wal_index == NULL) { - DEBUG_LOG("DEBUG: add_wal_index: WAL index is NULL\n"); - return -1; - } - - SBitmapInterface* bm = (SBitmapInterface*)skiplist_find(engine->wal_index, wal_offset); - if (!bm) { - DEBUG_LOG("DEBUG: add_wal_index: Creating new bitmap for WAL offset %lu\n", wal_offset); - bm = bitmap_interface_create(); - if (!bm) { - DEBUG_LOG("DEBUG: add_wal_index: Failed to create bitmap\n"); - return -1; - } - skiplist_insert(engine->wal_index, wal_offset, bm); - DEBUG_LOG("DEBUG: add_wal_index: Inserted new bitmap at WAL offset %lu\n", wal_offset); - } else { - DEBUG_LOG("DEBUG: add_wal_index: Found existing bitmap for WAL offset %lu\n", wal_offset); - } - - bm->add(bm->bitmap, block_id); - DEBUG_LOG("DEBUG: add_wal_index: Added block %lu to bitmap\n", block_id); - - return 0; -} - - -SBitmapEngine* bitmap_engine_init(void) { - SBitmapEngine* engine = (SBitmapEngine*)malloc(sizeof(SBitmapEngine)); - if (engine == NULL) { - return NULL; - } - - // 初始化位图 - engine->dirty_blocks = bitmap_interface_create(); - engine->new_blocks = bitmap_interface_create(); - engine->deleted_blocks = bitmap_interface_create(); - - if (engine->dirty_blocks == NULL || engine->new_blocks == NULL || engine->deleted_blocks == NULL) { - bitmap_engine_destroy(engine); - return NULL; - } - - // 初始化元数据映射 - engine->metadata_map_size = 1000000; // 默认大小 - engine->metadata_map = (SBlockMetadataNode**)calloc(engine->metadata_map_size, sizeof(SBlockMetadataNode*)); - if (engine->metadata_map == NULL) { - bitmap_engine_destroy(engine); - return NULL; - } - - // 初始化索引 - engine->time_index_head = NULL; - engine->wal_index_head = NULL; - - // 初始化跳表索引 - engine->time_index = skiplist_create(); - engine->wal_index = skiplist_create(); - - // 初始化统计信息 - engine->total_blocks = 0; - engine->dirty_count = 0; - engine->new_count = 0; - engine->deleted_count = 0; - engine->metadata_count = 0; - - // 初始化线程同步 - if (pthread_mutex_init(&engine->mutex, NULL) != 0) { - bitmap_engine_destroy(engine); - return NULL; - } - - if (pthread_rwlock_init(&engine->rwlock, NULL) != 0) { - pthread_mutex_destroy(&engine->mutex); - bitmap_engine_destroy(engine); - return NULL; - } - - return engine; -} - -void bitmap_engine_destroy(SBitmapEngine* engine) { - if (engine == NULL) { - return; - } - - // 销毁位图 - if (engine->dirty_blocks) { - bitmap_interface_destroy(engine->dirty_blocks); - } - if (engine->new_blocks) { - bitmap_interface_destroy(engine->new_blocks); - } - if (engine->deleted_blocks) { - bitmap_interface_destroy(engine->deleted_blocks); - } - - // 销毁元数据映射 - if (engine->metadata_map) { - for (uint32_t i = 0; i < engine->metadata_map_size; i++) { - SBlockMetadataNode* node = engine->metadata_map[i]; - while (node != NULL) { - SBlockMetadataNode* next = node->next; - free(node); - node = next; - } - } - free(engine->metadata_map); - } - - // 销毁时间索引 - STimeIndexNode* time_node = engine->time_index_head; - while (time_node != NULL) { - STimeIndexNode* next = time_node->next; - if (time_node->block_ids) { - bitmap_interface_destroy(time_node->block_ids); - } - free(time_node); - time_node = next; - } - - // 销毁WAL索引 - SWalIndexNode* wal_node = engine->wal_index_head; - while (wal_node != NULL) { - SWalIndexNode* next = wal_node->next; - if (wal_node->block_ids) { - bitmap_interface_destroy(wal_node->block_ids); - } - free(wal_node); - wal_node = next; - } - - // 先释放跳表中节点的位图值，再销毁跳表本身（skiplist不负责释放value） - if (engine->time_index) { - skiplist_node_t* node = engine->time_index->header->forward[0]; - while (node) { - if (node->value) { - bitmap_interface_destroy((SBitmapInterface*)node->value); - node->value = NULL; - } - node = node->forward[0]; - } - skiplist_destroy(engine->time_index); - } - if (engine->wal_index) { - skiplist_node_t* node = engine->wal_index->header->forward[0]; - while (node) { - if (node->value) { - bitmap_interface_destroy((SBitmapInterface*)node->value); - node->value = NULL; - } - node = node->forward[0]; - } - skiplist_destroy(engine->wal_index); - } - - // 销毁线程同步 - pthread_mutex_destroy(&engine->mutex); - pthread_rwlock_destroy(&engine->rwlock); - - free(engine); -} - -int32_t bitmap_engine_mark_dirty(SBitmapEngine* engine, uint64_t block_id, - uint64_t wal_offset, int64_t timestamp) { - if (engine == NULL) { - return ERR_INVALID_PARAM; - } - - pthread_rwlock_wrlock(&engine->rwlock); - - // 获取当前块状态（如果存在） - EBlockState current_state = BLOCK_STATE_CLEAN; // 默认状态 - SBlockMetadataNode* existing_node = find_block_metadata(engine, block_id); - if (existing_node != NULL) { - current_state = existing_node->metadata.state; - } - - // 验证状态转换 - if (bitmap_engine_validate_state_transition(current_state, BLOCK_STATE_DIRTY) != 0) { - pthread_rwlock_unlock(&engine->rwlock); - return ERR_INVALID_STATE_TRANS; - } - - // 创建或更新元数据 - SBlockMetadata metadata; - metadata.block_id = block_id; - metadata.wal_offset = wal_offset; - metadata.timestamp = timestamp; - metadata.state = BLOCK_STATE_DIRTY; - - if (insert_block_metadata(engine, &metadata) != 0) { - pthread_rwlock_unlock(&engine->rwlock); - return -1; - } - - // 添加到位图 - engine->dirty_blocks->add(engine->dirty_blocks->bitmap, block_id); - - // 添加到索引 - add_time_index(engine, timestamp, block_id); - add_wal_index(engine, wal_offset, block_id); - - // 更新统计信息 - engine->dirty_count++; - engine->total_blocks++; - - pthread_rwlock_unlock(&engine->rwlock); - return 0; -} - -int32_t bitmap_engine_mark_new(SBitmapEngine* engine, uint64_t block_id, - uint64_t wal_offset, int64_t timestamp) { - if (engine == NULL) { - return ERR_INVALID_PARAM; - } - - pthread_rwlock_wrlock(&engine->rwlock); - - // 获取当前块状态（如果存在） - EBlockState current_state = BLOCK_STATE_CLEAN; // 默认状态 - SBlockMetadataNode* existing_node = find_block_metadata(engine, block_id); - if (existing_node != NULL) { - current_state = existing_node->metadata.state; - } - - // 验证状态转换 - if (bitmap_engine_validate_state_transition(current_state, BLOCK_STATE_NEW) != 0) { - pthread_rwlock_unlock(&engine->rwlock); - return ERR_INVALID_STATE_TRANS; - } - - // 创建或更新元数据 - SBlockMetadata metadata; - metadata.block_id = block_id; - metadata.wal_offset = wal_offset; - metadata.timestamp = timestamp; - metadata.state = BLOCK_STATE_NEW; - - if (insert_block_metadata(engine, &metadata) != 0) { - pthread_rwlock_unlock(&engine->rwlock); - return -1; - } - - // 从dirty_blocks中移除（如果存在） - if (current_state == BLOCK_STATE_DIRTY) { - engine->dirty_blocks->remove(engine->dirty_blocks->bitmap, block_id); - engine->dirty_count--; - } - - // 添加到位图 - engine->new_blocks->add(engine->new_blocks->bitmap, block_id); - - // 添加到索引 - add_time_index(engine, timestamp, block_id); - add_wal_index(engine, wal_offset, block_id); - - // 更新统计信息 - engine->new_count++; - if (existing_node == NULL) { - engine->total_blocks++; // 只有新块才增加总数 - } - - pthread_rwlock_unlock(&engine->rwlock); - return 0; -} - -int32_t bitmap_engine_mark_deleted(SBitmapEngine* engine, uint64_t block_id, - uint64_t wal_offset, int64_t timestamp) { - if (engine == NULL) { - return ERR_INVALID_PARAM; - } - - pthread_mutex_lock(&engine->mutex); - - // 获取当前块状态（如果存在） - EBlockState current_state = BLOCK_STATE_CLEAN; // 默认状态 - SBlockMetadataNode* existing_node = find_block_metadata(engine, block_id); - if (existing_node != NULL) { - current_state = existing_node->metadata.state; - } - - // 验证状态转换 - if (bitmap_engine_validate_state_transition(current_state, BLOCK_STATE_DELETED) != 0) { - pthread_mutex_unlock(&engine->mutex); - return ERR_INVALID_STATE_TRANS; - } - - // 创建或更新元数据 - SBlockMetadata metadata; - metadata.block_id = block_id; - metadata.wal_offset = wal_offset; - metadata.timestamp = timestamp; - metadata.state = BLOCK_STATE_DELETED; - - if (insert_block_metadata(engine, &metadata) != 0) { - pthread_mutex_unlock(&engine->mutex); - return -1; - } - - // 从其他位图中移除（如果存在） - if (current_state == BLOCK_STATE_DIRTY) { - engine->dirty_blocks->remove(engine->dirty_blocks->bitmap, block_id); - engine->dirty_count--; - } else if (current_state == BLOCK_STATE_NEW) { - engine->new_blocks->remove(engine->new_blocks->bitmap, block_id); - engine->new_count--; - } - - // 添加到位图 - engine->deleted_blocks->add(engine->deleted_blocks->bitmap, block_id); - - // 添加到索引 - add_time_index(engine, timestamp, block_id); - add_wal_index(engine, wal_offset, block_id); - - // 更新统计信息 - engine->deleted_count++; - if (existing_node == NULL) { - engine->total_blocks++; // 只有新块才增加总数 - } - - pthread_mutex_unlock(&engine->mutex); - return 0; -} - -int32_t bitmap_engine_clear_block(SBitmapEngine* engine, uint64_t block_id) { - if (engine == NULL) { - return ERR_INVALID_PARAM; - } - - pthread_mutex_lock(&engine->mutex); - - // 获取当前块状态 - SBlockMetadataNode* existing_node = find_block_metadata(engine, block_id); - if (existing_node == NULL) { - pthread_mutex_unlock(&engine->mutex); - return ERR_BLOCK_NOT_FOUND; - } - - EBlockState current_state = existing_node->metadata.state; - - // 验证状态转换（清除块相当于转换为CLEAN状态） - if (bitmap_engine_validate_state_transition(current_state, BLOCK_STATE_CLEAN) != 0) { - pthread_mutex_unlock(&engine->mutex); - return ERR_INVALID_STATE_TRANS; - } - - // 从位图中移除 - engine->dirty_blocks->remove(engine->dirty_blocks->bitmap, block_id); - engine->new_blocks->remove(engine->new_blocks->bitmap, block_id); - engine->deleted_blocks->remove(engine->deleted_blocks->bitmap, block_id); - - // 删除元数据 - remove_block_metadata(engine, block_id); - - // 更新统计信息 - engine->dirty_count = engine->dirty_blocks->cardinality(engine->dirty_blocks->bitmap); - engine->new_count = engine->new_blocks->cardinality(engine->new_blocks->bitmap); - engine->deleted_count = engine->deleted_blocks->cardinality(engine->deleted_blocks->bitmap); - engine->total_blocks = engine->metadata_count; - - pthread_mutex_unlock(&engine->mutex); - return 0; -} - -uint32_t bitmap_engine_get_dirty_blocks_by_time(SBitmapEngine* engine, - int64_t start_time, int64_t end_time, - uint64_t* block_ids, uint32_t max_count) { - if (engine == NULL || block_ids == NULL || max_count == 0) { - return 0; - } - - pthread_rwlock_rdlock(&engine->rwlock); - - SBitmapInterface* result = bitmap_interface_create(); - uint32_t count = 0; - - // 跳表范围查询（使用文件级静态回调函数） - - SRangeQueryCtx ctx = { .engine_ptr = engine, .result_ptr = result }; - skiplist_range_query(engine->time_index, start_time, end_time, false, bitmap_range_accumulate_cb, &ctx); - - // 获取结果 - count = result->to_array(result->bitmap, block_ids, max_count); - - bitmap_interface_destroy(result); - pthread_rwlock_unlock(&engine->rwlock); - - return count; -} - -uint32_t bitmap_engine_get_dirty_blocks_by_wal(SBitmapEngine* engine, - uint64_t start_offset, uint64_t end_offset, - uint64_t* block_ids, uint32_t max_count) { - if (engine == NULL || block_ids == NULL || max_count == 0) { - return 0; - } - - DEBUG_LOG("DEBUG: bitmap_engine_get_dirty_blocks_by_wal: engine=%p, start_offset=%lu, end_offset=%lu\n", - engine, start_offset, end_offset); - - pthread_rwlock_rdlock(&engine->rwlock); - - // 检查WAL索引是否为空 - if (engine->wal_index == NULL) { - DEBUG_LOG("DEBUG: WAL index is NULL\n"); - pthread_rwlock_unlock(&engine->rwlock); - return 0; - } - - // 检查跳表大小 - DEBUG_LOG("DEBUG: WAL index size: %d\n", engine->wal_index->size); - - // 使用更安全的策略：直接收集所有匹配的块ID，而不是进行位图操作 - uint64_t* temp_block_ids = (uint64_t*)malloc(sizeof(uint64_t) * max_count * 2); // 分配更多空间 - if (!temp_block_ids) { - DEBUG_LOG("DEBUG: Failed to allocate temp block IDs\n"); - pthread_rwlock_unlock(&engine->rwlock); - return 0; - } - - uint32_t temp_count = 0; - - // 遍历WAL索引，收集所有匹配的块ID - skiplist_node_t* current = engine->wal_index->header->forward[0]; - while (current && current != engine->wal_index->header && temp_count < max_count * 2) { - if (current->key >= start_offset && current->key <= end_offset) { - SBitmapInterface* bm = (SBitmapInterface*)current->value; - if (bm && bm->bitmap && bm->to_array) { - // 获取当前WAL偏移量对应的块ID - uint64_t temp_array[100]; // 临时数组 - uint32_t block_count = bm->to_array(bm->bitmap, temp_array, 100); - - // 添加到临时结果中 - for (uint32_t i = 0; i < block_count && temp_count < max_count * 2; i++) { - temp_block_ids[temp_count++] = temp_array[i]; - } - } - } - current = current->forward[0]; - } - - DEBUG_LOG("DEBUG: Collected %u temporary block IDs\n", temp_count); - - // 去重并复制到结果数组 - uint32_t final_count = 0; - for (uint32_t i = 0; i < temp_count && final_count < max_count; i++) { - uint64_t block_id = temp_block_ids[i]; - bool duplicate = false; - - // 检查是否重复 - for (uint32_t j = 0; j < final_count; j++) { - if (block_ids[j] == block_id) { - duplicate = true; - break; - } - } - - if (!duplicate) { - block_ids[final_count++] = block_id; - } - } - - DEBUG_LOG("DEBUG: Final result: %u unique block IDs\n", final_count); - - // 输出找到的块ID - for (uint32_t i = 0; i < final_count && i < 10; i++) { - DEBUG_LOG("DEBUG: block_ids[%u] = %lu\n", i, block_ids[i]); - } - - // 清理临时内存 - free(temp_block_ids); - - pthread_rwlock_unlock(&engine->rwlock); - - return final_count; -} - -int32_t bitmap_engine_get_block_metadata(SBitmapEngine* engine, uint64_t block_id, - SBlockMetadata* metadata) { - if (engine == NULL || metadata == NULL) { - DEBUG_LOG("DEBUG: bitmap_engine_get_block_metadata: engine=%p, metadata=%p\n", engine, metadata); - return -1; - } - - DEBUG_LOG("DEBUG: bitmap_engine_get_block_metadata: block_id=%lu\n", block_id); - - pthread_rwlock_rdlock(&engine->rwlock); - - SBlockMetadataNode* node = find_block_metadata(engine, block_id); - if (node == NULL) { - DEBUG_LOG("DEBUG: find_block_metadata returned NULL for block_id=%lu\n", block_id); - pthread_rwlock_unlock(&engine->rwlock); - return -1; - } - - DEBUG_LOG("DEBUG: Found metadata for block_id=%lu, state=%d\n", block_id, node->metadata.state); - - *metadata = node->metadata; - pthread_rwlock_unlock(&engine->rwlock); - - return 0; -} - - -void bitmap_engine_get_stats(SBitmapEngine* engine, uint64_t* total_blocks, - uint64_t* dirty_count, uint64_t* new_count, uint64_t* deleted_count) { - if (engine == NULL) { - return; - } - - pthread_mutex_lock(&engine->mutex); - - if (total_blocks) *total_blocks = engine->total_blocks; - if (dirty_count) *dirty_count = engine->dirty_count; - if (new_count) *new_count = engine->new_count; - if (deleted_count) *deleted_count = engine->deleted_count; - - pthread_mutex_unlock(&engine->mutex); -} - - - -// 状态转换验证实现 - -// 状态转换规则矩阵 -// 行：当前状态，列：目标状态 -// 1表示允许转换，0表示不允许转换 -static const int8_t STATE_TRANSITION_MATRIX[4][4] = { - // CLEAN DIRTY NEW DELETED - { 0, 1, 1, 1 }, // CLEAN - { 1, 0, 1, 1 }, // DIRTY (允许转换为NEW) - { 0, 1, 0, 1 }, // NEW - { 0, 0, 0, 0 } // DELETED (不可转换为任何状态) -}; - -int32_t bitmap_engine_validate_state_transition(EBlockState current_state, EBlockState target_state) { - // 检查状态值是否有效 - if (current_state < 0 || current_state >= 4 || target_state < 0 || target_state >= 4) { - return ERR_INVALID_STATE_TRANS; - } - - // 检查转换是否允许 - if (STATE_TRANSITION_MATRIX[current_state][target_state]) { - return 0; // 允许转换 - } else { - return ERR_INVALID_STATE_TRANS; // 不允许转换 - } -} - -const char* bitmap_engine_get_state_transition_error(EBlockState current_state, EBlockState target_state) { - static char error_msg[256]; - - const char* state_names[] = {"CLEAN", "DIRTY", "NEW", "DELETED"}; - - if (current_state < 0 || current_state >= 4 || target_state < 0 || target_state >= 4) { - snprintf(error_msg, sizeof(error_msg), "Invalid state values: current=%d, target=%d", - current_state, target_state); - return error_msg; - } - - if (STATE_TRANSITION_MATRIX[current_state][target_state]) { - snprintf(error_msg, sizeof(error_msg), "State transition from %s to %s is valid", - state_names[current_state], state_names[target_state]); - return error_msg; - } - - // 根据具体的不允许转换情况提供详细错误信息 - if (current_state == BLOCK_STATE_DELETED) { - snprintf(error_msg, sizeof(error_msg), - "Cannot transition from DELETED state to %s state. DELETED blocks cannot be modified.", - state_names[target_state]); - } else if (current_state == BLOCK_STATE_CLEAN && target_state == BLOCK_STATE_NEW) { - snprintf(error_msg, sizeof(error_msg), - "Cannot transition from CLEAN to NEW state. CLEAN blocks must first become DIRTY."); - } else if (current_state == BLOCK_STATE_CLEAN && target_state == BLOCK_STATE_DELETED) { - snprintf(error_msg, sizeof(error_msg), - "Cannot transition from CLEAN to DELETED state. CLEAN blocks must first become DIRTY."); - } else if (current_state == BLOCK_STATE_NEW && target_state == BLOCK_STATE_CLEAN) { - snprintf(error_msg, sizeof(error_msg), - "Cannot transition from NEW to CLEAN state. NEW blocks can only become DIRTY or DELETED."); - } else { - snprintf(error_msg, sizeof(error_msg), - "Invalid state transition from %s to %s", - state_names[current_state], state_names[target_state]); - } - - return error_msg; -} - -int32_t bitmap_engine_get_block_state(SBitmapEngine* engine, uint64_t block_id, EBlockState* state) { - if (engine == NULL || state == NULL) { - return ERR_INVALID_PARAM; - } - - pthread_mutex_lock(&engine->mutex); - - SBlockMetadataNode* node = find_block_metadata(engine, block_id); - if (node == NULL) { - pthread_mutex_unlock(&engine->mutex); - return ERR_BLOCK_NOT_FOUND; - } - - *state = node->metadata.state; - - pthread_mutex_unlock(&engine->mutex); - return 0; -} \ No newline at end of file diff --git a/plugins/incremental_bitmap/src/event_interceptor.c b/plugins/incremental_bitmap/src/event_interceptor.c deleted file mode 100644 index 88da2b04a651..000000000000 --- a/plugins/incremental_bitmap/src/event_interceptor.c +++ /dev/null @@ -1,406 +0,0 @@ -#include "event_interceptor.h" -#include "storage_engine_interface.h" -#include "ring_buffer.h" -#include "bitmap_engine.h" -#include -#include -#include -#include -#include -#include - -// 回调线程参数 -typedef struct { - SEventInterceptor* interceptor; - uint32_t thread_id; - bool running; -} SCallbackThreadParam; - -// 获取当前时间戳（纳秒） -static int64_t get_current_timestamp() { - struct timespec ts; - clock_gettime(CLOCK_REALTIME, &ts); - return (int64_t)ts.tv_sec * 1000000000LL + (int64_t)ts.tv_nsec; -} - -// 回调线程函数 -static void* callback_thread_func(void* arg) { - SEventInterceptor* interceptor = (SEventInterceptor*)arg; - - while (!interceptor->stop_threads) { - // 从环形队列中获取事件 - void* event_ptr; - int32_t result = ring_buffer_dequeue_blocking(interceptor->event_buffer, &event_ptr, 1000); // 1秒超时 - - if (result == 0) { - SBlockEvent* event = (SBlockEvent*)event_ptr; - - // 更新位图引擎 - switch (event->event_type) { - case EVENT_BLOCK_CREATE: - bitmap_engine_mark_new(interceptor->bitmap_engine, event->block_id, event->wal_offset, event->timestamp); - break; - case EVENT_BLOCK_UPDATE: - bitmap_engine_mark_dirty(interceptor->bitmap_engine, event->block_id, event->wal_offset, event->timestamp); - break; - case EVENT_BLOCK_FLUSH: - bitmap_engine_clear_block(interceptor->bitmap_engine, event->block_id); - break; - case EVENT_BLOCK_DELETE: - bitmap_engine_mark_deleted(interceptor->bitmap_engine, event->block_id, event->wal_offset, event->timestamp); - break; - default: - break; - } - - // 处理事件 - if (interceptor->config.callback) { - interceptor->config.callback(event, interceptor->config.callback_user_data); - } - - // 更新统计信息 - pthread_mutex_lock(&interceptor->mutex); - interceptor->events_processed++; - pthread_mutex_unlock(&interceptor->mutex); - - // 释放事件内存 - free(event); - } - } - - return NULL; -} - -SEventInterceptor* event_interceptor_init(const SEventInterceptorConfig* config, - SBitmapEngine* bitmap_engine) { - if (!config || !bitmap_engine) { - return NULL; - } - - SEventInterceptor* interceptor = (SEventInterceptor*)malloc(sizeof(SEventInterceptor)); - if (!interceptor) { - return NULL; - } - - // 复制配置 - memcpy(&interceptor->config, config, sizeof(SEventInterceptorConfig)); - interceptor->bitmap_engine = bitmap_engine; - interceptor->buffer_size = config->event_buffer_size; - interceptor->thread_count = config->callback_threads; - interceptor->stop_threads = false; - interceptor->events_processed = 0; - interceptor->events_dropped = 0; - interceptor->storage_interface = NULL; - - // 初始化事件缓冲区（环形队列） - interceptor->event_buffer = ring_buffer_init(interceptor->buffer_size); - if (!interceptor->event_buffer) { - free(interceptor); - return NULL; - } - - // 初始化互斥锁 - if (pthread_mutex_init(&interceptor->mutex, NULL) != 0) { - ring_buffer_destroy(interceptor->event_buffer); - free(interceptor); - return NULL; - } - - // 初始化条件变量 - if (pthread_cond_init(&interceptor->condition, NULL) != 0) { - pthread_mutex_destroy(&interceptor->mutex); - ring_buffer_destroy(interceptor->event_buffer); - free(interceptor); - return NULL; - } - - // 分配回调线程数组 - interceptor->callback_threads = (pthread_t*)malloc(sizeof(pthread_t) * interceptor->thread_count); - if (!interceptor->callback_threads) { - pthread_cond_destroy(&interceptor->condition); - pthread_mutex_destroy(&interceptor->mutex); - ring_buffer_destroy(interceptor->event_buffer); - free(interceptor); - return NULL; - } - - // 初始化线程ID为0 - for (uint32_t i = 0; i < interceptor->thread_count; i++) { - interceptor->callback_threads[i] = 0; - } - - return interceptor; -} - -void event_interceptor_destroy(SEventInterceptor* interceptor) { - if (!interceptor) { - return; - } - - // 停止所有线程（如果正在运行） - if (!interceptor->stop_threads) { - event_interceptor_stop(interceptor); - } - - // 销毁存储引擎接口 - if (interceptor->storage_interface) { - interceptor->storage_interface->destroy(); - interceptor->storage_interface = NULL; - } - - // 销毁环形队列 - if (interceptor->event_buffer) { - ring_buffer_destroy(interceptor->event_buffer); - } - - // 销毁条件变量 - pthread_cond_destroy(&interceptor->condition); - - // 销毁互斥锁 - pthread_mutex_destroy(&interceptor->mutex); - - // 释放线程数组 - if (interceptor->callback_threads) { - free(interceptor->callback_threads); - } - - free(interceptor); -} - -int32_t event_interceptor_start(SEventInterceptor* interceptor) { - if (!interceptor) { - return -1; - } - - pthread_mutex_lock(&interceptor->mutex); - - if (interceptor->stop_threads) { - // 已经启动 - pthread_mutex_unlock(&interceptor->mutex); - return 0; - } - - interceptor->stop_threads = false; - - // 创建回调线程 - for (uint32_t i = 0; i < interceptor->thread_count; i++) { - if (pthread_create(&interceptor->callback_threads[i], NULL, - callback_thread_func, interceptor) != 0) { - // 创建失败，停止已创建的线程 - for (uint32_t j = 0; j < i; j++) { - pthread_join(interceptor->callback_threads[j], NULL); - } - pthread_mutex_unlock(&interceptor->mutex); - return -1; - } - } - - pthread_mutex_unlock(&interceptor->mutex); - return 0; -} - -int32_t event_interceptor_stop(SEventInterceptor* interceptor) { - if (!interceptor) { - return -1; - } - - pthread_mutex_lock(&interceptor->mutex); - - if (interceptor->stop_threads) { - // 已经停止 - pthread_mutex_unlock(&interceptor->mutex); - return 0; - } - - interceptor->stop_threads = true; - - // 等待所有线程结束（只有在线程已经创建的情况下） - for (uint32_t i = 0; i < interceptor->thread_count; i++) { - if (interceptor->callback_threads[i] != 0) { - pthread_join(interceptor->callback_threads[i], NULL); - } - } - - pthread_mutex_unlock(&interceptor->mutex); - return 0; -} - -int32_t event_interceptor_on_block_create(SEventInterceptor* interceptor, - uint64_t block_id, uint64_t wal_offset, int64_t timestamp) { - if (!interceptor || !interceptor->config.enable_interception) { - return 0; - } - - SBlockEvent* event = (SBlockEvent*)malloc(sizeof(SBlockEvent)); - if (!event) { - return -1; - } - - event->event_type = EVENT_BLOCK_CREATE; - event->block_id = block_id; - event->wal_offset = wal_offset; - event->timestamp = timestamp; - event->user_data = NULL; - - int32_t result = ring_buffer_enqueue(interceptor->event_buffer, event); - if (result != 0) { - pthread_mutex_lock(&interceptor->mutex); - interceptor->events_dropped++; - pthread_mutex_unlock(&interceptor->mutex); - free(event); - } - - return result; -} - -int32_t event_interceptor_on_block_update(SEventInterceptor* interceptor, - uint64_t block_id, uint64_t wal_offset, int64_t timestamp) { - if (!interceptor || !interceptor->config.enable_interception) { - return 0; - } - - SBlockEvent* event = (SBlockEvent*)malloc(sizeof(SBlockEvent)); - if (!event) { - return -1; - } - - event->event_type = EVENT_BLOCK_UPDATE; - event->block_id = block_id; - event->wal_offset = wal_offset; - event->timestamp = timestamp; - event->user_data = NULL; - - int32_t result = ring_buffer_enqueue(interceptor->event_buffer, event); - if (result != 0) { - pthread_mutex_lock(&interceptor->mutex); - interceptor->events_dropped++; - pthread_mutex_unlock(&interceptor->mutex); - free(event); - } - - return result; -} - -int32_t event_interceptor_on_block_flush(SEventInterceptor* interceptor, - uint64_t block_id, uint64_t wal_offset, int64_t timestamp) { - if (!interceptor || !interceptor->config.enable_interception) { - return 0; - } - - SBlockEvent* event = (SBlockEvent*)malloc(sizeof(SBlockEvent)); - if (!event) { - return -1; - } - - event->event_type = EVENT_BLOCK_FLUSH; - event->block_id = block_id; - event->wal_offset = wal_offset; - event->timestamp = timestamp; - event->user_data = NULL; - - int32_t result = ring_buffer_enqueue(interceptor->event_buffer, event); - if (result != 0) { - pthread_mutex_lock(&interceptor->mutex); - interceptor->events_dropped++; - pthread_mutex_unlock(&interceptor->mutex); - free(event); - } - - return result; -} - -int32_t event_interceptor_on_block_delete(SEventInterceptor* interceptor, - uint64_t block_id, uint64_t wal_offset, int64_t timestamp) { - if (!interceptor || !interceptor->config.enable_interception) { - return 0; - } - - SBlockEvent* event = (SBlockEvent*)malloc(sizeof(SBlockEvent)); - if (!event) { - return -1; - } - - event->event_type = EVENT_BLOCK_DELETE; - event->block_id = block_id; - event->wal_offset = wal_offset; - event->timestamp = timestamp; - event->user_data = NULL; - - int32_t result = ring_buffer_enqueue(interceptor->event_buffer, event); - if (result != 0) { - pthread_mutex_lock(&interceptor->mutex); - interceptor->events_dropped++; - pthread_mutex_unlock(&interceptor->mutex); - free(event); - } - - return result; -} - -void event_interceptor_get_stats(SEventInterceptor* interceptor, - uint64_t* events_processed, uint64_t* events_dropped) { - if (!interceptor) { - return; - } - - pthread_mutex_lock(&interceptor->mutex); - - if (events_processed) { - *events_processed = interceptor->events_processed; - } - if (events_dropped) { - *events_dropped = interceptor->events_dropped; - } - - pthread_mutex_unlock(&interceptor->mutex); -} - -// 存储引擎接口管理 -int32_t event_interceptor_set_storage_interface(SEventInterceptor* interceptor, - SStorageEngineInterface* interface) { - if (!interceptor || !interface) { - return -1; - } - - // 如果已有接口，先销毁 - if (interceptor->storage_interface) { - interceptor->storage_interface->destroy(); - } - - interceptor->storage_interface = interface; - return 0; -} - -int32_t event_interceptor_install_storage_interception(SEventInterceptor* interceptor) { - if (!interceptor || !interceptor->storage_interface) { - return -1; - } - - return interceptor->storage_interface->install_interception(); -} - -int32_t event_interceptor_uninstall_storage_interception(SEventInterceptor* interceptor) { - if (!interceptor || !interceptor->storage_interface) { - return -1; - } - - return interceptor->storage_interface->uninstall_interception(); -} - -// 测试用的手动事件触发 -int32_t event_interceptor_trigger_test_event(SEventInterceptor* interceptor, - EBlockEventType event_type, - uint64_t block_id, uint64_t wal_offset, int64_t timestamp) { - switch (event_type) { - case EVENT_BLOCK_CREATE: - return event_interceptor_on_block_create(interceptor, block_id, wal_offset, timestamp); - case EVENT_BLOCK_UPDATE: - return event_interceptor_on_block_update(interceptor, block_id, wal_offset, timestamp); - case EVENT_BLOCK_FLUSH: - return event_interceptor_on_block_flush(interceptor, block_id, wal_offset, timestamp); - case EVENT_BLOCK_DELETE: - return event_interceptor_on_block_delete(interceptor, block_id, wal_offset, timestamp); - default: - return -1; - } -} \ No newline at end of file diff --git a/plugins/incremental_bitmap/src/incremental_backup_tool.c b/plugins/incremental_bitmap/src/incremental_backup_tool.c deleted file mode 100644 index 4f66aa22de0e..000000000000 --- a/plugins/incremental_bitmap/src/incremental_backup_tool.c +++ /dev/null @@ -1,362 +0,0 @@ -#include "bitmap_engine.h" -#include "event_interceptor.h" -#include "backup_coordinator.h" -#include "storage_engine_interface.h" -#include -#include -#include -#include -#include -#include - -// 线程数自适应与覆盖： -// 优先使用环境变量 IB_CALLBACK_THREADS（正整数）； -// 否则按 min(2 * 在线CPU核数, 64) 自适应。 -static uint32_t get_adaptive_callback_threads(void) { - const char* env_value = getenv("IB_CALLBACK_THREADS"); - if (env_value && *env_value) { - long parsed = strtol(env_value, NULL, 10); - if (parsed > 0 && parsed < 1000000) { - printf("[并发配置] 使用环境变量 IB_CALLBACK_THREADS=%ld\n", parsed); - return (uint32_t)parsed; - } else { - printf("[并发配置] 环境变量 IB_CALLBACK_THREADS=\"%s\" 非法，忽略并采用自适应\n", env_value); - } - } - - long cores = sysconf(_SC_NPROCESSORS_ONLN); - if (cores <= 0) cores = 1; - long adaptive = cores * 2; - if (adaptive > 64) adaptive = 64; - if (adaptive < 1) adaptive = 1; - printf("[并发配置] Detected cores=%ld, using callback_threads=%ld (source=auto)\n", cores, adaptive); - return (uint32_t)adaptive; -} - -// 前向声明回调，确保在配置结构体使用前可见 -static void backup_event_callback(const SBlockEvent* event, void* user_data); - -// 增量备份工具配置 -typedef struct { - char* source_host; - int source_port; - char* database; - char* backup_path; - char* bitmap_cache_path; - int64_t since_timestamp; - uint32_t batch_size; - bool enable_compression; - bool enable_encryption; -} SIncrementalBackupConfig; - -// 增量备份工具状态 -typedef struct { - SIncrementalBackupConfig config; - SBitmapEngine* bitmap_engine; - SEventInterceptor* event_interceptor; - SBackupCoordinator* backup_coordinator; - SStorageEngineInterface* storage_interface; - bool is_running; - uint64_t total_blocks; - uint64_t processed_blocks; - uint64_t failed_blocks; -} SIncrementalBackupTool; - -// 创建增量备份工具 -SIncrementalBackupTool* incremental_backup_tool_create(const SIncrementalBackupConfig* config) { - if (!config) { - return NULL; - } - - SIncrementalBackupTool* tool = malloc(sizeof(SIncrementalBackupTool)); - if (!tool) { - return NULL; - } - - memset(tool, 0, sizeof(SIncrementalBackupTool)); - memcpy(&tool->config, config, sizeof(SIncrementalBackupConfig)); - - // 初始化位图引擎（当前API无配置参数） - tool->bitmap_engine = bitmap_engine_init(); - if (!tool->bitmap_engine) { - free(tool); - return NULL; - } - - // 初始化事件拦截器 - SEventInterceptorConfig interceptor_config = { - .enable_interception = true, - .event_buffer_size = 10000, - .callback_threads = get_adaptive_callback_threads(), - .callback = backup_event_callback, - .callback_user_data = tool - }; - - tool->event_interceptor = event_interceptor_init(&interceptor_config, tool->bitmap_engine); - if (!tool->event_interceptor) { - bitmap_engine_destroy(tool->bitmap_engine); - free(tool); - return NULL; - } - - // 初始化备份协调器 - SBackupConfig backup_config = { - .batch_size = config->batch_size, - .max_retries = 3, - .retry_interval_ms = 1000, - .timeout_ms = 30000, - .enable_compression = config->enable_compression, - .enable_encryption = config->enable_encryption, - .backup_path = config->backup_path, - .temp_path = "/tmp" - }; - - tool->backup_coordinator = backup_coordinator_init(tool->bitmap_engine, &backup_config); - if (!tool->backup_coordinator) { - event_interceptor_destroy(tool->event_interceptor); - bitmap_engine_destroy(tool->bitmap_engine); - free(tool); - return NULL; - } - - // 获取存储引擎接口 - tool->storage_interface = get_storage_engine_interface("tdengine"); - if (tool->storage_interface) { - event_interceptor_set_storage_interface(tool->event_interceptor, tool->storage_interface); - } - - tool->is_running = false; - return tool; -} - -// 事件回调函数 -static void backup_event_callback(const SBlockEvent* event, void* user_data) { - SIncrementalBackupTool* tool = (SIncrementalBackupTool*)user_data; - - // 将事件转换为存储事件 - SStorageEvent storage_event = { - .event_type = (EStorageEventType)event->event_type, - .block_id = event->block_id, - .wal_offset = event->wal_offset, - .timestamp = event->timestamp, - .user_data = NULL - }; - - // 触发存储引擎事件 - if (tool->storage_interface) { - tool->storage_interface->trigger_event(&storage_event); - } - - // 更新位图引擎 - switch (event->event_type) { - case EVENT_BLOCK_CREATE: - bitmap_engine_mark_new(tool->bitmap_engine, event->block_id, - event->wal_offset, event->timestamp); - break; - case EVENT_BLOCK_UPDATE: - bitmap_engine_mark_dirty(tool->bitmap_engine, event->block_id, - event->wal_offset, event->timestamp); - break; - case EVENT_BLOCK_FLUSH: - // 清除块状态表示已刷新 - bitmap_engine_clear_block(tool->bitmap_engine, event->block_id); - break; - case EVENT_BLOCK_DELETE: - bitmap_engine_mark_deleted(tool->bitmap_engine, event->block_id, - event->wal_offset, event->timestamp); - break; - } -} - -// 启动增量备份 -int32_t incremental_backup_tool_start(SIncrementalBackupTool* tool) { - if (!tool || tool->is_running) { - return -1; - } - - // 回调在创建时已经设置，此处无需更新配置 - - // 启动事件拦截器 - int32_t result = event_interceptor_start(tool->event_interceptor); - if (result != 0) { - return result; - } - - // 安装存储引擎拦截 - if (tool->storage_interface) { - result = event_interceptor_install_storage_interception(tool->event_interceptor); - if (result != 0) { - event_interceptor_stop(tool->event_interceptor); - return result; - } - } - - tool->is_running = true; - printf("[增量备份] 工具启动成功\n"); - return 0; -} - -// 执行增量备份 -int32_t incremental_backup_tool_backup(SIncrementalBackupTool* tool, int64_t since_timestamp) { - if (!tool) { - return -1; - } - - printf("[增量备份] 开始执行增量备份，时间戳: %ld\n", since_timestamp); - - // 获取增量块 - uint32_t max_ids = tool->config.batch_size > 0 ? tool->config.batch_size : 10000; - uint64_t* block_ids = (uint64_t*)calloc(max_ids, sizeof(uint64_t)); - if (!block_ids) return -1; - - int64_t end_time = (int64_t)time(NULL) * 1000000000LL; // ns - uint32_t block_count = backup_coordinator_get_dirty_blocks_by_time( - tool->backup_coordinator, since_timestamp, end_time, block_ids, max_ids); - - if (block_count == 0) { - printf("[增量备份] 没有发现增量数据\n"); - return 0; - } - - printf("[增量备份] 发现 %u 个增量块\n", block_count); - - // 将块ID转换为块信息（此处不加载实际数据，只聚合元信息） - uint64_t total_size = 0; - for (uint32_t i = 0; i < block_count; i++) { - SBlockMetadata md; - if (bitmap_engine_get_block_metadata(tool->bitmap_engine, block_ids[i], &md) == 0) { - total_size += md.wal_offset ? (uint64_t)1024 : 0; // 估算大小占位 - } - } - - // 更新统计信息 - tool->total_blocks += block_count; - tool->processed_blocks += block_count; - printf("[增量备份] 备份完成: 处理=%u, 估算大小=%lu bytes\n", block_count, total_size); - - // 清理内存 - free(block_ids); - - return 0; -} - -// 生成taosdump兼容的增量备份脚本 -int32_t incremental_backup_tool_generate_taosdump_script(SIncrementalBackupTool* tool, - const char* script_path) { - if (!tool || !script_path) { - return -1; - } - - FILE* file = fopen(script_path, "w"); - if (!file) { - return -1; - } - - // 生成bash脚本 - fprintf(file, "#!/bin/bash\n\n"); - fprintf(file, "# TDengine增量备份脚本 - 由位图插件生成\n"); - fprintf(file, "# 生成时间: %s\n\n", ctime(&(time_t){time(NULL)})); - - fprintf(file, "SOURCE_HOST=%s\n", tool->config.source_host); - fprintf(file, "SOURCE_PORT=%d\n", tool->config.source_port); - fprintf(file, "DATABASE=%s\n", tool->config.database); - fprintf(file, "BACKUP_PATH=%s\n", tool->config.backup_path); - fprintf(file, "SINCE_TIMESTAMP=%ld\n\n", tool->config.since_timestamp); - - // 1. 使用位图插件检测增量块 - fprintf(file, "echo \"步骤1: 检测增量数据块...\"\n"); - fprintf(file, "./incremental_bitmap_tool --detect \\\n"); - fprintf(file, " --host $SOURCE_HOST --port $SOURCE_PORT \\\n"); - fprintf(file, " --database $DATABASE \\\n"); - fprintf(file, " --since $SINCE_TIMESTAMP \\\n"); - fprintf(file, " --output incremental_blocks.json\n\n"); - - // 2. 使用taosdump备份增量数据 - fprintf(file, "echo \"步骤2: 使用taosdump备份增量数据...\"\n"); - fprintf(file, "taosdump -h $SOURCE_HOST -P $SOURCE_PORT \\\n"); - fprintf(file, " -D $DATABASE \\\n"); - fprintf(file, " -S $SINCE_TIMESTAMP \\\n"); - fprintf(file, " -o $BACKUP_PATH/incremental_$(date +%%Y%%m%%d_%%H%%M%%S)\n\n"); - - // 3. 使用位图插件验证备份完整性 - fprintf(file, "echo \"步骤3: 验证备份完整性...\"\n"); - fprintf(file, "./incremental_bitmap_tool --verify \\\n"); - fprintf(file, " --backup $BACKUP_PATH \\\n"); - fprintf(file, " --blocks incremental_blocks.json \\\n"); - fprintf(file, " --report backup_verification_report.json\n\n"); - - fprintf(file, "echo \"增量备份完成!\"\n"); - - fclose(file); - - // 设置执行权限 - chmod(script_path, 0755); - - printf("[增量备份] 生成taosdump脚本: %s\n", script_path); - return 0; -} - -// 停止增量备份 -int32_t incremental_backup_tool_stop(SIncrementalBackupTool* tool) { - if (!tool || !tool->is_running) { - return -1; - } - - // 停止事件拦截器 - event_interceptor_stop(tool->event_interceptor); - - // 卸载存储引擎拦截 - if (tool->storage_interface) { - event_interceptor_uninstall_storage_interception(tool->event_interceptor); - } - - tool->is_running = false; - printf("[增量备份] 工具已停止\n"); - return 0; -} - -// 销毁增量备份工具 -void incremental_backup_tool_destroy(SIncrementalBackupTool* tool) { - if (!tool) { - return; - } - - if (tool->is_running) { - incremental_backup_tool_stop(tool); - } - - if (tool->backup_coordinator) { - backup_coordinator_destroy(tool->backup_coordinator); - } - - if (tool->event_interceptor) { - event_interceptor_destroy(tool->event_interceptor); - } - - if (tool->bitmap_engine) { - bitmap_engine_destroy(tool->bitmap_engine); - } - - free(tool); -} - -// 获取备份统计信息 -void incremental_backup_tool_get_stats(SIncrementalBackupTool* tool, - uint64_t* total_blocks, - uint64_t* processed_blocks, - uint64_t* failed_blocks) { - if (!tool) { - return; - } - - if (total_blocks) { - *total_blocks = tool->total_blocks; - } - if (processed_blocks) { - *processed_blocks = tool->processed_blocks; - } - if (failed_blocks) { - *failed_blocks = tool->failed_blocks; - } -} - diff --git a/plugins/incremental_bitmap/src/observability.c b/plugins/incremental_bitmap/src/observability.c deleted file mode 100644 index 006b45d161aa..000000000000 --- a/plugins/incremental_bitmap/src/observability.c +++ /dev/null @@ -1,272 +0,0 @@ -#include "observability.h" -#include -#include -#include - -// 可观测性指标更新函数 -void update_observability_metrics(SObservabilityMetrics* metrics) { - if (!metrics) { - return; - } - - // 更新最后更新时间 - metrics->last_update_time = time(NULL) * 1000; // 转换为毫秒 - - // 这里可以添加更多的指标更新逻辑 - // 例如从系统获取内存使用情况等 -} - -// 可观测性指标打印函数 -void print_observability_metrics(const SObservabilityMetrics* metrics) { - if (!metrics) { - printf("可观测性指标: NULL\n"); - return; - } - - printf("\n=== TDengine 增量位图插件可观测性指标 ===\n"); - - // 速率指标 - printf("📊 速率指标:\n"); - printf(" 事件处理速率: %lu 事件/秒\n", metrics->events_per_second); - printf(" 消息消费速率: %lu 消息/秒\n", metrics->messages_per_second); - printf(" 数据吞吐量: %lu 字节/秒\n", metrics->bytes_per_second); - - // 滞后指标 - printf("\n⏱️ 滞后指标:\n"); - printf(" 消费者滞后时间: %ld ms\n", metrics->consumer_lag_ms); - printf(" Offset滞后数量: %lu\n", metrics->offset_lag); - printf(" 处理延迟: %ld ms\n", metrics->processing_delay_ms); - - // 丢弃指标 - printf("\n❌ 丢弃指标:\n"); - printf(" 丢弃事件数: %lu\n", metrics->events_dropped); - printf(" 丢弃消息数: %lu\n", metrics->messages_dropped); - printf(" 解析错误数: %lu\n", metrics->parse_errors); - - // 重试指标 - printf("\n🔄 重试指标:\n"); - printf(" 连接重试次数: %lu\n", metrics->connection_retries); - printf(" 订阅重试次数: %lu\n", metrics->subscription_retries); - printf(" 提交重试次数: %lu\n", metrics->commit_retries); - - // 队列水位 - printf("\n📦 队列水位:\n"); - printf(" 环形队列使用率: %u%%\n", metrics->ring_buffer_usage); - printf(" 环形队列容量: %u\n", metrics->ring_buffer_capacity); - printf(" 事件队列大小: %u\n", metrics->event_queue_size); - - // 内存指标 - printf("\n💾 内存指标:\n"); - printf(" 总内存使用: %zu 字节 (%.2f MB)\n", - metrics->memory_usage_bytes, - metrics->memory_usage_bytes / (1024.0 * 1024.0)); - printf(" 位图内存使用: %zu 字节 (%.2f MB)\n", - metrics->bitmap_memory_bytes, - metrics->bitmap_memory_bytes / (1024.0 * 1024.0)); - printf(" 元数据内存使用: %zu 字节 (%.2f MB)\n", - metrics->metadata_memory_bytes, - metrics->metadata_memory_bytes / (1024.0 * 1024.0)); - - // 时间戳 - printf("\n⏰ 时间信息:\n"); - printf(" 最后更新时间: %ld\n", metrics->last_update_time); - printf(" 运行时间: %ld 秒 (%.2f 小时)\n", - metrics->uptime_seconds, - metrics->uptime_seconds / 3600.0); - - printf("\n==========================================\n"); -} - -// 格式化可观测性指标为JSON字符串 -void format_observability_metrics_json(const SObservabilityMetrics* metrics, char* buffer, size_t buffer_size) { - if (!metrics || !buffer || buffer_size == 0) { - return; - } - - snprintf(buffer, buffer_size, - "{" - "\"rate\":{" - "\"events_per_second\":%lu," - "\"messages_per_second\":%lu," - "\"bytes_per_second\":%lu" - "}," - "\"lag\":{" - "\"consumer_lag_ms\":%ld," - "\"offset_lag\":%lu," - "\"processing_delay_ms\":%ld" - "}," - "\"dropped\":{" - "\"events_dropped\":%lu," - "\"messages_dropped\":%lu," - "\"parse_errors\":%lu" - "}," - "\"retries\":{" - "\"connection_retries\":%lu," - "\"subscription_retries\":%lu," - "\"commit_retries\":%lu" - "}," - "\"queue\":{" - "\"ring_buffer_usage\":%u," - "\"ring_buffer_capacity\":%u," - "\"event_queue_size\":%u" - "}," - "\"memory\":{" - "\"memory_usage_bytes\":%zu," - "\"bitmap_memory_bytes\":%zu," - "\"metadata_memory_bytes\":%zu" - "}," - "\"time\":{" - "\"last_update_time\":%ld," - "\"uptime_seconds\":%ld" - "}" - "}", - metrics->events_per_second, - metrics->messages_per_second, - metrics->bytes_per_second, - metrics->consumer_lag_ms, - metrics->offset_lag, - metrics->processing_delay_ms, - metrics->events_dropped, - metrics->messages_dropped, - metrics->parse_errors, - metrics->connection_retries, - metrics->subscription_retries, - metrics->commit_retries, - metrics->ring_buffer_usage, - metrics->ring_buffer_capacity, - metrics->event_queue_size, - metrics->memory_usage_bytes, - metrics->bitmap_memory_bytes, - metrics->metadata_memory_bytes, - metrics->last_update_time, - metrics->uptime_seconds - ); -} - -// 格式化可观测性指标为Prometheus格式 -void format_observability_metrics_prometheus(const SObservabilityMetrics* metrics, char* buffer, size_t buffer_size) { - if (!metrics || !buffer || buffer_size == 0) { - return; - } - - size_t offset = 0; - int written; - - // 速率指标 - written = snprintf(buffer + offset, buffer_size - offset, - "# HELP tdengine_events_per_second Events processed per second\n" - "# TYPE tdengine_events_per_second gauge\n" - "tdengine_events_per_second %lu\n" - "# HELP tdengine_messages_per_second Messages consumed per second\n" - "# TYPE tdengine_messages_per_second gauge\n" - "tdengine_messages_per_second %lu\n" - "# HELP tdengine_bytes_per_second Data throughput in bytes per second\n" - "# TYPE tdengine_bytes_per_second gauge\n" - "tdengine_bytes_per_second %lu\n", - metrics->events_per_second, - metrics->messages_per_second, - metrics->bytes_per_second - ); - if (written < 0 || (size_t)written >= buffer_size - offset) return; - offset += written; - - // 滞后指标 - written = snprintf(buffer + offset, buffer_size - offset, - "# HELP tdengine_consumer_lag_ms Consumer lag in milliseconds\n" - "# TYPE tdengine_consumer_lag_ms gauge\n" - "tdengine_consumer_lag_ms %ld\n" - "# HELP tdengine_offset_lag Offset lag count\n" - "# TYPE tdengine_offset_lag gauge\n" - "tdengine_offset_lag %lu\n" - "# HELP tdengine_processing_delay_ms Processing delay in milliseconds\n" - "# TYPE tdengine_processing_delay_ms gauge\n" - "tdengine_processing_delay_ms %ld\n", - metrics->consumer_lag_ms, - metrics->offset_lag, - metrics->processing_delay_ms - ); - if (written < 0 || (size_t)written >= buffer_size - offset) return; - offset += written; - - // 丢弃指标 - written = snprintf(buffer + offset, buffer_size - offset, - "# HELP tdengine_events_dropped Total events dropped\n" - "# TYPE tdengine_events_dropped counter\n" - "tdengine_events_dropped %lu\n" - "# HELP tdengine_messages_dropped Total messages dropped\n" - "# TYPE tdengine_messages_dropped counter\n" - "tdengine_messages_dropped %lu\n" - "# HELP tdengine_parse_errors Total parse errors\n" - "# TYPE tdengine_parse_errors counter\n" - "tdengine_parse_errors %lu\n", - metrics->events_dropped, - metrics->messages_dropped, - metrics->parse_errors - ); - if (written < 0 || (size_t)written >= buffer_size - offset) return; - offset += written; - - // 重试指标 - written = snprintf(buffer + offset, buffer_size - offset, - "# HELP tdengine_connection_retries Total connection retries\n" - "# TYPE tdengine_connection_retries counter\n" - "tdengine_connection_retries %lu\n" - "# HELP tdengine_subscription_retries Total subscription retries\n" - "# TYPE tdengine_subscription_retries counter\n" - "tdengine_subscription_retries %lu\n" - "# HELP tdengine_commit_retries Total commit retries\n" - "# TYPE tdengine_commit_retries counter\n" - "tdengine_commit_retries %lu\n", - metrics->connection_retries, - metrics->subscription_retries, - metrics->commit_retries - ); - if (written < 0 || (size_t)written >= buffer_size - offset) return; - offset += written; - - // 队列指标 - written = snprintf(buffer + offset, buffer_size - offset, - "# HELP tdengine_ring_buffer_usage Ring buffer usage percentage\n" - "# TYPE tdengine_ring_buffer_usage gauge\n" - "tdengine_ring_buffer_usage %u\n" - "# HELP tdengine_ring_buffer_capacity Ring buffer capacity\n" - "# TYPE tdengine_ring_buffer_capacity gauge\n" - "tdengine_ring_buffer_capacity %u\n" - "# HELP tdengine_event_queue_size Event queue size\n" - "# TYPE tdengine_event_queue_size gauge\n" - "tdengine_event_queue_size %u\n", - metrics->ring_buffer_usage, - metrics->ring_buffer_capacity, - metrics->event_queue_size - ); - if (written < 0 || (size_t)written >= buffer_size - offset) return; - offset += written; - - // 内存指标 - written = snprintf(buffer + offset, buffer_size - offset, - "# HELP tdengine_memory_usage_bytes Total memory usage in bytes\n" - "# TYPE tdengine_memory_usage_bytes gauge\n" - "tdengine_memory_usage_bytes %zu\n" - "# HELP tdengine_bitmap_memory_bytes Bitmap memory usage in bytes\n" - "# TYPE tdengine_bitmap_memory_bytes gauge\n" - "tdengine_bitmap_memory_bytes %zu\n" - "# HELP tdengine_metadata_memory_bytes Metadata memory usage in bytes\n" - "# TYPE tdengine_metadata_memory_bytes gauge\n" - "tdengine_metadata_memory_bytes %zu\n", - metrics->memory_usage_bytes, - metrics->bitmap_memory_bytes, - metrics->metadata_memory_bytes - ); - if (written < 0 || (size_t)written >= buffer_size - offset) return; - offset += written; - - // 时间指标 - written = snprintf(buffer + offset, buffer_size - offset, - "# HELP tdengine_uptime_seconds Total uptime in seconds\n" - "# TYPE tdengine_uptime_seconds gauge\n" - "tdengine_uptime_seconds %ld\n", - metrics->uptime_seconds - ); - if (written < 0 || (size_t)written >= buffer_size - offset) return; - offset += written; -} diff --git a/plugins/incremental_bitmap/src/ring_buffer.c b/plugins/incremental_bitmap/src/ring_buffer.c deleted file mode 100644 index f966ba75231f..000000000000 --- a/plugins/incremental_bitmap/src/ring_buffer.c +++ /dev/null @@ -1,330 +0,0 @@ -/* - * Copyright (c) 2024 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "ring_buffer.h" -#include -#include -#include -#include -#include // Added for printf - -SRingBuffer* ring_buffer_init(uint32_t capacity) { - if (capacity == 0) { - return NULL; - } - - SRingBuffer* ring_buffer = (SRingBuffer*)malloc(sizeof(SRingBuffer)); - if (!ring_buffer) { - return NULL; - } - - ring_buffer->buffer = (void**)malloc(sizeof(void*) * capacity); - if (!ring_buffer->buffer) { - free(ring_buffer); - return NULL; - } - - ring_buffer->capacity = capacity; - ring_buffer->size = 0; - ring_buffer->head = 0; - ring_buffer->tail = 0; - ring_buffer->enqueue_count = 0; - ring_buffer->dequeue_count = 0; - ring_buffer->overflow_count = 0; - - // 初始化互斥锁和条件变量 - if (pthread_mutex_init(&ring_buffer->mutex, NULL) != 0) { - free(ring_buffer->buffer); - free(ring_buffer); - return NULL; - } - - if (pthread_cond_init(&ring_buffer->not_empty, NULL) != 0) { - pthread_mutex_destroy(&ring_buffer->mutex); - free(ring_buffer->buffer); - free(ring_buffer); - return NULL; - } - - if (pthread_cond_init(&ring_buffer->not_full, NULL) != 0) { - pthread_cond_destroy(&ring_buffer->not_empty); - pthread_mutex_destroy(&ring_buffer->mutex); - free(ring_buffer->buffer); - free(ring_buffer); - return NULL; - } - - return ring_buffer; -} - -void ring_buffer_destroy(SRingBuffer* ring_buffer) { - if (!ring_buffer) { - return; - } - - pthread_cond_destroy(&ring_buffer->not_full); - pthread_cond_destroy(&ring_buffer->not_empty); - pthread_mutex_destroy(&ring_buffer->mutex); - - free(ring_buffer->buffer); - free(ring_buffer); -} - -int32_t ring_buffer_enqueue(SRingBuffer* ring_buffer, void* item) { - if (!ring_buffer || !item) { - return -1; - } - - pthread_mutex_lock(&ring_buffer->mutex); - - if (ring_buffer->size >= ring_buffer->capacity) { - ring_buffer->overflow_count++; - pthread_mutex_unlock(&ring_buffer->mutex); - return -1; // 队列已满 - } - - ring_buffer->buffer[ring_buffer->tail] = item; - ring_buffer->tail = (ring_buffer->tail + 1) % ring_buffer->capacity; - ring_buffer->size++; - ring_buffer->enqueue_count++; - - // 通知等待的消费者 - pthread_cond_signal(&ring_buffer->not_empty); - - pthread_mutex_unlock(&ring_buffer->mutex); - return 0; -} - -int32_t ring_buffer_enqueue_blocking(SRingBuffer* ring_buffer, void* item, uint32_t timeout_ms) { - if (!ring_buffer || !item) { - return -1; - } - - pthread_mutex_lock(&ring_buffer->mutex); - int success = 0; - // 等待队列有空间 - while (ring_buffer->size >= ring_buffer->capacity) { - if (timeout_ms == 0) { - // 无限等待 - pthread_cond_wait(&ring_buffer->not_full, &ring_buffer->mutex); - } else { - // 有限等待 - struct timespec ts; - clock_gettime(CLOCK_REALTIME, &ts); - ts.tv_nsec += (timeout_ms % 1000) * 1000000; - ts.tv_sec += timeout_ms / 1000 + ts.tv_nsec / 1000000000; - ts.tv_nsec %= 1000000000; - int ret = pthread_cond_timedwait(&ring_buffer->not_full, &ring_buffer->mutex, &ts); - if (ret == ETIMEDOUT) { - ring_buffer->overflow_count++; - pthread_mutex_unlock(&ring_buffer->mutex); - return -1; // 超时 - } - } - } - // 只有在有空间时才写入 item - ring_buffer->buffer[ring_buffer->tail] = item; - ring_buffer->tail = (ring_buffer->tail + 1) % ring_buffer->capacity; - ring_buffer->size++; - ring_buffer->enqueue_count++; - // 通知等待的消费者 - pthread_cond_signal(&ring_buffer->not_empty); - pthread_mutex_unlock(&ring_buffer->mutex); - return 0; -} - -int32_t ring_buffer_dequeue(SRingBuffer* ring_buffer, void** item) { - if (!ring_buffer || !item) { - return -1; - } - - pthread_mutex_lock(&ring_buffer->mutex); - - if (ring_buffer->size == 0) { - pthread_mutex_unlock(&ring_buffer->mutex); - return -1; // 队列为空 - } - - *item = ring_buffer->buffer[ring_buffer->head]; - ring_buffer->buffer[ring_buffer->head] = NULL; - ring_buffer->head = (ring_buffer->head + 1) % ring_buffer->capacity; - ring_buffer->size--; - ring_buffer->dequeue_count++; - - // 通知等待的生产者 - pthread_cond_signal(&ring_buffer->not_full); - - pthread_mutex_unlock(&ring_buffer->mutex); - return 0; -} - -int32_t ring_buffer_dequeue_blocking(SRingBuffer* ring_buffer, void** item, uint32_t timeout_ms) { - if (!ring_buffer || !item) { - return -1; - } - - pthread_mutex_lock(&ring_buffer->mutex); - - // 等待队列有数据 - while (ring_buffer->size == 0) { - if (timeout_ms == 0) { - // 无限等待 - pthread_cond_wait(&ring_buffer->not_empty, &ring_buffer->mutex); - } else { - // 有限等待 - struct timespec ts; - clock_gettime(CLOCK_REALTIME, &ts); - ts.tv_nsec += (timeout_ms % 1000) * 1000000; - ts.tv_sec += timeout_ms / 1000 + ts.tv_nsec / 1000000000; - ts.tv_nsec %= 1000000000; - - int ret = pthread_cond_timedwait(&ring_buffer->not_empty, &ring_buffer->mutex, &ts); - if (ret == ETIMEDOUT) { - pthread_mutex_unlock(&ring_buffer->mutex); - return -1; // 超时 - } - } - } - - *item = ring_buffer->buffer[ring_buffer->head]; - ring_buffer->buffer[ring_buffer->head] = NULL; - ring_buffer->head = (ring_buffer->head + 1) % ring_buffer->capacity; - ring_buffer->size--; - ring_buffer->dequeue_count++; - - // 通知等待的生产者 - pthread_cond_signal(&ring_buffer->not_full); - - pthread_mutex_unlock(&ring_buffer->mutex); - return 0; -} - -int32_t ring_buffer_peek(SRingBuffer* ring_buffer, void** item) { - if (!ring_buffer || !item) { - return -1; - } - - pthread_mutex_lock(&ring_buffer->mutex); - - if (ring_buffer->size == 0) { - pthread_mutex_unlock(&ring_buffer->mutex); - return -1; // 队列为空 - } - - *item = ring_buffer->buffer[ring_buffer->head]; - - pthread_mutex_unlock(&ring_buffer->mutex); - return 0; -} - -void ring_buffer_clear(SRingBuffer* ring_buffer, void (*free_func)(void*)) { - if (!ring_buffer) { - return; - } - - pthread_mutex_lock(&ring_buffer->mutex); - - if (free_func) { - uint32_t i = ring_buffer->head; - uint32_t count = ring_buffer->size; - while (count > 0) { - if (ring_buffer->buffer[i]) { - free_func(ring_buffer->buffer[i]); - ring_buffer->buffer[i] = NULL; - } - i = (i + 1) % ring_buffer->capacity; - count--; - } - } - - ring_buffer->size = 0; - ring_buffer->head = 0; - ring_buffer->tail = 0; - - // 通知所有等待的线程 - pthread_cond_broadcast(&ring_buffer->not_full); - - pthread_mutex_unlock(&ring_buffer->mutex); -} - -ERingBufferState ring_buffer_get_state(SRingBuffer* ring_buffer) { - if (!ring_buffer) { - return RING_BUFFER_EMPTY; - } - - pthread_mutex_lock(&ring_buffer->mutex); - ERingBufferState state; - - if (ring_buffer->size == 0) { - state = RING_BUFFER_EMPTY; - } else if (ring_buffer->size >= ring_buffer->capacity) { - state = RING_BUFFER_FULL; - } else { - state = RING_BUFFER_PARTIAL; - } - - pthread_mutex_unlock(&ring_buffer->mutex); - return state; -} - -uint32_t ring_buffer_get_size(SRingBuffer* ring_buffer) { - if (!ring_buffer) { - return 0; - } - - pthread_mutex_lock(&ring_buffer->mutex); - uint32_t size = ring_buffer->size; - pthread_mutex_unlock(&ring_buffer->mutex); - - return size; -} - -uint32_t ring_buffer_get_capacity(SRingBuffer* ring_buffer) { - return ring_buffer ? ring_buffer->capacity : 0; -} - -bool ring_buffer_is_empty(SRingBuffer* ring_buffer) { - return ring_buffer_get_size(ring_buffer) == 0; -} - -bool ring_buffer_is_full(SRingBuffer* ring_buffer) { - if (!ring_buffer) { - return true; - } - - pthread_mutex_lock(&ring_buffer->mutex); - bool is_full = (ring_buffer->size >= ring_buffer->capacity); - pthread_mutex_unlock(&ring_buffer->mutex); - - return is_full; -} - -void ring_buffer_get_stats(SRingBuffer* ring_buffer, uint64_t* enqueue_count, - uint64_t* dequeue_count, uint64_t* overflow_count) { - if (!ring_buffer) { - if (enqueue_count) *enqueue_count = 0; - if (dequeue_count) *dequeue_count = 0; - if (overflow_count) *overflow_count = 0; - return; - } - - pthread_mutex_lock(&ring_buffer->mutex); - - if (enqueue_count) *enqueue_count = ring_buffer->enqueue_count; - if (dequeue_count) *dequeue_count = ring_buffer->dequeue_count; - if (overflow_count) *overflow_count = ring_buffer->overflow_count; - - pthread_mutex_unlock(&ring_buffer->mutex); -} \ No newline at end of file diff --git a/plugins/incremental_bitmap/src/roaring_bitmap.c b/plugins/incremental_bitmap/src/roaring_bitmap.c deleted file mode 100644 index 48cfc729ebbd..000000000000 --- a/plugins/incremental_bitmap/src/roaring_bitmap.c +++ /dev/null @@ -1,325 +0,0 @@ -#include "../include/bitmap_interface.h" -#include -#include -#include -#include -#include -#include -#include - -// RoaringBitmap 适配器实现 -typedef struct SRoaringBitmap { - roaring64_bitmap_t* roaring_bitmap; - pthread_mutex_t mutex; // 添加互斥锁用于线程安全 -} SRoaringBitmap; - -// 内部函数声明 - 使用 tdengine_ 前缀避免与 CRoaring 冲突 -static void tdengine_roaring_add(void* bitmap, uint64_t value); -static void tdengine_roaring_remove(void* bitmap, uint64_t value); -static bool tdengine_roaring_contains(void* bitmap, uint64_t value); -static uint32_t tdengine_roaring_cardinality(void* bitmap); -static void tdengine_roaring_clear(void* bitmap); -static void tdengine_roaring_union_with(void* bitmap, const void* other); -static void tdengine_roaring_intersect_with(void* bitmap, const void* other); -static void tdengine_roaring_subtract(void* bitmap, const void* other); -static uint32_t tdengine_roaring_to_array(void* bitmap, uint64_t* array, uint32_t max_count); -static size_t tdengine_roaring_serialized_size(void* bitmap); -static int32_t tdengine_roaring_serialize(void* bitmap, void* buffer, size_t buffer_size); -static int32_t tdengine_roaring_deserialize(void** bitmap, const void* buffer, size_t buffer_size); -static void tdengine_roaring_destroy(void* bitmap); -static size_t tdengine_roaring_memory_usage(void* bitmap); -static void* tdengine_roaring_create(void); -static void* tdengine_roaring_clone(const void* bitmap); - -// 实现函数 -static void tdengine_roaring_add(void* bitmap, uint64_t value) { - SRoaringBitmap* rb = (SRoaringBitmap*)bitmap; - if (rb && rb->roaring_bitmap) { - pthread_mutex_lock(&rb->mutex); - roaring64_bitmap_add(rb->roaring_bitmap, value); - pthread_mutex_unlock(&rb->mutex); - } -} - -static void tdengine_roaring_remove(void* bitmap, uint64_t value) { - SRoaringBitmap* rb = (SRoaringBitmap*)bitmap; - if (rb && rb->roaring_bitmap) { - pthread_mutex_lock(&rb->mutex); - roaring64_bitmap_remove(rb->roaring_bitmap, value); - pthread_mutex_unlock(&rb->mutex); - } -} - -static bool tdengine_roaring_contains(void* bitmap, uint64_t value) { - SRoaringBitmap* rb = (SRoaringBitmap*)bitmap; - if (rb && rb->roaring_bitmap) { - pthread_mutex_lock(&rb->mutex); - bool result = roaring64_bitmap_contains(rb->roaring_bitmap, value); - pthread_mutex_unlock(&rb->mutex); - return result; - } - return false; -} - -static uint32_t tdengine_roaring_cardinality(void* bitmap) { - SRoaringBitmap* rb = (SRoaringBitmap*)bitmap; - if (rb && rb->roaring_bitmap) { - pthread_mutex_lock(&rb->mutex); - uint32_t result = (uint32_t)roaring64_bitmap_get_cardinality(rb->roaring_bitmap); - pthread_mutex_unlock(&rb->mutex); - return result; - } - return 0; -} - -static void tdengine_roaring_clear(void* bitmap) { - SRoaringBitmap* rb = (SRoaringBitmap*)bitmap; - if (rb && rb->roaring_bitmap) { - pthread_mutex_lock(&rb->mutex); - roaring64_bitmap_clear(rb->roaring_bitmap); - pthread_mutex_unlock(&rb->mutex); - } -} - -static void tdengine_roaring_union_with(void* bitmap, const void* other) { - SRoaringBitmap* rb = (SRoaringBitmap*)bitmap; - SRoaringBitmap* other_rb = (SRoaringBitmap*)other; - - // 严格的参数验证 - if (!rb || !rb->roaring_bitmap || !other_rb || !other_rb->roaring_bitmap) { - printf("DEBUG: tdengine_roaring_union_with: Invalid parameters\n"); - return; - } - - // 验证位图对象的完整性 - if (rb->roaring_bitmap == (void*)0x7d6 || other_rb->roaring_bitmap == (void*)0x7d6) { - printf("DEBUG: tdengine_roaring_union_with: Corrupted bitmap detected\n"); - return; - } - - pthread_mutex_lock(&rb->mutex); - - // 再次验证位图是否仍然有效 - if (!rb->roaring_bitmap || !other_rb->roaring_bitmap) { - printf("DEBUG: tdengine_roaring_union_with: Bitmap became invalid during lock\n"); - pthread_mutex_unlock(&rb->mutex); - return; - } - - // 使用安全的并集操作 - roaring64_bitmap_or_inplace(rb->roaring_bitmap, other_rb->roaring_bitmap); - - pthread_mutex_unlock(&rb->mutex); -} - -static void tdengine_roaring_intersect_with(void* bitmap, const void* other) { - SRoaringBitmap* rb = (SRoaringBitmap*)bitmap; - SRoaringBitmap* other_rb = (SRoaringBitmap*)other; - if (rb && rb->roaring_bitmap && other_rb && other_rb->roaring_bitmap) { - pthread_mutex_lock(&rb->mutex); - roaring64_bitmap_and_inplace(rb->roaring_bitmap, other_rb->roaring_bitmap); - pthread_mutex_unlock(&rb->mutex); - } -} - -static void tdengine_roaring_subtract(void* bitmap, const void* other) { - SRoaringBitmap* rb = (SRoaringBitmap*)bitmap; - SRoaringBitmap* other_rb = (SRoaringBitmap*)other; - if (rb && rb->roaring_bitmap && other_rb && other_rb->roaring_bitmap) { - pthread_mutex_lock(&rb->mutex); - roaring64_bitmap_andnot_inplace(rb->roaring_bitmap, other_rb->roaring_bitmap); - pthread_mutex_unlock(&rb->mutex); - } -} - -static uint32_t tdengine_roaring_to_array(void* bitmap, uint64_t* array, uint32_t max_count) { - SRoaringBitmap* rb = (SRoaringBitmap*)bitmap; - if (!rb || !rb->roaring_bitmap || !array) { - return 0; - } - - pthread_mutex_lock(&rb->mutex); - uint64_t cardinality = roaring64_bitmap_get_cardinality(rb->roaring_bitmap); - uint32_t count = (uint32_t)(cardinality < max_count ? cardinality : max_count); - - if (count > 0) { - roaring64_bitmap_to_uint64_array(rb->roaring_bitmap, array); - } - pthread_mutex_unlock(&rb->mutex); - - return count; -} - -static size_t tdengine_roaring_serialized_size(void* bitmap) { - SRoaringBitmap* rb = (SRoaringBitmap*)bitmap; - if (rb && rb->roaring_bitmap) { - pthread_mutex_lock(&rb->mutex); - size_t size = roaring64_bitmap_portable_size_in_bytes(rb->roaring_bitmap); - pthread_mutex_unlock(&rb->mutex); - return size; - } - return 0; -} - -static int32_t tdengine_roaring_serialize(void* bitmap, void* buffer, size_t buffer_size) { - SRoaringBitmap* rb = (SRoaringBitmap*)bitmap; - if (!rb || !rb->roaring_bitmap || !buffer) { - return -1; - } - - pthread_mutex_lock(&rb->mutex); - size_t serialized_size = roaring64_bitmap_portable_size_in_bytes(rb->roaring_bitmap); - - if (buffer_size < serialized_size) { - pthread_mutex_unlock(&rb->mutex); - return -1; - } - - size_t bytes_written = roaring64_bitmap_portable_serialize(rb->roaring_bitmap, (char*)buffer); - pthread_mutex_unlock(&rb->mutex); - - return (bytes_written == serialized_size) ? 0 : -1; -} - -static int32_t tdengine_roaring_deserialize(void** bitmap, const void* buffer, size_t buffer_size) { - if (!bitmap || !buffer) { - return -1; - } - - SRoaringBitmap* rb = malloc(sizeof(SRoaringBitmap)); - if (!rb) { - return -1; - } - - // 初始化互斥锁 - if (pthread_mutex_init(&rb->mutex, NULL) != 0) { - free(rb); - return -1; - } - - pthread_mutex_lock(&rb->mutex); - rb->roaring_bitmap = roaring64_bitmap_portable_deserialize_safe(buffer, buffer_size); - pthread_mutex_unlock(&rb->mutex); - - if (!rb->roaring_bitmap) { - pthread_mutex_destroy(&rb->mutex); - free(rb); - return -1; - } - - *bitmap = rb; - return 0; -} - -static void tdengine_roaring_destroy(void* bitmap) { - SRoaringBitmap* rb = (SRoaringBitmap*)bitmap; - if (rb) { - if (rb->roaring_bitmap) { - roaring64_bitmap_free(rb->roaring_bitmap); - } - pthread_mutex_destroy(&rb->mutex); - free(rb); - } -} - -static size_t tdengine_roaring_memory_usage(void* bitmap) { - SRoaringBitmap* rb = (SRoaringBitmap*)bitmap; - if (rb && rb->roaring_bitmap) { - pthread_mutex_lock(&rb->mutex); - roaring64_statistics_t stats; - roaring64_bitmap_statistics(rb->roaring_bitmap, &stats); - size_t memory_usage = stats.n_bytes_array_containers + - stats.n_bytes_run_containers + - stats.n_bytes_bitset_containers; - pthread_mutex_unlock(&rb->mutex); - return memory_usage; - } - return 0; -} - -static void* tdengine_roaring_create(void) { - SRoaringBitmap* rb = malloc(sizeof(SRoaringBitmap)); - if (!rb) { - return NULL; - } - - // 初始化互斥锁 - if (pthread_mutex_init(&rb->mutex, NULL) != 0) { - free(rb); - return NULL; - } - - rb->roaring_bitmap = roaring64_bitmap_create(); - if (!rb->roaring_bitmap) { - pthread_mutex_destroy(&rb->mutex); - free(rb); - return NULL; - } - - return rb; -} - -static void* tdengine_roaring_clone(const void* bitmap) { - SRoaringBitmap* src_rb = (SRoaringBitmap*)bitmap; - if (!src_rb || !src_rb->roaring_bitmap) { - return NULL; - } - - SRoaringBitmap* dst_rb = malloc(sizeof(SRoaringBitmap)); - if (!dst_rb) { - return NULL; - } - - // 初始化互斥锁 - if (pthread_mutex_init(&dst_rb->mutex, NULL) != 0) { - free(dst_rb); - return NULL; - } - - pthread_mutex_lock(&src_rb->mutex); - dst_rb->roaring_bitmap = roaring64_bitmap_copy(src_rb->roaring_bitmap); - pthread_mutex_unlock(&src_rb->mutex); - - if (!dst_rb->roaring_bitmap) { - pthread_mutex_destroy(&dst_rb->mutex); - free(dst_rb); - return NULL; - } - - return dst_rb; -} - -// 工厂函数 -SBitmapInterface* roaring_bitmap_interface_create(void) { - SBitmapInterface* interface = malloc(sizeof(SBitmapInterface)); - if (!interface) { - return NULL; - } - - // 创建位图实例 - interface->bitmap = tdengine_roaring_create(); - if (!interface->bitmap) { - free(interface); - return NULL; - } - - // 设置函数指针 - interface->add = tdengine_roaring_add; - interface->remove = tdengine_roaring_remove; - interface->contains = tdengine_roaring_contains; - interface->cardinality = tdengine_roaring_cardinality; - interface->clear = tdengine_roaring_clear; - interface->union_with = tdengine_roaring_union_with; - interface->intersect_with = tdengine_roaring_intersect_with; - interface->subtract = tdengine_roaring_subtract; - interface->to_array = tdengine_roaring_to_array; - interface->serialized_size = tdengine_roaring_serialized_size; - interface->serialize = tdengine_roaring_serialize; - interface->deserialize = tdengine_roaring_deserialize; - interface->destroy = tdengine_roaring_destroy; - interface->memory_usage = tdengine_roaring_memory_usage; - interface->create = tdengine_roaring_create; - interface->clone = tdengine_roaring_clone; - - return interface; -} \ No newline at end of file diff --git a/plugins/incremental_bitmap/src/simple_bitmap.c b/plugins/incremental_bitmap/src/simple_bitmap.c deleted file mode 100644 index b7222b98a82b..000000000000 --- a/plugins/incremental_bitmap/src/simple_bitmap.c +++ /dev/null @@ -1,428 +0,0 @@ -#include "../include/bitmap_interface.h" -#include -#include -#include -#include -#include - -// 简单位图实现（用于开发阶段） -typedef struct SSimpleBitmap { - uint64_t* values; // 存储的值数组 - uint32_t count; // 当前元素数量 - uint32_t capacity; // 数组容量 - bool sorted; // 是否已排序 -} SSimpleBitmap; - -// 内部函数声明 -static void simple_bitmap_add(void* bitmap, uint64_t value); -static void simple_bitmap_remove(void* bitmap, uint64_t value); -static bool simple_bitmap_contains(void* bitmap, uint64_t value); -static uint32_t simple_bitmap_cardinality(void* bitmap); -static void simple_bitmap_clear(void* bitmap); -static void simple_bitmap_union_with(void* bitmap, const void* other); -static void simple_bitmap_intersect_with(void* bitmap, const void* other); -static void simple_bitmap_subtract(void* bitmap, const void* other); -static uint32_t simple_bitmap_to_array(void* bitmap, uint64_t* array, uint32_t max_count); -static size_t simple_bitmap_serialized_size(void* bitmap); -static int32_t simple_bitmap_serialize(void* bitmap, void* buffer, size_t buffer_size); -static int32_t simple_bitmap_deserialize(void** bitmap, const void* buffer, size_t buffer_size); -static void simple_bitmap_destroy(void* bitmap); -static size_t simple_bitmap_memory_usage(void* bitmap); -static void* simple_bitmap_create(void); -static void* simple_bitmap_clone(const void* bitmap); - -// 辅助函数：二分查找 -static int32_t binary_search(const uint64_t* array, uint32_t count, uint64_t value) { - int32_t left = 0; - int32_t right = count - 1; - - while (left <= right) { - int32_t mid = left + (right - left) / 2; - if (array[mid] == value) { - return mid; - } else if (array[mid] < value) { - left = mid + 1; - } else { - right = mid - 1; - } - } - return -1; -} - -// 辅助函数：插入排序 -static void insert_sort(uint64_t* array, uint32_t count) { - for (uint32_t i = 1; i < count; i++) { - uint64_t key = array[i]; - int32_t j = i - 1; - - while (j >= 0 && array[j] > key) { - array[j + 1] = array[j]; - j--; - } - array[j + 1] = key; - } -} - -// 辅助函数：确保容量足够 -static bool ensure_capacity(SSimpleBitmap* bitmap, uint32_t min_capacity) { - if (bitmap->capacity >= min_capacity) { - return true; - } - - uint32_t new_capacity = bitmap->capacity * 2; - if (new_capacity < min_capacity) { - new_capacity = min_capacity; - } - - // 添加安全检查 - if (bitmap->values == NULL) { - // printf("DEBUG: ensure_capacity: bitmap->values is NULL\n"); - return false; - } - - uint64_t* new_values = realloc(bitmap->values, new_capacity * sizeof(uint64_t)); - if (!new_values) { - // printf("DEBUG: ensure_capacity: realloc failed\n"); - return false; - } - - // 检查realloc是否移动了内存块 - if (new_values != bitmap->values) { - // printf("DEBUG: ensure_capacity: memory moved from %p to %p\n", bitmap->values, new_values); - } - - bitmap->values = new_values; - bitmap->capacity = new_capacity; - - // 添加验证 - if (bitmap->values == NULL) { - // printf("DEBUG: ensure_capacity: bitmap->values became NULL after update\n"); - return false; - } - - return true; -} - -// 基本操作实现 -static void simple_bitmap_add(void* bitmap, uint64_t value) { - SSimpleBitmap* sb = (SSimpleBitmap*)bitmap; - - // 添加安全检查 - if (sb == NULL) { - // printf("DEBUG: simple_bitmap_add: sb is NULL\n"); - return; - } - - if (sb->values == NULL) { - // printf("DEBUG: simple_bitmap_add: sb->values is NULL\n"); - return; - } - - // 检查是否已存在 - if (simple_bitmap_contains(bitmap, value)) { - return; - } - - // 确保容量足够 - if (!ensure_capacity(sb, sb->count + 1)) { - // printf("DEBUG: simple_bitmap_add: ensure_capacity failed\n"); - return; - } - - // 再次检查values指针 - if (sb->values == NULL) { - // printf("DEBUG: simple_bitmap_add: sb->values became NULL after ensure_capacity\n"); - return; - } - - // 添加到数组末尾 - sb->values[sb->count++] = value; - sb->sorted = false; -} - -static void simple_bitmap_remove(void* bitmap, uint64_t value) { - SSimpleBitmap* sb = (SSimpleBitmap*)bitmap; - - // 如果未排序，先排序 - if (!sb->sorted) { - insert_sort(sb->values, sb->count); - sb->sorted = true; - } - - // 查找并删除 - int32_t index = binary_search(sb->values, sb->count, value); - if (index >= 0) { - // 移动后面的元素 - for (uint32_t i = index; i < sb->count - 1; i++) { - sb->values[i] = sb->values[i + 1]; - } - sb->count--; - } -} - -static bool simple_bitmap_contains(void* bitmap, uint64_t value) { - SSimpleBitmap* sb = (SSimpleBitmap*)bitmap; - - // 如果未排序，先排序 - if (!sb->sorted) { - insert_sort(sb->values, sb->count); - sb->sorted = true; - } - - return binary_search(sb->values, sb->count, value) >= 0; -} - -static uint32_t simple_bitmap_cardinality(void* bitmap) { - SSimpleBitmap* sb = (SSimpleBitmap*)bitmap; - return sb->count; -} - -static void simple_bitmap_clear(void* bitmap) { - SSimpleBitmap* sb = (SSimpleBitmap*)bitmap; - sb->count = 0; - sb->sorted = true; -} - -// 集合操作实现 -static void simple_bitmap_union_with(void* bitmap, const void* other) { - SSimpleBitmap* sb = (SSimpleBitmap*)bitmap; - const SSimpleBitmap* other_sb = (const SSimpleBitmap*)other; - - for (uint32_t i = 0; i < other_sb->count; i++) { - simple_bitmap_add(bitmap, other_sb->values[i]); - } -} - -static void simple_bitmap_intersect_with(void* bitmap, const void* other) { - SSimpleBitmap* sb = (SSimpleBitmap*)bitmap; - const SSimpleBitmap* other_sb = (const SSimpleBitmap*)other; - - // 创建临时数组存储交集结果 - uint64_t* temp = malloc(sb->count * sizeof(uint64_t)); - if (!temp) return; - - uint32_t temp_count = 0; - for (uint32_t i = 0; i < sb->count; i++) { - if (simple_bitmap_contains((void*)other, sb->values[i])) { - temp[temp_count++] = sb->values[i]; - } - } - - // 更新原数组 - memcpy(sb->values, temp, temp_count * sizeof(uint64_t)); - sb->count = temp_count; - sb->sorted = false; - - free(temp); -} - -static void simple_bitmap_subtract(void* bitmap, const void* other) { - SSimpleBitmap* sb = (SSimpleBitmap*)bitmap; - const SSimpleBitmap* other_sb = (const SSimpleBitmap*)other; - - for (uint32_t i = 0; i < other_sb->count; i++) { - simple_bitmap_remove(bitmap, other_sb->values[i]); - } -} - -// 迭代操作实现 -static uint32_t simple_bitmap_to_array(void* bitmap, uint64_t* array, uint32_t max_count) { - SSimpleBitmap* sb = (SSimpleBitmap*)bitmap; - - uint32_t copy_count = (sb->count < max_count) ? sb->count : max_count; - memcpy(array, sb->values, copy_count * sizeof(uint64_t)); - - return copy_count; -} - -// 序列化操作实现 -static size_t simple_bitmap_serialized_size(void* bitmap) { - SSimpleBitmap* sb = (SSimpleBitmap*)bitmap; - return sizeof(uint32_t) + sb->count * sizeof(uint64_t); -} - -static int32_t simple_bitmap_serialize(void* bitmap, void* buffer, size_t buffer_size) { - SSimpleBitmap* sb = (SSimpleBitmap*)bitmap; - size_t required_size = simple_bitmap_serialized_size(bitmap); - - if (buffer_size < required_size) { - return -1; - } - - // 写入元素数量 - memcpy(buffer, &sb->count, sizeof(uint32_t)); - - // 写入元素数组 - if (sb->count > 0) { - memcpy((char*)buffer + sizeof(uint32_t), sb->values, sb->count * sizeof(uint64_t)); - } - - return 0; -} - -static int32_t simple_bitmap_deserialize(void** bitmap, const void* buffer, size_t buffer_size) { - if (buffer_size < sizeof(uint32_t)) { - return -1; - } - - // 读取元素数量 - uint32_t count; - memcpy(&count, buffer, sizeof(uint32_t)); - - size_t expected_size = sizeof(uint32_t) + count * sizeof(uint64_t); - if (buffer_size < expected_size) { - return -1; - } - - // 创建新的位图 - SSimpleBitmap* sb = simple_bitmap_create(); - if (!sb) { - return -1; - } - - // 确保容量足够 - if (!ensure_capacity(sb, count)) { - simple_bitmap_destroy(sb); - return -1; - } - - // 读取元素数组 - if (count > 0) { - memcpy(sb->values, (char*)buffer + sizeof(uint32_t), count * sizeof(uint64_t)); - } - - sb->count = count; - sb->sorted = false; - - *bitmap = sb; - return 0; -} - -// 内存管理实现 -static void simple_bitmap_destroy(void* bitmap) { - SSimpleBitmap* sb = (SSimpleBitmap*)bitmap; - if (sb) { - free(sb->values); - free(sb); - } -} - -static size_t simple_bitmap_memory_usage(void* bitmap) { - SSimpleBitmap* sb = (SSimpleBitmap*)bitmap; - return sizeof(SSimpleBitmap) + sb->capacity * sizeof(uint64_t); -} - -static void* simple_bitmap_create(void) { - SSimpleBitmap* sb = malloc(sizeof(SSimpleBitmap)); - if (!sb) { - return NULL; - } - - sb->values = malloc(16 * sizeof(uint64_t)); // 初始容量16 - if (!sb->values) { - free(sb); - return NULL; - } - - sb->count = 0; - sb->capacity = 16; - sb->sorted = true; - - return sb; -} - -static void* simple_bitmap_clone(const void* bitmap) { - const SSimpleBitmap* src = (const SSimpleBitmap*)bitmap; - SSimpleBitmap* dst = simple_bitmap_create(); - if (!dst) { - return NULL; - } - - // 确保容量足够 - if (!ensure_capacity(dst, src->count)) { - simple_bitmap_destroy(dst); - return NULL; - } - - // 复制数据 - memcpy(dst->values, src->values, src->count * sizeof(uint64_t)); - dst->count = src->count; - dst->sorted = src->sorted; - - return dst; -} - -// 创建位图接口 -SBitmapInterface* bitmap_interface_create(void) { - // 检查是否强制使用简单位图实现 - const char* use_simple = getenv("TDENGINE_USE_SIMPLE_BITMAP"); - if (use_simple && (strcmp(use_simple, "1") == 0 || strcmp(use_simple, "true") == 0)) { - // 强制使用简单位图实现 - SBitmapInterface* interface = malloc(sizeof(SBitmapInterface)); - if (!interface) { - return NULL; - } - - // 创建位图实例 - interface->bitmap = simple_bitmap_create(); - if (!interface->bitmap) { - free(interface); - return NULL; - } - - // 设置函数指针 - interface->add = simple_bitmap_add; - interface->remove = simple_bitmap_remove; - interface->contains = simple_bitmap_contains; - interface->cardinality = simple_bitmap_cardinality; - interface->clear = simple_bitmap_clear; - interface->union_with = simple_bitmap_union_with; - interface->intersect_with = simple_bitmap_intersect_with; - interface->subtract = simple_bitmap_subtract; - interface->to_array = simple_bitmap_to_array; - interface->serialized_size = simple_bitmap_serialized_size; - interface->serialize = simple_bitmap_serialize; - interface->deserialize = simple_bitmap_deserialize; - interface->destroy = simple_bitmap_destroy; - interface->memory_usage = simple_bitmap_memory_usage; - interface->create = simple_bitmap_create; - interface->clone = simple_bitmap_clone; - - return interface; - } - - // 默认使用 RoaringBitmap 实现 - extern SBitmapInterface* roaring_bitmap_interface_create(void); - return roaring_bitmap_interface_create(); -} - -void bitmap_interface_destroy(SBitmapInterface* interface) { - if (!interface) { - return; - } - - // printf("DEBUG: bitmap_interface_destroy: interface=%p, interface->bitmap=%p\n", - // interface, interface->bitmap); - - // 验证位图对象的完整性 - if (interface->bitmap == (void*)0x7d6) { - // printf("DEBUG: bitmap_interface_destroy: Corrupted bitmap detected, skipping destruction\n"); - // 清理接口结构体，但不销毁损坏的位图 - free(interface); - return; - } - - if (interface->bitmap) { - // 验证位图对象是否仍然有效 - if (interface->bitmap < (void*)0x1000) { - // printf("DEBUG: bitmap_interface_destroy: Invalid bitmap address, skipping destruction\n"); - free(interface); - return; - } - - // 安全地销毁位图 - interface->destroy(interface->bitmap); - } - - free(interface); -} \ No newline at end of file diff --git a/plugins/incremental_bitmap/src/skiplist.c b/plugins/incremental_bitmap/src/skiplist.c deleted file mode 100644 index 161a29e7e045..000000000000 --- a/plugins/incremental_bitmap/src/skiplist.c +++ /dev/null @@ -1,210 +0,0 @@ -#include "../include/skiplist.h" -#include -#include -#include -#include - -static int random_level() { - int level = 1; - while (((double)rand() / RAND_MAX) < SKIPLIST_P && level < SKIPLIST_MAX_LEVEL) - level++; - return level; -} - -// 内存池分配/回收 -static skiplist_node_t* node_pool_alloc(skiplist_t* sl) { - if (sl->free_nodes) { - skiplist_node_t* node = sl->free_nodes; - sl->free_nodes = node->forward[0]; - sl->free_count--; - return node; - } - return (skiplist_node_t*)calloc(1, sizeof(skiplist_node_t)); -} - -static void node_pool_free(skiplist_t* sl, skiplist_node_t* node) { - node->forward[0] = sl->free_nodes; - sl->free_nodes = node; - sl->free_count++; -} - -void skiplist_node_pool_clear(skiplist_t* sl) { - skiplist_node_t* node = sl->free_nodes; - while (node) { - skiplist_node_t* next = node->forward[0]; - free(node); - node = next; - } - sl->free_nodes = NULL; - sl->free_count = 0; -} - -skiplist_t* skiplist_create() { - skiplist_t* sl = (skiplist_t*)calloc(1, sizeof(skiplist_t)); - sl->level = 1; - sl->size = 0; - sl->header = (skiplist_node_t*)calloc(1, sizeof(skiplist_node_t)); - sl->tail = NULL; - pthread_rwlock_init(&sl->rwlock, NULL); - sl->free_nodes = NULL; - sl->free_count = 0; - return sl; -} - -void skiplist_destroy(skiplist_t* sl) { - if (!sl) return; - skiplist_node_t* node = sl->header->forward[0]; - while (node) { - skiplist_node_t* next = node->forward[0]; - // 注意：这里不释放value，由调用者负责 - free(node); - node = next; - } - free(sl->header); - skiplist_node_pool_clear(sl); - pthread_rwlock_destroy(&sl->rwlock); - free(sl); -} - -void skiplist_insert(skiplist_t* sl, uint64_t key, void* value) { - pthread_rwlock_wrlock(&sl->rwlock); - skiplist_node_t* update[SKIPLIST_MAX_LEVEL]; - skiplist_node_t* x = sl->header; - for (int i = sl->level - 1; i >= 0; i--) { - while (x->forward[i] && x->forward[i]->key < key) { - x = x->forward[i]; - } - update[i] = x; - } - x = x->forward[0]; - if (x && x->key == key) { - // 注意：这里不释放旧value，由调用者负责 - x->value = value; - pthread_rwlock_unlock(&sl->rwlock); - return; - } - int lvl = random_level(); - if (lvl > sl->level) { - for (int i = sl->level; i < lvl; i++) { - update[i] = sl->header; - } - sl->level = lvl; - } - x = node_pool_alloc(sl); - x->key = key; - x->value = value; - x->level = lvl; - for (int i = 0; i < lvl; i++) { - x->forward[i] = update[i]->forward[i]; - update[i]->forward[i] = x; - } - // backward指针 - x->backward = (update[0] == sl->header) ? NULL : update[0]; - if (x->forward[0]) x->forward[0]->backward = x; - if (!x->forward[0]) sl->tail = x; - sl->size++; - pthread_rwlock_unlock(&sl->rwlock); -} - -void* skiplist_find(skiplist_t* sl, uint64_t key) { - pthread_rwlock_rdlock(&sl->rwlock); - skiplist_node_t* x = sl->header; - for (int i = sl->level - 1; i >= 0; i--) { - while (x->forward[i] && x->forward[i]->key < key) { - x = x->forward[i]; - } - } - x = x->forward[0]; - void* result = NULL; - if (x && x->key == key) { - result = x->value; - } - pthread_rwlock_unlock(&sl->rwlock); - return result; -} - -void skiplist_remove(skiplist_t* sl, uint64_t key) { - pthread_rwlock_wrlock(&sl->rwlock); - skiplist_node_t* update[SKIPLIST_MAX_LEVEL]; - skiplist_node_t* x = sl->header; - for (int i = sl->level - 1; i >= 0; i--) { - while (x->forward[i] && x->forward[i]->key < key) { - x = x->forward[i]; - } - update[i] = x; - } - x = x->forward[0]; - if (x && x->key == key) { - for (int i = 0; i < sl->level; i++) { - if (update[i]->forward[i] != x) break; - update[i]->forward[i] = x->forward[i]; - } - if (x->forward[0]) x->forward[0]->backward = x->backward; - if (sl->tail == x) sl->tail = x->backward; - // 注意：这里不释放value，由调用者负责 - node_pool_free(sl, x); - while (sl->level > 1 && sl->header->forward[sl->level - 1] == NULL) { - sl->level--; - } - sl->size--; - } - pthread_rwlock_unlock(&sl->rwlock); -} - -void skiplist_range_query(skiplist_t* sl, uint64_t start, uint64_t end, bool reverse, skiplist_range_cb cb, void* user_data) { - pthread_rwlock_rdlock(&sl->rwlock); - if (!reverse) { - skiplist_node_t* x = sl->header; - for (int i = sl->level - 1; i >= 0; i--) { - while (x->forward[i] && x->forward[i]->key < start) { - x = x->forward[i]; - } - } - x = x->forward[0]; - while (x && x->key <= end) { - cb(x->key, x->value, user_data); - x = x->forward[0]; - } - } else { - skiplist_node_t* x = sl->tail; - while (x && x->key >= start) { - if (x->key <= end) cb(x->key, x->value, user_data); - if (x->key == start) break; - x = x->backward; - } - } - pthread_rwlock_unlock(&sl->rwlock); -} - -void skiplist_rdlock(skiplist_t* sl) { - pthread_rwlock_rdlock(&sl->rwlock); -} -void skiplist_wrlock(skiplist_t* sl) { - pthread_rwlock_wrlock(&sl->rwlock); -} -void skiplist_unlock(skiplist_t* sl) { - pthread_rwlock_unlock(&sl->rwlock); -} - -// 内存使用统计 -uint64_t skiplist_get_memory_usage(skiplist_t* sl) { - if (!sl) return 0; - uint64_t total_bytes = 0; - total_bytes += sizeof(skiplist_t); - total_bytes += sizeof(skiplist_node_t); - skiplist_node_t* node = sl->header->forward[0]; - while (node) { - total_bytes += sizeof(skiplist_node_t); - // 位图内存大小（抽象接口） - if (node->value && sl->value_mem_usage) { - total_bytes += sl->value_mem_usage(node->value); - } - node = node->forward[0]; - } - skiplist_node_t* free_node = sl->free_nodes; - while (free_node) { - total_bytes += sizeof(skiplist_node_t); - free_node = free_node->forward[0]; - } - return total_bytes; -} \ No newline at end of file diff --git a/plugins/incremental_bitmap/src/storage_engine_interface.c b/plugins/incremental_bitmap/src/storage_engine_interface.c deleted file mode 100644 index 4d622b331a72..000000000000 --- a/plugins/incremental_bitmap/src/storage_engine_interface.c +++ /dev/null @@ -1,197 +0,0 @@ -#include "storage_engine_interface.h" -#include -#include -#include - -// 存储引擎接口注册表 -typedef struct { - char* name; - StorageEngineInterfaceFactory factory; -} SStorageEngineRegistry; - -// 全局注册表 -static SStorageEngineRegistry* g_registry = NULL; -static uint32_t g_registry_size = 0; -static uint32_t g_registry_capacity = 0; -static pthread_mutex_t g_registry_mutex = PTHREAD_MUTEX_INITIALIZER; - -// 默认存储引擎接口（空实现） -static int32_t default_init(const SStorageEngineConfig* config) { - (void)config; // 避免未使用参数警告 - return 0; -} - -static void default_destroy(void) { - // 空实现 -} - -static int32_t default_install_interception(void) { - return 0; // 成功但不做任何事 -} - -static int32_t default_uninstall_interception(void) { - return 0; // 成功但不做任何事 -} - -static int32_t default_trigger_event(const SStorageEvent* event) { - (void)event; // 避免未使用参数警告 - return 0; -} - -static int32_t default_get_stats(uint64_t* events_processed, uint64_t* events_dropped) { - if (events_processed) *events_processed = 0; - if (events_dropped) *events_dropped = 0; - return 0; -} - -static bool default_is_supported(void) { - return false; // 默认不支持 -} - -static const char* default_get_engine_name(void) { - return "default"; -} - -// 默认存储引擎接口 -static SStorageEngineInterface g_default_interface = { - .init = default_init, - .destroy = default_destroy, - .install_interception = default_install_interception, - .uninstall_interception = default_uninstall_interception, - .trigger_event = default_trigger_event, - .get_stats = default_get_stats, - .is_supported = default_is_supported, - .get_engine_name = default_get_engine_name -}; - -int32_t register_storage_engine_interface(const char* name, StorageEngineInterfaceFactory factory) { - if (!name || !factory) { - return -1; - } - - pthread_mutex_lock(&g_registry_mutex); - - // 检查是否已存在 - for (uint32_t i = 0; i < g_registry_size; i++) { - if (strcmp(g_registry[i].name, name) == 0) { - // 更新现有注册 - g_registry[i].factory = factory; - pthread_mutex_unlock(&g_registry_mutex); - return 0; - } - } - - // 扩展注册表 - if (g_registry_size >= g_registry_capacity) { - uint32_t new_capacity = g_registry_capacity == 0 ? 4 : g_registry_capacity * 2; - SStorageEngineRegistry* new_registry = realloc(g_registry, - new_capacity * sizeof(SStorageEngineRegistry)); - if (!new_registry) { - pthread_mutex_unlock(&g_registry_mutex); - return -1; - } - g_registry = new_registry; - g_registry_capacity = new_capacity; - } - - // 添加新注册 - g_registry[g_registry_size].name = strdup(name); - g_registry[g_registry_size].factory = factory; - g_registry_size++; - - pthread_mutex_unlock(&g_registry_mutex); - return 0; -} - -// 内部: 按名称从注册表获取（不存在则返回NULL） -static SStorageEngineInterface* get_by_name_or_null(const char* name) { - if (!name) return NULL; - pthread_mutex_lock(&g_registry_mutex); - for (uint32_t i = 0; i < g_registry_size; i++) { - if (strcmp(g_registry[i].name, name) == 0) { - SStorageEngineInterface* interface = g_registry[i].factory(); - pthread_mutex_unlock(&g_registry_mutex); - return interface; - } - } - pthread_mutex_unlock(&g_registry_mutex); - return NULL; -} - -// 自动选择逻辑： -// 1) 若环境变量 USE_MOCK=1 或 STORAGE_ENGINE=mock，则返回 mock -// 2) 否则优先 tdengine_tmq，若未注册则回退 mock -// 3) 再回退 default -static SStorageEngineInterface* get_auto_selected_interface(void) { - const char* use_mock = getenv("USE_MOCK"); - const char* engine_env = getenv("STORAGE_ENGINE"); - if ((use_mock && strcmp(use_mock, "1") == 0) || (engine_env && strcmp(engine_env, "mock") == 0)) { - SStorageEngineInterface* mock = get_by_name_or_null("mock"); - return mock ? mock : get_default_storage_engine_interface(); - } - // 优先 TMQ - SStorageEngineInterface* tmq = get_by_name_or_null("tdengine_tmq"); - if (tmq) return tmq; - // 回退 mock - SStorageEngineInterface* mock = get_by_name_or_null("mock"); - if (mock) return mock; - return get_default_storage_engine_interface(); -} - -SStorageEngineInterface* get_storage_engine_interface(const char* name) { - if (!name || strcmp(name, "auto") == 0) { - return get_auto_selected_interface(); - } - - SStorageEngineInterface* found = get_by_name_or_null(name); - if (found) return found; - return get_auto_selected_interface(); -} - -SStorageEngineInterface* get_default_storage_engine_interface(void) { - return &g_default_interface; -} - -int32_t list_storage_engine_interfaces(char** names, uint32_t max_count, uint32_t* actual_count) { - if (!names || !actual_count) { - return -1; - } - - pthread_mutex_lock(&g_registry_mutex); - - uint32_t count = g_registry_size < max_count ? g_registry_size : max_count; - - for (uint32_t i = 0; i < count; i++) { - names[i] = strdup(g_registry[i].name); - if (!names[i]) { - // 清理已分配的内存 - for (uint32_t j = 0; j < i; j++) { - free(names[j]); - } - pthread_mutex_unlock(&g_registry_mutex); - return -1; - } - } - - *actual_count = count; - pthread_mutex_unlock(&g_registry_mutex); - return 0; -} - -// 清理函数（用于程序退出时） -void cleanup_storage_engine_registry(void) { - pthread_mutex_lock(&g_registry_mutex); - - if (g_registry) { - for (uint32_t i = 0; i < g_registry_size; i++) { - free(g_registry[i].name); - } - free(g_registry); - g_registry = NULL; - g_registry_size = 0; - g_registry_capacity = 0; - } - - pthread_mutex_unlock(&g_registry_mutex); -} - diff --git a/plugins/incremental_bitmap/src/tdengine_storage_engine.c b/plugins/incremental_bitmap/src/tdengine_storage_engine.c deleted file mode 100644 index 916898e9e614..000000000000 --- a/plugins/incremental_bitmap/src/tdengine_storage_engine.c +++ /dev/null @@ -1,1064 +0,0 @@ -#include "storage_engine_interface.h" -#include "event_interceptor.h" -#include "bitmap_engine.h" -#include "ring_buffer.h" -#include "skiplist.h" -#include -#include -#include -#include -#include -#include -#include - -// TMQ 头文件 - 需要链接 TDengine 客户端库 -#include - -// 本地时间戳工具，避免依赖 TDengine 的时间精度宏 -static inline int64_t now_monotonic_ns(void) { - struct timespec ts; - clock_gettime(CLOCK_REALTIME, &ts); - return (int64_t)ts.tv_sec * 1000000000LL + (int64_t)ts.tv_nsec; -} - -static inline int64_t now_monotonic_ms(void) { - struct timespec ts; - clock_gettime(CLOCK_REALTIME, &ts); - return (int64_t)ts.tv_sec * 1000LL + (int64_t)(ts.tv_nsec / 1000000LL); -} - -// 前向声明，避免隐式声明导致的编译错误 -static int32_t tdengine_uninstall_interception(void); - -// TMQ 配置和状态管理 -typedef struct { - // TMQ 连接相关 - tmq_t* consumer; - tmq_conf_t* config; - char* topic_name; - char* group_id; - - // 消费者线程 - pthread_t consumer_thread; - pthread_mutex_t thread_mutex; - bool thread_running; - bool thread_should_stop; - - // 配置参数 - char* server_ip; - int32_t server_port; - char* username; - char* password; - char* database; - - // Offset 提交策略 - bool auto_commit; - bool at_least_once; // true=至少一次, false=至多一次 - int64_t commit_interval_ms; - - // 统计信息 - uint64_t events_processed; - uint64_t events_dropped; - uint64_t messages_consumed; - uint64_t offset_commits; - int64_t last_commit_time; - - // 可观测性指标 - SObservabilityMetrics observability_metrics; - int64_t start_time; // 启动时间 - int64_t last_metrics_update; // 最后指标更新时间 - uint64_t last_events_count; // 上次事件计数 - uint64_t last_messages_count; // 上次消息计数 - uint64_t last_bytes_count; // 上次字节计数 - - // 重试统计 - uint64_t connection_retries; - uint64_t subscription_retries; - uint64_t commit_retries; - - // 错误统计 - uint64_t parse_errors; - - // 事件处理相关 - int64_t last_event_timestamp; // 最后事件时间戳 - SEventInterceptor* event_interceptor; // 事件拦截器 - SBitmapEngine* bitmap_engine; // 位图引擎 - - // 事件回调 - StorageEventCallback event_callback; - void* callback_user_data; - - // 互斥锁保护 - pthread_mutex_t mutex; - pthread_rwlock_t config_rwlock; - - // 状态标志 - bool initialized; - bool interception_installed; -} STmqContext; - -static STmqContext g_tmq_context = { - .consumer = NULL, - .config = NULL, - .topic_name = NULL, - .group_id = NULL, - .consumer_thread = 0, - .thread_mutex = PTHREAD_MUTEX_INITIALIZER, - .thread_running = false, - .thread_should_stop = false, - .server_ip = NULL, - .server_port = 0, - .username = NULL, - .password = NULL, - .database = NULL, - .auto_commit = false, - .at_least_once = true, - .commit_interval_ms = 0, - .events_processed = 0, - .events_dropped = 0, - .messages_consumed = 0, - .offset_commits = 0, - .last_commit_time = 0, - .observability_metrics = {0}, - .start_time = 0, - .last_metrics_update = 0, - .last_events_count = 0, - .last_messages_count = 0, - .last_bytes_count = 0, - .connection_retries = 0, - .subscription_retries = 0, - .commit_retries = 0, - .parse_errors = 0, - .last_event_timestamp = 0, - .event_interceptor = NULL, - .bitmap_engine = NULL, - .event_callback = NULL, - .callback_user_data = NULL, - .mutex = PTHREAD_MUTEX_INITIALIZER, - .config_rwlock = PTHREAD_RWLOCK_INITIALIZER, - .initialized = false, - .interception_installed = false -}; - -// 默认配置值 -#define DEFAULT_SERVER_IP "localhost" -#define DEFAULT_SERVER_PORT 6030 -#define DEFAULT_USERNAME "root" -#define DEFAULT_PASSWORD "taosdata" -#define DEFAULT_DATABASE "test_bitmap_db" -#define DEFAULT_TOPIC_NAME "incremental_backup_topic" -#define DEFAULT_GROUP_ID "incremental_backup_group" -#define DEFAULT_COMMIT_INTERVAL_MS 1000 -#define DEFAULT_POLL_TIMEOUT_MS 100 -#define DEFAULT_CONNECT_TIMEOUT_MS 5000 - -// 重试控制 -#define TMQ_CREATE_RETRY_MAX 5 -#define TMQ_SUBSCRIBE_RETRY_MAX 5 - -// TMQ 配置初始化 -static int32_t init_tmq_config(void) { - g_tmq_context.config = tmq_conf_new(); - if (!g_tmq_context.config) { - printf("[TMQ] 创建配置失败\n"); - return -1; - } - - // 允许通过环境变量覆盖基础连接参数（提升部署兼容性） - // TD_CONNECT_IP, TD_CONNECT_PORT, TD_USERNAME, TD_PASSWORD, TD_DATABASE, TD_TOPIC_NAME, TD_GROUP_ID - const char* env_ip = getenv("TD_CONNECT_IP"); - if (env_ip && *env_ip) { - snprintf(g_tmq_context.server_ip, sizeof(g_tmq_context.server_ip), "%s", env_ip); - } - const char* env_port = getenv("TD_CONNECT_PORT"); - if (env_port && *env_port) { - int p = atoi(env_port); - if (p > 0 && p < 65536) g_tmq_context.server_port = p; - } - const char* env_user = getenv("TD_USERNAME"); - if (env_user && *env_user) { - snprintf(g_tmq_context.username, sizeof(g_tmq_context.username), "%s", env_user); - } - const char* env_pass = getenv("TD_PASSWORD"); - if (env_pass && *env_pass) { - snprintf(g_tmq_context.password, sizeof(g_tmq_context.password), "%s", env_pass); - } - const char* env_db = getenv("TD_DATABASE"); - if (env_db && *env_db) { - snprintf(g_tmq_context.database, sizeof(g_tmq_context.database), "%s", env_db); - } - const char* env_topic = getenv("TD_TOPIC_NAME"); - if (env_topic && *env_topic) { - snprintf(g_tmq_context.topic_name, sizeof(g_tmq_context.topic_name), "%s", env_topic); - } - const char* env_group = getenv("TD_GROUP_ID"); - if (env_group && *env_group) { - snprintf(g_tmq_context.group_id, sizeof(g_tmq_context.group_id), "%s", env_group); - } - - // 设置基本连接参数 - if (tmq_conf_set(g_tmq_context.config, "td.connect.ip", g_tmq_context.server_ip) != TMQ_CONF_OK) { - printf("[TMQ] 设置服务器IP失败\n"); - return -1; - } - - char port_str[16]; - snprintf(port_str, sizeof(port_str), "%d", g_tmq_context.server_port); - if (tmq_conf_set(g_tmq_context.config, "td.connect.port", port_str) != TMQ_CONF_OK) { - printf("[TMQ] 设置服务器端口失败\n"); - return -1; - } - - if (tmq_conf_set(g_tmq_context.config, "td.connect.user", g_tmq_context.username) != TMQ_CONF_OK) { - printf("[TMQ] 设置用户名失败\n"); - return -1; - } - - if (tmq_conf_set(g_tmq_context.config, "td.connect.pass", g_tmq_context.password) != TMQ_CONF_OK) { - printf("[TMQ] 设置密码失败\n"); - return -1; - } - - // database 在不同版本中可能无此键；尝试多种键名，失败则仅警告不致命 - if (g_tmq_context.database && strlen(g_tmq_context.database) > 0) { - int db_set_ok = 0; - if (tmq_conf_set(g_tmq_context.config, "td.connect.database", g_tmq_context.database) == TMQ_CONF_OK) { - db_set_ok = 1; - } else if (tmq_conf_set(g_tmq_context.config, "td.connect.db", g_tmq_context.database) == TMQ_CONF_OK) { - db_set_ok = 1; - } - if (!db_set_ok) { - printf("[TMQ] 警告: 数据库配置键不被支持，跳过设置 (value=%s)\n", g_tmq_context.database); - } - } - - // 设置消费者组参数 - if (tmq_conf_set(g_tmq_context.config, "group.id", g_tmq_context.group_id) != TMQ_CONF_OK) { - printf("[TMQ] 设置消费者组ID失败\n"); - return -1; - } - - // 设置 offset 提交策略 - if (g_tmq_context.auto_commit) { - if (tmq_conf_set(g_tmq_context.config, "enable.auto.commit", "true") != TMQ_CONF_OK) { - printf("[TMQ] 启用自动提交失败\n"); - return -1; - } - } else { - if (tmq_conf_set(g_tmq_context.config, "enable.auto.commit", "false") != TMQ_CONF_OK) { - printf("[TMQ] 禁用自动提交失败\n"); - return -1; - } - } - - // 设置 offset 重置策略 - const char* offset_reset = g_tmq_context.at_least_once ? "earliest" : "latest"; - if (tmq_conf_set(g_tmq_context.config, "auto.offset.reset", offset_reset) != TMQ_CONF_OK) { - printf("[TMQ] 设置offset重置策略失败\n"); - return -1; - } - - // 连接与请求超时（尝试多种可能的配置键名），支持环境变量覆盖键名 - char timeout_buf[16]; - snprintf(timeout_buf, sizeof(timeout_buf), "%d", DEFAULT_CONNECT_TIMEOUT_MS); - - const char* key_conn_timeout = getenv("TMQ_KEY_CONNECT_TIMEOUT"); - int timeout_set_ok = 0; - if (key_conn_timeout && *key_conn_timeout) { - // 显式指定键名 - if (tmq_conf_set(g_tmq_context.config, key_conn_timeout, timeout_buf) == TMQ_CONF_OK) { - timeout_set_ok = 1; - } - } else { - // 尝试常见键名集合 - if (tmq_conf_set(g_tmq_context.config, "connect.timeout", timeout_buf) == TMQ_CONF_OK) { - timeout_set_ok = 1; - } else if (tmq_conf_set(g_tmq_context.config, "td.connect.timeout", timeout_buf) == TMQ_CONF_OK) { - timeout_set_ok = 1; - } else if (tmq_conf_set(g_tmq_context.config, "connection.timeout", timeout_buf) == TMQ_CONF_OK) { - timeout_set_ok = 1; - } - } - if (!timeout_set_ok) { - printf("[TMQ] 警告: 未能设置连接超时配置 (尝试键=%s, 值=%s)\n", key_conn_timeout ? key_conn_timeout : "auto", timeout_buf); - } - - // 请求超时配置 - const char* key_req_timeout = getenv("TMQ_KEY_REQUEST_TIMEOUT"); - int req_ok = 0; - if (key_req_timeout && *key_req_timeout) { - if (tmq_conf_set(g_tmq_context.config, key_req_timeout, timeout_buf) == TMQ_CONF_OK) req_ok = 1; - } else { - if (tmq_conf_set(g_tmq_context.config, "request.timeout.ms", timeout_buf) == TMQ_CONF_OK) { - req_ok = 1; - } else if (tmq_conf_set(g_tmq_context.config, "td.request.timeout", timeout_buf) == TMQ_CONF_OK) { - req_ok = 1; - } else if (tmq_conf_set(g_tmq_context.config, "request.timeout", timeout_buf) == TMQ_CONF_OK) { - req_ok = 1; - } - } - if (!req_ok) { - // 非致命，仅记录 - printf("[TMQ] 警告: 未能设置请求超时配置 (尝试键=%s, 值=%s)\n", key_req_timeout ? key_req_timeout : "auto", timeout_buf); - } - // 设定客户端ID（便于排查） - if (tmq_conf_set(g_tmq_context.config, "client.id", "backup-consumer-1") != TMQ_CONF_OK) { - // 非致命 - } - - printf("[TMQ] 配置初始化成功\n"); - return 0; -} - -// TMQ 消费者创建和连接 -static int32_t create_tmq_consumer(void) { - char errstr[512]; - - // 带重试创建消费者 - int attempt = 0; - while (attempt < TMQ_CREATE_RETRY_MAX) { - g_tmq_context.consumer = tmq_consumer_new(g_tmq_context.config, errstr, sizeof(errstr)); - if (g_tmq_context.consumer) break; - printf("[TMQ] 创建消费者失败(%d/%d): %s\n", attempt + 1, TMQ_CREATE_RETRY_MAX, errstr); - int backoff_ms = 200 * (1 << attempt); - struct timespec ts = { .tv_sec = backoff_ms / 1000, .tv_nsec = (backoff_ms % 1000) * 1000000L }; - nanosleep(&ts, NULL); - attempt++; - } - if (!g_tmq_context.consumer) { - printf("[TMQ] 创建消费者失败: %s\n", errstr); - return -1; - } - - // 创建主题列表 - tmq_list_t* topic_list = tmq_list_new(); - if (!topic_list) { - printf("[TMQ] 创建主题列表失败\n"); - return -1; - } - if (tmq_list_append(topic_list, g_tmq_context.topic_name) != 0) { - printf("[TMQ] 添加主题到列表失败\n"); - tmq_list_destroy(topic_list); - return -1; - } - - // 带重试订阅 - attempt = 0; - int sub_ok = 0; - while (attempt < TMQ_SUBSCRIBE_RETRY_MAX) { - int sub_result = tmq_subscribe(g_tmq_context.consumer, topic_list); - if (sub_result == 0) { - sub_ok = 1; - break; - } - printf("[TMQ] 订阅主题失败(%d/%d): 错误码=%d, 主题=%s\n", - attempt + 1, TMQ_SUBSCRIBE_RETRY_MAX, sub_result, g_tmq_context.topic_name); - int backoff_ms = 200 * (1 << attempt); - struct timespec ts = { .tv_sec = backoff_ms / 1000, .tv_nsec = (backoff_ms % 1000) * 1000000L }; - nanosleep(&ts, NULL); - attempt++; - } - tmq_list_destroy(topic_list); - if (!sub_ok) { - printf("[TMQ] 订阅主题失败\n"); - return -1; - } - - printf("[TMQ] 消费者创建和订阅成功\n"); - return 0; -} - -// TMQ 消息解析为存储事件 -static SStorageEvent* parse_tmq_message(TAOS_RES* msg) { - if (!msg) { - return NULL; - } - - SStorageEvent* event = malloc(sizeof(SStorageEvent)); - if (!event) { - printf("[TMQ] 分配事件内存失败\n"); - return NULL; - } - - // 获取消息类型 - tmq_res_t res_type = tmq_get_res_type(msg); - - // 根据消息类型映射事件类型 - switch (res_type) { - case TMQ_RES_DATA: - event->event_type = STORAGE_EVENT_BLOCK_UPDATE; - break; - case TMQ_RES_TABLE_META: - event->event_type = STORAGE_EVENT_BLOCK_CREATE; - break; - case TMQ_RES_METADATA: - event->event_type = STORAGE_EVENT_BLOCK_FLUSH; - break; - case TMQ_RES_RAWDATA: - event->event_type = STORAGE_EVENT_BLOCK_UPDATE; - break; - default: - event->event_type = STORAGE_EVENT_BLOCK_UPDATE; - break; - } - - // 获取块ID（从表名或其他标识符生成） - const char* table_name = tmq_get_table_name(msg); - if (table_name) { - // 简单的哈希算法生成块ID - uint64_t hash = 0; - for (int i = 0; table_name[i]; i++) { - hash = hash * 31 + table_name[i]; - } - event->block_id = hash; - } else { - event->block_id = 0; - } - - // 获取 WAL offset - event->wal_offset = tmq_get_vgroup_offset(msg); - - // 获取时间戳 - int64_t current_time = now_monotonic_ns(); - event->timestamp = current_time; - - event->user_data = NULL; - - return event; -} - -// Offset 提交策略实现 -static int32_t commit_offset(const char* topic_name, int32_t vg_id, int64_t offset, bool sync) { - if (!g_tmq_context.consumer) { - return -1; - } - - int32_t result; - if (sync) { - result = tmq_commit_offset_sync(g_tmq_context.consumer, topic_name, vg_id, offset); - } else { - // 异步提交 - tmq_commit_offset_async(g_tmq_context.consumer, topic_name, vg_id, offset, NULL, NULL); - result = 0; // 异步提交总是返回成功 - } - - if (result == 0) { - pthread_mutex_lock(&g_tmq_context.mutex); - g_tmq_context.offset_commits++; - g_tmq_context.last_commit_time = now_monotonic_ms(); - pthread_mutex_unlock(&g_tmq_context.mutex); - - if (sync) { - printf("[TMQ] Offset 同步提交成功: topic=%s, vg_id=%d, offset=%ld\n", - topic_name, vg_id, offset); - } else { - printf("[TMQ] Offset 异步提交成功: topic=%s, vg_id=%d, offset=%ld\n", - topic_name, vg_id, offset); - } - } else { - printf("[TMQ] Offset 提交失败: topic=%s, vg_id=%d, offset=%ld, error=%d\n", - topic_name, vg_id, offset, result); - } - - return result; -} - -// TMQ 消费线程主函数 -static void* tmq_consumer_thread(void* arg) { - (void)arg; // 避免未使用参数警告 - - printf("[TMQ] 消费线程启动\n"); - - while (g_tmq_context.thread_running && !g_tmq_context.thread_should_stop) { - // 轮询消息 - TAOS_RES* msg = tmq_consumer_poll(g_tmq_context.consumer, DEFAULT_POLL_TIMEOUT_MS); - - if (!msg) { - // 超时，继续轮询 - continue; - } - - // 解析消息为存储事件 - SStorageEvent* event = parse_tmq_message(msg); - if (event) { - // 调用事件回调 - if (g_tmq_context.event_callback) { - g_tmq_context.event_callback(event, g_tmq_context.callback_user_data); - } - - // 更新统计信息 - pthread_mutex_lock(&g_tmq_context.mutex); - g_tmq_context.events_processed++; - g_tmq_context.messages_consumed++; - pthread_mutex_unlock(&g_tmq_context.mutex); - - // 根据策略提交 offset - if (!g_tmq_context.auto_commit) { - const char* topic_name = tmq_get_topic_name(msg); - int32_t vg_id = tmq_get_vgroup_id(msg); - int64_t offset = tmq_get_vgroup_offset(msg); - - if (g_tmq_context.at_least_once) { - // 至少一次：同步提交 - commit_offset(topic_name, vg_id, offset, true); - } else { - // 至多一次：异步提交 - commit_offset(topic_name, vg_id, offset, false); - } - } - - free(event); - } else { - pthread_mutex_lock(&g_tmq_context.mutex); - g_tmq_context.events_dropped++; - pthread_mutex_unlock(&g_tmq_context.mutex); - printf("[TMQ] 消息解析失败\n"); - } - - // 释放消息资源 - taos_free_result(msg); - - // 检查是否需要定期提交 offset - if (g_tmq_context.auto_commit && g_tmq_context.commit_interval_ms > 0) { - int64_t current_time = now_monotonic_ms(); - if (current_time - g_tmq_context.last_commit_time >= g_tmq_context.commit_interval_ms) { - // 这里可以实现批量提交逻辑 - g_tmq_context.last_commit_time = current_time; - } - } - } - - printf("[TMQ] 消费线程退出\n"); - return NULL; -} - -// TDengine存储引擎实现函数 -static int32_t tdengine_init(const SStorageEngineConfig* config) { - if (!config) { - return -1; - } - - pthread_mutex_lock(&g_tmq_context.mutex); - - // 保存回调函数 - g_tmq_context.event_callback = config->event_callback; - g_tmq_context.callback_user_data = config->callback_user_data; - - // 初始化默认配置 - g_tmq_context.server_ip = strdup(DEFAULT_SERVER_IP); - g_tmq_context.server_port = DEFAULT_SERVER_PORT; - g_tmq_context.username = strdup(DEFAULT_USERNAME); - g_tmq_context.password = strdup(DEFAULT_PASSWORD); - g_tmq_context.database = strdup(DEFAULT_DATABASE); - g_tmq_context.topic_name = strdup(DEFAULT_TOPIC_NAME); - g_tmq_context.group_id = strdup(DEFAULT_GROUP_ID); - - // 从环境变量或配置中获取自定义设置 - const char* env_topic = getenv("TMQ_TOPIC_NAME"); - const char* env_group = getenv("TMQ_GROUP_ID"); - const char* env_server = getenv("TMQ_SERVER_ADDR"); - - if (env_topic && strlen(env_topic) > 0) { - free(g_tmq_context.topic_name); - g_tmq_context.topic_name = strdup(env_topic); - printf("[TMQ] 使用环境变量主题: %s\n", g_tmq_context.topic_name); - } - - if (env_group && strlen(env_group) > 0) { - free(g_tmq_context.group_id); - g_tmq_context.group_id = strdup(env_group); - printf("[TMQ] 使用环境变量组ID: %s\n", g_tmq_context.group_id); - } - - if (env_server && strlen(env_server) > 0) { - // 解析服务器地址 (格式: ip:port) - char* colon = strchr(env_server, ':'); - if (colon) { - *colon = '\0'; - free(g_tmq_context.server_ip); - g_tmq_context.server_ip = strdup(env_server); - g_tmq_context.server_port = atoi(colon + 1); - printf("[TMQ] 使用环境变量服务器: %s:%d\n", g_tmq_context.server_ip, g_tmq_context.server_port); - } - } - - // 设置 offset 提交策略 - g_tmq_context.auto_commit = false; // 默认手动提交 - g_tmq_context.at_least_once = true; // 默认至少一次 - g_tmq_context.commit_interval_ms = DEFAULT_COMMIT_INTERVAL_MS; - - // 初始化统计信息 - g_tmq_context.events_processed = 0; - g_tmq_context.events_dropped = 0; - g_tmq_context.messages_consumed = 0; - g_tmq_context.offset_commits = 0; - g_tmq_context.last_commit_time = 0; - - // 初始化线程状态 - g_tmq_context.thread_running = false; - g_tmq_context.thread_should_stop = false; - - g_tmq_context.initialized = true; - g_tmq_context.start_time = now_monotonic_ms(); - g_tmq_context.last_metrics_update = g_tmq_context.start_time; - pthread_mutex_unlock(&g_tmq_context.mutex); - - printf("[TMQ] TDengine存储引擎初始化成功\n"); - return 0; -} - -static void tdengine_destroy(void) { - // 先停止消费线程 - if (g_tmq_context.thread_running) { - tdengine_uninstall_interception(); - } - - pthread_mutex_lock(&g_tmq_context.mutex); - - // 清理 TMQ 资源 - if (g_tmq_context.consumer) { - tmq_consumer_close(g_tmq_context.consumer); - g_tmq_context.consumer = NULL; - } - - if (g_tmq_context.config) { - tmq_conf_destroy(g_tmq_context.config); - g_tmq_context.config = NULL; - } - - // 释放字符串资源 - if (g_tmq_context.server_ip) { - free(g_tmq_context.server_ip); - g_tmq_context.server_ip = NULL; - } - if (g_tmq_context.username) { - free(g_tmq_context.username); - g_tmq_context.username = NULL; - } - if (g_tmq_context.password) { - free(g_tmq_context.password); - g_tmq_context.password = NULL; - } - if (g_tmq_context.database) { - free(g_tmq_context.database); - g_tmq_context.database = NULL; - } - if (g_tmq_context.topic_name) { - free(g_tmq_context.topic_name); - g_tmq_context.topic_name = NULL; - } - if (g_tmq_context.group_id) { - free(g_tmq_context.group_id); - g_tmq_context.group_id = NULL; - } - - g_tmq_context.initialized = false; - pthread_mutex_unlock(&g_tmq_context.mutex); - - printf("[TMQ] TDengine存储引擎销毁完成\n"); -} - -static int32_t tdengine_install_interception(void) { - pthread_mutex_lock(&g_tmq_context.mutex); - if (!g_tmq_context.initialized) { - pthread_mutex_unlock(&g_tmq_context.mutex); - return -1; - } - - // 初始化 TMQ 配置 - if (init_tmq_config() != 0) { - pthread_mutex_unlock(&g_tmq_context.mutex); - return -1; - } - - // 创建 TMQ 消费者 - if (create_tmq_consumer() != 0) { - pthread_mutex_unlock(&g_tmq_context.mutex); - return -1; - } - - // 启动消费线程 - g_tmq_context.thread_running = true; - g_tmq_context.thread_should_stop = false; - - if (pthread_create(&g_tmq_context.consumer_thread, NULL, tmq_consumer_thread, NULL) != 0) { - printf("[TMQ] 创建消费线程失败\n"); - g_tmq_context.thread_running = false; - pthread_mutex_unlock(&g_tmq_context.mutex); - return -1; - } - - g_tmq_context.interception_installed = true; - pthread_mutex_unlock(&g_tmq_context.mutex); - - printf("[TMQ] 事件拦截安装成功，消费线程已启动\n"); - return 0; -} - -static int32_t tdengine_uninstall_interception(void) { - pthread_mutex_lock(&g_tmq_context.mutex); - - if (g_tmq_context.thread_running) { - // 通知线程停止 - g_tmq_context.thread_should_stop = true; - pthread_mutex_unlock(&g_tmq_context.mutex); - - // 等待线程退出 - if (pthread_join(g_tmq_context.consumer_thread, NULL) != 0) { - printf("[TMQ] 等待消费线程退出失败\n"); - } - - pthread_mutex_lock(&g_tmq_context.mutex); - g_tmq_context.thread_running = false; - } - - g_tmq_context.interception_installed = false; - pthread_mutex_unlock(&g_tmq_context.mutex); - - printf("[TMQ] 事件拦截卸载成功\n"); - return 0; -} - -static int32_t tdengine_trigger_event(const SStorageEvent* event) { - if (!event) { - return -1; - } - - pthread_mutex_lock(&g_tmq_context.mutex); - if (!g_tmq_context.interception_installed) { - pthread_mutex_unlock(&g_tmq_context.mutex); - return -1; - } - - g_tmq_context.events_processed++; - pthread_mutex_unlock(&g_tmq_context.mutex); - - printf("[TMQ] 触发事件: 类型=%d, 块ID=%lu, WAL偏移量=%lu, 时间戳=%ld\n", - event->event_type, event->block_id, event->wal_offset, event->timestamp); - - // 调用回调函数 - if (g_tmq_context.event_callback) { - g_tmq_context.event_callback(event, g_tmq_context.callback_user_data); - } - - return 0; -} - -static int32_t tdengine_get_stats(uint64_t* events_processed, uint64_t* events_dropped) { - pthread_mutex_lock(&g_tmq_context.mutex); - - if (events_processed) { - *events_processed = g_tmq_context.events_processed; - } - if (events_dropped) { - *events_dropped = g_tmq_context.events_dropped; - } - - pthread_mutex_unlock(&g_tmq_context.mutex); - return 0; -} - -static bool tdengine_is_supported(void) { - // 检查 TDengine 客户端库是否可用 - // 这里可以添加更详细的检查逻辑 - return true; -} - -static const char* tdengine_get_engine_name(void) { - return "tdengine_tmq"; -} - -// 前向声明 -static int32_t tdengine_get_observability_metrics(void* engine, SObservabilityMetrics* metrics); -static int32_t tdengine_reset_stats(void* engine); - -// TDengine存储引擎接口 -static SStorageEngineInterface g_tdengine_interface = { - .init = tdengine_init, - .destroy = tdengine_destroy, - .install_interception = tdengine_install_interception, - .uninstall_interception = tdengine_uninstall_interception, - .trigger_event = tdengine_trigger_event, - .get_stats = tdengine_get_stats, - .is_supported = tdengine_is_supported, - .get_engine_name = tdengine_get_engine_name, - .get_observability_metrics = tdengine_get_observability_metrics, - .reset_stats = tdengine_reset_stats -}; - -// TDengine存储引擎工厂函数 -SStorageEngineInterface* tdengine_storage_engine_create(void) { - return &g_tdengine_interface; -} - -// 便捷函数：注册TDengine存储引擎 -int32_t register_tdengine_storage_engine(void) { - extern int32_t register_storage_engine_interface(const char* name, StorageEngineInterfaceFactory factory); - return register_storage_engine_interface("tdengine_tmq", tdengine_storage_engine_create); -} - -// 配置 TMQ 参数的高级接口 -int32_t tdengine_set_tmq_config(const char* server_ip, int32_t server_port, - const char* username, const char* password, - const char* database, const char* topic_name, - const char* group_id) { - if (!g_tmq_context.initialized) { - return -1; - } - - pthread_rwlock_wrlock(&g_tmq_context.config_rwlock); - - if (server_ip) { - free(g_tmq_context.server_ip); - g_tmq_context.server_ip = strdup(server_ip); - } - if (server_port > 0) { - g_tmq_context.server_port = server_port; - } - if (username) { - free(g_tmq_context.username); - g_tmq_context.username = strdup(username); - } - if (password) { - free(g_tmq_context.password); - g_tmq_context.password = strdup(password); - } - if (database) { - free(g_tmq_context.database); - g_tmq_context.database = strdup(database); - } - if (topic_name) { - free(g_tmq_context.topic_name); - g_tmq_context.topic_name = strdup(topic_name); - } - if (group_id) { - free(g_tmq_context.group_id); - g_tmq_context.group_id = strdup(group_id); - } - - pthread_rwlock_unlock(&g_tmq_context.config_rwlock); - - printf("[TMQ] 配置更新成功\n"); - return 0; -} - -// 设置 offset 提交策略 -int32_t tdengine_set_commit_strategy(bool auto_commit, bool at_least_once, int64_t commit_interval_ms) { - if (!g_tmq_context.initialized) { - return -1; -} - - pthread_mutex_lock(&g_tmq_context.mutex); - g_tmq_context.auto_commit = auto_commit; - g_tmq_context.at_least_once = at_least_once; - g_tmq_context.commit_interval_ms = commit_interval_ms; - pthread_mutex_unlock(&g_tmq_context.mutex); - - printf("[TMQ] 提交策略设置: auto_commit=%s, at_least_once=%s, interval=%ldms\n", - auto_commit ? "true" : "false", at_least_once ? "true" : "false", commit_interval_ms); - return 0; -} - -// 获取详细的统计信息 -int32_t tdengine_get_detailed_stats(uint64_t* events_processed, uint64_t* events_dropped, - uint64_t* messages_consumed, uint64_t* offset_commits, - int64_t* last_commit_time) { - // 未初始化时返回错误，符合错误处理测试预期 - pthread_mutex_lock(&g_tmq_context.mutex); - if (!g_tmq_context.initialized) { - pthread_mutex_unlock(&g_tmq_context.mutex); - return -1; - } - - if (events_processed) *events_processed = g_tmq_context.events_processed; - if (events_dropped) *events_dropped = g_tmq_context.events_dropped; - if (messages_consumed) *messages_consumed = g_tmq_context.messages_consumed; - if (offset_commits) *offset_commits = g_tmq_context.offset_commits; - if (last_commit_time) *last_commit_time = g_tmq_context.last_commit_time; - - pthread_mutex_unlock(&g_tmq_context.mutex); - return 0; -} - -// 可观测性指标收集和更新函数 -static void update_observability_metrics_internal(void) { - if (!g_tmq_context.initialized) { - return; - } - - pthread_mutex_lock(&g_tmq_context.mutex); - - int64_t current_time = now_monotonic_ms(); - int64_t time_diff_ms = current_time - g_tmq_context.last_metrics_update; - - if (time_diff_ms > 0) { - // 计算速率指标 - uint64_t events_diff = g_tmq_context.events_processed - g_tmq_context.last_events_count; - uint64_t messages_diff = g_tmq_context.messages_consumed - g_tmq_context.last_messages_count; - uint64_t bytes_diff = 0; // TODO: 实现字节计数跟踪 - - g_tmq_context.observability_metrics.events_per_second = - (events_diff * 1000) / time_diff_ms; - g_tmq_context.observability_metrics.messages_per_second = - (messages_diff * 1000) / time_diff_ms; - g_tmq_context.observability_metrics.bytes_per_second = - (bytes_diff * 1000) / time_diff_ms; - - // 更新计数 - g_tmq_context.last_events_count = g_tmq_context.events_processed; - g_tmq_context.last_messages_count = g_tmq_context.messages_consumed; - g_tmq_context.last_metrics_update = current_time; - } - - // 更新丢弃指标 - g_tmq_context.observability_metrics.events_dropped = g_tmq_context.events_dropped; - g_tmq_context.observability_metrics.messages_dropped = g_tmq_context.events_dropped; - g_tmq_context.observability_metrics.parse_errors = g_tmq_context.parse_errors; - - // 更新重试指标 - g_tmq_context.observability_metrics.connection_retries = g_tmq_context.connection_retries; - g_tmq_context.observability_metrics.subscription_retries = g_tmq_context.subscription_retries; - g_tmq_context.observability_metrics.commit_retries = g_tmq_context.commit_retries; - - // 更新队列水位 - 从事件拦截器获取真实数据 - if (g_tmq_context.event_interceptor && g_tmq_context.event_interceptor->event_buffer) { - SRingBuffer* ring_buffer = (SRingBuffer*)g_tmq_context.event_interceptor->event_buffer; - uint32_t current_size = ring_buffer_get_size(ring_buffer); - uint32_t capacity = ring_buffer_get_capacity(ring_buffer); - - g_tmq_context.observability_metrics.ring_buffer_usage = - capacity > 0 ? (current_size * 100) / capacity : 0; - g_tmq_context.observability_metrics.ring_buffer_capacity = capacity; - g_tmq_context.observability_metrics.event_queue_size = current_size; - } else { - g_tmq_context.observability_metrics.ring_buffer_usage = 0; - g_tmq_context.observability_metrics.ring_buffer_capacity = 0; - g_tmq_context.observability_metrics.event_queue_size = 0; - } - - // 更新内存指标 - 从位图引擎获取真实数据 - size_t total_memory = sizeof(STmqContext); - size_t bitmap_memory = 0; - size_t metadata_memory = 0; - - // 计算位图引擎内存使用 - if (g_tmq_context.bitmap_engine) { - // 计算位图内存使用 - if (g_tmq_context.bitmap_engine->dirty_blocks) { - bitmap_memory += g_tmq_context.bitmap_engine->dirty_blocks->memory_usage( - g_tmq_context.bitmap_engine->dirty_blocks->bitmap); - } - if (g_tmq_context.bitmap_engine->new_blocks) { - bitmap_memory += g_tmq_context.bitmap_engine->new_blocks->memory_usage( - g_tmq_context.bitmap_engine->new_blocks->bitmap); - } - if (g_tmq_context.bitmap_engine->deleted_blocks) { - bitmap_memory += g_tmq_context.bitmap_engine->deleted_blocks->memory_usage( - g_tmq_context.bitmap_engine->deleted_blocks->bitmap); - } - - // 计算元数据内存使用 - metadata_memory = g_tmq_context.bitmap_engine->metadata_map_size * sizeof(SBlockMetadataNode*); - metadata_memory += g_tmq_context.bitmap_engine->metadata_count * sizeof(SBlockMetadataNode); - - // 计算跳表索引内存使用 - if (g_tmq_context.bitmap_engine->time_index) { - metadata_memory += skiplist_get_memory_usage(g_tmq_context.bitmap_engine->time_index); - } - if (g_tmq_context.bitmap_engine->wal_index) { - metadata_memory += skiplist_get_memory_usage(g_tmq_context.bitmap_engine->wal_index); - } - } - - total_memory += bitmap_memory + metadata_memory; - total_memory += g_tmq_context.events_processed * sizeof(SStorageEvent); - - g_tmq_context.observability_metrics.memory_usage_bytes = total_memory; - g_tmq_context.observability_metrics.bitmap_memory_bytes = bitmap_memory; - g_tmq_context.observability_metrics.metadata_memory_bytes = metadata_memory; - - // 更新时间戳 - g_tmq_context.observability_metrics.last_update_time = current_time; - g_tmq_context.observability_metrics.uptime_seconds = - (current_time - g_tmq_context.start_time) / 1000; - - // 计算滞后指标 - 基于事件处理延迟估算 - if (g_tmq_context.events_processed > 0) { - // 估算消费者滞后时间（基于事件处理速率） - int64_t avg_processing_time = time_diff_ms > 0 ? - (g_tmq_context.events_processed * 1000) / time_diff_ms : 0; - g_tmq_context.observability_metrics.consumer_lag_ms = avg_processing_time; - - // 估算Offset滞后数量（基于队列大小） - g_tmq_context.observability_metrics.offset_lag = - g_tmq_context.observability_metrics.event_queue_size; - - // 计算处理延迟（基于事件时间戳） - if (g_tmq_context.last_event_timestamp > 0) { - g_tmq_context.observability_metrics.processing_delay_ms = - current_time - g_tmq_context.last_event_timestamp; - } else { - g_tmq_context.observability_metrics.processing_delay_ms = 0; - } - } else { - g_tmq_context.observability_metrics.consumer_lag_ms = 0; - g_tmq_context.observability_metrics.offset_lag = 0; - g_tmq_context.observability_metrics.processing_delay_ms = 0; - } - - pthread_mutex_unlock(&g_tmq_context.mutex); -} - -// 获取可观测性指标 -static int32_t tdengine_get_observability_metrics(void* engine, SObservabilityMetrics* metrics) { - if (!g_tmq_context.initialized || !metrics) { - return -1; - } - - // 更新指标 - update_observability_metrics_internal(); - - // 复制指标数据 - pthread_mutex_lock(&g_tmq_context.mutex); - memcpy(metrics, &g_tmq_context.observability_metrics, sizeof(SObservabilityMetrics)); - pthread_mutex_unlock(&g_tmq_context.mutex); - - return 0; -} - -// 重置统计信息 -static int32_t tdengine_reset_stats(void* engine) { - if (!g_tmq_context.initialized) { - return -1; - } - - pthread_mutex_lock(&g_tmq_context.mutex); - - // 重置基本统计 - g_tmq_context.events_processed = 0; - g_tmq_context.events_dropped = 0; - g_tmq_context.messages_consumed = 0; - g_tmq_context.offset_commits = 0; - - // 重置重试统计 - g_tmq_context.connection_retries = 0; - g_tmq_context.subscription_retries = 0; - g_tmq_context.commit_retries = 0; - - // 重置错误统计 - g_tmq_context.parse_errors = 0; - - // 重置可观测性指标 - memset(&g_tmq_context.observability_metrics, 0, sizeof(SObservabilityMetrics)); - - // 重置计数 - g_tmq_context.last_events_count = 0; - g_tmq_context.last_messages_count = 0; - g_tmq_context.last_bytes_count = 0; - g_tmq_context.last_metrics_update = now_monotonic_ms(); - - pthread_mutex_unlock(&g_tmq_context.mutex); - - return 0; -} - diff --git a/plugins/incremental_bitmap/taosx_plugin/CMakeLists.txt b/plugins/incremental_bitmap/taosx_plugin/CMakeLists.txt deleted file mode 100644 index aae5f6bf7287..000000000000 --- a/plugins/incremental_bitmap/taosx_plugin/CMakeLists.txt +++ /dev/null @@ -1,66 +0,0 @@ -cmake_minimum_required(VERSION 3.16) - -# taosX Plugin Interface - Independent build target -# Default disabled, enabled via BUILD_TAOSX_PLUGIN option - -# Use the option defined in parent CMakeLists.txt -if(BUILD_TAOSX_PLUGIN) - message(STATUS "Building taosX plugin interface") - - # Set C standard - set(CMAKE_C_STANDARD 11) - - # Plugin source files - set(TAOSX_PLUGIN_SOURCES - taosx_plugin_interface.c - ) - - # Create shared library for taosX plugin - add_library(taosx_incremental_bitmap_plugin SHARED ${TAOSX_PLUGIN_SOURCES}) - - # Set output properties - set_target_properties(taosx_incremental_bitmap_plugin PROPERTIES - LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../build - RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../build - VERSION 0.1.0 - SOVERSION 0 - ) - - # Include directories - target_include_directories(taosx_incremental_bitmap_plugin - PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} - ) - - # Link libraries (minimal dependencies for taosX compatibility) - target_link_libraries(taosx_incremental_bitmap_plugin - PRIVATE pthread - ) - - # Compiler flags for plugin compatibility - target_compile_definitions(taosx_incremental_bitmap_plugin - PRIVATE TAOSX_PLUGIN_BUILD=1 - ) - - # Installation rules - install(TARGETS taosx_incremental_bitmap_plugin - LIBRARY DESTINATION lib/taosx/plugins - RUNTIME DESTINATION bin - ) - - # Create a simple test executable - add_executable(test_taosx_plugin_interface - ${CMAKE_CURRENT_SOURCE_DIR}/test_taosx_plugin.c - ) - - target_link_libraries(test_taosx_plugin_interface - PRIVATE taosx_incremental_bitmap_plugin - ) - - # Set test output directory - set_target_properties(test_taosx_plugin_interface PROPERTIES - RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../build - ) - -else() - message(STATUS "taosX plugin interface disabled (use -DBUILD_TAOSX_PLUGIN=ON to enable)") -endif() diff --git a/plugins/incremental_bitmap/taosx_plugin/taosx_plugin_interface.c b/plugins/incremental_bitmap/taosx_plugin/taosx_plugin_interface.c deleted file mode 100644 index d587dbd6a693..000000000000 --- a/plugins/incremental_bitmap/taosx_plugin/taosx_plugin_interface.c +++ /dev/null @@ -1,93 +0,0 @@ -#include "taosx_plugin_interface.h" -#include - -static volatile int g_initialized = 0; -static volatile int g_running = 0; - -static struct { - uint64_t start_time_epoch_s; - uint64_t events_processed; - uint64_t events_dropped; - uint64_t error_count; - size_t memory_usage_bytes; - uint32_t ring_buffer_usage_percent; -} g_state; - -// Minimal helpers (no OS deps beyond standard C) -static uint64_t fake_now_epoch_seconds(void) { - return 0; // Tech preview: keep deterministic in open env -} - -const char *taosx_plugin_get_name(void) { - return "incremental-bitmap-plugin"; -} - -const char *taosx_plugin_get_version(void) { - return "0.1.0-tech-preview"; -} - -const char *taosx_plugin_get_capabilities(void) { - return "bitmap-index,events-abi"; -} - -int taosx_plugin_init(void) { - if (g_initialized) return TAOSX_PLUGIN_ERR_ALREADY_RUNNING; - memset((void*)&g_state, 0, sizeof(g_state)); - g_state.start_time_epoch_s = fake_now_epoch_seconds(); - g_initialized = 1; - return TAOSX_PLUGIN_OK; -} - -int taosx_plugin_shutdown(void) { - if (!g_initialized) return TAOSX_PLUGIN_ERR_NOT_INITIALIZED; - g_running = 0; - g_initialized = 0; - return TAOSX_PLUGIN_OK; -} - -int taosx_plugin_configure(const TaosX_Config *config) { - if (!g_initialized) return TAOSX_PLUGIN_ERR_NOT_INITIALIZED; - if (!config && config != NULL) return TAOSX_PLUGIN_ERR_INVALID_ARG; - // Accept all key-values; no-op for tech preview. - (void)config; - return TAOSX_PLUGIN_OK; -} - -int taosx_plugin_start(void) { - if (!g_initialized) return TAOSX_PLUGIN_ERR_NOT_INITIALIZED; - if (g_running) return TAOSX_PLUGIN_ERR_ALREADY_RUNNING; - g_running = 1; - return TAOSX_PLUGIN_OK; -} - -int taosx_plugin_stop(void) { - if (!g_initialized) return TAOSX_PLUGIN_ERR_NOT_INITIALIZED; - g_running = 0; - return TAOSX_PLUGIN_OK; -} - -int taosx_plugin_on_block_event(const TaosX_BlockEvent *event) { - if (!g_initialized) return TAOSX_PLUGIN_ERR_NOT_INITIALIZED; - if (!g_running) return TAOSX_PLUGIN_ERR_INTERNAL; - if (event == NULL) return TAOSX_PLUGIN_ERR_INVALID_ARG; - // Count as processed; drop rules not implemented in tech preview. - (void)event; - g_state.events_processed += 1; - return TAOSX_PLUGIN_OK; -} - -int taosx_plugin_get_stats(TaosX_PluginStats *out_stats) { - if (!g_initialized) return TAOSX_PLUGIN_ERR_NOT_INITIALIZED; - if (out_stats == NULL) return TAOSX_PLUGIN_ERR_INVALID_ARG; - memset(out_stats, 0, sizeof(*out_stats)); - out_stats->uptime_seconds = fake_now_epoch_seconds() - g_state.start_time_epoch_s; - out_stats->error_count = g_state.error_count; - out_stats->events_processed = g_state.events_processed; - out_stats->events_dropped = g_state.events_dropped; - out_stats->memory_usage_bytes = g_state.memory_usage_bytes; - out_stats->ring_buffer_usage_percent = g_state.ring_buffer_usage_percent; - return TAOSX_PLUGIN_OK; -} - - - diff --git a/plugins/incremental_bitmap/taosx_plugin/taosx_plugin_interface.h b/plugins/incremental_bitmap/taosx_plugin/taosx_plugin_interface.h deleted file mode 100644 index 2e0d9bf3ffcb..000000000000 --- a/plugins/incremental_bitmap/taosx_plugin/taosx_plugin_interface.h +++ /dev/null @@ -1,86 +0,0 @@ -#ifndef TAOSX_PLUGIN_INTERFACE_H -#define TAOSX_PLUGIN_INTERFACE_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -// Tech Preview: Minimal, self-contained C ABI for future taosX integration. -// No dependency on enterprise headers; kept binary-stable and optional. - -typedef struct { - const char *key; - const char *value; -} TaosX_KVPair; - -typedef struct { - const TaosX_KVPair *items; - size_t count; -} TaosX_Config; - -typedef struct { - // Basic lifecycle/health - uint64_t uptime_seconds; - uint64_t error_count; - // Basic throughput - uint64_t events_processed; - uint64_t events_dropped; - // Memory and buffering - size_t memory_usage_bytes; - uint32_t ring_buffer_usage_percent; -} TaosX_PluginStats; - -typedef enum { - TAOSX_PLUGIN_OK = 0, - TAOSX_PLUGIN_ERR_INVALID_ARG = -1, - TAOSX_PLUGIN_ERR_NOT_INITIALIZED = -2, - TAOSX_PLUGIN_ERR_ALREADY_RUNNING = -3, - TAOSX_PLUGIN_ERR_INTERNAL = -100 -} TaosX_PluginCode; - -// Version/capabilities are simple strings for forward compatibility. -// Example: "1.0.0" and "bitmap-index,wal-follow". -const char *taosx_plugin_get_name(void); -const char *taosx_plugin_get_version(void); -const char *taosx_plugin_get_capabilities(void); - -// Lifecycle -int taosx_plugin_init(void); -int taosx_plugin_shutdown(void); - -// Configuration and run controls -int taosx_plugin_configure(const TaosX_Config *config); -int taosx_plugin_start(void); -int taosx_plugin_stop(void); - -// Event bridge (placeholder: taosX will pass storage events here) -typedef enum { - TAOSX_EVENT_BLOCK_CREATE = 1, - TAOSX_EVENT_BLOCK_UPDATE = 2, - TAOSX_EVENT_BLOCK_FLUSH = 3 -} TaosX_EventType; - -typedef struct { - uint64_t block_id; - int64_t wal_offset; - int64_t timestamp_ns; - TaosX_EventType event_type; -} TaosX_BlockEvent; - -int taosx_plugin_on_block_event(const TaosX_BlockEvent *event); - -// Stats -int taosx_plugin_get_stats(TaosX_PluginStats *out_stats); - -#ifdef __cplusplus -} -#endif - -#endif // TAOSX_PLUGIN_INTERFACE_H - - - diff --git a/plugins/incremental_bitmap/taosx_plugin/test_taosx_plugin.c b/plugins/incremental_bitmap/taosx_plugin/test_taosx_plugin.c deleted file mode 100644 index d93593d9a8e2..000000000000 --- a/plugins/incremental_bitmap/taosx_plugin/test_taosx_plugin.c +++ /dev/null @@ -1,228 +0,0 @@ -#include -#include -#include -#include -#include "taosx_plugin_interface.h" - -// Test helper macros -#define TEST_ASSERT(condition, message) \ - do { \ - if (!(condition)) { \ - fprintf(stderr, "❌ Test failed: %s\n", message); \ - return -1; \ - } else { \ - printf("✅ %s\n", message); \ - } \ - } while(0) - -#define TEST_SUCCESS(message) \ - printf("✅ %s\n", message) - -// Test functions -static int test_plugin_info(void); -static int test_plugin_lifecycle(void); -static int test_plugin_configuration(void); -static int test_plugin_events(void); -static int test_plugin_stats(void); - -int main() { - printf("==========================================\n"); - printf(" taosX Plugin Interface Test Suite\n"); - printf("==========================================\n\n"); - - int total_tests = 0; - int passed_tests = 0; - int failed_tests = 0; - - // Test list - struct { - const char* name; - int (*test_func)(void); - } tests[] = { - {"Plugin Information", test_plugin_info}, - {"Plugin Lifecycle", test_plugin_lifecycle}, - {"Plugin Configuration", test_plugin_configuration}, - {"Plugin Events", test_plugin_events}, - {"Plugin Statistics", test_plugin_stats} - }; - - total_tests = sizeof(tests) / sizeof(tests[0]); - - // Run tests - for (int i = 0; i < total_tests; i++) { - printf("Running test: %s\n", tests[i].name); - printf("------------------------------------------\n"); - - int result = tests[i].test_func(); - if (result == 0) { - passed_tests++; - TEST_SUCCESS(tests[i].name); - } else { - failed_tests++; - printf("❌ Test failed: %s\n", tests[i].name); - } - printf("\n"); - } - - // Print results - printf("==========================================\n"); - printf(" Test Results\n"); - printf("==========================================\n"); - printf("Total tests: %d\n", total_tests); - printf("Passed: %d\n", passed_tests); - printf("Failed: %d\n", failed_tests); - printf("Success rate: %.1f%%\n", (double)passed_tests / total_tests * 100); - - if (failed_tests == 0) { - printf("\n🎉 All tests passed!\n"); - return 0; - } else { - printf("\n❌ Some tests failed!\n"); - return 1; - } -} - -static int test_plugin_info(void) { - const char* name = taosx_plugin_get_name(); - const char* version = taosx_plugin_get_version(); - const char* capabilities = taosx_plugin_get_capabilities(); - - TEST_ASSERT(name != NULL, "Plugin name is not null"); - TEST_ASSERT(version != NULL, "Plugin version is not null"); - TEST_ASSERT(capabilities != NULL, "Plugin capabilities is not null"); - - printf(" Plugin name: %s\n", name); - printf(" Plugin version: %s\n", version); - printf(" Plugin capabilities: %s\n", capabilities); - - return 0; -} - -static int test_plugin_lifecycle(void) { - // Test initialization - int rc = taosx_plugin_init(); - TEST_ASSERT(rc == TAOSX_PLUGIN_OK, "Plugin initialization succeeds"); - - // Test double initialization (should fail) - rc = taosx_plugin_init(); - TEST_ASSERT(rc == TAOSX_PLUGIN_ERR_ALREADY_RUNNING, "Double initialization fails correctly"); - - // Test shutdown - rc = taosx_plugin_shutdown(); - TEST_ASSERT(rc == TAOSX_PLUGIN_OK, "Plugin shutdown succeeds"); - - // Test shutdown after shutdown (should fail) - rc = taosx_plugin_shutdown(); - TEST_ASSERT(rc == TAOSX_PLUGIN_ERR_NOT_INITIALIZED, "Shutdown after shutdown fails correctly"); - - return 0; -} - -static int test_plugin_configuration(void) { - // Initialize plugin first - int rc = taosx_plugin_init(); - TEST_ASSERT(rc == TAOSX_PLUGIN_OK, "Plugin initialization for config test"); - - // Test configuration with NULL config (should succeed) - rc = taosx_plugin_configure(NULL); - TEST_ASSERT(rc == TAOSX_PLUGIN_OK, "NULL configuration is accepted"); - - // Test configuration with valid config - TaosX_KVPair pairs[] = { - {"key1", "value1"}, - {"key2", "value2"} - }; - TaosX_Config config = { - .items = pairs, - .count = 2 - }; - - rc = taosx_plugin_configure(&config); - TEST_ASSERT(rc == TAOSX_PLUGIN_OK, "Valid configuration is accepted"); - - // Cleanup - taosx_plugin_shutdown(); - - return 0; -} - -static int test_plugin_events(void) { - // Initialize plugin - int rc = taosx_plugin_init(); - TEST_ASSERT(rc == TAOSX_PLUGIN_OK, "Plugin initialization for event test"); - - // Test event handling without starting (should fail) - TaosX_BlockEvent event = { - .block_id = 123, - .wal_offset = 456, - .timestamp_ns = 789, - .event_type = TAOSX_EVENT_BLOCK_CREATE - }; - - rc = taosx_plugin_on_block_event(&event); - TEST_ASSERT(rc == TAOSX_PLUGIN_ERR_INTERNAL, "Event handling without start fails correctly"); - - // Start plugin - rc = taosx_plugin_start(); - TEST_ASSERT(rc == TAOSX_PLUGIN_OK, "Plugin start succeeds"); - - // Test event handling - rc = taosx_plugin_on_block_event(&event); - TEST_ASSERT(rc == TAOSX_PLUGIN_OK, "Event handling succeeds"); - - // Test NULL event (should fail) - rc = taosx_plugin_on_block_event(NULL); - TEST_ASSERT(rc == TAOSX_PLUGIN_ERR_INVALID_ARG, "NULL event handling fails correctly"); - - // Stop plugin - rc = taosx_plugin_stop(); - TEST_ASSERT(rc == TAOSX_PLUGIN_OK, "Plugin stop succeeds"); - - // Cleanup - taosx_plugin_shutdown(); - - return 0; -} - -static int test_plugin_stats(void) { - // Initialize and start plugin - int rc = taosx_plugin_init(); - TEST_ASSERT(rc == TAOSX_PLUGIN_OK, "Plugin initialization for stats test"); - - rc = taosx_plugin_start(); - TEST_ASSERT(rc == TAOSX_PLUGIN_OK, "Plugin start for stats test"); - - // Process some events to generate stats - TaosX_BlockEvent event = { - .block_id = 1, - .wal_offset = 100, - .timestamp_ns = 1000, - .event_type = TAOSX_EVENT_BLOCK_CREATE - }; - - for (int i = 0; i < 5; i++) { - rc = taosx_plugin_on_block_event(&event); - TEST_ASSERT(rc == TAOSX_PLUGIN_OK, "Event processing for stats"); - } - - // Get stats - TaosX_PluginStats stats; - rc = taosx_plugin_get_stats(&stats); - TEST_ASSERT(rc == TAOSX_PLUGIN_OK, "Stats retrieval succeeds"); - - printf(" Events processed: %lu\n", stats.events_processed); - printf(" Events dropped: %lu\n", stats.events_dropped); - printf(" Error count: %lu\n", stats.error_count); - printf(" Memory usage: %zu bytes\n", stats.memory_usage_bytes); - printf(" Ring buffer usage: %u%%\n", stats.ring_buffer_usage_percent); - - // Test NULL stats (should fail) - rc = taosx_plugin_get_stats(NULL); - TEST_ASSERT(rc == TAOSX_PLUGIN_ERR_INVALID_ARG, "NULL stats retrieval fails correctly"); - - // Cleanup - taosx_plugin_stop(); - taosx_plugin_shutdown(); - - return 0; -} diff --git a/plugins/incremental_bitmap/test/e2e_consistency.c b/plugins/incremental_bitmap/test/e2e_consistency.c deleted file mode 100644 index 04183a3ff8f9..000000000000 --- a/plugins/incremental_bitmap/test/e2e_consistency.c +++ /dev/null @@ -1,171 +0,0 @@ -#include "e2e_consistency.h" -#include "e2e_perf.h" - -#include -#include -#include -#include -#include - -static int path_exists(const char *path) { - struct stat st; - return (path && stat(path, &st) == 0); -} - -static int is_directory(const char *path) { - struct stat st; - return (path && stat(path, &st) == 0 && S_ISDIR(st.st_mode)); -} - -static int count_files_in_dir(const char *dir_path, const char *suffix) { - DIR *dir = opendir(dir_path); - if (!dir) return 0; - - int count = 0; - struct dirent *entry; - while ((entry = readdir(dir)) != NULL) { - if (entry->d_type == DT_REG) { // 常规文件 - if (!suffix || strstr(entry->d_name, suffix)) { - count++; - } - } - } - closedir(dir); - return count; -} - -int e2e_validate_consistency(const E2EConsistencyConfig *config, - char *error_buffer, - size_t error_buffer_length) { - if (!config || !config->data_path || !config->snapshot_path || !config->recovery_path) { - if (error_buffer && error_buffer_length > 0) { - snprintf(error_buffer, error_buffer_length, "invalid config or paths"); - } - return -1; - } - - if (!path_exists(config->data_path)) { - if (error_buffer && error_buffer_length > 0) { - snprintf(error_buffer, error_buffer_length, "data path not found: %s", config->data_path); - } - return -2; - } - if (!path_exists(config->snapshot_path)) { - if (error_buffer && error_buffer_length > 0) { - snprintf(error_buffer, error_buffer_length, "snapshot path not found: %s", config->snapshot_path); - } - return -3; - } - if (!path_exists(config->recovery_path)) { - if (error_buffer && error_buffer_length > 0) { - snprintf(error_buffer, error_buffer_length, "recovery path not found: %s", config->recovery_path); - } - return -4; - } - - return 0; -} - -int e2e_validate_consistency_detailed(const E2EConsistencyConfig *config, - E2EConsistencyReport *report) { - if (!config || !report) return -1; - - // 初始化报告 - memset(report, 0, sizeof(*report)); - - E2EPerfTimer timer; - e2e_perf_timer_start(&timer); - - // 检查路径 - if (!is_directory(config->data_path)) { - report->missing_files++; - snprintf(report->details, sizeof(report->details), - "data directory missing: %s", config->data_path); - return -1; - } - - if (!is_directory(config->snapshot_path)) { - report->missing_files++; - snprintf(report->details, sizeof(report->details), - "snapshot directory missing: %s", config->snapshot_path); - return -2; - } - - if (!is_directory(config->recovery_path)) { - report->missing_files++; - snprintf(report->details, sizeof(report->details), - "recovery directory missing: %s", config->recovery_path); - return -3; - } - - // 统计文件数量 - report->snapshots_checked = count_files_in_dir(config->snapshot_path, ".snapshot"); - report->recovery_points_checked = count_files_in_dir(config->recovery_path, ".recovery"); - - // 模拟数据块比对 - report->data_blocks_compared = report->snapshots_checked * 1000; // 假设每个快照1000个块 - report->consistency_errors = 0; // 模拟无错误 - - e2e_perf_timer_stop(&timer); - report->validation_time_ms = e2e_perf_elapsed_ms(&timer); - - snprintf(report->details, sizeof(report->details), - "validation completed successfully"); - - return 0; -} - -int e2e_compare_snapshot_data(const char *snapshot_file, - const char *reference_file, - uint64_t *blocks_compared, - uint64_t *mismatches) { - if (!snapshot_file || !reference_file || !blocks_compared || !mismatches) { - return -1; - } - - *blocks_compared = 0; - *mismatches = 0; - - // 检查文件存在性 - if (!path_exists(snapshot_file) || !path_exists(reference_file)) { - return -2; - } - - // 模拟文件比对逻辑 - struct stat st1, st2; - if (stat(snapshot_file, &st1) != 0 || stat(reference_file, &st2) != 0) { - return -3; - } - - // 简单的大小比对 - *blocks_compared = (st1.st_size + 1023) / 1024; // 按1KB块计算 - if (st1.st_size != st2.st_size) { - *mismatches = 1; - } - - return 0; -} - -void e2e_print_consistency_report(const E2EConsistencyReport *report) { - if (!report) return; - - printf("==========================================\n"); - printf(" E2E Consistency Validation Report\n"); - printf("==========================================\n"); - printf("Snapshots Checked: %llu\n", (unsigned long long)report->snapshots_checked); - printf("Recovery Points: %llu\n", (unsigned long long)report->recovery_points_checked); - printf("Data Blocks Compared: %llu\n", (unsigned long long)report->data_blocks_compared); - printf("Consistency Errors: %llu\n", (unsigned long long)report->consistency_errors); - printf("Missing Files: %llu\n", (unsigned long long)report->missing_files); - printf("Validation Time: %.2f ms\n", report->validation_time_ms); - printf("Details: %s\n", report->details); - - if (report->consistency_errors == 0 && report->missing_files == 0) { - printf("Status: ✓ PASSED\n"); - } else { - printf("Status: ✗ FAILED\n"); - } - printf("==========================================\n"); -} - - diff --git a/plugins/incremental_bitmap/test/e2e_perf.c b/plugins/incremental_bitmap/test/e2e_perf.c deleted file mode 100644 index 28ef3ae139d9..000000000000 --- a/plugins/incremental_bitmap/test/e2e_perf.c +++ /dev/null @@ -1,49 +0,0 @@ -#include "e2e_perf.h" - -#include -#include - -static uint64_t now_ms(void) { -#if defined(CLOCK_MONOTONIC) - struct timespec ts; - clock_gettime(CLOCK_MONOTONIC, &ts); - return (uint64_t)ts.tv_sec * 1000ULL + (uint64_t)ts.tv_nsec / 1000000ULL; -#else - struct timespec ts; - clock_gettime(CLOCK_REALTIME, &ts); - return (uint64_t)ts.tv_sec * 1000ULL + (uint64_t)ts.tv_nsec / 1000000ULL; -#endif -} - -void e2e_perf_timer_start(E2EPerfTimer *timer) { - if (!timer) return; - timer->start_ms = now_ms(); - timer->end_ms = timer->start_ms; -} - -void e2e_perf_timer_stop(E2EPerfTimer *timer) { - if (!timer) return; - timer->end_ms = now_ms(); -} - -double e2e_perf_elapsed_ms(const E2EPerfTimer *timer) { - if (!timer) return 0.0; - if (timer->end_ms < timer->start_ms) return 0.0; - return (double)(timer->end_ms - timer->start_ms); -} - -double e2e_perf_throughput_per_sec(uint64_t items, const E2EPerfTimer *timer) { - double ms = e2e_perf_elapsed_ms(timer); - if (ms <= 0.0) return 0.0; - return (double)items / (ms / 1000.0); -} - -void e2e_perf_print_summary(const char *label, uint64_t items, const E2EPerfTimer *timer) { - double ms = e2e_perf_elapsed_ms(timer); - double tps = e2e_perf_throughput_per_sec(items, timer); - printf("[PERF] %s: items=%llu, elapsed=%.2f ms, throughput=%.2f ops/s\n", - label ? label : "E2E", (unsigned long long)items, ms, tps); -} - - - diff --git a/plugins/incremental_bitmap/test/mock_storage_engine.c b/plugins/incremental_bitmap/test/mock_storage_engine.c deleted file mode 100644 index dce9c1e5ae13..000000000000 --- a/plugins/incremental_bitmap/test/mock_storage_engine.c +++ /dev/null @@ -1,144 +0,0 @@ -#include "storage_engine_interface.h" -#include -#include -#include -#include - -// Mock存储引擎状态 -typedef struct { - bool initialized; - bool interception_installed; - uint64_t events_processed; - uint64_t events_dropped; - StorageEventCallback event_callback; - void* callback_user_data; - pthread_mutex_t mutex; -} SMockStorageEngine; - -static SMockStorageEngine g_mock_engine = { - .initialized = false, - .interception_installed = false, - .events_processed = 0, - .events_dropped = 0, - .mutex = PTHREAD_MUTEX_INITIALIZER -}; - -// Mock存储引擎实现函数 -static int32_t mock_init(const SStorageEngineConfig* config) { - if (!config) return -1; - - pthread_mutex_lock(&g_mock_engine.mutex); - g_mock_engine.initialized = true; - g_mock_engine.events_processed = 0; - g_mock_engine.events_dropped = 0; - g_mock_engine.event_callback = config->event_callback; - g_mock_engine.callback_user_data = config->callback_user_data; - pthread_mutex_unlock(&g_mock_engine.mutex); - - printf("[Mock] 存储引擎初始化成功\n"); - return 0; -} - -static void mock_destroy(void) { - pthread_mutex_lock(&g_mock_engine.mutex); - g_mock_engine.initialized = false; - g_mock_engine.interception_installed = false; - pthread_mutex_unlock(&g_mock_engine.mutex); - - printf("[Mock] 存储引擎销毁完成\n"); -} - -static int32_t mock_install_interception(void) { - pthread_mutex_lock(&g_mock_engine.mutex); - if (!g_mock_engine.initialized) { - pthread_mutex_unlock(&g_mock_engine.mutex); - return -1; - } - - g_mock_engine.interception_installed = true; - pthread_mutex_unlock(&g_mock_engine.mutex); - - printf("[Mock] 事件拦截安装成功\n"); - return 0; -} - -static int32_t mock_uninstall_interception(void) { - pthread_mutex_lock(&g_mock_engine.mutex); - g_mock_engine.interception_installed = false; - pthread_mutex_unlock(&g_mock_engine.mutex); - - printf("[Mock] 事件拦截卸载成功\n"); - return 0; -} - -static int32_t mock_trigger_event(const SStorageEvent* event) { - if (!event) { - return -1; - } - - pthread_mutex_lock(&g_mock_engine.mutex); - if (!g_mock_engine.interception_installed) { - pthread_mutex_unlock(&g_mock_engine.mutex); - return -1; - } - - g_mock_engine.events_processed++; - StorageEventCallback cb = g_mock_engine.event_callback; - void* user = g_mock_engine.callback_user_data; - pthread_mutex_unlock(&g_mock_engine.mutex); - - printf("[Mock] 触发事件: 类型=%d, 块ID=%lu, WAL偏移量=%lu, 时间戳=%ld\n", - event->event_type, event->block_id, event->wal_offset, event->timestamp); - - if (cb) { - cb(event, user); - } - - return 0; -} - -static int32_t mock_get_stats(uint64_t* events_processed, uint64_t* events_dropped) { - pthread_mutex_lock(&g_mock_engine.mutex); - - if (events_processed) { - *events_processed = g_mock_engine.events_processed; - } - if (events_dropped) { - *events_dropped = g_mock_engine.events_dropped; - } - - pthread_mutex_unlock(&g_mock_engine.mutex); - return 0; -} - -static bool mock_is_supported(void) { - return true; // Mock引擎总是支持的 -} - -static const char* mock_get_engine_name(void) { - return "mock"; -} - -// Mock存储引擎接口 -static SStorageEngineInterface g_mock_interface = { - .init = mock_init, - .destroy = mock_destroy, - .install_interception = mock_install_interception, - .uninstall_interception = mock_uninstall_interception, - .trigger_event = mock_trigger_event, - .get_stats = mock_get_stats, - .is_supported = mock_is_supported, - .get_engine_name = mock_get_engine_name -}; - -// Mock存储引擎工厂函数 -SStorageEngineInterface* mock_storage_engine_create(void) { - return &g_mock_interface; -} - -// 便捷函数：注册Mock存储引擎 -int32_t register_mock_storage_engine(void) { - extern int32_t register_storage_engine_interface(const char* name, StorageEngineInterfaceFactory factory); - return register_storage_engine_interface("mock", mock_storage_engine_create); -} - diff --git a/plugins/incremental_bitmap/test/pitr_e2e_test.c b/plugins/incremental_bitmap/test/pitr_e2e_test.c deleted file mode 100644 index 447ff8033890..000000000000 --- a/plugins/incremental_bitmap/test/pitr_e2e_test.c +++ /dev/null @@ -1,1504 +0,0 @@ -#include "pitr_e2e_test.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// 默认测试配置 - 优化为1GB以内 -const SPitrTestConfig PITR_DEFAULT_CONFIG = { - .snapshot_interval_ms = 2000, // 2秒间隔 - .recovery_points = 5, // 5个恢复点 - .data_block_count = 1000, // 1000个数据块（减少90%） - .concurrent_writers = 2, // 2个并发写入线程 - .test_duration_seconds = 60, // 1分钟测试 - .enable_disorder_test = true, // 启用乱序测试 - .enable_deletion_test = true, // 启用删除测试 - .test_data_path = "./pitr_test_data", // 使用相对路径 - .snapshot_path = "./pitr_snapshots", - .recovery_path = "./pitr_recovery" -}; - -// PITR测试器内部结构 -struct SPitrTester { - SPitrTestConfig config; - SPitrTestStatus status; - SBitmapEngine* bitmap_engine; - SBackupCoordinator* backup_coordinator; - - // 快照管理 - SSnapshotInfo* snapshots; - uint32_t snapshot_count; - uint32_t max_snapshots; - - // 恢复点管理 - SRecoveryPoint* recovery_points; - uint32_t recovery_count; - uint32_t max_recovery_points; - - // 测试数据 - char* test_data_buffer; - size_t test_data_size; - - // 线程管理 - pthread_t* writer_threads; - pthread_mutex_t data_mutex; - pthread_rwlock_t snapshot_rwlock; - - // 时间管理 - int64_t test_start_time; - int64_t last_snapshot_time; - - // 统计信息 - uint64_t total_blocks_processed; - uint64_t total_events_processed; - uint64_t total_deletions_processed; -}; - -// 内部函数声明 -static int create_directories(const char* path); -static int cleanup_directories(const char* path); -static void* writer_thread_function(void* arg); -static int64_t get_current_timestamp_ms(void); -static uint64_t generate_block_id(uint32_t thread_id, uint64_t sequence); -static int create_snapshot(SPitrTester* tester, int64_t timestamp); -static int verify_snapshot_consistency(SPitrTester* tester, const SSnapshotInfo* snapshot); - -// 前向声明 -static int process_disorder_events(SPitrTester* tester, double disorder_ratio); -static int process_deletion_events(SPitrTester* tester, uint64_t deletion_count); -static int validate_data_size_limits(const SPitrTestConfig* config); -static void print_data_size_warning(const SPitrTestConfig* config); -static int monitor_runtime_data_usage(const SPitrTestConfig* config, const char* test_path); -static int validate_test_paths(const SPitrTestConfig* config); - -// 创建PITR测试器 -SPitrTester* pitr_tester_create(const SPitrTestConfig* config) { - if (!config) { - fprintf(stderr, "Error: Invalid configuration\n"); - return NULL; - } - - // 配置验证（静默模式） - - SPitrTester* tester = calloc(1, sizeof(SPitrTester)); - if (!tester) { - fprintf(stderr, "Error: Failed to allocate memory for tester\n"); - return NULL; - } - - // 复制配置 - memcpy(&tester->config, config, sizeof(SPitrTestConfig)); - - // 手动复制字符串指针（确保指针有效性） - tester->config.test_data_path = config->test_data_path; - tester->config.snapshot_path = config->snapshot_path; - tester->config.recovery_path = config->recovery_path; - - // 基础配置校验：必须提供有效路径 - if (!tester->config.test_data_path || !tester->config.snapshot_path || !tester->config.recovery_path || - tester->config.test_data_path[0] == '\0' || tester->config.snapshot_path[0] == '\0' || tester->config.recovery_path[0] == '\0') { - // 静默处理无效配置（这是预期的测试行为） - free(tester); - return NULL; - } - - // 路径安全校验 - if (validate_test_paths(&tester->config) != 0) { - fprintf(stderr, "Error: Test paths validation failed\n"); - free(tester); - return NULL; - } - - // 🔒 强制数据量检查 - 必须保证测试数据量在1GB以内 - printf("🔍 开始数据量安全检查...\n"); - if (validate_data_size_limits(config) != 0) { - fprintf(stderr, "❌ 数据量检查失败！测试被阻止运行。\n"); - fprintf(stderr, " 请调整配置参数，确保数据量在 %.1f GB 以内。\n", (double)MAX_DATA_SIZE_GB); - free(tester); - return NULL; - } - - // 打印数据量警告（如果接近限制） - print_data_size_warning(config); - - // 初始化状态 - memset(&tester->status, 0, sizeof(SPitrTestStatus)); - tester->status.test_passed = true; - - // 创建目录 - // fprintf(stderr, "[PITR] creating directories: data='%s' snap='%s' rec='%s'\n", - // config->test_data_path ? config->test_data_path : "(null)", - // config->snapshot_path ? config->snapshot_path : "(null)", - // config->recovery_path ? config->recovery_path : "(null)"); - int rc_data = create_directories(config->test_data_path); - int rc_snap = create_directories(config->snapshot_path); - int rc_recv = create_directories(config->recovery_path); - if (rc_data != 0 || rc_snap != 0 || rc_recv != 0) { - fprintf(stderr, "Error: Failed to create directories (data=%d snap=%d rec=%d)\n", rc_data, rc_snap, rc_recv); - free(tester); - return NULL; - } - - // 初始化位图引擎 - tester->bitmap_engine = bitmap_engine_init(); - if (!tester->bitmap_engine) { - fprintf(stderr, "Error: Failed to initialize bitmap engine\n"); - cleanup_directories(config->test_data_path); - cleanup_directories(config->snapshot_path); - cleanup_directories(config->recovery_path); - free(tester); - return NULL; - } - - // 初始化备份协调器 - SBackupConfig backup_config = { - .batch_size = 1000, - .max_retries = 3, - .retry_interval_ms = 1000, - .timeout_ms = 5000, - .enable_compression = true, - .enable_encryption = false, - .backup_path = (char*)config->recovery_path, - .temp_path = "/tmp" - }; - - tester->backup_coordinator = backup_coordinator_init(tester->bitmap_engine, &backup_config); - if (!tester->backup_coordinator) { - fprintf(stderr, "Error: Failed to initialize backup coordinator\n"); - bitmap_engine_destroy(tester->bitmap_engine); - cleanup_directories(config->test_data_path); - cleanup_directories(config->snapshot_path); - cleanup_directories(config->recovery_path); - free(tester); - return NULL; - } - - // 初始化快照管理 - tester->max_snapshots = config->recovery_points * 2; // 预留空间 - tester->snapshots = calloc(tester->max_snapshots, sizeof(SSnapshotInfo)); - if (!tester->snapshots) { - fprintf(stderr, "Error: Failed to allocate memory for snapshots\n"); - backup_coordinator_destroy(tester->backup_coordinator); - bitmap_engine_destroy(tester->bitmap_engine); - cleanup_directories(config->test_data_path); - cleanup_directories(config->snapshot_path); - cleanup_directories(config->recovery_path); - free(tester); - return NULL; - } - - // 初始化恢复点管理 - tester->max_recovery_points = config->recovery_points; - tester->recovery_points = calloc(tester->max_recovery_points, sizeof(SRecoveryPoint)); - if (!tester->recovery_points) { - fprintf(stderr, "Error: Failed to allocate memory for recovery points\n"); - free(tester->snapshots); - backup_coordinator_destroy(tester->backup_coordinator); - bitmap_engine_destroy(tester->bitmap_engine); - cleanup_directories(config->test_data_path); - cleanup_directories(config->snapshot_path); - cleanup_directories(config->recovery_path); - free(tester); - return NULL; - } - - // 初始化线程管理 - tester->writer_threads = calloc(config->concurrent_writers, sizeof(pthread_t)); - if (!tester->writer_threads) { - fprintf(stderr, "Error: Failed to allocate memory for writer threads\n"); - free(tester->recovery_points); - free(tester->snapshots); - backup_coordinator_destroy(tester->backup_coordinator); - bitmap_engine_destroy(tester->bitmap_engine); - cleanup_directories(config->test_data_path); - cleanup_directories(config->snapshot_path); - cleanup_directories(config->recovery_path); - free(tester); - return NULL; - } - - // 初始化同步原语 - if (pthread_mutex_init(&tester->data_mutex, NULL) != 0 || - pthread_rwlock_init(&tester->snapshot_rwlock, NULL) != 0) { - fprintf(stderr, "Error: Failed to initialize synchronization primitives\n"); - free(tester->writer_threads); - free(tester->recovery_points); - free(tester->snapshots); - backup_coordinator_destroy(tester->backup_coordinator); - bitmap_engine_destroy(tester->bitmap_engine); - cleanup_directories(config->test_data_path); - cleanup_directories(config->snapshot_path); - cleanup_directories(config->recovery_path); - free(tester); - return NULL; - } - - // 启动备份协调器 - if (backup_coordinator_start(tester->backup_coordinator) != 0) { - fprintf(stderr, "Error: Failed to start backup coordinator\n"); - pthread_rwlock_destroy(&tester->snapshot_rwlock); - pthread_mutex_destroy(&tester->data_mutex); - free(tester->writer_threads); - free(tester->recovery_points); - free(tester->snapshots); - backup_coordinator_destroy(tester->backup_coordinator); - bitmap_engine_destroy(tester->bitmap_engine); - cleanup_directories(config->test_data_path); - cleanup_directories(config->snapshot_path); - cleanup_directories(config->recovery_path); - free(tester); - return NULL; - } - - printf("PITR tester created successfully\n"); - return tester; -} - -// 销毁PITR测试器 -void pitr_tester_destroy(SPitrTester* tester) { - if (!tester) return; - - // 停止备份协调器 - if (tester->backup_coordinator) { - backup_coordinator_stop(tester->backup_coordinator); - backup_coordinator_destroy(tester->backup_coordinator); - } - - // 销毁位图引擎 - if (tester->bitmap_engine) { - bitmap_engine_destroy(tester->bitmap_engine); - } - - // 销毁同步原语 - pthread_rwlock_destroy(&tester->snapshot_rwlock); - pthread_mutex_destroy(&tester->data_mutex); - - // 清理目录 - if (tester->config.test_data_path) { - cleanup_directories(tester->config.test_data_path); - } - if (tester->config.snapshot_path) { - cleanup_directories(tester->config.snapshot_path); - } - if (tester->config.recovery_path) { - cleanup_directories(tester->config.recovery_path); - } - - // 释放内存 - free(tester->writer_threads); - free(tester->recovery_points); - free(tester->snapshots); - free(tester->test_data_buffer); - free(tester); - - printf("PITR tester destroyed\n"); -} - -// 运行完整的PITR E2E测试 -int pitr_tester_run_full_test(SPitrTester* tester) { - if (!tester) { - return PITR_TEST_INVALID_CONFIG; - } - - printf("Starting full PITR E2E test...\n"); - - tester->test_start_time = get_current_timestamp_ms(); - tester->last_snapshot_time = tester->test_start_time; - - // 1. 创建测试数据 - printf("Step 1: Creating test data...\n"); - - // 运行时数据量监控 - if (monitor_runtime_data_usage(&tester->config, tester->config.test_data_path) != 0) { - strcpy(tester->status.error_message, "Runtime data usage monitoring failed"); - tester->status.test_passed = false; - return PITR_TEST_FAILED; - } - - if (pitr_create_test_data(tester->config.test_data_path, - tester->config.data_block_count, - tester->config.concurrent_writers) != 0) { - strcpy(tester->status.error_message, "Failed to create test data"); - tester->status.test_passed = false; - return PITR_TEST_FAILED; - } - - // 2. 运行快照测试 - printf("Step 2: Running snapshot test...\n"); - if (pitr_tester_run_snapshot_test(tester) != PITR_TEST_SUCCESS) { - strcpy(tester->status.error_message, "Snapshot test failed"); - tester->status.test_passed = false; - return PITR_TEST_FAILED; - } - - // 3. 运行恢复测试 - printf("Step 3: Running recovery test...\n"); - if (pitr_tester_run_recovery_test(tester) != PITR_TEST_SUCCESS) { - strcpy(tester->status.error_message, "Recovery test failed"); - tester->status.test_passed = false; - return PITR_TEST_FAILED; - } - - // 4. 运行乱序测试（如果启用） - if (tester->config.enable_disorder_test) { - printf("Step 4: Running disorder test...\n"); - if (pitr_tester_run_disorder_test(tester) != PITR_TEST_SUCCESS) { - strcpy(tester->status.error_message, "Disorder test failed"); - tester->status.test_passed = false; - return PITR_TEST_FAILED; - } - } - - // 5. 运行删除一致性测试（如果启用） - if (tester->config.enable_deletion_test) { - printf("Step 5: Running deletion consistency test...\n"); - if (pitr_tester_run_deletion_consistency_test(tester) != PITR_TEST_SUCCESS) { - strcpy(tester->status.error_message, "Deletion consistency test failed"); - tester->status.test_passed = false; - return PITR_TEST_FAILED; - } - } - - // 6. 运行边界条件测试 - printf("Step 6: Running boundary test...\n"); - if (pitr_tester_run_boundary_test(tester) != PITR_TEST_SUCCESS) { - strcpy(tester->status.error_message, "Boundary test failed"); - tester->status.test_passed = false; - return PITR_TEST_FAILED; - } - - // 更新测试状态 - tester->status.total_test_time_ms = get_current_timestamp_ms() - tester->test_start_time; - tester->status.test_passed = true; - - printf("Full PITR E2E test completed successfully in %lu ms\n", - tester->status.total_test_time_ms); - - return PITR_TEST_SUCCESS; -} - -// 运行快照创建测试 -int pitr_tester_run_snapshot_test(SPitrTester* tester) { - if (!tester) { - return PITR_TEST_INVALID_CONFIG; - } - - printf("Running snapshot test...\n"); - - int64_t current_time = get_current_timestamp_ms(); - uint32_t snapshots_to_create = tester->config.recovery_points; - - for (uint32_t i = 0; i < snapshots_to_create; i++) { - // 等待到下一个快照时间（简化：只等待很短时间用于测试） - if (i > 0) { - usleep(100000); // 只等待100ms用于测试 - } - - // 创建快照 - if (create_snapshot(tester, get_current_timestamp_ms()) != 0) { - fprintf(stderr, "Error: Failed to create snapshot %u\n", i); - return PITR_TEST_FAILED; - } - - // 快照创建后检查数据量 - if (monitor_runtime_data_usage(&tester->config, tester->config.snapshot_path) != 0) { - fprintf(stderr, "Warning: Data usage monitoring failed after snapshot %u\n", i); - } - - tester->last_snapshot_time = get_current_timestamp_ms(); - tester->status.snapshots_created++; - - printf("Created snapshot %u/%u at timestamp %ld\n", - i + 1, snapshots_to_create, tester->last_snapshot_time); - } - - printf("Snapshot test completed: %lu snapshots created\n", tester->status.snapshots_created); - return PITR_TEST_SUCCESS; -} - -// 运行时间点恢复测试 -int pitr_tester_run_recovery_test(SPitrTester* tester) { - if (!tester) { - return PITR_TEST_INVALID_CONFIG; - } - - printf("Running recovery test...\n"); - - // 验证所有快照 - for (uint32_t i = 0; i < tester->snapshot_count; i++) { - if (!verify_snapshot_consistency(tester, &tester->snapshots[i])) { - fprintf(stderr, "Error: Snapshot %u consistency check failed\n", i); - return PITR_TEST_FAILED; - } - } - - // 创建恢复点 - for (uint32_t i = 0; i < tester->config.recovery_points && i < tester->snapshot_count; i++) { - SRecoveryPoint* recovery_point = &tester->recovery_points[tester->recovery_count]; - - recovery_point->recovery_id = tester->recovery_count; - recovery_point->recovery_timestamp = tester->snapshots[i].timestamp; - recovery_point->base_snapshot = &tester->snapshots[i]; - recovery_point->expected_block_count = tester->snapshots[i].block_count; - - // 执行恢复 - if (backup_coordinator_get_incremental_blocks(tester->backup_coordinator, - 0, recovery_point->recovery_timestamp, - NULL, 0) >= 0) { - recovery_point->recovery_successful = true; - tester->status.recovery_points_verified++; - } else { - recovery_point->recovery_successful = false; - fprintf(stderr, "Warning: Recovery point %u failed\n", i); - } - - tester->recovery_count++; - printf("Created recovery point %u/%u at timestamp %ld\n", - i + 1, tester->config.recovery_points, recovery_point->recovery_timestamp); - } - - printf("Recovery test completed: %lu recovery points verified\n", - tester->status.recovery_points_verified); - - return PITR_TEST_SUCCESS; -} - -// 运行乱序数据处理测试 -int pitr_tester_run_disorder_test(SPitrTester* tester) { - if (!tester) { - return PITR_TEST_INVALID_CONFIG; - } - - printf("Running disorder test...\n"); - - // 模拟不同比例的乱序数据 - double disorder_ratios[] = {0.1, 0.3, 0.5, 0.7, 0.9}; - uint32_t ratio_count = sizeof(disorder_ratios) / sizeof(disorder_ratios[0]); - - for (uint32_t i = 0; i < ratio_count; i++) { - printf("Testing disorder ratio: %.1f%%\n", disorder_ratios[i] * 100); - - if (process_disorder_events(tester, disorder_ratios[i]) != 0) { - fprintf(stderr, "Error: Disorder test failed for ratio %.1f%%\n", - disorder_ratios[i] * 100); - return PITR_TEST_FAILED; - } - - // 验证数据一致性 - SDataConsistencyResult result; - if (pitr_tester_verify_consistency(tester, get_current_timestamp_ms(), &result) == 0) { - if (result.is_consistent) { - printf("Data consistency verified for disorder ratio %.1f%%\n", - disorder_ratios[i] * 100); - } else { - fprintf(stderr, "Warning: Data inconsistency detected for disorder ratio %.1f%%\n", - disorder_ratios[i] * 100); - } - } - } - - printf("Disorder test completed successfully\n"); - return PITR_TEST_SUCCESS; -} - -// 运行删除覆盖一致性测试 -int pitr_tester_run_deletion_consistency_test(SPitrTester* tester) { - if (!tester) { - return PITR_TEST_INVALID_CONFIG; - } - - printf("Running deletion consistency test...\n"); - - // 模拟不同数量的删除操作 - uint64_t deletion_counts[] = {100, 500, 1000, 5000}; - uint32_t count_count = sizeof(deletion_counts) / sizeof(deletion_counts[0]); - - for (uint32_t i = 0; i < count_count; i++) { - printf("Testing deletion count: %lu\n", deletion_counts[i]); - - if (process_deletion_events(tester, deletion_counts[i]) != 0) { - fprintf(stderr, "Error: Deletion test failed for count %lu\n", deletion_counts[i]); - return PITR_TEST_FAILED; - } - - // 验证删除后的一致性 - SDataConsistencyResult result; - if (pitr_tester_verify_consistency(tester, get_current_timestamp_ms(), &result) == 0) { - if (result.is_consistent) { - printf("Data consistency verified after %lu deletions\n", deletion_counts[i]); - } else { - fprintf(stderr, "Warning: Data inconsistency detected after %lu deletions\n", - deletion_counts[i]); - } - } - } - - printf("Deletion consistency test completed successfully\n"); - return PITR_TEST_SUCCESS; -} - -// 运行边界条件测试 -int pitr_tester_run_boundary_test(SPitrTester* tester) { - if (!tester) { - return PITR_TEST_INVALID_CONFIG; - } - - printf("Running boundary test...\n"); - - // 测试边界值 - 移除UINT64_MAX避免系统崩溃 - uint64_t boundary_block_counts[] = {0, 1, 100, 1000000, 10000000}; // 最大1000万块，约80MB - uint32_t boundary_count = sizeof(boundary_block_counts) / sizeof(boundary_block_counts[0]); - - for (uint32_t i = 0; i < boundary_count; i++) { - printf("Testing boundary block count: %lu\n", boundary_block_counts[i]); - - // 创建边界测试数据 - if (pitr_create_test_data(tester->config.test_data_path, - boundary_block_counts[i], 1) != 0) { - // fprintf(stderr, "Warning: Failed to create test data for boundary count %lu\n", - // boundary_block_counts[i]); - continue; - } - - // 验证边界条件 - if (boundary_block_counts[i] == 0) { - // 空数据测试 - if (tester->bitmap_engine) { - // 使用正确的函数名和参数 - uint64_t block_ids[1]; - uint32_t count = bitmap_engine_get_dirty_blocks_by_wal(tester->bitmap_engine, 0, 1, block_ids, 1); - if (count != 0) { - fprintf(stderr, "Error: Empty data test failed\n"); - return PITR_TEST_FAILED; - } - } - } else if (boundary_block_counts[i] == UINT64_MAX) { - // 最大数据测试 - // 这里应该测试内存限制和性能边界 - printf("Maximum data test completed\n"); - } - } - - // 测试时间边界 - int64_t time_boundaries[] = {0, 1, INT64_MAX}; - uint32_t time_count = sizeof(time_boundaries) / sizeof(time_boundaries[0]); - - for (uint32_t i = 0; i < time_count; i++) { - printf("Testing time boundary: %ld\n", time_boundaries[i]); - - // 验证时间边界处理 - if (time_boundaries[i] == 0) { - // 零时间测试 - if (create_snapshot(tester, 0) != 0) { - fprintf(stderr, "Warning: Zero timestamp snapshot creation failed\n"); - } - } - } - - printf("Boundary test completed successfully\n"); - return PITR_TEST_SUCCESS; -} - -// 获取测试状态 -int pitr_tester_get_status(SPitrTester* tester, SPitrTestStatus* status) { - if (!tester || !status) { - return PITR_TEST_INVALID_CONFIG; - } - - memcpy(status, &tester->status, sizeof(SPitrTestStatus)); - return PITR_TEST_SUCCESS; -} - -// 获取快照列表 -int pitr_tester_get_snapshots(SPitrTester* tester, SSnapshotInfo* snapshots, uint32_t max_count, uint32_t* actual_count) { - if (!tester || !snapshots || !actual_count) { - return PITR_TEST_INVALID_CONFIG; - } - - uint32_t count = (max_count < tester->snapshot_count) ? max_count : tester->snapshot_count; - memcpy(snapshots, tester->snapshots, count * sizeof(SSnapshotInfo)); - *actual_count = count; - - return PITR_TEST_SUCCESS; -} - -// 获取恢复点列表 -int pitr_tester_get_recovery_points(SPitrTester* tester, SRecoveryPoint* recovery_points, uint32_t max_count, uint32_t* actual_count) { - if (!tester || !recovery_points || !actual_count) { - return PITR_TEST_INVALID_CONFIG; - } - - uint32_t count = (max_count < tester->recovery_count) ? max_count : tester->recovery_count; - memcpy(recovery_points, tester->recovery_points, count * sizeof(SRecoveryPoint)); - *actual_count = count; - - return PITR_TEST_SUCCESS; -} - -// 验证数据一致性 -int pitr_tester_verify_consistency(SPitrTester* tester, int64_t timestamp, SDataConsistencyResult* result) { - if (!tester || !result) { - return PITR_TEST_INVALID_CONFIG; - } - - // 这里应该实现实际的数据一致性检查逻辑 - // 目前返回模拟结果 - result->expected_blocks = tester->config.data_block_count; - result->actual_blocks = tester->config.data_block_count; - result->mismatched_blocks = 0; - result->missing_blocks = 0; - result->extra_blocks = 0; - result->consistency_percentage = 100.0; - result->is_consistent = true; - strcpy(result->details, "Data consistency check completed successfully"); - - tester->status.data_consistency_checks++; - return PITR_TEST_SUCCESS; -} - -// 生成测试报告 -int pitr_tester_generate_report(SPitrTester* tester, const char* report_path) { - if (!tester || !report_path) { - return PITR_TEST_INVALID_CONFIG; - } - - FILE* report_file = fopen(report_path, "w"); - if (!report_file) { - fprintf(stderr, "Error: Failed to open report file: %s\n", report_path); - return PITR_TEST_FAILED; - } - - fprintf(report_file, "PITR E2E Test Report\n"); - fprintf(report_file, "===================\n\n"); - - fprintf(report_file, "Test Configuration:\n"); - fprintf(report_file, "- Snapshot Interval: %u ms\n", tester->config.snapshot_interval_ms); - fprintf(report_file, "- Recovery Points: %u\n", tester->config.recovery_points); - fprintf(report_file, "- Data Block Count: %lu\n", tester->config.data_block_count); - fprintf(report_file, "- Concurrent Writers: %u\n", tester->config.concurrent_writers); - fprintf(report_file, "- Test Duration: %u seconds\n", tester->config.test_duration_seconds); - fprintf(report_file, "- Disorder Test: %s\n", tester->config.enable_disorder_test ? "Enabled" : "Disabled"); - fprintf(report_file, "- Deletion Test: %s\n", tester->config.enable_deletion_test ? "Enabled" : "Disabled"); - - fprintf(report_file, "\nTest Results:\n"); - fprintf(report_file, "- Snapshots Created: %lu\n", tester->status.snapshots_created); - fprintf(report_file, "- Recovery Points Verified: %lu\n", tester->status.recovery_points_verified); - fprintf(report_file, "- Data Consistency Checks: %lu\n", tester->status.data_consistency_checks); - fprintf(report_file, "- Disorder Events Handled: %lu\n", tester->status.disorder_handled); - fprintf(report_file, "- Deletion Operations Handled: %lu\n", tester->status.deletion_handled); - fprintf(report_file, "- Total Test Time: %lu ms\n", tester->status.total_test_time_ms); - fprintf(report_file, "- Test Status: %s\n", tester->status.test_passed ? "PASSED" : "FAILED"); - - if (!tester->status.test_passed) { - fprintf(report_file, "- Error Message: %s\n", tester->status.error_message); - } - - fprintf(report_file, "\nSnapshots:\n"); - for (uint32_t i = 0; i < tester->snapshot_count; i++) { - fprintf(report_file, "- Snapshot %u: ID=%lu, Time=%ld, Blocks=%lu, Size=%lu bytes, Valid=%s\n", - i, tester->snapshots[i].snapshot_id, tester->snapshots[i].timestamp, - tester->snapshots[i].block_count, tester->snapshots[i].data_size_bytes, - tester->snapshots[i].is_valid ? "Yes" : "No"); - } - - fprintf(report_file, "\nRecovery Points:\n"); - for (uint32_t i = 0; i < tester->recovery_count; i++) { - fprintf(report_file, "- Recovery Point %u: ID=%lu, Time=%ld, Success=%s\n", - i, tester->recovery_points[i].recovery_id, - tester->recovery_points[i].recovery_timestamp, - tester->recovery_points[i].recovery_successful ? "Yes" : "No"); - } - - fclose(report_file); - printf("Test report generated: %s\n", report_path); - - return PITR_TEST_SUCCESS; -} - -// 重置测试器状态 -int pitr_tester_reset(SPitrTester* tester) { - if (!tester) { - return PITR_TEST_INVALID_CONFIG; - } - - printf("Resetting PITR tester...\n"); - - // 重置状态 - memset(&tester->status, 0, sizeof(SPitrTestStatus)); - tester->status.test_passed = true; - - // 重置快照和恢复点 - tester->snapshot_count = 0; - tester->recovery_count = 0; - - // 重置时间 - tester->test_start_time = 0; - tester->last_snapshot_time = 0; - - // 重置统计 - tester->total_blocks_processed = 0; - tester->total_events_processed = 0; - tester->total_deletions_processed = 0; - - printf("PITR tester reset completed\n"); - return PITR_TEST_SUCCESS; -} - -// 内部函数实现 - -// 创建目录 -static int create_directories(const char* path) { - if (!path) return -1; - - char* path_copy = strdup(path); - if (!path_copy) return -1; - - char* token = strtok(path_copy, "/"); - char current_path[512] = ""; - - // 处理绝对路径：如果原始路径以'/'开头，先放入根目录 - if (path[0] == '/') { - // 初始化为根目录 - strncpy(current_path, "/", sizeof(current_path) - 1); - current_path[sizeof(current_path) - 1] = '\0'; - } - - while (token) { - // 跳过当前目录标记 '.' - if (strcmp(token, ".") != 0 && strlen(token) > 0) { - if (strlen(current_path) > 1 || (strlen(current_path) == 1 && current_path[0] == '/')) { - // 不是初始空串时追加分隔符；当 current_path == "." 时也需要加分隔符 - size_t len = strlen(current_path); - if (!((len == 1 && current_path[0] == '/'))) { - strncat(current_path, "/", sizeof(current_path) - strlen(current_path) - 1); - } - } else if (strlen(current_path) == 1 && current_path[0] == '.') { - // 从 "." 继续拼接时，先补一个 '/' - strncat(current_path, "/", sizeof(current_path) - strlen(current_path) - 1); - } - strncat(current_path, token, sizeof(current_path) - strlen(current_path) - 1); - } else { - // 仅为 '.' 时，跳过本轮 mkdir - token = strtok(NULL, "/"); - continue; - } - - if (strlen(current_path) > 0) { - if (mkdir(current_path, 0755) != 0 && errno != EEXIST) { - fprintf(stderr, "mkdir failed: '%s' (from '%s'): %s\n", current_path, path, strerror(errno)); - free(path_copy); - return -1; - } - } - - token = strtok(NULL, "/"); - } - - free(path_copy); - return 0; -} - -// 清理目录 -static int cleanup_directories(const char* path) { - if (!path) return -1; - - // 简单的目录清理，实际应用中可能需要递归删除 - printf("Cleaning up directory: %s\n", path); - return 0; -} - -// 获取当前时间戳（毫秒） -static int64_t get_current_timestamp_ms(void) { - struct timespec ts; - clock_gettime(CLOCK_MONOTONIC, &ts); - return (int64_t)ts.tv_sec * 1000 + (int64_t)ts.tv_nsec / 1000000; -} - -// 生成块ID -static uint64_t generate_block_id(uint32_t thread_id, uint64_t sequence) { - return ((uint64_t)thread_id << 32) | sequence; -} - -// 创建快照 -static int create_snapshot(SPitrTester* tester, int64_t timestamp) { - if (!tester || tester->snapshot_count >= tester->max_snapshots) { - return -1; - } - - SSnapshotInfo* snapshot = &tester->snapshots[tester->snapshot_count]; - - snapshot->timestamp = timestamp; - snapshot->snapshot_id = tester->snapshot_count; - snapshot->block_count = tester->config.data_block_count; - snapshot->data_size_bytes = snapshot->block_count * 1024; // 假设每个块1KB - snapshot->is_valid = true; - - // 确保快照目录存在 - if (mkdir(tester->config.snapshot_path, 0755) != 0 && errno != EEXIST) { - fprintf(stderr, "Error: Failed to create snapshot directory %s: %s\n", - tester->config.snapshot_path, strerror(errno)); - return -1; - } - - // 生成快照文件路径 - snprintf(snapshot->metadata_path, sizeof(snapshot->metadata_path), - "%s/snapshot_%lu_metadata.bin", tester->config.snapshot_path, snapshot->snapshot_id); - snprintf(snapshot->data_path, sizeof(snapshot->data_path), - "%s/snapshot_%lu_data.bin", tester->config.snapshot_path, snapshot->snapshot_id); - - // 创建快照元数据文件 - FILE* metadata_file = fopen(snapshot->metadata_path, "w"); - if (!metadata_file) { - fprintf(stderr, "Error: Failed to create metadata file %s: %s\n", - snapshot->metadata_path, strerror(errno)); - return -1; - } - - // 写入快照元数据 - fprintf(metadata_file, "Snapshot ID: %lu\n", snapshot->snapshot_id); - fprintf(metadata_file, "Timestamp: %ld\n", snapshot->timestamp); - fprintf(metadata_file, "Block Count: %lu\n", snapshot->block_count); - fprintf(metadata_file, "Data Size: %lu bytes\n", snapshot->data_size_bytes); - fclose(metadata_file); - - // 创建快照数据文件 - FILE* data_file = fopen(snapshot->data_path, "w"); - if (!data_file) { - fprintf(stderr, "Error: Failed to create data file %s: %s\n", - snapshot->data_path, strerror(errno)); - return -1; - } - - // 写入模拟数据（每个块1KB） - char block_data[1024]; - memset(block_data, 0, sizeof(block_data)); - for (uint64_t i = 0; i < snapshot->block_count; i++) { - if (fwrite(block_data, sizeof(block_data), 1, data_file) != 1) { - fprintf(stderr, "Error: Failed to write block data\n"); - fclose(data_file); - return -1; - } - } - fclose(data_file); - - tester->snapshot_count++; - return 0; -} - -// 验证快照一致性 -static int verify_snapshot_consistency(SPitrTester* tester, const SSnapshotInfo* snapshot) { - if (!tester || !snapshot) return 0; - - // 检查文件是否存在且可读 - struct stat st; - if (stat(snapshot->metadata_path, &st) != 0) { - fprintf(stderr, "Warning: Metadata file not found: %s\n", snapshot->metadata_path); - return 0; - } - - if (stat(snapshot->data_path, &st) != 0) { - fprintf(stderr, "Warning: Data file not found: %s\n", snapshot->data_path); - return 0; - } - - // 检查文件大小是否合理 - if (st.st_size == 0) { - fprintf(stderr, "Warning: File size is 0: %s\n", snapshot->data_path); - return 0; - } - - // 这里应该实现实际的快照一致性检查逻辑 - // 目前只是基本的文件存在性检查 - return 1; -} - -// 处理乱序事件 -static int process_disorder_events(SPitrTester* tester, double disorder_ratio) { - if (!tester) return -1; - - // 模拟乱序事件处理 - uint64_t total_events = 1000; - uint64_t disorder_events = (uint64_t)(total_events * disorder_ratio); - uint64_t ordered_events = total_events - disorder_events; - - tester->status.disorder_handled += disorder_events; - tester->total_events_processed += total_events; - - printf("Processed %lu events (%.1f%% disorder)\n", total_events, disorder_ratio * 100); - - return 0; -} - -// 处理删除事件 -static int process_deletion_events(SPitrTester* tester, uint64_t deletion_count) { - if (!tester) return -1; - - // 模拟删除操作处理 - uint64_t successful_deletions = (uint64_t)(deletion_count * 0.95); // 95%成功率 - uint64_t failed_deletions = deletion_count - successful_deletions; - - tester->status.deletion_handled += successful_deletions; - tester->total_deletions_processed += deletion_count; - - printf("Processed %lu deletions (%lu successful, %lu failed)\n", - deletion_count, successful_deletions, failed_deletions); - - return 0; -} - -// 测试辅助函数实现 - -// 创建测试数据 -int pitr_create_test_data(const char* data_path, uint64_t block_count, uint32_t concurrent_writers) { - if (!data_path || block_count == 0 || concurrent_writers == 0) { - return -1; - } - - // 检查数据量是否合理（避免系统崩溃） - uint64_t max_safe_blocks = 100000000; // 最大1亿块，约800MB - if (block_count > max_safe_blocks) { - fprintf(stderr, "Error: Block count %lu exceeds safe limit %lu\n", block_count, max_safe_blocks); - return -1; - } - - printf("Creating test data: %lu blocks, %u writers\n", block_count, concurrent_writers); - - // 确保目录存在 - if (mkdir(data_path, 0755) != 0 && errno != EEXIST) { - fprintf(stderr, "Error: Failed to create directory %s: %s\n", data_path, strerror(errno)); - return -1; - } - - // 创建测试数据文件 - char data_file_path[512]; - snprintf(data_file_path, sizeof(data_file_path), "%s/test_data.bin", data_path); - - FILE* data_file = fopen(data_file_path, "w"); - if (!data_file) { - fprintf(stderr, "Error: Failed to create test data file %s: %s\n", data_file_path, strerror(errno)); - return -1; - } - - // 写入测试数据 - for (uint64_t i = 0; i < block_count; i++) { - uint64_t block_data = i; - if (fwrite(&block_data, sizeof(block_data), 1, data_file) != 1) { - fprintf(stderr, "Error: Failed to write block data\n"); - fclose(data_file); - return -1; - } - } - - fclose(data_file); - printf("Test data created successfully: %s\n", data_file_path); - - return 0; -} - -// 验证快照完整性 -bool pitr_verify_snapshot_integrity(const SSnapshotInfo* snapshot) { - if (!snapshot) return false; - - // 检查文件是否存在且可读 - struct stat st; - if (stat(snapshot->metadata_path, &st) != 0 || stat(snapshot->data_path, &st) != 0) { - return false; - } - - // 检查文件大小是否合理 - if (st.st_size == 0) { - return false; - } - - return true; -} - -// 时间测量工具实现 - -// 开始计时 -SPitrTimer* pitr_timer_start(const char* operation_name) { - if (!operation_name) return NULL; - - SPitrTimer* timer = malloc(sizeof(SPitrTimer)); - if (!timer) return NULL; - - strncpy(timer->operation_name, operation_name, sizeof(timer->operation_name) - 1); - timer->operation_name[sizeof(timer->operation_name) - 1] = '\0'; - - clock_gettime(CLOCK_MONOTONIC, &timer->start_time); - return timer; -} - -// 停止计时 -void pitr_timer_stop(SPitrTimer* timer) { - if (!timer) return; - - clock_gettime(CLOCK_MONOTONIC, &timer->end_time); -} - -// 获取耗时（毫秒） -double pitr_timer_get_elapsed_ms(SPitrTimer* timer) { - if (!timer) return 0.0; - - double elapsed = (timer->end_time.tv_sec - timer->start_time.tv_sec) * 1000.0 + - (timer->end_time.tv_nsec - timer->start_time.tv_nsec) / 1000000.0; - - return elapsed; -} - -// 打印计时结果 -void pitr_timer_print_result(SPitrTimer* timer) { - if (!timer) return; - - double elapsed = pitr_timer_get_elapsed_ms(timer); - printf("Operation '%s' took %.2f ms\n", timer->operation_name, elapsed); - - free(timer); -} - -// 性能基准测试 -int pitr_run_performance_benchmark(const char* test_name, void (*test_func)(void*), void* test_data, uint32_t iterations) { - if (!test_name || !test_func || iterations == 0) { - return -1; - } - - printf("Running performance benchmark: %s (%u iterations)\n", test_name, iterations); - - SPitrTimer* timer = pitr_timer_start(test_name); - if (!timer) { - return -1; - } - - // 运行测试函数 - for (uint32_t i = 0; i < iterations; i++) { - test_func(test_data); - } - - pitr_timer_stop(timer); - double elapsed = pitr_timer_get_elapsed_ms(timer); - - printf("Benchmark completed: %s took %.2f ms for %u iterations (%.2f ms/iteration)\n", - test_name, elapsed, iterations, elapsed / iterations); - - pitr_timer_print_result(timer); - - return 0; -} - -// 内存使用监控 -int pitr_monitor_memory_usage(size_t* peak_memory, size_t* current_memory) { - if (!peak_memory || !current_memory) { - return -1; - } - - // 这里实现实际的内存监控逻辑 - // 目前返回模拟值 - *peak_memory = 1024 * 1024; // 1MB - *current_memory = 512 * 1024; // 512KB - - printf("Memory monitoring: peak=%zu bytes, current=%zu bytes\n", *peak_memory, *current_memory); - - return 0; -} - -// ============================================================================ -// 数据量检查函数实现 - 必须保证测试数据量在1GB以内 -// ============================================================================ - -// 估算测试数据大小 -static int64_t estimate_test_data_size(const SPitrTestConfig* config) { - if (!config) return 0; - - // 基础数据块大小 - int64_t base_data_size = (int64_t)config->data_block_count * ESTIMATED_BLOCK_SIZE_BYTES; - - // 快照开销 - int64_t snapshot_overhead = (int64_t)(base_data_size * ESTIMATED_SNAPSHOT_OVERHEAD * config->recovery_points); - - // 恢复点开销 - int64_t recovery_overhead = (int64_t)(base_data_size * ESTIMATED_RECOVERY_OVERHEAD * config->recovery_points); - - // 元数据开销（每个快照和恢复点的元数据） - int64_t metadata_overhead = (int64_t)((config->recovery_points * 2) * 1024); // 每个元数据约1KB - - // 总估算大小 - int64_t total_size = base_data_size + snapshot_overhead + recovery_overhead + metadata_overhead; - - return total_size; -} - -// 运行时数据量监控 -static int monitor_runtime_data_usage(const SPitrTestConfig* config, const char* test_path) { - if (!config || !test_path) return 0; - - struct statvfs vfs; - if (statvfs(test_path, &vfs) != 0) { - fprintf(stderr, "⚠️ 警告: 无法检查磁盘空间使用情况\n"); - return 0; - } - - uint64_t free_space_bytes = (uint64_t)vfs.f_bavail * vfs.f_frsize; - double free_space_gb = (double)free_space_bytes / (1024 * 1024 * 1024); - - // 如果可用空间少于100MB，发出警告 - if (free_space_gb < 0.1) { - fprintf(stderr, "🚨 严重警告: 磁盘空间不足！可用空间: %.2f GB\n", free_space_gb); - return -1; - } else if (free_space_gb < 0.5) { - printf("⚠️ 警告: 磁盘空间较低，可用空间: %.2f GB\n", free_space_gb); - } - - return 0; -} - -// 检查测试路径安全性 -static int validate_test_paths(const SPitrTestConfig* config) { - if (!config) return -1; - - // 检查路径是否为空 - if (!config->test_data_path || !config->snapshot_path || !config->recovery_path) { - return -1; // 路径为空，静默返回错误 - } - - // 检查是否包含系统重要路径 - const char* dangerous_paths[] = { - "/", "/etc", "/usr", "/var", "/home", "/root", "/tmp", "/opt" - }; - - for (int i = 0; i < sizeof(dangerous_paths) / sizeof(dangerous_paths[0]); i++) { - // 检查绝对路径匹配（必须以危险路径开头） - if ((strncmp(config->test_data_path, dangerous_paths[i], strlen(dangerous_paths[i])) == 0 && - strlen(config->test_data_path) >= strlen(dangerous_paths[i])) || - (strncmp(config->snapshot_path, dangerous_paths[i], strlen(dangerous_paths[i])) == 0 && - strlen(config->snapshot_path) >= strlen(dangerous_paths[i])) || - (strncmp(config->recovery_path, dangerous_paths[i], strlen(dangerous_paths[i])) == 0 && - strlen(config->recovery_path) >= strlen(dangerous_paths[i]))) { - fprintf(stderr, "❌ 错误: 测试路径包含系统重要目录: %s\n", dangerous_paths[i]); - fprintf(stderr, " 请使用相对路径或安全的测试目录\n"); - return -1; - } - } - - // 检查路径长度 - if (strlen(config->test_data_path) > 200 || - strlen(config->snapshot_path) > 200 || - strlen(config->recovery_path) > 200) { - fprintf(stderr, "❌ 错误: 测试路径过长，可能存在安全风险\n"); - return -1; - } - - return 0; -} - -// 验证数据量限制 -static int validate_data_size_limits(const SPitrTestConfig* config) { - if (!config) { - fprintf(stderr, "❌ 错误: 无效的测试配置\n"); - return -1; - } - - int64_t estimated_size = estimate_test_data_size(config); - - printf("📊 数据量检查:\n"); - printf(" 配置的数据块数量: %lu\n", config->data_block_count); - printf(" 恢复点数量: %u\n", config->recovery_points); - printf(" 估算数据大小: %.2f MB (%.2f GB)\n", - (double)estimated_size / (1024 * 1024), - (double)estimated_size / (1024 * 1024 * 1024)); - - if (estimated_size > MAX_DATA_SIZE_BYTES) { - fprintf(stderr, "❌ 错误: 估算数据大小 %.2f GB 超过限制 %.1d GB\n", - (double)estimated_size / (1024 * 1024 * 1024), MAX_DATA_SIZE_GB); - fprintf(stderr, " 测试被阻止运行！请调整配置参数。\n"); - return -1; - } - - printf("✅ 数据量检查通过，测试可以安全运行\n"); - return 0; -} - -// 打印数据量警告 -static void print_data_size_warning(const SPitrTestConfig* config) { - if (!config) return; - - int64_t estimated_size = estimate_test_data_size(config); - double size_gb = (double)estimated_size / (1024 * 1024 * 1024); - - if (size_gb > 0.5) { // 如果超过500MB，给出警告 - printf("⚠️ 警告: 当前配置估算数据大小为 %.2f GB\n", size_gb); - printf(" 建议使用轻量级配置以减少数据量:\n"); - printf(" - 数据块数量: 500\n"); - printf(" - 恢复点数量: 3\n"); - printf(" - 测试时长: 30秒\n"); - } -} - -// ============================================================================ -// 缺失的PITR函数实现 -// ============================================================================ - -// 时间点恢复测试 -int pitr_tester_run_time_point_recovery(SPitrTester* tester, int64_t timestamp) { - if (!tester) { - return PITR_TEST_INVALID_CONFIG; - } - - printf("Running time point recovery test at timestamp %ld...\n", timestamp); - - // 查找最接近的恢复点 - SRecoveryPoint* best_recovery_point = NULL; - int64_t min_time_diff = INT64_MAX; - - for (uint32_t i = 0; i < tester->recovery_count; i++) { - int64_t time_diff = abs(tester->recovery_points[i].recovery_timestamp - timestamp); - if (time_diff < min_time_diff) { - min_time_diff = time_diff; - best_recovery_point = &tester->recovery_points[i]; - } - } - - if (!best_recovery_point) { - // 如果没有恢复点，创建一个新的 - if (tester->recovery_count >= tester->max_recovery_points) { - fprintf(stderr, "Error: No more recovery points available\n"); - return PITR_TEST_FAILED; - } - - best_recovery_point = &tester->recovery_points[tester->recovery_count]; - best_recovery_point->recovery_id = tester->recovery_count; - best_recovery_point->recovery_timestamp = timestamp; - best_recovery_point->expected_block_count = tester->config.data_block_count; - best_recovery_point->base_snapshot = NULL; - best_recovery_point->recovery_successful = false; - - tester->recovery_count++; - } - - // 执行时间点恢复 - if (backup_coordinator_get_incremental_blocks(tester->backup_coordinator, - 0, timestamp, - NULL, 0) >= 0) { - best_recovery_point->recovery_successful = true; - tester->status.recovery_points_verified++; - - printf("Time point recovery successful at timestamp %ld\n", timestamp); - return PITR_TEST_SUCCESS; - } else { - fprintf(stderr, "Error: Time point recovery failed at timestamp %ld\n", timestamp); - return PITR_TEST_FAILED; - } -} - -// 数据一致性验证 -int pitr_tester_verify_data_consistency(SPitrTester* tester) { - if (!tester) { - return PITR_TEST_INVALID_CONFIG; - } - - printf("Running data consistency verification...\n"); - - // 验证所有快照的一致性 - for (uint32_t i = 0; i < tester->snapshot_count; i++) { - if (!verify_snapshot_consistency(tester, &tester->snapshots[i])) { - fprintf(stderr, "Error: Snapshot %u consistency check failed\n", i); - return PITR_TEST_FAILED; - } - } - - // 验证所有恢复点的一致性 - for (uint32_t i = 0; i < tester->recovery_count; i++) { - if (!tester->recovery_points[i].recovery_successful) { - fprintf(stderr, "Error: Recovery point %u is not successful\n", i); - return PITR_TEST_FAILED; - } - } - - // 验证位图引擎状态 - if (tester->bitmap_engine) { - // 检查位图引擎的基本状态 - uint64_t total_blocks = 0; - if (tester->bitmap_engine->dirty_blocks) { - total_blocks += tester->bitmap_engine->dirty_blocks->cardinality( - tester->bitmap_engine->dirty_blocks->bitmap); - } - if (tester->bitmap_engine->new_blocks) { - total_blocks += tester->bitmap_engine->new_blocks->cardinality( - tester->bitmap_engine->new_blocks->bitmap); - } - if (tester->bitmap_engine->deleted_blocks) { - total_blocks += tester->bitmap_engine->deleted_blocks->cardinality( - tester->bitmap_engine->deleted_blocks->bitmap); - } - - printf("Bitmap engine consistency verified: total blocks = %lu\n", total_blocks); - } - - printf("Data consistency verification completed successfully\n"); - return PITR_TEST_SUCCESS; -} - -// 边界条件测试 -int pitr_tester_run_boundary_tests(SPitrTester* tester) { - if (!tester) { - return PITR_TEST_INVALID_CONFIG; - } - - printf("Running boundary condition tests...\n"); - - // 测试空数据 - printf("Testing empty data scenario...\n"); - if (pitr_create_test_data(tester->config.test_data_path, 0, 1) != 0) { - fprintf(stderr, "Warning: Failed to create empty test data\n"); - } else { - // 验证空数据下的位图引擎行为 - if (tester->bitmap_engine) { - uint64_t block_ids[1]; - uint32_t count = bitmap_engine_get_dirty_blocks_by_wal(tester->bitmap_engine, 0, 1, block_ids, 1); - if (count != 0) { - fprintf(stderr, "Error: Empty data test failed\n"); - return PITR_TEST_FAILED; - } - } - printf("Empty data test passed\n"); - } - - // 测试单块数据 - printf("Testing single block scenario...\n"); - if (pitr_create_test_data(tester->config.test_data_path, 1, 1) != 0) { - fprintf(stderr, "Warning: Failed to create single block test data\n"); - } else { - printf("Single block test passed\n"); - } - - // 测试时间边界 - printf("Testing time boundary scenarios...\n"); - int64_t time_boundaries[] = {0, 1, INT64_MAX}; - uint32_t time_count = sizeof(time_boundaries) / sizeof(time_boundaries[0]); - - for (uint32_t i = 0; i < time_count; i++) { - printf("Testing time boundary: %ld\n", time_boundaries[i]); - - if (time_boundaries[i] == 0) { - // 零时间测试 - if (create_snapshot(tester, 0) != 0) { - fprintf(stderr, "Warning: Zero timestamp snapshot creation failed\n"); - } else { - printf("Zero timestamp test passed\n"); - } - } - } - - // 测试并发边界 - 使用配置中的数据块数量，避免硬编码 - printf("Testing concurrency boundary scenarios...\n"); - uint32_t concurrency_levels[] = {1, 2, 4, 8}; - uint32_t concurrency_count = sizeof(concurrency_levels) / sizeof(concurrency_levels[0]); - - for (uint32_t i = 0; i < concurrency_count; i++) { - printf("Testing concurrency level: %u\n", concurrency_levels[i]); - - // 创建测试数据 - 使用配置中的数据块数量，而不是硬编码的100 - uint64_t test_block_count = (tester->config.data_block_count < 100) ? - tester->config.data_block_count : 100; - if (pitr_create_test_data(tester->config.test_data_path, test_block_count, concurrency_levels[i]) != 0) { - fprintf(stderr, "Warning: Failed to create test data for concurrency level %u\n", concurrency_levels[i]); - continue; - } - - printf("Concurrency level %u test passed\n", concurrency_levels[i]); - } - - printf("Boundary condition tests completed successfully\n"); - return PITR_TEST_SUCCESS; -} - -// 性能测试 -int pitr_tester_run_performance_test(SPitrTester* tester) { - if (!tester) { - return PITR_TEST_INVALID_CONFIG; - } - - printf("Running performance test...\n"); - - // 测试快照创建性能 - printf("Testing snapshot creation performance...\n"); - SPitrTimer* timer = pitr_timer_start("Snapshot Creation"); - if (!timer) { - fprintf(stderr, "Error: Failed to create timer\n"); - return PITR_TEST_FAILED; - } - - if (create_snapshot(tester, get_current_timestamp_ms()) != 0) { - fprintf(stderr, "Error: Failed to create snapshot for performance test\n"); - pitr_timer_print_result(timer); - return PITR_TEST_FAILED; - } - - pitr_timer_stop(timer); - double snapshot_time = pitr_timer_get_elapsed_ms(timer); - printf("Snapshot creation took %.2f ms\n", snapshot_time); - pitr_timer_print_result(timer); - - // 测试位图操作性能 - 使用配置中的数据块数量，避免硬编码 - printf("Testing bitmap operations performance...\n"); - timer = pitr_timer_start("Bitmap Operations"); - if (!timer) { - fprintf(stderr, "Error: Failed to create timer\n"); - return PITR_TEST_FAILED; - } - - if (tester->bitmap_engine) { - // 执行一些位图操作 - 使用配置中的数据块数量，而不是硬编码的1000 - uint64_t test_operations = (tester->config.data_block_count < 1000) ? - tester->config.data_block_count : 1000; - for (uint64_t i = 0; i < test_operations; i++) { - bitmap_engine_mark_dirty(tester->bitmap_engine, i, i * 1000, get_current_timestamp_ms()); - } - - // 查询位图状态 - uint64_t block_ids[1000]; - uint32_t count = bitmap_engine_get_dirty_blocks_by_time(tester->bitmap_engine, - 0, get_current_timestamp_ms(), - block_ids, 1000); - printf("Bitmap operations completed: %u blocks found\n", count); - } - - pitr_timer_stop(timer); - double bitmap_time = pitr_timer_get_elapsed_ms(timer); - printf("Bitmap operations took %.2f ms\n", bitmap_time); - pitr_timer_print_result(timer); - - // 测试恢复性能 - printf("Testing recovery performance...\n"); - timer = pitr_timer_start("Recovery Operations"); - if (!timer) { - fprintf(stderr, "Error: Failed to create timer\n"); - return PITR_TEST_FAILED; - } - - if (pitr_tester_run_time_point_recovery(tester, get_current_timestamp_ms()) != 0) { - fprintf(stderr, "Error: Recovery performance test failed\n"); - pitr_timer_print_result(timer); - return PITR_TEST_FAILED; - } - - pitr_timer_stop(timer); - double recovery_time = pitr_timer_get_elapsed_ms(timer); - printf("Recovery operations took %.2f ms\n", recovery_time); - pitr_timer_print_result(timer); - - // 性能总结 - printf("\nPerformance Test Summary:\n"); - printf("- Snapshot Creation: %.2f ms\n", snapshot_time); - printf("- Bitmap Operations: %.2f ms\n", bitmap_time); - printf("- Recovery Operations: %.2f ms\n", recovery_time); - printf("- Total Performance Test Time: %.2f ms\n", snapshot_time + bitmap_time + recovery_time); - - printf("Performance test completed successfully\n"); - return PITR_TEST_SUCCESS; -} diff --git a/plugins/incremental_bitmap/test/test_abstraction_layer.c b/plugins/incremental_bitmap/test/test_abstraction_layer.c deleted file mode 100644 index 1e61b0af14ad..000000000000 --- a/plugins/incremental_bitmap/test/test_abstraction_layer.c +++ /dev/null @@ -1,380 +0,0 @@ -#include -/* - * Copyright (c) 2024 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "../include/bitmap_interface.h" -#include "../include/roaring_bitmap.h" -#include -#include -#include -#include -#include -#include - -// 测试辅助函数 -static void print_test_result(const char* test_name, bool passed) { - printf("[%s] %s\n", passed ? "PASS" : "FAIL", test_name); -} - -// 测试1: 工厂模式测试 -static void test_factory_pattern() { - printf("\n=== 测试1: 工厂模式测试 ===\n"); - - // 测试默认实现（应该是RoaringBitmap） - SBitmapInterface* default_interface = bitmap_interface_create(); - assert(default_interface != NULL); - assert(default_interface->add != NULL); - assert(default_interface->remove != NULL); - assert(default_interface->contains != NULL); - print_test_result("默认工厂创建", true); - - // 测试基本操作 - default_interface->add(default_interface->bitmap, 100); - assert(default_interface->contains(default_interface->bitmap, 100) == true); - assert(default_interface->cardinality(default_interface->bitmap) == 1); - print_test_result("默认实现基本操作", true); - - // 清理 - default_interface->destroy(default_interface->bitmap); - free(default_interface); -} - -// 测试2: 环境变量控制测试 -static void test_environment_control() { - printf("\n=== 测试2: 环境变量控制测试 ===\n"); - - // 设置环境变量强制使用SimpleBitmap - setenv("TDENGINE_USE_SIMPLE_BITMAP", "1", 1); - - SBitmapInterface* simple_interface = bitmap_interface_create(); - assert(simple_interface != NULL); - print_test_result("环境变量控制SimpleBitmap", true); - - // 测试基本操作 - simple_interface->add(simple_interface->bitmap, 200); - assert(simple_interface->contains(simple_interface->bitmap, 200) == true); - assert(simple_interface->cardinality(simple_interface->bitmap) == 1); - print_test_result("SimpleBitmap基本操作", true); - - // 清理 - simple_interface->destroy(simple_interface->bitmap); - free(simple_interface); - - // 恢复环境变量 - unsetenv("TDENGINE_USE_SIMPLE_BITMAP"); -} - -// 测试3: RoaringBitmap直接创建测试 -static void test_roaring_bitmap_direct() { - printf("\n=== 测试3: RoaringBitmap直接创建测试 ===\n"); - - SBitmapInterface* roaring_interface = roaring_bitmap_interface_create(); - assert(roaring_interface != NULL); - print_test_result("RoaringBitmap直接创建", true); - - // 测试基本操作 - roaring_interface->add(roaring_interface->bitmap, 300); - assert(roaring_interface->contains(roaring_interface->bitmap, 300) == true); - assert(roaring_interface->cardinality(roaring_interface->bitmap) == 1); - print_test_result("RoaringBitmap基本操作", true); - - // 清理 - roaring_interface->destroy(roaring_interface->bitmap); - free(roaring_interface); -} - -// 测试4: 接口一致性测试 -static void test_interface_consistency() { - printf("\n=== 测试4: 接口一致性测试 ===\n"); - - // 测试SimpleBitmap - setenv("TDENGINE_USE_SIMPLE_BITMAP", "1", 1); - SBitmapInterface* simple_interface = bitmap_interface_create(); - - // 测试RoaringBitmap - unsetenv("TDENGINE_USE_SIMPLE_BITMAP"); - SBitmapInterface* roaring_interface = bitmap_interface_create(); - - // 验证两个接口都有相同的函数指针 - assert(simple_interface->add != NULL); - assert(simple_interface->remove != NULL); - assert(simple_interface->contains != NULL); - assert(simple_interface->cardinality != NULL); - assert(simple_interface->clone != NULL); - assert(simple_interface->destroy != NULL); - - assert(roaring_interface->add != NULL); - assert(roaring_interface->remove != NULL); - assert(roaring_interface->contains != NULL); - assert(roaring_interface->cardinality != NULL); - assert(roaring_interface->clone != NULL); - assert(roaring_interface->destroy != NULL); - - print_test_result("接口一致性验证", true); - - // 清理 - simple_interface->destroy(simple_interface->bitmap); - free(simple_interface); - roaring_interface->destroy(roaring_interface->bitmap); - free(roaring_interface); -} - -// 测试5: 性能对比测试 -static void test_performance_comparison() { - printf("\n=== 测试5: 性能对比测试 ===\n"); - - const int test_count = 100000; - - // 测试SimpleBitmap性能 - setenv("TDENGINE_USE_SIMPLE_BITMAP", "1", 1); - SBitmapInterface* simple_interface = bitmap_interface_create(); - - clock_t start_time = clock(); - for (int i = 0; i < test_count; i++) { - simple_interface->add(simple_interface->bitmap, i); - } - clock_t simple_add_time = clock() - start_time; - - start_time = clock(); - for (int i = 0; i < test_count; i++) { - simple_interface->contains(simple_interface->bitmap, i); - } - clock_t simple_query_time = clock() - start_time; - - printf("SimpleBitmap - 添加: %.2f ms, 查询: %.2f ms\n", - (double)simple_add_time * 1000 / CLOCKS_PER_SEC, - (double)simple_query_time * 1000 / CLOCKS_PER_SEC); - - simple_interface->destroy(simple_interface->bitmap); - free(simple_interface); - - // 测试RoaringBitmap性能 - unsetenv("TDENGINE_USE_SIMPLE_BITMAP"); - SBitmapInterface* roaring_interface = bitmap_interface_create(); - - start_time = clock(); - for (int i = 0; i < test_count; i++) { - roaring_interface->add(roaring_interface->bitmap, i); - } - clock_t roaring_add_time = clock() - start_time; - - start_time = clock(); - for (int i = 0; i < test_count; i++) { - roaring_interface->contains(roaring_interface->bitmap, i); - } - clock_t roaring_query_time = clock() - start_time; - - printf("RoaringBitmap - 添加: %.2f ms, 查询: %.2f ms\n", - (double)roaring_add_time * 1000 / CLOCKS_PER_SEC, - (double)roaring_query_time * 1000 / CLOCKS_PER_SEC); - - roaring_interface->destroy(roaring_interface->bitmap); - free(roaring_interface); - - print_test_result("性能对比测试", true); -} - -// 测试6: 内存使用对比测试 -static void test_memory_usage_comparison() { - printf("\n=== 测试6: 内存使用对比测试 ===\n"); - - const int test_count = 10000; - - // 测试SimpleBitmap内存使用 - setenv("TDENGINE_USE_SIMPLE_BITMAP", "1", 1); - SBitmapInterface* simple_interface = bitmap_interface_create(); - - for (int i = 0; i < test_count; i++) { - simple_interface->add(simple_interface->bitmap, i); - } - - size_t simple_memory = simple_interface->memory_usage(simple_interface->bitmap); - printf("SimpleBitmap内存使用: %zu bytes\n", simple_memory); - - simple_interface->destroy(simple_interface->bitmap); - free(simple_interface); - - // 测试RoaringBitmap内存使用 - unsetenv("TDENGINE_USE_SIMPLE_BITMAP"); - SBitmapInterface* roaring_interface = bitmap_interface_create(); - - for (int i = 0; i < test_count; i++) { - roaring_interface->add(roaring_interface->bitmap, i); - } - - size_t roaring_memory = roaring_interface->memory_usage(roaring_interface->bitmap); - printf("RoaringBitmap内存使用: %zu bytes\n", roaring_memory); - - roaring_interface->destroy(roaring_interface->bitmap); - free(roaring_interface); - - print_test_result("内存使用对比测试", true); -} - -// 测试7: 序列化/反序列化测试 -static void test_serialization() { - printf("\n=== 测试7: 序列化/反序列化测试 ===\n"); - - // 测试SimpleBitmap序列化 - setenv("TDENGINE_USE_SIMPLE_BITMAP", "1", 1); - SBitmapInterface* simple_interface = bitmap_interface_create(); - - // 添加测试数据 - for (int i = 0; i < 100; i++) { - simple_interface->add(simple_interface->bitmap, i); - } - - // 序列化 - size_t simple_size = simple_interface->serialized_size(simple_interface->bitmap); - void* simple_data = malloc(simple_size); - int32_t ser_ret = simple_interface->serialize(simple_interface->bitmap, simple_data, simple_size); - assert(ser_ret == 0); - - // 反序列化 - SBitmapInterface* simple_interface2 = bitmap_interface_create(); - void* tmp_bitmap = NULL; - int32_t deser_ret = simple_interface2->deserialize(&tmp_bitmap, simple_data, simple_size); - assert(deser_ret == 0); - simple_interface2->destroy(simple_interface2->bitmap); - simple_interface2->bitmap = tmp_bitmap; - - // 验证数据一致性 - for (int i = 0; i < 100; i++) { - assert(simple_interface2->contains(simple_interface2->bitmap, i) == true); - } - - print_test_result("SimpleBitmap序列化测试", true); - - // 清理 - free(simple_data); - simple_interface->destroy(simple_interface->bitmap); - free(simple_interface); - simple_interface2->destroy(simple_interface2->bitmap); - free(simple_interface2); - - // 测试RoaringBitmap序列化 - unsetenv("TDENGINE_USE_SIMPLE_BITMAP"); - SBitmapInterface* roaring_interface = bitmap_interface_create(); - - // 添加测试数据 - for (int i = 0; i < 100; i++) { - roaring_interface->add(roaring_interface->bitmap, i); - } - - // 序列化 - size_t roaring_size = roaring_interface->serialized_size(roaring_interface->bitmap); - void* roaring_data = malloc(roaring_size); - ser_ret = roaring_interface->serialize(roaring_interface->bitmap, roaring_data, roaring_size); - assert(ser_ret == 0); - - // 反序列化 - SBitmapInterface* roaring_interface2 = bitmap_interface_create(); - tmp_bitmap = NULL; - deser_ret = roaring_interface2->deserialize(&tmp_bitmap, roaring_data, roaring_size); - assert(deser_ret == 0); - roaring_interface2->destroy(roaring_interface2->bitmap); - roaring_interface2->bitmap = tmp_bitmap; - - // 验证数据一致性 - for (int i = 0; i < 100; i++) { - assert(roaring_interface2->contains(roaring_interface2->bitmap, i) == true); - } - - print_test_result("RoaringBitmap序列化测试", true); - - // 清理 - free(roaring_data); - roaring_interface->destroy(roaring_interface->bitmap); - free(roaring_interface); - roaring_interface2->destroy(roaring_interface2->bitmap); - free(roaring_interface2); -} - -// 测试8: 克隆功能测试 -static void test_clone_functionality() { - printf("\n=== 测试8: 克隆功能测试 ===\n"); - - // 测试SimpleBitmap克隆 - setenv("TDENGINE_USE_SIMPLE_BITMAP", "1", 1); - SBitmapInterface* simple_interface = bitmap_interface_create(); - - // 添加测试数据 - for (int i = 0; i < 50; i++) { - simple_interface->add(simple_interface->bitmap, i); - } - - // 克隆 - void* simple_clone_bitmap = simple_interface->clone(simple_interface->bitmap); - SBitmapInterface* simple_clone = bitmap_interface_create(); - simple_clone->destroy(simple_clone->bitmap); - simple_clone->bitmap = simple_clone_bitmap; - - // 验证克隆数据一致性 - for (int i = 0; i < 50; i++) { - assert(simple_clone->contains(simple_clone->bitmap, i) == true); - } - - print_test_result("SimpleBitmap克隆测试", true); - - // 清理 - simple_interface->destroy(simple_interface->bitmap); - free(simple_interface); - simple_clone->destroy(simple_clone->bitmap); - free(simple_clone); - - // 测试RoaringBitmap克隆 - unsetenv("TDENGINE_USE_SIMPLE_BITMAP"); - SBitmapInterface* roaring_interface = bitmap_interface_create(); - - // 添加测试数据 - for (int i = 0; i < 50; i++) { - roaring_interface->add(roaring_interface->bitmap, i); - } - - // 克隆 - void* roaring_clone_bitmap = roaring_interface->clone(roaring_interface->bitmap); - SBitmapInterface* roaring_clone = bitmap_interface_create(); - roaring_clone->destroy(roaring_clone->bitmap); - roaring_clone->bitmap = roaring_clone_bitmap; - - // 验证克隆数据一致性 - for (int i = 0; i < 50; i++) { - assert(roaring_clone->contains(roaring_clone->bitmap, i) == true); - } - - print_test_result("RoaringBitmap克隆测试", true); - - // 清理 - roaring_interface->destroy(roaring_interface->bitmap); - free(roaring_interface); - roaring_clone->destroy(roaring_clone->bitmap); - free(roaring_clone); -} - -int main() { - printf("开始抽象层接口测试...\n"); - - test_factory_pattern(); - test_environment_control(); - test_roaring_bitmap_direct(); - test_interface_consistency(); - test_performance_comparison(); - test_memory_usage_comparison(); - test_serialization(); - test_clone_functionality(); - - printf("\n所有抽象层接口测试完成！\n"); - return 0; -} diff --git a/plugins/incremental_bitmap/test/test_backup_coordinator.c b/plugins/incremental_bitmap/test/test_backup_coordinator.c deleted file mode 100644 index 53b4c645f3ff..000000000000 --- a/plugins/incremental_bitmap/test/test_backup_coordinator.c +++ /dev/null @@ -1,507 +0,0 @@ -#include "../include/backup_coordinator.h" -#include "../include/bitmap_engine.h" -#include "../include/event_interceptor.h" -#include -#include -#include -#include -#include -#include - -// 测试辅助函数 -static void print_test_result(const char* test_name, bool passed) { - printf("[%s] %s\n", passed ? "PASS" : "FAIL", test_name); -} - -static int64_t get_current_timestamp() { - struct timespec ts; - clock_gettime(CLOCK_REALTIME, &ts); - return (int64_t)ts.tv_sec * 1000000000LL + (int64_t)ts.tv_nsec; -} - -// 测试1: 基本初始化和销毁 -static void test_basic_init_destroy() { - printf("\n=== 测试1: 基本初始化和销毁 ===\n"); - - // 创建位图引擎 - SBitmapEngine* bitmap_engine = bitmap_engine_init(); - assert(bitmap_engine != NULL); - - // 创建事件拦截器 - SEventInterceptorConfig interceptor_config = { - .enable_interception = true, - .event_buffer_size = 1000, - .callback_threads = 2, - .callback = NULL, - .callback_user_data = NULL - }; - - SEventInterceptor* event_interceptor = event_interceptor_init(&interceptor_config, bitmap_engine); - assert(event_interceptor != NULL); - - // 创建备份协同器 - SBackupConfig config = { - .batch_size = 100, - .max_retries = 3, - .retry_interval_ms = 1000, - .timeout_ms = 5000, - .enable_compression = true, - .enable_encryption = false, - .backup_path = NULL, - .temp_path = NULL - }; - - SBackupCoordinator* coordinator = backup_coordinator_init(bitmap_engine, &config); - assert(coordinator != NULL); - print_test_result("初始化备份协同器", true); - - backup_coordinator_destroy(coordinator); - event_interceptor_destroy(event_interceptor); - bitmap_engine_destroy(bitmap_engine); - print_test_result("销毁备份协同器", true); -} - -// 测试2: 获取脏块 -static void test_get_dirty_blocks() { - printf("\n=== 测试2: 获取脏块 ===\n"); - - // 创建组件 - SBitmapEngine* bitmap_engine = bitmap_engine_init(); - assert(bitmap_engine != NULL); - - SEventInterceptorConfig interceptor_config = { - .enable_interception = true, - .event_buffer_size = 1000, - .callback_threads = 2, - .callback = NULL, - .callback_user_data = NULL - }; - - SEventInterceptor* event_interceptor = event_interceptor_init(&interceptor_config, bitmap_engine); - assert(event_interceptor != NULL); - - SBackupConfig config = { - .batch_size = 100, - .max_retries = 3, - .retry_interval_ms = 1000, - .timeout_ms = 5000, - .enable_compression = true, - .enable_encryption = false, - .backup_path = NULL, - .temp_path = NULL - }; - - SBackupCoordinator* coordinator = backup_coordinator_init(bitmap_engine, &config); - assert(coordinator != NULL); - - int64_t timestamp = get_current_timestamp(); - - // 添加一些脏块 - bitmap_engine_mark_dirty(bitmap_engine, 1001, 1000, timestamp); - bitmap_engine_mark_dirty(bitmap_engine, 1002, 2000, timestamp + 1000); - bitmap_engine_mark_dirty(bitmap_engine, 1003, 3000, timestamp + 2000); - bitmap_engine_mark_dirty(bitmap_engine, 1004, 4000, timestamp + 3000); - - // 简单测试：检查位图引擎是否正确标记了脏块 - uint64_t total_blocks, dirty_count, new_count, deleted_count; - bitmap_engine_get_stats(bitmap_engine, &total_blocks, &dirty_count, &new_count, &deleted_count); - - printf("总块数: %lu, 脏块数: %lu\n", total_blocks, dirty_count); - assert(dirty_count >= 4); - print_test_result("标记脏块", true); - - // 测试基本功能：获取所有脏块 - uint64_t block_ids[10]; - uint32_t count = bitmap_engine_get_dirty_blocks_by_wal(bitmap_engine, 0, UINT64_MAX, block_ids, 10); - - printf("找到 %u 个脏块\n", count); - assert(count >= 4); - print_test_result("获取脏块", true); - - backup_coordinator_destroy(coordinator); - event_interceptor_destroy(event_interceptor); - bitmap_engine_destroy(bitmap_engine); -} - -// 测试3: 创建和销毁游标 -static void test_cursor_operations() { - printf("\n=== 测试3: 创建和销毁游标 ===\n"); - - // 创建组件 - SBitmapEngine* bitmap_engine = bitmap_engine_init(); - assert(bitmap_engine != NULL); - - SEventInterceptorConfig interceptor_config = { - .enable_interception = true, - .event_buffer_size = 1000, - .callback_threads = 2, - .callback = NULL, - .callback_user_data = NULL - }; - - SEventInterceptor* event_interceptor = event_interceptor_init(&interceptor_config, bitmap_engine); - assert(event_interceptor != NULL); - - SBackupConfig config = { - .batch_size = 100, - .max_retries = 3, - .retry_interval_ms = 1000, - .timeout_ms = 5000, - .enable_compression = true, - .enable_encryption = false, - .backup_path = NULL, - .temp_path = NULL - }; - - SBackupCoordinator* coordinator = backup_coordinator_init(bitmap_engine, &config); - assert(coordinator != NULL); - - int64_t start_time = get_current_timestamp(); - int64_t end_time = start_time + 1000000; // 1秒后 - - // 设置游标 - SBackupCursor cursor = { - .type = BACKUP_CURSOR_TYPE_TIME, - .time_cursor = start_time, - .wal_cursor = 1000, - .block_count = 0, - .last_update_time = start_time - }; - - int32_t result = backup_coordinator_set_cursor(coordinator, &cursor); - assert(result == 0); - print_test_result("设置游标", true); - - // 验证游标 - SBackupCursor retrieved_cursor; - result = backup_coordinator_get_cursor(coordinator, &retrieved_cursor); - assert(result == 0); - assert(retrieved_cursor.type == BACKUP_CURSOR_TYPE_TIME); - assert(retrieved_cursor.time_cursor == start_time); - print_test_result("获取游标", true); - - backup_coordinator_destroy(coordinator); - event_interceptor_destroy(event_interceptor); - bitmap_engine_destroy(bitmap_engine); -} - -// 测试4: 批量获取增量数据 -static void test_get_next_batch() { - printf("\n=== 测试4: 批量获取增量数据 ===\n"); - - // 创建组件 - SBitmapEngine* bitmap_engine = bitmap_engine_init(); - assert(bitmap_engine != NULL); - - SEventInterceptorConfig interceptor_config = { - .enable_interception = true, - .event_buffer_size = 1000, - .callback_threads = 2, - .callback = NULL, - .callback_user_data = NULL - }; - - SEventInterceptor* event_interceptor = event_interceptor_init(&interceptor_config, bitmap_engine); - assert(event_interceptor != NULL); - - SBackupConfig config = { - .batch_size = 100, - .max_retries = 3, - .retry_interval_ms = 1000, - .timeout_ms = 5000, - .enable_compression = true, - .enable_encryption = false, - .backup_path = NULL, - .temp_path = NULL - }; - - SBackupCoordinator* coordinator = backup_coordinator_init(bitmap_engine, &config); - assert(coordinator != NULL); - - int64_t timestamp = get_current_timestamp(); - - // 添加一些块 - for (int i = 0; i < 10; i++) { - uint64_t block_id = 2000 + i; - bitmap_engine_mark_dirty(bitmap_engine, block_id, block_id * 10, timestamp + i); - } - - // 获取增量块 - SIncrementalBlock blocks[5]; - uint32_t count = backup_coordinator_get_incremental_blocks(coordinator, 20000, 30000, blocks, 5); - - assert(count > 0); - print_test_result("批量获取增量数据", true); - - // 验证块信息 - for (uint32_t i = 0; i < count; i++) { - assert(blocks[i].block_id >= 2000 && blocks[i].block_id < 2010); - } - backup_coordinator_destroy(coordinator); - event_interceptor_destroy(event_interceptor); - bitmap_engine_destroy(bitmap_engine); -} - -// 测试5: 估算备份大小 -static void test_estimate_size() { - printf("\n=== 测试5: 估算备份大小 ===\n"); - - // 创建组件 - SBitmapEngine* bitmap_engine = bitmap_engine_init(); - assert(bitmap_engine != NULL); - - SEventInterceptorConfig interceptor_config = { - .enable_interception = true, - .event_buffer_size = 1000, - .callback_threads = 2, - .callback = NULL, - .callback_user_data = NULL - }; - - SEventInterceptor* event_interceptor = event_interceptor_init(&interceptor_config, bitmap_engine); - assert(event_interceptor != NULL); - - SBackupConfig config = { - .batch_size = 100, - .max_retries = 3, - .retry_interval_ms = 1000, - .timeout_ms = 5000, - .enable_compression = true, - .enable_encryption = false, - .backup_path = NULL, - .temp_path = NULL - }; - - SBackupCoordinator* coordinator = backup_coordinator_init(bitmap_engine, &config); - assert(coordinator != NULL); - - int64_t timestamp = get_current_timestamp(); - - // 添加一些块 - for (int i = 0; i < 20; i++) { - uint64_t block_id = 3000 + i; - bitmap_engine_mark_dirty(bitmap_engine, block_id, block_id * 10, timestamp + i); - } - - // 估算备份大小 - uint64_t estimated_size = backup_coordinator_estimate_backup_size(coordinator, 30000, 50000); - - assert(estimated_size > 0); - print_test_result("估算备份大小", true); - - printf("估算大小: %lu 字节\n", estimated_size); - - backup_coordinator_destroy(coordinator); - event_interceptor_destroy(event_interceptor); - bitmap_engine_destroy(bitmap_engine); -} - -// 测试6: 生成元数据 -static void test_generate_metadata() { - printf("\n=== 测试6: 生成元数据 ===\n"); - - // 创建组件 - SBitmapEngine* bitmap_engine = bitmap_engine_init(); - assert(bitmap_engine != NULL); - - SEventInterceptorConfig interceptor_config = { - .enable_interception = true, - .event_buffer_size = 1000, - .callback_threads = 2, - .callback = NULL, - .callback_user_data = NULL - }; - - SEventInterceptor* event_interceptor = event_interceptor_init(&interceptor_config, bitmap_engine); - assert(event_interceptor != NULL); - - SBackupConfig config = { - .batch_size = 100, - .max_retries = 3, - .retry_interval_ms = 1000, - .timeout_ms = 5000, - .enable_compression = true, - .enable_encryption = false, - .backup_path = NULL, - .temp_path = NULL - }; - - SBackupCoordinator* coordinator = backup_coordinator_init(bitmap_engine, &config); - assert(coordinator != NULL); - - // 获取统计信息 - SBackupStats stats; - int32_t result = backup_coordinator_get_stats(coordinator, &stats); - - assert(result == 0); - print_test_result("获取统计信息", true); - - printf("总块数: %lu, 已处理块数: %lu\n", stats.total_blocks, stats.processed_blocks); - - backup_coordinator_destroy(coordinator); - event_interceptor_destroy(event_interceptor); - bitmap_engine_destroy(bitmap_engine); -} - -// 测试7: 验证备份完整性 -static void test_validate_backup() { - printf("\n=== 测试7: 验证备份完整性 ===\n"); - - // 创建组件 - SBitmapEngine* bitmap_engine = bitmap_engine_init(); - assert(bitmap_engine != NULL); - - SEventInterceptorConfig interceptor_config = { - .enable_interception = true, - .event_buffer_size = 1000, - .callback_threads = 2, - .callback = NULL, - .callback_user_data = NULL - }; - - SEventInterceptor* event_interceptor = event_interceptor_init(&interceptor_config, bitmap_engine); - assert(event_interceptor != NULL); - - SBackupConfig config = { - .batch_size = 100, - .max_retries = 3, - .retry_interval_ms = 1000, - .timeout_ms = 5000, - .enable_compression = true, - .enable_encryption = false, - .backup_path = NULL, - .temp_path = NULL - }; - - SBackupCoordinator* coordinator = backup_coordinator_init(bitmap_engine, &config); - assert(coordinator != NULL); - - int64_t timestamp = get_current_timestamp(); - - // 添加一些块 - for (int i = 0; i < 5; i++) { - uint64_t block_id = 4000 + i; - bitmap_engine_mark_dirty(bitmap_engine, block_id, block_id * 10, timestamp + i); - } - - // 获取增量块进行验证 - SIncrementalBlock blocks[5]; - uint32_t count = backup_coordinator_get_incremental_blocks(coordinator, 40000, 50000, blocks, 5); - - assert(count > 0); - print_test_result("验证备份完整性", true); - - backup_coordinator_destroy(coordinator); - event_interceptor_destroy(event_interceptor); - bitmap_engine_destroy(bitmap_engine); -} - -// 测试8: 统计信息 -static void test_statistics() { - printf("\n=== 测试8: 统计信息 ===\n"); - - // 创建组件 - SBitmapEngine* bitmap_engine = bitmap_engine_init(); - assert(bitmap_engine != NULL); - - SEventInterceptorConfig interceptor_config = { - .enable_interception = true, - .event_buffer_size = 1000, - .callback_threads = 2, - .callback = NULL, - .callback_user_data = NULL - }; - - SEventInterceptor* event_interceptor = event_interceptor_init(&interceptor_config, bitmap_engine); - assert(event_interceptor != NULL); - - SBackupConfig config = { - .batch_size = 100, - .max_retries = 3, - .retry_interval_ms = 1000, - .timeout_ms = 5000, - .enable_compression = true, - .enable_encryption = false, - .backup_path = NULL, - .temp_path = NULL - }; - - SBackupCoordinator* coordinator = backup_coordinator_init(bitmap_engine, &config); - assert(coordinator != NULL); - - // 获取统计信息 - SBackupStats stats; - int32_t result = backup_coordinator_get_stats(coordinator, &stats); - assert(result == 0); - - printf("总块数: %lu, 总大小: %lu 字节, 开始时间: %ld\n", - stats.total_blocks, stats.total_size, stats.start_time); - print_test_result("统计信息", true); - - backup_coordinator_destroy(coordinator); - event_interceptor_destroy(event_interceptor); - bitmap_engine_destroy(bitmap_engine); -} - -// 测试插件接口功能 -// 测试9: 插件接口测试 -static void test_plugin_interface() { - printf("\n=== 测试9: 插件接口测试 ===\n"); - - // 测试插件接口获取 - SBackupPluginInterface* interface = backup_plugin_get_interface(); - assert(interface != NULL); - print_test_result("插件接口获取", true); - - // 测试插件初始化 - int32_t result = interface->init("test_config"); - assert(result == 0); - print_test_result("插件初始化", true); - - // 测试获取脏块（在没有标记脏块的情况下，应该返回0） - uint64_t block_ids[10]; - uint32_t count = interface->get_dirty_blocks(1000, 10000, block_ids, 10); - printf("获取到 %u 个脏块\n", count); - // 在没有标记脏块的情况下，返回0是正常的 - print_test_result("插件获取脏块", true); - - // 测试估算备份大小（在没有脏块的情况下，应该返回0） - uint64_t size = interface->estimate_backup_size(1000, 10000); - printf("估算备份大小: %lu 字节\n", size); - // 在没有脏块的情况下，返回0是正常的 - print_test_result("插件估算备份大小", true); - - // 测试获取统计信息 - SBackupStats stats; - result = interface->get_stats(&stats); - assert(result == 0); - print_test_result("插件获取统计信息", true); - - // 测试重置统计信息 - result = interface->reset_stats(); - assert(result == 0); - print_test_result("插件重置统计信息", true); - - // 测试插件销毁 - interface->destroy(); - print_test_result("插件销毁", true); -} - - -int main() { - printf("开始备份协同器测试...\n"); - - test_basic_init_destroy(); - test_get_dirty_blocks(); - test_cursor_operations(); - test_get_next_batch(); - test_estimate_size(); - test_statistics(); - test_validate_backup(); - test_statistics(); - test_plugin_interface(); - - - printf("\n所有测试完成！\n"); - return 0; -} \ No newline at end of file diff --git a/plugins/incremental_bitmap/test/test_bitmap_engine_core.c b/plugins/incremental_bitmap/test/test_bitmap_engine_core.c deleted file mode 100644 index fae3dbb22bf3..000000000000 --- a/plugins/incremental_bitmap/test/test_bitmap_engine_core.c +++ /dev/null @@ -1,301 +0,0 @@ -#include - -#include "../include/bitmap_engine.h" -#include "../include/bitmap_interface.h" -#include -#include -#include -#include -#include -#include - -// 测试辅助函数 -static void print_test_result(const char* test_name, bool passed) { - printf("[%s] %s\n", passed ? "PASS" : "FAIL", test_name); -} - -static int64_t get_current_timestamp() { - struct timespec ts; - clock_gettime(CLOCK_REALTIME, &ts); - return (int64_t)ts.tv_sec * 1000000000LL + (int64_t)ts.tv_nsec; -} - -// 测试1: 位图引擎基本初始化和销毁 -static void test_bitmap_engine_basic() { - printf("\n=== 测试1: 位图引擎基本初始化和销毁 ===\n"); - - // 测试初始化 - SBitmapEngine* engine = bitmap_engine_init(); - assert(engine != NULL); - print_test_result("位图引擎初始化", true); - - // 测试销毁 - bitmap_engine_destroy(engine); - print_test_result("位图引擎销毁", true); -} - -// 测试2: 位图操作功能 -static void test_bitmap_operations() { - printf("\n=== 测试2: 位图操作功能 ===\n"); - - SBitmapEngine* engine = bitmap_engine_init(); - assert(engine != NULL); - - // 测试标记脏块 - uint64_t test_values[] = {1, 10, 100, 1000, 10000}; - int test_count = sizeof(test_values) / sizeof(test_values[0]); - - for (int i = 0; i < test_count; i++) { - int32_t result = bitmap_engine_mark_dirty(engine, test_values[i], i * 1000, get_current_timestamp()); - assert(result == 0); - } - print_test_result("标记脏块", true); - - // 测试查询块状态 - for (int i = 0; i < test_count; i++) { - EBlockState state; - int32_t result = bitmap_engine_get_block_state(engine, test_values[i], &state); - assert(result == 0); - assert(state == BLOCK_STATE_DIRTY); - } - print_test_result("查询块状态", true); - - // 测试清除块状态 - int32_t result = bitmap_engine_clear_block(engine, 100); - assert(result == 0); - - EBlockState state; - result = bitmap_engine_get_block_state(engine, 100, &state); - assert(result == ERR_BLOCK_NOT_FOUND); - print_test_result("清除块状态", true); - - bitmap_engine_destroy(engine); -} - -// 测试3: 状态转换测试 -static void test_state_transitions() { - printf("\n=== 测试3: 状态转换测试 ===\n"); - - SBitmapEngine* engine = bitmap_engine_init(); - assert(engine != NULL); - - uint64_t block_id = 12345; - int64_t timestamp = get_current_timestamp(); - - // 测试 CLEAN -> DIRTY - int32_t result = bitmap_engine_mark_dirty(engine, block_id, 1000, timestamp); - assert(result == 0); - - EBlockState state; - result = bitmap_engine_get_block_state(engine, block_id, &state); - assert(result == 0); - assert(state == BLOCK_STATE_DIRTY); - print_test_result("CLEAN -> DIRTY 转换", true); - - // 测试 DIRTY -> NEW - result = bitmap_engine_mark_new(engine, block_id, 2000, timestamp); - printf("mark_new result: %d\n", result); - if (result != 0) { - EBlockState current_state; - bitmap_engine_get_block_state(engine, block_id, ¤t_state); - printf("Current state before mark_new: %d\n", current_state); - printf("Error: %s\n", bitmap_engine_get_state_transition_error(current_state, BLOCK_STATE_NEW)); - } - assert(result == 0); - - result = bitmap_engine_get_block_state(engine, block_id, &state); - assert(result == 0); - assert(state == BLOCK_STATE_NEW); - print_test_result("DIRTY -> NEW 转换", true); - - // 测试 NEW -> DELETED - result = bitmap_engine_mark_deleted(engine, block_id, 3000, timestamp); - printf("mark_deleted result: %d\n", result); - if (result != 0) { - EBlockState current_state; - bitmap_engine_get_block_state(engine, block_id, ¤t_state); - printf("Current state before mark_deleted: %d\n", current_state); - printf("Error: %s\n", bitmap_engine_get_state_transition_error(current_state, BLOCK_STATE_DELETED)); - } - assert(result == 0); - - result = bitmap_engine_get_block_state(engine, block_id, &state); - assert(result == 0); - assert(state == BLOCK_STATE_DELETED); - print_test_result("NEW -> DELETED 转换", true); - - bitmap_engine_destroy(engine); -} - -// 测试4: 统计信息测试 -static void test_statistics() { - printf("\n=== 测试4: 统计信息测试 ===\n"); - - SBitmapEngine* engine = bitmap_engine_init(); - assert(engine != NULL); - - // 添加一些测试数据 - for (int i = 0; i < 10; i++) { - bitmap_engine_mark_dirty(engine, i, i * 1000, get_current_timestamp()); - } - - for (int i = 10; i < 20; i++) { - bitmap_engine_mark_new(engine, i, i * 1000, get_current_timestamp()); - } - - for (int i = 20; i < 25; i++) { - bitmap_engine_mark_deleted(engine, i, i * 1000, get_current_timestamp()); - } - - // 获取统计信息 - uint64_t total_blocks, dirty_count, new_count, deleted_count; - bitmap_engine_get_stats(engine, &total_blocks, &dirty_count, &new_count, &deleted_count); - - printf("统计信息: 总块数=%lu, 脏块数=%lu, 新块数=%lu, 删除块数=%lu\n", - total_blocks, dirty_count, new_count, deleted_count); - - assert(dirty_count == 10); - assert(new_count == 10); - assert(deleted_count == 5); - print_test_result("统计信息测试", true); - - bitmap_engine_destroy(engine); -} - -// 测试5: 性能基准测试 -static void test_performance_benchmark() { - printf("\n=== 测试5: 性能基准测试 ===\n"); - - SBitmapEngine* engine = bitmap_engine_init(); - assert(engine != NULL); - - const int benchmark_count = 100000; - int64_t start_time = get_current_timestamp(); - - // 添加性能测试 - for (int i = 0; i < benchmark_count; i++) { - bitmap_engine_mark_dirty(engine, i, i * 1000, get_current_timestamp()); - } - - int64_t add_time = get_current_timestamp() - start_time; - printf("添加 %d 个脏块耗时: %ld ns (%.2f ops/ms)\n", - benchmark_count, add_time, (double)benchmark_count / (add_time / 1000000.0)); - - // 查询性能测试 - start_time = get_current_timestamp(); - for (int i = 0; i < benchmark_count; i++) { - EBlockState state; - bitmap_engine_get_block_state(engine, i, &state); - } - - int64_t query_time = get_current_timestamp() - start_time; - printf("查询 %d 个块状态耗时: %ld ns (%.2f ops/ms)\n", - benchmark_count, query_time, (double)benchmark_count / (query_time / 1000000.0)); - - print_test_result("性能基准测试", true); - - bitmap_engine_destroy(engine); -} - -// 测试6: 并发安全测试 -static void* concurrent_mark_thread(void* arg) { - SBitmapEngine* engine = (SBitmapEngine*)arg; - - for (int i = 0; i < 1000; i++) { - bitmap_engine_mark_dirty(engine, i, i * 1000, get_current_timestamp()); - } - - return NULL; -} - -static void test_concurrent_safety() { - printf("\n=== 测试6: 并发安全测试 ===\n"); - - SBitmapEngine* engine = bitmap_engine_init(); - assert(engine != NULL); - - const int thread_count = 4; - pthread_t threads[thread_count]; - - // 创建多个线程并发标记 - for (int i = 0; i < thread_count; i++) { - int result = pthread_create(&threads[i], NULL, concurrent_mark_thread, engine); - assert(result == 0); - } - - // 等待所有线程完成 - for (int i = 0; i < thread_count; i++) { - pthread_join(threads[i], NULL); - } - - print_test_result("并发安全测试", true); - - bitmap_engine_destroy(engine); -} - -// 测试7: 时间范围查询测试 -static void test_time_range_query() { - printf("\n=== 测试7: 时间范围查询测试 ===\n"); - - SBitmapEngine* engine = bitmap_engine_init(); - assert(engine != NULL); - - int64_t base_time = get_current_timestamp(); - - // 添加不同时间戳的块 - for (int i = 0; i < 100; i++) { - bitmap_engine_mark_dirty(engine, i, i * 1000, base_time + i * 1000000); - } - - // 查询时间范围内的块 - uint64_t block_ids[50]; - uint32_t count = bitmap_engine_get_dirty_blocks_by_time(engine, - base_time + 10 * 1000000, - base_time + 20 * 1000000, - block_ids, 50); - - printf("时间范围内找到 %u 个块\n", count); - assert(count > 0); - print_test_result("时间范围查询测试", true); - - bitmap_engine_destroy(engine); -} - -// 测试8: WAL范围查询测试 -static void test_wal_range_query() { - printf("\n=== 测试8: WAL范围查询测试 ===\n"); - - SBitmapEngine* engine = bitmap_engine_init(); - assert(engine != NULL); - - // 添加不同WAL偏移量的块 - for (int i = 0; i < 100; i++) { - bitmap_engine_mark_dirty(engine, i, i * 1000, get_current_timestamp()); - } - - // 查询WAL范围内的块 - uint64_t block_ids[50]; - uint32_t count = bitmap_engine_get_dirty_blocks_by_wal(engine, 5000, 15000, block_ids, 50); - - printf("WAL范围内找到 %u 个块\n", count); - assert(count > 0); - print_test_result("WAL范围查询测试", true); - - bitmap_engine_destroy(engine); -} - -int main() { - printf("开始位图引擎核心功能测试...\n"); - - test_bitmap_engine_basic(); - test_bitmap_operations(); - test_state_transitions(); - test_statistics(); - test_performance_benchmark(); - test_concurrent_safety(); - test_time_range_query(); - test_wal_range_query(); - - printf("\n所有位图引擎核心功能测试完成！\n"); - return 0; -} diff --git a/plugins/incremental_bitmap/test/test_consistency_minimal.c b/plugins/incremental_bitmap/test/test_consistency_minimal.c deleted file mode 100644 index fc6294d96621..000000000000 --- a/plugins/incremental_bitmap/test/test_consistency_minimal.c +++ /dev/null @@ -1,141 +0,0 @@ -#include "pitr_e2e_test.h" -#include -#include -#include -#include -#include - -// 最小化一致性验证配置 -static SPitrTestConfig minimal_consistency_config = { - .snapshot_interval_ms = 500, // 0.5秒间隔 - .recovery_points = 2, // 2个恢复点 - .data_block_count = 50, // 50个数据块 - .concurrent_writers = 1, // 1个并发写入线程 - .test_duration_seconds = 10, // 10秒测试 - .enable_disorder_test = false, // 禁用复杂测试 - .enable_deletion_test = false, // 禁用复杂测试 - .test_data_path = "./build/pitr_test_data", - .snapshot_path = "./build/pitr_snapshots", - .recovery_path = "./build/pitr_recovery" -}; - -// 创建最小化一致性验证数据 -int create_minimal_consistency_data() { - printf("🔧 创建最小化一致性验证数据...\n"); - - // 创建PITR测试器 - SPitrTester* tester = pitr_tester_create(&minimal_consistency_config); - if (!tester) { - fprintf(stderr, "❌ 无法创建PITR测试器\n"); - return -1; - } - - printf("✅ PITR测试器创建成功\n"); - - // 运行快照测试 - printf("📸 创建快照数据...\n"); - int rc = pitr_tester_run_snapshot_test(tester); - if (rc != 0) { - fprintf(stderr, "❌ 快照测试失败: %d\n", rc); - pitr_tester_destroy(tester); - return -1; - } - - printf("✅ 快照数据创建成功\n"); - - // 运行恢复测试 - printf("🔄 创建恢复点数据...\n"); - rc = pitr_tester_run_recovery_test(tester); - if (rc != 0) { - fprintf(stderr, "❌ 恢复测试失败: %d\n", rc); - pitr_tester_destroy(tester); - return -1; - } - - printf("✅ 恢复点数据创建成功\n"); - - // 获取测试状态 - SPitrTestStatus status; - rc = pitr_tester_get_status(tester, &status); - if (rc == 0) { - printf("📊 测试状态: 快照=%lu, 恢复点=%lu, 一致性检查=%lu\n", - status.snapshots_created, status.recovery_points_verified, - status.data_consistency_checks); - } - - // 清理 - pitr_tester_destroy(tester); - - printf("✅ 最小化一致性验证数据创建完成\n"); - return 0; -} - -// 验证生成的数据文件 -int verify_consistency_files() { - printf("🔍 验证一致性文件...\n"); - - const char* paths[] = { - "./build/pitr_test_data", - "./build/pitr_snapshots", - "./build/pitr_recovery" - }; - - for (int i = 0; i < 3; i++) { - struct stat st; - if (stat(paths[i], &st) == 0) { - printf("✅ 目录存在: %s\n", paths[i]); - - // 检查快照文件 - if (i == 1) { // snapshots目录 - char snapshot_file[256]; - for (int j = 0; j < 3; j++) { - snprintf(snapshot_file, sizeof(snapshot_file), - "%s/snapshot_%d_data.bin", paths[i], j); - if (stat(snapshot_file, &st) == 0) { - printf("✅ 快照文件存在: %s (%ld bytes)\n", - snapshot_file, st.st_size); - } - } - } - } else { - printf("❌ 目录不存在: %s\n", paths[i]); - return -1; - } - } - - printf("✅ 一致性文件验证完成\n"); - return 0; -} - -int main() { - printf("==========================================\n"); - printf(" 最小化一致性验证数据生成器\n"); - printf("==========================================\n"); - - // 创建目录 - printf("📁 创建测试目录...\n"); - mkdir("./build/pitr_test_data", 0755); - mkdir("./build/pitr_snapshots", 0755); - mkdir("./build/pitr_recovery", 0755); - - // 创建最小化一致性数据 - int rc = create_minimal_consistency_data(); - if (rc != 0) { - fprintf(stderr, "❌ 创建一致性数据失败\n"); - return 1; - } - - // 验证生成的文件 - rc = verify_consistency_files(); - if (rc != 0) { - fprintf(stderr, "❌ 文件验证失败\n"); - return 1; - } - - printf("\n==========================================\n"); - printf("✅ 最小化一致性验证数据生成成功！\n"); - printf("现在可以运行 test_e2e_tdengine_real 进行一致性验证\n"); - printf("==========================================\n"); - - return 0; -} diff --git a/plugins/incremental_bitmap/test/test_e2e_tdengine_real.c b/plugins/incremental_bitmap/test/test_e2e_tdengine_real.c deleted file mode 100644 index b093ed222565..000000000000 --- a/plugins/incremental_bitmap/test/test_e2e_tdengine_real.c +++ /dev/null @@ -1,309 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include "../include/storage_engine_interface.h" -#include "../include/e2e_consistency.h" -#include "../include/e2e_perf.h" - -// E2E测试配置结构 -typedef struct { - const char* topic_name; - const char* group_id; - const char* server_addr; - int test_duration_sec; - int expected_events; - uint64_t max_data_volume_mb; // 最大数据量限制(MB) - uint64_t max_export_operations; // 最大导出操作次数 - bool enable_real_performance; // 是否启用真实性能测试 -} E2ETestConfig; - -// E2E统计结构 -typedef struct { - int events_received; - int events_processed; - int bitmap_updates; - int export_operations; - uint64_t total_data_size; - pthread_mutex_t stats_mutex; -} E2EStats; - -// 全局统计变量 -static E2EStats g_e2e_stats = {0}; -static pthread_mutex_t g_stats_mutex = PTHREAD_MUTEX_INITIALIZER; - -// 前向声明 -static void print_step(const char* step); -static int run_tmq_pipeline_test(const E2ETestConfig* config, E2EPerfTimer* timer); -static void e2e_event_callback(const SStorageEvent* event, void* user_data); -static int run_closed_loop_taosdump_comparison(void); - -// 打印步骤信息 -static void print_step(const char* step) { - printf("\n==========================================\n"); - printf(" %s\n", step); - printf("==========================================\n"); -} - -// 事件回调函数 -static void e2e_event_callback(const SStorageEvent* event, void* user_data) { - if (!event || !user_data) return; - - const E2ETestConfig* config = (const E2ETestConfig*)user_data; - - // 使用原子操作减少锁竞争 - __sync_fetch_and_add(&g_e2e_stats.events_received, 1); - __sync_fetch_and_add(&g_e2e_stats.events_processed, 1); - __sync_fetch_and_add(&g_e2e_stats.bitmap_updates, 1); - - // 批量处理导出操作，减少锁竞争 - static __thread int local_export_count = 0; - local_export_count++; - - if (local_export_count >= 100) { // 每100个事件批量处理一次导出 - pthread_mutex_lock(&g_stats_mutex); - if (g_e2e_stats.export_operations < config->max_export_operations) { - g_e2e_stats.export_operations += local_export_count; - g_e2e_stats.total_data_size += local_export_count * 1024; - } - pthread_mutex_unlock(&g_stats_mutex); - local_export_count = 0; - } - - // 移除人工延迟，让位图引擎展示真实性能 - // if (!config->enable_real_performance) { - // usleep(100); // 移除这个延迟 - // } -} - -// TMQ管道测试 -static int run_tmq_pipeline_test(const E2ETestConfig* config, E2EPerfTimer* timer) { - if (!config || !timer) return -1; - - e2e_perf_timer_start(timer); - - // 获取存储引擎接口 - SStorageEngineInterface* engine = get_storage_engine_interface("tdengine"); - if (!engine) { - printf(" [ERROR] 无法获取TDengine存储引擎接口\n"); - return -1; - } - - // 初始化存储引擎配置 - 优化性能 - SStorageEngineConfig engine_config = { - .enable_interception = true, - .event_callback = (StorageEventCallback)e2e_event_callback, - .callback_user_data = (void*)config, - .event_buffer_size = 10000, // 增大缓冲区 - .callback_threads = 4 // 使用4个线程并行处理 - }; - - // 初始化引擎 - int rc = engine->init(&engine_config); - if (rc != 0) { - printf(" [ERROR] 存储引擎初始化失败: %d\n", rc); - return rc; - } - - // 安装事件拦截 - rc = engine->install_interception(); - if (rc != 0) { - printf(" [ERROR] 事件拦截安装失败: %d\n", rc); - engine->destroy(); - return rc; - } - - // 批量事件处理 - 优化性能 - const int batch_size = 1000; // 每批处理1000个事件 - for (int batch = 0; batch < (config->expected_events + batch_size - 1) / batch_size; batch++) { - int start_idx = batch * batch_size; - int end_idx = (start_idx + batch_size < config->expected_events) ? - start_idx + batch_size : config->expected_events; - - // 批量生成事件 - for (int i = start_idx; i < end_idx; i++) { - SStorageEvent event = { - .block_id = i, - .wal_offset = i * 1000, - .timestamp = time(NULL) * 1000 + i, - .event_type = 0 // STORAGE_EVENT_WRITE = 0 - }; - - // 调用事件回调 - e2e_event_callback(&event, (void*)config); - } - - // 批量检查数据量限制 - pthread_mutex_lock(&g_stats_mutex); - uint64_t current_data_mb = g_e2e_stats.total_data_size / (1024 * 1024); - pthread_mutex_unlock(&g_stats_mutex); - - if (current_data_mb >= config->max_data_volume_mb) { - printf(" [INFO] 达到数据量限制 (%lu MB)，停止生成更多数据\n", config->max_data_volume_mb); - break; - } - } - - // 清理 - engine->uninstall_interception(); - engine->destroy(); - - e2e_perf_timer_stop(timer); - return 0; -} - -// 运行闭环验证：调用现有 taosdump 对比测试（二进制应与本测试同目录） -static int run_closed_loop_taosdump_comparison(void) { - printf("\n[闭环] 真实数据→导出→对比\n"); - // 将输出重定向，便于脚本进一步校验 - int rc = system("./test_taosdump_comparison > /tmp/test_taosdump_comparison_inline.log 2>&1"); - if (rc == 0) { - printf(" 闭环验证: ✓ PASSED (日志: /tmp/test_taosdump_comparison_inline.log)\n"); - return 0; - } else { - printf(" 闭环验证: ✗ FAILED (日志: /tmp/test_taosdump_comparison_inline.log, rc=%d)\n", rc); - return -1; - } -} - -int main(void) { - print_step("Real TDengine E2E Test"); - - // 设置TMQ环境变量 - setenv("TMQ_TOPIC_NAME", "test_bitmap_events_topic", 1); - setenv("TMQ_GROUP_ID", "e2e_test_group", 1); - setenv("TMQ_SERVER_ADDR", "127.0.0.1:6030", 1); - - // 注册TDengine存储引擎 - extern SStorageEngineInterface* tdengine_storage_engine_create(void); - if (register_storage_engine_interface("tdengine", tdengine_storage_engine_create) != 0) { - printf(" [ERROR] 注册TDengine存储引擎失败\n"); - return 1; - } - - // 测试配置 - 优化性能测试 - E2ETestConfig config = { - .topic_name = "test_bitmap_events_topic", - .group_id = "e2e_test_group", - .server_addr = "127.0.0.1:6030", - .test_duration_sec = 30, - .expected_events = 100000, // 增加事件数量以测试性能 - .max_data_volume_mb = 100, // 限制最大数据量为100MB - .max_export_operations = 10000, // 增加导出操作次数 - .enable_real_performance = true // 启用真实性能测试 - }; - - printf("[1/3] 事件→位图→协调器→导出\n"); - E2EPerfTimer pipeline_timer; - int rc = run_tmq_pipeline_test(&config, &pipeline_timer); - if (rc == 0) { - pthread_mutex_lock(&g_stats_mutex); - int events_rx = g_e2e_stats.events_received; - int events_proc = g_e2e_stats.events_processed; - int bitmap_ups = g_e2e_stats.bitmap_updates; - int exports = g_e2e_stats.export_operations; - uint64_t total_data_size = g_e2e_stats.total_data_size; - pthread_mutex_unlock(&g_stats_mutex); - - e2e_perf_print_summary("TMQ Pipeline", events_proc, &pipeline_timer); - printf(" 统计: 接收=%d, 处理=%d, 位图更新=%d, 导出=%d\n", - events_rx, events_proc, bitmap_ups, exports); - printf(" 数据量: %lu bytes (%.2f MB)\n", - total_data_size, total_data_size / (1024.0 * 1024.0)); - } else { - printf(" [ERROR] TMQ链路测试失败\n"); - } - - printf("\n[2/3] 数据一致性验证\n"); - - E2EConsistencyConfig cfg = { - .data_path = "./build/pitr_test_data", - .snapshot_path = "./build/pitr_snapshots", - .recovery_path = "./build/pitr_recovery", - }; - - E2EConsistencyReport consistency_report; - int consistency_rc; - - // 检查是否存在一致性验证数据，如果不存在则生成 - struct stat st; - if (stat("./build/pitr_snapshots", &st) != 0) { - printf(" [INFO] 未发现一致性验证数据，正在生成最小化测试数据...\n"); - printf(" 运行: ./build/test_consistency_minimal\n"); - - // 尝试运行最小化一致性数据生成器 - int gen_rc = system("./build/test_consistency_minimal"); - if (gen_rc != 0) { - printf(" [WARNING] 一致性数据生成失败，跳过一致性验证\n"); - printf(" 说明: 核心TMQ管道功能已验证，一致性测试需要PITR环境\n"); - consistency_rc = 0; // 标记为通过 - } else { - printf(" [INFO] 一致性验证数据生成成功，开始验证...\n"); - consistency_rc = e2e_validate_consistency_detailed(&cfg, &consistency_report); - } - } else { - consistency_rc = e2e_validate_consistency_detailed(&cfg, &consistency_report); - } - if (consistency_rc == 0) { - e2e_print_consistency_report(&consistency_report); - } else { - printf(" [WARNING] 一致性验证失败: 路径不存在或不可访问 (rc=%d)\n", consistency_rc); - printf(" 说明: 核心TMQ管道功能已验证，一致性测试需要完整PITR环境\n"); - } - - // 闭环验证（小数据集）：依赖现有对比测试完整跑通导出与比对 - int closed_loop_rc = run_closed_loop_taosdump_comparison(); - - printf("\n[3/3] 端到端性能汇总\n"); - E2EPerfTimer total_timer; - e2e_perf_timer_start(&total_timer); - - // 模拟各阶段性能计时 - printf(" 性能分析:\n"); - - // 事件处理阶段性能 - double pipeline_ms = e2e_perf_elapsed_ms(&pipeline_timer); - pthread_mutex_lock(&g_stats_mutex); - int total_events = g_e2e_stats.events_processed; - pthread_mutex_unlock(&g_stats_mutex); - - if (total_events > 0 && pipeline_ms > 0) { - double event_tps = e2e_perf_throughput_per_sec(total_events, &pipeline_timer); - printf(" - 事件处理: %.2f ms, %d 事件, %.2f events/sec\n", - pipeline_ms, total_events, event_tps); - printf(" - 平均延迟: %.3f ms/event\n", pipeline_ms / total_events); - } - - // 一致性验证阶段性能 - if (consistency_rc == 0) { - printf(" - 一致性验证: %.2f ms, %llu 快照, %llu 恢复点\n", - consistency_report.validation_time_ms, - (unsigned long long)consistency_report.snapshots_checked, - (unsigned long long)consistency_report.recovery_points_checked); - } - - e2e_perf_timer_stop(&total_timer); - double total_ms = e2e_perf_elapsed_ms(&total_timer); - - printf(" 总耗时: %.2f ms\n", total_ms); - - // 整体测试状态 - printf("\n==========================================\n"); - printf(" E2E Test Summary\n"); - printf("==========================================\n"); - printf("TMQ Pipeline: %s\n", rc == 0 ? "✓ PASSED" : "✗ FAILED"); - printf("Consistency Check: %s\n", consistency_rc == 0 ? "✓ PASSED" : "⚠ SKIPPED"); - printf("Performance Analysis: ✓ COMPLETED\n"); - printf("Data Volume Control: ✓ ENABLED\n"); - printf("Closed Loop (Export): %s\n", closed_loop_rc == 0 ? "✓ PASSED" : "✗ FAILED"); - - bool overall_pass = (rc == 0) && (consistency_rc == 0 || consistency_rc < 0) && (closed_loop_rc == 0); - printf("Overall Status: %s\n", overall_pass ? "✓ PASSED" : "⚠ PARTIAL"); - printf("==========================================\n"); - - return overall_pass ? 0 : 1; -} diff --git a/plugins/incremental_bitmap/test/test_event_interceptor.c b/plugins/incremental_bitmap/test/test_event_interceptor.c deleted file mode 100644 index 5f86b1fff0e6..000000000000 --- a/plugins/incremental_bitmap/test/test_event_interceptor.c +++ /dev/null @@ -1,400 +0,0 @@ -#include - -#include "../include/event_interceptor.h" -#include "../include/bitmap_engine.h" -#include -#include -#include -#include -#include -#include - -// 全局变量用于测试 -static uint64_t g_event_count = 0; -static pthread_mutex_t g_event_mutex = PTHREAD_MUTEX_INITIALIZER; - -// 事件回调函数 -static void test_event_callback(const SBlockEvent* event, void* user_data) { - pthread_mutex_lock(&g_event_mutex); - g_event_count++; - printf("收到事件: 类型=%d, 块ID=%lu, WAL偏移量=%lu, 时间戳=%ld\n", - event->event_type, event->block_id, event->wal_offset, event->timestamp); - pthread_mutex_unlock(&g_event_mutex); -} - -// 测试辅助函数 -static void print_test_result(const char* test_name, bool passed) { - printf("[%s] %s\n", passed ? "PASS" : "FAIL", test_name); -} - -static int64_t get_current_timestamp() { - struct timespec ts; - clock_gettime(CLOCK_REALTIME, &ts); - return (int64_t)ts.tv_sec * 1000000000LL + (int64_t)ts.tv_nsec; -} - -// 测试1: 基本初始化和销毁 -static void test_basic_init_destroy() { - printf("\n=== 测试1: 基本初始化和销毁 ===\n"); - - // 创建位图引擎 - - SBitmapEngine* bitmap_engine = bitmap_engine_init(); - assert(bitmap_engine != NULL); - - // 创建事件拦截器 - SEventInterceptorConfig config = { - .enable_interception = true, - .event_buffer_size = 1000, - .callback_threads = 2, - .callback = test_event_callback, - .callback_user_data = NULL - }; - - SEventInterceptor* interceptor = event_interceptor_init(&config, bitmap_engine); - assert(interceptor != NULL); - print_test_result("初始化事件拦截器", true); - - event_interceptor_destroy(interceptor); - bitmap_engine_destroy(bitmap_engine); - print_test_result("销毁事件拦截器", true); -} - -// 测试2: 事件处理 -static void test_event_processing() { - printf("\n=== 测试2: 事件处理 ===\n"); - - // 创建位图引擎 - - SBitmapEngine* bitmap_engine = bitmap_engine_init(); - assert(bitmap_engine != NULL); - - // 创建事件拦截器 - SEventInterceptorConfig config = { - .enable_interception = true, - .event_buffer_size = 1000, - .callback_threads = 2, - .callback = test_event_callback, - .callback_user_data = NULL - }; - - SEventInterceptor* interceptor = event_interceptor_init(&config, bitmap_engine); - assert(interceptor != NULL); - - // 启动拦截器 - int32_t result = event_interceptor_start(interceptor); - assert(result == 0); - print_test_result("启动事件拦截器", true); - - // 重置事件计数 - pthread_mutex_lock(&g_event_mutex); - g_event_count = 0; - pthread_mutex_unlock(&g_event_mutex); - - int64_t timestamp = get_current_timestamp(); - - // 发送各种事件 - event_interceptor_on_block_create(interceptor, 1001, 1000, timestamp); - event_interceptor_on_block_update(interceptor, 1002, 2000, timestamp + 1000); - event_interceptor_on_block_flush(interceptor, 1003, 3000, timestamp + 2000); - event_interceptor_on_block_delete(interceptor, 1004, 4000, timestamp + 3000); - - // 等待事件处理 - usleep(100000); // 100ms - - // 验证事件计数 - pthread_mutex_lock(&g_event_mutex); - assert(g_event_count == 4); - pthread_mutex_unlock(&g_event_mutex); - print_test_result("事件处理", true); - - // 停止拦截器 - result = event_interceptor_stop(interceptor); - assert(result == 0); - print_test_result("停止事件拦截器", true); - - event_interceptor_destroy(interceptor); - bitmap_engine_destroy(bitmap_engine); -} - -// 测试3: 位图引擎集成 -static void test_bitmap_engine_integration() { - printf("\n=== 测试3: 位图引擎集成 ===\n"); - - // 创建位图引擎 - - SBitmapEngine* bitmap_engine = bitmap_engine_init(); - assert(bitmap_engine != NULL); - - // 创建事件拦截器 - SEventInterceptorConfig config = { - .enable_interception = true, - .event_buffer_size = 1000, - .callback_threads = 2, - .callback = NULL, // 不使用回调 - .callback_user_data = NULL - }; - - SEventInterceptor* interceptor = event_interceptor_init(&config, bitmap_engine); - assert(interceptor != NULL); - - // 启动事件拦截器 - int32_t result = event_interceptor_start(interceptor); - assert(result == 0); - - int64_t timestamp = get_current_timestamp(); - - // 发送事件并验证位图引擎状态 - event_interceptor_on_block_create(interceptor, 2001, 1000, timestamp); - event_interceptor_on_block_update(interceptor, 2002, 2000, timestamp + 1000); - - // 等待事件处理完成 - usleep(100000); // 100ms - - // 验证位图引擎统计信息 - uint64_t total_blocks, dirty_count, new_count, deleted_count; - bitmap_engine_get_stats(bitmap_engine, &total_blocks, &dirty_count, &new_count, &deleted_count); - - printf("实际 total_blocks = %lu\n", total_blocks); - assert(total_blocks == 2); - assert(dirty_count == 1); - assert(new_count == 1); - print_test_result("位图引擎集成", true); - - event_interceptor_destroy(interceptor); - bitmap_engine_destroy(bitmap_engine); -} - -// 测试4: 事件缓冲区溢出 -static void test_event_buffer_overflow() { - printf("\n=== 测试4: 事件缓冲区溢出 ===\n"); - - // 创建位图引擎 - - SBitmapEngine* bitmap_engine = bitmap_engine_init(); - assert(bitmap_engine != NULL); - - // 创建小缓冲区的事件拦截器 - SEventInterceptorConfig config = { - .enable_interception = true, - .event_buffer_size = 5, // 很小的缓冲区 - .callback_threads = 1, - .callback = NULL, // 不使用回调，让缓冲区填满 - .callback_user_data = NULL - }; - - SEventInterceptor* interceptor = event_interceptor_init(&config, bitmap_engine); - assert(interceptor != NULL); - - int64_t timestamp = get_current_timestamp(); - - // 发送超过缓冲区大小的事件 - for (int i = 0; i < 10; i++) { - int32_t result = event_interceptor_on_block_update(interceptor, 3000 + i, 1000 + i, timestamp + i); - if (i >= 5) { - // 缓冲区满后应该返回错误 - assert(result != 0); - } - } - - print_test_result("事件缓冲区溢出处理", true); - - event_interceptor_destroy(interceptor); - bitmap_engine_destroy(bitmap_engine); -} - -// 测试5: 并发事件处理 -static void* concurrent_event_thread(void* arg) { - SEventInterceptor* interceptor = (SEventInterceptor*)arg; - int64_t timestamp = get_current_timestamp(); - - for (int i = 0; i < 50; i++) { - uint64_t block_id = 4000 + i; - event_interceptor_on_block_update(interceptor, block_id, block_id * 10, timestamp + i); - } - - return NULL; -} - -static void test_concurrent_event_processing() { - printf("\n=== 测试5: 并发事件处理 ===\n"); - - // 创建位图引擎 - - SBitmapEngine* bitmap_engine = bitmap_engine_init(); - assert(bitmap_engine != NULL); - - // 创建事件拦截器 - SEventInterceptorConfig config = { - .enable_interception = true, - .event_buffer_size = 1000, - .callback_threads = 4, - .callback = test_event_callback, - .callback_user_data = NULL - }; - - SEventInterceptor* interceptor = event_interceptor_init(&config, bitmap_engine); - assert(interceptor != NULL); - - // 启动拦截器 - event_interceptor_start(interceptor); - - // 重置事件计数 - pthread_mutex_lock(&g_event_mutex); - g_event_count = 0; - pthread_mutex_unlock(&g_event_mutex); - - // 创建多个线程并发发送事件 - pthread_t threads[4]; - for (int i = 0; i < 4; i++) { - pthread_create(&threads[i], NULL, concurrent_event_thread, interceptor); - } - - // 等待所有线程完成 - for (int i = 0; i < 4; i++) { - pthread_join(threads[i], NULL); - } - - // 等待事件处理完成 - usleep(500000); // 500ms - - // 验证事件计数 - pthread_mutex_lock(&g_event_mutex); - assert(g_event_count == 200); // 4个线程 * 50个事件 - pthread_mutex_unlock(&g_event_mutex); - print_test_result("并发事件处理", true); - - // 停止拦截器 - event_interceptor_stop(interceptor); - event_interceptor_destroy(interceptor); - bitmap_engine_destroy(bitmap_engine); -} - -// 测试6: 统计信息 -static void test_statistics() { - printf("\n=== 测试6: 统计信息 ===\n"); - - // 创建位图引擎 - - SBitmapEngine* bitmap_engine = bitmap_engine_init(); - assert(bitmap_engine != NULL); - - // 创建事件拦截器 - SEventInterceptorConfig config = { - .enable_interception = true, - .event_buffer_size = 1000, - .callback_threads = 2, - .callback = NULL, - .callback_user_data = NULL - }; - - SEventInterceptor* interceptor = event_interceptor_init(&config, bitmap_engine); - assert(interceptor != NULL); - - int64_t timestamp = get_current_timestamp(); - - // 发送一些事件 - event_interceptor_on_block_create(interceptor, 5001, 1000, timestamp); - event_interceptor_on_block_update(interceptor, 5002, 2000, timestamp + 1000); - event_interceptor_on_block_update(interceptor, 5003, 3000, timestamp + 2000); - - // 获取统计信息 - uint64_t events_processed, events_dropped; - event_interceptor_get_stats(interceptor, &events_processed, &events_dropped); - - print_test_result("统计信息", true); - - event_interceptor_destroy(interceptor); - bitmap_engine_destroy(bitmap_engine); -} - -// 测试7: 性能测试 -static void test_performance() { - printf("\n=== 测试7: 性能测试 ===\n"); - - // 创建位图引擎 - SBitmapEngine* bitmap_engine = bitmap_engine_init(); - assert(bitmap_engine != NULL); - - // 创建事件拦截器 - 增加缓冲区大小和线程数 - SEventInterceptorConfig config = { - .enable_interception = true, - .event_buffer_size = 50000, // 增加缓冲区大小 - .callback_threads = 8, // 增加线程数 - .callback = NULL, // 不使用回调以提高性能 - .callback_user_data = NULL - }; - - SEventInterceptor* interceptor = event_interceptor_init(&config, bitmap_engine); - assert(interceptor != NULL); - - // 启动事件拦截器 - int32_t result = event_interceptor_start(interceptor); - assert(result == 0); - - int64_t start_time = get_current_timestamp(); - - // 批量发送事件 - 减少事件数量以避免过载 - const int event_count = 1000; // 减少到1000个事件 - for (int i = 0; i < event_count; i++) { - uint64_t block_id = 6000 + i; - event_interceptor_on_block_update(interceptor, block_id, block_id * 10, start_time + i); - } - - int64_t end_time = get_current_timestamp(); - double duration_ms = (end_time - start_time) / 1000000.0; - - // 等待事件处理完成 - 增加等待时间 - usleep(1000000); // 1秒 - - // 再次等待，直到所有事件都被处理 - int wait_count = 0; - uint64_t events_processed = 0, events_dropped = 0; - while (wait_count < 100) { // 最多等待10秒 - event_interceptor_get_stats(interceptor, &events_processed, &events_dropped); - - if (events_processed >= event_count) { - break; // 所有事件都已处理 - } - - usleep(100000); // 100ms - wait_count++; - } - - // 验证结果 - uint64_t total_blocks, dirty_count, new_count, deleted_count; - bitmap_engine_get_stats(bitmap_engine, &total_blocks, &dirty_count, &new_count, &deleted_count); - - printf("性能测试结果: 处理了 %lu 个事件，耗时 %.2f ms\n", events_processed, duration_ms); - printf("位图引擎统计: 总块数=%lu, 脏块数=%lu, 新块数=%lu, 删除块数=%lu\n", - total_blocks, dirty_count, new_count, deleted_count); - printf("丢弃事件数: %lu\n", events_dropped); - - // 验证所有事件都被处理了 - assert(events_processed == event_count); - assert(events_dropped == 0); - - // 验证位图引擎中的块数量 - assert(total_blocks == event_count); - assert(dirty_count == event_count); - - print_test_result("性能测试", true); - - event_interceptor_destroy(interceptor); - bitmap_engine_destroy(bitmap_engine); -} - -int main() { - printf("开始事件拦截器测试...\n"); - - test_basic_init_destroy(); - test_event_processing(); - test_bitmap_engine_integration(); - test_event_buffer_overflow(); - test_concurrent_event_processing(); - test_statistics(); - test_performance(); - - printf("\n所有测试完成！\n"); - return 0; -} \ No newline at end of file diff --git a/plugins/incremental_bitmap/test/test_fault_injection.c b/plugins/incremental_bitmap/test/test_fault_injection.c deleted file mode 100644 index a17de354857a..000000000000 --- a/plugins/incremental_bitmap/test/test_fault_injection.c +++ /dev/null @@ -1,623 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "../include/storage_engine_interface.h" -#include "../include/observability.h" - -// 测试宏定义 -#define TEST_ASSERT(condition, message) do { \ - total_tests++; \ - if (condition) { \ - passed_tests++; \ - printf("✓ %s\n", message); \ - } else { \ - failed_tests++; \ - printf("❌ %s\n", message); \ - } \ -} while(0) - -// 测试计数器 -static int total_tests = 0; -static int passed_tests = 0; -static int failed_tests = 0; - -// 故障注入状态 -typedef struct { - bool connection_failure; - bool subscription_failure; - bool parse_failure; - bool commit_failure; - int failure_count; - int max_failures; - pthread_mutex_t mutex; -} SFaultInjectionState; - -static SFaultInjectionState g_fault_state = { - .connection_failure = false, - .subscription_failure = false, - .parse_failure = false, - .commit_failure = false, - .failure_count = 0, - .max_failures = 3, - .mutex = PTHREAD_MUTEX_INITIALIZER -}; - -// 模拟连接状态 -typedef struct { - bool connected; - int reconnect_attempts; - int max_reconnect_attempts; - time_t last_connect_time; - time_t last_disconnect_time; - pthread_mutex_t mutex; -} SConnectionState; - -static SConnectionState g_connection_state = { - .connected = true, - .reconnect_attempts = 0, - .max_reconnect_attempts = 5, - .last_connect_time = 0, - .last_disconnect_time = 0, - .mutex = PTHREAD_MUTEX_INITIALIZER -}; - -// 故障注入控制函数 -void inject_connection_failure(bool enable, int max_failures) { - pthread_mutex_lock(&g_fault_state.mutex); - g_fault_state.connection_failure = enable; - g_fault_state.max_failures = max_failures; - g_fault_state.failure_count = 0; - pthread_mutex_unlock(&g_fault_state.mutex); -} - -void inject_subscription_failure(bool enable, int max_failures) { - pthread_mutex_lock(&g_fault_state.mutex); - g_fault_state.subscription_failure = enable; - g_fault_state.max_failures = max_failures; - g_fault_state.failure_count = 0; - pthread_mutex_unlock(&g_fault_state.mutex); -} - -void inject_parse_failure(bool enable, int max_failures) { - pthread_mutex_lock(&g_fault_state.mutex); - g_fault_state.parse_failure = enable; - g_fault_state.max_failures = max_failures; - g_fault_state.failure_count = 0; - pthread_mutex_unlock(&g_fault_state.mutex); -} - -void inject_commit_failure(bool enable, int max_failures) { - pthread_mutex_lock(&g_fault_state.mutex); - g_fault_state.commit_failure = enable; - g_fault_state.max_failures = max_failures; - g_fault_state.failure_count = 0; - pthread_mutex_unlock(&g_fault_state.mutex); -} - -// 模拟连接函数 -int mock_connect() { - pthread_mutex_lock(&g_fault_state.mutex); - bool should_fail = g_fault_state.connection_failure && - (g_fault_state.max_failures == 0 || g_fault_state.failure_count < g_fault_state.max_failures); - if (should_fail) { - g_fault_state.failure_count++; - pthread_mutex_unlock(&g_fault_state.mutex); - printf("[FAULT] 连接失败 (第 %d 次)\n", g_fault_state.failure_count); - return -1; - } - pthread_mutex_unlock(&g_fault_state.mutex); - - pthread_mutex_lock(&g_connection_state.mutex); - g_connection_state.connected = true; - g_connection_state.last_connect_time = time(NULL); - g_connection_state.reconnect_attempts = 0; - pthread_mutex_unlock(&g_connection_state.mutex); - - printf("[MOCK] 连接成功\n"); - return 0; -} - -// 模拟断开连接 -void mock_disconnect() { - pthread_mutex_lock(&g_connection_state.mutex); - g_connection_state.connected = false; - g_connection_state.last_disconnect_time = time(NULL); - pthread_mutex_unlock(&g_connection_state.mutex); - printf("[MOCK] 连接断开\n"); -} - -// 模拟重连函数 -int mock_reconnect() { - pthread_mutex_lock(&g_connection_state.mutex); - g_connection_state.reconnect_attempts++; - int attempts = g_connection_state.reconnect_attempts; - pthread_mutex_unlock(&g_connection_state.mutex); - - printf("[MOCK] 重连尝试 %d/%d\n", attempts, g_connection_state.max_reconnect_attempts); - - if (attempts > g_connection_state.max_reconnect_attempts) { - printf("[MOCK] 重连失败，超过最大尝试次数\n"); - return -1; - } - - // 模拟重连延迟 - usleep(100000); // 100ms - - // 不调用mock_connect，直接返回成功，避免重置计数器 - pthread_mutex_lock(&g_connection_state.mutex); - g_connection_state.connected = true; - g_connection_state.last_connect_time = time(NULL); - pthread_mutex_unlock(&g_connection_state.mutex); - - return 0; -} - -// 模拟订阅函数 -int mock_subscribe(const char* topic, int vg_id) { - pthread_mutex_lock(&g_fault_state.mutex); - bool should_fail = g_fault_state.subscription_failure && - g_fault_state.failure_count < g_fault_state.max_failures; - if (should_fail) { - g_fault_state.failure_count++; - pthread_mutex_unlock(&g_fault_state.mutex); - printf("[FAULT] 订阅失败 (第 %d 次): topic=%s, vg_id=%d\n", - g_fault_state.failure_count, topic, vg_id); - return -1; - } - pthread_mutex_unlock(&g_fault_state.mutex); - - printf("[MOCK] 订阅成功: topic=%s, vg_id=%d\n", topic, vg_id); - return 0; -} - -// 模拟解析函数 -int mock_parse_message(const char* message, size_t length) { - pthread_mutex_lock(&g_fault_state.mutex); - bool should_fail = g_fault_state.parse_failure && - g_fault_state.failure_count < g_fault_state.max_failures; - if (should_fail) { - g_fault_state.failure_count++; - pthread_mutex_unlock(&g_fault_state.mutex); - printf("[FAULT] 解析失败 (第 %d 次): length=%zu\n", g_fault_state.failure_count, length); - return -1; - } - pthread_mutex_unlock(&g_fault_state.mutex); - - printf("[MOCK] 解析成功: length=%zu\n", length); - return 0; -} - -// 模拟提交函数 -int mock_commit_offset(const char* topic, int vg_id, int64_t offset, bool sync) { - pthread_mutex_lock(&g_fault_state.mutex); - bool should_fail = g_fault_state.commit_failure && - g_fault_state.failure_count < g_fault_state.max_failures; - if (should_fail) { - g_fault_state.failure_count++; - pthread_mutex_unlock(&g_fault_state.mutex); - printf("[FAULT] 提交失败 (第 %d 次): topic=%s, vg_id=%d, offset=%ld\n", - g_fault_state.failure_count, topic, vg_id, offset); - return -1; - } - pthread_mutex_unlock(&g_fault_state.mutex); - - printf("[MOCK] 提交成功: topic=%s, vg_id=%d, offset=%ld, sync=%s\n", - topic, vg_id, offset, sync ? "true" : "false"); - return 0; -} - -// 测试1: 断线重连测试 -static void test_connection_recovery() { - printf("\n=== 测试1: 断线重连测试 ===\n"); - - // 重置状态 - pthread_mutex_lock(&g_connection_state.mutex); - g_connection_state.connected = true; - g_connection_state.reconnect_attempts = 0; - pthread_mutex_unlock(&g_connection_state.mutex); - - // 测试正常连接 - int result = mock_connect(); - TEST_ASSERT(result == 0, "正常连接成功"); - - // 模拟断开连接 - mock_disconnect(); - pthread_mutex_lock(&g_connection_state.mutex); - bool disconnected = !g_connection_state.connected; - pthread_mutex_unlock(&g_connection_state.mutex); - TEST_ASSERT(disconnected, "连接状态正确断开"); - - // 测试重连成功 - result = mock_reconnect(); - TEST_ASSERT(result == 0, "重连成功"); - - // 测试重连失败（超过最大尝试次数） - mock_disconnect(); - int max_attempts_reached = 0; - for (int i = 0; i < 10; i++) { - int result = mock_reconnect(); - if (result == -1) { - max_attempts_reached = 1; - break; - } - } - TEST_ASSERT(max_attempts_reached == 1, "重连尝试次数正确"); - - // 重置连接状态 - pthread_mutex_lock(&g_connection_state.mutex); - g_connection_state.connected = true; - g_connection_state.reconnect_attempts = 0; - pthread_mutex_unlock(&g_connection_state.mutex); - - printf("✓ 断线重连测试完成\n"); -} - -// 测试2: 订阅失败重试测试 -static void test_subscription_retry() { - printf("\n=== 测试2: 订阅失败重试测试 ===\n"); - - // 注入订阅失败 - inject_subscription_failure(true, 3); - - int retry_count = 0; - const int max_retries = 5; - - for (int i = 0; i < max_retries; i++) { - int result = mock_subscribe("test_topic", 1); - if (result == 0) { - printf("订阅在第 %d 次尝试时成功\n", i + 1); - break; - } - retry_count++; - usleep(100000); // 100ms延迟 - } - - TEST_ASSERT(retry_count >= 3, "订阅失败重试次数正确"); - TEST_ASSERT(retry_count <= max_retries, "重试次数在合理范围内"); - - // 重置故障注入 - inject_subscription_failure(false, 0); - - printf("✓ 订阅失败重试测试完成\n"); -} - -// 测试3: 解析失败处理测试 -static void test_parse_failure_handling() { - printf("\n=== 测试3: 解析失败处理测试 ===\n"); - - // 注入解析失败 - inject_parse_failure(true, 2); - - const char* test_messages[] = { - "valid_message_1", - "valid_message_2", - "valid_message_3", - "valid_message_4" - }; - - int success_count = 0; - int failure_count = 0; - - for (int i = 0; i < 4; i++) { - int result = mock_parse_message(test_messages[i], strlen(test_messages[i])); - if (result == 0) { - success_count++; - } else { - failure_count++; - } - } - - TEST_ASSERT(failure_count == 2, "解析失败次数正确"); - TEST_ASSERT(success_count == 2, "解析成功次数正确"); - - // 重置故障注入 - inject_parse_failure(false, 0); - - printf("✓ 解析失败处理测试完成\n"); -} - -// 测试4: 提交失败重试机制测试 -static void test_commit_retry_mechanism() { - printf("\n=== 测试4: 提交失败重试机制测试 ===\n"); - - // 注入提交失败 - inject_commit_failure(true, 3); - - int retry_count = 0; - const int max_retries = 10; - bool final_success = false; - - for (int i = 0; i < max_retries; i++) { - int result = mock_commit_offset("test_topic", 1, 1000 + i, true); - if (result == 0) { - final_success = true; - printf("提交在第 %d 次尝试时成功\n", i + 1); - break; - } - retry_count++; - usleep(50000); // 50ms延迟 - } - - TEST_ASSERT(retry_count >= 3, "提交失败重试次数正确"); - TEST_ASSERT(final_success, "最终提交成功"); - - // 重置故障注入 - inject_commit_failure(false, 0); - - printf("✓ 提交失败重试机制测试完成\n"); -} - -// 测试5: 混合故障场景测试 -static void test_mixed_fault_scenarios() { - printf("\n=== 测试5: 混合故障场景测试 ===\n"); - - // 同时注入多种故障 - inject_connection_failure(true, 2); - inject_subscription_failure(true, 1); - inject_parse_failure(true, 1); - inject_commit_failure(true, 2); - - // 模拟完整的处理流程 - int step = 0; - - // 步骤1: 连接 - step++; - int result = mock_connect(); - if (result != 0) { - result = mock_reconnect(); - if (result != 0) { - result = mock_reconnect(); // 再次重试 - } - } - TEST_ASSERT(result == 0, "混合故障场景下连接成功"); - - // 步骤2: 订阅 - step++; - result = mock_subscribe("test_topic", 1); - if (result != 0) { - // 如果订阅失败，重试一次 - result = mock_subscribe("test_topic", 1); - } - TEST_ASSERT(result == 0, "混合故障场景下订阅成功"); - - // 步骤3: 解析消息 - step++; - result = mock_parse_message("test_message", 12); - TEST_ASSERT(result == 0, "混合故障场景下解析成功"); - - // 步骤4: 提交offset - step++; - result = mock_commit_offset("test_topic", 1, 1000, false); - TEST_ASSERT(result == 0, "混合故障场景下提交成功"); - - // 重置所有故障注入 - inject_connection_failure(false, 0); - inject_subscription_failure(false, 0); - inject_parse_failure(false, 0); - inject_commit_failure(false, 0); - - printf("✓ 混合故障场景测试完成\n"); -} - -// 测试6: 故障恢复时间测试 -static void test_fault_recovery_time() { - printf("\n=== 测试6: 故障恢复时间测试 ===\n"); - - // 测试连接恢复时间 - inject_connection_failure(true, 1); - - time_t start_time = time(NULL); - int result = mock_connect(); - if (result != 0) { - result = mock_reconnect(); - } - time_t end_time = time(NULL); - - int recovery_time = (int)(end_time - start_time); - TEST_ASSERT(result == 0, "故障恢复后连接成功"); - TEST_ASSERT(recovery_time >= 0, "恢复时间计算正确"); - - printf("连接恢复时间: %d 秒\n", recovery_time); - - // 重置故障注入 - inject_connection_failure(false, 0); - - printf("✓ 故障恢复时间测试完成\n"); -} - -// 线程函数：并发处理故障 -static void* fault_thread_func(void* arg) { - int thread_id = *(int*)arg; - for (int i = 0; i < 5; i++) { - // 随机注入故障 - if (rand() % 3 == 0) { - inject_connection_failure(true, 1); - } - if (rand() % 3 == 0) { - inject_subscription_failure(true, 1); - } - - // 尝试连接和订阅 - int result = mock_connect(); - if (result != 0) { - mock_reconnect(); - } - - mock_subscribe("test_topic", thread_id); - - usleep(10000); // 10ms延迟 - } - return NULL; -} - -// 测试7: 并发故障处理测试 -static void test_concurrent_fault_handling() { - printf("\n=== 测试7: 并发故障处理测试 ===\n"); - - const int thread_count = 3; - pthread_t threads[thread_count]; - int thread_ids[thread_count]; - - // 创建线程 - for (int i = 0; i < thread_count; i++) { - thread_ids[i] = i; - int result = pthread_create(&threads[i], NULL, fault_thread_func, &thread_ids[i]); - TEST_ASSERT(result == 0, "故障处理线程创建成功"); - } - - // 等待所有线程完成 - for (int i = 0; i < thread_count; i++) { - pthread_join(threads[i], NULL); - } - - printf("✓ 并发故障处理测试完成\n"); -} - -// 测试8: 故障统计和监控测试 -static void test_fault_statistics_monitoring() { - printf("\n=== 测试8: 故障统计和监控测试 ===\n"); - - int total_failures = 0; - - // 测试连接故障 - inject_connection_failure(true, 2); - for (int i = 0; i < 3; i++) { - if (mock_connect() != 0) { - total_failures++; - } - } - inject_connection_failure(false, 0); - - // 测试订阅故障 - inject_subscription_failure(true, 1); - for (int i = 0; i < 2; i++) { - if (mock_subscribe("test_topic", 1) != 0) { - total_failures++; - } - } - inject_subscription_failure(false, 0); - - // 测试解析故障 - inject_parse_failure(true, 1); - for (int i = 0; i < 2; i++) { - if (mock_parse_message("test", 4) != 0) { - total_failures++; - } - } - inject_parse_failure(false, 0); - - // 测试提交故障 - inject_commit_failure(true, 1); - for (int i = 0; i < 2; i++) { - if (mock_commit_offset("test_topic", 1, 1000, true) != 0) { - total_failures++; - } - } - inject_commit_failure(false, 0); - - TEST_ASSERT(total_failures >= 4, "故障统计正确"); - printf("总故障次数: %d\n", total_failures); - - printf("✓ 故障统计和监控测试完成\n"); -} - -// 测试9: 边界条件故障测试 -static void test_boundary_fault_conditions() { - printf("\n=== 测试9: 边界条件故障测试 ===\n"); - - // 测试最大重试次数边界 - inject_connection_failure(true, 0); // 立即失败 - int result = mock_connect(); - TEST_ASSERT(result == -1, "零重试次数时立即失败"); - - // 重置故障注入 - inject_connection_failure(false, 0); - - // 重置故障计数器 - pthread_mutex_lock(&g_fault_state.mutex); - g_fault_state.failure_count = 0; - pthread_mutex_unlock(&g_fault_state.mutex); - - // 测试大量重试 - inject_connection_failure(true, 10); - int success_count = 0; - for (int i = 0; i < 15; i++) { - if (mock_connect() == 0) { - success_count++; - } - } - TEST_ASSERT(success_count > 0, "大量重试后能成功"); - - // 重置所有故障注入 - inject_connection_failure(false, 0); - inject_subscription_failure(false, 0); - inject_parse_failure(false, 0); - inject_commit_failure(false, 0); - - printf("✓ 边界条件故障测试完成\n"); -} - -// 测试10: 故障恢复验证测试 -static void test_fault_recovery_verification() { - printf("\n=== 测试10: 故障恢复验证测试 ===\n"); - - // 验证故障注入状态重置 - inject_connection_failure(true, 5); - inject_connection_failure(false, 0); - - int result = mock_connect(); - TEST_ASSERT(result == 0, "故障注入重置后连接正常"); - - // 验证连接状态恢复 - pthread_mutex_lock(&g_connection_state.mutex); - bool connected = g_connection_state.connected; - pthread_mutex_unlock(&g_connection_state.mutex); - TEST_ASSERT(connected, "连接状态正确恢复"); - - // 验证重连计数器重置 - pthread_mutex_lock(&g_connection_state.mutex); - int attempts = g_connection_state.reconnect_attempts; - pthread_mutex_unlock(&g_connection_state.mutex); - TEST_ASSERT(attempts == 0, "重连计数器正确重置"); - - printf("✓ 故障恢复验证测试完成\n"); -} - -int main() { - printf("开始故障注入测试...\n"); - - // 初始化随机数种子 - srand(time(NULL)); - - test_connection_recovery(); - test_subscription_retry(); - test_parse_failure_handling(); - test_commit_retry_mechanism(); - test_mixed_fault_scenarios(); - test_fault_recovery_time(); - test_concurrent_fault_handling(); - test_fault_statistics_monitoring(); - test_boundary_fault_conditions(); - test_fault_recovery_verification(); - - printf("\n=== 测试结果汇总 ===\n"); - printf("总测试数: %d\n", total_tests); - printf("通过测试: %d\n", passed_tests); - printf("失败测试: %d\n", failed_tests); - printf("通过率: %.2f%%\n", (double)passed_tests / total_tests * 100); - - if (failed_tests == 0) { - printf("\n🎉 所有故障注入测试通过！系统容错能力验证完成！\n"); - return 0; - } else { - printf("\n❌ 有 %d 个测试失败，需要修复！\n", failed_tests); - return 1; - } -} diff --git a/plugins/incremental_bitmap/test/test_observability_comprehensive.c b/plugins/incremental_bitmap/test/test_observability_comprehensive.c deleted file mode 100644 index daf4aa57fd58..000000000000 --- a/plugins/incremental_bitmap/test/test_observability_comprehensive.c +++ /dev/null @@ -1,609 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include "../include/storage_engine_interface.h" -#include "../include/observability.h" -#include "../include/bitmap_engine.h" -#include "../include/event_interceptor.h" -#include "../include/ring_buffer.h" - -// 测试计数器 -static int total_tests = 0; -static int passed_tests = 0; -static int failed_tests = 0; - -// 测试宏 -#define TEST_ASSERT(condition, message) do { \ - total_tests++; \ - if (condition) { \ - passed_tests++; \ - printf("✓ %s\n", message); \ - } else { \ - failed_tests++; \ - printf("✗ %s\n", message); \ - } \ -} while(0) - -// 测试1: 可观测性指标结构体完整性 -static void test_observability_metrics_structure() { - printf("\n=== 测试1: 可观测性指标结构体完整性 ===\n"); - - SObservabilityMetrics metrics; - memset(&metrics, 0, sizeof(metrics)); - - // 测试结构体大小 - TEST_ASSERT(sizeof(SObservabilityMetrics) > 0, "结构体大小大于0"); - TEST_ASSERT(sizeof(SObservabilityMetrics) < 10000, "结构体大小合理"); - - // 测试字段访问 - metrics.events_per_second = 1000; - metrics.messages_per_second = 1500; - metrics.bytes_per_second = 1024000; - metrics.consumer_lag_ms = 50; - metrics.offset_lag = 100; - metrics.processing_delay_ms = 10; - metrics.events_dropped = 5; - metrics.messages_dropped = 3; - metrics.parse_errors = 2; - metrics.connection_retries = 1; - metrics.subscription_retries = 0; - metrics.commit_retries = 2; - metrics.ring_buffer_usage = 75; - metrics.ring_buffer_capacity = 10000; - metrics.event_queue_size = 7500; - metrics.memory_usage_bytes = 1048576; - metrics.bitmap_memory_bytes = 524288; - metrics.metadata_memory_bytes = 262144; - metrics.last_update_time = 1234567890; - metrics.uptime_seconds = 3600; - - TEST_ASSERT(metrics.events_per_second == 1000, "events_per_second字段正确"); - TEST_ASSERT(metrics.messages_per_second == 1500, "messages_per_second字段正确"); - TEST_ASSERT(metrics.bytes_per_second == 1024000, "bytes_per_second字段正确"); - TEST_ASSERT(metrics.consumer_lag_ms == 50, "consumer_lag_ms字段正确"); - TEST_ASSERT(metrics.offset_lag == 100, "offset_lag字段正确"); - TEST_ASSERT(metrics.processing_delay_ms == 10, "processing_delay_ms字段正确"); - TEST_ASSERT(metrics.events_dropped == 5, "events_dropped字段正确"); - TEST_ASSERT(metrics.messages_dropped == 3, "messages_dropped字段正确"); - TEST_ASSERT(metrics.parse_errors == 2, "parse_errors字段正确"); - TEST_ASSERT(metrics.connection_retries == 1, "connection_retries字段正确"); - TEST_ASSERT(metrics.subscription_retries == 0, "subscription_retries字段正确"); - TEST_ASSERT(metrics.commit_retries == 2, "commit_retries字段正确"); - TEST_ASSERT(metrics.ring_buffer_usage == 75, "ring_buffer_usage字段正确"); - TEST_ASSERT(metrics.ring_buffer_capacity == 10000, "ring_buffer_capacity字段正确"); - TEST_ASSERT(metrics.event_queue_size == 7500, "event_queue_size字段正确"); - TEST_ASSERT(metrics.memory_usage_bytes == 1048576, "memory_usage_bytes字段正确"); - TEST_ASSERT(metrics.bitmap_memory_bytes == 524288, "bitmap_memory_bytes字段正确"); - TEST_ASSERT(metrics.metadata_memory_bytes == 262144, "metadata_memory_bytes字段正确"); - TEST_ASSERT(metrics.last_update_time == 1234567890, "last_update_time字段正确"); - TEST_ASSERT(metrics.uptime_seconds == 3600, "uptime_seconds字段正确"); -} - -// 测试2: 可观测性指标更新函数 -static void test_observability_metrics_update() { - printf("\n=== 测试2: 可观测性指标更新函数 ===\n"); - - SObservabilityMetrics metrics; - memset(&metrics, 0, sizeof(metrics)); - - // 测试NULL指针处理 - update_observability_metrics(NULL); - TEST_ASSERT(1, "NULL指针处理正常"); - - // 测试正常更新 - int64_t before_time = time(NULL); - update_observability_metrics(&metrics); - int64_t after_time = time(NULL); - - TEST_ASSERT(metrics.last_update_time > 0, "更新时间戳正确设置"); - TEST_ASSERT(metrics.last_update_time >= before_time * 1000, "更新时间戳合理"); - TEST_ASSERT(metrics.last_update_time <= after_time * 1000, "更新时间戳合理"); -} - -// 测试3: 可观测性指标打印函数 -static void test_observability_metrics_print() { - printf("\n=== 测试3: 可观测性指标打印函数 ===\n"); - - SObservabilityMetrics metrics; - memset(&metrics, 0, sizeof(metrics)); - - // 设置测试数据 - metrics.events_per_second = 2000; - metrics.messages_per_second = 2500; - metrics.bytes_per_second = 2048000; - metrics.consumer_lag_ms = 25; - metrics.offset_lag = 50; - metrics.processing_delay_ms = 5; - metrics.events_dropped = 10; - metrics.messages_dropped = 5; - metrics.parse_errors = 3; - metrics.connection_retries = 2; - metrics.subscription_retries = 1; - metrics.commit_retries = 3; - metrics.ring_buffer_usage = 80; - metrics.ring_buffer_capacity = 10000; - metrics.event_queue_size = 8000; - metrics.memory_usage_bytes = 2097152; - metrics.bitmap_memory_bytes = 1048576; - metrics.metadata_memory_bytes = 524288; - metrics.last_update_time = 1234567890; - metrics.uptime_seconds = 7200; - - // 测试NULL指针处理 - print_observability_metrics(NULL); - TEST_ASSERT(1, "NULL指针处理正常"); - - // 测试正常打印 - printf("调用 print_observability_metrics 函数:\n"); - print_observability_metrics(&metrics); - TEST_ASSERT(1, "打印函数正常执行"); -} - -// 测试4: JSON格式化函数 -static void test_json_formatting() { - printf("\n=== 测试4: JSON格式化函数 ===\n"); - - SObservabilityMetrics metrics; - memset(&metrics, 0, sizeof(metrics)); - - // 设置测试数据 - metrics.events_per_second = 1000; - metrics.messages_per_second = 1500; - metrics.bytes_per_second = 1024000; - metrics.consumer_lag_ms = 50; - metrics.offset_lag = 100; - metrics.processing_delay_ms = 10; - metrics.events_dropped = 5; - metrics.messages_dropped = 3; - metrics.parse_errors = 2; - metrics.connection_retries = 1; - metrics.subscription_retries = 0; - metrics.commit_retries = 2; - metrics.ring_buffer_usage = 75; - metrics.ring_buffer_capacity = 10000; - metrics.event_queue_size = 7500; - metrics.memory_usage_bytes = 1048576; - metrics.bitmap_memory_bytes = 524288; - metrics.metadata_memory_bytes = 262144; - metrics.last_update_time = 1234567890; - metrics.uptime_seconds = 3600; - - // 测试NULL指针处理 - format_observability_metrics_json(NULL, NULL, 0); - TEST_ASSERT(1, "NULL指针处理正常"); - - // 测试空缓冲区 - format_observability_metrics_json(&metrics, NULL, 0); - TEST_ASSERT(1, "空缓冲区处理正常"); - - // 测试小缓冲区 - char small_buffer[10]; - format_observability_metrics_json(&metrics, small_buffer, sizeof(small_buffer)); - TEST_ASSERT(1, "小缓冲区处理正常"); - - // 测试正常格式化 - char json_buffer[4096]; - format_observability_metrics_json(&metrics, json_buffer, sizeof(json_buffer)); - - TEST_ASSERT(strlen(json_buffer) > 0, "JSON字符串非空"); - TEST_ASSERT(strstr(json_buffer, "\"events_per_second\":1000") != NULL, "JSON包含events_per_second"); - TEST_ASSERT(strstr(json_buffer, "\"memory_usage_bytes\":1048576") != NULL, "JSON包含memory_usage_bytes"); - TEST_ASSERT(strstr(json_buffer, "\"ring_buffer_usage\":75") != NULL, "JSON包含ring_buffer_usage"); - TEST_ASSERT(strstr(json_buffer, "\"rate\":") != NULL, "JSON包含rate字段"); - TEST_ASSERT(strstr(json_buffer, "\"lag\":") != NULL, "JSON包含lag字段"); - TEST_ASSERT(strstr(json_buffer, "\"dropped\":") != NULL, "JSON包含dropped字段"); - TEST_ASSERT(strstr(json_buffer, "\"retries\":") != NULL, "JSON包含retries字段"); - TEST_ASSERT(strstr(json_buffer, "\"queue\":") != NULL, "JSON包含queue字段"); - TEST_ASSERT(strstr(json_buffer, "\"memory\":") != NULL, "JSON包含memory字段"); - TEST_ASSERT(strstr(json_buffer, "\"time\":") != NULL, "JSON包含time字段"); - - printf("JSON输出: %s\n", json_buffer); -} - -// 测试5: Prometheus格式化函数 -static void test_prometheus_formatting() { - printf("\n=== 测试5: Prometheus格式化函数 ===\n"); - - SObservabilityMetrics metrics; - memset(&metrics, 0, sizeof(metrics)); - - // 设置测试数据 - metrics.events_per_second = 1000; - metrics.messages_per_second = 1500; - metrics.bytes_per_second = 1024000; - metrics.consumer_lag_ms = 50; - metrics.offset_lag = 100; - metrics.processing_delay_ms = 10; - metrics.events_dropped = 5; - metrics.messages_dropped = 3; - metrics.parse_errors = 2; - metrics.connection_retries = 1; - metrics.subscription_retries = 0; - metrics.commit_retries = 2; - metrics.ring_buffer_usage = 75; - metrics.ring_buffer_capacity = 10000; - metrics.event_queue_size = 7500; - metrics.memory_usage_bytes = 1048576; - metrics.bitmap_memory_bytes = 524288; - metrics.metadata_memory_bytes = 262144; - metrics.last_update_time = 1234567890; - metrics.uptime_seconds = 3600; - - // 测试NULL指针处理 - format_observability_metrics_prometheus(NULL, NULL, 0); - TEST_ASSERT(1, "NULL指针处理正常"); - - // 测试空缓冲区 - format_observability_metrics_prometheus(&metrics, NULL, 0); - TEST_ASSERT(1, "空缓冲区处理正常"); - - // 测试小缓冲区 - char small_buffer[10]; - format_observability_metrics_prometheus(&metrics, small_buffer, sizeof(small_buffer)); - TEST_ASSERT(1, "小缓冲区处理正常"); - - // 测试正常格式化 - char prometheus_buffer[8192]; - format_observability_metrics_prometheus(&metrics, prometheus_buffer, sizeof(prometheus_buffer)); - - TEST_ASSERT(strlen(prometheus_buffer) > 0, "Prometheus字符串非空"); - TEST_ASSERT(strstr(prometheus_buffer, "tdengine_events_per_second 1000") != NULL, "包含events_per_second指标"); - TEST_ASSERT(strstr(prometheus_buffer, "tdengine_memory_usage_bytes 1048576") != NULL, "包含memory_usage_bytes指标"); - TEST_ASSERT(strstr(prometheus_buffer, "tdengine_ring_buffer_usage 75") != NULL, "包含ring_buffer_usage指标"); - TEST_ASSERT(strstr(prometheus_buffer, "# HELP") != NULL, "包含HELP注释"); - TEST_ASSERT(strstr(prometheus_buffer, "# TYPE") != NULL, "包含TYPE注释"); - TEST_ASSERT(strstr(prometheus_buffer, "gauge") != NULL, "包含gauge类型"); - TEST_ASSERT(strstr(prometheus_buffer, "counter") != NULL, "包含counter类型"); - - printf("Prometheus输出:\n%s\n", prometheus_buffer); -} - -// 测试6: 内存使用计算 -static void test_memory_usage_calculation() { - printf("\n=== 测试6: 内存使用计算 ===\n"); - - // 创建位图引擎 - SBitmapEngine* engine = bitmap_engine_init(); - TEST_ASSERT(engine != NULL, "位图引擎创建成功"); - - // 添加一些测试数据 - for (int i = 0; i < 1000; i++) { - int result = bitmap_engine_mark_dirty(engine, i, i * 100, i * 1000); - TEST_ASSERT(result == 0, "标记脏块成功"); - } - - // 创建事件拦截器 - SEventInterceptorConfig config = { - .enable_interception = true, - .event_buffer_size = 1000, - .callback_threads = 2, - .callback = NULL, - .callback_user_data = NULL - }; - - SEventInterceptor* interceptor = event_interceptor_init(&config, engine); - TEST_ASSERT(interceptor != NULL, "事件拦截器创建成功"); - - // 模拟一些事件 - for (int i = 0; i < 100; i++) { - int result = event_interceptor_on_block_create(interceptor, i, i * 100, i * 1000); - TEST_ASSERT(result == 0, "事件处理成功"); - } - - // 创建可观测性指标 - SObservabilityMetrics metrics; - memset(&metrics, 0, sizeof(metrics)); - - // 手动计算内存使用 - size_t total_memory = sizeof(void*) * 3; // 基本结构体大小 - size_t bitmap_memory = 0; - size_t metadata_memory = 0; - - // 计算位图内存使用 - if (engine->dirty_blocks) { - bitmap_memory += engine->dirty_blocks->memory_usage(engine->dirty_blocks->bitmap); - } - if (engine->new_blocks) { - bitmap_memory += engine->new_blocks->memory_usage(engine->new_blocks->bitmap); - } - if (engine->deleted_blocks) { - bitmap_memory += engine->deleted_blocks->memory_usage(engine->deleted_blocks->bitmap); - } - - // 计算元数据内存使用 - metadata_memory = engine->metadata_map_size * sizeof(void*); - metadata_memory += engine->metadata_count * sizeof(void*); - - total_memory += bitmap_memory + metadata_memory; - - TEST_ASSERT(bitmap_memory > 0, "位图内存使用大于0"); - TEST_ASSERT(metadata_memory > 0, "元数据内存使用大于0"); - TEST_ASSERT(total_memory > bitmap_memory + metadata_memory, "总内存使用合理"); - - printf("计算的内存使用:\n"); - printf(" 总内存: %zu 字节\n", total_memory); - printf(" 位图内存: %zu 字节\n", bitmap_memory); - printf(" 元数据内存: %zu 字节\n", metadata_memory); - - // 清理资源 - event_interceptor_destroy(interceptor); - bitmap_engine_destroy(engine); -} - -// 测试7: 队列水位监控 -static void test_queue_watermark_monitoring() { - printf("\n=== 测试7: 队列水位监控 ===\n"); - - // 创建环形队列 - SRingBuffer* ring_buffer = ring_buffer_init(100); - TEST_ASSERT(ring_buffer != NULL, "环形队列创建成功"); - - // 测试空队列 - uint32_t size = ring_buffer_get_size(ring_buffer); - uint32_t capacity = ring_buffer_get_capacity(ring_buffer); - uint32_t usage = capacity > 0 ? (size * 100) / capacity : 0; - - TEST_ASSERT(size == 0, "空队列大小为0"); - TEST_ASSERT(capacity == 100, "队列容量正确"); - TEST_ASSERT(usage == 0, "空队列使用率为0"); - - // 添加一些数据 - for (int i = 0; i < 50; i++) { - int* data = malloc(sizeof(int)); - *data = i; - int result = ring_buffer_enqueue(ring_buffer, data); - TEST_ASSERT(result == 0, "数据入队成功"); - } - - size = ring_buffer_get_size(ring_buffer); - usage = capacity > 0 ? (size * 100) / capacity : 0; - - TEST_ASSERT(size == 50, "半满队列大小正确"); - TEST_ASSERT(usage == 50, "半满队列使用率正确"); - - // 填满队列 - for (int i = 50; i < 100; i++) { - int* data = malloc(sizeof(int)); - *data = i; - int result = ring_buffer_enqueue(ring_buffer, data); - TEST_ASSERT(result == 0, "数据入队成功"); - } - - size = ring_buffer_get_size(ring_buffer); - usage = capacity > 0 ? (size * 100) / capacity : 0; - - TEST_ASSERT(size == 100, "满队列大小正确"); - TEST_ASSERT(usage == 100, "满队列使用率正确"); - - // 测试队列满时的行为 - int* extra_data = malloc(sizeof(int)); - *extra_data = 999; - int result = ring_buffer_enqueue(ring_buffer, extra_data); - TEST_ASSERT(result != 0, "满队列拒绝新数据"); - free(extra_data); - - // 清理资源 - ring_buffer_clear(ring_buffer, free); - ring_buffer_destroy(ring_buffer); -} - -// 测试8: 性能指标计算 -static void test_performance_metrics_calculation() { - printf("\n=== 测试8: 性能指标计算 ===\n"); - - SObservabilityMetrics metrics; - memset(&metrics, 0, sizeof(metrics)); - - // 模拟性能数据 - metrics.events_per_second = 10000; - metrics.messages_per_second = 15000; - metrics.bytes_per_second = 10485760; // 10MB/s - metrics.consumer_lag_ms = 100; - metrics.offset_lag = 1000; - metrics.processing_delay_ms = 5; - metrics.ring_buffer_usage = 60; - metrics.memory_usage_bytes = 52428800; // 50MB - - // 计算性能指标 - double events_per_ms = metrics.events_per_second / 1000.0; - double messages_per_ms = metrics.messages_per_second / 1000.0; - double mb_per_second = metrics.bytes_per_second / (1024.0 * 1024.0); - double memory_mb = metrics.memory_usage_bytes / (1024.0 * 1024.0); - - TEST_ASSERT(events_per_ms > 0, "事件处理速率大于0"); - TEST_ASSERT(messages_per_ms > 0, "消息处理速率大于0"); - TEST_ASSERT(mb_per_second > 0, "数据吞吐量大于0"); - TEST_ASSERT(memory_mb > 0, "内存使用大于0"); - TEST_ASSERT(metrics.ring_buffer_usage <= 100, "队列使用率合理"); - TEST_ASSERT(metrics.processing_delay_ms >= 0, "处理延迟非负"); - - printf("性能指标:\n"); - printf(" 事件处理: %.2f 事件/毫秒\n", events_per_ms); - printf(" 消息处理: %.2f 消息/毫秒\n", messages_per_ms); - printf(" 数据吞吐: %.2f MB/秒\n", mb_per_second); - printf(" 内存使用: %.2f MB\n", memory_mb); - printf(" 队列使用率: %u%%\n", metrics.ring_buffer_usage); - printf(" 处理延迟: %ld ms\n", metrics.processing_delay_ms); -} - -// 测试9: 边界条件测试 -static void test_edge_cases() { - printf("\n=== 测试9: 边界条件测试 ===\n"); - - SObservabilityMetrics metrics; - memset(&metrics, 0, sizeof(metrics)); - - // 测试最大值 - metrics.events_per_second = UINT64_MAX; - metrics.messages_per_second = UINT64_MAX; - metrics.bytes_per_second = UINT64_MAX; - metrics.consumer_lag_ms = INT64_MAX; - metrics.offset_lag = UINT64_MAX; - metrics.processing_delay_ms = INT64_MAX; - metrics.events_dropped = UINT64_MAX; - metrics.messages_dropped = UINT64_MAX; - metrics.parse_errors = UINT64_MAX; - metrics.connection_retries = UINT64_MAX; - metrics.subscription_retries = UINT64_MAX; - metrics.commit_retries = UINT64_MAX; - metrics.ring_buffer_usage = UINT32_MAX; - metrics.ring_buffer_capacity = UINT32_MAX; - metrics.event_queue_size = UINT32_MAX; - metrics.memory_usage_bytes = SIZE_MAX; - metrics.bitmap_memory_bytes = SIZE_MAX; - metrics.metadata_memory_bytes = SIZE_MAX; - metrics.last_update_time = INT64_MAX; - metrics.uptime_seconds = INT64_MAX; - - // 测试JSON格式化 - char json_buffer[4096]; - format_observability_metrics_json(&metrics, json_buffer, sizeof(json_buffer)); - TEST_ASSERT(strlen(json_buffer) > 0, "最大值JSON格式化成功"); - - // 测试Prometheus格式化 - char prometheus_buffer[8192]; - format_observability_metrics_prometheus(&metrics, prometheus_buffer, sizeof(prometheus_buffer)); - TEST_ASSERT(strlen(prometheus_buffer) > 0, "最大值Prometheus格式化成功"); - - // 测试最小值 - memset(&metrics, 0, sizeof(metrics)); - format_observability_metrics_json(&metrics, json_buffer, sizeof(json_buffer)); - TEST_ASSERT(strlen(json_buffer) > 0, "最小值JSON格式化成功"); - - format_observability_metrics_prometheus(&metrics, prometheus_buffer, sizeof(prometheus_buffer)); - TEST_ASSERT(strlen(prometheus_buffer) > 0, "最小值Prometheus格式化成功"); -} - -// 线程函数 - 移到函数外部 -static void* thread_func(void* arg) { - SObservabilityMetrics metrics; - char buffer[1024]; - int iterations = *(int*)arg; - - for (int i = 0; i < iterations; i++) { - memset(&metrics, 0, sizeof(metrics)); - metrics.events_per_second = i; - metrics.memory_usage_bytes = i * 1024; - - update_observability_metrics(&metrics); - format_observability_metrics_json(&metrics, buffer, sizeof(buffer)); - format_observability_metrics_prometheus(&metrics, buffer, sizeof(buffer)); - } - - return NULL; -} - -// 测试10: 并发安全性测试 -static void test_concurrency_safety() { - printf("\n=== 测试10: 并发安全性测试 ===\n"); - - // 创建多个线程同时访问可观测性指标 - const int num_threads = 8; // 减少线程数量 - const int iterations = 100; // 减少迭代次数 - pthread_t threads[num_threads]; - int thread_arg = iterations; - - // 创建线程 - for (int i = 0; i < num_threads; i++) { - int result = pthread_create(&threads[i], NULL, thread_func, &thread_arg); - TEST_ASSERT(result == 0, "线程创建成功"); - } - - // 等待所有线程完成 - for (int i = 0; i < num_threads; i++) { - pthread_join(threads[i], NULL); - } - - TEST_ASSERT(1, "并发访问测试完成"); -} - -// 测试11: 内存泄漏测试 -static void test_memory_leaks() { - printf("\n=== 测试11: 内存泄漏测试 ===\n"); - - // 多次调用格式化函数，检查内存使用 - SObservabilityMetrics metrics; - memset(&metrics, 0, sizeof(metrics)); - - // 设置一些测试数据 - metrics.events_per_second = 1000; - metrics.memory_usage_bytes = 1048576; - metrics.ring_buffer_usage = 50; - - char json_buffer[4096]; - char prometheus_buffer[8192]; - - // 多次调用格式化函数 - for (int i = 0; i < 1000; i++) { - format_observability_metrics_json(&metrics, json_buffer, sizeof(json_buffer)); - format_observability_metrics_prometheus(&metrics, prometheus_buffer, sizeof(prometheus_buffer)); - } - - TEST_ASSERT(1, "内存泄漏测试完成"); -} - -// 测试12: 错误恢复测试 -static void test_error_recovery() { - printf("\n=== 测试12: 错误恢复测试 ===\n"); - - // 测试各种错误情况下的恢复能力 - SObservabilityMetrics metrics; - memset(&metrics, 0, sizeof(metrics)); - - // 测试NULL指针 - update_observability_metrics(NULL); - print_observability_metrics(NULL); - format_observability_metrics_json(NULL, NULL, 0); - format_observability_metrics_prometheus(NULL, NULL, 0); - - // 测试空缓冲区 - format_observability_metrics_json(&metrics, NULL, 0); - format_observability_metrics_prometheus(&metrics, NULL, 0); - - // 测试小缓冲区 - char small_buffer[1]; - format_observability_metrics_json(&metrics, small_buffer, 1); - format_observability_metrics_prometheus(&metrics, small_buffer, 1); - - // 测试正常情况 - char buffer[1024]; - format_observability_metrics_json(&metrics, buffer, sizeof(buffer)); - format_observability_metrics_prometheus(&metrics, buffer, sizeof(buffer)); - - TEST_ASSERT(1, "错误恢复测试完成"); -} - -int main() { - printf("开始全面可观测性指标测试...\n"); - - test_observability_metrics_structure(); - test_observability_metrics_update(); - test_observability_metrics_print(); - test_json_formatting(); - test_prometheus_formatting(); - test_memory_usage_calculation(); - test_queue_watermark_monitoring(); - test_performance_metrics_calculation(); - test_edge_cases(); - test_concurrency_safety(); - test_memory_leaks(); - test_error_recovery(); - - printf("\n=== 测试结果汇总 ===\n"); - printf("总测试数: %d\n", total_tests); - printf("通过测试: %d\n", passed_tests); - printf("失败测试: %d\n", failed_tests); - printf("通过率: %.2f%%\n", (double)passed_tests / total_tests * 100); - - if (failed_tests == 0) { - printf("\n🎉 所有测试通过！可观测性指标功能验证完成！\n"); - return 0; - } else { - printf("\n❌ 有 %d 个测试失败，需要修复！\n", failed_tests); - return 1; - } -} diff --git a/plugins/incremental_bitmap/test/test_observability_enhanced.c b/plugins/incremental_bitmap/test/test_observability_enhanced.c deleted file mode 100644 index 3a957faa8df3..000000000000 --- a/plugins/incremental_bitmap/test/test_observability_enhanced.c +++ /dev/null @@ -1,403 +0,0 @@ -#include -#include -#include -#include -#include -#include "../include/storage_engine_interface.h" -#include "../include/observability.h" -#include "../include/bitmap_engine.h" -#include "../include/event_interceptor.h" -#include "../include/ring_buffer.h" - -// 模拟事件处理函数 -static void mock_event_processor(void* user_data) { - // 模拟事件处理延迟 - usleep(1000); // 1ms延迟 -} - -// 测试1: 基础可观测性指标结构体 -static void test_basic_observability_metrics() { - printf("=== 测试1: 基础可观测性指标结构体 ===\n"); - - SObservabilityMetrics metrics; - memset(&metrics, 0, sizeof(metrics)); - - // 设置测试数据 - metrics.events_per_second = 1000; - metrics.messages_per_second = 1500; - metrics.bytes_per_second = 1024000; - metrics.consumer_lag_ms = 50; - metrics.offset_lag = 100; - metrics.processing_delay_ms = 10; - metrics.events_dropped = 5; - metrics.messages_dropped = 3; - metrics.parse_errors = 2; - metrics.connection_retries = 1; - metrics.subscription_retries = 0; - metrics.commit_retries = 2; - metrics.ring_buffer_usage = 75; - metrics.ring_buffer_capacity = 10000; - metrics.event_queue_size = 7500; - metrics.memory_usage_bytes = 1048576; - metrics.bitmap_memory_bytes = 524288; - metrics.metadata_memory_bytes = 262144; - metrics.last_update_time = 1234567890; - metrics.uptime_seconds = 3600; - - // 验证结构体大小 - printf("结构体大小: %zu 字节\n", sizeof(SObservabilityMetrics)); - assert(sizeof(SObservabilityMetrics) > 0); - - // 验证字段访问 - assert(metrics.events_per_second == 1000); - assert(metrics.messages_per_second == 1500); - assert(metrics.consumer_lag_ms == 50); - assert(metrics.ring_buffer_usage == 75); - assert(metrics.memory_usage_bytes == 1048576); - - printf("✓ 基础可观测性指标结构体测试通过\n"); -} - -// 测试2: 可观测性指标打印函数 -static void test_observability_metrics_print() { - printf("\n=== 测试2: 可观测性指标打印函数 ===\n"); - - SObservabilityMetrics metrics; - memset(&metrics, 0, sizeof(metrics)); - - // 设置测试数据 - metrics.events_per_second = 2000; - metrics.messages_per_second = 2500; - metrics.bytes_per_second = 2048000; - metrics.consumer_lag_ms = 25; - metrics.offset_lag = 50; - metrics.processing_delay_ms = 5; - metrics.events_dropped = 10; - metrics.messages_dropped = 5; - metrics.parse_errors = 3; - metrics.connection_retries = 2; - metrics.subscription_retries = 1; - metrics.commit_retries = 3; - metrics.ring_buffer_usage = 80; - metrics.ring_buffer_capacity = 10000; - metrics.event_queue_size = 8000; - metrics.memory_usage_bytes = 2097152; - metrics.bitmap_memory_bytes = 1048576; - metrics.metadata_memory_bytes = 524288; - metrics.last_update_time = 1234567890; - metrics.uptime_seconds = 7200; - - // 调用打印函数 - printf("调用 print_observability_metrics 函数:\n"); - print_observability_metrics(&metrics); - - printf("✓ 可观测性指标打印函数测试通过\n"); -} - -// 测试3: JSON格式化函数 -static void test_json_formatting() { - printf("\n=== 测试3: JSON格式化函数 ===\n"); - - SObservabilityMetrics metrics; - memset(&metrics, 0, sizeof(metrics)); - - // 设置测试数据 - metrics.events_per_second = 1000; - metrics.messages_per_second = 1500; - metrics.bytes_per_second = 1024000; - metrics.consumer_lag_ms = 50; - metrics.offset_lag = 100; - metrics.processing_delay_ms = 10; - metrics.events_dropped = 5; - metrics.messages_dropped = 3; - metrics.parse_errors = 2; - metrics.connection_retries = 1; - metrics.subscription_retries = 0; - metrics.commit_retries = 2; - metrics.ring_buffer_usage = 75; - metrics.ring_buffer_capacity = 10000; - metrics.event_queue_size = 7500; - metrics.memory_usage_bytes = 1048576; - metrics.bitmap_memory_bytes = 524288; - metrics.metadata_memory_bytes = 262144; - metrics.last_update_time = 1234567890; - metrics.uptime_seconds = 3600; - - // 格式化JSON - char json_buffer[4096]; - format_observability_metrics_json(&metrics, json_buffer, sizeof(json_buffer)); - - printf("JSON格式输出:\n%s\n", json_buffer); - - // 验证JSON包含关键字段 - assert(strstr(json_buffer, "\"events_per_second\":1000") != NULL); - assert(strstr(json_buffer, "\"memory_usage_bytes\":1048576") != NULL); - assert(strstr(json_buffer, "\"ring_buffer_usage\":75") != NULL); - - printf("✓ JSON格式化函数测试通过\n"); -} - -// 测试4: Prometheus格式化函数 -static void test_prometheus_formatting() { - printf("\n=== 测试4: Prometheus格式化函数 ===\n"); - - SObservabilityMetrics metrics; - memset(&metrics, 0, sizeof(metrics)); - - // 设置测试数据 - metrics.events_per_second = 1000; - metrics.messages_per_second = 1500; - metrics.bytes_per_second = 1024000; - metrics.consumer_lag_ms = 50; - metrics.offset_lag = 100; - metrics.processing_delay_ms = 10; - metrics.events_dropped = 5; - metrics.messages_dropped = 3; - metrics.parse_errors = 2; - metrics.connection_retries = 1; - metrics.subscription_retries = 0; - metrics.commit_retries = 2; - metrics.ring_buffer_usage = 75; - metrics.ring_buffer_capacity = 10000; - metrics.event_queue_size = 7500; - metrics.memory_usage_bytes = 1048576; - metrics.bitmap_memory_bytes = 524288; - metrics.metadata_memory_bytes = 262144; - metrics.last_update_time = 1234567890; - metrics.uptime_seconds = 3600; - - // 格式化Prometheus - char prometheus_buffer[8192]; - format_observability_metrics_prometheus(&metrics, prometheus_buffer, sizeof(prometheus_buffer)); - - printf("Prometheus格式输出:\n%s\n", prometheus_buffer); - - // 验证Prometheus格式包含关键指标 - assert(strstr(prometheus_buffer, "tdengine_events_per_second 1000") != NULL); - assert(strstr(prometheus_buffer, "tdengine_memory_usage_bytes 1048576") != NULL); - assert(strstr(prometheus_buffer, "tdengine_ring_buffer_usage 75") != NULL); - - printf("✓ Prometheus格式化函数测试通过\n"); -} - -// 测试5: 内存使用指标计算 -static void test_memory_usage_calculation() { - printf("\n=== 测试5: 内存使用指标计算 ===\n"); - - // 创建位图引擎 - SBitmapEngine* engine = bitmap_engine_init(); - assert(engine != NULL); - - // 添加一些测试数据 - for (int i = 0; i < 1000; i++) { - bitmap_engine_mark_dirty(engine, i, i * 100, i * 1000); - } - - // 创建事件拦截器 - SEventInterceptorConfig config = { - .enable_interception = true, - .event_buffer_size = 1000, - .callback_threads = 2, - .callback = NULL, - .callback_user_data = NULL - }; - - SEventInterceptor* interceptor = event_interceptor_init(&config, engine); - assert(interceptor != NULL); - - // 模拟一些事件 - for (int i = 0; i < 100; i++) { - event_interceptor_on_block_create(interceptor, i, i * 100, i * 1000); - } - - // 创建可观测性指标 - SObservabilityMetrics metrics; - memset(&metrics, 0, sizeof(metrics)); - - // 手动计算内存使用 - size_t total_memory = sizeof(void*) * 3; // 基本结构体大小 - size_t bitmap_memory = 0; - size_t metadata_memory = 0; - - // 计算位图内存使用 - if (engine->dirty_blocks) { - bitmap_memory += engine->dirty_blocks->memory_usage(engine->dirty_blocks->bitmap); - } - if (engine->new_blocks) { - bitmap_memory += engine->new_blocks->memory_usage(engine->new_blocks->bitmap); - } - if (engine->deleted_blocks) { - bitmap_memory += engine->deleted_blocks->memory_usage(engine->deleted_blocks->bitmap); - } - - // 计算元数据内存使用 - metadata_memory = engine->metadata_map_size * sizeof(void*); - metadata_memory += engine->metadata_count * sizeof(void*); - - total_memory += bitmap_memory + metadata_memory; - - printf("计算的内存使用:\n"); - printf(" 总内存: %zu 字节\n", total_memory); - printf(" 位图内存: %zu 字节\n", bitmap_memory); - printf(" 元数据内存: %zu 字节\n", metadata_memory); - - // 验证内存使用合理 - assert(bitmap_memory > 0); - assert(metadata_memory > 0); - assert(total_memory > bitmap_memory + metadata_memory); - - // 清理资源 - event_interceptor_destroy(interceptor); - bitmap_engine_destroy(engine); - - printf("✓ 内存使用指标计算测试通过\n"); -} - -// 测试6: 队列水位监控 -static void test_queue_watermark_monitoring() { - printf("\n=== 测试6: 队列水位监控 ===\n"); - - // 创建环形队列 - SRingBuffer* ring_buffer = ring_buffer_init(100); - assert(ring_buffer != NULL); - - // 测试空队列 - uint32_t size = ring_buffer_get_size(ring_buffer); - uint32_t capacity = ring_buffer_get_capacity(ring_buffer); - uint32_t usage = capacity > 0 ? (size * 100) / capacity : 0; - - printf("空队列状态:\n"); - printf(" 大小: %u\n", size); - printf(" 容量: %u\n", capacity); - printf(" 使用率: %u%%\n", usage); - - assert(size == 0); - assert(capacity == 100); - assert(usage == 0); - - // 添加一些数据 - for (int i = 0; i < 50; i++) { - int* data = malloc(sizeof(int)); - *data = i; - ring_buffer_enqueue(ring_buffer, data); - } - - size = ring_buffer_get_size(ring_buffer); - usage = capacity > 0 ? (size * 100) / capacity : 0; - - printf("半满队列状态:\n"); - printf(" 大小: %u\n", size); - printf(" 容量: %u\n", capacity); - printf(" 使用率: %u%%\n", usage); - - assert(size == 50); - assert(usage == 50); - - // 填满队列 - for (int i = 50; i < 100; i++) { - int* data = malloc(sizeof(int)); - *data = i; - ring_buffer_enqueue(ring_buffer, data); - } - - size = ring_buffer_get_size(ring_buffer); - usage = capacity > 0 ? (size * 100) / capacity : 0; - - printf("满队列状态:\n"); - printf(" 大小: %u\n", size); - printf(" 容量: %u\n", capacity); - printf(" 使用率: %u%%\n", usage); - - assert(size == 100); - assert(usage == 100); - - // 清理资源 - ring_buffer_clear(ring_buffer, free); - ring_buffer_destroy(ring_buffer); - - printf("✓ 队列水位监控测试通过\n"); -} - -// 测试7: 性能指标计算 -static void test_performance_metrics_calculation() { - printf("\n=== 测试7: 性能指标计算 ===\n"); - - SObservabilityMetrics metrics; - memset(&metrics, 0, sizeof(metrics)); - - // 模拟性能数据 - metrics.events_per_second = 10000; - metrics.messages_per_second = 15000; - metrics.bytes_per_second = 10485760; // 10MB/s - metrics.consumer_lag_ms = 100; - metrics.offset_lag = 1000; - metrics.processing_delay_ms = 5; - metrics.ring_buffer_usage = 60; - metrics.memory_usage_bytes = 52428800; // 50MB - - // 计算性能指标 - double events_per_ms = metrics.events_per_second / 1000.0; - double messages_per_ms = metrics.messages_per_second / 1000.0; - double mb_per_second = metrics.bytes_per_second / (1024.0 * 1024.0); - double memory_mb = metrics.memory_usage_bytes / (1024.0 * 1024.0); - - printf("性能指标:\n"); - printf(" 事件处理: %.2f 事件/毫秒\n", events_per_ms); - printf(" 消息处理: %.2f 消息/毫秒\n", messages_per_ms); - printf(" 数据吞吐: %.2f MB/秒\n", mb_per_second); - printf(" 内存使用: %.2f MB\n", memory_mb); - printf(" 队列使用率: %u%%\n", metrics.ring_buffer_usage); - printf(" 处理延迟: %ld ms\n", metrics.processing_delay_ms); - - // 验证性能指标合理 - assert(events_per_ms > 0); - assert(messages_per_ms > 0); - assert(mb_per_second > 0); - assert(memory_mb > 0); - assert(metrics.ring_buffer_usage <= 100); - assert(metrics.processing_delay_ms >= 0); - - printf("✓ 性能指标计算测试通过\n"); -} - -// 测试8: 错误处理 -static void test_error_handling() { - printf("\n=== 测试8: 错误处理 ===\n"); - - // 测试NULL指针处理 - print_observability_metrics(NULL); - - char buffer[100]; - format_observability_metrics_json(NULL, buffer, sizeof(buffer)); - format_observability_metrics_prometheus(NULL, buffer, sizeof(buffer)); - - // 测试空缓冲区 - SObservabilityMetrics metrics; - memset(&metrics, 0, sizeof(metrics)); - - format_observability_metrics_json(&metrics, NULL, 0); - format_observability_metrics_prometheus(&metrics, NULL, 0); - - // 测试小缓冲区 - char small_buffer[10]; - format_observability_metrics_json(&metrics, small_buffer, sizeof(small_buffer)); - format_observability_metrics_prometheus(&metrics, small_buffer, sizeof(small_buffer)); - - printf("✓ 错误处理测试通过\n"); -} - -int main() { - printf("开始增强可观测性指标测试...\n\n"); - - test_basic_observability_metrics(); - test_observability_metrics_print(); - test_json_formatting(); - test_prometheus_formatting(); - test_memory_usage_calculation(); - test_queue_watermark_monitoring(); - test_performance_metrics_calculation(); - test_error_handling(); - - printf("\n🎉 所有增强可观测性指标测试通过！\n"); - return 0; -} diff --git a/plugins/incremental_bitmap/test/test_observability_interface.c b/plugins/incremental_bitmap/test/test_observability_interface.c deleted file mode 100644 index 87c5265cea25..000000000000 --- a/plugins/incremental_bitmap/test/test_observability_interface.c +++ /dev/null @@ -1,148 +0,0 @@ -#include -#include -#include -#include "../include/storage_engine_interface.h" -#include "../include/observability.h" - -// 测试可观测性指标结构体 -static void test_observability_metrics_struct() { - printf("=== 测试可观测性指标结构体 ===\n"); - - // 创建并初始化指标结构体 - SObservabilityMetrics metrics; - memset(&metrics, 0, sizeof(metrics)); - - // 设置测试数据 - metrics.events_per_second = 1000; - metrics.messages_per_second = 1500; - metrics.bytes_per_second = 1024000; - metrics.consumer_lag_ms = 50; - metrics.offset_lag = 100; - metrics.processing_delay_ms = 10; - metrics.events_dropped = 5; - metrics.messages_dropped = 3; - metrics.parse_errors = 2; - metrics.connection_retries = 1; - metrics.subscription_retries = 0; - metrics.commit_retries = 2; - metrics.ring_buffer_usage = 75; - metrics.ring_buffer_capacity = 10000; - metrics.event_queue_size = 7500; - metrics.memory_usage_bytes = 1048576; - metrics.bitmap_memory_bytes = 524288; - metrics.metadata_memory_bytes = 262144; - metrics.last_update_time = 1234567890; - metrics.uptime_seconds = 3600; - - // 验证结构体大小（确保没有填充问题） - printf("结构体大小: %zu 字节\n", sizeof(SObservabilityMetrics)); - assert(sizeof(SObservabilityMetrics) > 0); - - // 验证字段访问 - assert(metrics.events_per_second == 1000); - assert(metrics.messages_per_second == 1500); - assert(metrics.consumer_lag_ms == 50); - assert(metrics.ring_buffer_usage == 75); - assert(metrics.memory_usage_bytes == 1048576); - - printf("✓ 可观测性指标结构体测试通过\n"); -} - -// 测试可观测性指标打印函数 -static void test_observability_metrics_print() { - printf("\n=== 测试可观测性指标打印函数 ===\n"); - - SObservabilityMetrics metrics; - memset(&metrics, 0, sizeof(metrics)); - - // 设置一些测试数据 - metrics.events_per_second = 2000; - metrics.messages_per_second = 2500; - metrics.consumer_lag_ms = 25; - metrics.events_dropped = 10; - metrics.ring_buffer_usage = 80; - metrics.memory_usage_bytes = 2097152; - metrics.uptime_seconds = 7200; - - // 调用打印函数 - printf("调用 print_observability_metrics 函数...\n"); - print_observability_metrics(&metrics); - - printf("✓ 可观测性指标打印函数测试通过\n"); -} - -// 测试存储引擎接口中的可观测性函数指针 -static void test_storage_engine_observability_interface() { - printf("\n=== 测试存储引擎接口中的可观测性函数 ===\n"); - - // 创建存储引擎接口结构体 - SStorageEngineInterface interface; - memset(&interface, 0, sizeof(interface)); - - // 验证函数指针类型 - printf("get_observability_metrics 函数指针类型: %zu 字节\n", - sizeof(interface.get_observability_metrics)); - - // 验证函数指针可以设置为NULL - interface.get_observability_metrics = NULL; - assert(interface.get_observability_metrics == NULL); - - printf("✓ 存储引擎接口中的可观测性函数测试通过\n"); -} - -// 测试可观测性指标更新函数 -static void test_observability_metrics_update() { - printf("\n=== 测试可观测性指标更新函数 ===\n"); - - SObservabilityMetrics metrics; - memset(&metrics, 0, sizeof(metrics)); - - // 调用更新函数 - printf("调用 update_observability_metrics 函数...\n"); - update_observability_metrics(&metrics); - - printf("✓ 可观测性指标更新函数测试通过\n"); -} - -// 测试可观测性指标的内存布局 -static void test_observability_metrics_memory_layout() { - printf("\n=== 测试可观测性指标内存布局 ===\n"); - - SObservabilityMetrics metrics; - memset(&metrics, 0, sizeof(metrics)); - - // 验证字段偏移量 - printf("events_per_second 偏移量: %zu\n", - (char*)&metrics.events_per_second - (char*)&metrics); - printf("messages_per_second 偏移量: %zu\n", - (char*)&metrics.messages_per_second - (char*)&metrics); - printf("consumer_lag_ms 偏移量: %zu\n", - (char*)&metrics.consumer_lag_ms - (char*)&metrics); - printf("ring_buffer_usage 偏移量: %zu\n", - (char*)&metrics.ring_buffer_usage - (char*)&metrics); - printf("memory_usage_bytes 偏移量: %zu\n", - (char*)&metrics.memory_usage_bytes - (char*)&metrics); - - // 验证所有偏移量都是合理的 - assert((char*)&metrics.events_per_second - (char*)&metrics >= 0); - assert((char*)&metrics.messages_per_second - (char*)&metrics >= 0); - assert((char*)&metrics.consumer_lag_ms - (char*)&metrics >= 0); - assert((char*)&metrics.ring_buffer_usage - (char*)&metrics >= 0); - assert((char*)&metrics.memory_usage_bytes - (char*)&metrics >= 0); - - printf("✓ 可观测性指标内存布局测试通过\n"); -} - -int main() { - printf("开始可观测性接口测试...\n\n"); - - test_observability_metrics_struct(); - test_observability_metrics_print(); - test_storage_engine_observability_interface(); - test_observability_metrics_update(); - test_observability_metrics_memory_layout(); - - printf("\n🎉 所有可观测性接口测试通过！\n"); - return 0; -} - diff --git a/plugins/incremental_bitmap/test/test_offset_semantics.c b/plugins/incremental_bitmap/test/test_offset_semantics.c deleted file mode 100644 index 256674e79732..000000000000 --- a/plugins/incremental_bitmap/test/test_offset_semantics.c +++ /dev/null @@ -1,450 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include "../include/storage_engine_interface.h" -#include "../include/observability.h" - -// 测试宏定义 -#define TEST_ASSERT(condition, message) do { \ - total_tests++; \ - if (condition) { \ - passed_tests++; \ - printf("✓ %s\n", message); \ - } else { \ - failed_tests++; \ - printf("❌ %s\n", message); \ - } \ -} while(0) - -// 测试计数器 -static int total_tests = 0; -static int passed_tests = 0; -static int failed_tests = 0; - -// 模拟TMQ上下文 -typedef struct { - bool auto_commit; - bool at_least_once; - int64_t last_commit_time; - uint64_t offset_commits; - uint64_t commit_retries; - pthread_mutex_t mutex; - int64_t committed_offsets[100]; // 模拟已提交的offset - int committed_count; -} MockTmqContext; - -static MockTmqContext g_mock_tmq = {0}; - -// 模拟Offset提交函数 -static int32_t mock_commit_offset(const char* topic_name, int32_t vg_id, int64_t offset, bool sync) { - pthread_mutex_lock(&g_mock_tmq.mutex); - - // 模拟提交延迟 - if (sync) { - usleep(1000); // 1ms延迟 - } - - // 检查offset是否有效 - if (offset < 0) { - pthread_mutex_unlock(&g_mock_tmq.mutex); - return -1; - } - - // 模拟重复提交检查 - bool is_duplicate = false; - for (int i = 0; i < g_mock_tmq.committed_count; i++) { - if (g_mock_tmq.committed_offsets[i] == offset) { - is_duplicate = true; - break; - } - } - - // 记录新的offset（如果不是重复的） - if (!is_duplicate && g_mock_tmq.committed_count < 100) { - g_mock_tmq.committed_offsets[g_mock_tmq.committed_count++] = offset; - } - - // 总是增加提交计数（包括重复提交） - g_mock_tmq.offset_commits++; - g_mock_tmq.last_commit_time = time(NULL) * 1000; - - pthread_mutex_unlock(&g_mock_tmq.mutex); - - printf("[MOCK] Offset提交: topic=%s, vg_id=%d, offset=%ld, sync=%s\n", - topic_name, vg_id, offset, sync ? "true" : "false"); - - return 0; -} - -// 初始化模拟TMQ上下文 -static void init_mock_tmq() { - memset(&g_mock_tmq, 0, sizeof(g_mock_tmq)); - pthread_mutex_init(&g_mock_tmq.mutex, NULL); - g_mock_tmq.auto_commit = false; - g_mock_tmq.at_least_once = true; -} - -// 清理模拟TMQ上下文 -static void cleanup_mock_tmq() { - pthread_mutex_destroy(&g_mock_tmq.mutex); -} - -// 测试1: 同步提交测试 -static void test_sync_commit() { - printf("\n=== 测试1: 同步提交测试 ===\n"); - - init_mock_tmq(); - - // 测试正常同步提交 - int32_t result = mock_commit_offset("test_topic", 1, 100, true); - TEST_ASSERT(result == 0, "同步提交成功"); - TEST_ASSERT(g_mock_tmq.offset_commits == 1, "提交计数正确"); - - // 测试无效offset - result = mock_commit_offset("test_topic", 1, -1, true); - TEST_ASSERT(result == -1, "无效offset被拒绝"); - TEST_ASSERT(g_mock_tmq.offset_commits == 1, "无效提交不影响计数"); - - // 测试重复提交 - result = mock_commit_offset("test_topic", 1, 100, true); - TEST_ASSERT(result == 0, "重复提交返回成功"); - TEST_ASSERT(g_mock_tmq.offset_commits == 2, "重复提交计数增加"); - - cleanup_mock_tmq(); -} - -// 测试2: 异步提交测试 -static void test_async_commit() { - printf("\n=== 测试2: 异步提交测试 ===\n"); - - init_mock_tmq(); - - // 测试异步提交 - int32_t result = mock_commit_offset("test_topic", 1, 200, false); - TEST_ASSERT(result == 0, "异步提交成功"); - - // 异步提交应该立即返回，但实际提交可能延迟 - usleep(2000); // 等待2ms - - TEST_ASSERT(g_mock_tmq.offset_commits == 1, "异步提交计数正确"); - - // 测试多个异步提交 - for (int i = 0; i < 5; i++) { - result = mock_commit_offset("test_topic", 1, 300 + i, false); - TEST_ASSERT(result == 0, "批量异步提交成功"); - } - - usleep(5000); // 等待5ms - TEST_ASSERT(g_mock_tmq.offset_commits == 6, "批量异步提交计数正确"); - - cleanup_mock_tmq(); -} - -// 测试3: 至少一次语义验证 -static void test_at_least_once_semantics() { - printf("\n=== 测试3: 至少一次语义验证 ===\n"); - - init_mock_tmq(); - g_mock_tmq.at_least_once = true; - - // 模拟至少一次语义：同步提交确保消息被处理 - int64_t test_offsets[] = {100, 101, 102, 103, 104}; - int offset_count = sizeof(test_offsets) / sizeof(test_offsets[0]); - - for (int i = 0; i < offset_count; i++) { - // 模拟消息处理 - printf("处理消息 offset=%ld\n", test_offsets[i]); - - // 至少一次：同步提交 - int32_t result = mock_commit_offset("test_topic", 1, test_offsets[i], true); - TEST_ASSERT(result == 0, "至少一次语义同步提交成功"); - } - - TEST_ASSERT(g_mock_tmq.offset_commits == offset_count, "所有offset都被提交"); - - // 验证提交顺序 - for (int i = 0; i < offset_count; i++) { - bool found = false; - for (int j = 0; j < g_mock_tmq.committed_count; j++) { - if (g_mock_tmq.committed_offsets[j] == test_offsets[i]) { - found = true; - break; - } - } - TEST_ASSERT(found, "offset被正确记录"); - } - - cleanup_mock_tmq(); -} - -// 测试4: 至多一次语义验证 -static void test_at_most_once_semantics() { - printf("\n=== 测试4: 至多一次语义验证 ===\n"); - - init_mock_tmq(); - g_mock_tmq.at_least_once = false; - - // 模拟至多一次语义：异步提交，可能丢失但不会重复 - int64_t test_offsets[] = {200, 201, 202, 203, 204}; - int offset_count = sizeof(test_offsets) / sizeof(test_offsets[0]); - - for (int i = 0; i < offset_count; i++) { - // 模拟消息处理 - printf("处理消息 offset=%ld\n", test_offsets[i]); - - // 至多一次：异步提交 - int32_t result = mock_commit_offset("test_topic", 1, test_offsets[i], false); - TEST_ASSERT(result == 0, "至多一次语义异步提交成功"); - } - - // 异步提交可能丢失，但不会重复处理 - TEST_ASSERT(g_mock_tmq.offset_commits <= offset_count, "提交数不超过消息数"); - - cleanup_mock_tmq(); -} - -// 测试5: 断点恢复测试 -static void test_checkpoint_recovery() { - printf("\n=== 测试5: 断点恢复测试 ===\n"); - - init_mock_tmq(); - - // 模拟初始状态：已提交到offset 150 - g_mock_tmq.committed_offsets[0] = 150; - g_mock_tmq.committed_count = 1; - g_mock_tmq.offset_commits = 1; - - // 模拟重启后的恢复 - printf("模拟系统重启，从offset 150恢复\n"); - - // 验证恢复后的状态 - TEST_ASSERT(g_mock_tmq.committed_count == 1, "恢复后保留已提交状态"); - TEST_ASSERT(g_mock_tmq.committed_offsets[0] == 150, "恢复后offset正确"); - - // 继续处理新消息 - int64_t new_offsets[] = {151, 152, 153, 154, 155}; - int new_count = sizeof(new_offsets) / sizeof(new_offsets[0]); - - for (int i = 0; i < new_count; i++) { - int32_t result = mock_commit_offset("test_topic", 1, new_offsets[i], true); - TEST_ASSERT(result == 0, "恢复后新消息提交成功"); - } - - TEST_ASSERT(g_mock_tmq.offset_commits == new_count + 1, "恢复后提交计数正确"); - - cleanup_mock_tmq(); -} - -// 测试6: 幂等性验证 -static void test_idempotency() { - printf("\n=== 测试6: 幂等性验证 ===\n"); - - init_mock_tmq(); - - int64_t test_offset = 300; - - // 第一次提交 - int32_t result1 = mock_commit_offset("test_topic", 1, test_offset, true); - TEST_ASSERT(result1 == 0, "第一次提交成功"); - int commit_count_1 = g_mock_tmq.offset_commits; - - // 重复提交相同offset - int32_t result2 = mock_commit_offset("test_topic", 1, test_offset, true); - TEST_ASSERT(result2 == 0, "重复提交返回成功"); - int commit_count_2 = g_mock_tmq.offset_commits; - - // 再次重复提交 - int32_t result3 = mock_commit_offset("test_topic", 1, test_offset, false); - TEST_ASSERT(result3 == 0, "异步重复提交返回成功"); - int commit_count_3 = g_mock_tmq.offset_commits; - - // 验证幂等性：多次提交相同offset不会产生副作用 - TEST_ASSERT(commit_count_1 == 1, "第一次提交计数正确"); - TEST_ASSERT(commit_count_2 == 2, "重复提交计数增加"); - TEST_ASSERT(commit_count_3 == 3, "异步重复提交计数增加"); - - // 验证offset只被记录一次 - int recorded_count = 0; - for (int i = 0; i < g_mock_tmq.committed_count; i++) { - if (g_mock_tmq.committed_offsets[i] == test_offset) { - recorded_count++; - } - } - TEST_ASSERT(recorded_count == 1, "相同offset只被记录一次"); - - cleanup_mock_tmq(); -} - -// 线程函数：并发提交offset - 移到函数外部 -static void* thread_func(void* arg) { - int thread_id = *(int*)arg; - for (int i = 0; i < 10; i++) { // 固定10次提交 - int64_t offset = thread_id * 1000 + i; - mock_commit_offset("test_topic", thread_id, offset, (i % 2) == 0); - usleep(100); // 小延迟 - } - return NULL; -} - -// 测试7: 并发提交测试 -static void test_concurrent_commit() { - printf("\n=== 测试7: 并发提交测试 ===\n"); - - init_mock_tmq(); - - const int thread_count = 5; - const int commits_per_thread = 10; - pthread_t threads[thread_count]; - - // 创建线程 - int thread_ids[thread_count]; - for (int i = 0; i < thread_count; i++) { - thread_ids[i] = i; - int result = pthread_create(&threads[i], NULL, thread_func, &thread_ids[i]); - TEST_ASSERT(result == 0, "线程创建成功"); - } - - // 等待所有线程完成 - for (int i = 0; i < thread_count; i++) { - pthread_join(threads[i], NULL); - } - - // 验证并发提交结果 - int expected_commits = thread_count * commits_per_thread; - TEST_ASSERT(g_mock_tmq.offset_commits == expected_commits, "并发提交计数正确"); - TEST_ASSERT(g_mock_tmq.committed_count <= expected_commits, "提交记录数合理"); - - cleanup_mock_tmq(); -} - -// 测试8: 边界条件测试 -static void test_edge_cases() { - printf("\n=== 测试8: 边界条件测试 ===\n"); - - init_mock_tmq(); - - // 测试最大offset值 - int64_t max_offset = INT64_MAX; - int32_t result = mock_commit_offset("test_topic", 1, max_offset, true); - TEST_ASSERT(result == 0, "最大offset提交成功"); - - // 测试0 offset - result = mock_commit_offset("test_topic", 1, 0, true); - TEST_ASSERT(result == 0, "0 offset提交成功"); - - // 测试负数offset - result = mock_commit_offset("test_topic", 1, -1, true); - TEST_ASSERT(result == -1, "负数offset被拒绝"); - - // 测试超大vg_id - result = mock_commit_offset("test_topic", 999999, 100, true); - TEST_ASSERT(result == 0, "大vg_id提交成功"); - - // 测试空topic名称 - result = mock_commit_offset("", 1, 100, true); - TEST_ASSERT(result == 0, "空topic名称处理正常"); - - cleanup_mock_tmq(); -} - -// 测试9: 性能测试 -static void test_performance() { - printf("\n=== 测试9: 性能测试 ===\n"); - - init_mock_tmq(); - - const int test_count = 1000; - clock_t start_time, end_time; - double sync_time, async_time; - - // 测试同步提交性能 - start_time = clock(); - for (int i = 0; i < test_count; i++) { - mock_commit_offset("test_topic", 1, i, true); - } - end_time = clock(); - sync_time = ((double)(end_time - start_time)) / CLOCKS_PER_SEC; - - // 重置计数器 - g_mock_tmq.offset_commits = 0; - g_mock_tmq.committed_count = 0; - - // 测试异步提交性能 - start_time = clock(); - for (int i = 0; i < test_count; i++) { - mock_commit_offset("test_topic", 1, i, false); - } - end_time = clock(); - async_time = ((double)(end_time - start_time)) / CLOCKS_PER_SEC; - - printf("同步提交 %d 次耗时: %.3f 秒\n", test_count, sync_time); - printf("异步提交 %d 次耗时: %.3f 秒\n", test_count, async_time); - - TEST_ASSERT(sync_time > 0, "同步提交性能测试完成"); - TEST_ASSERT(async_time > 0, "异步提交性能测试完成"); - TEST_ASSERT(async_time < sync_time, "异步提交比同步提交快"); - - cleanup_mock_tmq(); -} - -// 测试10: 错误恢复测试 -static void test_error_recovery() { - printf("\n=== 测试10: 错误恢复测试 ===\n"); - - init_mock_tmq(); - - // 模拟提交失败后的重试 - int retry_count = 0; - const int max_retries = 3; - - for (int retry = 0; retry < max_retries; retry++) { - int32_t result = mock_commit_offset("test_topic", 1, 500, true); - if (result == 0) { - printf("第 %d 次重试成功\n", retry + 1); - break; - } else { - retry_count++; - printf("第 %d 次重试失败，继续重试\n", retry + 1); - usleep(1000); // 重试延迟 - } - } - - TEST_ASSERT(retry_count <= max_retries, "重试次数在合理范围内"); - TEST_ASSERT(g_mock_tmq.offset_commits > 0, "最终提交成功"); - - cleanup_mock_tmq(); -} - -int main() { - printf("开始Offset语义验证测试...\n"); - - test_sync_commit(); - test_async_commit(); - test_at_least_once_semantics(); - test_at_most_once_semantics(); - test_checkpoint_recovery(); - test_idempotency(); - test_concurrent_commit(); - test_edge_cases(); - test_performance(); - test_error_recovery(); - - printf("\n=== 测试结果汇总 ===\n"); - printf("总测试数: %d\n", total_tests); - printf("通过测试: %d\n", passed_tests); - printf("失败测试: %d\n", failed_tests); - printf("通过率: %.2f%%\n", (double)passed_tests / total_tests * 100); - - if (failed_tests == 0) { - printf("\n🎉 所有Offset语义验证测试通过！\n"); - return 0; - } else { - printf("\n❌ 有 %d 个测试失败，需要修复！\n", failed_tests); - return 1; - } -} diff --git a/plugins/incremental_bitmap/test/test_offset_semantics_real.c b/plugins/incremental_bitmap/test/test_offset_semantics_real.c deleted file mode 100644 index 91806a7f04a0..000000000000 --- a/plugins/incremental_bitmap/test/test_offset_semantics_real.c +++ /dev/null @@ -1,757 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "../include/storage_engine_interface.h" -#include "../include/observability.h" -#include "../include/tdengine_storage_engine.h" - -// 测试宏定义 -#define TEST_ASSERT(condition, message) do { \ - total_tests++; \ - if (condition) { \ - passed_tests++; \ - printf("\xe2\x9c\x93 %s\n", message); \ - } else { \ - failed_tests++; \ - printf("\xe2\x9d\x8c %s\n", message); \ - } \ -} while(0) - -// 测试计数器 -static int total_tests = 0; -static int passed_tests = 0; -static int failed_tests = 0; - -// 全局变量 -static volatile bool g_running = true; -static int64_t g_last_committed_offset = -1; -static pthread_mutex_t g_offset_mutex = PTHREAD_MUTEX_INITIALIZER; -static uint64_t g_total_events_processed = 0; -static uint64_t g_total_events_committed = 0; - -// 信号处理 -static void signal_handler(int sig) { - printf("\n\xe6\x94\xb6\xe5\x88\xb0\xe4\xbf\xa1\xe5\x8f\xb7 %d\xef\xbc\x8c\xe6\xad\xa3\xe5\x9c\xa8\xe4\xbc\x98\xe9\x9b\x85\xe5\x85\xb3\xe9\x97\xad...\n", sig); - g_running = false; -} - -// 获取当前时间戳（毫秒） -static int64_t get_current_time_ms() { - struct timeval tv; - gettimeofday(&tv, NULL); - return tv.tv_sec * 1000LL + tv.tv_usec / 1000LL; -} - -// 事件回调函数 -static void test_event_callback(const SStorageEvent* event, void* user_data) { - if (!event || !user_data) return; - - int* event_count = (int*)user_data; - (*event_count)++; - - pthread_mutex_lock(&g_offset_mutex); - g_total_events_processed++; - - // 记录最新的offset - if (event->wal_offset > g_last_committed_offset) { - g_last_committed_offset = event->wal_offset; - } - pthread_mutex_unlock(&g_offset_mutex); - - printf("[\xe4\xba\x8b\xe4\xbb\xb6] block_id=%lu, wal_offset=%ld, timestamp=%ld, type=%d\n", - event->block_id, event->wal_offset, event->timestamp, event->event_type); -} - -// 等待事件处理 -static bool wait_for_events(int expected_count, int timeout_ms) { - int start_time = get_current_time_ms(); - int current_count = 0; - - while (current_count < expected_count && (get_current_time_ms() - start_time) < timeout_ms) { - usleep(10000); // 10ms - pthread_mutex_lock(&g_offset_mutex); - current_count = g_total_events_processed; - pthread_mutex_unlock(&g_offset_mutex); - } - - return current_count >= expected_count; -} - -// 测试1: 真实TMQ连接和配置测试 -static void test_real_tmq_connection() { - printf("\n=== \xe6\xb5\x8b\xe8\xaf\x951: \xe7\x9c\x9f\xe5\xae\x9eTMQ\xe8\xbf\x9e\xe6\x8e\xa5\xe5\x92\x8c\xe9\x85\x8d\xe7\xbd\xae\xe6\xb5\x8b\xe8\xaf\x95 ===\n"); - - // 设置TMQ配置 - int32_t result = tdengine_set_tmq_config( - "127.0.0.1", 6030, "root", "taosdata", - "test_db", "test_topic", "test_group" - ); - TEST_ASSERT(result == 0, "TMQ\xe9\x85\x8d\xe7\xbd\xae\xe8\xae\xbe\xe7\xbd\xae\xe6\x88\x90\xe5\x8a\x9f"); - - // 获取存储引擎接口 - SStorageEngineInterface* engine = get_storage_engine_interface("auto"); - TEST_ASSERT(engine != NULL, "\xe8\x8e\xb7\xe5\x8f\x96\xe5\xad\x98\xe5\x82\xa8\xe5\xbc\x95\xe6\x93\x8e\xe6\x8e\xa5\xe5\x8f\xa3\xe6\x88\x90\xe5\x8a\x9f"); - - // 初始化存储引擎 - SStorageEngineConfig config = { - .enable_interception = true, - .event_callback = test_event_callback, - .callback_user_data = &(int){0}, - .event_buffer_size = 1000, - .callback_threads = 2 - }; - - result = engine->init(&config); - TEST_ASSERT(result == 0, "\xe5\xad\x98\xe5\x82\xa8\xe5\xbc\x95\xe6\x93\x8e\xe5\x88\x9d\xe5\xa7\x8b\xe5\x8c\x96\xe6\x88\x90\xe5\x8a\x9f"); - - // 安装事件拦截 - result = engine->install_interception(); - TEST_ASSERT(result == 0, "\xe4\xba\x8b\xe4\xbb\xb6\xe6\x8b\xa6\xe6\x88\xaa\xe5\xae\x89\xe8\xa3\x85\xe6\x88\x90\xe5\x8a\x9f"); - - // 等待连接建立 - printf("\xe7\xad\x89\xe5\xbe\x85TMQ\xe8\xbf\x9e\xe6\x8e\xa5\xe5\xbb\xba\xe7\xab\x8b...\n"); - sleep(2); - - // 获取可观测性指标 - SObservabilityMetrics metrics; - result = engine->get_observability_metrics(engine, &metrics); - TEST_ASSERT(result == 0, "\xe8\x8e\xb7\xe5\x8f\x96\xe5\x8f\xaf\xe8\xa7\x82\xe6\xb5\x8b\xe6\x80\xa7\xe6\x8c\x87\xe6\xa0\x87\xe6\x88\x90\xe5\x8a\x9f"); - - printf("\xe8\xbf\x9e\xe6\x8e\xa5\xe7\x8a\xb6\xe6\x80\x81: uptime=%lds, memory=%zu bytes\n", - metrics.uptime_seconds, metrics.memory_usage_bytes); - - // 清理 - engine->uninstall_interception(); - engine->destroy(); -} - -// 测试2: 同步提交语义验证 -static void test_sync_commit_semantics() { - printf("\n=== \xe6\xb5\x8b\xe8\xaf\x952: \xe5\x90\x8c\xe6\xad\xa5\xe6\x8f\x90\xe4\xba\xa4\xe8\xaf\xad\xe4\xb9\x89\xe9\xaa\x8c\xe8\xaf\x81 ===\n"); - - // 设置TMQ配置 - int32_t result = tdengine_set_tmq_config( - "127.0.0.1", 6030, "root", "taosdata", - "test_db", "test_topic", "test_group_sync" - ); - TEST_ASSERT(result == 0, "TMQ\xe9\x85\x8d\xe7\xbd\x8e\xe8\xae\xbe\xe7\xbd\xae\xe6\x88\x90\xe5\x8a\x9f"); - - // 设置提交策略：同步提交，至少一次语义 - result = tdengine_set_commit_strategy(true, true, 1000); - TEST_ASSERT(result == 0, "\xe5\x90\x8c\xe6\xad\xa5\xe6\x8f\x90\xe4\xba\xa4\xe7\xad\x96\xe7\x95\xa5\xe8\xae\xbe\xe7\xbd\xae\xe6\x88\x90\xe5\x8a\x9f"); - - // 获取存储引擎接口 - SStorageEngineInterface* engine = get_storage_engine_interface("auto"); - TEST_ASSERT(engine != NULL, "\xe8\x8e\xb7\xe5\x8f\x96\xe5\xad\x98\xe5\x82\xa8\xe5\xbc\x95\xe6\x93\x8e\xe6\x8e\xa5\xe5\x8f\xa3\xe6\x88\x90\xe5\x8a\x9f"); - - // 初始化存储引擎 - StorageEventCallback callback = test_event_callback; - int event_count = 0; - SStorageEngineConfig config = { - .enable_interception = true, - .event_callback = callback, - .callback_user_data = &event_count, - .event_buffer_size = 1000, - .callback_threads = 2 - }; - - result = engine->init(&config); - TEST_ASSERT(result == 0, "\xe5\xad\x98\xe5\x82\xa8\xe5\xbc\x95\xe6\x93\x8e\xe5\x88\x9d\xe5\xa7\x8b\xe5\x8c\x96\xe6\x88\x90\xe5\x8a\x9f"); - - // 安装事件拦截 - result = engine->install_interception(); - TEST_ASSERT(result == 0, "\xe4\xba\x8b\xe4\xbb\xb6\xe6\x8b\xa6\xe6\x88\xaa\xe5\xae\x89\xe8\xa3\x85\xe6\x88\x90\xe5\x8a\x9f"); - - // 等待连接建立 - printf("\xe7\xad\x89\xe5\xbe\x85TMQ\xe8\xbf\x9e\xe6\x8e\xa5\xe5\xbb\xba\xe7\xab\x8b...\n"); - sleep(2); - - // 记录初始状态 - int64_t initial_offset = g_last_committed_offset; - uint64_t initial_events = g_total_events_processed; - - // 等待事件处理 - printf("\xe7\xad\x89\xe5\xbe\x85\xe4\xba\x8b\xe4\xbb\xb6\xe5\xa4\x84\xe7\x90\x86...\n"); - if (wait_for_events(initial_events + 5, 10000)) { - printf("\xe6\x8e\xa5\xe6\x94\xb6\xe5\x88\xb0 %d \xe4\xb8\xaa\xe6\x96\xb0\xe4\xba\x8b\xe4\xbb\xb6\n", (int)(g_total_events_processed - initial_events)); - - // 验证同步提交：最新offset应该被提交 - int64_t current_offset = g_last_committed_offset; - TEST_ASSERT(current_offset > initial_offset, "\xe5\x90\x8c\xe6\xad\xa5\xe6\x8f\x90\xe4\xba\xa4\xe5\x90\x8eoffset\xe5\x89\x8d\xe8\xbf\x9b"); - - // 获取详细统计 - uint64_t events_processed, events_dropped, messages_consumed, offset_commits; - int64_t last_commit_time; - result = tdengine_get_detailed_stats( - &events_processed, &events_dropped, &messages_consumed, - &offset_commits, &last_commit_time - ); - TEST_ASSERT(result == 0, "\xe8\x8e\xb7\xe5\x8f\x96\xe8\xaf\xa6\xe7\xbb\x86\xe7\xbb\x9f\xe8\xae\xa1\xe6\x88\x90\xe5\x8a\x9f"); - - printf("\xe7\xbb\x9f\xe8\xae\xa1: \xe5\xa4\x84\xe7\x90\x86=%lu, \xe4\xb8\xa2\xe5\xbc\x83=%lu, \xe6\xb6\x88\xe8\xb4\xb9=%lu, \xe6\x8f\x90\xe4\xba\xa4=%lu, \xe6\x9c\x80\xe5\x90\x8e\xe6\x8f\x90\xe4\xba\xa4=%ld\n", - events_processed, events_dropped, messages_consumed, offset_commits, last_commit_time); - - TEST_ASSERT(offset_commits > 0, "\xe6\x9c\x89offset\xe8\xa2\xab\xe6\x8f\x90\xe4\xba\xa4"); - TEST_ASSERT(last_commit_time > 0, "\xe6\x9c\x80\xe5\x90\x8e\xe6\x8f\x90\xe4\xba\xa4\xe6\x97\xb6\xe9\x97\xb4\xe6\x9c\x89\xe6\x95\x88"); - - } else { - printf("\xe7\xad\x89\xe5\xbe\x85\xe4\xba\x8b\xe4\xbb\xb6\xe8\xb6\x85\xe6\x97\xb6\xef\xbc\x8c\xe8\xb7\xb3\xe8\xbf\x87\xe5\x90\x8c\xe6\xad\xa5\xe6\x8f\x90\xe4\xba\xa4\xe9\xaa\x8c\xe8\xaf\x81\n"); - } - - // 清理 - engine->uninstall_interception(); - engine->destroy(); -} - -// 测试3: 异步提交语义验证 -static void test_async_commit_semantics() { - printf("\n=== \xe6\xb5\x8b\xe8\xaf\x953: \xe5\xbc\x82\xe6\ad\xa5\xe6\x8f\x90\xe4\xba\xa4\xe8\xaf\xad\xe4\xb9\x89\xe9\xaa\x8c\xe8\xaf\x81 ===\n"); - - // 设置TMQ配置 - int32_t result = tdengine_set_tmq_config( - "127.0.0.1", 6030, "root", "taosdata", - "test_db", "test_topic", "test_group_async" - ); - TEST_ASSERT(result == 0, "TMQ\xe9\x85\x8d\xe7\xbd\x8e\xe8\xae\xbe\xe7\xbd\xae\xe6\x88\x90\xe5\x8a\x9f"); - - // 设置提交策略：异步提交，至多一次语义 - result = tdengine_set_commit_strategy(false, false, 500); - TEST_ASSERT(result == 0, "\xe5\xbc\x82\xe6\ad\xa5\xe6\x8f\x90\xe4\xba\xa4\xe7\xad\x96\xe7\x95\xa5\xe8\xae\xbe\xe7\xbd\xae\xe6\x88\x90\xe5\x8a\x9f"); - - // 获取存储引擎接口 - SStorageEngineInterface* engine = get_storage_engine_interface("auto"); - TEST_ASSERT(engine != NULL, "\xe8\x8e\xb7\xe5\x8f\x96\xe5\xad\x98\xe5\x82\xa8\xe5\xbc\x95\xe6\x93\x8e\xe6\x8e\xa5\xe5\x8f\xa3\xe6\x88\x90\xe5\x8a\x9f"); - - // 初始化存储引擎 - StorageEventCallback callback = test_event_callback; - int event_count = 0; - SStorageEngineConfig config = { - .enable_interception = true, - .event_callback = callback, - .callback_user_data = &event_count, - .event_buffer_size = 1000, - .callback_threads = 2 - }; - - result = engine->init(&config); - TEST_ASSERT(result == 0, "\xe5\xad\x98\xe5\x82\xa8\xe5\xbc\x95\xe6\x93\x8e\xe5\x88\x9d\xe5\xa7\x8b\xe5\x8c\x96\xe6\x88\x90\xe5\x8a\x9f"); - - // 安装事件拦截 - result = engine->install_interception(); - TEST_ASSERT(result == 0, "\xe4\xba\x8b\xe4\xbb\xb6\xe6\x8b\xa6\xe6\x88\xaa\xe5\xae\x89\xe8\xa3\x85\xe6\x88\x90\xe5\x8a\x9f"); - - // 等待连接建立 - printf("\xe7\xad\x89\xe5\xbe\x85TMQ\xe8\xbf\x9e\xe6\x8e\xa5\xe5\xbb\xba\xe7\xab\x8b...\n"); - sleep(2); - - // 记录初始状态 - int64_t initial_offset = g_last_committed_offset; - uint64_t initial_events = g_total_events_processed; - - // 等待事件处理 - printf("\xe7\xad\x89\xe5\xbe\x85\xe4\xba\x8b\xe4\xbb\xb6\xe5\xa4\x84\xe7\x90\x86...\n"); - if (wait_for_events(initial_events + 5, 10000)) { - printf("\xe6\x8e\xa5\xe6\x94\xb6\xe5\x88\xb0 %d \xe4\xb8\xaa\xe6\x96\xb0\xe4\xba\x8b\xe4\xbb\xb6\n", (int)(g_total_events_processed - initial_events)); - - // 验证异步提交：offset可能延迟提交 - int64_t current_offset = g_last_committed_offset; - TEST_ASSERT(current_offset >= initial_offset, "\xe5\xbc\x82\xe6\ad\xa5\xe6\x8f\x90\xe4\xba\xa4\xe5\x90\x8eoffset\xe6\x9c\xaa\xe5\x9b\x9e\xe9\x80\x80"); - - // 获取可观测性指标 - SObservabilityMetrics metrics; - result = engine->get_observability_metrics(engine, &metrics); - TEST_ASSERT(result == 0, "\xe8\x8e\xb7\xe5\x8f\x96\xe5\x8f\xaf\xe8\xa7\x82\xe6\xb5\x8b\xe6\x80\xa7\xe6\x8c\x87\xe6\xa0\x87\xe6\x88\x90\xe5\x8a\x9f"); - - printf("\xe5\xbc\x82\xe6\ad\xa5\xe6\x8f\x90\xe4\xba\xa4\xe6\x8c\x87\xe6\xa0\x87: \xe4\xba\x8b\xe4\xbb\xb6/\xe7\xa7\x92=%lu, \xe6\xb6\x88\xe6\x81\xaf/\xe7\xa7\x92=%lu, \xe6\xb6\x88\xe8\xb4\xb9\xe6\xbb\x9e\xe5\x90\x8e=%ldms\n", - metrics.events_per_second, metrics.messages_per_second, metrics.consumer_lag_ms); - - // 异步提交可能有一些延迟 - TEST_ASSERT(metrics.events_per_second >= 0, "\xe4\xba\x8b\xe4\xbb\xb6\xe9\x80\x9f\xe7\x8e\x87\xe6\x9c\x89\xe6\x95\x88"); - - } else { - printf("\xe7\xad\x89\xe5\xbe\x85\xe4\xba\x8b\xe4\xbb\xb6\xe8\xb6\x85\xe6\x97\xb6\xef\xbc\x8c\xe8\xb7\xb3\xe8\xbf\x87\xe5\xbc\x82\xe6\xad\xa5\xe6\x8f\x90\xe4\xba\xa4\xe9\xaa\x8c\xe8\xaf\x81\n"); - } - - // 清理 - engine->uninstall_interception(); - engine->destroy(); -} - -// 测试4: 至少一次语义验证 -static void test_at_least_once_semantics() { - printf("\n=== \xe6\xb5\x8b\xe8\xaf\x954: \xe8\x87\xb3\xe5\xb0\x91\xe4\xb8\x80\xe6\xac\xa1\xe8\xaf\xad\xe4\xb9\x89\xe9\xaa\x8c\xe8\xaf\x81 ===\n"); - - // 设置TMQ配置 - int32_t result = tdengine_set_tmq_config( - "127.0.0.1", 6030, "root", "taosdata", - "test_db", "test_topic", "test_group_at_least_once" - ); - TEST_ASSERT(result == 0, "TMQ\xe9\x85\x8d\xe7\xbd\x8e\xe8\xae\xbe\xe7\xbd\xae\xe6\x88\x90\xe5\x8a\x9f"); - - // 设置提交策略：同步提交，至少一次语义 - result = tdengine_set_commit_strategy(true, true, 1000); - TEST_ASSERT(result == 0, "\xe8\x87\xb3\xe5\xb0\x91\xe4\xb8\x80\xe6\xac\xa1\xe8\xaf\xad\xe4\xb9\x89\xe7\xad\x96\xe7\x95\xa5\xe8\xae\xbe\xe7\xbd\xae\xe6\x88\x90\xe5\x8a\x9f"); - - // 获取存储引擎接口 - SStorageEngineInterface* engine = get_storage_engine_interface("auto"); - TEST_ASSERT(engine != NULL, "\xe8\x8e\xb7\xe5\x8f\x96\xe5\xad\x98\xe5\x82\xa8\xe5\xbc\x95\xe6\x93\x8e\xe6\x8e\xa5\xe5\x8f\xa3\xe6\x88\x90\xe5\x8a\x9f"); - - // 初始化存储引擎 - StorageEventCallback callback = test_event_callback; - int event_count = 0; - SStorageEngineConfig config = { - .enable_interception = true, - .event_callback = callback, - .callback_user_data = &event_count, - .event_buffer_size = 1000, - .callback_threads = 2 - }; - - result = engine->init(&config); - TEST_ASSERT(result == 0, "\xe5\xad\x98\xe5\x82\xa8\xe5\xbc\x95\xe6\x93\x8e\xe5\x88\x9d\xe5\xa7\x8b\xe5\x8c\x96\xe6\x88\x90\xe5\x8a\x9f"); - - // 安装事件拦截 - result = engine->install_interception(); - TEST_ASSERT(result == 0, "\xe4\xba\x8b\xe4\xbb\xb6\xe6\x8b\xa6\xe6\x88\xaa\xe5\xae\x89\xe8\xa3\x85\xe6\x88\x90\xe5\x8a\x9f"); - - // 等待连接建立 - printf("\xe7\xad\x89\xe5\xbe\x85TMQ\xe8\xbf\x9e\xe6\x8e\xa5\xe5\xbb\xba\xe7\xab\x8b...\n"); - sleep(2); - - // 记录初始状态 - uint64_t initial_events = g_total_events_processed; - int64_t initial_offset = g_last_committed_offset; - - // 等待事件处理 - printf("\xe7\xad\x89\xe5\xbe\x85\xe4\xba\x8b\xe4\xbb\xb6\xe5\xa4\x84\xe7\x90\x86...\n"); - if (wait_for_events(initial_events + 10, 15000)) { - uint64_t processed_events = g_total_events_processed - initial_events; - printf("\xe6\x8e\xa5\xe6\x94\xb6\xe5\x88\xb0 %lu \xe4\xb8\xaa\xe6\x96\xb0\xe4\xba\x8b\xe4\xbb\xb6\n", processed_events); - - // 验证至少一次语义：所有事件都应该被处理 - TEST_ASSERT(processed_events > 0, "\xe6\x9c\x89\xe4\xba\x8b\xe4\xbb\xb6\xe8\xa2\xab\xe5\xa4\x84\xe7\x90\x86"); - - // 获取详细统计 - uint64_t events_processed, events_dropped, messages_consumed, offset_commits; - int64_t last_commit_time; - result = tdengine_get_detailed_stats( - &events_processed, &events_dropped, &messages_consumed, - &offset_commits, &last_commit_time - ); - TEST_ASSERT(result == 0, "\xe8\x8e\xb7\xe5\x8f\x96\xe8\xaf\xa6\xe7\xbb\x86\xe7\xbb\x9f\xe8\xae\xa1\xe6\x88\x90\xe5\x8a\x9f"); - - // 至少一次语义：丢弃的事件应该很少 - double drop_rate = (double)events_dropped / (events_processed + events_dropped); - printf("\xe4\xba\x8b\xe4\xbb\xb6\xe4\xb8\xa2\xe5\xbc\x83\xe7\x8e\x87: %.2f%% (%lu/%lu)\n", - drop_rate * 100, events_dropped, events_processed + events_dropped); - - TEST_ASSERT(drop_rate < 0.1, "\xe4\xba\x8b\xe4\xbb\xb6\xe4\xb8\xa2\xe5\xbc\x83\xe7\x8e\x87\xe4\xbd\x8e\xe4\xba\x8e10%"); - TEST_ASSERT(offset_commits > 0, "\xe6\x9c\x89offset\xe8\xa2\xab\xe6\x8f\x90\xe4\xba\xa4"); - - } else { - printf("\xe7\xad\x89\xe5\xbe\x85\xe4\xba\x8b\xe4\xbb\xb6\xe8\xb6\x85\xe6\x97\xb6\xef\xbc\x8c\xe8\xb7\xb3\xe8\xbf\x87\xe8\x87\xb3\xe5\xb0\x91\xe4\xb8\x80\xe6\xac\xa1\xe8\xaf\xad\xe4\xb9\x89\xe9\xaa\x8c\xe8\xaf\x81\n"); - } - - // 清理 - engine->uninstall_interception(); - engine->destroy(); -} - -// 测试5: 至多一次语义验证 -static void test_at_most_once_semantics() { - printf("\n=== \xe6\xb5\x8b\xe8\xaf\x955: \xe8\x87\xb3\xe5\xa4\x9a\xe4\xb8\x80\xe6\xac\xa1\xe8\xaf\xad\xe4\xb9\x89\xe9\xaa\x8c\xe8\xaf\x81 ===\n"); - - // 设置TMQ配置 - int32_t result = tdengine_set_tmq_config( - "127.0.0.1", 6030, "root", "taosdata", - "test_db", "test_topic", "test_group_at_most_once" - ); - TEST_ASSERT(result == 0, "TMQ\xe9\x85\x8d\xe7\xbd\x8e\xe8\xae\xbe\xe7\xbd\xae\xe6\x88\x90\xe5\x8a\x9f"); - - // 设置提交策略：异步提交，至多一次语义 - result = tdengine_set_commit_strategy(false, false, 500); - TEST_ASSERT(result == 0, "\xe8\x87\xb3\xe5\xa4\x9a\xe4\xb8\x80\xe6\xac\xa1\xe8\xaf\xad\xe4\xb9\x89\xe7\xad\x96\xe7\x95\xa5\xe8\xae\xbe\xe7\xbd\xae\xe6\x88\x90\xe5\x8a\x9f"); - - // 获取存储引擎接口 - SStorageEngineInterface* engine = get_storage_engine_interface("auto"); - TEST_ASSERT(engine != NULL, "\xe8\x8e\xb7\xe5\x8f\x96\xe5\xad\x98\xe5\x82\xa8\xe5\xbc\x95\xe6\x93\x8e\xe6\x8e\xa5\xe5\x8f\xa3\xe6\x88\x90\xe5\x8a\x9f"); - - // 初始化存储引擎 - StorageEventCallback callback = test_event_callback; - int event_count = 0; - SStorageEngineConfig config = { - .enable_interception = true, - .event_callback = callback, - .callback_user_data = &event_count, - .event_buffer_size = 1000, - .callback_threads = 2 - }; - - result = engine->init(&config); - TEST_ASSERT(result == 0, "\xe5\xad\x98\xe5\x82\xa8\xe5\xbc\x95\xe6\x93\x8e\xe5\x88\x9d\xe5\xa7\x8b\xe5\x8c\x96\xe6\x88\x90\xe5\x8a\x9f"); - - // 安装事件拦截 - result = engine->install_interception(); - TEST_ASSERT(result == 0, "\xe4\xba\x8b\xe4\xbb\xb6\xe6\x8b\xa6\xe6\x88\xaa\xe5\xae\x89\xe8\xa3\x85\xe6\x88\x90\xe5\x8a\x9f"); - - // 等待连接建立 - printf("\xe7\xad\x89\xe5\xbe\x85TMQ\xe8\xbf\x9e\xe6\x8e\xa5\xe5\xbb\xba\xe7\xab\x8b...\n"); - sleep(2); - - // 记录初始状态 - uint64_t initial_events = g_total_events_processed; - int64_t initial_offset = g_last_committed_offset; - - // 等待事件处理 - printf("\xe7\xad\x89\xe5\xbe\x85\xe4\xba\x8b\xe4\xbb\xb6\xe5\xa4\x84\xe7\x90\x86...\n"); - if (wait_for_events(initial_events + 10, 15000)) { - uint64_t processed_events = g_total_events_processed - initial_events; - printf("\xe6\x8e\xa5\xe6\x94\xb6\xe5\x88\xb0 %lu \xe4\xb8\xaa\xe6\x96\xb0\xe4\xba\x8b\xe4\xbb\xb6\n", processed_events); - - // 验证至多一次语义：可能丢失事件，但不会重复处理 - TEST_ASSERT(processed_events >= 0, "\xe4\xba\x8b\xe4\xbb\xb6\xe5\xa4\x84\xe7\x90\x86\xe8\xae\xa1\xe6\x95\xb0\xe6\x9c\x89\xe6\x95\x88"); - - // 获取可观测性指标 - SObservabilityMetrics metrics; - result = engine->get_observability_metrics(engine, &metrics); - TEST_ASSERT(result == 0, "\xe8\x8e\xb7\xe5\x8f\x96\xe5\x8f\xaf\xe8\xa7\x82\xe6\xb5\x8b\xe6\x80\xa7\xe6\x8c\x87\xe6\xa0\x87\xe6\x88\x90\xe5\x8a\x9f"); - - printf("\xe8\x87\xb3\xe5\xa4\x9a\xe4\xb8\x80\xe6\xac\xa1\xe8\xaf\xad\xe4\xb9\x89\xe6\x8c\x87\xe6\xa0\x87: \xe4\xba\x8b\xe4\xbb\xb6/\xe7\xa7\x92=%lu, \xe6\xb6\x88\xe6\x81\xaf/\xe7\xa7\x92=%lu, \xe6\xb6\x88\xe8\xb4\xb9\xe6\xbb\x9e\xe5\x90\x8e=%ldms\n", - metrics.events_per_second, metrics.messages_per_second, metrics.consumer_lag_ms); - - // 至多一次语义：处理延迟可能较高，但不会重复 - TEST_ASSERT(metrics.processing_delay_ms >= 0, "\xe5\xa4\x84\xe7\x90\x86\xe5\xbb\xb6\xe8\xbf\x9f\xe6\x9c\x89\xe6\x95\x88"); - - } else { - printf("\xe7\xad\x89\xe5\xbe\x85\xe4\xba\x8b\xe4\xbb\xb6\xe8\xb6\x85\xe6\x97\xb6\xef\xbc\x8c\xe8\xb7\xb3\xe8\xbf\x87\xe8\x87\xb3\xe5\xa4\x9a\xe4\xb8\x80\xe6\xac\xa1\xe8\xaf\xad\xe4\xb9\x89\xe9\xaa\x8c\xe8\xaf\x81\n"); - } - - // 清理 - engine->uninstall_interception(); - engine->destroy(); -} - -// 测试6: 断点恢复测试 -static void test_checkpoint_recovery() { - printf("\n=== \xe6\xb5\x8b\xe8\xaf\x956: \xe6\x96\xad\xe7\x82\xb9\xe6\x81\xa2\xe5\xa4\x8d\xe6\xb5\x8b\xe8\xaf\x95 ===\n"); - - // 设置TMQ配置 - int32_t result = tdengine_set_tmq_config( - "127.0.0.1", 6030, "root", "taosdata", - "test_db", "test_topic", "test_group_recovery" - ); - TEST_ASSERT(result == 0, "TMQ\xe9\x85\x8d\xe7\xbd\x8e\xe8\xae\xbe\xe7\xbd\xae\xe6\x88\x90\xe5\x8a\x9f"); - - // 设置提交策略：同步提交，至少一次语义 - result = tdengine_set_commit_strategy(true, true, 1000); - TEST_ASSERT(result == 0, "\xe6\x96\xad\xe7\x82\xb9\xe6\x81\xa2\xe5\xa4\x8d\xe7\xad\x96\xe7\x95\xa5\xe8\xae\xbe\xe7\xbd\xae\xe6\x88\x90\xe5\x8a\x9f"); - - // 第一次运行：建立连接并处理事件 - printf("\xe7\xac\xac\xe4\xb8\x80\xe6\xac\xa1\xe8\xbf\x90\xe8\xa1\x8c\xef\xbc\x9a\xe5\xbb\xba\xe7\xab\x8b\xe8\xbf\x9e\xe6\x8e\xa5...\n"); - SStorageEngineInterface* engine1 = get_storage_engine_interface("auto"); - TEST_ASSERT(engine1 != NULL, "\xe8\x8e\xb7\xe5\x8f\x96\xe5\xad\x98\xe5\x82\xa8\xe5\xbc\x95\xe6\x93\x8e\xe6\x8e\xa5\xe5\x8f\xa3\xe6\x88\x90\xe5\x8a\x9f"); - - StorageEventCallback callback = test_event_callback; - int event_count1 = 0; - SStorageEngineConfig config = { - .enable_interception = true, - .event_callback = callback, - .callback_user_data = &event_count1, - .event_buffer_size = 1000, - .callback_threads = 2 - }; - - result = engine1->init(&config); - TEST_ASSERT(result == 0, "\xe5\xad\x98\xe5\x82\xa8\xe5\xbc\x95\xe6\x93\x8e\xe5\x88\x9d\xe5\xa7\x8b\xe5\x8c\x96\xe6\x88\x90\xe5\x8a\x9f"); - - result = engine1->install_interception(); - TEST_ASSERT(result == 0, "\xe4\xba\x8b\xe4\xbb\xb6\xe6\x8b\xa6\xe6\x88\xaa\xe5\xae\x89\xe8\xa3\x85\xe6\x88\x90\xe5\x8a\x9f"); - - // 等待连接建立和事件处理 - printf("\xe7\xad\x89\xe5\xbe\x85\xe4\xba\x8b\xe4\xbb\xb6\xe5\xa4\x84\xe7\x90\x86...\n"); - sleep(3); - - // 记录第一次运行的状态 - int64_t first_run_offset = g_last_committed_offset; - uint64_t first_run_events = g_total_events_processed; - - printf("\xe7\xac\xac\xe4\xb8\x80\xe6\xac\xa1\xe8\xbf\x90\xe8\xa1\x8c\xe7\xbb\x93\xe6\x9e\x9c: offset=%ld, events=%lu\n", first_run_offset, first_run_events); - - // 清理第一次运行 - engine1->uninstall_interception(); - engine1->destroy(); - - // 等待一段时间 - printf("\xe7\xad\x895\xe7\xa7\x92\xe5\x90\x8e\xe9\x87\x8d\xe6\x96\xb0\xe8\xbf\x9e\xe6\x8e\xa5...\n"); - sleep(5); - - // 第二次运行：模拟重启后的恢复 - printf("\xe7\xac\xac\xe4\xba\x8c\xe6\xac\xa1\xe8\xbf\x90\xe8\xa1\x8c\xef\xbc\x9a\xe6\xa8\xa1\xe6\x8b\x9f\xe9\x87\x8d\xe5\x90\xaf\xe6\x81\xa2\xe5\xa4\x8d...\n"); - SStorageEngineInterface* engine2 = get_storage_engine_interface("auto"); - TEST_ASSERT(engine2 != NULL, "\xe8\x8e\xb7\xe5\x8f\x96\xe5\xad\x98\xe5\x82\xa8\xe5\xbc\x95\xe6\x93\x8e\xe6\x8e\xa5\xe5\x8f\xa3\xe6\x88\x90\xe5\x8a\x9f"); - - int event_count2 = 0; - config.callback_user_data = &event_count2; - - result = engine2->init(&config); - TEST_ASSERT(result == 0, "\xe5\xad\x98\xe5\x82\xa8\xe5\xbc\x95\xe6\x93\x8e\xe9\x87\x8d\xe6\x96\xb0\xe5\x88\x9d\xe5\xa7\x8b\xe5\x8c\x96\xe6\x88\x90\xe5\x8a\x9f"); - - result = engine2->install_interception(); - TEST_ASSERT(result == 0, "\xe4\xba\x8b\xe4\xbb\xb6\xe6\x8b\xa6\xe6\x88\xaa\xe9\x87\x8d\xe6\x96\xb0\xe5\xae\x89\xe8\xa3\x85\xe6\x88\x90\xe5\x8a\x9f"); - - // 等待连接建立和事件处理 - printf("\xe7\xad\x89\xe5\xbe\x85\xe4\xba\x8b\xe4\xbb\xb6\xe5\xa4\x84\xe7\x90\x86...\n"); - sleep(3); - - // 记录第二次运行的状态 - int64_t second_run_offset = g_last_committed_offset; - uint64_t second_run_events = g_total_events_processed; - - printf("\xe7\xac\xac\xe4\xba\x8c\xe6\xac\xa1\xe8\xbf\x90\xe8\xa1\x8c\xe7\xbb\x93\xe6\x9e\x9c: offset=%ld, events=%lu\n", second_run_offset, second_run_events); - - // 验证断点恢复：第二次运行应该从上次的offset继续 - if (first_run_offset > 0) { - TEST_ASSERT(second_run_offset >= first_run_offset, "\xe6\x81\xa2\xe5\xa4\x8doffset\xe6\x9c\xaa\xe5\x9b\x9e\xe9\x80\x80"); - printf("\xe6\x96\xad\xe7\x82\xb9\xe6\x81\xa2\xe5\xa4\x8d\xe6\x88\x90\xe5\x8a\x9f: \xe4\xbb\x8eoffset %ld \xe7\xbb\xad\xe7\xbb\xad\n", first_run_offset); - } else { - printf("\xe9\xa6\x96\xe6\xac\xa1\xe8\xbf\x90\xe8\xa1\x8c\xe6\x97\xa0\xe6\x9c\x89\xe6\x95\x88offset\xef\xbc\x8c\xe8\xb7\xb3\xe8\xbf\x87\xe6\x96\xad\xe7\x82\xb9\xe6\x81\xa2\xe5\xa4\x8d\xe9\xaa\x8c\xe8\xaf\x81\n"); - } - - // 清理第二次运行 - engine2->uninstall_interception(); - engine2->destroy(); -} - -// 测试7: 性能基准测试 -static void test_performance_benchmark() { - printf("\n=== \xe6\xb5\x8b\xe8\xaf\x957: \xe6\x80\xa7\xe8\x83\xbd\xe5\x9f\xba\xe5\x87\x86\xe6\xb5\x8b\xe8\xaf\x95 ===\n"); - - // 设置TMQ配置 - int32_t result = tdengine_set_tmq_config( - "127.0.0.1", 6030, "root", "taosdata", - "test_db", "test_topic", "test_group_perf" - ); - TEST_ASSERT(result == 0, "TMQ\xe9\x85\x8d\xe7\xbd\x8e\xe8\xae\xbe\xe7\xbd\xae\xe6\x88\x90\xe5\x8a\x9f"); - - // 设置提交策略：异步提交，至多一次语义（性能优先） - result = tdengine_set_commit_strategy(false, false, 100); - TEST_ASSERT(result == 0, "\xe6\x80\xa7\xe8\x83\xbd\xe6\xb5\x8b\xe8\xaf\x95\xe7\xad\x96\xe7\x95\xa5\xe8\xae\xbe\xe7\xbd\xae\xe6\x88\x90\xe5\x8a\x9f"); - - // 获取存储引擎接口 - SStorageEngineInterface* engine = get_storage_engine_interface("auto"); - TEST_ASSERT(engine != NULL, "\xe8\x8e\xb7\xe5\x8f\x96\xe5\xad\x98\xe5\x82\xa8\xe5\xbc\x95\xe6\x93\x8e\xe6\x8e\xa5\xe5\x8f\xa3\xe6\x88\x90\xe5\x8a\x9f"); - - // 初始化存储引擎 - StorageEventCallback callback = test_event_callback; - int event_count = 0; - SStorageEngineConfig config = { - .enable_interception = true, - .event_callback = callback, - .callback_user_data = &event_count, - .event_buffer_size = 10000, - .callback_threads = 4 - }; - - result = engine->init(&config); - TEST_ASSERT(result == 0, "\xe5\xad\x98\xe5\x82\xa8\xe5\xbc\x95\xe6\x93\x8e\xe5\x88\x9d\xe5\xa7\x8b\xe5\x8c\x96\xe6\x88\x90\xe5\x8a\x9f"); - - // 安装事件拦截 - result = engine->install_interception(); - TEST_ASSERT(result == 0, "\xe4\xba\x8b\xe4\xbb\xb6\xe6\x8b\xa6\xe6\x88\xaa\xe5\xae\x89\xe8\xa3\x85\xe6\x88\x90\xe5\x8a\x9f"); - - // 等待连接建立 - printf("\xe7\xad\x89\xe5\xbe\x85TMQ\xe8\xbf\x9e\xe6\x8e\xa5\xe5\xbb\xba\xe7\xab\x8b...\n"); - sleep(2); - - // 记录性能测试开始时间 - int64_t start_time = get_current_time_ms(); - uint64_t start_events = g_total_events_processed; - - // 运行性能测试 - printf("\xe8\xbf\x90\xe8\xa1\x8c\xe6\x80\xa7\xe8\x83\xbd\xe6\xb5\x8b\xe8\xaf\x9530\xe7\xa7\x92...\n"); - int test_duration = 30; - int elapsed = 0; - - while (elapsed < test_duration && g_running) { - sleep(5); - elapsed += 5; - - // 获取当前性能指标 - SObservabilityMetrics metrics; - result = engine->get_observability_metrics(engine, &metrics); - if (result == 0) { - printf("\xe6\x80\xa7\xe8\x83\xbd\xe6\x8c\x87\xe6\xa0\x87 [%ds]: \xe4\xba\x8b\xe4\xbb\xb6/\xe7\xa7\x92=%lu, \xe6\xb6\x88\xe6\x81\xaf/\xe7\xa7\x92=%lu, \xe5\x86\x85\xe5\xad\x98=%zu bytes, \xe9\x98\x9f\xe5\x88\x97\xe4\xbd\xbf\xe7\x94\xa8=%u%%\n", - elapsed, metrics.events_per_second, metrics.messages_per_second, - metrics.memory_usage_bytes, metrics.ring_buffer_usage); - } - } - - // 记录性能测试结束时间 - int64_t end_time = get_current_time_ms(); - uint64_t end_events = g_total_events_processed; - - // 计算性能指标 - int64_t total_duration = end_time - start_time; - uint64_t total_events = end_events - start_events; - - if (total_duration > 0) { - double events_per_second = (double)total_events / (total_duration / 1000.0); - printf("\xe6\x80\xa7\xe8\x83\xbd\xe6\xb5\x8b\xe8\xaf\x95\xe7\xbb\x93\xe6\x9e\x9c:\n"); - printf(" \xe6\x80\xbb\xe6\x97\xb6\xe9\x95\xbf: %ld ms\n", total_duration); - printf(" \xe6\x80\xbb\xe4\xba\x8b\xe4\xbb\xb6: %lu\n", total_events); - printf(" \xe4\xba\x8b\xe4\xbb\xb6\xe9\x80\x9f\xe7\x8e\x87: %.2f events/sec\n", events_per_second); - - // 验证性能指标 - TEST_ASSERT(total_events >= 0, "\xe4\xba\x8b\xe4\xbb\xb6\xe8\xae\xa1\xe6\x95\xb0\xe6\x9c\x89\xe6\x95\x88"); - TEST_ASSERT(events_per_second >= 0, "\xe4\xba\x8b\xe4\xbb\xb6\xe9\x80\x9f\xe7\x8e\x87\xe6\x9c\x89\xe6\x95\x88"); - - // 获取最终性能指标 - SObservabilityMetrics final_metrics; - result = engine->get_observability_metrics(engine, &final_metrics); - if (result == 0) { - printf(" \xe6\x9c\x80\xe7\xbb\x88\xe5\x86\x85\xe5\xad\x98\xe4\xbd\xbf\xe7\x94\xa8: %zu bytes\n", final_metrics.memory_usage_bytes); - printf(" \xe6\x9c\x80\xe7\xbb\x88\xe9\x98\x9f\xe5\x88\x97\xe4\xbd\xbf\xe7\x94\xa8: %u%%\n", final_metrics.ring_buffer_usage); - - TEST_ASSERT(final_metrics.memory_usage_bytes > 0, "\xe5\x86\x85\xe5\xad\x98\xe4\xbd\xbf\xe7\x94\xa8\xe6\x9c\x89\xe6\x95\x88"); - TEST_ASSERT(final_metrics.ring_buffer_usage <= 100, "\xe9\x98\x9f\xe5\x88\x97\xe4\xbd\xbf\xe7\x94\xa8\xe7\x8e\x87\xe5\x90\x88\xe7\x90\x86"); - } - } else { - printf("\xe6\x80\xa7\xe8\x83\xbd\xe6\xb5\x8b\xe8\xaf\x95\xe6\x97\xb6\xe9\x97\xb4\xe8\xbf\x87\xe7\x9f\xad\xef\xbc\x8c\xe8\xb7\xb3\xe8\xbf\x87\xe6\x80\xa7\xe8\x83\xbd\xe9\xaa\x8c\xe8\xaf\x81\n"); - } - - // 清理 - engine->uninstall_interception(); - engine->destroy(); -} - -// 测试8: 错误处理和恢复测试 -static void test_error_handling_and_recovery() { - printf("\n=== \xe6\xb5\x8b\xe8\xaf\x958: \xe9\x94\x99\xe8\xaf\xaf\xe5\xa4\x84\xe7\x90\x86\xe5\x92\x8c\xe6\x81\xa2\xe5\xa4\x8d\xe6\xb5\x8b\xe8\xaf\x95 ===\n"); - - // 测试无效配置 - printf("\xe6\xb5\x8b\xe8\xaf\x95\xe6\x97\xa0\xe6\x95\x88\xe9\x85\x8d\xe7\xbd\xae...\n"); - int32_t result = tdengine_set_tmq_config( - "invalid_host", 9999, "invalid_user", "invalid_pass", - "invalid_db", "invalid_topic", "invalid_group" - ); - // 注意：无效配置可能不会立即失败，而是在连接时失败 - - // 设置有效配置 - result = tdengine_set_tmq_config( - "127.0.0.1", 6030, "root", "taosdata", - "test_db", "test_topic", "test_group_error" - ); - TEST_ASSERT(result == 0, "\xe6\x9c\x89\xe6\x95\x88TMQ\xe9\x85\x8d\xe7\xbd\x8e\xe8\xae\xbe\xe7\xbd\xae\xe6\x88\x90\xe5\x8a\x9f"); - - // 获取存储引擎接口 - SStorageEngineInterface* engine = get_storage_engine_interface("auto"); - TEST_ASSERT(engine != NULL, "\xe8\x8e\xb7\xe5\x8f\x96\xe5\xad\x98\xe5\x82\xa8\xe5\xbc\x95\xe6\x93\x8e\xe6\x8e\xa5\xe5\x8f\xa3\xe6\x88\x90\xe5\x8a\x9f"); - - // 初始化存储引擎 - StorageEventCallback callback = test_event_callback; - int event_count = 0; - SStorageEngineConfig config = { - .enable_interception = true, - .event_callback = callback, - .callback_user_data = &event_count, - .event_buffer_size = 1000, - .callback_threads = 2 - }; - - result = engine->init(&config); - TEST_ASSERT(result == 0, "\xe5\xad\x98\xe5\x82\xa8\xe5\xbc\x95\xe6\x93\x8e\xe5\x88\x9d\xe5\xa7\x8b\xe5\x8c\x96\xe6\x88\x90\xe5\x8a\x9f"); - - // 安装事件拦截 - result = engine->install_interception(); - TEST_ASSERT(result == 0, "\xe4\xba\x8b\xe4\xbb\xb6\xe6\x8b\xa6\xe6\x88\xaa\xe5\xae\x89\xe8\xa3\x85\xe6\x88\x90\xe5\x8a\x9f"); - - // 等待连接建立 - printf("\xe7\xad\x89\xe5\xbe\x85TMQ\xe8\xbf\x9e\xe6\x8e\xa5\xe5\xbb\xba\xe7\xab\x8b...\n"); - sleep(2); - - // 获取错误统计 - uint64_t events_processed, events_dropped, messages_consumed, offset_commits; - int64_t last_commit_time; - result = tdengine_get_detailed_stats( - &events_processed, &events_dropped, &messages_consumed, - &offset_commits, &last_commit_time - ); - TEST_ASSERT(result == 0, "\xe8\x8e\xb7\xe5\x8f\x96\xe9\x94\x99\xe8\xaf\xaf\xe7\xbb\x9f\xe8\xae\xa1\xe6\x88\x90\xe5\x8a\x9f"); - - printf("\xe9\x94\x99\xe8\xaf\xaf\xe7\xbb\x9f\xe8\xae\xa1: \xe5\xa4\x84\xe7\x90\x86=%lu, \xe4\xb8\xa2\xe5\xbc\x83=%lu, \xe6\xb6\x88\xe8\xb4\xb9=%lu, \xe6\x8f\x90\xe4\xba\xa4=%lu\n", - events_processed, events_dropped, messages_consumed, offset_commits); - - // 验证错误处理：即使有错误，系统应该继续运行 - TEST_ASSERT(events_processed >= 0, "\xe5\xa4\x84\xe7\x90\x86\xe4\xba\x8b\xe4\xbb\xb6\xe8\xae\xa1\xe6\x95\xb0\xe6\x9c\x89\xe6\x95\x88"); - TEST_ASSERT(events_dropped >= 0, "\xe4\xb8\xa2\xe5\xbc\x83\xe4\xba\x8b\xe4\xbb\xb6\xe8\xae\xa1\xe6\x95\xb0\xe6\x9c\x89\xe6\x95\x88"); - - // 获取可观测性指标 - SObservabilityMetrics metrics; - result = engine->get_observability_metrics(engine, &metrics); - TEST_ASSERT(result == 0, "\xe8\x8e\xb7\xe5\x8f\x96\xe5\x8f\xaf\xe8\xa7\x82\xe6\xb5\x8b\xe6\x80\xa7\xe6\x8c\x87\xe6\xa0\x87\xe6\x88\x90\xe5\x8a\x9f"); - - printf("\xe9\x94\x99\xe8\xaf\xaf\xe6\x81\xa2\xe5\xa4\x8d\xe6\x8c\x87\xe6\xa0\x87: \xe8\xbf\x9e\xe6\x8e\xa5\xe9\x87\x8d\xe8\xaf\x95=%lu, \xe8\xae\xa2\xe9\x98\x85\xe9\x87\x8d\xe8\xaf\x95=%lu, \xe6\x8f\x90\xe4\xba\xa4\xe9\x87\x8d\xe8\xaf\x95=%lu, \xe8\xa7\xa3\xe6\x9e\x90\xe9\x94\x99\xe8\xaf\xaf=%lu\n", - metrics.connection_retries, metrics.subscription_retries, - metrics.commit_retries, metrics.parse_errors); - - // 验证重试机制 - TEST_ASSERT(metrics.connection_retries >= 0, "\xe8\xbf\x9e\xe6\x8e\xa5\xe9\x87\x8d\xe8\xaf\x95\xe8\xae\xa1\xe6\x95\xb0\xe6\x9c\x89\xe6\x95\x88"); - TEST_ASSERT(metrics.subscription_retries >= 0, "\xe8\xae\xa2\xe9\x98\x85\xe9\x87\x8d\xe8\xaf\x95\xe8\xae\xa1\xe6\x95\xb0\xe6\x9c\x89\xe6\x95\x88"); - TEST_ASSERT(metrics.commit_retries >= 0, "\xe6\x8f\x90\xe4\xba\xa4\xe9\x87\x8d\xe8\xaf\x95\xe8\xae\xa1\xe6\x95\xb0\xe6\x9c\x89\xe6\x95\x88"); - TEST_ASSERT(metrics.parse_errors >= 0, "\xe8\xa7\xa3\xe6\x9e\x90\xe9\x94\x99\xe8\xaf\xaf\xe8\xae\xa1\xe6\x95\xb0\xe6\x9c\x89\xe6\x95\x88"); - - // 清理 - engine->uninstall_interception(); - engine->destroy(); -} - -int main() { - printf("\xe5\xbc\x80\xe5\xa7\x8b\xe7\x9c\x9f\xe5\xae\x9eTDengine TMQ Offset\xe8\xaf\xad\xe4\xb9\x89\xe9\xaa\x8c\xe8\xaf\x81\xe6\xb5\x8b\xe8\xaf\x95...\n"); - printf("\xe6\xb3\xa8\xe6\x84\x8f\xef\xbc\x9a\xe6\xad\xa4\xe6\xb5\x8b\xe8\xaf\x95\xe9\x9c\x80\xe8\xa6\x81\xe8\xbf\x90\xe8\xa1\x8c\xe4\xb8\xad\xe7\x9a\x84TDengine\xe5\xae\x9e\xe4\xbe\x8b\xe5\x92\x8cTMQ topic\n"); - - // 设置信号处理 - signal(SIGINT, signal_handler); - signal(SIGTERM, signal_handler); - - // 初始化全局变量 - pthread_mutex_init(&g_offset_mutex, NULL); - g_total_events_processed = 0; - g_total_events_committed = 0; - g_last_committed_offset = -1; - - // 运行测试 - test_real_tmq_connection(); - test_sync_commit_semantics(); - test_async_commit_semantics(); - test_at_least_once_semantics(); - test_at_most_once_semantics(); - test_checkpoint_recovery(); - test_performance_benchmark(); - test_error_handling_and_recovery(); - - // 清理 - pthread_mutex_destroy(&g_offset_mutex); - - printf("\n=== \xe7\x9c\x9f\xe5\xae\x9eTDengine\xe6\xb5\x8b\xe8\xaf\x95\xe7\xbb\xbb\xe6\x9e\x9c\xe6\xb1\x87\xe6\x80\xbb ===\n"); - printf("\xe6\x80\xbb\xe6\xb5\x8b\xe8\xaf\x95\xe6\x95\xb0: %d\n", total_tests); - printf("\xe9\x80\x9a\xe8\xbf\x87\xe6\xb5\x8b\xe8\xaf\x95: %d\n", passed_tests); - printf("\xe5\xa4\xb1\xe8\xb4\xa5\xe6\xb5\x8b\xe8\xaf\x95: %d\n", failed_tests); - printf("\xe9\x80\x9a\xe8\xbf\x87\xe7\x8e\x87: %.2f%%\n", (double)passed_tests / total_tests * 100); - - if (failed_tests == 0) { - printf("\n\xf0\x9f\x8e\x89 \xe6\x89\x80\xe6\x9c\x89\xe7\x9c\x9f\xe5\xae\x9eTDengine Offset\xe8\xaf\xad\xe4\xb9\x89\xe9\xaa\x8c\xe8\xaf\x81\xe6\xb5\x8b\xe8\xaf\x95\xe9\x80\x9a\xe8\xbf\x87\xef\xbc\x81\n"); - return 0; - } else { - printf("\n\xe2\x9d\x8c \xe6\x9c\x89 %d \xe4\xb8\xaa\xe6\xb5\x8b\xe8\xaf\x95\xe5\xa4\xb1\xe8\xb4\xa5\xef\xbc\x8c\xe9\x9c\x80\xe8\xa6\x81\xe4\xbf\xae\xe5\xa4\x8d\xef\xbc\x81\n", failed_tests); - return 1; - } -} - - diff --git a/plugins/incremental_bitmap/test/test_offset_semantics_realtime.c b/plugins/incremental_bitmap/test/test_offset_semantics_realtime.c deleted file mode 100644 index 0d0ef9e2b47c..000000000000 --- a/plugins/incremental_bitmap/test/test_offset_semantics_realtime.c +++ /dev/null @@ -1,493 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -// 测试配置 -#define TEST_TOPIC "test_offset_topic" -#define TEST_VGROUP_ID 1 -#define MAX_MESSAGES 100 -#define MAX_RETRIES 3 -#define COMMIT_INTERVAL 100 // 每100条消息提交一次 - -// 全局测试状态 -typedef struct { - int64_t last_committed_offset; - int64_t current_offset; - int message_count; - int commit_count; - int retry_count; - pthread_mutex_t mutex; - pthread_cond_t cond; -} TestState; - -TestState g_test_state; - -// 测试结果统计 -typedef struct { - int total_tests; - int passed_tests; - int failed_tests; - double success_rate; -} TestResults; - -TestResults g_results = {0, 0, 0, 0.0}; - -// 测试断言宏 -#define TEST_ASSERT(condition, message) do { \ - g_results.total_tests++; \ - if (condition) { \ - g_results.passed_tests++; \ - printf("✓ %s\n", message); \ - } else { \ - g_results.failed_tests++; \ - printf("❌ %s\n", message); \ - } \ -} while(0) - -// 初始化测试状态 -void init_test_state() { - memset(&g_test_state, 0, sizeof(TestState)); - pthread_mutex_init(&g_test_state.mutex, NULL); - pthread_cond_init(&g_test_state.cond, NULL); - g_test_state.last_committed_offset = -1; - g_test_state.current_offset = 0; -} - -// 清理测试状态 -void cleanup_test_state() { - pthread_mutex_destroy(&g_test_state.mutex); - pthread_cond_destroy(&g_test_state.cond); -} - -// 模拟消息处理 -void process_message(int64_t offset, const char* message) { - pthread_mutex_lock(&g_test_state.mutex); - g_test_state.current_offset = offset; - g_test_state.message_count++; - - // 模拟消息处理延迟 - usleep(1000); // 1ms - - pthread_mutex_unlock(&g_test_state.mutex); -} - -// 模拟offset提交 -int commit_offset(int64_t offset, bool sync) { - pthread_mutex_lock(&g_test_state.mutex); - - // 模拟提交延迟 - usleep(1000); // 1ms - - // 检查offset是否有效 - if (offset <= g_test_state.last_committed_offset) { - pthread_mutex_unlock(&g_test_state.mutex); - return -1; // 无效的offset - } - - g_test_state.last_committed_offset = offset; - g_test_state.commit_count++; - - pthread_mutex_unlock(&g_test_state.mutex); - - printf("[COMMIT] offset=%ld, sync=%s, committed_count=%d\n", - offset, sync ? "true" : "false", g_test_state.commit_count); - - return 0; -} - -// 测试1: 同步提交测试 -void test_sync_commit() { - printf("\n=== 测试1: 同步提交测试 ===\n"); - - // 重置状态 - g_test_state.last_committed_offset = -1; - g_test_state.commit_count = 0; - - // 测试同步提交 - for (int i = 0; i < 10; i++) { - int64_t offset = i * 10; - int result = commit_offset(offset, true); - TEST_ASSERT(result == 0, "同步提交成功"); - TEST_ASSERT(g_test_state.last_committed_offset == offset, "offset正确记录"); - } - - TEST_ASSERT(g_test_state.commit_count == 10, "提交次数正确"); - printf("✓ 同步提交测试完成\n"); -} - -// 测试2: 异步提交测试 -void test_async_commit() { - printf("\n=== 测试2: 异步提交测试 ===\n"); - - // 重置状态 - g_test_state.last_committed_offset = -1; - g_test_state.commit_count = 0; - - // 测试异步提交 - for (int i = 0; i < 10; i++) { - int64_t offset = i * 10; - int result = commit_offset(offset, false); - TEST_ASSERT(result == 0, "异步提交成功"); - TEST_ASSERT(g_test_state.last_committed_offset == offset, "offset正确记录"); - } - - TEST_ASSERT(g_test_state.commit_count == 10, "提交次数正确"); - printf("✓ 异步提交测试完成\n"); -} - -// 测试3: 至少一次语义验证 -void test_at_least_once_semantics() { - printf("\n=== 测试3: 至少一次语义验证 ===\n"); - - // 重置状态 - g_test_state.last_committed_offset = -1; - g_test_state.commit_count = 0; - g_test_state.message_count = 0; - - // 模拟消息处理流程 - for (int i = 0; i < 20; i++) { - int64_t offset = i * 5; - char message[64]; - snprintf(message, sizeof(message), "message_%ld", offset); - - // 处理消息 - process_message(offset, message); - - // 每5条消息提交一次 - if ((i + 1) % 5 == 0) { - int result = commit_offset(offset, true); - TEST_ASSERT(result == 0, "offset提交成功"); - } - } - - // 验证至少一次语义 - TEST_ASSERT(g_test_state.message_count == 20, "消息处理数量正确"); - TEST_ASSERT(g_test_state.commit_count == 4, "提交次数正确"); - TEST_ASSERT(g_test_state.last_committed_offset == 95, "最后提交的offset正确"); - - printf("✓ 至少一次语义验证完成\n"); -} - -// 测试4: 至多一次语义验证 -void test_at_most_once_semantics() { - printf("\n=== 测试4: 至多一次语义验证 ===\n"); - - // 重置状态 - g_test_state.last_committed_offset = -1; - g_test_state.commit_count = 0; - - // 测试重复提交相同的offset - int64_t test_offset = 100; - - // 第一次提交 - int result1 = commit_offset(test_offset, true); - TEST_ASSERT(result1 == 0, "第一次提交成功"); - - // 重复提交相同offset - int result2 = commit_offset(test_offset, true); - TEST_ASSERT(result2 == -1, "重复提交被拒绝"); - - // 验证提交状态 - TEST_ASSERT(g_test_state.commit_count == 1, "只提交了一次"); - TEST_ASSERT(g_test_state.last_committed_offset == test_offset, "offset状态正确"); - - printf("✓ 至多一次语义验证完成\n"); -} - -// 测试5: 断点恢复测试 -void test_checkpoint_recovery() { - printf("\n=== 测试5: 断点恢复测试 ===\n"); - - // 重置状态 - g_test_state.last_committed_offset = -1; - g_test_state.commit_count = 0; - g_test_state.message_count = 0; - - // 模拟正常处理流程 - for (int i = 0; i < 15; i++) { - int64_t offset = i * 10; - char message[64]; - snprintf(message, sizeof(message), "message_%ld", offset); - - process_message(offset, message); - - // 每3条消息提交一次 - if ((i + 1) % 3 == 0) { - commit_offset(offset, true); - } - } - - // 模拟系统重启，从最后提交的offset恢复 - int64_t recovery_offset = g_test_state.last_committed_offset; - printf("[RECOVERY] 从offset %ld 恢复\n", recovery_offset); - - // 继续处理消息 - for (int i = 15; i < 25; i++) { - int64_t offset = i * 10; - char message[64]; - snprintf(message, sizeof(message), "message_%ld", offset); - - process_message(offset, message); - - // 每3条消息提交一次 - if ((i + 1) % 3 == 0) { - commit_offset(offset, true); - } - } - - // 验证恢复后的状态 - printf("恢复后状态: message_count=%d, last_committed_offset=%ld\n", - g_test_state.message_count, g_test_state.last_committed_offset); - TEST_ASSERT(g_test_state.message_count == 25, "恢复后消息总数正确"); - TEST_ASSERT(g_test_state.last_committed_offset == 230, "恢复后最后offset正确"); - - printf("✓ 断点恢复测试完成\n"); -} - -// 测试6: 幂等性验证 -void test_idempotency() { - printf("\n=== 测试6: 幂等性验证 ===\n"); - - // 重置状态 - g_test_state.last_committed_offset = -1; - g_test_state.commit_count = 0; - - // 测试幂等性：多次提交相同的offset序列 - int64_t offsets[] = {1000, 1001, 1002, 1003, 1004}; - int offset_count = sizeof(offsets) / sizeof(offsets[0]); - - // 第一次提交 - for (int i = 0; i < offset_count; i++) { - int result = commit_offset(offsets[i], true); - TEST_ASSERT(result == 0, "第一次提交成功"); - } - - int first_commit_count = g_test_state.commit_count; - - // 重复提交相同序列 - for (int i = 0; i < offset_count; i++) { - int result = commit_offset(offsets[i], true); - TEST_ASSERT(result == -1, "重复提交被拒绝"); - } - - // 验证幂等性 - printf("幂等性验证: commit_count=%d, first_commit_count=%d, last_committed_offset=%ld\n", - g_test_state.commit_count, first_commit_count, g_test_state.last_committed_offset); - TEST_ASSERT(g_test_state.commit_count == first_commit_count, "重复提交不影响状态"); - TEST_ASSERT(g_test_state.last_committed_offset == 1004, "最后offset保持不变"); - - printf("✓ 幂等性验证完成\n"); -} - -// 测试7: 并发提交测试 -void* concurrent_commit_thread(void* arg) { - int thread_id = *(int*)arg; - - for (int i = 0; i < 10; i++) { - int64_t offset = thread_id * 1000 + i; - int result = commit_offset(offset, false); - - if (result == 0) { - printf("[THREAD-%d] 提交成功: offset=%ld\n", thread_id, offset); - } else { - printf("[THREAD-%d] 提交失败: offset=%ld\n", thread_id, offset); - } - - usleep(1000); // 1ms延迟 - } - - return NULL; -} - -void test_concurrent_commit() { - printf("\n=== 测试7: 并发提交测试 ===\n"); - - // 重置状态 - g_test_state.last_committed_offset = -1; - g_test_state.commit_count = 0; - - // 创建多个并发线程 - pthread_t threads[3]; - int thread_ids[3] = {1, 2, 3}; - - for (int i = 0; i < 3; i++) { - int result = pthread_create(&threads[i], NULL, concurrent_commit_thread, &thread_ids[i]); - TEST_ASSERT(result == 0, "线程创建成功"); - } - - // 等待所有线程完成 - for (int i = 0; i < 3; i++) { - pthread_join(threads[i], NULL); - } - - // 验证并发提交结果 - TEST_ASSERT(g_test_state.commit_count > 0, "并发提交成功"); - TEST_ASSERT(g_test_state.last_committed_offset > 0, "最后offset正确"); - - printf("✓ 并发提交测试完成\n"); -} - -// 测试8: 边界条件测试 -void test_boundary_conditions() { - printf("\n=== 测试8: 边界条件测试 ===\n"); - - // 重置状态 - g_test_state.last_committed_offset = -1; - g_test_state.commit_count = 0; - - // 测试边界offset值 - int64_t boundary_offsets[] = {0, 1, INT64_MAX - 1, INT64_MAX}; - int boundary_count = sizeof(boundary_offsets) / sizeof(boundary_offsets[0]); - - printf("开始测试 %d 个边界值\n", boundary_count); - - for (int i = 0; i < boundary_count; i++) { - int64_t offset = boundary_offsets[i]; - printf("测试边界offset[%d]: %ld\n", i, offset); - - int result = commit_offset(offset, true); - printf("提交结果: %d\n", result); - - if (offset == 0) { - printf("测试offset=0: result=%d\n", result); - TEST_ASSERT(result == 0, "offset=0提交成功"); - } else if (offset == INT64_MAX) { - // INT64_MAX可能超出范围，但我们的实现应该能处理 - printf("测试offset=INT64_MAX: result=%d\n", result); - if (result == 0) { - printf("✓ offset=INT64_MAX提交成功\n"); - TEST_ASSERT(true, "offset=INT64_MAX提交成功"); - } else { - printf("✓ offset=INT64_MAX提交被拒绝（预期）\n"); - TEST_ASSERT(true, "offset=INT64_MAX提交被拒绝（预期）"); - } - } else { - // 对于其他边界值，检查是否成功提交 - printf("测试其他边界值: offset=%ld, result=%d\n", offset, result); - if (result == 0) { - printf("✓ offset=%ld提交成功\n", offset); - TEST_ASSERT(true, "边界offset提交成功"); - } else { - printf("✓ offset=%ld提交被拒绝（可能是重复提交）\n", offset); - TEST_ASSERT(true, "边界offset提交被拒绝（可能是重复提交）"); - } - } - - printf("当前提交状态: count=%d, last_offset=%ld\n", - g_test_state.commit_count, g_test_state.last_committed_offset); - } - - printf("✓ 边界条件测试完成\n"); -} - -// 测试9: 性能测试 -void test_performance() { - printf("\n=== 测试9: 性能测试 ===\n"); - - // 重置状态 - g_test_state.last_committed_offset = -1; - g_test_state.commit_count = 0; - - // 性能测试：批量提交 - const int batch_size = 1000; - clock_t start_time = clock(); - - for (int i = 0; i < batch_size; i++) { - int64_t offset = i; - commit_offset(offset, false); - } - - clock_t end_time = clock(); - double elapsed_time = ((double)(end_time - start_time)) / CLOCKS_PER_SEC; - double throughput = batch_size / elapsed_time; - - printf("批量提交 %d 个offset，耗时: %.3f 秒\n", batch_size, elapsed_time); - printf("吞吐量: %.2f ops/sec\n", throughput); - - TEST_ASSERT(throughput > 1000, "性能满足要求"); - TEST_ASSERT(g_test_state.commit_count == batch_size, "批量提交数量正确"); - - printf("✓ 性能测试完成\n"); -} - -// 测试10: 错误恢复测试 -void test_error_recovery() { - printf("\n=== 测试10: 错误恢复测试 ===\n"); - - // 重置状态 - g_test_state.last_committed_offset = -1; - g_test_state.commit_count = 0; - - // 模拟错误情况：提交无效offset - int64_t invalid_offsets[] = {-1, -100, -1000}; - int invalid_count = sizeof(invalid_offsets) / sizeof(invalid_offsets[0]); - - for (int i = 0; i < invalid_count; i++) { - int64_t offset = invalid_offsets[i]; - int result = commit_offset(offset, true); - TEST_ASSERT(result == -1, "无效offset被拒绝"); - } - - // 验证状态未受影响 - TEST_ASSERT(g_test_state.commit_count == 0, "无效提交不影响状态"); - TEST_ASSERT(g_test_state.last_committed_offset == -1, "最后offset状态正确"); - - // 恢复正常提交 - int result = commit_offset(100, true); - TEST_ASSERT(result == 0, "恢复正常提交"); - - printf("✓ 错误恢复测试完成\n"); -} - -// 打印测试结果 -void print_test_results() { - printf("\n=== 测试结果汇总 ===\n"); - printf("总测试数: %d\n", g_results.total_tests); - printf("通过测试: %d\n", g_results.passed_tests); - printf("失败测试: %d\n", g_results.failed_tests); - - if (g_results.total_tests > 0) { - g_results.success_rate = (double)g_results.passed_tests / g_results.total_tests * 100.0; - printf("通过率: %.2f%%\n", g_results.success_rate); - } - - if (g_results.failed_tests == 0) { - printf("\n🎉 所有Offset语义验证测试通过！系统一致性验证完成！\n"); - } else { - printf("\n❌ 有 %d 个测试失败，需要修复！\n", g_results.failed_tests); - } -} - -// 主函数 -int main() { - printf("开始真实的TDengine Offset语义验证测试...\n"); - printf("测试环境: TDengine %s\n", taos_get_client_info()); - - // 初始化测试状态 - init_test_state(); - - // 执行所有测试 - test_sync_commit(); - test_async_commit(); - test_at_least_once_semantics(); - test_at_most_once_semantics(); - test_checkpoint_recovery(); - test_idempotency(); - test_concurrent_commit(); - test_boundary_conditions(); - test_performance(); - test_error_recovery(); - - // 打印测试结果 - print_test_results(); - - // 清理资源 - cleanup_test_state(); - - return (g_results.failed_tests == 0) ? 0 : 1; -} diff --git a/plugins/incremental_bitmap/test/test_pitr_e2e.c b/plugins/incremental_bitmap/test/test_pitr_e2e.c deleted file mode 100644 index 7324b1575cb3..000000000000 --- a/plugins/incremental_bitmap/test/test_pitr_e2e.c +++ /dev/null @@ -1,990 +0,0 @@ -#include "pitr_e2e_test.h" -#include -#include -#include -#include -#include -#include -#include - -// 测试宏 -#define TEST_ASSERT(condition, message) \ - do { \ - if (!(condition)) { \ - fprintf(stderr, "Test failed: %s\n", message); \ - return -1; \ - } \ - } while(0) - -#define TEST_SUCCESS(message) \ - printf("✓ %s\n", message) - -// 测试配置 -static SPitrTestConfig test_config = { - .snapshot_interval_ms = 1000, // 1秒间隔（测试用） - .recovery_points = 5, // 5个恢复点 - .data_block_count = 1000, // 1000个数据块 - .concurrent_writers = 2, // 2个并发写入线程 - .test_duration_seconds = 60, // 1分钟测试 - .enable_disorder_test = true, // 启用乱序测试 - .enable_deletion_test = true, // 启用删除测试 - .test_data_path = "./pitr_test_data", - .snapshot_path = "./pitr_snapshots", - .recovery_path = "./pitr_recovery" -}; - -// 性能监控结构 -typedef struct { - int64_t start_time; - int64_t end_time; - size_t peak_memory; - size_t current_memory; - uint64_t operations_count; - double operations_per_second; -} SPerformanceMetrics; - -// 全局性能指标 -static SPerformanceMetrics g_performance_metrics = {0}; - -// 测试函数声明 -static int test_pitr_tester_creation(void); -static int test_snapshot_functionality(void); -static int test_recovery_functionality(void); -static int test_disorder_handling(void); -static int test_deletion_consistency(void); -static int test_boundary_conditions(void); -static int test_full_e2e_workflow(void); -static int test_performance_benchmarks(void); -static int test_error_handling(void); -static int test_memory_management(void); -static int test_concurrent_operations(void); -static int test_data_persistence(void); -static int test_failure_recovery(void); -static int test_stress_testing(void); -static int test_integration_scenarios(void); - -// 辅助函数声明 -static int64_t get_current_timestamp_ms(void); -static void start_performance_monitoring(void); -static void stop_performance_monitoring(void); -static void update_performance_metrics(uint64_t operations); -static void print_performance_summary(void); -static void generate_detailed_test_report(int total_tests, int passed_tests, int failed_tests); - -// 主测试函数 -int main() { - printf("==========================================\n"); - printf(" PITR E2E Test Suite\n"); - printf("==========================================\n\n"); - - // 启动性能监控 - start_performance_monitoring(); - - int total_tests = 0; - int passed_tests = 0; - int failed_tests = 0; - - // 测试列表 - 完整版本 - struct { - const char* name; - int (*test_func)(void); - } tests[] = { - {"PITR Tester Creation", test_pitr_tester_creation}, - {"Snapshot Functionality", test_snapshot_functionality}, - {"Recovery Functionality", test_recovery_functionality}, - {"Disorder Handling", test_disorder_handling}, - {"Deletion Consistency", test_deletion_consistency}, - {"Boundary Conditions", test_boundary_conditions}, - {"Full E2E Workflow", test_full_e2e_workflow}, - {"Performance Benchmarks", test_performance_benchmarks}, - {"Error Handling", test_error_handling}, - {"Memory Management", test_memory_management}, - {"Concurrent Operations", test_concurrent_operations}, - {"Data Persistence", test_data_persistence}, - {"Failure Recovery", test_failure_recovery}, - {"Stress Testing", test_stress_testing}, - {"Integration Scenarios", test_integration_scenarios} - }; - - // 运行所有测试 - for (size_t i = 0; i < sizeof(tests) / sizeof(tests[0]); i++) { - printf("Running test: %s\n", tests[i].name); - printf("------------------------------------------\n"); - - total_tests++; - int result = tests[i].test_func(); - - if (result == 0) { - passed_tests++; - printf("✓ Test passed: %s\n", tests[i].name); - } else { - failed_tests++; - printf("✗ Test failed: %s\n", tests[i].name); - } - - printf("\n"); - } - - // 测试结果汇总 - printf("==========================================\n"); - printf(" Test Results\n"); - printf("==========================================\n"); - printf("Total tests: %d\n", total_tests); - printf("Passed: %d\n", passed_tests); - printf("Failed: %d\n", failed_tests); - printf("Success rate: %.1f%%\n", (double)passed_tests / total_tests * 100); - - // 停止性能监控 - stop_performance_monitoring(); - - // 打印性能摘要 - print_performance_summary(); - - // 生成详细测试报告（包含真实统计） - generate_detailed_test_report(total_tests, passed_tests, failed_tests); - - if (failed_tests == 0) { - printf("\n🎉 All tests passed!\n"); - return 0; - } else { - printf("\n❌ %d test(s) failed\n", failed_tests); - return 1; - } -} - -// 测试PITR测试器创建 -static int test_pitr_tester_creation(void) { - printf("Testing PITR tester creation...\n"); - - // 测试1: 正常创建 - SPitrTester* tester = pitr_tester_create(&PITR_DEFAULT_CONFIG); - TEST_ASSERT(tester != NULL, "Failed to create PITR tester"); - TEST_SUCCESS("PITR tester created successfully"); - - // 测试2: 获取状态 - SPitrTestStatus status; - int result = pitr_tester_get_status(tester, &status); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Failed to get tester status"); - TEST_ASSERT(status.test_passed == true, "Initial test status should be true"); - TEST_SUCCESS("Status retrieval works correctly"); - - // 测试3: 重置测试器 - result = pitr_tester_reset(tester); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Failed to reset tester"); - TEST_SUCCESS("Tester reset works correctly"); - - // 测试4: 销毁测试器 - pitr_tester_destroy(tester); - TEST_SUCCESS("PITR tester destroyed successfully"); - - // 测试5: 无效配置处理 - printf("Testing invalid config handling...\n"); - SPitrTestConfig invalid_config = {0}; - SPitrTester* invalid_tester = pitr_tester_create(&invalid_config); - if (invalid_tester == NULL) { - TEST_SUCCESS("Invalid config correctly rejected"); - } else { - fprintf(stderr, "Test failed: Invalid config should have been rejected\n"); - pitr_tester_destroy(invalid_tester); - return -1; - } - - printf("test_pitr_tester_creation completed successfully\n"); - - return 0; -} - -// 测试快照功能 -static int test_snapshot_functionality(void) { - printf("Testing snapshot functionality...\n"); - - SPitrTester* tester = pitr_tester_create(&PITR_DEFAULT_CONFIG); - TEST_ASSERT(tester != NULL, "Failed to create PITR tester"); - - // 测试1: 运行快照测试 - int result = pitr_tester_run_snapshot_test(tester); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Snapshot test failed"); - TEST_SUCCESS("Snapshot test completed successfully"); - - // 测试2: 验证快照数量 - SPitrTestStatus status; - int status_result = pitr_tester_get_status(tester, &status); - TEST_ASSERT(status_result == PITR_TEST_SUCCESS, "Failed to get status"); - TEST_ASSERT(status.snapshots_created == PITR_DEFAULT_CONFIG.recovery_points, - "Snapshot count mismatch"); - TEST_SUCCESS("Snapshot count verification passed"); - - // 测试3: 获取快照列表 - SSnapshotInfo snapshots[10]; - uint32_t actual_count = 0; - result = pitr_tester_get_snapshots(tester, snapshots, 10, &actual_count); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Failed to get snapshots"); - TEST_ASSERT(actual_count == PITR_DEFAULT_CONFIG.recovery_points, "Snapshot count mismatch"); - TEST_SUCCESS("Snapshot list retrieval works correctly"); - - // 测试4: 验证快照完整性 - for (uint32_t i = 0; i < actual_count; i++) { - bool is_valid = pitr_verify_snapshot_integrity(&snapshots[i]); - TEST_ASSERT(is_valid, "Snapshot integrity check failed"); - } - TEST_SUCCESS("All snapshots passed integrity check"); - - // 测试5: 验证快照时间戳 - for (uint32_t i = 1; i < actual_count; i++) { - TEST_ASSERT(snapshots[i].timestamp > snapshots[i-1].timestamp, - "Snapshot timestamps should be monotonically increasing"); - } - TEST_SUCCESS("Snapshot timestamp ordering verified"); - - pitr_tester_destroy(tester); - return 0; -} - -// 测试恢复功能 -static int test_recovery_functionality(void) { - printf("Testing recovery functionality...\n"); - - SPitrTester* tester = pitr_tester_create(&PITR_DEFAULT_CONFIG); - TEST_ASSERT(tester != NULL, "Failed to create PITR tester"); - - // 先创建快照 - int result = pitr_tester_run_snapshot_test(tester); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Snapshot test failed"); - - // 测试1: 运行恢复测试 - result = pitr_tester_run_recovery_test(tester); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Recovery test failed"); - TEST_SUCCESS("Recovery test completed successfully"); - - // 测试2: 验证恢复点数量 - SPitrTestStatus status; - int status_result = pitr_tester_get_status(tester, &status); - TEST_ASSERT(status_result == PITR_TEST_SUCCESS, "Failed to get status"); - TEST_ASSERT(status.recovery_points_verified > 0, "No recovery points verified"); - TEST_SUCCESS("Recovery points verification passed"); - - // 测试3: 获取恢复点列表 - SRecoveryPoint recovery_points[10]; - uint32_t actual_count = 0; - result = pitr_tester_get_recovery_points(tester, recovery_points, 10, &actual_count); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Failed to get recovery points"); - TEST_ASSERT(actual_count > 0, "No recovery points found"); - TEST_SUCCESS("Recovery points list retrieval works correctly"); - - // 测试4: 验证恢复点一致性 - for (uint32_t i = 0; i < actual_count; i++) { - TEST_ASSERT(recovery_points[i].base_snapshot != NULL, "Base snapshot is NULL"); - TEST_ASSERT(recovery_points[i].recovery_timestamp > 0, "Invalid recovery timestamp"); - TEST_ASSERT(recovery_points[i].expected_block_count > 0, "Invalid expected block count"); - } - TEST_SUCCESS("Recovery points consistency verified"); - - // 测试5: 验证数据一致性 - SDataConsistencyResult consistency_result; - result = pitr_tester_verify_consistency(tester, get_current_timestamp_ms(), &consistency_result); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Data consistency verification failed"); - TEST_ASSERT(consistency_result.is_consistent, "Data consistency check failed"); - TEST_SUCCESS("Data consistency verification passed"); - - pitr_tester_destroy(tester); - return 0; -} - -// 测试乱序处理 -static int test_disorder_handling(void) { - printf("Testing disorder handling...\n"); - - SPitrTester* tester = pitr_tester_create(&PITR_DEFAULT_CONFIG); - TEST_ASSERT(tester != NULL, "Failed to create PITR tester"); - - // 测试1: 运行乱序测试 - int result = pitr_tester_run_disorder_test(tester); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Disorder test failed"); - TEST_SUCCESS("Disorder test completed successfully"); - - // 测试2: 验证乱序事件处理 - SPitrTestStatus status; - int status_result = pitr_tester_get_status(tester, &status); - TEST_ASSERT(status_result == PITR_TEST_SUCCESS, "Failed to get status"); - TEST_ASSERT(status.disorder_handled > 0, "No disorder events handled"); - TEST_SUCCESS("Disorder event handling verified"); - - // 测试3: 重复运行以验证计数递增（最小可验证实现） - SPitrTestStatus status_before; - TEST_ASSERT(pitr_tester_get_status(tester, &status_before) == PITR_TEST_SUCCESS, "Failed to get status before rerun"); - uint64_t handled_before = status_before.disorder_handled; - result = pitr_tester_run_disorder_test(tester); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Disorder test rerun failed"); - SPitrTestStatus status_after; - TEST_ASSERT(pitr_tester_get_status(tester, &status_after) == PITR_TEST_SUCCESS, "Failed to get status after rerun"); - TEST_ASSERT(status_after.disorder_handled >= handled_before, "Disorder handled counter did not increase or remain valid"); - TEST_SUCCESS("Disorder handling counter validated"); - - // 测试4: 验证乱序后的数据一致性 - SDataConsistencyResult consistency_result; - result = pitr_tester_verify_consistency(tester, get_current_timestamp_ms(), &consistency_result); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Post-disorder consistency check failed"); - TEST_SUCCESS("Post-disorder data consistency verified"); - - pitr_tester_destroy(tester); - return 0; -} - -// 测试删除一致性 -static int test_deletion_consistency(void) { - printf("Testing deletion consistency...\n"); - - SPitrTester* tester = pitr_tester_create(&PITR_DEFAULT_CONFIG); - TEST_ASSERT(tester != NULL, "Failed to create PITR tester"); - - // 测试1: 运行删除一致性测试 - int result = pitr_tester_run_deletion_consistency_test(tester); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Deletion consistency test failed"); - TEST_SUCCESS("Deletion consistency test completed successfully"); - - // 测试2: 验证删除操作处理 - SPitrTestStatus status; - int status_result = pitr_tester_get_status(tester, &status); - TEST_ASSERT(status_result == PITR_TEST_SUCCESS, "Failed to get status"); - TEST_ASSERT(status.deletion_handled > 0, "No deletion operations handled"); - TEST_SUCCESS("Deletion operation handling verified"); - - // 测试3: 重复运行以验证删除计数递增（最小可验证实现） - SPitrTestStatus del_status_before; - TEST_ASSERT(pitr_tester_get_status(tester, &del_status_before) == PITR_TEST_SUCCESS, "Failed to get status before deletion rerun"); - uint64_t deletion_before = del_status_before.deletion_handled; - result = pitr_tester_run_deletion_consistency_test(tester); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Deletion consistency rerun failed"); - SPitrTestStatus del_status_after; - TEST_ASSERT(pitr_tester_get_status(tester, &del_status_after) == PITR_TEST_SUCCESS, "Failed to get status after deletion rerun"); - TEST_ASSERT(del_status_after.deletion_handled >= deletion_before, "Deletion handled counter did not increase or remain valid"); - TEST_SUCCESS("Deletion handling counter validated"); - - // 测试4: 验证删除后的数据一致性 - SDataConsistencyResult consistency_result; - result = pitr_tester_verify_consistency(tester, get_current_timestamp_ms(), &consistency_result); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Post-deletion consistency check failed"); - TEST_SUCCESS("Post-deletion data consistency verified"); - - pitr_tester_destroy(tester); - return 0; -} - -// 测试边界条件 -static int test_boundary_conditions(void) { - printf("Testing boundary conditions...\n"); - - SPitrTester* tester = pitr_tester_create(&PITR_DEFAULT_CONFIG); - TEST_ASSERT(tester != NULL, "Failed to create PITR tester"); - - // 测试1: 运行边界条件测试 - int result = pitr_tester_run_boundary_test(tester); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Boundary test failed"); - TEST_SUCCESS("Boundary test completed successfully"); - - // 测试2: 边界块数量测试 - uint64_t boundary_counts[] = {0, 1, UINT64_MAX}; - for (size_t i = 0; i < sizeof(boundary_counts) / sizeof(boundary_counts[0]); i++) { - if (boundary_counts[i] == UINT64_MAX) { - // 跳过最大值的实际测试，因为内存可能不足 - printf("Skipping UINT64_MAX test due to memory constraints\n"); - continue; - } - - result = pitr_create_test_data(PITR_DEFAULT_CONFIG.test_data_path, boundary_counts[i], 1); - if (result == 0) { - TEST_SUCCESS("Boundary block count test passed"); - } else { - printf("Warning: Boundary block count %lu test failed (expected for some cases)\n", - boundary_counts[i]); - } - } - - // 测试3: 边界时间戳测试 - int64_t boundary_timestamps[] = {0, 1, INT64_MAX}; - for (size_t i = 0; i < sizeof(boundary_timestamps) / sizeof(boundary_timestamps[0]); i++) { - if (boundary_timestamps[i] == INT64_MAX) { - // 跳过最大值的实际测试 - printf("Skipping INT64_MAX timestamp test\n"); - continue; - } - - // 测试边界时间戳处理 - if (boundary_timestamps[i] == 0) { - // 零时间戳应该被正确处理 - TEST_SUCCESS("Zero timestamp handling verified"); - } - } - - // 测试4: 空配置测试 - SPitrTestConfig empty_config = {0}; - SPitrTester* empty_tester = pitr_tester_create(&empty_config); - if (empty_tester == NULL) { - TEST_SUCCESS("Empty config rejection works correctly"); - } else { - pitr_tester_destroy(empty_tester); - printf("Warning: Empty config was accepted (may be valid)\n"); - } - - pitr_tester_destroy(tester); - return 0; -} - -// 测试完整E2E工作流 -static int test_full_e2e_workflow(void) { - printf("Testing full E2E workflow...\n"); - - // 确保测试目录存在 - if (mkdir(PITR_DEFAULT_CONFIG.test_data_path, 0755) != 0 && errno != EEXIST) { - printf("Warning: Failed to create test data directory, continuing...\n"); - } - if (mkdir(PITR_DEFAULT_CONFIG.snapshot_path, 0755) != 0 && errno != EEXIST) { - printf("Warning: Failed to create snapshot directory, continuing...\n"); - } - if (mkdir(PITR_DEFAULT_CONFIG.recovery_path, 0755) != 0 && errno != EEXIST) { - printf("Warning: Failed to create recovery directory, continuing...\n"); - } - - SPitrTester* tester = pitr_tester_create(&PITR_DEFAULT_CONFIG); - TEST_ASSERT(tester != NULL, "Failed to create PITR tester"); - - // 测试1: 运行完整E2E测试 - int result = pitr_tester_run_full_test(tester); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Full E2E test failed"); - TEST_SUCCESS("Full E2E test completed successfully"); - - // 测试2: 验证测试状态 - SPitrTestStatus status; - pitr_tester_get_status(tester, &status); - TEST_ASSERT(status.test_passed == true, "Full test should pass"); - TEST_ASSERT(status.snapshots_created > 0, "Should create snapshots"); - TEST_ASSERT(status.recovery_points_verified > 0, "Should verify recovery points"); - TEST_ASSERT(status.data_consistency_checks > 0, "Should perform consistency checks"); - TEST_ASSERT(status.total_test_time_ms > 0, "Should record test time"); - TEST_SUCCESS("Full test status verification passed"); - - // 测试3: 生成测试报告 - const char* report_path = "/tmp/pitr_test_report.txt"; - result = pitr_tester_generate_report(tester, report_path); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Failed to generate test report"); - - // 验证报告文件存在 - FILE* report_file = fopen(report_path, "r"); - TEST_ASSERT(report_file != NULL, "Test report file not found"); - fclose(report_file); - TEST_SUCCESS("Test report generation works correctly"); - - // 测试4: 重置和重新运行 - result = pitr_tester_reset(tester); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Failed to reset tester"); - - // 重新运行一个简单的测试 - result = pitr_tester_run_snapshot_test(tester); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Re-run test failed after reset"); - TEST_SUCCESS("Tester reset and re-run works correctly"); - - pitr_tester_destroy(tester); - return 0; -} - -// 测试性能基准 -static int test_performance_benchmarks(void) { - printf("Testing performance benchmarks...\n"); - - // 测试1: 时间测量工具 - SPitrTimer* timer = pitr_timer_start("test_operation"); - TEST_ASSERT(timer != NULL, "Failed to create timer"); - - // 模拟一些工作 - usleep(10000); // 10ms - - pitr_timer_stop(timer); - double elapsed = pitr_timer_get_elapsed_ms(timer); - TEST_ASSERT(elapsed >= 9.0 && elapsed <= 15.0, "Timer accuracy check failed"); - TEST_SUCCESS("Timer accuracy verified"); - - pitr_timer_print_result(timer); - - // 测试2: 性能基准测试函数 - int test_data = 42; - int iterations = 1000; - - int result = pitr_run_performance_benchmark("test_benchmark", - NULL, &test_data, iterations); - // 注意：这个函数可能还没有完全实现，所以结果可能不准确 - TEST_SUCCESS("Performance benchmark function called"); - - // 测试3: 内存使用监控 - size_t peak_memory, current_memory; - result = pitr_monitor_memory_usage(&peak_memory, ¤t_memory); - // 注意：这个函数可能还没有完全实现 - TEST_SUCCESS("Memory monitoring function called"); - - return 0; -} - -// 测试错误处理 -static int test_error_handling(void) { - printf("Testing error handling...\n"); - - // 测试1: NULL指针处理 - SPitrTester* null_tester = NULL; - int result = pitr_tester_get_status(null_tester, NULL); - TEST_ASSERT(result == PITR_TEST_INVALID_CONFIG, "NULL pointer should return invalid config"); - TEST_SUCCESS("NULL pointer handling works correctly"); - - // 测试2: 无效配置处理 - SPitrTestConfig invalid_config = {0}; - invalid_config.data_block_count = 0; // 无效值 - SPitrTester* invalid_tester = pitr_tester_create(&invalid_config); - if (invalid_tester == NULL) { - TEST_SUCCESS("Invalid config rejection works correctly"); - } else { - pitr_tester_destroy(invalid_tester); - printf("Warning: Invalid config was accepted\n"); - } - - // 测试3: 错误状态处理 - SPitrTester* tester = pitr_tester_create(&PITR_DEFAULT_CONFIG); - if (tester) { - // 模拟错误状态 - // 注意：这里不能直接访问内部结构，应该通过API设置错误状态 - // 目前只是测试API调用，所以跳过这个测试 - printf("Warning: Direct status modification test skipped (not supported by current API)\n"); - - // 由于不能直接修改状态，这里只测试基本的API调用 - SPitrTestStatus status; - result = pitr_tester_get_status(tester, &status); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Failed to get status"); - TEST_SUCCESS("Status retrieval works correctly"); - - pitr_tester_destroy(tester); - } - - return 0; -} - -// 测试内存管理 -static int test_memory_management(void) { - printf("Testing memory management...\n"); - - // 测试1: 内存分配和释放 - SPitrTester* tester = pitr_tester_create(&PITR_DEFAULT_CONFIG); - TEST_ASSERT(tester != NULL, "Failed to create PITR tester"); - - // 多次创建和销毁，检查内存泄漏 - for (int i = 0; i < 5; i++) { - SPitrTester* temp_tester = pitr_tester_create(&PITR_DEFAULT_CONFIG); - TEST_ASSERT(temp_tester != NULL, "Failed to create temporary tester"); - - // 运行一个简单测试 - int result = pitr_tester_run_snapshot_test(temp_tester); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Temporary tester test failed"); - - pitr_tester_destroy(temp_tester); - TEST_SUCCESS("Temporary tester creation/destruction cycle completed"); - } - - // 测试2: 大内存分配测试 - SPitrTestConfig large_config = PITR_DEFAULT_CONFIG; - large_config.data_block_count = 100000; // 10万个块 - - SPitrTester* large_tester = pitr_tester_create(&large_config); - if (large_tester) { - TEST_SUCCESS("Large memory allocation succeeded"); - pitr_tester_destroy(large_tester); - } else { - printf("Warning: Large memory allocation failed (may be expected on low-memory systems)\n"); - } - - // 测试3: 内存压力测试 - SPitrTester* testers[10]; - int created_count = 0; - - for (int i = 0; i < 10; i++) { - testers[i] = pitr_tester_create(&PITR_DEFAULT_CONFIG); - if (testers[i]) { - created_count++; - } else { - break; - } - } - - printf("Created %d testers under memory pressure\n", created_count); - TEST_SUCCESS("Memory pressure test completed"); - - // 清理 - for (int i = 0; i < created_count; i++) { - pitr_tester_destroy(testers[i]); - } - - pitr_tester_destroy(tester); - return 0; -} - -// 并发操作测试 -static int test_concurrent_operations(void) { - printf("Testing concurrent operations...\n"); - - SPitrTester* tester = pitr_tester_create(&PITR_DEFAULT_CONFIG); - TEST_ASSERT(tester != NULL, "Failed to create PITR tester"); - - // 测试1: 并发快照创建 - printf("Testing concurrent snapshot creation...\n"); - int result = pitr_tester_run_snapshot_test(tester); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Concurrent snapshot test failed"); - TEST_SUCCESS("Concurrent snapshot creation works correctly"); - - // 测试2: 并发恢复操作 - printf("Testing concurrent recovery operations...\n"); - result = pitr_tester_run_recovery_test(tester); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Concurrent recovery test failed"); - TEST_SUCCESS("Concurrent recovery operations work correctly"); - - // 测试3: 并发数据一致性检查 - printf("Testing concurrent consistency checks...\n"); - SDataConsistencyResult consistency_result; - result = pitr_tester_verify_consistency(tester, get_current_timestamp_ms(), &consistency_result); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Concurrent consistency check failed"); - TEST_SUCCESS("Concurrent consistency checks work correctly"); - - pitr_tester_destroy(tester); - return 0; -} - -// 数据持久化测试 -static int test_data_persistence(void) { - printf("Testing data persistence...\n"); - - // 确保测试目录存在 - if (mkdir(PITR_DEFAULT_CONFIG.test_data_path, 0755) != 0 && errno != EEXIST) { - printf("Warning: Failed to create test data directory, continuing...\n"); - } - - SPitrTester* tester = pitr_tester_create(&PITR_DEFAULT_CONFIG); - TEST_ASSERT(tester != NULL, "Failed to create PITR tester"); - - // 测试1: 创建测试数据 - printf("Creating test data for persistence test...\n"); - int result = pitr_create_test_data(PITR_DEFAULT_CONFIG.test_data_path, 100, 1); - TEST_ASSERT(result == 0, "Failed to create test data"); - TEST_SUCCESS("Test data created successfully"); - - // 测试2: 创建快照 - printf("Creating snapshots for persistence test...\n"); - result = pitr_tester_run_snapshot_test(tester); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Snapshot creation failed"); - TEST_SUCCESS("Snapshots created successfully"); - - // 测试3: 验证快照持久化 - printf("Verifying snapshot persistence...\n"); - SSnapshotInfo snapshots[10]; - uint32_t actual_count = 0; - result = pitr_tester_get_snapshots(tester, snapshots, 10, &actual_count); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Failed to get snapshots"); - TEST_ASSERT(actual_count > 0, "No snapshots found"); - - // 验证快照文件存在 - for (uint32_t i = 0; i < actual_count; i++) { - bool is_valid = pitr_verify_snapshot_integrity(&snapshots[i]); - TEST_ASSERT(is_valid, "Snapshot persistence verification failed"); - } - TEST_SUCCESS("All snapshots persisted correctly"); - - // 测试4: 验证恢复点持久化 - printf("Verifying recovery point persistence...\n"); - SRecoveryPoint recovery_points[10]; - uint32_t recovery_count = 0; - result = pitr_tester_get_recovery_points(tester, recovery_points, 10, &recovery_count); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Failed to get recovery points"); - TEST_SUCCESS("Recovery points persisted correctly"); - - pitr_tester_destroy(tester); - return 0; -} - -// 故障恢复测试 -static int test_failure_recovery(void) { - printf("Testing failure recovery...\n"); - - SPitrTester* tester = pitr_tester_create(&PITR_DEFAULT_CONFIG); - TEST_ASSERT(tester != NULL, "Failed to create PITR tester"); - - // 测试1: 模拟快照创建失败 - printf("Testing snapshot creation failure recovery...\n"); - // 这里可以模拟磁盘空间不足等场景 - int result = pitr_tester_run_snapshot_test(tester); - if (result == PITR_TEST_SUCCESS) { - TEST_SUCCESS("Snapshot creation succeeded (no failure simulated)"); - } else { - printf("Warning: Snapshot creation failed as expected\n"); - } - - // 测试2: 模拟恢复操作失败 - printf("Testing recovery operation failure recovery...\n"); - result = pitr_tester_run_recovery_test(tester); - if (result == PITR_TEST_SUCCESS) { - TEST_SUCCESS("Recovery operation succeeded (no failure simulated)"); - } else { - printf("Warning: Recovery operation failed as expected\n"); - } - - // 测试3: 模拟数据一致性检查失败 - printf("Testing consistency check failure recovery...\n"); - SDataConsistencyResult consistency_result; - result = pitr_tester_verify_consistency(tester, get_current_timestamp_ms(), &consistency_result); - if (result == PITR_TEST_SUCCESS) { - TEST_SUCCESS("Consistency check succeeded (no failure simulated)"); - } else { - printf("Warning: Consistency check failed as expected\n"); - } - - pitr_tester_destroy(tester); - return 0; -} - -// 压力测试 -static int test_stress_testing(void) { - printf("Testing stress scenarios...\n"); - - // 测试1: 高并发压力测试 - printf("Testing high concurrency stress...\n"); - SPitrTestConfig stress_config = PITR_DEFAULT_CONFIG; - stress_config.concurrent_writers = 10; // 增加并发写入线程 - stress_config.data_block_count = 10000; // 增加数据块数量 - - SPitrTester* stress_tester = pitr_tester_create(&stress_config); - if (stress_tester) { - int result = pitr_tester_run_snapshot_test(stress_tester); - if (result == PITR_TEST_SUCCESS) { - TEST_SUCCESS("High concurrency stress test passed"); - } else { - printf("Warning: High concurrency stress test failed\n"); - } - pitr_tester_destroy(stress_tester); - } else { - printf("Warning: Failed to create stress tester (may be expected on low-resource systems)\n"); - } - - // 测试2: 大数据量压力测试 - printf("Testing large data volume stress...\n"); - SPitrTestConfig large_config = PITR_DEFAULT_CONFIG; - large_config.data_block_count = 100000; // 10万个数据块 - - SPitrTester* large_tester = pitr_tester_create(&large_config); - if (large_tester) { - int result = pitr_tester_run_snapshot_test(large_tester); - if (result == PITR_TEST_SUCCESS) { - TEST_SUCCESS("Large data volume stress test passed"); - } else { - printf("Warning: Large data volume stress test failed\n"); - } - pitr_tester_destroy(large_tester); - } else { - printf("Warning: Failed to create large data tester (may be expected on low-resource systems)\n"); - } - - // 测试3: 长时间运行压力测试 - printf("Testing long-running stress...\n"); - SPitrTestConfig long_config = PITR_DEFAULT_CONFIG; - long_config.test_duration_seconds = 300; // 5分钟测试 - - SPitrTester* long_tester = pitr_tester_create(&long_config); - if (long_tester) { - // 运行一个较短的测试来验证长时间运行能力 - int result = pitr_tester_run_snapshot_test(long_tester); - if (result == PITR_TEST_SUCCESS) { - TEST_SUCCESS("Long-running stress test preparation passed"); - } else { - printf("Warning: Long-running stress test preparation failed\n"); - } - pitr_tester_destroy(long_tester); - } else { - printf("Warning: Failed to create long-running tester\n"); - } - - return 0; -} - -// 集成场景测试 -static int test_integration_scenarios(void) { - printf("Testing integration scenarios...\n"); - - // 确保测试目录存在 - if (mkdir(PITR_DEFAULT_CONFIG.test_data_path, 0755) != 0 && errno != EEXIST) { - printf("Warning: Failed to create test data directory, continuing...\n"); - } - if (mkdir(PITR_DEFAULT_CONFIG.snapshot_path, 0755) != 0 && errno != EEXIST) { - printf("Warning: Failed to create snapshot directory, continuing...\n"); - } - if (mkdir(PITR_DEFAULT_CONFIG.recovery_path, 0755) != 0 && errno != EEXIST) { - printf("Warning: Failed to create recovery directory, continuing...\n"); - } - - // 测试1: 完整工作流集成测试 - printf("Testing complete workflow integration...\n"); - SPitrTester* tester = pitr_tester_create(&PITR_DEFAULT_CONFIG); - TEST_ASSERT(tester != NULL, "Failed to create PITR tester"); - - // 运行完整的集成测试 - int result = pitr_tester_run_full_test(tester); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Complete workflow integration test failed"); - TEST_SUCCESS("Complete workflow integration test passed"); - - // 测试2: 多阶段集成测试 - printf("Testing multi-stage integration...\n"); - - // 重置测试器状态，为多阶段测试做准备 - result = pitr_tester_reset(tester); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Failed to reset PITR tester for multi-stage test"); - - // 阶段1: 快照创建 - result = pitr_tester_run_snapshot_test(tester); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Stage 1: Snapshot creation failed"); - - // 阶段2: 恢复验证 - result = pitr_tester_run_recovery_test(tester); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Stage 2: Recovery verification failed"); - TEST_SUCCESS("Stage 2: Recovery verification passed"); - - // 阶段3: 数据一致性检查 - SDataConsistencyResult consistency_result; - result = pitr_tester_verify_consistency(tester, get_current_timestamp_ms(), &consistency_result); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Stage 3: Data consistency check failed"); - TEST_SUCCESS("Stage 3: Data consistency check passed"); - - TEST_SUCCESS("Multi-stage integration test completed"); - - // 测试3: 报告生成集成测试 - printf("Testing report generation integration...\n"); - const char* report_path = "/tmp/pitr_integration_report.txt"; - result = pitr_tester_generate_report(tester, report_path); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Report generation integration failed"); - - // 验证报告文件 - FILE* report_file = fopen(report_path, "r"); - TEST_ASSERT(report_file != NULL, "Integration report file not found"); - fclose(report_file); - TEST_SUCCESS("Report generation integration test passed"); - - pitr_tester_destroy(tester); - return 0; -} - -// 性能监控函数实现 - -// 启动性能监控 -static void start_performance_monitoring(void) { - g_performance_metrics.start_time = get_current_timestamp_ms(); - g_performance_metrics.peak_memory = 0; - g_performance_metrics.current_memory = 0; - g_performance_metrics.operations_count = 0; - g_performance_metrics.operations_per_second = 0.0; - - printf("Performance monitoring started at %ld ms\n", g_performance_metrics.start_time); -} - -// 停止性能监控 -static void stop_performance_monitoring(void) { - g_performance_metrics.end_time = get_current_timestamp_ms(); - - // 计算操作每秒 - int64_t total_time_ms = g_performance_metrics.end_time - g_performance_metrics.start_time; - if (total_time_ms > 0) { - g_performance_metrics.operations_per_second = - (double)g_performance_metrics.operations_count / (total_time_ms / 1000.0); - } - - printf("Performance monitoring stopped at %ld ms\n", g_performance_metrics.end_time); -} - -// 更新性能指标 -static void update_performance_metrics(uint64_t operations) { - g_performance_metrics.operations_count += operations; - - // 这里可以添加内存使用监控 - // 注意：实际的内存监控需要系统特定的实现 - printf("Performance metrics updated: %lu operations\n", operations); -} - -// 打印性能摘要 -static void print_performance_summary(void) { - printf("\n==========================================\n"); - printf(" Performance Summary\n"); - printf("==========================================\n"); - - int64_t total_time_ms = g_performance_metrics.end_time - g_performance_metrics.start_time; - printf("Total test time: %ld ms (%.2f seconds)\n", - total_time_ms, total_time_ms / 1000.0); - - printf("Total operations: %lu\n", g_performance_metrics.operations_count); - printf("Operations per second: %.2f\n", g_performance_metrics.operations_per_second); - - if (g_performance_metrics.peak_memory > 0) { - printf("Peak memory usage: %zu bytes (%.2f MB)\n", - g_performance_metrics.peak_memory, - g_performance_metrics.peak_memory / (1024.0 * 1024.0)); - } - - if (g_performance_metrics.current_memory > 0) { - printf("Current memory usage: %zu bytes (%.2f MB)\n", - g_performance_metrics.current_memory, - g_performance_metrics.current_memory / (1024.0 * 1024.0)); - } -} - -// 生成详细测试报告 -static void generate_detailed_test_report(int total_tests, int passed_tests, int failed_tests) { - const char* report_path = "/tmp/pitr_detailed_report.txt"; - FILE* report_file = fopen(report_path, "w"); - if (!report_file) { - printf("Warning: Failed to create detailed test report\n"); - return; - } - - fprintf(report_file, "PITR E2E Detailed Test Report\n"); - fprintf(report_file, "================================\n\n"); - - // 测试配置信息 - fprintf(report_file, "Test Configuration:\n"); - fprintf(report_file, "- Snapshot Interval: %u ms\n", PITR_DEFAULT_CONFIG.snapshot_interval_ms); - fprintf(report_file, "- Recovery Points: %u\n", PITR_DEFAULT_CONFIG.recovery_points); - fprintf(report_file, "- Data Block Count: %lu\n", PITR_DEFAULT_CONFIG.data_block_count); - fprintf(report_file, "- Concurrent Writers: %u\n", PITR_DEFAULT_CONFIG.concurrent_writers); - fprintf(report_file, "- Test Duration: %u seconds\n", PITR_DEFAULT_CONFIG.test_duration_seconds); - fprintf(report_file, "- Disorder Test: %s\n", PITR_DEFAULT_CONFIG.enable_disorder_test ? "Enabled" : "Disabled"); - fprintf(report_file, "- Deletion Test: %s\n", PITR_DEFAULT_CONFIG.enable_deletion_test ? "Enabled" : "Disabled"); - - // 性能指标 - fprintf(report_file, "\nPerformance Metrics:\n"); - int64_t total_time_ms = g_performance_metrics.end_time - g_performance_metrics.start_time; - fprintf(report_file, "- Total Test Time: %ld ms (%.2f seconds)\n", - total_time_ms, total_time_ms / 1000.0); - fprintf(report_file, "- Total Operations: %lu\n", g_performance_metrics.operations_count); - fprintf(report_file, "- Operations per Second: %.2f\n", g_performance_metrics.operations_per_second); - - // 测试结果 - fprintf(report_file, "\nTest Results:\n"); - double success_rate = (total_tests > 0) ? (100.0 * (double)passed_tests / (double)total_tests) : 0.0; - fprintf(report_file, "- Total: %d\n", total_tests); - fprintf(report_file, "- Passed: %d\n", passed_tests); - fprintf(report_file, "- Failed: %d\n", failed_tests); - fprintf(report_file, "- Success Rate: %.1f%%\n", success_rate); - fprintf(report_file, "- Performance monitoring active throughout testing\n"); - - // 建议和改进 - fprintf(report_file, "\nRecommendations:\n"); - fprintf(report_file, "- Consider running stress tests in production-like environments\n"); - fprintf(report_file, "- Monitor memory usage during long-running operations\n"); - fprintf(report_file, "- Validate performance metrics against production requirements\n"); - - fclose(report_file); - printf("Detailed test report generated: %s\n", report_path); -} - -// 辅助函数 -static int64_t get_current_timestamp_ms(void) { - struct timespec ts; - clock_gettime(CLOCK_MONOTONIC, &ts); - return (int64_t)ts.tv_sec * 1000 + (int64_t)ts.tv_nsec / 1000000; -} diff --git a/plugins/incremental_bitmap/test/test_pitr_e2e_debug.c b/plugins/incremental_bitmap/test/test_pitr_e2e_debug.c deleted file mode 100644 index 2af4e75a7499..000000000000 --- a/plugins/incremental_bitmap/test/test_pitr_e2e_debug.c +++ /dev/null @@ -1,81 +0,0 @@ -#include -#include -#include -#include -#include -#include - -// 包含必要的头文件 -#include "pitr_e2e_test.h" - -// 简化的测试函数 -static int test_pitr_tester_creation_debug(void) { - printf("Testing PITR tester creation...\n"); - - // 测试1: 正常创建 - printf("Creating tester with PITR_DEFAULT_CONFIG...\n"); - SPitrTester* tester = pitr_tester_create(&PITR_DEFAULT_CONFIG); - if (tester == NULL) { - printf("❌ Failed to create PITR tester\n"); - return -1; - } - printf("✓ PITR tester created successfully\n"); - - // 测试2: 获取状态 - printf("Getting tester status...\n"); - SPitrTestStatus status; - int result = pitr_tester_get_status(tester, &status); - if (result != PITR_TEST_SUCCESS) { - printf("❌ Failed to get tester status\n"); - pitr_tester_destroy(tester); - return -1; - } - printf("✓ Status retrieval works correctly\n"); - - // 测试3: 重置测试器 - printf("Resetting tester...\n"); - result = pitr_tester_reset(tester); - if (result != PITR_TEST_SUCCESS) { - printf("❌ Failed to reset tester\n"); - pitr_tester_destroy(tester); - return -1; - } - printf("✓ Tester reset works correctly\n"); - - // 测试4: 销毁测试器 - printf("Destroying tester...\n"); - pitr_tester_destroy(tester); - printf("✓ PITR tester destroyed successfully\n"); - - // 测试5: 无效配置处理 - printf("Testing invalid config handling...\n"); - SPitrTestConfig invalid_config = {0}; - SPitrTester* invalid_tester = pitr_tester_create(&invalid_config); - if (invalid_tester == NULL) { - printf("✓ Invalid config correctly rejected\n"); - } else { - printf("❌ Invalid config should have been rejected\n"); - pitr_tester_destroy(invalid_tester); - return -1; - } - - printf("test_pitr_tester_creation_debug completed successfully\n"); - return 0; -} - -// 主函数 -int main() { - printf("==========================================\n"); - printf(" PITR E2E Debug Test\n"); - printf("==========================================\n\n"); - - int result = test_pitr_tester_creation_debug(); - - if (result == 0) { - printf("\n🎉 Debug test passed!\n"); - } else { - printf("\n❌ Debug test failed!\n"); - } - - return result; -} diff --git a/plugins/incremental_bitmap/test/test_pitr_e2e_simple.c b/plugins/incremental_bitmap/test/test_pitr_e2e_simple.c deleted file mode 100644 index 60b05d195dc4..000000000000 --- a/plugins/incremental_bitmap/test/test_pitr_e2e_simple.c +++ /dev/null @@ -1,171 +0,0 @@ -#include "pitr_e2e_test.h" -#include -#include -#include -#include -#include - -// 测试宏 -#define TEST_ASSERT(condition, message) \ - do { \ - if (!(condition)) { \ - fprintf(stderr, "Test failed: %s\n", message); \ - return -1; \ - } \ - } while(0) - -#define TEST_SUCCESS(message) \ - printf("✓ %s\n", message) - -// 简化的测试配置 -static SPitrTestConfig simple_test_config = { - .snapshot_interval_ms = 1000, // 1秒间隔 - .recovery_points = 3, // 3个恢复点 - .data_block_count = 100, // 100个数据块 - .concurrent_writers = 1, // 1个并发写入线程 - .test_duration_seconds = 30, // 30秒测试 - .enable_disorder_test = false, // 禁用复杂测试 - .enable_deletion_test = false, // 禁用复杂测试 - .test_data_path = "/tmp/pitr_simple_test", - .snapshot_path = "/tmp/pitr_simple_snapshots", - .recovery_path = "/tmp/pitr_simple_recovery" -}; - -// 简化的测试函数 -static int test_basic_pitr_functionality(void); -static int test_config_validation(void); -static int test_memory_management(void); - -// 主测试函数 -int main() { - printf("==========================================\n"); - printf(" PITR E2E Simple Test Suite\n"); - printf("==========================================\n\n"); - - int total_tests = 0; - int passed_tests = 0; - int failed_tests = 0; - - // 测试列表 - struct { - const char* name; - int (*test_func)(void); - } tests[] = { - {"Basic PITR Functionality", test_basic_pitr_functionality}, - {"Config Validation", test_config_validation}, - {"Memory Management", test_memory_management} - }; - - // 运行所有测试 - for (size_t i = 0; i < sizeof(tests) / sizeof(tests[0]); i++) { - printf("Running test: %s\n", tests[i].name); - printf("------------------------------------------\n"); - - total_tests++; - int result = tests[i].test_func(); - - if (result == 0) { - passed_tests++; - printf("✓ Test passed: %s\n", tests[i].name); - } else { - failed_tests++; - printf("✗ Test failed: %s\n", tests[i].name); - } - - printf("\n"); - } - - // 测试结果汇总 - printf("==========================================\n"); - printf(" Test Results\n"); - printf("==========================================\n"); - printf("Total tests: %d\n", total_tests); - printf("Passed: %d\n", passed_tests); - printf("Failed: %d\n", failed_tests); - printf("Success rate: %.1f%%\n", (double)passed_tests / total_tests * 100); - - if (failed_tests == 0) { - printf("\n🎉 All tests passed!\n"); - return 0; - } else { - printf("\n❌ %d test(s) failed\n", failed_tests); - return 1; - } -} - -// 测试基本PITR功能 -static int test_basic_pitr_functionality(void) { - printf("Testing basic PITR functionality...\n"); - - // 测试1: 创建测试器 - SPitrTester* tester = pitr_tester_create(&simple_test_config); - if (tester == NULL) { - printf("Warning: PITR tester creation failed (may be expected in mock mode)\n"); - return 0; // 在mock模式下，这可能是预期的 - } - TEST_SUCCESS("PITR tester created successfully"); - - // 测试2: 获取状态 - SPitrTestStatus status; - int result = pitr_tester_get_status(tester, &status); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Failed to get tester status"); - TEST_SUCCESS("Status retrieval works correctly"); - - // 测试3: 重置测试器 - result = pitr_tester_reset(tester); - TEST_ASSERT(result == PITR_TEST_SUCCESS, "Failed to reset tester"); - TEST_SUCCESS("Tester reset works correctly"); - - // 测试4: 销毁测试器 - pitr_tester_destroy(tester); - TEST_SUCCESS("PITR tester destroyed successfully"); - - return 0; -} - -// 测试配置验证 -static int test_config_validation(void) { - printf("Testing configuration validation...\n"); - - // 测试1: 空配置 - SPitrTestConfig empty_config = {0}; - SPitrTester* empty_tester = pitr_tester_create(&empty_config); - if (empty_tester == NULL) { - TEST_SUCCESS("Empty config rejection works correctly"); - } else { - printf("Warning: Empty config was accepted\n"); - pitr_tester_destroy(empty_tester); - } - - // 测试2: 无效配置 - SPitrTestConfig invalid_config = simple_test_config; - invalid_config.data_block_count = 0; // 无效值 - SPitrTester* invalid_tester = pitr_tester_create(&invalid_config); - if (invalid_tester == NULL) { - TEST_SUCCESS("Invalid config rejection works correctly"); - } else { - printf("Warning: Invalid config was accepted\n"); - pitr_tester_destroy(invalid_tester); - } - - return 0; -} - -// 测试内存管理 -static int test_memory_management(void) { - printf("Testing memory management...\n"); - - // 测试1: 多次创建和销毁 - for (int i = 0; i < 3; i++) { - SPitrTester* temp_tester = pitr_tester_create(&simple_test_config); - if (temp_tester) { - TEST_SUCCESS("Temporary tester creation succeeded"); - pitr_tester_destroy(temp_tester); - TEST_SUCCESS("Temporary tester destruction succeeded"); - } else { - printf("Warning: Temporary tester creation failed (may be expected in mock mode)\n"); - } - } - - return 0; -} diff --git a/plugins/incremental_bitmap/test/test_pitr_pr_review.c b/plugins/incremental_bitmap/test/test_pitr_pr_review.c deleted file mode 100644 index e659b3cbf89a..000000000000 --- a/plugins/incremental_bitmap/test/test_pitr_pr_review.c +++ /dev/null @@ -1,142 +0,0 @@ -#include "pitr_e2e_test.h" -#include -#include -#include - -// PR Review专用的PITR测试程序 -// 使用轻量级配置，确保数据量在500MB以内 - -int main() { - printf("🚀 开始PITR PR Review测试...\n"); - printf("📋 测试目标: 验证PITR功能在轻量级配置下的正确性\n"); - printf("💾 数据量限制: 500MB以内\n"); - printf("⏱️ 测试时长: 30秒以内\n\n"); - - // 使用PR Review专用配置 - SPitrTestConfig config = PITR_PR_REVIEW_CONFIG; - - // 🔒 强制数据量检查 - printf("🔍 开始配置验证...\n"); - if (pitr_validate_config_for_pr_review(&config) != 0) { - fprintf(stderr, "❌ 配置验证失败！测试被阻止运行。\n"); - return -1; - } - - // 创建PITR测试器 - printf("🔧 创建PITR测试器...\n"); - SPitrTester* tester = pitr_tester_create(&config); - if (!tester) { - fprintf(stderr, "❌ 创建PITR测试器失败！\n"); - return -1; - } - - printf("✅ PITR测试器创建成功\n"); - - // 运行基础功能测试 - printf("\n🧪 运行基础功能测试...\n"); - - // 测试1: 快照创建 - printf(" 测试1: 快照创建...\n"); - int64_t timestamp = get_current_timestamp_ms(); - if (create_snapshot(tester, timestamp) != 0) { - fprintf(stderr, "❌ 快照创建失败！\n"); - pitr_tester_destroy(tester); - return -1; - } - printf(" ✅ 快照创建成功\n"); - - // 测试2: 快照一致性验证 - printf(" 测试2: 快照一致性验证...\n"); - if (verify_snapshot_consistency(tester, &tester->snapshots[0]) != 0) { - fprintf(stderr, "❌ 快照一致性验证失败！\n"); - pitr_tester_destroy(tester); - return -1; - } - printf(" ✅ 快照一致性验证成功\n"); - - // 测试3: 乱序数据处理 - if (config.enable_disorder_test) { - printf(" 测试3: 乱序数据处理...\n"); - if (process_disorder_events(tester, 0.1) != 0) { // 10%乱序 - fprintf(stderr, "❌ 乱序数据处理失败！\n"); - pitr_tester_destroy(tester); - return -1; - } - printf(" ✅ 乱序数据处理成功\n"); - } - - // 测试4: 删除事件处理 - if (config.enable_deletion_test) { - printf(" 测试4: 删除事件处理...\n"); - if (process_deletion_events(tester, 50) != 0) { // 50个删除事件 - fprintf(stderr, "❌ 删除事件处理失败！\n"); - pitr_tester_destroy(tester); - return -1; - } - printf(" ✅ 删除事件处理成功\n"); - } - - // 测试5: 时间点恢复 - printf(" 测试5: 时间点恢复...\n"); - if (pitr_tester_run_time_point_recovery(tester, timestamp) != 0) { - fprintf(stderr, "❌ 时间点恢复失败！\n"); - pitr_tester_destroy(tester); - return -1; - } - printf(" ✅ 时间点恢复成功\n"); - - // 测试6: 数据一致性验证 - printf(" 测试6: 数据一致性验证...\n"); - if (pitr_tester_verify_data_consistency(tester) != 0) { - fprintf(stderr, "❌ 数据一致性验证失败！\n"); - pitr_tester_destroy(tester); - return -1; - } - printf(" ✅ 数据一致性验证成功\n"); - - // 测试7: 边界条件测试 - printf(" 测试7: 边界条件测试...\n"); - if (pitr_tester_run_boundary_tests(tester) != 0) { - fprintf(stderr, "❌ 边界条件测试失败！\n"); - pitr_tester_destroy(tester); - return -1; - } - printf(" ✅ 边界条件测试成功\n"); - - // 测试8: 性能基准测试 - printf(" 测试8: 性能基准测试...\n"); - if (pitr_run_performance_benchmark("PITR_PR_Review", - (void (*)(void*))pitr_tester_run_performance_test, - tester, 3) != 0) { - fprintf(stderr, "❌ 性能基准测试失败！\n"); - pitr_tester_destroy(tester); - return -1; - } - printf(" ✅ 性能基准测试成功\n"); - - // 获取测试状态 - SPitrTestStatus status; - if (pitr_tester_get_status(tester, &status) == 0) { - printf("\n📊 测试状态总结:\n"); - printf(" 测试通过: %s\n", status.test_passed ? "✅ 是" : "❌ 否"); - printf(" 总快照数: %u\n", status.total_snapshots); - printf(" 总恢复点数: %u\n", status.total_recovery_points); - printf(" 总数据块数: %lu\n", status.total_blocks_processed); - printf(" 总事件数: %lu\n", status.total_events_processed); - printf(" 总删除数: %lu\n", status.total_deletions_processed); - printf(" 测试耗时: %.2f 秒\n", status.test_duration_seconds); - } - - // 清理资源 - printf("\n🧹 清理测试资源...\n"); - pitr_tester_destroy(tester); - - printf("\n🎉 PITR PR Review测试完成！\n"); - printf("✅ 所有测试通过\n"); - printf("💾 数据量控制在500MB以内\n"); - printf("⏱️ 测试时长控制在30秒以内\n"); - printf("🚀 可以安全提交PR！\n"); - - return 0; -} - diff --git a/plugins/incremental_bitmap/test/test_retry_mechanism.c b/plugins/incremental_bitmap/test/test_retry_mechanism.c deleted file mode 100644 index 4fefc08c74e8..000000000000 --- a/plugins/incremental_bitmap/test/test_retry_mechanism.c +++ /dev/null @@ -1,244 +0,0 @@ -/* - * Copyright (c) 2024 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include -#include -#include -#include -#include -#include "../include/backup_coordinator.h" - -// 模拟失败的操作 -static int32_t mock_failing_operation(void* user_data) { - int* attempt_count = (int*)user_data; - (*attempt_count)++; - - // 前3次失败，第4次成功 - if (*attempt_count < 4) { - return BACKUP_ERROR_NETWORK; - } - - return BACKUP_SUCCESS; -} - -// 模拟不可重试的操作 -static int32_t mock_non_retryable_operation(void* user_data) { - return BACKUP_ERROR_INVALID_PARAM; -} - -// 测试重试机制 -void test_retry_mechanism() { - printf("Testing retry mechanism...\n"); - - // 初始化重试上下文 - SRetryContext retry_ctx; - int32_t result = backup_retry_context_init(&retry_ctx, 5, 1); - assert(result == BACKUP_SUCCESS); - - // 测试成功的重试 - int attempt_count = 0; - result = backup_execute_with_retry(&retry_ctx, mock_failing_operation, &attempt_count); - assert(result == BACKUP_SUCCESS); - assert(attempt_count == 4); - assert(retry_ctx.state == RETRY_STATE_SUCCESS); - - printf("✓ Retry mechanism test passed\n"); - - // 测试不可重试的错误 - result = backup_execute_with_retry(&retry_ctx, mock_non_retryable_operation, NULL); - assert(result == BACKUP_ERROR_INVALID_PARAM); - assert(retry_ctx.state == RETRY_STATE_FAILED); - - printf("✓ Non-retryable error test passed\n"); - - // 清理 - backup_retry_context_destroy(&retry_ctx); -} - -// 测试错误记录 -void test_error_recording() { - printf("Testing error recording...\n"); - - // 创建配置 - SBackupCoordinatorConfig config = { - .max_blocks_per_batch = 1000, - .batch_timeout_ms = 5000, - .enable_compression = true, - .enable_encryption = false, - .encryption_key = NULL, - .error_retry_max = 3, - .error_retry_interval = 1, - .error_store_path = "/tmp", - .enable_error_logging = true, - .error_buffer_size = 1000, - .backup_path = NULL, - .backup_max_size = 1024 * 1024 * 1024, - .compression_level = 1 - }; - - // 创建协同器（简化版本，不依赖完整的引擎） - SBackupCoordinator* coordinator = (SBackupCoordinator*)malloc(sizeof(SBackupCoordinator)); - assert(coordinator != NULL); - - // 手动初始化协同器 - coordinator->config = config; - coordinator->bitmap_engine = NULL; - coordinator->event_interceptor = NULL; - coordinator->active_cursor = NULL; - coordinator->last_error_message = NULL; - coordinator->error_count = 0; - coordinator->retry_count = 0; - coordinator->total_backup_blocks = 0; - coordinator->total_backup_size = 0; - coordinator->backup_duration_ms = 0; - - // 初始化重试上下文 - backup_retry_context_init(&coordinator->retry_context, - config.error_retry_max, - config.error_retry_interval); - - // 记录错误 - backup_record_error(coordinator, BACKUP_ERROR_NETWORK, "Network connection failed"); - - // 验证错误信息 - const char* error_msg = backup_get_last_error(coordinator); - assert(strstr(error_msg, "Network connection failed") != NULL); - - // 验证错误统计 - uint64_t error_count, retry_count; - backup_get_error_stats(coordinator, &error_count, &retry_count); - assert(error_count == 1); - - // 清除错误 - backup_clear_error(coordinator); - error_msg = backup_get_last_error(coordinator); - assert(strcmp(error_msg, "Success") == 0); - - printf("✓ Error recording test passed\n"); - - // 清理 - backup_coordinator_destroy(coordinator); -} - -// 测试插件接口 -void test_plugin_interface() { - printf("Testing plugin interface...\n"); - - // 初始化插件 - int32_t result = backup_plugin_init("test_config", 11); - assert(result == 0); - - // 测试错误统计接口 - uint64_t error_count, retry_count; - backup_plugin_get_error_stats(&error_count, &retry_count); - assert(error_count == 0); - assert(retry_count == 0); - - // 测试错误信息接口 - const char* error_msg = backup_plugin_get_last_error(); - assert(strcmp(error_msg, "Success") == 0); - - // 清理插件 - backup_plugin_cleanup(); - - printf("✓ Plugin interface test passed\n"); -} - -// 测试带重试的文件写入 -void test_file_write_with_retry() { - printf("Testing file write with retry...\n"); - - // 创建配置 - SBackupCoordinatorConfig config = { - .max_blocks_per_batch = 1000, - .batch_timeout_ms = 5000, - .enable_compression = true, - .enable_encryption = false, - .encryption_key = NULL, - .error_retry_max = 3, - .error_retry_interval = 1, - .error_store_path = "/tmp", - .enable_error_logging = true, - .error_buffer_size = 1000, - .backup_path = NULL, - .backup_max_size = 1024 * 1024 * 1024, - .compression_level = 1 - }; - - // 创建协同器（简化版本） - SBackupCoordinator* coordinator = (SBackupCoordinator*)malloc(sizeof(SBackupCoordinator)); - assert(coordinator != NULL); - - // 手动初始化协同器 - coordinator->config = config; - coordinator->bitmap_engine = NULL; - coordinator->event_interceptor = NULL; - coordinator->active_cursor = NULL; - coordinator->last_error_message = NULL; - coordinator->error_count = 0; - coordinator->retry_count = 0; - coordinator->total_backup_blocks = 0; - coordinator->total_backup_size = 0; - coordinator->backup_duration_ms = 0; - - // 初始化重试上下文 - backup_retry_context_init(&coordinator->retry_context, - config.error_retry_max, - config.error_retry_interval); - - // 测试文件写入 - const char* test_data = "Hello, World!"; - const char* test_file = "/tmp/test_backup_file.txt"; - - int32_t result = backup_write_file_with_retry(coordinator, test_file, test_data, strlen(test_data)); - assert(result == BACKUP_SUCCESS); - - // 验证文件内容 - FILE* fp = fopen(test_file, "r"); - assert(fp != NULL); - - char buffer[100]; - size_t read_size = fread(buffer, 1, sizeof(buffer), fp); - fclose(fp); - - assert(read_size == strlen(test_data)); - assert(strcmp(buffer, test_data) == 0); - - // 清理测试文件 - unlink(test_file); - - printf("✓ File write with retry test passed\n"); - - // 清理 - backup_coordinator_destroy(coordinator); -} - -int main() { - printf("Starting retry mechanism and error handling tests...\n\n"); - - printf("Running retry mechanism test...\n"); - test_retry_mechanism(); - printf("Retry mechanism test completed.\n"); - - printf("Running error recording test...\n"); - test_error_recording(); - printf("Error recording test completed.\n"); - - // test_plugin_interface(); // 暂时跳过插件接口测试 - // test_file_write_with_retry(); // 暂时跳过文件写入测试 - - printf("\nCore tests passed! ✓\n"); - return 0; -} \ No newline at end of file diff --git a/plugins/incremental_bitmap/test/test_ring_buffer.c b/plugins/incremental_bitmap/test/test_ring_buffer.c deleted file mode 100644 index 849ffbe5e87e..000000000000 --- a/plugins/incremental_bitmap/test/test_ring_buffer.c +++ /dev/null @@ -1,191 +0,0 @@ -#include -#include -#include -#include -#include -#include "../include/ring_buffer.h" - -#define TEST_BUFFER_SIZE 10 -#define TEST_ITERATIONS 1000 - -// 测试数据结构 -typedef struct { - int id; - char data[64]; -} TestData; - -// 生产者线程函数 -void* producer_thread(void* arg) { - SRingBuffer* ring_buffer = (SRingBuffer*)arg; - - for (int i = 0; i < TEST_ITERATIONS; i++) { - TestData* data = (TestData*)malloc(sizeof(TestData)); - data->id = i; - snprintf(data->data, sizeof(data->data), "Producer data %d", i); - - int ret = ring_buffer_enqueue_blocking(ring_buffer, data, 1000); - if (ret != 0) { - printf("Producer failed to enqueue item %d\n", i); - free(data); - } else { - printf("Producer enqueued item %d\n", i); - } - - usleep(1000); // 1ms delay - } - - return NULL; -} - -// 消费者线程函数 -void* consumer_thread(void* arg) { - SRingBuffer* ring_buffer = (SRingBuffer*)arg; - - for (int i = 0; i < TEST_ITERATIONS; i++) { - void* item; - int ret = ring_buffer_dequeue_blocking(ring_buffer, &item, 1000); - if (ret != 0) { - printf("Consumer failed to dequeue item %d\n", i); - } else { - TestData* data = (TestData*)item; - printf("Consumer dequeued item %d: %s\n", data->id, data->data); - free(data); - } - - usleep(2000); // 2ms delay - } - - return NULL; -} - -// 测试基本功能 -void test_basic_functionality() { - printf("=== Testing Basic Functionality ===\n"); - - SRingBuffer* ring_buffer = ring_buffer_init(5); - assert(ring_buffer != NULL); - - // 测试空队列状态 - assert(ring_buffer_is_empty(ring_buffer)); - assert(!ring_buffer_is_full(ring_buffer)); - assert(ring_buffer_get_size(ring_buffer) == 0); - assert(ring_buffer_get_capacity(ring_buffer) == 5); - - // 测试入队 - int* item1 = malloc(sizeof(int)); - *item1 = 42; - assert(ring_buffer_enqueue(ring_buffer, item1) == 0); - assert(!ring_buffer_is_empty(ring_buffer)); - assert(ring_buffer_get_size(ring_buffer) == 1); - - // 测试查看队首元素 - void* peek_item; - assert(ring_buffer_peek(ring_buffer, &peek_item) == 0); - assert(*(int*)peek_item == 42); - assert(ring_buffer_get_size(ring_buffer) == 1); // 大小不变 - - // 测试出队 - void* dequeue_item; - assert(ring_buffer_dequeue(ring_buffer, &dequeue_item) == 0); - assert(*(int*)dequeue_item == 42); - assert(ring_buffer_is_empty(ring_buffer)); - free(dequeue_item); - - // 测试队列满的情况 - for (int i = 0; i < 5; i++) { - int* item = malloc(sizeof(int)); - *item = i; - assert(ring_buffer_enqueue(ring_buffer, item) == 0); - } - assert(ring_buffer_is_full(ring_buffer)); - assert(ring_buffer_get_size(ring_buffer) == 5); - - // 测试队列满时入队失败 - int* overflow_item = malloc(sizeof(int)); - *overflow_item = 999; - assert(ring_buffer_enqueue(ring_buffer, overflow_item) != 0); - free(overflow_item); - - // 清空队列 - ring_buffer_clear(ring_buffer, free); - assert(ring_buffer_is_empty(ring_buffer)); - - ring_buffer_destroy(ring_buffer); - printf("Basic functionality test passed!\n"); -} - -// 测试多线程并发 -void test_concurrent_access() { - printf("=== Testing Concurrent Access ===\n"); - - SRingBuffer* ring_buffer = ring_buffer_init(TEST_BUFFER_SIZE); - assert(ring_buffer != NULL); - - pthread_t producer, consumer; - - // 创建生产者和消费者线程 - assert(pthread_create(&producer, NULL, producer_thread, ring_buffer) == 0); - assert(pthread_create(&consumer, NULL, consumer_thread, ring_buffer) == 0); - - // 等待线程完成 - pthread_join(producer, NULL); - pthread_join(consumer, NULL); - - // 验证队列为空 - assert(ring_buffer_is_empty(ring_buffer)); - - // 获取统计信息 - uint64_t enqueue_count, dequeue_count, overflow_count; - ring_buffer_get_stats(ring_buffer, &enqueue_count, &dequeue_count, &overflow_count); - printf("Stats: enqueue=%lu, dequeue=%lu, overflow=%lu\n", - enqueue_count, dequeue_count, overflow_count); - - ring_buffer_destroy(ring_buffer); - printf("Concurrent access test passed!\n"); -} - -// 测试超时功能 -void test_timeout_functionality() { - printf("=== Testing Timeout Functionality ===\n"); - - SRingBuffer* ring_buffer = ring_buffer_init(1); - assert(ring_buffer != NULL); - - // 填充队列 - int* item1 = malloc(sizeof(int)); - *item1 = 1; - assert(ring_buffer_enqueue(ring_buffer, item1) == 0); - - // 测试入队超时 - int* item2 = malloc(sizeof(int)); - *item2 = 2; - int ret = ring_buffer_enqueue_blocking(ring_buffer, item2, 100); // 100ms timeout - assert(ret != 0); // 应该超时 - free(item2); - - // 测试出队超时 - void* dequeue_item; - ret = ring_buffer_dequeue(ring_buffer, &dequeue_item); - assert(ret == 0); - if (dequeue_item == item1) { - free(dequeue_item); - item1 = NULL; - dequeue_item = NULL; - } - - ret = ring_buffer_dequeue_blocking(ring_buffer, &dequeue_item, 100); // 100ms timeout - assert(ret != 0); // 应该超时 - - ring_buffer_clear(ring_buffer, free); - ring_buffer_destroy(ring_buffer); - printf("Timeout functionality test passed!\n"); -} - -int main() { - printf("Starting Ring Buffer Tests...\n"); - test_basic_functionality(); - test_concurrent_access(); - test_timeout_functionality(); - printf("All tests passed!\n"); - return 0; -} \ No newline at end of file diff --git a/plugins/incremental_bitmap/test/test_roaring_bitmap_specific.c b/plugins/incremental_bitmap/test/test_roaring_bitmap_specific.c deleted file mode 100644 index 01f7b95e8501..000000000000 --- a/plugins/incremental_bitmap/test/test_roaring_bitmap_specific.c +++ /dev/null @@ -1,392 +0,0 @@ -#include -/* - * Copyright (c) 2024 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "../include/bitmap_interface.h" -#include "../include/roaring_bitmap.h" -#include -#include -#include -#include -#include -#include - -// 测试辅助函数 -static void print_test_result(const char* test_name, bool passed) { - printf("[%s] %s\n", passed ? "PASS" : "FAIL", test_name); -} - -static int64_t get_current_timestamp() { - struct timespec ts; - clock_gettime(CLOCK_REALTIME, &ts); - return (int64_t)ts.tv_sec * 1000000000LL + (int64_t)ts.tv_nsec; -} - -// 测试1: RoaringBitmap基本功能测试 -static void test_roaring_basic_functionality() { - printf("\n=== 测试1: RoaringBitmap基本功能测试 ===\n"); - - SBitmapInterface* roaring_interface = roaring_bitmap_interface_create(); - assert(roaring_interface != NULL); - - // 测试添加元素 - roaring_interface->add(roaring_interface->bitmap, 1); - roaring_interface->add(roaring_interface->bitmap, 1000); - roaring_interface->add(roaring_interface->bitmap, 1000000); - - // 测试查询元素 - assert(roaring_interface->contains(roaring_interface->bitmap, 1) == true); - assert(roaring_interface->contains(roaring_interface->bitmap, 1000) == true); - assert(roaring_interface->contains(roaring_interface->bitmap, 1000000) == true); - assert(roaring_interface->contains(roaring_interface->bitmap, 999) == false); - - // 测试基数 - assert(roaring_interface->cardinality(roaring_interface->bitmap) == 3); - - print_test_result("RoaringBitmap基本功能", true); - - roaring_interface->destroy(roaring_interface->bitmap); - free(roaring_interface); -} - -// 测试2: RoaringBitmap稀疏数据测试 -static void test_roaring_sparse_data() { - printf("\n=== 测试2: RoaringBitmap稀疏数据测试 ===\n"); - - SBitmapInterface* roaring_interface = roaring_bitmap_interface_create(); - assert(roaring_interface != NULL); - - // 添加稀疏数据（大间隔） - for (int i = 0; i < 1000; i++) { - roaring_interface->add(roaring_interface->bitmap, i * 10000); - } - - // 验证数据 - for (int i = 0; i < 1000; i++) { - assert(roaring_interface->contains(roaring_interface->bitmap, i * 10000) == true); - } - - assert(roaring_interface->cardinality(roaring_interface->bitmap) == 1000); - - print_test_result("RoaringBitmap稀疏数据", true); - - roaring_interface->destroy(roaring_interface->bitmap); - free(roaring_interface); -} - -// 测试3: RoaringBitmap密集数据测试 -static void test_roaring_dense_data() { - printf("\n=== 测试3: RoaringBitmap密集数据测试 ===\n"); - - SBitmapInterface* roaring_interface = roaring_bitmap_interface_create(); - assert(roaring_interface != NULL); - - // 添加密集数据（连续） - for (int i = 0; i < 100000; i++) { - roaring_interface->add(roaring_interface->bitmap, i); - } - - // 验证数据 - for (int i = 0; i < 100000; i++) { - assert(roaring_interface->contains(roaring_interface->bitmap, i) == true); - } - - assert(roaring_interface->cardinality(roaring_interface->bitmap) == 100000); - - print_test_result("RoaringBitmap密集数据", true); - - roaring_interface->destroy(roaring_interface->bitmap); - free(roaring_interface); -} - -// 测试4: RoaringBitmap混合数据测试 -static void test_roaring_mixed_data() { - printf("\n=== 测试4: RoaringBitmap混合数据测试 ===\n"); - - SBitmapInterface* roaring_interface = roaring_bitmap_interface_create(); - assert(roaring_interface != NULL); - - // 添加混合数据（既有稀疏又有密集） - // 密集区域1 - for (int i = 0; i < 1000; i++) { - roaring_interface->add(roaring_interface->bitmap, i); - } - - // 稀疏区域 - for (int i = 0; i < 100; i++) { - roaring_interface->add(roaring_interface->bitmap, 10000 + i * 1000); - } - - // 密集区域2 - for (int i = 0; i < 2000; i++) { - roaring_interface->add(roaring_interface->bitmap, 100000 + i); - } - - // 验证数据 - for (int i = 0; i < 1000; i++) { - assert(roaring_interface->contains(roaring_interface->bitmap, i) == true); - } - - for (int i = 0; i < 100; i++) { - assert(roaring_interface->contains(roaring_interface->bitmap, 10000 + i * 1000) == true); - } - - for (int i = 0; i < 2000; i++) { - assert(roaring_interface->contains(roaring_interface->bitmap, 100000 + i) == true); - } - - print_test_result("RoaringBitmap混合数据", true); - - roaring_interface->destroy(roaring_interface->bitmap); - free(roaring_interface); -} - -// 测试5: RoaringBitmap内存效率测试 -static void test_roaring_memory_efficiency() { - printf("\n=== 测试5: RoaringBitmap内存效率测试 ===\n"); - - SBitmapInterface* roaring_interface = roaring_bitmap_interface_create(); - assert(roaring_interface != NULL); - - // 添加大量稀疏数据 - for (uint64_t i = 0; i < 10000; i++) { - roaring_interface->add(roaring_interface->bitmap, i * 1000000ULL); - } - - // 稀疏数据内存使用 - size_t memory_usage = roaring_interface->memory_usage(roaring_interface->bitmap); - printf("稀疏数据内存使用: %zu bytes\n", memory_usage); - - // 清除数据 - roaring_interface->destroy(roaring_interface->bitmap); - free(roaring_interface); - - // 重新创建，添加密集数据 - roaring_interface = roaring_bitmap_interface_create(); - for (int i = 0; i < 10000; i++) { - roaring_interface->add(roaring_interface->bitmap, i); - } - - memory_usage = roaring_interface->memory_usage(roaring_interface->bitmap); - printf("密集数据内存使用: %zu bytes\n", memory_usage); - - print_test_result("RoaringBitmap内存效率", true); - - roaring_interface->destroy(roaring_interface->bitmap); - free(roaring_interface); -} - -// 测试6: RoaringBitmap性能基准测试 -static void test_roaring_performance_benchmark() { - printf("\n=== 测试6: RoaringBitmap性能基准测试 ===\n"); - - SBitmapInterface* roaring_interface = roaring_bitmap_interface_create(); - assert(roaring_interface != NULL); - - const int test_count = 1000000; - int64_t start_time = get_current_timestamp(); - - // 添加性能测试 - for (int i = 0; i < test_count; i++) { - roaring_interface->add(roaring_interface->bitmap, i); - } - - int64_t add_time = get_current_timestamp() - start_time; - printf("添加 %d 个元素耗时: %ld ns (%.2f ops/ms)\n", - test_count, add_time, (double)test_count / (add_time / 1000000.0)); - - // 查询性能测试 - start_time = get_current_timestamp(); - for (int i = 0; i < test_count; i++) { - roaring_interface->contains(roaring_interface->bitmap, i); - } - - int64_t query_time = get_current_timestamp() - start_time; - printf("查询 %d 个元素耗时: %ld ns (%.2f ops/ms)\n", - test_count, query_time, (double)test_count / (query_time / 1000000.0)); - - // 基数计算性能测试 - start_time = get_current_timestamp(); - uint64_t cardinality = roaring_interface->cardinality(roaring_interface->bitmap); - int64_t cardinality_time = get_current_timestamp() - start_time; - printf("基数计算耗时: %ld ns, 结果: %lu\n", cardinality_time, cardinality); - - print_test_result("RoaringBitmap性能基准", true); - - roaring_interface->destroy(roaring_interface->bitmap); - free(roaring_interface); -} - -// 测试7: RoaringBitmap并发安全测试 -static void* concurrent_roaring_thread(void* arg) { - SBitmapInterface* roaring_interface = (SBitmapInterface*)arg; - - for (int i = 0; i < 10000; i++) { - roaring_interface->add(roaring_interface->bitmap, i); - } - - return NULL; -} - -static void test_roaring_concurrent_safety() { - printf("\n=== 测试7: RoaringBitmap并发安全测试 ===\n"); - - SBitmapInterface* roaring_interface = roaring_bitmap_interface_create(); - assert(roaring_interface != NULL); - - const int thread_count = 8; - pthread_t threads[thread_count]; - - // 创建多个线程并发添加 - for (int i = 0; i < thread_count; i++) { - int result = pthread_create(&threads[i], NULL, concurrent_roaring_thread, roaring_interface); - assert(result == 0); - } - - // 等待所有线程完成 - for (int i = 0; i < thread_count; i++) { - pthread_join(threads[i], NULL); - } - - print_test_result("RoaringBitmap并发安全", true); - - roaring_interface->destroy(roaring_interface->bitmap); - free(roaring_interface); -} - -// 测试8: RoaringBitmap序列化性能测试 -static void test_roaring_serialization_performance() { - printf("\n=== 测试8: RoaringBitmap序列化性能测试 ===\n"); - - SBitmapInterface* roaring_interface = roaring_bitmap_interface_create(); - assert(roaring_interface != NULL); - - // 添加测试数据 - for (int i = 0; i < 100000; i++) { - roaring_interface->add(roaring_interface->bitmap, i); - } - - // 序列化性能测试 - int64_t start_time = get_current_timestamp(); - size_t serialized_size = roaring_interface->serialized_size(roaring_interface->bitmap); - void* serialized_data = malloc(serialized_size); - int32_t ser_ret = roaring_interface->serialize(roaring_interface->bitmap, serialized_data, serialized_size); - int64_t serialize_time = get_current_timestamp() - start_time; - - printf("序列化耗时: %ld ns, 数据大小: %zu bytes\n", serialize_time, serialized_size); - - // 反序列化性能测试 - SBitmapInterface* roaring_interface2 = roaring_bitmap_interface_create(); - void* tmp_bitmap = NULL; - start_time = get_current_timestamp(); - int32_t deser_ret = roaring_interface2->deserialize(&tmp_bitmap, serialized_data, serialized_size); - int64_t deserialize_time = get_current_timestamp() - start_time; - roaring_interface2->destroy(roaring_interface2->bitmap); - roaring_interface2->bitmap = tmp_bitmap; - - printf("反序列化耗时: %ld ns\n", deserialize_time); - - assert(ser_ret == 0); - assert(deser_ret == 0); - - // 验证数据一致性 - for (int i = 0; i < 100000; i++) { - assert(roaring_interface2->contains(roaring_interface2->bitmap, i) == true); - } - - print_test_result("RoaringBitmap序列化性能", true); - - // 清理 - free(serialized_data); - roaring_interface->destroy(roaring_interface->bitmap); - free(roaring_interface); - roaring_interface2->destroy(roaring_interface2->bitmap); - free(roaring_interface2); -} - -// 测试9: RoaringBitmap与SimpleBitmap对比测试 -static void test_roaring_vs_simple_comparison() { - printf("\n=== 测试9: RoaringBitmap与SimpleBitmap对比测试 ===\n"); - - const int test_count = 100000; - - // 测试RoaringBitmap - SBitmapInterface* roaring_interface = roaring_bitmap_interface_create(); - - int64_t start_time = get_current_timestamp(); - for (int i = 0; i < test_count; i++) { - roaring_interface->add(roaring_interface->bitmap, i); - } - int64_t roaring_add_time = get_current_timestamp() - start_time; - - start_time = get_current_timestamp(); - for (int i = 0; i < test_count; i++) { - roaring_interface->contains(roaring_interface->bitmap, i); - } - int64_t roaring_query_time = get_current_timestamp() - start_time; - - size_t roaring_memory = roaring_interface->memory_usage(roaring_interface->bitmap); - - roaring_interface->destroy(roaring_interface->bitmap); - free(roaring_interface); - - // 测试SimpleBitmap - setenv("TDENGINE_USE_SIMPLE_BITMAP", "1", 1); - SBitmapInterface* simple_interface = bitmap_interface_create(); - - start_time = get_current_timestamp(); - for (int i = 0; i < test_count; i++) { - simple_interface->add(simple_interface->bitmap, i); - } - int64_t simple_add_time = get_current_timestamp() - start_time; - - start_time = get_current_timestamp(); - for (int i = 0; i < test_count; i++) { - simple_interface->contains(simple_interface->bitmap, i); - } - int64_t simple_query_time = get_current_timestamp() - start_time; - - size_t simple_memory = simple_interface->memory_usage(simple_interface->bitmap); - - simple_interface->destroy(simple_interface->bitmap); - free(simple_interface); - unsetenv("TDENGINE_USE_SIMPLE_BITMAP"); - - // 输出对比结果 - printf("性能对比结果:\n"); - printf("RoaringBitmap - 添加: %ld ns, 查询: %ld ns, 内存: %zu bytes\n", - roaring_add_time, roaring_query_time, roaring_memory); - printf("SimpleBitmap - 添加: %ld ns, 查询: %ld ns, 内存: %zu bytes\n", - simple_add_time, simple_query_time, simple_memory); - - print_test_result("RoaringBitmap与SimpleBitmap对比", true); -} - -int main() { - printf("开始RoaringBitmap专用测试...\n"); - - test_roaring_basic_functionality(); - test_roaring_sparse_data(); - test_roaring_dense_data(); - test_roaring_mixed_data(); - test_roaring_memory_efficiency(); - test_roaring_performance_benchmark(); - test_roaring_concurrent_safety(); - test_roaring_serialization_performance(); - test_roaring_vs_simple_comparison(); - - printf("\n所有RoaringBitmap专用测试完成！\n"); - return 0; -} diff --git a/plugins/incremental_bitmap/test/test_skiplist.c b/plugins/incremental_bitmap/test/test_skiplist.c deleted file mode 100644 index 74a40b0aa5b8..000000000000 --- a/plugins/incremental_bitmap/test/test_skiplist.c +++ /dev/null @@ -1,330 +0,0 @@ -#include "../include/skiplist.h" -#include -#include -#include -#include -#include - -// 测试结果统计 -static int total_tests = 0; -static int passed_tests = 0; -static int failed_tests = 0; - -// 测试用全局互斥锁：保护对非线程安全跳表的并发写 -static pthread_mutex_t g_skiplist_mutex = PTHREAD_MUTEX_INITIALIZER; - -// 测试宏 -#define TEST_ASSERT(condition, message) do { \ - total_tests++; \ - if (condition) { \ - passed_tests++; \ - printf("✓ %s\n", message); \ - } else { \ - failed_tests++; \ - printf("✗ %s\n", message); \ - } \ -} while(0) - -// 简单的键值结构用于测试 -typedef struct { - uint64_t key; - char* value; -} TestEntry; - -// 释放节点值的回调（文件作用域） -static void free_entry_callback(uint64_t key, void* value, void* user_data) { - (void)key; - if (value) { - TestEntry* e = (TestEntry*)value; - free(e->value); - free(e); - if (user_data) { - int* c = (int*)user_data; - (*c)++; - } - } -} - -// 范围查询回调函数 -static void range_query_callback(uint64_t key, void* value, void* user_data) { - int* count_ptr = (int*)user_data; - (*count_ptr)++; -} - -// 测试基本操作 -void test_basic_operations() { - printf("\n=== 测试跳表基本操作 ===\n"); - - skiplist_t* skiplist = skiplist_create(); - TEST_ASSERT(skiplist != NULL, "创建跳表"); - - // 测试空跳表 - TEST_ASSERT(skiplist->size == 0, "空跳表大小为0"); - TEST_ASSERT(skiplist->size == 0, "空跳表为空"); - - // 测试插入元素 - TestEntry* entry1 = malloc(sizeof(TestEntry)); - entry1->key = 100; - entry1->value = strdup("value100"); - - skiplist_insert(skiplist, entry1->key, entry1); - TEST_ASSERT(skiplist->size == 1, "插入后大小为1"); - TEST_ASSERT(skiplist->size > 0, "插入后不为空"); - - // 测试查找元素 - TestEntry* found = (TestEntry*)skiplist_find(skiplist, entry1->key); - TEST_ASSERT(found != NULL, "查找元素成功"); - TEST_ASSERT(found->key == 100, "查找结果键值正确"); - TEST_ASSERT(strcmp(found->value, "value100") == 0, "查找结果值正确"); - - // 测试查找不存在的元素 - uint64_t not_found_key = 200; - TestEntry* not_found = (TestEntry*)skiplist_find(skiplist, not_found_key); - TEST_ASSERT(not_found == NULL, "查找不存在的元素返回NULL"); - - // 测试插入多个元素 - TestEntry* entry2 = malloc(sizeof(TestEntry)); - entry2->key = 50; - entry2->value = strdup("value50"); - - TestEntry* entry3 = malloc(sizeof(TestEntry)); - entry3->key = 150; - entry3->value = strdup("value150"); - - skiplist_insert(skiplist, entry2->key, entry2); - skiplist_insert(skiplist, entry3->key, entry3); - - TEST_ASSERT(skiplist->size == 3, "插入多个元素后大小正确"); - - // 测试删除元素 - skiplist_remove(skiplist, entry2->key); - TEST_ASSERT(skiplist->size == 2, "删除后大小正确"); - - // 验证删除后无法找到 - TestEntry* should_not_find = (TestEntry*)skiplist_find(skiplist, entry2->key); - TEST_ASSERT(should_not_find == NULL, "删除后无法找到元素"); - - // 清理 - free(entry1->value); - free(entry1); - free(entry2->value); - free(entry2); - free(entry3->value); - free(entry3); - skiplist_destroy(skiplist); -} - -// 测试范围查询 -void test_range_queries() { - printf("\n=== 测试范围查询 ===\n"); - - skiplist_t* skiplist = skiplist_create(); - - // 插入测试数据 - TestEntry* entries[10]; - for (int i = 0; i < 10; i++) { - entries[i] = malloc(sizeof(TestEntry)); - entries[i]->key = i * 10; // 0, 10, 20, ..., 90 - entries[i]->value = malloc(20); - sprintf(entries[i]->value, "value%d", i * 10); - skiplist_insert(skiplist, entries[i]->key, entries[i]); - } - - TEST_ASSERT(skiplist->size == 10, "插入10个元素"); - - // 测试范围查询 [20, 60] - uint64_t start_key = 20; - uint64_t end_key = 60; - - int count = 0; - skiplist_range_query(skiplist, start_key, end_key, false, range_query_callback, &count); - - TEST_ASSERT(count == 5, "范围查询返回5个元素"); // 20, 30, 40, 50, 60 - - // 测试空范围查询 - uint64_t empty_start = 100; - uint64_t empty_end = 200; - int empty_count = 0; - skiplist_range_query(skiplist, empty_start, empty_end, false, range_query_callback, &empty_count); - - TEST_ASSERT(empty_count == 0, "空范围查询返回0个元素"); - - // 清理 - for (int i = 0; i < 10; i++) { - free(entries[i]->value); - free(entries[i]); - } - skiplist_destroy(skiplist); -} - -// 测试迭代器 -void test_iterators() { - printf("\n=== 测试迭代器 ===\n"); - - skiplist_t* skiplist = skiplist_create(); - - // 插入测试数据 - TestEntry* entries[5]; - for (int i = 0; i < 5; i++) { - entries[i] = malloc(sizeof(TestEntry)); - entries[i]->key = i * 10; // 0, 10, 20, 30, 40 - entries[i]->value = malloc(20); - sprintf(entries[i]->value, "value%d", i * 10); - skiplist_insert(skiplist, entries[i]->key, entries[i]); - } - - // 测试正向迭代 - int forward_count = 0; - skiplist_range_query(skiplist, 0, 100, false, range_query_callback, &forward_count); - - TEST_ASSERT(forward_count == 5, "正向迭代返回5个元素"); - - // 测试反向迭代 - int reverse_count = 0; - skiplist_range_query(skiplist, 0, 100, true, range_query_callback, &reverse_count); - - TEST_ASSERT(reverse_count == 5, "反向迭代返回5个元素"); - - // 清理 - for (int i = 0; i < 5; i++) { - free(entries[i]->value); - free(entries[i]); - } - skiplist_destroy(skiplist); -} - -// 测试内存管理 -void test_memory_management() { - printf("\n=== 测试内存管理 ===\n"); - - skiplist_t* skiplist = skiplist_create(); - - size_t initial_memory = skiplist_get_memory_usage(skiplist); - TEST_ASSERT(initial_memory > 0, "初始内存使用量大于0"); - - // 插入大量元素 - TestEntry* entries[100]; - for (int i = 0; i < 100; i++) { - entries[i] = malloc(sizeof(TestEntry)); - entries[i]->key = i; - entries[i]->value = malloc(20); - sprintf(entries[i]->value, "value%d", i); - skiplist_insert(skiplist, entries[i]->key, entries[i]); - } - - size_t after_insert_memory = skiplist_get_memory_usage(skiplist); - TEST_ASSERT(after_insert_memory > initial_memory, "插入后内存使用量增加"); - - // 删除所有元素 - for (int i = 0; i < 100; i++) { - skiplist_remove(skiplist, entries[i]->key); - free(entries[i]->value); - free(entries[i]); - } - - TEST_ASSERT(skiplist->size == 0, "删除所有元素后为空"); - - skiplist_destroy(skiplist); -} - -// 并发插入参数 -typedef struct { - skiplist_t* skiplist; - uint64_t base_key; -} ConcurrentArgs; - -// 并发插入测试函数（使用不重叠键区间，避免覆盖导致的旧值泄漏） -void* test_concurrent_insert(void* arg) { - ConcurrentArgs* args = (ConcurrentArgs*)arg; - skiplist_t* skiplist = args->skiplist; - uint64_t base = args->base_key; - - for (int i = 0; i < 100; i++) { - TestEntry* entry = malloc(sizeof(TestEntry)); - entry->key = base + (uint64_t)i; - entry->value = malloc(20); - sprintf(entry->value, "value%lu", (unsigned long)entry->key); - pthread_mutex_lock(&g_skiplist_mutex); - skiplist_insert(skiplist, entry->key, entry); - pthread_mutex_unlock(&g_skiplist_mutex); - } - - return NULL; -} - -// 测试并发性 -void test_concurrency() { - printf("\n=== 测试并发性 ===\n"); - - skiplist_t* skiplist = skiplist_create(); - - // 创建多个线程进行并发插入（分配不重叠键区间） - pthread_t threads[4]; - ConcurrentArgs args[4]; - for (int i = 0; i < 4; i++) { - args[i].skiplist = skiplist; - args[i].base_key = (uint64_t)i * 1000ULL; // 0,1000,2000,3000 - pthread_create(&threads[i], NULL, test_concurrent_insert, &args[i]); - } - - // 等待所有线程完成 - for (int i = 0; i < 4; i++) { - pthread_join(threads[i], NULL); - } - - // 验证所有元素都被插入（检查每个分区的首元素） - for (int i = 0; i < 4; i++) { - uint64_t key = (uint64_t)i * 1000ULL; - TestEntry* found = (TestEntry*)skiplist_find(skiplist, key); - TEST_ASSERT(found != NULL, "并发插入的元素存在"); - } - - // 释放所有节点的值，避免内存泄漏 - int free_count = 0; - skiplist_range_query(skiplist, 0, UINT64_MAX, false, free_entry_callback, &free_count); - - skiplist_destroy(skiplist); -} - -// 测试边界情况 -void test_edge_cases() { - printf("\n=== 测试边界情况 ===\n"); - - skiplist_t* skiplist = skiplist_create(); - - // 测试插入NULL值 - skiplist_insert(skiplist, 1, NULL); - TEST_ASSERT(skiplist->size == 1, "插入NULL值成功"); - - // 测试查找不存在的键 - TestEntry* null_find = (TestEntry*)skiplist_find(skiplist, 999); - TEST_ASSERT(null_find == NULL, "查找不存在的键返回NULL"); - - // 测试删除不存在的键 - uint64_t not_exist_key = 999; - skiplist_remove(skiplist, not_exist_key); - TEST_ASSERT(skiplist->size == 1, "删除不存在的键不影响大小"); - - // 测试空跳表的范围查询 - int empty_count = 0; - skiplist_range_query(skiplist, 0, 100, false, range_query_callback, &empty_count); - - TEST_ASSERT(empty_count == 1, "空跳表范围查询返回1个元素（NULL值）"); - - skiplist_destroy(skiplist); -} - -int main() { - printf("开始跳表测试...\n"); - - test_basic_operations(); - test_range_queries(); - test_iterators(); - test_memory_management(); - test_concurrency(); - test_edge_cases(); - - printf("\n=== 测试结果 ===\n"); - - return (failed_tests == 0) ? 0 : 1; -} \ No newline at end of file diff --git a/plugins/incremental_bitmap/test/test_state_transitions.c b/plugins/incremental_bitmap/test/test_state_transitions.c deleted file mode 100644 index 8f9f5ee2c410..000000000000 --- a/plugins/incremental_bitmap/test/test_state_transitions.c +++ /dev/null @@ -1,346 +0,0 @@ -#include "../include/bitmap_engine.h" -#include -#include -#include -#include - -// 测试配置 -#define TEST_BLOCK_ID 1001 -#define TEST_WAL_OFFSET 1000 -#define TEST_TIMESTAMP 1000000000 - -// 状态名称数组 -static const char* STATE_NAMES[] = {"CLEAN", "DIRTY", "NEW", "DELETED"}; - -// 打印状态转换结果 -static void print_transition_result(EBlockState from, EBlockState to, int32_t result) { - const char* result_str = (result == 0) ? "✅ 成功" : "❌ 失败"; - printf(" %s -> %s: %s", STATE_NAMES[from], STATE_NAMES[to], result_str); - - if (result != 0) { - printf(" (错误: %s)", bitmap_engine_get_state_transition_error(from, to)); - } - printf("\n"); -} - -// 测试1: 验证状态转换规则矩阵 -static int test_state_transition_matrix() { - printf("=== 测试1: 状态转换规则矩阵验证 ===\n"); - - int matched = 0; - int total = 0; - - // 与引擎中的期望矩阵保持一致 - // 行：当前状态，列：目标状态；1=允许，0=禁止 - const int8_t expected[4][4] = { - /* CLEAN DIRTY NEW DELETED */ - /* CLEAN */ { 0, 1, 1, 1 }, - /* DIRTY */ { 1, 0, 1, 1 }, - /* NEW */ { 0, 1, 0, 1 }, - /* DELETED*/ { 0, 0, 0, 0 } - }; - - for (EBlockState from = BLOCK_STATE_CLEAN; from <= BLOCK_STATE_DELETED; from++) { - for (EBlockState to = BLOCK_STATE_CLEAN; to <= BLOCK_STATE_DELETED; to++) { - total++; - int32_t result = bitmap_engine_validate_state_transition(from, to); - int expect_allow = expected[from][to]; - int is_allow = (result == 0); - printf(" 验证转换: %s -> %s: 期望=%s 实际=%s\n", - STATE_NAMES[from], STATE_NAMES[to], expect_allow ? "允许" : "禁止", - is_allow ? "允许" : "禁止"); - if (expect_allow == is_allow) matched++; - } - } - - printf(" 总计: %d/%d 与期望矩阵匹配\n", matched, total); - return (matched == total) ? 0 : -1; -} - -// 测试2: 合法状态转换测试 -static int test_valid_transitions() { - printf("=== 测试2: 合法状态转换测试 ===\n"); - - SBitmapEngine* engine = bitmap_engine_init(); - if (engine == NULL) { - printf("❌ 引擎初始化失败\n"); - return -1; - } - - int passed = 0; - int total = 0; - - // 测试1: CLEAN -> DIRTY (合法) - printf(" 测试 CLEAN -> DIRTY:\n"); - total++; - int32_t result = bitmap_engine_mark_dirty(engine, TEST_BLOCK_ID, TEST_WAL_OFFSET, TEST_TIMESTAMP); - print_transition_result(BLOCK_STATE_CLEAN, BLOCK_STATE_DIRTY, result); - if (result == 0) passed++; - - // 测试2: DIRTY -> CLEAN (合法) - printf(" 测试 DIRTY -> CLEAN:\n"); - total++; - result = bitmap_engine_clear_block(engine, TEST_BLOCK_ID); - print_transition_result(BLOCK_STATE_DIRTY, BLOCK_STATE_CLEAN, result); - if (result == 0) passed++; - - // 测试3: CLEAN -> NEW (合法) - printf(" 测试 CLEAN -> NEW:\n"); - total++; - result = bitmap_engine_mark_new(engine, TEST_BLOCK_ID + 1, TEST_WAL_OFFSET + 1, TEST_TIMESTAMP + 1); - print_transition_result(BLOCK_STATE_CLEAN, BLOCK_STATE_NEW, result); - if (result == 0) passed++; - - // 测试4: NEW -> DIRTY (合法) - printf(" 测试 NEW -> DIRTY:\n"); - total++; - result = bitmap_engine_mark_dirty(engine, TEST_BLOCK_ID + 1, TEST_WAL_OFFSET + 2, TEST_TIMESTAMP + 2); - print_transition_result(BLOCK_STATE_NEW, BLOCK_STATE_DIRTY, result); - if (result == 0) passed++; - - // 测试5: DIRTY -> DELETED (合法) - printf(" 测试 DIRTY -> DELETED:\n"); - total++; - result = bitmap_engine_mark_deleted(engine, TEST_BLOCK_ID + 1, TEST_WAL_OFFSET + 3, TEST_TIMESTAMP + 3); - print_transition_result(BLOCK_STATE_DIRTY, BLOCK_STATE_DELETED, result); - if (result == 0) passed++; - - // 测试6: NEW -> DELETED (合法) - printf(" 测试 NEW -> DELETED:\n"); - total++; - result = bitmap_engine_mark_deleted(engine, TEST_BLOCK_ID + 2, TEST_WAL_OFFSET + 4, TEST_TIMESTAMP + 4); - print_transition_result(BLOCK_STATE_NEW, BLOCK_STATE_DELETED, result); - if (result == 0) passed++; - - printf(" 总计: %d/%d 合法转换测试通过\n", passed, total); - - bitmap_engine_destroy(engine); - return (passed == total) ? 0 : -1; -} - -// 测试3: 非法状态转换测试 -static int test_invalid_transitions() { - printf("=== 测试3: 非法状态转换测试 ===\n"); - - SBitmapEngine* engine = bitmap_engine_init(); - if (engine == NULL) { - printf("❌ 引擎初始化失败\n"); - return -1; - } - - int passed = 0; - int total = 0; - - // 测试1: CLEAN -> NEW (根据当前引擎规则：合法) - printf(" 测试 CLEAN -> NEW (应该成功):\n"); - total++; - int32_t result = bitmap_engine_mark_new(engine, TEST_BLOCK_ID + 10, TEST_WAL_OFFSET, TEST_TIMESTAMP); - if (result == 0) { - printf(" ✅ 按规则允许该转换\n"); - passed++; - } else { - printf(" ❌ 错误地拒绝了合法转换: %s\n", bitmap_engine_get_state_transition_error(BLOCK_STATE_CLEAN, BLOCK_STATE_NEW)); - } - - // 测试2: CLEAN -> DELETED (根据当前引擎规则：合法) - printf(" 测试 CLEAN -> DELETED (应该成功):\n"); - total++; - result = bitmap_engine_mark_deleted(engine, TEST_BLOCK_ID + 11, TEST_WAL_OFFSET, TEST_TIMESTAMP); - if (result == 0) { - printf(" ✅ 按规则允许该转换\n"); - passed++; - } else { - printf(" ❌ 错误地拒绝了合法转换: %s\n", bitmap_engine_get_state_transition_error(BLOCK_STATE_CLEAN, BLOCK_STATE_DELETED)); - } - - // 测试3: 先创建NEW块，然后尝试转换为CLEAN (非法) - printf(" 测试 NEW -> CLEAN (应该失败):\n"); - bitmap_engine_mark_new(engine, TEST_BLOCK_ID + 12, TEST_WAL_OFFSET, TEST_TIMESTAMP); - total++; - result = bitmap_engine_clear_block(engine, TEST_BLOCK_ID + 12); - if (result == ERR_INVALID_STATE_TRANS) { - printf(" ✅ 正确拒绝非法转换: %s\n", bitmap_engine_get_state_transition_error(BLOCK_STATE_NEW, BLOCK_STATE_CLEAN)); - passed++; - } else { - printf(" ❌ 错误地允许了非法转换\n"); - } - - // 测试4: 先创建DELETED块，然后尝试任何转换 (非法) - printf(" 测试 DELETED -> DIRTY (应该失败):\n"); - bitmap_engine_mark_deleted(engine, TEST_BLOCK_ID + 13, TEST_WAL_OFFSET, TEST_TIMESTAMP); - total++; - result = bitmap_engine_mark_dirty(engine, TEST_BLOCK_ID + 13, TEST_WAL_OFFSET + 1, TEST_TIMESTAMP + 1); - if (result == ERR_INVALID_STATE_TRANS) { - printf(" ✅ 正确拒绝非法转换: %s\n", bitmap_engine_get_state_transition_error(BLOCK_STATE_DELETED, BLOCK_STATE_DIRTY)); - passed++; - } else { - printf(" ❌ 错误地允许了非法转换\n"); - } - - // 测试5: 尝试清除不存在的块 - printf(" 测试清除不存在的块 (应该失败):\n"); - total++; - result = bitmap_engine_clear_block(engine, 99999); - if (result == ERR_BLOCK_NOT_FOUND) { - printf(" ✅ 正确返回块未找到错误\n"); - passed++; - } else { - printf(" ❌ 错误的错误码: %d\n", result); - } - - printf(" 总计: %d/%d 非法转换测试通过\n", passed, total); - - bitmap_engine_destroy(engine); - return (passed == total) ? 0 : -1; -} - -// 测试4: 状态查询测试 -static int test_state_query() { - printf("=== 测试4: 状态查询测试 ===\n"); - - SBitmapEngine* engine = bitmap_engine_init(); - if (engine == NULL) { - printf("❌ 引擎初始化失败\n"); - return -1; - } - - int passed = 0; - int total = 0; - - // 测试1: 查询不存在的块 - printf(" 测试查询不存在的块:\n"); - total++; - EBlockState state; - int32_t result = bitmap_engine_get_block_state(engine, 99999, &state); - if (result == ERR_BLOCK_NOT_FOUND) { - printf(" ✅ 正确返回块未找到错误\n"); - passed++; - } else { - printf(" ❌ 错误的错误码: %d\n", result); - } - - // 测试2: 创建块并查询状态 - printf(" 测试创建DIRTY块并查询状态:\n"); - total++; - result = bitmap_engine_mark_dirty(engine, TEST_BLOCK_ID + 20, TEST_WAL_OFFSET, TEST_TIMESTAMP); - if (result == 0) { - result = bitmap_engine_get_block_state(engine, TEST_BLOCK_ID + 20, &state); - if (result == 0 && state == BLOCK_STATE_DIRTY) { - printf(" ✅ 正确查询到DIRTY状态\n"); - passed++; - } else { - printf(" ❌ 状态查询失败或状态错误: %d\n", state); - } - } else { - printf(" ❌ 创建块失败\n"); - } - - // 测试3: 状态转换后查询 - printf(" 测试状态转换后查询:\n"); - total++; - result = bitmap_engine_mark_deleted(engine, TEST_BLOCK_ID + 20, TEST_WAL_OFFSET + 1, TEST_TIMESTAMP + 1); - if (result == 0) { - result = bitmap_engine_get_block_state(engine, TEST_BLOCK_ID + 20, &state); - if (result == 0 && state == BLOCK_STATE_DELETED) { - printf(" ✅ 正确查询到DELETED状态\n"); - passed++; - } else { - printf(" ❌ 状态查询失败或状态错误: %d\n", state); - } - } else { - printf(" ❌ 状态转换失败\n"); - } - - printf(" 总计: %d/%d 状态查询测试通过\n", passed, total); - - bitmap_engine_destroy(engine); - return (passed == total) ? 0 : -1; -} - -// 测试5: 边界条件测试 -static int test_edge_cases() { - printf("=== 测试5: 边界条件测试 ===\n"); - - int passed = 0; - int total = 0; - - // 测试1: 无效状态值 - printf(" 测试无效状态值:\n"); - total++; - int32_t result = bitmap_engine_validate_state_transition(-1, BLOCK_STATE_DIRTY); - if (result == ERR_INVALID_STATE_TRANS) { - printf(" ✅ 正确拒绝无效状态值\n"); - passed++; - } else { - printf(" ❌ 错误地接受了无效状态值\n"); - } - - // 测试2: 超出范围的状态值 - total++; - result = bitmap_engine_validate_state_transition(BLOCK_STATE_DELETED + 1, BLOCK_STATE_DIRTY); - if (result == ERR_INVALID_STATE_TRANS) { - printf(" ✅ 正确拒绝超出范围的状态值\n"); - passed++; - } else { - printf(" ❌ 错误地接受了超出范围的状态值\n"); - } - - // 测试3: NULL参数 - printf(" 测试NULL参数:\n"); - total++; - result = bitmap_engine_get_block_state(NULL, TEST_BLOCK_ID, NULL); - if (result == ERR_INVALID_PARAM) { - printf(" ✅ 正确拒绝NULL参数\n"); - passed++; - } else { - printf(" ❌ 错误地接受了NULL参数\n"); - } - - printf(" 总计: %d/%d 边界条件测试通过\n", passed, total); - return (passed == total) ? 0 : -1; -} - -int main() { - printf("=== 状态转换验证测试程序 ===\n\n"); - - // 运行所有测试 - int test_results[] = { - test_state_transition_matrix(), - test_valid_transitions(), - test_invalid_transitions(), - test_state_query(), - test_edge_cases() - }; - - // 输出测试结果 - printf("=== 测试结果汇总 ===\n"); - const char* test_names[] = { - "状态转换规则矩阵验证", - "合法状态转换测试", - "非法状态转换测试", - "状态查询测试", - "边界条件测试" - }; - - int passed = 0; - int total = sizeof(test_results) / sizeof(test_results[0]); - - for (int i = 0; i < total; i++) { - if (test_results[i] == 0) { - printf("%s: 通过\n", test_names[i]); - passed++; - } else { - printf("%s: 失败\n", test_names[i]); - } - } - - printf("\n总计: %d/%d 测试通过\n", passed, total); - - if (passed == total) { - printf("所有状态转换验证测试通过！状态机设计正确。\n"); - return 0; - } else { - printf("部分测试失败，请检查状态转换实现。\n"); - return 1; - } -} \ No newline at end of file diff --git a/plugins/incremental_bitmap/test/test_storage_engine_interface.c b/plugins/incremental_bitmap/test/test_storage_engine_interface.c deleted file mode 100644 index 15d82f6c9a36..000000000000 --- a/plugins/incremental_bitmap/test/test_storage_engine_interface.c +++ /dev/null @@ -1,229 +0,0 @@ -#include "../include/storage_engine_interface.h" -#include "../include/event_interceptor.h" -#include "../include/bitmap_engine.h" -#include -#include -#include -#include -#include - -// 测试辅助函数 -static void print_test_result(const char* test_name, bool passed) { - printf("[%s] %s\n", passed ? "PASS" : "FAIL", test_name); -} - -static int64_t get_current_timestamp() { - struct timespec ts; - clock_gettime(CLOCK_REALTIME, &ts); - return (int64_t)ts.tv_sec * 1000000000LL + (int64_t)ts.tv_nsec; -} - -// 事件回调函数 -static void test_event_callback(const SBlockEvent* event, void* user_data) { - printf("收到事件: 类型=%d, 块ID=%lu, WAL偏移量=%lu, 时间戳=%ld\n", - event->event_type, event->block_id, event->wal_offset, event->timestamp); -} - -// 测试1: 存储引擎接口注册和获取 -static void test_interface_registration() { - printf("\n=== 测试1: 存储引擎接口注册和获取 ===\n"); - - // 注册Mock存储引擎 - extern int32_t register_mock_storage_engine(void); - int32_t result = register_mock_storage_engine(); - assert(result == 0); - print_test_result("注册Mock存储引擎", true); - - // 获取Mock存储引擎接口 - SStorageEngineInterface* mock_interface = get_storage_engine_interface("mock"); - assert(mock_interface != NULL); - print_test_result("获取Mock存储引擎接口", true); - - // 检查接口名称 - const char* name = mock_interface->get_engine_name(); - assert(strcmp(name, "mock") == 0); - print_test_result("检查接口名称", true); - - // 检查是否支持 - bool supported = mock_interface->is_supported(); - assert(supported == true); - print_test_result("检查支持状态", true); -} - -// 测试2: 存储引擎接口初始化和配置 -static void test_interface_initialization() { - printf("\n=== 测试2: 存储引擎接口初始化和配置 ===\n"); - - // 获取Mock存储引擎接口 - SStorageEngineInterface* mock_interface = get_storage_engine_interface("mock"); - assert(mock_interface != NULL); - - // 配置存储引擎 - SStorageEngineConfig config = { - .enable_interception = true, - .event_callback = test_event_callback, - .callback_user_data = NULL, - .event_buffer_size = 1000, - .callback_threads = 2 - }; - - // 初始化存储引擎 - int32_t result = mock_interface->init(&config); - assert(result == 0); - print_test_result("初始化存储引擎", true); - - // 安装事件拦截 - result = mock_interface->install_interception(); - assert(result == 0); - print_test_result("安装事件拦截", true); - - // 卸载事件拦截 - result = mock_interface->uninstall_interception(); - assert(result == 0); - print_test_result("卸载事件拦截", true); - - // 销毁存储引擎 - mock_interface->destroy(); - print_test_result("销毁存储引擎", true); -} - -// 测试3: 事件触发和统计 -static void test_event_triggering() { - printf("\n=== 测试3: 事件触发和统计 ===\n"); - - // 获取Mock存储引擎接口 - SStorageEngineInterface* mock_interface = get_storage_engine_interface("mock"); - assert(mock_interface != NULL); - - // 初始化并安装拦截 - SStorageEngineConfig config = { - .enable_interception = true, - .event_callback = test_event_callback, - .callback_user_data = NULL, - .event_buffer_size = 1000, - .callback_threads = 2 - }; - - mock_interface->init(&config); - mock_interface->install_interception(); - - // 触发事件 - SStorageEvent event = { - .event_type = STORAGE_EVENT_BLOCK_UPDATE, - .block_id = 12345, - .wal_offset = 1000, - .timestamp = get_current_timestamp(), - .user_data = NULL - }; - - int32_t result = mock_interface->trigger_event(&event); - assert(result == 0); - print_test_result("触发事件", true); - - // 获取统计信息 - uint64_t events_processed, events_dropped; - result = mock_interface->get_stats(&events_processed, &events_dropped); - assert(result == 0); - assert(events_processed == 1); - assert(events_dropped == 0); - print_test_result("获取统计信息", true); - - // 清理 - mock_interface->uninstall_interception(); - mock_interface->destroy(); -} - -// 测试4: 与事件拦截器集成 -static void test_event_interceptor_integration() { - printf("\n=== 测试4: 与事件拦截器集成 ===\n"); - - // 创建位图引擎 - SBitmapEngine* bitmap_engine = bitmap_engine_init(); - assert(bitmap_engine != NULL); - - // 创建事件拦截器 - SEventInterceptorConfig interceptor_config = { - .enable_interception = true, - .event_buffer_size = 1000, - .callback_threads = 2, - .callback = test_event_callback, - .callback_user_data = NULL - }; - - SEventInterceptor* interceptor = event_interceptor_init(&interceptor_config, bitmap_engine); - assert(interceptor != NULL); - - // 获取Mock存储引擎接口 - SStorageEngineInterface* mock_interface = get_storage_engine_interface("mock"); - assert(mock_interface != NULL); - - // 设置存储引擎接口 - int32_t result = event_interceptor_set_storage_interface(interceptor, mock_interface); - assert(result == 0); - print_test_result("设置存储引擎接口", true); - - // 安装存储引擎拦截 - result = event_interceptor_install_storage_interception(interceptor); - assert(result == 0); - print_test_result("安装存储引擎拦截", true); - - // 启动事件拦截器 - result = event_interceptor_start(interceptor); - assert(result == 0); - print_test_result("启动事件拦截器", true); - - // 手动触发测试事件 - result = event_interceptor_trigger_test_event(interceptor, EVENT_BLOCK_UPDATE, - 12345, 1000, get_current_timestamp()); - assert(result == 0); - print_test_result("手动触发测试事件", true); - - // 等待事件处理 - usleep(100000); // 100ms - - // 获取统计信息 - uint64_t events_processed, events_dropped; - event_interceptor_get_stats(interceptor, &events_processed, &events_dropped); - printf("事件拦截器统计: 处理=%lu, 丢弃=%lu\n", events_processed, events_dropped); - print_test_result("事件拦截器统计", true); - - // 停止和清理 - event_interceptor_stop(interceptor); - event_interceptor_uninstall_storage_interception(interceptor); - event_interceptor_destroy(interceptor); - bitmap_engine_destroy(bitmap_engine); -} - -// 测试5: 接口列表功能 -static void test_interface_listing() { - printf("\n=== 测试5: 接口列表功能 ===\n"); - - // 列出所有可用的存储引擎接口 - char* names[10]; - uint32_t actual_count; - int32_t result = list_storage_engine_interfaces(names, 10, &actual_count); - assert(result == 0); - assert(actual_count >= 1); // 至少应该有mock接口 - - printf("可用的存储引擎接口:\n"); - for (uint32_t i = 0; i < actual_count; i++) { - printf(" - %s\n", names[i]); - free(names[i]); // 释放内存 - } - - print_test_result("列出存储引擎接口", true); -} - -int main() { - printf("开始存储引擎接口测试...\n"); - - test_interface_registration(); - test_interface_initialization(); - test_event_triggering(); - test_event_interceptor_integration(); - test_interface_listing(); - - printf("\n所有测试完成！\n"); - return 0; -} - diff --git a/plugins/incremental_bitmap/test/test_taosdump_comparison.c b/plugins/incremental_bitmap/test/test_taosdump_comparison.c deleted file mode 100644 index bc140b3d66ec..000000000000 --- a/plugins/incremental_bitmap/test/test_taosdump_comparison.c +++ /dev/null @@ -1,359 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include "../include/bitmap_engine.h" -#include "../include/backup_coordinator.h" - -// 测试配置 -#define TEST_DB_NAME "test_bitmap_db" -#define TEST_STB_NAME "test_stable" -#define TEST_CHILD_COUNT 5 -#define TEST_ROWS_PER_CHILD 1000 -#define BACKUP_DIR "/tmp/taosdump_comparison_test" - -// 测试结果统计 -typedef struct { - int total_tests; - int passed_tests; - int failed_tests; - double bitmap_detection_time_ms; - double taosdump_backup_time_ms; - uint64_t bitmap_detected_blocks; - uint64_t taosdump_backup_size; -} TestStats; - -static TestStats g_test_stats = {0}; - -// 测试宏定义 -#define TEST_ASSERT(condition, message) do { \ - g_test_stats.total_tests++; \ - if (condition) { \ - g_test_stats.passed_tests++; \ - printf("✓ %s\n", message); \ - } else { \ - g_test_stats.failed_tests++; \ - printf("❌ %s\n", message); \ - } \ -} while(0) - -// 获取当前时间戳（毫秒） -static int64_t get_current_time_ms() { - struct timespec ts; - clock_gettime(CLOCK_MONOTONIC, &ts); - return ts.tv_sec * 1000LL + ts.tv_nsec / 1000000LL; -} - -// 创建测试数据 -static int create_test_data() { - printf("\n=== 创建测试数据 ===\n"); - - // 连接到TDengine - TAOS* taos = taos_connect("127.0.0.1", "root", "taosdata", NULL, 6030); - if (!taos) { - printf(" [ERROR] 无法连接到TDengine\n"); - return -1; - } - printf(" [INFO] 成功连接到TDengine\n"); - - // 创建数据库 - char sql[512]; - snprintf(sql, sizeof(sql), "CREATE DATABASE IF NOT EXISTS %s", TEST_DB_NAME); - TAOS_RES* result = taos_query(taos, sql); - if (result == NULL) { - printf(" [ERROR] 创建数据库失败: %s\n", taos_errstr(taos)); - taos_close(taos); - return -1; - } - taos_free_result(result); - printf(" [INFO] 数据库创建/连接成功\n"); - - // 切换到测试数据库 - snprintf(sql, sizeof(sql), "USE %s", TEST_DB_NAME); - result = taos_query(taos, sql); - if (result == NULL) { - printf(" [ERROR] 切换数据库失败: %s\n", taos_errstr(taos)); - taos_close(taos); - return -1; - } - taos_free_result(result); - printf(" [INFO] 切换到数据库: %s\n", TEST_DB_NAME); - - // 创建超级表 - snprintf(sql, sizeof(sql), - "CREATE STABLE IF NOT EXISTS %s (ts TIMESTAMP, value DOUBLE, tag1 INT) " - "TAGS (location BINARY(20), device_id INT)", - TEST_STB_NAME); - result = taos_query(taos, sql); - if (result == NULL) { - printf(" [ERROR] 创建超级表失败: %s\n", taos_errstr(taos)); - taos_close(taos); - return -1; - } - taos_free_result(result); - printf(" [INFO] 超级表创建成功: %s\n", TEST_STB_NAME); - - // 创建子表并插入数据 - for (int i = 0; i < TEST_CHILD_COUNT; i++) { - char child_table[64]; - snprintf(child_table, sizeof(child_table), "%s_%d", TEST_STB_NAME, i); - - // 创建子表 - snprintf(sql, sizeof(sql), - "CREATE TABLE IF NOT EXISTS %s USING %s " - "TAGS ('location_%d', %d)", - child_table, TEST_STB_NAME, i, i); - result = taos_query(taos, sql); - if (result == NULL) { - printf(" [ERROR] 创建子表失败: %s\n", taos_errstr(taos)); - continue; - } - taos_free_result(result); - - // 插入测试数据 - for (int j = 0; j < TEST_ROWS_PER_CHILD; j++) { - snprintf(sql, sizeof(sql), - "INSERT INTO %s VALUES (%ld, %f, %d)", - child_table, - time(NULL) * 1000 + j, - (double)(i * 100 + j), - j); - result = taos_query(taos, sql); - if (result == NULL) { - printf(" [WARNING] 插入数据失败: %s\n", taos_errstr(taos)); - } else { - taos_free_result(result); - } - } - - printf(" 创建子表 %s，插入 %d 行数据\n", child_table, TEST_ROWS_PER_CHILD); - } - - taos_close(taos); - printf(" [SUCCESS] 测试数据创建完成\n"); - return 0; -} - -// 测试位图插件增量检测 -static int test_bitmap_incremental_detection() { - printf("\n=== 测试位图插件增量检测 ===\n"); - - int64_t start_time = get_current_time_ms(); - - // 初始化位图引擎 - SBitmapEngine* bitmap_engine = bitmap_engine_init(); - if (!bitmap_engine) { - printf(" [ERROR] 位图引擎初始化失败\n"); - return -1; - } - - // 创建备份配置 - SBackupConfig backup_config = { - .batch_size = 1000, - .max_retries = 3, - .retry_interval_ms = 1000, - .timeout_ms = 30000, - .enable_compression = true, - .enable_encryption = false, - .backup_path = "/tmp/backup", - .temp_path = "/tmp/temp" - }; - - // 初始化备份协调器 - SBackupCoordinator* coordinator = backup_coordinator_init(bitmap_engine, &backup_config); - if (!coordinator) { - printf(" [ERROR] 备份协调器初始化失败\n"); - bitmap_engine_destroy(bitmap_engine); - return -1; - } - - // 模拟一些块变更事件 - for (int i = 0; i < 100; i++) { - uint64_t block_id = i; - uint64_t wal_offset = i * 1000; - int64_t timestamp = time(NULL) * 1000000000LL + i * 1000000LL; - - bitmap_engine_mark_dirty(bitmap_engine, block_id, wal_offset, timestamp); - } - - // 获取增量块 - uint64_t block_ids[100]; - uint32_t max_count = 100; - uint32_t count = backup_coordinator_get_dirty_blocks(coordinator, 0, 100000, block_ids, max_count); - - int64_t end_time = get_current_time_ms(); - g_test_stats.bitmap_detection_time_ms = end_time - start_time; - g_test_stats.bitmap_detected_blocks = count; - - printf(" 位图插件检测到 %u 个增量块\n", count); - printf(" 检测耗时: %.2f ms\n", g_test_stats.bitmap_detection_time_ms); - - // 清理资源 - backup_coordinator_destroy(coordinator); - bitmap_engine_destroy(bitmap_engine); - - TEST_ASSERT(count > 0, "位图插件成功检测到增量块"); - return 0; -} - -// 测试taosdump备份 -static int test_taosdump_backup() { - printf("\n=== 测试taosdump备份 ===\n"); - - // 创建备份目录 - char cmd[512]; - snprintf(cmd, sizeof(cmd), "rm -rf %s && mkdir -p %s", BACKUP_DIR, BACKUP_DIR); - system(cmd); - - int64_t start_time = get_current_time_ms(); - - // 执行taosdump备份 - snprintf(cmd, sizeof(cmd), - "taosdump -h 127.0.0.1 -P 6030 -D %s -o %s > /dev/null 2>&1", - TEST_DB_NAME, BACKUP_DIR); - int result = system(cmd); - - int64_t end_time = get_current_time_ms(); - g_test_stats.taosdump_backup_time_ms = end_time - start_time; - - if (result == 0) { - // 计算备份大小 - struct stat st; - if (stat(BACKUP_DIR, &st) == 0) { - g_test_stats.taosdump_backup_size = st.st_size; - } - - printf(" taosdump备份成功\n"); - printf(" 备份耗时: %.2f ms\n", g_test_stats.taosdump_backup_time_ms); - printf(" 备份大小: %lu bytes\n", g_test_stats.taosdump_backup_size); - - TEST_ASSERT(1, "taosdump备份成功完成"); - } else { - printf(" [ERROR] taosdump备份失败\n"); - TEST_ASSERT(0, "taosdump备份失败"); - } - - return result; -} - -// 生成协作脚本 -static int generate_collaboration_script() { - printf("\n=== 生成位图插件与taosdump协作脚本 ===\n"); - - FILE* script = fopen("/tmp/bitmap_taosdump_collaboration.sh", "w"); - if (!script) { - printf(" [ERROR] 无法创建协作脚本\n"); - return -1; - } - - fprintf(script, "#!/bin/bash\n\n"); - fprintf(script, "# 位图插件与taosdump协作脚本\n"); - fprintf(script, "# 生成时间: %s\n\n", ctime(&(time_t){time(NULL)})); - - fprintf(script, "echo \"步骤1: 使用位图插件检测增量数据...\"\n"); - fprintf(script, "./incremental_bitmap_tool --detect \\\n"); - fprintf(script, " --host 127.0.0.1 --port 6030 \\\n"); - fprintf(script, " --database %s \\\n", TEST_DB_NAME); - fprintf(script, " --since $(date -d '1 hour ago' +%%s) \\\n"); - fprintf(script, " --output incremental_blocks.json\n\n"); - - fprintf(script, "echo \"步骤2: 使用taosdump备份增量数据...\"\n"); - fprintf(script, "taosdump -h 127.0.0.1 -P 6030 \\\n"); - fprintf(script, " -D %s \\\n", TEST_DB_NAME); - fprintf(script, " -S $(date -d '1 hour ago' +%%s) \\\n"); - fprintf(script, " -o /backup/incremental_$(date +%%Y%%m%%d_%%H%%M%%S)\n\n"); - - fprintf(script, "echo \"步骤3: 验证备份完整性...\"\n"); - fprintf(script, "./incremental_bitmap_tool --verify \\\n"); - fprintf(script, " --backup /backup/ \\\n"); - fprintf(script, " --blocks incremental_blocks.json \\\n"); - fprintf(script, " --report backup_verification_report.json\n\n"); - - fprintf(script, "echo \"协作备份完成！\"\n"); - - fclose(script); - - // 设置执行权限 - chmod("/tmp/bitmap_taosdump_collaboration.sh", 0755); - - printf(" 协作脚本已生成: /tmp/bitmap_taosdump_collaboration.sh\n"); - TEST_ASSERT(1, "协作脚本生成成功"); - - return 0; -} - -// 性能对比分析 -static void performance_comparison_analysis() { - printf("\n=== 性能对比分析 ===\n"); - - printf("位图插件增量检测:\n"); - printf(" - 检测时间: %.2f ms\n", g_test_stats.bitmap_detection_time_ms); - printf(" - 检测块数: %lu\n", g_test_stats.bitmap_detected_blocks); - printf(" - 检测速率: %.2f blocks/ms\n", - (double)g_test_stats.bitmap_detected_blocks / g_test_stats.bitmap_detection_time_ms); - - printf("\ntaosdump备份:\n"); - printf(" - 备份时间: %.2f ms\n", g_test_stats.taosdump_backup_time_ms); - printf(" - 备份大小: %lu bytes\n", g_test_stats.taosdump_backup_size); - printf(" - 备份速率: %.2f bytes/ms\n", - (double)g_test_stats.taosdump_backup_size / g_test_stats.taosdump_backup_time_ms); - - printf("\n协作优势:\n"); - printf(" - 位图插件提供精确的增量检测\n"); - printf(" - taosdump提供稳定的数据导出\n"); - printf(" - 两者协作，各司其职\n"); - printf(" - 避免全量备份，提升效率\n"); -} - -// 打印测试总结 -static void print_test_summary() { - printf("\n==========================================\n"); - printf(" taosdump增量对比测试总结\n"); - printf("==========================================\n"); - printf("总测试数: %d\n", g_test_stats.total_tests); - printf("通过测试: %d\n", g_test_stats.passed_tests); - printf("失败测试: %d\n", g_test_stats.failed_tests); - printf("通过率: %.1f%%\n", - (double)g_test_stats.passed_tests / g_test_stats.total_tests * 100); - - printf("\n测试结论:\n"); - if (g_test_stats.failed_tests == 0) { - printf("✅ 位图插件与taosdump协作测试全部通过\n"); - printf("✅ 位图插件提供高效的增量检测\n"); - printf("✅ taosdump提供稳定的数据备份\n"); - printf("✅ 两者协作模式验证成功\n"); - } else { - printf("⚠️ 部分测试失败，需要进一步优化\n"); - } -} - -int main() { - printf("开始taosdump增量对比测试...\n"); - - // 1. 创建测试数据 - if (create_test_data() != 0) { - printf(" [ERROR] 测试数据创建失败，跳过后续测试\n"); - return 1; - } - - // 2. 测试位图插件增量检测 - test_bitmap_incremental_detection(); - - // 3. 测试taosdump备份 - test_taosdump_backup(); - - // 4. 生成协作脚本 - generate_collaboration_script(); - - // 5. 性能对比分析 - performance_comparison_analysis(); - - // 6. 打印测试总结 - print_test_summary(); - - return (g_test_stats.failed_tests == 0) ? 0 : 1; -} diff --git a/plugins/incremental_bitmap/test/test_taosdump_integration.c b/plugins/incremental_bitmap/test/test_taosdump_integration.c deleted file mode 100644 index 3962b24deee9..000000000000 --- a/plugins/incremental_bitmap/test/test_taosdump_integration.c +++ /dev/null @@ -1,225 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -#include "bitmap_engine.h" -#include "event_interceptor.h" -#include "backup_coordinator.h" -#include "storage_engine_interface.h" - -// 测试taosdump集成功能 -static void test_taosdump_integration() { - printf("\n=== 测试taosdump集成功能 ===\n"); - - // 1. 初始化位图引擎 - SBitmapEngine* bitmap_engine = bitmap_engine_init(); - assert(bitmap_engine != NULL); - printf("[PASS] 位图引擎初始化\n"); - - // 2. 初始化事件拦截器 - SEventInterceptorConfig interceptor_config = { - .enable_interception = true, - .event_buffer_size = 1000, - .callback_threads = 2, - .callback = NULL, - .callback_user_data = NULL - }; - - SEventInterceptor* event_interceptor = event_interceptor_init(&interceptor_config, bitmap_engine); - assert(event_interceptor != NULL); - printf("[PASS] 事件拦截器初始化\n"); - - // 3. 初始化备份协调器 - SBackupConfig backup_config = { - .batch_size = 100, - .max_retries = 3, - .retry_interval_ms = 1000, - .timeout_ms = 5000, - .enable_compression = true, - .enable_encryption = false, - .backup_path = "/tmp/tdengine_backup", - .temp_path = "/tmp" - }; - - SBackupCoordinator* coordinator = backup_coordinator_init(bitmap_engine, &backup_config); - assert(coordinator != NULL); - printf("[PASS] 备份协调器初始化\n"); - - // 4. 模拟数据写入和块变更事件 - printf("\n--- 模拟数据写入 ---\n"); - - // 模拟块1001-1004的变更 - for (int i = 1; i <= 4; i++) { - uint64_t block_id = 1000 + i; - uint64_t wal_offset = i * 1000; - int64_t timestamp = time(NULL) * 1000000000LL + i * 1000000LL; // 纳秒时间戳 - - // 标记块为脏状态 - bitmap_engine_mark_dirty(bitmap_engine, block_id, wal_offset, timestamp); - printf("标记块 %lu 为脏状态 (WAL偏移量: %lu, 时间戳: %ld)\n", - block_id, wal_offset, timestamp); - } - - // 5. 测试增量块检测 - printf("\n--- 测试增量块检测 ---\n"); - - uint64_t start_wal = 0; - uint64_t end_wal = 5000; - uint64_t block_ids[10]; - uint32_t max_count = 10; - - uint32_t count = backup_coordinator_get_dirty_blocks(coordinator, start_wal, end_wal, block_ids, max_count); - printf("在WAL范围 [%lu, %lu] 内找到 %u 个脏块:\n", start_wal, end_wal, count); - - for (uint32_t i = 0; i < count; i++) { - printf(" 块ID: %lu\n", block_ids[i]); - } - - assert(count > 0); - printf("[PASS] 增量块检测\n"); - - // 6. 测试taosdump兼容的时间范围查询 - printf("\n--- 测试taosdump兼容的时间范围查询 ---\n"); - - int64_t start_time = time(NULL) * 1000000000LL; // 当前时间 - int64_t end_time = start_time + 1000000000LL; // 1秒后 - - uint32_t time_count = backup_coordinator_get_dirty_blocks_by_time(coordinator, start_time, end_time, block_ids, max_count); - printf("在时间范围 [%ld, %ld] 内找到 %u 个脏块:\n", start_time, end_time, time_count); - - for (uint32_t i = 0; i < time_count; i++) { - printf(" 块ID: %lu\n", block_ids[i]); - } - - printf("[PASS] 时间范围查询\n"); - - // 7. 生成taosdump兼容的备份脚本 - printf("\n--- 生成taosdump兼容的备份脚本 ---\n"); - - // 创建备份目录 - system("mkdir -p /tmp/tdengine_backup"); - - // 生成备份脚本 - FILE* script = fopen("/tmp/tdengine_backup/backup_script.sh", "w"); - if (script) { - fprintf(script, "#!/bin/bash\n\n"); - fprintf(script, "# TDengine增量备份脚本 - 由位图插件生成\n"); - fprintf(script, "# 生成时间: %s\n\n", ctime(&(time_t){time(NULL)})); - - fprintf(script, "SOURCE_HOST=localhost\n"); - fprintf(script, "SOURCE_PORT=6030\n"); - fprintf(script, "DATABASE=test_db\n"); - fprintf(script, "BACKUP_PATH=/tmp/tdengine_backup\n"); - fprintf(script, "SINCE_TIMESTAMP=%ld\n\n", start_time / 1000000000LL); - - // 使用taosdump备份增量数据 - fprintf(script, "echo \"使用taosdump备份增量数据...\"\n"); - fprintf(script, "taosdump -h $SOURCE_HOST -P $SOURCE_PORT \\\n"); - fprintf(script, " -D $DATABASE \\\n"); - fprintf(script, " -S $SINCE_TIMESTAMP \\\n"); - fprintf(script, " -o $BACKUP_PATH/incremental_$(date +%%Y%%m%%d_%%H%%M%%S)\n\n"); - - fprintf(script, "echo \"增量备份完成!\"\n"); - - fclose(script); - - // 设置执行权限 - chmod("/tmp/tdengine_backup/backup_script.sh", 0755); - - printf("生成备份脚本: /tmp/tdengine_backup/backup_script.sh\n"); - printf("[PASS] 备份脚本生成\n"); - } else { - printf("[FAIL] 无法生成备份脚本\n"); - } - - // 8. 测试备份大小估算 - printf("\n--- 测试备份大小估算 ---\n"); - - uint64_t estimated_size = backup_coordinator_estimate_backup_size(coordinator, start_wal, end_wal); - printf("估算备份大小: %lu 字节\n", estimated_size); - - assert(estimated_size > 0); - printf("[PASS] 备份大小估算\n"); - - // 9. 测试统计信息 - printf("\n--- 测试统计信息 ---\n"); - - SBackupStats stats; - int32_t result = backup_coordinator_get_stats(coordinator, &stats); - assert(result == 0); - - printf("总块数: %lu, 总大小: %lu 字节, 开始时间: %ld\n", - stats.total_blocks, stats.total_size, stats.start_time); - printf("[PASS] 统计信息获取\n"); - - // 10. 清理资源 - printf("\n--- 清理资源 ---\n"); - - backup_coordinator_destroy(coordinator); - event_interceptor_destroy(event_interceptor); - bitmap_engine_destroy(bitmap_engine); - - printf("[PASS] 资源清理\n"); - - printf("\n=== taosdump集成测试完成 ===\n"); -} - -// 测试taosdump命令模拟 -static void test_taosdump_command_simulation() { - printf("\n=== 测试taosdump命令模拟 ===\n"); - - // 模拟taosdump的增量备份命令 - printf("模拟taosdump增量备份命令:\n"); - printf("taosdump -h localhost -P 6030 -D test_db -S %ld -o /tmp/backup\n", time(NULL)); - - // 检查我们的插件是否能提供taosdump需要的信息 - printf("\n我们的插件提供的信息:\n"); - printf("1. 增量块检测: 通过位图引擎识别变更的数据块\n"); - printf("2. 时间范围查询: 支持taosdump的-S和-E参数\n"); - printf("3. 块元数据: 提供块大小、状态等信息\n"); - printf("4. 备份脚本生成: 自动生成taosdump兼容的备份命令\n"); - - printf("[PASS] taosdump命令模拟\n"); -} - -// 测试增量备份工作流 -static void test_incremental_backup_workflow() { - printf("\n=== 测试增量备份工作流 ===\n"); - - printf("完整的增量备份工作流:\n"); - printf("1. 启动位图引擎，监控数据变更\n"); - printf("2. 事件拦截器捕获块变更事件\n"); - printf("3. 备份协调器管理增量块信息\n"); - printf("4. 生成taosdump兼容的备份脚本\n"); - printf("5. 执行taosdump命令进行实际备份\n"); - printf("6. 验证备份完整性\n"); - - printf("\n与taosdump的协作点:\n"); - printf("- 时间范围: 插件提供精确的时间范围\n"); - printf("- 数据库选择: 插件识别变更的数据库\n"); - printf("- 增量检测: 插件只备份变更的数据\n"); - printf("- 性能优化: 避免全量备份，提升效率\n"); - - printf("[PASS] 增量备份工作流\n"); -} - -int main() { - printf("开始taosdump集成测试...\n"); - - test_taosdump_integration(); - test_taosdump_command_simulation(); - test_incremental_backup_workflow(); - - printf("\n所有测试完成！\n"); - printf("\n下一步操作:\n"); - printf("1. 安装taosdump: 参考上述安装方法\n"); - printf("2. 运行实际测试: 使用真实的TDengine实例\n"); - printf("3. 验证集成效果: 检查备份脚本和taosdump输出\n"); - - return 0; -} - diff --git a/plugins/incremental_bitmap/test/test_tmq_integration.c b/plugins/incremental_bitmap/test/test_tmq_integration.c deleted file mode 100644 index e3409b1b527d..000000000000 --- a/plugins/incremental_bitmap/test/test_tmq_integration.c +++ /dev/null @@ -1,327 +0,0 @@ -#include -#include -#include -#include -#ifndef USE_MOCK -#include -#endif -#include -#include - -#include "../include/storage_engine_interface.h" -#ifndef USE_MOCK -#include "../include/tdengine_storage_engine.h" -#endif - -// 测试事件回调函数 -static void test_event_callback(const SStorageEvent* event, void* user_data) { - printf("[TEST] 收到事件: 类型=%d, 块ID=%lu, WAL偏移量=%lu, 时间戳=%ld\n", - event->event_type, event->block_id, event->wal_offset, event->timestamp); - - // 更新事件计数（允许NULL，避免崩溃） - if (user_data) { - int* event_count = (int*)user_data; - (*event_count)++; - } -} - -// 测试 TMQ 配置设置 -static void test_tmq_config(void) { - printf("\n=== 测试 TMQ 配置设置 ===\n"); - - // 先初始化引擎，再设置配置 - SStorageEngineInterface* engine = get_storage_engine_interface("tdengine_tmq"); - assert(engine != NULL); - - SStorageEngineConfig cfg = { - .enable_interception = false, - .event_callback = test_event_callback, - .callback_user_data = NULL, - .event_buffer_size = 256, - .callback_threads = 1 - }; - int32_t init_ret = engine->init(&cfg); - assert(init_ret == 0); - - // 设置自定义配置（仅 TMQ 下编译） - int32_t result = 0; -#ifndef USE_MOCK - result = tdengine_set_tmq_config( - "192.168.1.100", // 自定义服务器IP - 6042, // 自定义端口 - "admin", // 自定义用户名 - "password123", // 自定义密码 - "testdb", // 自定义数据库 - "custom_topic", // 自定义主题 - "custom_group" // 自定义消费者组 - ); - - assert(result == 0); - printf("✓ TMQ 配置设置成功\n"); - - // 设置提交策略 - result = tdengine_set_commit_strategy( - false, // 手动提交 - true, // 至少一次 - 2000 // 2秒间隔 - ); - - assert(result == 0); - printf("✓ 提交策略设置成功\n"); -#else - printf("[TEST] 跳过 TMQ 配置设置（USE_MOCK=ON）\n"); -#endif - - // 清理本测试的初始化 - engine->destroy(); -} - -// 测试存储引擎生命周期 -static void test_storage_engine_lifecycle(void) { - printf("\n=== 测试存储引擎生命周期 ===\n"); - - // 获取自动选择的存储引擎接口（优先 TMQ，不可用时回退 mock） - SStorageEngineInterface* engine = get_storage_engine_interface("auto"); - assert(engine != NULL); - printf("✓ 获取存储引擎接口成功: %s\n", engine->get_engine_name()); - - // 检查是否支持 - bool supported = engine->is_supported(); - assert(supported == true); - printf("✓ 存储引擎支持检查通过\n"); - - const char* name = engine->get_engine_name(); - printf("✓ 引擎名称: %s\n", name); - - // 初始化存储引擎 - int event_count = 0; - SStorageEngineConfig config = { - .enable_interception = true, - .event_callback = test_event_callback, - .callback_user_data = &event_count, - .event_buffer_size = 1024, - .callback_threads = 2 - }; - - int32_t result = engine->init(&config); - assert(result == 0); - printf("✓ TMQ 存储引擎初始化成功\n"); - - // 安装事件拦截 - result = engine->install_interception(); - assert(result == 0); - printf("✓ 事件拦截安装成功\n"); - - // 在 TDengine 中准备数据库/主题/表并写入一条数据（仅在 TMQ 模式下执行） - if (strcmp(name, "tdengine_tmq") == 0) { -#ifndef USE_MOCK - TAOS* tconn = taos_connect("localhost", "root", "taosdata", NULL, 6030); - if (tconn) { - const char* stmts[] = { - "CREATE DATABASE IF NOT EXISTS test", \ - "USE test", \ - "CREATE STABLE IF NOT EXISTS meters (ts TIMESTAMP, v INT) TAGS (t INT)", \ - "CREATE TABLE IF NOT EXISTS d0 USING meters TAGS (1)", \ - "CREATE TOPIC IF NOT EXISTS incremental_backup_topic AS DATABASE test", \ - "INSERT INTO d0 VALUES (now, 1)" - }; - for (size_t i = 0; i < sizeof(stmts)/sizeof(stmts[0]); ++i) { - TAOS_RES* r = taos_query(tconn, stmts[i]); - taos_free_result(r); - } - taos_close(tconn); - } else { - printf("[TEST] 警告: 无法连接 TDengine，可能影响消息消费验证\n"); - } - (void)tconn; // 避免编译器在不同宏下的未使用警告 -#else - printf("[TEST] 跳过真实 TDengine 初始化（USE_MOCK=ON）\n"); -#endif - } - - // 等待一段时间让消费线程运行 - printf("等待 3 秒让消费线程运行...\n"); - sleep(3); - - // 在 mock 模式下手动触发一次事件，确保至少一次事件回调 - if (strcmp(name, "mock") == 0) { - SStorageEvent mock_event = { - .event_type = STORAGE_EVENT_BLOCK_UPDATE, - .block_id = 42, - .wal_offset = 0, - .timestamp = 0, - .user_data = NULL - }; - int32_t tr = engine->trigger_event(&mock_event); - assert(tr == 0); - } - - // 获取统计信息 - uint64_t events_processed, events_dropped; - result = engine->get_stats(&events_processed, &events_dropped); - assert(result == 0); - printf("✓ 统计信息获取成功: 处理=%lu, 丢弃=%lu\n", events_processed, events_dropped); - - // 获取详细统计信息（仅 TMQ 模式才有 tmq 专有统计） - if (strcmp(name, "tdengine_tmq") == 0) { -#ifndef USE_MOCK - uint64_t messages_consumed, offset_commits; - int64_t last_commit_time; - result = tdengine_get_detailed_stats(&events_processed, &events_dropped, - &messages_consumed, &offset_commits, &last_commit_time); - assert(result == 0); - printf("✓ 详细统计信息获取成功:\n"); - printf(" 事件处理: %lu\n", events_processed); - printf(" 事件丢弃: %lu\n", events_dropped); - printf(" 消息消费: %lu\n", messages_consumed); - printf(" Offset提交: %lu\n", offset_commits); - printf(" 最后提交时间: %ld\n", last_commit_time); -#else - printf("[TEST] 跳过 TMQ 详细统计（USE_MOCK=ON）\n"); -#endif - } - // 至少应有一次事件回调 - assert(event_count > 0); - - // 卸载事件拦截 - result = engine->uninstall_interception(); - assert(result == 0); - printf("✓ 事件拦截卸载成功\n"); - - // 销毁存储引擎 - engine->destroy(); - printf("✓ TMQ 存储引擎销毁成功\n"); -} - -// 测试事件触发 -static void test_event_trigger(void) { - printf("\n=== 测试事件触发 ===\n"); - - // 获取自动选择的存储引擎接口 - SStorageEngineInterface* engine = get_storage_engine_interface("auto"); - assert(engine != NULL); - - // 初始化 - int event_count = 0; - SStorageEngineConfig config = { - .enable_interception = true, - .event_callback = test_event_callback, - .callback_user_data = &event_count, - .event_buffer_size = 1024, - .callback_threads = 1 - }; - - int32_t result = engine->init(&config); - assert(result == 0); - - // 安装事件拦截 - result = engine->install_interception(); - assert(result == 0); - - // 手动触发事件 - SStorageEvent test_event = { - .event_type = STORAGE_EVENT_BLOCK_UPDATE, - .block_id = 12345, - .wal_offset = 67890, - .timestamp = 1234567890, - .user_data = NULL - }; - - result = engine->trigger_event(&test_event); - assert(result == 0); - printf("✓ 手动事件触发成功\n"); - - // 等待事件处理 - sleep(1); - - // 清理 - engine->uninstall_interception(); - engine->destroy(); -} - -// 测试错误处理 -static void test_error_handling(void) { - printf("\n=== 测试错误处理 ===\n"); - // 选择当前引擎 - SStorageEngineInterface* engine = get_storage_engine_interface("auto"); - assert(engine != NULL); - const char* cur = engine->get_engine_name(); - - // TMQ 专有错误路径：仅在 TMQ 模式下验证 - if (strcmp(cur, "tdengine_tmq") == 0) { -#ifndef USE_MOCK - int32_t result = tdengine_set_tmq_config("localhost", 6041, "user", "pass", "db", "topic", "group"); - assert(result == -1); - printf("✓ 未初始化时的配置设置正确拒绝\n"); - - result = tdengine_set_commit_strategy(false, true, 1000); - assert(result == -1); - printf("✓ 未初始化时的策略设置正确拒绝\n"); - - result = tdengine_get_detailed_stats(NULL, NULL, NULL, NULL, NULL); - assert(result == -1); - printf("✓ 未初始化时的统计获取正确拒绝\n"); - (void)result; -#else - printf("[TEST] 跳过 TMQ 专有未初始化错误测试（USE_MOCK=ON）\n"); -#endif - } else { - printf("[TEST] 跳过 TMQ 专有未初始化错误测试（当前为 mock）\n"); - } - - // 测试空指针参数 - int32_t result = engine->init(NULL); - assert(result == -1); // 应该失败 - printf("✓ 空配置参数正确拒绝\n"); - - // 测试未安装拦截时的操作 - SStorageEngineConfig config = { - .enable_interception = true, - .event_callback = test_event_callback, - .callback_user_data = NULL, - .event_buffer_size = 1024, - .callback_threads = 1 - }; - - result = engine->init(&config); - assert(result == 0); - - SStorageEvent test_event = { - .event_type = STORAGE_EVENT_BLOCK_UPDATE, - .block_id = 1, - .wal_offset = 1, - .timestamp = 1, - .user_data = NULL - }; - - result = engine->trigger_event(&test_event); - assert(result == -1); // 应该失败，因为拦截未安装 - printf("✓ 未安装拦截时的事件触发正确拒绝\n"); - - // 清理 - engine->destroy(); -} - -int main(void) { - printf("开始 TMQ 集成测试...\n"); - - // 注册 TMQ 与 Mock 存储引擎，auto 将按可用性选择 - int32_t result = 0; -#ifndef USE_MOCK - result = register_tdengine_storage_engine(); - assert(result == 0); -#endif - extern int32_t register_mock_storage_engine(void); - result = register_mock_storage_engine(); - assert(result == 0); - printf("✓ 存储引擎注册成功（TMQ + Mock）\n"); - - // 运行测试 - test_tmq_config(); - test_storage_engine_lifecycle(); - test_event_trigger(); - test_error_handling(); - - printf("\n🎉 所有 TMQ 集成测试通过！\n"); - return 0; -}