From 5348aa074e11496ff8f8e5055056b4821a81c62e Mon Sep 17 00:00:00 2001 From: chenBright Date: Sun, 23 Nov 2025 11:28:41 +0800 Subject: [PATCH] Fix compilation error with protobuf 24 --- .../compile-with-bazel-protobuf/action.yml | 14 + .../change_bzlmod_protobuf_version.sh | 81 ++++++ .github/workflows/ci-linux.yml | 251 +++++------------- .github/workflows/ci-macos.yml | 66 ----- MODULE.bazel | 5 +- src/json2pb/json_to_pb.cpp | 2 +- src/json2pb/pb_to_json.cpp | 2 +- 7 files changed, 163 insertions(+), 258 deletions(-) create mode 100644 .github/actions/compile-with-bazel-protobuf/action.yml create mode 100644 .github/actions/compile-with-bazel-protobuf/change_bzlmod_protobuf_version.sh delete mode 100644 .github/workflows/ci-macos.yml diff --git a/.github/actions/compile-with-bazel-protobuf/action.yml b/.github/actions/compile-with-bazel-protobuf/action.yml new file mode 100644 index 0000000000..bb2e9fdb55 --- /dev/null +++ b/.github/actions/compile-with-bazel-protobuf/action.yml @@ -0,0 +1,14 @@ +inputs: + protobuf-version: + description: version of protobuf + required: true + env: + description: action environment + required: false +runs: + using: "composite" + steps: + - run: | + bash ./.github/actions/compile-with-bazel-protobuf/change_bzlmod_protobuf_version.sh ${{inputs.protobuf-version}} + bazel build --verbose_failures --action_env="${{inputs.env}}" -- //... -//example/... + shell: bash diff --git a/.github/actions/compile-with-bazel-protobuf/change_bzlmod_protobuf_version.sh b/.github/actions/compile-with-bazel-protobuf/change_bzlmod_protobuf_version.sh new file mode 100644 index 0000000000..efcf9a9738 --- /dev/null +++ b/.github/actions/compile-with-bazel-protobuf/change_bzlmod_protobuf_version.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +if [ $# -ne 1 ]; then + echo 'Usage: $0 ' + echo 'Example: $0 21.7' + exit 1 +fi + +NEW_VERSION="$1" +MODULE_FILE='MODULE.bazel' + +if [ ! -f "$MODULE_FILE" ]; then + echo "Error: $MODULE_FILE not found" + exit 1 +fi + +TEMP_FILE=$(mktemp) + +# Scan file to find relevant lines. +bazel_dep_line_num=0 +override_exists=false +current_line_num=0 + +echo "Scanning $MODULE_FILE..." + +while read -r line; do + current_line_num=$((current_line_num + 1)) + + # Skip comment lines. + if [[ "$line" =~ ^[[:space:]]*# ]]; then + continue + fi + + # Check if this is protobuf bazel_dep line. + if [[ "$line" =~ bazel_dep.*name[[:space:]]*=[[:space:]]*[\'\"]*protobuf[\'\"]*.*version ]]; then + bazel_dep_line_num=$current_line_num + echo "Found bazel_dep for protobuf at line $current_line_num" + fi + + # Check if protobuf single_version_override exists. + if [[ "$line" =~ single_version_override.*module_name[[:space:]]*=[[:space:]]*[\'\"]*protobuf[\'\"]*.*version ]]; then + override_exists=true + echo "Found existing single_version_override for protobuf at line $current_line_num" + fi +done < "$MODULE_FILE" + +# If bazel_dep protobuf line not found, returns error. +if [ "$bazel_dep_line_num" -eq 0 ]; then + echo 'Error: bazel_dep for protobuf not found in MODULE.bazel' + exit 1 +fi + +# Process file. +echo "Processing $MODULE_FILE..." +current_line_num=0 +while read -r line; do + current_line_num=$((current_line_num + 1)) + + # If this is single_version_override protobuf line and not a comment, replace version + if [[ "$line" =~ single_version_override.*module_name[[:space:]]*=[[:space:]]*[\'\"]*protobuf[\'\"]*.*version ]] && ! [[ "$line" =~ ^[[:space:]]*# ]]; then + # Replace version using single quote format uniformly + new_line=$(echo "$line" | sed "s/version[[:space:]]*=[[:space:]]*['\"][^'\"]*['\"]/version = '$NEW_VERSION'/") + echo "$new_line" + else + echo "$line" + # If no override exists and current line is bazel_dep protobuf, add single_version_override after it + if [ "$override_exists" = false ] && [ "$current_line_num" -eq "$bazel_dep_line_num" ]; then + echo "single_version_override(module_name = 'protobuf', version = '$NEW_VERSION')" + fi + fi +done < "$MODULE_FILE" > "$TEMP_FILE" + +# Replace original file directly +mv "$TEMP_FILE" "$MODULE_FILE" + +echo "Successfully updated protobuf version to $NEW_VERSION in $MODULE_FILE" + +cat "$MODULE_FILE" + +# Clean up temporary file +trap "rm -f $TEMP_FILE" EXIT diff --git a/.github/workflows/ci-linux.yml b/.github/workflows/ci-linux.yml index f2d6d69287..72ad05608b 100644 --- a/.github/workflows/ci-linux.yml +++ b/.github/workflows/ci-linux.yml @@ -15,222 +15,95 @@ env: # https://github.com/actions/runner-images jobs: - compile-with-make: + gcc-compile-with-bazel: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 - - uses: ./.github/actions/install-all-dependencies - - - name: gcc with default options - uses: ./.github/actions/compile-with-make + - name: protobuf 23.1 + uses: ./.github/actions/compile-with-bazel-protobuf with: - options: --headers=/usr/include --libs=/usr/lib /usr/lib64 --cc=gcc --cxx=g++ --werror + protobuf-version: 23.1 + env: CC=gcc - - name: gcc with all options - uses: ./.github/actions/compile-with-make + - name: protobuf 24.4 + uses: ./.github/actions/compile-with-bazel-protobuf with: - options: --headers=/usr/include --libs=/usr/lib /usr/lib64 --cc=gcc --cxx=g++ --werror --with-thrift --with-glog --with-rdma --with-debug-bthread-sche-safety --with-debug-lock --with-bthread-tracer --with-asan + protobuf-version: 24.4 + env: CC=gcc - - name: clang with default options - uses: ./.github/actions/compile-with-make + - name: protobuf 25.5 + uses: ./.github/actions/compile-with-bazel-protobuf with: - options: --headers=/usr/include --libs=/usr/lib /usr/lib64 --cc=clang --cxx=clang++ --werror + protobuf-version: 25.5 + env: CC=gcc - - name: clang with all options - uses: ./.github/actions/compile-with-make + - name: protobuf 27.5 + uses: ./.github/actions/compile-with-bazel-protobuf with: - options: --headers=/usr/include --libs=/usr/lib /usr/lib64 --cc=clang --cxx=clang++ --werror --with-thrift --with-glog --with-rdma --with-debug-bthread-sche-safety --with-debug-lock --with-bthread-tracer --with-asan - - compile-with-cmake: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/install-all-dependencies - - - name: gcc with default options - run: | - export CC=gcc && export CXX=g++ - mkdir gcc_build && cd gcc_build && cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 .. - make -j ${{env.proc_num}} && make clean - - - name: gcc with all options - run: | - export CC=gcc && export CXX=g++ - mkdir gcc_build_all && cd gcc_build_all - cmake -DWITH_MESALINK=OFF -DWITH_GLOG=ON -DWITH_THRIFT=ON -DWITH_RDMA=ON -DWITH_DEBUG_BTHREAD_SCHE_SAFETY=ON -DWITH_DEBUG_LOCK=ON -DWITH_BTHREAD_TRACER=ON -DWITH_ASAN=ON -DCMAKE_POLICY_VERSION_MINIMUM=3.5 .. - make -j ${{env.proc_num}} && make clean - - - name: clang with default options - run: | - export CC=clang && export CXX=clang++ - mkdir clang_build && cd clang_build && cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 .. - make -j ${{env.proc_num}} && make clean - - - name: clang with all options - run: | - export CC=clang && export CXX=clang++ - mkdir clang_build_all && cd clang_build_all - cmake -DWITH_MESALINK=OFF -DWITH_GLOG=ON -DWITH_THRIFT=ON -DWITH_RDMA=ON -DWITH_DEBUG_BTHREAD_SCHE_SAFETY=ON -DWITH_DEBUG_LOCK=ON -DWITH_BTHREAD_TRACER=ON -DWITH_ASAN=ON -DCMAKE_POLICY_VERSION_MINIMUM=3.5 .. - make -j ${{env.proc_num}} && make clean - - gcc-compile-with-make-protobuf: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/install-essential-dependencies + protobuf-version: 27.5 + env: CC=gcc - - name: protobuf 3.5.1 - uses: ./.github/actions/compile-with-make-protobuf + - name: protobuf 28.3 + uses: ./.github/actions/compile-with-bazel-protobuf with: - protobuf-version: 3.5.1 - protobuf-cpp-version: 3.5.1 - protobuf-install-dir: /protobuf-3.5.1 - config-brpc-options: --cc=gcc --cxx=g++ --werror + protobuf-version: 28.3 + env: CC=gcc - - name: protobuf 3.12.4 - uses: ./.github/actions/compile-with-make-protobuf + - name: protobuf 29.5 + uses: ./.github/actions/compile-with-bazel-protobuf with: - protobuf-version: 3.12.4 - protobuf-cpp-version: 3.12.4 - protobuf-install-dir: /protobuf-3.12.4 - config-brpc-options: --cc=gcc --cxx=g++ --werror + protobuf-version: 29.5 + env: CC=gcc - - name: protobuf 21.12 - uses: ./.github/actions/compile-with-make-protobuf + - name: protobuf 30.2 + uses: ./.github/actions/compile-with-bazel-protobuf with: - protobuf-version: 21.12 - protobuf-cpp-version: 3.21.12 - protobuf-install-dir: /protobuf-3.21.12 - config-brpc-options: --cc=gcc --cxx=g++ --werror + protobuf-version: 30.2 + env: CC=gcc - gcc-compile-with-bazel: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v2 - - run: bazel build --verbose_failures -- //... -//example/... - - gcc-compile-with-boringssl: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v2 - - run: bazel build --verbose_failures --define with_mesalink=false --define with_glog=true --define with_thrift=true --define BRPC_WITH_BORINGSSL=true -- //... -//example/... - - gcc-compile-with-bazel-all-options: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v2 - - run: | - bazel build --verbose_failures \ - --define with_mesalink=false \ - --define with_glog=true \ - --define with_thrift=true \ - --define with_debug_bthread_sche_safety=true \ - --define with_debug_lock=true \ - --define with_asan=true \ - --define with_bthread_tracer=true \ - --define BRPC_WITH_NO_PTHREAD_MUTEX_HOOK=true \ - -- //... -//example/... - - clang-compile-with-make-protobuf: + clang-compile-with-bazel: runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 - - uses: ./.github/actions/install-essential-dependencies - - - name: protobuf 3.5.1 - uses: ./.github/actions/compile-with-make-protobuf + - name: protobuf 23.1 + uses: ./.github/actions/compile-with-bazel-protobuf with: - protobuf-version: 3.5.1 - protobuf-cpp-version: 3.5.1 - protobuf-install-dir: /protobuf-3.5.1 - config-brpc-options: --cc=clang --cxx=clang++ --werror + protobuf-version: 23.1 + env: CC=clang - - name: protobuf 3.12.4 - uses: ./.github/actions/compile-with-make-protobuf + - name: protobuf 24.4 + uses: ./.github/actions/compile-with-bazel-protobuf with: - protobuf-version: 3.12.4 - protobuf-cpp-version: 3.12.4 - protobuf-install-dir: /protobuf-3.12.4 - config-brpc-options: --cc=clang --cxx=clang++ --werror + protobuf-version: 24.4 + env: CC=clang - - name: protobuf 21.12 - uses: ./.github/actions/compile-with-make-protobuf + - name: protobuf 25.5 + uses: ./.github/actions/compile-with-bazel-protobuf with: - protobuf-version: 21.12 - protobuf-cpp-version: 3.21.12 - protobuf-install-dir: /protobuf-3.21.12 - config-brpc-options: --cc=clang --cxx=clang++ --werror + protobuf-version: 25.5 + env: CC=clang - clang-compile-with-bazel: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v2 - - run: bazel build --verbose_failures --action_env=CC=clang -- //... -//example/... + - name: protobuf 27.5 + uses: ./.github/actions/compile-with-bazel-protobuf + with: + protobuf-version: 27.5 + env: CC=clang - clang-compile-with-boringssl: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v2 - - run: bazel build --verbose_failures --action_env=CC=clang --define with_mesalink=false --define with_glog=true --define with_thrift=true --define BRPC_WITH_BORINGSSL=true -- //... -//example/... + - name: protobuf 28.3 + uses: ./.github/actions/compile-with-bazel-protobuf + with: + protobuf-version: 28.3 + env: CC=clang - clang-compile-with-bazel-all-options: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v2 - - run: | - bazel build --verbose_failures \ - --action_env=CC=clang \ - --define with_mesalink=false \ - --define with_glog=true \ - --define with_thrift=true \ - --define with_debug_bthread_sche_safety=true \ - --define with_debug_lock=true \ - --define with_asan=true \ - --define with_bthread_tracer=true \ - --define BRPC_WITH_NO_PTHREAD_MUTEX_HOOK=true \ - -- //... -//example/... - - clang-unittest: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/install-essential-dependencies - - uses: ./.github/actions/init-ut-make-config + - name: protobuf 29.5 + uses: ./.github/actions/compile-with-bazel-protobuf with: - options: --with-bthread-tracer - - name: compile tests - run: | - cat config.mk - cd test - make -j ${{env.proc_num}} - - name: run tests - run: | - cd test - sh ./run_tests.sh - - clang-unittest-asan: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v2 - - uses: ./.github/actions/install-essential-dependencies - - uses: ./.github/actions/init-ut-make-config + protobuf-version: 29.5 + env: CC=clang + + - name: protobuf 30.2 + uses: ./.github/actions/compile-with-bazel-protobuf with: - options: --with-bthread-tracer --with-asan - - name: compile tests - run: | - cat config.mk - cd test - make NEED_GPERFTOOLS=0 -j ${{env.proc_num}} - - name: run tests - run: | - cd test - sh ./run_tests.sh - - bazel-bvar-unittest: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v2 - - run: bazel test --verbose_failures //test:bvar_test - - run: bazel test --verbose_failures --define with_babylon_counter=true //test:bvar_test - - run: bazel test --verbose_failures --action_env=CC=clang //test:bvar_test - - run: bazel test --verbose_failures --action_env=CC=clang --define with_babylon_counter=true //test:bvar_test + protobuf-version: 30.2 + env: CC=clang + diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml deleted file mode 100644 index 61d45ac821..0000000000 --- a/.github/workflows/ci-macos.yml +++ /dev/null @@ -1,66 +0,0 @@ -name: Build on Macos - -on: - push: - branches: [ master ] - paths-ignore: - - '**.md' - pull_request: - branches: [ master ] - paths-ignore: - - '**.md' - -env: - proc_num: $(sysctl -n hw.logicalcpu) - -jobs: - compile-with-make-cmake-protobuf21: - runs-on: macos-latest # https://github.com/actions/runner-images - - steps: - - uses: actions/checkout@v2 - - - name: install dependences - run: | - brew install openssl gnu-getopt coreutils gflags leveldb protobuf@21 - - - name: compile with make - run: | - GETOPT_PATH=$(brew --prefix gnu-getopt)/bin - export PATH=$GETOPT_PATH:$PATH - ./config_brpc.sh --header="$(brew --prefix)/include" --libs="$(brew --prefix)/lib" - make -j ${{env.proc_num}} && make clean - - - name: compile with cmake - run: | - echo "CMAKE_PREFIX_PATH=$(brew --prefix protobuf@21)" - mkdir build && cd build && cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_PREFIX_PATH=$(brew --prefix protobuf@21) .. - make -j ${{env.proc_num}} && make clean - - compile-with-make-cmake-protobuf29: - runs-on: macos-latest # https://github.com/actions/runner-images - - steps: - - uses: actions/checkout@v2 - - - name: install dependences - run: | - brew install openssl gnu-getopt coreutils gflags leveldb protobuf@29 - - - name: compile with make - run: | - GETOPT_PATH=$(brew --prefix gnu-getopt)/bin - export PATH=$GETOPT_PATH:$PATH - ./config_brpc.sh --header="$(brew --prefix)/include" --libs="$(brew --prefix)/lib" - make -j ${{env.proc_num}} && make clean - - - name: compile with cmake - run: | - mkdir build && cd build && cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_PREFIX_PATH=$(brew --prefix protobuf@29) .. - make -j ${{env.proc_num}} && make clean - - compile-with-bazel: - runs-on: macos-latest # https://github.com/actions/runner-images - steps: - - uses: actions/checkout@v2 - - run: bazel build --verbose_failures -- //:brpc -//example/... diff --git a/MODULE.bazel b/MODULE.bazel index 19700bc411..5f45fec6c1 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -8,7 +8,7 @@ module( bazel_dep(name = 'abseil-cpp', version = '20210324.2', repo_name = 'com_google_absl') bazel_dep(name = 'bazel_skylib', version = '1.0.3') bazel_dep(name = 'boringssl', version = '0.0.0-20211025-d4f1ab9') -bazel_dep(name = 'protobuf', version = '27.3', repo_name = 'com_google_protobuf') +bazel_dep(name = 'protobuf', version = '27.2', repo_name = 'com_google_protobuf') bazel_dep(name = 'gflags', version = '2.2.2', repo_name = 'com_github_gflags_gflags') bazel_dep(name = 'glog', version = '0.5.0', repo_name = 'com_github_google_glog') bazel_dep(name = 'platforms', version = '0.0.4') @@ -19,6 +19,9 @@ bazel_dep(name = 'zlib', version = '1.3.1.bcr.5', repo_name = 'com_github_madler bazel_dep(name = 'libunwind', version = '1.8.1', repo_name = 'com_github_libunwind_libunwind') bazel_dep(name = 'babylon', version = '1.4.4') +single_version_override(module_name = 'rules_cc', version = '0.0.9') +single_version_override(module_name = 'rules_proto', version = '4.0.0') + # --registry=https://baidu.github.io/babylon/registry bazel_dep(name = 'leveldb', version = '1.23', repo_name = 'com_github_google_leveldb') single_version_override( diff --git a/src/json2pb/json_to_pb.cpp b/src/json2pb/json_to_pb.cpp index e758bdb3ab..866cb25700 100644 --- a/src/json2pb/json_to_pb.cpp +++ b/src/json2pb/json_to_pb.cpp @@ -553,7 +553,7 @@ bool JsonValueToProtoMessage(const BUTIL_RAPIDJSON_NAMESPACE::Value& json_value, for (int i = 0; i < descriptor->extension_range_count(); ++i) { const google::protobuf::Descriptor::ExtensionRange* ext_range = descriptor->extension_range(i); -#if GOOGLE_PROTOBUF_VERSION < 4025000 +#if GOOGLE_PROTOBUF_VERSION < 4024000 for (int tag_number = ext_range->start; tag_number < ext_range->end; ++tag_number) #else for (int tag_number = ext_range->start_number(); tag_number < ext_range->end_number(); ++tag_number) diff --git a/src/json2pb/pb_to_json.cpp b/src/json2pb/pb_to_json.cpp index f232226785..d211caf633 100644 --- a/src/json2pb/pb_to_json.cpp +++ b/src/json2pb/pb_to_json.cpp @@ -134,7 +134,7 @@ bool PbToJsonConverter::Convert(const google::protobuf::Message& message, Handle for (int i = 0; i < ext_range_count; ++i) { const google::protobuf::Descriptor::ExtensionRange* ext_range = descriptor->extension_range(i); -#if GOOGLE_PROTOBUF_VERSION < 4025000 +#if GOOGLE_PROTOBUF_VERSION < 4024000 for (int tag_number = ext_range->start; tag_number < ext_range->end; ++tag_number) #else for (int tag_number = ext_range->start_number(); tag_number < ext_range->end_number(); ++tag_number)