From 15ae7a8b116d3c8620d5048562b04a50de38ac79 Mon Sep 17 00:00:00 2001 From: Gagandeep Singh Date: Tue, 17 Feb 2026 13:39:27 +0530 Subject: [PATCH 1/7] Upgrade protobuf from 3.21.9 to 4.25.6 and gRPC from 1.46.3 to 1.62.1 Updates core dependencies to support newer features and bug fixes. --- WORKSPACE | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 683cc7e7f..b630f7924 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -103,9 +103,11 @@ http_archive( http_archive( name = "com_google_protobuf", - sha256 = "1add10f9bd92775b91f326da259f243881e904dd509367d5031d4c782ba82810", - strip_prefix = "protobuf-3.21.9", - urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.21.9.tar.gz"], + sha256 = "4e6727bc5d23177edefa3ad86fd2f5a92cd324151636212fd1f7f13aef3fd2b7", + strip_prefix = "protobuf-4.25.6", + urls = [ + "https://github.com/protocolbuffers/protobuf/archive/v4.25.6.tar.gz", + ], ) load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") @@ -176,9 +178,9 @@ http_archive( http_archive( name = "com_github_grpc_grpc", - urls = ["https://github.com/grpc/grpc/archive/v1.46.3.tar.gz"], - sha256 = "d6cbf22cb5007af71b61c6be316a79397469c58c82a942552a62e708bce60964", - strip_prefix = "grpc-1.46.3", + urls = ["https://github.com/grpc/grpc/archive/v1.62.1.tar.gz"], + sha256 = "c9f9ae6e4d6f40464ee9958be4068087881ed6aa37e30d0e64d40ed7be39dd01", + strip_prefix = "grpc-1.62.1", ) load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps") From 7e380b1c8ad04e8003b23e1b1ddff67440558c21 Mon Sep 17 00:00:00 2001 From: Gagandeep Singh Date: Tue, 17 Feb 2026 13:40:44 +0530 Subject: [PATCH 2/7] build: disable bzlmod for protobuf v4.25.6 compatibility Adds --noenable_bzlmod flag to work around protobuf v4.25.6 bzlmod issues. --- .bazelrc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.bazelrc b/.bazelrc index 0088def11..1b8dec2f9 100644 --- a/.bazelrc +++ b/.bazelrc @@ -14,6 +14,10 @@ # This macro is needed in order for mlmd to build with ZetaSQL which can only # be compiled upon c++17 or higher. +# Disable bzlmod for protobuf v4.25.6 compatibility +# https://github.com/protocolbuffers/protobuf/issues/14313 +common --noenable_bzlmod + build --cxxopt="-std=c++17" build --host_cxxopt="-std=c++17" From 91b70142c1839e09b62e9949db27688aab13ee83 Mon Sep 17 00:00:00 2001 From: Gagandeep Singh Date: Tue, 17 Feb 2026 13:41:28 +0530 Subject: [PATCH 3/7] build: implement custom py_proto_library for protobuf v4 compatibility - Add ProtoDescriptorInfo provider for proto descriptor files - Implement custom _py_proto_library_impl to handle proto compilation - Add proto_descriptor helper rule - Support gRPC plugin for service generation - Handle _virtual_imports paths from dependencies This replaces the deprecated protobuf.bzl rules with custom implementations compatible with protobuf v4. --- ml_metadata/ml_metadata.bzl | 210 ++++++++++++++++++++++++++++++++++-- 1 file changed, 201 insertions(+), 9 deletions(-) diff --git a/ml_metadata/ml_metadata.bzl b/ml_metadata/ml_metadata.bzl index a22aff948..e8f9ad977 100644 --- a/ml_metadata/ml_metadata.bzl +++ b/ml_metadata/ml_metadata.bzl @@ -17,7 +17,198 @@ This module contains build rules for ml_metadata in OSS. load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") -load("@com_google_protobuf//:protobuf.bzl", "cc_proto_library", "py_proto_library") + +# Custom provider for descriptor proto files +ProtoDescriptorInfo = provider( + fields = { + "direct_sources": "Direct proto source files", + "transitive_sources": "Transitive proto source files", + } +) + +# Helper rule to make proto files available without compilation +def _proto_descriptor_impl(ctx): + proto_sources = depset(direct = ctx.files.srcs) + return [ + DefaultInfo(files = proto_sources), + ProtoDescriptorInfo( + direct_sources = ctx.files.srcs, + transitive_sources = proto_sources, + ) + ] + +proto_descriptor = rule( + implementation = _proto_descriptor_impl, + attrs = { + "srcs": attr.label_list(allow_files = [".proto"]), + }, +) + +def _py_proto_library_impl(ctx): + proto_deps = ctx.attr.deps + use_grpc = ctx.attr.use_grpc_plugin + + all_sources = [] + py_infos = [] + all_proto_infos = [] + + for dep in proto_deps: + if ProtoInfo in dep: + all_sources.extend(dep[ProtoInfo].direct_sources) + all_proto_infos.append(dep[ProtoInfo]) + elif ProtoDescriptorInfo in dep: + # Handle proto_descriptor custom provider + all_sources.extend(dep[ProtoDescriptorInfo].direct_sources) + elif PyInfo in dep: + py_infos.append(dep[PyInfo]) + + workspace_sources = [] + for src in all_sources: + if not src.short_path.startswith("external/") and not src.short_path.startswith("../"): + workspace_sources.append(src) + + py_outputs = [] + for proto_src in workspace_sources: + basename = proto_src.basename[:-6] + py_outputs.append(ctx.actions.declare_file(basename + "_pb2.py")) + # Add grpc output file if grpc plugin is enabled + if use_grpc: + py_outputs.append(ctx.actions.declare_file(basename + "_pb2_grpc.py")) + + if py_outputs: + proto_path_args = ["--proto_path=."] + proto_paths = {".": True} + + for ws in workspace_sources: + ws_dir = "/".join(ws.short_path.split("/")[:-1]) + if ws_dir and ws_dir not in proto_paths: + proto_paths[ws_dir] = True + proto_path_args.append("--proto_path=" + ws_dir) + + # Extract proto paths from dependencies + for proto_info in all_proto_infos: + # Extract _virtual_imports paths from direct and transitive sources + for src in proto_info.direct_sources: + src_path = src.path + if "_virtual_imports" in src_path: + # Extract path up to and including _virtual_imports/XXX + parts = src_path.split("/_virtual_imports/") + if len(parts) == 2: + virtual_import_path = parts[0] + "/_virtual_imports/" + parts[1].split("/")[0] + if virtual_import_path not in proto_paths: + proto_paths[virtual_import_path] = True + proto_path_args.append("--proto_path=" + virtual_import_path) + + # Also process transitive sources + for src in proto_info.transitive_sources.to_list(): + src_path = src.path + if "_virtual_imports" in src_path: + # Extract path up to and including _virtual_imports/XXX + parts = src_path.split("/_virtual_imports/") + if len(parts) == 2: + virtual_import_path = parts[0] + "/_virtual_imports/" + parts[1].split("/")[0] + if virtual_import_path not in proto_paths: + proto_paths[virtual_import_path] = True + proto_path_args.append("--proto_path=" + virtual_import_path) + + proto_file_args = [src.short_path for src in workspace_sources] + + # Build protoc arguments + protoc_args = ["--python_out=" + ctx.bin_dir.path] + + # Add grpc plugin if enabled + tools = [] + if use_grpc and ctx.executable._grpc_plugin: + protoc_args.append("--grpc_python_out=" + ctx.bin_dir.path) + protoc_args.append("--plugin=protoc-gen-grpc_python=" + ctx.executable._grpc_plugin.path) + tools.append(ctx.executable._grpc_plugin) + + ctx.actions.run( + inputs = depset( + direct = workspace_sources, + transitive = [ + dep[ProtoInfo].transitive_sources + for dep in proto_deps + if ProtoInfo in dep + ] + ), + outputs = py_outputs, + executable = ctx.executable._protoc, + arguments = protoc_args + proto_path_args + proto_file_args, + tools = tools, + mnemonic = "ProtocPython", + ) + + all_transitive_sources = [depset(py_outputs)] + all_imports = [depset([ctx.bin_dir.path])] if py_outputs else [] + + for py_info in py_infos: + all_transitive_sources.append(py_info.transitive_sources) + if hasattr(py_info, "imports"): + all_imports.append(py_info.imports) + + return [ + DefaultInfo(files = depset(py_outputs)), + PyInfo( + transitive_sources = depset(transitive = all_transitive_sources), + imports = depset(transitive = all_imports), + has_py2_only_sources = False, + has_py3_only_sources = True, + ), + ] + +_py_proto_library_rule = rule( + implementation = _py_proto_library_impl, + attrs = { + "deps": attr.label_list( + providers = [[ProtoInfo], [PyInfo]], + ), + "use_grpc_plugin": attr.bool( + default = False, + doc = "Whether to use the gRPC plugin to generate service stubs", + ), + "_protoc": attr.label( + default = "@com_google_protobuf//:protoc", + executable = True, + cfg = "exec", + ), + "_grpc_plugin": attr.label( + default = "@com_github_grpc_grpc//src/compiler:grpc_python_plugin", + executable = True, + cfg = "exec", + ), + }, + provides = [PyInfo], +) + +# Wrapper for cc_proto_library to maintain compatibility with Protobuf 4.x. +def cc_proto_library( + name, + srcs = [], + deps = [], + cc_libs = [], + protoc = None, + default_runtime = None, + use_grpc_plugin = None, + testonly = 0, + visibility = None, + **kwargs): + _ignore = [cc_libs, protoc, default_runtime, use_grpc_plugin, kwargs] + + native.proto_library( + name = name + "_proto", + srcs = srcs, + deps = [d + "_proto" if not d.startswith("@") else d for d in deps], + testonly = testonly, + visibility = visibility, + ) + + native.cc_proto_library( + name = name, + deps = [":" + name + "_proto"], + testonly = testonly, + visibility = visibility, + ) def ml_metadata_cc_test( name, @@ -87,17 +278,18 @@ def ml_metadata_proto_library_py( oss_deps = [], use_grpc_plugin = False): """Opensource py_proto_library.""" - _ignore = [proto_library, api_version, oss_deps] - py_proto_library( + _ignore = [api_version, srcs] + if not proto_library: + fail("proto_library parameter is required for ml_metadata_proto_library_py") + + actual_proto_library = ":" + proto_library + "_proto" + + _py_proto_library_rule( name = name, - srcs = srcs, - srcs_version = "PY2AND3", - deps = ["@com_google_protobuf//:well_known_types_py_pb2"] + deps + oss_deps, - default_runtime = "@com_google_protobuf//:protobuf_python", - protoc = "@com_google_protobuf//:protoc", + deps = [actual_proto_library] + deps + oss_deps, + use_grpc_plugin = use_grpc_plugin, visibility = visibility, testonly = testonly, - use_grpc_plugin = use_grpc_plugin, ) def ml_metadata_proto_library_go( From e5a12ab5198fcab2cbb642af70f2cd9e8ab359d1 Mon Sep 17 00:00:00 2001 From: Gagandeep Singh Date: Tue, 17 Feb 2026 13:41:52 +0530 Subject: [PATCH 4/7] build: update proto file references to use _py_pb2 suffix Changes proto library references from _pb2.py to _py_pb2 format to align with protobuf v4 naming conventions. --- ml_metadata/BUILD | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ml_metadata/BUILD b/ml_metadata/BUILD index c3ede2f05..4406fb8b5 100644 --- a/ml_metadata/BUILD +++ b/ml_metadata/BUILD @@ -29,10 +29,9 @@ selects.config_setting_group( ) _public_protos = [ - "//ml_metadata/simple_types/proto:simple_types_pb2.py", - "//ml_metadata/proto:metadata_store_pb2.py", - "//ml_metadata/proto:metadata_store_service_pb2.py", - "//ml_metadata/proto:metadata_store_service_pb2_grpc.py", + "//ml_metadata/simple_types/proto:simple_types_py_pb2", + "//ml_metadata/proto:metadata_store_py_pb2", + "//ml_metadata/proto:metadata_store_service_py_pb2", ] _py_extension = select({ From 062761643ee6633a49cd871b2d4cf7dde80230e8 Mon Sep 17 00:00:00 2001 From: Gagandeep Singh Date: Tue, 17 Feb 2026 13:42:11 +0530 Subject: [PATCH 5/7] build: add CMAKE_POLICY_VERSION_MINIMUM=3.5 to MySQL client build Ensures CMake compatibility by explicitly setting minimum policy version. --- ml_metadata/libmysqlclient.BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ml_metadata/libmysqlclient.BUILD b/ml_metadata/libmysqlclient.BUILD index 73cc65646..6092dab06 100644 --- a/ml_metadata/libmysqlclient.BUILD +++ b/ml_metadata/libmysqlclient.BUILD @@ -26,7 +26,7 @@ genrule( "cd $$TMP_DIR", "mkdir build", "cd build", - "cmake .. -DCMAKE_BUILD_TYPE=Release $${CMAKE_ICONV_FLAG-}", + "cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_POLICY_VERSION_MINIMUM=3.5 $${CMAKE_ICONV_FLAG-}", "cd ..", "cp -R ./build/* $$INSTALL_DIR", "rm -rf $$TMP_DIR", From 4a62d91b8d5bcd4ee8a12c3be430d97d4ac40633 Mon Sep 17 00:00:00 2001 From: Gagandeep Singh Date: Tue, 17 Feb 2026 13:45:19 +0530 Subject: [PATCH 6/7] build: update proto dependencies for protobuf v4 compatibility - Replace cc_wkt_protos with explicit proto targets (any_proto, struct_proto, descriptor_proto, field_mask_proto) - Add missing protobuf dependency to record_parsing_utils - Bump Bazel version from 6.1.0 to 6.5.0 in Dockerfile These changes ensure compatibility with protobuf v4.25.6's new dependency structure. --- ml_metadata/proto/BUILD | 13 ++++++++++--- ml_metadata/tools/docker_server/Dockerfile | 2 +- ml_metadata/util/BUILD | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/ml_metadata/proto/BUILD b/ml_metadata/proto/BUILD index 6fc019a41..7a9007072 100644 --- a/ml_metadata/proto/BUILD +++ b/ml_metadata/proto/BUILD @@ -21,7 +21,11 @@ licenses(["notice"]) ml_metadata_proto_library( name = "metadata_store_proto", srcs = ["metadata_store.proto"], - deps = ["@com_google_protobuf//:cc_wkt_protos"], + deps = [ + "@com_google_protobuf//:any_proto", + "@com_google_protobuf//:struct_proto", + "@com_google_protobuf//:descriptor_proto", + ], ) ml_metadata_proto_library( @@ -29,7 +33,10 @@ ml_metadata_proto_library( srcs = ["metadata_store_service.proto"], has_services = 1, cc_grpc_version = 1, - deps = [":metadata_store_proto"], + deps = [ + ":metadata_store_proto", + "@com_google_protobuf//:field_mask_proto", + ], ) ml_metadata_proto_library_py( @@ -69,6 +76,6 @@ ml_metadata_proto_library( name = "metadata_source_proto", srcs = ["metadata_source.proto"], deps = [ - "@com_google_protobuf//:cc_wkt_protos", + "@com_google_protobuf//:any_proto", ], ) diff --git a/ml_metadata/tools/docker_server/Dockerfile b/ml_metadata/tools/docker_server/Dockerfile index 7fefdb74b..33c00fecc 100644 --- a/ml_metadata/tools/docker_server/Dockerfile +++ b/ml_metadata/tools/docker_server/Dockerfile @@ -36,7 +36,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ python3-dev # Set up Bazel 6.1.0 -ENV BAZEL_VERSION 6.1.0 +ENV BAZEL_VERSION 6.5.0 WORKDIR / RUN mkdir /bazel && \ cd /bazel && \ diff --git a/ml_metadata/util/BUILD b/ml_metadata/util/BUILD index d03c42631..7549d32e7 100644 --- a/ml_metadata/util/BUILD +++ b/ml_metadata/util/BUILD @@ -92,7 +92,7 @@ cc_library( hdrs = ["record_parsing_utils.h"], deps = [ ":return_utils", - + "@com_google_protobuf//:protobuf", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", From 056509b12d429c1525a6042260e1331373a6afe2 Mon Sep 17 00:00:00 2001 From: Gagandeep Singh Date: Tue, 17 Feb 2026 15:58:00 +0530 Subject: [PATCH 7/7] chore: Bump platforms to 0.0.10 for compatibility --- WORKSPACE | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index b630f7924..eac095ebb 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -14,14 +14,11 @@ http_archive( ], ) -#Install bazel platform version 0.0.6 +#Install bazel platform version 0.0.10 http_archive( name = "platforms", - urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/0.0.6/platforms-0.0.6.tar.gz", - "https://github.com/bazelbuild/platforms/releases/download/0.0.6/platforms-0.0.6.tar.gz", - ], - sha256 = "5308fc1d8865406a49427ba24a9ab53087f17f5266a7aabbfc28823f3916e1ca", + url = "https://github.com/bazelbuild/platforms/releases/download/0.0.10/platforms-0.0.10.tar.gz", + sha256 = "218efe8ee736d26a3572663b374a253c012b716d8af0c07e842e82f238a0a7ee", ) # Install version 0.12.0 of rules_foreign_cc