diff --git a/src/BUILD b/src/BUILD index ea624f5e59..dac466c23e 100644 --- a/src/BUILD +++ b/src/BUILD @@ -56,7 +56,7 @@ cc_shared_library( name = "ovms_shared", dynamic_deps = [], static_deps = [ - #"//src:libovmsfilesystemfactory", // TODO missing symbols + #"//src/filesystem:libovmsfilesystemfactory", // TODO missing symbols "@tensorflow_serving//:__subpackages__", "@minitrace//:__subpackages__", "@com_github_gabime_spdlog//:__subpackages__", @@ -165,6 +165,32 @@ ovms_cc_library( name = "servable", hdrs = ["servable.hpp"], ) +ovms_cc_library( + name = "libovms_servable_definition", + hdrs = ["servable_definition.hpp"], +) +ovms_cc_library( + name = "libovms_single_version_servable_definition", + srcs = [ + "servable_definition_unload_guard.cpp", + "single_version_servable_definition.cpp", + ], + hdrs = [ + "servable_definition_unload_guard.hpp", + "single_version_servable_definition.hpp", + "tensorinfo_fwd.hpp", + ], + deps = [ + "servable", + "libovms_servable_definition", + "//src/dags:pipelinedefinitionstatus", + "libovms_tensorinfo", + "libovmslogging", + "libovmsmodelversion", + "libovmsstatus", + ], + visibility = ["//visibility:public"], +) cc_library( name = "ovms_header", hdrs = ["ovms.h"], @@ -314,8 +340,8 @@ ovms_cc_library( "@com_github_jarro2783_cxxopts//:cxxopts", "libovms_server_settings", "libovms_version", - "libovmsfilesystem", - "libovmslocalfilesystem", + "//src/filesystem:libovmsfilesystem", + "//src/filesystem:libovmslocalfilesystem", "//src/graph_export:graph_cli_parser", "//src/graph_export:rerank_graph_cli_parser", "//src/graph_export:embeddings_graph_cli_parser", @@ -354,7 +380,7 @@ ovms_cc_library( "libovmsmodelversion", "libovms_execution_context", "libovmslogging", - "libovmsmetrics", + "//src/metrics:libovmsmetrics", ], visibility = ["//visibility:public",], ) @@ -405,31 +431,24 @@ ovms_cc_library( visibility = ["//visibility:public",], ) ovms_cc_library( - name = "dags_nodestreamidguard", - hdrs = ["dags/nodestreamidguard.hpp",], - srcs = ["dags/nodestreamidguard.cpp",], + name = "modelinstance", + srcs = ["modelinstance.cpp"], deps = [ - "libovmsprofiler", - "libovms_ovinferrequestsqueue", - "model_metric_reporter", - "libovmsmodelversion", - "libovmslogging", - ], - visibility = ["//visibility:public",], -) -ovms_cc_library( - name = "dags_pipeline", - hdrs = ["dags/pipeline.hpp",], - srcs = ["dags/pipeline.cpp",], - deps = [ - "libovms_dags_aliases", - "libovms_dags_node", - "libovms_execution_context", - "libovmsprofiler", - "libovmslogging", + "modelinstance_h", + "anonymous_input_name", + "customloaders", + "executingstreamidguard", + "//src/filesystem:libovmsfilesystem", + "libovmslayout", + "libovmslayout_configuration", + "libovms_config", + "libovms_ov_utils", + "libovmsshape", + "libovmsstring_utils", + "libovmstensorfactory", + "libovmstimer", + "modelinstanceunloadguard", "libovmsstatus", - "libovms_dags_pipelineeventqueue", - "libovms_dags_nodesession", ], visibility = ["//visibility:public",], ) @@ -482,6 +501,123 @@ ovms_cc_library( ], visibility = ["//visibility:public",], ) +ovms_cc_library( + name = "copyable_object_wrapper", + hdrs = ["copyable_object_wrapper.hpp"], + visibility = ["//visibility:public"], +) +ovms_cc_library( + name = "libovms_mediapipe_kfs_executor", + srcs = [ + "kfs_frontend/kfs_graph_executor_impl.cpp", + "kfs_frontend/kfs_graph_executor_impl.hpp", + ], + deps = select({ + "//:not_disable_python": [ + "//src/python:libovmspythonmodule", + ], + "//:disable_python": [] + }) + [ + "//src/mediapipe_internal:libovms_mediapipe", + "predict_request_validation_utils", + "tfs_utils", + "libovms_kfs_utils", + "libovms_kfs_grpc_inference_service_h", + "//src/kfserving_api:kfserving_api_cpp", + "opencv_dep", + "@mediapipe//mediapipe/framework/formats:image_frame", + "@mediapipe//mediapipe/framework/formats:image_frame_opencv", + "@mediapipe//mediapipe/framework/formats:tensor", + ], + visibility = ["//visibility:public",], + additional_copts = COPTS_PYTHON + COPTS_MEDIAPIPE, + alwayslink = 1, +) +ovms_cc_library( + name = "libovms_mediapipe_http_executor", + srcs = [ + "http_frontend/http_graph_executor_impl.cpp", + "http_frontend/http_graph_executor_impl.hpp", + ], + deps = select({ + "//:not_disable_python": [ + "//src/python:libovmspythonmodule", + ], + "//:disable_python": [] + }) + [ + "//src/mediapipe_internal:libovms_mediapipe", + "httppayload", + "libhttp_async_writer_interface", + "opencv_dep", + "//src/port:rapidjson_stringbuffer", + "//src/port:rapidjson_writer", + "@mediapipe//mediapipe/framework/formats:image_frame", + "@mediapipe//mediapipe/framework/formats:image_frame_opencv", + "@mediapipe//mediapipe/framework/formats:tensor", + ], + visibility = ["//visibility:public",], + additional_copts = COPTS_PYTHON + COPTS_MEDIAPIPE, + alwayslink = 1, +) +ovms_cc_library( + name = "global_sequences_viewer", + hdrs = ["global_sequences_viewer.hpp"], + srcs = ["global_sequences_viewer.cpp"], + deps = [ + "libovmslogging", + "libovmsmodelversion", + "libovms_sequence_manager", + "libovmsstatus", + ], + visibility = ["//visibility:public"], +) +ovms_cc_library( + name = "rest_parser_utils", + hdrs = [ + "rest_parser.hpp", + "rest_utils.hpp", + ], + srcs = [ + "rest_parser.cpp", + "rest_utils.cpp", + ], + deps = [ + "@com_github_tencent_rapidjson//:rapidjson", + "@com_google_absl//absl/strings", + "@tensorflow_serving//tensorflow_serving/apis:prediction_service_cc_proto", + "@tensorflow_serving//tensorflow_serving/util:json_tensor", + "@org_tensorflow//tensorflow/core:framework", + "//src/kfserving_api:kfserving_api_cpp", + "libovms_kfs_grpc_inference_service_h", + "libovms_kfs_utils", + "libovms_tensorinfo", + "libovmsprecision", + "libovmsprofiler", + "libovmsstatus", + "libovmstimer", + "tfs_utils", + ], + visibility = ["//visibility:public"], +) +ovms_cc_library( + name = "customloaders", + hdrs = [ + "customloaders.hpp", + "customloaderconfig.hpp", + "customloaderinterface.hpp", + ], + srcs = ["customloaders.cpp"], + deps = [ + "@com_github_tencent_rapidjson//:rapidjson", + "//src/port:rapidjson_stringbuffer", + "//src/port:rapidjson_writer", + "//src/filesystem:libovmsfilesystem", + "libovmslogging", + "libovmsstatus", + "libovmsstring_utils", + ], + visibility = ["//visibility:public"], +) ovms_cc_library( name = "ovms_lib", hdrs = [ @@ -491,31 +627,19 @@ ovms_cc_library( "capi_frontend/capi.cpp", "cleaner_utils.cpp", "cleaner_utils.hpp", - "customloaderconfig.hpp", - "customloaders.hpp", - "customloaders.cpp", - "customloaderinterface.hpp", - "dags/dl_node.cpp", - "dags/dl_node.hpp", - "dags/dlnodesession.cpp", - "dags/dlnodesession.hpp", - "dags/entry_node.cpp", - "dags/entry_node.hpp", - "dags/exit_node.cpp", - "dags/exit_node.hpp", - "dags/exitnodesession.cpp", - "dags/exitnodesession.hpp", - "dags/gatherexitnodeinputhandler.hpp", - "dags/pipelinedefinition.hpp", - "dags/pipelinedefinition.cpp", - "dags/pipelinedefinitionunloadguard.cpp", - "dags/pipelinedefinitionunloadguard.hpp", - "dags/pipeline_factory.cpp", - "dags/pipeline_factory.hpp", + "//src/dags:entry_node.cpp", + "//src/dags:entry_node.hpp", + "//src/dags:exit_node.cpp", + "//src/dags:exit_node.hpp", + "//src/dags:exitnodesession.cpp", + "//src/dags:exitnodesession.hpp", + "//src/dags:gatherexitnodeinputhandler.hpp", + "//src/dags:pipelinedefinition.hpp", + "//src/dags:pipelinedefinition.cpp", + "//src/dags:pipeline_factory.cpp", + "//src/dags:pipeline_factory.hpp", "get_model_metadata_impl.cpp", "get_model_metadata_impl.hpp", - "global_sequences_viewer.hpp", - "global_sequences_viewer.cpp", "http_server.hpp", "http_server.cpp", "http_rest_api_handler.hpp", @@ -529,43 +653,19 @@ ovms_cc_library( "model.hpp", "modelmanager.cpp", "modelmanager.hpp", - "modelinstance.cpp", - "modelinstance.hpp", "model_service.hpp", "model_service.cpp", "prediction_service.cpp", "prediction_service.hpp", "prediction_service_utils.hpp", "predict_request_validation_utils.hpp", - "rest_parser.cpp", - "rest_parser.hpp", - "rest_utils.cpp", - "rest_utils.hpp", "servablemanagermodule.cpp", "servablemanagermodule.hpp", "server.cpp", "server.hpp", "statefulmodelinstance.cpp", "statefulmodelinstance.hpp", - ] + select({ - "//:not_disable_mediapipe": [ - "copyable_object_wrapper.hpp", - "mediapipe_internal/mediapipefactory.cpp", - "mediapipe_internal/mediapipefactory.hpp", - "mediapipe_internal/mediapipegraphconfig.hpp", - "mediapipe_internal/mediapipegraphconfig.cpp", - "mediapipe_internal/mediapipegraphdefinition.cpp", - "mediapipe_internal/mediapipegraphdefinition.hpp", - "mediapipe_internal/mediapipegraphexecutor.cpp", - "mediapipe_internal/mediapipegraphexecutor.hpp", - "mediapipe_internal/packettypes.hpp", - "kfs_frontend/kfs_graph_executor_impl.cpp", - "kfs_frontend/kfs_graph_executor_impl.hpp", - "http_frontend/http_graph_executor_impl.cpp", - "http_frontend/http_graph_executor_impl.hpp", - ], - "//:disable_mediapipe" : [], - }), + ], deps = select({ "//:not_disable_python": [ "//src/python:libovmspythonmodule", @@ -574,16 +674,29 @@ ovms_cc_library( }) + select({ "//conditions:default": [], "//:not_disable_mediapipe" : [ + "//src/mediapipe_internal:libovms_mediapipe", + "libovms_mediapipe_kfs_executor", + "libovms_mediapipe_http_executor", + "@mediapipe_calculators//:mediapipe_calculators", + "@mediapipe//mediapipe/graphs/holistic_tracking:holistic_tracking_to_render_data", + "@mediapipe//mediapipe/graphs/iris_tracking:iris_tracking_cpu_deps", + "@mediapipe//mediapipe/calculators/tensor:image_to_tensor_calculator", + "@mediapipe//mediapipe/modules/holistic_landmark:holistic_landmark_cpu", + "@mediapipe//mediapipe/calculators/geti/inference:inference_calculators", + "@mediapipe//mediapipe/calculators/geti/utils:utils", + "@mediapipe//mediapipe/calculators/geti/utils:emptylabel_calculators", + "@mediapipe//mediapipe/calculators/geti/serialization:calculators", + "//src/llm:openai_completions_api_handler", "//src/image_gen:image_gen_calculator", "//src/audio/speech_to_text:s2t_calculator", "//src/audio/text_to_speech:t2s_calculator", "//src/audio:audio_utils", "//src/image_gen:imagegen_init", - "//src/llm:openai_completions_api_handler", "//src/embeddings:embeddingscalculator_ov", "//src/rerank:rerankcalculator", "//src/rerank:rerankcalculator_ov", - "//src/llm:llmcalculator",], + "//src/llm:llmcalculator", + ], }) + select({ "//:enable_drogon": [ "libdrogon_http_server", @@ -600,6 +713,7 @@ ovms_cc_library( "libovms_capi_servable_metadata", "libmodelconfigjsonparser", "modelconfig", + "modelinstance", "libovmscapibuffer", "libovmscapi_utils", "capiinferenceparameter", @@ -607,38 +721,43 @@ ovms_cc_library( "inferenceresponse", "inferencetensor", "capi_dag_utils", + "customloaders", + "global_sequences_viewer", + "rest_parser_utils", "libovms_cliparser", "libovms_config", - "libovms_dags_pipelinedefinitionstatus", - "libovms_dags_aliases", - "dags_pipeline", - "custom_node", - "custom_node_library_manager", - "libovms_dags_node", - "node_library", - "node_library_utils", - "custom_node_library_internal_manager_wrapper", - "nodeinfo", + "//src/dags:pipelinedefinitionstatus", + "//src/dags:aliases", + "//src/dags:pipeline", + "//src/dags:dl_node", + "//src/dags:dlnodesession", + "//src/dags:custom_node", + "//src/dags:custom_node_library_manager", + "//src/dags:node", + "//src/dags:node_library", + "//src/dags:node_library_utils", + "//src/dags:custom_node_library_internal_manager_wrapper", + "//src/dags:nodeinfo", "ovms_exit_codes", - "dags_nodestreamidguard", - "libovms_dags_session_id", - "libovms_dags_pipelineeventqueue", - "libovms_dags_nodesessionmetadata", - "libovms_dags_nodesessionresult", - "libovms_dags_nodeinputhandler", - "custom_node_output_allocator", + "//src/dags:nodestreamidguard", + "//src/dags:session_id", + "//src/dags:pipelineeventqueue", + "//src/dags:nodesessionmetadata", + "//src/dags:nodesessionresult", + "//src/dags:nodeinputhandler", + "//src/dags:custom_node_output_allocator", "libovms_execution_context", "executingstreamidguard", - "libovmsfilesystem", - "libovmsfilesystemfactory_h", - "libovmsfilesystemfactory", # TODO missing symbols @atobisze when not in ovms_lib + "//src/filesystem:libovmsfilesystem", + "//src/filesystem:libovmsfilesystemfactory_h", + "//src/filesystem:libovmsfilesystemfactory", # TODO missing symbols @atobisze when not in ovms_lib "grpc_utils", "libovms_kfs_utils", "libovms_kfs_grpc_inference_service_h", "modelchangesubscription", - "libovmslocalfilesystem", # indirectly & directly through factory + "//src/filesystem:libovmslocalfilesystem", # indirectly & directly through factory "libovmslogging", - "libovmsmetrics", + "//src/metrics:libovmsmetrics", "model_metric_reporter", "network_utils", "libovms_ov_utils", @@ -665,7 +784,7 @@ ovms_cc_library( "libovms_queue", "libovms_ovinferrequestsqueue", "libovmstensor_utils", - "libovms_dags_tensormap", + "//src/dags:tensormap", "libovmsstring_utils", "libovmsmodelversioning", "httppayload", @@ -682,9 +801,15 @@ ovms_cc_library( "kfs_backend_impl", "tfs_backend_impl", "anonymous_input_name", + "libovms_servable_name_checker", + "//src/metrics:libovms_metric_provider", + "libovms_model_instance_provider", + "libovms_servable_definition", + "libovms_single_version_servable_definition", + "copyable_object_wrapper", ] + select({ "//:not_disable_cloud": [ - "libovmsazurefilesystem", + "//src/filesystem:libovmsazurefilesystem", ], "//:disable_cloud": [] }) @@ -694,22 +819,7 @@ ovms_cc_library( "@windows_opencl//:opencl", "@windows_opencl2//:opencl2", ], - }) - + select({ - "//conditions:default": [ - "@mediapipe_calculators//:mediapipe_calculators", # Need this dependencies here because we use ovms/src - cannot add in ovms_dependencies because we copy src directory later in Dockerfile - "@mediapipe//mediapipe/graphs/holistic_tracking:holistic_tracking_to_render_data", - "@mediapipe//mediapipe/graphs/iris_tracking:iris_tracking_cpu_deps", - "@mediapipe//mediapipe/calculators/tensor:image_to_tensor_calculator", - "@mediapipe//mediapipe/modules/holistic_landmark:holistic_landmark_cpu", - "libovmsmediapipe_utils", - "@mediapipe//mediapipe/calculators/geti/inference:inference_calculators", - "@mediapipe//mediapipe/calculators/geti/utils:utils", - "@mediapipe//mediapipe/calculators/geti/utils:emptylabel_calculators", - "@mediapipe//mediapipe/calculators/geti/serialization:calculators", - ], - "//:disable_mediapipe" : [], - }), + }), visibility = ["//visibility:public",], additional_copts = COPTS_PYTHON + COPTS_MEDIAPIPE + COPTS_DROGON + COPTS_OV_TRACE, additional_linkopts = select({ @@ -1072,7 +1182,7 @@ cc_binary( ], deps = [ "//src:ovms_lib", - "//src:libovmsfilesystemfactory", + "//src/filesystem:libovmsfilesystemfactory", ], linkstatic = True, ) @@ -1104,47 +1214,13 @@ ovms_cc_library( visibility = ["//visibility:public"], ) -# Unused -ovms_cc_library( # TODO split dependencies - name = "libovmshttpservermodule", - hdrs = [ - "http_server.hpp", - "http_rest_api_handler.hpp", - "httpservermodule.hpp", - ], - srcs = [ - "http_rest_api_handler.cpp", - "http_server.cpp", - "httpservermodule.cpp", - ], - deps = [ - "@com_github_jupp0r_prometheus_cpp//core", - "@mediapipe//mediapipe/framework:calculator_framework", - "@tensorflow_serving//tensorflow_serving/apis:prediction_service_cc_proto", - "@tensorflow_serving//tensorflow_serving/util:threadpool_executor", - "@tensorflow_serving//tensorflow_serving/util:json_tensor", - "libovms_module", - "libovmslogging", - "libovmsprofiler", - "libovmsstatus", - "libdrogon_http_server", - "//src/kfserving_api:kfserving_api_cpp", - ] + select({ - "//:not_disable_python": [ - "//src/python:libovmspythonmodule", - ], - "//:disable_python": [] - }), - visibility = ["//visibility:public"], -) - ovms_cc_library( name = "libovmsschema", hdrs = ["schema.hpp",], srcs = ["schema.cpp",], deps = [ "@com_github_tencent_rapidjson//:rapidjson", - "libovmsfilesystem", + "//src/filesystem:libovmsfilesystem", "libovmsstatus", "libovmslogging", "//src/port:rapidjson_stringbuffer", @@ -1153,54 +1229,28 @@ ovms_cc_library( additional_copts = COPTS_MEDIAPIPE + COPTS_DROGON, ) -ovms_cc_library( - name = "libovmsmetrics", - hdrs = [ - "metric.hpp", - "metric_config.hpp", - "metric_family.hpp", - "metric_module.hpp", - "metric_registry.hpp", - ], - srcs = [ - "metric.cpp", - "metric_config.cpp", - "metric_family.cpp", - "metric_registry.cpp", - "metric_module.cpp", - ], - deps = [ - "@com_github_jupp0r_prometheus_cpp//core", - "@com_github_tencent_rapidjson//:rapidjson", - "//src/port:rapidjson_stringbuffer", - "//src/port:rapidjson_writer", - "cpp_headers", - "libovms_module", - "libovmslogging", - "libovmsschema", - "libovmsstatus", - "libovmsstring_utils", - ], - visibility = ["//visibility:public"], -) ovms_cc_library( name = "libovmsstatus", hdrs = ["status.hpp",], srcs = ["status.cpp",], - deps = ["libovmslogging"], + deps = ["libovmslogging", + "@fmtlib",], visibility = ["//visibility:public"], ) ovms_cc_library( - name = "libovmsmediapipe_utils", - hdrs = ["mediapipe_internal/mediapipe_utils.hpp", - "mediapipe_internal/packettypes.hpp",], - srcs = ["mediapipe_internal/mediapipe_utils.cpp",], - deps = ["libovmsstring_utils", - "libovmsstatus", - "libovmslogging",], - visibility = ["//visibility:public",], + name = "libovms_servable_name_checker", + hdrs = ["servable_name_checker.hpp",], + deps = [], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "libovms_model_instance_provider", + hdrs = ["model_instance_provider.hpp",], + deps = ["libovmsmodelversion",], + visibility = ["//visibility:public"], ) ovms_cc_library( # make ovms_lib dependent, use share doptions @@ -1222,41 +1272,6 @@ ovms_cc_library( visibility = ["//visibility:public"], ) -ovms_cc_library( # make ovms_lib dependent, use share doptions - name = "libovmsazurefilesystem", - hdrs = ["azurefilesystem.hpp",], - srcs = [ - "azurefilesystem.cpp", - "azurestorage.cpp", - "azurestorage.hpp", - ], - deps = [ - "@azure//:storage", - "@cpprest//:sdk", - "libovmsfilesystem", - "libovmslogging", - "libovmsstring_utils", - "libovmsmodelversioning", - "libovmsstatus", - ], - visibility = ["//visibility:public"], -) -ovms_cc_library( - name = "libovmsgcsfilesystem", - hdrs = ["gcsfilesystem.hpp",], - srcs = [ - "gcsfilesystem.cpp", - ], - deps = [ - "@google_cloud_cpp//:storage", - "libovmsfilesystem", - "libovmslogging", - "libovmsstring_utils", - "libovmsmodelversioning", - "libovmsstatus", - ], - visibility = ["//visibility:public"], -) # TODO split ovms_cc_library( name = "libovmstensorfactory", @@ -1315,7 +1330,7 @@ ovms_cc_library( "libovmsprecision", "extractchoice", "requesttensorextractor", - "libovms_dags_pipelinedefinitionstatus", # TODO @atobisze split further + "//src/dags:pipelinedefinitionstatus", "predict_request_validation_utils", ], visibility = ["//visibility:public"], @@ -1340,7 +1355,7 @@ ovms_cc_library( srcs = ["capi_frontend/capi_dag_utils.cpp",], deps = [ "//third_party:openvino", - "libovms_dags_pipelinedefinitionstatus", + "//src/dags:pipelinedefinitionstatus", "ovms_header", "libovmslogging", "libovmscapi_utils", @@ -1620,7 +1635,7 @@ ovms_cc_library( srcs = ["modelconfig.cpp",], deps = [ "@com_github_tencent_rapidjson//:rapidjson", # TODO split into parser - "libovmsfilesystem", + "//src/filesystem:libovmsfilesystem", "libovmslayout_configuration", "libovmscolor_format_configuration", "libovmsprecision_configuration", @@ -1718,6 +1733,7 @@ ovms_cc_library( hdrs = ["modelversionstatus.hpp",], srcs = ["modelversionstatus.cpp",], deps = [ + "@fmtlib", "libovmslogging", "libovmsmodelversion", ], @@ -1731,65 +1747,12 @@ ovms_cc_library( ], visibility = ["//visibility:public"], ) -ovms_cc_library( - name = "libovms_dags_nodesessionresult", - hdrs = [ - "dags/nodesessionresult.hpp", - ], - srcs = [ - ], - deps = [], - visibility = ["//visibility:public"], -) -ovms_cc_library( - name = "libovms_dags_nodesessionmetadata", - hdrs = [ - "dags/nodesessionmetadata.hpp", - ], - srcs = [ - "dags/nodesessionmetadata.cpp", - ], - deps = [ - "libovmslogging", - "libovms_execution_context", - "libovms_dags_session_id", - ], - visibility = ["//visibility:public"], -) ovms_cc_library( name = "custom_node_interface", hdrs = ["custom_node_interface.h",], deps = [], visibility = ["//visibility:public"], ) -ovms_cc_library( - name = "libovms_dags_aliases", - hdrs = ["dags/aliases.hpp",], - deps = [], - visibility = ["//visibility:public"], -) -ovms_cc_library( - name = "nodeinfo", - hdrs = ["dags/nodeinfo.hpp",], - deps = [ - "libovms_threadsafequeue", - "libovmsmodelversion", - "libovms_tensorinfo", - "node_library", - "libovms_dags_aliases", - "libovmslogging", - ], - visibility = ["//visibility:public"], -) -ovms_cc_library( - name = "node_library", - hdrs = ["dags/node_library.hpp",], - srcs = ["dags/node_library.cpp",], - deps = [ - "custom_node_interface", - ], - visibility = ["//visibility:public"], -) ovms_cc_library( name = "grpc_utils", hdrs = ["grpc_utils.hpp",], @@ -1797,122 +1760,7 @@ ovms_cc_library( deps = [ "libovmsstatus", "@com_github_grpc_grpc//:grpc++", - "libovmslogging", - ], - visibility = ["//visibility:public"], -) -ovms_cc_library( - # TODO needs further splitting - name = "libovms_dags_pipelinedefinition", - hdrs = ["dags/pipelinedefinition.hpp",], - srcs = ["dags/pipelinedefinition.cpp",], - deps = [ - "@com_github_tencent_rapidjson//:rapidjson", - "libovms_kfs_utils", - "tfs_utils", - "libovms_dags_aliases", - "nodeinfo", - "libovms_dags_pipelinedefinitionstatus", - "libovms_tensorinfo", - "notifyreceiver", - "libovmsmodelversion", - ], - visibility = ["//visibility:public"], -) -ovms_cc_library( - name = "libovms_dags_pipelineeventqueue", - hdrs = ["dags/pipelineeventqueue.hpp",], - deps = [ - "libovms_threadsafequeue", - "libovms_dags_session_id", - ], - visibility = ["//visibility:public"], -) -ovms_cc_library( - name = "node_library_utils", - hdrs = ["dags/node_library_utils.hpp",], - srcs = ["dags/node_library_utils.cpp",], - deps = [ - "custom_node_interface", - "node_library", - "libovms_dags_tensormap", - "libovmsprecision", - "libovmsshape", - "libovmslayout", - "libovms_ov_utils", - "libovms_tensorinfo", - ], - visibility = ["//visibility:public"], -) -ovms_cc_library( - name = "custom_node_library_manager", - hdrs = ["dags/custom_node_library_manager.hpp",], - srcs = ["dags/custom_node_library_manager.cpp",], - deps = [ - "node_library", - "libovmsfilesystem", - "libovmslogging", - "libovmsstatus", - ], - visibility = ["//visibility:public"], -) -ovms_cc_library( - name = "custom_node_output_allocator", - hdrs = ["dags/custom_node_output_allocator.hpp",], - srcs = ["dags/custom_node_output_allocator.cpp",], - deps = [ - "custom_node_interface", - "node_library", - "libovmslogging", - ], - visibility = ["//visibility:public"], -) -ovms_cc_library( - name = "custom_node_library_internal_manager_wrapper", - hdrs = ["dags/custom_node_library_internal_manager_wrapper.hpp",], - deps = [ - "node_library", - ], - visibility = ["//visibility:public"], -) -ovms_cc_library( - name = "customnodesession", - hdrs = ["dags/customnodesession.hpp",], - srcs = ["dags/customnodesession.cpp",], - deps = [ - "custom_node_interface", - "libovms_dags_pipelineeventqueue", -# "nodeinfo", - "libovms_dags_node", - "node_library", - "node_library_utils", - "libovms_dags_nodeinputhandler", - "libovms_dags_nodesession", - "libovms_dags_tensormap", - "custom_node_output_allocator", - "libovmslogging", - "libovmsstatus", - "libovmstimer", - "libovmsprofiler", - ], - visibility = ["//visibility:public"], -) -ovms_cc_library( - name = "custom_node", - hdrs = ["dags/custom_node.hpp",], - srcs = ["dags/custom_node.cpp",], - deps = [ - "custom_node_interface", - "libovms_dags_pipelineeventqueue", - "nodeinfo", - "node_library", - "libovms_dags_node", - "node_library_utils", - "customnodesession", - "custom_node_output_allocator", - "custom_node_library_internal_manager_wrapper", - "libovmslogging", - "libovmsstatus", + "@fmtlib", ], visibility = ["//visibility:public"], ) @@ -2095,105 +1943,6 @@ ovms_cc_library( deps = [], visibility = ["//visibility:public"], ) -ovms_cc_library( - name = "libovms_dags_session_id", - hdrs = ["dags/session_id.hpp",], - deps = [], - visibility = ["//visibility:public"], -) -ovms_cc_library( - name = "libovms_dags_tensormap", - hdrs = ["dags/tensormap.hpp",], - deps = [ - "//third_party:openvino", - "libovmstensor_utils", - ], - visibility = ["//visibility:public"], -) -ovms_cc_library( - name = "libovms_dags_gathernodeinputhandler", - hdrs = ["dags/gathernodeinputhandler.hpp",], - srcs = ["dags/gathernodeinputhandler.cpp",], - deps = [ - "//third_party:openvino", - "libovmslogging", - "libovms_dags_tensormap", - "libovms_dags_nodeinputhandler", - "libovms_dags_nodesessionmetadata", - "libovms_dags_session_id", - "libovmsstatus", - "libovms_tensorinfo", - "libovmsprofiler", - "libovms_ov_utils", - "libovmsshape", - "libovmsprecision", - ], - visibility = ["//visibility:public"], -) -ovms_cc_library( - name = "libovms_dags_nodeinputhandler", - hdrs = ["dags/nodeinputhandler.hpp",], - srcs = ["dags/nodeinputhandler.cpp",], - deps = [ - "libovmslogging", - "libovms_dags_tensormap", - "libovms_dags_session_id", - "libovmstensor_utils", - "libovmstimer", - "libovmsstatus", - ], - visibility = ["//visibility:public"], -) -ovms_cc_library( - name = "libovms_dags_nodesession", - hdrs = ["dags/nodesession.hpp",], - srcs = ["dags/nodesession.cpp",], - deps = [ - "libovmslogging", - "libovms_dags_tensormap", - "libovms_dags_nodesessionmetadata", - "libovms_dags_gathernodeinputhandler", - "libovms_dags_session_id", - "libovms_dags_nodeinputhandler", - "libovmstensor_utils", - "libovmsstatus", - "libovmsshape", - "libovmsprecision", - ], - visibility = ["//visibility:public"], -) -ovms_cc_library( - name = "libovms_dags_node", - hdrs = ["dags/node.hpp",], - srcs = ["dags/node.cpp",], - deps = [ - "libovmslogging", - "libovmsmodelversionstatus", - "libovms_execution_context", - "libovms_dags_aliases", - "libovms_dags_pipelineeventqueue", - "libovms_dags_session_id", - "libovms_dags_tensormap", - "libovms_dags_nodesession", - "libovms_dags_nodesessionresult", - "libovms_dags_nodesessionmetadata", - "libovms_ov_utils", - "libovmsprofiler", - "libovmsshape", - "libovmsstatus", - ], - visibility = ["//visibility:public"], -) -ovms_cc_library( - name = "libovms_dags_pipelinedefinitionstatus", - hdrs = ["dags/pipelinedefinitionstatus.hpp",], - srcs = ["dags/pipelinedefinitionstatus.cpp",], - deps = [ - "libovmslogging", - "libovmsmodelversionstatus", - ], - visibility = ["//visibility:public"], -) ovms_cc_library( name = "libovmsoclutils", hdrs = ["ocl_utils.hpp", @@ -2207,73 +1956,6 @@ ovms_cc_library( ], visibility = ["//visibility:public"], ) -ovms_cc_library( - name = "libovmss3filesystem", - hdrs = ["s3filesystem.hpp",], - srcs = [ - "s3filesystem.cpp", - ], - deps = [ - "@aws-sdk-cpp//:aws-sdk-cpp", - "libovmsfilesystem", - "libovmslogging", - "libovmsstring_utils", - "libovmsmodelversioning", - "libovmsstatus", - ], - visibility = ["//visibility:public"], -) -ovms_cc_library( - name = "libovmsfilesystem", - hdrs = ["filesystem.hpp",], - srcs = ["filesystem.cpp",], - deps = [ - "libovmslogging", - "libovmsmodelversioning", - "libovmsstatus", - "libovmsstring_utils", - "@boringssl//:ssl"], - visibility = ["//visibility:public"], -) - -ovms_cc_library( - name = "libovmslocalfilesystem", - hdrs = ["localfilesystem.hpp",], - srcs = ["localfilesystem.cpp",], - deps = ["libovmsfilesystem", - "libovmsstatus", - "libovmslogging"], - visibility = ["//visibility:public"], -) - -ovms_cc_library( - name = "libovmsfilesystemfactory_h", - hdrs = ["filesystemfactory.hpp",], - deps = [], - visibility = ["//visibility:public"], -) - -ovms_cc_library( - name = "libovmsfilesystemfactory", - hdrs = [], - srcs = ["filesystemfactory.cpp",], - deps = ["libovmsfilesystem", - "libovmsfilesystemfactory_h", - "libovmsstatus", - "libovmslocalfilesystem", - "libovmslogging" - ] + select({ - "//:not_disable_cloud": [ - "libovmsazurefilesystem", - "libovmsgcsfilesystem", - "libovmss3filesystem", - ], - "//:disable_cloud": [] - }), - additional_copts = COPTS_CLOUD, - visibility = ["//visibility:public"], -) - ovms_cc_library( name = "libmodelconfigjsonparser", hdrs = [ @@ -2402,7 +2084,7 @@ cc_binary( ], deps = [ "//src:ovms_lib", - "//src:libovmsfilesystemfactory", + "//src/filesystem:libovmsfilesystemfactory", ], linkstatic = True, ) @@ -2431,7 +2113,7 @@ cc_binary( copts = COMMON_STATIC_LIBS_COPTS, deps = [ "//src:ovms_lib", - "//src:libovmsfilesystemfactory",] + "//src/filesystem:libovmsfilesystemfactory",] + select({ "//conditions:default": [ "//src:libcustom_node_east_ocr.so", @@ -2710,7 +2392,7 @@ cc_test( deps = [ "optimum-cli", "//src:ovms_lib", - "//src:libovmsfilesystemfactory", + "//src/filesystem:libovmsfilesystemfactory", "@cpp_httplib//:cpp_httplib", # server_test.cpp ":test_utils", ":test_test_with_temp_dir", @@ -3278,7 +2960,7 @@ ovms_cc_library( deps = ["//src:libovmslogging", "//src:libmodelconfigjsonparser", "//src:libovmsstatus", - "//src:libovmsfilesystem", + "//src/filesystem:libovmsfilesystem", "//src:libovmsmodelversioning", "//src:libovms_ovinferrequestsqueue", "@mediapipe//mediapipe/framework:calculator_framework", diff --git a/src/audio/speech_to_text/BUILD b/src/audio/speech_to_text/BUILD index ec08ba386b..ca8410e31a 100644 --- a/src/audio/speech_to_text/BUILD +++ b/src/audio/speech_to_text/BUILD @@ -26,7 +26,8 @@ ovms_cc_library( ovms_cc_library( name = "s2t_calculator", - srcs = ["s2t_calculator.cc"], + srcs = ["s2t_calculator.cc", + "stt_node_initializer.cpp"], deps = [ "@mediapipe//mediapipe/framework:calculator_framework", "//src:httppayload", @@ -39,6 +40,8 @@ ovms_cc_library( "//third_party:genai", "//src/audio:audio_utils", "//src:libmodelconfigjsonparser", + "//src/mediapipe_internal:node_initializer", + "//src:libovmsstring_utils", ], visibility = ["//visibility:public"], alwayslink = 1, diff --git a/src/audio/speech_to_text/stt_node_initializer.cpp b/src/audio/speech_to_text/stt_node_initializer.cpp new file mode 100644 index 0000000000..124e74becf --- /dev/null +++ b/src/audio/speech_to_text/stt_node_initializer.cpp @@ -0,0 +1,73 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include + +#include "src/mediapipe_internal/graph_side_packets.hpp" +#include "src/mediapipe_internal/node_initializer.hpp" +#include "src/stringutils.hpp" +#include "s2t_servable.hpp" +#include "mediapipe/framework/calculator.pb.h" +#include "src/audio/speech_to_text/s2t_calculator.pb.h" + +#include "src/logging.hpp" + +namespace ovms { +class SttNodeInitializer : public NodeInitializer { + static constexpr const char* CALCULATOR_NAME = "S2tCalculator"; + +public: + bool matches(const std::string& calculatorName) const override { + return endsWith(calculatorName, CALCULATOR_NAME); + } + Status initialize( + const ::mediapipe::CalculatorGraphConfig_Node& nodeConfig, + const std::string& graphName, + const std::string& basePath, + GraphSidePackets& sidePackets, + PythonBackend* /*pythonBackend*/) override { + auto& sttServableMap = sidePackets.sttServableMap; + if (!nodeConfig.node_options().size()) { + SPDLOG_ERROR("SpeechToText node missing options in graph: {}. ", graphName); + return StatusCode::LLM_NODE_MISSING_OPTIONS; + } + if (nodeConfig.name().empty()) { + SPDLOG_ERROR("SpeechToText node name is missing in graph: {}. ", graphName); + return StatusCode::LLM_NODE_MISSING_NAME; + } + std::string nodeName = nodeConfig.name(); + if (sttServableMap.find(nodeName) != sttServableMap.end()) { + SPDLOG_ERROR("SpeechToText node name: {} already used in graph: {}. ", nodeName, graphName); + return StatusCode::LLM_NODE_NAME_ALREADY_EXISTS; + } + mediapipe::S2tCalculatorOptions nodeOptions; + const auto& calculatorOptions = nodeConfig.node_options(0); + if (!calculatorOptions.UnpackTo(&nodeOptions)) { + SPDLOG_ERROR("Failed to unpack calculator options"); + return StatusCode::MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID; + } + auto servable = std::make_shared(nodeOptions, basePath); + sttServableMap.insert(std::pair>(nodeName, std::move(servable))); + return StatusCode::OK; + } +}; + +static bool sttNodeInitializerRegistered = []() { + NodeInitializerRegistry::instance().add(std::make_unique()); + return true; +}(); +} // namespace ovms diff --git a/src/audio/text_to_speech/BUILD b/src/audio/text_to_speech/BUILD index 854c67c78d..2a494f6e16 100644 --- a/src/audio/text_to_speech/BUILD +++ b/src/audio/text_to_speech/BUILD @@ -34,7 +34,8 @@ ovms_cc_library( ovms_cc_library( name = "t2s_calculator", - srcs = ["t2s_calculator.cc"], + srcs = ["t2s_calculator.cc", + "tts_node_initializer.cpp"], deps = [ "@mediapipe//mediapipe/framework:calculator_framework", "//src:httppayload", @@ -46,6 +47,8 @@ ovms_cc_library( ":t2s_servable", "//src/audio:audio_utils", "//src:libmodelconfigjsonparser", + "//src/mediapipe_internal:node_initializer", + "//src:libovmsstring_utils", ], visibility = ["//visibility:public"], alwayslink = 1, diff --git a/src/audio/text_to_speech/tts_node_initializer.cpp b/src/audio/text_to_speech/tts_node_initializer.cpp new file mode 100644 index 0000000000..6fb0bfdc79 --- /dev/null +++ b/src/audio/text_to_speech/tts_node_initializer.cpp @@ -0,0 +1,79 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include +#include + +#include "src/mediapipe_internal/graph_side_packets.hpp" +#include "src/mediapipe_internal/node_initializer.hpp" +#include "src/stringutils.hpp" +#include "t2s_servable.hpp" +#include "mediapipe/framework/calculator.pb.h" +#include "src/audio/text_to_speech/t2s_calculator.pb.h" + +#include "src/logging.hpp" + +namespace ovms { +class TtsNodeInitializer : public NodeInitializer { + static constexpr const char* CALCULATOR_NAME = "T2sCalculator"; + +public: + bool matches(const std::string& calculatorName) const override { + return endsWith(calculatorName, CALCULATOR_NAME); + } + Status initialize( + const ::mediapipe::CalculatorGraphConfig_Node& nodeConfig, + const std::string& graphName, + const std::string& basePath, + GraphSidePackets& sidePackets, + PythonBackend* /*pythonBackend*/) override { + auto& ttsServableMap = sidePackets.ttsServableMap; + if (!nodeConfig.node_options().size()) { + SPDLOG_ERROR("TextToSpeech node missing options in graph: {}. ", graphName); + return StatusCode::LLM_NODE_MISSING_OPTIONS; + } + if (nodeConfig.name().empty()) { + SPDLOG_ERROR("TextToSpeech node name is missing in graph: {}. ", graphName); + return StatusCode::LLM_NODE_MISSING_NAME; + } + std::string nodeName = nodeConfig.name(); + if (ttsServableMap.find(nodeName) != ttsServableMap.end()) { + SPDLOG_ERROR("TextToSpeech node name: {} already used in graph: {}. ", nodeName, graphName); + return StatusCode::LLM_NODE_NAME_ALREADY_EXISTS; + } + mediapipe::T2sCalculatorOptions nodeOptions; + const auto& calculatorOptions = nodeConfig.node_options(0); + if (!calculatorOptions.UnpackTo(&nodeOptions)) { + SPDLOG_ERROR("Failed to unpack calculator options"); + return StatusCode::MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID; + } + try { + auto servable = std::make_shared(nodeOptions.models_path(), nodeOptions.target_device(), nodeOptions.voices(), nodeOptions.plugin_config(), basePath); + ttsServableMap.insert(std::pair>(nodeName, std::move(servable))); + } catch (const std::runtime_error& e) { + SPDLOG_ERROR("TextToSpeech node name: {} initialization failed: {}. ", nodeName, e.what()); + return StatusCode::MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID; + } + return StatusCode::OK; + } +}; + +static bool ttsNodeInitializerRegistered = []() { + NodeInitializerRegistry::instance().add(std::make_unique()); + return true; +}(); +} // namespace ovms diff --git a/src/capi_frontend/capi.cpp b/src/capi_frontend/capi.cpp index f6a4b2b159..13e3dc3202 100644 --- a/src/capi_frontend/capi.cpp +++ b/src/capi_frontend/capi.cpp @@ -28,14 +28,11 @@ #pragma warning(pop) #include "../dags/pipeline.hpp" +#include "../dags/pipeline_factory.hpp" #include "../dags/pipelinedefinition.hpp" -#include "../dags/pipelinedefinitionstatus.hpp" -#include "../dags/pipelinedefinitionunloadguard.hpp" +#include "../servable_definition_unload_guard.hpp" #include "../execution_context.hpp" #include "../version.hpp" -#if (MEDIAPIPE_DISABLE == 0) -#include "../mediapipe_internal/mediapipegraphdefinition.hpp" -#endif #include "../model_service.hpp" #include "../modelinstance.hpp" #include "capi_request_utils.hpp" // TODO @atobisze must be before executor @@ -49,8 +46,11 @@ #include "../ovms.h" // NOLINT #include "../prediction_service.hpp" #include "../profiler.hpp" +#include "../dags/pipelinedefinitionstatus.hpp" +#include "../servable_definition.hpp" #include "../servablemanagermodule.hpp" #include "../server.hpp" +#include "../single_version_servable_definition.hpp" #include "../status.hpp" #include "../timer.hpp" #include "buffer.hpp" @@ -73,7 +73,7 @@ using ovms::ModelInstanceUnloadGuard; using ovms::ModelManager; using ovms::Pipeline; using ovms::PipelineDefinition; -using ovms::PipelineDefinitionUnloadGuard; +using ovms::ServableDefinitionUnloadGuard; using ovms::ServableManagerModule; using ovms::Server; using ovms::Status; @@ -120,10 +120,10 @@ static Status getPipeline(ovms::Server& server, const InferenceRequest* request, if (!status.ok()) { return status; } - return modelManager->createPipeline(pipelinePtr, request->getServableName(), request, response); + return modelManager->getPipelineFactory().create(pipelinePtr, request->getServableName(), request, response, *modelManager); } -static Status getPipelineDefinition(Server& server, const std::string& servableName, PipelineDefinition** pipelineDefinition, std::unique_ptr& unloadGuard) { +static Status getPipelineDefinition(Server& server, const std::string& servableName, PipelineDefinition** pipelineDefinition, std::unique_ptr& unloadGuard) { ModelManager* modelManager{nullptr}; Status status = getModelManager(server, &modelManager); if (!status.ok()) { @@ -1190,21 +1190,16 @@ DLL_PUBLIC OVMS_Status* OVMS_GetServableState(OVMS_Server* serverPtr, const char std::shared_ptr modelInstance = modelManager->findModelInstance(servableName, servableVersion); if (modelInstance == nullptr) { - SPDLOG_DEBUG("Requested model: {} does not exist. Searching for pipeline with that name...", servableName); - PipelineDefinition* pipelineDefinition = nullptr; - pipelineDefinition = modelManager->getPipelineFactory().findDefinitionByName(servableName); - if (!pipelineDefinition) { -#if (MEDIAPIPE_DISABLE == 0) - ovms::MediapipeGraphDefinition* mediapipeDefinition = modelManager->getMediapipeFactory().findDefinitionByName(servableName); - if (mediapipeDefinition) { - *state = convertToServableState(mediapipeDefinition->getStateCode()); - return nullptr; - } -#endif + SPDLOG_DEBUG("Requested model: {} does not exist. Searching for definition with that name...", servableName); + auto* definition = modelManager->findServableDefinition(servableName); + if (!definition) { return reinterpret_cast(new Status(StatusCode::MODEL_NAME_MISSING)); } - *state = convertToServableState(pipelineDefinition->getStateCode()); - + auto* svsd = dynamic_cast(definition); + if (!svsd) { + return reinterpret_cast(new Status(StatusCode::MODEL_NAME_MISSING)); + } + *state = convertToServableState(svsd->getStatus().getStateCode()); return nullptr; } if (!status.ok()) { @@ -1274,7 +1269,7 @@ DLL_PUBLIC OVMS_Status* OVMS_GetServableMetadata(OVMS_Server* serverPtr, const c if (status == StatusCode::MODEL_NAME_MISSING) { SPDLOG_DEBUG("Requested model: {} does not exist. Searching for pipeline with that name...", servableName); PipelineDefinition* pipelineDefinition = nullptr; - std::unique_ptr unloadGuard; + std::unique_ptr unloadGuard; status = getPipelineDefinition(server, servableName, &pipelineDefinition, unloadGuard); if (!status.ok() || !pipelineDefinition) { return reinterpret_cast(new Status(std::move(status))); diff --git a/src/capi_frontend/capi_request_utils.hpp b/src/capi_frontend/capi_request_utils.hpp index 7d22e1c7ce..565a17e417 100644 --- a/src/capi_frontend/capi_request_utils.hpp +++ b/src/capi_frontend/capi_request_utils.hpp @@ -19,10 +19,10 @@ #include #include "../ovms.h" // NOLINT +#include "src/logging.hpp" #include "../precision.hpp" #include "inferencerequest.hpp" #include "../shape.hpp" -#include "../logging.hpp" #include "../status.hpp" // TODO move impl @atobisze #include "../extractchoice.hpp" #include "../requesttensorextractor.hpp" diff --git a/src/cli_parser.cpp b/src/cli_parser.cpp index 4f968c5294..9437fd9755 100644 --- a/src/cli_parser.cpp +++ b/src/cli_parser.cpp @@ -31,8 +31,8 @@ #include "graph_export/s2t_graph_cli_parser.hpp" #include "graph_export/image_generation_graph_cli_parser.hpp" #include "ovms_exit_codes.hpp" -#include "filesystem.hpp" -#include "localfilesystem.hpp" +#include "filesystem/filesystem.hpp" +#include "filesystem/localfilesystem.hpp" #include "stringutils.hpp" #include "version.hpp" diff --git a/src/color_format_configuration.cpp b/src/color_format_configuration.cpp index 0b6640bc88..f1973f53da 100644 --- a/src/color_format_configuration.cpp +++ b/src/color_format_configuration.cpp @@ -18,6 +18,8 @@ #include #include +#include "logging.hpp" + namespace ovms { const char ColorFormatConfiguration::COLOR_FORMAT_DELIMITER = ':'; diff --git a/src/config_export_module/BUILD b/src/config_export_module/BUILD index 2cd0094bc4..cbad7badab 100644 --- a/src/config_export_module/BUILD +++ b/src/config_export_module/BUILD @@ -23,8 +23,8 @@ ovms_cc_library( deps = [ "@com_github_tencent_rapidjson//:rapidjson", "//src:libovms_server_settings", - "//src:libovmsfilesystem", - "//src:libovmslocalfilesystem", + "//src/filesystem:libovmsfilesystem", + "//src/filesystem:libovmslocalfilesystem", "//src:libovmslogging", "//src:libovmsschema", ], diff --git a/src/config_export_module/config_export.cpp b/src/config_export_module/config_export.cpp index 8e2e449564..eed6eba16a 100644 --- a/src/config_export_module/config_export.cpp +++ b/src/config_export_module/config_export.cpp @@ -24,8 +24,8 @@ #pragma warning(pop) #include "../capi_frontend/server_settings.hpp" -#include "src/filesystem.hpp" -#include "src/localfilesystem.hpp" +#include "src/filesystem/filesystem.hpp" +#include "src/filesystem/localfilesystem.hpp" #include "src/logging.hpp" #include "src/schema.hpp" #include "src/status.hpp" diff --git a/src/customloaderconfig.hpp b/src/customloaderconfig.hpp index 2bf5eee2ed..8f01bf6309 100644 --- a/src/customloaderconfig.hpp +++ b/src/customloaderconfig.hpp @@ -25,45 +25,45 @@ #pragma warning(pop) #include -#include "filesystem.hpp" +#include "filesystem/filesystem.hpp" #include "status.hpp" #include "stringutils.hpp" namespace ovms { /** - * @brief This class represents Custom Loader configuration - */ + * @brief This class represents Custom Loader configuration + */ class CustomLoaderConfig { private: /** - * @brief Custom Loader Name - */ + * @brief Custom Loader Name + */ std::string loaderName; /** - * @brief Custom Loader Library Path - */ + * @brief Custom Loader Library Path + */ std::string libraryPath; /** - * @brief Custom Loader Config Path - */ + * @brief Custom Loader Config Path + */ std::string loaderConfigFile; /** - * @brief Json config directory path - */ + * @brief Json config directory path + */ std::string rootDirectoryPath; public: /** - * @brief Construct a new Custom Loader Config object - * - * @param name - * @param libraryPath - * @param configPath - */ + * @brief Construct a new Custom Loader Config object + * + * @param name + * @param libraryPath + * @param configPath + */ CustomLoaderConfig(const std::string& loaderName = "", const std::string& libraryPath = "", const std::string& loaderConfigFile = "") : @@ -79,73 +79,73 @@ class CustomLoaderConfig { } /** - * @brief Get the name - * - * @return const std::string& - */ + * @brief Get the name + * + * @return const std::string& + */ const std::string& getLoaderName() const { return this->loaderName; } /** - * @brief Set the name - * - * @param name - */ + * @brief Set the name + * + * @param name + */ void setLoaderName(const std::string& loaderName) { this->loaderName = loaderName; } /** - * @brief Get the Library Path - * - * @return const std::string& - */ + * @brief Get the Library Path + * + * @return const std::string& + */ const std::string& getLibraryPath() const { return this->libraryPath; } /** - * @brief Set the Library Path - * - * @param libraryPath - */ + * @brief Set the Library Path + * + * @param libraryPath + */ void setLibraryPath(const std::string& libraryPath) { FileSystem::setPath(this->libraryPath, libraryPath, this->rootDirectoryPath); } /** - * @brief Set root directory path - * - * @param rootDirectoryPath - */ + * @brief Set root directory path + * + * @param rootDirectoryPath + */ void setRootDirectoryPath(const std::string& rootDirectoryPath) { this->rootDirectoryPath = rootDirectoryPath; } /** - * @brief Get the Config Path - * - * @return const std::string& - */ + * @brief Get the Config Path + * + * @return const std::string& + */ const std::string& getLoaderConfigFile() const { return this->loaderConfigFile; } /** - * @brief Set the Config Path - * - * @param configPath - */ + * @brief Set the Config Path + * + * @param configPath + */ void setLoaderConfigFile(const std::string& loaderConfigFile) { this->loaderConfigFile = loaderConfigFile; } /** * @brief Parses all settings from a JSON node - * - * @return Status - */ + * + * @return Status + */ Status parseNode(const rapidjson::Value& v) { try { this->setLoaderName(v["loader_name"].GetString()); diff --git a/src/dags/BUILD b/src/dags/BUILD new file mode 100644 index 0000000000..514a62a947 --- /dev/null +++ b/src/dags/BUILD @@ -0,0 +1,370 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +load("//:common_settings.bzl", "COMMON_STATIC_LIBS_COPTS", "ovms_cc_library") + +# Files still compiled as part of //src:ovms_lib (pending further extraction) +exports_files([ + "entry_node.cpp", + "entry_node.hpp", + "exit_node.cpp", + "exit_node.hpp", + "exitnodesession.cpp", + "exitnodesession.hpp", + "gatherexitnodeinputhandler.hpp", + "pipelinedefinition.hpp", + "pipelinedefinition.cpp", + "pipeline_factory.cpp", + "pipeline_factory.hpp", +]) + +ovms_cc_library( + name = "aliases", + hdrs = ["aliases.hpp"], + deps = [], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "session_id", + hdrs = ["session_id.hpp"], + deps = [], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "tensormap", + hdrs = ["tensormap.hpp"], + deps = [ + "//third_party:openvino", + "//src:libovmstensor_utils", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "nodesessionresult", + hdrs = ["nodesessionresult.hpp"], + deps = [], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "nodesessionmetadata", + hdrs = ["nodesessionmetadata.hpp"], + srcs = ["nodesessionmetadata.cpp"], + deps = [ + "//src:libovmslogging", + "//src:libovms_execution_context", + "session_id", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "pipelineeventqueue", + hdrs = ["pipelineeventqueue.hpp"], + deps = [ + "//src:libovms_threadsafequeue", + "session_id", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "nodeinputhandler", + hdrs = ["nodeinputhandler.hpp"], + srcs = ["nodeinputhandler.cpp"], + deps = [ + "//src:libovmslogging", + "tensormap", + "session_id", + "//src:libovmstensor_utils", + "//src:libovmstimer", + "//src:libovmsstatus", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "gathernodeinputhandler", + hdrs = ["gathernodeinputhandler.hpp"], + srcs = ["gathernodeinputhandler.cpp"], + deps = [ + "//third_party:openvino", + "//src:libovmslogging", + "tensormap", + "nodeinputhandler", + "nodesessionmetadata", + "session_id", + "//src:libovmsstatus", + "//src:libovms_tensorinfo", + "//src:libovmsprofiler", + "//src:libovms_ov_utils", + "//src:libovmsshape", + "//src:libovmsprecision", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "nodesession", + hdrs = ["nodesession.hpp"], + srcs = ["nodesession.cpp"], + deps = [ + "//src:libovmslogging", + "tensormap", + "nodesessionmetadata", + "gathernodeinputhandler", + "session_id", + "nodeinputhandler", + "//src:libovmstensor_utils", + "//src:libovmsstatus", + "//src:libovmsshape", + "//src:libovmsprecision", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "node", + hdrs = ["node.hpp"], + srcs = ["node.cpp"], + deps = [ + "//src:libovmslogging", + "//src:libovmsmodelversionstatus", + "//src:libovms_execution_context", + "aliases", + "pipelineeventqueue", + "session_id", + "tensormap", + "nodesession", + "nodesessionresult", + "nodesessionmetadata", + "//src:libovms_ov_utils", + "//src:libovmsprofiler", + "//src:libovmsshape", + "//src:libovmsstatus", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "pipelinedefinitionstatus", + hdrs = ["pipelinedefinitionstatus.hpp"], + srcs = ["pipelinedefinitionstatus.cpp"], + deps = [ + "//src:libovmslogging", + "//src:libovmsmodelversionstatus", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "pipeline", + hdrs = ["pipeline.hpp"], + srcs = ["pipeline.cpp"], + deps = [ + "aliases", + "node", + "//src:libovms_execution_context", + "//src:libovmsprofiler", + "//src:libovmslogging", + "//src:libovmsstatus", + "pipelineeventqueue", + "nodesession", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "nodestreamidguard", + hdrs = ["nodestreamidguard.hpp"], + srcs = ["nodestreamidguard.cpp"], + deps = [ + "//src:libovmsprofiler", + "//src:libovms_ovinferrequestsqueue", + "//src:model_metric_reporter", + "//src:libovmsmodelversion", + "//src:libovmslogging", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "dlnodesession", + hdrs = ["dlnodesession.hpp"], + srcs = ["dlnodesession.cpp"], + deps = [ + "//third_party:openvino", + "//src:libovms_model_instance_provider", + "//src:modelinstance_h", + "//src:modelinstanceunloadguard", + "//src:libovms_ov_utils", + "//src:libovmslogging", + "//src:libovmsprofiler", + "//src:libovmsmodelversion", + "//src:libovmsshape", + "//src:libovmsstatus", + "//src:libovms_tensorinfo", + "//src:libovmstimer", + "node", + "nodesession", + "nodeinputhandler", + "pipelineeventqueue", + "nodestreamidguard", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "dl_node", + hdrs = ["dl_node.hpp"], + srcs = ["dl_node.cpp"], + deps = [ + "//third_party:openvino", + "//src:executingstreamidguard", + "//src:modelinstance_h", + "//src:modelinstanceunloadguard", + "//src:libovms_model_instance_provider", + "//src:libovms_ov_utils", + "//src:libovmslogging", + "//src/metrics:libovmsmetrics", + "//src:libovmsmodelversion", + "//src:libovmstimer", + "node", + "nodesession", + "dlnodesession", + "nodestreamidguard", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "nodeinfo", + hdrs = ["nodeinfo.hpp"], + deps = [ + "@fmtlib", + "//src:libovms_threadsafequeue", + "//src:libovmsmodelversion", + "//src:libovms_tensorinfo", + "node_library", + "aliases", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "node_library", + hdrs = ["node_library.hpp"], + srcs = ["node_library.cpp"], + deps = [ + "//src:custom_node_interface", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "node_library_utils", + hdrs = ["node_library_utils.hpp"], + srcs = ["node_library_utils.cpp"], + deps = [ + "//src:custom_node_interface", + "node_library", + "tensormap", + "//src:libovmsprecision", + "//src:libovmsshape", + "//src:libovmslayout", + "//src:libovms_ov_utils", + "//src:libovms_tensorinfo", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "custom_node_output_allocator", + hdrs = ["custom_node_output_allocator.hpp"], + srcs = ["custom_node_output_allocator.cpp"], + deps = [ + "//src:custom_node_interface", + "node_library", + "//src:libovmslogging", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "custom_node_library_manager", + hdrs = ["custom_node_library_manager.hpp"], + srcs = ["custom_node_library_manager.cpp"], + deps = [ + "node_library", + "//src/filesystem:libovmsfilesystem", + "//src:libovmslogging", + "//src:libovmsstatus", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "custom_node_library_internal_manager_wrapper", + hdrs = ["custom_node_library_internal_manager_wrapper.hpp"], + deps = [ + "node_library", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "customnodesession", + hdrs = ["customnodesession.hpp"], + srcs = ["customnodesession.cpp"], + deps = [ + "//src:custom_node_interface", + "pipelineeventqueue", + "node", + "node_library", + "node_library_utils", + "nodeinputhandler", + "nodesession", + "tensormap", + "custom_node_output_allocator", + "//src:libovmslogging", + "//src:libovmsstatus", + "//src:libovmstimer", + "//src:libovmsprofiler", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "custom_node", + hdrs = ["custom_node.hpp"], + srcs = ["custom_node.cpp"], + deps = [ + "//src:custom_node_interface", + "pipelineeventqueue", + "nodeinfo", + "node_library", + "node", + "node_library_utils", + "customnodesession", + "custom_node_output_allocator", + "custom_node_library_internal_manager_wrapper", + "//src:libovmslogging", + "//src:libovmsstatus", + ], + visibility = ["//visibility:public"], +) diff --git a/src/dags/custom_node_library_manager.cpp b/src/dags/custom_node_library_manager.cpp index 0beb09cd24..8270bf8933 100644 --- a/src/dags/custom_node_library_manager.cpp +++ b/src/dags/custom_node_library_manager.cpp @@ -22,7 +22,7 @@ #include #endif -#include "../filesystem.hpp" +#include "src/filesystem/filesystem.hpp" #include "../logging.hpp" #include "../status.hpp" diff --git a/src/dags/dl_node.cpp b/src/dags/dl_node.cpp index 99bbb3cec1..65fafe09e0 100644 --- a/src/dags/dl_node.cpp +++ b/src/dags/dl_node.cpp @@ -21,13 +21,10 @@ #include "../executingstreamidguard.hpp" #include "../logging.hpp" -#include "../metric.hpp" +#include "src/metrics/metric.hpp" #include "../modelinstance.hpp" #include "../modelinstanceunloadguard.hpp" -#include "../modelmanager.hpp" #include "../ov_utils.hpp" -#include "../ovinferrequestsqueue.hpp" -#include "../prediction_service_utils.hpp" #include "../timer.hpp" #include "dlnodesession.hpp" #include "nodestreamidguard.hpp" @@ -50,7 +47,7 @@ Status DLNode::getRealOutputName(ModelInstance& model, const std::string& alias, DLNode::DLNode(const std::string& nodeName, const std::string& modelName, std::optional modelVersion, - ModelManager& modelManager, + ModelInstanceProvider& modelManager, std::unordered_map nodeOutputNameAlias, std::optional demultiplyCount, std::set gatherFromNode) : Node(nodeName, demultiplyCount, std::move(gatherFromNode)), diff --git a/src/dags/dl_node.hpp b/src/dags/dl_node.hpp index ccf7650ba2..5eaf5b1ec2 100644 --- a/src/dags/dl_node.hpp +++ b/src/dags/dl_node.hpp @@ -24,7 +24,6 @@ #include #include "../executingstreamidguard.hpp" -#include "../model_version_policy.hpp" // for model_version_t typename #include "../modelversion.hpp" #include "node.hpp" @@ -33,13 +32,13 @@ namespace ovms { class ModelInstance; class ModelInstanceUnloadGuard; class NodeStreamIdGuard; -class ModelManager; +class ModelInstanceProvider; class DLNode : public Node { protected: std::string modelName; std::optional modelVersion; - ModelManager& modelManager; + ModelInstanceProvider& modelManager; const std::unordered_map nodeOutputNameAlias; std::shared_ptr model; @@ -48,7 +47,7 @@ class DLNode : public Node { public: DLNode(const std::string& nodeName, const std::string& modelName, std::optional modelVersion, - ModelManager& modelManager, + ModelInstanceProvider& modelManager, std::unordered_map nodeOutputNameAlias = {}, std::optional demultiplyCount = std::nullopt, std::set gatherFromNode = {}); diff --git a/src/dags/dlnodesession.cpp b/src/dags/dlnodesession.cpp index 5d678ed8ba..25f7534aa0 100644 --- a/src/dags/dlnodesession.cpp +++ b/src/dags/dlnodesession.cpp @@ -20,9 +20,9 @@ #include #include "../logging.hpp" +#include "../model_instance_provider.hpp" #include "../modelinstance.hpp" #include "../modelinstanceunloadguard.hpp" -#include "../modelmanager.hpp" #include "../ov_utils.hpp" #include "../profiler.hpp" #include "../shape.hpp" @@ -33,13 +33,13 @@ #include "nodestreamidguard.hpp" namespace ovms { -DLNodeSession::DLNodeSession(const NodeSessionMetadata& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, ModelManager& manager, const std::string& modelName, model_version_t modelVersion) : +DLNodeSession::DLNodeSession(const NodeSessionMetadata& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, ModelInstanceProvider& manager, const std::string& modelName, model_version_t modelVersion) : NodeSession(metadata, nodeName, inputsCount, collapsingDetails), modelManager(manager), modelName(modelName), modelVersion(modelVersion) {} -DLNodeSession::DLNodeSession(const NodeSessionMetadata&& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, ModelManager& manager, const std::string& modelName, model_version_t modelVersion) : +DLNodeSession::DLNodeSession(const NodeSessionMetadata&& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, ModelInstanceProvider& manager, const std::string& modelName, model_version_t modelVersion) : NodeSession(std::move(metadata), nodeName, inputsCount, collapsingDetails), modelManager(manager), modelName(modelName), diff --git a/src/dags/dlnodesession.hpp b/src/dags/dlnodesession.hpp index dad8183a5a..5b15279629 100644 --- a/src/dags/dlnodesession.hpp +++ b/src/dags/dlnodesession.hpp @@ -27,7 +27,7 @@ namespace ovms { -class ModelManager; +class ModelInstanceProvider; class ModelInstance; class Node; class NodeStreamIdGuard; @@ -40,13 +40,13 @@ class DLNodeSession : public NodeSession { std::unique_ptr nodeStreamIdGuard; std::unique_ptr modelUnloadGuard; - ModelManager& modelManager; + ModelInstanceProvider& modelManager; const std::string& modelName; const model_version_t modelVersion; public: - DLNodeSession(const NodeSessionMetadata& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, ModelManager& manager, const std::string& modelName, model_version_t modelVersion); - DLNodeSession(const NodeSessionMetadata&& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, ModelManager& manager, const std::string& modelName, model_version_t modelVersion); + DLNodeSession(const NodeSessionMetadata& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, ModelInstanceProvider& manager, const std::string& modelName, model_version_t modelVersion); + DLNodeSession(const NodeSessionMetadata&& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, ModelInstanceProvider& manager, const std::string& modelName, model_version_t modelVersion); virtual ~DLNodeSession(); ov::InferRequest& getInferRequest(const uint32_t microseconds); diff --git a/src/dags/entry_node.hpp b/src/dags/entry_node.hpp index c146b5021f..018aabdde2 100644 --- a/src/dags/entry_node.hpp +++ b/src/dags/entry_node.hpp @@ -21,7 +21,6 @@ #include -#include "../logging.hpp" #include "../ovms.h" // NOLINT #include "../regularovtensorfactory.hpp" #include "../tensorinfo.hpp" diff --git a/src/dags/gatherexitnodeinputhandler.hpp b/src/dags/gatherexitnodeinputhandler.hpp index 9432e3b2ac..14ac2aa522 100644 --- a/src/dags/gatherexitnodeinputhandler.hpp +++ b/src/dags/gatherexitnodeinputhandler.hpp @@ -23,7 +23,6 @@ #include "../capi_frontend/capi_utils.hpp" #include "../capi_frontend/capi_dag_utils.hpp" #include "../kfs_frontend/kfs_utils.hpp" -#include "../logging.hpp" #include "../profiler.hpp" #include "../status.hpp" #include "../tfs_frontend/tfs_utils.hpp" diff --git a/src/dags/nodeinfo.hpp b/src/dags/nodeinfo.hpp index 503c9397df..6a20f31717 100644 --- a/src/dags/nodeinfo.hpp +++ b/src/dags/nodeinfo.hpp @@ -24,11 +24,12 @@ #include #include +#include + #include "../modelversion.hpp" #include "../tensorinfo.hpp" #include "aliases.hpp" #include "node_library.hpp" -#include "../logging.hpp" namespace ovms { diff --git a/src/dags/pipeline_factory.cpp b/src/dags/pipeline_factory.cpp index 003c627721..bc13f01e8f 100644 --- a/src/dags/pipeline_factory.cpp +++ b/src/dags/pipeline_factory.cpp @@ -15,6 +15,12 @@ //***************************************************************************** #include "pipeline_factory.hpp" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wall" +#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" +#pragma GCC diagnostic pop +#include "../kfs_frontend/kfs_utils.hpp" + #include "../capi_frontend/inferencerequest.hpp" #include "../capi_frontend/inferenceresponse.hpp" #include "../logging.hpp" diff --git a/src/dags/pipeline_factory.hpp b/src/dags/pipeline_factory.hpp index d8f447f669..4bb4c6b388 100644 --- a/src/dags/pipeline_factory.hpp +++ b/src/dags/pipeline_factory.hpp @@ -24,11 +24,6 @@ #include #include -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop -#include "../kfs_frontend/kfs_grpc_inference_service.hpp" #include "nodeinfo.hpp" namespace ovms { diff --git a/src/dags/pipelinedefinition.cpp b/src/dags/pipelinedefinition.cpp index 434859c2ad..dcc76405cb 100644 --- a/src/dags/pipelinedefinition.cpp +++ b/src/dags/pipelinedefinition.cpp @@ -20,12 +20,14 @@ #include #include "../logging.hpp" +#include "../model.hpp" #include "../model_metric_reporter.hpp" #include "../modelinstance.hpp" #include "../modelinstanceunloadguard.hpp" #include "../modelmanager.hpp" #include "../ov_utils.hpp" #include "../prediction_service_utils.hpp" +#include "../servable_definition_unload_guard.hpp" #include "../status.hpp" #include "custom_node.hpp" #include "custom_node_library_internal_manager_wrapper.hpp" @@ -36,7 +38,6 @@ #include "nodeinfo.hpp" #include "nodestreamidguard.hpp" #include "pipeline.hpp" -#include "pipelinedefinitionunloadguard.hpp" namespace ovms { const std::string PipelineDefinition::SCHEDULER_CLASS_NAME{"Pipeline"}; @@ -59,25 +60,19 @@ PipelineDefinition::PipelineDefinition(const std::string& pipelineName, const pipeline_connections_t& connections, MetricRegistry* registry, const MetricConfig* metricConfig) : - pipelineName(pipelineName), + SingleVersionServableDefinition(pipelineName), nodeInfos(nodeInfos), connections(connections), reporter(std::make_unique(metricConfig, registry, pipelineName, VERSION)), - status(SCHEDULER_CLASS_NAME, this->pipelineName) {} + status(SCHEDULER_CLASS_NAME, getName()) {} Status PipelineDefinition::validate(ModelManager& manager) { SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Started validation of pipeline: {}", getName()); ValidationResultNotifier notifier(status, loadedNotify); - if (manager.modelExists(this->pipelineName)) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Pipeline name: {} is already occupied by model.", pipelineName); + if (manager.servableExists(getName(), ServableType::Model | ServableType::Mediapipe)) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Pipeline name: {} is already occupied by model or mediapipe graph.", getName()); return StatusCode::PIPELINE_NAME_OCCUPIED; } -#if (MEDIAPIPE_DISABLE == 0) - if (manager.getMediapipeFactory().definitionExists(this->pipelineName)) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Pipeline name: {} is already occupied by mediapipe graph.", pipelineName); - return StatusCode::PIPELINE_NAME_OCCUPIED; - } -#endif Status validationResult = initializeNodeResources(manager); if (!validationResult.ok()) { return validationResult; @@ -118,7 +113,7 @@ Status PipelineDefinition::initializeNodeResources(ModelManager& manager) { auto params = createCustomNodeParamArray(nodeInfo.parameters); int paramsLength = nodeInfo.parameters.size(); if (!nodeInfo.library.isValid()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Pipeline: {} node: {} refers to invalid library", pipelineName, nodeInfo.nodeName); + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Pipeline: {} node: {} refers to invalid library", getName(), nodeInfo.nodeName); return StatusCode::PIPELINE_DEFINITION_INVALID_NODE_LIBRARY; } auto status = nodeInfo.library.initialize(&customNodeLibraryInternalManager, params.get(), paramsLength); @@ -188,50 +183,12 @@ void PipelineDefinition::retire(ModelManager& manager) { this->connections.clear(); } -Status PipelineDefinition::waitForLoaded(std::unique_ptr& unloadGuard, const uint32_t waitForLoadedTimeoutMicroseconds) { - unloadGuard = std::make_unique(*this); - - const uint32_t waitLoadedTimestepMicroseconds = 1000; - const uint32_t waitCheckpoints = waitForLoadedTimeoutMicroseconds / waitLoadedTimestepMicroseconds; - uint32_t waitCheckpointsCounter = waitCheckpoints; - std::mutex cvMtx; - std::unique_lock cvLock(cvMtx); - while (waitCheckpointsCounter-- != 0) { - if (status.isAvailable()) { - SPDLOG_DEBUG("Successfully waited for pipeline definition: {}", getName()); - return StatusCode::OK; - } - unloadGuard.reset(); - if (!status.canEndLoaded()) { - if (status.getStateCode() != PipelineDefinitionStateCode::RETIRED) { - SPDLOG_DEBUG("Waiting for pipeline definition: {} ended due to timeout.", getName()); - return StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET; - } else { - SPDLOG_DEBUG("Waiting for pipeline definition: {} ended since it failed to load.", getName()); - return StatusCode::PIPELINE_DEFINITION_NOT_LOADED_ANYMORE; - } - } - SPDLOG_DEBUG("Waiting for available state for pipeline: {}, with timestep: {}us timeout: {}us check count: {}", - getName(), waitLoadedTimestepMicroseconds, waitForLoadedTimeoutMicroseconds, waitCheckpointsCounter); - loadedNotify.wait_for(cvLock, - std::chrono::microseconds(waitLoadedTimestepMicroseconds), - [this]() { - return this->status.isAvailable() || - !this->status.canEndLoaded(); - }); - unloadGuard = std::make_unique(*this); - } - if (!status.isAvailable()) { - if (status.getStateCode() != PipelineDefinitionStateCode::RETIRED) { - SPDLOG_DEBUG("Waiting for pipeline definition: {} ended due to timeout.", getName()); - return StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET; - } else { - SPDLOG_DEBUG("Waiting for pipeline definition: {} ended since it failed to load.", getName()); - return StatusCode::PIPELINE_DEFINITION_NOT_LOADED_ANYMORE; - } - } - SPDLOG_DEBUG("Successfully waited for pipeline definition: {}", getName()); - return StatusCode::OK; +StatusCode PipelineDefinition::notLoadedYetCode() const { + return StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET; +} + +StatusCode PipelineDefinition::notLoadedAnymoreCode() const { + return StatusCode::PIPELINE_DEFINITION_NOT_LOADED_ANYMORE; } template @@ -239,7 +196,7 @@ Status PipelineDefinition::create(std::unique_ptr& pipeline, const RequestType* request, ResponseType* response, ModelManager& manager) { - std::unique_ptr unloadGuard; + std::unique_ptr unloadGuard; Status status = waitForLoaded(unloadGuard); if (!status.ok()) { return status; @@ -300,7 +257,7 @@ Status PipelineDefinition::create(std::unique_ptr& pipeline, } #pragma warning(push) #pragma warning(disable : 6011) - pipeline = std::make_unique(*entry, *exit, *this->reporter, pipelineName); + pipeline = std::make_unique(*entry, *exit, *this->reporter, getName()); #pragma warning(pop) for (auto& kv : nodes) { pipeline->push(std::move(kv.second)); @@ -515,7 +472,7 @@ class NodeValidator { } if (shape.size() < 3) { SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Node: {} demultiply cannot occur due to not enough shape dimensions: {}", - this->pipelineName, + pipelineName, demultiplicatorNodeInfo.nodeName, shape.size()); return StatusCode::PIPELINE_NOT_ENOUGH_SHAPE_DIMENSIONS_TO_DEMULTIPLY; @@ -525,7 +482,7 @@ class NodeValidator { auto demultiplyDimension = Dimension(demultiplicatorNodeInfo.demultiplyCount.value()); if (!shape[0].partiallyFitsInto(demultiplyDimension)) { SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Demultiply count: {} of node: {} does not match tensor first dimension value: {}", - this->pipelineName, + pipelineName, demultiplicatorNodeInfo.demultiplyCount.value(), demultiplicatorNodeInfo.nodeName, shape[0].toString()); @@ -533,14 +490,14 @@ class NodeValidator { } } else { SPDLOG_LOGGER_WARN(modelmanager_logger, "Pipeline: {}; Demultiply count: {} of node: {} is fixed while first dimension value of node library is not: {}. This pipeline may fail at execution stage.", - this->pipelineName, + pipelineName, demultiplicatorNodeInfo.demultiplyCount.value(), demultiplicatorNodeInfo.nodeName, shape[0].toString()); } } else if (!shape[0].isAny()) { SPDLOG_LOGGER_WARN(modelmanager_logger, "Pipeline: {}; Demultiply count: {} of node: {} is dynamic while first dimension value of gather node is not: {}. This pipeline may fail at execution stage.", - this->pipelineName, + pipelineName, demultiplicatorNodeInfo.demultiplyCount.value(), demultiplicatorNodeInfo.nodeName, shape[0].toString()); @@ -593,7 +550,7 @@ class NodeValidator { result = influenceShapeWithDemultiplexer(tensorInputShape, *demultiplicatorNode); if (!result.ok()) { SPDLOG_LOGGER_ERROR(dag_executor_logger, "Validation of pipeline: {} definition failed. Demultiply count: {} of gather_from node: {} does not match tensor first dimension value: {} of node: {}", - this->pipelineName, + pipelineName, demultiplicatorNode->demultiplyCount.value(), demultiplicatorNode->nodeName, tensorInputShape[1].toString(), @@ -602,7 +559,7 @@ class NodeValidator { } } else if (dependantNodeInfo.gatherFromNode.size() > 1) { SPDLOG_LOGGER_ERROR(modelmanager_logger, "Validation of pipeline: {} definition failed. Manual gathering from multiple nodes is not supported in node name: {}", - this->pipelineName, + pipelineName, dependantNodeInfo.nodeName); return StatusCode::PIPELINE_MANUAL_GATHERING_FROM_MULTIPLE_NODES_NOT_SUPPORTED; } @@ -761,7 +718,7 @@ class NodeValidator { dependantNodeInfo, this->inputsInfo, dependantNodeInfo.library.getInputsInfo, - this->pipelineName, + pipelineName, getCNLIMWrapperPtr(nodeResources.at(dependantNodeInfo.nodeName))); if (!result.ok()) { return result; @@ -770,7 +727,7 @@ class NodeValidator { dependantNodeInfo, this->outputsInfo, dependantNodeInfo.library.getOutputsInfo, - this->pipelineName, + pipelineName, getCNLIMWrapperPtr(nodeResources.at(dependantNodeInfo.nodeName))); if (!result.ok()) { return result; @@ -789,7 +746,7 @@ class NodeValidator { dependencyNodeInfo, this->dependencyInputsInfo, dependencyNodeInfo.library.getInputsInfo, - this->pipelineName, + pipelineName, getCNLIMWrapperPtr(nodeResources.at(dependencyNodeInfo.nodeName))); if (!result.ok()) { return result; @@ -798,7 +755,7 @@ class NodeValidator { dependencyNodeInfo, this->dependencyOutputsInfo, dependencyNodeInfo.library.getOutputsInfo, - this->pipelineName, + pipelineName, getCNLIMWrapperPtr(nodeResources.at(dependencyNodeInfo.nodeName))); if (!result.ok()) { return result; @@ -887,7 +844,7 @@ class NodeValidator { }; Status PipelineDefinition::validateNode(ModelManager& manager, const NodeInfo& dependantNodeInfo, const bool isMultiBatchAllowed) { - NodeValidator validator(this->pipelineName, manager, dependantNodeInfo, connections, nodeInfos, nodeResources, isMultiBatchAllowed); + NodeValidator validator(getName(), manager, dependantNodeInfo, connections, nodeInfos, nodeResources, isMultiBatchAllowed); return validator.validate(); } @@ -1038,22 +995,22 @@ Status PipelineDefinition::validateNodes(ModelManager& manager) { [](const NodeInfo& info) { return info.kind == NodeKind::EXIT; }); if (entryNodeCount <= 0) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "PipelineDefinition: {} is missing request node", pipelineName); + SPDLOG_LOGGER_ERROR(modelmanager_logger, "PipelineDefinition: {} is missing request node", getName()); return StatusCode::PIPELINE_MISSING_ENTRY_OR_EXIT; } if (exitNodeCount <= 0) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "PipelineDefinition: {} is missing response node", pipelineName); + SPDLOG_LOGGER_ERROR(modelmanager_logger, "PipelineDefinition: {} is missing response node", getName()); return StatusCode::PIPELINE_MISSING_ENTRY_OR_EXIT; } if (entryNodeCount > 1) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "PipelineDefinition: {} has multiple request nodes", pipelineName); + SPDLOG_LOGGER_ERROR(modelmanager_logger, "PipelineDefinition: {} has multiple request nodes", getName()); return StatusCode::PIPELINE_MULTIPLE_ENTRY_NODES; } if (exitNodeCount > 1) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "PipelineDefinition: {} has multiple response nodes", pipelineName); + SPDLOG_LOGGER_ERROR(modelmanager_logger, "PipelineDefinition: {} has multiple response nodes", getName()); return StatusCode::PIPELINE_MULTIPLE_EXIT_NODES; } @@ -1070,7 +1027,7 @@ Status PipelineDefinition::validateNodes(ModelManager& manager) { this->nodeInfos.end(), [](const NodeInfo& info) { return info.demultiplyCount.has_value(); }); if (isAnyNodeDynamicDemultiplexer && (demultiplexerCount > 1)) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "PipelineDefinition: {} has multiple demultiplexers with at least one dynamic.", pipelineName); + SPDLOG_LOGGER_ERROR(modelmanager_logger, "PipelineDefinition: {} has multiple demultiplexers with at least one dynamic.", getName()); return StatusCode::NOT_IMPLEMENTED; } @@ -1081,7 +1038,7 @@ Status PipelineDefinition::validateNodes(ModelManager& manager) { }; if (std::count_if(nodeInfos.begin(), nodeInfos.end(), findByName) > 1) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "PipelineDefinition: {} has multiple nodes with name: {}", pipelineName, node.nodeName); + SPDLOG_LOGGER_ERROR(modelmanager_logger, "PipelineDefinition: {} has multiple nodes with name: {}", getName(), node.nodeName); return StatusCode::PIPELINE_NODE_NAME_DUPLICATE; } @@ -1376,7 +1333,7 @@ Shape PipelineDefinition::getNodeGatherShape(const NodeInfo& info) const { someNodeInfo, nodeOutputsInfo, someNodeInfo.library.getOutputsInfo, - this->pipelineName, + getName(), getCNLIMWrapperPtr(nodeResources.at(someNodeInfo.nodeName))); if (!result.ok()) { SPDLOG_ERROR("Failed to read node: {} library metadata with error: {}", nodeName, result.string()); diff --git a/src/dags/pipelinedefinition.hpp b/src/dags/pipelinedefinition.hpp index b45527dcfc..33322dfad6 100644 --- a/src/dags/pipelinedefinition.hpp +++ b/src/dags/pipelinedefinition.hpp @@ -15,8 +15,6 @@ //***************************************************************************** #pragma once -#include -#include #include #include #include @@ -26,10 +24,10 @@ #include #include -#include "../kfs_frontend/kfs_utils.hpp" -#include "../tfs_frontend/tfs_utils.hpp" +#include "../model_metric_reporter.hpp" #include "../modelversion.hpp" #include "../notifyreceiver.hpp" +#include "../single_version_servable_definition.hpp" #include "../tensorinfo.hpp" #include "aliases.hpp" #include "nodeinfo.hpp" @@ -40,15 +38,12 @@ struct CNLIMWrapper; class MetricConfig; class MetricRegistry; class ModelManager; -class ServableMetricReporter; class NodeValidator; class Pipeline; -class PipelineDefinitionUnloadGuard; class Status; -class PipelineDefinition : public NotifyReceiver { +class PipelineDefinition : public SingleVersionServableDefinition, public NotifyReceiver { friend NodeValidator; - friend PipelineDefinitionUnloadGuard; struct ValidationResultNotifier { ValidationResultNotifier(PipelineDefinitionStatus& status, std::condition_variable& loadedNotify) : status(status), @@ -68,7 +63,6 @@ class PipelineDefinition : public NotifyReceiver { std::condition_variable& loadedNotify; }; - const std::string pipelineName; std::vector nodeInfos; std::map> nodeResources = {}; pipeline_connections_t connections; @@ -79,11 +73,6 @@ class PipelineDefinition : public NotifyReceiver { private: mutable std::shared_mutex metadataMtx; - std::atomic requestsHandlesCounter = 0; - std::condition_variable loadedNotify; - - // Pipelines are not versioned and any available definition has constant version equal 1. - static constexpr model_version_t VERSION = 1; std::unique_ptr reporter; @@ -99,7 +88,6 @@ class PipelineDefinition : public NotifyReceiver { Shape getNodeGatherShape(const NodeInfo& info) const; public: - static constexpr uint64_t WAIT_FOR_LOADED_DEFAULT_TIMEOUT_MICROSECONDS = 500000; PipelineDefinition(const std::string& pipelineName, const std::vector& nodeInfos, const pipeline_connections_t& connections, @@ -129,15 +117,15 @@ class PipelineDefinition : public NotifyReceiver { std::vector calculateNodeInfosDiff(const std::vector& nodeInfos); void deinitializeNodeResources(const std::vector& nodeInfosDiff); - const std::string& getName() const override { return pipelineName; } + const std::string& getName() const override { return SingleVersionServableDefinition::getName(); } const PipelineDefinitionStateCode getStateCode() const { return status.getStateCode(); } - const model_version_t getVersion() const { return VERSION; } + bool isAvailable() const override { return status.isAvailable(); } void receiveNotification(const std::string& ownerDetails) override { this->status.handle(UsedModelChangedEvent(ownerDetails)); } - const PipelineDefinitionStatus& getStatus() const { + const PipelineDefinitionStatus& getStatus() const override { return this->status; } @@ -148,15 +136,15 @@ class PipelineDefinition : public NotifyReceiver { void makeSubscriptions(ModelManager& manager); void resetSubscriptions(ModelManager& manager); - ServableMetricReporter& getMetricReporter() const { return *this->reporter; } + ServableMetricReporter& getMetricReporter() const override { return *this->reporter; } protected: Status updateInputsInfo(const ModelManager& manager); Status updateOutputsInfo(const ModelManager& manager); public: - const tensor_map_t getInputsInfo() const; - const tensor_map_t getOutputsInfo() const; + const tensor_map_t getInputsInfo() const override; + const tensor_map_t getOutputsInfo() const override; private: static Status getCustomNodeMetadata(const NodeInfo& customNodeInfo, tensor_map_t& inputsInfo, metadata_fn callback, const std::string& pipelineName, void* customNodeLibraryInternalManager); @@ -175,16 +163,10 @@ class PipelineDefinition : public NotifyReceiver { const Aliases& aliases, const Shape& gatherShape) const; - void increaseRequestsHandlesCount() { - ++requestsHandlesCounter; - } - - void decreaseRequestsHandlesCount() { - --requestsHandlesCounter; - } + StatusCode notLoadedYetCode() const override; + StatusCode notLoadedAnymoreCode() const override; public: static const std::string SCHEDULER_CLASS_NAME; - Status waitForLoaded(std::unique_ptr& unloadGuard, const uint32_t waitForLoadedTimeoutMicroseconds = WAIT_FOR_LOADED_DEFAULT_TIMEOUT_MICROSECONDS); }; } // namespace ovms diff --git a/src/embeddings/BUILD b/src/embeddings/BUILD index 267b796354..c0ce6afc77 100644 --- a/src/embeddings/BUILD +++ b/src/embeddings/BUILD @@ -68,7 +68,8 @@ mediapipe_proto_library( ovms_cc_library( name = "embeddingscalculator_ov", hdrs = [], - srcs = ["embeddings_calculator_ov.cc"], + srcs = ["embeddings_calculator_ov.cc", + "embeddings_node_initializer.cpp"], deps = [ "@mediapipe//mediapipe/framework:calculator_framework", "@com_github_tencent_rapidjson//:rapidjson", @@ -85,6 +86,8 @@ ovms_cc_library( "//src:libovms_execution_context", ":embeddings_api", "//third_party:openvino", + "//src/mediapipe_internal:node_initializer", + "//src:libovmsstring_utils", ], visibility = ["//visibility:public"], alwayslink = 1, diff --git a/src/embeddings/embeddings_node_initializer.cpp b/src/embeddings/embeddings_node_initializer.cpp new file mode 100644 index 0000000000..400b7e9584 --- /dev/null +++ b/src/embeddings/embeddings_node_initializer.cpp @@ -0,0 +1,80 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include + +#include "src/mediapipe_internal/graph_side_packets.hpp" +#include "src/mediapipe_internal/node_initializer.hpp" +#include "src/stringutils.hpp" +#include "embeddings_servable.hpp" +#include "mediapipe/framework/calculator.pb.h" +#include "src/embeddings/embeddings_calculator_ov.pb.h" + +#include "src/logging.hpp" + +namespace ovms { +class EmbeddingsNodeInitializer : public NodeInitializer { + static constexpr const char* CALCULATOR_NAME = "EmbeddingsCalculatorOV"; + +public: + bool matches(const std::string& calculatorName) const override { + return endsWith(calculatorName, CALCULATOR_NAME); + } + Status initialize( + const ::mediapipe::CalculatorGraphConfig_Node& nodeConfig, + const std::string& graphName, + const std::string& basePath, + GraphSidePackets& sidePackets, + PythonBackend* /*pythonBackend*/) override { + auto& embeddingsServableMap = sidePackets.embeddingsServableMap; + if (!nodeConfig.node_options().size()) { + SPDLOG_ERROR("Embeddings node missing options in graph: {}. ", graphName); + return StatusCode::LLM_NODE_MISSING_OPTIONS; + } + if (nodeConfig.name().empty()) { + SPDLOG_ERROR("Embeddings node name is missing in graph: {}. ", graphName); + return StatusCode::LLM_NODE_MISSING_NAME; + } + std::string nodeName = nodeConfig.name(); + if (embeddingsServableMap.find(nodeName) != embeddingsServableMap.end()) { + SPDLOG_ERROR("Embeddings node name: {} already used in graph: {}. ", nodeName, graphName); + return StatusCode::LLM_NODE_NAME_ALREADY_EXISTS; + } + mediapipe::EmbeddingsCalculatorOVOptions nodeOptions; + nodeConfig.node_options(0).UnpackTo(&nodeOptions); + auto servable = std::make_shared( + nodeOptions.models_path(), + nodeOptions.target_device(), + nodeOptions.plugin_config(), + basePath, + nodeOptions.pooling(), + nodeOptions.normalize_embeddings()); + servable->initialize( + nodeOptions.models_path(), + nodeOptions.target_device(), + nodeOptions.plugin_config(), + basePath); + embeddingsServableMap.insert(std::pair>(nodeName, std::move(servable))); + return StatusCode::OK; + } +}; + +static bool embeddingsNodeInitializerRegistered = []() { + NodeInitializerRegistry::instance().add(std::make_unique()); + return true; +}(); +} // namespace ovms diff --git a/src/embeddings/embeddings_servable.hpp b/src/embeddings/embeddings_servable.hpp index 851bc334ca..60bc759f6d 100644 --- a/src/embeddings/embeddings_servable.hpp +++ b/src/embeddings/embeddings_servable.hpp @@ -17,9 +17,12 @@ #include "../sidepacket_servable.hpp" #include "src/embeddings/embeddings_calculator_ov.pb.h" -#include "../filesystem.hpp" +#include "src/filesystem/filesystem.hpp" +#pragma warning(push) +#pragma warning(disable : 6313) #include #include +#pragma warning(pop) #include #include #include diff --git a/src/filesystem/BUILD b/src/filesystem/BUILD new file mode 100644 index 0000000000..2c9c2cebd0 --- /dev/null +++ b/src/filesystem/BUILD @@ -0,0 +1,126 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +load("//:common_settings.bzl", "ovms_cc_library") + +COPTS_CLOUD = select({ + "//conditions:default": ["-DCLOUD_DISABLE=1"], + "//:not_disable_cloud" : ["-DCLOUD_DISABLE=0"], +}) + +ovms_cc_library( + name = "libovmsfilesystem", + hdrs = ["filesystem.hpp",], + srcs = ["filesystem.cpp",], + deps = [ + "//src:libovmslogging", + "//src:libovmsmodelversioning", + "//src:libovmsstatus", + "//src:libovmsstring_utils", + "@boringssl//:ssl"], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "libovmslocalfilesystem", + hdrs = ["localfilesystem.hpp",], + srcs = ["localfilesystem.cpp",], + deps = ["libovmsfilesystem", + "//src:libovmsstatus", + "//src:libovmslogging"], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "libovmsfilesystemfactory_h", + hdrs = ["filesystemfactory.hpp",], + deps = [], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "libovmsfilesystemfactory", + hdrs = [], + srcs = ["filesystemfactory.cpp",], + deps = ["libovmsfilesystem", + "libovmsfilesystemfactory_h", + "//src:libovmsstatus", + "libovmslocalfilesystem", + "//src:libovmslogging" + ] + select({ + "//:not_disable_cloud": [ + "libovmsazurefilesystem", + "libovmsgcsfilesystem", + "libovmss3filesystem", + ], + "//:disable_cloud": [] + }), + additional_copts = COPTS_CLOUD, + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "libovmsazurefilesystem", + hdrs = ["azurefilesystem.hpp",], + srcs = [ + "azurefilesystem.cpp", + "azurestorage.cpp", + "azurestorage.hpp", + ], + deps = [ + "@azure//:storage", + "@cpprest//:sdk", + "libovmsfilesystem", + "//src:libovmslogging", + "//src:libovmsstring_utils", + "//src:libovmsmodelversioning", + "//src:libovmsstatus", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "libovmsgcsfilesystem", + hdrs = ["gcsfilesystem.hpp",], + srcs = [ + "gcsfilesystem.cpp", + ], + deps = [ + "@google_cloud_cpp//:storage", + "libovmsfilesystem", + "//src:libovmslogging", + "//src:libovmsstring_utils", + "//src:libovmsmodelversioning", + "//src:libovmsstatus", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "libovmss3filesystem", + hdrs = ["s3filesystem.hpp",], + srcs = [ + "s3filesystem.cpp", + ], + deps = [ + "@aws-sdk-cpp//:aws-sdk-cpp", + "libovmsfilesystem", + "//src:libovmslogging", + "//src:libovmsstring_utils", + "//src:libovmsmodelversioning", + "//src:libovmsstatus", + ], + visibility = ["//visibility:public"], +) diff --git a/src/azurefilesystem.cpp b/src/filesystem/azurefilesystem.cpp similarity index 99% rename from src/azurefilesystem.cpp rename to src/filesystem/azurefilesystem.cpp index 3016d2994b..66c750afa2 100644 --- a/src/azurefilesystem.cpp +++ b/src/filesystem/azurefilesystem.cpp @@ -20,8 +20,8 @@ #include #include -#include "logging.hpp" -#include "stringutils.hpp" +#include "src/logging.hpp" +#include "src/stringutils.hpp" namespace ovms { diff --git a/src/azurefilesystem.hpp b/src/filesystem/azurefilesystem.hpp similarity index 59% rename from src/azurefilesystem.hpp rename to src/filesystem/azurefilesystem.hpp index 3b0d26370f..7f520b8292 100644 --- a/src/azurefilesystem.hpp +++ b/src/filesystem/azurefilesystem.hpp @@ -23,7 +23,7 @@ #include "azurestorage.hpp" #include "filesystem.hpp" -#include "status.hpp" +#include "src/status.hpp" namespace ovms { @@ -32,130 +32,130 @@ namespace as = azure::storage; class AzureFileSystem : public FileSystem { public: /** - * @brief Construct a new AzureFileSystem object - * - */ + * @brief Construct a new AzureFileSystem object + * + */ AzureFileSystem(); /** - * @brief Destroy the AzureFileSystem object - * - */ + * @brief Destroy the AzureFileSystem object + * + */ virtual ~AzureFileSystem(); /** - * @brief Check if given path or file exists - * - * @param path - * @param exists - * @return StatusCode - */ + * @brief Check if given path or file exists + * + * @param path + * @param exists + * @return StatusCode + */ StatusCode fileExists(const std::string& path, bool* exists) override; /** - * @brief Check if given path is a directory - * - * @param path - * @param is_dir - * @return StatusCode - */ + * @brief Check if given path is a directory + * + * @param path + * @param is_dir + * @return StatusCode + */ StatusCode isDirectory(const std::string& path, bool* is_dir) override; /** - * @brief Get the files and directories in given directory - * - * @param path - * @param contents - * @return StatusCode - */ + * @brief Get the files and directories in given directory + * + * @param path + * @param contents + * @return StatusCode + */ StatusCode getDirectoryContents(const std::string& path, files_list_t* contents) override; /** - * @brief Get only directories in given directory - * - * @param path - * @param subdirs - * @return StatusCode - */ + * @brief Get only directories in given directory + * + * @param path + * @param subdirs + * @return StatusCode + */ StatusCode getDirectorySubdirs(const std::string& path, files_list_t* subdirs) override; /** - * @brief Get only files in given directory - * - * @param path - * @param files - * @return StatusCode - */ + * @brief Get only files in given directory + * + * @param path + * @param files + * @return StatusCode + */ StatusCode getDirectoryFiles(const std::string& path, files_list_t* files) override; /** - * @brief Read the content of the given file into a string - * - * @param path - * @param contents - * @return StatusCode - */ + * @brief Read the content of the given file into a string + * + * @param path + * @param contents + * @return StatusCode + */ StatusCode readTextFile(const std::string& path, std::string* contents) override; /** - * @brief Download a remote directory - * - * @param path - * @param local_path - * @return StatusCode - */ + * @brief Download a remote directory + * + * @param path + * @param local_path + * @return StatusCode + */ StatusCode downloadFileFolder(const std::string& path, const std::string& local_path) override; /** - * @brief Download selected model versions - * - * @param path - * @param local_path - * @param versions - * @return StatusCode - */ + * @brief Download selected model versions + * + * @param path + * @param local_path + * @param versions + * @return StatusCode + */ StatusCode downloadModelVersions(const std::string& path, std::string* local_path, const std::vector& versions) override; StatusCode fileModificationTime(const std::string& path, int64_t* mtime_ns); /** - * @brief Delete a folder - * - * @param path - * @return StatusCode - */ + * @brief Delete a folder + * + * @param path + * @return StatusCode + */ StatusCode deleteFileFolder(const std::string& path) override; private: /** - * - * @brief - * - * @param remote_path - * @param local_path - */ + * + * @brief + * + * @param remote_path + * @param local_path + */ StatusCode downloadFile(const std::string& remote_path, const std::string& local_path); /** - * @brief Download a remote directory to a callee-provided local_path. - * - * @param path - * @param local_path - * @return StatusCode - */ + * @brief Download a remote directory to a callee-provided local_path. + * + * @param path + * @param local_path + * @return StatusCode + */ StatusCode downloadFileFolderTo(const std::string& path, const std::string& local_path); /** - * @brief - * - */ + * @brief + * + */ as::cloud_storage_account account_; }; diff --git a/src/azurestorage.cpp b/src/filesystem/azurestorage.cpp similarity index 99% rename from src/azurestorage.cpp rename to src/filesystem/azurestorage.cpp index 38e5542f63..ecda652a56 100644 --- a/src/azurestorage.cpp +++ b/src/filesystem/azurestorage.cpp @@ -19,7 +19,7 @@ #include #include "azurefilesystem.hpp" -#include "logging.hpp" +#include "src/logging.hpp" namespace ovms { diff --git a/src/azurestorage.hpp b/src/filesystem/azurestorage.hpp similarity index 99% rename from src/azurestorage.hpp rename to src/filesystem/azurestorage.hpp index f45b2e8084..ea0eb0e33a 100644 --- a/src/azurestorage.hpp +++ b/src/filesystem/azurestorage.hpp @@ -21,8 +21,7 @@ #include #include -#include "logging.hpp" -#include "status.hpp" +#include "src/status.hpp" #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wall" #pragma GCC diagnostic ignored "-Wunknown-pragmas" diff --git a/src/filesystem.cpp b/src/filesystem/filesystem.cpp similarity index 99% rename from src/filesystem.cpp rename to src/filesystem/filesystem.cpp index 3f323c8b4a..edafa435e9 100644 --- a/src/filesystem.cpp +++ b/src/filesystem/filesystem.cpp @@ -21,7 +21,7 @@ #include #include #endif -#include "stringutils.hpp" +#include "src/stringutils.hpp" namespace ovms { diff --git a/src/filesystem.hpp b/src/filesystem/filesystem.hpp similarity index 99% rename from src/filesystem.hpp rename to src/filesystem/filesystem.hpp index d9fa879383..fb658e8f0e 100644 --- a/src/filesystem.hpp +++ b/src/filesystem/filesystem.hpp @@ -24,10 +24,10 @@ #include #include -#include "logging.hpp" -#include "model_version_policy.hpp" +#include "src/logging.hpp" +#include "src/model_version_policy.hpp" #include "openssl/md5.h" -#include "status.hpp" +#include "src/status.hpp" namespace ovms { diff --git a/src/filesystemfactory.cpp b/src/filesystem/filesystemfactory.cpp similarity index 100% rename from src/filesystemfactory.cpp rename to src/filesystem/filesystemfactory.cpp diff --git a/src/filesystemfactory.hpp b/src/filesystem/filesystemfactory.hpp similarity index 100% rename from src/filesystemfactory.hpp rename to src/filesystem/filesystemfactory.hpp diff --git a/src/gcsfilesystem.cpp b/src/filesystem/gcsfilesystem.cpp similarity index 99% rename from src/gcsfilesystem.cpp rename to src/filesystem/gcsfilesystem.cpp index 5091ce3fb0..d43f02f898 100644 --- a/src/gcsfilesystem.cpp +++ b/src/filesystem/gcsfilesystem.cpp @@ -31,8 +31,8 @@ #include #include -#include "logging.hpp" -#include "stringutils.hpp" +#include "src/logging.hpp" +#include "src/stringutils.hpp" namespace ovms { diff --git a/src/gcsfilesystem.hpp b/src/filesystem/gcsfilesystem.hpp similarity index 59% rename from src/gcsfilesystem.hpp rename to src/filesystem/gcsfilesystem.hpp index 6d7ff66141..3a6f8f6dda 100644 --- a/src/gcsfilesystem.hpp +++ b/src/filesystem/gcsfilesystem.hpp @@ -22,141 +22,141 @@ #include "google/cloud/storage/client.h" #include "filesystem.hpp" -#include "status.hpp" +#include "src/status.hpp" namespace ovms { class GCSFileSystem : public FileSystem { public: /** - * @brief Construct a new GCSFileSystem object - * - */ + * @brief Construct a new GCSFileSystem object + * + */ GCSFileSystem(); /** - * @brief Construct a new GCSFileSystem object - * - * @param options - */ + * @brief Construct a new GCSFileSystem object + * + * @param options + */ GCSFileSystem(const google::cloud::storage::v1::ClientOptions& options); /** - * @brief Destroy the GCSFileSystem object - * - */ + * @brief Destroy the GCSFileSystem object + * + */ virtual ~GCSFileSystem(); /** - * @brief Check if given path or file exists - * - * @param path - * @param exists - * @return StatusCode - */ + * @brief Check if given path or file exists + * + * @param path + * @param exists + * @return StatusCode + */ StatusCode fileExists(const std::string& path, bool* exists) override; /** - * @brief Check if given path is a directory - * - * @param path - * @param is_dir - * @return StatusCode - */ + * @brief Check if given path is a directory + * + * @param path + * @param is_dir + * @return StatusCode + */ StatusCode isDirectory(const std::string& path, bool* is_dir) override; /** - * @brief Get the files and directories in given directory - * - * @param path - * @param contents - * @return StatusCode - */ + * @brief Get the files and directories in given directory + * + * @param path + * @param contents + * @return StatusCode + */ StatusCode getDirectoryContents(const std::string& path, files_list_t* contents) override; /** - * @brief Get only directories in given directory - * - * @param path - * @param subdirs - * @return StatusCode - */ + * @brief Get only directories in given directory + * + * @param path + * @param subdirs + * @return StatusCode + */ StatusCode getDirectorySubdirs(const std::string& path, files_list_t* subdirs) override; /** - * @brief Get only files in given directory - * - * @param path - * @param files - * @return StatusCode - */ + * @brief Get only files in given directory + * + * @param path + * @param files + * @return StatusCode + */ StatusCode getDirectoryFiles(const std::string& path, files_list_t* files) override; /** - * @brief Read the content of the given file into a string - * - * @param path - * @param contents - * @return StatusCode - */ + * @brief Read the content of the given file into a string + * + * @param path + * @param contents + * @return StatusCode + */ StatusCode readTextFile(const std::string& path, std::string* contents) override; /** - * @brief Download a remote directory - * - * @param path - * @param local_path - * @return StatusCode - */ + * @brief Download a remote directory + * + * @param path + * @param local_path + * @return StatusCode + */ StatusCode downloadFileFolder(const std::string& path, const std::string& local_path) override; /** * @brief Download selected model versions - * - * @param path - * @param local_path - * @param versions - * @return StatusCode + * + * @param path + * @param local_path + * @param versions + * @return StatusCode */ StatusCode downloadModelVersions(const std::string& path, std::string* local_path, const std::vector& versions) override; /** - * @brief Delete a folder - * - * @param path - * @return StatusCode - */ + * @brief Delete a folder + * + * @param path + * @return StatusCode + */ StatusCode deleteFileFolder(const std::string& path) override; private: /** - * @brief - * - * @param path - * @param bucket - * @param object - * @return StatusCode - */ + * @brief + * + * @param path + * @param bucket + * @param object + * @return StatusCode + */ StatusCode parsePath(const std::string& path, std::string* bucket, std::string* object); /** - * - * @brief - * - * @param remote_path - * @param local_path - */ + * + * @brief + * + * @param remote_path + * @param local_path + */ StatusCode downloadFile(const std::string& remote_path, const std::string& local_path); /** - * @brief - * - */ + * @brief + * + */ google::cloud::storage::Client client_; }; diff --git a/src/localfilesystem.cpp b/src/filesystem/localfilesystem.cpp similarity index 99% rename from src/localfilesystem.cpp rename to src/filesystem/localfilesystem.cpp index 6ee21311a1..e25fc48927 100644 --- a/src/localfilesystem.cpp +++ b/src/filesystem/localfilesystem.cpp @@ -20,7 +20,7 @@ #include #include -#include "logging.hpp" +#include "src/logging.hpp" #if defined(__APPLE__) || defined(__NetBSD__) #define st_mtim st_mtimespec diff --git a/src/localfilesystem.hpp b/src/filesystem/localfilesystem.hpp similarity index 76% rename from src/localfilesystem.hpp rename to src/filesystem/localfilesystem.hpp index a41d32f11c..f1ad82bd37 100644 --- a/src/localfilesystem.hpp +++ b/src/filesystem/localfilesystem.hpp @@ -19,7 +19,7 @@ #include #include "filesystem.hpp" -#include "status.hpp" +#include "src/status.hpp" namespace ovms { @@ -27,106 +27,106 @@ class LocalFileSystem : public FileSystem { public: /** * @brief Destroy the Local File System object - * + * */ ~LocalFileSystem() {} /** * @brief Check if given path or file exists - * - * @param path - * @param exists + * + * @param path + * @param exists * @return Status */ static Status exists(const std::string& path, bool* exists); /** * @brief Check if given path is a directory - * - * @param path - * @param is_dir - * @return Status + * + * @param path + * @param is_dir + * @return Status */ static Status isDir(const std::string& path, bool* is_dir); /** * @brief Check if given path or file exists - * - * @param path - * @param exists - * @return StatusCode + * + * @param path + * @param exists + * @return StatusCode */ StatusCode fileExists(const std::string& path, bool* exists) override; /** * @brief Check if given path is a directory - * - * @param path - * @param is_dir - * @return StatusCode + * + * @param path + * @param is_dir + * @return StatusCode */ StatusCode isDirectory(const std::string& path, bool* is_dir) override; /** * @brief Get the files and directories in given directory - * - * @param path - * @param contents - * @return StatusCode + * + * @param path + * @param contents + * @return StatusCode */ StatusCode getDirectoryContents(const std::string& path, files_list_t* contents) override; /** * @brief Get only directories in given directory - * - * @param path - * @param subdirs - * @return StatusCode + * + * @param path + * @param subdirs + * @return StatusCode */ StatusCode getDirectorySubdirs(const std::string& path, files_list_t* subdirs) override; /** * @brief Get only files in given directory - * - * @param path - * @param files - * @return StatusCode + * + * @param path + * @param files + * @return StatusCode */ StatusCode getDirectoryFiles(const std::string& path, files_list_t* files) override; /** * @brief Read the content of the given file into a string - * - * @param path - * @param contents - * @return StatusCode + * + * @param path + * @param contents + * @return StatusCode */ StatusCode readTextFile(const std::string& path, std::string* contents) override; /** * @brief Download a remote directory - * - * @param path - * @param local_path - * @return StatusCode + * + * @param path + * @param local_path + * @return StatusCode */ StatusCode downloadFileFolder(const std::string& path, const std::string& local_path) override; /** * @brief Download selected model versions - * - * @param path - * @param local_path - * @param versions - * @return StatusCode + * + * @param path + * @param local_path + * @param versions + * @return StatusCode */ StatusCode downloadModelVersions(const std::string& path, std::string* local_path, const std::vector& versions) override; /** * @brief Delete a folder - * - * @param path - * @return StatusCode + * + * @param path + * @return StatusCode */ StatusCode deleteFileFolder(const std::string& path) override; }; diff --git a/src/s3filesystem.cpp b/src/filesystem/s3filesystem.cpp similarity index 99% rename from src/s3filesystem.cpp rename to src/filesystem/s3filesystem.cpp index a08e6d41fa..c86ffe9b75 100644 --- a/src/s3filesystem.cpp +++ b/src/filesystem/s3filesystem.cpp @@ -40,8 +40,8 @@ #include #include -#include "logging.hpp" -#include "stringutils.hpp" +#include "src/logging.hpp" +#include "src/stringutils.hpp" namespace ovms { diff --git a/src/s3filesystem.hpp b/src/filesystem/s3filesystem.hpp similarity index 75% rename from src/s3filesystem.hpp rename to src/filesystem/s3filesystem.hpp index 650d06b906..c1dbd3e940 100644 --- a/src/s3filesystem.hpp +++ b/src/filesystem/s3filesystem.hpp @@ -23,7 +23,7 @@ #include #include "filesystem.hpp" -#include "status.hpp" +#include "src/status.hpp" namespace ovms { @@ -31,119 +31,119 @@ class S3FileSystem : public FileSystem { public: /** * @brief Construct a new S3FileSystem object - * - * @param options - * @param s3_path + * + * @param options + * @param s3_path */ S3FileSystem(const Aws::SDKOptions& options, const std::string& s3_path); /** * @brief Destroy the S3FileSystem object - * + * */ ~S3FileSystem(); /** * @brief Check if given path or file exists - * - * @param path - * @param exists - * @return StatusCode + * + * @param path + * @param exists + * @return StatusCode */ StatusCode fileExists(const std::string& path, bool* exists) override; /** * @brief Check if given path is a directory - * - * @param path - * @param is_dir - * @return StatusCode + * + * @param path + * @param is_dir + * @return StatusCode */ StatusCode isDirectory(const std::string& path, bool* is_dir) override; /** * @brief Get the files and directories in given directory - * - * @param path - * @param contents - * @return StatusCode + * + * @param path + * @param contents + * @return StatusCode */ StatusCode getDirectoryContents(const std::string& path, files_list_t* contents) override; /** * @brief Get only directories in given directory - * - * @param path - * @param subdirs - * @return StatusCode + * + * @param path + * @param subdirs + * @return StatusCode */ StatusCode getDirectorySubdirs(const std::string& path, files_list_t* subdirs) override; /** * @brief Get only files in given directory - * - * @param path - * @param files - * @return StatusCode + * + * @param path + * @param files + * @return StatusCode */ StatusCode getDirectoryFiles(const std::string& path, files_list_t* files) override; /** * @brief Read the content of the given file into a string - * - * @param path - * @param contents - * @return StatusCode + * + * @param path + * @param contents + * @return StatusCode */ StatusCode readTextFile(const std::string& path, std::string* contents) override; /** * @brief Download a remote directory - * - * @param path - * @param local_path - * @return StatusCode + * + * @param path + * @param local_path + * @return StatusCode */ StatusCode downloadFileFolder(const std::string& path, const std::string& local_path) override; /** * @brief Download selected model versions - * - * @param path - * @param local_path - * @param versions - * @return StatusCode + * + * @param path + * @param local_path + * @param versions + * @return StatusCode */ StatusCode downloadModelVersions(const std::string& path, std::string* local_path, const std::vector& versions) override; /** * @brief Delete a folder - * - * @param path - * @return StatusCode + * + * @param path + * @return StatusCode */ StatusCode deleteFileFolder(const std::string& path) override; private: /** - * @brief - * - * @param path - * @param bucket - * @param object - * @return StatusCode + * @brief + * + * @param path + * @param bucket + * @param object + * @return StatusCode */ StatusCode parsePath(const std::string& path, std::string* bucket, std::string* object); /** - * @brief - * + * @brief + * */ Aws::SDKOptions options_; /** - * @brief - * + * @brief + * */ Aws::S3::S3Client client_; std::regex s3_regex_; diff --git a/src/get_model_metadata_impl.cpp b/src/get_model_metadata_impl.cpp index 195df6da95..ae4202649c 100644 --- a/src/get_model_metadata_impl.cpp +++ b/src/get_model_metadata_impl.cpp @@ -17,10 +17,11 @@ #include +#include "dags/pipeline_factory.hpp" #include "dags/pipelinedefinition.hpp" -#include "dags/pipelinedefinitionstatus.hpp" -#include "dags/pipelinedefinitionunloadguard.hpp" +#include "servable_definition_unload_guard.hpp" #include "execution_context.hpp" +#include "model.hpp" #include "modelinstance.hpp" #include "modelinstanceunloadguard.hpp" #include "modelmanager.hpp" @@ -168,7 +169,7 @@ Status GetModelMetadataImpl::buildResponse( const ModelManager& manager) { // 0 meaning immediately return unload guard if possible, otherwise do not wait for available state - std::unique_ptr unloadGuard; + std::unique_ptr unloadGuard; auto status = pipelineDefinition.waitForLoaded(unloadGuard, 0); if (!status.ok()) { return status; diff --git a/src/graph_export/BUILD b/src/graph_export/BUILD index 47dc407a1e..58f286f8cf 100644 --- a/src/graph_export/BUILD +++ b/src/graph_export/BUILD @@ -27,8 +27,8 @@ ovms_cc_library( "@ovms//src:libovmslogging", "@ovms//src:libovms_server_settings", "@ovms//src:libovms_module", - "@ovms//src:libovmsfilesystem", - "@ovms//src:libovmslocalfilesystem", + "@ovms//src/filesystem:libovmsfilesystem", + "@ovms//src/filesystem:libovmslocalfilesystem", "@com_github_tencent_rapidjson//:rapidjson", "@ovms//src:libovmsschema", "@ovms//src:libovms_version", diff --git a/src/graph_export/graph_export.cpp b/src/graph_export/graph_export.cpp index dadbd57777..b98a18a966 100644 --- a/src/graph_export/graph_export.cpp +++ b/src/graph_export/graph_export.cpp @@ -32,8 +32,8 @@ #include "../capi_frontend/server_settings.hpp" #include "../config.hpp" -#include "../filesystem.hpp" -#include "../localfilesystem.hpp" +#include "src/filesystem/filesystem.hpp" +#include "src/filesystem/localfilesystem.hpp" #include "../logging.hpp" #include "../status.hpp" #include "../stringutils.hpp" diff --git a/src/grpc_utils.hpp b/src/grpc_utils.hpp index d9222f68f1..17baed5c3e 100644 --- a/src/grpc_utils.hpp +++ b/src/grpc_utils.hpp @@ -15,9 +15,9 @@ //***************************************************************************** #pragma once #include -#include -#include "logging.hpp" +#include +#include namespace ovms { class Status; diff --git a/src/http_rest_api_handler.cpp b/src/http_rest_api_handler.cpp index 33a81cb429..49bfd8e07a 100644 --- a/src/http_rest_api_handler.cpp +++ b/src/http_rest_api_handler.cpp @@ -38,16 +38,17 @@ #include "config.hpp" #include "dags/pipeline.hpp" +#include "dags/pipeline_factory.hpp" #include "dags/pipelinedefinition.hpp" -#include "dags/pipelinedefinitionunloadguard.hpp" +#include "servable_definition_unload_guard.hpp" #include "execution_context.hpp" -#include "filesystem.hpp" +#include "filesystem/filesystem.hpp" #include "get_model_metadata_impl.hpp" #include "grpcservermodule.hpp" #include "kfs_frontend/kfs_grpc_inference_service.hpp" #include "kfs_frontend/kfs_utils.hpp" -#include "metric_module.hpp" -#include "metric_registry.hpp" +#include "metrics/metric_module.hpp" +#include "metrics/metric_registry.hpp" #include "model_metric_reporter.hpp" #include "model_service.hpp" #include "modelinstance.hpp" @@ -68,6 +69,7 @@ #include "http_payload.hpp" #include "http_frontend/http_client_connection.hpp" #include "http_frontend/http_graph_executor_impl.hpp" +#include "mediapipe_internal/mediapipefactory.hpp" #include "mediapipe_internal/mediapipegraphexecutor.hpp" #endif @@ -1151,7 +1153,7 @@ Status HttpRestApiHandler::processPredictRequest( if (this->modelManager.modelExists(modelName)) { SPDLOG_DEBUG("Found model with name: {}. Searching for requested version...", modelName); status = processSingleModelRequest(modelName, modelVersion, request, requestOrder, responseProto, reporterOut); - } else if (this->modelManager.pipelineDefinitionExists(modelName)) { + } else if (this->modelManager.servableExists(modelName, ServableType::Pipeline)) { SPDLOG_DEBUG("Found pipeline with name: {}", modelName); status = processPipelineRequest(modelName, request, requestOrder, responseProto, reporterOut); } else { @@ -1245,7 +1247,7 @@ Status HttpRestApiHandler::getPipelineInputsAndReporter(const std::string& model if (!pipelineDefinition) { return StatusCode::MODEL_MISSING; } - std::unique_ptr unloadGuard; + std::unique_ptr unloadGuard; Status status = pipelineDefinition->waitForLoaded(unloadGuard); if (!status.ok()) { return status; @@ -1286,7 +1288,7 @@ Status HttpRestApiHandler::processPipelineRequest(const std::string& modelName, tensorflow::serving::PredictRequest& requestProto = requestParser.getProto(); requestProto.mutable_model_spec()->set_name(modelName); - status = this->modelManager.createPipeline(pipelinePtr, modelName, &requestProto, &responseProto); + status = this->modelManager.getPipelineFactory().create(pipelinePtr, modelName, &requestProto, &responseProto, this->modelManager); if (!status.ok()) { INCREMENT_IF_ENABLED(reporterOut->getInferRequestMetric(executionContext, false)); return status; diff --git a/src/image_gen/BUILD b/src/image_gen/BUILD index 38c44983f5..9c17c72431 100644 --- a/src/image_gen/BUILD +++ b/src/image_gen/BUILD @@ -38,7 +38,7 @@ ovms_cc_library( deps = [ "//src:libovmslogging", "//src:libmodelconfigjsonparser", - "//src:libovmsfilesystem", + "//src/filesystem:libovmsfilesystem", "//src:libovmsstring_utils", "imagegenutils", "imagegenpipelineargs", @@ -75,7 +75,8 @@ ovms_cc_library( ovms_cc_library( name = "image_gen_calculator", - srcs = ["http_image_gen_calculator.cc"], + srcs = ["http_image_gen_calculator.cc", + "image_gen_node_initializer.cpp"], deps = [ "@mediapipe//mediapipe/framework:calculator_framework", "//src:httppayload", @@ -84,7 +85,11 @@ ovms_cc_library( ":pipelines", "//src:image_conversion", ":imagegenutils", - "//third_party:genai",], + "//third_party:genai", + ":imagegen_init", + "//src/mediapipe_internal:node_initializer", + "//src:libovmsstring_utils", + "//third_party:openvino",], visibility = ["//visibility:public"], alwayslink = 1, ) diff --git a/src/image_gen/image_gen_node_initializer.cpp b/src/image_gen/image_gen_node_initializer.cpp new file mode 100644 index 0000000000..72e680fb72 --- /dev/null +++ b/src/image_gen/image_gen_node_initializer.cpp @@ -0,0 +1,84 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include +#include + +#include + +#include "src/mediapipe_internal/graph_side_packets.hpp" +#include "src/mediapipe_internal/node_initializer.hpp" +#include "src/stringutils.hpp" +#include "imagegen_init.hpp" +#include "pipelines.hpp" +#include "mediapipe/framework/calculator.pb.h" + +#include "src/logging.hpp" + +namespace ovms { +class ImageGenNodeInitializer : public NodeInitializer { + static constexpr const char* CALCULATOR_NAME = "ImageGenCalculator"; + +public: + bool matches(const std::string& calculatorName) const override { + return endsWith(calculatorName, CALCULATOR_NAME); + } + Status initialize( + const ::mediapipe::CalculatorGraphConfig_Node& nodeConfig, + const std::string& graphName, + const std::string& basePath, + GraphSidePackets& sidePackets, + PythonBackend* /*pythonBackend*/) override { + auto& imageGenPipelinesMap = sidePackets.imageGenPipelinesMap; + if (!nodeConfig.node_options().size()) { + SPDLOG_ERROR("Image Gen node missing options in graph: {}. ", graphName); + return StatusCode::LLM_NODE_MISSING_OPTIONS; + } + if (nodeConfig.name().empty()) { + SPDLOG_ERROR("Image Gen node name is missing in graph: {}. ", graphName); + return StatusCode::LLM_NODE_MISSING_NAME; + } + std::string nodeName = nodeConfig.name(); + if (imageGenPipelinesMap.find(nodeName) != imageGenPipelinesMap.end()) { + SPDLOG_ERROR("Image Gen node name: {} already used in graph: {}. ", nodeName, graphName); + return StatusCode::LLM_NODE_NAME_ALREADY_EXISTS; + } + auto statusOrArgs = prepareImageGenPipelineArgs(nodeConfig.node_options(0), basePath); + if (std::holds_alternative(statusOrArgs)) { + SPDLOG_ERROR("Failed to prepare Image Gen pipeline args for node: {}. Error: {}", graphName, std::get(statusOrArgs).string()); + return std::get(statusOrArgs); + } + std::shared_ptr servable; + try { + servable = std::make_shared(std::get(statusOrArgs)); + } catch (ov::Exception& e) { + SPDLOG_ERROR("Failed to create Image Generation pipelines: {}. Error: {}", graphName, e.what()); + return StatusCode::INTERNAL_ERROR; + } catch (...) { + SPDLOG_ERROR("Failed to create Image Generation pipelines: {}. Unknown error", graphName); + return StatusCode::INTERNAL_ERROR; + } + imageGenPipelinesMap.insert(std::pair>(nodeName, std::move(servable))); + return StatusCode::OK; + } +}; + +static bool imageGenNodeInitializerRegistered = []() { + NodeInitializerRegistry::instance().add(std::make_unique()); + return true; +}(); +} // namespace ovms diff --git a/src/image_gen/imagegen_init.cpp b/src/image_gen/imagegen_init.cpp index a96cbd764c..d25bf9dfdd 100644 --- a/src/image_gen/imagegen_init.cpp +++ b/src/image_gen/imagegen_init.cpp @@ -20,10 +20,12 @@ #include #include +#include + #include "absl/strings/str_replace.h" #include "absl/strings/ascii.h" -#include "src/filesystem.hpp" +#include "src/filesystem/filesystem.hpp" #include "src/image_gen/image_gen_calculator.pb.h" #include "src/json_parser.hpp" #include "src/logging.hpp" diff --git a/src/inference_request_common.hpp b/src/inference_request_common.hpp index a0f79f2460..b6405827e5 100644 --- a/src/inference_request_common.hpp +++ b/src/inference_request_common.hpp @@ -22,7 +22,6 @@ #include #include -#include "logging.hpp" #include "shape.hpp" #include "anonymous_input_name.hpp" #include "status.hpp" diff --git a/src/kfs_frontend/kfs_grpc_inference_service.cpp b/src/kfs_frontend/kfs_grpc_inference_service.cpp index e3499caa9f..e6ebb2bb0d 100644 --- a/src/kfs_frontend/kfs_grpc_inference_service.cpp +++ b/src/kfs_frontend/kfs_grpc_inference_service.cpp @@ -26,9 +26,8 @@ #include "kfs_utils.hpp" #include "kfs_request_utils.hpp" #include "../dags/pipeline.hpp" -#include "../dags/pipelinedefinition.hpp" +#include "../dags/pipeline_factory.hpp" #include "../dags/pipelinedefinitionstatus.hpp" -#include "../dags/pipelinedefinitionunloadguard.hpp" #include "../execution_context.hpp" #include "../grpc_utils.hpp" #if (MEDIAPIPE_DISABLE == 0) @@ -36,19 +35,22 @@ // kfs_graph_executor_impl needs to be included before mediapipegraphexecutor // because it contains functions required by graph execution template #include "kfs_graph_executor_impl.hpp" -#include "../mediapipe_internal/mediapipegraphdefinition.hpp" #include "../mediapipe_internal/mediapipegraphexecutor.hpp" // clang-format on #endif -#include "../metric.hpp" +#include "src/metrics/metric.hpp" +#include "../model.hpp" #include "../modelinstance.hpp" #include "../deserialization_main.hpp" #include "../inference_executor.hpp" #include "../modelinstanceunloadguard.hpp" #include "../modelmanager.hpp" #include "../ovinferrequestsqueue.hpp" +#include "../servable_definition.hpp" +#include "../servable_definition_unload_guard.hpp" #include "../servablemanagermodule.hpp" #include "../server.hpp" +#include "../single_version_servable_definition.hpp" #include "../status.hpp" #include "../stringutils.hpp" #include "../tensorinfo.hpp" @@ -85,7 +87,7 @@ Status KFSInferenceServiceImpl::getPipeline(const KFSRequest* request, KFSResponse* response, std::unique_ptr& pipelinePtr) { OVMS_PROFILE_FUNCTION(); - return this->modelManager.createPipeline(pipelinePtr, request->model_name(), request, response); + return this->modelManager.getPipelineFactory().create(pipelinePtr, request->model_name(), request, response, this->modelManager); } const std::string PLATFORM = "OpenVINO"; @@ -118,25 +120,18 @@ Status KFSInferenceServiceImpl::getModelReady(const KFSGetModelStatusRequest* re auto model = manager.findModelByName(name); SPDLOG_DEBUG("ModelReady requested name: {}, version: {}", name, versionString); if (model == nullptr) { - SPDLOG_DEBUG("ModelReady requested model {} is missing, trying to find pipeline with such name", name); - auto pipelineDefinition = manager.getPipelineFactory().findDefinitionByName(name); - if (!pipelineDefinition) { -#if (MEDIAPIPE_DISABLE == 0) - SPDLOG_DEBUG("ModelReady requested pipeline {} is missing, trying to find mediapipe with such name", name); - auto mediapipeGraphDefinition = manager.getMediapipeFactory().findDefinitionByName(name); - if (!mediapipeGraphDefinition) { - return StatusCode::MODEL_NAME_MISSING; - } - auto status = buildResponse(*mediapipeGraphDefinition, response); - INCREMENT_IF_ENABLED(mediapipeGraphDefinition->getMetricReporter().getModelReadyMetric(executionContext, status.ok())); - return status; -#else + SPDLOG_DEBUG("ModelReady requested model {} is missing, trying to find definition with such name", name); + auto* definition = manager.findServableDefinition(name); + if (!definition) { return StatusCode::MODEL_NAME_MISSING; -#endif } - auto status = buildResponse(*pipelineDefinition, response); - INCREMENT_IF_ENABLED(pipelineDefinition->getMetricReporter().getModelReadyMetric(executionContext, status.ok())); - return status; + auto* svsd = dynamic_cast(definition); + if (!svsd) { + return StatusCode::MODEL_NAME_MISSING; + } + response->set_ready(svsd->isAvailable()); + INCREMENT_IF_ENABLED(svsd->getMetricReporter().getModelReadyMetric(executionContext, true)); + return StatusCode::OK; } std::shared_ptr instance = nullptr; if (!versionString.empty()) { @@ -201,24 +196,17 @@ Status KFSInferenceServiceImpl::ModelMetadataImpl(::grpc::ServerContext* context auto model = this->modelManager.findModelByName(name); SPDLOG_DEBUG("ModelMetadata requested name: {}, version: {}", name, versionString); if (model == nullptr) { - SPDLOG_DEBUG("GetModelMetadata: Model {} is missing, trying to find pipeline with such name", name); - auto pipelineDefinition = this->modelManager.getPipelineFactory().findDefinitionByName(name); - if (!pipelineDefinition) { -#if (MEDIAPIPE_DISABLE == 0) - SPDLOG_DEBUG("GetModelMetadata: Pipeline {} is missing, trying to find mediapipe with such name", name); - auto mediapipeGraphDefinition = this->modelManager.getMediapipeFactory().findDefinitionByName(name); - if (!mediapipeGraphDefinition) { - return StatusCode::MODEL_NAME_MISSING; - } - auto status = buildResponse(*mediapipeGraphDefinition, response); - INCREMENT_IF_ENABLED(mediapipeGraphDefinition->getMetricReporter().getModelMetadataMetric(executionContext, status.ok())); - return status; -#else - return Status(StatusCode::MODEL_NAME_MISSING); -#endif + SPDLOG_DEBUG("GetModelMetadata: Model {} is missing, trying to find definition with such name", name); + auto* definition = this->modelManager.findServableDefinition(name); + if (!definition) { + return StatusCode::MODEL_NAME_MISSING; + } + auto* svsd = dynamic_cast(definition); + if (!svsd) { + return StatusCode::MODEL_NAME_MISSING; } - auto status = buildResponse(*pipelineDefinition, response); - INCREMENT_IF_ENABLED(pipelineDefinition->getMetricReporter().getModelMetadataMetric(executionContext, status.ok())); + auto status = buildResponse(*svsd, response); + INCREMENT_IF_ENABLED(svsd->getMetricReporter().getModelMetadataMetric(executionContext, status.ok())); return status; } std::shared_ptr instance = nullptr; @@ -369,24 +357,13 @@ Status KFSInferenceServiceImpl::buildResponse( } Status KFSInferenceServiceImpl::buildResponse( - PipelineDefinition& pipelineDefinition, - KFSGetModelStatusResponse* response) { - bool isReady = pipelineDefinition.getStatus().isAvailable(); - SPDLOG_DEBUG("Creating ModelReady response for pipeline: {}; ready: {}", pipelineDefinition.getName(), isReady); - response->set_ready(isReady); - return StatusCode::OK; -} - -#if (MEDIAPIPE_DISABLE == 0) -Status KFSInferenceServiceImpl::buildResponse( - MediapipeGraphDefinition& definition, + SingleVersionServableDefinition& definition, KFSGetModelStatusResponse* response) { bool isReady = definition.getStatus().isAvailable(); - SPDLOG_DEBUG("Creating ModelReady response for mediapipe: {}; ready: {}", definition.getName(), isReady); + SPDLOG_DEBUG("Creating ModelReady response for definition: {}; ready: {}", definition.getName(), isReady); response->set_ready(isReady); return StatusCode::OK; } -#endif static void addReadyVersions(Model& model, model_version_t versionAvailableDuringInitialCheck, @@ -443,61 +420,33 @@ KFSInferenceServiceImpl::KFSInferenceServiceImpl(const Server& server) : } Status KFSInferenceServiceImpl::buildResponse( - PipelineDefinition& pipelineDefinition, + SingleVersionServableDefinition& definition, KFSModelMetadataResponse* response) { - std::unique_ptr unloadGuard; + std::unique_ptr unloadGuard; // 0 meaning immediately return unload guard if possible, otherwise do not wait for available state - auto status = pipelineDefinition.waitForLoaded(unloadGuard, 0); + auto status = definition.waitForLoaded(unloadGuard, 0); if (!status.ok()) { return status; } response->Clear(); - response->set_name(pipelineDefinition.getName()); + response->set_name(definition.getName()); response->add_versions("1"); response->set_platform(PLATFORM); - for (const auto& input : pipelineDefinition.getInputsInfo()) { + for (const auto& input : definition.getInputsInfo()) { convert(input, response->add_inputs()); } - for (const auto& output : pipelineDefinition.getOutputsInfo()) { + for (const auto& output : definition.getOutputsInfo()) { convert(output, response->add_outputs()); } return StatusCode::OK; } -#if (MEDIAPIPE_DISABLE == 0) -Status KFSInferenceServiceImpl::buildResponse( - MediapipeGraphDefinition& mediapipeGraphDefinition, - KFSModelMetadataResponse* response) { - std::unique_ptr unloadGuard; - // 0 meaning immediately return unload guard if possible, otherwise do not wait for available state - auto status = mediapipeGraphDefinition.waitForLoaded(unloadGuard, 0); - if (!status.ok()) { - return status; - } - - response->Clear(); - response->set_name(mediapipeGraphDefinition.getName()); - response->add_versions("1"); - response->set_platform(PLATFORM); - - for (const auto& input : mediapipeGraphDefinition.getInputsInfo()) { - convert(input, response->add_inputs()); - } - - for (const auto& output : mediapipeGraphDefinition.getOutputsInfo()) { - convert(output, response->add_outputs()); - } - - return StatusCode::OK; -} -#endif - void KFSInferenceServiceImpl::convert( const std::pair>& from, KFSModelMetadataResponse::TensorMetadata* to) { diff --git a/src/kfs_frontend/kfs_grpc_inference_service.hpp b/src/kfs_frontend/kfs_grpc_inference_service.hpp index 2e787d8d47..ad5e34b756 100644 --- a/src/kfs_frontend/kfs_grpc_inference_service.hpp +++ b/src/kfs_frontend/kfs_grpc_inference_service.hpp @@ -20,7 +20,7 @@ #include #include -#include +#include #include "kfs_utils.hpp" #include "src/kfserving_api/grpc_predict_v2.grpc.pb.h" @@ -34,7 +34,6 @@ struct KFSModelExtraMetadata { namespace ovms { struct ExecutionContext; -class MediapipeGraphDefinition; class Model; class ModelInstance; class ModelInstanceUnloadGuard; @@ -42,9 +41,9 @@ class ModelManager; class ServableMetricReporter; class Pipeline; class Server; +class SingleVersionServableDefinition; class Status; class TensorInfo; -class PipelineDefinition; class KFSInferenceServiceImpl : public GRPCInferenceService::Service { protected: @@ -66,11 +65,9 @@ class KFSInferenceServiceImpl : public GRPCInferenceService::Service { ::grpc::Status ModelInfer(::grpc::ServerContext* context, const KFSRequest* request, KFSResponse* response) override; ::grpc::Status ModelStreamInfer(::grpc::ServerContext* context, ::grpc::ServerReaderWriter<::inference::ModelStreamInferResponse, ::inference::ModelInferRequest>* stream) override; static Status buildResponse(Model& model, ModelInstance& instance, KFSModelMetadataResponse* response, KFSModelExtraMetadata& extraMetadata); - static Status buildResponse(PipelineDefinition& pipelineDefinition, KFSModelMetadataResponse* response); + static Status buildResponse(SingleVersionServableDefinition& definition, KFSModelMetadataResponse* response); static Status buildResponse(std::shared_ptr instance, KFSGetModelStatusResponse* response); - static Status buildResponse(PipelineDefinition& pipelineDefinition, KFSGetModelStatusResponse* response); - static Status buildResponse(MediapipeGraphDefinition& pipelineDefinition, KFSGetModelStatusResponse* response); - static Status buildResponse(MediapipeGraphDefinition& mediapipeGraphDefinition, KFSModelMetadataResponse* response); + static Status buildResponse(SingleVersionServableDefinition& definition, KFSGetModelStatusResponse* response); static void convert(const std::pair>& from, KFSModelMetadataResponse::TensorMetadata* to); static Status getModelReady(const KFSGetModelStatusRequest* request, KFSGetModelStatusResponse* response, const ModelManager& manager, ExecutionContext executionContext); diff --git a/src/kfs_frontend/validation.hpp b/src/kfs_frontend/validation.hpp index 5dab096d64..b88e422b0a 100644 --- a/src/kfs_frontend/validation.hpp +++ b/src/kfs_frontend/validation.hpp @@ -24,7 +24,6 @@ #include "kfs_utils.hpp" #include "../precision.hpp" #include "../predict_request_validation_utils.hpp" -#include "../logging.hpp" #include "../profiler.hpp" #include "../tensorinfo.hpp" #include "../status.hpp" diff --git a/src/llm/BUILD b/src/llm/BUILD index ae37d936ca..de61450eb0 100644 --- a/src/llm/BUILD +++ b/src/llm/BUILD @@ -19,7 +19,8 @@ load("//:common_settings.bzl", "ovms_cc_library", "PYBIND_DEPS", "COPTS_PYTHON") ovms_cc_library( name = "llmcalculator", - srcs = ["http_llm_calculator.cc"], + srcs = ["http_llm_calculator.cc", + "llm_node_initializer.cpp"], deps = [ "//third_party:openvino", "@mediapipe//mediapipe/framework:calculator_framework", @@ -28,7 +29,10 @@ ovms_cc_library( "//src:libovmsprofiler", ":genai_servables", "//src:httppayload", - "//third_party:genai",], + "//third_party:genai", + "//src/mediapipe_internal:node_initializer", + "//src:libovmslogging", + "//src:libovmsstring_utils",], visibility = ["//visibility:public"], additional_copts = COPTS_PYTHON, alwayslink = 1, # needed, so the calculator can be registered by MediaPipe @@ -54,7 +58,7 @@ ovms_cc_library( "//src:libovmsprofiler", "//third_party:curl", "//src:image_conversion", - "//src:libovmsfilesystem", + "//src/filesystem:libovmsfilesystem", "@stb//:image", ":openai_request", ":output_parsers", @@ -226,10 +230,10 @@ ovms_cc_library( "//src:libmodelconfigjsonparser", "//src:libovmslogging", "//src:libovmsstatus", - "//src:libovmsmediapipe_utils", + "//src/mediapipe_internal:mediapipe_utils", "//src:libovmsprofiler", "//src:libovmsstring_utils", - "//src:libovmsfilesystem", + "//src/filesystem:libovmsfilesystem", "//src/tokenize:tokenize_parser", "llmcalculator_cc_proto", ":openai_completions_api_handler", diff --git a/src/llm/apis/openai_completions.cpp b/src/llm/apis/openai_completions.cpp index f8afc07134..4d4e45833b 100644 --- a/src/llm/apis/openai_completions.cpp +++ b/src/llm/apis/openai_completions.cpp @@ -25,11 +25,13 @@ #include #include +#include + #include "openai_json_response.hpp" #include "../../logging.hpp" #include "../../profiler.hpp" -#include "../../filesystem.hpp" +#include "src/filesystem/filesystem.hpp" #pragma warning(push) #pragma warning(disable : 6001 4324 6385 6386) #include "absl/strings/escaping.h" diff --git a/src/llm/llm_node_initializer.cpp b/src/llm/llm_node_initializer.cpp new file mode 100644 index 0000000000..b8ab217300 --- /dev/null +++ b/src/llm/llm_node_initializer.cpp @@ -0,0 +1,72 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include + +#include "src/mediapipe_internal/graph_side_packets.hpp" +#include "src/mediapipe_internal/node_initializer.hpp" +#include "src/stringutils.hpp" +#include "servable.hpp" +#include "servable_initializer.hpp" +#include "mediapipe/framework/calculator.pb.h" + +#include "src/logging.hpp" + +namespace ovms { +class LlmNodeInitializer : public NodeInitializer { + static constexpr const char* CALCULATOR_NAME = "LLMCalculator"; + +public: + bool matches(const std::string& calculatorName) const override { + return endsWith(calculatorName, CALCULATOR_NAME); + } + Status initialize( + const ::mediapipe::CalculatorGraphConfig_Node& nodeConfig, + const std::string& graphName, + const std::string& basePath, + GraphSidePackets& sidePackets, + PythonBackend* /*pythonBackend*/) override { + auto& genAiServableMap = sidePackets.genAiServableMap; + if (!nodeConfig.node_options().size()) { + SPDLOG_ERROR("LLM node missing options in graph: {}. ", graphName); + return StatusCode::LLM_NODE_MISSING_OPTIONS; + } + if (nodeConfig.name().empty()) { + SPDLOG_ERROR("LLM node name is missing in graph: {}. ", graphName); + return StatusCode::LLM_NODE_MISSING_NAME; + } + std::string nodeName = nodeConfig.name(); + if (genAiServableMap.find(nodeName) != genAiServableMap.end()) { + SPDLOG_ERROR("LLM node name: {} already used in graph: {}. ", nodeName, graphName); + return StatusCode::LLM_NODE_NAME_ALREADY_EXISTS; + } + std::shared_ptr servable; + Status status = initializeGenAiServable(servable, nodeConfig, basePath); + if (!status.ok()) { + SPDLOG_ERROR("Failed to process LLM node graph {}", graphName); + return status; + } + genAiServableMap.insert(std::pair>(nodeName, std::move(servable))); + return StatusCode::OK; + } +}; + +static bool llmNodeInitializerRegistered = []() { + NodeInitializerRegistry::instance().add(std::make_unique()); + return true; +}(); +} // namespace ovms diff --git a/src/llm/servable_initializer.cpp b/src/llm/servable_initializer.cpp index 27344ebb3b..3b36f1ef4d 100644 --- a/src/llm/servable_initializer.cpp +++ b/src/llm/servable_initializer.cpp @@ -39,7 +39,7 @@ #include "../logging.hpp" #include "../mediapipe_internal/mediapipe_utils.hpp" #include "../status.hpp" -#include "../filesystem.hpp" +#include "src/filesystem/filesystem.hpp" #include "../stringutils.hpp" #include "language_model/continuous_batching/servable.hpp" #include "language_model/continuous_batching/servable_initializer.hpp" diff --git a/src/logging.cpp b/src/logging.cpp index e89fce9a07..c0974c3a4e 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -15,6 +15,9 @@ //***************************************************************************** #include "logging.hpp" +#include +#include + #if (MEDIAPIPE_DISABLE == 0) #include #endif diff --git a/src/logging.hpp b/src/logging.hpp index 011458fe49..98b842f4d3 100644 --- a/src/logging.hpp +++ b/src/logging.hpp @@ -18,9 +18,6 @@ #include #include -#include -#include -#include #include namespace ovms { diff --git a/src/mediapipe_internal/BUILD b/src/mediapipe_internal/BUILD new file mode 100644 index 0000000000..a01a96b807 --- /dev/null +++ b/src/mediapipe_internal/BUILD @@ -0,0 +1,95 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +load("//:common_settings.bzl", "COPTS_PYTHON", "COPTS_MEDIAPIPE", "ovms_cc_library") + +ovms_cc_library( + name = "node_initializer", + srcs = [ + "node_initializer.cpp", + ], + hdrs = [ + "node_initializer.hpp", + "graph_side_packets.hpp", + ], + deps = [ + "//src:libovmsstatus", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "mediapipe_utils", + hdrs = [ + "mediapipe_utils.hpp", + "packettypes.hpp", + ], + srcs = [ + "mediapipe_utils.cpp", + ], + deps = [ + "//src:libovmsstring_utils", + "//src:libovmsstatus", + "//src:libovmslogging", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "libovms_mediapipe", + srcs = [ + "mediapipefactory.cpp", + "mediapipefactory.hpp", + "mediapipegraphconfig.hpp", + "mediapipegraphconfig.cpp", + "mediapipegraphdefinition.cpp", + "mediapipegraphdefinition.hpp", + "mediapipegraphexecutor.cpp", + "mediapipegraphexecutor.hpp", + ], + deps = select({ + "//:not_disable_python": [ + "//src/python:libovmspythonmodule", + ], + "//:disable_python": [] + }) + [ + ":node_initializer", + ":mediapipe_utils", + "//src/dags:pipelinedefinitionstatus", + "//src:libovms_single_version_servable_definition", + "//src:libovms_tensorinfo", + "//src:libovmslogging", + "//src:libovmsstatus", + "//src:model_metric_reporter", + "//src/metrics:libovmsmetrics", + "//src:libovms_ov_utils", + "//src:libovmsstring_utils", + "//src:libovms_servable_name_checker", + "//src/metrics:libovms_metric_provider", + "//src/filesystem:libovmsfilesystem", + "//src:libovms_version", + "//src:libovms_execution_context", + "//src:libovmstimer", + "//src:libovmsprofiler", + "@com_github_tencent_rapidjson//:rapidjson", + "//src/port:rapidjson_stringbuffer", + "//src/port:rapidjson_writer", + "@mediapipe//mediapipe/framework:calculator_graph", + "@mediapipe//mediapipe/framework/port:parse_text_proto", + ], + visibility = ["//visibility:public"], + additional_copts = COPTS_PYTHON + COPTS_MEDIAPIPE, + alwayslink = 1, +) diff --git a/src/mediapipe_internal/graph_side_packets.hpp b/src/mediapipe_internal/graph_side_packets.hpp new file mode 100644 index 0000000000..b9cdb147c9 --- /dev/null +++ b/src/mediapipe_internal/graph_side_packets.hpp @@ -0,0 +1,65 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once +#include +#include +#include + +namespace ovms { +class PythonNodeResources; +class GenAiServable; +struct ImageGenerationPipelines; +struct EmbeddingsServable; +struct RerankServable; +struct SttServable; +class TtsServable; + +using PythonNodeResourcesMap = std::unordered_map>; +using GenAiServableMap = std::unordered_map>; +using RerankServableMap = std::unordered_map>; +using SttServableMap = std::unordered_map>; +using TtsServableMap = std::unordered_map>; +using EmbeddingsServableMap = std::unordered_map>; +using ImageGenerationPipelinesMap = std::unordered_map>; + +struct GraphSidePackets { + PythonNodeResourcesMap pythonNodeResourcesMap; + GenAiServableMap genAiServableMap; + ImageGenerationPipelinesMap imageGenPipelinesMap; + EmbeddingsServableMap embeddingsServableMap; + RerankServableMap rerankServableMap; + SttServableMap sttServableMap; + TtsServableMap ttsServableMap; + void clear() { + pythonNodeResourcesMap.clear(); + genAiServableMap.clear(); + imageGenPipelinesMap.clear(); + embeddingsServableMap.clear(); + rerankServableMap.clear(); + sttServableMap.clear(); + ttsServableMap.clear(); + } + bool empty() { + return (pythonNodeResourcesMap.empty() && + genAiServableMap.empty() && + imageGenPipelinesMap.empty() && + embeddingsServableMap.empty() && + rerankServableMap.empty() && + sttServableMap.empty() && + ttsServableMap.empty()); + } +}; +} // namespace ovms diff --git a/src/mediapipe_internal/mediapipefactory.cpp b/src/mediapipe_internal/mediapipefactory.cpp index aa3689ae31..21dcdb89da 100644 --- a/src/mediapipe_internal/mediapipefactory.cpp +++ b/src/mediapipe_internal/mediapipefactory.cpp @@ -25,16 +25,9 @@ #include #include #include -#pragma warning(push) -#pragma warning(disable : 6001 4324 6385 6326 6308 6387 6246) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop -#pragma warning(pop) -#include "../kfs_frontend/kfs_grpc_inference_service.hpp" #include "../logging.hpp" -#include "../modelmanager.hpp" +#include "src/metrics/metric_provider.hpp" +#include "../servable_name_checker.hpp" #include "../status.hpp" #include "../stringutils.hpp" #pragma warning(push) @@ -62,13 +55,14 @@ MediapipeFactory::MediapipeFactory(PythonBackend* pythonBackend) { Status MediapipeFactory::createDefinition(const std::string& pipelineName, const MediapipeGraphConfig& config, - ModelManager& manager) { + MetricProvider& metrics, + const ServableNameChecker& checker) { if (definitionExists(pipelineName)) { SPDLOG_LOGGER_ERROR(modelmanager_logger, "Mediapipe graph definition: {} is already created", pipelineName); return StatusCode::PIPELINE_DEFINITION_ALREADY_EXIST; } - std::shared_ptr graphDefinition = std::make_shared(pipelineName, config, manager.getMetricRegistry(), &manager.getMetricConfig(), pythonBackend); - auto stat = graphDefinition->validate(manager); + std::shared_ptr graphDefinition = std::make_shared(pipelineName, config, metrics.getMetricRegistry(), &metrics.getMetricConfig(), pythonBackend); + auto stat = graphDefinition->validate(checker); if (stat.getCode() == StatusCode::MEDIAPIPE_GRAPH_NAME_OCCUPIED) { return stat; } @@ -94,19 +88,18 @@ MediapipeGraphDefinition* MediapipeFactory::findDefinitionByName(const std::stri Status MediapipeFactory::reloadDefinition(const std::string& name, const MediapipeGraphConfig& config, - ModelManager& manager) { + const ServableNameChecker& checker) { auto mgd = findDefinitionByName(name); if (mgd == nullptr) { SPDLOG_LOGGER_ERROR(modelmanager_logger, "Requested to reload mediapipe graph definition but it does not exist: {}", name); return StatusCode::INTERNAL_ERROR; } SPDLOG_LOGGER_INFO(modelmanager_logger, "Reloading mediapipe graph: {}", name); - return mgd->reload(manager, config); + return mgd->reload(checker, config); } Status MediapipeFactory::create(std::unique_ptr& pipeline, - const std::string& name, - ModelManager& manager) const { + const std::string& name) const { std::shared_lock lock(definitionsMtx); auto it = definitions.find(name); if (it == definitions.end()) { @@ -117,17 +110,17 @@ Status MediapipeFactory::create(std::unique_ptr& pipelin return definition.create(pipeline); } -void MediapipeFactory::retireOtherThan(std::set&& graphsInConfigFile, ModelManager& manager) { +void MediapipeFactory::retireOtherThan(std::set&& graphsInConfigFile) { std::for_each(definitions.begin(), definitions.end(), - [&graphsInConfigFile, &manager](auto& nameDefinitionPair) { + [&graphsInConfigFile](auto& nameDefinitionPair) { if (graphsInConfigFile.find(nameDefinitionPair.second->getName()) == graphsInConfigFile.end() && nameDefinitionPair.second->getStateCode() != PipelineDefinitionStateCode::RETIRED) { - nameDefinitionPair.second->retire(manager); + nameDefinitionPair.second->retire(); } }); } -Status MediapipeFactory::revalidatePipelines(ModelManager&) { +Status MediapipeFactory::revalidatePipelines() { SPDLOG_LOGGER_WARN(modelmanager_logger, "revalidation of mediapipe graphs not implemented yet"); return StatusCode::OK; } diff --git a/src/mediapipe_internal/mediapipefactory.hpp b/src/mediapipe_internal/mediapipefactory.hpp index e48146b0f0..0d03fcc7a4 100644 --- a/src/mediapipe_internal/mediapipefactory.hpp +++ b/src/mediapipe_internal/mediapipefactory.hpp @@ -25,18 +25,10 @@ #include #include -#pragma warning(push) -#pragma warning(disable : 6001 4324 6308 6387 6246) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wall" -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" -#pragma GCC diagnostic pop -#pragma warning(pop) -#include "../kfs_frontend/kfs_grpc_inference_service.hpp" - namespace ovms { -class ModelManager; +class MetricProvider; +class ServableNameChecker; class Status; class MediapipeGraphConfig; class MediapipeGraphDefinition; @@ -53,30 +45,22 @@ class MediapipeFactory { MediapipeFactory(PythonBackend* pythonBackend = nullptr); Status createDefinition(const std::string& pipelineName, const MediapipeGraphConfig& config, - ModelManager& manager); + MetricProvider& metrics, + const ServableNameChecker& checker); bool definitionExists(const std::string& name) const; -private: - template - Status createInternal(std::unique_ptr& pipeline, - const std::string& name, - const RequestType* request, - ResponseType* response, - ModelManager& manager) const; - public: Status create(std::unique_ptr& pipeline, - const std::string& name, - ModelManager& manager) const; + const std::string& name) const; MediapipeGraphDefinition* findDefinitionByName(const std::string& name) const; Status reloadDefinition(const std::string& pipelineName, const MediapipeGraphConfig& config, - ModelManager& manager); + const ServableNameChecker& checker); - void retireOtherThan(std::set&& pipelinesInConfigFile, ModelManager& manager); - Status revalidatePipelines(ModelManager&); + void retireOtherThan(std::set&& pipelinesInConfigFile); + Status revalidatePipelines(); const std::vector getMediapipePipelinesNames() const; const std::vector getNamesOfAvailableMediapipePipelines() const; ~MediapipeFactory(); diff --git a/src/mediapipe_internal/mediapipegraphconfig.cpp b/src/mediapipe_internal/mediapipegraphconfig.cpp index 448da4e1b8..d767942535 100644 --- a/src/mediapipe_internal/mediapipegraphconfig.cpp +++ b/src/mediapipe_internal/mediapipegraphconfig.cpp @@ -24,7 +24,7 @@ #pragma warning(pop) #include -#include "../filesystem.hpp" +#include "src/filesystem/filesystem.hpp" #include "../status.hpp" namespace ovms { diff --git a/src/mediapipe_internal/mediapipegraphdefinition.cpp b/src/mediapipe_internal/mediapipegraphdefinition.cpp index 9047765e75..81cd0ce725 100644 --- a/src/mediapipe_internal/mediapipegraphdefinition.cpp +++ b/src/mediapipe_internal/mediapipegraphdefinition.cpp @@ -25,19 +25,12 @@ #include #include "../execution_context.hpp" -#include "../filesystem.hpp" -#include "../kfs_frontend/kfs_utils.hpp" -#include "../kfs_frontend/kfs_request_utils.hpp" -#include "../deserialization_main.hpp" -#include "../metric.hpp" +#include "src/filesystem/filesystem.hpp" +#include "src/metrics/metric.hpp" #include "../model_metric_reporter.hpp" -#include "../modelmanager.hpp" #include "../ov_utils.hpp" -#include "../llm/servable.hpp" -#include "../llm/servable_initializer.hpp" -#if (PYTHON_DISABLE == 0) -#include "../python/pythonnoderesources.hpp" -#endif +#include "../servable_definition_unload_guard.hpp" +#include "../servable_name_checker.hpp" #include "../status.hpp" #include "../stringutils.hpp" #include "../tensorinfo.hpp" @@ -47,24 +40,12 @@ #include "mediapipe/framework/port/status.h" #include "mediapipe_utils.hpp" #include "mediapipegraphexecutor.hpp" -#include "src/embeddings/embeddings_calculator_ov.pb.h" -#include "src/rerank/rerank_calculator_ov.pb.h" - -#include "src/image_gen/pipelines.hpp" -#include "src/image_gen/imagegen_init.hpp" -#include "src/image_gen/image_gen_calculator.pb.h" +#include "node_initializer.hpp" namespace ovms { MediapipeGraphConfig MediapipeGraphDefinition::MGC; const std::string MediapipeGraphDefinition::SCHEDULER_CLASS_NAME{"Mediapipe"}; -const std::string MediapipeGraphDefinition::PYTHON_NODE_CALCULATOR_NAME{"PythonExecutorCalculator"}; -const std::string MediapipeGraphDefinition::LLM_NODE_CALCULATOR_NAME{"LLMCalculator"}; -const std::string MediapipeGraphDefinition::IMAGE_GEN_CALCULATOR_NAME{"ImageGenCalculator"}; -const std::string MediapipeGraphDefinition::STT_NODE_CALCULATOR_NAME{"S2tCalculator"}; -const std::string MediapipeGraphDefinition::TTS_NODE_CALCULATOR_NAME{"T2sCalculator"}; -const std::string MediapipeGraphDefinition::EMBEDDINGS_NODE_CALCULATOR_NAME{"EmbeddingsCalculatorOV"}; -const std::string MediapipeGraphDefinition::RERANK_NODE_CALCULATOR_NAME{"RerankCalculatorOV"}; MediapipeGraphDefinition::~MediapipeGraphDefinition() = default; @@ -127,14 +108,14 @@ Status MediapipeGraphDefinition::dryInitializeTest() { } return StatusCode::OK; } -Status MediapipeGraphDefinition::validate(ModelManager& manager) { +Status MediapipeGraphDefinition::validate(const ServableNameChecker& checker) { SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Started validation of mediapipe: {}", getName()); if (!this->sidePacketMaps.empty()) { SPDLOG_ERROR("Internal Error: MediaPipe definition is in unexpected state."); return StatusCode::INTERNAL_ERROR; } ValidationResultNotifier notifier(this->status, this->loadedNotify); - if (manager.modelExists(this->getName()) || manager.pipelineDefinitionExists(this->getName())) { + if (checker.servableExists(this->getName(), ServableType::Model | ServableType::Pipeline)) { SPDLOG_LOGGER_ERROR(modelmanager_logger, "Mediapipe graph name: {} is already occupied by model or pipeline.", this->getName()); return StatusCode::MEDIAPIPE_GRAPH_NAME_OCCUPIED; } @@ -192,8 +173,8 @@ MediapipeGraphDefinition::MediapipeGraphDefinition(const std::string name, MetricRegistry* registry, const MetricConfig* metricConfig, PythonBackend* pythonBackend) : - name(name), - status(SCHEDULER_CLASS_NAME, this->name), + SingleVersionServableDefinition(name), + status(SCHEDULER_CLASS_NAME, getName()), pythonBackend(pythonBackend), reporter(std::make_unique(metricConfig, registry, name)) { mgconfig = config; @@ -254,7 +235,7 @@ Status MediapipeGraphDefinition::createOutputsInfo() { } Status MediapipeGraphDefinition::create(std::unique_ptr& pipeline) { - std::unique_ptr unloadGuard; + std::unique_ptr unloadGuard; Status status = waitForLoaded(unloadGuard); if (!status.ok()) { SPDLOG_DEBUG("Failed to execute mediapipe graph: {} since it is not available", getName()); @@ -332,7 +313,7 @@ Status MediapipeGraphDefinition::setStreamTypes() { return StatusCode::OK; } -Status MediapipeGraphDefinition::reload(ModelManager& manager, const MediapipeGraphConfig& config) { +Status MediapipeGraphDefinition::reload(const ServableNameChecker& checker, const MediapipeGraphConfig& config) { // block creating new unloadGuards this->status.handle(ReloadEvent()); while (requestsHandlesCounter > 0) { @@ -340,10 +321,10 @@ Status MediapipeGraphDefinition::reload(ModelManager& manager, const MediapipeGr } this->mgconfig = config; this->sidePacketMaps.clear(); - return validate(manager); + return validate(checker); } -void MediapipeGraphDefinition::retire(ModelManager& manager) { +void MediapipeGraphDefinition::retire() { this->sidePacketMaps.clear(); this->status.handle(RetireEvent()); } @@ -356,276 +337,38 @@ bool MediapipeGraphDefinition::isReloadRequired(const MediapipeGraphConfig& conf return getMediapipeGraphConfig().isReloadRequired(config); } -Status MediapipeGraphDefinition::waitForLoaded(std::unique_ptr& unloadGuard, const uint32_t waitForLoadedTimeoutMicroseconds) { - unloadGuard = std::make_unique(*this); - - const uint32_t waitLoadedTimestepMicroseconds = 1000; - const uint32_t waitCheckpoints = waitForLoadedTimeoutMicroseconds / waitLoadedTimestepMicroseconds; - uint32_t waitCheckpointsCounter = waitCheckpoints; - std::mutex cvMtx; - std::unique_lock cvLock(cvMtx); - while (waitCheckpointsCounter-- != 0) { - if (status.isAvailable()) { - SPDLOG_DEBUG("Successfully waited for mediapipe definition: {}", getName()); - return StatusCode::OK; - } - unloadGuard.reset(); - if (!status.canEndLoaded()) { - if (status.getStateCode() != PipelineDefinitionStateCode::RETIRED) { - SPDLOG_DEBUG("Waiting for mediapipe definition: {} ended due to timeout.", getName()); - return StatusCode::MEDIAPIPE_DEFINITION_NOT_LOADED_YET; - } else { - SPDLOG_DEBUG("Waiting for mediapipe definition: {} ended since it failed to load.", getName()); - return StatusCode::MEDIAPIPE_DEFINITION_NOT_LOADED_ANYMORE; - } - } - SPDLOG_DEBUG("Waiting for available state for mediapipe: {}, with timestep: {}us timeout: {}us check count: {}", - getName(), waitLoadedTimestepMicroseconds, waitForLoadedTimeoutMicroseconds, waitCheckpointsCounter); - loadedNotify.wait_for(cvLock, - std::chrono::microseconds(waitLoadedTimestepMicroseconds), - [this]() { - return this->status.isAvailable() || - !this->status.canEndLoaded(); - }); - unloadGuard = std::make_unique(*this); - } - if (!status.isAvailable()) { - if (status.getStateCode() != PipelineDefinitionStateCode::RETIRED) { - SPDLOG_DEBUG("Waiting for mediapipe definition: {} ended due to timeout.", getName()); - return StatusCode::MEDIAPIPE_DEFINITION_NOT_LOADED_YET; - } else { - SPDLOG_DEBUG("Waiting for mediapipe definition: {} ended since it failed to load.", getName()); - return StatusCode::MEDIAPIPE_DEFINITION_NOT_LOADED_ANYMORE; - } - } - SPDLOG_DEBUG("Successfully waited for mediapipe definition: {}", getName()); - return StatusCode::OK; +StatusCode MediapipeGraphDefinition::notLoadedYetCode() const { + return StatusCode::MEDIAPIPE_DEFINITION_NOT_LOADED_YET; } -template -class ResourcesCleaningGuard { -public: - bool shouldCleanup{true}; - T& resources; - ResourcesCleaningGuard(T& resources) : - resources(resources) {} - ~ResourcesCleaningGuard() { - if (shouldCleanup) { - resources.clear(); - } - } - void disableCleaning() { - shouldCleanup = false; - } -}; +StatusCode MediapipeGraphDefinition::notLoadedAnymoreCode() const { + return StatusCode::MEDIAPIPE_DEFINITION_NOT_LOADED_ANYMORE; +} Status MediapipeGraphDefinition::initializeNodes() { SPDLOG_INFO("MediapipeGraphDefinition initializing graph nodes"); - for (int i = 0; i < config.node().size(); i++) { -#if (PYTHON_DISABLE == 0) - auto& pythonNodeResourcesMap = this->sidePacketMaps.pythonNodeResourcesMap; - if (config.node(i).calculator() == PYTHON_NODE_CALCULATOR_NAME) { - ResourcesCleaningGuard pythonResourcesCleaningGuard(pythonNodeResourcesMap); - if (!config.node(i).node_options().size()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Python node missing options in graph: {}. ", this->name); - return StatusCode::PYTHON_NODE_MISSING_OPTIONS; - } - if (config.node(i).name().empty()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Python node name is missing in graph: {}. ", this->name); - return StatusCode::PYTHON_NODE_MISSING_NAME; - } - std::string nodeName = config.node(i).name(); - if (pythonNodeResourcesMap.find(nodeName) != pythonNodeResourcesMap.end()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Python node name: {} already used in graph: {}. ", nodeName, this->name); - return StatusCode::PYTHON_NODE_NAME_ALREADY_EXISTS; - } - - std::shared_ptr nodeResources = nullptr; - Status status = PythonNodeResources::createPythonNodeResources(nodeResources, config.node(i), pythonBackend, mgconfig.getBasePath()); - if (nodeResources == nullptr || !status.ok()) { - SPDLOG_ERROR("Failed to process python node graph {}", this->name); - return status; - } - - pythonNodeResourcesMap.insert(std::pair>(nodeName, std::move(nodeResources))); - pythonResourcesCleaningGuard.disableCleaning(); - } -#endif - // Passed to both calculators that require LLM Engine (gRPC KServe & HTTP OpenAI) - if (endsWith(config.node(i).calculator(), LLM_NODE_CALCULATOR_NAME)) { - auto& genAiServableMap = this->sidePacketMaps.genAiServableMap; - ResourcesCleaningGuard genAiServablesCleaningGuard(genAiServableMap); - if (!config.node(i).node_options().size()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "LLM node missing options in graph: {}. ", this->name); - return StatusCode::LLM_NODE_MISSING_OPTIONS; - } - if (config.node(i).name().empty()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "LLM node name is missing in graph: {}. ", this->name); - return StatusCode::LLM_NODE_MISSING_NAME; - } - std::string nodeName = config.node(i).name(); - if (genAiServableMap.find(nodeName) != genAiServableMap.end()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "LLM node name: {} already used in graph: {}. ", nodeName, this->name); - return StatusCode::LLM_NODE_NAME_ALREADY_EXISTS; - } - std::shared_ptr servable; - Status status = initializeGenAiServable(servable, config.node(i), mgconfig.getBasePath()); - if (!status.ok()) { - SPDLOG_ERROR("Failed to process LLM node graph {}", this->name); - return status; - } - genAiServableMap.insert(std::pair>(nodeName, std::move(servable))); - genAiServablesCleaningGuard.disableCleaning(); + bool success = false; + struct CleanupGuard { + GraphSidePackets& maps; + bool& success; + ~CleanupGuard() { + if (!success) + maps.clear(); } - // Passed to both calculators that require Image Generation pipelines - if (endsWith(config.node(i).calculator(), IMAGE_GEN_CALCULATOR_NAME)) { - auto& imageGenPipelinesMap = this->sidePacketMaps.imageGenPipelinesMap; - ResourcesCleaningGuard guard(imageGenPipelinesMap); - if (!config.node(i).node_options().size()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Image Gen node missing options in graph: {}. ", this->name); - return StatusCode::LLM_NODE_MISSING_OPTIONS; // TODO: create new error code - } - if (config.node(i).name().empty()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Image Gen node name is missing in graph: {}. ", this->name); - return StatusCode::LLM_NODE_MISSING_NAME; - } - std::string nodeName = config.node(i).name(); - if (imageGenPipelinesMap.find(nodeName) != imageGenPipelinesMap.end()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Image Gen node name: {} already used in graph: {}. ", nodeName, this->name); - return StatusCode::LLM_NODE_NAME_ALREADY_EXISTS; - } - auto statusOrArgs = prepareImageGenPipelineArgs(config.node(i).node_options(0), mgconfig.getBasePath()); - if (std::holds_alternative(statusOrArgs)) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to prepare Image Gen pipeline args for node: {}. Error: {}", this->name, std::get(statusOrArgs).string()); - return std::get(statusOrArgs); - } - std::shared_ptr servable; - try { - servable = std::make_shared(std::get(statusOrArgs)); - } catch (ov::Exception& e) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to create Image Generation pipelines: {}. Error: {}", this->name, e.what()); - return StatusCode::INTERNAL_ERROR; - } catch (...) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to create Image Generation pipelines: {}. Unknown error", this->name); - return StatusCode::INTERNAL_ERROR; - } - imageGenPipelinesMap.insert(std::pair>(nodeName, std::move(servable))); - guard.disableCleaning(); - } - if (endsWith(config.node(i).calculator(), EMBEDDINGS_NODE_CALCULATOR_NAME)) { - auto& embeddingsServableMap = this->sidePacketMaps.embeddingsServableMap; - ResourcesCleaningGuard embeddingsServablesCleaningGuard(embeddingsServableMap); - if (!config.node(i).node_options().size()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Embeddings node missing options in graph: {}. ", this->name); - return StatusCode::LLM_NODE_MISSING_OPTIONS; - } - if (config.node(i).name().empty()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Embeddings node name is missing in graph: {}. ", this->name); - return StatusCode::LLM_NODE_MISSING_NAME; - } - std::string nodeName = config.node(i).name(); - if (embeddingsServableMap.find(nodeName) != embeddingsServableMap.end()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Embeddings node name: {} already used in graph: {}. ", nodeName, this->name); - return StatusCode::LLM_NODE_NAME_ALREADY_EXISTS; - } - mediapipe::EmbeddingsCalculatorOVOptions nodeOptions; - config.node(i).node_options(0).UnpackTo(&nodeOptions); - std::shared_ptr servable = std::make_shared( - nodeOptions.models_path(), - nodeOptions.target_device(), - nodeOptions.plugin_config(), - mgconfig.getBasePath(), - nodeOptions.pooling(), - nodeOptions.normalize_embeddings()); - servable->initialize( - nodeOptions.models_path(), - nodeOptions.target_device(), - nodeOptions.plugin_config(), - mgconfig.getBasePath()); - embeddingsServableMap.insert(std::pair>(nodeName, std::move(servable))); - embeddingsServablesCleaningGuard.disableCleaning(); - } - if (endsWith(config.node(i).calculator(), RERANK_NODE_CALCULATOR_NAME)) { - auto& rerankServableMap = this->sidePacketMaps.rerankServableMap; - ResourcesCleaningGuard rerankServablesCleaningGuard(rerankServableMap); - if (!config.node(i).node_options().size()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Rerank node missing options in graph: {}. ", this->name); - return StatusCode::LLM_NODE_MISSING_OPTIONS; - } - if (config.node(i).name().empty()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Rerank node name is missing in graph: {}. ", this->name); - return StatusCode::LLM_NODE_MISSING_NAME; - } - std::string nodeName = config.node(i).name(); - if (rerankServableMap.find(nodeName) != rerankServableMap.end()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Rerank node name: {} already used in graph: {}. ", nodeName, this->name); - return StatusCode::LLM_NODE_NAME_ALREADY_EXISTS; - } - mediapipe::RerankCalculatorOVOptions nodeOptions; - config.node(i).node_options(0).UnpackTo(&nodeOptions); - std::shared_ptr servable = std::make_shared(nodeOptions.models_path(), nodeOptions.target_device(), nodeOptions.plugin_config(), mgconfig.getBasePath()); - servable->initialize(nodeOptions.models_path(), nodeOptions.target_device(), nodeOptions.plugin_config(), mgconfig.getBasePath()); - rerankServableMap.insert(std::pair>(nodeName, std::move(servable))); - rerankServablesCleaningGuard.disableCleaning(); - } - if (endsWith(config.node(i).calculator(), STT_NODE_CALCULATOR_NAME)) { - auto& sttServableMap = this->sidePacketMaps.sttServableMap; - ResourcesCleaningGuard sttServablesCleaningGuard(sttServableMap); - if (!config.node(i).node_options().size()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "SpeechToText node missing options in graph: {}. ", this->name); - return StatusCode::LLM_NODE_MISSING_OPTIONS; - } - if (config.node(i).name().empty()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "SpeechToText node name is missing in graph: {}. ", this->name); - return StatusCode::LLM_NODE_MISSING_NAME; - } - std::string nodeName = config.node(i).name(); - if (sttServableMap.find(nodeName) != sttServableMap.end()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "SpeechToText node name: {} already used in graph: {}. ", nodeName, this->name); - return StatusCode::LLM_NODE_NAME_ALREADY_EXISTS; - } - mediapipe::S2tCalculatorOptions nodeOptions; - auto& calculatorOptions = config.node(i).node_options(0); - if (!calculatorOptions.UnpackTo(&nodeOptions)) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to unpack calculator options"); - return StatusCode::MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID; - } - std::shared_ptr servable = std::make_shared(nodeOptions, mgconfig.getBasePath()); - sttServableMap.insert(std::pair>(nodeName, std::move(servable))); - sttServablesCleaningGuard.disableCleaning(); - } - if (endsWith(config.node(i).calculator(), TTS_NODE_CALCULATOR_NAME)) { - auto& ttsServableMap = this->sidePacketMaps.ttsServableMap; - ResourcesCleaningGuard ttsServablesCleaningGuard(ttsServableMap); - if (!config.node(i).node_options().size()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "TextToSpeech node missing options in graph: {}. ", this->name); - return StatusCode::LLM_NODE_MISSING_OPTIONS; - } - if (config.node(i).name().empty()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "TextToSpeech node name is missing in graph: {}. ", this->name); - return StatusCode::LLM_NODE_MISSING_NAME; - } - std::string nodeName = config.node(i).name(); - if (ttsServableMap.find(nodeName) != ttsServableMap.end()) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "TextToSpeech node name: {} already used in graph: {}. ", nodeName, this->name); - return StatusCode::LLM_NODE_NAME_ALREADY_EXISTS; - } - mediapipe::T2sCalculatorOptions nodeOptions; - auto& calculatorOptions = config.node(i).node_options(0); - if (!calculatorOptions.UnpackTo(&nodeOptions)) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to unpack calculator options"); - return StatusCode::MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID; - } - try { - std::shared_ptr servable = std::make_shared(nodeOptions.models_path(), nodeOptions.target_device(), nodeOptions.voices(), nodeOptions.plugin_config(), mgconfig.getBasePath()); - ttsServableMap.insert(std::pair>(nodeName, std::move(servable))); - ttsServablesCleaningGuard.disableCleaning(); - } catch (const std::runtime_error& e) { - SPDLOG_LOGGER_ERROR(modelmanager_logger, "TextToSpeech node name: {} initialization failed: {}. ", nodeName, e.what()); - return StatusCode::MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID; + } guard{sidePacketMaps, success}; + + auto& registry = NodeInitializerRegistry::instance(); + for (int i = 0; i < config.node().size(); i++) { + for (const auto& initializer : registry.all()) { + if (initializer->matches(config.node(i).calculator())) { + Status status = initializer->initialize(config.node(i), getName(), mgconfig.getBasePath(), sidePacketMaps, pythonBackend); + if (!status.ok()) { + return status; + } } } } + success = true; return StatusCode::OK; } } // namespace ovms diff --git a/src/mediapipe_internal/mediapipegraphdefinition.hpp b/src/mediapipe_internal/mediapipegraphdefinition.hpp index 14c9e0679f..32b01a9266 100644 --- a/src/mediapipe_internal/mediapipegraphdefinition.hpp +++ b/src/mediapipe_internal/mediapipegraphdefinition.hpp @@ -14,92 +14,41 @@ // limitations under the License. //***************************************************************************** #pragma once -#include #include #include #include -#include #include #include #include #include #include "../dags/pipelinedefinitionstatus.hpp" -#include "../kfs_frontend/kfs_grpc_inference_service.hpp" -#include "../kfs_frontend/kfs_utils.hpp" -#include "../metric.hpp" -#include "../tensorinfo.hpp" +#include "src/metrics/metric.hpp" +#include "../model_metric_reporter.hpp" +#include "../single_version_servable_definition.hpp" +#include "../tensorinfo_fwd.hpp" #pragma warning(push) #pragma warning(disable : 4005 4309 6001 6385 6386 6326 6011 4005 4456 6246) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" #include "mediapipe/framework/calculator_graph.h" -#include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status.h" #pragma GCC diagnostic pop #pragma warning(pop) #include "mediapipegraphconfig.hpp" +#include "graph_side_packets.hpp" #include "packettypes.hpp" -#include "../sidepacket_servable.hpp" -#include "../embeddings/embeddings_servable.hpp" -#include "../rerank/rerank_servable.hpp" -#include "../audio/speech_to_text/s2t_servable.hpp" -#include "../audio/text_to_speech/t2s_servable.hpp" - namespace ovms { -class MediapipeGraphDefinitionUnloadGuard; class MetricConfig; class MetricRegistry; -class MediapipeServableMetricReporter; -class ModelManager; +class ServableNameChecker; class MediapipeGraphExecutor; class Status; class PythonBackend; -class PythonNodeResources; -class GenAiServable; -struct ImageGenerationPipelines; -using PythonNodeResourcesMap = std::unordered_map>; -using GenAiServableMap = std::unordered_map>; -using RerankServableMap = std::unordered_map>; -using SttServableMap = std::unordered_map>; -using TtsServableMap = std::unordered_map>; -using EmbeddingsServableMap = std::unordered_map>; -using ImageGenerationPipelinesMap = std::unordered_map>; - -struct GraphSidePackets { - PythonNodeResourcesMap pythonNodeResourcesMap; - GenAiServableMap genAiServableMap; - ImageGenerationPipelinesMap imageGenPipelinesMap; - EmbeddingsServableMap embeddingsServableMap; - RerankServableMap rerankServableMap; - SttServableMap sttServableMap; - TtsServableMap ttsServableMap; - void clear() { - pythonNodeResourcesMap.clear(); - genAiServableMap.clear(); - imageGenPipelinesMap.clear(); - embeddingsServableMap.clear(); - rerankServableMap.clear(); - sttServableMap.clear(); - ttsServableMap.clear(); - } - bool empty() { - return (pythonNodeResourcesMap.empty() && - genAiServableMap.empty() && - imageGenPipelinesMap.empty() && - embeddingsServableMap.empty() && - rerankServableMap.empty() && - sttServableMap.empty() && - ttsServableMap.empty()); - } -}; - -class MediapipeGraphDefinition { - friend MediapipeGraphDefinitionUnloadGuard; +class MediapipeGraphDefinition : public SingleVersionServableDefinition { public: virtual ~MediapipeGraphDefinition(); MediapipeGraphDefinition(const std::string name, @@ -108,38 +57,25 @@ class MediapipeGraphDefinition { const MetricConfig* metricConfig = nullptr, PythonBackend* pythonBackend = nullptr); - const std::string& getName() const { return name; } - const PipelineDefinitionStatus& getStatus() const { + const PipelineDefinitionStatus& getStatus() const override { return this->status; } const PipelineDefinitionStateCode getStateCode() const { return status.getStateCode(); } - const model_version_t getVersion() const { return VERSION; } - const tensor_map_t getInputsInfo() const; - const tensor_map_t getOutputsInfo() const; + bool isAvailable() const override { return status.isAvailable(); } + const tensor_map_t getInputsInfo() const override; + const tensor_map_t getOutputsInfo() const override; const MediapipeGraphConfig& getMediapipeGraphConfig() const { return this->mgconfig; } - MediapipeServableMetricReporter& getMetricReporter() const { return *this->reporter; } + MediapipeServableMetricReporter& getMetricReporter() const override { return *this->reporter; } Status create(std::unique_ptr& pipeline); - Status reload(ModelManager& manager, const MediapipeGraphConfig& config); - Status validate(ModelManager& manager); - void retire(ModelManager& manager); + Status reload(const ServableNameChecker& checker, const MediapipeGraphConfig& config); + Status validate(const ServableNameChecker& checker); + void retire(); Status initializeNodes(); bool isReloadRequired(const MediapipeGraphConfig& config) const; - static constexpr uint64_t WAIT_FOR_LOADED_DEFAULT_TIMEOUT_MICROSECONDS = 500000; static const std::string SCHEDULER_CLASS_NAME; - static const std::string PYTHON_NODE_CALCULATOR_NAME; - static const std::string LLM_NODE_CALCULATOR_NAME; - static const std::string IMAGE_GEN_CALCULATOR_NAME; - static const std::string EMBEDDINGS_NODE_CALCULATOR_NAME; - static const std::string RERANK_NODE_CALCULATOR_NAME; - static const std::string STT_NODE_CALCULATOR_NAME; - static const std::string TTS_NODE_CALCULATOR_NAME; - Status waitForLoaded(std::unique_ptr& unloadGuard, const uint32_t waitForLoadedTimeoutMicroseconds = WAIT_FOR_LOADED_DEFAULT_TIMEOUT_MICROSECONDS); - - // Pipelines are not versioned and any available definition has constant version equal 1. - static constexpr model_version_t VERSION = 1; protected: GraphSidePackets sidePacketMaps; @@ -171,7 +107,6 @@ class MediapipeGraphDefinition { Status dryInitializeTest(); std::string chosenConfig; static MediapipeGraphConfig MGC; - const std::string name; bool passKfsRequestFlag; std::unordered_map inputTypes; @@ -185,17 +120,11 @@ class MediapipeGraphDefinition { Status createOutputsInfo(); Status createInputSidePacketsInfo(); - std::condition_variable loadedNotify; mutable std::shared_mutex metadataMtx; private: - void increaseRequestsHandlesCount() { - ++requestsHandlesCounter; - } - - void decreaseRequestsHandlesCount() { - --requestsHandlesCounter; - } + StatusCode notLoadedYetCode() const override; + StatusCode notLoadedAnymoreCode() const override; tensor_map_t inputsInfo; tensor_map_t outputsInfo; @@ -204,25 +133,8 @@ class MediapipeGraphDefinition { std::vector outputNames; std::vector inputSidePacketNames; - std::atomic requestsHandlesCounter = 0; - PythonBackend* pythonBackend; std::unique_ptr reporter; }; - -class MediapipeGraphDefinitionUnloadGuard { -public: - MediapipeGraphDefinitionUnloadGuard(MediapipeGraphDefinition& definition) : - definition(definition) { - definition.increaseRequestsHandlesCount(); - } - - ~MediapipeGraphDefinitionUnloadGuard() { - definition.decreaseRequestsHandlesCount(); - } - -private: - MediapipeGraphDefinition& definition; -}; } // namespace ovms diff --git a/src/mediapipe_internal/mediapipegraphexecutor.cpp b/src/mediapipe_internal/mediapipegraphexecutor.cpp index 93b53fdf8e..3c036a845a 100644 --- a/src/mediapipe_internal/mediapipegraphexecutor.cpp +++ b/src/mediapipe_internal/mediapipegraphexecutor.cpp @@ -31,8 +31,6 @@ #include "../python/python_backend.hpp" #endif -#include "../image_gen/pipelines.hpp" - namespace ovms { MediapipeGraphExecutor::MediapipeGraphExecutor( diff --git a/src/mediapipe_internal/mediapipegraphexecutor.hpp b/src/mediapipe_internal/mediapipegraphexecutor.hpp index c165469395..4d0b069f43 100644 --- a/src/mediapipe_internal/mediapipegraphexecutor.hpp +++ b/src/mediapipe_internal/mediapipegraphexecutor.hpp @@ -37,7 +37,7 @@ #pragma GCC diagnostic pop #pragma warning(pop) #include "mediapipe_utils.hpp" -#include "mediapipegraphdefinition.hpp" // for version in response and PythonNodeResourceMap +#include "graph_side_packets.hpp" #include "packettypes.hpp" namespace ovms { diff --git a/src/mediapipe_internal/node_initializer.cpp b/src/mediapipe_internal/node_initializer.cpp new file mode 100644 index 0000000000..3b2ae477e8 --- /dev/null +++ b/src/mediapipe_internal/node_initializer.cpp @@ -0,0 +1,33 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "node_initializer.hpp" + +#include + +namespace ovms { +NodeInitializerRegistry& NodeInitializerRegistry::instance() { + static NodeInitializerRegistry registry; + return registry; +} + +void NodeInitializerRegistry::add(std::unique_ptr initializer) { + initializers_.push_back(std::move(initializer)); +} + +const std::vector>& NodeInitializerRegistry::all() const { + return initializers_; +} +} // namespace ovms diff --git a/src/mediapipe_internal/node_initializer.hpp b/src/mediapipe_internal/node_initializer.hpp new file mode 100644 index 0000000000..e74bfadadb --- /dev/null +++ b/src/mediapipe_internal/node_initializer.hpp @@ -0,0 +1,53 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once +#include +#include +#include + +#include "src/status.hpp" + +namespace mediapipe { +class CalculatorGraphConfig_Node; +} + +namespace ovms { +struct GraphSidePackets; +class PythonBackend; + +class NodeInitializer { +public: + virtual ~NodeInitializer() = default; + virtual bool matches(const std::string& calculatorName) const = 0; + virtual Status initialize( + const ::mediapipe::CalculatorGraphConfig_Node& nodeConfig, + const std::string& graphName, + const std::string& basePath, + GraphSidePackets& sidePackets, + PythonBackend* pythonBackend) = 0; +}; + +class NodeInitializerRegistry { +public: + static NodeInitializerRegistry& instance(); + void add(std::unique_ptr initializer); + const std::vector>& all() const; + +private: + NodeInitializerRegistry() = default; + std::vector> initializers_; +}; +} // namespace ovms diff --git a/src/metrics/BUILD b/src/metrics/BUILD new file mode 100644 index 0000000000..21d7a87026 --- /dev/null +++ b/src/metrics/BUILD @@ -0,0 +1,54 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +load("//:common_settings.bzl", "ovms_cc_library") + +ovms_cc_library( + name = "libovmsmetrics", + hdrs = [ + "metric.hpp", + "metric_config.hpp", + "metric_family.hpp", + "metric_module.hpp", + "metric_registry.hpp", + ], + srcs = [ + "metric.cpp", + "metric_config.cpp", + "metric_family.cpp", + "metric_registry.cpp", + "metric_module.cpp", + ], + deps = [ + "@com_github_jupp0r_prometheus_cpp//core", + "@com_github_tencent_rapidjson//:rapidjson", + "//src/port:rapidjson_stringbuffer", + "//src/port:rapidjson_writer", + "//src:cpp_headers", + "//src:libovms_module", + "//src:libovmslogging", + "//src:libovmsschema", + "//src:libovmsstatus", + "//src:libovmsstring_utils", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "libovms_metric_provider", + hdrs = ["metric_provider.hpp",], + deps = [], + visibility = ["//visibility:public"], +) diff --git a/src/metric.cpp b/src/metrics/metric.cpp similarity index 100% rename from src/metric.cpp rename to src/metrics/metric.cpp diff --git a/src/metric.hpp b/src/metrics/metric.hpp similarity index 100% rename from src/metric.hpp rename to src/metrics/metric.hpp diff --git a/src/metric_config.cpp b/src/metrics/metric_config.cpp similarity index 98% rename from src/metric_config.cpp rename to src/metrics/metric_config.cpp index 8583d71207..fedfb4a806 100644 --- a/src/metric_config.cpp +++ b/src/metrics/metric_config.cpp @@ -27,10 +27,10 @@ #pragma warning(pop) #include -#include "logging.hpp" -#include "schema.hpp" -#include "status.hpp" -#include "stringutils.hpp" +#include "src/logging.hpp" +#include "src/schema.hpp" +#include "src/status.hpp" +#include "src/stringutils.hpp" #ifndef __linux__ // Workaround : https://github.com/Tencent/rapidjson/issues/1448 diff --git a/src/metric_config.hpp b/src/metrics/metric_config.hpp similarity index 98% rename from src/metric_config.hpp rename to src/metrics/metric_config.hpp index f867125694..bf1430f9c2 100644 --- a/src/metric_config.hpp +++ b/src/metrics/metric_config.hpp @@ -53,8 +53,8 @@ extern const std::string METRIC_NAME_REQUEST_LATENCY; class Status; /** - * @brief This class represents metrics configuration - */ + * @brief This class represents metrics configuration + */ class MetricConfig { public: bool metricsEnabled; diff --git a/src/metric_family.cpp b/src/metrics/metric_family.cpp similarity index 100% rename from src/metric_family.cpp rename to src/metrics/metric_family.cpp diff --git a/src/metric_family.hpp b/src/metrics/metric_family.hpp similarity index 100% rename from src/metric_family.hpp rename to src/metrics/metric_family.hpp diff --git a/src/metric_module.cpp b/src/metrics/metric_module.cpp similarity index 97% rename from src/metric_module.cpp rename to src/metrics/metric_module.cpp index a65771efee..608d83d423 100644 --- a/src/metric_module.cpp +++ b/src/metrics/metric_module.cpp @@ -16,7 +16,7 @@ #include "metric_module.hpp" #include "metric_registry.hpp" -#include "status.hpp" +#include "src/status.hpp" namespace ovms { diff --git a/src/metric_module.hpp b/src/metrics/metric_module.hpp similarity index 97% rename from src/metric_module.hpp rename to src/metrics/metric_module.hpp index 4e0d213547..cf67158e5b 100644 --- a/src/metric_module.hpp +++ b/src/metrics/metric_module.hpp @@ -16,7 +16,7 @@ #pragma once #include -#include "module.hpp" +#include "src/module.hpp" namespace ovms { diff --git a/src/dags/pipelinedefinitionunloadguard.hpp b/src/metrics/metric_provider.hpp similarity index 71% rename from src/dags/pipelinedefinitionunloadguard.hpp rename to src/metrics/metric_provider.hpp index c36466f21b..36824a8599 100644 --- a/src/dags/pipelinedefinitionunloadguard.hpp +++ b/src/metrics/metric_provider.hpp @@ -1,5 +1,5 @@ //***************************************************************************** -// Copyright 2020 Intel Corporation +// Copyright 2026 Intel Corporation // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,15 +16,15 @@ #pragma once namespace ovms { -class PipelineDefinition; -class PipelineDefinitionUnloadGuard { -public: - PipelineDefinitionUnloadGuard() = delete; - PipelineDefinitionUnloadGuard(PipelineDefinition& pipelineDefinition); - ~PipelineDefinitionUnloadGuard(); +class MetricConfig; +class MetricRegistry; -private: - PipelineDefinition& pipelineDefinition; +class MetricProvider { +public: + virtual ~MetricProvider() = default; + virtual MetricRegistry* getMetricRegistry() const = 0; + virtual const MetricConfig& getMetricConfig() const = 0; }; + } // namespace ovms diff --git a/src/metric_registry.cpp b/src/metrics/metric_registry.cpp similarity index 100% rename from src/metric_registry.cpp rename to src/metrics/metric_registry.cpp diff --git a/src/metric_registry.hpp b/src/metrics/metric_registry.hpp similarity index 100% rename from src/metric_registry.hpp rename to src/metrics/metric_registry.hpp diff --git a/src/model.cpp b/src/model.cpp index e5b6e00eb3..ae662429a9 100644 --- a/src/model.cpp +++ b/src/model.cpp @@ -23,9 +23,8 @@ #include "customloaderinterface.hpp" #include "customloaders.hpp" -#include "dags/pipelinedefinition.hpp" -#include "filesystem.hpp" -#include "localfilesystem.hpp" +#include "filesystem/filesystem.hpp" +#include "filesystem/localfilesystem.hpp" #include "logging.hpp" #include "modelinstance.hpp" #include "statefulmodelinstance.hpp" @@ -33,6 +32,10 @@ namespace ovms { +bool Model::isAvailable() const { + return getDefaultModelInstance() != nullptr; +} + static StatusCode downloadModels(std::shared_ptr& fs, ModelConfig& config, std::shared_ptr versions) { if (versions->size() == 0) { return StatusCode::OK; diff --git a/src/model.hpp b/src/model.hpp index b83525c8ed..b86de8b42c 100644 --- a/src/model.hpp +++ b/src/model.hpp @@ -27,6 +27,7 @@ #include "modelchangesubscription.hpp" #include "modelconfig.hpp" #include "modelversion.hpp" +#include "servable_definition.hpp" namespace ov { class Core; @@ -42,7 +43,7 @@ class MetricRegistry; class Status; /* * @brief This class represent inference models */ -class Model { +class Model : public ServableDefinition { private: /** * @brief Mutex for protecting concurrent modifying and accessing modelVersions @@ -136,10 +137,12 @@ class Model { * * @return model name */ - const std::string& getName() const { + const std::string& getName() const override { return name; } + bool isAvailable() const override; + const bool isStateful() const { return stateful; } diff --git a/src/model_instance_provider.hpp b/src/model_instance_provider.hpp new file mode 100644 index 0000000000..6788078189 --- /dev/null +++ b/src/model_instance_provider.hpp @@ -0,0 +1,39 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include + +#include "modelversion.hpp" + +namespace ovms { + +class ModelInstance; +class ModelInstanceUnloadGuard; +class Status; + +class ModelInstanceProvider { +public: + virtual ~ModelInstanceProvider() = default; + virtual Status getModelInstance( + const std::string& modelName, + model_version_t modelVersionId, + std::shared_ptr& modelInstance, + std::unique_ptr& modelInstanceUnloadGuardPtr) const = 0; +}; + +} // namespace ovms diff --git a/src/model_metric_reporter.cpp b/src/model_metric_reporter.cpp index 45923a8a9f..93d40140cc 100644 --- a/src/model_metric_reporter.cpp +++ b/src/model_metric_reporter.cpp @@ -20,9 +20,9 @@ #include "execution_context.hpp" #include "logging.hpp" -#include "metric_config.hpp" -#include "metric_family.hpp" -#include "metric_registry.hpp" +#include "metrics/metric_config.hpp" +#include "metrics/metric_family.hpp" +#include "metrics/metric_registry.hpp" namespace ovms { diff --git a/src/model_metric_reporter.hpp b/src/model_metric_reporter.hpp index f2238d7e7f..e300334742 100644 --- a/src/model_metric_reporter.hpp +++ b/src/model_metric_reporter.hpp @@ -15,14 +15,13 @@ //***************************************************************************** #pragma once -#include #include #include #include #include #include "execution_context.hpp" -#include "metric.hpp" +#include "metrics/metric.hpp" #include "modelversion.hpp" namespace ovms { @@ -30,7 +29,15 @@ namespace ovms { class MetricRegistry; class MetricConfig; -class ServableMetricReporter { +class StatusMetricReporter { +public: + virtual ~StatusMetricReporter() = default; + virtual std::unique_ptr& getModelReadyMetric(const ExecutionContext& context, bool success = true) = 0; + virtual std::unique_ptr& getModelMetadataMetric(const ExecutionContext& context, bool success = true) = 0; + virtual std::unique_ptr& getGetModelStatusRequestSuccessMetric(const ExecutionContext& context) = 0; +}; + +class ServableMetricReporter : public StatusMetricReporter { MetricRegistry* registry; protected: @@ -77,7 +84,7 @@ class ServableMetricReporter { std::unique_ptr requestTimeGrpc; std::unique_ptr requestTimeRest; - inline std::unique_ptr& getGetModelStatusRequestSuccessMetric(const ExecutionContext& context) { + inline std::unique_ptr& getGetModelStatusRequestSuccessMetric(const ExecutionContext& context) override { if (context.method != ExecutionContext::Method::GetModelStatus) { static std::unique_ptr empty = nullptr; return empty; // In case something calls it from ConfigReload/ConfigStatus methods @@ -123,7 +130,7 @@ class ServableMetricReporter { } } - inline std::unique_ptr& getModelMetadataMetric(const ExecutionContext& context, bool success = true) { + inline std::unique_ptr& getModelMetadataMetric(const ExecutionContext& context, bool success = true) override { if (context.interface == ExecutionContext::Interface::GRPC) { return success ? this->requestSuccessGrpcModelMetadata : this->requestFailGrpcModelMetadata; } else { @@ -131,7 +138,7 @@ class ServableMetricReporter { } } - inline std::unique_ptr& getModelReadyMetric(const ExecutionContext& context, bool success = true) { + inline std::unique_ptr& getModelReadyMetric(const ExecutionContext& context, bool success = true) override { if (context.interface == ExecutionContext::Interface::GRPC) { return success ? this->requestSuccessGrpcModelReady : this->requestFailGrpcModelReady; } else { @@ -153,7 +160,7 @@ class ModelMetricReporter : public ServableMetricReporter { ModelMetricReporter(const MetricConfig* metricConfig, MetricRegistry* registry, const std::string& modelName, model_version_t modelVersion); }; -class MediapipeServableMetricReporter { +class MediapipeServableMetricReporter : public StatusMetricReporter { MetricRegistry* registry; protected: @@ -300,7 +307,7 @@ class MediapipeServableMetricReporter { return nullptr; } - inline std::unique_ptr& getModelMetadataMetric(const ExecutionContext& context, bool success = true) { + inline std::unique_ptr& getModelMetadataMetric(const ExecutionContext& context, bool success = true) override { if (context.interface == ExecutionContext::Interface::GRPC) { return success ? this->requestSuccessGrpcModelMetadata : this->requestFailGrpcModelMetadata; } else { @@ -308,7 +315,7 @@ class MediapipeServableMetricReporter { } } - inline std::unique_ptr& getModelReadyMetric(const ExecutionContext& context, bool success = true) { + inline std::unique_ptr& getModelReadyMetric(const ExecutionContext& context, bool success = true) override { if (context.interface == ExecutionContext::Interface::GRPC) { return success ? this->requestSuccessGrpcModelReady : this->requestFailGrpcModelReady; } else { @@ -316,6 +323,11 @@ class MediapipeServableMetricReporter { } } + inline std::unique_ptr& getGetModelStatusRequestSuccessMetric(const ExecutionContext& context) override { + static std::unique_ptr empty{nullptr}; + return empty; + } + MediapipeServableMetricReporter(const MetricConfig* metricConfig, MetricRegistry* registry, const std::string& graphName); }; diff --git a/src/model_service.cpp b/src/model_service.cpp index 658c628f76..9d1c6080e2 100644 --- a/src/model_service.cpp +++ b/src/model_service.cpp @@ -33,17 +33,16 @@ #include "tensorflow_serving/apis/model_service.pb.h" #pragma GCC diagnostic pop -#include "dags/pipelinedefinition.hpp" +#include "dags/pipelinedefinitionstatus.hpp" #include "execution_context.hpp" #include "grpc_utils.hpp" -#if (MEDIAPIPE_DISABLE == 0) -#include "mediapipe_internal/mediapipefactory.hpp" -#include "mediapipe_internal/mediapipegraphdefinition.hpp" -#endif +#include "model.hpp" #include "modelinstance.hpp" #include "modelmanager.hpp" +#include "servable_definition.hpp" #include "servablemanagermodule.hpp" #include "server.hpp" +#include "single_version_servable_definition.hpp" #include "status.hpp" using google::protobuf::util::JsonPrintOptions; @@ -110,25 +109,17 @@ Status GetModelStatusImpl::getModelStatus( std::string requested_model_name = request->model_spec().name(); auto model_ptr = manager.findModelByName(requested_model_name); if (!model_ptr) { - SPDLOG_DEBUG("GetModelStatus: Model {} is missing, trying to find pipeline with such name", requested_model_name); - auto pipelineDefinition = manager.getPipelineFactory().findDefinitionByName(requested_model_name); - if (!pipelineDefinition) { -#if (MEDIAPIPE_DISABLE == 0) - auto mediapipeGraphDefinition = manager.getMediapipeFactory().findDefinitionByName(requested_model_name); - if (!mediapipeGraphDefinition) { - return StatusCode::MODEL_NAME_MISSING; - } - addStatusToResponse(response, mediapipeGraphDefinition->getVersion(), mediapipeGraphDefinition->getStatus()); - SPDLOG_DEBUG("model_service: response: {}", response->DebugString()); - SPDLOG_DEBUG("MODEL_STATUS created a response for {} - {}", requested_model_name, requested_version); - return StatusCode::OK; -#else + SPDLOG_DEBUG("GetModelStatus: Model {} is missing, trying to find definition with such name", requested_model_name); + auto* definition = manager.findServableDefinition(requested_model_name); + if (!definition) { return StatusCode::MODEL_NAME_MISSING; -#endif } - INCREMENT_IF_ENABLED(pipelineDefinition->getMetricReporter().getGetModelStatusRequestSuccessMetric(context)); - - addStatusToResponse(response, pipelineDefinition->getVersion(), pipelineDefinition->getStatus()); + auto* svsd = dynamic_cast(definition); + if (!svsd) { + return StatusCode::MODEL_NAME_MISSING; + } + INCREMENT_IF_ENABLED(svsd->getMetricReporter().getGetModelStatusRequestSuccessMetric(context)); + addStatusToResponse(response, svsd->getVersion(), svsd->getStatus()); SPDLOG_DEBUG("model_service: response: {}", response->DebugString()); SPDLOG_DEBUG("MODEL_STATUS created a response for {} - {}", requested_model_name, requested_version); return StatusCode::OK; @@ -186,34 +177,18 @@ Status GetModelStatusImpl::getAllModelsStatuses(std::map& pipelinesNames = manager.getPipelineFactory().getPipelinesNames(); - for (auto const& pipelineName : pipelinesNames) { - std::optional noValueModelVersion; - tensorflow::serving::GetModelStatusRequest request; - GetModelStatusImpl::createGrpcRequest(pipelineName, noValueModelVersion, &request); - tensorflow::serving::GetModelStatusResponse response; - auto status = GetModelStatusImpl::getModelStatus(&request, &response, manager, context); - if (status != StatusCode::OK) { - // Same situation like with models. - continue; - } - modelsStatusesTmp.insert({pipelineName, response}); - } -#if (MEDIAPIPE_DISABLE == 0) - const std::vector& mediapipePipelineNames = manager.getMediapipeFactory().getMediapipePipelinesNames(); - for (auto const& mediapipePipelineName : mediapipePipelineNames) { + const auto servableNames = manager.getServableDefinitionNames(); + for (const auto& servableName : servableNames) { std::optional noValueModelVersion; tensorflow::serving::GetModelStatusRequest request; - GetModelStatusImpl::createGrpcRequest(mediapipePipelineName, noValueModelVersion, &request); + GetModelStatusImpl::createGrpcRequest(servableName, noValueModelVersion, &request); tensorflow::serving::GetModelStatusResponse response; auto status = GetModelStatusImpl::getModelStatus(&request, &response, manager, context); if (status != StatusCode::OK) { - // Same situation like with models. continue; } - modelsStatusesTmp.insert({mediapipePipelineName, response}); + modelsStatusesTmp.insert({servableName, response}); } -#endif modelsStatuses.merge(modelsStatusesTmp); return StatusCode::OK; diff --git a/src/modelconfig.cpp b/src/modelconfig.cpp index d17e6fa6bf..2442b93fb3 100644 --- a/src/modelconfig.cpp +++ b/src/modelconfig.cpp @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -29,7 +30,8 @@ #include "src/port/rapidjson_writer.hpp" #pragma warning(pop) -#include "filesystem.hpp" +#include "anonymous_input_name.hpp" +#include "filesystem/filesystem.hpp" #include "json_parser.hpp" #include "logging.hpp" #include "model_version_policy.hpp" @@ -811,4 +813,31 @@ const std::string ModelConfig::getPath() const { return getLocalPath() + FileSystem::getOsSeparator() + std::to_string(version); } +bool ModelConfig::anyShapeSetToAuto() const { + for (const auto& [name, shapeInfo] : getShapes()) { + if (shapeInfo.shapeMode == AUTO) + return true; + } + return false; +} + +bool ModelConfig::isShapeAuto(const std::string& name) const { + auto it = getShapes().find(name); + if (it == getShapes().end()) { + it = getShapes().find(ANONYMOUS_INPUT_NAME); + } + if (it == getShapes().end()) { + return false; + } + return it->second.shapeMode == Mode::AUTO; +} + +bool ModelConfig::isShapeAnonymous() const { + return getShapes().size() == 1 && getShapes().begin()->first == ANONYMOUS_INPUT_NAME; +} + +bool ModelConfig::isShapeAnonymousFixed() const { + return isShapeAnonymous() && !isShapeAuto(ANONYMOUS_INPUT_NAME); +} + } // namespace ovms diff --git a/src/modelconfig.hpp b/src/modelconfig.hpp index a41463cb1d..59b575aa50 100644 --- a/src/modelconfig.hpp +++ b/src/modelconfig.hpp @@ -15,11 +15,9 @@ //***************************************************************************** #pragma once -#include #include #include #include -#include #include #include #include @@ -30,7 +28,6 @@ #include #pragma warning(pop) -#include "anonymous_input_name.hpp" #include "layout_configuration.hpp" #include "color_format_configuration.hpp" #include "precision_configuration.hpp" @@ -192,7 +189,6 @@ class ModelConfig { /** * @brief Allowed configurable layouts */ - static const std::set configAllowedLayouts; /** * @brief custom_loader_options config as map @@ -742,13 +738,7 @@ class ModelConfig { * * @return bool */ - bool anyShapeSetToAuto() const { - for (const auto& [name, shapeInfo] : getShapes()) { - if (shapeInfo.shapeMode == AUTO) - return true; - } - return false; - } + bool anyShapeSetToAuto() const; /** * @brief Get the shapes @@ -773,24 +763,9 @@ class ModelConfig { * * @return bool */ - bool isShapeAuto(const std::string& name) const { - auto it = getShapes().find(name); - if (it == getShapes().end()) { - it = getShapes().find(ANONYMOUS_INPUT_NAME); - } - if (it == getShapes().end()) { - return false; - } - return it->second.shapeMode == Mode::AUTO; - } - - bool isShapeAnonymous() const { - return getShapes().size() == 1 && getShapes().begin()->first == ANONYMOUS_INPUT_NAME; - } - - bool isShapeAnonymousFixed() const { - return isShapeAnonymous() && !isShapeAuto(ANONYMOUS_INPUT_NAME); - } + bool isShapeAuto(const std::string& name) const; + bool isShapeAnonymous() const; + bool isShapeAnonymousFixed() const; bool isCloudStored() const { return getLocalPath() != getBasePath(); diff --git a/src/modelinstance.cpp b/src/modelinstance.cpp index 49409fe109..f4620c8cfc 100644 --- a/src/modelinstance.cpp +++ b/src/modelinstance.cpp @@ -40,12 +40,11 @@ #include #include "anonymous_input_name.hpp" -#include "cleaner_utils.hpp" #include "config.hpp" #include "customloaderinterface.hpp" #include "customloaders.hpp" #include "executingstreamidguard.hpp" -#include "filesystem.hpp" +#include "filesystem/filesystem.hpp" #include "layout.hpp" #include "layout_configuration.hpp" #include "logging.hpp" @@ -65,6 +64,12 @@ #include "vaapitensorfactory.hpp" #endif +#ifdef _WIN32 +namespace ovms { +bool malloc_trim_win(); +} // namespace ovms +#endif + namespace { enum : unsigned int { GET_INFER_REQUEST, diff --git a/src/modelinstance.hpp b/src/modelinstance.hpp index a8b4067d3e..a8ce7633ec 100644 --- a/src/modelinstance.hpp +++ b/src/modelinstance.hpp @@ -30,7 +30,6 @@ #include -#include "logging.hpp" #include "model_metric_reporter.hpp" #include "modelchangesubscription.hpp" #include "modelconfig.hpp" diff --git a/src/modelmanager.cpp b/src/modelmanager.cpp index e1588e86f7..b4fd0b8da4 100644 --- a/src/modelmanager.cpp +++ b/src/modelmanager.cpp @@ -34,6 +34,7 @@ #endif #include +#include #pragma warning(push) #pragma warning(disable : 6313) #include @@ -55,18 +56,20 @@ #include "dags/pipeline.hpp" #include "dags/pipeline_factory.hpp" #include "dags/pipelinedefinition.hpp" -#include "filesystem.hpp" -#include "filesystemfactory.hpp" +#include "filesystem/filesystem.hpp" +#include "filesystem/filesystemfactory.hpp" #include "logging.hpp" #if (MEDIAPIPE_DISABLE == 0) #include "mediapipe_internal/mediapipefactory.hpp" #include "mediapipe_internal/mediapipegraphdefinition.hpp" #endif -#include "metric_config.hpp" -#include "metric_registry.hpp" +#include "metrics/metric_config.hpp" +#include "metrics/metric_registry.hpp" +#include "model.hpp" #include "modelinstance.hpp" // for logging #include "ov_utils.hpp" #include "schema.hpp" +#include "servable_definition.hpp" #include "stringutils.hpp" namespace ovms { @@ -79,8 +82,9 @@ const std::string DEFAULT_MODEL_CACHE_DIRECTORY = "/opt/cache"; #endif ModelManager::ModelManager(const std::string& modelCacheDirectory, MetricRegistry* registry, PythonBackend* pythonBackend) : ieCore(std::make_unique()), + pipelineFactory(std::make_unique()), #if (MEDIAPIPE_DISABLE == 0) - mediapipeFactory(pythonBackend), + mediapipeFactory(std::make_unique(pythonBackend)), #endif waitForModelLoadedTimeoutMs(DEFAULT_WAIT_FOR_MODEL_LOADED_TIMEOUT_MS), modelCacheDirectory(modelCacheDirectory), @@ -511,7 +515,7 @@ Status ModelManager::processMediapipeConfig(const MediapipeGraphConfig& config, MediapipeGraphDefinition* mediapipeGraphDefinition = factory.findDefinitionByName(config.getGraphName()); if (mediapipeGraphDefinition == nullptr) { SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Mediapipe graph:{} was not loaded so far. Triggering load", config.getGraphName()); - auto status = factory.createDefinition(config.getGraphName(), config, *this); + auto status = factory.createDefinition(config.getGraphName(), config, *this, *this); return status; } if (mediapipeGraphDefinition->isReloadRequired(config)) { @@ -692,7 +696,7 @@ Status ModelManager::loadCustomNodeLibrariesConfig(rapidjson::Document& configJs Status ModelManager::loadMediapipeGraphsConfig(std::vector& mediapipesInConfigFile) { if (mediapipesInConfigFile.size() == 0) { SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Configuration file doesn't have mediapipe property."); - mediapipeFactory.retireOtherThan({}, *this); + mediapipeFactory->retireOtherThan({}); return StatusCode::OK; } std::set mediapipesInConfigFileNames; @@ -701,13 +705,13 @@ Status ModelManager::loadMediapipeGraphsConfig(std::vector for (const auto& mediapipeGraphConfig : mediapipesInConfigFile) { mediapipesInConfigFileNames.insert(mediapipeGraphConfig.getGraphName()); } - mediapipeFactory.retireOtherThan(std::move(mediapipesInConfigFileNames), *this); + mediapipeFactory->retireOtherThan(std::move(mediapipesInConfigFileNames)); std::set mediapipesAlreadyLoaded; for (const auto& mediapipeGraphConfig : mediapipesInConfigFile) { if (spdlog::default_logger_raw()->level() <= spdlog::level::debug) { mediapipeGraphConfig.logGraphConfigContent(); } - auto status = processMediapipeConfig(mediapipeGraphConfig, mediapipesAlreadyLoaded, mediapipeFactory); + auto status = processMediapipeConfig(mediapipeGraphConfig, mediapipesAlreadyLoaded, *mediapipeFactory); if (status != StatusCode::OK) { IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); } @@ -725,18 +729,18 @@ Status ModelManager::loadPipelinesConfig(rapidjson::Document& configJson) { const auto itrp = configJson.FindMember("pipeline_config_list"); if (itrp == configJson.MemberEnd() || !itrp->value.IsArray()) { SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Configuration file doesn't have pipelines property."); - pipelineFactory.retireOtherThan({}, *this); + pipelineFactory->retireOtherThan({}, *this); return StatusCode::OK; } std::set pipelinesInConfigFile; Status firstErrorStatus = StatusCode::OK; for (const auto& pipelineConfig : itrp->value.GetArray()) { - auto status = processPipelineConfig(configJson, pipelineConfig, pipelinesInConfigFile, pipelineFactory, *this); + auto status = processPipelineConfig(configJson, pipelineConfig, pipelinesInConfigFile, *pipelineFactory, *this); if (status != StatusCode::OK) { IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); } } - pipelineFactory.retireOtherThan(std::move(pipelinesInConfigFile), *this); + pipelineFactory->retireOtherThan(std::move(pipelinesInConfigFile), *this); return firstErrorStatus; } @@ -909,18 +913,11 @@ Status ModelManager::loadModels(const rapidjson::Value::MemberIterator& modelsCo modelConfig.setCacheDir(this->modelCacheDirectory); const auto& modelName = modelConfig.getName(); - if (pipelineDefinitionExists(modelName)) { + if (servableExists(modelName, ServableType::Pipeline | ServableType::Mediapipe)) { IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(StatusCode::MODEL_NAME_OCCUPIED); - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Model name: {} is already occupied by pipeline definition.", modelName); + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Model name: {} is already occupied by pipeline or mediapipe graph definition.", modelName); continue; } -#if (MEDIAPIPE_DISABLE == 0) - if (mediapipeFactory.definitionExists(modelName)) { - IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(StatusCode::MODEL_NAME_OCCUPIED); - SPDLOG_LOGGER_ERROR(modelmanager_logger, "Model name: {} is already occupied by mediapipe graph definition.", modelName); - continue; - } -#endif if (modelsInConfigFile.find(modelName) != modelsInConfigFile.end()) { IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(StatusCode::MODEL_NAME_OCCUPIED); SPDLOG_LOGGER_WARN(modelmanager_logger, "Duplicated model names: {} defined in config file. Only first definition will be loaded.", modelName); @@ -1180,7 +1177,7 @@ Status ModelManager::updateConfigurationWithoutConfigFile() { reloadNeeded = true; } } - status = pipelineFactory.revalidatePipelines(*this); + status = pipelineFactory->revalidatePipelines(*this); if (!status.ok()) { IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); } @@ -1421,6 +1418,10 @@ Status ModelManager::checkStatefulFlagChange(const std::string& modelName, bool return StatusCode::OK; } +std::shared_ptr ModelManager::modelFactory(const std::string& name, const bool isStateful) { + return std::make_shared(name, isStateful, &this->globalSequencesViewer); +} + std::shared_ptr ModelManager::getModelIfExistCreateElse(const std::string& modelName, const bool isStateful) { std::unique_lock modelsLock(modelsMtx); auto modelIt = models.find(modelName); @@ -1724,7 +1725,7 @@ const std::vector ModelManager::getNamesOfAvailableModels() const { Status ModelManager::createPipeline(std::unique_ptr& graph, const std::string& name) { #if (MEDIAPIPE_DISABLE == 0) - return this->mediapipeFactory.create(graph, name, *this); + return this->mediapipeFactory->create(graph, name); #else SPDLOG_ERROR("Mediapipe support was disabled during build process..."); return StatusCode::INTERNAL_ERROR; @@ -1734,4 +1735,58 @@ Status ModelManager::createPipeline(std::unique_ptr& gra void ModelManager::setRootDirectoryPath(const std::string& configFileFullPath) { FileSystem::setRootDirectoryPath(this->rootDirectoryPath, configFileFullPath); } + +bool ModelManager::servableExists(const std::string& name, ServableType check) const { + if (hasFlag(check, ServableType::Model) && findModelByName(name) != nullptr) { + return true; + } + if (hasFlag(check, ServableType::Pipeline) && pipelineFactory->definitionExists(name)) { + return true; + } +#if (MEDIAPIPE_DISABLE == 0) + if (hasFlag(check, ServableType::Mediapipe) && mediapipeFactory->definitionExists(name)) { + return true; + } +#endif + return false; +} + +const PipelineFactory& ModelManager::getPipelineFactory() const { + return *pipelineFactory; +} + +// Returns raw pointer - safe because definitions (Model, PipelineDefinition, +// MediapipeGraphDefinition) are never removed from their maps during server +// lifetime. They only transition to RETIRED state. This matches the existing +// contract of PipelineFactory::findDefinitionByName and +// MediapipeFactory::findDefinitionByName which also return raw pointers. +ServableDefinition* ModelManager::findServableDefinition(const std::string& name) const { + auto model = findModelByName(name); + if (model) { + return model.get(); + } + auto* pipelineDefinition = pipelineFactory->findDefinitionByName(name); + if (pipelineDefinition) { + return pipelineDefinition; + } +#if (MEDIAPIPE_DISABLE == 0) + auto* mediapipeDefinition = mediapipeFactory->findDefinitionByName(name); + if (mediapipeDefinition) { + return mediapipeDefinition; + } +#endif + return nullptr; +} + +std::vector ModelManager::getServableDefinitionNames() const { + std::vector names; + auto pipelineNames = pipelineFactory->getPipelinesNames(); + names.insert(names.end(), pipelineNames.begin(), pipelineNames.end()); +#if (MEDIAPIPE_DISABLE == 0) + auto mediapipeNames = mediapipeFactory->getMediapipePipelinesNames(); + names.insert(names.end(), mediapipeNames.begin(), mediapipeNames.end()); +#endif + return names; +} + } // namespace ovms diff --git a/src/modelmanager.hpp b/src/modelmanager.hpp index f7e79c76ed..eaf6383034 100644 --- a/src/modelmanager.hpp +++ b/src/modelmanager.hpp @@ -26,23 +26,23 @@ #include #include -#include #pragma warning(push) #pragma warning(disable : 6313) #include #pragma warning(pop) -#include -#include -#include "dags/pipeline_factory.hpp" #include "global_sequences_viewer.hpp" -#if (MEDIAPIPE_DISABLE == 0) -#include "mediapipe_internal/mediapipefactory.hpp" -#endif -#include "metric_config.hpp" -#include "model.hpp" +#include "metrics/metric_config.hpp" +#include "metrics/metric_provider.hpp" +#include "model_instance_provider.hpp" +#include "modelconfig.hpp" +#include "servable_name_checker.hpp" #include "status.hpp" +namespace ov { +class Core; +} // namespace ov + namespace ovms { const uint32_t DEFAULT_WAIT_FOR_MODEL_LOADED_TIMEOUT_MS = 10000; @@ -54,16 +54,24 @@ struct ModelsSettingsImpl; class CustomLoaderConfig; class CustomNodeLibraryManager; class MetricRegistry; +class Model; class ModelConfig; class FileSystem; +class MediapipeFactory; +class MediapipeGraphConfig; class MediapipeGraphExecutor; +class ModelInstance; +class ServableDefinition; +class ModelInstanceUnloadGuard; +class Pipeline; +class PipelineFactory; struct FunctorSequenceCleaner; struct FunctorResourcesCleaner; class PythonBackend; /** * @brief Model manager is managing the list of model topologies enabled for serving and their versions. */ -class ModelManager { +class ModelManager : public ServableNameChecker, public MetricProvider, public ModelInstanceProvider { public: /** * @brief A default constructor is private @@ -79,14 +87,14 @@ class ModelManager { /** * @brief A collection of models - * + * */ std::map> models; std::unique_ptr ieCore; - PipelineFactory pipelineFactory; + std::unique_ptr pipelineFactory; #if (MEDIAPIPE_DISABLE == 0) - MediapipeFactory mediapipeFactory; + std::unique_ptr mediapipeFactory; #endif std::unique_ptr customNodeLibraryManager; std::vector> resources = {}; @@ -161,13 +169,13 @@ class ModelManager { std::thread cleanerThread; /** - * @brief Metrics config - */ + * @brief Metrics config + */ MetricConfig metricConfig; /** - * @brief Metrics config was loaded flag - */ + * @brief Metrics config was loaded flag + */ bool metricConfigLoadedOnce = false; /** @@ -182,7 +190,7 @@ class ModelManager { /** * @brief A current configurations of models - * + * */ std::unordered_map servedModelConfigs; @@ -219,8 +227,8 @@ class ModelManager { private: /** - * @brief last md5sum of configfile - */ + * @brief last md5sum of configfile + */ std::string lastConfigFileMD5; /** @@ -254,7 +262,7 @@ class ModelManager { const std::string getFullPath(const std::string& pathToCheck) const; /** - * @brief Get the config root path + * @brief Get the config root path * * @return const std::string& */ @@ -291,13 +299,13 @@ class ModelManager { /** * @brief Destroy the Model Manager object - * + * */ virtual ~ModelManager(); /** * @brief Gets config filename - * + * * @return config filename */ bool isStartedWithConfigFile() { @@ -306,7 +314,7 @@ class ModelManager { /** * @brief Gets models collection - * + * * @return models collection */ const std::map>& getModels() { @@ -320,13 +328,11 @@ class ModelManager { */ void startCleaner(); - const PipelineFactory& getPipelineFactory() const { - return pipelineFactory; - } + const PipelineFactory& getPipelineFactory() const; #if (MEDIAPIPE_DISABLE == 0) const MediapipeFactory& getMediapipeFactory() const { - return mediapipeFactory; + return *mediapipeFactory; } #endif @@ -337,14 +343,14 @@ class ModelManager { * * @param name of the model to search for * - * @return pointer to Model or nullptr if not found + * @return pointer to Model or nullptr if not found */ const std::shared_ptr findModelByName(const std::string& name) const; Status getModelInstance(const std::string& modelName, ovms::model_version_t modelVersionId, std::shared_ptr& modelInstance, - std::unique_ptr& modelInstanceUnloadGuardPtr) const; + std::unique_ptr& modelInstanceUnloadGuardPtr) const override; const bool modelExists(const std::string& name) const { if (findModelByName(name) == nullptr) @@ -359,27 +365,16 @@ class ModelManager { * @param name of the model to search for * @param version of the model to search for or 0 if default * - * @return pointer to ModelInstance or nullptr if not found + * @return pointer to ModelInstance or nullptr if not found */ const std::shared_ptr findModelInstance(const std::string& name, model_version_t version = 0) const; - template - Status createPipeline(std::unique_ptr& pipeline, - const std::string& name, - const RequestType* request, - ResponseType* response) { - return pipelineFactory.create(pipeline, name, request, response, *this); - } Status createPipeline(std::unique_ptr& graph, const std::string& name); - const bool pipelineDefinitionExists(const std::string& name) const { - return pipelineFactory.definitionExists(name); - } - /** * @brief Starts model manager using provided config file - * + * * @param filename * @return status */ @@ -387,17 +382,17 @@ class ModelManager { /** * @brief Starts model manager using command line arguments - * - * @return Status + * + * @return Status */ Status startFromConfig(); /** - * @brief Get the metric config - * - * @return const std::string& - */ - const MetricConfig& getMetricConfig() const { + * @brief Get the metric config + * + * @return const std::string& + */ + const MetricConfig& getMetricConfig() const override { return this->metricConfig; } @@ -405,33 +400,33 @@ class ModelManager { Status loadMetricsConfig(rapidjson::Document& configJson); /** - * @brief Set the metric config - * - * @param metricConfig - */ + * @brief Set the metric config + * + * @param metricConfig + */ void setMetricConfig(const MetricConfig& metricConfig) { this->metricConfig = metricConfig; } /** * @brief Reload model versions located in base path - * + * * @param ModelConfig config - * + * * @return status */ Status reloadModelWithVersions(ModelConfig& config); /** * @brief Starts model manager using ovms::Config - * + * * @return status */ Status start(const Config& config); /** * @brief Starts monitoring as new thread - * + * */ void startWatcher(bool watchConfigFile); @@ -442,20 +437,18 @@ class ModelManager { /** * @brief Factory for creating a model - * - * @return std::shared_ptr + * + * @return std::shared_ptr */ - virtual std::shared_ptr modelFactory(const std::string& name, const bool isStateful) { - return std::make_shared(name, isStateful, &this->globalSequencesViewer); - } + virtual std::shared_ptr modelFactory(const std::string& name, const bool isStateful); /** * @brief Reads available versions from given filesystem - * - * @param fs - * @param base - * @param versions - * @return Status + * + * @param fs + * @param base + * @param versions + * @return Status */ virtual Status readAvailableVersions( std::shared_ptr& fs, @@ -488,9 +481,9 @@ class ModelManager { /** * @brief Reads models from configuration file - * + * * @param jsonFilename configuration file - * @return Status + * @return Status */ Status loadConfig(); @@ -504,7 +497,13 @@ class ModelManager { */ void cleanupResources(); - MetricRegistry* getMetricRegistry() const { return this->metricRegistry; } + bool servableExists(const std::string& name, ServableType check = ServableType::All) const override; + + ServableDefinition* findServableDefinition(const std::string& name) const; + + std::vector getServableDefinitionNames() const; + + MetricRegistry* getMetricRegistry() const override { return this->metricRegistry; } }; void cleanerRoutine(uint32_t resourcesCleanupInterval, FunctorResourcesCleaner& functorResourcesCleaner, uint32_t sequenceCleanerInterval, FunctorSequenceCleaner& functorSequenceCleaner, std::future& cleanerExitSignal); diff --git a/src/modelversionstatus.hpp b/src/modelversionstatus.hpp index e46fe4be96..87dded0350 100644 --- a/src/modelversionstatus.hpp +++ b/src/modelversionstatus.hpp @@ -19,8 +19,9 @@ #include #include +#include + #include "modelversion.hpp" -#include "logging.hpp" // note: think about using https://github.com/Neargye/magic_enum when compatible compiler is supported. diff --git a/src/precision_configuration.cpp b/src/precision_configuration.cpp index 58b0dbcd9d..83365f9130 100644 --- a/src/precision_configuration.cpp +++ b/src/precision_configuration.cpp @@ -18,6 +18,8 @@ #include #include +#include "logging.hpp" + namespace ovms { const char PrecisionConfiguration::PRECISION_DELIMITER = ':'; diff --git a/src/predict_request_validation_utils_impl.hpp b/src/predict_request_validation_utils_impl.hpp index 57707ffc7c..14eff89771 100644 --- a/src/predict_request_validation_utils_impl.hpp +++ b/src/predict_request_validation_utils_impl.hpp @@ -22,7 +22,6 @@ #include #include -#include "logging.hpp" #include "shape.hpp" #include "anonymous_input_name.hpp" #include "status.hpp" diff --git a/src/prediction_service.cpp b/src/prediction_service.cpp index 74e23583d1..e211ab1ccc 100644 --- a/src/prediction_service.cpp +++ b/src/prediction_service.cpp @@ -33,6 +33,7 @@ #include "tfs_frontend/tfs_request_utils.hpp" #include "dags/pipeline.hpp" +#include "dags/pipeline_factory.hpp" #include "execution_context.hpp" #include "get_model_metadata_impl.hpp" #include "grpc_utils.hpp" @@ -88,7 +89,7 @@ Status PredictionServiceImpl::getPipeline(const PredictRequest* request, PredictResponse* response, std::unique_ptr& pipelinePtr) { OVMS_PROFILE_FUNCTION(); - return this->modelManager.createPipeline(pipelinePtr, request->model_spec().name(), request, response); + return this->modelManager.getPipelineFactory().create(pipelinePtr, request->model_spec().name(), request, response, this->modelManager); } grpc::Status ovms::PredictionServiceImpl::Predict( diff --git a/src/prediction_service_utils.hpp b/src/prediction_service_utils.hpp index 788141d0a4..21769d08c1 100644 --- a/src/prediction_service_utils.hpp +++ b/src/prediction_service_utils.hpp @@ -27,7 +27,6 @@ #include "kfs_frontend/kfs_grpc_inference_service.hpp" #include "extractchoice.hpp" #include "requesttensorextractor.hpp" -#include "logging.hpp" #include "shape.hpp" #include "status.hpp" diff --git a/src/profilermodule.hpp b/src/profilermodule.hpp index 9927b75ab8..de4eaa86a8 100644 --- a/src/profilermodule.hpp +++ b/src/profilermodule.hpp @@ -17,7 +17,6 @@ #include #include -#include "logging.hpp" #include "module.hpp" namespace ovms { diff --git a/src/pull_module/BUILD b/src/pull_module/BUILD index 7b6b0b0588..538e5d318d 100644 --- a/src/pull_module/BUILD +++ b/src/pull_module/BUILD @@ -32,8 +32,8 @@ ovms_cc_library( deps = [ "@ovms//src:libovmsstatus", "@ovms//src:libovmslogging", - "@ovms//src:libovmsfilesystem", - "@ovms//src:libovmslocalfilesystem", + "@ovms//src/filesystem:libovmsfilesystem", + "@ovms//src/filesystem:libovmslocalfilesystem", ], visibility = ["//visibility:public"], ) @@ -45,8 +45,8 @@ ovms_cc_library( deps = [ "@libgit2_engine//:libgit2_engine", "@ovms//src:libovmslogging", - "@ovms//src:libovmsfilesystem", - "@ovms//src:libovmslocalfilesystem", + "@ovms//src/filesystem:libovmsfilesystem", + "@ovms//src/filesystem:libovmslocalfilesystem", "@ovms//src:libovmsstatus", "@ovms//src:libovmsstring_utils", ":cmd_exec", @@ -66,8 +66,8 @@ ovms_cc_library( "@ovms//src:libovmsstatus", "@ovms//src:libovmsstring_utils", "@ovms//src:libovms_server_settings", - "@ovms//src:libovmsfilesystem", - "@ovms//src:libovmslocalfilesystem", + "@ovms//src/filesystem:libovmsfilesystem", + "@ovms//src/filesystem:libovmslocalfilesystem", "@ovms//src:libovms_version", ], visibility = ["//visibility:public"], @@ -85,8 +85,8 @@ ovms_cc_library( "@ovms//src:libovmsstatus", "@ovms//src:libovmsstring_utils", "@ovms//src:libovms_server_settings", - "@ovms//src:libovmsfilesystem", - "@ovms//src:libovmslocalfilesystem", + "@ovms//src/filesystem:libovmsfilesystem", + "@ovms//src/filesystem:libovmslocalfilesystem", ], visibility = ["//visibility:public"], ) diff --git a/src/pull_module/gguf_downloader.cpp b/src/pull_module/gguf_downloader.cpp index 1a6c5355fd..dd951e6147 100644 --- a/src/pull_module/gguf_downloader.cpp +++ b/src/pull_module/gguf_downloader.cpp @@ -24,8 +24,8 @@ #include #include "../capi_frontend/server_settings.hpp" -#include "../filesystem.hpp" -#include "../localfilesystem.hpp" +#include "src/filesystem/filesystem.hpp" +#include "src/filesystem/localfilesystem.hpp" #include "../logging.hpp" #include "../stringutils.hpp" #include "../status.hpp" diff --git a/src/pull_module/libgit2.cpp b/src/pull_module/libgit2.cpp index 1a7d515671..8d5cbf8d4a 100644 --- a/src/pull_module/libgit2.cpp +++ b/src/pull_module/libgit2.cpp @@ -36,8 +36,8 @@ #include #include "cmd_exec.hpp" -#include "../filesystem.hpp" -#include "../localfilesystem.hpp" +#include "src/filesystem/filesystem.hpp" +#include "src/filesystem/localfilesystem.hpp" #include "../logging.hpp" #include "../stringutils.hpp" #include "../status.hpp" diff --git a/src/pull_module/model_downloader.cpp b/src/pull_module/model_downloader.cpp index 4b67958a73..cdba53e75d 100644 --- a/src/pull_module/model_downloader.cpp +++ b/src/pull_module/model_downloader.cpp @@ -17,8 +17,8 @@ #include "model_downloader.hpp" -#include "../filesystem.hpp" -#include "../localfilesystem.hpp" +#include "src/filesystem/filesystem.hpp" +#include "src/filesystem/localfilesystem.hpp" #include "../logging.hpp" namespace ovms { diff --git a/src/pull_module/optimum_export.cpp b/src/pull_module/optimum_export.cpp index c41c614920..f57e2448dc 100644 --- a/src/pull_module/optimum_export.cpp +++ b/src/pull_module/optimum_export.cpp @@ -19,8 +19,8 @@ #include #include "../capi_frontend/server_settings.hpp" -#include "../filesystem.hpp" -#include "../localfilesystem.hpp" +#include "src/filesystem/filesystem.hpp" +#include "src/filesystem/localfilesystem.hpp" #include "../logging.hpp" #include "../stringutils.hpp" #include "../status.hpp" diff --git a/src/python/BUILD b/src/python/BUILD index f4fd4c571e..8b5cb2e70d 100644 --- a/src/python/BUILD +++ b/src/python/BUILD @@ -41,7 +41,7 @@ mediapipe_proto_library( ovms_cc_library( name = "utils", hdrs = ["utils.hpp",], - srcs = [], + srcs = [], deps = PYBIND_DEPS + [ "//src:libovmslogging", ], @@ -71,7 +71,7 @@ ovms_cc_library( "@mediapipe//mediapipe/framework:calculator_framework", "//src:libovmslogging", "//src:libovmsstatus", - "//src:libovmsmediapipe_utils", + "//src/mediapipe_internal:mediapipe_utils", "pythonexecutorcalculator_cc_proto", "utils", ], @@ -98,7 +98,8 @@ ovms_cc_library( ovms_cc_library( name = "pythonexecutorcalculator", - srcs = ["python_executor_calculator.cc",], + srcs = ["python_executor_calculator.cc", + "python_node_initializer.cpp",], deps = PYBIND_DEPS + [ "//third_party:openvino", "@mediapipe//mediapipe/framework:calculator_framework", @@ -106,6 +107,8 @@ ovms_cc_library( "pythonexecutorcalculator_cc_proto", "pythonbackend", "pythonnoderesources", + "//src/mediapipe_internal:node_initializer", + "//src:libovmslogging", ], visibility = ["//visibility:private"], alwayslink = 1, @@ -131,7 +134,7 @@ ovms_cc_library( "//src:cpp_headers", # TODO make ovmslib depend on that "//src:libovmslogging", "//src:libovms_module", - "//src:libovmsmediapipe_utils", + "//src/mediapipe_internal:mediapipe_utils", "ovmspytensor", "pytensorovtensorconvertercalculator", "pythonbackend", diff --git a/src/python/python_node_initializer.cpp b/src/python/python_node_initializer.cpp new file mode 100644 index 0000000000..f6dea17378 --- /dev/null +++ b/src/python/python_node_initializer.cpp @@ -0,0 +1,70 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include + +#include "src/mediapipe_internal/graph_side_packets.hpp" +#include "src/mediapipe_internal/node_initializer.hpp" +#include "pythonnoderesources.hpp" +#include "mediapipe/framework/calculator.pb.h" + +#include "src/logging.hpp" + +namespace ovms { +class PythonNodeInitializer : public NodeInitializer { + static constexpr const char* CALCULATOR_NAME = "PythonExecutorCalculator"; + +public: + bool matches(const std::string& calculatorName) const override { + return calculatorName == CALCULATOR_NAME; + } + Status initialize( + const ::mediapipe::CalculatorGraphConfig_Node& nodeConfig, + const std::string& graphName, + const std::string& basePath, + GraphSidePackets& sidePackets, + PythonBackend* pythonBackend) override { + auto& pythonNodeResourcesMap = sidePackets.pythonNodeResourcesMap; + if (!nodeConfig.node_options().size()) { + SPDLOG_ERROR("Python node missing options in graph: {}. ", graphName); + return StatusCode::PYTHON_NODE_MISSING_OPTIONS; + } + if (nodeConfig.name().empty()) { + SPDLOG_ERROR("Python node name is missing in graph: {}. ", graphName); + return StatusCode::PYTHON_NODE_MISSING_NAME; + } + std::string nodeName = nodeConfig.name(); + if (pythonNodeResourcesMap.find(nodeName) != pythonNodeResourcesMap.end()) { + SPDLOG_ERROR("Python node name: {} already used in graph: {}. ", nodeName, graphName); + return StatusCode::PYTHON_NODE_NAME_ALREADY_EXISTS; + } + std::shared_ptr nodeResources = nullptr; + Status status = PythonNodeResources::createPythonNodeResources(nodeResources, nodeConfig, pythonBackend, basePath); + if (nodeResources == nullptr || !status.ok()) { + SPDLOG_ERROR("Failed to process python node graph {}", graphName); + return status; + } + pythonNodeResourcesMap.insert(std::pair>(nodeName, std::move(nodeResources))); + return StatusCode::OK; + } +}; + +static bool pythonNodeInitializerRegistered = []() { + NodeInitializerRegistry::instance().add(std::make_unique()); + return true; +}(); +} // namespace ovms diff --git a/src/python/utils.hpp b/src/python/utils.hpp index 13708be639..6e8b4f511d 100644 --- a/src/python/utils.hpp +++ b/src/python/utils.hpp @@ -18,6 +18,9 @@ #include #include + +#include "src/logging.hpp" + #pragma warning(push) #pragma warning(disable : 6326 28182 6011 28020) #include @@ -25,7 +28,6 @@ #include #pragma warning(pop) -#include "../logging.hpp" namespace py = pybind11; using namespace py::literals; diff --git a/src/rerank/BUILD b/src/rerank/BUILD index 7f3b1a6ec9..8d642b2c80 100644 --- a/src/rerank/BUILD +++ b/src/rerank/BUILD @@ -30,7 +30,8 @@ mediapipe_proto_library( ovms_cc_library( name = "rerank_servable", hdrs = ["rerank_servable.hpp"], - deps = ["//src:sidepacket_servable",], + deps = ["//src:sidepacket_servable", + "//src/port:rapidjson_document",], visibility = ["//visibility:public"], alwayslink = 1, ) @@ -65,7 +66,8 @@ ovms_cc_library( ovms_cc_library( name = "rerankcalculator_ov", - srcs = ["rerank_calculator_ov.cc"], + srcs = ["rerank_calculator_ov.cc", + "rerank_node_initializer.cpp"], deps = [ "@mediapipe//mediapipe/framework:calculator_framework", "@com_github_tencent_rapidjson//:rapidjson", @@ -81,6 +83,8 @@ ovms_cc_library( "//src:executingstreamidguard", "//src:libovms_execution_context", "//src/tokenize:tokenize_parser", + "//src/mediapipe_internal:node_initializer", + "//src:libovmsstring_utils", ], visibility = ["//visibility:public"], alwayslink = 1, diff --git a/src/rerank/rerank_node_initializer.cpp b/src/rerank/rerank_node_initializer.cpp new file mode 100644 index 0000000000..8349f4b05a --- /dev/null +++ b/src/rerank/rerank_node_initializer.cpp @@ -0,0 +1,70 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include + +#include "src/mediapipe_internal/graph_side_packets.hpp" +#include "src/mediapipe_internal/node_initializer.hpp" +#include "src/stringutils.hpp" +#include "rerank_servable.hpp" +#include "mediapipe/framework/calculator.pb.h" +#include "src/rerank/rerank_calculator_ov.pb.h" + +#include "src/logging.hpp" + +namespace ovms { +class RerankNodeInitializer : public NodeInitializer { + static constexpr const char* CALCULATOR_NAME = "RerankCalculatorOV"; + +public: + bool matches(const std::string& calculatorName) const override { + return endsWith(calculatorName, CALCULATOR_NAME); + } + Status initialize( + const ::mediapipe::CalculatorGraphConfig_Node& nodeConfig, + const std::string& graphName, + const std::string& basePath, + GraphSidePackets& sidePackets, + PythonBackend* /*pythonBackend*/) override { + auto& rerankServableMap = sidePackets.rerankServableMap; + if (!nodeConfig.node_options().size()) { + SPDLOG_ERROR("Rerank node missing options in graph: {}. ", graphName); + return StatusCode::LLM_NODE_MISSING_OPTIONS; + } + if (nodeConfig.name().empty()) { + SPDLOG_ERROR("Rerank node name is missing in graph: {}. ", graphName); + return StatusCode::LLM_NODE_MISSING_NAME; + } + std::string nodeName = nodeConfig.name(); + if (rerankServableMap.find(nodeName) != rerankServableMap.end()) { + SPDLOG_ERROR("Rerank node name: {} already used in graph: {}. ", nodeName, graphName); + return StatusCode::LLM_NODE_NAME_ALREADY_EXISTS; + } + mediapipe::RerankCalculatorOVOptions nodeOptions; + nodeConfig.node_options(0).UnpackTo(&nodeOptions); + auto servable = std::make_shared(nodeOptions.models_path(), nodeOptions.target_device(), nodeOptions.plugin_config(), basePath); + servable->initialize(nodeOptions.models_path(), nodeOptions.target_device(), nodeOptions.plugin_config(), basePath); + rerankServableMap.insert(std::pair>(nodeName, std::move(servable))); + return StatusCode::OK; + } +}; + +static bool rerankNodeInitializerRegistered = []() { + NodeInitializerRegistry::instance().add(std::make_unique()); + return true; +}(); +} // namespace ovms diff --git a/src/rerank/rerank_servable.hpp b/src/rerank/rerank_servable.hpp index 15e23983c4..be28f27677 100644 --- a/src/rerank/rerank_servable.hpp +++ b/src/rerank/rerank_servable.hpp @@ -16,9 +16,13 @@ #pragma once #include "../sidepacket_servable.hpp" -#include "../filesystem.hpp" +#include "src/filesystem/filesystem.hpp" +#pragma warning(push) +#pragma warning(disable : 6313) +#include #include #include +#pragma warning(pop) #include #include #include diff --git a/src/schema.cpp b/src/schema.cpp index 7bf2bf5a1e..b9db36d103 100644 --- a/src/schema.cpp +++ b/src/schema.cpp @@ -31,7 +31,7 @@ #include #endif -#include "filesystem.hpp" +#include "filesystem/filesystem.hpp" #include "logging.hpp" #include "status.hpp" diff --git a/src/servable.hpp b/src/servable.hpp index 7af7f1c9af..6c6e5858d2 100644 --- a/src/servable.hpp +++ b/src/servable.hpp @@ -18,7 +18,6 @@ #include #include "modelversion.hpp" -#include "tensorinfo.hpp" namespace ovms { class Servable { diff --git a/src/servable_definition.hpp b/src/servable_definition.hpp new file mode 100644 index 0000000000..8586bae346 --- /dev/null +++ b/src/servable_definition.hpp @@ -0,0 +1,29 @@ +//***************************************************************************** +// Copyright 2025 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include + +namespace ovms { + +class ServableDefinition { +public: + virtual ~ServableDefinition() = default; + virtual const std::string& getName() const = 0; + virtual bool isAvailable() const = 0; +}; + +} // namespace ovms diff --git a/src/dags/pipelinedefinitionunloadguard.cpp b/src/servable_definition_unload_guard.cpp similarity index 63% rename from src/dags/pipelinedefinitionunloadguard.cpp rename to src/servable_definition_unload_guard.cpp index fc4c6964fd..5911c2f01f 100644 --- a/src/dags/pipelinedefinitionunloadguard.cpp +++ b/src/servable_definition_unload_guard.cpp @@ -1,5 +1,5 @@ //***************************************************************************** -// Copyright 2020 Intel Corporation +// Copyright 2026 Intel Corporation // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -13,17 +13,19 @@ // See the License for the specific language governing permissions and // limitations under the License. //***************************************************************************** -#include "pipelinedefinitionunloadguard.hpp" +#include "servable_definition_unload_guard.hpp" -#include "pipelinedefinition.hpp" +#include "single_version_servable_definition.hpp" namespace ovms { -PipelineDefinitionUnloadGuard::PipelineDefinitionUnloadGuard(PipelineDefinition& pipelineDefinition) : - pipelineDefinition(pipelineDefinition) { - pipelineDefinition.increaseRequestsHandlesCount(); + +ServableDefinitionUnloadGuard::ServableDefinitionUnloadGuard(SingleVersionServableDefinition& definition) : + definition(definition) { + definition.increaseRequestsHandlesCount(); } -PipelineDefinitionUnloadGuard::~PipelineDefinitionUnloadGuard() { - pipelineDefinition.decreaseRequestsHandlesCount(); +ServableDefinitionUnloadGuard::~ServableDefinitionUnloadGuard() { + definition.decreaseRequestsHandlesCount(); } + } // namespace ovms diff --git a/src/servable_definition_unload_guard.hpp b/src/servable_definition_unload_guard.hpp new file mode 100644 index 0000000000..2caf00ebf4 --- /dev/null +++ b/src/servable_definition_unload_guard.hpp @@ -0,0 +1,32 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +namespace ovms { + +class SingleVersionServableDefinition; + +class ServableDefinitionUnloadGuard { +public: + ServableDefinitionUnloadGuard() = delete; + ServableDefinitionUnloadGuard(SingleVersionServableDefinition& definition); + ~ServableDefinitionUnloadGuard(); + +private: + SingleVersionServableDefinition& definition; +}; + +} // namespace ovms diff --git a/src/servable_name_checker.hpp b/src/servable_name_checker.hpp new file mode 100644 index 0000000000..373dd940c2 --- /dev/null +++ b/src/servable_name_checker.hpp @@ -0,0 +1,46 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include + +namespace ovms { + +enum class ServableType : uint8_t { + Model = 1 << 0, + Pipeline = 1 << 1, + Mediapipe = 1 << 2, + All = Model | Pipeline | Mediapipe +}; + +inline ServableType operator|(ServableType a, ServableType b) { + using T = std::underlying_type_t; + return static_cast(static_cast(a) | static_cast(b)); +} + +inline bool hasFlag(ServableType value, ServableType flag) { + using T = std::underlying_type_t; + return (static_cast(value) & static_cast(flag)) != 0; +} + +class ServableNameChecker { +public: + virtual ~ServableNameChecker() = default; + virtual bool servableExists(const std::string& name, ServableType check = ServableType::All) const = 0; +}; + +} // namespace ovms diff --git a/src/servablemanagermodule.cpp b/src/servablemanagermodule.cpp index 247b3bf0da..3c9e8ad291 100644 --- a/src/servablemanagermodule.cpp +++ b/src/servablemanagermodule.cpp @@ -20,7 +20,7 @@ #include "config.hpp" #include "logging.hpp" -#include "metric_module.hpp" +#include "metrics/metric_module.hpp" #include "modelmanager.hpp" #include "server.hpp" #if (PYTHON_DISABLE == 0) diff --git a/src/servables_config_manager_module/BUILD b/src/servables_config_manager_module/BUILD index 8c3a041204..86e8fc9721 100644 --- a/src/servables_config_manager_module/BUILD +++ b/src/servables_config_manager_module/BUILD @@ -21,7 +21,7 @@ ovms_cc_library( hdrs = ["listmodels.hpp",], srcs = ["listmodels.cpp",], deps = [ - "//src:libovmsfilesystem", + "//src/filesystem:libovmsfilesystem", "//src:libovmslogging", "//src:modelextensions", ], diff --git a/src/servables_config_manager_module/listmodels.cpp b/src/servables_config_manager_module/listmodels.cpp index 9344450333..96bcaad33e 100644 --- a/src/servables_config_manager_module/listmodels.cpp +++ b/src/servables_config_manager_module/listmodels.cpp @@ -19,7 +19,7 @@ #include #include -#include "src/filesystem.hpp" +#include "src/filesystem/filesystem.hpp" #include "src/modelextensions.hpp" #include "src/logging.hpp" diff --git a/src/server.cpp b/src/server.cpp index ec0a7e4b10..144ca3ec35 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -56,7 +56,7 @@ #include "httpservermodule.hpp" #include "kfs_frontend/kfs_grpc_inference_service.hpp" #include "logging.hpp" -#include "metric_module.hpp" +#include "metrics/metric_module.hpp" #include "model_service.hpp" #include "modelmanager.hpp" #include "ovms_exit_codes.hpp" diff --git a/src/sidepacket_servable.cpp b/src/sidepacket_servable.cpp index c06b6f30d4..3e8dea1554 100644 --- a/src/sidepacket_servable.cpp +++ b/src/sidepacket_servable.cpp @@ -31,7 +31,7 @@ #include "status.hpp" #include "config.hpp" -#include "filesystem.hpp" +#include "filesystem/filesystem.hpp" using namespace ov::genai; using namespace ov; diff --git a/src/single_version_servable_definition.cpp b/src/single_version_servable_definition.cpp new file mode 100644 index 0000000000..d1b8109402 --- /dev/null +++ b/src/single_version_servable_definition.cpp @@ -0,0 +1,74 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "single_version_servable_definition.hpp" + +#include +#include + +#include + +#include "dags/pipelinedefinitionstatus.hpp" +#include "servable_definition_unload_guard.hpp" + +namespace ovms { + +Status SingleVersionServableDefinition::waitForLoaded(std::unique_ptr& guard, const uint32_t waitForLoadedTimeoutMicroseconds) { + guard = std::make_unique(*this); + + const uint32_t waitLoadedTimestepMicroseconds = 1000; + const uint32_t waitCheckpoints = waitForLoadedTimeoutMicroseconds / waitLoadedTimestepMicroseconds; + uint32_t waitCheckpointsCounter = waitCheckpoints; + std::mutex cvMtx; + std::unique_lock cvLock(cvMtx); + while (waitCheckpointsCounter-- != 0) { + if (getStatus().isAvailable()) { + SPDLOG_DEBUG("Successfully waited for definition: {}", getName()); + return StatusCode::OK; + } + guard.reset(); + if (!getStatus().canEndLoaded()) { + if (getStatus().getStateCode() != PipelineDefinitionStateCode::RETIRED) { + SPDLOG_DEBUG("Waiting for definition: {} ended due to timeout.", getName()); + return notLoadedYetCode(); + } else { + SPDLOG_DEBUG("Waiting for definition: {} ended since it failed to load.", getName()); + return notLoadedAnymoreCode(); + } + } + SPDLOG_DEBUG("Waiting for available state for definition: {}, with timestep: {}us timeout: {}us check count: {}", + getName(), waitLoadedTimestepMicroseconds, waitForLoadedTimeoutMicroseconds, waitCheckpointsCounter); + loadedNotify.wait_for(cvLock, + std::chrono::microseconds(waitLoadedTimestepMicroseconds), + [this]() { + return this->getStatus().isAvailable() || + !this->getStatus().canEndLoaded(); + }); + guard = std::make_unique(*this); + } + if (!getStatus().isAvailable()) { + if (getStatus().getStateCode() != PipelineDefinitionStateCode::RETIRED) { + SPDLOG_DEBUG("Waiting for definition: {} ended due to timeout.", getName()); + return notLoadedYetCode(); + } else { + SPDLOG_DEBUG("Waiting for definition: {} ended since it failed to load.", getName()); + return notLoadedAnymoreCode(); + } + } + SPDLOG_DEBUG("Successfully waited for definition: {}", getName()); + return StatusCode::OK; +} + +} // namespace ovms diff --git a/src/single_version_servable_definition.hpp b/src/single_version_servable_definition.hpp new file mode 100644 index 0000000000..e1fee7becb --- /dev/null +++ b/src/single_version_servable_definition.hpp @@ -0,0 +1,67 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include + +#include "modelversion.hpp" +#include "servable.hpp" +#include "servable_definition.hpp" +#include "status.hpp" +#include "tensorinfo_fwd.hpp" + +namespace ovms { + +class PipelineDefinitionStatus; +class ServableDefinitionUnloadGuard; +class StatusMetricReporter; +enum class PipelineDefinitionStateCode; + +class SingleVersionServableDefinition : public ServableDefinition, public Servable { + friend class ServableDefinitionUnloadGuard; + +public: + static constexpr model_version_t VERSION = 1; + static constexpr uint64_t WAIT_FOR_LOADED_DEFAULT_TIMEOUT_MICROSECONDS = 500000; + + SingleVersionServableDefinition(const std::string& name) : + Servable(name, VERSION) {} + + const std::string& getName() const override { return Servable::getName(); } + model_version_t getVersion() const override { return Servable::getVersion(); } + + virtual const PipelineDefinitionStatus& getStatus() const = 0; + virtual const tensor_map_t getInputsInfo() const = 0; + virtual const tensor_map_t getOutputsInfo() const = 0; + virtual StatusMetricReporter& getMetricReporter() const = 0; + Status waitForLoaded(std::unique_ptr& guard, + uint32_t waitForLoadedTimeoutMicroseconds = WAIT_FOR_LOADED_DEFAULT_TIMEOUT_MICROSECONDS); + +protected: + std::atomic requestsHandlesCounter = 0; + std::condition_variable loadedNotify; + + void increaseRequestsHandlesCount() { ++requestsHandlesCounter; } + void decreaseRequestsHandlesCount() { --requestsHandlesCounter; } + + virtual StatusCode notLoadedYetCode() const = 0; + virtual StatusCode notLoadedAnymoreCode() const = 0; +}; + +} // namespace ovms diff --git a/src/status.hpp b/src/status.hpp index 9c0624afec..04c6e251c2 100644 --- a/src/status.hpp +++ b/src/status.hpp @@ -21,7 +21,7 @@ #include #include -#include "logging.hpp" +#include namespace ovms { diff --git a/src/tensor_conversion.hpp b/src/tensor_conversion.hpp index 4b23a734b1..c5eb537062 100644 --- a/src/tensor_conversion.hpp +++ b/src/tensor_conversion.hpp @@ -21,6 +21,7 @@ #include #include "deps/opencv.hpp" +#include "logging.hpp" #include "precision.hpp" #include "predict_request_validation_utils_impl.hpp" #include "profiler.hpp" diff --git a/src/tensor_conversion_common.cpp b/src/tensor_conversion_common.cpp index ad073b1740..328705baa4 100644 --- a/src/tensor_conversion_common.cpp +++ b/src/tensor_conversion_common.cpp @@ -20,6 +20,7 @@ #include #include "deps/opencv.hpp" +#include "logging.hpp" #include "precision.hpp" #include "predict_request_validation_utils_impl.hpp" #include "profiler.hpp" diff --git a/src/tensorinfo_fwd.hpp b/src/tensorinfo_fwd.hpp new file mode 100644 index 0000000000..6d261eb845 --- /dev/null +++ b/src/tensorinfo_fwd.hpp @@ -0,0 +1,27 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include + +namespace ovms { + +class TensorInfo; +using tensor_map_t = std::map>; + +} // namespace ovms diff --git a/src/test/azurefilesystem_test.cpp b/src/test/azurefilesystem_test.cpp index 75b667f4df..f46de1ef6d 100644 --- a/src/test/azurefilesystem_test.cpp +++ b/src/test/azurefilesystem_test.cpp @@ -18,7 +18,7 @@ #include "spdlog/spdlog.h" -#include "../azurefilesystem.hpp" +#include "src/filesystem/azurefilesystem.hpp" #include "gtest/gtest.h" using namespace ovms; diff --git a/src/test/c_api_stress_tests.cpp b/src/test/c_api_stress_tests.cpp index 673e1b2d68..4b134cd8d3 100644 --- a/src/test/c_api_stress_tests.cpp +++ b/src/test/c_api_stress_tests.cpp @@ -27,7 +27,7 @@ #include "../dags/pipeline_factory.hpp" #include "../dags/pipelinedefinition.hpp" #include "../get_model_metadata_impl.hpp" -#include "../localfilesystem.hpp" +#include "src/filesystem/localfilesystem.hpp" #include "../logging.hpp" #include "../model_service.hpp" #include "../modelconfig.hpp" diff --git a/src/test/c_api_tests.cpp b/src/test/c_api_tests.cpp index 576bd33b6c..226d672690 100644 --- a/src/test/c_api_tests.cpp +++ b/src/test/c_api_tests.cpp @@ -36,7 +36,7 @@ #include "../capi_frontend/capi_dag_utils.hpp" #include "../capi_frontend/servablemetadata.hpp" #include "../dags/pipelinedefinitionstatus.hpp" -#include "../metric_module.hpp" +#include "src/metrics/metric_module.hpp" #include "../ovms.h" #include "../servablemanagermodule.hpp" #include "../server.hpp" diff --git a/src/test/config_export_full_test.cpp b/src/test/config_export_full_test.cpp index edec5b6989..3fb363db51 100644 --- a/src/test/config_export_full_test.cpp +++ b/src/test/config_export_full_test.cpp @@ -24,7 +24,7 @@ #include "light_test_utils.hpp" #include "platform_utils.hpp" #include "test_with_temp_dir.hpp" -#include "../filesystem.hpp" +#include "src/filesystem/filesystem.hpp" #include "../module_names.hpp" #include "../server.hpp" #include "../status.hpp" diff --git a/src/test/config_export_test.cpp b/src/test/config_export_test.cpp index 87382c509f..20d6af46da 100644 --- a/src/test/config_export_test.cpp +++ b/src/test/config_export_test.cpp @@ -19,7 +19,7 @@ #include #include -#include "src/filesystem.hpp" +#include "src/filesystem/filesystem.hpp" #include "src/status.hpp" #include "src/stringutils.hpp" #include "src/capi_frontend/server_settings.hpp" diff --git a/src/test/constructor_enabled_model_manager.hpp b/src/test/constructor_enabled_model_manager.hpp index 3dba6c624d..9ceeaf970d 100644 --- a/src/test/constructor_enabled_model_manager.hpp +++ b/src/test/constructor_enabled_model_manager.hpp @@ -17,7 +17,7 @@ #include -#include "src/metric_registry.hpp" +#include "src/metrics/metric_registry.hpp" #include "src/modelmanager.hpp" class ConstructorEnabledModelManager : public ovms::ModelManager { diff --git a/src/test/custom_loader_test.cpp b/src/test/custom_loader_test.cpp index f8d926402a..9fe0148699 100644 --- a/src/test/custom_loader_test.cpp +++ b/src/test/custom_loader_test.cpp @@ -28,7 +28,7 @@ #include "../executingstreamidguard.hpp" #include "../get_model_metadata_impl.hpp" -#include "../localfilesystem.hpp" +#include "src/filesystem/localfilesystem.hpp" #include "../model.hpp" #include "../model_service.hpp" #include "../modelinstance.hpp" @@ -57,7 +57,7 @@ using testing::UnorderedElementsAre; using namespace ovms; -/* +/* ------------------------------------------------ AFTER SAMPLE CUSTOM LOADER REMOVAL BELOW CONFIGURATIONS ARE NOT USED REMOVE THIS ENTIRE FILE ONCE THE FEATURE IS REMOVED @@ -621,7 +621,7 @@ TEST_F(TestCustomLoader, CustomLoaderConfigMultiplePropertiesInCustomLoaderOptio } // Functional Validation -/* +/* ------------------------------------------------ AFTER SAMPLE CUSTOM LOADER REMOVAL BELOW TESTS ARE NOT VALID REMOVE THIS ENTIRE FILE ONCE THE FEATURE IS REMOVED diff --git a/src/test/embeddingsnode_test.cpp b/src/test/embeddingsnode_test.cpp index 92a852a502..6443bba525 100644 --- a/src/test/embeddingsnode_test.cpp +++ b/src/test/embeddingsnode_test.cpp @@ -19,6 +19,7 @@ #include #include "../http_rest_api_handler.hpp" +#include "../mediapipe_internal/mediapipefactory.hpp" #include "../servablemanagermodule.hpp" #include "../server.hpp" #include "rapidjson/document.h" diff --git a/src/test/ensemble_config_change_stress.cpp b/src/test/ensemble_config_change_stress.cpp index 7fa5a70d31..225a2ab703 100644 --- a/src/test/ensemble_config_change_stress.cpp +++ b/src/test/ensemble_config_change_stress.cpp @@ -27,7 +27,7 @@ #include "../kfs_frontend/kfs_graph_executor_impl.hpp" #endif #include "../kfs_frontend/kfs_utils.hpp" -#include "../localfilesystem.hpp" +#include "src/filesystem/localfilesystem.hpp" #include "../logging.hpp" #include "../model_service.hpp" #include "../modelconfig.hpp" diff --git a/src/test/ensemble_flow_custom_node_tests.cpp b/src/test/ensemble_flow_custom_node_tests.cpp index 01a81d827e..7d26ed7b91 100644 --- a/src/test/ensemble_flow_custom_node_tests.cpp +++ b/src/test/ensemble_flow_custom_node_tests.cpp @@ -45,9 +45,11 @@ #include "../dags/node_library_utils.hpp" #include "../dags/nodestreamidguard.hpp" #include "../dags/pipeline.hpp" +#include "../dags/pipeline_factory.hpp" #include "../dags/pipelinedefinition.hpp" #include "../execution_context.hpp" -#include "../metric_registry.hpp" +#include "src/metrics/metric_registry.hpp" +#include "../model.hpp" #include "../model_metric_reporter.hpp" #include "../modelinstance.hpp" #include "../modelinstanceunloadguard.hpp" @@ -1520,7 +1522,7 @@ TEST_F(EnsembleFlowCustomNodeLoadConfigThenExecuteTest, AddSubCustomNode) { std::unique_ptr pipeline; this->prepareRequest(inputValues); this->loadCorrectConfiguration(); - ASSERT_EQ(manager.createPipeline(pipeline, pipelineName, &request, &response), StatusCode::OK); + ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); this->checkResponseForCorrectConfiguration(); } @@ -1573,17 +1575,17 @@ TEST_F(EnsembleFlowCustomNodeLoadConfigThenExecuteTest, ReferenceMissingLibraryT // Loading correct configuration is required for test to pass. // This is due to fact that when OVMS loads pipeline definition for the first time and fails, its status is RETIRED. this->loadCorrectConfiguration(); - ASSERT_EQ(manager.createPipeline(pipeline, pipelineName, &request, &response), StatusCode::OK); + ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); this->checkResponseForCorrectConfiguration(); response.Clear(); this->loadConfiguration(pipelineCustomNodeReferenceMissingLibraryConfig, StatusCode::PIPELINE_DEFINITION_INVALID_NODE_LIBRARY); - ASSERT_EQ(manager.createPipeline(pipeline, pipelineName, &request, &response), StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET); + ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET); response.Clear(); this->loadCorrectConfiguration(); - ASSERT_EQ(manager.createPipeline(pipeline, pipelineName, &request, &response), StatusCode::OK); + ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); this->checkResponseForCorrectConfiguration(); } @@ -1634,18 +1636,18 @@ TEST_F(EnsembleFlowCustomNodeLoadConfigThenExecuteTest, ReferenceLibraryWithExec // Loading correct configuration is required for test to pass. // This is due to fact that when OVMS loads pipeline definition for the first time and fails, its status is RETIRED. this->loadCorrectConfiguration(); - ASSERT_EQ(manager.createPipeline(pipeline, pipelineName, &request, &response), StatusCode::OK); + ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); this->checkResponseForCorrectConfiguration(); response.Clear(); this->loadConfiguration(pipelineCustomNodeReferenceLibraryWithExecutionErrorMissingParamsLibraryConfig); - ASSERT_EQ(manager.createPipeline(pipeline, pipelineName, &request, &response), StatusCode::OK); + ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::NODE_LIBRARY_EXECUTION_FAILED); response.Clear(); this->loadCorrectConfiguration(); - ASSERT_EQ(manager.createPipeline(pipeline, pipelineName, &request, &response), StatusCode::OK); + ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); this->checkResponseForCorrectConfiguration(); } @@ -1697,18 +1699,18 @@ TEST_F(EnsembleFlowCustomNodeLoadConfigThenExecuteTest, MissingRequiredNodeParam // Loading correct configuration is required for test to pass. // This is due to fact that when OVMS loads pipeline definition for the first time and fails, its status is RETIRED. this->loadCorrectConfiguration(); - ASSERT_EQ(manager.createPipeline(pipeline, pipelineName, &request, &response), StatusCode::OK); + ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); this->checkResponseForCorrectConfiguration(); response.Clear(); this->loadConfiguration(pipelineCustomNodeMissingParametersConfig); - ASSERT_EQ(manager.createPipeline(pipeline, pipelineName, &request, &response), StatusCode::OK); + ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::NODE_LIBRARY_EXECUTION_FAILED); response.Clear(); this->loadCorrectConfiguration(); - ASSERT_EQ(manager.createPipeline(pipeline, pipelineName, &request, &response), StatusCode::OK); + ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); this->checkResponseForCorrectConfiguration(); } @@ -1761,17 +1763,17 @@ TEST_F(EnsembleFlowCustomNodeLoadConfigThenExecuteTest, ReferenceLibraryWithRest // Loading correct configuration is required for test to pass. // This is due to fact that when OVMS loads pipeline definition for the first time and fails, its status is RETIRED. this->loadCorrectConfiguration(); - ASSERT_EQ(manager.createPipeline(pipeline, pipelineName, &request, &response), StatusCode::OK); + ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); this->checkResponseForCorrectConfiguration(); response.Clear(); this->loadConfiguration(pipelineCustomNodeLibraryNotEscapedPathConfig, StatusCode::PIPELINE_DEFINITION_INVALID_NODE_LIBRARY); - ASSERT_EQ(manager.createPipeline(pipeline, pipelineName, &request, &response), StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET); + ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET); response.Clear(); this->loadCorrectConfiguration(); - ASSERT_EQ(manager.createPipeline(pipeline, pipelineName, &request, &response), StatusCode::OK); + ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); this->checkResponseForCorrectConfiguration(); } @@ -1920,7 +1922,7 @@ TEST_F(EnsembleFlowCustomNodeAndDemultiplexerLoadConfigThenExecuteTest, JustDiff this->prepareRequest(request, input, differentOpsInputName); this->prepareRequest(request, factors, differentOpsFactorsName); this->loadConfiguration(pipelineCustomNodeDifferentOperationsConfig); - ASSERT_EQ(manager.createPipeline(pipeline, pipelineName, &request, &response), StatusCode::OK); + ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); std::vector expectedOutput(4 * DUMMY_MODEL_OUTPUT_SIZE); @@ -2011,7 +2013,7 @@ TEST_F(EnsembleFlowCustomNodeAndDemultiplexerLoadConfigThenExecuteTest, Differen this->prepareRequest(request, input, differentOpsInputName); this->prepareRequest(request, factors, differentOpsFactorsName); this->loadConfiguration(pipelineCustomNodeDifferentOperationsThenDummyConfig); - ASSERT_EQ(manager.createPipeline(pipeline, pipelineName, &request, &response), StatusCode::OK); + ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); std::vector expectedOutput(4 * DUMMY_MODEL_OUTPUT_SIZE); prepareDifferentOpsExpectedOutput(expectedOutput, input, factors); @@ -2093,7 +2095,7 @@ TEST_F(EnsembleFlowCustomNodeAndDemultiplexerLoadConfigThenExecuteTest, Differen this->prepareRequest(request, input, differentOpsInputName); this->prepareRequest(request, factors, differentOpsFactorsName); this->loadConfiguration(pipelineCustomNodeDifferentOperations2OutputsConfig); - ASSERT_EQ(manager.createPipeline(pipeline, pipelineName, &request, &response), StatusCode::OK); + ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); std::vector expectedOutput(4 * DUMMY_MODEL_OUTPUT_SIZE); @@ -2207,7 +2209,7 @@ TEST_F(EnsembleFlowCustomNodeAndDemultiplexerLoadConfigThenExecuteTest, Differen this->prepareRequest(request, input, differentOpsInputName); this->prepareRequest(request, factors, differentOpsFactorsName); this->loadConfiguration(pipelineCustomNodeDifferentOperationsThenDummyThenChooseMaximumConfig); - ASSERT_EQ(manager.createPipeline(pipeline, pipelineName, &request, &response), StatusCode::OK); + ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); std::vector expectedOutput(4 * DUMMY_MODEL_OUTPUT_SIZE); @@ -2322,7 +2324,7 @@ TEST_F(EnsembleFlowCustomNodeAndDemultiplexerLoadConfigThenExecuteTest, Differen this->prepareRequest(request, input, differentOpsInputName); this->prepareRequest(request, factors, differentOpsFactorsName); this->loadConfiguration(pipelineCustomNodeDifferentOperationsThenDummyThenChooseMaximumThenDummyConfig); - ASSERT_EQ(manager.createPipeline(pipeline, pipelineName, &request, &response), StatusCode::OK); + ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); std::vector expectedOutput(4 * DUMMY_MODEL_OUTPUT_SIZE); @@ -2415,7 +2417,7 @@ TEST_F(EnsembleFlowCustomNodeAndDemultiplexerLoadConfigThenExecuteTest, Demultip this->prepareRequest(request, input, differentOpsInputName, {4, 1, 10}); this->loadConfiguration(demultiplyThenDummyThenChooseMaximumConfig); - ASSERT_EQ(manager.createPipeline(pipeline, pipelineName, &request, &response), StatusCode::OK); + ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); auto status = pipeline->execute(DEFAULT_TEST_CONTEXT); ASSERT_EQ(status, StatusCode::OK) << status.string(); @@ -4474,7 +4476,7 @@ TEST_F(EnsembleFlowCustomNodeAndDynamicDemultiplexerLoadConfigThenExecuteTest, J std::vector input{static_cast(dynamicDemultiplyCount), 1, 2, 3, 4, 5, 6, 7, 8, 9}; this->prepareRequest(request, input, differentOpsInputName); this->loadConfiguration(pipelineCustomNodeDynamicDemultiplexThenDummyConfig); - ASSERT_EQ(manager.createPipeline(pipeline, pipelineName, &request, &response), StatusCode::OK); + ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); std::vector expectedOutput(dynamicDemultiplyCount * DUMMY_MODEL_OUTPUT_SIZE); @@ -4667,7 +4669,7 @@ TEST_F(EnsembleFlowCustomNodeAndDynamicDemultiplexerLoadConfigThenExecuteTest, D std::iota(input.begin(), input.end(), 42); this->prepareRequest(request, input, differentOpsInputName, {dynamicDemultiplyCount, 1, 10}); this->loadConfiguration(pipelineEntryNodeDynamicDemultiplexThenDummyConfig); - ASSERT_EQ(manager.createPipeline(pipeline, pipelineName, &request, &response), StatusCode::OK); + ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); std::vector expectedOutput = input; @@ -4741,7 +4743,7 @@ TEST_F(EnsembleFlowCustomNodeAndDynamicDemultiplexerLoadConfigThenExecuteTest, D std::iota(input.begin(), input.end(), 42); this->prepareRequest(request, input, pipelineInputName, {3, 5, DUMMY_MODEL_INPUT_SIZE}); this->loadConfiguration(pipelineEntryNodeDemultiplexThenDummyConfig); - ASSERT_EQ(manager.createPipeline(pipeline, pipelineName, &request, &response), StatusCode::OK); + ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); std::vector expectedOutput = input; @@ -4774,7 +4776,7 @@ TEST_F(EnsembleFlowCustomNodeAndDynamicDemultiplexerLoadConfigThenExecuteTest, D std::vector input{static_cast(dynamicDemultiplyCount), 1, 2, 3, 4, 5, 6, 7, 8, 9}; this->prepareRequest(request, input, differentOpsInputName); this->loadConfiguration(pipelineCustomNodeDynamicDemultiplexThenDummyConfig); - ASSERT_EQ(manager.createPipeline(pipeline, pipelineName, &request, &response), StatusCode::OK); + ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); auto status = pipeline->execute(DEFAULT_TEST_CONTEXT); ASSERT_EQ(status, StatusCode::PIPELINE_TOO_LARGE_DIMENSION_SIZE_TO_DEMULTIPLY) << status.string(); } @@ -4870,7 +4872,7 @@ TEST_F(EnsembleFlowCustomNodeAndDemultiplexerLoadConfigThenExecuteTest, Differen this->prepareRequest(request, input, differentOpsInputName); this->prepareRequest(request, factors, differentOpsFactorsName); this->loadConfiguration(pipelineCustomNodeDifferentOperationsThenDummyThenChooseMaximumNotInOrderConfig); - ASSERT_EQ(manager.createPipeline(pipeline, pipelineName, &request, &response), StatusCode::OK); + ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); std::vector expectedOutput(4 * DUMMY_MODEL_OUTPUT_SIZE); @@ -4887,7 +4889,7 @@ TEST_F(EnsembleFlowCustomNodeAndDynamicDemultiplexerLoadConfigThenExecuteTest, D std::vector input{static_cast(dynamicDemultiplyCount), 1, 2, 3, 4, 5, 6, 7, 8, 9}; this->prepareRequest(request, input, differentOpsInputName); this->loadConfiguration(pipelineCustomNodeDynamicDemultiplexThenDummyConfig); - ASSERT_EQ(manager.createPipeline(pipeline, pipelineName, &request, &response), StatusCode::OK); + ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::PIPELINE_DEMULTIPLEXER_NO_RESULTS); } @@ -4897,7 +4899,7 @@ TEST_F(EnsembleFlowCustomNodeAndDynamicDemultiplexerLoadConfigThenExecuteTest, D std::vector input{static_cast(dynamicDemultiplyCount), 1, 2, 3, 4, 5, 6, 7, 8, 9}; this->prepareRequest(request, input, differentOpsInputName); this->loadConfiguration(pipelineCustomNodeDynamicDemultiplexThenDummyConfig); - ASSERT_EQ(manager.createPipeline(pipeline, pipelineName, &request, &response), StatusCode::OK); + ASSERT_EQ(manager.getPipelineFactory().create(pipeline, pipelineName, &request, &response, manager), StatusCode::OK); ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); std::vector expectedOutput(dynamicDemultiplyCount * DUMMY_MODEL_OUTPUT_SIZE); diff --git a/src/test/ensemble_mapping_config_tests.cpp b/src/test/ensemble_mapping_config_tests.cpp index 23e6a84a4c..5c34f11f0c 100644 --- a/src/test/ensemble_mapping_config_tests.cpp +++ b/src/test/ensemble_mapping_config_tests.cpp @@ -26,6 +26,7 @@ #include "../execution_context.hpp" #include "../get_model_metadata_impl.hpp" #include "../kfs_frontend/kfs_grpc_inference_service.hpp" +#include "../model.hpp" #include "../model_metric_reporter.hpp" #include "../modelconfig.hpp" #include "../modelmanager.hpp" diff --git a/src/test/ensemble_tests.cpp b/src/test/ensemble_tests.cpp index 1eb83595d4..923cd9d69c 100644 --- a/src/test/ensemble_tests.cpp +++ b/src/test/ensemble_tests.cpp @@ -37,9 +37,11 @@ #include "../dags/pipelinedefinition.hpp" #include "../tfs_frontend/deserialization.hpp" #include "../inference_executor.hpp" -#include "../localfilesystem.hpp" +#include "src/filesystem/localfilesystem.hpp" #include "../logging.hpp" -#include "../metric_registry.hpp" +#include "../mediapipe_internal/mediapipefactory.hpp" +#include "src/metrics/metric_registry.hpp" +#include "../model.hpp" #include "../model_metric_reporter.hpp" #include "../modelconfig.hpp" #include "../modelinstance.hpp" @@ -225,10 +227,11 @@ class EnsembleFlowTest : public TestWithTempDir { ConstructorEnabledModelManager managerWithDummyModel; managerWithDummyModel.loadConfig(fileToReload); std::unique_ptr pipeline; - auto status = managerWithDummyModel.createPipeline(pipeline, + auto status = managerWithDummyModel.getPipelineFactory().create(pipeline, "pipeline1Dummy", &request, - &response); + &response, + managerWithDummyModel); ASSERT_EQ(status, ovms::StatusCode::PIPELINE_DEFINITION_NAME_MISSING) << status.string(); } @@ -965,9 +968,36 @@ TEST_F(EnsembleFlowTest, ExecutePipelineWithBatchSizeAny) { const int batchSize = 3; proto.mutable_tensor_shape()->mutable_dim(0)->set_size(batchSize); requestData = { - -5, -4, -3, -2, -1, 1, 2, 3, 4, 5, // batch 1 - -15, -14, -13, -12, -11, 11, 12, 13, 14, 15, // batch 2 - -25, -24, -23, -22, -21, 21, 22, 23, 24, 25, // batch 3 + -5, + -4, + -3, + -2, + -1, + 1, + 2, + 3, + 4, + 5, // batch 1 + -15, + -14, + -13, + -12, + -11, + 11, + 12, + 13, + 14, + 15, // batch 2 + -25, + -24, + -23, + -22, + -21, + 21, + 22, + 23, + 24, + 25, // batch 3 }; proto.mutable_tensor_content()->assign((char*)requestData.data(), requestData.size() * sizeof(float)); @@ -1014,9 +1044,36 @@ TEST_F(EnsembleFlowTest, ExecutePipelineWithBatchSizeRange) { int batchSize = 3; proto.mutable_tensor_shape()->mutable_dim(0)->set_size(batchSize); requestData = { - -5, -4, -3, -2, -1, 1, 2, 3, 4, 5, // batch 1 - -15, -14, -13, -12, -11, 11, 12, 13, 14, 15, // batch 2 - -25, -24, -23, -22, -21, 21, 22, 23, 24, 25, // batch 3 + -5, + -4, + -3, + -2, + -1, + 1, + 2, + 3, + 4, + 5, // batch 1 + -15, + -14, + -13, + -12, + -11, + 11, + 12, + 13, + 14, + 15, // batch 2 + -25, + -24, + -23, + -22, + -21, + 21, + 22, + 23, + 24, + 25, // batch 3 }; proto.mutable_tensor_content()->assign((char*)requestData.data(), requestData.size() * sizeof(float)); @@ -1086,7 +1143,11 @@ TEST_F(EnsembleFlowTest, ExecutePipelineWithShapeAny) { tensorflow::TensorProto& proto = (*request.mutable_inputs())[customPipelineInputName]; proto.mutable_tensor_shape()->mutable_dim(1)->set_size(5); std::vector requestData = { - -5, -4, -3, -2, -1, // batch 1 + -5, + -4, + -3, + -2, + -1, // batch 1 }; proto.mutable_tensor_content()->assign((char*)requestData.data(), requestData.size() * sizeof(float)); @@ -1147,7 +1208,11 @@ TEST_F(EnsembleFlowTest, ExecutePipelineWithShapeRange) { tensorflow::TensorProto& proto = (*request.mutable_inputs())[customPipelineInputName]; proto.mutable_tensor_shape()->mutable_dim(1)->set_size(5); std::vector requestData = { - -5, -4, -3, -2, -1, // batch 1 + -5, + -4, + -3, + -2, + -1, // batch 1 }; proto.mutable_tensor_content()->assign((char*)requestData.data(), requestData.size() * sizeof(float)); @@ -1416,9 +1481,36 @@ TEST_F(EnsembleFlowTest, DISABLED_ExecutePipelineWithDynamicBatchSize) { const int batchSize = 3; proto.mutable_tensor_shape()->mutable_dim(0)->set_size(batchSize); requestData = { - -5, -4, -3, -2, -1, 1, 2, 3, 4, 5, // batch 1 - -15, -14, -13, -12, -11, 11, 12, 13, 14, 15, // batch 2 - -25, -24, -23, -22, -21, 21, 22, 23, 24, 25, // batch 3 + -5, + -4, + -3, + -2, + -1, + 1, + 2, + 3, + 4, + 5, // batch 1 + -15, + -14, + -13, + -12, + -11, + 11, + 12, + 13, + 14, + 15, // batch 2 + -25, + -24, + -23, + -22, + -21, + 21, + 22, + 23, + 24, + 25, // batch 3 }; proto.mutable_tensor_content()->assign((char*)requestData.data(), requestData.size() * sizeof(float)); @@ -1466,7 +1558,11 @@ TEST_F(EnsembleFlowTest, DISABLED_ExecutePipelineWithDynamicShape) { tensorflow::TensorProto& proto = (*request.mutable_inputs())[customPipelineInputName]; proto.mutable_tensor_shape()->mutable_dim(1)->set_size(5); std::vector requestData = { - -5, -4, -3, -2, -1, // batch 1 + -5, + -4, + -3, + -2, + -1, // batch 1 }; proto.mutable_tensor_content()->assign((char*)requestData.data(), requestData.size() * sizeof(float)); @@ -2886,10 +2982,11 @@ TEST_F(EnsembleFlowTest, PipelineFactoryCreationWithInputOutputsMappings) { ConstructorEnabledModelManager managerWithDummyModel; managerWithDummyModel.loadConfig(fileToReload); std::unique_ptr pipeline; - auto status = managerWithDummyModel.createPipeline(pipeline, + auto status = managerWithDummyModel.getPipelineFactory().create(pipeline, "pipeline1Dummy", &request, - &response); + &response, + managerWithDummyModel); ASSERT_EQ(status, ovms::StatusCode::OK) << status.string(); ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); const int dummySeriallyConnectedCount = 1; @@ -2959,10 +3056,11 @@ TEST_F(EnsembleFlowTest, PipelineFactoryCreationWithInputOutputsMappings2Paralle ConstructorEnabledModelManager managerWithDummyModel; managerWithDummyModel.loadConfig(fileToReload); std::unique_ptr pipeline; - auto status = managerWithDummyModel.createPipeline(pipeline, + auto status = managerWithDummyModel.getPipelineFactory().create(pipeline, "pipeline1Dummy", &request, - &response); + &response, + managerWithDummyModel); ASSERT_EQ(status, ovms::StatusCode::OK) << status.string(); ASSERT_EQ(pipeline->execute(DEFAULT_TEST_CONTEXT), StatusCode::OK); ASSERT_EQ(response.outputs().count(customPipelineOutputName), 1); @@ -4362,7 +4460,7 @@ TEST_F(EnsembleFlowTest, MediapipeConfigModelWithSameNamePipeline) { ASSERT_FALSE(manager.getMediapipeFactory().definitionExists(MEDIAPIPE_DUMMY_NAME)); - ASSERT_TRUE(manager.pipelineDefinitionExists(MEDIAPIPE_DUMMY_NAME)); + ASSERT_TRUE(manager.servableExists(MEDIAPIPE_DUMMY_NAME, ServableType::Pipeline)); } #endif TEST_F(EnsembleFlowTest, PipelineConfigModelWithSameName) { diff --git a/src/test/gather_node_test.cpp b/src/test/gather_node_test.cpp index b2ba912d48..3e6d2cb716 100644 --- a/src/test/gather_node_test.cpp +++ b/src/test/gather_node_test.cpp @@ -197,9 +197,9 @@ static const char* configDummy1BsDummy2Bs = R"( class DLNodeSessionWithGetInputsExposed : public DLNodeSession { public: - DLNodeSessionWithGetInputsExposed(const NodeSessionMetadata& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, ModelManager& manager, const std::string& modelName, model_version_t modelVersion) : + DLNodeSessionWithGetInputsExposed(const NodeSessionMetadata& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, ModelInstanceProvider& manager, const std::string& modelName, model_version_t modelVersion) : DLNodeSession(metadata, nodeName, inputsCount, collapsingDetails, manager, modelName, modelVersion) {} - DLNodeSessionWithGetInputsExposed(const NodeSessionMetadata&& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, ModelManager& manager, const std::string& modelName, model_version_t modelVersion) : + DLNodeSessionWithGetInputsExposed(const NodeSessionMetadata&& metadata, const std::string& nodeName, uint32_t inputsCount, const CollapseDetails& collapsingDetails, ModelInstanceProvider& manager, const std::string& modelName, model_version_t modelVersion) : DLNodeSession(std::move(metadata), nodeName, inputsCount, collapsingDetails, manager, modelName, modelVersion) {} const auto& getInputs() const { diff --git a/src/test/gcsfilesystem_test.cpp b/src/test/gcsfilesystem_test.cpp index 00b869178c..76a2481426 100644 --- a/src/test/gcsfilesystem_test.cpp +++ b/src/test/gcsfilesystem_test.cpp @@ -18,7 +18,7 @@ #include "spdlog/spdlog.h" -#include "../gcsfilesystem.hpp" +#include "src/filesystem/gcsfilesystem.hpp" #include "gtest/gtest.h" using namespace ovms; diff --git a/src/test/get_mediapipe_graph_metadata_response_test.cpp b/src/test/get_mediapipe_graph_metadata_response_test.cpp index 4b70491273..bbbae9472c 100644 --- a/src/test/get_mediapipe_graph_metadata_response_test.cpp +++ b/src/test/get_mediapipe_graph_metadata_response_test.cpp @@ -30,7 +30,7 @@ #include "../get_model_metadata_impl.hpp" #include "../grpcservermodule.hpp" #include "../kfs_frontend/kfs_grpc_inference_service.hpp" -#include "../localfilesystem.hpp" +#include "src/filesystem/localfilesystem.hpp" #include "../model.hpp" #include "../model_service.hpp" #include "../modelinstance.hpp" diff --git a/src/test/graph_export_test.cpp b/src/test/graph_export_test.cpp index 777792e7d3..468d2e7917 100644 --- a/src/test/graph_export_test.cpp +++ b/src/test/graph_export_test.cpp @@ -23,7 +23,7 @@ #include "light_test_utils.hpp" #include "../capi_frontend/server_settings.hpp" #include "../graph_export/graph_export.hpp" -#include "../filesystem.hpp" +#include "src/filesystem/filesystem.hpp" #include "../status.hpp" #include "../version.hpp" diff --git a/src/test/http_openai_handler_test.cpp b/src/test/http_openai_handler_test.cpp index 2aea78eaae..fc6fc5706a 100644 --- a/src/test/http_openai_handler_test.cpp +++ b/src/test/http_openai_handler_test.cpp @@ -14,6 +14,7 @@ // limitations under the License. //***************************************************************************** #include +#include #include #include #include diff --git a/src/test/http_rest_api_handler_test.cpp b/src/test/http_rest_api_handler_test.cpp index 711dbb966f..c640012344 100644 --- a/src/test/http_rest_api_handler_test.cpp +++ b/src/test/http_rest_api_handler_test.cpp @@ -17,7 +17,7 @@ #include "../config.hpp" #include "../http_rest_api_handler.hpp" -#include "../localfilesystem.hpp" +#include "src/filesystem/localfilesystem.hpp" #include "../logging.hpp" #include "../modelmanager.hpp" #include "../servablemanagermodule.hpp" @@ -470,10 +470,7 @@ TEST_F(ConfigReload, removeModelFromDiskWhenLoaded) { // On windows, the test expects inability to remove model #ifdef _WIN32 - EXPECT_THROW({ - std::filesystem::remove_all(getGenericFullPathForSrcTest("/tmp/dummy")); - }, - std::filesystem::filesystem_error); + EXPECT_THROW({ std::filesystem::remove_all(getGenericFullPathForSrcTest("/tmp/dummy")); }, std::filesystem::filesystem_error); #else // On linux, removing the model should be possible even if ENABLE_MMAP=YES std::filesystem::remove_all(getGenericFullPathForSrcTest("/tmp/dummy")); diff --git a/src/test/listdirectorymodels_test.cpp b/src/test/listdirectorymodels_test.cpp index a08561b49a..6fff6b6043 100644 --- a/src/test/listdirectorymodels_test.cpp +++ b/src/test/listdirectorymodels_test.cpp @@ -21,7 +21,7 @@ #include #include "test_with_temp_dir.hpp" -#include "src/filesystem.hpp" +#include "src/filesystem/filesystem.hpp" #include "src/pull_module/hf_pull_model_module.hpp" #include "src/pull_module/libgit2.hpp" #include "src/servables_config_manager_module/listmodels.hpp" diff --git a/src/test/llm/assisted_decoding_test.cpp b/src/test/llm/assisted_decoding_test.cpp index e265cd6b68..9c870a1f63 100644 --- a/src/test/llm/assisted_decoding_test.cpp +++ b/src/test/llm/assisted_decoding_test.cpp @@ -22,6 +22,8 @@ #include #include +#include + #include #include #include diff --git a/src/test/llm/llmnode_test.cpp b/src/test/llm/llmnode_test.cpp index 55f87fa720..0cbcfa2f2d 100644 --- a/src/test/llm/llmnode_test.cpp +++ b/src/test/llm/llmnode_test.cpp @@ -22,6 +22,8 @@ #include #include +#include + #include #include #include diff --git a/src/test/llm/llmtemplate_test.cpp b/src/test/llm/llmtemplate_test.cpp index 8cdf565cba..1fdb5cd0d0 100644 --- a/src/test/llm/llmtemplate_test.cpp +++ b/src/test/llm/llmtemplate_test.cpp @@ -28,7 +28,7 @@ #pragma warning(pop) #include -#include "../../filesystem.hpp" +#include "src/filesystem/filesystem.hpp" #include "../../http_payload.hpp" #include "../../http_rest_api_handler.hpp" #include "../../httpservermodule.hpp" diff --git a/src/test/localfilesystem_test.cpp b/src/test/localfilesystem_test.cpp index 6aa22f1abc..e53e9885d8 100644 --- a/src/test/localfilesystem_test.cpp +++ b/src/test/localfilesystem_test.cpp @@ -25,8 +25,8 @@ #include #include -#include "../filesystem.hpp" -#include "../localfilesystem.hpp" +#include "src/filesystem/filesystem.hpp" +#include "src/filesystem/localfilesystem.hpp" using namespace testing; using ::testing::UnorderedElementsAre; diff --git a/src/test/mediapipe_framework_test.cpp b/src/test/mediapipe_framework_test.cpp index 85abfbd519..e028dbd269 100644 --- a/src/test/mediapipe_framework_test.cpp +++ b/src/test/mediapipe_framework_test.cpp @@ -31,8 +31,8 @@ #include "../kfs_frontend/kfs_grpc_inference_service.hpp" #include "../mediapipe_internal/mediapipefactory.hpp" #include "../mediapipe_internal/mediapipegraphdefinition.hpp" -#include "../metric_config.hpp" -#include "../metric_module.hpp" +#include "src/metrics/metric_config.hpp" +#include "src/metrics/metric_module.hpp" #include "../model_service.hpp" #include "../precision.hpp" #include "../servablemanagermodule.hpp" diff --git a/src/test/mediapipeflow_test.cpp b/src/test/mediapipeflow_test.cpp index 55b6ab96ed..cb07d4a52f 100644 --- a/src/test/mediapipeflow_test.cpp +++ b/src/test/mediapipeflow_test.cpp @@ -36,6 +36,7 @@ #pragma GCC diagnostic pop #include "../config.hpp" +#include "../dags/pipeline_factory.hpp" #include "../dags/pipelinedefinition.hpp" #include "../grpcservermodule.hpp" #include "../http_rest_api_handler.hpp" @@ -45,8 +46,9 @@ #include "../mediapipe_internal/mediapipefactory.hpp" #include "../mediapipe_internal/mediapipegraphdefinition.hpp" #include "../mediapipe_internal/mediapipegraphexecutor.hpp" -#include "../metric_config.hpp" -#include "../metric_module.hpp" +#include "src/metrics/metric_config.hpp" +#include "src/metrics/metric_module.hpp" +#include "../model.hpp" #include "../model_service.hpp" #include "../ovms_exit_codes.hpp" #include "../precision.hpp" @@ -1487,8 +1489,6 @@ TEST_F(MediapipeStreamFlowAddTest, Infer) { TEST_F(MediapipeStreamFlowAddTest, InferOnUnloadedGraph) { const ovms::Module* grpcModule = server.getModule(ovms::GRPC_SERVER_MODULE_NAME); KFSInferenceServiceImpl& impl = dynamic_cast(grpcModule)->getKFSGrpcImpl(); - const ServableManagerModule* smm = dynamic_cast(server.getModule(SERVABLE_MANAGER_MODULE_NAME)); - ModelManager& modelManager = smm->getServableManager(); auto* definition = this->getMPDefinitionByName(this->modelName); ASSERT_NE(definition, nullptr); @@ -1532,10 +1532,10 @@ TEST_F(MediapipeStreamFlowAddTest, InferOnUnloadedGraph) { checkAddResponse("out", this->requestData1[2], this->requestData1[2], this->request[2], msg.infer_response(), 1, 1, this->modelName); return true; }); - std::thread unloader([&startUnloading, &finishedUnloading, &definition, &modelManager]() { + std::thread unloader([&startUnloading, &finishedUnloading, &definition]() { // Wait till first response notifies that we should start unloading startUnloading.get_future().get(); - definition->retire(modelManager); + definition->retire(); // Notify second request to arrive because we unloaded the graph finishedUnloading.set_value(); }); @@ -1654,11 +1654,9 @@ TEST_F(MediapipeStreamFlowAddTest, InferOnReloadedGraph) { TEST_F(MediapipeStreamFlowAddTest, NegativeShouldNotReachInferDueToRetiredGraph) { const ovms::Module* grpcModule = server.getModule(ovms::GRPC_SERVER_MODULE_NAME); KFSInferenceServiceImpl& impl = dynamic_cast(grpcModule)->getKFSGrpcImpl(); - const ServableManagerModule* smm = dynamic_cast(server.getModule(SERVABLE_MANAGER_MODULE_NAME)); - ModelManager& modelManager = smm->getServableManager(); auto* definition = this->getMPDefinitionByName(this->modelName); ASSERT_NE(definition, nullptr); - definition->retire(modelManager); + definition->retire(); // Opening new stream, expect graph to be unavailable MockedServerReaderWriter<::inference::ModelStreamInferResponse, ::inference::ModelInferRequest> stream; @@ -3716,10 +3714,10 @@ TEST(WhitelistRegistered, MediapipeCalculatorsList) { // Expected when building with python "CalculatorRunnerSinkCalculator", "CalculatorRunnerSourceCalculator", - "PyTensorOvTensorConverterCalculator", // integral OVMS calculator - "PythonExecutorCalculator", // integral OVMS calculator + "PyTensorOvTensorConverterCalculator", // integral OVMS calculator + "PythonExecutorCalculator", // integral OVMS calculator #endif - "HttpLLMCalculator", // integral OVMS calculator + "HttpLLMCalculator", // integral OVMS calculator "OpenAIChatCompletionsMockCalculator", // OVMS test calculator "AddHeaderCalculator", "AddNumbersMultiInputsOutputsTestCalculator", diff --git a/src/test/metric_config_test.cpp b/src/test/metric_config_test.cpp index ac2317a624..23184bdf73 100644 --- a/src/test/metric_config_test.cpp +++ b/src/test/metric_config_test.cpp @@ -20,8 +20,8 @@ #include #include "../config.hpp" -#include "../metric_config.hpp" -#include "../metric_registry.hpp" +#include "src/metrics/metric_config.hpp" +#include "src/metrics/metric_registry.hpp" #include "../model_metric_reporter.hpp" #include "../modelconfig.hpp" #include "../modelinstance.hpp" diff --git a/src/test/metrics_flow_test.cpp b/src/test/metrics_flow_test.cpp index f2defd4e9a..61349c8b64 100644 --- a/src/test/metrics_flow_test.cpp +++ b/src/test/metrics_flow_test.cpp @@ -30,8 +30,8 @@ #include "../get_model_metadata_impl.hpp" #include "../http_rest_api_handler.hpp" #include "../kfs_frontend/kfs_grpc_inference_service.hpp" -#include "../metric_config.hpp" -#include "../metric_module.hpp" +#include "src/metrics/metric_config.hpp" +#include "src/metrics/metric_module.hpp" #include "../model_service.hpp" #include "../precision.hpp" #include "../prediction_service.hpp" diff --git a/src/test/metrics_test.cpp b/src/test/metrics_test.cpp index 37dedb6277..04cab0fff0 100644 --- a/src/test/metrics_test.cpp +++ b/src/test/metrics_test.cpp @@ -22,9 +22,9 @@ #include #include -#include "../metric.hpp" -#include "../metric_family.hpp" -#include "../metric_registry.hpp" +#include "src/metrics/metric.hpp" +#include "src/metrics/metric_family.hpp" +#include "src/metrics/metric_registry.hpp" using namespace ovms; diff --git a/src/test/model_test.cpp b/src/test/model_test.cpp index ce1f908819..8b6e9dad95 100644 --- a/src/test/model_test.cpp +++ b/src/test/model_test.cpp @@ -18,7 +18,7 @@ #include #include -#include "../filesystem.hpp" +#include "src/filesystem/filesystem.hpp" #include "../model.hpp" #include "../modelmanager.hpp" #include "mockmodelinstancechangingstates.hpp" diff --git a/src/test/modelmanager_test.cpp b/src/test/modelmanager_test.cpp index d51efeb433..0c56e0218d 100644 --- a/src/test/modelmanager_test.cpp +++ b/src/test/modelmanager_test.cpp @@ -24,7 +24,7 @@ #include "../config.hpp" #include "../dags/custom_node_library_internal_manager_wrapper.hpp" #include "../dags/node_library.hpp" -#include "../localfilesystem.hpp" +#include "src/filesystem/localfilesystem.hpp" #include "../logging.hpp" #include "../model.hpp" #include "../modelinstanceunloadguard.hpp" diff --git a/src/test/ov_utils_test.cpp b/src/test/ov_utils_test.cpp index 4bc44ca01b..2f71624386 100644 --- a/src/test/ov_utils_test.cpp +++ b/src/test/ov_utils_test.cpp @@ -19,7 +19,7 @@ #include #include -#include "../filesystem.hpp" +#include "src/filesystem/filesystem.hpp" #include "../modelinstance.hpp" #include "../ov_utils.hpp" #include "test_utils.hpp" diff --git a/src/test/ovmsconfig_test.cpp b/src/test/ovmsconfig_test.cpp index 5e6f694f56..6b76d0e2fa 100644 --- a/src/test/ovmsconfig_test.cpp +++ b/src/test/ovmsconfig_test.cpp @@ -26,7 +26,7 @@ #include "../capi_frontend/server_settings.hpp" #include "../utils/env_guard.hpp" #include "../config.hpp" -#include "../filesystem.hpp" +#include "src/filesystem/filesystem.hpp" #include "../ovms_exit_codes.hpp" #include "../systeminfo.hpp" #include "test_utils.hpp" diff --git a/src/test/pull_gguf_hf_model_test.cpp b/src/test/pull_gguf_hf_model_test.cpp index a8366a167a..ffb5350fa2 100644 --- a/src/test/pull_gguf_hf_model_test.cpp +++ b/src/test/pull_gguf_hf_model_test.cpp @@ -25,8 +25,8 @@ #include "test_utils.hpp" #include "test_with_temp_dir.hpp" #include "gguf_environment.hpp" -#include "../filesystem.hpp" -#include "../localfilesystem.hpp" +#include "src/filesystem/filesystem.hpp" +#include "src/filesystem/localfilesystem.hpp" #include "src/pull_module/gguf_downloader.hpp" #include "src/modelextensions.hpp" diff --git a/src/test/pull_hf_model_test.cpp b/src/test/pull_hf_model_test.cpp index 1fbd0798f6..55199f6971 100644 --- a/src/test/pull_hf_model_test.cpp +++ b/src/test/pull_hf_model_test.cpp @@ -31,7 +31,7 @@ #include "src/test/test_utils.hpp" #include "src/test/test_file_utils.hpp" #include "src/test/test_with_temp_dir.hpp" -#include "src/filesystem.hpp" +#include "src/filesystem/filesystem.hpp" #include "src/pull_module/hf_pull_model_module.hpp" #include "src/pull_module/libgit2.hpp" #include "src/pull_module/optimum_export.hpp" diff --git a/src/test/pythonnode_test.cpp b/src/test/pythonnode_test.cpp index 54c9acbfa1..5b31146d19 100644 --- a/src/test/pythonnode_test.cpp +++ b/src/test/pythonnode_test.cpp @@ -36,8 +36,8 @@ #include "../mediapipe_internal/mediapipefactory.hpp" #include "../mediapipe_internal/mediapipegraphdefinition.hpp" #include "../mediapipe_internal/mediapipegraphexecutor.hpp" -#include "../metric_config.hpp" -#include "../metric_module.hpp" +#include "src/metrics/metric_config.hpp" +#include "src/metrics/metric_module.hpp" #include "../model_service.hpp" #include "../precision.hpp" #include "../python/pythoninterpretermodule.hpp" diff --git a/src/test/server_test.cpp b/src/test/server_test.cpp index bc74335164..0665772689 100644 --- a/src/test/server_test.cpp +++ b/src/test/server_test.cpp @@ -25,7 +25,7 @@ #include "../cleaner_utils.hpp" #include "../dags/node_library.hpp" #include "../kfs_frontend/kfs_grpc_inference_service.hpp" -#include "../localfilesystem.hpp" +#include "src/filesystem/localfilesystem.hpp" #include "../logging.hpp" #include "../model.hpp" #include "../modelinstanceunloadguard.hpp" diff --git a/src/test/streaming_test.cpp b/src/test/streaming_test.cpp index 02e7c4178a..5af193b6c3 100644 --- a/src/test/streaming_test.cpp +++ b/src/test/streaming_test.cpp @@ -29,6 +29,7 @@ #include "../status.hpp" #include "../stringutils.hpp" #include "mediapipe/framework/port/integral_types.h" +#include "../mediapipe_internal/mediapipefactory.hpp" #include "constructor_enabled_model_manager.hpp" #include "platform_utils.hpp" #include "test_utils.hpp" diff --git a/src/test/stress_test_utils.hpp b/src/test/stress_test_utils.hpp index ccbdd60758..2acda0ec72 100644 --- a/src/test/stress_test_utils.hpp +++ b/src/test/stress_test_utils.hpp @@ -40,7 +40,7 @@ #include "../dags/pipelinedefinition.hpp" #include "../get_model_metadata_impl.hpp" #include "../kfs_frontend/kfs_utils.hpp" -#include "../localfilesystem.hpp" +#include "src/filesystem/localfilesystem.hpp" #include "../logging.hpp" #include "../model_service.hpp" #include "../modelconfig.hpp" @@ -1726,7 +1726,7 @@ class ConfigChangeStressTest : public TestWithTempDir { RequestType request = preparePipelinePredictRequest(request2); ovms::Status createPipelineStatus = StatusCode::UNKNOWN_ERROR; if (typeid(ServableType) == typeid(ovms::Pipeline)) { - createPipelineStatus = this->manager->createPipeline(pipelinePtr, pipelineName, &request, &response); + createPipelineStatus = this->manager->getPipelineFactory().create(pipelinePtr, pipelineName, &request, &response, *(this->manager)); #if (MEDIAPIPE_DISABLE == 0) } else if (typeid(ServableType) == typeid(ovms::MediapipeGraphExecutor)) { mediacreate(executorPtr, *(this->manager), request, response, createPipelineStatus); diff --git a/src/test/test_utils.cpp b/src/test/test_utils.cpp index a80f924f9f..5c21e15158 100644 --- a/src/test/test_utils.cpp +++ b/src/test/test_utils.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include diff --git a/src/test/test_utils.hpp b/src/test/test_utils.hpp index 879ab1313e..7f5c80202d 100644 --- a/src/test/test_utils.hpp +++ b/src/test/test_utils.hpp @@ -48,7 +48,7 @@ #include "../mediapipe_internal/mediapipegraphdefinition.hpp" #include "../mediapipe_internal/mediapipegraphexecutor.hpp" #endif -#include "../metric_registry.hpp" +#include "src/metrics/metric_registry.hpp" #include "../modelinstance.hpp" #include "../modelmanager.hpp" #include "../shape.hpp" diff --git a/src/tfs_frontend/serialization.hpp b/src/tfs_frontend/serialization.hpp index f6925206d1..1d227bc976 100644 --- a/src/tfs_frontend/serialization.hpp +++ b/src/tfs_frontend/serialization.hpp @@ -29,7 +29,6 @@ #pragma GCC diagnostic pop #include "../profiler.hpp" -#include "../logging.hpp" #include "../status.hpp" #include "../serialization_common.hpp" #include "../tensorinfo.hpp" diff --git a/src/tfs_frontend/tfs_request_utils.hpp b/src/tfs_frontend/tfs_request_utils.hpp index d040e7a0a9..e6f55d451d 100644 --- a/src/tfs_frontend/tfs_request_utils.hpp +++ b/src/tfs_frontend/tfs_request_utils.hpp @@ -26,7 +26,6 @@ #include "../extractchoice.hpp" #include "../requesttensorextractor.hpp" #include "../statefulrequestprocessor.hpp" -#include "../logging.hpp" #include "../profiler.hpp" #include "../shape.hpp" #include "../status.hpp"