[RPD-321] Update MLFlow image and version

pyproject.toml

@@ -43,7 +43,7 @@ matcha = "matcha_ml.cli.cli:app"
 
 [tool.poetry.dependencies]
 python = "^3.8"
-typer = {extras = ["all"], version = "^0.7.0"}
+typer = {extras = ["all"], version = "^0.9.0"}
 python-terraform = "^0.10.1"
 azure-identity = "^1.12.0"
 azure-mgmt-resource = "^23.0.0"

src/matcha_ml/infrastructure/default/data_version_control_storage/main.tf

@@ -18,5 +18,5 @@ resource "azurerm_storage_account" "storageaccount" {
 resource "azurerm_storage_container" "storagecontainer" {
   name                  = "${var.prefix}dvcstore"
   storage_account_name  = azurerm_storage_account.storageaccount.name
-  container_access_type = "container"
+  container_access_type = "private"
 }

src/matcha_ml/infrastructure/default/mlflow_module/main.tf

@@ -9,8 +9,19 @@ resource "helm_release" "mlflow_tracking" {
   # Change type from "ClusterIP" to "LoadBalancer"
   set {
     name  = "service.type"
-    value = "LoadBalancer"
+    value = "ClusterIP"
   }
+
+  set {
+    name = "image.repository"
+    value = "ghcr.io/mlflow/mlflow"
+  }
+
+  set {
+    name = "image.tag"
+    value = "v2.6.0"
+  }
+
   # set proxied access to artifact storage
   set {
     name  = "artifactRoot.proxiedArtifactStorage"

src/matcha_ml/infrastructure/default/mlflow_module/output.tf

@@ -1,4 +0,0 @@
-output "mlflow_tracking_url" {
-  description = "The tracking URL for MLFlow dashboard"
-  value = "http://${data.kubernetes_service.mlflow_tracking.status.0.load_balancer.0.ingress.0.ip}:${data.kubernetes_service.mlflow_tracking.spec.0.port.0.port}"
-}

src/matcha_ml/infrastructure/default/output.tf

@@ -1,8 +1,3 @@
-output "experiment_tracker_mlflow_tracking_url" {
-  description = "The URL for the MLflow tracking server"
-  value       = module.mlflow.mlflow_tracking_url
-}
-
 output "experiment_tracker_mlflow_azure_connection_string" {
   description = "The Azure connection string for the MLflow artifact storage"
   value       = module.storage.primary_connection_string
@@ -26,11 +21,6 @@ output "orchestrator_aks_k8s_context" {
   value       = local.kubectl_context
 }
 
-output "pipeline_zenml_server_url" {
-  description = "The URL for the ZenServer API server"
-  value       = module.zenserver.zenserver_url
-}
-
 output "pipeline_zenml_server_username" {
   description = "The username for accessing the ZenServer API server"
   value       = module.zenserver.zenserver_username

src/matcha_ml/infrastructure/default/storage/main.tf

@@ -18,5 +18,5 @@ resource "azurerm_storage_account" "storageaccount" {
 resource "azurerm_storage_container" "storagecontainer" {
   name                  = "${var.prefix}store"
   storage_account_name  = azurerm_storage_account.storageaccount.name
-  container_access_type = "container"
+  container_access_type = "private"
 }

src/matcha_ml/infrastructure/default/zen_server/outputs.tf

@@ -1,8 +1,3 @@
-output "zenserver_url" {
-  description = "The URL for the ZenML server"
-  value = "http://${data.kubernetes_service.zen_server.status.0.load_balancer.0.ingress.0.ip}"
-}
-
 output "zenserver_username" {
   description = "The username used to access the ZenML server"
   value = var.username
@@ -13,7 +8,3 @@ output "zenserver_password" {
   value     = var.password
   sensitive = true
 }
-
-# output "zenserver_url" {
-#   value = var.create_ingress_controller ? "https://${data.kubernetes_service.ingress-controller[0].status.0.load_balancer.0.ingress.0.ip}.nip.io/${var.ingress_path}" : "https://${var.ingress_controller_hostname}.nip.io/${var.ingress_path}"
-# }

src/matcha_ml/infrastructure/default/zen_server/zenml_helm/values.yaml

@@ -250,7 +250,7 @@ zenml:
   secretEnvironment: {}
 
   service:
-    type: LoadBalancer  # changed from ClusterIP
+    type: ClusterIP  # changed from ClusterIP
     port: 80
 
   ingress:

src/matcha_ml/infrastructure/llm/aks/main.tf

@@ -6,7 +6,7 @@ resource "azurerm_kubernetes_cluster" "main" {
 
   default_node_pool {
     name    = "default"
-    vm_size = "Standard_DS3_v2"
+    vm_size = "Standard_NC8as_T4_v3"
 
     enable_auto_scaling = true
     max_count           = 3

src/matcha_ml/infrastructure/llm/chroma/chroma_helm/values.yaml

@@ -1,7 +1,7 @@
 # values.yaml
 image:
   repository: ghcr.io/chroma-core/chroma
-  tag: 0.4.3
+  tag: latest
 
 resources:
   requests:

src/matcha_ml/infrastructure/llm/data_version_control_storage/main.tf

@@ -18,5 +18,5 @@ resource "azurerm_storage_account" "storageaccount" {
 resource "azurerm_storage_container" "storagecontainer" {
   name                  = "${var.prefix}dvcstore"
   storage_account_name  = azurerm_storage_account.storageaccount.name
-  container_access_type = "container"
+  container_access_type = "private"
 }

src/matcha_ml/infrastructure/llm/mlflow_module/main.tf

@@ -9,7 +9,17 @@ resource "helm_release" "mlflow_tracking" {
   # Change type from "ClusterIP" to "LoadBalancer"
   set {
     name  = "service.type"
-    value = "LoadBalancer"
+    value = "ClusterIP"
+  }
+
+  set {
+    name = "image.repository"
+    value = "ghcr.io/mlflow/mlflow"
+  }
+
+  set {
+    name = "image.tag"
+    value = "v2.6.0"
   }
   # set proxied access to artifact storage
   set {

src/matcha_ml/infrastructure/llm/mlflow_module/output.tf

@@ -1,4 +0,0 @@
-output "mlflow_tracking_url" {
-  description = "The tracking URL for MLFlow dashboard"
-  value = "http://${data.kubernetes_service.mlflow_tracking.status.0.load_balancer.0.ingress.0.ip}:${data.kubernetes_service.mlflow_tracking.spec.0.port.0.port}"
-}

src/matcha_ml/infrastructure/llm/output.tf

@@ -1,8 +1,3 @@
-output "experiment_tracker_mlflow_tracking_url" {
-  description = "The URL for the MLflow tracking server"
-  value       = module.mlflow.mlflow_tracking_url
-}
-
 output "experiment_tracker_mlflow_azure_connection_string" {
   description = "The Azure connection string for the MLflow artifact storage"
   value       = module.storage.primary_connection_string
@@ -26,11 +21,6 @@ output "orchestrator_aks_k8s_context" {
   value       = local.kubectl_context
 }
 
-output "pipeline_zenml_server_url" {
-  description = "The URL for the ZenServer API server"
-  value       = module.zenserver.zenserver_url
-}
-
 output "pipeline_zenml_server_username" {
   description = "The username for accessing the ZenServer API server"
   value       = module.zenserver.zenserver_username

src/matcha_ml/infrastructure/llm/storage/main.tf

@@ -18,5 +18,5 @@ resource "azurerm_storage_account" "storageaccount" {
 resource "azurerm_storage_container" "storagecontainer" {
   name                  = "${var.prefix}store"
   storage_account_name  = azurerm_storage_account.storageaccount.name
-  container_access_type = "container"
+  container_access_type = "private"
 }

src/matcha_ml/infrastructure/llm/zen_server/zenml_helm/values.yaml

@@ -250,7 +250,7 @@ zenml:
   secretEnvironment: {}
 
   service:
-    type: LoadBalancer  # changed from ClusterIP
+    type: ClusterIP  # changed from ClusterIP
     port: 80
 
   ingress:

src/matcha_ml/infrastructure/remote_state_storage/state_storage/main.tf

@@ -16,5 +16,5 @@ resource "azurerm_storage_account" "statestorageaccount" {
 resource "azurerm_storage_container" "statestoragecontainer" {
   name                  = "${var.prefix}statestore"
   storage_account_name  = azurerm_storage_account.statestorageaccount.name
-  container_access_type = "container"
+  container_access_type = "private"
 }

src/matcha_ml/services/azure_service.py

@@ -1,4 +1,6 @@
 """The Azure Service interface."""
+import json
+import subprocess
 from subprocess import DEVNULL
 from typing import Dict, List, Optional, Set, cast
 
@@ -75,9 +77,11 @@ def _subscription_id(self) -> str:
         Returns:
             str: the subscription id.
         """
-        subscriptions = self._client.subscriptions.list()
-        if subscriptions:
-            return str(list(subscriptions)[0].subscription_id)
+        subscripion_id = json.loads(
+            subprocess.check_output("az account show", shell=True).decode("utf-8")
+        ).get("id")
+        if subscripion_id:
+            return str(subscripion_id)
         else:
             raise MatchaAuthenticationError(
                 "no subscriptions found - you at least one subscription active in your Azure account."