From 88c8b57f43a9e6feb4c0c8ed8b32bffc2e8392ff Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 1 Nov 2019 12:51:51 +0530 Subject: [PATCH 001/134] cks: initial functionality Signed-off-by: Abhishek Kumar --- .../apache/cloudstack/api/ApiConstants.java | 8 + client/pom.xml | 5 + .../META-INF/db/schema-41300to41400.sql | 97 + .../integrations/kubernetes-service/pom.xml | 132 + .../scripts/create-binaries-iso.sh | 79 + .../kubernetescluster/KubernetesCluster.java | 123 + .../KubernetesClusterDetailsVO.java | 84 + .../KubernetesClusterEventTypes.java | 25 + .../KubernetesClusterManagerImpl.java | 2593 +++++++++++++++++ .../KubernetesClusterService.java | 56 + .../KubernetesClusterVO.java | 330 +++ .../KubernetesClusterVmMap.java | 27 + .../KubernetesClusterVmMapVO.java | 76 + .../KubernetesServiceConfig.java | 75 + .../dao/KubernetesClusterDao.java | 34 + .../dao/KubernetesClusterDaoImpl.java | 113 + .../dao/KubernetesClusterDetailsDao.java | 28 + .../dao/KubernetesClusterDetailsDaoImpl.java | 32 + .../dao/KubernetesClusterVmMapDao.java | 26 + .../dao/KubernetesClusterVmMapDaoImpl.java | 46 + .../KubernetesSupportedVersion.java | 32 + .../KubernetesSupportedVersionVO.java | 113 + .../KubernetesVersionEventTypes.java | 23 + .../KubernetesVersionManagerImpl.java | 230 ++ .../KubernetesVersionService.java | 32 + .../dao/KubernetesSupportedVersionDao.java | 27 + .../KubernetesSupportedVersionDaoImpl.java | 39 + .../AddKubernetesSupportedVersionCmd.java | 126 + .../DeleteKubernetesSupportedVersionCmd.java | 111 + .../CreateKubernetesClusterCmd.java | 289 ++ .../DeleteKubernetesClusterCmd.java | 124 + .../GetKubernetesClusterConfigCmd.java | 92 + .../ListKubernetesClustersCmd.java | 93 + .../ScaleKubernetesClusterCmd.java | 142 + .../StartKubernetesClusterCmd.java | 123 + .../StopKubernetesClusterCmd.java | 124 + .../ListKubernetesSupportedVersionsCmd.java | 91 + .../KubernetesClusterConfigResponse.java | 61 + .../response/KubernetesClusterResponse.java | 287 ++ .../KubernetesSupportedVersionResponse.java | 114 + .../kubernetes-service/module.properties | 15 + .../spring-kubernetes-service-context.xml | 34 + .../src/main/resources/conf/k8s-master.yml | 238 ++ .../src/main/resources/conf/k8s-node.yml | 191 ++ plugins/pom.xml | 1 + tools/apidoc/gen_toc.py | 2 + ui/plugins/cks/cks.css | 31 + ui/plugins/cks/cks.js | 1296 ++++++++ ui/plugins/cks/config.js | 25 + ui/plugins/cks/icon.png | Bin 0 -> 1208 bytes ui/plugins/plugins.js | 3 +- 51 files changed, 8097 insertions(+), 1 deletion(-) create mode 100644 plugins/integrations/kubernetes-service/pom.xml create mode 100644 plugins/integrations/kubernetes-service/scripts/create-binaries-iso.sh create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesCluster.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterDetailsVO.java create mode 100755 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterEventTypes.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterService.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVO.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVmMap.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVmMapVO.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesServiceConfig.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDao.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDaoImpl.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDetailsDao.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDetailsDaoImpl.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterVmMapDao.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterVmMapDaoImpl.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesSupportedVersion.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesSupportedVersionVO.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionEventTypes.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionService.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/dao/KubernetesSupportedVersionDao.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/dao/KubernetesSupportedVersionDaoImpl.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/AddKubernetesSupportedVersionCmd.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/DeleteKubernetesSupportedVersionCmd.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/CreateKubernetesClusterCmd.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/DeleteKubernetesClusterCmd.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/GetKubernetesClusterConfigCmd.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/ListKubernetesClustersCmd.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/ScaleKubernetesClusterCmd.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/StartKubernetesClusterCmd.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/StopKubernetesClusterCmd.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetesversion/ListKubernetesSupportedVersionsCmd.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterConfigResponse.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java create mode 100644 plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/module.properties create mode 100644 plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml create mode 100644 plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml create mode 100644 plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml create mode 100644 ui/plugins/cks/cks.css create mode 100644 ui/plugins/cks/cks.js create mode 100644 ui/plugins/cks/config.js create mode 100644 ui/plugins/cks/icon.png diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index fb44a8a11f48..4c274cdde3f1 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -742,6 +742,14 @@ public class ApiConstants { public static final String TARGET_ID = "targetid"; public static final String VOLUME_IDS = "volumeids"; + public static final String KUBERNETES_VERSION_ID = "kubernetesversionid"; + public static final String CONSOLE_END_POINT = "consoleendpoint"; + public static final String DOCKER_REGISTRY_USER_NAME = "dockerregistryusername"; + public static final String DOCKER_REGISTRY_PASSWORD = "dockerregistrypassword"; + public static final String DOCKER_REGISTRY_URL = "dockerregistryurl"; + public static final String DOCKER_REGISTRY_EMAIL = "dockerregistryemail"; + public static final String NODE_ROOT_DISK_SIZE = "noderootdisksize"; + public enum HostDetails { all, capacity, events, stats, min; } diff --git a/client/pom.xml b/client/pom.xml index 3f1e0763d05c..c68f42a56f66 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -479,6 +479,11 @@ cloud-plugin-integrations-prometheus-exporter ${project.version} + + org.apache.cloudstack + cloud-plugin-integrations-kubernetes-service + ${project.version} + diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql b/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql index 137dd2c1a320..63c0384c2ac9 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql @@ -34,3 +34,100 @@ UPDATE `cloud`.`guest_os` SET `category_id`='4' WHERE `id`=284 AND display_name= UPDATE `cloud`.`guest_os` SET `category_id`='4' WHERE `id`=285 AND display_name="Red Hat Enterprise Linux 7.6"; UPDATE `cloud`.`guest_os` SET `category_id`='4' WHERE `id`=286 AND display_name="Red Hat Enterprise Linux 8.0"; +-- Kubernetes service +CREATE TABLE IF NOT EXISTS `cloud`.`kubernetes_cluster` ( + `id` bigint unsigned NOT NULL auto_increment COMMENT 'id', + `uuid` varchar(40) DEFAULT NULL, + `name` varchar(255) NOT NULL, + `description` varchar(4096) COMMENT 'display text for this kubernetes cluster', + `zone_id` bigint unsigned NOT NULL COMMENT 'zone id', + `kubernetes_version_id` bigint unsigned NOT NULL COMMENT 'kubernetes version id for the cluster', + `service_offering_id` bigint unsigned COMMENT 'service offering id for the cluster VM', + `template_id` bigint unsigned COMMENT 'vm_template.id', + `network_id` bigint unsigned COMMENT 'network this kubernetes cluster uses', + `node_count` bigint NOT NULL default '0', + `account_id` bigint unsigned NOT NULL COMMENT 'owner of this cluster', + `domain_id` bigint unsigned NOT NULL COMMENT 'owner of this cluster', + `state` char(32) NOT NULL COMMENT 'current state of this cluster', + `key_pair` varchar(40), + `cores` bigint unsigned NOT NULL COMMENT 'number of cores', + `memory` bigint unsigned NOT NULL COMMENT 'total memory', + `node_root_disk_size` bigint(20) unsigned DEFAULT 0 COMMENT 'root disk size of root disk for each node', + `endpoint` varchar(255) COMMENT 'url endpoint of the kubernetes cluster manager api access', + `console_endpoint` varchar(255) COMMENT 'url for the kubernetes cluster manager dashbaord', + `created` datetime NOT NULL COMMENT 'date created', + `removed` datetime COMMENT 'date removed if not null', + `gc` tinyint unsigned NOT NULL DEFAULT 1 COMMENT 'gc this kubernetes cluster or not', + + PRIMARY KEY(`id`), + CONSTRAINT `fk_cluster__zone_id` FOREIGN KEY `fk_cluster__zone_id`(`zone_id`) REFERENCES `data_center` (`id`) ON DELETE CASCADE, + CONSTRAINT `fk_cluster__kubernetes_version_id` FOREIGN KEY `fk_cluster__kubernetes_version_id`(`kubernetes_version_id`) REFERENCES `kubernetes_supported_version` (`id`) ON DELETE CASCADE, + CONSTRAINT `fk_cluster__service_offering_id` FOREIGN KEY `fk_cluster__service_offering_id`(`service_offering_id`) REFERENCES `service_offering`(`id`) ON DELETE CASCADE, + CONSTRAINT `fk_cluster__template_id` FOREIGN KEY `fk_cluster__template_id`(`template_id`) REFERENCES `vm_template`(`id`) ON DELETE CASCADE, + CONSTRAINT `fk_cluster__network_id` FOREIGN KEY `fk_cluster__network_id`(`network_id`) REFERENCES `networks`(`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE IF NOT EXISTS `cloud`.`kubernetes_cluster_vm_map` ( + `id` bigint unsigned NOT NULL auto_increment COMMENT 'id', + `cluster_id` bigint unsigned NOT NULL COMMENT 'cluster id', + `vm_id` bigint unsigned NOT NULL COMMENT 'vm id', + + PRIMARY KEY(`id`), + CONSTRAINT `fk_kubernetes_cluster_vm_map__cluster_id` FOREIGN KEY `fk_kubernetes_cluster_vm_map__cluster_id`(`cluster_id`) REFERENCES `kubernetes_cluster`(`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE IF NOT EXISTS `cloud`.`kubernetes_cluster_details` ( + `id` bigint unsigned NOT NULL auto_increment COMMENT 'id', + `cluster_id` bigint unsigned NOT NULL COMMENT 'kubernetes cluster id', + `name` varchar(255) NOT NULL, + `value` varchar(10240) NOT NULL, + `display` tinyint(1) NOT NULL DEFAULT '1' COMMENT 'True if the detail can be displayed to the end user', + + PRIMARY KEY(`id`), + CONSTRAINT `fk_kubernetes_cluster_details__cluster_id` FOREIGN KEY `fk_kubernetes_cluster_details__cluster_id`(`cluster_id`) REFERENCES `kubernetes_cluster`(`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE IF NOT EXISTS `cloud`.`kubernetes_supported_version` ( + `id` bigint unsigned NOT NULL auto_increment COMMENT 'id', + `uuid` varchar(40) DEFAULT NULL COMMENT 'uuid', + `name` varchar(255) NOT NULL COMMENT 'kubernetes version name', + `iso_id` bigint unsigned NOT NULL COMMENT 'kubernetes version binary ISO id', + `zone_id` bigint unsigned DEFAULT NULL COMMENT 'zone id in which kubernetes version is available', + `created` datetime NOT NULL COMMENT 'date created', + `removed` datetime COMMENT 'date removed if not null', + + PRIMARY KEY(`id`), + CONSTRAINT `fk_kubernetes_supported_version__iso_id` FOREIGN KEY `fk_kubernetes_supported_version__iso_id`(`iso_id`) REFERENCES `vm_template`(`id`) ON DELETE CASCADE, + CONSTRAINT `fk_kubernetes_supported_version__zone_id` FOREIGN KEY `fk_kubernetes_supported_version__zone_id`(`zone_id`) REFERENCES `data_center` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', +'cloud.kubernetes.cluster.template.name', "Kubernetes-Service-Template", 'Name of the template to be used for creating Kubernetes cluster nodes', 'Kubernetes-Service-Template', NULL, NULL, 0); + +INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', +'cloud.kubernetes.cluster.binaries.iso.name', 'Kubernetes-Service-Binaries-ISO' , 'Name of the ISO that contains Kubernetes binaries and docker images for offline installation.', 'Kubernetes-Service-Binaries-ISO', NULL , NULL, 0); + +INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', +'cloud.kubernetes.cluster.master.cloudconfig', '/etc/cloudstack/management/k8s-master.yml' , 'file location path of the cloud config used for creating kubernetes cluster master node', '/etc/cloudstack/management/k8s-master.yml', NULL , NULL, 0); + +INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', +'cloud.kubernetes.cluster.node.cloudconfig', '/etc/cloudstack/management/k8s-node.yml', 'file location path of the cloud config used for creating kubernetes cluster node', '/etc/cloudstack/management/k8s-node.yml', NULL , NULL, 0); + +INSERT IGNORE INTO `cloud`.`network_offerings` (name, uuid, unique_name, display_text, nw_rate, mc_rate, traffic_type, tags, system_only, specify_vlan, service_offering_id, conserve_mode, created,availability, dedicated_lb_service, shared_source_nat_service, sort_key, redundant_router_service, state, guest_type, elastic_ip_service, eip_associate_public_ip, elastic_lb_service, specify_ip_ranges, inline,is_persistent,internal_lb, public_lb, egress_default_policy, concurrent_connections, keep_alive_enabled, supports_streched_l2, `default`, removed) VALUES ('DefaultNetworkOfferingforKubernetesService', UUID(), 'DefaultNetworkOfferingforKubernetesService', 'Network Offering used for CloudStack kubernetes service', NULL,NULL,'Guest',NULL,0,0,NULL,1,now(),'Required',1,0,0,0,'Enabled','Isolated',0,1,0,0,0,0,0,1,1,NULL,0,0,0,NULL); + +UPDATE `cloud`.`network_offerings` SET removed=NULL WHERE unique_name='DefaultNetworkOfferingforKubernetesService'; + +SET @k8snetwork = (select id from network_offerings where name='DefaultNetworkOfferingforKubernetesService' and removed IS NULL); +INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@k8snetwork, 'Dhcp','VirtualRouter',now()); +INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@k8snetwork, 'Dns','VirtualRouter',now()); +INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@k8snetwork, 'Firewall','VirtualRouter',now()); +INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@k8snetwork, 'Gateway','VirtualRouter',now()); +INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@k8snetwork, 'Lb','VirtualRouter',now()); +INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@k8snetwork, 'PortForwarding','VirtualRouter',now()); +INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@k8snetwork, 'SourceNat','VirtualRouter',now()); +INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@k8snetwork, 'StaticNat','VirtualRouter',now()); +INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@k8snetwork, 'UserData','VirtualRouter',now()); +INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@k8snetwork, 'Vpn','VirtualRouter',now()); + +INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', +'cloud.kubernetes.cluster.network.offering', 'DefaultNetworkOfferingforKubernetesService' , 'Network Offering used for CloudStack kubernetes service', 'DefaultNetworkOfferingforKubernetesService', NULL , NULL, 0); diff --git a/plugins/integrations/kubernetes-service/pom.xml b/plugins/integrations/kubernetes-service/pom.xml new file mode 100644 index 000000000000..800f9b5f9fa2 --- /dev/null +++ b/plugins/integrations/kubernetes-service/pom.xml @@ -0,0 +1,132 @@ + + + + 4.0.0 + cloud-plugin-integrations-kubernetes-service + Apache CloudStack Plugin - Kubernetes Service + + org.apache.cloudstack + cloudstack-plugins + 4.14.0.0-SNAPSHOT + ../../pom.xml + + + + org.apache.cloudstack + cloud-core + ${project.version} + + + org.apache.cloudstack + cloud-framework-db + ${project.version} + + + org.apache.cloudstack + cloud-framework-ca + ${project.version} + + + org.apache.cloudstack + cloud-framework-security + ${project.version} + + + org.apache.cloudstack + cloud-engine-schema + ${project.version} + + + org.apache.cloudstack + cloud-engine-api + ${project.version} + + + org.apache.cloudstack + cloud-engine-components-api + ${project.version} + + + org.apache.cloudstack + cloud-framework-managed-context + ${project.version} + + + org.eclipse.persistence + javax.persistence + ${cs.jpa.version} + + + com.google.code.gson + gson + ${cs.gson.version} + + + com.google.guava + guava + ${cs.guava.version} + + + log4j + log4j + ${cs.log4j.version} + + + org.springframework + spring-context + ${org.springframework.version} + + + org.springframework + spring-aop + ${org.springframework.version} + + + org.springframework + spring-beans + ${org.springframework.version} + + + org.springframework + spring-test + ${org.springframework.version} + + + commons-codec + commons-codec + ${cs.codec.version} + + + org.hamcrest + hamcrest-library + ${cs.hamcrest.version} + test + + + org.bouncycastle + bcprov-jdk15on + ${cs.bcprov.version} + + + joda-time + joda-time + ${cs.joda-time.version} + + + diff --git a/plugins/integrations/kubernetes-service/scripts/create-binaries-iso.sh b/plugins/integrations/kubernetes-service/scripts/create-binaries-iso.sh new file mode 100644 index 000000000000..41a5ef690f8a --- /dev/null +++ b/plugins/integrations/kubernetes-service/scripts/create-binaries-iso.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +RELEASE="v1.11.4" +start_dir="$PWD" +iso_dir="${start_dir}/iso" +working_dir="${iso_dir}/${RELEASE}" +mkdir -p "${working_dir}" + +CNI_VERSION="v0.7.1" +echo "Downloading CNI ${CNI_VERSION}..." +cni_dir="${working_dir}/cni/${CNI_VERSION}" +mkdir -p "${cni_dir}" +curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" -o "${cni_dir}/cni-plugins-amd64-${CNI_VERSION}.tgz" + +CRICTL_VERSION="v1.11.1" +echo "Downloading CRI tools ${CRICTL_VERSION}..." +crictl_dir="${working_dir}/cri-tools/${CRICTL_VERSION}" +mkdir -p "${crictl_dir}" +curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" -o "${crictl_dir}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" + +echo "Downloading Kubernetes tools ${RELEASE}..." +k8s_dir="${working_dir}/k8s" +mkdir -p "${k8s_dir}" +cd "${k8s_dir}" +curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} +kubeadm_file_permissions=`stat --format '%a' kubeadm` +chmod +x kubeadm + +echo "Downloading kubelet.service ${RELEASE}..." +cd $start_dir +kubelet_service_file="${working_dir}/kubelet.service" +touch "${kubelet_service_file}" +curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > ${kubelet_service_file} +echo "Downloading 10-kubeadm.conf ${RELEASE}..." +kubeadm_conf_file="${working_dir}/10-kubeadm.conf" +touch "${kubeadm_conf_file}" +curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > ${kubeadm_conf_file} + +echo "Fetching k8s docker images..." +docker -v +if [ $? -ne 0 ]; then + echo "Installing docker..." + sudo apt update && sudo apt install docker.io -y + sudo systemctl enable docker && sudo systemctl start docker +fi +mkdir -p "${working_dir}/docker" +output=`${k8s_dir}/kubeadm config images list` +while read -r line; do + echo "Downloading docker image $line ---" + sudo docker pull "$line" + image_name=`echo "$line" | grep -oE "[^/]+$"` + sudo docker save "$line" > "${working_dir}/docker/$image_name.tar" +done <<< "$output" + +echo "Restore kubeadm permissions..." +if [ "${kubeadm_file_permissions}" -eq "" ]; then + kubeadm_file_permissions=644 +fi +chmod ${kubeadm_file_permissions} "${working_dir}/k8s/kubeadm" + +mkisofs -o setup.iso -J -R -l "${iso_dir}" + +rm -rf "${iso_dir}" diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesCluster.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesCluster.java new file mode 100644 index 000000000000..439e299247a9 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesCluster.java @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetescluster; + +import org.apache.cloudstack.acl.ControlledEntity; +import org.apache.cloudstack.api.Displayable; +import org.apache.cloudstack.api.Identity; +import org.apache.cloudstack.api.InternalIdentity; + +import com.cloud.utils.fsm.StateMachine2; + +/** + * KubernetesCluster describes the properties of kubernetes cluster + * + */ +public interface KubernetesCluster extends ControlledEntity, com.cloud.utils.fsm.StateObject, Identity, InternalIdentity, Displayable { + + enum Event { + StartRequested, + StopRequested, + DestroyRequested, + RecoveryRequested, + ScaleUpRequested, + ScaleDownRequested, + OperationSucceeded, + OperationFailed, + CreateFailed, + FaultsDetected; + } + + enum State { + Created("Initial State of kubernetes cluster. At this state its just a logical/DB entry with no resources consumed"), + Starting("Resources needed for kubernetes cluster are being provisioned"), + Running("Necessary resources are provisioned and kubernetes cluster is in operational ready state to launch kubernetess"), + Stopping("Ephermal resources for the kubernetes cluster are being destroyed"), + Stopped("All ephermal resources for the kubernetes cluster are destroyed, Kubernetes cluster may still have ephermal resource like persistent volumens provisioned"), + Scaling("Transient state in which resoures are either getting scaled up/down"), + Alert("State to represent kubernetes clusters which are not in expected desired state (operationally in active control place, stopped cluster VM's etc)."), + Recovering("State in which kubernetes cluster is recovering from alert state"), + Destroyed("End state of kubernetes cluster in which all resources are destroyed, cluster will not be useable further"), + Destroying("State in which resources for the kubernetes cluster is getting cleaned up or yet to be cleaned up by garbage collector"), + Error("State of the failed to create kubernetes clusters"); + + protected static final StateMachine2 s_fsm = new StateMachine2(); + + public static StateMachine2 getStateMachine() { return s_fsm; } + + static { + s_fsm.addTransition(State.Created, Event.StartRequested, State.Starting); + + s_fsm.addTransition(State.Starting, Event.OperationSucceeded, State.Running); + s_fsm.addTransition(State.Starting, Event.OperationFailed, State.Alert); + s_fsm.addTransition(State.Starting, Event.CreateFailed, State.Error); + + s_fsm.addTransition(State.Running, Event.StopRequested, State.Stopping); + s_fsm.addTransition(State.Stopping, Event.OperationSucceeded, State.Stopped); + s_fsm.addTransition(State.Stopping, Event.OperationFailed, State.Alert); + + s_fsm.addTransition(State.Stopped, Event.StartRequested, State.Starting); + + s_fsm.addTransition(State.Running, Event.FaultsDetected, State.Alert); + + s_fsm.addTransition(State.Running, Event.ScaleUpRequested, State.Scaling); + s_fsm.addTransition(State.Running, Event.ScaleDownRequested, State.Scaling); + s_fsm.addTransition(State.Scaling, Event.OperationSucceeded, State.Running); + s_fsm.addTransition(State.Scaling, Event.OperationFailed, State.Alert); + + s_fsm.addTransition(State.Alert, Event.RecoveryRequested, State.Recovering); + s_fsm.addTransition(State.Recovering, Event.OperationSucceeded, State.Running); + s_fsm.addTransition(State.Recovering, Event.OperationFailed, State.Alert); + + s_fsm.addTransition(State.Running, Event.DestroyRequested, State.Destroying); + s_fsm.addTransition(State.Stopped, Event.DestroyRequested, State.Destroying); + s_fsm.addTransition(State.Alert, Event.DestroyRequested, State.Destroying); + s_fsm.addTransition(State.Error, Event.DestroyRequested, State.Destroying); + + s_fsm.addTransition(State.Destroying, Event.DestroyRequested, State.Destroying); + s_fsm.addTransition(State.Destroying, Event.OperationSucceeded, State.Destroyed); + s_fsm.addTransition(State.Destroying, Event.OperationFailed, State.Destroying); + + } + String _description; + + State(String description) { + _description = description; + } + } + + long getId(); + String getName(); + String getDescription(); + long getZoneId(); + long getKubernetesVersionId(); + long getServiceOfferingId(); + long getTemplateId(); + long getNetworkId(); + long getDomainId(); + long getAccountId(); + long getNodeCount(); + String getKeyPair(); + long getCores(); + long getMemory(); + long getNodeRootDiskSize(); + String getEndpoint(); + String getConsoleEndpoint(); + boolean isCheckForGc(); + @Override + State getState(); +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterDetailsVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterDetailsVO.java new file mode 100644 index 000000000000..29bdd48f65d7 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterDetailsVO.java @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetescluster; + + +import javax.persistence.Column; + +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; + +import org.apache.cloudstack.api.ResourceDetail; + +@Entity +@Table(name = "kubernetes_cluster_details") +public class KubernetesClusterDetailsVO implements ResourceDetail { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "cluster_id") + private long resourceId; + + @Column(name = "name") + private String name; + + @Column(name = "value", length = 10240) + private String value; + + @Column(name = "display") + private boolean display; + + public KubernetesClusterDetailsVO() { + } + + public KubernetesClusterDetailsVO(long id, String name, String value, boolean display) { + this.resourceId = id; + this.name = name; + this.value = value; + this.display = display; + } + + @Override + public long getId() { + return id; + } + + @Override + public String getName() { + return name; + } + + @Override + public String getValue() { + return value; + } + + @Override + public long getResourceId() { + return resourceId; + } + + @Override + public boolean isDisplay() { + return display; + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterEventTypes.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterEventTypes.java new file mode 100755 index 000000000000..6d651fef0de5 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterEventTypes.java @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetescluster; + +public class KubernetesClusterEventTypes { + public static final String EVENT_KUBERNETES_CLUSTER_CREATE = "KUBERNETES.CLUSTER.CREATE"; + public static final String EVENT_KUBERNETES_CLUSTER_DELETE = "KUBERNETES.CLUSTER.DELETE"; + public static final String EVENT_KUBERNETES_CLUSTER_START = "KUBERNETES.CLUSTER.START"; + public static final String EVENT_KUBERNETES_CLUSTER_STOP = "KUBERNETES.CLUSTER.STOP"; + public static final String EVENT_KUBERNETES_CLUSTER_SCALE = "KUBERNETES.CLUSTER.SCALE"; +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java new file mode 100644 index 000000000000..fac214271763 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java @@ -0,0 +1,2593 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetescluster; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.math.BigInteger; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.MalformedURLException; +import java.net.Socket; +import java.net.URL; +import java.net.UnknownHostException; +import java.nio.charset.Charset; +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; + +import org.apache.cloudstack.acl.ControlledEntity; +import org.apache.cloudstack.acl.SecurityChecker; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.user.firewall.CreateFirewallRuleCmd; +import org.apache.cloudstack.api.command.user.kubernetescluster.CreateKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetescluster.DeleteKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetescluster.GetKubernetesClusterConfigCmd; +import org.apache.cloudstack.api.command.user.kubernetescluster.ListKubernetesClustersCmd; +import org.apache.cloudstack.api.command.user.kubernetescluster.ScaleKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetescluster.StartKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetescluster.StopKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.vm.StartVMCmd; +import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse; +import org.apache.cloudstack.api.response.KubernetesClusterResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.ca.CAManager; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; +import org.apache.cloudstack.framework.ca.Certificate; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.utils.security.CertUtils; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.io.IOUtils; +import org.apache.log4j.Logger; + +import com.cloud.api.ApiDBUtils; +import com.cloud.capacity.CapacityManager; +import com.cloud.dc.ClusterDetailsDao; +import com.cloud.dc.ClusterDetailsVO; +import com.cloud.dc.ClusterVO; +import com.cloud.dc.DataCenter; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.ClusterDao; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.deploy.DeployDestination; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.InsufficientServerCapacityException; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ManagementServerException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.PermissionDeniedException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.host.Host.Type; +import com.cloud.host.HostVO; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.kubernetescluster.dao.KubernetesClusterDao; +import com.cloud.kubernetescluster.dao.KubernetesClusterDetailsDao; +import com.cloud.kubernetescluster.dao.KubernetesClusterVmMapDao; +import com.cloud.kubernetesversion.KubernetesSupportedVersion; +import com.cloud.kubernetesversion.dao.KubernetesSupportedVersionDao; +import com.cloud.network.IpAddress; +import com.cloud.network.IpAddressManager; +import com.cloud.network.Network; +import com.cloud.network.Network.Service; +import com.cloud.network.NetworkModel; +import com.cloud.network.NetworkService; +import com.cloud.network.PhysicalNetwork; +import com.cloud.network.dao.FirewallRulesDao; +import com.cloud.network.dao.IPAddressDao; +import com.cloud.network.dao.IPAddressVO; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; +import com.cloud.network.dao.PhysicalNetworkDao; +import com.cloud.network.firewall.FirewallService; +import com.cloud.network.rules.FirewallRule; +import com.cloud.network.rules.FirewallRuleVO; +import com.cloud.network.rules.PortForwardingRuleVO; +import com.cloud.network.rules.RulesService; +import com.cloud.network.rules.dao.PortForwardingRulesDao; +import com.cloud.offering.NetworkOffering; +import com.cloud.offering.ServiceOffering; +import com.cloud.offerings.NetworkOfferingVO; +import com.cloud.offerings.dao.NetworkOfferingDao; +import com.cloud.offerings.dao.NetworkOfferingServiceMapDao; +import com.cloud.org.Grouping; +import com.cloud.resource.ResourceManager; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.storage.Storage; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VMTemplateZoneVO; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VMTemplateZoneDao; +import com.cloud.template.TemplateApiService; +import com.cloud.template.VirtualMachineTemplate; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.AccountService; +import com.cloud.user.SSHKeyPairVO; +import com.cloud.user.User; +import com.cloud.user.dao.AccountDao; +import com.cloud.user.dao.SSHKeyPairDao; +import com.cloud.uservm.UserVm; +import com.cloud.utils.Pair; +import com.cloud.utils.component.ComponentContext; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.concurrency.NamedThreadFactory; +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GlobalLock; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.db.TransactionCallbackNoReturn; +import com.cloud.utils.db.TransactionCallbackWithException; +import com.cloud.utils.db.TransactionStatus; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.fsm.NoTransitionException; +import com.cloud.utils.fsm.StateMachine2; +import com.cloud.utils.net.Ip; +import com.cloud.utils.ssh.SshHelper; +import com.cloud.vm.Nic; +import com.cloud.vm.ReservationContext; +import com.cloud.vm.ReservationContextImpl; +import com.cloud.vm.UserVmManager; +import com.cloud.vm.UserVmService; +import com.cloud.vm.UserVmVO; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.dao.UserVmDao; +import com.cloud.vm.dao.VMInstanceDao; +import com.google.common.base.Strings; + +public class KubernetesClusterManagerImpl extends ManagerBase implements KubernetesClusterService { + + private static final Logger LOGGER = Logger.getLogger(KubernetesClusterManagerImpl.class); + + protected StateMachine2 _stateMachine = KubernetesCluster.State.getStateMachine(); + + ScheduledExecutorService _gcExecutor; + ScheduledExecutorService _stateScanner; + + @Inject + protected KubernetesClusterDao kubernetesClusterDao; + @Inject + protected KubernetesClusterVmMapDao kubernetesClusterVmMapDao; + @Inject + protected KubernetesClusterDetailsDao kubernetesClusterDetailsDao; + @Inject + protected KubernetesSupportedVersionDao kubernetesSupportedVersionDao; + @Inject + protected CAManager caManager; + @Inject + protected SSHKeyPairDao sshKeyPairDao; + @Inject + protected DataCenterDao dataCenterDao; + @Inject + protected ClusterDao clusterDao; + @Inject + protected ClusterDetailsDao clusterDetailsDao; + @Inject + protected ServiceOfferingDao serviceOfferingDao; + @Inject + protected VMTemplateDao templateDao; + @Inject + protected TemplateApiService templateService; + @Inject + protected VMTemplateZoneDao templateZoneDao; + @Inject + protected AccountService accountService; + @Inject + protected AccountDao accountDao; + @Inject + protected AccountManager accountManager; + @Inject + protected VMInstanceDao vmInstanceDao; + @Inject + protected UserVmDao userVmDao; + @Inject + protected UserVmService userVmService; + @Inject + protected UserVmManager userVmManager; + @Inject + protected ConfigurationDao globalConfigDao; + @Inject + protected NetworkOfferingDao networkOfferingDao; + @Inject + protected NetworkService networkService; + @Inject + protected NetworkModel networkModel; + @Inject + protected PhysicalNetworkDao physicalNetworkDao; + @Inject + protected NetworkOrchestrationService networkMgr; + @Inject + protected NetworkDao networkDao; + @Inject + protected IPAddressDao ipAddressDao; + @Inject + protected PortForwardingRulesDao portForwardingRulesDao; + @Inject + protected FirewallService firewallService; + @Inject + protected RulesService rulesService; + @Inject + protected NetworkOfferingServiceMapDao networkOfferingServiceMapDao; + @Inject + protected CapacityManager capacityManager; + @Inject + protected ResourceManager resourceManager; + @Inject + protected FirewallRulesDao firewallRulesDao; + @Inject + protected IpAddressManager ipAddressManager; + + @Override + public KubernetesCluster findById(final Long id) { + return kubernetesClusterDao.findById(id); + } + + @Override + public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) + throws InsufficientCapacityException, ResourceAllocationException, ManagementServerException { + final String name = cmd.getName(); + final String displayName = cmd.getDisplayName(); + final Long zoneId = cmd.getZoneId(); + final Long kubernetesVersionId = cmd.getKubernetesVersionId(); + final Long serviceOfferingId = cmd.getServiceOfferingId(); + final Account owner = accountService.getActiveAccountById(cmd.getEntityOwnerId()); + final Long networkId = cmd.getNetworkId(); + final String sshKeyPair= cmd.getSSHKeyPairName(); + final Long clusterSize = cmd.getClusterSize(); + final String dockerRegistryUserName = cmd.getDockerRegistryUserName(); + final String dockerRegistryPassword = cmd.getDockerRegistryPassword(); + final String dockerRegistryUrl = cmd.getDockerRegistryUrl(); + final String dockerRegistryEmail = cmd.getDockerRegistryEmail(); + final Long nodeRootDiskSize = cmd.getNodeRootDiskSize(); + + if (name == null || name.isEmpty()) { + throw new InvalidParameterValueException("Invalid name for the kubernetes cluster name:" + name); + } + + if (clusterSize < 1 || clusterSize > 100) { + throw new InvalidParameterValueException("invalid cluster size " + clusterSize); + } + + DataCenter zone = dataCenterDao.findById(zoneId); + if (zone == null) { + throw new InvalidParameterValueException("Unable to find zone by id:" + zoneId); + } + + if (Grouping.AllocationState.Disabled == zone.getAllocationState()) { + throw new PermissionDeniedException("Cannot perform this operation, Zone:" + zone.getId() + " is currently disabled."); + } + + final KubernetesSupportedVersion kubernetesVersion = kubernetesSupportedVersionDao.findById(kubernetesVersionId); + if (kubernetesVersion == null) { + throw new InvalidParameterValueException("Unable to find Kubernetes by version in supported versions"); + } + + if (kubernetesVersion.getZoneId() != null && kubernetesVersion.getZoneId() != zone.getId()) { + throw new InvalidParameterValueException(String.format("Kubernetes version ID: %s is not available for zone ID: %s", kubernetesVersion.getUuid(), zone.getUuid())); + } + + ServiceOffering serviceOffering = serviceOfferingDao.findById(serviceOfferingId); + if (serviceOffering == null) { + throw new InvalidParameterValueException("No service offering with id:" + serviceOfferingId); + } else { + } + + if (sshKeyPair != null && !sshKeyPair.isEmpty()) { + SSHKeyPairVO sshKeyPairVO = sshKeyPairDao.findByName(owner.getAccountId(), owner.getDomainId(), sshKeyPair); + if (sshKeyPairVO == null) { + throw new InvalidParameterValueException("Given SSH key pair with name:" + sshKeyPair + " was not found for the account " + owner.getAccountName()); + } + } + + if (!isKubernetesServiceConfigured(zone)) { + throw new ManagementServerException("Kubernetes service has not been configured properly to provision kubernetes clusters."); + } + + VMTemplateVO template = templateDao.findByTemplateName(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesClusterTemplateName.key())); + List listZoneTemplate = templateZoneDao.listByZoneTemplate(zone.getId(), template.getId()); + if (listZoneTemplate == null || listZoneTemplate.isEmpty()) { + LOGGER.warn("The template:" + template.getId() + " is not available for use in zone:" + zoneId + " to provision kubernetes cluster name:" + name); + throw new ManagementServerException("Kubernetes service has not been configured properly to provision kubernetes clusters."); + } + + if (!validateServiceOffering(serviceOfferingDao.findById(serviceOfferingId))) { + throw new InvalidParameterValueException("This service offering is not suitable for k8s cluster, service offering id is " + networkId); + } + + validateDockerRegistryParams(dockerRegistryUserName, dockerRegistryPassword, dockerRegistryUrl, dockerRegistryEmail); + + plan(clusterSize, zoneId, serviceOfferingDao.findById(serviceOfferingId)); + + Network network = null; + if (networkId != null) { + if (kubernetesClusterDao.listByNetworkId(networkId).isEmpty()) { + network = networkService.getNetwork(networkId); + if (network == null) { + throw new InvalidParameterValueException("Unable to find network by ID " + networkId); + } + if (!validateNetwork(network)) { + throw new InvalidParameterValueException("This network is not suitable for k8s cluster, network id is " + networkId); + } + networkModel.checkNetworkPermissions(owner, network); + } else { + throw new InvalidParameterValueException("This network is already under use by another k8s cluster, network id is " + networkId); + } + } else { // user has not specified network in which cluster VM's to be provisioned, so create a network for kubernetes cluster + NetworkOfferingVO networkOffering = networkOfferingDao.findByUniqueName( + globalConfigDao.getValue(KubernetesServiceConfig.KubernetesClusterNetworkOffering.key())); + + long physicalNetworkId = networkModel.findPhysicalNetworkId(zone.getId(), networkOffering.getTags(), networkOffering.getTrafficType()); + PhysicalNetwork physicalNetwork = physicalNetworkDao.findById(physicalNetworkId); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Creating network for account " + owner + " from the network offering id=" + + networkOffering.getId() + " as a part of cluster: " + name + " deployment process"); + } + + try { + network = networkMgr.createGuestNetwork(networkOffering.getId(), name + "-network", owner.getAccountName() + "-network", + null, null, null, false, null, owner, null, physicalNetwork, zone.getId(), ControlledEntity.ACLType.Account, null, null, null, null, true, null, null); + } catch (Exception e) { + LOGGER.warn("Unable to create a network for the kubernetes cluster due to " + e); + throw new ManagementServerException("Unable to create a network for the kubernetes cluster."); + } + } + + final Network defaultNetwork = network; + final VMTemplateVO finalTemplate = template; + final long cores = serviceOffering.getCpu() * clusterSize; + final long memory = serviceOffering.getRamSize() * clusterSize; + + final KubernetesClusterVO cluster = Transaction.execute(new TransactionCallback() { + @Override + public KubernetesClusterVO doInTransaction(TransactionStatus status) { + KubernetesClusterVO newCluster = new KubernetesClusterVO(name, displayName, zoneId, kubernetesVersion.getId(), + serviceOfferingId, finalTemplate.getId(), defaultNetwork.getId(), owner.getDomainId(), + owner.getAccountId(), clusterSize, KubernetesCluster.State.Created, sshKeyPair, cores, memory, nodeRootDiskSize, "", ""); + kubernetesClusterDao.persist(newCluster); + return newCluster; + } + }); + + Transaction.execute(new TransactionCallbackNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + List details = new ArrayList<>(); + if (!Strings.isNullOrEmpty(dockerRegistryUserName)) { + details.add(new KubernetesClusterDetailsVO(cluster.getId(), ApiConstants.DOCKER_REGISTRY_USER_NAME, dockerRegistryUserName, true)); + } + if (!Strings.isNullOrEmpty(dockerRegistryPassword)) { + details.add(new KubernetesClusterDetailsVO(cluster.getId(), ApiConstants.DOCKER_REGISTRY_PASSWORD, dockerRegistryPassword, false)); + } + if (!Strings.isNullOrEmpty(dockerRegistryUrl)) { + details.add(new KubernetesClusterDetailsVO(cluster.getId(), ApiConstants.DOCKER_REGISTRY_URL, dockerRegistryUrl, true)); + } + if (!Strings.isNullOrEmpty(dockerRegistryEmail)) { + details.add(new KubernetesClusterDetailsVO(cluster.getId(), ApiConstants.DOCKER_REGISTRY_EMAIL, dockerRegistryEmail, true)); + } + details.add(new KubernetesClusterDetailsVO(cluster.getId(), ApiConstants.USERNAME, "admin", true)); + SecureRandom random = new SecureRandom(); + String randomPassword = new BigInteger(130, random).toString(32); + details.add(new KubernetesClusterDetailsVO(cluster.getId(), ApiConstants.PASSWORD, randomPassword, false)); + details.add(new KubernetesClusterDetailsVO(cluster.getId(), "networkCleanup", String.valueOf(networkId == null), true)); + kubernetesClusterDetailsDao.saveDetails(details); + } + }); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("A kubernetes cluster name:" + name + " id:" + cluster.getId() + " has been created."); + } + + return cluster; + } + + + // Start operation can be performed at two diffrent life stages of kubernetes cluster. First when a freshly created cluster + // in which case there are no resources provisisioned for the kubernetes cluster. So during start all the resources + // are provisioned from scratch. Second kind of start, happens on Stopped kubernetes cluster, in which all resources + // are provisioned (like volumes, nics, networks etc). It just that VM's are not in running state. So just + // start the VM's (which can possibly implicitly start the network also). + @Override + public boolean startKubernetesCluster(long kubernetesClusterId, boolean onCreate) throws ManagementServerException, + ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException { + + if (onCreate) { + // Start for kubernetes cluster in 'Created' state + return startKubernetesClusterOnCreate(kubernetesClusterId); + } else { + // Start for kubernetes cluster in 'Stopped' state. Resources are already provisioned, just need to be started + return startStoppedKubernetesCluster(kubernetesClusterId); + } + } + + // perform a cold start (which will provision resources as well) + private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) throws ManagementServerException { + + // Starting a kubernetes cluster has below workflow + // - start the network + // - provision the master /node VM + // - provision node VM's (as many as cluster size) + // - update the book keeping data of the VM's provisioned for the cluster + // - setup networking (add Firewall and PF rules) + // - wait till kubernetes API server on master VM to come up + // - wait till addon services (dashboard etc) to come up + // - update API and dashboard URL endpoints in kubernetes cluster details + + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Starting kubernetes cluster: " + kubernetesCluster.getName()); + } + + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.StartRequested); + + Account account = accountDao.findById(kubernetesCluster.getAccountId()); + + DeployDestination dest = null; + try { + dest = plan(kubernetesClusterId, kubernetesCluster.getZoneId()); + } catch (InsufficientCapacityException e) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + LOGGER.warn("Provisioning the cluster failed due to insufficient capacity in the kubernetes cluster: " + kubernetesCluster.getName() + " due to " + e); + throw new ManagementServerException("Provisioning the cluster failed due to insufficient capacity in the kubernetes cluster: " + kubernetesCluster.getName(), e); + } + final ReservationContext context = new ReservationContextImpl(null, null, null, account); + + try { + networkMgr.startNetwork(kubernetesCluster.getNetworkId(), dest, context); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Network:" + kubernetesCluster.getNetworkId() + " is started for the kubernetes cluster: " + kubernetesCluster.getName()); + } + } catch (RuntimeException e) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + LOGGER.warn("Starting the network failed as part of starting kubernetes cluster " + kubernetesCluster.getName() + " due to " + e); + throw new ManagementServerException("Failed to start the network while creating kubernetes cluster name:" + kubernetesCluster.getName(), e); + } catch (Exception e) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + LOGGER.warn("Starting the network failed as part of starting kubernetes cluster " + kubernetesCluster.getName() + " due to " + e); + throw new ManagementServerException("Failed to start the network while creating kubernetes cluster name:" + kubernetesCluster.getName(), e); + } + + IPAddressVO publicIp = null; + List ips = ipAddressDao.listByAssociatedNetwork(kubernetesCluster.getNetworkId(), true); + if (ips == null || ips.isEmpty()) { + LOGGER.warn("Network:" + kubernetesCluster.getNetworkId() + " for the kubernetes cluster name:" + kubernetesCluster.getName() + " does not have " + + "public IP's associated with it. So aborting kubernetes cluster start."); + throw new ManagementServerException("Failed to start the network while creating kubernetes cluster name:" + kubernetesCluster.getName()); + } + publicIp = ips.get(0); + + List clusterVMIds = new ArrayList<>(); + + UserVm k8sMasterVM = null; + try { + k8sMasterVM = createKubernetesMaster(kubernetesCluster, ips); + + final long clusterId = kubernetesCluster.getId(); + final long masterVmId = k8sMasterVM.getId(); + Transaction.execute(new TransactionCallback() { + @Override + public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { + KubernetesClusterVmMapVO newClusterVmMap = new KubernetesClusterVmMapVO(clusterId, masterVmId); + kubernetesClusterVmMapDao.persist(newClusterVmMap); + return newClusterVmMap; + } + }); + + startKubernetesVM(k8sMasterVM, kubernetesCluster); + clusterVMIds.add(k8sMasterVM.getId()); + k8sMasterVM = userVmDao.findById(k8sMasterVM.getId()); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Provisioned the master VM's in to the kubernetes cluster name:" + kubernetesCluster.getName()); + } + } catch (RuntimeException e) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + LOGGER.warn("Provisioning the master VM' failed in the kubernetes cluster: " + kubernetesCluster.getName() + " due to " + e); + throw new ManagementServerException("Provisioning the master VM' failed in the kubernetes cluster: " + kubernetesCluster.getName(), e); + } catch (Exception e) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + LOGGER.warn("Provisioning the master VM' failed in the kubernetes cluster: " + kubernetesCluster.getName() + " due to " + e); + throw new ManagementServerException("Provisioning the master VM' failed in the kubernetes cluster: " + kubernetesCluster.getName(), e); + } + + String masterIP = k8sMasterVM.getPrivateIpAddress(); + + for (int i = 1; i <= kubernetesCluster.getNodeCount(); i++) { + UserVm vm = null; + try { + vm = createKubernetesNode(kubernetesCluster, masterIP, i); + final long nodeVmId = vm.getId(); + KubernetesClusterVmMapVO clusterNodeVmMap = Transaction.execute(new TransactionCallback() { + @Override + public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { + KubernetesClusterVmMapVO newClusterVmMap = new KubernetesClusterVmMapVO(kubernetesClusterId, nodeVmId); + kubernetesClusterVmMapDao.persist(newClusterVmMap); + return newClusterVmMap; + } + }); + startKubernetesVM(vm, kubernetesCluster); + clusterVMIds.add(vm.getId()); + + vm = userVmDao.findById(vm.getId()); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Provisioned a node VM in to the kubernetes cluster: " + kubernetesCluster.getName()); + } + } catch (RuntimeException e) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + LOGGER.warn("Provisioning the node VM failed in the kubernetes cluster " + kubernetesCluster.getName() + " due to " + e); + throw new ManagementServerException("Provisioning the node VM failed in the kubernetes cluster " + kubernetesCluster.getName(), e); + } catch (Exception e) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + LOGGER.warn("Provisioning the node VM failed in the kubernetes cluster " + kubernetesCluster.getName() + " due to " + e); + throw new ManagementServerException("Provisioning the node VM failed in the kubernetes cluster " + kubernetesCluster.getName(), e); + } + } + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Kubernetes cluster : " + kubernetesCluster.getName() + " VM's are successfully provisioned."); + } + + setupKubernetesClusterNetworkRules(publicIp, account, kubernetesClusterId, clusterVMIds); + attachIsoKubernetesVMs(kubernetesClusterId, clusterVMIds); + + int retryCounter = 0; + int maxRetries = 30; + boolean k8sApiServerSetup = false; + + while (retryCounter < maxRetries) { + try (Socket socket = new Socket()) { + socket.connect(new InetSocketAddress(publicIp.getAddress().addr(), 6443), 10000); + k8sApiServerSetup = true; + kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + kubernetesCluster.setEndpoint("https://" + publicIp.getAddress() + ":6443/"); + kubernetesClusterDao.update(kubernetesCluster.getId(), kubernetesCluster); + break; + } catch (IOException e) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Waiting for kubernetes cluster: " + kubernetesCluster.getName() + " API endpoint to be available. retry: " + retryCounter + "/" + maxRetries); + } + try { + Thread.sleep(60000); + } catch (InterruptedException ex) { + } + retryCounter++; + } + } + + boolean k8sKubeConfigCopied = false; + if (k8sApiServerSetup) { + Runtime r = Runtime.getRuntime(); + retryCounter = 0; + maxRetries = 5; + String kubeConfig = ""; + while (retryCounter < maxRetries && kubeConfig.isEmpty()) { + try { + Boolean devel = Boolean.valueOf(globalConfigDao.getValue("developer")); + String keyFile = String.format("%s/.ssh/id_rsa", System.getProperty("user.home")); + if (devel) { + keyFile += ".cloud"; + } + File pkFile = new File(keyFile); + Pair result = SshHelper.sshExecute(publicIp.getAddress().addr(), 2222, "core", + pkFile, null, "sudo cat /etc/kubernetes/admin.conf", + 10000, 10000, 10000); + + if (result.first() && !Strings.isNullOrEmpty(result.second())) { + kubeConfig = result.second(); + kubeConfig = kubeConfig.replace(String.format("server: https://%s:6443", k8sMasterVM.getPrivateIpAddress()), + String.format("server: https://%s:6443", publicIp.getAddress().addr())); + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "kubeConfigData", Base64.encodeBase64String(kubeConfig.getBytes(Charset.forName("UTF-8"))), false); + k8sKubeConfigCopied = true; + break; + } + } catch (Exception e) { + LOGGER.warn("Failed to retrieve kube-config file for cluster with ID " + kubernetesCluster.getUuid() + ": " + e); + } + retryCounter++; + } + } + + if (k8sKubeConfigCopied) { + retryCounter = 0; + maxRetries = 30; + // Dashbaord service is a docker image downloaded at run time. + // So wait for some time and check if dashbaord service is up running. + while (retryCounter < maxRetries) { + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Waiting for dashboard service for the kubernetes cluster: " + kubernetesCluster.getName() + + " to come up. Attempt: " + retryCounter + " of max retries " + maxRetries); + } + + if (isAddOnServiceRunning(kubernetesCluster.getId(), "kubernetes-dashboard")) { + + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationSucceeded); + + kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + kubernetesCluster.setConsoleEndpoint("https://" + publicIp.getAddress() + ":6443/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy#!/overview?namespace=_all"); + kubernetesClusterDao.update(kubernetesCluster.getId(), kubernetesCluster); + + detachIsoKubernetesVMs(kubernetesClusterId, clusterVMIds); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Kubernetes cluster name:" + kubernetesCluster.getName() + " is successfully started"); + } + + return true; + } + + try { + Thread.sleep(10000); + } catch (InterruptedException ex) { + } + retryCounter++; + } + LOGGER.warn("Failed to setup kubernetes cluster " + kubernetesCluster.getName() + " in usable state as" + + " unable to bring dashboard add on service up"); + } else { + LOGGER.warn("Failed to setup kubernetes cluster " + kubernetesCluster.getName() + " in usable state as" + + " unable to bring the API server up"); + } + + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + + detachIsoKubernetesVMs(kubernetesClusterId, clusterVMIds); + + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, + "Failed to deploy kubernetes cluster: " + kubernetesCluster.getId() + " as unable to setup up in usable state"); + } + + private boolean startStoppedKubernetesCluster(long kubernetesClusterId) throws ManagementServerException, + ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException { + + final KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + if (kubernetesCluster == null) { + throw new ManagementServerException("Failed to find kubernetes cluster id: " + kubernetesClusterId); + } + + if (kubernetesCluster.getRemoved() != null) { + throw new ManagementServerException("Kubernetes cluster id:" + kubernetesClusterId + " is already deleted."); + } + + if (kubernetesCluster.getState().equals(KubernetesCluster.State.Running)) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Kubernetes cluster id: " + kubernetesClusterId + " is already Running."); + } + return true; + } + + if (kubernetesCluster.getState().equals(KubernetesCluster.State.Starting)) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Kubernetes cluster id: " + kubernetesClusterId + " is getting started."); + } + return true; + } + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Starting kubernetes cluster: " + kubernetesCluster.getName()); + } + + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.StartRequested); + + for (final KubernetesClusterVmMapVO vmMapVO : kubernetesClusterVmMapDao.listByClusterId(kubernetesClusterId)) { + final UserVmVO vm = userVmDao.findById(vmMapVO.getVmId()); + try { + if (vm == null) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + throw new ManagementServerException("Failed to start all VMs in kubernetes cluster id: " + kubernetesClusterId); + } + startKubernetesVM(vm, kubernetesCluster); + } catch (ServerApiException ex) { + LOGGER.warn("Failed to start VM in kubernetes cluster id:" + kubernetesClusterId + " due to " + ex); + // dont bail out here. proceed further to stop the reset of the VM's + } + } + + for (final KubernetesClusterVmMapVO vmMapVO : kubernetesClusterVmMapDao.listByClusterId(kubernetesClusterId)) { + final UserVmVO vm = userVmDao.findById(vmMapVO.getVmId()); + if (vm == null || !vm.getState().equals(VirtualMachine.State.Running)) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + throw new ManagementServerException("Failed to start all VMs in kubernetes cluster id: " + kubernetesClusterId); + } + } + + InetAddress address = null; + try { + address = InetAddress.getByName(new URL(kubernetesCluster.getEndpoint()).getHost()); + } catch (MalformedURLException | UnknownHostException ex) { + // API end point is generated by CCS, so this situation should not arise. + LOGGER.warn("Kubernetes cluster id:" + kubernetesClusterId + " has invalid api endpoint. Can not " + + "verify if cluster is in ready state."); + throw new ManagementServerException("Can not verify if kubernetes cluster id:" + kubernetesClusterId + " is in usable state."); + } + + // wait for fixed time for K8S api server to be avaialble + int retryCounter = 0; + int maxRetries = 10; + boolean k8sApiServerSetup = false; + while (retryCounter < maxRetries) { + try (Socket socket = new Socket()) { + socket.connect(new InetSocketAddress(address.getHostAddress(), 6443), 10000); + k8sApiServerSetup = true; + break; + } catch (IOException e) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Waiting for kubernetes cluster: " + kubernetesCluster.getName() + " API endpoint to be available. retry: " + retryCounter + "/" + maxRetries); + } + try { + Thread.sleep(50000); + } catch (InterruptedException ex) { + } + retryCounter++; + } + } + + if (!k8sApiServerSetup) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + throw new ManagementServerException("Failed to setup kubernetes cluster id: " + kubernetesClusterId + " is usable state."); + } + + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationSucceeded); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(" Kubernetes cluster name:" + kubernetesCluster.getName() + " is successfully started."); + } + return true; + } + + // Open up firewall port 6443, secure port on which kubernetes API server is running. Also create port-forwarding + // rule to forward public IP traffic to master VM private IP + // Open up firewall ports 2222 to 2222+n for SSH access. Also create port-forwarding + // rule to forward public IP traffic to all node VM private IP + private void setupKubernetesClusterNetworkRules(IPAddressVO publicIp, Account account, long kubernetesClusterId, + List clusterVMIds) throws ManagementServerException { + + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + + List sourceCidrList = new ArrayList(); + sourceCidrList.add("0.0.0.0/0"); + + try { + CreateFirewallRuleCmd rule = new CreateFirewallRuleCmd(); + rule = ComponentContext.inject(rule); + + Field addressField = rule.getClass().getDeclaredField("ipAddressId"); + addressField.setAccessible(true); + addressField.set(rule, publicIp.getId()); + + Field protocolField = rule.getClass().getDeclaredField("protocol"); + protocolField.setAccessible(true); + protocolField.set(rule, "TCP"); + + Field startPortField = rule.getClass().getDeclaredField("publicStartPort"); + startPortField.setAccessible(true); + startPortField.set(rule, new Integer(6443)); + + Field endPortField = rule.getClass().getDeclaredField("publicEndPort"); + endPortField.setAccessible(true); + endPortField.set(rule, new Integer(6443)); + + Field cidrField = rule.getClass().getDeclaredField("cidrlist"); + cidrField.setAccessible(true); + cidrField.set(rule, sourceCidrList); + + firewallService.createIngressFirewallRule(rule); + firewallService.applyIngressFwRules(publicIp.getId(), account); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Provisioned firewall rule to open up port 6443 on " + publicIp.getAddress() + + " for cluster " + kubernetesCluster.getName()); + } + } catch (RuntimeException rte) { + LOGGER.warn("Failed to provision firewall rules for the kubernetes cluster: " + kubernetesCluster.getName() + + " due to exception: " + getStackTrace(rte)); + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + throw new ManagementServerException("Failed to provision firewall rules for the kubernetes " + + "cluster: " + kubernetesCluster.getName(), rte); + } catch (Exception e) { + LOGGER.warn("Failed to provision firewall rules for the kubernetes cluster: " + kubernetesCluster.getName() + + " due to exception: " + getStackTrace(e)); + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + throw new ManagementServerException("Failed to provision firewall rules for the kubernetes " + + "cluster: " + kubernetesCluster.getName()); + } + + try { + CreateFirewallRuleCmd rule = new CreateFirewallRuleCmd(); + rule = ComponentContext.inject(rule); + + Field addressField = rule.getClass().getDeclaredField("ipAddressId"); + addressField.setAccessible(true); + addressField.set(rule, publicIp.getId()); + + Field protocolField = rule.getClass().getDeclaredField("protocol"); + protocolField.setAccessible(true); + protocolField.set(rule, "TCP"); + + Field startPortField = rule.getClass().getDeclaredField("publicStartPort"); + startPortField.setAccessible(true); + startPortField.set(rule, new Integer(2222)); + + Field endPortField = rule.getClass().getDeclaredField("publicEndPort"); + endPortField.setAccessible(true); + endPortField.set(rule, new Integer(2222 + clusterVMIds.size() - 1)); // clusterVMIds contains all nodes including master + + Field cidrField = rule.getClass().getDeclaredField("cidrlist"); + cidrField.setAccessible(true); + cidrField.set(rule, sourceCidrList); + + firewallService.createIngressFirewallRule(rule); + firewallService.applyIngressFwRules(publicIp.getId(), account); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Provisioned firewall rule to open up port 2222 on " + publicIp.getAddress() + + " for cluster " + kubernetesCluster.getName()); + } + } catch (RuntimeException rte) { + LOGGER.warn("Failed to provision firewall rules for the kubernetes cluster: " + kubernetesCluster.getName() + + " due to exception: " + getStackTrace(rte)); + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + throw new ManagementServerException("Failed to provision firewall rules for the kubernetes " + + "cluster: " + kubernetesCluster.getName(), rte); + } catch (Exception e) { + LOGGER.warn("Failed to provision firewall rules for the kubernetes cluster: " + kubernetesCluster.getName() + + " due to exception: " + getStackTrace(e)); + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + throw new ManagementServerException("Failed to provision firewall rules for the kubernetes " + + "cluster: " + kubernetesCluster.getName()); + } + + Nic masterVmNic = networkModel.getNicInNetwork(clusterVMIds.get(0), kubernetesCluster.getNetworkId()); + // handle Nic interface method change between releases 4.5 and 4.6 and above through reflection + Method m = null; + try { + m = Nic.class.getMethod("getIp4Address"); + } catch (NoSuchMethodException e1) { + try { + m = Nic.class.getMethod("getIPv4Address"); + } catch (NoSuchMethodException e2) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + throw new ManagementServerException("Failed to activate port forwarding rules for the cluster: " + kubernetesCluster.getName()); + } + } + Ip masterIp = null; + try { + masterIp = new Ip(m.invoke(masterVmNic).toString()); + } catch (InvocationTargetException | IllegalAccessException ie) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + throw new ManagementServerException("Failed to activate port forwarding rules for the cluster: " + kubernetesCluster.getName()); + } + final Ip masterIpFinal = masterIp; + final long publicIpId = publicIp.getId(); + final long networkId = kubernetesCluster.getNetworkId(); + final long accountId = account.getId(); + final long domainId = account.getDomainId(); + final long masterVmIdFinal = clusterVMIds.get(0); + + try { + PortForwardingRuleVO pfRule = Transaction.execute(new TransactionCallbackWithException() { + @Override + public PortForwardingRuleVO doInTransaction(TransactionStatus status) throws NetworkRuleConflictException { + PortForwardingRuleVO newRule = + new PortForwardingRuleVO(null, publicIpId, + 6443, 6443, + masterIpFinal, + 6443, 6443, + "tcp", networkId, accountId, domainId, masterVmIdFinal); + newRule.setDisplay(true); + newRule.setState(FirewallRule.State.Add); + newRule = portForwardingRulesDao.persist(newRule); + return newRule; + } + }); + rulesService.applyPortForwardingRules(publicIp.getId(), account); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Provisioning port forwarding rule from port 6443 on " + publicIp.getAddress() + + " to the master VM IP :" + masterIpFinal + " in kubernetes cluster " + kubernetesCluster.getName()); + } + } catch (RuntimeException rte) { + LOGGER.warn("Failed to activate port forwarding rules for the kubernetes cluster " + kubernetesCluster.getName() + " due to " + rte); + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + throw new ManagementServerException("Failed to activate port forwarding rules for the cluster: " + kubernetesCluster.getName(), rte); + } catch (Exception e) { + LOGGER.warn("Failed to activate port forwarding rules for the kubernetes cluster " + kubernetesCluster.getName() + " due to " + e); + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + throw new ManagementServerException("Failed to activate port forwarding rules for the cluster: " + kubernetesCluster.getName(), e); + } + + for (int i = 0; i < clusterVMIds.size(); ++i) { + long vmId = clusterVMIds.get(i); + Nic vmNic = networkModel.getNicInNetwork(vmId, kubernetesCluster.getNetworkId()); + Ip vmIp = null; + try { + vmIp = new Ip(m.invoke(vmNic).toString()); + } catch (InvocationTargetException | IllegalAccessException ie) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + throw new ManagementServerException("Failed to activate SSH port forwarding rules for the cluster: " + kubernetesCluster.getName()); + } + final Ip vmIpFinal = vmIp; + final long vmIdFinal = vmId; + final int srcPortFinal = 2222 + i; + + try { + PortForwardingRuleVO pfRule = Transaction.execute(new TransactionCallbackWithException() { + @Override + public PortForwardingRuleVO doInTransaction(TransactionStatus status) throws NetworkRuleConflictException { + PortForwardingRuleVO newRule = + new PortForwardingRuleVO(null, publicIpId, + srcPortFinal, srcPortFinal, + vmIpFinal, + 22, 22, + "tcp", networkId, accountId, domainId, vmIdFinal); + newRule.setDisplay(true); + newRule.setState(FirewallRule.State.Add); + newRule = portForwardingRulesDao.persist(newRule); + return newRule; + } + }); + rulesService.applyPortForwardingRules(publicIp.getId(), account); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Provisioning SSH port forwarding rule from port 2222 to 22 on " + publicIp.getAddress() + + " to the VM IP :" + vmIpFinal + " in kubernetes cluster " + kubernetesCluster.getName()); + } + } catch (RuntimeException rte) { + LOGGER.warn("Failed to activate SSH port forwarding rules for the kubernetes cluster " + kubernetesCluster.getName() + " due to " + rte); + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + throw new ManagementServerException("Failed to activate SSH port forwarding rules for the cluster: " + kubernetesCluster.getName(), rte); + } catch (Exception e) { + LOGGER.warn("Failed to activate SSH port forwarding rules for the kubernetes cluster " + kubernetesCluster.getName() + " due to " + e); + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + throw new ManagementServerException("Failed to activate SSH port forwarding rules for the cluster: " + kubernetesCluster.getName(), e); + } + } + } + + // Open up firewall ports 2222 to 2222+n for SSH access. Also create port-forwarding + // rule to forward public IP traffic to all node VM private IP. Existing node VMs before scaling + // will already be having these rules + private void scaleKubernetesClusterNetworkRules(IPAddressVO publicIp, Account account, long kubernetesClusterId, + List clusterVMIds) throws ManagementServerException { + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + + List sourceCidrList = new ArrayList(); + sourceCidrList.add("0.0.0.0/0"); + boolean firewallRuleFound = false; + int existingFirewallRuleSourcePortEnd = 2222; + List firewallRules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(publicIp.getId(), FirewallRule.Purpose.Firewall); + for (FirewallRuleVO firewallRule : firewallRules) { + if (firewallRule.getSourcePortStart() == 2222) { + firewallRuleFound = true; + existingFirewallRuleSourcePortEnd = firewallRule.getSourcePortEnd(); + firewallService.revokeIngressFwRule(firewallRule.getId(), true); + break; + } + } + if (!firewallRuleFound) { + throw new ManagementServerException("Firewall rule for node SSH access can't be provisioned!"); + } + try { + CreateFirewallRuleCmd rule = new CreateFirewallRuleCmd(); + rule = ComponentContext.inject(rule); + + Field addressField = rule.getClass().getDeclaredField("ipAddressId"); + addressField.setAccessible(true); + addressField.set(rule, publicIp.getId()); + + Field protocolField = rule.getClass().getDeclaredField("protocol"); + protocolField.setAccessible(true); + protocolField.set(rule, "TCP"); + + Field startPortField = rule.getClass().getDeclaredField("publicStartPort"); + startPortField.setAccessible(true); + startPortField.set(rule, new Integer(2222)); + + Field endPortField = rule.getClass().getDeclaredField("publicEndPort"); + endPortField.setAccessible(true); + endPortField.set(rule, new Integer(2222 + (int)kubernetesCluster.getNodeCount())); + + Field cidrField = rule.getClass().getDeclaredField("cidrlist"); + cidrField.setAccessible(true); + cidrField.set(rule, sourceCidrList); + + firewallService.createIngressFirewallRule(rule); + firewallService.applyIngressFwRules(publicIp.getId(), account); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Provisioned firewall rule to open up port 2222 on " + publicIp.getAddress() + + " for cluster " + kubernetesCluster.getName()); + } + } catch (RuntimeException rte) { + LOGGER.warn("Failed to provision firewall rules for the kubernetes cluster: " + kubernetesCluster.getName() + + " due to exception: " + getStackTrace(rte)); + throw new ManagementServerException("Failed to provision firewall rules for the kubernetes " + + "cluster: " + kubernetesCluster.getName(), rte); + } catch (Exception e) { + LOGGER.warn("Failed to provision firewall rules for the kubernetes cluster: " + kubernetesCluster.getName() + + " due to exception: " + getStackTrace(e)); + throw new ManagementServerException("Failed to provision firewall rules for the kubernetes " + + "cluster: " + kubernetesCluster.getName()); + } + + if (clusterVMIds != null && !clusterVMIds.isEmpty()) { // Upscaling, add new port-forwarding rules + // handle Nic interface method change between releases 4.5 and 4.6 and above through reflection + Method m = null; + try { + m = Nic.class.getMethod("getIp4Address"); + } catch (NoSuchMethodException e1) { + try { + m = Nic.class.getMethod("getIPv4Address"); + } catch (NoSuchMethodException e2) { + throw new ManagementServerException("Failed to activate port forwarding rules for the cluster: " + kubernetesCluster.getName()); + } + } + + // Apply port forwarding only to new VMs + final long publicIpId = publicIp.getId(); + final long networkId = kubernetesCluster.getNetworkId(); + final long accountId = account.getId(); + final long domainId = account.getDomainId(); + for (int i = 0; i < clusterVMIds.size(); ++i) { + long vmId = clusterVMIds.get(i); + Nic vmNic = networkModel.getNicInNetwork(vmId, kubernetesCluster.getNetworkId()); + Ip vmIp = null; + try { + vmIp = new Ip(m.invoke(vmNic).toString()); + } catch (InvocationTargetException | IllegalAccessException ie) { + throw new ManagementServerException("Failed to activate SSH port forwarding rules for the cluster: " + kubernetesCluster.getName()); + } + final Ip vmIpFinal = vmIp; + final long vmIdFinal = vmId; + final int srcPortFinal = existingFirewallRuleSourcePortEnd + 1 + i; + + try { + PortForwardingRuleVO pfRule = Transaction.execute(new TransactionCallbackWithException() { + @Override + public PortForwardingRuleVO doInTransaction(TransactionStatus status) throws NetworkRuleConflictException { + PortForwardingRuleVO newRule = + new PortForwardingRuleVO(null, publicIpId, + srcPortFinal, srcPortFinal, + vmIpFinal, + 22, 22, + "tcp", networkId, accountId, domainId, vmIdFinal); + newRule.setDisplay(true); + newRule.setState(FirewallRule.State.Add); + newRule = portForwardingRulesDao.persist(newRule); + return newRule; + } + }); + rulesService.applyPortForwardingRules(publicIp.getId(), account); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Provisioning SSH port forwarding rule from port 2222 to 22 on " + publicIp.getAddress() + + " to the VM IP :" + vmIpFinal + " in kubernetes cluster " + kubernetesCluster.getName()); + } + } catch (RuntimeException rte) { + LOGGER.warn("Failed to activate SSH port forwarding rules for the kubernetes cluster " + kubernetesCluster.getName() + " due to " + rte); + throw new ManagementServerException("Failed to activate SSH port forwarding rules for the cluster: " + kubernetesCluster.getName(), rte); + } catch (Exception e) { + LOGGER.warn("Failed to activate SSH port forwarding rules for the kubernetes cluster " + kubernetesCluster.getName() + " due to " + e); + throw new ManagementServerException("Failed to activate SSH port forwarding rules for the cluster: " + kubernetesCluster.getName(), e); + } + } + } + } + + public boolean validateNetwork(Network network) { + NetworkOffering networkOffering = networkOfferingDao.findById(network.getNetworkOfferingId()); + if (networkOffering.isSystemOnly()) { + throw new InvalidParameterValueException("This network is for system use only, network id " + network.getId()); + } + if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.UserData)) { + throw new InvalidParameterValueException("This network does not support userdata that is required for k8s, network id " + network.getId()); + } + if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.Firewall)) { + throw new InvalidParameterValueException("This network does not support firewall that is required for k8s, network id " + network.getId()); + } + if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.PortForwarding)) { + throw new InvalidParameterValueException("This network does not support port forwarding that is required for k8s, network id " + network.getId()); + } + if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.Dhcp)) { + throw new InvalidParameterValueException("This network does not support dhcp that is required for k8s, network id " + network.getId()); + } + + List addrs = networkModel.listPublicIpsAssignedToGuestNtwk(network.getId(), true); + IPAddressVO sourceNatIp = null; + if (addrs.isEmpty()) { + throw new InvalidParameterValueException("The network id:" + network.getId() + " does not have source NAT ip assoicated with it. " + + "To provision a Kubernetes Cluster, a isolated network with source NAT is required."); + } else { + for (IpAddress addr : addrs) { + if (addr.isSourceNat()) { + sourceNatIp = ipAddressDao.findById(addr.getId()); + } + } + if (sourceNatIp == null) { + throw new InvalidParameterValueException("The network id:" + network.getId() + " does not have source NAT ip assoicated with it. " + + "To provision a Kubernetes Cluster, a isolated network with source NAT is required."); + } + } + List rules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(sourceNatIp.getId(), FirewallRule.Purpose.Firewall); + for (FirewallRuleVO rule : rules) { + Integer startPort = rule.getSourcePortStart(); + Integer endPort = rule.getSourcePortEnd(); + LOGGER.debug("Network rule : " + startPort + " " + endPort); + if (startPort <= 6443 && 6443 <= endPort) { + throw new InvalidParameterValueException("The network id:" + network.getId() + " has conflicting firewall rules to provision" + + " kubernetes cluster."); + } + } + + rules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(sourceNatIp.getId(), FirewallRule.Purpose.PortForwarding); + for (FirewallRuleVO rule : rules) { + Integer startPort = rule.getSourcePortStart(); + Integer endPort = rule.getSourcePortEnd(); + LOGGER.debug("Network rule : " + startPort + " " + endPort); + if (startPort <= 6443 && 6443 <= endPort) { + throw new InvalidParameterValueException("The network id:" + network.getId() + " has conflicting port forwarding rules to provision" + + " kubernetes cluster."); + } + } + return true; + } + + private boolean validateServiceOffering(ServiceOffering serviceOffering) { + if (serviceOffering.isDynamic()) { + throw new InvalidParameterValueException(String.format("Custom service offerings are not supported for creating clusters, service offering ID: %s", serviceOffering.getUuid())); + } + if (serviceOffering.getCpu() < 2 || serviceOffering.getRamSize() < 2048) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster cannot be created with service offering ID: %s, kubernetes cluster template(CoreOS) needs minimum 2 vCPUs and 2 GB RAM", serviceOffering.getUuid())); + } + return true; + } + + private void validateDockerRegistryParams(final String dockerRegistryUserName, + final String dockerRegistryPassword, + final String dockerRegistryUrl, + final String dockerRegistryEmail) { + // if no params related to docker registry specified then nothing to validate so return true + if ((dockerRegistryUserName == null || dockerRegistryUserName.isEmpty()) && + (dockerRegistryPassword == null || dockerRegistryPassword.isEmpty()) && + (dockerRegistryUrl == null || dockerRegistryUrl.isEmpty()) && + (dockerRegistryEmail == null || dockerRegistryEmail.isEmpty())) { + return; + } + + // all params related to docker registry must be specified or nothing + if (!((dockerRegistryUserName != null && !dockerRegistryUserName.isEmpty()) && + (dockerRegistryPassword != null && !dockerRegistryPassword.isEmpty()) && + (dockerRegistryUrl != null && !dockerRegistryUrl.isEmpty()) && + (dockerRegistryEmail != null && !dockerRegistryEmail.isEmpty()))) { + throw new InvalidParameterValueException("All the docker private registry parameters (username, password, url, email) required are specified"); + } + + try { + URL url = new URL(dockerRegistryUrl); + } catch (MalformedURLException e) { + throw new InvalidParameterValueException("Invalid docker registry url specified"); + } + + Pattern VALID_EMAIL_ADDRESS_REGEX = Pattern.compile("^[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,6}$", Pattern.CASE_INSENSITIVE); + Matcher matcher = VALID_EMAIL_ADDRESS_REGEX.matcher(dockerRegistryEmail); + if (!matcher.find()) { + throw new InvalidParameterValueException("Invalid docker registry email specified"); + } + } + + public DeployDestination plan(final long clusterSize, final long dcId, final ServiceOffering offering) throws InsufficientServerCapacityException { + final int cpu_requested = offering.getCpu() * offering.getSpeed(); + final long ram_requested = offering.getRamSize() * 1024L * 1024L; + List hosts = resourceManager.listAllHostsInOneZoneByType(Type.Routing, dcId); + final Map> hosts_with_resevered_capacity = new ConcurrentHashMap>(); + for (HostVO h : hosts) { + hosts_with_resevered_capacity.put(h.getUuid(), new Pair(h, 0)); + } + boolean suitable_host_found = false; + for (int i = 1; i <= clusterSize + 1; i++) { + suitable_host_found = false; + for (Map.Entry> hostEntry : hosts_with_resevered_capacity.entrySet()) { + Pair hp = hostEntry.getValue(); + HostVO h = hp.first(); + int reserved = hp.second(); + reserved++; + ClusterVO cluster = clusterDao.findById(h.getClusterId()); + ClusterDetailsVO cluster_detail_cpu = clusterDetailsDao.findDetail(cluster.getId(), "cpuOvercommitRatio"); + ClusterDetailsVO cluster_detail_ram = clusterDetailsDao.findDetail(cluster.getId(), "memoryOvercommitRatio"); + Float cpuOvercommitRatio = Float.parseFloat(cluster_detail_cpu.getValue()); + Float memoryOvercommitRatio = Float.parseFloat(cluster_detail_ram.getValue()); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Checking host " + h.getId() + " for capacity already reserved " + reserved); + } + if (capacityManager.checkIfHostHasCapacity(h.getId(), cpu_requested * reserved, ram_requested * reserved, false, cpuOvercommitRatio, memoryOvercommitRatio, true)) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Found host " + h.getId() + " has enough capacity cpu = " + cpu_requested * reserved + " ram =" + ram_requested * reserved); + } + hostEntry.setValue(new Pair(h, reserved)); + suitable_host_found = true; + break; + } + } + if (suitable_host_found) { + continue; + } else { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Suitable hosts not found in datacenter " + dcId + " for node " + i); + } + break; + } + } + if (suitable_host_found) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Suitable hosts found in datacenter " + dcId + " creating deployment destination"); + } + return new DeployDestination(dataCenterDao.findById(dcId), null, null, null); + } + String msg = String.format("Cannot find enough capacity for kubernetes cluster(requested cpu=%1$s memory=%2$s)", + cpu_requested * clusterSize, ram_requested * clusterSize); + LOGGER.warn(msg); + throw new InsufficientServerCapacityException(msg, DataCenter.class, dcId); + } + + public DeployDestination plan(final KubernetesCluster kubernetesCluster) throws InsufficientServerCapacityException { + return plan(kubernetesCluster.getId(), kubernetesCluster.getZoneId()); + } + + public DeployDestination plan(final long kubernetesClusterId, final long dcId) throws InsufficientServerCapacityException { + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + ServiceOffering offering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Checking deployment destination for kubernetesClusterId= " + kubernetesClusterId + " in dcId=" + dcId); + } + + return plan(kubernetesCluster.getNodeCount() + 1, dcId, offering); + } + + @Override + public boolean stopKubernetesCluster(long kubernetesClusterId) throws ManagementServerException { + + final KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + if (kubernetesCluster == null) { + throw new ManagementServerException("Failed to find kubernetes cluster id: " + kubernetesClusterId); + } + + if (kubernetesCluster.getRemoved() != null) { + throw new ManagementServerException("Kubernetes cluster id:" + kubernetesClusterId + " is already deleted."); + } + + if (kubernetesCluster.getState().equals(KubernetesCluster.State.Stopped)) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Kubernetes cluster id: " + kubernetesClusterId + " is already stopped."); + } + return true; + } + + if (kubernetesCluster.getState().equals(KubernetesCluster.State.Stopping)) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Kubernetes cluster id: " + kubernetesClusterId + " is getting stopped."); + } + return true; + } + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Stopping kubernetes cluster: " + kubernetesCluster.getName()); + } + + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.StopRequested); + + for (final KubernetesClusterVmMapVO vmMapVO : kubernetesClusterVmMapDao.listByClusterId(kubernetesClusterId)) { + final UserVmVO vm = userVmDao.findById(vmMapVO.getVmId()); + try { + if (vm == null) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + throw new ManagementServerException("Failed to start all VMs in kubernetes cluster id: " + kubernetesClusterId); + } + stopClusterVM(vmMapVO); + } catch (ServerApiException ex) { + LOGGER.warn("Failed to stop VM in kubernetes cluster id:" + kubernetesClusterId + " due to " + ex); + // dont bail out here. proceed further to stop the reset of the VM's + } + } + + for (final KubernetesClusterVmMapVO vmMapVO : kubernetesClusterVmMapDao.listByClusterId(kubernetesClusterId)) { + final UserVmVO vm = userVmDao.findById(vmMapVO.getVmId()); + if (vm == null || !vm.getState().equals(VirtualMachine.State.Stopped)) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + throw new ManagementServerException("Failed to stop all VMs in kubernetes cluster id: " + kubernetesClusterId); + } + } + + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationSucceeded); + return true; + } + + private boolean isAddOnServiceRunning(Long clusterId, String svcName) { + + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(clusterId); + + //FIXME: whole logic needs revamp. Assumption that management server has public network access is not practical + IPAddressVO publicIp = null; + List ips = ipAddressDao.listByAssociatedNetwork(kubernetesCluster.getNetworkId(), true); + publicIp = ips.get(0); + + Runtime r = Runtime.getRuntime(); + int nodePort = 0; + try { + Boolean devel = Boolean.valueOf(globalConfigDao.getValue("developer")); + String keyFile = String.format("%s/.ssh/id_rsa", System.getProperty("user.home")); + if (devel) { + keyFile += ".cloud"; + } + File pkFile = new File(keyFile); + Pair result = SshHelper.sshExecute(publicIp.getAddress().addr(), 2222, "core", + pkFile, null, "sudo kubectl get pods --namespace=kube-system", + 10000, 10000, 10000); + if (result.first() && !Strings.isNullOrEmpty(result.second())) { + String[] lines = result.second().split("\n"); + for (String line : + lines) { + if (line.contains(svcName) && line.contains("Running")) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Service :" + svcName + " for the kubernetes cluster " + + kubernetesCluster.getName() + " is running"); + } + return true; + } + } + } + } catch (Exception e) { + LOGGER.warn("KUBECTL: " + e); + } + return false; + } + + @Override + public boolean deleteKubernetesCluster(Long kubernetesClusterId) throws ManagementServerException { + + KubernetesClusterVO cluster = kubernetesClusterDao.findById(kubernetesClusterId); + if (cluster == null) { + throw new InvalidParameterValueException("Invalid cluster id specified"); + } + + CallContext ctx = CallContext.current(); + Account caller = ctx.getCallingAccount(); + + accountManager.checkAccess(caller, SecurityChecker.AccessType.OperateEntry, false, cluster); + + return cleanupKubernetesClusterResources(kubernetesClusterId); + } + + private boolean cleanupKubernetesClusterResources(Long kubernetesClusterId) throws ManagementServerException { + + KubernetesClusterVO cluster = kubernetesClusterDao.findById(kubernetesClusterId); + + if (!(cluster.getState().equals(KubernetesCluster.State.Running) + || cluster.getState().equals(KubernetesCluster.State.Stopped) + || cluster.getState().equals(KubernetesCluster.State.Alert) + || cluster.getState().equals(KubernetesCluster.State.Error) + || cluster.getState().equals(KubernetesCluster.State.Destroying))) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Cannot perform delete operation on cluster:" + cluster.getName() + " in state " + cluster.getState()); + } + throw new PermissionDeniedException("Cannot perform delete operation on cluster: " + cluster.getName() + " in state " + cluster.getState()); + } + + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.DestroyRequested); + + boolean failedVmDestroy = false; + List clusterVMs = kubernetesClusterVmMapDao.listByClusterId(cluster.getId()); + if ((clusterVMs != null) && !clusterVMs.isEmpty()) { + for (KubernetesClusterVmMapVO clusterVM : clusterVMs) { + long vmID = clusterVM.getVmId(); + + // delete only if VM exists and is not removed + UserVmVO userVM = userVmDao.findById(vmID); + if (userVM == null || userVM.isRemoved()) { + continue; + } + + try { + UserVm vm = userVmService.destroyVm(vmID, true); + if (!VirtualMachine.State.Expunging.equals(vm.getState())) { + LOGGER.warn(String.format("VM '%s' with uuid '%s' should have been expunging by now but is '%s'... retrying..." + , vm.getInstanceName() + , vm.getUuid() + , vm.getState().toString())); + } + vm = userVmService.expungeVm(vmID); + if (!VirtualMachine.State.Expunging.equals(vm.getState())) { + LOGGER.error(String.format("VM '%s' is now in state '%s'. I will probably fail at deleting it's cluster." + , vm.getInstanceName() + , vm.getState().toString())); + } + kubernetesClusterVmMapDao.expunge(clusterVM.getId()); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Destroyed VM: " + userVM.getInstanceName() + " as part of cluster: " + cluster.getName() + " destroy."); + } + } catch (Exception e) { + failedVmDestroy = true; + LOGGER.warn("Failed to destroy VM :" + userVM.getInstanceName() + " part of the cluster: " + cluster.getName() + + " due to " + e); + LOGGER.warn("Moving on with destroying remaining resources provisioned for the cluster: " + cluster.getName()); + } + } + } + boolean cleanupNetwork = true; + try { + final KubernetesClusterDetailsVO clusterDetails = kubernetesClusterDetailsDao.findDetail(kubernetesClusterId, "networkCleanup"); + cleanupNetwork = Boolean.parseBoolean(clusterDetails.getValue()); + } catch (Exception e) {} + + // if there are VM's that were not expunged, we can not delete the network + if (!failedVmDestroy) { + if (cleanupNetwork) { + if(clusterVMs!=null && !clusterVMs.isEmpty()) { // Wait for few seconds to get all VMs really expunged + final int maxRetries = 3; + int retryCounter = 0; + while (retryCounter < maxRetries) { + boolean allVMsRemoved = true; + for (KubernetesClusterVmMap clusterVM : clusterVMs) { + UserVmVO userVM = userVmDao.findById(clusterVM.getVmId()); + if (userVM != null && !userVM.isRemoved()) { + allVMsRemoved = false; + break; + } + } + if (allVMsRemoved) { + break; + } + try { + Thread.sleep(10000); + } catch (InterruptedException ie) {} + retryCounter++; + } + } + NetworkVO network = null; + try { + network = networkDao.findById(cluster.getNetworkId()); + if (network != null && network.getRemoved() == null) { + Account owner = accountManager.getAccount(network.getAccountId()); + User callerUser = accountManager.getActiveUser(CallContext.current().getCallingUserId()); + ReservationContext context = new ReservationContextImpl(null, null, callerUser, owner); + boolean networkDestroyed = networkMgr.destroyNetwork(cluster.getNetworkId(), context, true); + if (!networkDestroyed) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Failed to destroy network: " + cluster.getNetworkId() + + " as part of cluster: " + cluster.getName() + " destroy"); + } + processFailedNetworkDelete(kubernetesClusterId); + throw new ManagementServerException("Failed to delete the network as part of kubernetes cluster name:" + cluster.getName() + " clean up"); + } + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Destroyed network: " + network.getName() + " as part of cluster: " + cluster.getName() + " destroy"); + } + } + } catch (Exception e) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Failed to destroy network: " + cluster.getNetworkId() + + " as part of cluster: " + cluster.getName() + " destroy due to " + e); + } + processFailedNetworkDelete(kubernetesClusterId); + throw new ManagementServerException("Failed to delete the network as part of kubernetes cluster name:" + cluster.getName() + " clean up"); + } + } + } else { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("There are VM's that are not expunged in kubernetes cluster " + cluster.getName()); + } + processFailedNetworkDelete(kubernetesClusterId); + throw new ManagementServerException("Failed to destroy one or more VM's as part of kubernetes cluster name:" + cluster.getName() + " clean up"); + } + + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationSucceeded); + + cluster = kubernetesClusterDao.findById(kubernetesClusterId); + cluster.setCheckForGc(false); + kubernetesClusterDao.update(cluster.getId(), cluster); + + kubernetesClusterDao.remove(cluster.getId()); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Kubernetes cluster name:" + cluster.getName() + " is successfully deleted"); + } + + return true; + } + + private void processFailedNetworkDelete(long kubernetesClusterId) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + KubernetesClusterVO cluster = kubernetesClusterDao.findById(kubernetesClusterId); + cluster.setCheckForGc(true); + kubernetesClusterDao.update(cluster.getId(), cluster); + } + + private UserVm createKubernetesMaster(final KubernetesClusterVO kubernetesCluster, final List ips) throws ManagementServerException, + ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException { + + UserVm masterVm = null; + + DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); + ServiceOffering serviceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); + VirtualMachineTemplate template = templateDao.findById(kubernetesCluster.getTemplateId()); + + List networkIds = new ArrayList(); + networkIds.add(kubernetesCluster.getNetworkId()); + + Account owner = accountDao.findById(kubernetesCluster.getAccountId()); + + final String masterIp = ipAddressManager.acquireGuestIpAddress(networkDao.findById(kubernetesCluster.getNetworkId()), null); + Network.IpAddresses addrs = new Network.IpAddresses(masterIp, null); + + long rootDiskSize = kubernetesCluster.getNodeRootDiskSize(); + + Map customParameterMap = new HashMap(); + if (rootDiskSize > 0) { + customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); + } + + String hostName = kubernetesCluster.getName() + "-k8s-master"; + + String k8sMasterConfig = null; + try { + k8sMasterConfig = readResourceFile("/conf/k8s-master.yml"); + + final String apiServerCert = "{{ k8s_master.apiserver.crt }}"; + final String apiServerKey = "{{ k8s_master.apiserver.key }}"; + final String caCert = "{{ k8s_master.ca.crt }}"; + final String msSshPubKey = "{{ k8s_master.ms.ssh.pub.key }}"; + final String clusterToken = "{{ k8s_master.cluster.token }}"; + final String clusterIp = "{{ k8s_master.cluster.ip }}"; + + + final List addresses = new ArrayList<>(); + addresses.add(masterIp); + for (final IPAddressVO ip : ips) { + addresses.add(ip.getAddress().addr()); + } + + final Certificate certificate = caManager.issueCertificate(null, Arrays.asList(hostName, "kubernetes", + "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.default.svc.cluster.local"), + addresses, 3650, null); + + final String tlsClientCert = CertUtils.x509CertificateToPem(certificate.getClientCertificate()); + final String tlsPrivateKey = CertUtils.privateKeyToPem(certificate.getPrivateKey()); + final String tlsCaCert = CertUtils.x509CertificatesToPem(certificate.getCaCertificates()); + + k8sMasterConfig = k8sMasterConfig.replace(apiServerCert, tlsClientCert.replace("\n", "\n ")); + k8sMasterConfig = k8sMasterConfig.replace(apiServerKey, tlsPrivateKey.replace("\n", "\n ")); + k8sMasterConfig = k8sMasterConfig.replace(caCert, tlsCaCert.replace("\n", "\n ")); + + String pubKey = "- \"" + globalConfigDao.getValue("ssh.publickey") + "\""; + + String sshKeyPair = kubernetesCluster.getKeyPair(); + if (!Strings.isNullOrEmpty(sshKeyPair)) { + SSHKeyPairVO sshkp = sshKeyPairDao.findByName(owner.getAccountId(), owner.getDomainId(), sshKeyPair); + if (sshkp != null) { + pubKey += "\n - \"" + sshkp.getPublicKey() + "\""; + } + } + k8sMasterConfig = k8sMasterConfig.replace(msSshPubKey, pubKey); + + k8sMasterConfig = k8sMasterConfig.replace(clusterToken, generateClusterToken(kubernetesCluster)); + k8sMasterConfig = k8sMasterConfig.replace(clusterIp, String.format("--apiserver-cert-extra-sans=%s", ips.get(0).getAddress().toString())); + } catch (RuntimeException e) { + LOGGER.error("Failed to read kubernetes master configuration file due to " + e); + throw new ManagementServerException("Failed to read kubernetes master configuration file", e); + } catch (Exception e) { + LOGGER.error("Failed to read kubernetes master configuration file due to " + e); + throw new ManagementServerException("Failed to read kubernetes master configuration file", e); + } + + String base64UserData = Base64.encodeBase64String(k8sMasterConfig.getBytes(Charset.forName("UTF-8"))); + masterVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, + hostName, kubernetesCluster.getDescription(), null, null, null, + null, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), + null, addrs, null, null, null, customParameterMap, null, null, null, null); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Created master VM: " + hostName + " in the kubernetes cluster: " + kubernetesCluster.getName()); + } + + return masterVm; + } + + private UserVm createKubernetesNode(KubernetesClusterVO kubernetesCluster, String masterIp, int nodeInstance) throws ManagementServerException, + ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException { + + UserVm nodeVm = null; + + DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); + ServiceOffering serviceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); + VirtualMachineTemplate template = templateDao.findById(kubernetesCluster.getTemplateId()); + + List networkIds = new ArrayList(); + networkIds.add(kubernetesCluster.getNetworkId()); + + Account owner = accountDao.findById(kubernetesCluster.getAccountId()); + + Network.IpAddresses addrs = new Network.IpAddresses(null, null); + + long rootDiskSize = kubernetesCluster.getNodeRootDiskSize(); + + Map customParameterMap = new HashMap(); + if (rootDiskSize > 0) { + customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); + } + + String hostName = kubernetesCluster.getName() + "-k8s-node-" + String.valueOf(nodeInstance); + + String k8sNodeConfig = null; + try { + k8sNodeConfig = readResourceFile("/conf/k8s-node.yml"); + String masterIPString = "{{ k8s_master.default_ip }}"; + final String clusterTokenString = "{{ k8s_master.cluster.token }}"; + + k8sNodeConfig = k8sNodeConfig.replace(masterIPString, masterIp); + k8sNodeConfig = k8sNodeConfig.replace(clusterTokenString, generateClusterToken(kubernetesCluster)); + + /* genarate /.docker/config.json file on the nodes only if kubernetes cluster is created to + * use docker private registry */ + String dockerUserName = null; + String dockerPassword = null; + String dockerRegistryUrl = null; + String dockerRegistryEmail = null; + List details = kubernetesClusterDetailsDao.listDetails(kubernetesCluster.getId()); + for (KubernetesClusterDetailsVO detail : details) { + if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_USER_NAME)) { + dockerUserName = detail.getValue(); + } + if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_PASSWORD)) { + dockerPassword = detail.getValue(); + } + if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_URL)) { + dockerRegistryUrl = detail.getValue(); + } + if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_EMAIL)) { + dockerRegistryEmail = detail.getValue(); + } + } + if (!Strings.isNullOrEmpty(dockerUserName) && !Strings.isNullOrEmpty(dockerPassword)) { + // do write file for /.docker/config.json through the code instead of k8s-node.yml as we can no make a section + // optional or conditionally applied + String dockerConfigString = "write-files:\n" + + " - path: /.docker/config.json\n" + + " owner: core:core\n" + + " permissions: '0644'\n" + + " content: |\n" + + " {\n" + + " \"auths\": {\n" + + " {{docker.url}}: {\n" + + " \"auth\": {{docker.secret}},\n" + + " \"email\": {{docker.email}}\n" + + " }\n" + + " }\n" + + " }"; + k8sNodeConfig = k8sNodeConfig.replace("write-files:", dockerConfigString); + String dockerUrl = "{{docker.url}}"; + String dockerAuth = "{{docker.secret}}"; + String dockerEmail = "{{docker.email}}"; + String usernamePassword = dockerUserName + ":" + dockerPassword; + String base64Auth = Base64.encodeBase64String(usernamePassword.getBytes(Charset.forName("UTF-8"))); + k8sNodeConfig = k8sNodeConfig.replace(dockerUrl, "\"" + dockerRegistryUrl + "\""); + k8sNodeConfig = k8sNodeConfig.replace(dockerAuth, "\"" + base64Auth + "\""); + k8sNodeConfig = k8sNodeConfig.replace(dockerEmail, "\"" + dockerRegistryEmail + "\""); + } + } catch (Exception e) { + LOGGER.warn("Failed to read node configuration file due to " + e); + throw new ManagementServerException("Failed to read cluster node configuration file.", e); + } + + String base64UserData = Base64.encodeBase64String(k8sNodeConfig.getBytes(Charset.forName("UTF-8"))); + + nodeVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, + hostName, kubernetesCluster.getDescription(), null, null, null, + null, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), + null, addrs, null, null, null, customParameterMap, null, null, null, null); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Created cluster node VM: " + hostName + " in the kubernetes cluster: " + kubernetesCluster.getName()); + } + + return nodeVm; + } + + private void startKubernetesVM(final UserVm vm, final KubernetesClusterVO kubernetesCluster) throws ServerApiException { + + try { + StartVMCmd startVm = new StartVMCmd(); + startVm = ComponentContext.inject(startVm); + Field f = startVm.getClass().getDeclaredField("id"); + f.setAccessible(true); + f.set(startVm, vm.getId()); + userVmService.startVirtualMachine(startVm); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Started VM in the kubernetes cluster: " + kubernetesCluster.getName()); + } + } catch (ConcurrentOperationException ex) { + LOGGER.warn("Failed to start VM in the kubernetes cluster name:" + kubernetesCluster.getName() + " due to Exception: ", ex); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to start VM in the kubernetes cluster name:" + kubernetesCluster.getName(), ex); + } catch (ResourceUnavailableException ex) { + LOGGER.warn("Failed to start VM in the kubernetes cluster name:" + kubernetesCluster.getName() + " due to Exception: ", ex); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to start VM in the kubernetes cluster name:" + kubernetesCluster.getName(), ex); + } catch (InsufficientCapacityException ex) { + LOGGER.warn("Failed to start VM in the kubernetes cluster name:" + kubernetesCluster.getName() + " due to Exception: ", ex); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to start VM in the kubernetes cluster name:" + kubernetesCluster.getName(), ex); + } catch (RuntimeException ex) { + LOGGER.warn("Failed to start VM in the kubernetes cluster name:" + kubernetesCluster.getName() + " due to Exception: ", ex); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to start VM in the kubernetes cluster name:" + kubernetesCluster.getName(), ex); + } catch (Exception ex) { + LOGGER.warn("Failed to start VM in the kubernetes cluster name:" + kubernetesCluster.getName() + " due to Exception: ", ex); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to start VM in the kubernetes cluster name:" + kubernetesCluster.getName(), ex); + } + + UserVm startVm = userVmDao.findById(vm.getId()); + if (!startVm.getState().equals(VirtualMachine.State.Running)) { + LOGGER.warn("Failed to start VM instance."); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to start VM instance in kubernetes cluster " + kubernetesCluster.getName()); + } + } + + private void attachIsoKubernetesVMs(long kubernetesClusterId, List clusterVMIds) throws ServerApiException { + KubernetesCluster kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + KubernetesSupportedVersion version = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); + if (version == null) { + LOGGER.error(String .format("Unable to find Kubernetes version for cluster ID: %s", kubernetesCluster.getUuid())); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String .format("Unable to find Kubernetes version for cluster ID: %s", kubernetesCluster.getUuid())); + } + VMTemplateVO iso = templateDao.findById(version.getIsoId()); + if (iso == null) { + LOGGER.error(String.format("Unable to attach ISO to Kubernetes cluster ID: %s. Binaries ISO not found.", kubernetesCluster.getUuid())); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Unable to attach ISO to Kubernetes cluster ID: %s. Binaries ISO not found.", kubernetesCluster.getUuid())); + } + if (!iso.getFormat().equals(Storage.ImageFormat.ISO)) { + LOGGER.error(String.format("Unable to attach ISO to Kubernetes cluster ID: %s. Invalid Binaries ISO.", kubernetesCluster.getUuid())); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Unable to attach ISO to Kubernetes cluster ID: %s. Invalid Binaries ISO.", kubernetesCluster.getUuid())); + } + if (!iso.getState().equals(VirtualMachineTemplate.State.Active)) { + LOGGER.error(String.format("Unable to attach ISO to Kubernetes cluster ID: %s. Binaries ISO not active.", kubernetesCluster.getUuid())); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Unable to attach ISO to Kubernetes cluster ID: %s. Binaries ISO not active.", kubernetesCluster.getUuid())); + } + for (int i = 0; i < clusterVMIds.size(); ++i) { + UserVm vm = userVmDao.findById(clusterVMIds.get(i)); + try { + templateService.attachIso(iso.getId(), vm.getId()); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Attached binaries ISO for VM: %s in cluster: %s", vm.getUuid(), kubernetesCluster.getName())); + } + } catch (CloudRuntimeException ex) { + LOGGER.warn(String.format("Failed to attach binaries ISO for VM: %s in the kubernetes cluster name: %s due to Exception: ", vm.getDisplayName(), kubernetesCluster.getName()), ex); + // throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to attach binaries ISO for VM: %s in the kubernetes cluster name: %s", vm.getDisplayName(), kubernetesCluster.getName()), ex); + } + } + } + + private void detachIsoKubernetesVMs(long kubernetesClusterId, List clusterVMIds) throws ServerApiException { + KubernetesCluster kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + for (int i = 0; i < clusterVMIds.size(); ++i) { + UserVm vm = userVmDao.findById(clusterVMIds.get(i)); + + try { + templateService.detachIso(vm.getId()); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Started VM in the kubernetes cluster: " + kubernetesCluster.getName()); + } + } catch (CloudRuntimeException ex) { + LOGGER.warn(String.format("Failed to detach binaries ISO for VM: %s in the kubernetes cluster name: %s due to Exception: ", vm.getDisplayName(), kubernetesCluster.getName()), ex); + // throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to attach binaries ISO for VM: %s in the kubernetes cluster name: %s", vm.getDisplayName(), kubernetesCluster.getName()), ex); + } + } + } + + private void stopClusterVM(final KubernetesClusterVmMapVO vmMapVO) throws ServerApiException { + try { + userVmService.stopVirtualMachine(vmMapVO.getVmId(), false); + } catch (ConcurrentOperationException ex) { + LOGGER.warn("Failed to stop kubernetes cluster VM due to Exception: ", ex); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); + } + } + + @Override + public boolean scaleKubernetesCluster(ScaleKubernetesClusterCmd cmd) throws ManagementServerException, ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException { + final long kubernetesClusterId = cmd.getId(); + final Long serviceOfferingId = cmd.getServiceOfferingId(); + final Long clusterSize = cmd.getClusterSize(); + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + if (kubernetesCluster == null) { + throw new InvalidParameterValueException("Failed to find kubernetes cluster id: " + kubernetesClusterId); + } + + Account caller = CallContext.current().getCallingAccount(); + + accountManager.checkAccess(caller, SecurityChecker.AccessType.OperateEntry, false, kubernetesCluster); + + if (serviceOfferingId == null && clusterSize == null) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster id: %s cannot be scaled, either a new service offering or a new cluster size must be passed", kubernetesCluster.getUuid())); + } + + ServiceOffering serviceOffering = null; + if (serviceOfferingId != null) { + serviceOffering = serviceOfferingDao.findById(serviceOfferingId); + if (serviceOffering == null) { + throw new InvalidParameterValueException("Failed to find service offering id: " + serviceOfferingId); + } else { + if (serviceOffering.isDynamic()) { + throw new InvalidParameterValueException(String.format("Custom service offerings are not supported for kubernetes clusters. Kubernetes cluster ID: %s, service offering ID: %s", kubernetesCluster.getUuid(), serviceOffering.getUuid())); + } + if (serviceOffering.getCpu() < 2 || serviceOffering.getRamSize() < 2048) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be scaled with service offering ID: %s, kubernetes cluster template(CoreOS) needs minimum 2 vCPUs and 2 GB RAM", kubernetesCluster.getUuid(), serviceOffering.getUuid())); + } + } + } + + if (kubernetesCluster.getRemoved() != null) { + throw new ManagementServerException("Kubernetes cluster id:" + kubernetesCluster.getUuid() + " is already deleted."); + } + + if (!(kubernetesCluster.getState().equals(KubernetesCluster.State.Created) || + kubernetesCluster.getState().equals(KubernetesCluster.State.Running) || + kubernetesCluster.getState().equals(KubernetesCluster.State.Stopped))) { + throw new PermissionDeniedException(String.format("Kubernetes cluster id: %s is in %s state", kubernetesCluster.getUuid(), kubernetesCluster.getState().toString())); + } + + if (clusterSize != null) { + if (kubernetesCluster.getState().equals(KubernetesCluster.State.Stopped)) { // Cannot scale stopped cluster currently for cluster size + throw new PermissionDeniedException(String.format("Kubernetes cluster id: %s is in %s state", kubernetesCluster.getUuid(), kubernetesCluster.getState().toString())); + } + if (clusterSize < 1) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster id: %s cannot be scaled for size, %d", kubernetesCluster.getUuid(), clusterSize)); + } + } + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Scaling kubernetes cluster: " + kubernetesCluster.getName()); + } + + final KubernetesCluster.State clusterState = kubernetesCluster.getState(); + final long originalNodeCount = kubernetesCluster.getNodeCount(); + + final ServiceOffering existingServiceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); + if (existingServiceOffering == null) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, service offering for the kubernetes cluster not found!", kubernetesCluster.getUuid())); + } + final boolean serviceOfferingScalingNeeded = serviceOffering != null && serviceOffering.getId() != existingServiceOffering.getId(); + final boolean clusterSizeScalingNeeded = clusterSize != null && clusterSize != originalNodeCount; + if (serviceOfferingScalingNeeded) { + List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + if (vmList == null || vmList.isEmpty() || vmList.size() - 1 < originalNodeCount) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, it is in unstable state as not enough existing VM instances found!", kubernetesCluster.getUuid())); + } else { + for (KubernetesClusterVmMapVO vmMapVO : vmList) { + VMInstanceVO vmInstance = vmInstanceDao.findById(vmMapVO.getVmId()); + if (vmInstance != null && vmInstance.getState().equals(VirtualMachine.State.Running) && vmInstance.getHypervisorType() != Hypervisor.HypervisorType.XenServer && vmInstance.getHypervisorType() != Hypervisor.HypervisorType.VMware && vmInstance.getHypervisorType() != Hypervisor.HypervisorType.Simulator) { + LOGGER.info("Scaling the VM dynamically is not supported for VMs running on Hypervisor " + vmInstance.getHypervisorType()); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, scaling kubernetes cluster with running VMs on hypervisor %s is not supported!", kubernetesCluster.getUuid(), vmInstance.getHypervisorType())); + } + } + } + if (serviceOffering.getRamSize() < existingServiceOffering.getRamSize() || + serviceOffering.getCpu()*serviceOffering.getSpeed() < existingServiceOffering.getCpu()*existingServiceOffering.getSpeed()) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, service offering for the kubernetes cluster cannot be scaled down!", kubernetesCluster.getUuid())); + } + + // ToDo: Check capacity with new service offering at this point, how? + + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.ScaleUpRequested); + + final long size = (clusterSize == null ? kubernetesCluster.getNodeCount() : clusterSize); + final long cores = serviceOffering.getCpu() * size; + final long memory = serviceOffering.getRamSize() * size; + kubernetesCluster = Transaction.execute(new TransactionCallback() { + @Override + public KubernetesClusterVO doInTransaction(TransactionStatus status) { + KubernetesClusterVO updatedCluster = kubernetesClusterDao.createForUpdate(kubernetesClusterId); + updatedCluster.setNodeCount(size); + updatedCluster.setCores(cores); + updatedCluster.setMemory(memory); + updatedCluster.setServiceOfferingId(serviceOfferingId); + kubernetesClusterDao.persist(updatedCluster); + return updatedCluster; + } + }); + if (kubernetesCluster == null) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, unable to update kubernetes cluster!", kubernetesCluster.getUuid())); + } + kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + final long tobeScaledVMCount = Math.min(vmList.size(), size+1); + for (long i=0; i()); + } catch (Exception e) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, unable to scale cluster VM ID: %s! %s", kubernetesCluster.getUuid(), userVM.getUuid(), e.getMessage()), e); + } + if (!result) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, unable to scale cluster VM ID: %s!", kubernetesCluster.getUuid(), userVM.getUuid())); + } + } + } + + if (clusterSizeScalingNeeded) { + // Check capacity and transition state + final long newVmRequiredCount = clusterSize - originalNodeCount; + final ServiceOffering clusterServiceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); + if (clusterServiceOffering == null) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, service offering for the kubernetes cluster not found!", kubernetesCluster.getUuid())); + } + if (newVmRequiredCount > 0) { + if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.ScaleUpRequested); + } + try { + if (clusterState.equals(KubernetesCluster.State.Running)) { + plan(newVmRequiredCount, kubernetesCluster.getZoneId(), clusterServiceOffering); + } else { + plan(kubernetesCluster.getNodeCount() + newVmRequiredCount, kubernetesCluster.getZoneId(), clusterServiceOffering); + } + } catch (InsufficientCapacityException e) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + LOGGER.warn("Scaling the cluster failed due to insufficient capacity in the kubernetes cluster: " + kubernetesCluster.getName() + " due to " + e); + throw new ServerApiException(ApiErrorCode.INSUFFICIENT_CAPACITY_ERROR, "Provisioning the cluster failed due to insufficient capacity in the kubernetes cluster: " + kubernetesCluster.getName(), e); + } + } else { + if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.ScaleDownRequested); + } + } + + if (!serviceOfferingScalingNeeded) { // Else already updated + // Update KubernetesClusterVO + final long cores = clusterServiceOffering.getCpu() * clusterSize; + final long memory = clusterServiceOffering.getRamSize() * clusterSize; + + kubernetesCluster = Transaction.execute(new TransactionCallback() { + @Override + public KubernetesClusterVO doInTransaction(TransactionStatus status) { + KubernetesClusterVO updatedCluster = kubernetesClusterDao.createForUpdate(kubernetesClusterId); + updatedCluster.setNodeCount(clusterSize); + updatedCluster.setCores(cores); + updatedCluster.setMemory(memory); + kubernetesClusterDao.persist(updatedCluster); + return updatedCluster; + } + }); + if (kubernetesCluster == null) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, unable to update kubernetes cluster!", kubernetesCluster.getUuid())); + } + kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + } + + // Perform size scaling + if (clusterState.equals(KubernetesCluster.State.Running)) { + List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + if (vmList == null || vmList.isEmpty() || vmList.size() - 1 < originalNodeCount) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, it is in unstable state as not enough existing VM instances found!", kubernetesCluster.getUuid())); + } + IPAddressVO publicIp = null; + List ips = ipAddressDao.listByAssociatedNetwork(kubernetesCluster.getNetworkId(), true); + if (ips == null || ips.isEmpty() || ips.get(0) == null) { + LOGGER.warn("Network:" + kubernetesCluster.getNetworkId() + " for the kubernetes cluster name:" + kubernetesCluster.getName() + " does not have " + + "public IP's associated with it. So aborting kubernetes cluster scaling."); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, unable to connect the network!", kubernetesCluster.getUuid())); + } + publicIp = ips.get(0); + Boolean devel = Boolean.valueOf(globalConfigDao.getValue("developer")); + String keyFile = String.format("%s/.ssh/id_rsa", System.getProperty("user.home")); + if (devel) { + keyFile += ".cloud"; + } + File pkFile = new File(keyFile); + Account account = accountDao.findById(kubernetesCluster.getAccountId()); + if (newVmRequiredCount < 0) { // downscale + int i = vmList.size() - 1; + while (i > 1 && vmList.size() > clusterSize + 1) { // Reverse order as first VM will be k8s master + KubernetesClusterVmMapVO vmMapVO = vmList.get(i); + UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId()); + + // Gracefully remove-delete k8s node + int retryCounter = 0; + int maxRetries = 3; + while (retryCounter < maxRetries) { + retryCounter++; + try { + Pair result = SshHelper.sshExecute(publicIp.getAddress().addr(), 2222, "core", + pkFile, null, String.format("sudo kubectl drain %s --ignore-daemonsets --delete-local-data", userVM.getHostName()), + 10000, 10000, 30000); + if (!result.first()) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Draining kubernetes node unsuccessful"); + } + result = SshHelper.sshExecute(publicIp.getAddress().addr(), 2222, "core", + pkFile, null, String.format("sudo kubectl delete node %s", userVM.getHostName()), + 10000, 10000, 30000); + if (!result.first()) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Deleting kubernetes node unsuccessful"); + } + break; + } catch (Exception e) { + if (retryCounter < maxRetries) { + try { + Thread.sleep(30000); + } catch (InterruptedException ie) {} + } else { + LOGGER.warn("Failed to remove kubernetes node while scaling kubernetes cluster with ID " + kubernetesCluster.getUuid() + ": " + e); + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, unable to remove kubernetes node!" + e, kubernetesCluster.getUuid())); + } + } + } + + // Remove port-forwarding network rules + List pfRules = portForwardingRulesDao.listByNetwork(kubernetesCluster.getNetworkId()); + for (PortForwardingRuleVO pfRule : pfRules) { + if (pfRule.getVirtualMachineId() == userVM.getId()) { + portForwardingRulesDao.remove(pfRule.getId()); + break; + } + } + rulesService.applyPortForwardingRules(publicIp.getId(), account); + + // Expunge VM + UserVm vm = userVmService.destroyVm(userVM.getId(), true); + if (!VirtualMachine.State.Expunging.equals(vm.getState())) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, VM '%s' is now in state '%s'." + , kubernetesCluster.getUuid() + , vm.getInstanceName() + , vm.getState().toString())); + } + vm = userVmService.expungeVm(userVM.getId()); + if (!VirtualMachine.State.Expunging.equals(vm.getState())) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, VM '%s' is now in state '%s'." + , kubernetesCluster.getUuid() + , vm.getInstanceName() + , vm.getState().toString())); + } + + // Expunge cluster VMMapVO + kubernetesClusterVmMapDao.expunge(vmMapVO.getId()); + + i--; + } + + // Scale network rules to update firewall rule + try { + scaleKubernetesClusterNetworkRules(publicIp, account, kubernetesClusterId, null); + } catch (Exception e) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, unable to update network rules!", kubernetesCluster.getUuid()), e); + } + } else { // upscale, same node count handled above + UserVmVO masterVm = userVmDao.findById(vmList.get(0).getVmId()); + String masterIP = masterVm.getPrivateIpAddress(); + List clusterVMIds = new ArrayList<>(); + + // Create new node VMs + for (int i = (int) originalNodeCount + 1; i <= clusterSize; i++) { + UserVm vm = null; + try { + vm = createKubernetesNode(kubernetesCluster, masterIP, i); + final long nodeVmId = vm.getId(); + KubernetesClusterVmMapVO clusterNodeVmMap = Transaction.execute(new TransactionCallback() { + @Override + public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { + KubernetesClusterVmMapVO newClusterVmMap = new KubernetesClusterVmMapVO(kubernetesClusterId, nodeVmId); + kubernetesClusterVmMapDao.persist(newClusterVmMap); + return newClusterVmMap; + } + }); + startKubernetesVM(vm, kubernetesCluster); + clusterVMIds.add(vm.getId()); + + vm = userVmDao.findById(vm.getId()); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Provisioned a node VM in to the kubernetes cluster: " + kubernetesCluster.getName()); + } + } catch (Exception e) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, provisioning the node VM failed in the kubernetes cluster!", kubernetesCluster.getUuid()), e); + } + } + + // Scale network rules to update firewall rule and add port-forwarding rules + try { + scaleKubernetesClusterNetworkRules(publicIp, account, kubernetesClusterId, clusterVMIds); + } catch (Exception e) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, unable to update network rules!", kubernetesCluster.getUuid()), e); + } + + // Attach binaries ISO to new VMs + attachIsoKubernetesVMs(kubernetesClusterId, clusterVMIds); + + // Check if new nodes are added in k8s cluster + int retryCounter = 0; + int maxRetries = 30*4; // Max wait for 30 mins as online install can take time, same as while creating cluster + while (retryCounter < maxRetries) { + try { + Pair result = SshHelper.sshExecute(publicIp.getAddress().addr(), 2222, "core", + pkFile, null, "sudo kubectl get nodes -o json | jq \".items[].metadata.name\" | wc -l", + 20000, 10000, 30000); + if (result.first()) { + int nodesCount = 0; + try { + nodesCount = Integer.parseInt(result.second().trim()); + } catch (Exception e) { + } + if (nodesCount == kubernetesCluster.getNodeCount() + 1) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("All nodes kubernetes cluster: " + kubernetesCluster.getName() + " are ready now, scaling successful!"); + } + break; + } + } + } catch (Exception e) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Waiting for kubernetes cluster: " + kubernetesCluster.getName() + " API endpoint to be available. retry: " + retryCounter + "/" + maxRetries); + } + } + try { + Thread.sleep(15000); + } catch (InterruptedException ex) { + } + retryCounter++; + } + + // Detach binaries ISO from new VMs + detachIsoKubernetesVMs(kubernetesClusterId, clusterVMIds); + + // Throw exception if nodes count for k8s cluster timed out + if (retryCounter >= maxRetries) { // Scaling failed + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Scaling unsuccessful for kubernetes cluster: " + kubernetesCluster.getName() + ". Kubernetes cluster does not have desired number of nodes in ready state."); + } + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, unable to have desired number of nodes in ready state!", kubernetesCluster.getUuid())); + } + } + } + } + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationSucceeded); + return true; + } + + @Override + public ListResponse listKubernetesClusters(ListKubernetesClustersCmd cmd) { + CallContext ctx = CallContext.current(); + Account caller = ctx.getCallingAccount(); + + ListResponse response = new ListResponse(); + + List responsesList = new ArrayList(); + SearchCriteria sc = kubernetesClusterDao.createSearchCriteria(); + + String state = cmd.getState(); + if (state != null && !state.isEmpty()) { + if (!KubernetesCluster.State.Running.toString().equals(state) && + !KubernetesCluster.State.Stopped.toString().equals(state) && + !KubernetesCluster.State.Destroyed.toString().equals(state)) { + throw new InvalidParameterValueException("Invalid value for cluster state is specified"); + } + } + + if (cmd.getId() != null) { + KubernetesClusterVO cluster = kubernetesClusterDao.findById(cmd.getId()); + if (cluster == null) { + throw new InvalidParameterValueException("Invalid cluster id specified"); + } + accountManager.checkAccess(caller, SecurityChecker.AccessType.ListEntry, false, cluster); + responsesList.add(createKubernetesClusterResponse(cmd.getId())); + } else { + Filter searchFilter = new Filter(KubernetesClusterVO.class, "id", true, cmd.getStartIndex(), cmd.getPageSizeVal()); + + if (state != null && !state.isEmpty()) { + sc.addAnd("state", SearchCriteria.Op.EQ, state); + } + + if (accountManager.isNormalUser(caller.getId())) { + sc.addAnd("accountId", SearchCriteria.Op.EQ, caller.getAccountId()); + } else if (accountManager.isDomainAdmin(caller.getId())) { + sc.addAnd("domainId", SearchCriteria.Op.EQ, caller.getDomainId()); + } + + String name = cmd.getName(); + if (name != null && !name.isEmpty()) { + sc.addAnd("name", SearchCriteria.Op.LIKE, name); + } + + List kubernetesClusters = kubernetesClusterDao.search(sc, searchFilter); + for (KubernetesClusterVO cluster : kubernetesClusters) { + KubernetesClusterResponse clusterReponse = createKubernetesClusterResponse(cluster.getId()); + responsesList.add(clusterReponse); + } + } + response.setResponses(responsesList); + return response; + } + + public KubernetesClusterConfigResponse getKubernetesClusterConfig(GetKubernetesClusterConfigCmd cmd) { + KubernetesClusterConfigResponse response = new KubernetesClusterConfigResponse(); + KubernetesCluster kubernetesCluster = kubernetesClusterDao.findById(cmd.getId()); + if (kubernetesCluster != null) { + response.setId(kubernetesCluster.getUuid()); + response.setName(kubernetesCluster.getName()); + String configData = ""; + try { + configData = new String(Base64.decodeBase64(kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "kubeConfigData").getValue())); + } catch (Exception e) {} + } + response.setObjectName("clusterconfig"); + return response; + } + + public KubernetesClusterResponse createKubernetesClusterResponse(long kubernetesClusterId) { + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + KubernetesClusterResponse response = new KubernetesClusterResponse(); + response.setObjectName("kubernetescluster"); + response.setId(kubernetesCluster.getUuid()); + response.setName(kubernetesCluster.getName()); + response.setDescription(kubernetesCluster.getDescription()); + DataCenterVO zone = ApiDBUtils.findZoneById(kubernetesCluster.getZoneId()); + response.setZoneId(zone.getUuid()); + response.setZoneName(zone.getName()); + response.setClusterSize(String.valueOf(kubernetesCluster.getNodeCount())); + VMTemplateVO template = ApiDBUtils.findTemplateById(kubernetesCluster.getTemplateId()); + response.setTemplateId(template.getUuid()); + ServiceOfferingVO offering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); + response.setServiceOfferingId(offering.getUuid()); + response.setServiceOfferingName(offering.getName()); + response.setKeypair(kubernetesCluster.getKeyPair()); + response.setState(kubernetesCluster.getState().toString()); + response.setCores(String.valueOf(kubernetesCluster.getCores())); + response.setMemory(String.valueOf(kubernetesCluster.getMemory())); + NetworkVO ntwk = networkDao.findByIdIncludingRemoved(kubernetesCluster.getNetworkId()); + response.setEndpoint(kubernetesCluster.getEndpoint()); + response.setNetworkId(ntwk.getUuid()); + response.setAssociatedNetworkName(ntwk.getName()); + response.setConsoleEndpoint(kubernetesCluster.getConsoleEndpoint()); + List vmIds = new ArrayList(); + List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + if (vmList != null && !vmList.isEmpty()) { + for (KubernetesClusterVmMapVO vmMapVO : vmList) { + UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId()); + if (userVM != null) { + vmIds.add(userVM.getUuid()); + } + } + } + response.setVirtualMachineIds(vmIds); + return response; + } + + private String readResourceFile(String resource) throws IOException { + InputStream in = Thread.currentThread().getContextClassLoader().getResourceAsStream(resource); + return IOUtils.toString(Thread.currentThread().getContextClassLoader().getResourceAsStream(resource), Charset.defaultCharset().name()); + } + + protected boolean stateTransitTo(long kubernetesClusterId, KubernetesCluster.Event e) { + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + try { + return _stateMachine.transitTo(kubernetesCluster, e, null, kubernetesClusterDao); + } catch (NoTransitionException nte) { + LOGGER.warn("Failed to transistion state of the kubernetes cluster: " + kubernetesCluster.getName() + + " in state " + kubernetesCluster.getState().toString() + " on event " + e.toString()); + return false; + } + } + + private static String getStackTrace(final Throwable throwable) { + final StringWriter sw = new StringWriter(); + final PrintWriter pw = new PrintWriter(sw, true); + throwable.printStackTrace(pw); + return sw.getBuffer().toString(); + } + + private boolean isKubernetesServiceConfigured(DataCenter zone) { + + String templateName = globalConfigDao.getValue(KubernetesServiceConfig.KubernetesClusterTemplateName.key()); + if (templateName == null || templateName.isEmpty()) { + LOGGER.warn("Global setting " + KubernetesServiceConfig.KubernetesClusterTemplateName.key() + " is empty." + + "Template name need to be specified, for kubernetes service to function."); + return false; + } + + final VMTemplateVO template = templateDao.findByTemplateName(templateName); + if (template == null) { + LOGGER.warn("Unable to find the template:" + templateName + " to be used for provisioning cluster"); + return false; + } + + String networkOfferingName = globalConfigDao.getValue(KubernetesServiceConfig.KubernetesClusterNetworkOffering.key()); + if (networkOfferingName == null || networkOfferingName.isEmpty()) { + LOGGER.warn("global setting " + KubernetesServiceConfig.KubernetesClusterNetworkOffering.key() + " is empty. " + + "Admin has not yet specified the network offering to be used for provisioning isolated network for the cluster."); + return false; + } + + NetworkOfferingVO networkOffering = networkOfferingDao.findByUniqueName(networkOfferingName); + if (networkOffering == null) { + LOGGER.warn("Network offering with name :" + networkOfferingName + " specified by admin is not found."); + return false; + } + + if (networkOffering.getState() == NetworkOffering.State.Disabled) { + LOGGER.warn("Network offering :" + networkOfferingName + "is not enabled."); + return false; + } + + List services = networkOfferingServiceMapDao.listServicesForNetworkOffering(networkOffering.getId()); + if (services == null || services.isEmpty() || !services.contains("SourceNat")) { + LOGGER.warn("Network offering :" + networkOfferingName + " does not have necessary services to provision kubernetes cluster"); + return false; + } + + if (!networkOffering.isEgressDefaultPolicy()) { + LOGGER.warn("Network offering :" + networkOfferingName + "has egress default policy turned off should be on to provision kubernetes cluster."); + return false; + } + + long physicalNetworkId = networkModel.findPhysicalNetworkId(zone.getId(), networkOffering.getTags(), networkOffering.getTrafficType()); + PhysicalNetwork physicalNetwork = physicalNetworkDao.findById(physicalNetworkId); + if (physicalNetwork == null) { + LOGGER.warn("Unable to find physical network with id: " + physicalNetworkId + " and tag: " + networkOffering.getTags()); + return false; + } + + return true; + } + + @Override + public List> getCommands() { + List> cmdList = new ArrayList>(); + cmdList.add(CreateKubernetesClusterCmd.class); + cmdList.add(StartKubernetesClusterCmd.class); + cmdList.add(StopKubernetesClusterCmd.class); + cmdList.add(DeleteKubernetesClusterCmd.class); + cmdList.add(ListKubernetesClustersCmd.class); + cmdList.add(GetKubernetesClusterConfigCmd.class); + cmdList.add(ScaleKubernetesClusterCmd.class); + return cmdList; + } + + // Garbage collector periodically run through the kubernetes clusters marked for GC. For each kubernetes cluster + // marked for GC, attempt is made to destroy cluster. + public class KubernetesClusterGarbageCollector extends ManagedContextRunnable { + @Override + protected void runInContext() { + GlobalLock gcLock = GlobalLock.getInternLock("KubernetesCluster.GC.Lock"); + try { + if (gcLock.lock(3)) { + try { + reallyRun(); + } finally { + gcLock.unlock(); + } + } + } finally { + gcLock.releaseRef(); + } + } + + public void reallyRun() { + try { + List kubernetesClusters = kubernetesClusterDao.findKubernetesClustersToGarbageCollect(); + for (KubernetesCluster kubernetesCluster : kubernetesClusters) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Running kubernetes cluster garbage collector on kubernetes cluster name:" + kubernetesCluster.getName()); + } + try { + if (cleanupKubernetesClusterResources(kubernetesCluster.getId())) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Kubernetes cluster: " + kubernetesCluster.getName() + " is successfully garbage collected"); + } + } else { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Kubernetes cluster: " + kubernetesCluster.getName() + " failed to get" + + " garbage collected. Will be attempted to garbage collected in next run"); + } + } + } catch (RuntimeException e) { + LOGGER.debug("Faied to destroy kubernetes cluster name:" + kubernetesCluster.getName() + " during GC due to " + e); + // proceed furhter with rest of the kubernetes cluster garbage collection + } catch (Exception e) { + LOGGER.debug("Faied to destroy kubernetes cluster name:" + kubernetesCluster.getName() + " during GC due to " + e); + // proceed furhter with rest of the kubernetes cluster garbage collection + } + } + } catch (Exception e) { + LOGGER.warn("Caught exception while running kubernetes cluster gc: ", e); + } + } + } + + /* Kubernetes cluster scanner checks if the kubernetes cluster is in desired state. If it detects kubernetes cluster + is not in desired state, it will trigger an event and marks the kubernetes cluster to be 'Alert' state. For e.g a + kubernetes cluster in 'Running' state should mean all the cluster of node VM's in the custer should be running and + number of the node VM's should be of cluster size, and the master node VM's is running. It is possible due to + out of band changes by user or hosts going down, we may end up one or more VM's in stopped state. in which case + scanner detects these changes and marks the cluster in 'Alert' state. Similarly cluster in 'Stopped' state means + all the cluster VM's are in stopped state any mismatch in states should get picked up by kubernetes cluster and + mark the kubernetes cluster to be 'Alert' state. Through recovery API, or reconciliation clusters in 'Alert' will + be brought back to known good state or desired state. + */ + public class KubernetesClusterStatusScanner extends ManagedContextRunnable { + @Override + protected void runInContext() { + GlobalLock gcLock = GlobalLock.getInternLock("KubernetesCluster.State.Scanner.Lock"); + try { + if (gcLock.lock(3)) { + try { + reallyRun(); + } finally { + gcLock.unlock(); + } + } + } finally { + gcLock.releaseRef(); + } + } + + public void reallyRun() { + try { + + // run through kubernetes clusters in 'Running' state and ensure all the VM's are Running in the cluster + List runningKubernetesClusters = kubernetesClusterDao.findKubernetesClustersInState(KubernetesCluster.State.Running); + for (KubernetesCluster kubernetesCluster : runningKubernetesClusters) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Running kubernetes cluster state scanner on kubernetes cluster name:" + kubernetesCluster.getName()); + } + try { + if (!isClusterInDesiredState(kubernetesCluster, VirtualMachine.State.Running)) { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.FaultsDetected); + } + } catch (Exception e) { + LOGGER.warn("Failed to run through VM states of kubernetes cluster due to " + e); + } + } + + // run through kubernetes clusters in 'Stopped' state and ensure all the VM's are Stopped in the cluster + List stoppedKubernetesClusters = kubernetesClusterDao.findKubernetesClustersInState(KubernetesCluster.State.Stopped); + for (KubernetesCluster kubernetesCluster : stoppedKubernetesClusters) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Running kubernetes cluster state scanner on kubernetes cluster name:" + kubernetesCluster.getName() + " for state " + KubernetesCluster.State.Stopped); + } + try { + if (!isClusterInDesiredState(kubernetesCluster, VirtualMachine.State.Stopped)) { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.FaultsDetected); + } + } catch (Exception e) { + LOGGER.warn("Failed to run through VM states of kubernetes cluster due to " + e); + } + } + + // run through kubernetes clusters in 'Alert' state and reconcile state as 'Running' if the VM's are running + List alertKubernetesClusters = kubernetesClusterDao.findKubernetesClustersInState(KubernetesCluster.State.Alert); + for (KubernetesCluster kubernetesCluster : alertKubernetesClusters) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Running kubernetes cluster state scanner on kubernetes cluster name:" + kubernetesCluster.getName() + " for state " + KubernetesCluster.State.Alert); + } + try { + if (isClusterInDesiredState(kubernetesCluster, VirtualMachine.State.Running)) { + // mark the cluster to be running + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.RecoveryRequested); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); + } + } catch (Exception e) { + LOGGER.warn("Failed to run through VM states of kubernetes cluster status scanner due to " + e); + } + } + + } catch (RuntimeException e) { + LOGGER.warn("Caught exception while running kubernetes cluster state scanner.", e); + } catch (Exception e) { + LOGGER.warn("Caught exception while running kubernetes cluster state scanner.", e); + } + } + } + + // checks if kubernetes cluster is in desired state + boolean isClusterInDesiredState(KubernetesCluster kubernetesCluster, VirtualMachine.State state) { + List clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + + // check if all the VM's are in same state + for (KubernetesClusterVmMapVO clusterVm : clusterVMs) { + VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(clusterVm.getVmId()); + if (vm.getState() != state) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Found VM in the kubernetes cluster: " + kubernetesCluster.getName() + + " in state: " + vm.getState().toString() + " while expected to be in state: " + state.toString() + + " So moving the cluster to Alert state for reconciliation."); + } + return false; + } + } + + // check cluster is running at desired capacity include master node as well, so count should be cluster size + 1 + if (clusterVMs.size() != (kubernetesCluster.getNodeCount() + 1)) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Found only " + clusterVMs.size() + " VM's in the kubernetes cluster: " + kubernetesCluster.getName() + + " in state: " + state.toString() + " While expected number of VM's to " + + " be in state: " + state.toString() + " is " + (kubernetesCluster.getNodeCount() + 1) + + " So moving the cluster to Alert state for reconciliation."); + } + return false; + } + return true; + } + + @Override + public boolean start() { + _gcExecutor.scheduleWithFixedDelay(new KubernetesClusterGarbageCollector(), 300, 300, TimeUnit.SECONDS); + _stateScanner.scheduleWithFixedDelay(new KubernetesClusterStatusScanner(), 300, 30, TimeUnit.SECONDS); + + return true; + } + + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + _name = name; + _configParams = params; + _gcExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("Kubernetes-Cluster-Scavenger")); + _stateScanner = Executors.newScheduledThreadPool(1, new NamedThreadFactory("Kubernetes-Cluster-State-Scanner")); + + return true; + } + + private String generateClusterToken(KubernetesCluster kubernetesCluster) { + if (kubernetesCluster == null) return ""; + String token = kubernetesCluster.getUuid(); + token = token.replaceAll("-", ""); + token = token.substring(0, 22); + token = token.substring(0, 6) + "." + token.substring(6); + return token; + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterService.java new file mode 100644 index 000000000000..48b26178c802 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterService.java @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetescluster; + +import org.apache.cloudstack.api.command.user.kubernetescluster.CreateKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetescluster.GetKubernetesClusterConfigCmd; +import org.apache.cloudstack.api.command.user.kubernetescluster.ListKubernetesClustersCmd; +import org.apache.cloudstack.api.command.user.kubernetescluster.ScaleKubernetesClusterCmd; +import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse; +import org.apache.cloudstack.api.response.KubernetesClusterResponse; +import org.apache.cloudstack.api.response.ListResponse; + +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.ManagementServerException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.utils.component.PluggableService; + +public interface KubernetesClusterService extends PluggableService { + + KubernetesCluster findById(final Long id); + + KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) throws InsufficientCapacityException, + ResourceAllocationException, ManagementServerException; + + boolean startKubernetesCluster(long kubernetesClusterId, boolean onCreate) throws ManagementServerException, + ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException; + + boolean stopKubernetesCluster(long kubernetesClusterId) throws ManagementServerException; + + boolean deleteKubernetesCluster(Long kubernetesClusterId) throws ManagementServerException; + + ListResponse listKubernetesClusters(ListKubernetesClustersCmd cmd); + + KubernetesClusterConfigResponse getKubernetesClusterConfig(GetKubernetesClusterConfigCmd cmd); + + KubernetesClusterResponse createKubernetesClusterResponse(long kubernetesClusterId); + + boolean scaleKubernetesCluster(ScaleKubernetesClusterCmd cmd) throws ManagementServerException, + ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException; + +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVO.java new file mode 100644 index 000000000000..80a046e7bec1 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVO.java @@ -0,0 +1,330 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetescluster; + +import java.util.Date; +import java.util.UUID; + + +import javax.persistence.Column; + +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; + +import com.cloud.utils.db.GenericDao; + +@Entity +@Table(name = "kubernetes_cluster") +public class KubernetesClusterVO implements KubernetesCluster { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "uuid") + private String uuid; + + @Column(name = "name") + private String name; + + @Column(name = "description", length = 4096) + private String description; + + @Column(name = "zone_id") + private long zoneId; + + @Column(name = "kubernetes_version_id") + private long kubernetesVersionId; + + @Column(name = "service_offering_id") + private long serviceOfferingId; + + @Column(name = "template_id") + private long templateId; + + @Column(name = "network_id") + private long networkId; + + @Column(name = "domain_id") + private long domainId; + + @Column(name = "account_id") + private long accountId; + + @Column(name = "node_count") + private long nodeCount; + + @Column(name = "cores") + private long cores; + + @Column(name = "memory") + private long memory; + + @Column(name = "node_root_disk_size") + private long nodeRootDiskSize; + + @Column(name = "state") + private State state; + + @Column(name = "key_pair") + private String keyPair; + + @Column(name = "endpoint") + private String endpoint; + + @Column(name = "console_endpoint") + private String consoleEndpoint; + + @Column(name = GenericDao.CREATED_COLUMN) + private Date created; + + @Column(name = GenericDao.REMOVED_COLUMN) + private Date removed; + + @Column(name = "gc") + private boolean checkForGc; + + @Override + public long getId() { + return id; + } + + public void setId(long id) { + this.id = id; + } + + @Override + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + @Override + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + @Override + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + @Override + public long getZoneId() { + return zoneId; + } + + public void setZoneId(long zoneId) { + this.zoneId = zoneId; + } + + @Override + public long getKubernetesVersionId() { + return kubernetesVersionId; + } + + public void setKubernetesVersionId(long kubernetesVersionId) { + this.kubernetesVersionId = kubernetesVersionId; + } + + @Override + public long getServiceOfferingId() { + return serviceOfferingId; + } + + public void setServiceOfferingId(long serviceOfferingId) { + this.serviceOfferingId = serviceOfferingId; + } + + @Override + public long getTemplateId() { + return templateId; + } + + public void setTemplateId(long templateId) { + this.templateId = templateId; + } + + @Override + public long getNetworkId() { + return networkId; + } + + public void setNetworkId(long networkId) { + this.networkId = networkId; + } + + @Override + public long getDomainId() { + return domainId; + } + + public void setDomainId(long domainId) { + this.domainId = domainId; + } + + @Override + public long getAccountId() { + return accountId; + } + + public void setAccountId(long accountId) { + this.accountId = accountId; + } + + @Override + public long getNodeCount() { + return nodeCount; + } + + public void setNodeCount(long nodeCount) { + this.nodeCount = nodeCount; + } + + @Override + public long getCores() { + return cores; + } + + public void setCores(long cores) { + this.cores = cores; + } + + @Override + public long getMemory() { + return memory; + } + + public void setMemory(long memory) { + this.memory = memory; + } + + @Override + public long getNodeRootDiskSize() { + return nodeRootDiskSize; + } + + public void setNodeRootDiskSize(long nodeRootDiskSize) { + this.nodeRootDiskSize = nodeRootDiskSize; + } + + @Override + public State getState() { + return state; + } + + public void setState(State state) { + this.state = state; + } + + @Override + public String getEndpoint() { + return endpoint; + } + + public void setEndpoint(String endpoint) { + this.endpoint = endpoint; + } + + public String getKeyPair() { + return keyPair; + } + + public void setKeyPair(String keyPair) { + this.keyPair = keyPair; + } + + @Override + public String getConsoleEndpoint() { + return consoleEndpoint; + } + + public void setConsoleEndpoint(String consoleEndpoint) { + this.consoleEndpoint = consoleEndpoint; + } + + @Override + public boolean isDisplay() { + return true; + } + + + public Date getRemoved() { + if (removed == null) + return null; + return new Date(removed.getTime()); + } + + @Override + public boolean isCheckForGc() { + return checkForGc; + } + + public void setCheckForGc(boolean check) { + checkForGc = check; + } + + public KubernetesClusterVO() { + + } + + public KubernetesClusterVO(String name, String description, long zoneId, long kubernetesVersionId, long serviceOfferingId, long templateId, + long networkId, long domainId, long accountId, long nodeCount, State state, + String keyPair, long cores, long memory, Long nodeRootDiskSize, String endpoint, String consoleEndpoint) { + this.uuid = UUID.randomUUID().toString(); + this.name = name; + this.description = description; + this.zoneId = zoneId; + this.kubernetesVersionId = kubernetesVersionId; + this.serviceOfferingId = serviceOfferingId; + this.templateId = templateId; + this.networkId = networkId; + this.domainId = domainId; + this.accountId = accountId; + this.nodeCount = nodeCount; + this.state = state; + this.keyPair = keyPair; + this.cores = cores; + this.memory = memory; + if (nodeRootDiskSize != null && nodeRootDiskSize > 0) { + this.nodeRootDiskSize = nodeRootDiskSize; + } + this.endpoint = endpoint; + this.consoleEndpoint = consoleEndpoint; + this.checkForGc = false; + } + + @Override + public Class getEntityType() { + return KubernetesCluster.class; + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVmMap.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVmMap.java new file mode 100644 index 000000000000..d2d56f2c902a --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVmMap.java @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetescluster; + +/** + * VirtualMachine describes the properties held by a virtual machine + * + */ +public interface KubernetesClusterVmMap { + long getId(); + long getClusterId(); + long getVmId(); +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVmMapVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVmMapVO.java new file mode 100644 index 000000000000..64d0bba3fc86 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVmMapVO.java @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetescluster; + +import javax.persistence.Column; + +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; + +@Entity +@Table(name = "kubernetes_cluster_vm_map") +public class KubernetesClusterVmMapVO implements KubernetesClusterVmMap { + + @Override + public long getId() { + return id; + } + + @Override + public long getClusterId() { + return clusterId; + + } + + public void setClusterId(long clusterId) { + + this.clusterId = clusterId; + } + + @Override + public long getVmId() { + return vmId; + } + + public void setVmId(long vmId) { + + this.vmId = vmId; + } + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + long id; + + @Column(name = "cluster_id") + long clusterId; + + @Column(name = "vm_id") + long vmId; + + public KubernetesClusterVmMapVO() { + + } + + public KubernetesClusterVmMapVO(long clusterId, long vmId) { + this.vmId = vmId; + this.clusterId = clusterId; + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesServiceConfig.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesServiceConfig.java new file mode 100644 index 000000000000..2c4e2330d7cd --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesServiceConfig.java @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetescluster; + +import com.cloud.server.ManagementServer; + +public enum KubernetesServiceConfig { + + KubernetesClusterTemplateName("Advanced", ManagementServer.class, String.class, "cloud.kubernetes.cluster.template.name", "Kubernetes-Service-Template", "Name of the template to be used for creating Kubernetes cluster nodes", null, null), + KubernetesClusterNetworkOffering("Advanced", ManagementServer.class, String.class, "cloud.kubernetes.cluster.network.offering", "DefaultNetworkOfferingforKubernetesService", "Name of the network offering that will be used to create isolated network in which Kubernetes cluster VMs will be launched.", null, null), + KubernetesClusterBinariesIsoName("Advanced", ManagementServer.class, String.class, "cloud.kubernetes.cluster.binaries.iso.name", "Kubernetes-Service-Binaries-ISO", "Name of the ISO that contains Kubernetes binaries and docker images for offline installation.", null, null); + + + private final String _category; + private final Class _componentClass; + private final Class _type; + private final String _name; + private final String _defaultValue; + private final String _description; + private final String _range; + private final String _scope; + + private KubernetesServiceConfig(String category, Class componentClass, Class type, String name, String defaultValue, String description, String range, String scope) { + _category = category; + _componentClass = componentClass; + _type = type; + _name = name; + _defaultValue = defaultValue; + _description = description; + _range = range; + _scope = scope; + } + + public String getCategory() { + return _category; + } + + public String key() { + return _name; + } + + public String getDescription() { + return _description; + } + + public String getDefaultValue() { + return _defaultValue; + } + + public Class getType() { + return _type; + } + + public Class getComponentClass() { + return _componentClass; + } + + public String getScope() { + return _scope; + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDao.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDao.java new file mode 100644 index 000000000000..3192c94cd67a --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDao.java @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetescluster.dao; + +import java.util.List; + +import com.cloud.kubernetescluster.KubernetesCluster; +import com.cloud.kubernetescluster.KubernetesClusterVO; +import com.cloud.utils.db.GenericDao; +import com.cloud.utils.fsm.StateDao; + +public interface KubernetesClusterDao extends GenericDao, + StateDao { + + List listByAccount(long accountId); + List findKubernetesClustersToGarbageCollect(); + List findKubernetesClustersInState(KubernetesCluster.State state); + List listByNetworkId(long networkId); + List listAllByKubernetesVersion(long kubernetesVersionId); +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDaoImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDaoImpl.java new file mode 100644 index 000000000000..4a0fd1897315 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDaoImpl.java @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetescluster.dao; + +import java.util.List; + +import org.springframework.stereotype.Component; + +import com.cloud.kubernetescluster.KubernetesCluster; +import com.cloud.kubernetescluster.KubernetesCluster.Event; +import com.cloud.kubernetescluster.KubernetesClusterVO; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.TransactionLegacy; + +@Component +public class KubernetesClusterDaoImpl extends GenericDaoBase implements KubernetesClusterDao { + + private final SearchBuilder AccountIdSearch; + private final SearchBuilder GarbageCollectedSearch; + private final SearchBuilder StateSearch; + private final SearchBuilder SameNetworkSearch; + private final SearchBuilder KubernetesVersionSearch; + + public KubernetesClusterDaoImpl() { + AccountIdSearch = createSearchBuilder(); + AccountIdSearch.and("account", AccountIdSearch.entity().getAccountId(), SearchCriteria.Op.EQ); + AccountIdSearch.done(); + + GarbageCollectedSearch = createSearchBuilder(); + GarbageCollectedSearch.and("gc", GarbageCollectedSearch.entity().isCheckForGc(), SearchCriteria.Op.EQ); + GarbageCollectedSearch.and("state", GarbageCollectedSearch.entity().getState(), SearchCriteria.Op.NEQ); + GarbageCollectedSearch.done(); + + StateSearch = createSearchBuilder(); + StateSearch.and("state", StateSearch.entity().getState(), SearchCriteria.Op.EQ); + StateSearch.done(); + + SameNetworkSearch = createSearchBuilder(); + SameNetworkSearch.and("network_id", SameNetworkSearch.entity().getNetworkId(), SearchCriteria.Op.EQ); + SameNetworkSearch.done(); + + KubernetesVersionSearch = createSearchBuilder(); + KubernetesVersionSearch.and("kubernetesVersionId", KubernetesVersionSearch.entity().getKubernetesVersionId(), SearchCriteria.Op.EQ); + KubernetesVersionSearch.done(); + } + + @Override + public List listByAccount(long accountId) { + SearchCriteria sc = AccountIdSearch.create(); + sc.setParameters("account", accountId); + return listBy(sc, null); + } + + @Override + public List findKubernetesClustersToGarbageCollect() { + SearchCriteria sc = GarbageCollectedSearch.create(); + sc.setParameters("gc", true); + sc.setParameters("state", KubernetesCluster.State.Destroying); + return listBy(sc); + } + + @Override + public List findKubernetesClustersInState(KubernetesCluster.State state) { + SearchCriteria sc = StateSearch.create(); + sc.setParameters("state", state); + return listBy(sc); + } + + @Override + public boolean updateState(KubernetesCluster.State currentState, Event event, KubernetesCluster.State nextState, + KubernetesCluster vo, Object data) { + // TODO: ensure this update is correct + TransactionLegacy txn = TransactionLegacy.currentTxn(); + txn.start(); + + KubernetesClusterVO ccVo = (KubernetesClusterVO)vo; + ccVo.setState(nextState); + super.update(ccVo.getId(), ccVo); + + txn.commit(); + return true; + } + + @Override + public List listByNetworkId(long networkId) { + SearchCriteria sc = SameNetworkSearch.create(); + sc.setParameters("network_id", networkId); + return this.listBy(sc); + } + + @Override + public List listAllByKubernetesVersion(long kubernetesVersionId) { + SearchCriteria sc = SameNetworkSearch.create(); + sc.setParameters("kubernetesVersionId", kubernetesVersionId); + return this.listBy(sc); + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDetailsDao.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDetailsDao.java new file mode 100644 index 000000000000..109114f578ea --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDetailsDao.java @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetescluster.dao; + + +import org.apache.cloudstack.resourcedetail.ResourceDetailsDao; + +import com.cloud.kubernetescluster.KubernetesClusterDetailsVO; +import com.cloud.utils.db.GenericDao; + + +public interface KubernetesClusterDetailsDao extends GenericDao, ResourceDetailsDao { + +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDetailsDaoImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDetailsDaoImpl.java new file mode 100644 index 000000000000..fe751009e27b --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDetailsDaoImpl.java @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetescluster.dao; + +import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; +import org.springframework.stereotype.Component; + +import com.cloud.kubernetescluster.KubernetesClusterDetailsVO; + + +@Component +public class KubernetesClusterDetailsDaoImpl extends ResourceDetailsDaoBase implements KubernetesClusterDetailsDao { + + @Override + public void addDetail(long resourceId, String key, String value, boolean display) { + super.addDetail(new KubernetesClusterDetailsVO(resourceId, key, value, display)); + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterVmMapDao.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterVmMapDao.java new file mode 100644 index 000000000000..4b8143e91840 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterVmMapDao.java @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetescluster.dao; + +import com.cloud.kubernetescluster.KubernetesClusterVmMapVO; +import com.cloud.utils.db.GenericDao; + +import java.util.List; + +public interface KubernetesClusterVmMapDao extends GenericDao { + public List listByClusterId(long clusterId); +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterVmMapDaoImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterVmMapDaoImpl.java new file mode 100644 index 000000000000..f32474cf22f5 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterVmMapDaoImpl.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetescluster.dao; + +import java.util.List; + +import org.springframework.stereotype.Component; + +import com.cloud.kubernetescluster.KubernetesClusterVmMapVO; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + + +@Component +public class KubernetesClusterVmMapDaoImpl extends GenericDaoBase implements KubernetesClusterVmMapDao { + + private final SearchBuilder clusterIdSearch; + + public KubernetesClusterVmMapDaoImpl() { + clusterIdSearch = createSearchBuilder(); + clusterIdSearch.and("clusterId", clusterIdSearch.entity().getClusterId(), SearchCriteria.Op.EQ); + clusterIdSearch.done(); + } + + @Override + public List listByClusterId(long clusterId) { + SearchCriteria sc = clusterIdSearch.create(); + sc.setParameters("clusterId", clusterId); + return listBy(sc, null); + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesSupportedVersion.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesSupportedVersion.java new file mode 100644 index 000000000000..da74cd83b4ab --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesSupportedVersion.java @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.kubernetesversion; + +import org.apache.cloudstack.api.Identity; +import org.apache.cloudstack.api.InternalIdentity; + +/** + * KubernetesSupportedVersion describes the properties of supported kubernetes version + * + */ +public interface KubernetesSupportedVersion extends InternalIdentity, Identity { + long getId(); + String getName(); + long getIsoId(); + Long getZoneId(); +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesSupportedVersionVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesSupportedVersionVO.java new file mode 100644 index 000000000000..00a26a4481c3 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesSupportedVersionVO.java @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.kubernetesversion; + +import java.util.Date; +import java.util.UUID; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +import com.cloud.utils.db.GenericDao; + +@Entity +@Table(name = "kubernetes_supported_version") +public class KubernetesSupportedVersionVO implements KubernetesSupportedVersion { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "uuid") + private String uuid; + + @Column(name = "name") + private String name; + + @Column(name = "iso_id") + private long isoId; + + @Column(name = "zone_id") + private Long zoneId; + + @Column(name = GenericDao.CREATED_COLUMN) + Date created; + + @Column(name = GenericDao.REMOVED_COLUMN) + Date removed; + + public KubernetesSupportedVersionVO() { + this.uuid = UUID.randomUUID().toString(); + } + + public KubernetesSupportedVersionVO(String name, long isoId, Long zoneId) { + this.uuid = UUID.randomUUID().toString(); + this.name = name; + this.isoId = isoId; + this.zoneId = zoneId; + } + + @Override + public long getId() { + return id; + } + + @Override + public String getUuid() { + return uuid; + } + + @Override + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + @Override + public long getIsoId() { + return isoId; + } + + public void setIsoId(long isoId) { + this.isoId = isoId; + } + + @Override + public Long getZoneId() { + return zoneId; + } + + public void setZoneId(Long zoneId) { + this.zoneId = zoneId; + } + + public Date getCreated() { + return created; + } + + public Date getRemoved() { + return removed; + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionEventTypes.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionEventTypes.java new file mode 100644 index 000000000000..6f067e736035 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionEventTypes.java @@ -0,0 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.kubernetesversion; + +public class KubernetesVersionEventTypes { + public static final String EVENT_KUBERNETES_VERSION_ADD = "KUBERNETES.VERSION.ADD"; + public static final String EVENT_KUBERNETES_VERSION_DELETE = "KUBERNETES.VERSION.DELETE"; +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java new file mode 100644 index 000000000000..4870d58ec74a --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java @@ -0,0 +1,230 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.kubernetesversion; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; + +import javax.inject.Inject; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.admin.kubernetesversion.AddKubernetesSupportedVersionCmd; +import org.apache.cloudstack.api.command.admin.kubernetesversion.DeleteKubernetesSupportedVersionCmd; +import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; +import org.apache.cloudstack.api.command.user.iso.RegisterIsoCmd; +import org.apache.cloudstack.api.command.user.kubernetesversion.ListKubernetesSupportedVersionsCmd; +import org.apache.cloudstack.api.response.KubernetesSupportedVersionResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.log4j.Logger; + +import com.cloud.api.ApiDBUtils; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.event.ActionEvent; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.kubernetescluster.KubernetesClusterVO; +import com.cloud.kubernetescluster.dao.KubernetesClusterDao; +import com.cloud.kubernetesversion.dao.KubernetesSupportedVersionDao; +import com.cloud.storage.Storage; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VMTemplateZoneVO; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VMTemplateZoneDao; +import com.cloud.template.TemplateApiService; +import com.cloud.template.VirtualMachineTemplate; +import com.cloud.utils.component.ComponentContext; +import com.cloud.utils.component.ManagerBase; +import com.google.common.base.Strings; + +public class KubernetesVersionManagerImpl extends ManagerBase implements KubernetesVersionService { + public static final Logger LOGGER = Logger.getLogger(KubernetesVersionManagerImpl.class.getName()); + + @Inject + private KubernetesSupportedVersionDao kubernetesSupportedVersionDao; + @Inject + private KubernetesClusterDao kubernetesClusterDao; + @Inject + private VMTemplateDao templateDao; + @Inject + private VMTemplateZoneDao templateZoneDao; + @Inject + private DataCenterDao dataCenterDao; + @Inject + private TemplateApiService templateService; + + private KubernetesSupportedVersionResponse createKubernetesSupportedVersionResponse(final KubernetesSupportedVersion kubernetesSupportedVersion) { + KubernetesSupportedVersionResponse response = new KubernetesSupportedVersionResponse(); + response.setObjectName("kubernetessupportedversion"); + response.setId(kubernetesSupportedVersion.getUuid()); + response.setName(kubernetesSupportedVersion.getName()); + DataCenterVO zone = ApiDBUtils.findZoneById(kubernetesSupportedVersion.getZoneId()); + if (zone != null) { + response.setZoneId(zone.getUuid()); + response.setZoneName(zone.getName()); + } + VMTemplateVO template = ApiDBUtils.findTemplateById(kubernetesSupportedVersion.getIsoId()); + response.setIsoId(template.getUuid()); + response.setIsoName(template.getName()); + response.setIsoState(template.getState().toString()); + return response; + } + + @Override + public ListResponse listKubernetesSupportedVersions(final ListKubernetesSupportedVersionsCmd cmd) { + final Long versionId = cmd.getId(); + List responseList = new ArrayList<>(); + if (versionId != null) { + KubernetesSupportedVersionVO version = kubernetesSupportedVersionDao.findById(versionId); + if (version != null) { + responseList.add(createKubernetesSupportedVersionResponse(version)); + } + } else { + List versions = kubernetesSupportedVersionDao.listAll(); + for (KubernetesSupportedVersionVO version : versions) { + responseList.add(createKubernetesSupportedVersionResponse(version)); + } + } + ListResponse response = new ListResponse<>(); + response.setResponses(responseList); + return response; + } + + @Override + @ActionEvent(eventType = KubernetesVersionEventTypes.EVENT_KUBERNETES_VERSION_ADD, eventDescription = "Adding Kubernetes supported version") + public KubernetesSupportedVersionResponse addKubernetesSupportedVersion(final AddKubernetesSupportedVersionCmd cmd) { + final String name = cmd.getName(); + final Long zoneId = cmd.getZoneId(); + final Long isoId = cmd.getIsoId(); + final String isoUrl = cmd.getUrl(); + final String isoChecksum = cmd.getChecksum(); + if (Strings.isNullOrEmpty(name)) { + throw new InvalidParameterValueException("Name cannot be empty to add a new supported Kubernetes version"); + } + if (Strings.isNullOrEmpty(isoUrl) && (isoId == null || isoId <= 0)) { + throw new InvalidParameterValueException(String.format("Either %s or %s paramter must be passed to add a new supported Kubernetes version", "isourl", ApiConstants.ISO_ID)); + } + + if (!Strings.isNullOrEmpty(isoUrl) && isoId != null && isoId > 0) { + throw new InvalidParameterValueException(String.format("Both %s and %s parameters can not be passed simultaneously to add a new supported Kubernetes version", "isourl", ApiConstants.ISO_ID)); + } + + VMTemplateVO template = null; + if (isoId != null) { + template = templateDao.findById(isoId); + } + if (template == null) { // register new ISO + VirtualMachineTemplate vmTemplate = null; + try { + String isoName = String.format("%s-Kubernetes-Binaries-ISO", name); + RegisterIsoCmd registerIsoCmd = new RegisterIsoCmd(); + registerIsoCmd = ComponentContext.inject(registerIsoCmd); + Field f = registerIsoCmd.getClass().getDeclaredField("name"); + f.setAccessible(true); + f.set(registerIsoCmd, isoName); + f = registerIsoCmd.getClass().getDeclaredField("displayText"); + f.setAccessible(true); + f.set(registerIsoCmd, isoName); + f = registerIsoCmd.getClass().getDeclaredField("bootable"); + f.setAccessible(true); + f.set(registerIsoCmd, false); + f = registerIsoCmd.getClass().getDeclaredField("publicIso"); + f.setAccessible(true); + f.set(registerIsoCmd, true); + f = registerIsoCmd.getClass().getDeclaredField("url"); + f.setAccessible(true); + f.set(registerIsoCmd, isoUrl); + if (Strings.isNullOrEmpty(isoChecksum)) { + f = registerIsoCmd.getClass().getDeclaredField("checksum"); + f.setAccessible(true); + f.set(registerIsoCmd, isoChecksum); + } + vmTemplate = templateService.registerIso(registerIsoCmd); + } catch (Exception ex) { + LOGGER.error(String.format("Unable to register binaries ISO for supported kubernetes version, %s", name), ex); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Unable to register binaries ISO for supported kubernetes version, %s", name)); + } + template = templateDao.findById(vmTemplate.getId()); + } else { + if (!template.getFormat().equals(Storage.ImageFormat.ISO)) { + throw new InvalidParameterValueException(String.format("%s is not an ISO", template.getUuid())); + } + if (!template.isPublicTemplate()) { + throw new InvalidParameterValueException(String.format("ISO ID: %s is not public", template.getUuid())); + } + if (!template.isCrossZones() && zoneId == null) { + throw new InvalidParameterValueException(String.format("ISO ID: %s is not available across zones", template.getUuid())); + } + if (!template.isCrossZones() && zoneId != null) { + List templatesZoneVOs = templateZoneDao.listByZoneTemplate(zoneId, template.getId()); + if (templatesZoneVOs.isEmpty()) { + DataCenterVO zone = dataCenterDao.findById(zoneId); + throw new InvalidParameterValueException(String.format("ISO ID: %s is not available for zone ID: %s", template.getUuid(), zone.getUuid())); + } + } + } + KubernetesSupportedVersionVO supportedVersionVO = new KubernetesSupportedVersionVO(name, template.getId(), zoneId); + supportedVersionVO = kubernetesSupportedVersionDao.persist(supportedVersionVO); + return createKubernetesSupportedVersionResponse(supportedVersionVO); + } + + @Override + @ActionEvent(eventType = KubernetesVersionEventTypes.EVENT_KUBERNETES_VERSION_DELETE, eventDescription = "Deleting Kubernetes supported version", async = true) + public boolean deleteKubernetesSupportedVersion(final DeleteKubernetesSupportedVersionCmd cmd) { + final Long versionId = cmd.getId(); + final Boolean isDeleteIso = cmd.isDeleteIso(); + KubernetesSupportedVersion version = kubernetesSupportedVersionDao.findById(versionId); + if (version == null) { + throw new InvalidParameterValueException("Invalid Kubernetes version id specified"); + } + List clusters = kubernetesClusterDao.listAllByKubernetesVersion(versionId); + if (clusters.size() > 0) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Unable to delete Kubernetes version ID: %s. Exisiting clusters currently using the version.", version.getUuid())); + } + + VMTemplateVO template = templateDao.findById(version.getId()); + if (template == null) { + LOGGER.warn(String.format("Unable to find ISO associated with supported Kubernetes version ID: %s", version.getUuid())); + } + if (isDeleteIso && template != null) { // Delete ISO + try { + DeleteIsoCmd deleteIsoCmd = new DeleteIsoCmd(); + deleteIsoCmd = ComponentContext.inject(deleteIsoCmd); + Field f = deleteIsoCmd.getClass().getDeclaredField("id"); + f.setAccessible(true); + f.set(deleteIsoCmd, template.getId()); + templateService.deleteIso(deleteIsoCmd); + } catch (Exception ex) { + LOGGER.error(String.format("Unable to delete binaries ISO ID: %s associated with supported kubernetes version ID: %s", template.getUuid(), version.getUuid()), ex); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Unable to delete binaries ISO ID: %s associated with supported kubernetes version ID: %s", template.getUuid(), version.getUuid())); + } + } + return kubernetesSupportedVersionDao.remove(version.getId()); + } + + @Override + public List> getCommands() { + List> cmdList = new ArrayList>(); + cmdList.add(AddKubernetesSupportedVersionCmd.class); + cmdList.add(ListKubernetesSupportedVersionsCmd.class); + cmdList.add(DeleteKubernetesSupportedVersionCmd.class); + return cmdList; + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionService.java new file mode 100644 index 000000000000..3e0b978c2fea --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionService.java @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.kubernetesversion; + +import org.apache.cloudstack.api.command.admin.kubernetesversion.AddKubernetesSupportedVersionCmd; +import org.apache.cloudstack.api.command.admin.kubernetesversion.DeleteKubernetesSupportedVersionCmd; +import org.apache.cloudstack.api.command.user.kubernetesversion.ListKubernetesSupportedVersionsCmd; +import org.apache.cloudstack.api.response.KubernetesSupportedVersionResponse; +import org.apache.cloudstack.api.response.ListResponse; + +import com.cloud.utils.component.PluggableService; + +public interface KubernetesVersionService extends PluggableService { + ListResponse listKubernetesSupportedVersions(ListKubernetesSupportedVersionsCmd cmd); + KubernetesSupportedVersionResponse addKubernetesSupportedVersion(AddKubernetesSupportedVersionCmd cmd); + boolean deleteKubernetesSupportedVersion(DeleteKubernetesSupportedVersionCmd cmd); +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/dao/KubernetesSupportedVersionDao.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/dao/KubernetesSupportedVersionDao.java new file mode 100644 index 000000000000..551c3e30673c --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/dao/KubernetesSupportedVersionDao.java @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.kubernetesversion.dao; + +import java.util.List; + +import com.cloud.kubernetesversion.KubernetesSupportedVersionVO; +import com.cloud.utils.db.GenericDao; + +public interface KubernetesSupportedVersionDao extends GenericDao { + List listAllInZone(long dataCenterId); +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/dao/KubernetesSupportedVersionDaoImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/dao/KubernetesSupportedVersionDaoImpl.java new file mode 100644 index 000000000000..ab866a6ea58f --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/dao/KubernetesSupportedVersionDaoImpl.java @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.kubernetesversion.dao; + +import java.util.List; + +import com.cloud.kubernetesversion.KubernetesSupportedVersionVO; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchCriteria; + +public class KubernetesSupportedVersionDaoImpl extends GenericDaoBase implements KubernetesSupportedVersionDao { + public KubernetesSupportedVersionDaoImpl() { + } + + @Override + public List listAllInZone(long dataCenterId) { + SearchCriteria sc = createSearchCriteria(); + SearchCriteria scc = createSearchCriteria(); + scc.addOr("zoneId", SearchCriteria.Op.EQ, dataCenterId); + scc.addOr("zoneId", SearchCriteria.Op.NULL); + sc.addAnd("zoneId", SearchCriteria.Op.SC, scc); + return listBy(sc); + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/AddKubernetesSupportedVersionCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/AddKubernetesSupportedVersionCmd.java new file mode 100644 index 000000000000..2569f36d3670 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/AddKubernetesSupportedVersionCmd.java @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.kubernetesversion; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.user.UserCmd; +import org.apache.cloudstack.api.response.KubernetesSupportedVersionResponse; +import org.apache.cloudstack.api.response.TemplateResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.kubernetesversion.KubernetesSupportedVersion; +import com.cloud.kubernetesversion.KubernetesVersionService; + +@APICommand(name = AddKubernetesSupportedVersionCmd.APINAME, + description = "Add a supported Kubernetes version", + responseObject = KubernetesSupportedVersionResponse.class, + responseView = ResponseObject.ResponseView.Restricted, + entityType = {KubernetesSupportedVersion.class}, + authorized = {RoleType.Admin}) +public class AddKubernetesSupportedVersionCmd extends BaseCmd implements UserCmd { + public static final Logger LOGGER = Logger.getLogger(AddKubernetesSupportedVersionCmd.class.getName()); + public static final String APINAME = "addKubernetesSupportedVersion"; + + @Inject + private KubernetesVersionService kubernetesVersionService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, + description = "the name of the Kubernetes supported version") + private String name; + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, + entityType = ZoneResponse.class, + description = "the ID of the zone in which Kubernetes supported version will be available") + private Long zoneId; + + @Parameter(name = ApiConstants.ISO_ID, type = CommandType.UUID, + entityType = TemplateResponse.class, + description = "the ID of the binaries ISO for Kubernetes supported version") + private Long isoId; + + @Parameter(name = ApiConstants.URL, type = CommandType.STRING, + description = "the URL of the binaries ISO for Kubernetes supported version") + private String url; + + @Parameter(name = ApiConstants.CHECKSUM, type = CommandType.STRING, + description = "the checksum value of the binaries ISO. " + ApiConstants.CHECKSUM_PARAMETER_PREFIX_DESCRIPTION) + private String checksum; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + + public String getName() { + return name; + } + + public Long getZoneId() { + return zoneId; + } + + public Long getIsoId() { + return isoId; + } + + public String getUrl() { + return url; + } + + public String getChecksum() { + return checksum; + } + + @Override + public String getCommandName() { + return APINAME.toLowerCase() + "response"; + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccountId(); + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + KubernetesSupportedVersionResponse response = kubernetesVersionService.addKubernetesSupportedVersion(this); + response.setResponseName(getCommandName()); + setResponseObject(response); + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/DeleteKubernetesSupportedVersionCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/DeleteKubernetesSupportedVersionCmd.java new file mode 100644 index 000000000000..40c9157efd08 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/DeleteKubernetesSupportedVersionCmd.java @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.kubernetesversion; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.KubernetesSupportedVersionResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.kubernetesversion.KubernetesSupportedVersion; +import com.cloud.kubernetesversion.KubernetesVersionEventTypes; +import com.cloud.kubernetesversion.KubernetesVersionService; + +@APICommand(name = DeleteKubernetesSupportedVersionCmd.APINAME, + description = "Deletes a Kubernetes cluster", + responseObject = SuccessResponse.class, + entityType = {KubernetesSupportedVersion.class}, + authorized = {RoleType.Admin}) +public class DeleteKubernetesSupportedVersionCmd extends BaseAsyncCmd { + public static final Logger LOGGER = Logger.getLogger(DeleteKubernetesSupportedVersionCmd.class.getName()); + public static final String APINAME = "deleteKubernetesSupportedVersion"; + + @Inject + private KubernetesVersionService kubernetesVersionService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, + entityType = KubernetesSupportedVersionResponse.class, + description = "the ID of the Kubernetes supported version", + required = true) + private Long id; + @Parameter(name = "deleteIso", type = CommandType.BOOLEAN, + description = "true if ISO associated with the Kubernetes version to be deleted else false. Default is false") + private Boolean deleteIso; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + public Long getId() { + return id; + } + + public Boolean isDeleteIso() { + return deleteIso; + } + + @Override + public String getCommandName() { + return APINAME.toLowerCase() + "response"; + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccountId(); + } + + @Override + public String getEventType() { + return KubernetesVersionEventTypes.EVENT_KUBERNETES_VERSION_DELETE; + } + + @Override + public String getEventDescription() { + return "Deleting Kubernetes supported version " + getId(); + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + boolean result = kubernetesVersionService.deleteKubernetesSupportedVersion(this); + if (result) { + SuccessResponse response = new SuccessResponse(getCommandName()); + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to delete supported Kubernetes version"); + } + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/CreateKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/CreateKubernetesClusterCmd.java new file mode 100644 index 000000000000..bd083f31c7e1 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/CreateKubernetesClusterCmd.java @@ -0,0 +1,289 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.kubernetescluster; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.acl.SecurityChecker.AccessType; +import org.apache.cloudstack.api.ACL; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandJobType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCreateCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject.ResponseView; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.KubernetesClusterResponse; +import org.apache.cloudstack.api.response.KubernetesSupportedVersionResponse; +import org.apache.cloudstack.api.response.NetworkResponse; +import org.apache.cloudstack.api.response.ProjectResponse; +import org.apache.cloudstack.api.response.ServiceOfferingResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.ManagementServerException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.kubernetescluster.KubernetesCluster; +import com.cloud.kubernetescluster.KubernetesClusterEventTypes; +import com.cloud.kubernetescluster.KubernetesClusterService; + +@APICommand(name = CreateKubernetesClusterCmd.APINAME, + description = "Creates a Kubernetes cluster", + responseObject = KubernetesClusterResponse.class, + responseView = ResponseView.Restricted, + entityType = {KubernetesCluster.class}, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = true, + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { + public static final Logger LOGGER = Logger.getLogger(CreateKubernetesClusterCmd.class.getName()); + public static final String APINAME = "createKubernetesCluster"; + + @Inject + public KubernetesClusterService kubernetesClusterService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, description = "name for the Kubernetes cluster") + private String name; + + @Parameter(name = ApiConstants.DESCRIPTION, type = CommandType.STRING, description = "description for the Kubernetes cluster") + private String description; + + @ACL(accessType = AccessType.UseEntry) + @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, required = true, + description = "availability zone in which Kubernetes cluster to be launched") + private Long zoneId; + + @ACL(accessType = AccessType.UseEntry) + @Parameter(name = ApiConstants.KUBERNETES_VERSION_ID, type = CommandType.UUID, entityType = KubernetesSupportedVersionResponse.class, required = true, + description = "Kubernetes version with which cluster to be launched") + private Long kubernetesVersionId; + + @ACL(accessType = AccessType.UseEntry) + @Parameter(name = ApiConstants.SERVICE_OFFERING_ID, type = CommandType.UUID, entityType = ServiceOfferingResponse.class, + required = true, description = "the ID of the service offering for the virtual machines in the cluster.") + private Long serviceOfferingId; + + @ACL(accessType = AccessType.UseEntry) + @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "an optional account for the" + + " virtual machine. Must be used with domainId.") + private String accountName; + + @ACL(accessType = AccessType.UseEntry) + @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, + description = "an optional domainId for the virtual machine. If the account parameter is used, domainId must also be used.") + private Long domainId; + + @ACL(accessType = AccessType.UseEntry) + @Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, + description = "Deploy cluster for the project") + private Long projectId; + + @ACL(accessType = AccessType.UseEntry) + @Parameter(name = ApiConstants.NETWORK_ID, type = CommandType.UUID, entityType = NetworkResponse.class, + description = "Network in which Kubernetes cluster is to be launched") + private Long networkId; + + @ACL(accessType = AccessType.UseEntry) + @Parameter(name = ApiConstants.SSH_KEYPAIR, type = CommandType.STRING, + description = "name of the ssh key pair used to login to the virtual machines") + private String sshKeyPairName; + + @Parameter(name=ApiConstants.SIZE, type = CommandType.LONG, + required = true, description = "number of Kubernetes cluster nodes") + private Long clusterSize; + + @Parameter(name = ApiConstants.DOCKER_REGISTRY_USER_NAME, type = CommandType.STRING, + description = "user name for the docker image private registry") + private String dockerRegistryUserName; + + @Parameter(name = ApiConstants.DOCKER_REGISTRY_PASSWORD, type = CommandType.STRING, + description = "password for the docker image private registry") + private String dockerRegistryPassword; + + @Parameter(name = ApiConstants.DOCKER_REGISTRY_URL, type = CommandType.STRING, + description = "URL for the docker image private registry") + private String dockerRegistryUrl; + + @Parameter(name = ApiConstants.DOCKER_REGISTRY_EMAIL, type = CommandType.STRING, + description = "email of the docker image private registry user") + private String dockerRegistryEmail; + + @Parameter(name = ApiConstants.NODE_ROOT_DISK_SIZE, type = CommandType.LONG, + description = "root disk size of root disk for each node") + private Long nodeRootDiskSize; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public String getAccountName() { + if (accountName == null) { + return CallContext.current().getCallingAccount().getAccountName(); + } + return accountName; + } + + public String getDisplayName() { + return description; + } + + public Long getDomainId() { + if (domainId == null) { + return CallContext.current().getCallingAccount().getDomainId(); + } + return domainId; + } + + public Long getServiceOfferingId() { + return serviceOfferingId; + } + + public Long getZoneId() { + return zoneId; + } + + public Long getKubernetesVersionId() { + return kubernetesVersionId; + } + + public Long getNetworkId() { return networkId;} + + public String getName() { + return name; + } + + public String getSSHKeyPairName() { + return sshKeyPairName; + } + + public Long getClusterSize() { + return clusterSize; + } + + public String getDockerRegistryUserName() { + return dockerRegistryUserName; + } + + public String getDockerRegistryPassword() { + return dockerRegistryPassword; + } + + public String getDockerRegistryUrl() { + return dockerRegistryUrl; + } + + public String getDockerRegistryEmail() { + return dockerRegistryEmail; + } + + public Long getNodeRootDiskSize() { + return nodeRootDiskSize; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public String getCommandName() { + return APINAME.toLowerCase() + "response"; + } + + public static String getResultObjectName() { + return "kubernetescluster"; + } + + @Override + public long getEntityOwnerId() { + Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + if (accountId == null) { + return CallContext.current().getCallingAccount().getId(); + } + + return accountId; + } + + @Override + public String getEventType() { + return KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_CREATE; + } + + @Override + public String getCreateEventType() { + return KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_CREATE; + } + + @Override + public String getCreateEventDescription() { + return "creating Kubernetes cluster"; + } + + @Override + public String getEventDescription() { + return "creating Kubernetes cluster. Cluster Id: " + getEntityId(); + } + + @Override + public ApiCommandJobType getInstanceType() { + return ApiCommandJobType.VirtualMachine; + } + + @Override + public void execute() { + try { + kubernetesClusterService.startKubernetesCluster(getEntityId(), true); + KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(getEntityId()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (InsufficientCapacityException | ResourceUnavailableException | ResourceAllocationException ex) { + LOGGER.warn("Failed to deploy Kubernetes cluster:" + getEntityUuid() + " due to " + ex.getMessage()); + throw new ServerApiException(ApiErrorCode.RESOURCE_ALLOCATION_ERROR, + "Failed to deploy Kubernetes cluster:" + getEntityUuid(), ex); + } catch (ManagementServerException ex) { + LOGGER.warn("Failed to deploy Kubernetes cluster:" + getEntityUuid() + " due to " + ex.getMessage()); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, + "Failed to deploy Kubernetes cluster:" + getEntityUuid(), ex); + } + } + + @Override + public void create() throws ResourceAllocationException { + try { + KubernetesCluster cluster = kubernetesClusterService.createKubernetesCluster(this); + if (cluster != null) { + setEntityId(cluster.getId()); + setEntityUuid(cluster.getUuid()); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create Kubernetes cluster"); + } + } catch (ConcurrentOperationException | InsufficientCapacityException | ManagementServerException me) { + LOGGER.error("Exception: ", me); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, me.getMessage()); + } + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/DeleteKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/DeleteKubernetesClusterCmd.java new file mode 100644 index 000000000000..ef78cbefb3a4 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/DeleteKubernetesClusterCmd.java @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.kubernetescluster; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.KubernetesClusterResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +import com.cloud.kubernetescluster.KubernetesClusterEventTypes; +import com.cloud.kubernetescluster.KubernetesCluster; +import com.cloud.kubernetescluster.KubernetesClusterService; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; + +@APICommand(name = DeleteKubernetesClusterCmd.APINAME, + description = "Deletes a Kubernetes cluster", + responseObject = SuccessResponse.class, + entityType = {KubernetesCluster.class}, + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class DeleteKubernetesClusterCmd extends BaseAsyncCmd { + public static final Logger LOGGER = Logger.getLogger(DeleteKubernetesClusterCmd.class.getName()); + public static final String APINAME = "deleteKubernetesCluster"; + + @Inject + public KubernetesClusterService kubernetesClusterService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, + type = CommandType.UUID, + entityType = KubernetesClusterResponse.class, + required = true, + description = "the ID of the Kubernetes cluster") + private Long id; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + public KubernetesCluster validateRequest() { + if (getId() == null || getId() < 1L) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Invalid Kubernetes cluster ID provided"); + } + final KubernetesCluster kubernetesCluster = kubernetesClusterService.findById(getId()); + if (kubernetesCluster == null) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Given Kubernetes cluster was not found"); + } + return kubernetesCluster; + } + + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, + ServerApiException, ConcurrentOperationException, ResourceAllocationException, + NetworkRuleConflictException { + final KubernetesCluster kubernetesCluster = validateRequest(); + try { + kubernetesClusterService.deleteKubernetesCluster(id); + SuccessResponse response = new SuccessResponse(getCommandName()); + setResponseObject(response); + } catch (Exception e) { + LOGGER.warn("Failed to delete vm Kubernetes cluster due to " + e); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to delete Kubernetes cluster ID: %s. %s", kubernetesCluster.getUuid(), e.getMessage()), e); + } + } + + @Override + public String getCommandName() { + return APINAME.toLowerCase() + "response"; + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } + + + @Override + public String getEventType() { + return KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_DELETE; + } + + @Override + public String getEventDescription() { + return "Deleting Kubernetes cluster. Cluster Id: " + getId(); + } + +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/GetKubernetesClusterConfigCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/GetKubernetesClusterConfigCmd.java new file mode 100644 index 000000000000..0520aeaf4b0b --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/GetKubernetesClusterConfigCmd.java @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.kubernetescluster; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse; +import org.apache.cloudstack.api.response.KubernetesClusterResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +import com.cloud.kubernetescluster.KubernetesClusterService; +import com.cloud.user.Account; + + +@APICommand(name = GetKubernetesClusterConfigCmd.APINAME, + description = "Get Kubernetes cluster config", + responseObject = KubernetesClusterConfigResponse.class, + responseView = ResponseObject.ResponseView.Restricted, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = true, + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class GetKubernetesClusterConfigCmd extends BaseCmd { + public static final Logger LOGGER = Logger.getLogger(GetKubernetesClusterConfigCmd.class.getName()); + public static final String APINAME = "getKubernetesClusterConfig"; + + @Inject + public KubernetesClusterService kubernetesClusterService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, + entityType = KubernetesClusterResponse.class, + description = "the ID of the Kubernetes cluster") + private Long id; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public long getEntityOwnerId() { + Account account = CallContext.current().getCallingAccount(); + if (account != null) { + return account.getId(); + } + + return Account.ACCOUNT_ID_SYSTEM; // no account info given, parent this command to SYSTEM so ERROR events are tracked + } + + @Override + public String getCommandName() { + return APINAME.toLowerCase() + "response"; + } + + @Override + public void execute() { + + KubernetesClusterConfigResponse response = kubernetesClusterService.getKubernetesClusterConfig(this); + response.setResponseName(getCommandName()); + setResponseObject(response); + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/ListKubernetesClustersCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/ListKubernetesClustersCmd.java new file mode 100644 index 000000000000..73833e19c2be --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/ListKubernetesClustersCmd.java @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.kubernetescluster; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.response.KubernetesClusterResponse; +import org.apache.log4j.Logger; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject.ResponseView; +import org.apache.cloudstack.api.response.ListResponse; +import com.cloud.kubernetescluster.KubernetesClusterService; + +@APICommand(name = ListKubernetesClustersCmd.APINAME, + description = "Lists Kubernetes clusters", + responseObject = KubernetesClusterResponse.class, + responseView = ResponseView.Restricted, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = true, + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class ListKubernetesClustersCmd extends BaseListCmd { + public static final Logger LOGGER = Logger.getLogger(ListKubernetesClustersCmd.class.getName()); + public static final String APINAME = "listKubernetesClusters"; + + @Inject + public KubernetesClusterService kubernetesClusterService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, + entityType = KubernetesClusterResponse.class, + description = "the ID of the Kubernetes cluster") + private Long id; + + @Parameter(name = ApiConstants.STATE, type = CommandType.STRING, description = "state of the Kubernetes cluster") + private String state; + + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "name of the Kubernetes cluster" + + " (a substring match is made against the parameter value, data for all matching Kubernetes clusters will be returned)") + private String name; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + public String getState() { + return state; + } + + public String getName() { + return name; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public String getCommandName() { + return APINAME.toLowerCase() + "response"; + } + + @Override + public void execute() { + ListResponse response = kubernetesClusterService.listKubernetesClusters(this); + response.setResponseName(getCommandName()); + setResponseObject(response); + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/ScaleKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/ScaleKubernetesClusterCmd.java new file mode 100644 index 000000000000..16340a08738f --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/ScaleKubernetesClusterCmd.java @@ -0,0 +1,142 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.kubernetescluster; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.acl.SecurityChecker; +import org.apache.cloudstack.api.ACL; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.KubernetesClusterResponse; +import org.apache.cloudstack.api.response.ServiceOfferingResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +import com.cloud.kubernetescluster.KubernetesClusterEventTypes; +import com.cloud.kubernetescluster.KubernetesCluster; +import com.cloud.kubernetescluster.KubernetesClusterService; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.ManagementServerException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; + +@APICommand(name = ScaleKubernetesClusterCmd.APINAME, + description = "Scales a created or running Kubernetes cluster", + responseObject = KubernetesClusterResponse.class, + responseView = ResponseObject.ResponseView.Restricted, + entityType = {KubernetesCluster.class}, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = true, + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class ScaleKubernetesClusterCmd extends BaseAsyncCmd { + public static final Logger LOGGER = Logger.getLogger(StartKubernetesClusterCmd.class.getName()); + public static final String APINAME = "scaleKubernetesCluster"; + + @Inject + public KubernetesClusterService kubernetesClusterService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, + entityType = KubernetesClusterResponse.class, + description = "the ID of the Kubernetes cluster") + private Long id; + + @ACL(accessType = SecurityChecker.AccessType.UseEntry) + @Parameter(name = ApiConstants.SERVICE_OFFERING_ID, type = CommandType.UUID, entityType = ServiceOfferingResponse.class, + description = "the ID of the service offering for the virtual machines in the cluster.") + private Long serviceOfferingId; + + @Parameter(name=ApiConstants.SIZE, type = CommandType.LONG, + description = "number of Kubernetes cluster nodes") + private Long clusterSize; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + public Long getServiceOfferingId() { + return serviceOfferingId; + } + + public Long getClusterSize() { + return clusterSize; + } + + @Override + public String getEventType() { + return KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_SCALE; + } + + @Override + public String getEventDescription() { + return "Scaling Kubernetes cluster id: " + getId(); + } + + @Override + public String getCommandName() { + return APINAME.toLowerCase() + "response"; + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + public KubernetesCluster validateRequest() { + if (getId() == null || getId() < 1L) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Invalid Kubernetes cluster ID provided"); + } + final KubernetesCluster kubernetesCluster = kubernetesClusterService.findById(getId()); + if (kubernetesCluster == null) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Given Kubernetes cluster was not found"); + } + return kubernetesCluster; + } + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + final KubernetesCluster kubernetesCluster = validateRequest(); + try { + kubernetesClusterService.scaleKubernetesCluster(this); + final KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(getId()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (InsufficientCapacityException | ResourceUnavailableException | ManagementServerException ex) { + LOGGER.warn("Failed to scale Kubernetes cluster:" + kubernetesCluster.getUuid() + " due to " + ex.getMessage()); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, + "Failed to scale Kubernetes cluster:" + kubernetesCluster.getUuid(), ex); + } + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/StartKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/StartKubernetesClusterCmd.java new file mode 100644 index 000000000000..d0c700effa05 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/StartKubernetesClusterCmd.java @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.kubernetescluster; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.KubernetesClusterResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.ManagementServerException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.kubernetescluster.KubernetesCluster; +import com.cloud.kubernetescluster.KubernetesClusterEventTypes; +import com.cloud.kubernetescluster.KubernetesClusterService; + +@APICommand(name = StartKubernetesClusterCmd.APINAME, description = "Starts a stopped Kubernetes cluster", + responseObject = KubernetesClusterResponse.class, + responseView = ResponseObject.ResponseView.Restricted, + entityType = {KubernetesCluster.class}, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = true, + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class StartKubernetesClusterCmd extends BaseAsyncCmd { + public static final Logger LOGGER = Logger.getLogger(StartKubernetesClusterCmd.class.getName()); + public static final String APINAME = "startKubernetesCluster"; + + @Inject + public KubernetesClusterService kubernetesClusterService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, + entityType = KubernetesClusterResponse.class, + description = "the ID of the Kubernetes cluster") + private Long id; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + @Override + public String getEventType() { + return KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_START; + } + + @Override + public String getEventDescription() { + return "Starting Kubernetes cluster id: " + getId(); + } + + @Override + public String getCommandName() { + return APINAME.toLowerCase() + "response"; + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + public KubernetesCluster validateRequest() { + if (getId() == null || getId() < 1L) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Invalid Kubernetes cluster ID provided"); + } + final KubernetesCluster kubernetesCluster = kubernetesClusterService.findById(getId()); + if (kubernetesCluster == null) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Given Kubernetes cluster was not found"); + } + return kubernetesCluster; + } + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + final KubernetesCluster kubernetesCluster = validateRequest(); + try { + kubernetesClusterService.startKubernetesCluster(getId().longValue(), false); + final KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(getId()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (InsufficientCapacityException | ResourceUnavailableException | ManagementServerException ex) { + LOGGER.warn("Failed to start Kubernetes cluster:" + kubernetesCluster.getUuid() + " due to " + ex.getMessage()); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, + "Failed to start Kubernetes cluster:" + kubernetesCluster.getUuid(), ex); + } + } + +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/StopKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/StopKubernetesClusterCmd.java new file mode 100644 index 000000000000..c02eab523c74 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/StopKubernetesClusterCmd.java @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.kubernetescluster; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.KubernetesClusterResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.ManagementServerException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.kubernetescluster.KubernetesCluster; +import com.cloud.kubernetescluster.KubernetesClusterEventTypes; +import com.cloud.kubernetescluster.KubernetesClusterService; + +@APICommand(name = StopKubernetesClusterCmd.APINAME, description = "Stops a running Kubernetes cluster", + responseObject = SuccessResponse.class, + responseView = ResponseObject.ResponseView.Restricted, + entityType = {KubernetesCluster.class}, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = true, + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class StopKubernetesClusterCmd extends BaseAsyncCmd { + public static final Logger LOGGER = Logger.getLogger(StopKubernetesClusterCmd.class.getName()); + public static final String APINAME = "stopKubernetesCluster"; + + @Inject + public KubernetesClusterService kubernetesClusterService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, + entityType = KubernetesClusterResponse.class, + description = "the ID of the Kubernetes cluster") + private Long id; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + @Override + public String getEventType() { + return KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_STOP; + } + + @Override + public String getEventDescription() { + return "Stopping Kubernetes cluster id: " + getId(); + } + + @Override + public String getCommandName() { + return APINAME.toLowerCase() + "response"; + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + public KubernetesCluster validateRequest() { + if (getId() == null || getId() < 1L) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Invalid Kubernetes cluster ID provided"); + } + final KubernetesCluster kubernetesCluster = kubernetesClusterService.findById(getId()); + if (kubernetesCluster == null) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Given Kubernetes cluster was not found"); + } + return kubernetesCluster; + } + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + final KubernetesCluster kubernetesCluster = validateRequest(); + try { + final boolean result = kubernetesClusterService.stopKubernetesCluster(getId()); + final SuccessResponse response = new SuccessResponse(getCommandName()); + response.setSuccess(result); + setResponseObject(response); + } catch (ManagementServerException ex) { + LOGGER.warn("Failed to stop Kubernetes cluster:" + kubernetesCluster.getUuid() + " due to " + ex.getMessage()); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, + "Failed to stop Kubernetes cluster:" + kubernetesCluster.getUuid(), ex); + } + } + +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetesversion/ListKubernetesSupportedVersionsCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetesversion/ListKubernetesSupportedVersionsCmd.java new file mode 100644 index 000000000000..57917323dfe0 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetesversion/ListKubernetesSupportedVersionsCmd.java @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.user.kubernetesversion; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.KubernetesSupportedVersionResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.log4j.Logger; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.kubernetesversion.KubernetesVersionService; + +@APICommand(name = ListKubernetesSupportedVersionsCmd.APINAME, + description = "Lists container clusters", + responseObject = KubernetesSupportedVersionResponse.class, + responseView = ResponseObject.ResponseView.Restricted, + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class ListKubernetesSupportedVersionsCmd extends BaseListCmd { + public static final Logger LOGGER = Logger.getLogger(ListKubernetesSupportedVersionsCmd.class.getName()); + public static final String APINAME = "listKubernetesSupportedVersions"; + + @Inject + private KubernetesVersionService kubernetesVersionService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, + entityType = KubernetesSupportedVersionResponse.class, + description = "the ID of the Kubernetes supported version") + private Long id; + + @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, + entityType = ZoneResponse.class, + description = "the ID of the zone in which Kubernetes supported version will be available") + private Long zoneId; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + public Long getId() { + return id; + } + + public Long getZoneId() { + return zoneId; + } + + @Override + public String getCommandName() { + return APINAME.toLowerCase() + "response"; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + ListResponse response = kubernetesVersionService.listKubernetesSupportedVersions(this); + response.setResponseName(getCommandName()); + setResponseObject(response); + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterConfigResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterConfigResponse.java new file mode 100644 index 000000000000..0308518b9981 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterConfigResponse.java @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.response; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; + +public class KubernetesClusterConfigResponse extends BaseResponse { + @SerializedName(ApiConstants.ID) + @Param(description = "the id of the container cluster") + private String id; + + @SerializedName(ApiConstants.NAME) + @Param(description = "Name of the container cluster") + private String name; + + @SerializedName("configdata") + @Param(description = "the config data of the cluster") + private String configData; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getConfigData() { + return configData; + } + + public void setConfigData(String configData) { + this.configData = configData; + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java new file mode 100644 index 000000000000..a734413ae3bd --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java @@ -0,0 +1,287 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.response; + +import java.util.List; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; +import org.apache.cloudstack.api.EntityReference; + +import com.cloud.kubernetescluster.KubernetesCluster; +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; + +@SuppressWarnings("unused") +@EntityReference(value = {KubernetesCluster.class}) +public class KubernetesClusterResponse extends BaseResponse implements ControlledEntityResponse { + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getZoneId() { + return zoneId; + } + + public void setZoneId(String zoneId) { + this.zoneId = zoneId; + } + + public String getZoneName() { + return zoneName; + } + + public void setZoneName(String zoneName) { + this.zoneName = zoneName; + } + + public String getServiceOfferingId() { + return serviceOfferingId; + } + + public void setServiceOfferingId(String serviceOfferingId) { + this.serviceOfferingId = serviceOfferingId; + } + + public String getTemplateId() { + return templateId; + } + + public void setTemplateId(String templateId) { + this.templateId = templateId; + } + + public String getNetworkId() { + return networkId; + } + + public void setNetworkId(String networkId) { + this.networkId = networkId; + } + + public String getKeypair() { + return keypair; + } + + public void setKeypair(String keypair) { + this.keypair = keypair; + } + + public String getClusterSize() { + return clusterSize; + } + + public void setClusterSize(String clusterSize) { + this.clusterSize = clusterSize; + } + + public String getCores() { + return cores; + } + + public void setCores(String cores) { + this.cores = cores; + } + + public String getMemory() { + return memory; + } + + public void setMemory(String memory) { + this.memory = memory; + } + + public String getState() { return state;} + + public void setState(String state) {this.state = state;} + + public String getEndpoint() { return endpoint;} + + public void setEndpoint(String endpoint) {this.endpoint = endpoint;} + + public String getConsoleEndpoint() { return consoleendpoint;} + + public void setConsoleEndpoint(String consoleendpoint) {this.consoleendpoint = consoleendpoint;} + + public String getId() { + return this.id; + } + + public void setId(String id) { + this.id = id; + } + + public String getServiceOfferingName() { + return serviceOfferingName; + } + + public void setServiceOfferingName(String serviceOfferingName) { + this.serviceOfferingName = serviceOfferingName; + } + + public void setVirtualMachineIds(List virtualMachineIds) { this.virtualMachineIds = virtualMachineIds;}; + + public void setUsername(String userName) { this.username = userName;} + + public String getPassword() { + return password; + } + + public void setPassword(String password) { this.password = password;} + + public String getUsername() { + return username; + } + + public List getVirtualMachineIds() { + return virtualMachineIds; + } + + public String getConsoleendpoint() { + return consoleendpoint; + } + + @SerializedName(ApiConstants.ID) + @Param(description = "the id of the Kubernetes cluster") + private String id; + + @SerializedName(ApiConstants.NAME) + @Param(description = "Name of the Kubernetes cluster") + private String name; + + @SerializedName(ApiConstants.DESCRIPTION) + @Param(description = "Description of the Kubernetes cluster") + private String description; + + @SerializedName(ApiConstants.ZONE_ID) + @Param(description = "zone id") + private String zoneId; + + @SerializedName(ApiConstants.ZONE_NAME) + @Param(description = "zone name") + private String zoneName; + + @SerializedName(ApiConstants.SERVICE_OFFERING_ID) + @Param(description = "Service Offering id") + private String serviceOfferingId; + + @SerializedName("serviceofferingname") + @Param(description = "the name of the service offering of the virtual machine") + private String serviceOfferingName; + + @SerializedName(ApiConstants.TEMPLATE_ID) + @Param(description = "template id") + private String templateId; + + @SerializedName(ApiConstants.NETWORK_ID) + @Param(description = "network id details") + private String networkId; + + @SerializedName(ApiConstants.ASSOCIATED_NETWORK_NAME) + @Param(description = "the name of the Network associated with the IP address") + private String associatedNetworkName; + + public String getAssociatedNetworkName() { + return associatedNetworkName; + } + + public void setAssociatedNetworkName(String associatedNetworkName) { + this.associatedNetworkName = associatedNetworkName; + } + + @SerializedName(ApiConstants.SSH_KEYPAIR) + @Param(description = "keypair details") + private String keypair; + + @SerializedName(ApiConstants.SIZE) + @Param(description = "cluster size") + private String clusterSize; + + @SerializedName(ApiConstants.STATE) + @Param(description = "state of the cluster") + private String state; + + @SerializedName(ApiConstants.CPU_NUMBER) + @Param(description = "cluster cpu cores") + private String cores; + + @SerializedName(ApiConstants.MEMORY) + @Param(description = "cluster size") + private String memory; + + @SerializedName(ApiConstants.END_POINT) + @Param(description = "URL end point for the cluster") + private String endpoint; + + @SerializedName(ApiConstants.CONSOLE_END_POINT) + @Param(description = "URL end point for the cluster UI") + private String consoleendpoint; + + @SerializedName(ApiConstants.VIRTUAL_MACHINE_IDS) + @Param(description = "the list of virtualmachine ids associated with this Kubernetes cluster") + private List virtualMachineIds; + + @SerializedName(ApiConstants.USERNAME) + @Param(description = "Username with which Kubernetes cluster is setup") + private String username; + + @SerializedName(ApiConstants.PASSWORD) + @Param(description = "Password with which Kubernetes cluster is setup") + private String password; + + public KubernetesClusterResponse() { + + } + + @Override + public void setAccountName(String accountName) { + + } + + @Override + public void setProjectId(String projectId) { + + } + + @Override + public void setProjectName(String projectName) { + + } + + @Override + public void setDomainId(String domainId) { + + } + + @Override + public void setDomainName(String domainName) { + + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java new file mode 100644 index 000000000000..be704406f179 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.response; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; +import org.apache.cloudstack.api.EntityReference; + +import com.cloud.kubernetesversion.KubernetesSupportedVersion; +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; + +@SuppressWarnings("unused") +@EntityReference(value = {KubernetesSupportedVersion.class}) +public class KubernetesSupportedVersionResponse extends BaseResponse { + @SerializedName(ApiConstants.ID) + @Param(description = "the id of the Kubernetes supported version") + private String id; + + @SerializedName(ApiConstants.NAME) + @Param(description = "Name of the Kubernetes supported version") + private String name; + + @SerializedName(ApiConstants.ISO_ID) + @Param(description = "the id of the binaries ISO for Kubernetes supported version") + private String isoId; + + @SerializedName("isoname") + @Param(description = "the name of the binaries ISO for Kubernetes supported version") + private String isoName; + + @SerializedName("isostate") + @Param(description = "the state of the binaries ISO for Kubernetes supported version") + private String isoState; + + @SerializedName(ApiConstants.ZONE_ID) + @Param(description = "the id of the zone in which Kubernetes supported version is available") + private String zoneId; + + @SerializedName(ApiConstants.ZONE_NAME) + @Param(description = "the name of the zone in which Kubernetes supported version is available") + private String zoneName; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getIsoId() { + return isoId; + } + + public void setIsoId(String isoId) { + this.isoId = isoId; + } + + public String getIsoName() { + return isoName; + } + + public void setIsoName(String isoName) { + this.isoName = isoName; + } + + public String getIsoState() { + return isoState; + } + + public void setIsoState(String isoState) { + this.isoState = isoState; + } + + public String getZoneId() { + return zoneId; + } + + public void setZoneId(String zoneId) { + this.zoneId = zoneId; + } + + public String getZoneName() { + return zoneName; + } + + public void setZoneName(String zoneName) { + this.zoneName = zoneName; + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/module.properties b/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/module.properties new file mode 100644 index 000000000000..4fe375e1e3c2 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/module.properties @@ -0,0 +1,15 @@ +# Copyright 2016 ShapeBlue Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +name=kubernetes-service +parent=compute diff --git a/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml b/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml new file mode 100644 index 000000000000..f56c04c35450 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml @@ -0,0 +1,34 @@ + + + + + + + + + + + diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml new file mode 100644 index 000000000000..ea9788dfdc3c --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml @@ -0,0 +1,238 @@ +#cloud-config + +--- +ssh_authorized_keys: + {{ k8s_master.ms.ssh.pub.key }} + +write-files: + - path: /etc/conf.d/nfs + permissions: '0644' + content: | + OPTS_RPC_MOUNTD="" + + - path: /etc/kubernetes/pki/cloudstack/ca.crt + permissions: '0644' + content: | + {{ k8s_master.ca.crt }} + + - path: /etc/kubernetes/pki/cloudstack/apiserver.crt + permissions: '0644' + content: | + {{ k8s_master.apiserver.crt }} + + - path: /etc/kubernetes/pki/cloudstack/apiserver.key + permissions: '0600' + content: | + {{ k8s_master.apiserver.key }} + + - path: /opt/bin/setup-kube-system + permissions: 0700 + owner: root:root + content: | + #!/bin/bash -e + + export PATH=$PATH:/opt/bin + + RELEASE="v1.11.4" + ISO_MOUNT_DIR=/mnt/k8sdisk + BINARIES_DIR=${ISO_MOUNT_DIR}/${RELEASE} + CNI_VERSION="v0.7.1" + CRICTL_VERSION="v1.11.1" + ATTEMPT_OFFLINE_INSTALL=true + setup_complete=false + + OFFLINE_INSTALL_ATTEMPT_SLEEP=5 + MAX_OFFLINE_INSTALL_ATTEMPTS=36 + offline_attempts=1 + MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 + crucial_cmd_attempts=1 + while true; do + if (( "$offline_attempts" > "$MAX_OFFLINE_INSTALL_ATTEMPTS" )); then + echo "Warning: Offline install timed out!" + break + fi + set +e + output=`blkid -o device -t TYPE=iso9660` + set -e + if [ "$output" != "" ]; then + while read -r line; do + if [ ! -d "${ISO_MOUNT_DIR}" ]; then + mkdir "${ISO_MOUNT_DIR}" + fi + retval=0 + set +e + mount -o ro "${line}" "${ISO_MOUNT_DIR}" + retval=$? + set -e + if [ $retval -eq 0 ]; then + if [ -d "$BINARIES_DIR" ]; then + break + else + umount "${line}" && rmdir "${ISO_MOUNT_DIR}" + fi + fi + done <<< "$output" + fi + if [ -d "$BINARIES_DIR" ]; then + break + fi + echo "Waiting for Binaries directory $BINARIES_DIR to be available, sleeping for $OFFLINE_INSTALL_ATTEMPT_SLEEP seconds, attempt: $offline_attempts" + sleep $OFFLINE_INSTALL_ATTEMPT_SLEEP + offline_attempts=$[$offline_attempts + 1] + done + + if [ -d "$BINARIES_DIR" ]; then + ### Binaries available offline ### + echo "Installing binaries from ${BINARIES_DIR}" + mkdir -p /opt/cni/bin + tar -f "${BINARIES_DIR}/cni/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" -C /opt/cni/bin -xz + + mkdir -p /opt/bin + tar -f "${BINARIES_DIR}/cri-tools/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" -C /opt/bin -xz + + mkdir -p /opt/bin + cd /opt/bin + cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} /opt/bin + chmod +x {kubeadm,kubelet,kubectl} + + sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service > /etc/systemd/system/kubelet.service + mkdir -p /etc/systemd/system/kubelet.service.d + sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + + output=`ls ${BINARIES_DIR}/docker/` + if [ "$output" != "" ]; then + while read -r line; do + crucial_cmd_attempts=1 + while true; do + if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Loading docker image ${BINARIES_DIR}/docker/$line failed!" + break; + fi + retval=0 + set +e + docker load < "${BINARIES_DIR}/docker/$line" + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] + done + done <<< "$output" + setup_complete=true + fi + umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" + fi + if [ "$setup_complete" = false ]; then + ### Binaries not available offline ### + echo "Warning: ${BINARIES_DIR} not found. Will get binaries and docker images from Internet." + mkdir -p /opt/cni/bin + curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz + + mkdir -p /opt/bin + curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" | tar -C /opt/bin -xz + + mkdir -p /opt/bin + cd /opt/bin + curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} + chmod +x {kubeadm,kubelet,kubectl} + + curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service + mkdir -p /etc/systemd/system/kubelet.service.d + curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + fi + + systemctl enable kubelet && systemctl start kubelet + modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 + + crucial_cmd_attempts=1 + while true; do + if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Error: kubeadm pull images failed!" + exit 1 + fi + retval=0 + set +e + kubeadm config images pull + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] + done + crucial_cmd_attempts=1 + while true; do + if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Error: kubeadm init failed!" + exit 1 + fi + retval=0 + set +e + kubeadm init --pod-network-cidr=10.244.0.0/16 --token {{ k8s_master.cluster.token }} {{ k8s_master.cluster.ip }} + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] + done + + - path: /opt/bin/deploy-kube-system + permissions: 0700 + owner: root:root + content: | + #!/bin/bash -e + if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then + echo "setup-kube-system is running!" + exit 1 + fi + export PATH=$PATH:/opt/bin + export KUBECONFIG=/etc/kubernetes/admin.conf + + mkdir -p /root/.kube + cp -i /etc/kubernetes/admin.conf /root/.kube/config + chown $(id -u):$(id -g) /root/.kube/config + echo export PATH=\$PATH:/opt/bin >> /root/.bashrc + + kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/v0.10.0/Documentation/kube-flannel.yml + kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.0/src/deploy/recommended/kubernetes-dashboard.yaml + + kubectl create rolebinding admin-binding --role=admin --user=admin || true + kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin --user=admin || true + kubectl create clusterrolebinding kubernetes-dashboard --clusterrole=cluster-admin --serviceaccount=kube-system:kubernetes-dashboard || true + +coreos: + units: + - name: docker.service + command: start + enable: true + + - name: setup-kube-system.service + command: start + content: | + [Unit] + Requires=docker.service + After=docker.service + + [Service] + Type=simple + StartLimitInterval=0 + ExecStart=/opt/bin/setup-kube-system + + - name: deploy-kube-system.service + command: start + content: | + [Unit] + After=setup-kube-system.service + + [Service] + Type=simple + StartLimitInterval=0 + Restart=on-failure + ExecStartPre=/usr/bin/curl -k https://127.0.0.1:6443/version + ExecStart=/opt/bin/deploy-kube-system + + update: + group: stable + reboot-strategy: off diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml new file mode 100644 index 000000000000..4c1361cee47f --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -0,0 +1,191 @@ +#cloud-config + +--- +write-files: + - path: /opt/bin/setup-kube-system + permissions: 0700 + owner: root:root + content: | + #!/bin/bash -e + + export PATH=$PATH:/opt/bin + + RELEASE="v1.11.4" + ISO_MOUNT_DIR=/mnt/k8sdisk + BINARIES_DIR=${ISO_MOUNT_DIR}/${RELEASE} + CNI_VERSION="v0.7.1" + CRICTL_VERSION="v1.11.1" + ATTEMPT_OFFLINE_INSTALL=true + setup_complete=false + + OFFLINE_INSTALL_ATTEMPT_SLEEP=5 + MAX_OFFLINE_INSTALL_ATTEMPTS=36 + offline_attempts=1 + MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 + crucial_cmd_attempts=1 + while true; do + if (( "$offline_attempts" > "$MAX_OFFLINE_INSTALL_ATTEMPTS" )); then + echo "Warning: Offline install timed out!" + break + fi + set +e + output=`blkid -o device -t TYPE=iso9660` + set -e + if [ "$output" != "" ]; then + while read -r line; do + if [ ! -d "${ISO_MOUNT_DIR}" ]; then + mkdir "${ISO_MOUNT_DIR}" + fi + retval=0 + set +e + mount -o ro "${line}" "${ISO_MOUNT_DIR}" + retval=$? + set -e + if [ $retval -eq 0 ]; then + if [ -d "$BINARIES_DIR" ]; then + break + else + umount "${line}" && rmdir "${ISO_MOUNT_DIR}" + fi + fi + done <<< "$output" + fi + if [ -d "$BINARIES_DIR" ]; then + break + fi + echo "Waiting for Binaries directory $BINARIES_DIR to be available, sleeping for $OFFLINE_INSTALL_ATTEMPT_SLEEP seconds, attempt: $offline_attempts" + sleep $OFFLINE_INSTALL_ATTEMPT_SLEEP + offline_attempts=$[$offline_attempts + 1] + done + + if [ -d "$BINARIES_DIR" ]; then + ### Binaries available offline ### + echo "Installing binaries from ${BINARIES_DIR}" + mkdir -p /opt/cni/bin + tar -f "${BINARIES_DIR}/cni/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" -C /opt/cni/bin -xz + + mkdir -p /opt/bin + tar -f "${BINARIES_DIR}/cri-tools/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" -C /opt/bin -xz + + mkdir -p /opt/bin + cd /opt/bin + cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} /opt/bin + chmod +x {kubeadm,kubelet,kubectl} + + sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service > /etc/systemd/system/kubelet.service + mkdir -p /etc/systemd/system/kubelet.service.d + sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + + output=`ls ${BINARIES_DIR}/docker/` + if [ "$output" != "" ]; then + while read -r line; do + crucial_cmd_attempts=1 + while true; do + if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Loading docker image ${BINARIES_DIR}/docker/$line failed!" + break; + fi + retval=0 + set +e + docker load < "${BINARIES_DIR}/docker/$line" + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] + done + done <<< "$output" + setup_complete=true + fi + umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" + fi + if [ "$setup_complete" = false ]; then + ### Binaries not available offline ### + echo "Warning: ${BINARIES_DIR} not found. Will get binaries and docker images from Internet." + mkdir -p /opt/cni/bin + curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz + + mkdir -p /opt/bin + curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" | tar -C /opt/bin -xz + + mkdir -p /opt/bin + cd /opt/bin + curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} + chmod +x {kubeadm,kubelet,kubectl} + + curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service + mkdir -p /etc/systemd/system/kubelet.service.d + curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + fi + + systemctl enable kubelet && systemctl start kubelet + modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 + + crucial_cmd_attempts=1 + while true; do + if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Error: kubeadm pull images failed!" + exit 1 + fi + retval=0 + set +e + kubeadm config images pull + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] + done + + - path: /opt/bin/deploy-kube-system + permissions: 0700 + owner: root:root + content: | + #!/bin/bash -e + if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then + echo "setup-kube-system is running!" + exit 1 + fi + modprobe ip_vs + modprobe ip_vs_wrr + modprobe ip_vs_sh + modprobe nf_conntrack_ipv4 + export PATH=$PATH:/opt/bin + kubeadm join {{ k8s_master.default_ip }}:6443 --token {{ k8s_master.cluster.token }} --discovery-token-unsafe-skip-ca-verification + +coreos: + units: + - name: docker.service + command: start + enable: true + + - name: setup-kube-system.service + command: start + content: | + [Unit] + Requires=docker.service + After=docker.service + + [Service] + Type=simple + StartLimitInterval=0 + ExecStart=/opt/bin/setup-kube-system + + - name: deploy-kube-system.service + command: start + content: | + [Unit] + After=setup-kube-system.service + + [Service] + Type=simple + StartLimitInterval=0 + Restart=on-failure + ExecStartPre=/usr/bin/curl -k https://{{ k8s_master.default_ip }}:6443/version + ExecStart=/opt/bin/deploy-kube-system + + update: + group: stable + reboot-strategy: off diff --git a/plugins/pom.xml b/plugins/pom.xml index 4a96b4d2a433..89b3f4f77c31 100755 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -86,6 +86,7 @@ integrations/cloudian integrations/prometheus + integrations/kubernetes-service metrics diff --git a/tools/apidoc/gen_toc.py b/tools/apidoc/gen_toc.py index 1c5cf27128a5..824dbef75dc8 100644 --- a/tools/apidoc/gen_toc.py +++ b/tools/apidoc/gen_toc.py @@ -189,6 +189,8 @@ 'Sioc' : 'Sioc', 'Diagnostics': 'Diagnostics', 'Management': 'Management', + 'KubernetesSupportedVersion': 'Kubernetes Service', + 'KubernetesCluster': 'Kubernetes Service', } diff --git a/ui/plugins/cks/cks.css b/ui/plugins/cks/cks.css new file mode 100644 index 000000000000..bcab05922472 --- /dev/null +++ b/ui/plugins/cks/cks.css @@ -0,0 +1,31 @@ +/* + * Copyright 2016 ShapeBlue Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +.downloadKubernetesClusterKubeConfig .icon { + background-position: -35px -125px; +} + +.downloadKubernetesClusterKubeConfig:hover .icon { + background-position: -35px -707px; +} + +.scaleKubernetesCluster .icon { + background-position: -264px -2px; +} + +.scaleKubernetesCluster:hover .icon { + background-position: -263px -583px; +} diff --git a/ui/plugins/cks/cks.js b/ui/plugins/cks/cks.js new file mode 100644 index 000000000000..03e45ed630b3 --- /dev/null +++ b/ui/plugins/cks/cks.js @@ -0,0 +1,1296 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +(function (cloudStack) { + + var rootCaCert = ""; + var downloadCaCert = function() { + var blob = new Blob([rootCaCert], {type: 'application/x-x509-ca-cert'}); + var filename = "cloudstack-ca.pem"; + if(window.navigator.msSaveOrOpenBlob) { + window.navigator.msSaveBlob(blob, filename); + } else{ + var elem = window.document.createElement('a'); + elem.href = window.URL.createObjectURL(blob); + elem.download = filename; + document.body.appendChild(elem) + elem.click(); + document.body.removeChild(elem); + } + }; + var clusterKubeConfig = ""; + var downloadClusterKubeConfig = function() { + var blob = new Blob([clusterKubeConfig], {type: 'text/plain'}); + var filename = "admin.conf"; + if(window.navigator.msSaveOrOpenBlob) { + window.navigator.msSaveBlob(blob, filename); + } else{ + var elem = window.document.createElement('a'); + elem.href = window.URL.createObjectURL(blob); + elem.download = filename; + document.body.appendChild(elem) + elem.click(); + document.body.removeChild(elem); + } + }; + cloudStack.plugins.cks = function(plugin) { + plugin.ui.addSection({ + id: 'cks', + title: 'Kubernetes Service', + preFilter: function(args) { + return true; + }, + showOnNavigation: true, + sectionSelect: { + label: 'label.select-view', + preFilter: function() { + return ['kubernetesclusters', 'kubernetesversions']; + } + }, + sections: { + kubernetesclusters: { + id: 'kubernetesclusters', + type: 'select', + title: "Clusters", + listView: { + filters: { + all: { + label: 'ui.listView.filters.all' + }, + running: { + label: 'state.Running' + }, + stopped: { + label: 'state.Stopped' + }, + destroyed: { + label: 'state.Destroyed' + } + }, + fields: { + name: { + label: 'label.name' + }, + zonename: { + label: 'label.zone.name' + }, + size : { + label: 'label.size' + }, + cpunumber: { + label: 'label.num.cpu.cores' + }, + memory: { + label: 'label.memory.mb' + }, + state: { + label: 'label.state', + indicator: { + 'Running': 'on', + 'Stopped': 'off', + 'Destroyed': 'off', + 'Error': 'off' + } + } + }, + advSearchFields: { + name: { + label: 'label.name' + }, + zoneid: { + label: 'label.zone', + select: function(args) { + $.ajax({ + url: createURL('listZones'), + data: { + listAll: true + }, + success: function(json) { + var zones = json.listzonesresponse.zone ? json.listzonesresponse.zone : []; + + args.response.success({ + data: $.map(zones, function(zone) { + return { + id: zone.id, + description: zone.name + }; + }) + }); + } + }); + } + }, + }, + // List view actions + actions: { + add: { + label: 'Add Kubernetes cluster', + createForm: { + title: 'Add Kubernetes cluster', + preFilter: cloudStack.preFilter.createTemplate, + fields: { + name: { + label: 'label.name', + //docID: 'Name of the cluster', + validation: { + required: true + } + }, + description: { + label: 'label.description', + //docID: 'helpKubernetesClusterDesc', + }, + zone: { + label: 'label.zone', + //docID: 'helpKubernetesClusterZone', + validation: { + required: true + }, + select: function(args) { + $.ajax({ + url: createURL("listZones&available=true"), + dataType: "json", + async: true, + success: function(json) { + var items = []; + var zoneObjs = json.listzonesresponse.zone; + if (zoneObjs != null) { + for (var i = 0; i < zoneObjs.length; i++) { + items.push({ + id: zoneObjs[i].id, + description: zoneObjs[i].name + }); + } + } + args.response.success({ + data: items + }); + } + }); + } + }, + kubernetesversion: { + label: 'Kubernetes version', + dependsOn: 'zone', + //docID: 'helpKubernetesClusterZone', + validation: { + required: true + }, + select: function(args) { + $.ajax({ + url: createURL("listKubernetesSupportedVersions"), + data: { zoneid: args.zone }, + dataType: "json", + async: true, + url: createURL("listKubernetesSupportedVersions"), + dataType: "json", + async: true, + success: function(json) { + var items = []; + var versionObjs = json.listkubernetessupportedversionsresponse.kubernetessupportedversion; + if (versionObjs != null) { + for (var i = 0; i < versionObjs.length; i++) { + if (versionObjs[i].isostate == 'Active') { + items.push({ + id: versionObjs[i].id, + description: versionObjs[i].name + }); + } + } + } + args.response.success({ + data: items + }); + } + }); + } + }, + serviceoffering: { + label: 'label.menu.service.offerings', + //docID: 'helpKubernetesClusterServiceOffering', + validation: { + required: true + }, + select: function(args) { + $.ajax({ + url: createURL("listServiceOfferings"), + dataType: "json", + async: true, + success: function(json) { + var offeringObjs = []; + var items = json.listserviceofferingsresponse.serviceoffering; + if (items != null) { + for (var i = 0; i < items.length; i++) { + offeringObjs.push({ + id: items[i].id, + description: items[i].name + }); + } + } + args.response.success({ + data: offeringObjs + }); + } + }); + } + }, + noderootdisksize: { + label: 'Node root disk size (in GB)', + //docID: 'helpKubernetesClusterNodeRootDiskSize', + validation: { + number: true + } + }, + network: { + label: 'label.network', + //docID: 'helpKubernetesClusterNetwork', + select: function(args) { + $.ajax({ + url: createURL("listNetworks"), + dataType: "json", + async: true, + success: function(json) { + var networkObjs = []; + networkObjs.push({ + id: "", + description: "" + }); + var items = json.listnetworksresponse.network; + if (items != null) { + for (var i = 0; i < items.length; i++) { + networkObjs.push({ + id: items[i].id, + description: items[i].name + }); + } + } + args.response.success({ + data: networkObjs + }); + } + }); + } + }, + size: { + label: 'Cluster size', + //docID: 'helpKubernetesClusterSize', + validation: { + required: true, + number: true + }, + }, + sshkeypair: { + label: 'label.ssh.key.pair', + //docID: 'helpKubernetesClusterSSH', + select: function(args) { + $.ajax({ + url: createURL("listSSHKeyPairs"), + dataType: "json", + async: true, + success: function(json) { + var keypairObjs = []; + keypairObjs.push({ + id: "", + description: "" + }); + var items = json.listsshkeypairsresponse.sshkeypair; + if (items != null) { + for (var i = 0; i < items.length; i++) { + keypairObjs.push({ + id: items[i].name, + description: items[i].name + }); + } + } + args.response.success({ + data: keypairObjs + }); + } + }); + } + }, + supportPrivateRegistry: { + label: 'Private Registry', + isBoolean: true, + isChecked: false, + }, + username: { + label: 'label.username', + dependsOn: 'supportPrivateRegistry', + validation: { + required: true + }, + isHidden: true + }, + password: { + label: 'label.password', + dependsOn: 'supportPrivateRegistry', + validation: { + required: true + }, + isHidden: true, + isPassword: true + }, + url: { + label: 'label.url', + dependsOn: 'supportPrivateRegistry', + validation: { + required: true + }, + isHidden: true, + }, + email: { + label: 'label.email', + dependsOn: 'supportPrivateRegistry', + validation: { + required: true + }, + isHidden: true, + } + } + }, + + action: function(args) { + var data = { + name: args.data.name, + description: args.data.description, + zoneid: args.data.zone, + kubernetesversionid: args.data.kubernetesversion, + serviceofferingid: args.data.serviceoffering, + size: args.data.size, + keypair: args.data.sshkeypair + }; + + if (args.data.noderootdisksize != null && args.data.noderootdisksize != "" && args.data.noderootdisksize > 0) { + $.extend(data, { + noderootdisksize: args.data.noderootdisksize + }); + } + + if (args.data.supportPrivateRegistry) { + $.extend(data, { + dockerregistryusername: args.data.username, + dockerregistrypassword: args.data.password, + dockerregistryurl: args.data.url, + dockerregistryemail: args.data.email + }); + } + + if (args.data.network != null && args.data.network.length > 0) { + $.extend(data, { + networkid: args.data.network + }); + } + $.ajax({ + url: createURL('createKubernetesCluster'), + data: data, + success: function(json) { + var jid = json.createkubernetesclusterresponse.jobid; + args.response.success({ + _custom: { + jobId: jid + } + }); + }, + error: function(XMLHttpResponse) { + var errorMsg = parseXMLHttpResponse(XMLHttpResponse); + args.response.error(errorMsg); + } + }); + }, + + + messages: { + notification: function(args) { + return 'Kubernetes Cluster Add'; + } + }, + notification: { + poll: pollAsyncJobResult + } + } + }, + dataProvider: function(args) { + var data = { + page: args.page, + pagesize: pageSize + }; + listViewDataProvider(args, data); + if (args.filterBy != null) { //filter dropdown + if (args.filterBy.kind != null) { + switch (args.filterBy.kind) { + case "all": + break; + case "running": + $.extend(data, { + state: 'Running' + }); + break; + case "stopped": + $.extend(data, { + state: 'Stopped' + }); + break; + case "destroyed": + $.extend(data, { + state: 'Destroyed' + }); + break; + } + } + } + + $.ajax({ + url: createURL("listKubernetesClusters"), + data: data, + dataType: "json", + sync: true, + success: function(json) { + var items = []; + if (json.listkubernetesclustersresponse.kubernetescluster != null) { + items = json.listkubernetesclustersresponse.kubernetescluster; + } + args.response.success({ + actionFilter: cksActionfilter, + data: items + }); + } + }); + }, + + detailView: { + name: 'Kubernetes cluster details', + isMaximized: true, + actions: { + start: { + label: 'Start Kubernetes Cluster', + action: function(args) { + $.ajax({ + url: createURL("startKubernetesCluster"), + data: {"id": args.context.kubernetesclusters[0].id}, + dataType: "json", + async: true, + success: function(json) { + var jid = json.startkubernetesclusterresponse.jobid; + args.response.success({ + _custom: { + jobId: jid + } + }); + } + }); + }, + messages: { + confirm: function(args) { + return 'Please confirm that you want to start this Kubernetes cluster.'; + }, + notification: function(args) { + return 'Started Kubernetes cluster.'; + } + }, + notification: { + poll: pollAsyncJobResult + } + }, + stop: { + label: 'Stop Kubernetes Cluster', + action: function(args) { + $.ajax({ + url: createURL("stopKubernetesCluster"), + data: {"id": args.context.kubernetesclusters[0].id}, + dataType: "json", + async: true, + success: function(json) { + var jid = json.stopkubernetesclusterresponse.jobid; + args.response.success({ + _custom: { + jobId: jid + } + }); + } + }); + }, + messages: { + confirm: function(args) { + return 'Please confirm that you want to stop this Kubernetes cluster.'; + }, + notification: function(args) { + return 'Stopped Kubernetes cluster.'; + } + }, + notification: { + poll: pollAsyncJobResult + } + }, + destroy: { + label: 'Destroy Cluster', + compactLabel: 'label.destroy', + createForm: { + title: 'Destroy Kubernetes Cluster', + desc: 'Destroy Kubernetes Cluster', + isWarning: true, + fields: { + } + }, + messages: { + confirm: function(args) { + return 'Please confirm that you want to destroy this Kubernetes cluster.'; + }, + notification: function(args) { + return 'Destroyed Kubernetes cluster.'; + } + }, + action: function(args) { + var data = { + id: args.context.kubernetesclusters[0].id, + expunge: true + }; + $.ajax({ + url: createURL('deleteKubernetesCluster'), + data: data, + dataType: "json", + async: true, + success: function(json) { + args.response.success({ + _custom: { + jobId: json.deletecontaierclusterresponse.jobid, + getUpdatedItem: function(json) { + return { 'toRemove': true }; + } + } + }); + } + }); + }, + notification: { + poll: pollAsyncJobResult + } + }, + downloadKubernetesClusterKubeConfig: { + label: 'Download Kubernetes Cluster Config', + messages: { + notification: function(args) { + return 'Download Kubernetes Cluster Config'; + } + }, + action: function(args) { + var data = { + id: args.context.kubernetesclusters[0].id + } + $.ajax({ + url: createURL("getKubernetesClusterConfig"), + dataType: "json", + data: data, + async: false, + success: function(json) { + var jsonObj; + if (json.getkubernetesclusterconfigresponse.clusterconfig != null && + json.getkubernetesclusterconfigresponse.clusterconfig.configdata != null ) { + jsonObj = json.getkubernetesclusterconfigresponse.clusterconfig; + clusterKubeConfig = jsonObj.configdata ; + } + } + }); + downloadClusterKubeConfig(); + args.response.success({}); + }, + notification: { + poll: function(args) { + args.complete(); + } + } + }, + scaleKubernetesCluster: { + label: 'Scale Kubernetes Cluster', + messages: { + notification: function(args) { + return 'Scale Kubernetes Cluster'; + } + }, + createForm: { + title: 'Scale Kubernetes Cluster', + desc: '', + preFilter: function(args) { + var options = args.$form.find('.form-item[rel=serviceoffering]').find('option'); + $.each(options, function(optionIndex, option) { + if ($(option).val() === args.context.kubernetesclusters[0].serviceofferingid) { + $(option).attr('selected','selected'); + } + }); + }, + fields: { + serviceoffering: { + label: 'label.menu.service.offerings', + //docID: 'helpKubernetesClusterServiceOffering', + validation: { + required: true + }, + select: function(args) { + $.ajax({ + url: createURL("listServiceOfferings"), + dataType: "json", + async: true, + success: function(json) { + var offeringObjs = []; + var items = json.listserviceofferingsresponse.serviceoffering; + if (items != null) { + for (var i = 0; i < items.length; i++) { + offeringObjs.push({ + id: items[i].id, + description: items[i].name + }); + } + } + args.response.success({ + data: offeringObjs + }); + } + }); + } + }, + size: { + label: 'Cluster size', + //docID: 'helpKubernetesClusterSize', + validation: { + required: true, + number: true + }, + } + } + }, + action: function(args) { + var data = { + id: args.context.kubernetesclusters[0].id, + serviceofferingid: args.data.serviceoffering, + size: args.data.size + }; + $.ajax({ + url: createURL('scaleKubernetesCluster'), + data: data, + dataType: "json", + success: function (json) { + var jid = json.scalekubernetesclusterresponse.jobid; + args.response.success({ + _custom: { + jobId: jid, + getActionFilter: function() { + return cksActionfilter; + } + } + }); + } + }); //end ajax + }, + notification: { + poll: pollAsyncJobResult + } + } + }, + tabs: { + // Details tab + details: { + title: 'label.details', + fields: [{ + id: { + label: 'label.id' + }, + name: { + label: 'label.name' + }, + zonename: { + label: 'label.zone.name' + }, + size : { + label: 'Cluster Size' + }, + cpunumber: { + label: 'label.num.cpu.cores' + }, + memory: { + label: 'label.memory.mb' + }, + state: { + label: 'label.state', + }, + serviceofferingname: { + label: 'label.compute.offering' + }, + associatednetworkname: { + label: 'label.network' + }, + keypair: { + label: 'label.ssh.key.pair' + }, + endpoint: { + label: 'API endpoint', + isCopyPaste: true + }, + consoleendpoint: { + label: 'Dashboard endpoint', + isCopyPaste: true + }, + username: { + label: 'label.username', + isCopyPaste: true + }, + password: { + label: 'label.password', + isCopyPaste: true + } + }], + + dataProvider: function(args) { + $.ajax({ + url: createURL("listKubernetesClusters&id=" + args.context.kubernetesclusters[0].id), + dataType: "json", + async: true, + success: function(json) { + var jsonObj; + if (json.listkubernetesclustersresponse.kubernetescluster != null && json.listkubernetesclustersresponse.kubernetescluster.length > 0) { + jsonObj = json.listkubernetesclustersresponse.kubernetescluster[0]; + } + args.response.success({ + actionFilter: cksActionfilter, + data: jsonObj + }); + } + }); + } + }, + console : { + title: 'Access', + custom : function (args) { + var showDashboard = function() { + var state = args.context.kubernetesclusters[0].state; + + if (state == "Created" || state == "Starting") { // Starting + return jQuery('

').html("Kubernetes cluster setup is under progress, please check again in few minutes."); + } + + if (state == "Stopped" || state == "Stopping") { // Starting + return jQuery('

').html("Kubernetes cluster setup is under progress, please check again in few minutes."); + } + + if (state == "Running") { // Running + var data = { + id: args.context.kubernetesclusters[0].id + } + $.ajax({ + url: createURL("getKubernetesClusterConfig"), + dataType: "json", + data: data, + async: true, + success: function(json) { + var jsonObj; + if (json.getkubernetesclusterconfigresponse.clusterconfig != null && + json.getkubernetesclusterconfigresponse.clusterconfig.configdata != null ) { + jsonObj = json.getkubernetesclusterconfigresponse.clusterconfig; + clusterKubeConfig = jsonObj.configdata ; + args.response.success({ + data: jsonObj + }); + } + } + }); + return jQuery('

').html("Access Kubernetes cluster
Download Config File

Use kubectl
kubectl --kubeconfig /custom/path/kube.config {COMMAND}

List pods
kubectl --kubeconfig /custom/path/kube.config get pods --all-namespaces
Access dashboard web UI
Run proxy locally
kubectl --kubeconfig /custom/path/kube.config proxy
Open URL in browser
http://localhost:8001/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/"); + // return jQuery('

').html("Access Kubernetes cluster
Download Config File

How to do this
kubectl --kubeconfig /custom/path/kube.config get pods"); + } + + return jQuery('

').html("Kubernetes cluster is not in a stable state, please check again in few minutes."); + }; + return showDashboard(); + } + }, + clusterinstances: { + title: 'Instances', + listView: { + section: 'clusterinstances', + fields: { + name: { + label: 'label.name', + truncate: true + }, + instancename: { + label: 'label.internal.name' + }, + displayname: { + label: 'label.display.name', + truncate: true + }, + ipaddress: { + label: 'label.ip.address' + }, + zonename: { + label: 'label.zone.name' + }, + state: { + label: 'label.state', + indicator: { + 'Running': 'on', + 'Stopped': 'off', + 'Destroyed': 'off', + 'Error': 'off' + } + } + }, + dataProvider: function(args) { + var data = {}; + listViewDataProvider(args, data); + + $.ajax({ + url: createURL("listKubernetesClusters"), + data: {"id": args.context.kubernetesclusters[0].id}, + success: function(json) { + var items = json.listkubernetesclustersresponse.kubernetescluster; + + var vmlist = []; + $.each(items, function(idx, item) { + if ("virtualmachineids" in item) { + vmlist = vmlist.concat(item.virtualmachineids); + } + }); + + $.extend(data, { + ids: vmlist.join() + }); + + if (data.ids.length == 0) { + args.response.success({ + data: [] + }); + } else { + $.ajax({ + url: createURL('listVirtualMachines'), + data: data, + success: function(json) { + var items = json.listvirtualmachinesresponse.virtualmachine; + if (items) { + $.each(items, function(idx, vm) { + if (vm.nic && vm.nic.length > 0 && vm.nic[0].ipaddress) { + items[idx].ipaddress = vm.nic[0].ipaddress; + } + }); + } + args.response.success({ + data: items + }); + }, + error: function(XMLHttpResponse) { + cloudStack.dialog.notice({ + message: parseXMLHttpResponse(XMLHttpResponse) + }); + args.response.error(); + } + }); + } + } + }); + }, + } + }, + firewall: { + title: 'label.firewall', + custom: function(args) { + $.ajax({ + url: createURL('listNetworks'), + data: {id: args.context.kubernetesclusters[0].networkid, listAll: true}, + async: false, + dataType: "json", + success: function(json) { + var network = json.listnetworksresponse.network; + $.extend(args.context, {"networks": [network]}); + } + }); + + $.ajax({ + url: createURL('listPublicIpAddresses'), + data: {associatedNetworkId: args.context.kubernetesclusters[0].networkid, listAll: true, forvirtualnetwork: true}, + async: false, + dataType: "json", + success: function(json) { + var ips = json.listpublicipaddressesresponse.publicipaddress; + var fwip = ips[0]; + $.each(ips, function(idx, ip) { + if (ip.issourcenat || ip.isstaticnat) { + fwip = ip; + return false; + } + }); + $.extend(args.context, {"ipAddresses": [fwip]}); + } + }); + return cloudStack.sections.network.sections.ipAddresses.listView.detailView.tabs.ipRules.custom(args); + }, + }, + } + } + } + }, + kubernetesversions: { + id: 'kubernetesversions', + type: 'select', + title: "Versions", + listView: { + fields: { + name: { + label: 'label.name' + }, + zonename: { + label: 'label.zone.name' + }, + isoname: { + label: 'ISO Name' + }, + isostate: { + label: 'ISO State' + } + }, + advSearchFields: { + name: { + label: 'label.name' + }, + zoneid: { + label: 'label.zone', + select: function(args) { + $.ajax({ + url: createURL('listZones'), + data: { + listAll: true + }, + success: function(json) { + var zones = json.listzonesresponse.zone ? json.listzonesresponse.zone : []; + + args.response.success({ + data: $.map(zones, function(zone) { + return { + id: zone.id, + description: zone.name + }; + }) + }); + } + }); + } + }, + }, + // List view actions + actions: { + add: { + label: 'Add Kubernetes version', + preFilter: function(args) { return isAdmin(); }, + createForm: { + title: 'Add Kubernetes version', + preFilter: cloudStack.preFilter.createTemplate, + fields: { + name: { + label: 'label.name', + //docID: 'Name of the cluster', + validation: { + required: true + } + }, + zone: { + label: 'label.zone', + //docID: 'helpKubernetesClusterZone', + validation: { + required: true + }, + select: function(args) { + $.ajax({ + url: createURL("listZones&available=true"), + dataType: "json", + async: true, + success: function(json) { + var items = []; + var zoneObjs = json.listzonesresponse.zone; + if (zoneObjs != null) { + for (var i = 0; i < zoneObjs.length; i++) { + items.push({ + id: zoneObjs[i].id, + description: zoneObjs[i].name + }); + } + } + items.sort(function(a, b) { + return a.description.localeCompare(b.description); + }); + items.unshift({ + id: -1, + description: "All Zones" + }); + args.response.success({ + data: items + }); + } + }); + } + }, + isoid: { + label: 'label.iso', + //docID: 'helpKubernetesClusterZone', + validation: { + required: true + }, + select: function(args) { + $.ajax({ + url: createURL("listIsos&ispublic=true&bootable=false"), + data: { zoneid: args.zone }, + dataType: "json", + async: true, + success: function(json) { + var items = []; + var isoObjs = json.listisosresponse.iso;; + if (isoObjs != null) { + for (var i = 0; i < isoObjs.length; i++) { + items.push({ + id: isoObjs[i].id, + description: isoObjs[i].name + }); + } + } + items.sort(function(a, b) { + return a.description.localeCompare(b.description); + }); + items.unshift({ + id: -1, + description: "Add new ISO" + }); + args.response.success({ + data: items + }); + } + }); + + args.$select.change(function() { + var $form = $(this).closest('form'); + var currentIsoId = $(this).val(); + if (currentIsoId < 0) { + $form.find('.form-item[rel=isourl]').css('display', 'inline-block'); + $form.find('.form-item[rel=isochecksum]').css('display', 'inline-block'); + } else { + $form.find('.form-item[rel=isourl]').hide(); + $form.find('.form-item[rel=isochecksum]').hide(); + } + }); + } + }, + isourl: { + label: 'label.url', + //docID: 'Name of the cluster', + isHidden: true + }, + isochecksum: { + label: 'label.checksum', + //docID: 'Name of the cluster', + isHidden: true + }, + } + }, + + action: function(args) { + var data = { + name: args.data.name, + }; + if (args.data.zone > 0) { + $.extend(data, { + zoneid: args.data.zone + }); + } + if (args.data.isoid < 0) { + if (args.data.isourl == null || args.data.isourl == '') { + cloudStack.dialog.notice({ + message: 'ISO URL is required to a new ISO' //_l('') + }); + return; + } + $.extend(data, { + url: args.data.isourl, + checksum: args.data.isochecksum + }); + } else { + $.extend(data, { + isoid: args.data.isoid + }); + } + $.ajax({ + url: createURL('addKubernetesSupportedVersion'), + data: data, + success: function(json) { + var jid = json.addKubernetesSupportedVersion.jobid; + args.response.success({ + _custom: { + jobId: jid + } + }); + }, + error: function(XMLHttpResponse) { + var errorMsg = parseXMLHttpResponse(XMLHttpResponse); + args.response.error(errorMsg); + } + }); + }, + + messages: { + notification: function(args) { + return 'Kubernetes Supported Version Add'; + } + }, + notification: { + poll: pollAsyncJobResult + } + } + }, + dataProvider: function(args) { + var data = { + page: args.page, + pagesize: pageSize + }; + listViewDataProvider(args, data); + $.ajax({ + url: createURL("listKubernetesSupportedVersions"), + data: data, + dataType: "json", + sync: true, + success: function(json) { + var items = []; + if (json.listkubernetessupportedversionsresponse.kubernetessupportedversion != null) { + items = json.listkubernetessupportedversionsresponse.kubernetessupportedversion; + } + args.response.success({ + data: items + }); + } + }); + }, + + detailView: { + name: 'Kubernetes version details', + isMaximized: true, + actions: { + destroy: { + label: 'Destroy Version', + compactLabel: 'label.destroy', + preFilter: function(args) { return isAdmin(); }, + createForm: { + title: 'Destroy Kubernetes Version', + desc: 'Destroy Kubernetes Version', + isWarning: true, + fields: { + } + }, + messages: { + confirm: function(args) { + return 'Please confirm that you want to destroy this Kubernetes version.'; + }, + notification: function(args) { + return 'Destroyed Kubernetes version.'; + } + }, + action: function(args) { + var data = { + id: args.context.kubernetesversions[0].id + }; + $.ajax({ + url: createURL('deleteKubernetesSupportedVersion'), + data: data, + dataType: "json", + async: true, + success: function(json) { + args.response.success({ + _custom: { + jobId: json.deletekubernetessupportedversionresponse.jobid, + getUpdatedItem: function(json) { + return { 'toRemove': true }; + } + } + }); + } + }); + }, + notification: { + poll: pollAsyncJobResult + } + } + }, + tabs: { + // Details tab + details: { + title: 'label.details', + fields: [{ + id: { + label: 'label.id' + }, + name: { + label: 'label.name' + }, + zonename: { + label: 'label.zone.name' + }, + isoid: { + label: 'ISO ID' + }, + isoname: { + label: 'ISO Name' + }, + isostate: { + label: 'ISO State' + } + }], + + dataProvider: function(args) { + $.ajax({ + url: createURL("listKubernetesSupportedVersions&id=" + args.context.kubernetesversions[0].id), + dataType: "json", + async: true, + success: function(json) { + var jsonObj; + if (json.listkubernetessupportedversionsresponse.kubernetessupportedversion != null && json.listkubernetessupportedversionsresponse.kubernetessupportedversion.length > 0) { + jsonObj = json.listkubernetessupportedversionsresponse.kubernetessupportedversion[0]; + } + args.response.success({ + data: jsonObj + }); + } + }); + } + } + } + } + } + }, + } + }); + }; + + var cksActionfilter = cloudStack.actionFilter.cksActionfilter = function(args) { + var jsonObj = args.context.item; + var allowedActions = []; + if (jsonObj.state != "Destroyed" && jsonObj.state != "Destroying") { + if (jsonObj.state == "Stopped") { + allowedActions.push("start"); + } else { + allowedActions.push("downloadKubernetesClusterKubeConfig"); + allowedActions.push("stop"); + } + if (jsonObj.state == "Created" || jsonObj.state == "Running") { + allowedActions.push("scaleKubernetesCluster"); + } + allowedActions.push("destroy"); + } + return allowedActions; + } + +}(cloudStack)); diff --git a/ui/plugins/cks/config.js b/ui/plugins/cks/config.js new file mode 100644 index 000000000000..a5ea16358b18 --- /dev/null +++ b/ui/plugins/cks/config.js @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +(function (cloudStack) { + cloudStack.plugins.cks.config = { + title: 'Kubernetes Service', + desc: 'Kubernetes Service', + externalLink: 'http://www.cloudstack.org/', + authorName: 'Apache CloudStack', + authorEmail: 'dev@cloudstack.apache.org' + }; +}(cloudStack)); diff --git a/ui/plugins/cks/icon.png b/ui/plugins/cks/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..1d049675c27178c9dac8ae5879c07963fd84c7a0 GIT binary patch literal 1208 zcmV;p1V{UcP)Sg)WwqAOj5q3NU;1Zb1AaZo4{5rx1-z&1%8 zy~f4>3xR3Cc;J}S@FBa|7GR~MClB-P@j!{BPy0iWlYrNO5@4#Nt?sV@mYUoN_L}59 zcCiueKLC6L%#*a)^Unfa13m|;?3S4-G8Cu-8i5}G0T%#w0mDt|6+kmk3Oonw2Cf5! z0CRxp=4Tf$+v6=(dM@zvzvWcH|KLFx5Wt20sDr?}z#Wn{SY`_FCa?!MU(y!$&j9WL z&akxr)&ljCzUkMfV{43xh{eFYDY;??i2e`15pf#uv3I)>n2{p-qjsGd;34;21#ELW`RKQt zEN}4GBuPK_w?(F#oNba80oMZU9-9ZO0T!j?8y%GrCEmB9^Ub00z|?_ot;v4AVZZ+j zxYAa_32|&negaVD|E~ZuCGC>b={bc!`9LUA>gW0tiQ9mi9hPwdoDVEZ6^vg3wUTzF z-nGfkvVN_8olA<`{5B$rTu^MQ%MG}`BJrHDH3tSA` z=ZrlQIJuYD^GRbN4umb^u$w%VUAZQ?*rscHnyFoQc3G z{#_0nVVU2YWIKR$lIr`KBZa`%z;fUn;B4SwU>`6$y?YTc*%=#Pzvm{r+9c_Bmo7v^ zg%#NkGytCfmjY9O5T#EM5-W>Y_2h?dHX9;2PjO-~n4|0dRjx_BLB% zqon2T`ye73Q*~Ij2djvvG>P{C)sj{@I=u%R4?H7jmd`NjoqIL|lO-LDhz8(67o4?{ zRt(q)F(R5=n(uJ7zr?%06PT79k|LrGc)~7J>4LJwF4!!oZlKTX4Yo=t@HTL#`WeI`~kcO%#qYJs4gEOq8XU= z55-@QR6Xc!Q6i$i74JCToJ?}YP722zz*i2h3!Hy0lho4}>-XhN<4w*7Rgzu;S|Vap zs($KD&KeQ1%NH2+z+;2PA`x+d&ld-Q3KxQZCqEik=W^#%}#m%0000 Date: Tue, 19 Nov 2019 14:55:51 +0530 Subject: [PATCH 002/134] made script dynamic Signed-off-by: Abhishek Kumar --- .../scripts/create-binaries-iso.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/plugins/integrations/kubernetes-service/scripts/create-binaries-iso.sh b/plugins/integrations/kubernetes-service/scripts/create-binaries-iso.sh index 41a5ef690f8a..b70fc8e3cc77 100644 --- a/plugins/integrations/kubernetes-service/scripts/create-binaries-iso.sh +++ b/plugins/integrations/kubernetes-service/scripts/create-binaries-iso.sh @@ -16,19 +16,26 @@ # specific language governing permissions and limitations # under the License. -RELEASE="v1.11.4" + +echo "Params $#" +if [ $# -lt 3 ]; then + echo "Invalid input. Valid usage: ./create-binaries-iso KUBERNETES_VERSION CNI_VERSION CRICTL_VERSION, eg: ./create-binaries-iso 1.11.4 0.7.1 1.11.1" + exit 1 +fi + +RELEASE="v${1}" start_dir="$PWD" iso_dir="${start_dir}/iso" working_dir="${iso_dir}/${RELEASE}" mkdir -p "${working_dir}" -CNI_VERSION="v0.7.1" +CNI_VERSION="v${2}" echo "Downloading CNI ${CNI_VERSION}..." cni_dir="${working_dir}/cni/${CNI_VERSION}" mkdir -p "${cni_dir}" curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" -o "${cni_dir}/cni-plugins-amd64-${CNI_VERSION}.tgz" -CRICTL_VERSION="v1.11.1" +CRICTL_VERSION="v${3}" echo "Downloading CRI tools ${CRICTL_VERSION}..." crictl_dir="${working_dir}/cri-tools/${CRICTL_VERSION}" mkdir -p "${crictl_dir}" @@ -76,4 +83,4 @@ chmod ${kubeadm_file_permissions} "${working_dir}/k8s/kubeadm" mkisofs -o setup.iso -J -R -l "${iso_dir}" -rm -rf "${iso_dir}" +rm -rf "${iso_dir}" \ No newline at end of file From 4fba0e8361667b0247fbc6d62ae0a77132733d15 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 19 Nov 2019 14:56:17 +0530 Subject: [PATCH 003/134] fix for delete cluster API response parsing Signed-off-by: Abhishek Kumar --- ui/plugins/cks/cks.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/plugins/cks/cks.js b/ui/plugins/cks/cks.js index 03e45ed630b3..56f453b715e5 100644 --- a/ui/plugins/cks/cks.js +++ b/ui/plugins/cks/cks.js @@ -566,7 +566,7 @@ success: function(json) { args.response.success({ _custom: { - jobId: json.deletecontaierclusterresponse.jobid, + jobId: json.deletekubernetesclusterresponse.jobid, getUpdatedItem: function(json) { return { 'toRemove': true }; } From a11e523cc98f776883daa40bff51911b7f45a74c Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 27 Nov 2019 12:38:58 +0530 Subject: [PATCH 004/134] kubernetes version - added semantic version field Fixed, renamed Kubernetes binaries ISO script Signed-off-by: Abhishek Kumar --- .../apache/cloudstack/api/ApiConstants.java | 5 +- .../META-INF/db/schema-41300to41400.sql | 1 + ...o.sh => create-kubernetes-binaries-iso.sh} | 26 +++++-- .../KubernetesSupportedVersion.java | 1 + .../KubernetesSupportedVersionVO.java | 15 ++++- .../KubernetesVersionManagerImpl.java | 67 ++++++++++++++++--- .../AddKubernetesSupportedVersionCmd.java | 16 +++++ .../DeleteKubernetesSupportedVersionCmd.java | 4 +- .../ListKubernetesSupportedVersionsCmd.java | 13 ++++ .../KubernetesSupportedVersionResponse.java | 12 ++++ ui/plugins/cks/cks.js | 33 +++++++-- 11 files changed, 167 insertions(+), 26 deletions(-) rename plugins/integrations/kubernetes-service/scripts/{create-binaries-iso.sh => create-kubernetes-binaries-iso.sh} (74%) diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 4c274cdde3f1..c95fcdea3522 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -742,12 +742,15 @@ public class ApiConstants { public static final String TARGET_ID = "targetid"; public static final String VOLUME_IDS = "volumeids"; - public static final String KUBERNETES_VERSION_ID = "kubernetesversionid"; public static final String CONSOLE_END_POINT = "consoleendpoint"; + public static final String DELETE_ISO = "deleteiso"; public static final String DOCKER_REGISTRY_USER_NAME = "dockerregistryusername"; public static final String DOCKER_REGISTRY_PASSWORD = "dockerregistrypassword"; public static final String DOCKER_REGISTRY_URL = "dockerregistryurl"; public static final String DOCKER_REGISTRY_EMAIL = "dockerregistryemail"; + public static final String KUBERNETES_VERSION = "kubernetesversion"; + public static final String KUBERNETES_VERSION_ID = "kubernetesversionid"; + public static final String MIN_KUBERNETES_VERSION = "minimumkubernetesversion"; public static final String NODE_ROOT_DISK_SIZE = "noderootdisksize"; public enum HostDetails { diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql b/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql index 63c0384c2ac9..7a84e855b00e 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql @@ -91,6 +91,7 @@ CREATE TABLE IF NOT EXISTS `cloud`.`kubernetes_supported_version` ( `id` bigint unsigned NOT NULL auto_increment COMMENT 'id', `uuid` varchar(40) DEFAULT NULL COMMENT 'uuid', `name` varchar(255) NOT NULL COMMENT 'kubernetes version name', + `kubernetes_version` varchar(32) NOT NULL COMMENT 'kubernetes semantic version', `iso_id` bigint unsigned NOT NULL COMMENT 'kubernetes version binary ISO id', `zone_id` bigint unsigned DEFAULT NULL COMMENT 'zone id in which kubernetes version is available', `created` datetime NOT NULL COMMENT 'date created', diff --git a/plugins/integrations/kubernetes-service/scripts/create-binaries-iso.sh b/plugins/integrations/kubernetes-service/scripts/create-kubernetes-binaries-iso.sh similarity index 74% rename from plugins/integrations/kubernetes-service/scripts/create-binaries-iso.sh rename to plugins/integrations/kubernetes-service/scripts/create-kubernetes-binaries-iso.sh index b70fc8e3cc77..2a69736e972f 100644 --- a/plugins/integrations/kubernetes-service/scripts/create-binaries-iso.sh +++ b/plugins/integrations/kubernetes-service/scripts/create-kubernetes-binaries-iso.sh @@ -18,28 +18,29 @@ echo "Params $#" -if [ $# -lt 3 ]; then - echo "Invalid input. Valid usage: ./create-binaries-iso KUBERNETES_VERSION CNI_VERSION CRICTL_VERSION, eg: ./create-binaries-iso 1.11.4 0.7.1 1.11.1" +if [ $# -lt 5 ]; then + echo "Invalid input. Valid usage: ./create-kubernetes-binaries-iso KUBERNETES_VERSION CNI_VERSION CRICTL_VERSION WEAVENET_NETWORK_YAML_CONFIG DASHBOARD_YAML_CONFIG" + echo "eg: ./create-kubernetes-binaries-iso 1.11.4 0.7.1 1.11.1 https://github.com/weaveworks/weave/releases/download/latest_release/weave-daemonset-k8s-1.11.yaml https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.0/src/deploy/recommended/kubernetes-dashboard.yaml" exit 1 fi RELEASE="v${1}" start_dir="$PWD" iso_dir="${start_dir}/iso" -working_dir="${iso_dir}/${RELEASE}" +working_dir="${iso_dir}/" mkdir -p "${working_dir}" CNI_VERSION="v${2}" echo "Downloading CNI ${CNI_VERSION}..." -cni_dir="${working_dir}/cni/${CNI_VERSION}" +cni_dir="${working_dir}/cni/" mkdir -p "${cni_dir}" -curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" -o "${cni_dir}/cni-plugins-amd64-${CNI_VERSION}.tgz" +curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" -o "${cni_dir}/cni-plugins-amd64.tgz" CRICTL_VERSION="v${3}" echo "Downloading CRI tools ${CRICTL_VERSION}..." -crictl_dir="${working_dir}/cri-tools/${CRICTL_VERSION}" +crictl_dir="${working_dir}/cri-tools/" mkdir -p "${crictl_dir}" -curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" -o "${crictl_dir}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" +curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" -o "${crictl_dir}/crictl-linux-amd64.tar.gz" echo "Downloading Kubernetes tools ${RELEASE}..." k8s_dir="${working_dir}/k8s" @@ -54,11 +55,22 @@ cd $start_dir kubelet_service_file="${working_dir}/kubelet.service" touch "${kubelet_service_file}" curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > ${kubelet_service_file} + echo "Downloading 10-kubeadm.conf ${RELEASE}..." kubeadm_conf_file="${working_dir}/10-kubeadm.conf" touch "${kubeadm_conf_file}" curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > ${kubeadm_conf_file} +NETWORK_CONFIG_URL="${4}" +echo "Downloading network config ${NETWORK_CONFIG_URL}" +network_conf_file="${working_dir}/network.yaml" +curl -sSL ${NETWORK_CONFIG_URL} -o ${network_conf_file} + +DASHBORAD_CONFIG_URL="${5}" +echo "Downloading dashboard config ${DASHBORAD_CONFIG_URL}" +dashboard_conf_file="${working_dir}/dashboard.yaml" +curl -sSL ${DASHBORAD_CONFIG_URL} -o ${dashboard_conf_file} + echo "Fetching k8s docker images..." docker -v if [ $? -ne 0 ]; then diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesSupportedVersion.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesSupportedVersion.java index da74cd83b4ab..3706f43b6172 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesSupportedVersion.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesSupportedVersion.java @@ -27,6 +27,7 @@ public interface KubernetesSupportedVersion extends InternalIdentity, Identity { long getId(); String getName(); + String getKubernetesVersion(); long getIsoId(); Long getZoneId(); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesSupportedVersionVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesSupportedVersionVO.java index 00a26a4481c3..d857d9aab097 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesSupportedVersionVO.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesSupportedVersionVO.java @@ -43,6 +43,9 @@ public class KubernetesSupportedVersionVO implements KubernetesSupportedVersion @Column(name = "name") private String name; + @Column(name = "kubernetes_version") + private String kubernetesVersion; + @Column(name = "iso_id") private long isoId; @@ -59,9 +62,10 @@ public KubernetesSupportedVersionVO() { this.uuid = UUID.randomUUID().toString(); } - public KubernetesSupportedVersionVO(String name, long isoId, Long zoneId) { + public KubernetesSupportedVersionVO(String name, String kubernetesVersion, long isoId, Long zoneId) { this.uuid = UUID.randomUUID().toString(); this.name = name; + this.kubernetesVersion = kubernetesVersion; this.isoId = isoId; this.zoneId = zoneId; } @@ -85,6 +89,15 @@ public void setName(String name) { this.name = name; } + @Override + public String getKubernetesVersion() { + return kubernetesVersion; + } + + public void setKubernetesVersion(String kubernetesVersion) { + this.kubernetesVersion = kubernetesVersion; + } + @Override public long getIsoId() { return isoId; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java index 4870d58ec74a..43f117d98be8 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java @@ -75,6 +75,7 @@ private KubernetesSupportedVersionResponse createKubernetesSupportedVersionRespo response.setObjectName("kubernetessupportedversion"); response.setId(kubernetesSupportedVersion.getUuid()); response.setName(kubernetesSupportedVersion.getName()); + response.setKubernetesVersion(kubernetesSupportedVersion.getKubernetesVersion()); DataCenterVO zone = ApiDBUtils.findZoneById(kubernetesSupportedVersion.getZoneId()); if (zone != null) { response.setZoneId(zone.getUuid()); @@ -87,21 +88,68 @@ private KubernetesSupportedVersionResponse createKubernetesSupportedVersionRespo return response; } + public static int compareKubernetesVersion(String v1, String v2) throws IllegalArgumentException { + if (Strings.isNullOrEmpty(v1) || Strings.isNullOrEmpty(v2)) { + throw new IllegalArgumentException(String.format("Invalid version comparision with versions %s, %s", v1, v2)); + } + if(!v1.matches("[0-9]+(\\.[0-9]+)*")) { + throw new IllegalArgumentException(String.format("Invalid version format, %s", v1)); + } + if(!v2.matches("[0-9]+(\\.[0-9]+)*")) { + throw new IllegalArgumentException(String.format("Invalid version format, %s", v2)); + } + String[] thisParts = v1.split("\\."); + String[] thatParts = v2.split("\\."); + int length = Math.max(thisParts.length, thatParts.length); + for(int i = 0; i < length; i++) { + int thisPart = i < thisParts.length ? + Integer.parseInt(thisParts[i]) : 0; + int thatPart = i < thatParts.length ? + Integer.parseInt(thatParts[i]) : 0; + if(thisPart < thatPart) + return -1; + if(thisPart > thatPart) + return 1; + } + return 0; + } + @Override public ListResponse listKubernetesSupportedVersions(final ListKubernetesSupportedVersionsCmd cmd) { final Long versionId = cmd.getId(); + final Long zoneId = cmd.getZoneId(); + final String minimumKubernetesVersion = cmd.getMinimumKubernetesVersion(); List responseList = new ArrayList<>(); + List versions = new ArrayList<>(); if (versionId != null) { KubernetesSupportedVersionVO version = kubernetesSupportedVersionDao.findById(versionId); - if (version != null) { - responseList.add(createKubernetesSupportedVersionResponse(version)); + if (version != null && (zoneId == null || version.getZoneId() == null || version.getZoneId().equals(zoneId))) { + versions.add(version); } } else { - List versions = kubernetesSupportedVersionDao.listAll(); - for (KubernetesSupportedVersionVO version : versions) { - responseList.add(createKubernetesSupportedVersionResponse(version)); + if (zoneId == null) { + versions = kubernetesSupportedVersionDao.listAll(); + } else { + versions = kubernetesSupportedVersionDao.listAllInZone(zoneId); } } + // Filter versions for minimum Kubernetes version + if (!Strings.isNullOrEmpty(minimumKubernetesVersion)) { + for (int i = versions.size() - 1; i >= 0; --i) { + KubernetesSupportedVersionVO version = versions.get(i); + try { + if (compareKubernetesVersion(minimumKubernetesVersion, version.getKubernetesVersion()) > 0) { + versions.remove(i); + } + } catch (Exception e) { + LOGGER.warn(String.format("Unable to compare Kubernetes version for supported version ID: %s with %s", version.getUuid(), minimumKubernetesVersion)); + versions.remove(i); + } + } + } + for (KubernetesSupportedVersionVO version : versions) { + responseList.add(createKubernetesSupportedVersionResponse(version)); + } ListResponse response = new ListResponse<>(); response.setResponses(responseList); return response; @@ -111,6 +159,7 @@ public ListResponse listKubernetesSupportedV @ActionEvent(eventType = KubernetesVersionEventTypes.EVENT_KUBERNETES_VERSION_ADD, eventDescription = "Adding Kubernetes supported version") public KubernetesSupportedVersionResponse addKubernetesSupportedVersion(final AddKubernetesSupportedVersionCmd cmd) { final String name = cmd.getName(); + final String kubernetesVersion = cmd.getKubernetesVersion(); final Long zoneId = cmd.getZoneId(); final Long isoId = cmd.getIsoId(); final String isoUrl = cmd.getUrl(); @@ -136,7 +185,7 @@ public KubernetesSupportedVersionResponse addKubernetesSupportedVersion(final Ad String isoName = String.format("%s-Kubernetes-Binaries-ISO", name); RegisterIsoCmd registerIsoCmd = new RegisterIsoCmd(); registerIsoCmd = ComponentContext.inject(registerIsoCmd); - Field f = registerIsoCmd.getClass().getDeclaredField("name"); + Field f = registerIsoCmd.getClass().getDeclaredField("isoName"); f.setAccessible(true); f.set(registerIsoCmd, isoName); f = registerIsoCmd.getClass().getDeclaredField("displayText"); @@ -180,7 +229,7 @@ public KubernetesSupportedVersionResponse addKubernetesSupportedVersion(final Ad } } } - KubernetesSupportedVersionVO supportedVersionVO = new KubernetesSupportedVersionVO(name, template.getId(), zoneId); + KubernetesSupportedVersionVO supportedVersionVO = new KubernetesSupportedVersionVO(name, kubernetesVersion, template.getId(), zoneId); supportedVersionVO = kubernetesSupportedVersionDao.persist(supportedVersionVO); return createKubernetesSupportedVersionResponse(supportedVersionVO); } @@ -189,7 +238,7 @@ public KubernetesSupportedVersionResponse addKubernetesSupportedVersion(final Ad @ActionEvent(eventType = KubernetesVersionEventTypes.EVENT_KUBERNETES_VERSION_DELETE, eventDescription = "Deleting Kubernetes supported version", async = true) public boolean deleteKubernetesSupportedVersion(final DeleteKubernetesSupportedVersionCmd cmd) { final Long versionId = cmd.getId(); - final Boolean isDeleteIso = cmd.isDeleteIso(); + final boolean isDeleteIso = cmd.isDeleteIso(); KubernetesSupportedVersion version = kubernetesSupportedVersionDao.findById(versionId); if (version == null) { throw new InvalidParameterValueException("Invalid Kubernetes version id specified"); @@ -199,7 +248,7 @@ public boolean deleteKubernetesSupportedVersion(final DeleteKubernetesSupportedV throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Unable to delete Kubernetes version ID: %s. Exisiting clusters currently using the version.", version.getUuid())); } - VMTemplateVO template = templateDao.findById(version.getId()); + VMTemplateVO template = templateDao.findById(version.getIsoId()); if (template == null) { LOGGER.warn(String.format("Unable to find ISO associated with supported Kubernetes version ID: %s", version.getUuid())); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/AddKubernetesSupportedVersionCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/AddKubernetesSupportedVersionCmd.java index 2569f36d3670..86511fa4b314 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/AddKubernetesSupportedVersionCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/AddKubernetesSupportedVersionCmd.java @@ -35,11 +35,13 @@ import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.NetworkRuleConflictException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.kubernetesversion.KubernetesSupportedVersion; import com.cloud.kubernetesversion.KubernetesVersionService; +import com.google.common.base.Strings; @APICommand(name = AddKubernetesSupportedVersionCmd.APINAME, description = "Add a supported Kubernetes version", @@ -61,6 +63,10 @@ public class AddKubernetesSupportedVersionCmd extends BaseCmd implements UserCmd description = "the name of the Kubernetes supported version") private String name; + @Parameter(name = ApiConstants.KUBERNETES_VERSION, type = CommandType.STRING, required = true, + description = "the semantic version of the Kubernetes") + private String kubernetesVersion; + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = ZoneResponse.class, description = "the ID of the zone in which Kubernetes supported version will be available") @@ -88,6 +94,16 @@ public String getName() { return name; } + public String getKubernetesVersion() { + if(Strings.isNullOrEmpty(kubernetesVersion)) { + throw new InvalidParameterValueException("Version can not be null"); + } + if(!kubernetesVersion.matches("[0-9]+(\\.[0-9]+)*")) { + throw new IllegalArgumentException("Invalid version format. Semantic version needed"); + } + return kubernetesVersion; + } + public Long getZoneId() { return zoneId; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/DeleteKubernetesSupportedVersionCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/DeleteKubernetesSupportedVersionCmd.java index 40c9157efd08..90f00dff9645 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/DeleteKubernetesSupportedVersionCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/DeleteKubernetesSupportedVersionCmd.java @@ -60,7 +60,7 @@ public class DeleteKubernetesSupportedVersionCmd extends BaseAsyncCmd { description = "the ID of the Kubernetes supported version", required = true) private Long id; - @Parameter(name = "deleteIso", type = CommandType.BOOLEAN, + @Parameter(name = ApiConstants.DELETE_ISO, type = CommandType.BOOLEAN, description = "true if ISO associated with the Kubernetes version to be deleted else false. Default is false") private Boolean deleteIso; @@ -72,7 +72,7 @@ public Long getId() { } public Boolean isDeleteIso() { - return deleteIso; + return deleteIso == null ? false : deleteIso; } @Override diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetesversion/ListKubernetesSupportedVersionsCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetesversion/ListKubernetesSupportedVersionsCmd.java index 57917323dfe0..2dd4f7f6e08d 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetesversion/ListKubernetesSupportedVersionsCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetesversion/ListKubernetesSupportedVersionsCmd.java @@ -37,6 +37,7 @@ import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.kubernetesversion.KubernetesVersionService; +import com.google.common.base.Strings; @APICommand(name = ListKubernetesSupportedVersionsCmd.APINAME, description = "Lists container clusters", @@ -63,6 +64,10 @@ public class ListKubernetesSupportedVersionsCmd extends BaseListCmd { description = "the ID of the zone in which Kubernetes supported version will be available") private Long zoneId; + @Parameter(name = ApiConstants.MIN_KUBERNETES_VERSION, type = CommandType.STRING, + description = "the minimum Kubernetes version") + private String minimumKubernetesVersion; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -74,6 +79,14 @@ public Long getZoneId() { return zoneId; } + public String getMinimumKubernetesVersion() { + if(!Strings.isNullOrEmpty(minimumKubernetesVersion) && + !minimumKubernetesVersion.matches("[0-9]+(\\.[0-9]+)*")) { + throw new IllegalArgumentException("Invalid version format"); + } + return minimumKubernetesVersion; + } + @Override public String getCommandName() { return APINAME.toLowerCase() + "response"; diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java index be704406f179..1203b488f25e 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java @@ -36,6 +36,10 @@ public class KubernetesSupportedVersionResponse extends BaseResponse { @Param(description = "Name of the Kubernetes supported version") private String name; + @SerializedName(ApiConstants.KUBERNETES_VERSION) + @Param(description = "Semantic Kubernetes version") + private String kubernetesVersion; + @SerializedName(ApiConstants.ISO_ID) @Param(description = "the id of the binaries ISO for Kubernetes supported version") private String isoId; @@ -72,6 +76,14 @@ public void setName(String name) { this.name = name; } + public String getKubernetesVersion() { + return kubernetesVersion; + } + + public void setKubernetesVersion(String kubernetesVersion) { + this.kubernetesVersion = kubernetesVersion; + } + public String getIsoId() { return isoId; } diff --git a/ui/plugins/cks/cks.js b/ui/plugins/cks/cks.js index 56f453b715e5..7e973af32956 100644 --- a/ui/plugins/cks/cks.js +++ b/ui/plugins/cks/cks.js @@ -948,6 +948,9 @@ name: { label: 'label.name' }, + kubernetesversion: { + label: 'Kubernetes version' + }, zonename: { label: 'label.zone.name' }, @@ -1002,6 +1005,13 @@ required: true } }, + version: { + label: 'Semantic version', + //docID: 'Name of the cluster', + validation: { + required: true + } + }, zone: { label: 'label.zone', //docID: 'helpKubernetesClusterZone', @@ -1103,6 +1113,7 @@ action: function(args) { var data = { name: args.data.name, + kubernetesversion: args.data.version, }; if (args.data.zone > 0) { $.extend(data, { @@ -1181,28 +1192,38 @@ isMaximized: true, actions: { destroy: { - label: 'Destroy Version', - compactLabel: 'label.destroy', + label: 'Delete Version', + compactLabel: 'label.delete', preFilter: function(args) { return isAdmin(); }, createForm: { - title: 'Destroy Kubernetes Version', - desc: 'Destroy Kubernetes Version', + title: 'Delete Kubernetes Version', + desc: 'Delete Kubernetes Version', isWarning: true, fields: { + deleteiso: { + label: 'Delete ISO', + isBoolean: true, + isChecked: false + }, } }, messages: { confirm: function(args) { - return 'Please confirm that you want to destroy this Kubernetes version.'; + return 'Please confirm that you want to delete this Kubernetes version.'; }, notification: function(args) { - return 'Destroyed Kubernetes version.'; + return 'Deleted Kubernetes version.'; } }, action: function(args) { var data = { id: args.context.kubernetesversions[0].id }; + if (args.data.deleteiso === 'on') { + $.extend(data, { + deleteiso: true + }); + } $.ajax({ url: createURL('deleteKubernetesSupportedVersion'), data: data, From 40954fdd9c4089c48f146de97f3febf08a83d2f1 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 27 Nov 2019 12:44:33 +0530 Subject: [PATCH 005/134] ha, multi-master support Support for multi-master Kubernetes cluster for Kubernetes version 1.16 and above Signed-off-by: Abhishek Kumar --- .../apache/cloudstack/api/ApiConstants.java | 3 +- .../META-INF/db/schema-41300to41400.sql | 1 + .../kubernetescluster/KubernetesCluster.java | 2 + .../KubernetesClusterManagerImpl.java | 422 +++++++++--------- .../KubernetesClusterVO.java | 20 +- .../CreateKubernetesClusterCmd.java | 13 +- .../GetKubernetesClusterConfigCmd.java | 1 - .../main/resources/conf/k8s-master-add.yml | 193 ++++++++ .../src/main/resources/conf/k8s-master.yml | 70 +-- .../src/main/resources/conf/k8s-node.yml | 54 +-- ui/plugins/cks/cks.js | 39 +- ui/scripts/sharedFunctions.js | 14 +- 12 files changed, 556 insertions(+), 276 deletions(-) create mode 100644 plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index c95fcdea3522..13ade27190f9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -207,8 +207,9 @@ public class ApiConstants { public static final String LOCK = "lock"; public static final String LUN = "lun"; public static final String LBID = "lbruleid"; - public static final String MAX = "max"; public static final String MAC_ADDRESS = "macaddress"; + public static final String MASTER_NODES = "masternodes"; + public static final String MAX = "max"; public static final String MAX_SNAPS = "maxsnaps"; public static final String MAX_CPU_NUMBER = "maxcpunumber"; public static final String MAX_MEMORY = "maxmemory"; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql b/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql index 7a84e855b00e..eb939adc2721 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql @@ -45,6 +45,7 @@ CREATE TABLE IF NOT EXISTS `cloud`.`kubernetes_cluster` ( `service_offering_id` bigint unsigned COMMENT 'service offering id for the cluster VM', `template_id` bigint unsigned COMMENT 'vm_template.id', `network_id` bigint unsigned COMMENT 'network this kubernetes cluster uses', + `master_node_count` bigint NOT NULL default '0', `node_count` bigint NOT NULL default '0', `account_id` bigint unsigned NOT NULL COMMENT 'owner of this cluster', `domain_id` bigint unsigned NOT NULL COMMENT 'owner of this cluster', diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesCluster.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesCluster.java index 439e299247a9..08edfe120cde 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesCluster.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesCluster.java @@ -110,7 +110,9 @@ enum State { long getNetworkId(); long getDomainId(); long getAccountId(); + long getMasterNodeCount(); long getNodeCount(); + long getTotalNodeCount(); String getKeyPair(); long getCores(); long getMemory(); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java index fac214271763..0e3626339d52 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java @@ -22,8 +22,6 @@ import java.io.PrintWriter; import java.io.StringWriter; import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.math.BigInteger; import java.net.InetAddress; import java.net.InetSocketAddress; @@ -32,6 +30,7 @@ import java.net.URL; import java.net.UnknownHostException; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.security.SecureRandom; import java.util.ArrayList; import java.util.Arrays; @@ -103,6 +102,7 @@ import com.cloud.kubernetescluster.dao.KubernetesClusterDetailsDao; import com.cloud.kubernetescluster.dao.KubernetesClusterVmMapDao; import com.cloud.kubernetesversion.KubernetesSupportedVersion; +import com.cloud.kubernetesversion.KubernetesVersionManagerImpl; import com.cloud.kubernetesversion.dao.KubernetesSupportedVersionDao; import com.cloud.network.IpAddress; import com.cloud.network.IpAddressManager; @@ -118,8 +118,10 @@ import com.cloud.network.dao.NetworkVO; import com.cloud.network.dao.PhysicalNetworkDao; import com.cloud.network.firewall.FirewallService; +import com.cloud.network.lb.LoadBalancingRulesService; import com.cloud.network.rules.FirewallRule; import com.cloud.network.rules.FirewallRuleVO; +import com.cloud.network.rules.LoadBalancer; import com.cloud.network.rules.PortForwardingRuleVO; import com.cloud.network.rules.RulesService; import com.cloud.network.rules.dao.PortForwardingRulesDao; @@ -163,6 +165,7 @@ import com.cloud.utils.fsm.NoTransitionException; import com.cloud.utils.fsm.StateMachine2; import com.cloud.utils.net.Ip; +import com.cloud.utils.net.NetUtils; import com.cloud.utils.ssh.SshHelper; import com.cloud.vm.Nic; import com.cloud.vm.ReservationContext; @@ -182,6 +185,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne protected StateMachine2 _stateMachine = KubernetesCluster.State.getStateMachine(); + public static final String MIN_KUBERNETES_VERSION_HA_SUPPORT = "1.16"; + ScheduledExecutorService _gcExecutor; ScheduledExecutorService _stateScanner; @@ -257,6 +262,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne protected FirewallRulesDao firewallRulesDao; @Inject protected IpAddressManager ipAddressManager; + @Inject + public LoadBalancingRulesService lbService; @Override public KubernetesCluster findById(final Long id) { @@ -274,6 +281,7 @@ public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) final Account owner = accountService.getActiveAccountById(cmd.getEntityOwnerId()); final Long networkId = cmd.getNetworkId(); final String sshKeyPair= cmd.getSSHKeyPairName(); + final Long masterNodeCount = cmd.getMasterNodes(); final Long clusterSize = cmd.getClusterSize(); final String dockerRegistryUserName = cmd.getDockerRegistryUserName(); final String dockerRegistryPassword = cmd.getDockerRegistryPassword(); @@ -285,8 +293,12 @@ public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) throw new InvalidParameterValueException("Invalid name for the kubernetes cluster name:" + name); } + if (masterNodeCount < 1 || masterNodeCount > 100) { + throw new InvalidParameterValueException("Invalid cluster master nodes count " + masterNodeCount); + } + if (clusterSize < 1 || clusterSize > 100) { - throw new InvalidParameterValueException("invalid cluster size " + clusterSize); + throw new InvalidParameterValueException("Invalid cluster size " + clusterSize); } DataCenter zone = dataCenterDao.findById(zoneId); @@ -298,13 +310,25 @@ public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) throw new PermissionDeniedException("Cannot perform this operation, Zone:" + zone.getId() + " is currently disabled."); } - final KubernetesSupportedVersion kubernetesVersion = kubernetesSupportedVersionDao.findById(kubernetesVersionId); - if (kubernetesVersion == null) { - throw new InvalidParameterValueException("Unable to find Kubernetes by version in supported versions"); + final KubernetesSupportedVersion clusterKubernetesVersion = kubernetesSupportedVersionDao.findById(kubernetesVersionId); + if (clusterKubernetesVersion == null) { + throw new InvalidParameterValueException("Unable to find given Kubernetes version in supported versions"); + } + if (clusterKubernetesVersion.getZoneId() != null && !clusterKubernetesVersion.getZoneId().equals(zone.getId())) { + throw new InvalidParameterValueException(String.format("Kubernetes version ID: %s is not available for zone ID: %s", clusterKubernetesVersion.getUuid(), zone.getUuid())); + } + if (masterNodeCount > 1 ) { + try { + if (KubernetesVersionManagerImpl.compareKubernetesVersion(clusterKubernetesVersion.getKubernetesVersion(), MIN_KUBERNETES_VERSION_HA_SUPPORT) < 0) { + throw new InvalidParameterValueException(String.format("HA support is available only for Kubernetes version %s and above. Given version ID: %s is %s", MIN_KUBERNETES_VERSION_HA_SUPPORT, clusterKubernetesVersion.getUuid(), clusterKubernetesVersion.getKubernetesVersion())); + } + } catch (Exception e) { + LOGGER.error(String.format("Unable to compare Kubernetes version for given version ID: %s with %s", clusterKubernetesVersion.getUuid(), MIN_KUBERNETES_VERSION_HA_SUPPORT), e); + } } - if (kubernetesVersion.getZoneId() != null && kubernetesVersion.getZoneId() != zone.getId()) { - throw new InvalidParameterValueException(String.format("Kubernetes version ID: %s is not available for zone ID: %s", kubernetesVersion.getUuid(), zone.getUuid())); + if (clusterKubernetesVersion.getZoneId() != null && clusterKubernetesVersion.getZoneId() != zone.getId()) { + throw new InvalidParameterValueException(String.format("Kubernetes version ID: %s is not available for zone ID: %s", clusterKubernetesVersion.getUuid(), zone.getUuid())); } ServiceOffering serviceOffering = serviceOfferingDao.findById(serviceOfferingId); @@ -337,7 +361,7 @@ public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) validateDockerRegistryParams(dockerRegistryUserName, dockerRegistryPassword, dockerRegistryUrl, dockerRegistryEmail); - plan(clusterSize, zoneId, serviceOfferingDao.findById(serviceOfferingId)); + plan(masterNodeCount + clusterSize, zoneId, serviceOfferingDao.findById(serviceOfferingId)); Network network = null; if (networkId != null) { @@ -376,15 +400,15 @@ public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) final Network defaultNetwork = network; final VMTemplateVO finalTemplate = template; - final long cores = serviceOffering.getCpu() * clusterSize; - final long memory = serviceOffering.getRamSize() * clusterSize; + final long cores = serviceOffering.getCpu() * (masterNodeCount + clusterSize); + final long memory = serviceOffering.getRamSize() * (masterNodeCount + clusterSize); final KubernetesClusterVO cluster = Transaction.execute(new TransactionCallback() { @Override public KubernetesClusterVO doInTransaction(TransactionStatus status) { - KubernetesClusterVO newCluster = new KubernetesClusterVO(name, displayName, zoneId, kubernetesVersion.getId(), + KubernetesClusterVO newCluster = new KubernetesClusterVO(name, displayName, zoneId, clusterKubernetesVersion.getId(), serviceOfferingId, finalTemplate.getId(), defaultNetwork.getId(), owner.getDomainId(), - owner.getAccountId(), clusterSize, KubernetesCluster.State.Created, sshKeyPair, cores, memory, nodeRootDiskSize, "", ""); + owner.getAccountId(), masterNodeCount, clusterSize, KubernetesCluster.State.Created, sshKeyPair, cores, memory, nodeRootDiskSize, "", ""); kubernetesClusterDao.persist(newCluster); return newCluster; } @@ -479,10 +503,6 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t if (LOGGER.isDebugEnabled()) { LOGGER.debug("Network:" + kubernetesCluster.getNetworkId() + " is started for the kubernetes cluster: " + kubernetesCluster.getName()); } - } catch (RuntimeException e) { - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); - LOGGER.warn("Starting the network failed as part of starting kubernetes cluster " + kubernetesCluster.getName() + " due to " + e); - throw new ManagementServerException("Failed to start the network while creating kubernetes cluster name:" + kubernetesCluster.getName(), e); } catch (Exception e) { stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); LOGGER.warn("Starting the network failed as part of starting kubernetes cluster " + kubernetesCluster.getName() + " due to " + e); @@ -521,18 +541,43 @@ public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("Provisioned the master VM's in to the kubernetes cluster name:" + kubernetesCluster.getName()); } - } catch (RuntimeException e) { - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); - LOGGER.warn("Provisioning the master VM' failed in the kubernetes cluster: " + kubernetesCluster.getName() + " due to " + e); - throw new ManagementServerException("Provisioning the master VM' failed in the kubernetes cluster: " + kubernetesCluster.getName(), e); } catch (Exception e) { stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); - LOGGER.warn("Provisioning the master VM' failed in the kubernetes cluster: " + kubernetesCluster.getName() + " due to " + e); + LOGGER.warn("Provisioning the master VM failed in the kubernetes cluster: " + kubernetesCluster.getName() + " due to " + e); throw new ManagementServerException("Provisioning the master VM' failed in the kubernetes cluster: " + kubernetesCluster.getName(), e); } String masterIP = k8sMasterVM.getPrivateIpAddress(); + if (kubernetesCluster.getMasterNodeCount() > 1) { + for (int i = 1; i < kubernetesCluster.getMasterNodeCount(); i++) { + UserVm vm = null; + try { + vm = createKubernetesAdditionalMaster(kubernetesCluster, publicIp.getAddress().toString(), i); + final long additionalVmId = vm.getId(); + KubernetesClusterVmMapVO clusterNodeVmMap = Transaction.execute(new TransactionCallback() { + @Override + public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { + KubernetesClusterVmMapVO newClusterVmMap = new KubernetesClusterVmMapVO(kubernetesClusterId, additionalVmId); + kubernetesClusterVmMapDao.persist(newClusterVmMap); + return newClusterVmMap; + } + }); + startKubernetesVM(vm, kubernetesCluster); + clusterVMIds.add(vm.getId()); + + vm = userVmDao.findById(vm.getId()); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Provisioned a node VM in to the kubernetes cluster: " + kubernetesCluster.getName()); + } + } catch (Exception e) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + LOGGER.warn("Provisioning the node VM failed in the kubernetes cluster " + kubernetesCluster.getName() + " due to " + e); + throw new ManagementServerException("Provisioning the node VM failed in the kubernetes cluster " + kubernetesCluster.getName(), e); + } + } + } + for (int i = 1; i <= kubernetesCluster.getNodeCount(); i++) { UserVm vm = null; try { @@ -553,10 +598,6 @@ public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("Provisioned a node VM in to the kubernetes cluster: " + kubernetesCluster.getName()); } - } catch (RuntimeException e) { - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); - LOGGER.warn("Provisioning the node VM failed in the kubernetes cluster " + kubernetesCluster.getName() + " due to " + e); - throw new ManagementServerException("Provisioning the node VM failed in the kubernetes cluster " + kubernetesCluster.getName(), e); } catch (Exception e) { stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); LOGGER.warn("Provisioning the node VM failed in the kubernetes cluster " + kubernetesCluster.getName() + " due to " + e); @@ -572,32 +613,35 @@ public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { attachIsoKubernetesVMs(kubernetesClusterId, clusterVMIds); int retryCounter = 0; - int maxRetries = 30; + int maxRetries = 15; boolean k8sApiServerSetup = false; while (retryCounter < maxRetries) { - try (Socket socket = new Socket()) { - socket.connect(new InetSocketAddress(publicIp.getAddress().addr(), 6443), 10000); - k8sApiServerSetup = true; - kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); - kubernetesCluster.setEndpoint("https://" + publicIp.getAddress() + ":6443/"); - kubernetesClusterDao.update(kubernetesCluster.getId(), kubernetesCluster); - break; - } catch (IOException e) { + try { + String versionOutput = IOUtils.toString(new URL(String.format("https://%s:%d/version", publicIp.getAddress().addr(), 6443)), StandardCharsets.UTF_8); + if (!Strings.isNullOrEmpty(versionOutput)) { + LOGGER.debug(String.format("Kubernetes cluster: %s API has been successfully provisioned, %s", kubernetesCluster.getUuid(), versionOutput)); + k8sApiServerSetup = true; + kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + kubernetesCluster.setEndpoint(String.format("https://%s:%d/", publicIp.getAddress().addr(), 6443)); + kubernetesClusterDao.update(kubernetesCluster.getId(), kubernetesCluster); + break; + } + } catch (Exception e) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("Waiting for kubernetes cluster: " + kubernetesCluster.getName() + " API endpoint to be available. retry: " + retryCounter + "/" + maxRetries); } - try { - Thread.sleep(60000); - } catch (InterruptedException ex) { - } - retryCounter++; } + try { + Thread.sleep(30000); + } catch (InterruptedException ie) { + LOGGER.error(String.format("Error while waiting for kubernetes cluster: %s API endpoint to be available", kubernetesCluster.getUuid()), ie); + } + retryCounter++; } boolean k8sKubeConfigCopied = false; if (k8sApiServerSetup) { - Runtime r = Runtime.getRuntime(); retryCounter = 0; maxRetries = 5; String kubeConfig = ""; @@ -619,6 +663,7 @@ public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { String.format("server: https://%s:6443", publicIp.getAddress().addr())); kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "kubeConfigData", Base64.encodeBase64String(kubeConfig.getBytes(Charset.forName("UTF-8"))), false); k8sKubeConfigCopied = true; + LOGGER.debug(String.format("Kubernetes cluster: %s kube-config has been successdully retrieved", kubernetesCluster.getUuid())); break; } } catch (Exception e) { @@ -640,7 +685,7 @@ public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { + " to come up. Attempt: " + retryCounter + " of max retries " + maxRetries); } - if (isAddOnServiceRunning(kubernetesCluster.getId(), "kubernetes-dashboard")) { + if (isAddOnServiceRunning(kubernetesCluster.getId(), "kubernetes-dashboard", "kubernetes-dashboard")) { stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationSucceeded); @@ -656,7 +701,6 @@ public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { return true; } - try { Thread.sleep(10000); } catch (InterruptedException ex) { @@ -675,7 +719,7 @@ public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { detachIsoKubernetesVMs(kubernetesClusterId, clusterVMIds); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, - "Failed to deploy kubernetes cluster: " + kubernetesCluster.getId() + " as unable to setup up in usable state"); + "Failed to deploy kubernetes cluster: " + kubernetesCluster.getUuid() + " as unable to setup up in usable state"); } private boolean startStoppedKubernetesCluster(long kubernetesClusterId) throws ManagementServerException, @@ -818,12 +862,6 @@ private void setupKubernetesClusterNetworkRules(IPAddressVO publicIp, Account ac LOGGER.debug("Provisioned firewall rule to open up port 6443 on " + publicIp.getAddress() + " for cluster " + kubernetesCluster.getName()); } - } catch (RuntimeException rte) { - LOGGER.warn("Failed to provision firewall rules for the kubernetes cluster: " + kubernetesCluster.getName() - + " due to exception: " + getStackTrace(rte)); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); - throw new ManagementServerException("Failed to provision firewall rules for the kubernetes " + - "cluster: " + kubernetesCluster.getName(), rte); } catch (Exception e) { LOGGER.warn("Failed to provision firewall rules for the kubernetes cluster: " + kubernetesCluster.getName() + " due to exception: " + getStackTrace(e)); @@ -863,12 +901,6 @@ private void setupKubernetesClusterNetworkRules(IPAddressVO publicIp, Account ac LOGGER.debug("Provisioned firewall rule to open up port 2222 on " + publicIp.getAddress() + " for cluster " + kubernetesCluster.getName()); } - } catch (RuntimeException rte) { - LOGGER.warn("Failed to provision firewall rules for the kubernetes cluster: " + kubernetesCluster.getName() - + " due to exception: " + getStackTrace(rte)); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); - throw new ManagementServerException("Failed to provision firewall rules for the kubernetes " + - "cluster: " + kubernetesCluster.getName(), rte); } catch (Exception e) { LOGGER.warn("Failed to provision firewall rules for the kubernetes cluster: " + kubernetesCluster.getName() + " due to exception: " + getStackTrace(e)); @@ -877,76 +909,39 @@ private void setupKubernetesClusterNetworkRules(IPAddressVO publicIp, Account ac "cluster: " + kubernetesCluster.getName()); } - Nic masterVmNic = networkModel.getNicInNetwork(clusterVMIds.get(0), kubernetesCluster.getNetworkId()); - // handle Nic interface method change between releases 4.5 and 4.6 and above through reflection - Method m = null; - try { - m = Nic.class.getMethod("getIp4Address"); - } catch (NoSuchMethodException e1) { - try { - m = Nic.class.getMethod("getIPv4Address"); - } catch (NoSuchMethodException e2) { - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); - throw new ManagementServerException("Failed to activate port forwarding rules for the cluster: " + kubernetesCluster.getName()); - } - } - Ip masterIp = null; + // Load balancer rule fo API access for master node VMs try { - masterIp = new Ip(m.invoke(masterVmNic).toString()); - } catch (InvocationTargetException | IllegalAccessException ie) { + LoadBalancer lb = lbService.createPublicLoadBalancerRule(null, "api-lb", "LB rule for API access", + 6443, 6443, 6443, 6443, + publicIp.getId(), NetUtils.TCP_PROTO, "roundrobin", kubernetesCluster.getNetworkId(), + kubernetesCluster.getAccountId(), false, NetUtils.TCP_PROTO, true); + + Map> vmIdIpMap = new HashMap<>(); + for (int i=0; i ips = new ArrayList<>(); + Nic masterVmNic = networkModel.getNicInNetwork(clusterVMIds.get(i), kubernetesCluster.getNetworkId()); + ips.add(masterVmNic.getIPv4Address()); + vmIdIpMap.put(clusterVMIds.get(i), ips); + } + lbService.assignToLoadBalancer(lb.getId(), null, vmIdIpMap); + } catch (Exception e) { + LOGGER.warn("Failed to provision load balancer rule for API access for the kubernetes cluster: " + kubernetesCluster.getName() + + " due to exception: " + getStackTrace(e)); stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); - throw new ManagementServerException("Failed to activate port forwarding rules for the cluster: " + kubernetesCluster.getName()); + throw new ManagementServerException("Failed to provision load balancer rule for API access for the kubernetes " + + "cluster: " + kubernetesCluster.getName()); } - final Ip masterIpFinal = masterIp; + + // Port forwarding rule fo SSH access on each node VM final long publicIpId = publicIp.getId(); final long networkId = kubernetesCluster.getNetworkId(); final long accountId = account.getId(); final long domainId = account.getDomainId(); - final long masterVmIdFinal = clusterVMIds.get(0); - - try { - PortForwardingRuleVO pfRule = Transaction.execute(new TransactionCallbackWithException() { - @Override - public PortForwardingRuleVO doInTransaction(TransactionStatus status) throws NetworkRuleConflictException { - PortForwardingRuleVO newRule = - new PortForwardingRuleVO(null, publicIpId, - 6443, 6443, - masterIpFinal, - 6443, 6443, - "tcp", networkId, accountId, domainId, masterVmIdFinal); - newRule.setDisplay(true); - newRule.setState(FirewallRule.State.Add); - newRule = portForwardingRulesDao.persist(newRule); - return newRule; - } - }); - rulesService.applyPortForwardingRules(publicIp.getId(), account); - - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Provisioning port forwarding rule from port 6443 on " + publicIp.getAddress() + - " to the master VM IP :" + masterIpFinal + " in kubernetes cluster " + kubernetesCluster.getName()); - } - } catch (RuntimeException rte) { - LOGGER.warn("Failed to activate port forwarding rules for the kubernetes cluster " + kubernetesCluster.getName() + " due to " + rte); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); - throw new ManagementServerException("Failed to activate port forwarding rules for the cluster: " + kubernetesCluster.getName(), rte); - } catch (Exception e) { - LOGGER.warn("Failed to activate port forwarding rules for the kubernetes cluster " + kubernetesCluster.getName() + " due to " + e); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); - throw new ManagementServerException("Failed to activate port forwarding rules for the cluster: " + kubernetesCluster.getName(), e); - } for (int i = 0; i < clusterVMIds.size(); ++i) { long vmId = clusterVMIds.get(i); Nic vmNic = networkModel.getNicInNetwork(vmId, kubernetesCluster.getNetworkId()); - Ip vmIp = null; - try { - vmIp = new Ip(m.invoke(vmNic).toString()); - } catch (InvocationTargetException | IllegalAccessException ie) { - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); - throw new ManagementServerException("Failed to activate SSH port forwarding rules for the cluster: " + kubernetesCluster.getName()); - } - final Ip vmIpFinal = vmIp; + final Ip vmIp = new Ip(vmNic.getIPv4Address()); final long vmIdFinal = vmId; final int srcPortFinal = 2222 + i; @@ -957,7 +952,7 @@ public PortForwardingRuleVO doInTransaction(TransactionStatus status) throws Net PortForwardingRuleVO newRule = new PortForwardingRuleVO(null, publicIpId, srcPortFinal, srcPortFinal, - vmIpFinal, + vmIp, 22, 22, "tcp", networkId, accountId, domainId, vmIdFinal); newRule.setDisplay(true); @@ -970,7 +965,7 @@ public PortForwardingRuleVO doInTransaction(TransactionStatus status) throws Net if (LOGGER.isDebugEnabled()) { LOGGER.debug("Provisioning SSH port forwarding rule from port 2222 to 22 on " + publicIp.getAddress() + - " to the VM IP :" + vmIpFinal + " in kubernetes cluster " + kubernetesCluster.getName()); + " to the VM IP :" + vmIp + " in kubernetes cluster " + kubernetesCluster.getName()); } } catch (RuntimeException rte) { LOGGER.warn("Failed to activate SSH port forwarding rules for the kubernetes cluster " + kubernetesCluster.getName() + " due to " + rte); @@ -1038,11 +1033,6 @@ private void scaleKubernetesClusterNetworkRules(IPAddressVO publicIp, Account ac LOGGER.debug("Provisioned firewall rule to open up port 2222 on " + publicIp.getAddress() + " for cluster " + kubernetesCluster.getName()); } - } catch (RuntimeException rte) { - LOGGER.warn("Failed to provision firewall rules for the kubernetes cluster: " + kubernetesCluster.getName() - + " due to exception: " + getStackTrace(rte)); - throw new ManagementServerException("Failed to provision firewall rules for the kubernetes " + - "cluster: " + kubernetesCluster.getName(), rte); } catch (Exception e) { LOGGER.warn("Failed to provision firewall rules for the kubernetes cluster: " + kubernetesCluster.getName() + " due to exception: " + getStackTrace(e)); @@ -1051,18 +1041,6 @@ private void scaleKubernetesClusterNetworkRules(IPAddressVO publicIp, Account ac } if (clusterVMIds != null && !clusterVMIds.isEmpty()) { // Upscaling, add new port-forwarding rules - // handle Nic interface method change between releases 4.5 and 4.6 and above through reflection - Method m = null; - try { - m = Nic.class.getMethod("getIp4Address"); - } catch (NoSuchMethodException e1) { - try { - m = Nic.class.getMethod("getIPv4Address"); - } catch (NoSuchMethodException e2) { - throw new ManagementServerException("Failed to activate port forwarding rules for the cluster: " + kubernetesCluster.getName()); - } - } - // Apply port forwarding only to new VMs final long publicIpId = publicIp.getId(); final long networkId = kubernetesCluster.getNetworkId(); @@ -1071,13 +1049,7 @@ private void scaleKubernetesClusterNetworkRules(IPAddressVO publicIp, Account ac for (int i = 0; i < clusterVMIds.size(); ++i) { long vmId = clusterVMIds.get(i); Nic vmNic = networkModel.getNicInNetwork(vmId, kubernetesCluster.getNetworkId()); - Ip vmIp = null; - try { - vmIp = new Ip(m.invoke(vmNic).toString()); - } catch (InvocationTargetException | IllegalAccessException ie) { - throw new ManagementServerException("Failed to activate SSH port forwarding rules for the cluster: " + kubernetesCluster.getName()); - } - final Ip vmIpFinal = vmIp; + final Ip vmIp = new Ip(vmNic.getIPv4Address()); final long vmIdFinal = vmId; final int srcPortFinal = existingFirewallRuleSourcePortEnd + 1 + i; @@ -1088,7 +1060,7 @@ public PortForwardingRuleVO doInTransaction(TransactionStatus status) throws Net PortForwardingRuleVO newRule = new PortForwardingRuleVO(null, publicIpId, srcPortFinal, srcPortFinal, - vmIpFinal, + vmIp, 22, 22, "tcp", networkId, accountId, domainId, vmIdFinal); newRule.setDisplay(true); @@ -1101,11 +1073,8 @@ public PortForwardingRuleVO doInTransaction(TransactionStatus status) throws Net if (LOGGER.isDebugEnabled()) { LOGGER.debug("Provisioning SSH port forwarding rule from port 2222 to 22 on " + publicIp.getAddress() + - " to the VM IP :" + vmIpFinal + " in kubernetes cluster " + kubernetesCluster.getName()); + " to the VM IP :" + vmIp + " in kubernetes cluster " + kubernetesCluster.getName()); } - } catch (RuntimeException rte) { - LOGGER.warn("Failed to activate SSH port forwarding rules for the kubernetes cluster " + kubernetesCluster.getName() + " due to " + rte); - throw new ManagementServerException("Failed to activate SSH port forwarding rules for the cluster: " + kubernetesCluster.getName(), rte); } catch (Exception e) { LOGGER.warn("Failed to activate SSH port forwarding rules for the kubernetes cluster " + kubernetesCluster.getName() + " due to " + e); throw new ManagementServerException("Failed to activate SSH port forwarding rules for the cluster: " + kubernetesCluster.getName(), e); @@ -1215,7 +1184,7 @@ private void validateDockerRegistryParams(final String dockerRegistryUserName, } } - public DeployDestination plan(final long clusterSize, final long dcId, final ServiceOffering offering) throws InsufficientServerCapacityException { + public DeployDestination plan(final long nodesCount, final long dcId, final ServiceOffering offering) throws InsufficientServerCapacityException { final int cpu_requested = offering.getCpu() * offering.getSpeed(); final long ram_requested = offering.getRamSize() * 1024L * 1024L; List hosts = resourceManager.listAllHostsInOneZoneByType(Type.Routing, dcId); @@ -1224,7 +1193,7 @@ public DeployDestination plan(final long clusterSize, final long dcId, final Ser hosts_with_resevered_capacity.put(h.getUuid(), new Pair(h, 0)); } boolean suitable_host_found = false; - for (int i = 1; i <= clusterSize + 1; i++) { + for (int i = 1; i <= nodesCount + 1; i++) { suitable_host_found = false; for (Map.Entry> hostEntry : hosts_with_resevered_capacity.entrySet()) { Pair hp = hostEntry.getValue(); @@ -1264,7 +1233,7 @@ public DeployDestination plan(final long clusterSize, final long dcId, final Ser return new DeployDestination(dataCenterDao.findById(dcId), null, null, null); } String msg = String.format("Cannot find enough capacity for kubernetes cluster(requested cpu=%1$s memory=%2$s)", - cpu_requested * clusterSize, ram_requested * clusterSize); + cpu_requested * nodesCount, ram_requested * nodesCount); LOGGER.warn(msg); throw new InsufficientServerCapacityException(msg, DataCenter.class, dcId); } @@ -1281,7 +1250,7 @@ public DeployDestination plan(final long kubernetesClusterId, final long dcId) t LOGGER.debug("Checking deployment destination for kubernetesClusterId= " + kubernetesClusterId + " in dcId=" + dcId); } - return plan(kubernetesCluster.getNodeCount() + 1, dcId, offering); + return plan(kubernetesCluster.getTotalNodeCount(), dcId, offering); } @Override @@ -1342,15 +1311,12 @@ public boolean stopKubernetesCluster(long kubernetesClusterId) throws Management return true; } - private boolean isAddOnServiceRunning(Long clusterId, String svcName) { - + private boolean isAddOnServiceRunning(Long clusterId, final String nameSpace, String svcName) { KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(clusterId); - //FIXME: whole logic needs revamp. Assumption that management server has public network access is not practical IPAddressVO publicIp = null; List ips = ipAddressDao.listByAssociatedNetwork(kubernetesCluster.getNetworkId(), true); publicIp = ips.get(0); - Runtime r = Runtime.getRuntime(); int nodePort = 0; try { @@ -1360,8 +1326,12 @@ private boolean isAddOnServiceRunning(Long clusterId, String svcName) { keyFile += ".cloud"; } File pkFile = new File(keyFile); + String cmd = "sudo kubectl get pods --all-namespaces"; + if (!Strings.isNullOrEmpty(nameSpace)) { + cmd = String.format("sudo kubectl get pods --namespace=%s", nameSpace); + } Pair result = SshHelper.sshExecute(publicIp.getAddress().addr(), 2222, "core", - pkFile, null, "sudo kubectl get pods --namespace=kube-system", + pkFile, null, cmd, 10000, 10000, 10000); if (result.first() && !Strings.isNullOrEmpty(result.second())) { String[] lines = result.second().split("\n"); @@ -1544,62 +1514,56 @@ private void processFailedNetworkDelete(long kubernetesClusterId) { private UserVm createKubernetesMaster(final KubernetesClusterVO kubernetesCluster, final List ips) throws ManagementServerException, ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException { - UserVm masterVm = null; - DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); ServiceOffering serviceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); VirtualMachineTemplate template = templateDao.findById(kubernetesCluster.getTemplateId()); - List networkIds = new ArrayList(); networkIds.add(kubernetesCluster.getNetworkId()); - Account owner = accountDao.findById(kubernetesCluster.getAccountId()); - final String masterIp = ipAddressManager.acquireGuestIpAddress(networkDao.findById(kubernetesCluster.getNetworkId()), null); Network.IpAddresses addrs = new Network.IpAddresses(masterIp, null); - long rootDiskSize = kubernetesCluster.getNodeRootDiskSize(); - Map customParameterMap = new HashMap(); if (rootDiskSize > 0) { customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); } - String hostName = kubernetesCluster.getName() + "-k8s-master"; - + boolean haSupported = false; + final KubernetesSupportedVersion version = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); + if (version != null) { + try { + if (KubernetesVersionManagerImpl.compareKubernetesVersion(version.getKubernetesVersion(), MIN_KUBERNETES_VERSION_HA_SUPPORT) >= 0) { + haSupported = true; + } + } catch (Exception e) { + LOGGER.error(String.format("Unable to compare Kubernetes version for cluster version ID: %s with %s", version.getUuid(), MIN_KUBERNETES_VERSION_HA_SUPPORT), e); + } + } String k8sMasterConfig = null; try { k8sMasterConfig = readResourceFile("/conf/k8s-master.yml"); - final String apiServerCert = "{{ k8s_master.apiserver.crt }}"; final String apiServerKey = "{{ k8s_master.apiserver.key }}"; final String caCert = "{{ k8s_master.ca.crt }}"; final String msSshPubKey = "{{ k8s_master.ms.ssh.pub.key }}"; final String clusterToken = "{{ k8s_master.cluster.token }}"; - final String clusterIp = "{{ k8s_master.cluster.ip }}"; - - + final String clusterInitArgsKey = "{{ k8s_master.cluster.initargs }}"; final List addresses = new ArrayList<>(); addresses.add(masterIp); for (final IPAddressVO ip : ips) { addresses.add(ip.getAddress().addr()); } - final Certificate certificate = caManager.issueCertificate(null, Arrays.asList(hostName, "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.default.svc.cluster.local"), addresses, 3650, null); - final String tlsClientCert = CertUtils.x509CertificateToPem(certificate.getClientCertificate()); final String tlsPrivateKey = CertUtils.privateKeyToPem(certificate.getPrivateKey()); final String tlsCaCert = CertUtils.x509CertificatesToPem(certificate.getCaCertificates()); - k8sMasterConfig = k8sMasterConfig.replace(apiServerCert, tlsClientCert.replace("\n", "\n ")); k8sMasterConfig = k8sMasterConfig.replace(apiServerKey, tlsPrivateKey.replace("\n", "\n ")); k8sMasterConfig = k8sMasterConfig.replace(caCert, tlsCaCert.replace("\n", "\n ")); - String pubKey = "- \"" + globalConfigDao.getValue("ssh.publickey") + "\""; - String sshKeyPair = kubernetesCluster.getKeyPair(); if (!Strings.isNullOrEmpty(sshKeyPair)) { SSHKeyPairVO sshkp = sshKeyPairDao.findByName(owner.getAccountId(), owner.getDomainId(), sshKeyPair); @@ -1608,64 +1572,93 @@ private UserVm createKubernetesMaster(final KubernetesClusterVO kubernetesCluste } } k8sMasterConfig = k8sMasterConfig.replace(msSshPubKey, pubKey); - k8sMasterConfig = k8sMasterConfig.replace(clusterToken, generateClusterToken(kubernetesCluster)); - k8sMasterConfig = k8sMasterConfig.replace(clusterIp, String.format("--apiserver-cert-extra-sans=%s", ips.get(0).getAddress().toString())); - } catch (RuntimeException e) { - LOGGER.error("Failed to read kubernetes master configuration file due to " + e); - throw new ManagementServerException("Failed to read kubernetes master configuration file", e); + String initArgs = ""; + if (haSupported) { + initArgs = String.format("--control-plane-endpoint %s:6443 --upload-certs --certificate-key %s ", + ips.get(0).getAddress().addr(), + generateClusterHACertificateKey(kubernetesCluster)); + } + initArgs += String.format("--apiserver-cert-extra-sans=%s", ips.get(0).getAddress().addr()); + k8sMasterConfig = k8sMasterConfig.replace(clusterInitArgsKey, initArgs); } catch (Exception e) { LOGGER.error("Failed to read kubernetes master configuration file due to " + e); throw new ManagementServerException("Failed to read kubernetes master configuration file", e); } - String base64UserData = Base64.encodeBase64String(k8sMasterConfig.getBytes(Charset.forName("UTF-8"))); masterVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, hostName, kubernetesCluster.getDescription(), null, null, null, null, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), null, addrs, null, null, null, customParameterMap, null, null, null, null); - if (LOGGER.isDebugEnabled()) { LOGGER.debug("Created master VM: " + hostName + " in the kubernetes cluster: " + kubernetesCluster.getName()); } - return masterVm; } - private UserVm createKubernetesNode(KubernetesClusterVO kubernetesCluster, String masterIp, int nodeInstance) throws ManagementServerException, + private UserVm createKubernetesAdditionalMaster(final KubernetesClusterVO kubernetesCluster, final String joinIp, final int additionalMasterNodeInstance) throws ManagementServerException, ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException { - UserVm nodeVm = null; - DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); ServiceOffering serviceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); VirtualMachineTemplate template = templateDao.findById(kubernetesCluster.getTemplateId()); - List networkIds = new ArrayList(); networkIds.add(kubernetesCluster.getNetworkId()); - Account owner = accountDao.findById(kubernetesCluster.getAccountId()); - Network.IpAddresses addrs = new Network.IpAddresses(null, null); - long rootDiskSize = kubernetesCluster.getNodeRootDiskSize(); - Map customParameterMap = new HashMap(); if (rootDiskSize > 0) { customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); } + String hostName = String.format("%s-k8s-master-%s", kubernetesCluster.getName(), additionalMasterNodeInstance); + String k8sMasterConfig = null; + try { + k8sMasterConfig = readResourceFile("/conf/k8s-master-add.yml"); + final String joinIpKey = "{{ k8s_master.join_ip }}"; + final String clusterTokenKey = "{{ k8s_master.cluster.token }}"; + final String clusterHACertificateKey = "{{ k8s_master.cluster.ha.certificate.key }}"; + k8sMasterConfig = k8sMasterConfig.replace(joinIpKey, joinIp); + k8sMasterConfig = k8sMasterConfig.replace(clusterTokenKey, generateClusterToken(kubernetesCluster)); + k8sMasterConfig = k8sMasterConfig.replace(clusterHACertificateKey, generateClusterHACertificateKey(kubernetesCluster)); + } catch (Exception e) { + LOGGER.warn("Failed to read node configuration file due to " + e); + throw new ManagementServerException("Failed to read cluster node configuration file.", e); + } + String base64UserData = Base64.encodeBase64String(k8sMasterConfig.getBytes(Charset.forName("UTF-8"))); + nodeVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, + hostName, kubernetesCluster.getDescription(), null, null, null, + null, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), + null, addrs, null, null, null, customParameterMap, null, null, null, null); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Created cluster additional master node VM: " + hostName + " in the kubernetes cluster: " + kubernetesCluster.getName()); + } + return nodeVm; + } - String hostName = kubernetesCluster.getName() + "-k8s-node-" + String.valueOf(nodeInstance); - + private UserVm createKubernetesNode(KubernetesClusterVO kubernetesCluster, String joinIp, int nodeInstance) throws ManagementServerException, + ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException { + UserVm nodeVm = null; + DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); + ServiceOffering serviceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); + VirtualMachineTemplate template = templateDao.findById(kubernetesCluster.getTemplateId()); + List networkIds = new ArrayList(); + networkIds.add(kubernetesCluster.getNetworkId()); + Account owner = accountDao.findById(kubernetesCluster.getAccountId()); + Network.IpAddresses addrs = new Network.IpAddresses(null, null); + long rootDiskSize = kubernetesCluster.getNodeRootDiskSize(); + Map customParameterMap = new HashMap(); + if (rootDiskSize > 0) { + customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); + } + String hostName = String.format("%s-k8s-node-%s", kubernetesCluster.getName(), nodeInstance); String k8sNodeConfig = null; try { k8sNodeConfig = readResourceFile("/conf/k8s-node.yml"); - String masterIPString = "{{ k8s_master.default_ip }}"; - final String clusterTokenString = "{{ k8s_master.cluster.token }}"; - - k8sNodeConfig = k8sNodeConfig.replace(masterIPString, masterIp); - k8sNodeConfig = k8sNodeConfig.replace(clusterTokenString, generateClusterToken(kubernetesCluster)); - + final String joinIpKey = "{{ k8s_master.join_ip }}"; + final String clusterTokenKey = "{{ k8s_master.cluster.token }}"; + k8sNodeConfig = k8sNodeConfig.replace(joinIpKey, joinIp); + k8sNodeConfig = k8sNodeConfig.replace(clusterTokenKey, generateClusterToken(kubernetesCluster)); /* genarate /.docker/config.json file on the nodes only if kubernetes cluster is created to * use docker private registry */ String dockerUserName = null; @@ -1704,31 +1697,27 @@ private UserVm createKubernetesNode(KubernetesClusterVO kubernetesCluster, Strin " }\n" + " }"; k8sNodeConfig = k8sNodeConfig.replace("write-files:", dockerConfigString); - String dockerUrl = "{{docker.url}}"; - String dockerAuth = "{{docker.secret}}"; - String dockerEmail = "{{docker.email}}"; - String usernamePassword = dockerUserName + ":" + dockerPassword; - String base64Auth = Base64.encodeBase64String(usernamePassword.getBytes(Charset.forName("UTF-8"))); - k8sNodeConfig = k8sNodeConfig.replace(dockerUrl, "\"" + dockerRegistryUrl + "\""); - k8sNodeConfig = k8sNodeConfig.replace(dockerAuth, "\"" + base64Auth + "\""); - k8sNodeConfig = k8sNodeConfig.replace(dockerEmail, "\"" + dockerRegistryEmail + "\""); + final String dockerUrlKey = "{{docker.url}}"; + final String dockerAuthKey = "{{docker.secret}}"; + final String dockerEmailKey = "{{docker.email}}"; + final String usernamePasswordKey = dockerUserName + ":" + dockerPassword; + String base64Auth = Base64.encodeBase64String(usernamePasswordKey.getBytes(Charset.forName("UTF-8"))); + k8sNodeConfig = k8sNodeConfig.replace(dockerUrlKey, "\"" + dockerRegistryUrl + "\""); + k8sNodeConfig = k8sNodeConfig.replace(dockerAuthKey, "\"" + base64Auth + "\""); + k8sNodeConfig = k8sNodeConfig.replace(dockerEmailKey, "\"" + dockerRegistryEmail + "\""); } } catch (Exception e) { LOGGER.warn("Failed to read node configuration file due to " + e); throw new ManagementServerException("Failed to read cluster node configuration file.", e); } - String base64UserData = Base64.encodeBase64String(k8sNodeConfig.getBytes(Charset.forName("UTF-8"))); - nodeVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, hostName, kubernetesCluster.getDescription(), null, null, null, null, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), null, addrs, null, null, null, customParameterMap, null, null, null, null); - if (LOGGER.isDebugEnabled()) { LOGGER.debug("Created cluster node VM: " + hostName + " in the kubernetes cluster: " + kubernetesCluster.getName()); } - return nodeVm; } @@ -1970,7 +1959,7 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { if (clusterState.equals(KubernetesCluster.State.Running)) { plan(newVmRequiredCount, kubernetesCluster.getZoneId(), clusterServiceOffering); } else { - plan(kubernetesCluster.getNodeCount() + newVmRequiredCount, kubernetesCluster.getZoneId(), clusterServiceOffering); + plan(kubernetesCluster.getTotalNodeCount() + newVmRequiredCount, kubernetesCluster.getZoneId(), clusterServiceOffering); } } catch (InsufficientCapacityException e) { stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); @@ -2264,6 +2253,7 @@ public KubernetesClusterConfigResponse getKubernetesClusterConfig(GetKubernetesC try { configData = new String(Base64.decodeBase64(kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "kubeConfigData").getValue())); } catch (Exception e) {} + response.setConfigData(configData); } response.setObjectName("clusterconfig"); return response; @@ -2552,7 +2542,7 @@ boolean isClusterInDesiredState(KubernetesCluster kubernetesCluster, VirtualMach } // check cluster is running at desired capacity include master node as well, so count should be cluster size + 1 - if (clusterVMs.size() != (kubernetesCluster.getNodeCount() + 1)) { + if (clusterVMs.size() < kubernetesCluster.getTotalNodeCount()) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("Found only " + clusterVMs.size() + " VM's in the kubernetes cluster: " + kubernetesCluster.getName() + " in state: " + state.toString() + " While expected number of VM's to " + @@ -2590,4 +2580,14 @@ private String generateClusterToken(KubernetesCluster kubernetesCluster) { token = token.substring(0, 6) + "." + token.substring(6); return token; } + + private String generateClusterHACertificateKey(KubernetesCluster kubernetesCluster) { + if (kubernetesCluster == null) return ""; + String uuid = kubernetesCluster.getUuid(); + StringBuilder token = new StringBuilder(uuid.replaceAll("-", "")); + while (token.length() < 64) { + token.append(token); + } + return token.toString().substring(0, 64); + } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVO.java index 80a046e7bec1..a9824f76d567 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVO.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVO.java @@ -69,6 +69,9 @@ public class KubernetesClusterVO implements KubernetesCluster { @Column(name = "account_id") private long accountId; + @Column(name = "master_node_count") + private long masterNodeCount; + @Column(name = "node_count") private long nodeCount; @@ -201,6 +204,15 @@ public void setAccountId(long accountId) { this.accountId = accountId; } + @Override + public long getMasterNodeCount() { + return masterNodeCount; + } + + public void setMasterNodeCount(long masterNodeCount) { + this.masterNodeCount = masterNodeCount; + } + @Override public long getNodeCount() { return nodeCount; @@ -210,6 +222,11 @@ public void setNodeCount(long nodeCount) { this.nodeCount = nodeCount; } + @Override + public long getTotalNodeCount() { + return this.masterNodeCount + this.nodeCount; + } + @Override public long getCores() { return cores; @@ -298,7 +315,7 @@ public KubernetesClusterVO() { } public KubernetesClusterVO(String name, String description, long zoneId, long kubernetesVersionId, long serviceOfferingId, long templateId, - long networkId, long domainId, long accountId, long nodeCount, State state, + long networkId, long domainId, long accountId, long masterNodeCount, long nodeCount, State state, String keyPair, long cores, long memory, Long nodeRootDiskSize, String endpoint, String consoleEndpoint) { this.uuid = UUID.randomUUID().toString(); this.name = name; @@ -310,6 +327,7 @@ public KubernetesClusterVO(String name, String description, long zoneId, long ku this.networkId = networkId; this.domainId = domainId; this.accountId = accountId; + this.masterNodeCount = masterNodeCount; this.nodeCount = nodeCount; this.state = state; this.keyPair = keyPair; diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/CreateKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/CreateKubernetesClusterCmd.java index bd083f31c7e1..04ba93087bcb 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/CreateKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/CreateKubernetesClusterCmd.java @@ -113,8 +113,12 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { description = "name of the ssh key pair used to login to the virtual machines") private String sshKeyPairName; + @Parameter(name=ApiConstants.MASTER_NODES, type = CommandType.LONG, + description = "number of Kubernetes cluster master nodes, default is 1") + private Long masterNodes; + @Parameter(name=ApiConstants.SIZE, type = CommandType.LONG, - required = true, description = "number of Kubernetes cluster nodes") + required = true, description = "number of Kubernetes cluster worker nodes") private Long clusterSize; @Parameter(name = ApiConstants.DOCKER_REGISTRY_USER_NAME, type = CommandType.STRING, @@ -181,6 +185,13 @@ public String getSSHKeyPairName() { return sshKeyPairName; } + public Long getMasterNodes() { + if (masterNodes == null) { + return 1L; + } + return masterNodes; + } + public Long getClusterSize() { return clusterSize; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/GetKubernetesClusterConfigCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/GetKubernetesClusterConfigCmd.java index 0520aeaf4b0b..2039c04473ba 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/GetKubernetesClusterConfigCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/GetKubernetesClusterConfigCmd.java @@ -84,7 +84,6 @@ public String getCommandName() { @Override public void execute() { - KubernetesClusterConfigResponse response = kubernetesClusterService.getKubernetesClusterConfig(this); response.setResponseName(getCommandName()); setResponseObject(response); diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml new file mode 100644 index 000000000000..e081dd6904e5 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml @@ -0,0 +1,193 @@ +#cloud-config + +--- +write-files: + - path: /opt/bin/setup-kube-system + permissions: 0700 + owner: root:root + content: | + #!/bin/bash -e + + export PATH=$PATH:/opt/bin + + ISO_MOUNT_DIR=/mnt/k8sdisk + BINARIES_DIR=${ISO_MOUNT_DIR}/ + ATTEMPT_ONLINE_INSTALL=false + setup_complete=false + + OFFLINE_INSTALL_ATTEMPT_SLEEP=5 + MAX_OFFLINE_INSTALL_ATTEMPTS=36 + offline_attempts=1 + MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 + crucial_cmd_attempts=1 + while true; do + if (( "$offline_attempts" > "$MAX_OFFLINE_INSTALL_ATTEMPTS" )); then + echo "Warning: Offline install timed out!" + break + fi + set +e + output=`blkid -o device -t TYPE=iso9660` + set -e + if [ "$output" != "" ]; then + while read -r line; do + if [ ! -d "${ISO_MOUNT_DIR}" ]; then + mkdir "${ISO_MOUNT_DIR}" + fi + retval=0 + set +e + mount -o ro "${line}" "${ISO_MOUNT_DIR}" + retval=$? + set -e + if [ $retval -eq 0 ]; then + if [ -d "$BINARIES_DIR" ]; then + break + else + umount "${line}" && rmdir "${ISO_MOUNT_DIR}" + fi + fi + done <<< "$output" + fi + if [ -d "$BINARIES_DIR" ]; then + break + fi + echo "Waiting for Binaries directory $BINARIES_DIR to be available, sleeping for $OFFLINE_INSTALL_ATTEMPT_SLEEP seconds, attempt: $offline_attempts" + sleep $OFFLINE_INSTALL_ATTEMPT_SLEEP + offline_attempts=$[$offline_attempts + 1] + done + + if [ -d "$BINARIES_DIR" ]; then + ### Binaries available offline ### + echo "Installing binaries from ${BINARIES_DIR}" + mkdir -p /opt/cni/bin + tar -f "${BINARIES_DIR}/cni/cni-plugins-amd64.tgz" -C /opt/cni/bin -xz + + mkdir -p /opt/bin + tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-amd64.tar.gz" -C /opt/bin -xz + + mkdir -p /opt/bin + cd /opt/bin + cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} /opt/bin + chmod +x {kubeadm,kubelet,kubectl} + + sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service > /etc/systemd/system/kubelet.service + mkdir -p /etc/systemd/system/kubelet.service.d + sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + + output=`ls ${BINARIES_DIR}/docker/` + if [ "$output" != "" ]; then + while read -r line; do + crucial_cmd_attempts=1 + while true; do + if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Loading docker image ${BINARIES_DIR}/docker/$line failed!" + break; + fi + retval=0 + set +e + docker load < "${BINARIES_DIR}/docker/$line" + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] + done + done <<< "$output" + setup_complete=true + fi + umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" + fi + if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then + ### Binaries not available offline ### + RELEASE="v1.16.3" + CNI_VERSION="v0.7.5" + CRICTL_VERSION="v1.16.0" + echo "Warning: ${BINARIES_DIR} not found. Will get binaries and docker images from Internet." + mkdir -p /opt/cni/bin + curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz + + mkdir -p /opt/bin + curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" | tar -C /opt/bin -xz + + mkdir -p /opt/bin + cd /opt/bin + curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} + chmod +x {kubeadm,kubelet,kubectl} + + curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service + mkdir -p /etc/systemd/system/kubelet.service.d + curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf + fi + + systemctl enable kubelet && systemctl start kubelet + modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 + + if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then + crucial_cmd_attempts=1 + while true; do + if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Warning: kubeadm pull images failed after multiple tries!" + break; + fi + retval=0 + set +e + kubeadm config images pull + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] + done + fi + + - path: /opt/bin/deploy-kube-system + permissions: 0700 + owner: root:root + content: | + #!/bin/bash -e + if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then + echo "setup-kube-system is running!" + exit 1 + fi + modprobe ip_vs + modprobe ip_vs_wrr + modprobe ip_vs_sh + modprobe nf_conntrack_ipv4 + export PATH=$PATH:/opt/bin + kubeadm join {{ k8s_master.join_ip }}:6443 --token {{ k8s_master.cluster.token }} --control-plane --certificate-key {{ k8s_master.cluster.ha.certificate.key }} --discovery-token-unsafe-skip-ca-verification + +coreos: + units: + - name: docker.service + command: start + enable: true + + - name: setup-kube-system.service + command: start + content: | + [Unit] + Requires=docker.service + After=docker.service + + [Service] + Type=simple + StartLimitInterval=0 + ExecStart=/opt/bin/setup-kube-system + + - name: deploy-kube-system.service + command: start + content: | + [Unit] + After=setup-kube-system.service + + [Service] + Type=simple + StartLimitInterval=0 + Restart=on-failure + ExecStartPre=/usr/bin/curl -k https://{{ k8s_master.join_ip }}:6443/version + ExecStart=/opt/bin/deploy-kube-system + + update: + group: stable + reboot-strategy: off diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml index ea9788dfdc3c..caac285ad158 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml @@ -33,12 +33,9 @@ write-files: export PATH=$PATH:/opt/bin - RELEASE="v1.11.4" ISO_MOUNT_DIR=/mnt/k8sdisk - BINARIES_DIR=${ISO_MOUNT_DIR}/${RELEASE} - CNI_VERSION="v0.7.1" - CRICTL_VERSION="v1.11.1" - ATTEMPT_OFFLINE_INSTALL=true + BINARIES_DIR=${ISO_MOUNT_DIR}/ + ATTEMPT_ONLINE_INSTALL=false setup_complete=false OFFLINE_INSTALL_ATTEMPT_SLEEP=5 @@ -85,10 +82,10 @@ write-files: ### Binaries available offline ### echo "Installing binaries from ${BINARIES_DIR}" mkdir -p /opt/cni/bin - tar -f "${BINARIES_DIR}/cni/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" -C /opt/cni/bin -xz + tar -f "${BINARIES_DIR}/cni/cni-plugins-amd64.tgz" -C /opt/cni/bin -xz mkdir -p /opt/bin - tar -f "${BINARIES_DIR}/cri-tools/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" -C /opt/bin -xz + tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-amd64.tar.gz" -C /opt/bin -xz mkdir -p /opt/bin cd /opt/bin @@ -123,8 +120,11 @@ write-files: fi umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" fi - if [ "$setup_complete" = false ]; then + if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then ### Binaries not available offline ### + RELEASE="v1.16.3" + CNI_VERSION="v0.7.5" + CRICTL_VERSION="v1.16.0" echo "Warning: ${BINARIES_DIR} not found. Will get binaries and docker images from Internet." mkdir -p /opt/cni/bin curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz @@ -145,22 +145,25 @@ write-files: systemctl enable kubelet && systemctl start kubelet modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 - crucial_cmd_attempts=1 - while true; do - if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then - echo "Error: kubeadm pull images failed!" - exit 1 - fi - retval=0 - set +e - kubeadm config images pull - retval=$? - set -e - if [ $retval -eq 0 ]; then - break; - fi - crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] - done + if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then + crucial_cmd_attempts=1 + while true; do + if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Warning: kubeadm pull images failed after multiple tries!" + break; + fi + retval=0 + set +e + kubeadm config images pull + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] + done + fi + crucial_cmd_attempts=1 while true; do if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then @@ -169,7 +172,7 @@ write-files: fi retval=0 set +e - kubeadm init --pod-network-cidr=10.244.0.0/16 --token {{ k8s_master.cluster.token }} {{ k8s_master.cluster.ip }} + kubeadm init --token {{ k8s_master.cluster.token }} {{ k8s_master.cluster.initargs }} retval=$? set -e if [ $retval -eq 0 ]; then @@ -183,6 +186,10 @@ write-files: owner: root:root content: | #!/bin/bash -e + + ISO_MOUNT_DIR=/mnt/k8sdisk + BINARIES_DIR=${ISO_MOUNT_DIR}/ + if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then echo "setup-kube-system is running!" exit 1 @@ -195,12 +202,19 @@ write-files: chown $(id -u):$(id -g) /root/.kube/config echo export PATH=\$PATH:/opt/bin >> /root/.bashrc - kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/v0.10.0/Documentation/kube-flannel.yml - kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.0/src/deploy/recommended/kubernetes-dashboard.yaml + if [ -d "$BINARIES_DIR" ]; then + ### Network, dashboard configs available offline ### + echo "Offline configs are available!" + kubectl apply -f ${BINARIES_DIR}/network.yaml + kubectl apply -f ${BINARIES_DIR}/dashborad.yaml + else + kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')" + kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-beta6/aio/deploy/recommended.yaml + fi kubectl create rolebinding admin-binding --role=admin --user=admin || true kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin --user=admin || true - kubectl create clusterrolebinding kubernetes-dashboard --clusterrole=cluster-admin --serviceaccount=kube-system:kubernetes-dashboard || true + kubectl create clusterrolebinding kubernetes-dashboard-ui --clusterrole=cluster-admin --serviceaccount=kubernetes-dashboard:kubernetes-dashboard || true coreos: units: diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index 4c1361cee47f..94b7918783ca 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -10,12 +10,9 @@ write-files: export PATH=$PATH:/opt/bin - RELEASE="v1.11.4" ISO_MOUNT_DIR=/mnt/k8sdisk - BINARIES_DIR=${ISO_MOUNT_DIR}/${RELEASE} - CNI_VERSION="v0.7.1" - CRICTL_VERSION="v1.11.1" - ATTEMPT_OFFLINE_INSTALL=true + BINARIES_DIR=${ISO_MOUNT_DIR}/ + ATTEMPT_ONLINE_INSTALL=false setup_complete=false OFFLINE_INSTALL_ATTEMPT_SLEEP=5 @@ -62,10 +59,10 @@ write-files: ### Binaries available offline ### echo "Installing binaries from ${BINARIES_DIR}" mkdir -p /opt/cni/bin - tar -f "${BINARIES_DIR}/cni/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" -C /opt/cni/bin -xz + tar -f "${BINARIES_DIR}/cni/cni-plugins-amd64.tgz" -C /opt/cni/bin -xz mkdir -p /opt/bin - tar -f "${BINARIES_DIR}/cri-tools/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" -C /opt/bin -xz + tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-amd64.tar.gz" -C /opt/bin -xz mkdir -p /opt/bin cd /opt/bin @@ -100,8 +97,11 @@ write-files: fi umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" fi - if [ "$setup_complete" = false ]; then + if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then ### Binaries not available offline ### + RELEASE="v1.16.3" + CNI_VERSION="v0.7.5" + CRICTL_VERSION="v1.16.0" echo "Warning: ${BINARIES_DIR} not found. Will get binaries and docker images from Internet." mkdir -p /opt/cni/bin curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz @@ -122,22 +122,24 @@ write-files: systemctl enable kubelet && systemctl start kubelet modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1 - crucial_cmd_attempts=1 - while true; do - if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then - echo "Error: kubeadm pull images failed!" - exit 1 - fi - retval=0 - set +e - kubeadm config images pull - retval=$? - set -e - if [ $retval -eq 0 ]; then - break; - fi - crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] - done + if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then + crucial_cmd_attempts=1 + while true; do + if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then + echo "Warning: kubeadm pull images failed after multiple tries!" + break; + fi + retval=0 + set +e + kubeadm config images pull + retval=$? + set -e + if [ $retval -eq 0 ]; then + break; + fi + crucial_cmd_attempts=$[$crucial_cmd_attempts + 1] + done + fi - path: /opt/bin/deploy-kube-system permissions: 0700 @@ -153,7 +155,7 @@ write-files: modprobe ip_vs_sh modprobe nf_conntrack_ipv4 export PATH=$PATH:/opt/bin - kubeadm join {{ k8s_master.default_ip }}:6443 --token {{ k8s_master.cluster.token }} --discovery-token-unsafe-skip-ca-verification + kubeadm join {{ k8s_master.join_ip }}:6443 --token {{ k8s_master.cluster.token }} --discovery-token-unsafe-skip-ca-verification coreos: units: @@ -183,7 +185,7 @@ coreos: Type=simple StartLimitInterval=0 Restart=on-failure - ExecStartPre=/usr/bin/curl -k https://{{ k8s_master.default_ip }}:6443/version + ExecStartPre=/usr/bin/curl -k https://{{ k8s_master.join_ip }}:6443/version ExecStart=/opt/bin/deploy-kube-system update: diff --git a/ui/plugins/cks/cks.js b/ui/plugins/cks/cks.js index 7e973af32956..325530f05301 100644 --- a/ui/plugins/cks/cks.js +++ b/ui/plugins/cks/cks.js @@ -140,7 +140,10 @@ label: 'Add Kubernetes cluster', createForm: { title: 'Add Kubernetes cluster', - preFilter: cloudStack.preFilter.createTemplate, + preFilter: function(args) { + args.$form.find('.form-item[rel=masternodes]').find('input[name=masternodes]').val('1'); + args.$form.find('.form-item[rel=size]').find('input[name=size]').val('1'); + }, fields: { name: { label: 'label.name', @@ -184,15 +187,16 @@ }, kubernetesversion: { label: 'Kubernetes version', - dependsOn: 'zone', + dependsOn: ['zone'], //docID: 'helpKubernetesClusterZone', validation: { required: true }, select: function(args) { + var filterData = { zoneid: args.zone }; $.ajax({ url: createURL("listKubernetesSupportedVersions"), - data: { zoneid: args.zone }, + data: filterData, dataType: "json", async: true, url: createURL("listKubernetesSupportedVersions"), @@ -284,12 +288,27 @@ }); } }, + multimaster: { + label: "HA (Multi-master)", + isBoolean: true, + isChecked: false, + }, + masternodes: { + label: 'Master nodes', + //docID: 'helpKubernetesClusterSize', + validation: { + required: true, + naturalnumber: true + }, + dependsOn: "multimaster", + isHidden: true, + }, size: { - label: 'Cluster size', + label: 'Cluster size (Worker nodes)', //docID: 'helpKubernetesClusterSize', validation: { required: true, - number: true + naturalnumber: true }, }, sshkeypair: { @@ -380,6 +399,14 @@ }); } + var masterNodes = 1; + if (args.data.multimaster === 'on') { + masterNodes = args.data.masternodes; + } + $.extend(data, { + masternodes: masterNodes + }); + if (args.data.supportPrivateRegistry) { $.extend(data, { dockerregistryusername: args.data.username, @@ -805,7 +832,7 @@ } } }); - return jQuery('

').html("Access Kubernetes cluster
Download Config File

Use kubectl
kubectl --kubeconfig /custom/path/kube.config {COMMAND}

List pods
kubectl --kubeconfig /custom/path/kube.config get pods --all-namespaces
Access dashboard web UI
Run proxy locally
kubectl --kubeconfig /custom/path/kube.config proxy
Open URL in browser
http://localhost:8001/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/"); + return jQuery('

').html("Access Kubernetes cluster
Download Config File

Use kubectl
kubectl --kubeconfig /custom/path/kube.config {COMMAND}

List pods
kubectl --kubeconfig /custom/path/kube.config get pods --all-namespaces
Access dashboard web UI
Run proxy locally
kubectl --kubeconfig /custom/path/kube.config proxy
Open URL in browser
http://localhost:8001/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/"); // return jQuery('

').html("Access Kubernetes cluster
Download Config File

How to do this
kubectl --kubeconfig /custom/path/kube.config get pods"); } diff --git a/ui/scripts/sharedFunctions.js b/ui/scripts/sharedFunctions.js index d3e6fe870bee..458bef855109 100644 --- a/ui/scripts/sharedFunctions.js +++ b/ui/scripts/sharedFunctions.js @@ -2709,13 +2709,25 @@ jQuery.validator.addMethod("ipv6CustomJqueryValidator", function(value, element) $.validator.addMethod("allzonesonly", function(value, element){ - if ((value.indexOf("-1") != -1) &&(value.length > 1)) + if ((value.indexOf("-1") != -1) && (value.length > 1)) return false; return true; }, "All Zones cannot be combined with any other zone"); + +$.validator.addMethod("naturalnumber", function(value, element){ + if (this.optional(element) && value.length == 0) + return true; + if (isNaN(value)) + return false; + value = parseInt(value); + return (typeof value === 'number') && (value > 0.0) && (Math.floor(value) === value) && value !== Infinity; + +}, +"Please enter a valid number, 1 or greater"); + cloudStack.createTemplateMethod = function (isSnapshot){ return { label: 'label.create.template', From 25168c25190647a6f99dd339217690fd4ad22e81 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 27 Nov 2019 12:56:57 +0530 Subject: [PATCH 006/134] added master nodes count in API response refactored response class Signed-off-by: Abhishek Kumar --- .../KubernetesClusterManagerImpl.java | 5 +- .../response/KubernetesClusterResponse.java | 183 +++++++++--------- ui/plugins/cks/cks.js | 3 + 3 files changed, 101 insertions(+), 90 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java index 0e3626339d52..914c533673b3 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java @@ -2262,14 +2262,15 @@ public KubernetesClusterConfigResponse getKubernetesClusterConfig(GetKubernetesC public KubernetesClusterResponse createKubernetesClusterResponse(long kubernetesClusterId) { KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); KubernetesClusterResponse response = new KubernetesClusterResponse(); - response.setObjectName("kubernetescluster"); + response.setObjectName(KubernetesCluster.class.getSimpleName().toLowerCase()); response.setId(kubernetesCluster.getUuid()); response.setName(kubernetesCluster.getName()); response.setDescription(kubernetesCluster.getDescription()); DataCenterVO zone = ApiDBUtils.findZoneById(kubernetesCluster.getZoneId()); response.setZoneId(zone.getUuid()); response.setZoneName(zone.getName()); - response.setClusterSize(String.valueOf(kubernetesCluster.getNodeCount())); + response.setMasterNodes(kubernetesCluster.getMasterNodeCount()); + response.setClusterSize(kubernetesCluster.getNodeCount()); VMTemplateVO template = ApiDBUtils.findTemplateById(kubernetesCluster.getTemplateId()); response.setTemplateId(template.getUuid()); ServiceOfferingVO offering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java index a734413ae3bd..7ac2fba4d4d6 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java @@ -29,6 +29,89 @@ @SuppressWarnings("unused") @EntityReference(value = {KubernetesCluster.class}) public class KubernetesClusterResponse extends BaseResponse implements ControlledEntityResponse { + @SerializedName(ApiConstants.ID) + @Param(description = "the id of the Kubernetes cluster") + private String id; + + @SerializedName(ApiConstants.NAME) + @Param(description = "Name of the Kubernetes cluster") + private String name; + + @SerializedName(ApiConstants.DESCRIPTION) + @Param(description = "Description of the Kubernetes cluster") + private String description; + + @SerializedName(ApiConstants.ZONE_ID) + @Param(description = "zone id") + private String zoneId; + + @SerializedName(ApiConstants.ZONE_NAME) + @Param(description = "zone name") + private String zoneName; + + @SerializedName(ApiConstants.SERVICE_OFFERING_ID) + @Param(description = "Service Offering id") + private String serviceOfferingId; + + @SerializedName("serviceofferingname") + @Param(description = "the name of the service offering of the virtual machine") + private String serviceOfferingName; + + @SerializedName(ApiConstants.TEMPLATE_ID) + @Param(description = "template id") + private String templateId; + + @SerializedName(ApiConstants.NETWORK_ID) + @Param(description = "network id details") + private String networkId; + + @SerializedName(ApiConstants.ASSOCIATED_NETWORK_NAME) + @Param(description = "the name of the Network associated with the IP address") + private String associatedNetworkName; + + @SerializedName(ApiConstants.SSH_KEYPAIR) + @Param(description = "keypair details") + private String keypair; + + @SerializedName(ApiConstants.MASTER_NODES) + @Param(description = "master nodes count") + private Long masterNodes; + + @SerializedName(ApiConstants.SIZE) + @Param(description = "cluster size") + private Long clusterSize; + + @SerializedName(ApiConstants.STATE) + @Param(description = "state of the cluster") + private String state; + + @SerializedName(ApiConstants.CPU_NUMBER) + @Param(description = "cluster cpu cores") + private String cores; + + @SerializedName(ApiConstants.MEMORY) + @Param(description = "cluster size") + private String memory; + + @SerializedName(ApiConstants.END_POINT) + @Param(description = "URL end point for the cluster") + private String endpoint; + + @SerializedName(ApiConstants.CONSOLE_END_POINT) + @Param(description = "URL end point for the cluster UI") + private String consoleEndpoint; + + @SerializedName(ApiConstants.VIRTUAL_MACHINE_IDS) + @Param(description = "the list of virtualmachine ids associated with this Kubernetes cluster") + private List virtualMachineIds; + + @SerializedName(ApiConstants.USERNAME) + @Param(description = "Username with which Kubernetes cluster is setup") + private String username; + + @SerializedName(ApiConstants.PASSWORD) + @Param(description = "Password with which Kubernetes cluster is setup") + private String password; public String getName() { return name; @@ -94,11 +177,19 @@ public void setKeypair(String keypair) { this.keypair = keypair; } - public String getClusterSize() { + public Long getMasterNodes() { + return masterNodes; + } + + public void setMasterNodes(Long masterNodes) { + this.masterNodes = masterNodes; + } + + public Long getClusterSize() { return clusterSize; } - public void setClusterSize(String clusterSize) { + public void setClusterSize(Long clusterSize) { this.clusterSize = clusterSize; } @@ -126,9 +217,9 @@ public void setMemory(String memory) { public void setEndpoint(String endpoint) {this.endpoint = endpoint;} - public String getConsoleEndpoint() { return consoleendpoint;} + public String getConsoleEndpoint() { return consoleEndpoint;} - public void setConsoleEndpoint(String consoleendpoint) {this.consoleendpoint = consoleendpoint;} + public void setConsoleEndpoint(String consoleEndpoint) {this.consoleEndpoint = consoleEndpoint;} public String getId() { return this.id; @@ -164,50 +255,6 @@ public List getVirtualMachineIds() { return virtualMachineIds; } - public String getConsoleendpoint() { - return consoleendpoint; - } - - @SerializedName(ApiConstants.ID) - @Param(description = "the id of the Kubernetes cluster") - private String id; - - @SerializedName(ApiConstants.NAME) - @Param(description = "Name of the Kubernetes cluster") - private String name; - - @SerializedName(ApiConstants.DESCRIPTION) - @Param(description = "Description of the Kubernetes cluster") - private String description; - - @SerializedName(ApiConstants.ZONE_ID) - @Param(description = "zone id") - private String zoneId; - - @SerializedName(ApiConstants.ZONE_NAME) - @Param(description = "zone name") - private String zoneName; - - @SerializedName(ApiConstants.SERVICE_OFFERING_ID) - @Param(description = "Service Offering id") - private String serviceOfferingId; - - @SerializedName("serviceofferingname") - @Param(description = "the name of the service offering of the virtual machine") - private String serviceOfferingName; - - @SerializedName(ApiConstants.TEMPLATE_ID) - @Param(description = "template id") - private String templateId; - - @SerializedName(ApiConstants.NETWORK_ID) - @Param(description = "network id details") - private String networkId; - - @SerializedName(ApiConstants.ASSOCIATED_NETWORK_NAME) - @Param(description = "the name of the Network associated with the IP address") - private String associatedNetworkName; - public String getAssociatedNetworkName() { return associatedNetworkName; } @@ -216,46 +263,6 @@ public void setAssociatedNetworkName(String associatedNetworkName) { this.associatedNetworkName = associatedNetworkName; } - @SerializedName(ApiConstants.SSH_KEYPAIR) - @Param(description = "keypair details") - private String keypair; - - @SerializedName(ApiConstants.SIZE) - @Param(description = "cluster size") - private String clusterSize; - - @SerializedName(ApiConstants.STATE) - @Param(description = "state of the cluster") - private String state; - - @SerializedName(ApiConstants.CPU_NUMBER) - @Param(description = "cluster cpu cores") - private String cores; - - @SerializedName(ApiConstants.MEMORY) - @Param(description = "cluster size") - private String memory; - - @SerializedName(ApiConstants.END_POINT) - @Param(description = "URL end point for the cluster") - private String endpoint; - - @SerializedName(ApiConstants.CONSOLE_END_POINT) - @Param(description = "URL end point for the cluster UI") - private String consoleendpoint; - - @SerializedName(ApiConstants.VIRTUAL_MACHINE_IDS) - @Param(description = "the list of virtualmachine ids associated with this Kubernetes cluster") - private List virtualMachineIds; - - @SerializedName(ApiConstants.USERNAME) - @Param(description = "Username with which Kubernetes cluster is setup") - private String username; - - @SerializedName(ApiConstants.PASSWORD) - @Param(description = "Password with which Kubernetes cluster is setup") - private String password; - public KubernetesClusterResponse() { } diff --git a/ui/plugins/cks/cks.js b/ui/plugins/cks/cks.js index 325530f05301..248bf89e22a4 100644 --- a/ui/plugins/cks/cks.js +++ b/ui/plugins/cks/cks.js @@ -740,6 +740,9 @@ zonename: { label: 'label.zone.name' }, + masternodes : { + label: 'Master nodes' + }, size : { label: 'Cluster Size' }, From 266738714a53298642749f2ebd28e0e9e3e6d101 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 27 Nov 2019 15:48:03 +0530 Subject: [PATCH 007/134] [wip] refactoring Signed-off-by: Abhishek Kumar --- .../KubernetesClusterManagerImpl.java | 1444 ++++++++--------- 1 file changed, 683 insertions(+), 761 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java index 914c533673b3..e26bd9e558a2 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java @@ -265,224 +265,117 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne @Inject public LoadBalancingRulesService lbService; - @Override - public KubernetesCluster findById(final Long id) { - return kubernetesClusterDao.findById(id); + private static String getStackTrace(final Throwable throwable) { + final StringWriter sw = new StringWriter(); + final PrintWriter pw = new PrintWriter(sw, true); + throwable.printStackTrace(pw); + return sw.getBuffer().toString(); } - @Override - public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) - throws InsufficientCapacityException, ResourceAllocationException, ManagementServerException { - final String name = cmd.getName(); - final String displayName = cmd.getDisplayName(); - final Long zoneId = cmd.getZoneId(); - final Long kubernetesVersionId = cmd.getKubernetesVersionId(); - final Long serviceOfferingId = cmd.getServiceOfferingId(); - final Account owner = accountService.getActiveAccountById(cmd.getEntityOwnerId()); - final Long networkId = cmd.getNetworkId(); - final String sshKeyPair= cmd.getSSHKeyPairName(); - final Long masterNodeCount = cmd.getMasterNodes(); - final Long clusterSize = cmd.getClusterSize(); - final String dockerRegistryUserName = cmd.getDockerRegistryUserName(); - final String dockerRegistryPassword = cmd.getDockerRegistryPassword(); - final String dockerRegistryUrl = cmd.getDockerRegistryUrl(); - final String dockerRegistryEmail = cmd.getDockerRegistryEmail(); - final Long nodeRootDiskSize = cmd.getNodeRootDiskSize(); - - if (name == null || name.isEmpty()) { - throw new InvalidParameterValueException("Invalid name for the kubernetes cluster name:" + name); - } - - if (masterNodeCount < 1 || masterNodeCount > 100) { - throw new InvalidParameterValueException("Invalid cluster master nodes count " + masterNodeCount); - } - - if (clusterSize < 1 || clusterSize > 100) { - throw new InvalidParameterValueException("Invalid cluster size " + clusterSize); - } - - DataCenter zone = dataCenterDao.findById(zoneId); - if (zone == null) { - throw new InvalidParameterValueException("Unable to find zone by id:" + zoneId); - } - - if (Grouping.AllocationState.Disabled == zone.getAllocationState()) { - throw new PermissionDeniedException("Cannot perform this operation, Zone:" + zone.getId() + " is currently disabled."); - } + private String readResourceFile(String resource) throws IOException { + InputStream in = Thread.currentThread().getContextClassLoader().getResourceAsStream(resource); + return IOUtils.toString(Thread.currentThread().getContextClassLoader().getResourceAsStream(resource), Charset.defaultCharset().name()); + } - final KubernetesSupportedVersion clusterKubernetesVersion = kubernetesSupportedVersionDao.findById(kubernetesVersionId); - if (clusterKubernetesVersion == null) { - throw new InvalidParameterValueException("Unable to find given Kubernetes version in supported versions"); - } - if (clusterKubernetesVersion.getZoneId() != null && !clusterKubernetesVersion.getZoneId().equals(zone.getId())) { - throw new InvalidParameterValueException(String.format("Kubernetes version ID: %s is not available for zone ID: %s", clusterKubernetesVersion.getUuid(), zone.getUuid())); + private boolean isKubernetesServiceConfigured(DataCenter zone) { + // Check Kubernetes VM template for zone + String templateName = globalConfigDao.getValue(KubernetesServiceConfig.KubernetesClusterTemplateName.key()); + if (templateName == null || templateName.isEmpty()) { + LOGGER.warn("Global setting " + KubernetesServiceConfig.KubernetesClusterTemplateName.key() + " is empty." + + "Template name need to be specified, for Kubernetes service to function."); + return false; } - if (masterNodeCount > 1 ) { - try { - if (KubernetesVersionManagerImpl.compareKubernetesVersion(clusterKubernetesVersion.getKubernetesVersion(), MIN_KUBERNETES_VERSION_HA_SUPPORT) < 0) { - throw new InvalidParameterValueException(String.format("HA support is available only for Kubernetes version %s and above. Given version ID: %s is %s", MIN_KUBERNETES_VERSION_HA_SUPPORT, clusterKubernetesVersion.getUuid(), clusterKubernetesVersion.getKubernetesVersion())); - } - } catch (Exception e) { - LOGGER.error(String.format("Unable to compare Kubernetes version for given version ID: %s with %s", clusterKubernetesVersion.getUuid(), MIN_KUBERNETES_VERSION_HA_SUPPORT), e); - } + final VMTemplateVO template = templateDao.findByTemplateName(templateName); + if (template == null) { + LOGGER.warn("Unable to find the template:" + templateName + " to be used for provisioning cluster"); + return false; } - - if (clusterKubernetesVersion.getZoneId() != null && clusterKubernetesVersion.getZoneId() != zone.getId()) { - throw new InvalidParameterValueException(String.format("Kubernetes version ID: %s is not available for zone ID: %s", clusterKubernetesVersion.getUuid(), zone.getUuid())); + // Check network offering + String networkOfferingName = globalConfigDao.getValue(KubernetesServiceConfig.KubernetesClusterNetworkOffering.key()); + if (networkOfferingName == null || networkOfferingName.isEmpty()) { + LOGGER.warn("global setting " + KubernetesServiceConfig.KubernetesClusterNetworkOffering.key() + " is empty. " + + "Admin has not yet specified the network offering to be used for provisioning isolated network for the cluster."); + return false; } - - ServiceOffering serviceOffering = serviceOfferingDao.findById(serviceOfferingId); - if (serviceOffering == null) { - throw new InvalidParameterValueException("No service offering with id:" + serviceOfferingId); - } else { + NetworkOfferingVO networkOffering = networkOfferingDao.findByUniqueName(networkOfferingName); + if (networkOffering == null) { + LOGGER.warn("Network offering with name :" + networkOfferingName + " specified by admin is not found."); + return false; } - - if (sshKeyPair != null && !sshKeyPair.isEmpty()) { - SSHKeyPairVO sshKeyPairVO = sshKeyPairDao.findByName(owner.getAccountId(), owner.getDomainId(), sshKeyPair); - if (sshKeyPairVO == null) { - throw new InvalidParameterValueException("Given SSH key pair with name:" + sshKeyPair + " was not found for the account " + owner.getAccountName()); - } + if (networkOffering.getState() == NetworkOffering.State.Disabled) { + LOGGER.warn("Network offering :" + networkOfferingName + "is not enabled."); + return false; } - - if (!isKubernetesServiceConfigured(zone)) { - throw new ManagementServerException("Kubernetes service has not been configured properly to provision kubernetes clusters."); + List services = networkOfferingServiceMapDao.listServicesForNetworkOffering(networkOffering.getId()); + if (services == null || services.isEmpty() || !services.contains("SourceNat")) { + LOGGER.warn("Network offering :" + networkOfferingName + " does not have necessary services to provision Kubernetes cluster"); + return false; } - - VMTemplateVO template = templateDao.findByTemplateName(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesClusterTemplateName.key())); - List listZoneTemplate = templateZoneDao.listByZoneTemplate(zone.getId(), template.getId()); - if (listZoneTemplate == null || listZoneTemplate.isEmpty()) { - LOGGER.warn("The template:" + template.getId() + " is not available for use in zone:" + zoneId + " to provision kubernetes cluster name:" + name); - throw new ManagementServerException("Kubernetes service has not been configured properly to provision kubernetes clusters."); + if (!networkOffering.isEgressDefaultPolicy()) { + LOGGER.warn("Network offering :" + networkOfferingName + "has egress default policy turned off should be on to provision Kubernetes cluster."); + return false; } - - if (!validateServiceOffering(serviceOfferingDao.findById(serviceOfferingId))) { - throw new InvalidParameterValueException("This service offering is not suitable for k8s cluster, service offering id is " + networkId); + long physicalNetworkId = networkModel.findPhysicalNetworkId(zone.getId(), networkOffering.getTags(), networkOffering.getTrafficType()); + PhysicalNetwork physicalNetwork = physicalNetworkDao.findById(physicalNetworkId); + if (physicalNetwork == null) { + LOGGER.warn("Unable to find physical network with ID: " + physicalNetworkId + " and tag: " + networkOffering.getTags()); + return false; } - validateDockerRegistryParams(dockerRegistryUserName, dockerRegistryPassword, dockerRegistryUrl, dockerRegistryEmail); - - plan(masterNodeCount + clusterSize, zoneId, serviceOfferingDao.findById(serviceOfferingId)); - - Network network = null; - if (networkId != null) { - if (kubernetesClusterDao.listByNetworkId(networkId).isEmpty()) { - network = networkService.getNetwork(networkId); - if (network == null) { - throw new InvalidParameterValueException("Unable to find network by ID " + networkId); - } - if (!validateNetwork(network)) { - throw new InvalidParameterValueException("This network is not suitable for k8s cluster, network id is " + networkId); - } - networkModel.checkNetworkPermissions(owner, network); - } else { - throw new InvalidParameterValueException("This network is already under use by another k8s cluster, network id is " + networkId); - } - } else { // user has not specified network in which cluster VM's to be provisioned, so create a network for kubernetes cluster - NetworkOfferingVO networkOffering = networkOfferingDao.findByUniqueName( - globalConfigDao.getValue(KubernetesServiceConfig.KubernetesClusterNetworkOffering.key())); - - long physicalNetworkId = networkModel.findPhysicalNetworkId(zone.getId(), networkOffering.getTags(), networkOffering.getTrafficType()); - PhysicalNetwork physicalNetwork = physicalNetworkDao.findById(physicalNetworkId); + return true; + } - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Creating network for account " + owner + " from the network offering id=" + - networkOffering.getId() + " as a part of cluster: " + name + " deployment process"); - } + private String generateClusterToken(KubernetesCluster kubernetesCluster) { + if (kubernetesCluster == null) return ""; + String token = kubernetesCluster.getUuid(); + token = token.replaceAll("-", ""); + token = token.substring(0, 22); + token = token.substring(0, 6) + "." + token.substring(6); + return token; + } - try { - network = networkMgr.createGuestNetwork(networkOffering.getId(), name + "-network", owner.getAccountName() + "-network", - null, null, null, false, null, owner, null, physicalNetwork, zone.getId(), ControlledEntity.ACLType.Account, null, null, null, null, true, null, null); - } catch (Exception e) { - LOGGER.warn("Unable to create a network for the kubernetes cluster due to " + e); - throw new ManagementServerException("Unable to create a network for the kubernetes cluster."); - } + private String generateClusterHACertificateKey(KubernetesCluster kubernetesCluster) { + if (kubernetesCluster == null) return ""; + String uuid = kubernetesCluster.getUuid(); + StringBuilder token = new StringBuilder(uuid.replaceAll("-", "")); + while (token.length() < 64) { + token.append(token); } + return token.toString().substring(0, 64); + } - final Network defaultNetwork = network; - final VMTemplateVO finalTemplate = template; - final long cores = serviceOffering.getCpu() * (masterNodeCount + clusterSize); - final long memory = serviceOffering.getRamSize() * (masterNodeCount + clusterSize); - - final KubernetesClusterVO cluster = Transaction.execute(new TransactionCallback() { - @Override - public KubernetesClusterVO doInTransaction(TransactionStatus status) { - KubernetesClusterVO newCluster = new KubernetesClusterVO(name, displayName, zoneId, clusterKubernetesVersion.getId(), - serviceOfferingId, finalTemplate.getId(), defaultNetwork.getId(), owner.getDomainId(), - owner.getAccountId(), masterNodeCount, clusterSize, KubernetesCluster.State.Created, sshKeyPair, cores, memory, nodeRootDiskSize, "", ""); - kubernetesClusterDao.persist(newCluster); - return newCluster; - } - }); - - Transaction.execute(new TransactionCallbackNoReturn() { + private KubernetesClusterVmMapVO addKubernetesClusterVm(final long kubernetesClusterId, final long vmId) { + return Transaction.execute(new TransactionCallback() { @Override - public void doInTransactionWithoutResult(TransactionStatus status) { - List details = new ArrayList<>(); - if (!Strings.isNullOrEmpty(dockerRegistryUserName)) { - details.add(new KubernetesClusterDetailsVO(cluster.getId(), ApiConstants.DOCKER_REGISTRY_USER_NAME, dockerRegistryUserName, true)); - } - if (!Strings.isNullOrEmpty(dockerRegistryPassword)) { - details.add(new KubernetesClusterDetailsVO(cluster.getId(), ApiConstants.DOCKER_REGISTRY_PASSWORD, dockerRegistryPassword, false)); - } - if (!Strings.isNullOrEmpty(dockerRegistryUrl)) { - details.add(new KubernetesClusterDetailsVO(cluster.getId(), ApiConstants.DOCKER_REGISTRY_URL, dockerRegistryUrl, true)); - } - if (!Strings.isNullOrEmpty(dockerRegistryEmail)) { - details.add(new KubernetesClusterDetailsVO(cluster.getId(), ApiConstants.DOCKER_REGISTRY_EMAIL, dockerRegistryEmail, true)); - } - details.add(new KubernetesClusterDetailsVO(cluster.getId(), ApiConstants.USERNAME, "admin", true)); - SecureRandom random = new SecureRandom(); - String randomPassword = new BigInteger(130, random).toString(32); - details.add(new KubernetesClusterDetailsVO(cluster.getId(), ApiConstants.PASSWORD, randomPassword, false)); - details.add(new KubernetesClusterDetailsVO(cluster.getId(), "networkCleanup", String.valueOf(networkId == null), true)); - kubernetesClusterDetailsDao.saveDetails(details); + public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { + KubernetesClusterVmMapVO newClusterVmMap = new KubernetesClusterVmMapVO(kubernetesClusterId, vmId); + kubernetesClusterVmMapDao.persist(newClusterVmMap); + return newClusterVmMap; } }); - - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("A kubernetes cluster name:" + name + " id:" + cluster.getId() + " has been created."); - } - - return cluster; } - - // Start operation can be performed at two diffrent life stages of kubernetes cluster. First when a freshly created cluster - // in which case there are no resources provisisioned for the kubernetes cluster. So during start all the resources - // are provisioned from scratch. Second kind of start, happens on Stopped kubernetes cluster, in which all resources - // are provisioned (like volumes, nics, networks etc). It just that VM's are not in running state. So just - // start the VM's (which can possibly implicitly start the network also). @Override - public boolean startKubernetesCluster(long kubernetesClusterId, boolean onCreate) throws ManagementServerException, - ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException { - - if (onCreate) { - // Start for kubernetes cluster in 'Created' state - return startKubernetesClusterOnCreate(kubernetesClusterId); - } else { - // Start for kubernetes cluster in 'Stopped' state. Resources are already provisioned, just need to be started - return startStoppedKubernetesCluster(kubernetesClusterId); - } + public KubernetesCluster findById(final Long id) { + return kubernetesClusterDao.findById(id); } // perform a cold start (which will provision resources as well) private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) throws ManagementServerException { - // Starting a kubernetes cluster has below workflow + // Starting a Kubernetes cluster has below workflow // - start the network // - provision the master /node VM // - provision node VM's (as many as cluster size) // - update the book keeping data of the VM's provisioned for the cluster // - setup networking (add Firewall and PF rules) - // - wait till kubernetes API server on master VM to come up + // - wait till Kubernetes API server on master VM to come up // - wait till addon services (dashboard etc) to come up - // - update API and dashboard URL endpoints in kubernetes cluster details + // - update API and dashboard URL endpoints in Kubernetes cluster details KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Starting kubernetes cluster: " + kubernetesCluster.getName()); - } + LOGGER.debug(String.format("Starting Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.StartRequested); @@ -492,29 +385,37 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t try { dest = plan(kubernetesClusterId, kubernetesCluster.getZoneId()); } catch (InsufficientCapacityException e) { + String msg = String.format("Provisioning the cluster failed due to insufficient capacity in the Kubernetes cluster: %s", kubernetesCluster.getUuid()); + LOGGER.error(msg, e); stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); - LOGGER.warn("Provisioning the cluster failed due to insufficient capacity in the kubernetes cluster: " + kubernetesCluster.getName() + " due to " + e); - throw new ManagementServerException("Provisioning the cluster failed due to insufficient capacity in the kubernetes cluster: " + kubernetesCluster.getName(), e); + throw new ManagementServerException(msg, e); } final ReservationContext context = new ReservationContextImpl(null, null, null, account); + Network network = networkDao.findById(kubernetesCluster.getNetworkId()); + if (network == null) { + String msg = String.format("Network for Kubernetes cluster ID: %s not found", kubernetesCluster.getUuid()); + LOGGER.warn(msg); + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + throw new ManagementServerException(msg); + } try { - networkMgr.startNetwork(kubernetesCluster.getNetworkId(), dest, context); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Network:" + kubernetesCluster.getNetworkId() + " is started for the kubernetes cluster: " + kubernetesCluster.getName()); - } + networkMgr.startNetwork(network.getId(), dest, context); + LOGGER.debug(String.format("Network ID: %s is started for the Kubernetes cluster ID: %s", network.getUuid(), kubernetesCluster.getUuid())); } catch (Exception e) { + String msg = String.format("Failed to start Kubernetes cluster ID: %s as unable to start associated network ID: %s" , kubernetesCluster.getUuid(), network.getUuid()); + LOGGER.error(msg, e); stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); - LOGGER.warn("Starting the network failed as part of starting kubernetes cluster " + kubernetesCluster.getName() + " due to " + e); - throw new ManagementServerException("Failed to start the network while creating kubernetes cluster name:" + kubernetesCluster.getName(), e); + throw new ManagementServerException(msg, e); } IPAddressVO publicIp = null; List ips = ipAddressDao.listByAssociatedNetwork(kubernetesCluster.getNetworkId(), true); if (ips == null || ips.isEmpty()) { - LOGGER.warn("Network:" + kubernetesCluster.getNetworkId() + " for the kubernetes cluster name:" + kubernetesCluster.getName() + " does not have " + - "public IP's associated with it. So aborting kubernetes cluster start."); - throw new ManagementServerException("Failed to start the network while creating kubernetes cluster name:" + kubernetesCluster.getName()); + String msg = String.format("Failed to start Kubernetes cluster ID: %s as no public IP found for associated network ID: %s" , kubernetesCluster.getUuid(), network.getUuid()); + LOGGER.warn(msg); + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + throw new ManagementServerException(msg); } publicIp = ips.get(0); @@ -523,28 +424,16 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t UserVm k8sMasterVM = null; try { k8sMasterVM = createKubernetesMaster(kubernetesCluster, ips); - - final long clusterId = kubernetesCluster.getId(); - final long masterVmId = k8sMasterVM.getId(); - Transaction.execute(new TransactionCallback() { - @Override - public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { - KubernetesClusterVmMapVO newClusterVmMap = new KubernetesClusterVmMapVO(clusterId, masterVmId); - kubernetesClusterVmMapDao.persist(newClusterVmMap); - return newClusterVmMap; - } - }); - + addKubernetesClusterVm(kubernetesCluster.getId(), k8sMasterVM.getId()); startKubernetesVM(k8sMasterVM, kubernetesCluster); clusterVMIds.add(k8sMasterVM.getId()); k8sMasterVM = userVmDao.findById(k8sMasterVM.getId()); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Provisioned the master VM's in to the kubernetes cluster name:" + kubernetesCluster.getName()); - } + LOGGER.debug(String.format("Provisioned the master VM ID: %s in to the Kubernetes cluster ID: %s", k8sMasterVM.getUuid(), kubernetesCluster.getUuid())); } catch (Exception e) { + String msg = String.format("Provisioning the master VM failed in the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); + LOGGER.warn(msg, e); stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); - LOGGER.warn("Provisioning the master VM failed in the kubernetes cluster: " + kubernetesCluster.getName() + " due to " + e); - throw new ManagementServerException("Provisioning the master VM' failed in the kubernetes cluster: " + kubernetesCluster.getName(), e); + throw new ManagementServerException(msg, e); } String masterIP = k8sMasterVM.getPrivateIpAddress(); @@ -554,26 +443,15 @@ public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { UserVm vm = null; try { vm = createKubernetesAdditionalMaster(kubernetesCluster, publicIp.getAddress().toString(), i); - final long additionalVmId = vm.getId(); - KubernetesClusterVmMapVO clusterNodeVmMap = Transaction.execute(new TransactionCallback() { - @Override - public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { - KubernetesClusterVmMapVO newClusterVmMap = new KubernetesClusterVmMapVO(kubernetesClusterId, additionalVmId); - kubernetesClusterVmMapDao.persist(newClusterVmMap); - return newClusterVmMap; - } - }); + addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId()); startKubernetesVM(vm, kubernetesCluster); clusterVMIds.add(vm.getId()); - - vm = userVmDao.findById(vm.getId()); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Provisioned a node VM in to the kubernetes cluster: " + kubernetesCluster.getName()); - } + LOGGER.debug(String.format("Provisioned additional master VM ID: %s in to the Kubernetes cluster ID: %s", vm.getUuid(), kubernetesCluster.getUuid())); } catch (Exception e) { + String msg = String.format("Provisioning additional master VM %d/%d failed in the Kubernetes cluster ID: %s", i+1, kubernetesCluster.getMasterNodeCount(), kubernetesCluster.getUuid()); + LOGGER.warn(msg, e); stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); - LOGGER.warn("Provisioning the node VM failed in the kubernetes cluster " + kubernetesCluster.getName() + " due to " + e); - throw new ManagementServerException("Provisioning the node VM failed in the kubernetes cluster " + kubernetesCluster.getName(), e); + throw new ManagementServerException(msg, e); } } } @@ -582,32 +460,18 @@ public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { UserVm vm = null; try { vm = createKubernetesNode(kubernetesCluster, masterIP, i); - final long nodeVmId = vm.getId(); - KubernetesClusterVmMapVO clusterNodeVmMap = Transaction.execute(new TransactionCallback() { - @Override - public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { - KubernetesClusterVmMapVO newClusterVmMap = new KubernetesClusterVmMapVO(kubernetesClusterId, nodeVmId); - kubernetesClusterVmMapDao.persist(newClusterVmMap); - return newClusterVmMap; - } - }); + addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId()); startKubernetesVM(vm, kubernetesCluster); clusterVMIds.add(vm.getId()); - - vm = userVmDao.findById(vm.getId()); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Provisioned a node VM in to the kubernetes cluster: " + kubernetesCluster.getName()); - } + LOGGER.debug(String.format("Provisioned node master VM ID: %s in to the Kubernetes cluster ID: %s", vm.getUuid(), kubernetesCluster.getUuid())); } catch (Exception e) { + String msg = String.format("Provisioning node VM %d/%d failed in the Kubernetes cluster ID: %s", i, kubernetesCluster.getNodeCount(), kubernetesCluster.getUuid()); + LOGGER.warn(msg, e); stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); - LOGGER.warn("Provisioning the node VM failed in the kubernetes cluster " + kubernetesCluster.getName() + " due to " + e); - throw new ManagementServerException("Provisioning the node VM failed in the kubernetes cluster " + kubernetesCluster.getName(), e); + throw new ManagementServerException(msg, e); } } - - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Kubernetes cluster : " + kubernetesCluster.getName() + " VM's are successfully provisioned."); - } + LOGGER.debug(String.format("Kubernetes cluster ID: %s VMs successfully provisioned", kubernetesCluster.getUuid())); setupKubernetesClusterNetworkRules(publicIp, account, kubernetesClusterId, clusterVMIds); attachIsoKubernetesVMs(kubernetesClusterId, clusterVMIds); @@ -620,7 +484,7 @@ public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { try { String versionOutput = IOUtils.toString(new URL(String.format("https://%s:%d/version", publicIp.getAddress().addr(), 6443)), StandardCharsets.UTF_8); if (!Strings.isNullOrEmpty(versionOutput)) { - LOGGER.debug(String.format("Kubernetes cluster: %s API has been successfully provisioned, %s", kubernetesCluster.getUuid(), versionOutput)); + LOGGER.debug(String.format("Kubernetes cluster ID: %s API has been successfully provisioned, %s", kubernetesCluster.getUuid(), versionOutput)); k8sApiServerSetup = true; kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); kubernetesCluster.setEndpoint(String.format("https://%s:%d/", publicIp.getAddress().addr(), 6443)); @@ -629,13 +493,13 @@ public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { } } catch (Exception e) { if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Waiting for kubernetes cluster: " + kubernetesCluster.getName() + " API endpoint to be available. retry: " + retryCounter + "/" + maxRetries); + LOGGER.debug("Waiting for Kubernetes cluster: " + kubernetesCluster.getName() + " API endpoint to be available. retry: " + retryCounter + "/" + maxRetries); } } try { Thread.sleep(30000); } catch (InterruptedException ie) { - LOGGER.error(String.format("Error while waiting for kubernetes cluster: %s API endpoint to be available", kubernetesCluster.getUuid()), ie); + LOGGER.error(String.format("Error while waiting for Kubernetes cluster: %s API endpoint to be available", kubernetesCluster.getUuid()), ie); } retryCounter++; } @@ -681,7 +545,7 @@ public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { while (retryCounter < maxRetries) { if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Waiting for dashboard service for the kubernetes cluster: " + kubernetesCluster.getName() + LOGGER.debug("Waiting for dashboard service for the Kubernetes cluster: " + kubernetesCluster.getName() + " to come up. Attempt: " + retryCounter + " of max retries " + maxRetries); } @@ -707,10 +571,10 @@ public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { } retryCounter++; } - LOGGER.warn("Failed to setup kubernetes cluster " + kubernetesCluster.getName() + " in usable state as" + + LOGGER.warn("Failed to setup Kubernetes cluster " + kubernetesCluster.getName() + " in usable state as" + " unable to bring dashboard add on service up"); } else { - LOGGER.warn("Failed to setup kubernetes cluster " + kubernetesCluster.getName() + " in usable state as" + + LOGGER.warn("Failed to setup Kubernetes cluster " + kubernetesCluster.getName() + " in usable state as" + " unable to bring the API server up"); } @@ -719,7 +583,7 @@ public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { detachIsoKubernetesVMs(kubernetesClusterId, clusterVMIds); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, - "Failed to deploy kubernetes cluster: " + kubernetesCluster.getUuid() + " as unable to setup up in usable state"); + "Failed to deploy Kubernetes cluster: " + kubernetesCluster.getUuid() + " as unable to setup up in usable state"); } private boolean startStoppedKubernetesCluster(long kubernetesClusterId) throws ManagementServerException, @@ -727,30 +591,24 @@ private boolean startStoppedKubernetesCluster(long kubernetesClusterId) throws M final KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); if (kubernetesCluster == null) { - throw new ManagementServerException("Failed to find kubernetes cluster id: " + kubernetesClusterId); + throw new ManagementServerException("Invalid Kubernetes cluster ID"); } if (kubernetesCluster.getRemoved() != null) { - throw new ManagementServerException("Kubernetes cluster id:" + kubernetesClusterId + " is already deleted."); + throw new ManagementServerException(String.format("Kubernetes cluster ID: %s is already deleted", kubernetesCluster.getUuid())); } if (kubernetesCluster.getState().equals(KubernetesCluster.State.Running)) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Kubernetes cluster id: " + kubernetesClusterId + " is already Running."); - } + LOGGER.debug(String.format("Kubernetes cluster ID: %s is in running state", kubernetesCluster.getUuid())); return true; } if (kubernetesCluster.getState().equals(KubernetesCluster.State.Starting)) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Kubernetes cluster id: " + kubernetesClusterId + " is getting started."); - } + LOGGER.debug(String.format("Kubernetes cluster ID: %s is already in starting state", kubernetesCluster.getUuid())); return true; } - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Starting kubernetes cluster: " + kubernetesCluster.getName()); - } + LOGGER.debug(String.format("Starting Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.StartRequested); @@ -759,11 +617,11 @@ private boolean startStoppedKubernetesCluster(long kubernetesClusterId) throws M try { if (vm == null) { stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); - throw new ManagementServerException("Failed to start all VMs in kubernetes cluster id: " + kubernetesClusterId); + throw new ManagementServerException("Failed to start all VMs in Kubernetes cluster ID: " + kubernetesClusterId); } startKubernetesVM(vm, kubernetesCluster); } catch (ServerApiException ex) { - LOGGER.warn("Failed to start VM in kubernetes cluster id:" + kubernetesClusterId + " due to " + ex); + LOGGER.warn("Failed to start VM in Kubernetes cluster ID:" + kubernetesClusterId + " due to " + ex); // dont bail out here. proceed further to stop the reset of the VM's } } @@ -772,7 +630,7 @@ private boolean startStoppedKubernetesCluster(long kubernetesClusterId) throws M final UserVmVO vm = userVmDao.findById(vmMapVO.getVmId()); if (vm == null || !vm.getState().equals(VirtualMachine.State.Running)) { stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); - throw new ManagementServerException("Failed to start all VMs in kubernetes cluster id: " + kubernetesClusterId); + throw new ManagementServerException("Failed to start all VMs in Kubernetes cluster ID: " + kubernetesClusterId); } } @@ -781,9 +639,9 @@ private boolean startStoppedKubernetesCluster(long kubernetesClusterId) throws M address = InetAddress.getByName(new URL(kubernetesCluster.getEndpoint()).getHost()); } catch (MalformedURLException | UnknownHostException ex) { // API end point is generated by CCS, so this situation should not arise. - LOGGER.warn("Kubernetes cluster id:" + kubernetesClusterId + " has invalid api endpoint. Can not " + + LOGGER.warn("Kubernetes cluster ID:" + kubernetesClusterId + " has invalid api endpoint. Can not " + "verify if cluster is in ready state."); - throw new ManagementServerException("Can not verify if kubernetes cluster id:" + kubernetesClusterId + " is in usable state."); + throw new ManagementServerException("Can not verify if Kubernetes cluster ID:" + kubernetesClusterId + " is in usable state."); } // wait for fixed time for K8S api server to be avaialble @@ -797,7 +655,7 @@ private boolean startStoppedKubernetesCluster(long kubernetesClusterId) throws M break; } catch (IOException e) { if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Waiting for kubernetes cluster: " + kubernetesCluster.getName() + " API endpoint to be available. retry: " + retryCounter + "/" + maxRetries); + LOGGER.debug("Waiting for Kubernetes cluster: " + kubernetesCluster.getName() + " API endpoint to be available. retry: " + retryCounter + "/" + maxRetries); } try { Thread.sleep(50000); @@ -809,17 +667,15 @@ private boolean startStoppedKubernetesCluster(long kubernetesClusterId) throws M if (!k8sApiServerSetup) { stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); - throw new ManagementServerException("Failed to setup kubernetes cluster id: " + kubernetesClusterId + " is usable state."); + throw new ManagementServerException("Failed to setup Kubernetes cluster ID: " + kubernetesClusterId + " is usable state."); } stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationSucceeded); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug(" Kubernetes cluster name:" + kubernetesCluster.getName() + " is successfully started."); - } + LOGGER.debug(String.format("Kubernetes cluster ID: %s successfully started", kubernetesCluster.getUuid())); return true; } - // Open up firewall port 6443, secure port on which kubernetes API server is running. Also create port-forwarding + // Open up firewall port 6443, secure port on which Kubernetes API server is running. Also create port-forwarding // rule to forward public IP traffic to master VM private IP // Open up firewall ports 2222 to 2222+n for SSH access. Also create port-forwarding // rule to forward public IP traffic to all node VM private IP @@ -858,15 +714,13 @@ private void setupKubernetesClusterNetworkRules(IPAddressVO publicIp, Account ac firewallService.createIngressFirewallRule(rule); firewallService.applyIngressFwRules(publicIp.getId(), account); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Provisioned firewall rule to open up port 6443 on " + publicIp.getAddress() + - " for cluster " + kubernetesCluster.getName()); - } + LOGGER.debug(String.format("Provisioned firewall rule to open up port 6443 on %s for Kubernetes cluster ID: %s", + publicIp.getAddress().addr(), kubernetesCluster.getUuid())); } catch (Exception e) { - LOGGER.warn("Failed to provision firewall rules for the kubernetes cluster: " + kubernetesCluster.getName() + LOGGER.warn("Failed to provision firewall rules for the Kubernetes cluster: " + kubernetesCluster.getName() + " due to exception: " + getStackTrace(e)); stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); - throw new ManagementServerException("Failed to provision firewall rules for the kubernetes " + + throw new ManagementServerException("Failed to provision firewall rules for the Kubernetes " + "cluster: " + kubernetesCluster.getName()); } @@ -902,10 +756,10 @@ private void setupKubernetesClusterNetworkRules(IPAddressVO publicIp, Account ac " for cluster " + kubernetesCluster.getName()); } } catch (Exception e) { - LOGGER.warn("Failed to provision firewall rules for the kubernetes cluster: " + kubernetesCluster.getName() + LOGGER.warn("Failed to provision firewall rules for the Kubernetes cluster: " + kubernetesCluster.getName() + " due to exception: " + getStackTrace(e)); stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); - throw new ManagementServerException("Failed to provision firewall rules for the kubernetes " + + throw new ManagementServerException("Failed to provision firewall rules for the Kubernetes " + "cluster: " + kubernetesCluster.getName()); } @@ -925,10 +779,10 @@ private void setupKubernetesClusterNetworkRules(IPAddressVO publicIp, Account ac } lbService.assignToLoadBalancer(lb.getId(), null, vmIdIpMap); } catch (Exception e) { - LOGGER.warn("Failed to provision load balancer rule for API access for the kubernetes cluster: " + kubernetesCluster.getName() + LOGGER.warn("Failed to provision load balancer rule for API access for the Kubernetes cluster: " + kubernetesCluster.getName() + " due to exception: " + getStackTrace(e)); stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); - throw new ManagementServerException("Failed to provision load balancer rule for API access for the kubernetes " + + throw new ManagementServerException("Failed to provision load balancer rule for API access for the Kubernetes " + "cluster: " + kubernetesCluster.getName()); } @@ -965,14 +819,14 @@ public PortForwardingRuleVO doInTransaction(TransactionStatus status) throws Net if (LOGGER.isDebugEnabled()) { LOGGER.debug("Provisioning SSH port forwarding rule from port 2222 to 22 on " + publicIp.getAddress() + - " to the VM IP :" + vmIp + " in kubernetes cluster " + kubernetesCluster.getName()); + " to the VM IP :" + vmIp + " in Kubernetes cluster " + kubernetesCluster.getName()); } } catch (RuntimeException rte) { - LOGGER.warn("Failed to activate SSH port forwarding rules for the kubernetes cluster " + kubernetesCluster.getName() + " due to " + rte); + LOGGER.warn("Failed to activate SSH port forwarding rules for the Kubernetes cluster " + kubernetesCluster.getName() + " due to " + rte); stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); throw new ManagementServerException("Failed to activate SSH port forwarding rules for the cluster: " + kubernetesCluster.getName(), rte); } catch (Exception e) { - LOGGER.warn("Failed to activate SSH port forwarding rules for the kubernetes cluster " + kubernetesCluster.getName() + " due to " + e); + LOGGER.warn("Failed to activate SSH port forwarding rules for the Kubernetes cluster " + kubernetesCluster.getName() + " due to " + e); stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); throw new ManagementServerException("Failed to activate SSH port forwarding rules for the cluster: " + kubernetesCluster.getName(), e); } @@ -1034,9 +888,9 @@ private void scaleKubernetesClusterNetworkRules(IPAddressVO publicIp, Account ac " for cluster " + kubernetesCluster.getName()); } } catch (Exception e) { - LOGGER.warn("Failed to provision firewall rules for the kubernetes cluster: " + kubernetesCluster.getName() + LOGGER.warn("Failed to provision firewall rules for the Kubernetes cluster: " + kubernetesCluster.getName() + " due to exception: " + getStackTrace(e)); - throw new ManagementServerException("Failed to provision firewall rules for the kubernetes " + + throw new ManagementServerException("Failed to provision firewall rules for the Kubernetes " + "cluster: " + kubernetesCluster.getName()); } @@ -1073,17 +927,17 @@ public PortForwardingRuleVO doInTransaction(TransactionStatus status) throws Net if (LOGGER.isDebugEnabled()) { LOGGER.debug("Provisioning SSH port forwarding rule from port 2222 to 22 on " + publicIp.getAddress() + - " to the VM IP :" + vmIp + " in kubernetes cluster " + kubernetesCluster.getName()); + " to the VM IP :" + vmIp + " in Kubernetes cluster " + kubernetesCluster.getName()); } } catch (Exception e) { - LOGGER.warn("Failed to activate SSH port forwarding rules for the kubernetes cluster " + kubernetesCluster.getName() + " due to " + e); + LOGGER.warn("Failed to activate SSH port forwarding rules for the Kubernetes cluster " + kubernetesCluster.getName() + " due to " + e); throw new ManagementServerException("Failed to activate SSH port forwarding rules for the cluster: " + kubernetesCluster.getName(), e); } } } } - public boolean validateNetwork(Network network) { + private boolean validateNetwork(Network network) { NetworkOffering networkOffering = networkOfferingDao.findById(network.getNetworkOfferingId()); if (networkOffering.isSystemOnly()) { throw new InvalidParameterValueException("This network is for system use only, network id " + network.getId()); @@ -1104,7 +958,7 @@ public boolean validateNetwork(Network network) { List addrs = networkModel.listPublicIpsAssignedToGuestNtwk(network.getId(), true); IPAddressVO sourceNatIp = null; if (addrs.isEmpty()) { - throw new InvalidParameterValueException("The network id:" + network.getId() + " does not have source NAT ip assoicated with it. " + + throw new InvalidParameterValueException("The network ID:" + network.getId() + " does not have source NAT ip assoicated with it. " + "To provision a Kubernetes Cluster, a isolated network with source NAT is required."); } else { for (IpAddress addr : addrs) { @@ -1113,7 +967,7 @@ public boolean validateNetwork(Network network) { } } if (sourceNatIp == null) { - throw new InvalidParameterValueException("The network id:" + network.getId() + " does not have source NAT ip assoicated with it. " + + throw new InvalidParameterValueException("The network ID:" + network.getId() + " does not have source NAT ip assoicated with it. " + "To provision a Kubernetes Cluster, a isolated network with source NAT is required."); } } @@ -1123,8 +977,8 @@ public boolean validateNetwork(Network network) { Integer endPort = rule.getSourcePortEnd(); LOGGER.debug("Network rule : " + startPort + " " + endPort); if (startPort <= 6443 && 6443 <= endPort) { - throw new InvalidParameterValueException("The network id:" + network.getId() + " has conflicting firewall rules to provision" + - " kubernetes cluster."); + throw new InvalidParameterValueException("The network ID:" + network.getId() + " has conflicting firewall rules to provision" + + " Kubernetes cluster."); } } @@ -1134,8 +988,8 @@ public boolean validateNetwork(Network network) { Integer endPort = rule.getSourcePortEnd(); LOGGER.debug("Network rule : " + startPort + " " + endPort); if (startPort <= 6443 && 6443 <= endPort) { - throw new InvalidParameterValueException("The network id:" + network.getId() + " has conflicting port forwarding rules to provision" + - " kubernetes cluster."); + throw new InvalidParameterValueException("The network ID:" + network.getId() + " has conflicting port forwarding rules to provision" + + " Kubernetes cluster."); } } return true; @@ -1146,7 +1000,7 @@ private boolean validateServiceOffering(ServiceOffering serviceOffering) { throw new InvalidParameterValueException(String.format("Custom service offerings are not supported for creating clusters, service offering ID: %s", serviceOffering.getUuid())); } if (serviceOffering.getCpu() < 2 || serviceOffering.getRamSize() < 2048) { - throw new InvalidParameterValueException(String.format("Kubernetes cluster cannot be created with service offering ID: %s, kubernetes cluster template(CoreOS) needs minimum 2 vCPUs and 2 GB RAM", serviceOffering.getUuid())); + throw new InvalidParameterValueException(String.format("Kubernetes cluster cannot be created with service offering ID: %s, Kubernetes cluster template(CoreOS) needs minimum 2 vCPUs and 2 GB RAM", serviceOffering.getUuid())); } return true; } @@ -1184,7 +1038,7 @@ private void validateDockerRegistryParams(final String dockerRegistryUserName, } } - public DeployDestination plan(final long nodesCount, final long dcId, final ServiceOffering offering) throws InsufficientServerCapacityException { + private DeployDestination plan(final long nodesCount, final long dcId, final ServiceOffering offering) throws InsufficientServerCapacityException { final int cpu_requested = offering.getCpu() * offering.getSpeed(); final long ram_requested = offering.getRamSize() * 1024L * 1024L; List hosts = resourceManager.listAllHostsInOneZoneByType(Type.Routing, dcId); @@ -1232,17 +1086,17 @@ public DeployDestination plan(final long nodesCount, final long dcId, final Serv } return new DeployDestination(dataCenterDao.findById(dcId), null, null, null); } - String msg = String.format("Cannot find enough capacity for kubernetes cluster(requested cpu=%1$s memory=%2$s)", + String msg = String.format("Cannot find enough capacity for Kubernetes cluster(requested cpu=%1$s memory=%2$s)", cpu_requested * nodesCount, ram_requested * nodesCount); LOGGER.warn(msg); throw new InsufficientServerCapacityException(msg, DataCenter.class, dcId); } - public DeployDestination plan(final KubernetesCluster kubernetesCluster) throws InsufficientServerCapacityException { + private DeployDestination plan(final KubernetesCluster kubernetesCluster) throws InsufficientServerCapacityException { return plan(kubernetesCluster.getId(), kubernetesCluster.getZoneId()); } - public DeployDestination plan(final long kubernetesClusterId, final long dcId) throws InsufficientServerCapacityException { + private DeployDestination plan(final long kubernetesClusterId, final long dcId) throws InsufficientServerCapacityException { KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); ServiceOffering offering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); @@ -1253,64 +1107,6 @@ public DeployDestination plan(final long kubernetesClusterId, final long dcId) t return plan(kubernetesCluster.getTotalNodeCount(), dcId, offering); } - @Override - public boolean stopKubernetesCluster(long kubernetesClusterId) throws ManagementServerException { - - final KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); - if (kubernetesCluster == null) { - throw new ManagementServerException("Failed to find kubernetes cluster id: " + kubernetesClusterId); - } - - if (kubernetesCluster.getRemoved() != null) { - throw new ManagementServerException("Kubernetes cluster id:" + kubernetesClusterId + " is already deleted."); - } - - if (kubernetesCluster.getState().equals(KubernetesCluster.State.Stopped)) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Kubernetes cluster id: " + kubernetesClusterId + " is already stopped."); - } - return true; - } - - if (kubernetesCluster.getState().equals(KubernetesCluster.State.Stopping)) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Kubernetes cluster id: " + kubernetesClusterId + " is getting stopped."); - } - return true; - } - - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Stopping kubernetes cluster: " + kubernetesCluster.getName()); - } - - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.StopRequested); - - for (final KubernetesClusterVmMapVO vmMapVO : kubernetesClusterVmMapDao.listByClusterId(kubernetesClusterId)) { - final UserVmVO vm = userVmDao.findById(vmMapVO.getVmId()); - try { - if (vm == null) { - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); - throw new ManagementServerException("Failed to start all VMs in kubernetes cluster id: " + kubernetesClusterId); - } - stopClusterVM(vmMapVO); - } catch (ServerApiException ex) { - LOGGER.warn("Failed to stop VM in kubernetes cluster id:" + kubernetesClusterId + " due to " + ex); - // dont bail out here. proceed further to stop the reset of the VM's - } - } - - for (final KubernetesClusterVmMapVO vmMapVO : kubernetesClusterVmMapDao.listByClusterId(kubernetesClusterId)) { - final UserVmVO vm = userVmDao.findById(vmMapVO.getVmId()); - if (vm == null || !vm.getState().equals(VirtualMachine.State.Stopped)) { - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); - throw new ManagementServerException("Failed to stop all VMs in kubernetes cluster id: " + kubernetesClusterId); - } - } - - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationSucceeded); - return true; - } - private boolean isAddOnServiceRunning(Long clusterId, final String nameSpace, String svcName) { KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(clusterId); //FIXME: whole logic needs revamp. Assumption that management server has public network access is not practical @@ -1339,7 +1135,7 @@ private boolean isAddOnServiceRunning(Long clusterId, final String nameSpace, St lines) { if (line.contains(svcName) && line.contains("Running")) { if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Service :" + svcName + " for the kubernetes cluster " + LOGGER.debug("Service :" + svcName + " for the Kubernetes cluster " + kubernetesCluster.getName() + " is running"); } return true; @@ -1352,26 +1148,8 @@ private boolean isAddOnServiceRunning(Long clusterId, final String nameSpace, St return false; } - @Override - public boolean deleteKubernetesCluster(Long kubernetesClusterId) throws ManagementServerException { - - KubernetesClusterVO cluster = kubernetesClusterDao.findById(kubernetesClusterId); - if (cluster == null) { - throw new InvalidParameterValueException("Invalid cluster id specified"); - } - - CallContext ctx = CallContext.current(); - Account caller = ctx.getCallingAccount(); - - accountManager.checkAccess(caller, SecurityChecker.AccessType.OperateEntry, false, cluster); - - return cleanupKubernetesClusterResources(kubernetesClusterId); - } - private boolean cleanupKubernetesClusterResources(Long kubernetesClusterId) throws ManagementServerException { - KubernetesClusterVO cluster = kubernetesClusterDao.findById(kubernetesClusterId); - if (!(cluster.getState().equals(KubernetesCluster.State.Running) || cluster.getState().equals(KubernetesCluster.State.Stopped) || cluster.getState().equals(KubernetesCluster.State.Alert) @@ -1382,9 +1160,7 @@ private boolean cleanupKubernetesClusterResources(Long kubernetesClusterId) thro } throw new PermissionDeniedException("Cannot perform delete operation on cluster: " + cluster.getName() + " in state " + cluster.getState()); } - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.DestroyRequested); - boolean failedVmDestroy = false; List clusterVMs = kubernetesClusterVmMapDao.listByClusterId(cluster.getId()); if ((clusterVMs != null) && !clusterVMs.isEmpty()) { @@ -1396,7 +1172,6 @@ private boolean cleanupKubernetesClusterResources(Long kubernetesClusterId) thro if (userVM == null || userVM.isRemoved()) { continue; } - try { UserVm vm = userVmService.destroyVm(vmID, true); if (!VirtualMachine.State.Expunging.equals(vm.getState())) { @@ -1467,7 +1242,7 @@ private boolean cleanupKubernetesClusterResources(Long kubernetesClusterId) thro " as part of cluster: " + cluster.getName() + " destroy"); } processFailedNetworkDelete(kubernetesClusterId); - throw new ManagementServerException("Failed to delete the network as part of kubernetes cluster name:" + cluster.getName() + " clean up"); + throw new ManagementServerException("Failed to delete the network as part of Kubernetes cluster name:" + cluster.getName() + " clean up"); } if (LOGGER.isDebugEnabled()) { LOGGER.debug("Destroyed network: " + network.getName() + " as part of cluster: " + cluster.getName() + " destroy"); @@ -1479,15 +1254,15 @@ private boolean cleanupKubernetesClusterResources(Long kubernetesClusterId) thro " as part of cluster: " + cluster.getName() + " destroy due to " + e); } processFailedNetworkDelete(kubernetesClusterId); - throw new ManagementServerException("Failed to delete the network as part of kubernetes cluster name:" + cluster.getName() + " clean up"); + throw new ManagementServerException("Failed to delete the network as part of Kubernetes cluster name:" + cluster.getName() + " clean up"); } } } else { if (LOGGER.isDebugEnabled()) { - LOGGER.debug("There are VM's that are not expunged in kubernetes cluster " + cluster.getName()); + LOGGER.debug("There are VM's that are not expunged in Kubernetes cluster " + cluster.getName()); } processFailedNetworkDelete(kubernetesClusterId); - throw new ManagementServerException("Failed to destroy one or more VM's as part of kubernetes cluster name:" + cluster.getName() + " clean up"); + throw new ManagementServerException("Failed to destroy one or more VM's as part of Kubernetes cluster name:" + cluster.getName() + " clean up"); } stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationSucceeded); @@ -1582,8 +1357,8 @@ private UserVm createKubernetesMaster(final KubernetesClusterVO kubernetesCluste initArgs += String.format("--apiserver-cert-extra-sans=%s", ips.get(0).getAddress().addr()); k8sMasterConfig = k8sMasterConfig.replace(clusterInitArgsKey, initArgs); } catch (Exception e) { - LOGGER.error("Failed to read kubernetes master configuration file due to " + e); - throw new ManagementServerException("Failed to read kubernetes master configuration file", e); + LOGGER.error("Failed to read Kubernetes master configuration file due to " + e); + throw new ManagementServerException("Failed to read Kubernetes master configuration file", e); } String base64UserData = Base64.encodeBase64String(k8sMasterConfig.getBytes(Charset.forName("UTF-8"))); masterVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, @@ -1591,7 +1366,7 @@ private UserVm createKubernetesMaster(final KubernetesClusterVO kubernetesCluste null, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), null, addrs, null, null, null, customParameterMap, null, null, null, null); if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Created master VM: " + hostName + " in the kubernetes cluster: " + kubernetesCluster.getName()); + LOGGER.debug("Created master VM: " + hostName + " in the Kubernetes cluster: " + kubernetesCluster.getName()); } return masterVm; } @@ -1631,7 +1406,7 @@ private UserVm createKubernetesAdditionalMaster(final KubernetesClusterVO kubern null, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), null, addrs, null, null, null, customParameterMap, null, null, null, null); if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Created cluster additional master node VM: " + hostName + " in the kubernetes cluster: " + kubernetesCluster.getName()); + LOGGER.debug("Created cluster additional master node VM: " + hostName + " in the Kubernetes cluster: " + kubernetesCluster.getName()); } return nodeVm; } @@ -1659,7 +1434,7 @@ private UserVm createKubernetesNode(KubernetesClusterVO kubernetesCluster, Strin final String clusterTokenKey = "{{ k8s_master.cluster.token }}"; k8sNodeConfig = k8sNodeConfig.replace(joinIpKey, joinIp); k8sNodeConfig = k8sNodeConfig.replace(clusterTokenKey, generateClusterToken(kubernetesCluster)); - /* genarate /.docker/config.json file on the nodes only if kubernetes cluster is created to + /* genarate /.docker/config.json file on the nodes only if Kubernetes cluster is created to * use docker private registry */ String dockerUserName = null; String dockerPassword = null; @@ -1716,7 +1491,7 @@ private UserVm createKubernetesNode(KubernetesClusterVO kubernetesCluster, Strin null, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), null, addrs, null, null, null, customParameterMap, null, null, null, null); if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Created cluster node VM: " + hostName + " in the kubernetes cluster: " + kubernetesCluster.getName()); + LOGGER.debug("Created cluster node VM: " + hostName + " in the Kubernetes cluster: " + kubernetesCluster.getName()); } return nodeVm; } @@ -1731,29 +1506,29 @@ private void startKubernetesVM(final UserVm vm, final KubernetesClusterVO kubern f.set(startVm, vm.getId()); userVmService.startVirtualMachine(startVm); if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Started VM in the kubernetes cluster: " + kubernetesCluster.getName()); + LOGGER.debug("Started VM in the Kubernetes cluster: " + kubernetesCluster.getName()); } } catch (ConcurrentOperationException ex) { - LOGGER.warn("Failed to start VM in the kubernetes cluster name:" + kubernetesCluster.getName() + " due to Exception: ", ex); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to start VM in the kubernetes cluster name:" + kubernetesCluster.getName(), ex); + LOGGER.warn("Failed to start VM in the Kubernetes cluster name:" + kubernetesCluster.getName() + " due to Exception: ", ex); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to start VM in the Kubernetes cluster name:" + kubernetesCluster.getName(), ex); } catch (ResourceUnavailableException ex) { - LOGGER.warn("Failed to start VM in the kubernetes cluster name:" + kubernetesCluster.getName() + " due to Exception: ", ex); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to start VM in the kubernetes cluster name:" + kubernetesCluster.getName(), ex); + LOGGER.warn("Failed to start VM in the Kubernetes cluster name:" + kubernetesCluster.getName() + " due to Exception: ", ex); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to start VM in the Kubernetes cluster name:" + kubernetesCluster.getName(), ex); } catch (InsufficientCapacityException ex) { - LOGGER.warn("Failed to start VM in the kubernetes cluster name:" + kubernetesCluster.getName() + " due to Exception: ", ex); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to start VM in the kubernetes cluster name:" + kubernetesCluster.getName(), ex); + LOGGER.warn("Failed to start VM in the Kubernetes cluster name:" + kubernetesCluster.getName() + " due to Exception: ", ex); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to start VM in the Kubernetes cluster name:" + kubernetesCluster.getName(), ex); } catch (RuntimeException ex) { - LOGGER.warn("Failed to start VM in the kubernetes cluster name:" + kubernetesCluster.getName() + " due to Exception: ", ex); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to start VM in the kubernetes cluster name:" + kubernetesCluster.getName(), ex); + LOGGER.warn("Failed to start VM in the Kubernetes cluster name:" + kubernetesCluster.getName() + " due to Exception: ", ex); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to start VM in the Kubernetes cluster name:" + kubernetesCluster.getName(), ex); } catch (Exception ex) { - LOGGER.warn("Failed to start VM in the kubernetes cluster name:" + kubernetesCluster.getName() + " due to Exception: ", ex); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to start VM in the kubernetes cluster name:" + kubernetesCluster.getName(), ex); + LOGGER.warn("Failed to start VM in the Kubernetes cluster name:" + kubernetesCluster.getName() + " due to Exception: ", ex); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to start VM in the Kubernetes cluster name:" + kubernetesCluster.getName(), ex); } UserVm startVm = userVmDao.findById(vm.getId()); if (!startVm.getState().equals(VirtualMachine.State.Running)) { LOGGER.warn("Failed to start VM instance."); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to start VM instance in kubernetes cluster " + kubernetesCluster.getName()); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to start VM instance in Kubernetes cluster " + kubernetesCluster.getName()); } } @@ -1785,36 +1560,421 @@ private void attachIsoKubernetesVMs(long kubernetesClusterId, List cluster LOGGER.debug(String.format("Attached binaries ISO for VM: %s in cluster: %s", vm.getUuid(), kubernetesCluster.getName())); } } catch (CloudRuntimeException ex) { - LOGGER.warn(String.format("Failed to attach binaries ISO for VM: %s in the kubernetes cluster name: %s due to Exception: ", vm.getDisplayName(), kubernetesCluster.getName()), ex); - // throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to attach binaries ISO for VM: %s in the kubernetes cluster name: %s", vm.getDisplayName(), kubernetesCluster.getName()), ex); + LOGGER.warn(String.format("Failed to attach binaries ISO for VM: %s in the Kubernetes cluster name: %s due to Exception: ", vm.getDisplayName(), kubernetesCluster.getName()), ex); + // throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to attach binaries ISO for VM: %s in the Kubernetes cluster name: %s", vm.getDisplayName(), kubernetesCluster.getName()), ex); + } + } + } + + private void detachIsoKubernetesVMs(long kubernetesClusterId, List clusterVMIds) throws ServerApiException { + KubernetesCluster kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + for (int i = 0; i < clusterVMIds.size(); ++i) { + UserVm vm = userVmDao.findById(clusterVMIds.get(i)); + + try { + templateService.detachIso(vm.getId()); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Started VM in the Kubernetes cluster: " + kubernetesCluster.getName()); + } + } catch (CloudRuntimeException ex) { + LOGGER.warn(String.format("Failed to detach binaries ISO for VM: %s in the Kubernetes cluster name: %s due to Exception: ", vm.getDisplayName(), kubernetesCluster.getName()), ex); + // throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to attach binaries ISO for VM: %s in the Kubernetes cluster name: %s", vm.getDisplayName(), kubernetesCluster.getName()), ex); + } + } + } + + private void stopClusterVM(final KubernetesClusterVmMapVO vmMapVO) throws ServerApiException { + try { + userVmService.stopVirtualMachine(vmMapVO.getVmId(), false); + } catch (ConcurrentOperationException ex) { + LOGGER.warn("Failed to stop Kubernetes cluster VM due to Exception: ", ex); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); + } + } + + @Override + public KubernetesClusterResponse createKubernetesClusterResponse(long kubernetesClusterId) { + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + KubernetesClusterResponse response = new KubernetesClusterResponse(); + response.setObjectName(KubernetesCluster.class.getSimpleName().toLowerCase()); + response.setId(kubernetesCluster.getUuid()); + response.setName(kubernetesCluster.getName()); + response.setDescription(kubernetesCluster.getDescription()); + DataCenterVO zone = ApiDBUtils.findZoneById(kubernetesCluster.getZoneId()); + response.setZoneId(zone.getUuid()); + response.setZoneName(zone.getName()); + response.setMasterNodes(kubernetesCluster.getMasterNodeCount()); + response.setClusterSize(kubernetesCluster.getNodeCount()); + VMTemplateVO template = ApiDBUtils.findTemplateById(kubernetesCluster.getTemplateId()); + response.setTemplateId(template.getUuid()); + ServiceOfferingVO offering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); + response.setServiceOfferingId(offering.getUuid()); + response.setServiceOfferingName(offering.getName()); + response.setKeypair(kubernetesCluster.getKeyPair()); + response.setState(kubernetesCluster.getState().toString()); + response.setCores(String.valueOf(kubernetesCluster.getCores())); + response.setMemory(String.valueOf(kubernetesCluster.getMemory())); + NetworkVO ntwk = networkDao.findByIdIncludingRemoved(kubernetesCluster.getNetworkId()); + response.setEndpoint(kubernetesCluster.getEndpoint()); + response.setNetworkId(ntwk.getUuid()); + response.setAssociatedNetworkName(ntwk.getName()); + response.setConsoleEndpoint(kubernetesCluster.getConsoleEndpoint()); + List vmIds = new ArrayList(); + List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + if (vmList != null && !vmList.isEmpty()) { + for (KubernetesClusterVmMapVO vmMapVO : vmList) { + UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId()); + if (userVM != null) { + vmIds.add(userVM.getUuid()); + } + } + } + response.setVirtualMachineIds(vmIds); + return response; + } + + protected boolean stateTransitTo(long kubernetesClusterId, KubernetesCluster.Event e) { + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + try { + return _stateMachine.transitTo(kubernetesCluster, e, null, kubernetesClusterDao); + } catch (NoTransitionException nte) { + LOGGER.warn("Failed to transistion state of the Kubernetes cluster: " + kubernetesCluster.getName() + + " in state " + kubernetesCluster.getState().toString() + " on event " + e.toString()); + return false; + } + } + + @Override + public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) + throws InsufficientCapacityException, ResourceAllocationException, ManagementServerException { + final String name = cmd.getName(); + final String displayName = cmd.getDisplayName(); + final Long zoneId = cmd.getZoneId(); + final Long kubernetesVersionId = cmd.getKubernetesVersionId(); + final Long serviceOfferingId = cmd.getServiceOfferingId(); + final Account owner = accountService.getActiveAccountById(cmd.getEntityOwnerId()); + final Long networkId = cmd.getNetworkId(); + final String sshKeyPair= cmd.getSSHKeyPairName(); + final Long masterNodeCount = cmd.getMasterNodes(); + final Long clusterSize = cmd.getClusterSize(); + final String dockerRegistryUserName = cmd.getDockerRegistryUserName(); + final String dockerRegistryPassword = cmd.getDockerRegistryPassword(); + final String dockerRegistryUrl = cmd.getDockerRegistryUrl(); + final String dockerRegistryEmail = cmd.getDockerRegistryEmail(); + final Long nodeRootDiskSize = cmd.getNodeRootDiskSize(); + + if (name == null || name.isEmpty()) { + throw new InvalidParameterValueException("Invalid name for the Kubernetes cluster name:" + name); + } + + if (masterNodeCount < 1 || masterNodeCount > 100) { + throw new InvalidParameterValueException("Invalid cluster master nodes count " + masterNodeCount); + } + + if (clusterSize < 1 || clusterSize > 100) { + throw new InvalidParameterValueException("Invalid cluster size " + clusterSize); + } + + DataCenter zone = dataCenterDao.findById(zoneId); + if (zone == null) { + throw new InvalidParameterValueException("Unable to find zone by ID:" + zoneId); + } + + if (Grouping.AllocationState.Disabled == zone.getAllocationState()) { + throw new PermissionDeniedException("Cannot perform this operation, Zone:" + zone.getId() + " is currently disabled."); + } + + final KubernetesSupportedVersion clusterKubernetesVersion = kubernetesSupportedVersionDao.findById(kubernetesVersionId); + if (clusterKubernetesVersion == null) { + throw new InvalidParameterValueException("Unable to find given Kubernetes version in supported versions"); + } + if (clusterKubernetesVersion.getZoneId() != null && !clusterKubernetesVersion.getZoneId().equals(zone.getId())) { + throw new InvalidParameterValueException(String.format("Kubernetes version ID: %s is not available for zone ID: %s", clusterKubernetesVersion.getUuid(), zone.getUuid())); + } + if (masterNodeCount > 1 ) { + try { + if (KubernetesVersionManagerImpl.compareKubernetesVersion(clusterKubernetesVersion.getKubernetesVersion(), MIN_KUBERNETES_VERSION_HA_SUPPORT) < 0) { + throw new InvalidParameterValueException(String.format("HA support is available only for Kubernetes version %s and above. Given version ID: %s is %s", MIN_KUBERNETES_VERSION_HA_SUPPORT, clusterKubernetesVersion.getUuid(), clusterKubernetesVersion.getKubernetesVersion())); + } + } catch (Exception e) { + LOGGER.error(String.format("Unable to compare Kubernetes version for given version ID: %s with %s", clusterKubernetesVersion.getUuid(), MIN_KUBERNETES_VERSION_HA_SUPPORT), e); + } + } + + if (clusterKubernetesVersion.getZoneId() != null && clusterKubernetesVersion.getZoneId() != zone.getId()) { + throw new InvalidParameterValueException(String.format("Kubernetes version ID: %s is not available for zone ID: %s", clusterKubernetesVersion.getUuid(), zone.getUuid())); + } + + ServiceOffering serviceOffering = serviceOfferingDao.findById(serviceOfferingId); + if (serviceOffering == null) { + throw new InvalidParameterValueException("No service offering with ID:" + serviceOfferingId); + } else { + } + + if (sshKeyPair != null && !sshKeyPair.isEmpty()) { + SSHKeyPairVO sshKeyPairVO = sshKeyPairDao.findByName(owner.getAccountId(), owner.getDomainId(), sshKeyPair); + if (sshKeyPairVO == null) { + throw new InvalidParameterValueException("Given SSH key pair with name:" + sshKeyPair + " was not found for the account " + owner.getAccountName()); + } + } + + if (!isKubernetesServiceConfigured(zone)) { + throw new ManagementServerException("Kubernetes service has not been configured properly to provision Kubernetes clusters."); + } + + VMTemplateVO template = templateDao.findByTemplateName(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesClusterTemplateName.key())); + List listZoneTemplate = templateZoneDao.listByZoneTemplate(zone.getId(), template.getId()); + if (listZoneTemplate == null || listZoneTemplate.isEmpty()) { + LOGGER.warn("The template:" + template.getId() + " is not available for use in zone:" + zoneId + " to provision Kubernetes cluster name:" + name); + throw new ManagementServerException("Kubernetes service has not been configured properly to provision Kubernetes clusters."); + } + + if (!validateServiceOffering(serviceOfferingDao.findById(serviceOfferingId))) { + throw new InvalidParameterValueException("This service offering is not suitable for k8s cluster, service offering id is " + networkId); + } + + validateDockerRegistryParams(dockerRegistryUserName, dockerRegistryPassword, dockerRegistryUrl, dockerRegistryEmail); + + plan(masterNodeCount + clusterSize, zoneId, serviceOfferingDao.findById(serviceOfferingId)); + + Network network = null; + if (networkId != null) { + if (kubernetesClusterDao.listByNetworkId(networkId).isEmpty()) { + network = networkService.getNetwork(networkId); + if (network == null) { + throw new InvalidParameterValueException("Unable to find network by ID " + networkId); + } + if (!validateNetwork(network)) { + throw new InvalidParameterValueException("This network is not suitable for k8s cluster, network id is " + networkId); + } + networkModel.checkNetworkPermissions(owner, network); + } else { + throw new InvalidParameterValueException("This network is already under use by another k8s cluster, network id is " + networkId); + } + } else { // user has not specified network in which cluster VM's to be provisioned, so create a network for Kubernetes cluster + NetworkOfferingVO networkOffering = networkOfferingDao.findByUniqueName( + globalConfigDao.getValue(KubernetesServiceConfig.KubernetesClusterNetworkOffering.key())); + + long physicalNetworkId = networkModel.findPhysicalNetworkId(zone.getId(), networkOffering.getTags(), networkOffering.getTrafficType()); + PhysicalNetwork physicalNetwork = physicalNetworkDao.findById(physicalNetworkId); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Creating network for account " + owner + " from the network offering id=" + + networkOffering.getId() + " as a part of cluster: " + name + " deployment process"); + } + + try { + network = networkMgr.createGuestNetwork(networkOffering.getId(), name + "-network", owner.getAccountName() + "-network", + null, null, null, false, null, owner, null, physicalNetwork, zone.getId(), ControlledEntity.ACLType.Account, null, null, null, null, true, null, null); + } catch (Exception e) { + LOGGER.warn("Unable to create a network for the Kubernetes cluster due to " + e); + throw new ManagementServerException("Unable to create a network for the Kubernetes cluster."); + } + } + + final Network defaultNetwork = network; + final VMTemplateVO finalTemplate = template; + final long cores = serviceOffering.getCpu() * (masterNodeCount + clusterSize); + final long memory = serviceOffering.getRamSize() * (masterNodeCount + clusterSize); + + final KubernetesClusterVO cluster = Transaction.execute(new TransactionCallback() { + @Override + public KubernetesClusterVO doInTransaction(TransactionStatus status) { + KubernetesClusterVO newCluster = new KubernetesClusterVO(name, displayName, zoneId, clusterKubernetesVersion.getId(), + serviceOfferingId, finalTemplate.getId(), defaultNetwork.getId(), owner.getDomainId(), + owner.getAccountId(), masterNodeCount, clusterSize, KubernetesCluster.State.Created, sshKeyPair, cores, memory, nodeRootDiskSize, "", ""); + kubernetesClusterDao.persist(newCluster); + return newCluster; + } + }); + + Transaction.execute(new TransactionCallbackNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + List details = new ArrayList<>(); + if (!Strings.isNullOrEmpty(dockerRegistryUserName)) { + details.add(new KubernetesClusterDetailsVO(cluster.getId(), ApiConstants.DOCKER_REGISTRY_USER_NAME, dockerRegistryUserName, true)); + } + if (!Strings.isNullOrEmpty(dockerRegistryPassword)) { + details.add(new KubernetesClusterDetailsVO(cluster.getId(), ApiConstants.DOCKER_REGISTRY_PASSWORD, dockerRegistryPassword, false)); + } + if (!Strings.isNullOrEmpty(dockerRegistryUrl)) { + details.add(new KubernetesClusterDetailsVO(cluster.getId(), ApiConstants.DOCKER_REGISTRY_URL, dockerRegistryUrl, true)); + } + if (!Strings.isNullOrEmpty(dockerRegistryEmail)) { + details.add(new KubernetesClusterDetailsVO(cluster.getId(), ApiConstants.DOCKER_REGISTRY_EMAIL, dockerRegistryEmail, true)); + } + details.add(new KubernetesClusterDetailsVO(cluster.getId(), ApiConstants.USERNAME, "admin", true)); + SecureRandom random = new SecureRandom(); + String randomPassword = new BigInteger(130, random).toString(32); + details.add(new KubernetesClusterDetailsVO(cluster.getId(), ApiConstants.PASSWORD, randomPassword, false)); + details.add(new KubernetesClusterDetailsVO(cluster.getId(), "networkCleanup", String.valueOf(networkId == null), true)); + kubernetesClusterDetailsDao.saveDetails(details); + } + }); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("A Kubernetes cluster name:" + name + " ID:" + cluster.getId() + " has been created."); + } + + return cluster; + } + + + // Start operation can be performed at two diffrent life stages of Kubernetes cluster. First when a freshly created cluster + // in which case there are no resources provisisioned for the Kubernetes cluster. So during start all the resources + // are provisioned from scratch. Second kind of start, happens on Stopped Kubernetes cluster, in which all resources + // are provisioned (like volumes, nics, networks etc). It just that VM's are not in running state. So just + // start the VM's (which can possibly implicitly start the network also). + @Override + public boolean startKubernetesCluster(long kubernetesClusterId, boolean onCreate) throws ManagementServerException, + ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException { + + if (onCreate) { + // Start for Kubernetes cluster in 'Created' state + return startKubernetesClusterOnCreate(kubernetesClusterId); + } else { + // Start for Kubernetes cluster in 'Stopped' state. Resources are already provisioned, just need to be started + return startStoppedKubernetesCluster(kubernetesClusterId); + } + } + + @Override + public boolean stopKubernetesCluster(long kubernetesClusterId) throws ManagementServerException { + + final KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + if (kubernetesCluster == null) { + throw new ManagementServerException("Failed to find Kubernetes cluster ID: " + kubernetesClusterId); + } + + if (kubernetesCluster.getRemoved() != null) { + throw new ManagementServerException("Kubernetes cluster ID:" + kubernetesClusterId + " is already deleted."); + } + + if (kubernetesCluster.getState().equals(KubernetesCluster.State.Stopped)) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Kubernetes cluster ID: " + kubernetesClusterId + " is already stopped."); + } + return true; + } + + if (kubernetesCluster.getState().equals(KubernetesCluster.State.Stopping)) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Kubernetes cluster ID: " + kubernetesClusterId + " is getting stopped."); + } + return true; + } + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Stopping Kubernetes cluster: " + kubernetesCluster.getName()); + } + + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.StopRequested); + + for (final KubernetesClusterVmMapVO vmMapVO : kubernetesClusterVmMapDao.listByClusterId(kubernetesClusterId)) { + final UserVmVO vm = userVmDao.findById(vmMapVO.getVmId()); + try { + if (vm == null) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + throw new ManagementServerException("Failed to start all VMs in Kubernetes cluster ID: " + kubernetesClusterId); + } + stopClusterVM(vmMapVO); + } catch (ServerApiException ex) { + LOGGER.warn("Failed to stop VM in Kubernetes cluster ID:" + kubernetesClusterId + " due to " + ex); + // dont bail out here. proceed further to stop the reset of the VM's + } + } + + for (final KubernetesClusterVmMapVO vmMapVO : kubernetesClusterVmMapDao.listByClusterId(kubernetesClusterId)) { + final UserVmVO vm = userVmDao.findById(vmMapVO.getVmId()); + if (vm == null || !vm.getState().equals(VirtualMachine.State.Stopped)) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + throw new ManagementServerException("Failed to stop all VMs in Kubernetes cluster ID: " + kubernetesClusterId); + } + } + + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationSucceeded); + return true; + } + + @Override + public boolean deleteKubernetesCluster(Long kubernetesClusterId) throws ManagementServerException { + KubernetesClusterVO cluster = kubernetesClusterDao.findById(kubernetesClusterId); + if (cluster == null) { + throw new InvalidParameterValueException("Invalid cluster id specified"); + } + CallContext ctx = CallContext.current(); + Account caller = ctx.getCallingAccount(); + accountManager.checkAccess(caller, SecurityChecker.AccessType.OperateEntry, false, cluster); + return cleanupKubernetesClusterResources(kubernetesClusterId); + } + + @Override + public ListResponse listKubernetesClusters(ListKubernetesClustersCmd cmd) { + final CallContext ctx = CallContext.current(); + final Account caller = ctx.getCallingAccount(); + final Long clusterId = cmd.getId(); + final String state = cmd.getState(); + final String name = cmd.getName(); + + List responsesList = new ArrayList(); + if (state != null && !state.isEmpty()) { + if (!KubernetesCluster.State.Running.toString().equals(state) && + !KubernetesCluster.State.Stopped.toString().equals(state) && + !KubernetesCluster.State.Destroyed.toString().equals(state)) { + throw new InvalidParameterValueException("Invalid value for Kubernetes cluster state is specified"); + } + } + if (clusterId != null) { + KubernetesClusterVO cluster = kubernetesClusterDao.findById(clusterId); + if (cluster == null) { + throw new InvalidParameterValueException("Invalid Kubernetes cluster ID specified"); + } + accountManager.checkAccess(caller, SecurityChecker.AccessType.ListEntry, false, cluster); + responsesList.add(createKubernetesClusterResponse(clusterId)); + } else { + SearchCriteria sc = kubernetesClusterDao.createSearchCriteria(); + Filter searchFilter = new Filter(KubernetesClusterVO.class, "id", true, cmd.getStartIndex(), cmd.getPageSizeVal()); + if (state != null && !state.isEmpty()) { + sc.addAnd("state", SearchCriteria.Op.EQ, state); + } + if (accountManager.isNormalUser(caller.getId())) { + sc.addAnd("accountId", SearchCriteria.Op.EQ, caller.getAccountId()); + } else if (accountManager.isDomainAdmin(caller.getId())) { + sc.addAnd("domainId", SearchCriteria.Op.EQ, caller.getDomainId()); } - } - } - - private void detachIsoKubernetesVMs(long kubernetesClusterId, List clusterVMIds) throws ServerApiException { - KubernetesCluster kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); - for (int i = 0; i < clusterVMIds.size(); ++i) { - UserVm vm = userVmDao.findById(clusterVMIds.get(i)); - - try { - templateService.detachIso(vm.getId()); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Started VM in the kubernetes cluster: " + kubernetesCluster.getName()); - } - } catch (CloudRuntimeException ex) { - LOGGER.warn(String.format("Failed to detach binaries ISO for VM: %s in the kubernetes cluster name: %s due to Exception: ", vm.getDisplayName(), kubernetesCluster.getName()), ex); - // throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to attach binaries ISO for VM: %s in the kubernetes cluster name: %s", vm.getDisplayName(), kubernetesCluster.getName()), ex); + if (name != null && !name.isEmpty()) { + sc.addAnd("name", SearchCriteria.Op.LIKE, name); + } + List kubernetesClusters = kubernetesClusterDao.search(sc, searchFilter); + for (KubernetesClusterVO cluster : kubernetesClusters) { + KubernetesClusterResponse clusterResponse = createKubernetesClusterResponse(cluster.getId()); + responsesList.add(clusterResponse); } } + ListResponse response = new ListResponse(); + response.setResponses(responsesList); + return response; } - private void stopClusterVM(final KubernetesClusterVmMapVO vmMapVO) throws ServerApiException { + public KubernetesClusterConfigResponse getKubernetesClusterConfig(GetKubernetesClusterConfigCmd cmd) { + final Long clusterId = cmd.getId(); + KubernetesCluster kubernetesCluster = kubernetesClusterDao.findById(clusterId); + if (kubernetesCluster == null) { + throw new InvalidParameterValueException("Invalid Kubernetes cluster ID specified"); + } + KubernetesClusterConfigResponse response = new KubernetesClusterConfigResponse(); + response.setId(kubernetesCluster.getUuid()); + response.setName(kubernetesCluster.getName()); + String configData = ""; try { - userVmService.stopVirtualMachine(vmMapVO.getVmId(), false); - } catch (ConcurrentOperationException ex) { - LOGGER.warn("Failed to stop kubernetes cluster VM due to Exception: ", ex); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); + configData = new String(Base64.decodeBase64(kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "kubeConfigData").getValue())); + } catch (Exception e) { + LOGGER.warn(String.format("Failed to retrieve Kubernetes config for cluster ID: %s", kubernetesCluster.getUuid()), e); } + response.setConfigData(configData); + response.setObjectName("clusterconfig"); + return response; } @Override @@ -1823,62 +1983,55 @@ public boolean scaleKubernetesCluster(ScaleKubernetesClusterCmd cmd) throws Mana final Long serviceOfferingId = cmd.getServiceOfferingId(); final Long clusterSize = cmd.getClusterSize(); KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); - if (kubernetesCluster == null) { - throw new InvalidParameterValueException("Failed to find kubernetes cluster id: " + kubernetesClusterId); + if (kubernetesCluster == null || kubernetesCluster.getRemoved() != null) { + throw new InvalidParameterValueException("Invalid Kubernetes cluster ID"); } Account caller = CallContext.current().getCallingAccount(); - accountManager.checkAccess(caller, SecurityChecker.AccessType.OperateEntry, false, kubernetesCluster); if (serviceOfferingId == null && clusterSize == null) { - throw new InvalidParameterValueException(String.format("Kubernetes cluster id: %s cannot be scaled, either a new service offering or a new cluster size must be passed", kubernetesCluster.getUuid())); + throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be scaled, either a new service offering or a new cluster size must be passed", kubernetesCluster.getUuid())); } ServiceOffering serviceOffering = null; if (serviceOfferingId != null) { serviceOffering = serviceOfferingDao.findById(serviceOfferingId); if (serviceOffering == null) { - throw new InvalidParameterValueException("Failed to find service offering id: " + serviceOfferingId); + throw new InvalidParameterValueException("Failed to find service offering ID: " + serviceOfferingId); } else { if (serviceOffering.isDynamic()) { - throw new InvalidParameterValueException(String.format("Custom service offerings are not supported for kubernetes clusters. Kubernetes cluster ID: %s, service offering ID: %s", kubernetesCluster.getUuid(), serviceOffering.getUuid())); + throw new InvalidParameterValueException(String.format("Custom service offerings are not supported for Kubernetes clusters. Kubernetes cluster ID: %s, service offering ID: %s", kubernetesCluster.getUuid(), serviceOffering.getUuid())); } if (serviceOffering.getCpu() < 2 || serviceOffering.getRamSize() < 2048) { - throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be scaled with service offering ID: %s, kubernetes cluster template(CoreOS) needs minimum 2 vCPUs and 2 GB RAM", kubernetesCluster.getUuid(), serviceOffering.getUuid())); + throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be scaled with service offering ID: %s, Kubernetes cluster template(CoreOS) needs minimum 2 vCPUs and 2 GB RAM", kubernetesCluster.getUuid(), serviceOffering.getUuid())); } } } - if (kubernetesCluster.getRemoved() != null) { - throw new ManagementServerException("Kubernetes cluster id:" + kubernetesCluster.getUuid() + " is already deleted."); - } - if (!(kubernetesCluster.getState().equals(KubernetesCluster.State.Created) || kubernetesCluster.getState().equals(KubernetesCluster.State.Running) || kubernetesCluster.getState().equals(KubernetesCluster.State.Stopped))) { - throw new PermissionDeniedException(String.format("Kubernetes cluster id: %s is in %s state", kubernetesCluster.getUuid(), kubernetesCluster.getState().toString())); + throw new PermissionDeniedException(String.format("Kubernetes cluster ID: %s is in %s state", kubernetesCluster.getUuid(), kubernetesCluster.getState().toString())); } if (clusterSize != null) { if (kubernetesCluster.getState().equals(KubernetesCluster.State.Stopped)) { // Cannot scale stopped cluster currently for cluster size - throw new PermissionDeniedException(String.format("Kubernetes cluster id: %s is in %s state", kubernetesCluster.getUuid(), kubernetesCluster.getState().toString())); + throw new PermissionDeniedException(String.format("Kubernetes cluster ID: %s is in %s state", kubernetesCluster.getUuid(), kubernetesCluster.getState().toString())); } if (clusterSize < 1) { - throw new InvalidParameterValueException(String.format("Kubernetes cluster id: %s cannot be scaled for size, %d", kubernetesCluster.getUuid(), clusterSize)); + throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be scaled for size, %d", kubernetesCluster.getUuid(), clusterSize)); } } - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Scaling kubernetes cluster: " + kubernetesCluster.getName()); - } + LOGGER.debug(String.format("Scaling Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); final KubernetesCluster.State clusterState = kubernetesCluster.getState(); final long originalNodeCount = kubernetesCluster.getNodeCount(); final ServiceOffering existingServiceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); if (existingServiceOffering == null) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, service offering for the kubernetes cluster not found!", kubernetesCluster.getUuid())); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, service offering for the Kubernetes cluster not found!", kubernetesCluster.getUuid())); } final boolean serviceOfferingScalingNeeded = serviceOffering != null && serviceOffering.getId() != existingServiceOffering.getId(); final boolean clusterSizeScalingNeeded = clusterSize != null && clusterSize != originalNodeCount; @@ -1886,19 +2039,19 @@ public boolean scaleKubernetesCluster(ScaleKubernetesClusterCmd cmd) throws Mana List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); if (vmList == null || vmList.isEmpty() || vmList.size() - 1 < originalNodeCount) { stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, it is in unstable state as not enough existing VM instances found!", kubernetesCluster.getUuid())); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, it is in unstable state as not enough existing VM instances found!", kubernetesCluster.getUuid())); } else { for (KubernetesClusterVmMapVO vmMapVO : vmList) { VMInstanceVO vmInstance = vmInstanceDao.findById(vmMapVO.getVmId()); if (vmInstance != null && vmInstance.getState().equals(VirtualMachine.State.Running) && vmInstance.getHypervisorType() != Hypervisor.HypervisorType.XenServer && vmInstance.getHypervisorType() != Hypervisor.HypervisorType.VMware && vmInstance.getHypervisorType() != Hypervisor.HypervisorType.Simulator) { LOGGER.info("Scaling the VM dynamically is not supported for VMs running on Hypervisor " + vmInstance.getHypervisorType()); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, scaling kubernetes cluster with running VMs on hypervisor %s is not supported!", kubernetesCluster.getUuid(), vmInstance.getHypervisorType())); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, scaling Kubernetes cluster with running VMs on hypervisor %s is not supported!", kubernetesCluster.getUuid(), vmInstance.getHypervisorType())); } } } if (serviceOffering.getRamSize() < existingServiceOffering.getRamSize() || - serviceOffering.getCpu()*serviceOffering.getSpeed() < existingServiceOffering.getCpu()*existingServiceOffering.getSpeed()) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, service offering for the kubernetes cluster cannot be scaled down!", kubernetesCluster.getUuid())); + serviceOffering.getCpu()*serviceOffering.getSpeed() < existingServiceOffering.getCpu()*existingServiceOffering.getSpeed()) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, service offering for the Kubernetes cluster cannot be scaled down!", kubernetesCluster.getUuid())); } // ToDo: Check capacity with new service offering at this point, how? @@ -1922,7 +2075,7 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { }); if (kubernetesCluster == null) { stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, unable to update kubernetes cluster!", kubernetesCluster.getUuid())); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, unable to update Kubernetes cluster!", kubernetesCluster.getUuid())); } kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); final long tobeScaledVMCount = Math.min(vmList.size(), size+1); @@ -1934,11 +2087,11 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { result = userVmManager.upgradeVirtualMachine(userVM.getId(), serviceOffering.getId(), new HashMap()); } catch (Exception e) { stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, unable to scale cluster VM ID: %s! %s", kubernetesCluster.getUuid(), userVM.getUuid(), e.getMessage()), e); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, unable to scale cluster VM ID: %s! %s", kubernetesCluster.getUuid(), userVM.getUuid(), e.getMessage()), e); } if (!result) { stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, unable to scale cluster VM ID: %s!", kubernetesCluster.getUuid(), userVM.getUuid())); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, unable to scale cluster VM ID: %s!", kubernetesCluster.getUuid(), userVM.getUuid())); } } } @@ -1948,8 +2101,10 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { final long newVmRequiredCount = clusterSize - originalNodeCount; final ServiceOffering clusterServiceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); if (clusterServiceOffering == null) { + String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, service offering not found", kubernetesCluster.getUuid()); + LOGGER.error(msg); stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, service offering for the kubernetes cluster not found!", kubernetesCluster.getUuid())); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } if (newVmRequiredCount > 0) { if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { @@ -1962,9 +2117,10 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { plan(kubernetesCluster.getTotalNodeCount() + newVmRequiredCount, kubernetesCluster.getZoneId(), clusterServiceOffering); } } catch (InsufficientCapacityException e) { + String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, insufficient capacity", kubernetesCluster.getUuid()); + LOGGER.error(msg); stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); - LOGGER.warn("Scaling the cluster failed due to insufficient capacity in the kubernetes cluster: " + kubernetesCluster.getName() + " due to " + e); - throw new ServerApiException(ApiErrorCode.INSUFFICIENT_CAPACITY_ERROR, "Provisioning the cluster failed due to insufficient capacity in the kubernetes cluster: " + kubernetesCluster.getName(), e); + throw new ServerApiException(ApiErrorCode.INSUFFICIENT_CAPACITY_ERROR, msg, e); } } else { if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { @@ -1989,8 +2145,10 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { } }); if (kubernetesCluster == null) { + String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, unable to update cluster", kubernetesCluster.getUuid()); + LOGGER.warn(msg); stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, unable to update kubernetes cluster!", kubernetesCluster.getUuid())); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); } @@ -1999,15 +2157,17 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { if (clusterState.equals(KubernetesCluster.State.Running)) { List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); if (vmList == null || vmList.isEmpty() || vmList.size() - 1 < originalNodeCount) { + String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, t is in unstable state as not enough existing VM instances found", kubernetesCluster.getUuid()); + LOGGER.error(msg); stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, it is in unstable state as not enough existing VM instances found!", kubernetesCluster.getUuid())); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } IPAddressVO publicIp = null; List ips = ipAddressDao.listByAssociatedNetwork(kubernetesCluster.getNetworkId(), true); if (ips == null || ips.isEmpty() || ips.get(0) == null) { - LOGGER.warn("Network:" + kubernetesCluster.getNetworkId() + " for the kubernetes cluster name:" + kubernetesCluster.getName() + " does not have " + - "public IP's associated with it. So aborting kubernetes cluster scaling."); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, unable to connect the network!", kubernetesCluster.getUuid())); + String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, unable to retrieve associated public IP", kubernetesCluster.getUuid()); + LOGGER.error(msg); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } publicIp = ips.get(0); Boolean devel = Boolean.valueOf(globalConfigDao.getValue("developer")); @@ -2033,13 +2193,13 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { pkFile, null, String.format("sudo kubectl drain %s --ignore-daemonsets --delete-local-data", userVM.getHostName()), 10000, 10000, 30000); if (!result.first()) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Draining kubernetes node unsuccessful"); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Draining Kubernetes node unsuccessful"); } result = SshHelper.sshExecute(publicIp.getAddress().addr(), 2222, "core", pkFile, null, String.format("sudo kubectl delete node %s", userVM.getHostName()), 10000, 10000, 30000); if (!result.first()) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Deleting kubernetes node unsuccessful"); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Deleting Kubernetes node unsuccessful"); } break; } catch (Exception e) { @@ -2048,9 +2208,10 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { Thread.sleep(30000); } catch (InterruptedException ie) {} } else { - LOGGER.warn("Failed to remove kubernetes node while scaling kubernetes cluster with ID " + kubernetesCluster.getUuid() + ": " + e); + String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, failed to remove Kubernetes node", kubernetesCluster.getUuid()); + LOGGER.warn(msg, e); stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, unable to remove kubernetes node!" + e, kubernetesCluster.getUuid())); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } } } @@ -2069,14 +2230,14 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { UserVm vm = userVmService.destroyVm(userVM.getId(), true); if (!VirtualMachine.State.Expunging.equals(vm.getState())) { stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, VM '%s' is now in state '%s'." + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, VM '%s' is now in state '%s'." , kubernetesCluster.getUuid() , vm.getInstanceName() , vm.getState().toString())); } vm = userVmService.expungeVm(userVM.getId()); if (!VirtualMachine.State.Expunging.equals(vm.getState())) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, VM '%s' is now in state '%s'." + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, VM '%s' is now in state '%s'." , kubernetesCluster.getUuid() , vm.getInstanceName() , vm.getState().toString())); @@ -2092,8 +2253,10 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { try { scaleKubernetesClusterNetworkRules(publicIp, account, kubernetesClusterId, null); } catch (Exception e) { + String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, unable to update network rules", kubernetesCluster.getUuid()); + LOGGER.error(msg, e); stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, unable to update network rules!", kubernetesCluster.getUuid()), e); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg, e); } } else { // upscale, same node count handled above UserVmVO masterVm = userVmDao.findById(vmList.get(0).getVmId()); @@ -2105,25 +2268,15 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { UserVm vm = null; try { vm = createKubernetesNode(kubernetesCluster, masterIP, i); - final long nodeVmId = vm.getId(); - KubernetesClusterVmMapVO clusterNodeVmMap = Transaction.execute(new TransactionCallback() { - @Override - public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { - KubernetesClusterVmMapVO newClusterVmMap = new KubernetesClusterVmMapVO(kubernetesClusterId, nodeVmId); - kubernetesClusterVmMapDao.persist(newClusterVmMap); - return newClusterVmMap; - } - }); + addKubernetesClusterVm(kubernetesCluster.getId(), vm .getId()); startKubernetesVM(vm, kubernetesCluster); clusterVMIds.add(vm.getId()); - - vm = userVmDao.findById(vm.getId()); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Provisioned a node VM in to the kubernetes cluster: " + kubernetesCluster.getName()); - } + LOGGER.debug(String.format("Provisioned a node VM in to the Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); } catch (Exception e) { + String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, unable to provision node VM in the cluster", kubernetesCluster.getUuid()); + LOGGER.error(msg, e); stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, provisioning the node VM failed in the kubernetes cluster!", kubernetesCluster.getUuid()), e); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg, e); } } @@ -2131,8 +2284,10 @@ public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { try { scaleKubernetesClusterNetworkRules(publicIp, account, kubernetesClusterId, clusterVMIds); } catch (Exception e) { + String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, unable to update network rules", kubernetesCluster.getUuid()); + LOGGER.error(msg, e); stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, unable to update network rules!", kubernetesCluster.getUuid()), e); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg, e); } // Attach binaries ISO to new VMs @@ -2140,33 +2295,27 @@ public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { // Check if new nodes are added in k8s cluster int retryCounter = 0; - int maxRetries = 30*4; // Max wait for 30 mins as online install can take time, same as while creating cluster + int maxRetries = 15; // Max wait for 30 mins as online install can take time, same as while creating cluster while (retryCounter < maxRetries) { try { Pair result = SshHelper.sshExecute(publicIp.getAddress().addr(), 2222, "core", pkFile, null, "sudo kubectl get nodes -o json | jq \".items[].metadata.name\" | wc -l", 20000, 10000, 30000); if (result.first()) { - int nodesCount = 0; - try { - nodesCount = Integer.parseInt(result.second().trim()); - } catch (Exception e) { - } - if (nodesCount == kubernetesCluster.getNodeCount() + 1) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("All nodes kubernetes cluster: " + kubernetesCluster.getName() + " are ready now, scaling successful!"); - } + int nodesCount = Integer.parseInt(result.second().trim()); + if (nodesCount == kubernetesCluster.getTotalNodeCount()) { + LOGGER.debug(String.format("Scaling finished successfully for Kubernetes cluster ID: %s, new nodes are ready now", kubernetesCluster.getUuid())); break; } } } catch (Exception e) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Waiting for kubernetes cluster: " + kubernetesCluster.getName() + " API endpoint to be available. retry: " + retryCounter + "/" + maxRetries); - } + LOGGER.warn(String.format("Failed to retrieve node count for Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); } try { - Thread.sleep(15000); + LOGGER.debug(String.format("Waiting 30s for scaling to finish and new nodes to become ready for Kubernetes cluster ID: %s. %d/%d", kubernetesCluster.getUuid(), retryCounter+1, maxRetries)); + Thread.sleep(30000); } catch (InterruptedException ex) { + LOGGER.warn(String.format("Failed to wait for 30s for scaling to finish and new nodes to become ready for Kubernetes cluster ID: %s. %d/%d", kubernetesCluster.getUuid(), retryCounter+1, maxRetries), ex); } retryCounter++; } @@ -2176,11 +2325,10 @@ public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { // Throw exception if nodes count for k8s cluster timed out if (retryCounter >= maxRetries) { // Scaling failed - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Scaling unsuccessful for kubernetes cluster: " + kubernetesCluster.getName() + ". Kubernetes cluster does not have desired number of nodes in ready state."); - } + String msg = String.format("Scaling unsuccessful for Kubernetes cluster ID: %s as it does not have desired number of nodes in ready state", kubernetesCluster.getUuid()); + LOGGER.warn(msg); stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling kubernetes cluster ID: %s failed, unable to have desired number of nodes in ready state!", kubernetesCluster.getUuid())); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } } } @@ -2189,193 +2337,6 @@ public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { return true; } - @Override - public ListResponse listKubernetesClusters(ListKubernetesClustersCmd cmd) { - CallContext ctx = CallContext.current(); - Account caller = ctx.getCallingAccount(); - - ListResponse response = new ListResponse(); - - List responsesList = new ArrayList(); - SearchCriteria sc = kubernetesClusterDao.createSearchCriteria(); - - String state = cmd.getState(); - if (state != null && !state.isEmpty()) { - if (!KubernetesCluster.State.Running.toString().equals(state) && - !KubernetesCluster.State.Stopped.toString().equals(state) && - !KubernetesCluster.State.Destroyed.toString().equals(state)) { - throw new InvalidParameterValueException("Invalid value for cluster state is specified"); - } - } - - if (cmd.getId() != null) { - KubernetesClusterVO cluster = kubernetesClusterDao.findById(cmd.getId()); - if (cluster == null) { - throw new InvalidParameterValueException("Invalid cluster id specified"); - } - accountManager.checkAccess(caller, SecurityChecker.AccessType.ListEntry, false, cluster); - responsesList.add(createKubernetesClusterResponse(cmd.getId())); - } else { - Filter searchFilter = new Filter(KubernetesClusterVO.class, "id", true, cmd.getStartIndex(), cmd.getPageSizeVal()); - - if (state != null && !state.isEmpty()) { - sc.addAnd("state", SearchCriteria.Op.EQ, state); - } - - if (accountManager.isNormalUser(caller.getId())) { - sc.addAnd("accountId", SearchCriteria.Op.EQ, caller.getAccountId()); - } else if (accountManager.isDomainAdmin(caller.getId())) { - sc.addAnd("domainId", SearchCriteria.Op.EQ, caller.getDomainId()); - } - - String name = cmd.getName(); - if (name != null && !name.isEmpty()) { - sc.addAnd("name", SearchCriteria.Op.LIKE, name); - } - - List kubernetesClusters = kubernetesClusterDao.search(sc, searchFilter); - for (KubernetesClusterVO cluster : kubernetesClusters) { - KubernetesClusterResponse clusterReponse = createKubernetesClusterResponse(cluster.getId()); - responsesList.add(clusterReponse); - } - } - response.setResponses(responsesList); - return response; - } - - public KubernetesClusterConfigResponse getKubernetesClusterConfig(GetKubernetesClusterConfigCmd cmd) { - KubernetesClusterConfigResponse response = new KubernetesClusterConfigResponse(); - KubernetesCluster kubernetesCluster = kubernetesClusterDao.findById(cmd.getId()); - if (kubernetesCluster != null) { - response.setId(kubernetesCluster.getUuid()); - response.setName(kubernetesCluster.getName()); - String configData = ""; - try { - configData = new String(Base64.decodeBase64(kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "kubeConfigData").getValue())); - } catch (Exception e) {} - response.setConfigData(configData); - } - response.setObjectName("clusterconfig"); - return response; - } - - public KubernetesClusterResponse createKubernetesClusterResponse(long kubernetesClusterId) { - KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); - KubernetesClusterResponse response = new KubernetesClusterResponse(); - response.setObjectName(KubernetesCluster.class.getSimpleName().toLowerCase()); - response.setId(kubernetesCluster.getUuid()); - response.setName(kubernetesCluster.getName()); - response.setDescription(kubernetesCluster.getDescription()); - DataCenterVO zone = ApiDBUtils.findZoneById(kubernetesCluster.getZoneId()); - response.setZoneId(zone.getUuid()); - response.setZoneName(zone.getName()); - response.setMasterNodes(kubernetesCluster.getMasterNodeCount()); - response.setClusterSize(kubernetesCluster.getNodeCount()); - VMTemplateVO template = ApiDBUtils.findTemplateById(kubernetesCluster.getTemplateId()); - response.setTemplateId(template.getUuid()); - ServiceOfferingVO offering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); - response.setServiceOfferingId(offering.getUuid()); - response.setServiceOfferingName(offering.getName()); - response.setKeypair(kubernetesCluster.getKeyPair()); - response.setState(kubernetesCluster.getState().toString()); - response.setCores(String.valueOf(kubernetesCluster.getCores())); - response.setMemory(String.valueOf(kubernetesCluster.getMemory())); - NetworkVO ntwk = networkDao.findByIdIncludingRemoved(kubernetesCluster.getNetworkId()); - response.setEndpoint(kubernetesCluster.getEndpoint()); - response.setNetworkId(ntwk.getUuid()); - response.setAssociatedNetworkName(ntwk.getName()); - response.setConsoleEndpoint(kubernetesCluster.getConsoleEndpoint()); - List vmIds = new ArrayList(); - List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); - if (vmList != null && !vmList.isEmpty()) { - for (KubernetesClusterVmMapVO vmMapVO : vmList) { - UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId()); - if (userVM != null) { - vmIds.add(userVM.getUuid()); - } - } - } - response.setVirtualMachineIds(vmIds); - return response; - } - - private String readResourceFile(String resource) throws IOException { - InputStream in = Thread.currentThread().getContextClassLoader().getResourceAsStream(resource); - return IOUtils.toString(Thread.currentThread().getContextClassLoader().getResourceAsStream(resource), Charset.defaultCharset().name()); - } - - protected boolean stateTransitTo(long kubernetesClusterId, KubernetesCluster.Event e) { - KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); - try { - return _stateMachine.transitTo(kubernetesCluster, e, null, kubernetesClusterDao); - } catch (NoTransitionException nte) { - LOGGER.warn("Failed to transistion state of the kubernetes cluster: " + kubernetesCluster.getName() - + " in state " + kubernetesCluster.getState().toString() + " on event " + e.toString()); - return false; - } - } - - private static String getStackTrace(final Throwable throwable) { - final StringWriter sw = new StringWriter(); - final PrintWriter pw = new PrintWriter(sw, true); - throwable.printStackTrace(pw); - return sw.getBuffer().toString(); - } - - private boolean isKubernetesServiceConfigured(DataCenter zone) { - - String templateName = globalConfigDao.getValue(KubernetesServiceConfig.KubernetesClusterTemplateName.key()); - if (templateName == null || templateName.isEmpty()) { - LOGGER.warn("Global setting " + KubernetesServiceConfig.KubernetesClusterTemplateName.key() + " is empty." + - "Template name need to be specified, for kubernetes service to function."); - return false; - } - - final VMTemplateVO template = templateDao.findByTemplateName(templateName); - if (template == null) { - LOGGER.warn("Unable to find the template:" + templateName + " to be used for provisioning cluster"); - return false; - } - - String networkOfferingName = globalConfigDao.getValue(KubernetesServiceConfig.KubernetesClusterNetworkOffering.key()); - if (networkOfferingName == null || networkOfferingName.isEmpty()) { - LOGGER.warn("global setting " + KubernetesServiceConfig.KubernetesClusterNetworkOffering.key() + " is empty. " + - "Admin has not yet specified the network offering to be used for provisioning isolated network for the cluster."); - return false; - } - - NetworkOfferingVO networkOffering = networkOfferingDao.findByUniqueName(networkOfferingName); - if (networkOffering == null) { - LOGGER.warn("Network offering with name :" + networkOfferingName + " specified by admin is not found."); - return false; - } - - if (networkOffering.getState() == NetworkOffering.State.Disabled) { - LOGGER.warn("Network offering :" + networkOfferingName + "is not enabled."); - return false; - } - - List services = networkOfferingServiceMapDao.listServicesForNetworkOffering(networkOffering.getId()); - if (services == null || services.isEmpty() || !services.contains("SourceNat")) { - LOGGER.warn("Network offering :" + networkOfferingName + " does not have necessary services to provision kubernetes cluster"); - return false; - } - - if (!networkOffering.isEgressDefaultPolicy()) { - LOGGER.warn("Network offering :" + networkOfferingName + "has egress default policy turned off should be on to provision kubernetes cluster."); - return false; - } - - long physicalNetworkId = networkModel.findPhysicalNetworkId(zone.getId(), networkOffering.getTags(), networkOffering.getTrafficType()); - PhysicalNetwork physicalNetwork = physicalNetworkDao.findById(physicalNetworkId); - if (physicalNetwork == null) { - LOGGER.warn("Unable to find physical network with id: " + physicalNetworkId + " and tag: " + networkOffering.getTags()); - return false; - } - - return true; - } - @Override public List> getCommands() { List> cmdList = new ArrayList>(); @@ -2389,7 +2350,7 @@ public List> getCommands() { return cmdList; } - // Garbage collector periodically run through the kubernetes clusters marked for GC. For each kubernetes cluster + // Garbage collector periodically run through the Kubernetes clusters marked for GC. For each Kubernetes cluster // marked for GC, attempt is made to destroy cluster. public class KubernetesClusterGarbageCollector extends ManagedContextRunnable { @Override @@ -2412,42 +2373,32 @@ public void reallyRun() { try { List kubernetesClusters = kubernetesClusterDao.findKubernetesClustersToGarbageCollect(); for (KubernetesCluster kubernetesCluster : kubernetesClusters) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Running kubernetes cluster garbage collector on kubernetes cluster name:" + kubernetesCluster.getName()); - } + LOGGER.debug(String.format("Running Kubernetes cluster garbage collector on Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); try { if (cleanupKubernetesClusterResources(kubernetesCluster.getId())) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Kubernetes cluster: " + kubernetesCluster.getName() + " is successfully garbage collected"); - } + LOGGER.debug(String.format("Garbage collection complete for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); } else { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Kubernetes cluster: " + kubernetesCluster.getName() + " failed to get" + - " garbage collected. Will be attempted to garbage collected in next run"); - } + LOGGER.warn(String.format("Garbage collection failed for Kubernetes cluster ID: %s, it will be attempted to garbage collected in next run", kubernetesCluster.getUuid())); } - } catch (RuntimeException e) { - LOGGER.debug("Faied to destroy kubernetes cluster name:" + kubernetesCluster.getName() + " during GC due to " + e); - // proceed furhter with rest of the kubernetes cluster garbage collection } catch (Exception e) { - LOGGER.debug("Faied to destroy kubernetes cluster name:" + kubernetesCluster.getName() + " during GC due to " + e); - // proceed furhter with rest of the kubernetes cluster garbage collection + LOGGER.warn(String.format("Failed to destroy Kubernetes cluster ID: %s during GC", kubernetesCluster.getUuid()), e); + // proceed further with rest of the Kubernetes cluster garbage collection } } } catch (Exception e) { - LOGGER.warn("Caught exception while running kubernetes cluster gc: ", e); + LOGGER.warn("Caught exception while running Kubernetes cluster gc: ", e); } } } - /* Kubernetes cluster scanner checks if the kubernetes cluster is in desired state. If it detects kubernetes cluster - is not in desired state, it will trigger an event and marks the kubernetes cluster to be 'Alert' state. For e.g a - kubernetes cluster in 'Running' state should mean all the cluster of node VM's in the custer should be running and + /* Kubernetes cluster scanner checks if the Kubernetes cluster is in desired state. If it detects Kubernetes cluster + is not in desired state, it will trigger an event and marks the Kubernetes cluster to be 'Alert' state. For e.g a + Kubernetes cluster in 'Running' state should mean all the cluster of node VM's in the custer should be running and number of the node VM's should be of cluster size, and the master node VM's is running. It is possible due to out of band changes by user or hosts going down, we may end up one or more VM's in stopped state. in which case scanner detects these changes and marks the cluster in 'Alert' state. Similarly cluster in 'Stopped' state means - all the cluster VM's are in stopped state any mismatch in states should get picked up by kubernetes cluster and - mark the kubernetes cluster to be 'Alert' state. Through recovery API, or reconciliation clusters in 'Alert' will + all the cluster VM's are in stopped state any mismatch in states should get picked up by Kubernetes cluster and + mark the Kubernetes cluster to be 'Alert' state. Through recovery API, or reconciliation clusters in 'Alert' will be brought back to known good state or desired state. */ public class KubernetesClusterStatusScanner extends ManagedContextRunnable { @@ -2469,42 +2420,41 @@ protected void runInContext() { public void reallyRun() { try { - - // run through kubernetes clusters in 'Running' state and ensure all the VM's are Running in the cluster + // run through Kubernetes clusters in 'Running' state and ensure all the VM's are Running in the cluster List runningKubernetesClusters = kubernetesClusterDao.findKubernetesClustersInState(KubernetesCluster.State.Running); for (KubernetesCluster kubernetesCluster : runningKubernetesClusters) { if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Running kubernetes cluster state scanner on kubernetes cluster name:" + kubernetesCluster.getName()); + LOGGER.debug("Running Kubernetes cluster state scanner on Kubernetes cluster name:" + kubernetesCluster.getName()); } try { if (!isClusterInDesiredState(kubernetesCluster, VirtualMachine.State.Running)) { stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.FaultsDetected); } } catch (Exception e) { - LOGGER.warn("Failed to run through VM states of kubernetes cluster due to " + e); + LOGGER.warn("Failed to run through VM states of Kubernetes cluster due to " + e); } } - // run through kubernetes clusters in 'Stopped' state and ensure all the VM's are Stopped in the cluster + // run through Kubernetes clusters in 'Stopped' state and ensure all the VM's are Stopped in the cluster List stoppedKubernetesClusters = kubernetesClusterDao.findKubernetesClustersInState(KubernetesCluster.State.Stopped); for (KubernetesCluster kubernetesCluster : stoppedKubernetesClusters) { if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Running kubernetes cluster state scanner on kubernetes cluster name:" + kubernetesCluster.getName() + " for state " + KubernetesCluster.State.Stopped); + LOGGER.debug("Running Kubernetes cluster state scanner on Kubernetes cluster name:" + kubernetesCluster.getName() + " for state " + KubernetesCluster.State.Stopped); } try { if (!isClusterInDesiredState(kubernetesCluster, VirtualMachine.State.Stopped)) { stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.FaultsDetected); } } catch (Exception e) { - LOGGER.warn("Failed to run through VM states of kubernetes cluster due to " + e); + LOGGER.warn("Failed to run through VM states of Kubernetes cluster due to " + e); } } - // run through kubernetes clusters in 'Alert' state and reconcile state as 'Running' if the VM's are running + // run through Kubernetes clusters in 'Alert' state and reconcile state as 'Running' if the VM's are running List alertKubernetesClusters = kubernetesClusterDao.findKubernetesClustersInState(KubernetesCluster.State.Alert); for (KubernetesCluster kubernetesCluster : alertKubernetesClusters) { if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Running kubernetes cluster state scanner on kubernetes cluster name:" + kubernetesCluster.getName() + " for state " + KubernetesCluster.State.Alert); + LOGGER.debug("Running Kubernetes cluster state scanner on Kubernetes cluster name:" + kubernetesCluster.getName() + " for state " + KubernetesCluster.State.Alert); } try { if (isClusterInDesiredState(kubernetesCluster, VirtualMachine.State.Running)) { @@ -2513,45 +2463,36 @@ public void reallyRun() { stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); } } catch (Exception e) { - LOGGER.warn("Failed to run through VM states of kubernetes cluster status scanner due to " + e); + LOGGER.warn("Failed to run through VM states of Kubernetes cluster status scanner due to " + e); } } - } catch (RuntimeException e) { - LOGGER.warn("Caught exception while running kubernetes cluster state scanner.", e); } catch (Exception e) { - LOGGER.warn("Caught exception while running kubernetes cluster state scanner.", e); + LOGGER.warn("Caught exception while running Kubernetes cluster state scanner.", e); } } } - // checks if kubernetes cluster is in desired state + // checks if Kubernetes cluster is in desired state boolean isClusterInDesiredState(KubernetesCluster kubernetesCluster, VirtualMachine.State state) { List clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + + // check cluster is running at desired capacity include master nodes as well + if (clusterVMs.size() < kubernetesCluster.getTotalNodeCount()) { + LOGGER.debug(String.format("Found only %d VMs in the Kubernetes cluster ID: %s while expected %d VMs to be in state: %s", + clusterVMs.size(), kubernetesCluster.getUuid(), kubernetesCluster.getTotalNodeCount(), state.toString())); + return false; + } // check if all the VM's are in same state for (KubernetesClusterVmMapVO clusterVm : clusterVMs) { VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(clusterVm.getVmId()); if (vm.getState() != state) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Found VM in the kubernetes cluster: " + kubernetesCluster.getName() + - " in state: " + vm.getState().toString() + " while expected to be in state: " + state.toString() + - " So moving the cluster to Alert state for reconciliation."); - } + LOGGER.debug(String.format("Found VM ID: %s in the Kubernetes cluster ID: %s in state: %s while expected to be in state: %s. So moving the cluster to Alert state for reconciliation", + vm.getUuid(), kubernetesCluster.getUuid(), vm.getState().toString(), state.toString())); return false; } } - - // check cluster is running at desired capacity include master node as well, so count should be cluster size + 1 - if (clusterVMs.size() < kubernetesCluster.getTotalNodeCount()) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Found only " + clusterVMs.size() + " VM's in the kubernetes cluster: " + kubernetesCluster.getName() + - " in state: " + state.toString() + " While expected number of VM's to " + - " be in state: " + state.toString() + " is " + (kubernetesCluster.getNodeCount() + 1) + - " So moving the cluster to Alert state for reconciliation."); - } - return false; - } return true; } @@ -2572,23 +2513,4 @@ public boolean configure(String name, Map params) throws Configu return true; } - - private String generateClusterToken(KubernetesCluster kubernetesCluster) { - if (kubernetesCluster == null) return ""; - String token = kubernetesCluster.getUuid(); - token = token.replaceAll("-", ""); - token = token.substring(0, 22); - token = token.substring(0, 6) + "." + token.substring(6); - return token; - } - - private String generateClusterHACertificateKey(KubernetesCluster kubernetesCluster) { - if (kubernetesCluster == null) return ""; - String uuid = kubernetesCluster.getUuid(); - StringBuilder token = new StringBuilder(uuid.replaceAll("-", "")); - while (token.length() < 64) { - token.append(token); - } - return token.toString().substring(0, 64); - } } From fecada1629a889f7b438546382ba54a4bf2738ec Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 29 Nov 2019 02:20:14 +0530 Subject: [PATCH 008/134] script usage, ouput fix Signed-off-by: Abhishek Kumar --- .../scripts/create-kubernetes-binaries-iso.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/plugins/integrations/kubernetes-service/scripts/create-kubernetes-binaries-iso.sh b/plugins/integrations/kubernetes-service/scripts/create-kubernetes-binaries-iso.sh index 2a69736e972f..559f6b7df805 100644 --- a/plugins/integrations/kubernetes-service/scripts/create-kubernetes-binaries-iso.sh +++ b/plugins/integrations/kubernetes-service/scripts/create-kubernetes-binaries-iso.sh @@ -16,11 +16,9 @@ # specific language governing permissions and limitations # under the License. - -echo "Params $#" if [ $# -lt 5 ]; then - echo "Invalid input. Valid usage: ./create-kubernetes-binaries-iso KUBERNETES_VERSION CNI_VERSION CRICTL_VERSION WEAVENET_NETWORK_YAML_CONFIG DASHBOARD_YAML_CONFIG" - echo "eg: ./create-kubernetes-binaries-iso 1.11.4 0.7.1 1.11.1 https://github.com/weaveworks/weave/releases/download/latest_release/weave-daemonset-k8s-1.11.yaml https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.0/src/deploy/recommended/kubernetes-dashboard.yaml" + echo "Invalid input. Valid usage: ./create-kubernetes-binaries-iso.sh KUBERNETES_VERSION CNI_VERSION CRICTL_VERSION WEAVENET_NETWORK_YAML_CONFIG DASHBOARD_YAML_CONFIG" + echo "eg: ./create-kubernetes-binaries-iso.sh 1.11.4 0.7.1 1.11.1 https://github.com/weaveworks/weave/releases/download/latest_release/weave-daemonset-k8s-1.11.yaml https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.0/src/deploy/recommended/kubernetes-dashboard.yaml" exit 1 fi @@ -93,6 +91,6 @@ if [ "${kubeadm_file_permissions}" -eq "" ]; then fi chmod ${kubeadm_file_permissions} "${working_dir}/k8s/kubeadm" -mkisofs -o setup.iso -J -R -l "${iso_dir}" +mkisofs -o "setup-${RELEASE}.iso" -J -R -l "${iso_dir}" rm -rf "${iso_dir}" \ No newline at end of file From 6fe97237ecc6b759012def103c3ce93c2e68527f Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 29 Nov 2019 02:32:05 +0530 Subject: [PATCH 009/134] upgrade cluster funtionality refactorings logging fixes fix for SSH public key on cluster nodes state scanner improvement list versions api improvement Signed-off-by: Abhishek Kumar --- .../apache/cloudstack/api/ApiConstants.java | 1 + .../kubernetescluster/KubernetesCluster.java | 34 ++- .../KubernetesClusterEventTypes.java | 1 + .../KubernetesClusterManagerImpl.java | 268 ++++++++++++++++-- .../KubernetesClusterService.java | 2 + .../KubernetesClusterVO.java | 5 + .../KubernetesVersionManagerImpl.java | 39 ++- .../DeleteKubernetesClusterCmd.java | 3 +- .../ScaleKubernetesClusterCmd.java | 12 +- .../StartKubernetesClusterCmd.java | 17 +- .../StopKubernetesClusterCmd.java | 5 +- .../UpgradeKubernetesClusterCmd.java | 134 +++++++++ .../ListKubernetesSupportedVersionsCmd.java | 9 + .../response/KubernetesClusterResponse.java | 24 ++ .../main/resources/conf/k8s-master-add.yml | 3 + .../src/main/resources/conf/k8s-master.yml | 4 +- .../src/main/resources/conf/k8s-node.yml | 3 + .../resources/script/upgrade-kubernetes.sh | 104 +++++++ ui/plugins/cks/cks.js | 83 +++++- 19 files changed, 695 insertions(+), 56 deletions(-) create mode 100644 plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/UpgradeKubernetesClusterCmd.java create mode 100644 plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 13ade27190f9..df858f8c3f0b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -752,6 +752,7 @@ public class ApiConstants { public static final String KUBERNETES_VERSION = "kubernetesversion"; public static final String KUBERNETES_VERSION_ID = "kubernetesversionid"; public static final String MIN_KUBERNETES_VERSION = "minimumkubernetesversion"; + public static final String MIN_KUBERNETES_VERSION_ID = "minimumkubernetesversionid"; public static final String NODE_ROOT_DISK_SIZE = "noderootdisksize"; public enum HostDetails { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesCluster.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesCluster.java index 08edfe120cde..7c76c052c570 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesCluster.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesCluster.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.kubernetescluster; +import java.util.Date; + import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.api.Displayable; import org.apache.cloudstack.api.Identity; @@ -24,7 +26,7 @@ import com.cloud.utils.fsm.StateMachine2; /** - * KubernetesCluster describes the properties of kubernetes cluster + * KubernetesCluster describes the properties of Kubernetes cluster * */ public interface KubernetesCluster extends ControlledEntity, com.cloud.utils.fsm.StateObject, Identity, InternalIdentity, Displayable { @@ -36,6 +38,7 @@ enum Event { RecoveryRequested, ScaleUpRequested, ScaleDownRequested, + UpgradeRequested, OperationSucceeded, OperationFailed, CreateFailed, @@ -43,17 +46,18 @@ enum Event { } enum State { - Created("Initial State of kubernetes cluster. At this state its just a logical/DB entry with no resources consumed"), - Starting("Resources needed for kubernetes cluster are being provisioned"), - Running("Necessary resources are provisioned and kubernetes cluster is in operational ready state to launch kubernetess"), - Stopping("Ephermal resources for the kubernetes cluster are being destroyed"), - Stopped("All ephermal resources for the kubernetes cluster are destroyed, Kubernetes cluster may still have ephermal resource like persistent volumens provisioned"), - Scaling("Transient state in which resoures are either getting scaled up/down"), - Alert("State to represent kubernetes clusters which are not in expected desired state (operationally in active control place, stopped cluster VM's etc)."), - Recovering("State in which kubernetes cluster is recovering from alert state"), - Destroyed("End state of kubernetes cluster in which all resources are destroyed, cluster will not be useable further"), - Destroying("State in which resources for the kubernetes cluster is getting cleaned up or yet to be cleaned up by garbage collector"), - Error("State of the failed to create kubernetes clusters"); + Created("Initial State of Kubernetes cluster. At this state its just a logical/DB entry with no resources consumed"), + Starting("Resources needed for Kubernetes cluster are being provisioned"), + Running("Necessary resources are provisioned and Kubernetes cluster is in operational ready state to launch Kubernetes"), + Stopping("Resources for the Kubernetes cluster are being destroyed"), + Stopped("All resources for the Kubernetes cluster are destroyed, Kubernetes cluster may still have ephemeral resource like persistent volumes provisioned"), + Scaling("Transient state in which resources are either getting scaled up/down"), + Upgrading("Transient state in which cluster is getting upgraded"), + Alert("State to represent Kubernetes clusters which are not in expected desired state (operationally in active control place, stopped cluster VM's etc)."), + Recovering("State in which Kubernetes cluster is recovering from alert state"), + Destroyed("End state of Kubernetes cluster in which all resources are destroyed, cluster will not be usable further"), + Destroying("State in which resources for the Kubernetes cluster is getting cleaned up or yet to be cleaned up by garbage collector"), + Error("State of the failed to create Kubernetes clusters"); protected static final StateMachine2 s_fsm = new StateMachine2(); @@ -65,6 +69,7 @@ enum State { s_fsm.addTransition(State.Starting, Event.OperationSucceeded, State.Running); s_fsm.addTransition(State.Starting, Event.OperationFailed, State.Alert); s_fsm.addTransition(State.Starting, Event.CreateFailed, State.Error); + s_fsm.addTransition(State.Starting, Event.StopRequested, State.Stopping); s_fsm.addTransition(State.Running, Event.StopRequested, State.Stopping); s_fsm.addTransition(State.Stopping, Event.OperationSucceeded, State.Stopped); @@ -79,6 +84,10 @@ enum State { s_fsm.addTransition(State.Scaling, Event.OperationSucceeded, State.Running); s_fsm.addTransition(State.Scaling, Event.OperationFailed, State.Alert); + s_fsm.addTransition(State.Running, Event.UpgradeRequested, State.Upgrading); + s_fsm.addTransition(State.Upgrading, Event.OperationSucceeded, State.Running); + s_fsm.addTransition(State.Upgrading, Event.OperationFailed, State.Alert); + s_fsm.addTransition(State.Alert, Event.RecoveryRequested, State.Recovering); s_fsm.addTransition(State.Recovering, Event.OperationSucceeded, State.Running); s_fsm.addTransition(State.Recovering, Event.OperationFailed, State.Alert); @@ -122,4 +131,5 @@ enum State { boolean isCheckForGc(); @Override State getState(); + Date getCreated(); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterEventTypes.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterEventTypes.java index 6d651fef0de5..a13740697540 100755 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterEventTypes.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterEventTypes.java @@ -22,4 +22,5 @@ public class KubernetesClusterEventTypes { public static final String EVENT_KUBERNETES_CLUSTER_START = "KUBERNETES.CLUSTER.START"; public static final String EVENT_KUBERNETES_CLUSTER_STOP = "KUBERNETES.CLUSTER.STOP"; public static final String EVENT_KUBERNETES_CLUSTER_SCALE = "KUBERNETES.CLUSTER.SCALE"; + public static final String EVENT_KUBERNETES_CLUSTER_UPGRADE = "KUBERNETES.CLUSTER.UPGRADE"; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java index e26bd9e558a2..845a619bbfaf 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java @@ -16,9 +16,10 @@ // under the License. package com.cloud.kubernetescluster; +import java.io.BufferedWriter; import java.io.File; +import java.io.FileWriter; import java.io.IOException; -import java.io.InputStream; import java.io.PrintWriter; import java.io.StringWriter; import java.lang.reflect.Field; @@ -34,9 +35,12 @@ import java.security.SecureRandom; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -61,6 +65,7 @@ import org.apache.cloudstack.api.command.user.kubernetescluster.ScaleKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetescluster.StartKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetescluster.StopKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetescluster.UpgradeKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.vm.StartVMCmd; import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse; import org.apache.cloudstack.api.response.KubernetesClusterResponse; @@ -73,6 +78,7 @@ import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.utils.security.CertUtils; import org.apache.commons.codec.binary.Base64; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.io.IOUtils; import org.apache.log4j.Logger; @@ -102,6 +108,7 @@ import com.cloud.kubernetescluster.dao.KubernetesClusterDetailsDao; import com.cloud.kubernetescluster.dao.KubernetesClusterVmMapDao; import com.cloud.kubernetesversion.KubernetesSupportedVersion; +import com.cloud.kubernetesversion.KubernetesSupportedVersionVO; import com.cloud.kubernetesversion.KubernetesVersionManagerImpl; import com.cloud.kubernetesversion.dao.KubernetesSupportedVersionDao; import com.cloud.network.IpAddress; @@ -273,8 +280,7 @@ private static String getStackTrace(final Throwable throwable) { } private String readResourceFile(String resource) throws IOException { - InputStream in = Thread.currentThread().getContextClassLoader().getResourceAsStream(resource); - return IOUtils.toString(Thread.currentThread().getContextClassLoader().getResourceAsStream(resource), Charset.defaultCharset().name()); + return IOUtils.toString(Objects.requireNonNull(Thread.currentThread().getContextClassLoader().getResourceAsStream(resource)), Charset.defaultCharset().name()); } private boolean isKubernetesServiceConfigured(DataCenter zone) { @@ -474,7 +480,7 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t LOGGER.debug(String.format("Kubernetes cluster ID: %s VMs successfully provisioned", kubernetesCluster.getUuid())); setupKubernetesClusterNetworkRules(publicIp, account, kubernetesClusterId, clusterVMIds); - attachIsoKubernetesVMs(kubernetesClusterId, clusterVMIds); + attachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); int retryCounter = 0; int maxRetries = 15; @@ -557,7 +563,7 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t kubernetesCluster.setConsoleEndpoint("https://" + publicIp.getAddress() + ":6443/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy#!/overview?namespace=_all"); kubernetesClusterDao.update(kubernetesCluster.getId(), kubernetesCluster); - detachIsoKubernetesVMs(kubernetesClusterId, clusterVMIds); + detachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); if (LOGGER.isDebugEnabled()) { LOGGER.debug("Kubernetes cluster name:" + kubernetesCluster.getName() + " is successfully started"); @@ -580,7 +586,7 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); - detachIsoKubernetesVMs(kubernetesClusterId, clusterVMIds); + detachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to deploy Kubernetes cluster: " + kubernetesCluster.getUuid() + " as unable to setup up in usable state"); @@ -1321,7 +1327,7 @@ private UserVm createKubernetesMaster(final KubernetesClusterVO kubernetesCluste final String apiServerCert = "{{ k8s_master.apiserver.crt }}"; final String apiServerKey = "{{ k8s_master.apiserver.key }}"; final String caCert = "{{ k8s_master.ca.crt }}"; - final String msSshPubKey = "{{ k8s_master.ms.ssh.pub.key }}"; + final String sshPubKey = "{{ k8s.ssh.pub.key }}"; final String clusterToken = "{{ k8s_master.cluster.token }}"; final String clusterInitArgsKey = "{{ k8s_master.cluster.initargs }}"; final List addresses = new ArrayList<>(); @@ -1346,7 +1352,7 @@ private UserVm createKubernetesMaster(final KubernetesClusterVO kubernetesCluste pubKey += "\n - \"" + sshkp.getPublicKey() + "\""; } } - k8sMasterConfig = k8sMasterConfig.replace(msSshPubKey, pubKey); + k8sMasterConfig = k8sMasterConfig.replace(sshPubKey, pubKey); k8sMasterConfig = k8sMasterConfig.replace(clusterToken, generateClusterToken(kubernetesCluster)); String initArgs = ""; if (haSupported) { @@ -1392,7 +1398,17 @@ private UserVm createKubernetesAdditionalMaster(final KubernetesClusterVO kubern k8sMasterConfig = readResourceFile("/conf/k8s-master-add.yml"); final String joinIpKey = "{{ k8s_master.join_ip }}"; final String clusterTokenKey = "{{ k8s_master.cluster.token }}"; + final String sshPubKey = "{{ k8s.ssh.pub.key }}"; final String clusterHACertificateKey = "{{ k8s_master.cluster.ha.certificate.key }}"; + String pubKey = "- \"" + globalConfigDao.getValue("ssh.publickey") + "\""; + String sshKeyPair = kubernetesCluster.getKeyPair(); + if (!Strings.isNullOrEmpty(sshKeyPair)) { + SSHKeyPairVO sshkp = sshKeyPairDao.findByName(owner.getAccountId(), owner.getDomainId(), sshKeyPair); + if (sshkp != null) { + pubKey += "\n - \"" + sshkp.getPublicKey() + "\""; + } + } + k8sMasterConfig = k8sMasterConfig.replace(sshPubKey, pubKey); k8sMasterConfig = k8sMasterConfig.replace(joinIpKey, joinIp); k8sMasterConfig = k8sMasterConfig.replace(clusterTokenKey, generateClusterToken(kubernetesCluster)); k8sMasterConfig = k8sMasterConfig.replace(clusterHACertificateKey, generateClusterHACertificateKey(kubernetesCluster)); @@ -1430,8 +1446,18 @@ private UserVm createKubernetesNode(KubernetesClusterVO kubernetesCluster, Strin String k8sNodeConfig = null; try { k8sNodeConfig = readResourceFile("/conf/k8s-node.yml"); + final String sshPubKey = "{{ k8s.ssh.pub.key }}"; final String joinIpKey = "{{ k8s_master.join_ip }}"; final String clusterTokenKey = "{{ k8s_master.cluster.token }}"; + String pubKey = "- \"" + globalConfigDao.getValue("ssh.publickey") + "\""; + String sshKeyPair = kubernetesCluster.getKeyPair(); + if (!Strings.isNullOrEmpty(sshKeyPair)) { + SSHKeyPairVO sshkp = sshKeyPairDao.findByName(owner.getAccountId(), owner.getDomainId(), sshKeyPair); + if (sshkp != null) { + pubKey += "\n - \"" + sshkp.getPublicKey() + "\""; + } + } + k8sNodeConfig = k8sNodeConfig.replace(sshPubKey, pubKey); k8sNodeConfig = k8sNodeConfig.replace(joinIpKey, joinIp); k8sNodeConfig = k8sNodeConfig.replace(clusterTokenKey, generateClusterToken(kubernetesCluster)); /* genarate /.docker/config.json file on the nodes only if Kubernetes cluster is created to @@ -1532,8 +1558,7 @@ private void startKubernetesVM(final UserVm vm, final KubernetesClusterVO kubern } } - private void attachIsoKubernetesVMs(long kubernetesClusterId, List clusterVMIds) throws ServerApiException { - KubernetesCluster kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + private void attachIsoKubernetesVMs(KubernetesCluster kubernetesCluster, List clusterVMIds) throws ServerApiException { KubernetesSupportedVersion version = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); if (version == null) { LOGGER.error(String .format("Unable to find Kubernetes version for cluster ID: %s", kubernetesCluster.getUuid())); @@ -1566,19 +1591,19 @@ private void attachIsoKubernetesVMs(long kubernetesClusterId, List cluster } } - private void detachIsoKubernetesVMs(long kubernetesClusterId, List clusterVMIds) throws ServerApiException { - KubernetesCluster kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + private void detachIsoKubernetesVMs(final KubernetesCluster kubernetesCluster, List clusterVMIds) throws ServerApiException { for (int i = 0; i < clusterVMIds.size(); ++i) { UserVm vm = userVmDao.findById(clusterVMIds.get(i)); - + boolean result = false; try { - templateService.detachIso(vm.getId()); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Started VM in the Kubernetes cluster: " + kubernetesCluster.getName()); - } + result = templateService.detachIso(vm.getId()); } catch (CloudRuntimeException ex) { - LOGGER.warn(String.format("Failed to detach binaries ISO for VM: %s in the Kubernetes cluster name: %s due to Exception: ", vm.getDisplayName(), kubernetesCluster.getName()), ex); - // throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to attach binaries ISO for VM: %s in the Kubernetes cluster name: %s", vm.getDisplayName(), kubernetesCluster.getName()), ex); + LOGGER.warn(String.format("Failed to detach binaries ISO from VM ID: %s in the Kubernetes cluster ID: %s ", vm.getUuid(), kubernetesCluster.getUuid()), ex); + } + if (result) { + LOGGER.debug(String.format("Detached Kubernetes binaries from VM ID: %s in the Kubernetes cluster ID: %s", vm.getUuid(), kubernetesCluster.getUuid())); + } else { + LOGGER.warn(String.format("Failed to detach binaries ISO from VM ID: %s in the Kubernetes cluster ID: %s ", vm.getUuid(), kubernetesCluster.getUuid())); } } } @@ -1610,6 +1635,11 @@ public KubernetesClusterResponse createKubernetesClusterResponse(long kubernetes ServiceOfferingVO offering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); response.setServiceOfferingId(offering.getUuid()); response.setServiceOfferingName(offering.getName()); + KubernetesSupportedVersionVO version = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); + if (version != null) { + response.setKubernetesVersionId(version.getUuid()); + response.setKubernetesVersionName(version.getName()); + } response.setKeypair(kubernetesCluster.getKeyPair()); response.setState(kubernetesCluster.getState().toString()); response.setCores(String.valueOf(kubernetesCluster.getCores())); @@ -1979,9 +2009,12 @@ public KubernetesClusterConfigResponse getKubernetesClusterConfig(GetKubernetesC @Override public boolean scaleKubernetesCluster(ScaleKubernetesClusterCmd cmd) throws ManagementServerException, ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException { - final long kubernetesClusterId = cmd.getId(); + final Long kubernetesClusterId = cmd.getId(); final Long serviceOfferingId = cmd.getServiceOfferingId(); final Long clusterSize = cmd.getClusterSize(); + if (kubernetesClusterId == null || kubernetesClusterId < 1L) { + throw new InvalidParameterValueException("Invalid Kubernetes cluster ID"); + } KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); if (kubernetesCluster == null || kubernetesCluster.getRemoved() != null) { throw new InvalidParameterValueException("Invalid Kubernetes cluster ID"); @@ -2291,7 +2324,7 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { } // Attach binaries ISO to new VMs - attachIsoKubernetesVMs(kubernetesClusterId, clusterVMIds); + attachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); // Check if new nodes are added in k8s cluster int retryCounter = 0; @@ -2321,7 +2354,7 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { } // Detach binaries ISO from new VMs - detachIsoKubernetesVMs(kubernetesClusterId, clusterVMIds); + detachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); // Throw exception if nodes count for k8s cluster timed out if (retryCounter >= maxRetries) { // Scaling failed @@ -2337,6 +2370,175 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { return true; } + @Override + public boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws ManagementServerException { + // Validate parameters + final Long kubernetesClusterId = cmd.getId(); + final Long upgradeVersionId = cmd.getKubernetesVersionId(); + if (kubernetesClusterId == null || kubernetesClusterId < 1L) { + throw new InvalidParameterValueException("Invalid Kubernetes cluster ID"); + } + if (upgradeVersionId == null || upgradeVersionId < 1L) { + throw new InvalidParameterValueException("Invalid Kubernetes version ID"); + } + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + if (kubernetesCluster == null || kubernetesCluster.getRemoved() != null) { + throw new InvalidParameterValueException("Invalid Kubernetes cluster ID"); + } + if (!KubernetesCluster.State.Running.equals(kubernetesCluster.getState())) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s is not in running state", kubernetesCluster.getUuid())); + } + KubernetesSupportedVersionVO upgradeVersion = kubernetesSupportedVersionDao.findById(upgradeVersionId); + if (upgradeVersion == null || upgradeVersion.getRemoved() != null) { + throw new InvalidParameterValueException("Invalid Kubernetes version ID"); + } + KubernetesSupportedVersionVO clusterVersion = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); + if (clusterVersion == null || clusterVersion.getRemoved() != null) { + throw new InvalidParameterValueException(String.format("Invalid Kubernetes version associated with cluster ID: %s", + kubernetesCluster.getUuid())); + } + if (KubernetesVersionManagerImpl.compareKubernetesVersion( + upgradeVersion.getKubernetesVersion(), clusterVersion.getKubernetesVersion()) <= 0) { + throw new InvalidParameterValueException(String.format("Invalid Kubernetes version associated with cluster ID: %s", + kubernetesCluster.getUuid())); + } + // Check upgradeVersion is either path upgrade or immediate minor upgrade + try { + KubernetesVersionManagerImpl.canUpgradeKubernetesVersion(clusterVersion.getKubernetesVersion(), upgradeVersion.getKubernetesVersion()); + } catch (IllegalArgumentException e) { + throw new InvalidParameterValueException(e.getMessage()); + } + + // Get public IP + IPAddressVO publicIp = null; + List ips = ipAddressDao.listByAssociatedNetwork(kubernetesCluster.getNetworkId(), true); + if (CollectionUtils.isEmpty(ips)) { + String msg = String.format("Upgrade failed for Kubernetes cluster ID: %s, unable to retrieve associated public IP", kubernetesCluster.getUuid()); + LOGGER.warn(msg); + throw new ManagementServerException(msg); + } + publicIp = ips.get(0); + + kubernetesCluster.setKubernetesVersionId(upgradeVersion.getId()); + List clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + if (CollectionUtils.isEmpty(clusterVMs)) { + String msg = String.format("Upgrade failed for Kubernetes cluster ID: %s, unable to retrieve VMs for cluster", kubernetesCluster.getUuid()); + throw new ManagementServerException(msg); + } + List vmIds = new ArrayList<>(); + for (KubernetesClusterVmMapVO vmMap : clusterVMs) { + vmIds.add(vmMap.getVmId()); + } + Collections.sort(vmIds); + + boolean devel = Boolean.parseBoolean(globalConfigDao.getValue("developer")); + String keyFile = String.format("%s/.ssh/id_rsa", System.getProperty("user.home")); + if (devel) { + keyFile += ".cloud"; + } + File pkFile = new File(keyFile); + + File upgradeScriptFile = null; + try { + String upgradeScriptData = readResourceFile("/script/upgrade-kubernetes.sh"); + upgradeScriptFile = File.createTempFile("upgrade-kuberntes", ".sh"); + BufferedWriter upgradeScriptFileWriter = new BufferedWriter(new FileWriter(upgradeScriptFile)); + upgradeScriptFileWriter.write(upgradeScriptData); + upgradeScriptFileWriter.close(); + } catch (IOException e) { + String msg = String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to prepare upgrade script", kubernetesCluster.getUuid()); + LOGGER.error(msg, e); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg, e); + } + + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.UpgradeRequested); + + // Attach ISO + attachIsoKubernetesVMs(kubernetesCluster, vmIds); + + // Upgrade master + Pair result = null; + for (int i = 0; i < vmIds.size(); ++i) { + UserVm vm = userVmDao.findById(vmIds.get(i)); + result = null; + try { + result = SshHelper.sshExecute(publicIp.getAddress().addr(), 2222, "core", pkFile, null, + String.format("sudo kubectl drain %s --ignore-daemonsets --delete-local-data", vm.getHostName()), + 10000, 10000, 60000); + } catch (Exception e) { + String msg = String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to drain Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()); + LOGGER.error(msg, e); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg, e); + } + if (!result.first()) { + String msg = String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to drain Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()); + LOGGER.error(String.format("%s. Output:\n%s", msg, result.second())); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); + } + try { + SshHelper.scpTo(publicIp.getAddress().addr(), 2222 + i, "core", pkFile, null, + "~/", upgradeScriptFile.getAbsolutePath(), "0755"); + + result = SshHelper.sshExecute(publicIp.getAddress().addr(), 2222 + i, "core", pkFile, null, + String.format("sudo ./%s %s", upgradeScriptFile.getName(), i == 0 ? upgradeVersion.getKubernetesVersion() : "''"), + 10000, 10000, 5 * 60 * 1000); + } catch (Exception e) { + String msg = String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to upgrade Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()); + LOGGER.error(msg, e); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg, e); + } + if (!result.first()) { + String msg = String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to upgrade Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()); + LOGGER.error(String.format("%s. Output:\n%s", msg, result.second())); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); + } + int retryCounter = 0; + int maxRetries = 3; + while (retryCounter < maxRetries) { + try { + result = SshHelper.sshExecute(publicIp.getAddress().addr(), 2222, "core", pkFile, null, + String.format("sudo kubectl uncordon %s", vm.getHostName()), + 10000, 10000, 30000); + } catch (Exception e) { + String msg = String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to uncordon Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()); + LOGGER.error(msg, e); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg, e); + } + if (result.first()) { + break; + } + try { + Thread.sleep(30000); + } catch (InterruptedException ie) { + LOGGER.warn(String.format("Error while waiting for uncordon Kubernetes cluster ID: %s node running on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()), ie); + } + retryCounter++; + } + if (!result.first()) { + String msg = String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to uncordon Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()); + LOGGER.error(String.format("%s. Output:\n%s", msg, result.second())); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); + } + } + + // Detach ISO + detachIsoKubernetesVMs(kubernetesCluster, vmIds); + + boolean updated = kubernetesClusterDao.update(kubernetesCluster.getId(), kubernetesCluster); + if (!updated) { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } else { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); + } + return updated; + } + @Override public List> getCommands() { List> cmdList = new ArrayList>(); @@ -2347,6 +2549,7 @@ public List> getCommands() { cmdList.add(ListKubernetesClustersCmd.class); cmdList.add(GetKubernetesClusterConfigCmd.class); cmdList.add(ScaleKubernetesClusterCmd.class); + cmdList.add(UpgradeKubernetesClusterCmd.class); return cmdList; } @@ -2467,8 +2670,25 @@ public void reallyRun() { } } + // run through Kubernetes clusters in 'Starting' state and reconcile state as 'Running' or 'Error' if the VM's are running + List startingKubernetesClusters = kubernetesClusterDao.findKubernetesClustersInState(KubernetesCluster.State.Starting); + for (KubernetesCluster kubernetesCluster : startingKubernetesClusters) { + if (!Strings.isNullOrEmpty(kubernetesCluster.getEndpoint()) || + (new Date()).getTime() - kubernetesCluster.getCreated().getTime() < 10*60*1000) { + continue; + } + LOGGER.debug(String.format("Running Kubernetes cluster state scanner on Kubernetes cluster ID: %s for state: %s", kubernetesCluster.getUuid(), KubernetesCluster.State.Starting.toString())); + try { + if (isClusterInDesiredState(kubernetesCluster, VirtualMachine.State.Running)) { + // mark the cluster to be running + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); + } + } catch (Exception e) { + LOGGER.warn(String.format("Failed to run through VM states of Kubernetes cluster ID: %s status scanner", kubernetesCluster.getUuid()), e); + } + } } catch (Exception e) { - LOGGER.warn("Caught exception while running Kubernetes cluster state scanner.", e); + LOGGER.warn("Caught exception while running Kubernetes cluster state scanner", e); } } } @@ -2477,7 +2697,6 @@ public void reallyRun() { boolean isClusterInDesiredState(KubernetesCluster kubernetesCluster, VirtualMachine.State state) { List clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); - // check cluster is running at desired capacity include master nodes as well if (clusterVMs.size() < kubernetesCluster.getTotalNodeCount()) { LOGGER.debug(String.format("Found only %d VMs in the Kubernetes cluster ID: %s while expected %d VMs to be in state: %s", @@ -2493,6 +2712,7 @@ boolean isClusterInDesiredState(KubernetesCluster kubernetesCluster, VirtualMach return false; } } + return true; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterService.java index 48b26178c802..7693af65108a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterService.java @@ -20,6 +20,7 @@ import org.apache.cloudstack.api.command.user.kubernetescluster.GetKubernetesClusterConfigCmd; import org.apache.cloudstack.api.command.user.kubernetescluster.ListKubernetesClustersCmd; import org.apache.cloudstack.api.command.user.kubernetescluster.ScaleKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetescluster.UpgradeKubernetesClusterCmd; import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse; import org.apache.cloudstack.api.response.KubernetesClusterResponse; import org.apache.cloudstack.api.response.ListResponse; @@ -53,4 +54,5 @@ boolean startKubernetesCluster(long kubernetesClusterId, boolean onCreate) throw boolean scaleKubernetesCluster(ScaleKubernetesClusterCmd cmd) throws ManagementServerException, ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException; + boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws ManagementServerException; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVO.java index a9824f76d567..4e957a81d3cc 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVO.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVO.java @@ -310,6 +310,11 @@ public void setCheckForGc(boolean check) { checkForGc = check; } + @Override + public Date getCreated() { + return created; + } + public KubernetesClusterVO() { } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java index 43f117d98be8..70fd1398762e 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java @@ -114,11 +114,48 @@ public static int compareKubernetesVersion(String v1, String v2) throws IllegalA return 0; } + public static boolean canUpgradeKubernetesVersion(String currentVersion, String upgradeVersion) throws IllegalArgumentException { + if (Strings.isNullOrEmpty(currentVersion) || Strings.isNullOrEmpty(upgradeVersion)) { + throw new IllegalArgumentException(String.format("Invalid version comparision with versions %s, %s", currentVersion, upgradeVersion)); + } + if(!currentVersion.matches("[0-9]+(\\.[0-9]+)*")) { + throw new IllegalArgumentException(String.format("Invalid version format, %s", currentVersion)); + } + if(!upgradeVersion.matches("[0-9]+(\\.[0-9]+)*")) { + throw new IllegalArgumentException(String.format("Invalid version format, %s", upgradeVersion)); + } + String[] thisParts = currentVersion.split("\\."); + if (thisParts.length < 3) { + throw new IllegalArgumentException(String.format("Invalid version format, %s", currentVersion)); + } + String[] thatParts = upgradeVersion.split("\\."); + if (thatParts.length < 3) { + throw new IllegalArgumentException(String.format("Invalid version format, %s", upgradeVersion)); + } + int majorVerDiff = Integer.parseInt(thatParts[0]) - Integer.parseInt(thisParts[0]); + int minorVerDiff = Integer.parseInt(thatParts[1]) - Integer.parseInt(thisParts[1]); + if (majorVerDiff != 0 || minorVerDiff != 1) { + throw new IllegalArgumentException(String.format("Kubernetes clusters can be upgraded between next minor or patch version releases, current version: %s, upgrade version: %s", currentVersion, upgradeVersion)); + } + return true; + } + @Override public ListResponse listKubernetesSupportedVersions(final ListKubernetesSupportedVersionsCmd cmd) { final Long versionId = cmd.getId(); final Long zoneId = cmd.getZoneId(); - final String minimumKubernetesVersion = cmd.getMinimumKubernetesVersion(); + String minimumKubernetesVersion = cmd.getMinimumKubernetesVersion(); + final Long minimumKubernetesVersionId = cmd.getMinimumKubernetesVersionId(); + if (!Strings.isNullOrEmpty(minimumKubernetesVersion) && minimumKubernetesVersionId != null) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Both parameters %s and %s can not be passed together", ApiConstants.MIN_KUBERNETES_VERSION, ApiConstants.MIN_KUBERNETES_VERSION_ID)); + } + if (minimumKubernetesVersionId != null) { + KubernetesSupportedVersionVO minVersion = kubernetesSupportedVersionDao.findById(minimumKubernetesVersionId); + if (minVersion == null) { + throw new InvalidParameterValueException(String.format("Invalid %s passed", ApiConstants.MIN_KUBERNETES_VERSION_ID)); + } + minimumKubernetesVersion = minVersion.getKubernetesVersion(); + } List responseList = new ArrayList<>(); List versions = new ArrayList<>(); if (versionId != null) { diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/DeleteKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/DeleteKubernetesClusterCmd.java index ef78cbefb3a4..280923a530eb 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/DeleteKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/DeleteKubernetesClusterCmd.java @@ -118,7 +118,8 @@ public String getEventType() { @Override public String getEventDescription() { - return "Deleting Kubernetes cluster. Cluster Id: " + getId(); + KubernetesCluster cluster = _entityMgr.findById(KubernetesCluster.class, getId()); + return String.format("Deleting Kubernetes cluster ID: %s", cluster.getUuid()); } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/ScaleKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/ScaleKubernetesClusterCmd.java index 16340a08738f..49642adb9852 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/ScaleKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/ScaleKubernetesClusterCmd.java @@ -98,7 +98,8 @@ public String getEventType() { @Override public String getEventDescription() { - return "Scaling Kubernetes cluster id: " + getId(); + KubernetesCluster cluster = _entityMgr.findById(KubernetesCluster.class, getId()); + return String.format("Scaling Kubernetes cluster ID: %s", cluster.getUuid()); } @Override @@ -114,6 +115,7 @@ public long getEntityOwnerId() { ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// + public KubernetesCluster validateRequest() { if (getId() == null || getId() < 1L) { throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Invalid Kubernetes cluster ID provided"); @@ -130,13 +132,13 @@ public void execute() throws ResourceUnavailableException, InsufficientCapacityE final KubernetesCluster kubernetesCluster = validateRequest(); try { kubernetesClusterService.scaleKubernetesCluster(this); - final KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(getId()); + final KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(kubernetesCluster.getId()); response.setResponseName(getCommandName()); setResponseObject(response); } catch (InsufficientCapacityException | ResourceUnavailableException | ManagementServerException ex) { - LOGGER.warn("Failed to scale Kubernetes cluster:" + kubernetesCluster.getUuid() + " due to " + ex.getMessage()); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, - "Failed to scale Kubernetes cluster:" + kubernetesCluster.getUuid(), ex); + String msg = String.format("Failed to scale Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); + LOGGER.error(msg, ex); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg, ex); } } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/StartKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/StartKubernetesClusterCmd.java index d0c700effa05..5d12a1163f96 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/StartKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/StartKubernetesClusterCmd.java @@ -58,7 +58,7 @@ public class StartKubernetesClusterCmd extends BaseAsyncCmd { //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// @Parameter(name = ApiConstants.ID, type = CommandType.UUID, - entityType = KubernetesClusterResponse.class, + entityType = KubernetesClusterResponse.class, required = true, description = "the ID of the Kubernetes cluster") private Long id; @@ -77,7 +77,8 @@ public String getEventType() { @Override public String getEventDescription() { - return "Starting Kubernetes cluster id: " + getId(); + KubernetesCluster cluster = _entityMgr.findById(KubernetesCluster.class, getId()); + return String.format("Starting Kubernetes cluster ID: %s", cluster.getUuid()); } @Override @@ -109,14 +110,14 @@ public KubernetesCluster validateRequest() { public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { final KubernetesCluster kubernetesCluster = validateRequest(); try { - kubernetesClusterService.startKubernetesCluster(getId().longValue(), false); - final KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(getId()); + kubernetesClusterService.startKubernetesCluster(kubernetesCluster.getId(), false); + final KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(kubernetesCluster.getId()); response.setResponseName(getCommandName()); setResponseObject(response); - } catch (InsufficientCapacityException | ResourceUnavailableException | ManagementServerException ex) { - LOGGER.warn("Failed to start Kubernetes cluster:" + kubernetesCluster.getUuid() + " due to " + ex.getMessage()); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, - "Failed to start Kubernetes cluster:" + kubernetesCluster.getUuid(), ex); + } catch (InsufficientCapacityException | ResourceUnavailableException | ManagementServerException ex) { + String msg = String.format("Failed to start Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); + LOGGER.error(msg, ex); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg, ex); } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/StopKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/StopKubernetesClusterCmd.java index c02eab523c74..85bc1023e1d0 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/StopKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/StopKubernetesClusterCmd.java @@ -59,7 +59,7 @@ public class StopKubernetesClusterCmd extends BaseAsyncCmd { //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// @Parameter(name = ApiConstants.ID, type = CommandType.UUID, - entityType = KubernetesClusterResponse.class, + entityType = KubernetesClusterResponse.class, required = true, description = "the ID of the Kubernetes cluster") private Long id; @@ -78,7 +78,8 @@ public String getEventType() { @Override public String getEventDescription() { - return "Stopping Kubernetes cluster id: " + getId(); + KubernetesCluster cluster = _entityMgr.findById(KubernetesCluster.class, getId()); + return String.format("Stopping Kubernetes cluster ID: %s", cluster.getUuid()); } @Override diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/UpgradeKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/UpgradeKubernetesClusterCmd.java new file mode 100644 index 000000000000..de198f5cc2ec --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/UpgradeKubernetesClusterCmd.java @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.user.kubernetescluster; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.KubernetesClusterResponse; +import org.apache.cloudstack.api.response.KubernetesSupportedVersionResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.ManagementServerException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.kubernetescluster.KubernetesCluster; +import com.cloud.kubernetescluster.KubernetesClusterEventTypes; +import com.cloud.kubernetescluster.KubernetesClusterService; + +@APICommand(name = UpgradeKubernetesClusterCmd.APINAME, description = "Upgrades a running Kubernetes cluster", + responseObject = KubernetesClusterResponse.class, + responseView = ResponseObject.ResponseView.Restricted, + entityType = {KubernetesCluster.class}, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = true, + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class UpgradeKubernetesClusterCmd extends BaseAsyncCmd { + public static final Logger LOGGER = Logger.getLogger(UpgradeKubernetesClusterCmd.class.getName()); + public static final String APINAME = "upgradeKubernetesCluster"; + + @Inject + public KubernetesClusterService kubernetesClusterService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, + entityType = KubernetesClusterResponse.class, required = true, + description = "the ID of the Kubernetes cluster") + private Long id; + + @Parameter(name = ApiConstants.KUBERNETES_VERSION_ID, type = CommandType.UUID, + entityType = KubernetesSupportedVersionResponse.class, required = true, + description = "the ID of the Kubernetes version for upgrade") + private Long kubernetesVersionId; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + public Long getKubernetesVersionId() { + return kubernetesVersionId; + } + + @Override + public String getEventType() { + return KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_UPGRADE; + } + + @Override + public String getEventDescription() { + KubernetesCluster cluster = _entityMgr.findById(KubernetesCluster.class, getId()); + return String.format("Upgrading Kubernetes cluster ID: %s", cluster.getUuid()); + } + + @Override + public String getCommandName() { + return APINAME.toLowerCase() + "response"; + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + public KubernetesCluster validateRequest() { + if (getId() == null || getId() < 1L) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Invalid Kubernetes cluster ID provided"); + } + final KubernetesCluster kubernetesCluster = kubernetesClusterService.findById(getId()); + if (kubernetesCluster == null) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Given Kubernetes cluster was not found"); + } + return kubernetesCluster; + } + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + final KubernetesCluster kubernetesCluster = validateRequest(); + try { + kubernetesClusterService.upgradeKubernetesCluster(this); + final KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(kubernetesCluster.getId()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (ManagementServerException ex) { + String msg = String.format("Failed to upgrade Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); + LOGGER.error(msg, ex); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg, ex); + } + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetesversion/ListKubernetesSupportedVersionsCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetesversion/ListKubernetesSupportedVersionsCmd.java index 2dd4f7f6e08d..d04018403c5d 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetesversion/ListKubernetesSupportedVersionsCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetesversion/ListKubernetesSupportedVersionsCmd.java @@ -68,6 +68,11 @@ public class ListKubernetesSupportedVersionsCmd extends BaseListCmd { description = "the minimum Kubernetes version") private String minimumKubernetesVersion; + @Parameter(name = ApiConstants.MIN_KUBERNETES_VERSION_ID, type = CommandType.UUID, + entityType = KubernetesSupportedVersionResponse.class, + description = "the ID of the minimum Kubernetes version") + private Long minimumKubernetesVersionId; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -87,6 +92,10 @@ public String getMinimumKubernetesVersion() { return minimumKubernetesVersion; } + public Long getMinimumKubernetesVersionId() { + return minimumKubernetesVersionId; + } + @Override public String getCommandName() { return APINAME.toLowerCase() + "response"; diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java index 7ac2fba4d4d6..888a8ad64a7f 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java @@ -69,6 +69,14 @@ public class KubernetesClusterResponse extends BaseResponse implements Controlle @Param(description = "the name of the Network associated with the IP address") private String associatedNetworkName; + @SerializedName(ApiConstants.KUBERNETES_VERSION_ID) + @Param(description = "the ID of the Kubernetes version for the cluster") + private String kubernetesVersionId; + + @SerializedName(ApiConstants.KUBERNETES_VERSION) + @Param(description = "the name of the Kubernetes version for the cluster") + private String kubernetesVersionName; + @SerializedName(ApiConstants.SSH_KEYPAIR) @Param(description = "keypair details") private String keypair; @@ -263,6 +271,22 @@ public void setAssociatedNetworkName(String associatedNetworkName) { this.associatedNetworkName = associatedNetworkName; } + public String getKubernetesVersionId() { + return kubernetesVersionId; + } + + public void setKubernetesVersionId(String kubernetesVersionId) { + this.kubernetesVersionId = kubernetesVersionId; + } + + public String getKubernetesVersionName() { + return kubernetesVersionName; + } + + public void setKubernetesVersionName(String kubernetesVersionName) { + this.kubernetesVersionName = kubernetesVersionName; + } + public KubernetesClusterResponse() { } diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml index e081dd6904e5..710e1fece898 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml @@ -1,6 +1,9 @@ #cloud-config --- +ssh_authorized_keys: + {{ k8s.ssh.pub.key }} + write-files: - path: /opt/bin/setup-kube-system permissions: 0700 diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml index caac285ad158..b27d9d8829d4 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml @@ -2,7 +2,7 @@ --- ssh_authorized_keys: - {{ k8s_master.ms.ssh.pub.key }} + {{ k8s.ssh.pub.key }} write-files: - path: /etc/conf.d/nfs @@ -206,7 +206,7 @@ write-files: ### Network, dashboard configs available offline ### echo "Offline configs are available!" kubectl apply -f ${BINARIES_DIR}/network.yaml - kubectl apply -f ${BINARIES_DIR}/dashborad.yaml + kubectl apply -f ${BINARIES_DIR}/dashboard.yaml else kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')" kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-beta6/aio/deploy/recommended.yaml diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index 94b7918783ca..a89ea6553605 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -1,6 +1,9 @@ #cloud-config --- +ssh_authorized_keys: + {{ k8s.ssh.pub.key }} + write-files: - path: /opt/bin/setup-kube-system permissions: 0700 diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh new file mode 100644 index 000000000000..e92d1a94732e --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh @@ -0,0 +1,104 @@ +#!/bin/bash -e +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +echo "Params $#" +if [ $# -lt 1 ]; then + echo "Invalid input. Valid usage: ./upgrade-kubernetes.sh UPGRADE_VERSION" + echo "eg: ./upgrade-kubernetes.sh 1.16.3" + exit 1 +fi +UPGRADE_VERSION="${1}" + +export PATH=$PATH:/opt/bin + +ISO_MOUNT_DIR=/mnt/k8sdisk +BINARIES_DIR=${ISO_MOUNT_DIR}/ + +OFFLINE_INSTALL_ATTEMPT_SLEEP=5 +MAX_OFFLINE_INSTALL_ATTEMPTS=10 +offline_attempts=1 +while true; do + if (( "$offline_attempts" > "$MAX_OFFLINE_INSTALL_ATTEMPTS" )); then + echo "Warning: Offline install timed out!" + break + fi + set +e + output=`blkid -o device -t TYPE=iso9660` + set -e + if [ "$output" != "" ]; then + while read -r line; do + if [ ! -d "${ISO_MOUNT_DIR}" ]; then + mkdir "${ISO_MOUNT_DIR}" + fi + retval=0 + set +e + mount -o ro "${line}" "${ISO_MOUNT_DIR}" + retval=$? + set -e + if [ $retval -eq 0 ]; then + if [ -d "$BINARIES_DIR" ]; then + break + else + umount "${line}" && rmdir "${ISO_MOUNT_DIR}" + fi + fi + done <<< "$output" + fi + if [ -d "$BINARIES_DIR" ]; then + break + fi + echo "Waiting for Binaries directory $BINARIES_DIR to be available, sleeping for $OFFLINE_INSTALL_ATTEMPT_SLEEP seconds, attempt: $offline_attempts" + sleep $OFFLINE_INSTALL_ATTEMPT_SLEEP + offline_attempts=$[$offline_attempts + 1] +done + +if [ -d "$BINARIES_DIR" ]; then + ### Binaries available offline ### + echo "Installing binaries from ${BINARIES_DIR}" + + cd /opt/bin + + cp ${BINARIES_DIR}/k8s/kubeadm /opt/bin + chmod +x kubeadm + + output=`ls ${BINARIES_DIR}/docker/` + if [ "$output" != "" ]; then + while read -r line; do + docker load < "${BINARIES_DIR}/docker/$line" + done <<< "$output" + fi + + tar -f "${BINARIES_DIR}/cni/cni-plugins-amd64.tgz" -C /opt/cni/bin -xz + tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-amd64.tar.gz" -C /opt/bin -xz + + if [ "${UPGRADE_VERSION}" != '' ]; then + kubeadm upgrade apply ${UPGRADE_VERSION} -y + fi + + systemctl stop kubelet + cp -a ${BINARIES_DIR}/k8s/{kubelet,kubectl} /opt/bin + chmod +x {kubelet,kubectl} + systemctl restart kubelet + + if [ "${UPGRADE_VERSION}" != '' ]; then + kubectl apply -f ${BINARIES_DIR}/network.yaml + kubectl apply -f ${BINARIES_DIR}/dashboard.yaml + fi + + umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" +fi \ No newline at end of file diff --git a/ui/plugins/cks/cks.js b/ui/plugins/cks/cks.js index 248bf89e22a4..55e974af2241 100644 --- a/ui/plugins/cks/cks.js +++ b/ui/plugins/cks/cks.js @@ -724,7 +724,84 @@ notification: { poll: pollAsyncJobResult } - } + }, + upgradeKubernetesCluster: { + label: 'Upgrade Kubernetes Cluster', + messages: { + notification: function(args) { + return 'Upgrade Kubernetes Cluster'; + } + }, + createForm: { + title: 'Upgrade Kubernetes Cluster', + desc: '', + preFilter: function(args) {}, + fields: { + kubernetesversion: { + label: 'Kubernetes version', + //docID: 'helpKubernetesClusterZone', + validation: { + required: true + }, + select: function(args) { + var filterData = { minimumkubernetesversionid: args.context.kubernetesclusters[0].kubernetesversionid }; + $.ajax({ + url: createURL("listKubernetesSupportedVersions"), + data: filterData, + dataType: "json", + async: true, + url: createURL("listKubernetesSupportedVersions"), + dataType: "json", + async: true, + success: function(json) { + var items = []; + var versionObjs = json.listkubernetessupportedversionsresponse.kubernetessupportedversion; + if (versionObjs != null) { + for (var i = 0; i < versionObjs.length; i++) { + if (versionObjs[i].id != args.context.kubernetesclusters[0].kubernetesversionid && + versionObjs[i].isostate == 'Active') { + items.push({ + id: versionObjs[i].id, + description: versionObjs[i].name + }); + } + } + } + args.response.success({ + data: items + }); + } + }); + } + }, + } + }, + action: function(args) { + var data = { + id: args.context.kubernetesclusters[0].id, + kubernetesversionid: args.data.kubernetesversion + }; + $.ajax({ + url: createURL('upgradeKubernetesCluster'), + data: data, + dataType: "json", + success: function (json) { + var jid = json.upgradekubernetesclusterresponse.jobid; + args.response.success({ + _custom: { + jobId: jid, + getActionFilter: function() { + return cksActionfilter; + } + } + }); + } + }); //end ajax + }, + notification: { + poll: pollAsyncJobResult + } + }, }, tabs: { // Details tab @@ -740,6 +817,9 @@ zonename: { label: 'label.zone.name' }, + kubernetesversion: { + label: 'Kubernetes version' + }, masternodes : { label: 'Master nodes' }, @@ -1338,6 +1418,7 @@ } if (jsonObj.state == "Created" || jsonObj.state == "Running") { allowedActions.push("scaleKubernetesCluster"); + allowedActions.push("upgradeKubernetesCluster"); } allowedActions.push("destroy"); } From a40e08d8aef82226effc3a4460ae2ae7198d8923 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 29 Nov 2019 02:39:55 +0530 Subject: [PATCH 010/134] ui error fix Signed-off-by: Abhishek Kumar --- ui/plugins/cks/cks.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/ui/plugins/cks/cks.js b/ui/plugins/cks/cks.js index 55e974af2241..73f977e79170 100644 --- a/ui/plugins/cks/cks.js +++ b/ui/plugins/cks/cks.js @@ -909,9 +909,6 @@ json.getkubernetesclusterconfigresponse.clusterconfig.configdata != null ) { jsonObj = json.getkubernetesclusterconfigresponse.clusterconfig; clusterKubeConfig = jsonObj.configdata ; - args.response.success({ - data: jsonObj - }); } } }); From 8baf8777889e7b9932e66d4ec856b79744992119 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 29 Nov 2019 17:07:45 +0530 Subject: [PATCH 011/134] fixes, refactoring Signed-off-by: Abhishek Kumar --- .../apache/cloudstack/api/ApiConstants.java | 1 + .../KubernetesClusterManagerImpl.java | 320 ++++++++---------- .../KubernetesVersionManagerImpl.java | 7 + .../KubernetesSupportedVersionResponse.java | 12 + ui/plugins/cks/cks.js | 25 +- ui/scripts/sharedFunctions.js | 15 +- 6 files changed, 192 insertions(+), 188 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index df858f8c3f0b..5efb70ae31df 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -754,6 +754,7 @@ public class ApiConstants { public static final String MIN_KUBERNETES_VERSION = "minimumkubernetesversion"; public static final String MIN_KUBERNETES_VERSION_ID = "minimumkubernetesversionid"; public static final String NODE_ROOT_DISK_SIZE = "noderootdisksize"; + public static final String SUPPORTS_HA = "supportsha"; public enum HostDetails { all, capacity, events, stats, min; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java index 845a619bbfaf..9bd9bca2ed09 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java @@ -25,9 +25,7 @@ import java.lang.reflect.Field; import java.math.BigInteger; import java.net.InetAddress; -import java.net.InetSocketAddress; import java.net.MalformedURLException; -import java.net.Socket; import java.net.URL; import java.net.UnknownHostException; import java.nio.charset.Charset; @@ -287,44 +285,42 @@ private boolean isKubernetesServiceConfigured(DataCenter zone) { // Check Kubernetes VM template for zone String templateName = globalConfigDao.getValue(KubernetesServiceConfig.KubernetesClusterTemplateName.key()); if (templateName == null || templateName.isEmpty()) { - LOGGER.warn("Global setting " + KubernetesServiceConfig.KubernetesClusterTemplateName.key() + " is empty." + - "Template name need to be specified, for Kubernetes service to function."); + LOGGER.warn(String.format("Global setting %s is empty. Template name need to be specified for Kubernetes service to function", KubernetesServiceConfig.KubernetesClusterTemplateName.key())); return false; } final VMTemplateVO template = templateDao.findByTemplateName(templateName); if (template == null) { - LOGGER.warn("Unable to find the template:" + templateName + " to be used for provisioning cluster"); + LOGGER.warn(String.format("Unable to find the template %s to be used for provisioning Kubernetes cluster", templateName)); return false; } // Check network offering String networkOfferingName = globalConfigDao.getValue(KubernetesServiceConfig.KubernetesClusterNetworkOffering.key()); if (networkOfferingName == null || networkOfferingName.isEmpty()) { - LOGGER.warn("global setting " + KubernetesServiceConfig.KubernetesClusterNetworkOffering.key() + " is empty. " + - "Admin has not yet specified the network offering to be used for provisioning isolated network for the cluster."); + LOGGER.warn(String.format("Global setting %s is empty. Admin has not yet specified the network offering to be used for provisioning isolated network for the cluster", KubernetesServiceConfig.KubernetesClusterNetworkOffering.key())); return false; } NetworkOfferingVO networkOffering = networkOfferingDao.findByUniqueName(networkOfferingName); if (networkOffering == null) { - LOGGER.warn("Network offering with name :" + networkOfferingName + " specified by admin is not found."); + LOGGER.warn(String.format("Unable to find the network offering %s to be used for provisioning Kubernetes cluster", networkOfferingName)); return false; } if (networkOffering.getState() == NetworkOffering.State.Disabled) { - LOGGER.warn("Network offering :" + networkOfferingName + "is not enabled."); + LOGGER.warn(String.format("Network offering ID: %s is not enabled", networkOffering.getUuid())); return false; } List services = networkOfferingServiceMapDao.listServicesForNetworkOffering(networkOffering.getId()); if (services == null || services.isEmpty() || !services.contains("SourceNat")) { - LOGGER.warn("Network offering :" + networkOfferingName + " does not have necessary services to provision Kubernetes cluster"); + LOGGER.warn(String.format("Network offering ID: %s does not have necessary services to provision Kubernetes cluster", networkOffering.getUuid())); return false; } if (!networkOffering.isEgressDefaultPolicy()) { - LOGGER.warn("Network offering :" + networkOfferingName + "has egress default policy turned off should be on to provision Kubernetes cluster."); + LOGGER.warn(String.format("Network offering ID: %s has egress default policy turned off should be on to provision Kubernetes cluster", networkOffering.getUuid())); return false; } long physicalNetworkId = networkModel.findPhysicalNetworkId(zone.getId(), networkOffering.getTags(), networkOffering.getTrafficType()); PhysicalNetwork physicalNetwork = physicalNetworkDao.findById(physicalNetworkId); if (physicalNetwork == null) { - LOGGER.warn("Unable to find physical network with ID: " + physicalNetworkId + " and tag: " + networkOffering.getTags()); + LOGGER.warn(String.format("Unable to find physical network with tag: %s", networkOffering.getTags())); return false; } @@ -381,6 +377,11 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + final DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); + if (zone == null) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Unable to find zone for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + } + LOGGER.debug(String.format("Starting Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.StartRequested); @@ -389,7 +390,7 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t DeployDestination dest = null; try { - dest = plan(kubernetesClusterId, kubernetesCluster.getZoneId()); + dest = plan(kubernetesCluster, zone); } catch (InsufficientCapacityException e) { String msg = String.format("Provisioning the cluster failed due to insufficient capacity in the Kubernetes cluster: %s", kubernetesCluster.getUuid()); LOGGER.error(msg, e); @@ -485,7 +486,6 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t int retryCounter = 0; int maxRetries = 15; boolean k8sApiServerSetup = false; - while (retryCounter < maxRetries) { try { String versionOutput = IOUtils.toString(new URL(String.format("https://%s:%d/version", publicIp.getAddress().addr(), 6443)), StandardCharsets.UTF_8); @@ -498,18 +498,20 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t break; } } catch (Exception e) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Waiting for Kubernetes cluster: " + kubernetesCluster.getName() + " API endpoint to be available. retry: " + retryCounter + "/" + maxRetries); - } + LOGGER.warn(String.format("API endpoint for Kubernetes cluster ID: %s not available. Attempt: %d/%d", kubernetesCluster.getUuid(), retryCounter+1, maxRetries), e); } try { Thread.sleep(30000); } catch (InterruptedException ie) { - LOGGER.error(String.format("Error while waiting for Kubernetes cluster: %s API endpoint to be available", kubernetesCluster.getUuid()), ie); + LOGGER.error(String.format("Error while waiting for Kubernetes cluster ID: %s API endpoint to be available", kubernetesCluster.getUuid()), ie); } retryCounter++; } + if (!k8sApiServerSetup) { + LOGGER.error(String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to provision API endpoint for the cluster", kubernetesCluster.getUuid())); + } + boolean k8sKubeConfigCopied = false; if (k8sApiServerSetup) { retryCounter = 0; @@ -533,63 +535,48 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t String.format("server: https://%s:6443", publicIp.getAddress().addr())); kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "kubeConfigData", Base64.encodeBase64String(kubeConfig.getBytes(Charset.forName("UTF-8"))), false); k8sKubeConfigCopied = true; - LOGGER.debug(String.format("Kubernetes cluster: %s kube-config has been successdully retrieved", kubernetesCluster.getUuid())); + LOGGER.debug(String.format("Kubernetes cluster ID: %s kube-config has been successdully retrieved", kubernetesCluster.getUuid())); break; } } catch (Exception e) { - LOGGER.warn("Failed to retrieve kube-config file for cluster with ID " + kubernetesCluster.getUuid() + ": " + e); + LOGGER.warn(String.format("Failed to retrieve kube-config file for Kubernetes cluster ID: %s. Attempt: %d/%d", kubernetesCluster.getUuid(), retryCounter+1, maxRetries), e); } retryCounter++; } + if (!k8sKubeConfigCopied) { + LOGGER.error(String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to retrieve kube-config for the cluster", kubernetesCluster.getUuid())); + } } if (k8sKubeConfigCopied) { retryCounter = 0; maxRetries = 30; - // Dashbaord service is a docker image downloaded at run time. - // So wait for some time and check if dashbaord service is up running. + // Check if dashboard service is up running. while (retryCounter < maxRetries) { - - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Waiting for dashboard service for the Kubernetes cluster: " + kubernetesCluster.getName() - + " to come up. Attempt: " + retryCounter + " of max retries " + maxRetries); - } - + LOGGER.debug(String.format("Checking dashboard service for the Kubernetes cluster ID: %s to come up. Attempt: %d/%d", kubernetesCluster.getUuid(), retryCounter+1, maxRetries)); if (isAddOnServiceRunning(kubernetesCluster.getId(), "kubernetes-dashboard", "kubernetes-dashboard")) { - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationSucceeded); - kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); kubernetesCluster.setConsoleEndpoint("https://" + publicIp.getAddress() + ":6443/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy#!/overview?namespace=_all"); kubernetesClusterDao.update(kubernetesCluster.getId(), kubernetesCluster); - detachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); - - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Kubernetes cluster name:" + kubernetesCluster.getName() + " is successfully started"); - } - + LOGGER.debug(String.format("Kubernetes cluster ID: %s is successfully started", kubernetesCluster.getUuid())); return true; } try { Thread.sleep(10000); } catch (InterruptedException ex) { + LOGGER.error(String.format("Error while waiting for Kubernetes cluster: %s API dashboard service to be available", kubernetesCluster.getUuid()), ex); } retryCounter++; } LOGGER.warn("Failed to setup Kubernetes cluster " + kubernetesCluster.getName() + " in usable state as" + " unable to bring dashboard add on service up"); - } else { - LOGGER.warn("Failed to setup Kubernetes cluster " + kubernetesCluster.getName() + " in usable state as" + - " unable to bring the API server up"); } stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); - detachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); - - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, - "Failed to deploy Kubernetes cluster: " + kubernetesCluster.getUuid() + " as unable to setup up in usable state"); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to deploy Kubernetes cluster ID: %s as unable to setup up in usable state", kubernetesCluster.getUuid())); } private boolean startStoppedKubernetesCluster(long kubernetesClusterId) throws ManagementServerException, @@ -644,36 +631,45 @@ private boolean startStoppedKubernetesCluster(long kubernetesClusterId) throws M try { address = InetAddress.getByName(new URL(kubernetesCluster.getEndpoint()).getHost()); } catch (MalformedURLException | UnknownHostException ex) { - // API end point is generated by CCS, so this situation should not arise. - LOGGER.warn("Kubernetes cluster ID:" + kubernetesClusterId + " has invalid api endpoint. Can not " + - "verify if cluster is in ready state."); - throw new ManagementServerException("Can not verify if Kubernetes cluster ID:" + kubernetesClusterId + " is in usable state."); + String msg = String.format("Kubernetes cluster ID: %s has invalid api endpoint. Can not verify if cluster is in ready state", kubernetesCluster.getUuid()); + LOGGER.warn(msg, ex); + throw new ManagementServerException(msg, ex); } - // wait for fixed time for K8S api server to be avaialble + IPAddressVO publicIp = null; + List ips = ipAddressDao.listByAssociatedNetwork(kubernetesCluster.getNetworkId(), true); + if (ips == null || ips.isEmpty() || ips.get(0) == null) { + String msg = String.format("Failed to start Kubernetes cluster ID: %s, unable to retrieve associated public IP", kubernetesCluster.getUuid()); + LOGGER.error(msg); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); + } + publicIp = ips.get(0); + int retryCounter = 0; int maxRetries = 10; boolean k8sApiServerSetup = false; while (retryCounter < maxRetries) { - try (Socket socket = new Socket()) { - socket.connect(new InetSocketAddress(address.getHostAddress(), 6443), 10000); - k8sApiServerSetup = true; - break; - } catch (IOException e) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Waiting for Kubernetes cluster: " + kubernetesCluster.getName() + " API endpoint to be available. retry: " + retryCounter + "/" + maxRetries); - } - try { - Thread.sleep(50000); - } catch (InterruptedException ex) { + try { + String versionOutput = IOUtils.toString(new URL(String.format("https://%s:%d/version", publicIp.getAddress().addr(), 6443)), StandardCharsets.UTF_8); + if (!Strings.isNullOrEmpty(versionOutput)) { + LOGGER.debug(String.format("Kubernetes cluster ID: %s API has been successfully provisioned, %s", kubernetesCluster.getUuid(), versionOutput)); + k8sApiServerSetup = true; + break; } - retryCounter++; + } catch (Exception e) { + LOGGER.warn(String.format("API endpoint for Kubernetes cluster ID: %s not available. Attempt: %d/%d", kubernetesCluster.getUuid(), retryCounter+1, maxRetries), e); } + try { + Thread.sleep(30000); + } catch (InterruptedException ie) { + LOGGER.error(String.format("Error while waiting for Kubernetes cluster ID: %s API endpoint to be available", kubernetesCluster.getUuid()), ie); + } + retryCounter++; } if (!k8sApiServerSetup) { stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); - throw new ManagementServerException("Failed to setup Kubernetes cluster ID: " + kubernetesClusterId + " is usable state."); + throw new ManagementServerException(String.format("Failed to setup Kubernetes cluster ID: %s is usable state", kubernetesCluster.getUuid())); } stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationSucceeded); @@ -889,15 +885,11 @@ private void scaleKubernetesClusterNetworkRules(IPAddressVO publicIp, Account ac firewallService.createIngressFirewallRule(rule); firewallService.applyIngressFwRules(publicIp.getId(), account); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Provisioned firewall rule to open up port 2222 on " + publicIp.getAddress() + - " for cluster " + kubernetesCluster.getName()); - } + LOGGER.debug(String.format("Provisioned firewall rule to open up port 2222 to %d on %s in Kubernetes cluster ID: %s", 2222 + (int)kubernetesCluster.getNodeCount(), publicIp.getAddress().addr(), kubernetesCluster.getName())); } catch (Exception e) { - LOGGER.warn("Failed to provision firewall rules for the Kubernetes cluster: " + kubernetesCluster.getName() - + " due to exception: " + getStackTrace(e)); - throw new ManagementServerException("Failed to provision firewall rules for the Kubernetes " + - "cluster: " + kubernetesCluster.getName()); + String msg = String.format("Failed to activate SSH firewall rules for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); + LOGGER.warn(msg, e); + throw new ManagementServerException(msg, e); } if (clusterVMIds != null && !clusterVMIds.isEmpty()) { // Upscaling, add new port-forwarding rules @@ -930,42 +922,38 @@ public PortForwardingRuleVO doInTransaction(TransactionStatus status) throws Net } }); rulesService.applyPortForwardingRules(publicIp.getId(), account); - - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Provisioning SSH port forwarding rule from port 2222 to 22 on " + publicIp.getAddress() + - " to the VM IP :" + vmIp + " in Kubernetes cluster " + kubernetesCluster.getName()); - } + LOGGER.debug(String.format("Provisioned SSH port forwarding rule from port %d to 22 on %s to the VM IP : %s in Kubernetes cluster ID: %s", srcPortFinal, publicIp.getAddress().addr(), vmIp.toString(), kubernetesCluster.getName())); } catch (Exception e) { - LOGGER.warn("Failed to activate SSH port forwarding rules for the Kubernetes cluster " + kubernetesCluster.getName() + " due to " + e); - throw new ManagementServerException("Failed to activate SSH port forwarding rules for the cluster: " + kubernetesCluster.getName(), e); + String msg = String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); + LOGGER.warn(msg, e); + throw new ManagementServerException(msg, e); } } } } - private boolean validateNetwork(Network network) { + private boolean validateNetwork(Network network, int clusterTotalNodeCount) { NetworkOffering networkOffering = networkOfferingDao.findById(network.getNetworkOfferingId()); if (networkOffering.isSystemOnly()) { - throw new InvalidParameterValueException("This network is for system use only, network id " + network.getId()); + throw new InvalidParameterValueException(String.format("Network ID: %s is for system use only", network.getUuid())); } if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.UserData)) { - throw new InvalidParameterValueException("This network does not support userdata that is required for k8s, network id " + network.getId()); + throw new InvalidParameterValueException(String.format("Network ID: %s does not support userdata that is required for Kubernetes cluster", network.getUuid())); } if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.Firewall)) { - throw new InvalidParameterValueException("This network does not support firewall that is required for k8s, network id " + network.getId()); + throw new InvalidParameterValueException(String.format("Network ID: %s does not support firewall that is required for Kubernetes cluster", network.getUuid())); } if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.PortForwarding)) { - throw new InvalidParameterValueException("This network does not support port forwarding that is required for k8s, network id " + network.getId()); + throw new InvalidParameterValueException(String.format("Network ID: %s does not support port forwarding that is required for Kubernetes cluster", network.getUuid())); } if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.Dhcp)) { - throw new InvalidParameterValueException("This network does not support dhcp that is required for k8s, network id " + network.getId()); + throw new InvalidParameterValueException(String.format("Network ID: %s does not support DHCP that is required for Kubernetes cluster", network.getUuid())); } List addrs = networkModel.listPublicIpsAssignedToGuestNtwk(network.getId(), true); IPAddressVO sourceNatIp = null; if (addrs.isEmpty()) { - throw new InvalidParameterValueException("The network ID:" + network.getId() + " does not have source NAT ip assoicated with it. " + - "To provision a Kubernetes Cluster, a isolated network with source NAT is required."); + throw new InvalidParameterValueException(String.format("Network ID: %s does not have a public IP associated with it. To provision a Kubernetes Cluster, source NAT IP is required", network.getUuid())); } else { for (IpAddress addr : addrs) { if (addr.isSourceNat()) { @@ -973,8 +961,7 @@ private boolean validateNetwork(Network network) { } } if (sourceNatIp == null) { - throw new InvalidParameterValueException("The network ID:" + network.getId() + " does not have source NAT ip assoicated with it. " + - "To provision a Kubernetes Cluster, a isolated network with source NAT is required."); + throw new InvalidParameterValueException(String.format("Network ID: %s does not have a source NAT IP associated with it. To provision a Kubernetes Cluster, source NAT IP is required", network.getUuid())); } } List rules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(sourceNatIp.getId(), FirewallRule.Purpose.Firewall); @@ -983,8 +970,10 @@ private boolean validateNetwork(Network network) { Integer endPort = rule.getSourcePortEnd(); LOGGER.debug("Network rule : " + startPort + " " + endPort); if (startPort <= 6443 && 6443 <= endPort) { - throw new InvalidParameterValueException("The network ID:" + network.getId() + " has conflicting firewall rules to provision" + - " Kubernetes cluster."); + throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting firewall rules to provision Kubernetes cluster for API access", network.getUuid())); + } + if (startPort <= 2222 && 2222+clusterTotalNodeCount <= endPort) { + throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting firewall rules to provision Kubernetes cluster for node VM SSH access", network.getUuid())); } } @@ -994,8 +983,10 @@ private boolean validateNetwork(Network network) { Integer endPort = rule.getSourcePortEnd(); LOGGER.debug("Network rule : " + startPort + " " + endPort); if (startPort <= 6443 && 6443 <= endPort) { - throw new InvalidParameterValueException("The network ID:" + network.getId() + " has conflicting port forwarding rules to provision" + - " Kubernetes cluster."); + throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting port forwarding rules to provision Kubernetes cluster for API access", network.getUuid())); + } + if (startPort <= 2222 && 2222+clusterTotalNodeCount <= endPort) { + throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting port forwarding rules to provision Kubernetes cluster for node VM SSH access", network.getUuid())); } } return true; @@ -1044,10 +1035,10 @@ private void validateDockerRegistryParams(final String dockerRegistryUserName, } } - private DeployDestination plan(final long nodesCount, final long dcId, final ServiceOffering offering) throws InsufficientServerCapacityException { + private DeployDestination plan(final long nodesCount, final DataCenter zone, final ServiceOffering offering) throws InsufficientServerCapacityException { final int cpu_requested = offering.getCpu() * offering.getSpeed(); final long ram_requested = offering.getRamSize() * 1024L * 1024L; - List hosts = resourceManager.listAllHostsInOneZoneByType(Type.Routing, dcId); + List hosts = resourceManager.listAllHostsInOneZoneByType(Type.Routing, zone.getId()); final Map> hosts_with_resevered_capacity = new ConcurrentHashMap>(); for (HostVO h : hosts) { hosts_with_resevered_capacity.put(h.getUuid(), new Pair(h, 0)); @@ -1065,13 +1056,9 @@ private DeployDestination plan(final long nodesCount, final long dcId, final Ser ClusterDetailsVO cluster_detail_ram = clusterDetailsDao.findDetail(cluster.getId(), "memoryOvercommitRatio"); Float cpuOvercommitRatio = Float.parseFloat(cluster_detail_cpu.getValue()); Float memoryOvercommitRatio = Float.parseFloat(cluster_detail_ram.getValue()); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Checking host " + h.getId() + " for capacity already reserved " + reserved); - } + LOGGER.debug(String.format("Checking host ID: %s for capacity already reserved %d", h.getUuid(), reserved)); if (capacityManager.checkIfHostHasCapacity(h.getId(), cpu_requested * reserved, ram_requested * reserved, false, cpuOvercommitRatio, memoryOvercommitRatio, true)) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Found host " + h.getId() + " has enough capacity cpu = " + cpu_requested * reserved + " ram =" + ram_requested * reserved); - } + LOGGER.debug(String.format("Found host ID: %s for with enough capacity, CPU=%d RAM=%d already reserved %d", cpu_requested * reserved, ram_requested * reserved)); hostEntry.setValue(new Pair(h, reserved)); suitable_host_found = true; break; @@ -1080,37 +1067,26 @@ private DeployDestination plan(final long nodesCount, final long dcId, final Ser if (suitable_host_found) { continue; } else { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Suitable hosts not found in datacenter " + dcId + " for node " + i); - } + LOGGER.debug(String.format("Suitable hosts not found in datacenter ID: %s for node %d", zone.getUuid(), i)); break; } } if (suitable_host_found) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Suitable hosts found in datacenter " + dcId + " creating deployment destination"); - } - return new DeployDestination(dataCenterDao.findById(dcId), null, null, null); + LOGGER.debug(String.format("Suitable hosts found in datacenter ID: %s, creating deployment destination", zone.getUuid())); + return new DeployDestination(zone, null, null, null); } String msg = String.format("Cannot find enough capacity for Kubernetes cluster(requested cpu=%1$s memory=%2$s)", cpu_requested * nodesCount, ram_requested * nodesCount); LOGGER.warn(msg); - throw new InsufficientServerCapacityException(msg, DataCenter.class, dcId); - } - - private DeployDestination plan(final KubernetesCluster kubernetesCluster) throws InsufficientServerCapacityException { - return plan(kubernetesCluster.getId(), kubernetesCluster.getZoneId()); + throw new InsufficientServerCapacityException(msg, DataCenter.class, zone.getId()); } - private DeployDestination plan(final long kubernetesClusterId, final long dcId) throws InsufficientServerCapacityException { - KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + private DeployDestination plan(final KubernetesCluster kubernetesCluster, final DataCenter zone) throws InsufficientServerCapacityException { ServiceOffering offering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Checking deployment destination for kubernetesClusterId= " + kubernetesClusterId + " in dcId=" + dcId); - } + LOGGER.debug(String.format("Checking deployment destination for Kubernetes cluster ID: %s in zone ID: %s", kubernetesCluster.getUuid(), zone.getUuid())); - return plan(kubernetesCluster.getTotalNodeCount(), dcId, offering); + return plan(kubernetesCluster.getTotalNodeCount(), zone, offering); } private boolean isAddOnServiceRunning(Long clusterId, final String nameSpace, String svcName) { @@ -1523,7 +1499,6 @@ private UserVm createKubernetesNode(KubernetesClusterVO kubernetesCluster, Strin } private void startKubernetesVM(final UserVm vm, final KubernetesClusterVO kubernetesCluster) throws ServerApiException { - try { StartVMCmd startVm = new StartVMCmd(); startVm = ComponentContext.inject(startVm); @@ -1534,18 +1509,6 @@ private void startKubernetesVM(final UserVm vm, final KubernetesClusterVO kubern if (LOGGER.isDebugEnabled()) { LOGGER.debug("Started VM in the Kubernetes cluster: " + kubernetesCluster.getName()); } - } catch (ConcurrentOperationException ex) { - LOGGER.warn("Failed to start VM in the Kubernetes cluster name:" + kubernetesCluster.getName() + " due to Exception: ", ex); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to start VM in the Kubernetes cluster name:" + kubernetesCluster.getName(), ex); - } catch (ResourceUnavailableException ex) { - LOGGER.warn("Failed to start VM in the Kubernetes cluster name:" + kubernetesCluster.getName() + " due to Exception: ", ex); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to start VM in the Kubernetes cluster name:" + kubernetesCluster.getName(), ex); - } catch (InsufficientCapacityException ex) { - LOGGER.warn("Failed to start VM in the Kubernetes cluster name:" + kubernetesCluster.getName() + " due to Exception: ", ex); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to start VM in the Kubernetes cluster name:" + kubernetesCluster.getName(), ex); - } catch (RuntimeException ex) { - LOGGER.warn("Failed to start VM in the Kubernetes cluster name:" + kubernetesCluster.getName() + " due to Exception: ", ex); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to start VM in the Kubernetes cluster name:" + kubernetesCluster.getName(), ex); } catch (Exception ex) { LOGGER.warn("Failed to start VM in the Kubernetes cluster name:" + kubernetesCluster.getName() + " due to Exception: ", ex); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to start VM in the Kubernetes cluster name:" + kubernetesCluster.getName(), ex); @@ -1668,8 +1631,7 @@ protected boolean stateTransitTo(long kubernetesClusterId, KubernetesCluster.Eve try { return _stateMachine.transitTo(kubernetesCluster, e, null, kubernetesClusterDao); } catch (NoTransitionException nte) { - LOGGER.warn("Failed to transistion state of the Kubernetes cluster: " + kubernetesCluster.getName() - + " in state " + kubernetesCluster.getState().toString() + " on event " + e.toString()); + LOGGER.warn(String.format("Failed to transition state of the Kubernetes cluster ID: %s in state %s on event %s", kubernetesCluster.getUuid(), kubernetesCluster.getState().toString(), e.toString()), nte); return false; } } @@ -1698,20 +1660,20 @@ public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) } if (masterNodeCount < 1 || masterNodeCount > 100) { - throw new InvalidParameterValueException("Invalid cluster master nodes count " + masterNodeCount); + throw new InvalidParameterValueException("Invalid cluster master nodes count: " + masterNodeCount); } if (clusterSize < 1 || clusterSize > 100) { - throw new InvalidParameterValueException("Invalid cluster size " + clusterSize); + throw new InvalidParameterValueException("Invalid cluster size: " + clusterSize); } DataCenter zone = dataCenterDao.findById(zoneId); if (zone == null) { - throw new InvalidParameterValueException("Unable to find zone by ID:" + zoneId); + throw new InvalidParameterValueException("Unable to find zone by ID: " + zoneId); } if (Grouping.AllocationState.Disabled == zone.getAllocationState()) { - throw new PermissionDeniedException("Cannot perform this operation, Zone:" + zone.getId() + " is currently disabled."); + throw new PermissionDeniedException(String.format("Cannot perform this operation, zone ID: %s is currently disabled", zone.getUuid())); } final KubernetesSupportedVersion clusterKubernetesVersion = kubernetesSupportedVersionDao.findById(kubernetesVersionId); @@ -1737,49 +1699,53 @@ public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) ServiceOffering serviceOffering = serviceOfferingDao.findById(serviceOfferingId); if (serviceOffering == null) { - throw new InvalidParameterValueException("No service offering with ID:" + serviceOfferingId); + throw new InvalidParameterValueException("No service offering with ID: " + serviceOfferingId); } else { } if (sshKeyPair != null && !sshKeyPair.isEmpty()) { SSHKeyPairVO sshKeyPairVO = sshKeyPairDao.findByName(owner.getAccountId(), owner.getDomainId(), sshKeyPair); if (sshKeyPairVO == null) { - throw new InvalidParameterValueException("Given SSH key pair with name:" + sshKeyPair + " was not found for the account " + owner.getAccountName()); + throw new InvalidParameterValueException(String.format("Given SSH key pair with name: %s was not found for the account %s", sshKeyPair, owner.getAccountName())); } } if (!isKubernetesServiceConfigured(zone)) { - throw new ManagementServerException("Kubernetes service has not been configured properly to provision Kubernetes clusters."); + throw new ManagementServerException("Kubernetes service has not been configured properly to provision Kubernetes clusters"); } VMTemplateVO template = templateDao.findByTemplateName(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesClusterTemplateName.key())); List listZoneTemplate = templateZoneDao.listByZoneTemplate(zone.getId(), template.getId()); if (listZoneTemplate == null || listZoneTemplate.isEmpty()) { - LOGGER.warn("The template:" + template.getId() + " is not available for use in zone:" + zoneId + " to provision Kubernetes cluster name:" + name); - throw new ManagementServerException("Kubernetes service has not been configured properly to provision Kubernetes clusters."); + String msg = String.format("The template ID: %s is not available for use in zone ID: %s to provision Kubernetes cluster name: %s", template.getUuid(), zone.getUuid(), name); + LOGGER.warn(msg); + throw new ManagementServerException(msg); } if (!validateServiceOffering(serviceOfferingDao.findById(serviceOfferingId))) { - throw new InvalidParameterValueException("This service offering is not suitable for k8s cluster, service offering id is " + networkId); + throw new InvalidParameterValueException("Given service offering ID: %s is not suitable for Kubernetes cluster"); } validateDockerRegistryParams(dockerRegistryUserName, dockerRegistryPassword, dockerRegistryUrl, dockerRegistryEmail); - plan(masterNodeCount + clusterSize, zoneId, serviceOfferingDao.findById(serviceOfferingId)); - Network network = null; if (networkId != null) { - if (kubernetesClusterDao.listByNetworkId(networkId).isEmpty()) { - network = networkService.getNetwork(networkId); - if (network == null) { - throw new InvalidParameterValueException("Unable to find network by ID " + networkId); - } - if (!validateNetwork(network)) { - throw new InvalidParameterValueException("This network is not suitable for k8s cluster, network id is " + networkId); + network = networkService.getNetwork(networkId); + if (network == null) { + throw new InvalidParameterValueException("Unable to find network with given ID"); + } + } + + plan(masterNodeCount + clusterSize, zone, serviceOfferingDao.findById(serviceOfferingId)); + + if (network != null) { + if (kubernetesClusterDao.listByNetworkId(network.getId()).isEmpty()) { + if (!validateNetwork(network, (int)(masterNodeCount+clusterSize))) { + throw new InvalidParameterValueException(String.format("Network ID: %s is not suitable for Kubernetes cluster", network.getUuid())); } networkModel.checkNetworkPermissions(owner, network); } else { - throw new InvalidParameterValueException("This network is already under use by another k8s cluster, network id is " + networkId); + throw new InvalidParameterValueException(String.format("Network ID: %s is already under use by another Kubernetes cluster", network.getUuid())); } } else { // user has not specified network in which cluster VM's to be provisioned, so create a network for Kubernetes cluster NetworkOfferingVO networkOffering = networkOfferingDao.findByUniqueName( @@ -1788,17 +1754,15 @@ public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) long physicalNetworkId = networkModel.findPhysicalNetworkId(zone.getId(), networkOffering.getTags(), networkOffering.getTrafficType()); PhysicalNetwork physicalNetwork = physicalNetworkDao.findById(physicalNetworkId); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Creating network for account " + owner + " from the network offering id=" + - networkOffering.getId() + " as a part of cluster: " + name + " deployment process"); - } + LOGGER.debug(String.format("Creating network for account ID: %s from the network offering ID: %s as part of Kubernetes cluster: %s deployment process", owner.getUuid(), networkOffering.getUuid(), name)); try { network = networkMgr.createGuestNetwork(networkOffering.getId(), name + "-network", owner.getAccountName() + "-network", null, null, null, false, null, owner, null, physicalNetwork, zone.getId(), ControlledEntity.ACLType.Account, null, null, null, null, true, null, null); } catch (Exception e) { - LOGGER.warn("Unable to create a network for the Kubernetes cluster due to " + e); - throw new ManagementServerException("Unable to create a network for the Kubernetes cluster."); + String msg = String.format("Unable to create network for the Kubernetes cluster: %s", name); + LOGGER.warn(msg, e); + throw new ManagementServerException(msg, e); } } @@ -1842,11 +1806,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { kubernetesClusterDetailsDao.saveDetails(details); } }); - - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("A Kubernetes cluster name:" + name + " ID:" + cluster.getId() + " has been created."); - } - + LOGGER.debug(String.format("Kubernetes cluster name: %s and ID: %s has been created", cluster.getName(), cluster.getUuid())); return cluster; } @@ -1874,30 +1834,24 @@ public boolean stopKubernetesCluster(long kubernetesClusterId) throws Management final KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); if (kubernetesCluster == null) { - throw new ManagementServerException("Failed to find Kubernetes cluster ID: " + kubernetesClusterId); + throw new ManagementServerException("Failed to find Kubernetes cluster with given ID"); } if (kubernetesCluster.getRemoved() != null) { - throw new ManagementServerException("Kubernetes cluster ID:" + kubernetesClusterId + " is already deleted."); + throw new ManagementServerException(String.format("Kubernetes cluster ID: %s is already deleted", kubernetesCluster.getUuid())); } if (kubernetesCluster.getState().equals(KubernetesCluster.State.Stopped)) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Kubernetes cluster ID: " + kubernetesClusterId + " is already stopped."); - } + LOGGER.debug(String.format("Kubernetes cluster ID: %s is already stopped", kubernetesCluster.getUuid())); return true; } if (kubernetesCluster.getState().equals(KubernetesCluster.State.Stopping)) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Kubernetes cluster ID: " + kubernetesClusterId + " is getting stopped."); - } + LOGGER.debug(String.format("Kubernetes cluster ID: %s is getting stopped", kubernetesCluster.getUuid())); return true; } - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Stopping Kubernetes cluster: " + kubernetesCluster.getName()); - } + LOGGER.debug(String.format("Stopping Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.StopRequested); @@ -1906,11 +1860,11 @@ public boolean stopKubernetesCluster(long kubernetesClusterId) throws Management try { if (vm == null) { stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); - throw new ManagementServerException("Failed to start all VMs in Kubernetes cluster ID: " + kubernetesClusterId); + throw new ManagementServerException(String.format("Failed to find all VMs in Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); } stopClusterVM(vmMapVO); } catch (ServerApiException ex) { - LOGGER.warn("Failed to stop VM in Kubernetes cluster ID:" + kubernetesClusterId + " due to " + ex); + LOGGER.warn(String.format("Failed to stop VM in Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), ex); // dont bail out here. proceed further to stop the reset of the VM's } } @@ -1919,7 +1873,7 @@ public boolean stopKubernetesCluster(long kubernetesClusterId) throws Management final UserVmVO vm = userVmDao.findById(vmMapVO.getVmId()); if (vm == null || !vm.getState().equals(VirtualMachine.State.Stopped)) { stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); - throw new ManagementServerException("Failed to stop all VMs in Kubernetes cluster ID: " + kubernetesClusterId); + throw new ManagementServerException(String.format("Failed to stop all VMs in Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); } } @@ -1952,7 +1906,7 @@ public ListResponse listKubernetesClusters(ListKubern if (!KubernetesCluster.State.Running.toString().equals(state) && !KubernetesCluster.State.Stopped.toString().equals(state) && !KubernetesCluster.State.Destroyed.toString().equals(state)) { - throw new InvalidParameterValueException("Invalid value for Kubernetes cluster state is specified"); + throw new InvalidParameterValueException("Invalid value for Kubernetes cluster state specified"); } } if (clusterId != null) { @@ -2019,6 +1973,10 @@ public boolean scaleKubernetesCluster(ScaleKubernetesClusterCmd cmd) throws Mana if (kubernetesCluster == null || kubernetesCluster.getRemoved() != null) { throw new InvalidParameterValueException("Invalid Kubernetes cluster ID"); } + final DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); + if (zone == null) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Unable to find zone for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + } Account caller = CallContext.current().getCallingAccount(); accountManager.checkAccess(caller, SecurityChecker.AccessType.OperateEntry, false, kubernetesCluster); @@ -2134,7 +2092,7 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { final long newVmRequiredCount = clusterSize - originalNodeCount; final ServiceOffering clusterServiceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); if (clusterServiceOffering == null) { - String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, service offering not found", kubernetesCluster.getUuid()); + String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, cluster service offering not found", kubernetesCluster.getUuid()); LOGGER.error(msg); stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); @@ -2145,12 +2103,12 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { } try { if (clusterState.equals(KubernetesCluster.State.Running)) { - plan(newVmRequiredCount, kubernetesCluster.getZoneId(), clusterServiceOffering); + plan(newVmRequiredCount, zone, clusterServiceOffering); } else { - plan(kubernetesCluster.getTotalNodeCount() + newVmRequiredCount, kubernetesCluster.getZoneId(), clusterServiceOffering); + plan(kubernetesCluster.getTotalNodeCount() + newVmRequiredCount, zone, clusterServiceOffering); } } catch (InsufficientCapacityException e) { - String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, insufficient capacity", kubernetesCluster.getUuid()); + String msg = String.format("Scaling failed for Kubernetes cluster ID: %s in zone ID: %s, insufficient capacity", kubernetesCluster.getUuid(), zone.getUuid()); LOGGER.error(msg); stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); throw new ServerApiException(ApiErrorCode.INSUFFICIENT_CAPACITY_ERROR, msg, e); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java index 70fd1398762e..e9a8db5c03df 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java @@ -40,6 +40,7 @@ import com.cloud.dc.dao.DataCenterDao; import com.cloud.event.ActionEvent; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.kubernetescluster.KubernetesClusterManagerImpl; import com.cloud.kubernetescluster.KubernetesClusterVO; import com.cloud.kubernetescluster.dao.KubernetesClusterDao; import com.cloud.kubernetesversion.dao.KubernetesSupportedVersionDao; @@ -81,6 +82,12 @@ private KubernetesSupportedVersionResponse createKubernetesSupportedVersionRespo response.setZoneId(zone.getUuid()); response.setZoneName(zone.getName()); } + if (compareKubernetesVersion(kubernetesSupportedVersion.getKubernetesVersion(), + KubernetesClusterManagerImpl.MIN_KUBERNETES_VERSION_HA_SUPPORT)>=0) { + response.setSupportsHA(true); + } else { + response.setSupportsHA(false); + } VMTemplateVO template = ApiDBUtils.findTemplateById(kubernetesSupportedVersion.getIsoId()); response.setIsoId(template.getUuid()); response.setIsoName(template.getName()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java index 1203b488f25e..26004899ebfb 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java @@ -60,6 +60,10 @@ public class KubernetesSupportedVersionResponse extends BaseResponse { @Param(description = "the name of the zone in which Kubernetes supported version is available") private String zoneName; + @SerializedName(ApiConstants.SUPPORTS_HA) + @Param(description = "whether Kubernetes supported version supports HA, multi-master") + private Boolean supportsHA; + public String getId() { return id; } @@ -123,4 +127,12 @@ public String getZoneName() { public void setZoneName(String zoneName) { this.zoneName = zoneName; } + + public Boolean isSupportsHA() { + return supportsHA; + } + + public void setSupportsHA(Boolean supportsHA) { + this.supportsHA = supportsHA; + } } diff --git a/ui/plugins/cks/cks.js b/ui/plugins/cks/cks.js index 73f977e79170..9ec199cdbbea 100644 --- a/ui/plugins/cks/cks.js +++ b/ui/plugins/cks/cks.js @@ -141,7 +141,7 @@ createForm: { title: 'Add Kubernetes cluster', preFilter: function(args) { - args.$form.find('.form-item[rel=masternodes]').find('input[name=masternodes]').val('1'); + args.$form.find('.form-item[rel=masternodes]').find('input[name=masternodes]').val('2'); args.$form.find('.form-item[rel=size]').find('input[name=size]').val('1'); }, fields: { @@ -193,6 +193,7 @@ required: true }, select: function(args) { + var versionObjs; var filterData = { zoneid: args.zone }; $.ajax({ url: createURL("listKubernetesSupportedVersions"), @@ -204,7 +205,7 @@ async: true, success: function(json) { var items = []; - var versionObjs = json.listkubernetessupportedversionsresponse.kubernetessupportedversion; + versionObjs = json.listkubernetessupportedversionsresponse.kubernetessupportedversion; if (versionObjs != null) { for (var i = 0; i < versionObjs.length; i++) { if (versionObjs[i].isostate == 'Active') { @@ -220,6 +221,21 @@ }); } }); + + args.$select.change(function() { + var $form = $(this).closest("form"); + $form.find('.form-item[rel=multimaster]').find('input[name=multimaster]').prop('checked', false); + $form.find('.form-item[rel=multimaster]').hide(); + $form.find('.form-item[rel=masternodes]').hide(); + var currentVersionId = $(this).val(); + if (currentVersionId != null && versionObjs != null) { + for (var i = 0; i < versionObjs.length; i++) { + if (currentVersionId == versionObjs[i].id && versionObjs[i].supportsha === true) { + $form.find('.form-item[rel=multimaster]').css('display', 'inline-block'); + } + } + } + }); } }, serviceoffering: { @@ -290,6 +306,7 @@ }, multimaster: { label: "HA (Multi-master)", + dependsOn: 'kubernetesversion', isBoolean: true, isChecked: false, }, @@ -298,7 +315,7 @@ //docID: 'helpKubernetesClusterSize', validation: { required: true, - naturalnumber: true + multiplecountnumber: true }, dependsOn: "multimaster", isHidden: true, @@ -1169,7 +1186,7 @@ async: true, success: function(json) { var items = []; - var isoObjs = json.listisosresponse.iso;; + var isoObjs = json.listisosresponse.iso; if (isoObjs != null) { for (var i = 0; i < isoObjs.length; i++) { items.push({ diff --git a/ui/scripts/sharedFunctions.js b/ui/scripts/sharedFunctions.js index 458bef855109..cc73ecb7a24e 100644 --- a/ui/scripts/sharedFunctions.js +++ b/ui/scripts/sharedFunctions.js @@ -2706,7 +2706,6 @@ jQuery.validator.addMethod("ipv6CustomJqueryValidator", function(value, element) return jQuery.validator.methods.ipv6.call(this, value, element); }, "The specified IPv6 address is invalid."); - $.validator.addMethod("allzonesonly", function(value, element){ if ((value.indexOf("-1") != -1) && (value.length > 1)) @@ -2716,18 +2715,28 @@ $.validator.addMethod("allzonesonly", function(value, element){ }, "All Zones cannot be combined with any other zone"); - $.validator.addMethod("naturalnumber", function(value, element){ if (this.optional(element) && value.length == 0) return true; if (isNaN(value)) return false; value = parseInt(value); - return (typeof value === 'number') && (value > 0.0) && (Math.floor(value) === value) && value !== Infinity; + return (typeof value === 'number') && (value > 0) && (Math.floor(value) === value) && value !== Infinity; }, "Please enter a valid number, 1 or greater"); +$.validator.addMethod("multiplecountnumber", function(value, element){ + if (this.optional(element) && value.length == 0) + return true; + if (isNaN(value)) + return false; + value = parseInt(value); + return (typeof value === 'number') && (value > 1) && (Math.floor(value) === value) && value !== Infinity; + +}, +"Please enter a valid number, 2 or greater"); + cloudStack.createTemplateMethod = function (isSnapshot){ return { label: 'label.create.template', From 906d69e0506df70afc8ae77ba06eb85cde07d539 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 2 Dec 2019 10:53:13 +0530 Subject: [PATCH 012/134] fix for cluster restart issue Signed-off-by: Abhishek Kumar --- .../KubernetesClusterManagerImpl.java | 2 +- .../src/main/resources/conf/k8s-master-add.yml | 14 ++++++++++++++ .../src/main/resources/conf/k8s-master.yml | 13 +++++++++++++ .../src/main/resources/conf/k8s-node.yml | 14 ++++++++++++++ 4 files changed, 42 insertions(+), 1 deletion(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java index 9bd9bca2ed09..f1155c527eef 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java @@ -1058,7 +1058,7 @@ private DeployDestination plan(final long nodesCount, final DataCenter zone, fin Float memoryOvercommitRatio = Float.parseFloat(cluster_detail_ram.getValue()); LOGGER.debug(String.format("Checking host ID: %s for capacity already reserved %d", h.getUuid(), reserved)); if (capacityManager.checkIfHostHasCapacity(h.getId(), cpu_requested * reserved, ram_requested * reserved, false, cpuOvercommitRatio, memoryOvercommitRatio, true)) { - LOGGER.debug(String.format("Found host ID: %s for with enough capacity, CPU=%d RAM=%d already reserved %d", cpu_requested * reserved, ram_requested * reserved)); + LOGGER.debug(String.format("Found host ID: %s for with enough capacity, CPU=%d RAM=%d", h.getUuid(), cpu_requested * reserved, ram_requested * reserved)); hostEntry.setValue(new Pair(h, reserved)); suitable_host_found = true; break; diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml index 710e1fece898..0d3de7d3e1e7 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml @@ -11,6 +11,11 @@ write-files: content: | #!/bin/bash -e + if [[ -f "/home/core/success" ]]; then + echo "Already provisioned!" + exit 0 + fi + export PATH=$PATH:/opt/bin ISO_MOUNT_DIR=/mnt/k8sdisk @@ -149,6 +154,12 @@ write-files: owner: root:root content: | #!/bin/bash -e + + if [[ -f "/home/core/success" ]]; then + echo "Already provisioned!" + exit 0 + fi + if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then echo "setup-kube-system is running!" exit 1 @@ -160,6 +171,9 @@ write-files: export PATH=$PATH:/opt/bin kubeadm join {{ k8s_master.join_ip }}:6443 --token {{ k8s_master.cluster.token }} --control-plane --certificate-key {{ k8s_master.cluster.ha.certificate.key }} --discovery-token-unsafe-skip-ca-verification + sudo touch /home/core/success + echo "true" > /home/core/success + coreos: units: - name: docker.service diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml index b27d9d8829d4..6696179f6147 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml @@ -31,6 +31,11 @@ write-files: content: | #!/bin/bash -e + if [[ -f "/home/core/success" ]]; then + echo "Already provisioned!" + exit 0 + fi + export PATH=$PATH:/opt/bin ISO_MOUNT_DIR=/mnt/k8sdisk @@ -187,6 +192,11 @@ write-files: content: | #!/bin/bash -e + if [[ -f "/home/core/success" ]]; then + echo "Already provisioned!" + exit 0 + fi + ISO_MOUNT_DIR=/mnt/k8sdisk BINARIES_DIR=${ISO_MOUNT_DIR}/ @@ -216,6 +226,9 @@ write-files: kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin --user=admin || true kubectl create clusterrolebinding kubernetes-dashboard-ui --clusterrole=cluster-admin --serviceaccount=kubernetes-dashboard:kubernetes-dashboard || true + sudo touch /home/core/success + echo "true" > /home/core/success + coreos: units: - name: docker.service diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index a89ea6553605..4f4ac73b11a2 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -11,6 +11,11 @@ write-files: content: | #!/bin/bash -e + if [[ -f "/home/core/success" ]]; then + echo "Already provisioned!" + exit 0 + fi + export PATH=$PATH:/opt/bin ISO_MOUNT_DIR=/mnt/k8sdisk @@ -149,6 +154,12 @@ write-files: owner: root:root content: | #!/bin/bash -e + + if [[ -f "/home/core/success" ]]; then + echo "Already provisioned!" + exit 0 + fi + if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then echo "setup-kube-system is running!" exit 1 @@ -160,6 +171,9 @@ write-files: export PATH=$PATH:/opt/bin kubeadm join {{ k8s_master.join_ip }}:6443 --token {{ k8s_master.cluster.token }} --discovery-token-unsafe-skip-ca-verification + sudo touch /home/core/success + echo "true" > /home/core/success + coreos: units: - name: docker.service From 7ce3104e2675ffe77bd0680c446fe445be83eb59 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 2 Dec 2019 15:00:10 +0530 Subject: [PATCH 013/134] fixed order of sql statements causing upgrade error Signed-off-by: Abhishek Kumar --- .../META-INF/db/schema-41300to41400.sql | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql b/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql index eb939adc2721..4fd33989869d 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql @@ -35,6 +35,21 @@ UPDATE `cloud`.`guest_os` SET `category_id`='4' WHERE `id`=285 AND display_name= UPDATE `cloud`.`guest_os` SET `category_id`='4' WHERE `id`=286 AND display_name="Red Hat Enterprise Linux 8.0"; -- Kubernetes service +CREATE TABLE IF NOT EXISTS `cloud`.`kubernetes_supported_version` ( + `id` bigint unsigned NOT NULL auto_increment COMMENT 'id', + `uuid` varchar(40) DEFAULT NULL COMMENT 'uuid', + `name` varchar(255) NOT NULL COMMENT 'kubernetes version name', + `kubernetes_version` varchar(32) NOT NULL COMMENT 'kubernetes semantic version', + `iso_id` bigint unsigned NOT NULL COMMENT 'kubernetes version binary ISO id', + `zone_id` bigint unsigned DEFAULT NULL COMMENT 'zone id in which kubernetes version is available', + `created` datetime NOT NULL COMMENT 'date created', + `removed` datetime COMMENT 'date removed if not null', + + PRIMARY KEY(`id`), + CONSTRAINT `fk_kubernetes_supported_version__iso_id` FOREIGN KEY `fk_kubernetes_supported_version__iso_id`(`iso_id`) REFERENCES `vm_template`(`id`) ON DELETE CASCADE, + CONSTRAINT `fk_kubernetes_supported_version__zone_id` FOREIGN KEY `fk_kubernetes_supported_version__zone_id`(`zone_id`) REFERENCES `data_center` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + CREATE TABLE IF NOT EXISTS `cloud`.`kubernetes_cluster` ( `id` bigint unsigned NOT NULL auto_increment COMMENT 'id', `uuid` varchar(40) DEFAULT NULL, @@ -88,21 +103,6 @@ CREATE TABLE IF NOT EXISTS `cloud`.`kubernetes_cluster_details` ( CONSTRAINT `fk_kubernetes_cluster_details__cluster_id` FOREIGN KEY `fk_kubernetes_cluster_details__cluster_id`(`cluster_id`) REFERENCES `kubernetes_cluster`(`id`) ON DELETE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8; -CREATE TABLE IF NOT EXISTS `cloud`.`kubernetes_supported_version` ( - `id` bigint unsigned NOT NULL auto_increment COMMENT 'id', - `uuid` varchar(40) DEFAULT NULL COMMENT 'uuid', - `name` varchar(255) NOT NULL COMMENT 'kubernetes version name', - `kubernetes_version` varchar(32) NOT NULL COMMENT 'kubernetes semantic version', - `iso_id` bigint unsigned NOT NULL COMMENT 'kubernetes version binary ISO id', - `zone_id` bigint unsigned DEFAULT NULL COMMENT 'zone id in which kubernetes version is available', - `created` datetime NOT NULL COMMENT 'date created', - `removed` datetime COMMENT 'date removed if not null', - - PRIMARY KEY(`id`), - CONSTRAINT `fk_kubernetes_supported_version__iso_id` FOREIGN KEY `fk_kubernetes_supported_version__iso_id`(`iso_id`) REFERENCES `vm_template`(`id`) ON DELETE CASCADE, - CONSTRAINT `fk_kubernetes_supported_version__zone_id` FOREIGN KEY `fk_kubernetes_supported_version__zone_id`(`zone_id`) REFERENCES `data_center` (`id`) ON DELETE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', 'cloud.kubernetes.cluster.template.name', "Kubernetes-Service-Template", 'Name of the template to be used for creating Kubernetes cluster nodes', 'Kubernetes-Service-Template', NULL, NULL, 0); From b7d24ed7eb3a97e15453a3154d7aa13d6c6b55a1 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 5 Dec 2019 12:12:12 +0530 Subject: [PATCH 014/134] server: added method for retrieving available public IP Signed-off-by: Abhishek Kumar --- .../com/cloud/network/IpAddressManager.java | 3 + .../cloud/network/IpAddressManagerImpl.java | 114 ++++++++++-------- 2 files changed, 69 insertions(+), 48 deletions(-) diff --git a/engine/components-api/src/main/java/com/cloud/network/IpAddressManager.java b/engine/components-api/src/main/java/com/cloud/network/IpAddressManager.java index a5512786bcd7..604e9c92603c 100644 --- a/engine/components-api/src/main/java/com/cloud/network/IpAddressManager.java +++ b/engine/components-api/src/main/java/com/cloud/network/IpAddressManager.java @@ -175,6 +175,9 @@ IpAddress allocateIp(Account ipOwner, boolean isSystem, Account caller, long cal PublicIp assignPublicIpAddressFromVlans(long dcId, Long podId, Account owner, VlanType type, List vlanDbIds, Long networkId, String requestedIp, boolean isSystem) throws InsufficientAddressCapacityException; + PublicIp getAvailablePublicIpAddressFromVlans(long dcId, Long podId, Account owner, VlanType type, List vlanDbIds, Long networkId, String requestedIp, boolean isSystem) + throws InsufficientAddressCapacityException; + @DB void allocateNicValues(NicProfile nic, DataCenter dc, VirtualMachineProfile vm, Network network, String requestedIpv4, String requestedIpv6) throws InsufficientVirtualNetworkCapacityException, InsufficientAddressCapacityException; diff --git a/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java b/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java index 817efcccb5ed..2ff632b68b0f 100644 --- a/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java +++ b/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java @@ -694,9 +694,23 @@ public PublicIp assignPublicIpAddressFromVlans(long dcId, Long podId, Account ow return fetchNewPublicIp(dcId, podId, vlanDbIds, owner, type, networkId, false, true, requestedIp, isSystem, null, null, false); } + @Override + public PublicIp getAvailablePublicIpAddressFromVlans(long dcId, Long podId, Account owner, VlanType type, List vlanDbIds, Long networkId, String requestedIp, boolean isSystem) + throws InsufficientAddressCapacityException { + return fetchNewPublicIp(dcId, podId, vlanDbIds, owner, type, networkId, false, false, false, requestedIp, isSystem, null, null, false); + } + + @DB + public PublicIp fetchNewPublicIp(final long dcId, final Long podId, final List vlanDbIds, final Account owner, final VlanType vlanUse, final Long guestNetworkId, + final boolean sourceNat, final boolean allocate, final String requestedIp, final boolean isSystem, final Long vpcId, final Boolean displayIp, final boolean forSystemVms) + throws InsufficientAddressCapacityException { + return fetchNewPublicIp(dcId, podId, vlanDbIds, owner, vlanUse, guestNetworkId, + sourceNat, true, allocate, requestedIp, isSystem, vpcId, displayIp, forSystemVms); + } + @DB public PublicIp fetchNewPublicIp(final long dcId, final Long podId, final List vlanDbIds, final Account owner, final VlanType vlanUse, final Long guestNetworkId, - final boolean sourceNat, final boolean assign, final String requestedIp, final boolean isSystem, final Long vpcId, final Boolean displayIp, final boolean forSystemVms) + final boolean sourceNat, final boolean assign, final boolean allocate, final String requestedIp, final boolean isSystem, final Long vpcId, final Boolean displayIp, final boolean forSystemVms) throws InsufficientAddressCapacityException { IPAddressVO addr = Transaction.execute(new TransactionCallbackWithException() { @Override @@ -808,63 +822,67 @@ public IPAddressVO doInTransaction(TransactionStatus status) throws Insufficient assert(addrs.size() == 1) : "Return size is incorrect: " + addrs.size(); - if (!fetchFromDedicatedRange && VlanType.VirtualNetwork.equals(vlanUse)) { - // Check that the maximum number of public IPs for the given accountId will not be exceeded - try { - _resourceLimitMgr.checkResourceLimit(owner, ResourceType.public_ip); - } catch (ResourceAllocationException ex) { - s_logger.warn("Failed to allocate resource of type " + ex.getResourceType() + " for account " + owner); - throw new AccountLimitException("Maximum number of public IP addresses for account: " + owner.getAccountName() + " has been exceeded."); - } - } - IPAddressVO finalAddr = null; - for (final IPAddressVO possibleAddr: addrs) { - if (possibleAddr.getState() != IpAddress.State.Free) { - continue; - } - final IPAddressVO addr = possibleAddr; - addr.setSourceNat(sourceNat); - addr.setAllocatedTime(new Date()); - addr.setAllocatedInDomainId(owner.getDomainId()); - addr.setAllocatedToAccountId(owner.getId()); - addr.setSystem(isSystem); - - if (displayIp != null) { - addr.setDisplay(displayIp); + if (assign) { + if (!fetchFromDedicatedRange && VlanType.VirtualNetwork.equals(vlanUse)) { + // Check that the maximum number of public IPs for the given accountId will not be exceeded + try { + _resourceLimitMgr.checkResourceLimit(owner, ResourceType.public_ip); + } catch (ResourceAllocationException ex) { + s_logger.warn("Failed to allocate resource of type " + ex.getResourceType() + " for account " + owner); + throw new AccountLimitException("Maximum number of public IP addresses for account: " + owner.getAccountName() + " has been exceeded."); + } } - if (vlanUse != VlanType.DirectAttached) { - addr.setAssociatedWithNetworkId(guestNetworkId); - addr.setVpcId(vpcId); - } - if (_ipAddressDao.lockRow(possibleAddr.getId(), true) != null) { - final IPAddressVO userIp = _ipAddressDao.findById(addr.getId()); - if (userIp.getState() == IpAddress.State.Free) { - addr.setState(IpAddress.State.Allocating); - if (_ipAddressDao.update(addr.getId(), addr)) { - finalAddr = addr; - break; + for (final IPAddressVO possibleAddr : addrs) { + if (possibleAddr.getState() != IpAddress.State.Free) { + continue; + } + final IPAddressVO addr = possibleAddr; + addr.setSourceNat(sourceNat); + addr.setAllocatedTime(new Date()); + addr.setAllocatedInDomainId(owner.getDomainId()); + addr.setAllocatedToAccountId(owner.getId()); + addr.setSystem(isSystem); + + if (displayIp != null) { + addr.setDisplay(displayIp); + } + + if (vlanUse != VlanType.DirectAttached) { + addr.setAssociatedWithNetworkId(guestNetworkId); + addr.setVpcId(vpcId); + } + if (_ipAddressDao.lockRow(possibleAddr.getId(), true) != null) { + final IPAddressVO userIp = _ipAddressDao.findById(addr.getId()); + if (userIp.getState() == IpAddress.State.Free) { + addr.setState(IpAddress.State.Allocating); + if (_ipAddressDao.update(addr.getId(), addr)) { + finalAddr = addr; + break; + } } } } - } - if (finalAddr == null) { - s_logger.error("Failed to fetch any free public IP address"); - throw new CloudRuntimeException("Failed to fetch any free public IP address"); - } + if (finalAddr == null) { + s_logger.error("Failed to fetch any free public IP address"); + throw new CloudRuntimeException("Failed to fetch any free public IP address"); + } - if (assign) { - markPublicIpAsAllocated(finalAddr); - } + if (allocate) { + markPublicIpAsAllocated(finalAddr); + } - final State expectedAddressState = assign ? State.Allocated : State.Allocating; - if (finalAddr.getState() != expectedAddressState) { - s_logger.error("Failed to fetch new public IP and get in expected state=" + expectedAddressState); - throw new CloudRuntimeException("Failed to fetch new public IP with expected state " + expectedAddressState); - } + final State expectedAddressState = allocate ? State.Allocated : State.Allocating; + if (finalAddr.getState() != expectedAddressState) { + s_logger.error("Failed to fetch new public IP and get in expected state=" + expectedAddressState); + throw new CloudRuntimeException("Failed to fetch new public IP with expected state " + expectedAddressState); + } + } else { + finalAddr = addrs.get(0); + } return finalAddr; } }); From b099b1764951e50a2196a9526989f150fd280fd1 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 5 Dec 2019 12:25:52 +0530 Subject: [PATCH 015/134] fixes Made service pugin configurable using global setting Shared network support changes Added plugin UI strings in English trasaltion file Set minimum Kubernetes version to 1.11 State scanner improvements Code structure, logging related refactorings Signed-off-by: Abhishek Kumar --- .../apache/cloudstack/api/ApiConstants.java | 1 + .../user/config/ListCapabilitiesCmd.java | 1 + .../api/response/CapabilitiesResponse.java | 8 + .../META-INF/db/schema-41300to41400.sql | 12 +- .../KubernetesClusterManagerImpl.java | 1093 ++++++++++------- .../KubernetesClusterService.java | 1 + .../KubernetesServiceConfig.java | 5 +- .../dao/KubernetesClusterDaoImpl.java | 2 +- .../KubernetesVersionManagerImpl.java | 28 +- .../KubernetesVersionService.java | 1 + .../CreateKubernetesClusterCmd.java | 8 + .../resources/script/upgrade-kubernetes.sh | 32 +- .../cloud/server/ManagementServerImpl.java | 5 +- ui/l10n/en.js | 34 + ui/plugins/cks/cks.js | 141 ++- 15 files changed, 851 insertions(+), 521 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 5efb70ae31df..ab47971c5469 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -745,6 +745,7 @@ public class ApiConstants { public static final String CONSOLE_END_POINT = "consoleendpoint"; public static final String DELETE_ISO = "deleteiso"; + public static final String EXTERNAL_LOAD_BALANCER_IP_ADDRESS = "externalloadbalanceripaddress"; public static final String DOCKER_REGISTRY_USER_NAME = "dockerregistryusername"; public static final String DOCKER_REGISTRY_PASSWORD = "dockerregistrypassword"; public static final String DOCKER_REGISTRY_URL = "dockerregistryurl"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/config/ListCapabilitiesCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/config/ListCapabilitiesCmd.java index 40d1a71e9662..948097a15ffe 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/config/ListCapabilitiesCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/config/ListCapabilitiesCmd.java @@ -60,6 +60,7 @@ public void execute() { response.setAllowUserViewDestroyedVM((Boolean)capabilities.get("allowUserViewDestroyedVM")); response.setAllowUserExpungeRecoverVM((Boolean)capabilities.get("allowUserExpungeRecoverVM")); response.setAllowUserViewAllDomainAccounts((Boolean)capabilities.get("allowUserViewAllDomainAccounts")); + response.setKubernetesServiceEnabled((Boolean)capabilities.get("kubernetesServiceEnabled")); if (capabilities.containsKey("apiLimitInterval")) { response.setApiLimitInterval((Integer)capabilities.get("apiLimitInterval")); } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/CapabilitiesResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/CapabilitiesResponse.java index 153d7dfca9ae..6646437fd64f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/CapabilitiesResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/CapabilitiesResponse.java @@ -88,6 +88,10 @@ public class CapabilitiesResponse extends BaseResponse { @Param(description = "true if users can see all accounts within the same domain, false otherwise") private boolean allowUserViewAllDomainAccounts; + @SerializedName("kubernetesserviceenabled") + @Param(description = "true if Kubernetes Service plugin is enabled, false otherwise") + private boolean kubernetesServiceEnabled; + public void setSecurityGroupsEnabled(boolean securityGroupsEnabled) { this.securityGroupsEnabled = securityGroupsEnabled; } @@ -151,4 +155,8 @@ public void setAllowUserExpungeRecoverVM(boolean allowUserExpungeRecoverVM) { public void setAllowUserViewAllDomainAccounts(boolean allowUserViewAllDomainAccounts) { this.allowUserViewAllDomainAccounts = allowUserViewAllDomainAccounts; } + + public void setKubernetesServiceEnabled(boolean kubernetesServiceEnabled) { + this.kubernetesServiceEnabled = kubernetesServiceEnabled; + } } \ No newline at end of file diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql b/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql index 4fd33989869d..cadf4d2172b2 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql @@ -35,6 +35,9 @@ UPDATE `cloud`.`guest_os` SET `category_id`='4' WHERE `id`=285 AND display_name= UPDATE `cloud`.`guest_os` SET `category_id`='4' WHERE `id`=286 AND display_name="Red Hat Enterprise Linux 8.0"; -- Kubernetes service +INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', +'cloud.kubernetes.service.enabled', 'false', 'Indicates whether Kubernetes Service plugin is enabled or not. Management server restart needed on change', 'false', NULL, NULL, 0); + CREATE TABLE IF NOT EXISTS `cloud`.`kubernetes_supported_version` ( `id` bigint unsigned NOT NULL auto_increment COMMENT 'id', `uuid` varchar(40) DEFAULT NULL COMMENT 'uuid', @@ -106,15 +109,6 @@ CREATE TABLE IF NOT EXISTS `cloud`.`kubernetes_cluster_details` ( INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', 'cloud.kubernetes.cluster.template.name', "Kubernetes-Service-Template", 'Name of the template to be used for creating Kubernetes cluster nodes', 'Kubernetes-Service-Template', NULL, NULL, 0); -INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', -'cloud.kubernetes.cluster.binaries.iso.name', 'Kubernetes-Service-Binaries-ISO' , 'Name of the ISO that contains Kubernetes binaries and docker images for offline installation.', 'Kubernetes-Service-Binaries-ISO', NULL , NULL, 0); - -INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', -'cloud.kubernetes.cluster.master.cloudconfig', '/etc/cloudstack/management/k8s-master.yml' , 'file location path of the cloud config used for creating kubernetes cluster master node', '/etc/cloudstack/management/k8s-master.yml', NULL , NULL, 0); - -INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', -'cloud.kubernetes.cluster.node.cloudconfig', '/etc/cloudstack/management/k8s-node.yml', 'file location path of the cloud config used for creating kubernetes cluster node', '/etc/cloudstack/management/k8s-node.yml', NULL , NULL, 0); - INSERT IGNORE INTO `cloud`.`network_offerings` (name, uuid, unique_name, display_text, nw_rate, mc_rate, traffic_type, tags, system_only, specify_vlan, service_offering_id, conserve_mode, created,availability, dedicated_lb_service, shared_source_nat_service, sort_key, redundant_router_service, state, guest_type, elastic_ip_service, eip_associate_public_ip, elastic_lb_service, specify_ip_ranges, inline,is_persistent,internal_lb, public_lb, egress_default_policy, concurrent_connections, keep_alive_enabled, supports_streched_l2, `default`, removed) VALUES ('DefaultNetworkOfferingforKubernetesService', UUID(), 'DefaultNetworkOfferingforKubernetesService', 'Network Offering used for CloudStack kubernetes service', NULL,NULL,'Guest',NULL,0,0,NULL,1,now(),'Required',1,0,0,0,'Enabled','Isolated',0,1,0,0,0,0,0,1,1,NULL,0,0,0,NULL); UPDATE `cloud`.`network_offerings` SET removed=NULL WHERE unique_name='DefaultNetworkOfferingforKubernetesService'; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java index f1155c527eef..debdffe6bad5 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java @@ -87,8 +87,13 @@ import com.cloud.dc.ClusterVO; import com.cloud.dc.DataCenter; import com.cloud.dc.DataCenterVO; +import com.cloud.dc.Pod; +import com.cloud.dc.Vlan; +import com.cloud.dc.VlanVO; import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.DataCenterDao; +import com.cloud.dc.dao.HostPodDao; +import com.cloud.dc.dao.VlanDao; import com.cloud.deploy.DeployDestination; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; @@ -116,6 +121,7 @@ import com.cloud.network.NetworkModel; import com.cloud.network.NetworkService; import com.cloud.network.PhysicalNetwork; +import com.cloud.network.addr.PublicIp; import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.IPAddressVO; @@ -190,8 +196,6 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne protected StateMachine2 _stateMachine = KubernetesCluster.State.getStateMachine(); - public static final String MIN_KUBERNETES_VERSION_HA_SUPPORT = "1.16"; - ScheduledExecutorService _gcExecutor; ScheduledExecutorService _stateScanner; @@ -268,7 +272,11 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne @Inject protected IpAddressManager ipAddressManager; @Inject - public LoadBalancingRulesService lbService; + protected LoadBalancingRulesService lbService; + @Inject + protected VlanDao vlanDao; + @Inject + protected HostPodDao hostPodDao; private static String getStackTrace(final Throwable throwable) { final StringWriter sw = new StringWriter(); @@ -323,10 +331,18 @@ private boolean isKubernetesServiceConfigured(DataCenter zone) { LOGGER.warn(String.format("Unable to find physical network with tag: %s", networkOffering.getTags())); return false; } - return true; } + private File getManagementServerSshPublicKeyFile() { + boolean devel = Boolean.parseBoolean(globalConfigDao.getValue("developer")); + String keyFile = String.format("%s/.ssh/id_rsa", System.getProperty("user.home")); + if (devel) { + keyFile += ".cloud"; + } + return new File(keyFile); + } + private String generateClusterToken(KubernetesCluster kubernetesCluster) { if (kubernetesCluster == null) return ""; String token = kubernetesCluster.getUuid(); @@ -357,9 +373,171 @@ public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { }); } - @Override - public KubernetesCluster findById(final Long id) { - return kubernetesClusterDao.findById(id); + private boolean isKubernetesClusterServerRunning(KubernetesCluster kubernetesCluster, String ipAddress, int retries, long waitDuration) { + int retryCounter = 0; + boolean k8sApiServerSetup = false; + while (retryCounter < retries) { + try { + String versionOutput = IOUtils.toString(new URL(String.format("https://%s:%d/version", ipAddress, 6443)), StandardCharsets.UTF_8); + if (!Strings.isNullOrEmpty(versionOutput)) { + LOGGER.debug(String.format("Kubernetes cluster ID: %s API has been successfully provisioned, %s", kubernetesCluster.getUuid(), versionOutput)); + k8sApiServerSetup = true; + break; + } + } catch (Exception e) { + LOGGER.warn(String.format("API endpoint for Kubernetes cluster ID: %s not available. Attempt: %d/%d", kubernetesCluster.getUuid(), retryCounter+1, retries), e); + } + try { + Thread.sleep(waitDuration); + } catch (InterruptedException ie) { + LOGGER.error(String.format("Error while waiting for Kubernetes cluster ID: %s API endpoint to be available", kubernetesCluster.getUuid()), ie); + } + retryCounter++; + } + return k8sApiServerSetup; + } + + private String getKubernetesClusterConfig(KubernetesCluster kubernetesCluster, String ipAddress, int port, int retries) { + int retryCounter = 0; + String kubeConfig = ""; + while (retryCounter < retries) { + try { + Pair result = SshHelper.sshExecute(ipAddress, port, "core", + getManagementServerSshPublicKeyFile(), null, "sudo cat /etc/kubernetes/admin.conf", + 10000, 10000, 10000); + + if (result.first() && !Strings.isNullOrEmpty(result.second())) { + kubeConfig = result.second(); + break; + } + } catch (Exception e) { + LOGGER.warn(String.format("Failed to retrieve kube-config file for Kubernetes cluster ID: %s. Attempt: %d/%d", kubernetesCluster.getUuid(), retryCounter+1, retries), e); + } + retryCounter++; + } + return kubeConfig; + } + + private boolean isDashboardServiceRunning(KubernetesCluster kubernetesCluster, String ipAddress, int port, int retries, long waitDuration) { + boolean running = false; + int retryCounter = 0; + // Check if dashboard service is up running. + while (retryCounter < retries) { + LOGGER.debug(String.format("Checking dashboard service for the Kubernetes cluster ID: %s to come up. Attempt: %d/%d", kubernetesCluster.getUuid(), retryCounter+1, retries)); + if (isAddOnServiceRunning(kubernetesCluster, ipAddress, port, "kubernetes-dashboard", "kubernetes-dashboard")) { + running = true; + break; + } + try { + Thread.sleep(waitDuration); + } catch (InterruptedException ex) { + LOGGER.error(String.format("Error while waiting for Kubernetes cluster: %s API dashboard service to be available", kubernetesCluster.getUuid()), ex); + } + retryCounter++; + } + return running; + } + + private Pair getKubernetesClusterServerIpSshPort(KubernetesCluster kubernetesCluster, UserVm masterVm) { + int port = 2222; + KubernetesClusterDetailsVO detail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS); + if (detail != null && !Strings.isNullOrEmpty(detail.getValue())) { + return new Pair<>(detail.getValue(), port); + } + Network network = networkDao.findById(kubernetesCluster.getNetworkId()); + if (network == null) { + LOGGER.warn(String.format("Network for Kubernetes cluster ID: %s cannot be found", kubernetesCluster.getUuid())); + return new Pair<>(null, port); + } + if (Network.GuestType.Isolated.equals(network.getGuestType())) { + List addresses = networkModel.listPublicIpsAssignedToGuestNtwk(network.getId(), true); + if (CollectionUtils.isEmpty(addresses)) { + LOGGER.warn(String.format("No public IP addresses found for network ID: %s, Kubernetes cluster ID: %s", network.getUuid(), kubernetesCluster.getUuid())); + return new Pair<>(null, port); + } + for (IpAddress address : addresses) { + if (address.isSourceNat()) { + return new Pair<>(address.getAddress().addr(), port); + } + } + LOGGER.warn(String.format("No source NAT IP addresses found for network ID: %s, Kubernetes cluster ID: %s", network.getUuid(), kubernetesCluster.getUuid())); + return new Pair<>(null, port); + } else if (Network.GuestType.Shared.equals(network.getGuestType())) { + port = 22; + if (masterVm == null) { + List clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + if (CollectionUtils.isEmpty(clusterVMs)) { + LOGGER.warn(String.format("Unable to retrieve VMs for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + return new Pair<>(null, port); + } + List vmIds = new ArrayList<>(); + for (KubernetesClusterVmMapVO vmMap : clusterVMs) { + vmIds.add(vmMap.getVmId()); + } + Collections.sort(vmIds); + masterVm = userVmDao.findById(vmIds.get(0)); + } + if (masterVm == null) { + LOGGER.warn(String.format("Unable to retrieve master VM for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + return new Pair<>(null, port); + } + return new Pair<>(masterVm.getPrivateIpAddress(), port); + } + LOGGER.warn(String.format("Unable to retrieve server IP address for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + return new Pair<>(null, port); + } + + private Pair getKubernetesClusterServerIpSshPort(KubernetesCluster kubernetesCluster) { + return getKubernetesClusterServerIpSshPort(kubernetesCluster, null); + } + + private int getKubernetesClusterReadyNodesCount(KubernetesCluster kubernetesCluster, String ipAddress, int port) throws Exception { + Pair result = SshHelper.sshExecute(ipAddress, port, + "core", getManagementServerSshPublicKeyFile(), null, + "sudo kubectl get nodes | awk '{if ($2 == \"Ready\") print $1}' | wc -l", + 10000, 10000, 20000); + if (result.first()) { + return Integer.parseInt(result.second().trim().replace("\"", "")); + } + return 0; + } + + private int getKubernetesClusterReadyNodesCount(KubernetesCluster kubernetesCluster) throws Exception { + Pair ipSshPort = getKubernetesClusterServerIpSshPort(kubernetesCluster); + String ipAddress = ipSshPort.first(); + int sshPort = ipSshPort.second(); + if (Strings.isNullOrEmpty(ipAddress)) { + String msg = String.format("No public IP found for Kubernetes cluster ID: %s" , kubernetesCluster.getUuid()); + LOGGER.warn(msg); + throw new ManagementServerException(msg); + } + return getKubernetesClusterReadyNodesCount(kubernetesCluster, ipAddress, sshPort); + } + + private boolean validateKubernetesClusterReadyNodesCount(KubernetesCluster kubernetesCluster, String ipAddress, int port, int retries, long waitDuration) { + int retryCounter = 0; + while (retryCounter < retries) { + // "sudo kubectl get nodes -o json | jq \".items[].metadata.name\" | wc -l" + LOGGER.debug(String.format("Checking ready nodes for the Kubernetes cluster ID: %s with total %d provisioned nodes. Attempt: %d/%d", kubernetesCluster.getUuid(), kubernetesCluster.getTotalNodeCount(), retryCounter+1, retries)); + try { + int nodesCount = getKubernetesClusterReadyNodesCount(kubernetesCluster, ipAddress, port); + if (nodesCount == kubernetesCluster.getTotalNodeCount()) { + LOGGER.debug(String.format("Kubernetes cluster ID: %s has %d ready now", kubernetesCluster.getUuid(), kubernetesCluster.getTotalNodeCount())); + return true; + } else { + LOGGER.debug(String.format("Kubernetes cluster ID: %s has total %d provisioned nodes while %d ready now", kubernetesCluster.getUuid(), kubernetesCluster.getTotalNodeCount(), nodesCount)); + } + } catch (Exception e) { + LOGGER.warn(String.format("Failed to retrieve ready node count for Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); + } + try { + Thread.sleep(waitDuration); + } catch (InterruptedException ex) { + LOGGER.warn(String.format("Error while waiting during Kubernetes cluster ID: %s ready node check. %d/%d", kubernetesCluster.getUuid(), retryCounter+1, retries), ex); + } + retryCounter++; + } + return false; } // perform a cold start (which will provision resources as well) @@ -376,16 +554,12 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t // - update API and dashboard URL endpoints in Kubernetes cluster details KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); - final DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); if (zone == null) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Unable to find zone for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); } - LOGGER.debug(String.format("Starting Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.StartRequested); - Account account = accountDao.findById(kubernetesCluster.getAccountId()); DeployDestination dest = null; @@ -416,21 +590,21 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t throw new ManagementServerException(msg, e); } - IPAddressVO publicIp = null; - List ips = ipAddressDao.listByAssociatedNetwork(kubernetesCluster.getNetworkId(), true); - if (ips == null || ips.isEmpty()) { - String msg = String.format("Failed to start Kubernetes cluster ID: %s as no public IP found for associated network ID: %s" , kubernetesCluster.getUuid(), network.getUuid()); + Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(kubernetesCluster); + String publicIpAddress = publicIpSshPort.first(); + if (Strings.isNullOrEmpty(publicIpAddress) && + (Network.GuestType.Isolated.equals(network.getGuestType()) || kubernetesCluster.getMasterNodeCount() > 1)) { // Shared network, single-master cluster won't have an IP yet + String msg = String.format("Failed to start Kubernetes cluster ID: %s as no public IP found for the cluster" , kubernetesCluster.getUuid()); LOGGER.warn(msg); stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); throw new ManagementServerException(msg); } - publicIp = ips.get(0); List clusterVMIds = new ArrayList<>(); UserVm k8sMasterVM = null; try { - k8sMasterVM = createKubernetesMaster(kubernetesCluster, ips); + k8sMasterVM = createKubernetesMaster(kubernetesCluster, dest.getPod(), network, account, publicIpAddress); addKubernetesClusterVm(kubernetesCluster.getId(), k8sMasterVM.getId()); startKubernetesVM(k8sMasterVM, kubernetesCluster); clusterVMIds.add(k8sMasterVM.getId()); @@ -443,13 +617,22 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t throw new ManagementServerException(msg, e); } - String masterIP = k8sMasterVM.getPrivateIpAddress(); + if (Strings.isNullOrEmpty(publicIpAddress)) { + publicIpSshPort = getKubernetesClusterServerIpSshPort(kubernetesCluster, k8sMasterVM); + publicIpAddress = publicIpSshPort.first(); + if (Strings.isNullOrEmpty(publicIpAddress)) { + String msg = String.format("Failed to start Kubernetes cluster ID: %s as no public IP found for the cluster", kubernetesCluster.getUuid()); + LOGGER.warn(msg); + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + throw new ManagementServerException(msg); + } + } if (kubernetesCluster.getMasterNodeCount() > 1) { for (int i = 1; i < kubernetesCluster.getMasterNodeCount(); i++) { UserVm vm = null; try { - vm = createKubernetesAdditionalMaster(kubernetesCluster, publicIp.getAddress().toString(), i); + vm = createKubernetesAdditionalMaster(kubernetesCluster, publicIpAddress, i); addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId()); startKubernetesVM(vm, kubernetesCluster); clusterVMIds.add(vm.getId()); @@ -466,7 +649,7 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t for (int i = 1; i <= kubernetesCluster.getNodeCount(); i++) { UserVm vm = null; try { - vm = createKubernetesNode(kubernetesCluster, masterIP, i); + vm = createKubernetesNode(kubernetesCluster, publicIpAddress, i); addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId()); startKubernetesVM(vm, kubernetesCluster); clusterVMIds.add(vm.getId()); @@ -480,127 +663,71 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t } LOGGER.debug(String.format("Kubernetes cluster ID: %s VMs successfully provisioned", kubernetesCluster.getUuid())); - setupKubernetesClusterNetworkRules(publicIp, account, kubernetesClusterId, clusterVMIds); + setupKubernetesClusterNetworkRules(kubernetesCluster, network, account, clusterVMIds); attachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); - int retryCounter = 0; - int maxRetries = 15; - boolean k8sApiServerSetup = false; - while (retryCounter < maxRetries) { - try { - String versionOutput = IOUtils.toString(new URL(String.format("https://%s:%d/version", publicIp.getAddress().addr(), 6443)), StandardCharsets.UTF_8); - if (!Strings.isNullOrEmpty(versionOutput)) { - LOGGER.debug(String.format("Kubernetes cluster ID: %s API has been successfully provisioned, %s", kubernetesCluster.getUuid(), versionOutput)); - k8sApiServerSetup = true; - kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); - kubernetesCluster.setEndpoint(String.format("https://%s:%d/", publicIp.getAddress().addr(), 6443)); - kubernetesClusterDao.update(kubernetesCluster.getId(), kubernetesCluster); - break; - } - } catch (Exception e) { - LOGGER.warn(String.format("API endpoint for Kubernetes cluster ID: %s not available. Attempt: %d/%d", kubernetesCluster.getUuid(), retryCounter+1, maxRetries), e); - } - try { - Thread.sleep(30000); - } catch (InterruptedException ie) { - LOGGER.error(String.format("Error while waiting for Kubernetes cluster ID: %s API endpoint to be available", kubernetesCluster.getUuid()), ie); - } - retryCounter++; - } - + boolean k8sApiServerSetup = isKubernetesClusterServerRunning(kubernetesCluster, publicIpAddress,15, 30000); if (!k8sApiServerSetup) { - LOGGER.error(String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to provision API endpoint for the cluster", kubernetesCluster.getUuid())); + String msg = String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to provision API endpoint for the cluster", kubernetesCluster.getUuid()); + LOGGER.error(msg); + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + detachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } + kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + kubernetesCluster.setEndpoint(String.format("https://%s:%d/", publicIpAddress, 6443)); + kubernetesClusterDao.update(kubernetesCluster.getId(), kubernetesCluster); + int sshPort = publicIpSshPort.second(); boolean k8sKubeConfigCopied = false; - if (k8sApiServerSetup) { - retryCounter = 0; - maxRetries = 5; - String kubeConfig = ""; - while (retryCounter < maxRetries && kubeConfig.isEmpty()) { - try { - Boolean devel = Boolean.valueOf(globalConfigDao.getValue("developer")); - String keyFile = String.format("%s/.ssh/id_rsa", System.getProperty("user.home")); - if (devel) { - keyFile += ".cloud"; - } - File pkFile = new File(keyFile); - Pair result = SshHelper.sshExecute(publicIp.getAddress().addr(), 2222, "core", - pkFile, null, "sudo cat /etc/kubernetes/admin.conf", - 10000, 10000, 10000); - - if (result.first() && !Strings.isNullOrEmpty(result.second())) { - kubeConfig = result.second(); - kubeConfig = kubeConfig.replace(String.format("server: https://%s:6443", k8sMasterVM.getPrivateIpAddress()), - String.format("server: https://%s:6443", publicIp.getAddress().addr())); - kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "kubeConfigData", Base64.encodeBase64String(kubeConfig.getBytes(Charset.forName("UTF-8"))), false); - k8sKubeConfigCopied = true; - LOGGER.debug(String.format("Kubernetes cluster ID: %s kube-config has been successdully retrieved", kubernetesCluster.getUuid())); - break; - } - } catch (Exception e) { - LOGGER.warn(String.format("Failed to retrieve kube-config file for Kubernetes cluster ID: %s. Attempt: %d/%d", kubernetesCluster.getUuid(), retryCounter+1, maxRetries), e); - } - retryCounter++; - } - if (!k8sKubeConfigCopied) { - LOGGER.error(String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to retrieve kube-config for the cluster", kubernetesCluster.getUuid())); - } + String kubeConfig = getKubernetesClusterConfig(kubernetesCluster, publicIpAddress, sshPort, 5); + if (!Strings.isNullOrEmpty(kubeConfig)) { + k8sKubeConfigCopied = true; } - - if (k8sKubeConfigCopied) { - retryCounter = 0; - maxRetries = 30; - // Check if dashboard service is up running. - while (retryCounter < maxRetries) { - LOGGER.debug(String.format("Checking dashboard service for the Kubernetes cluster ID: %s to come up. Attempt: %d/%d", kubernetesCluster.getUuid(), retryCounter+1, maxRetries)); - if (isAddOnServiceRunning(kubernetesCluster.getId(), "kubernetes-dashboard", "kubernetes-dashboard")) { - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationSucceeded); - kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); - kubernetesCluster.setConsoleEndpoint("https://" + publicIp.getAddress() + ":6443/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy#!/overview?namespace=_all"); - kubernetesClusterDao.update(kubernetesCluster.getId(), kubernetesCluster); - detachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); - LOGGER.debug(String.format("Kubernetes cluster ID: %s is successfully started", kubernetesCluster.getUuid())); - return true; - } - try { - Thread.sleep(10000); - } catch (InterruptedException ex) { - LOGGER.error(String.format("Error while waiting for Kubernetes cluster: %s API dashboard service to be available", kubernetesCluster.getUuid()), ex); - } - retryCounter++; - } - LOGGER.warn("Failed to setup Kubernetes cluster " + kubernetesCluster.getName() + " in usable state as" + - " unable to bring dashboard add on service up"); + if (!k8sKubeConfigCopied) { + String msg = String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to retrieve kube-config for the cluster", kubernetesCluster.getUuid()); + LOGGER.error(msg); + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + detachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } + kubeConfig = kubeConfig.replace(String.format("server: https://%s:6443", k8sMasterVM.getPrivateIpAddress()), + String.format("server: https://%s:6443", publicIpAddress)); + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "kubeConfigData", Base64.encodeBase64String(kubeConfig.getBytes(Charset.forName("UTF-8"))), false); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + boolean dashboardServiceRunning = isDashboardServiceRunning(kubernetesCluster, publicIpAddress, sshPort, 10, 20000); + if (!dashboardServiceRunning) { + String msg = String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to get Dashboard service running for the cluster", kubernetesCluster.getUuid()); + LOGGER.error(msg); + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + detachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); + } detachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to deploy Kubernetes cluster ID: %s as unable to setup up in usable state", kubernetesCluster.getUuid())); + kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + kubernetesCluster.setConsoleEndpoint("https://" + publicIpAddress + ":6443/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy#!/overview?namespace=_all"); + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationSucceeded); + kubernetesClusterDao.update(kubernetesCluster.getId(), kubernetesCluster); + return true; } private boolean startStoppedKubernetesCluster(long kubernetesClusterId) throws ManagementServerException, ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException { - final KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); if (kubernetesCluster == null) { throw new ManagementServerException("Invalid Kubernetes cluster ID"); } - if (kubernetesCluster.getRemoved() != null) { throw new ManagementServerException(String.format("Kubernetes cluster ID: %s is already deleted", kubernetesCluster.getUuid())); } - if (kubernetesCluster.getState().equals(KubernetesCluster.State.Running)) { LOGGER.debug(String.format("Kubernetes cluster ID: %s is in running state", kubernetesCluster.getUuid())); return true; } - if (kubernetesCluster.getState().equals(KubernetesCluster.State.Starting)) { LOGGER.debug(String.format("Kubernetes cluster ID: %s is already in starting state", kubernetesCluster.getUuid())); return true; } - LOGGER.debug(String.format("Starting Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.StartRequested); @@ -631,45 +758,56 @@ private boolean startStoppedKubernetesCluster(long kubernetesClusterId) throws M try { address = InetAddress.getByName(new URL(kubernetesCluster.getEndpoint()).getHost()); } catch (MalformedURLException | UnknownHostException ex) { - String msg = String.format("Kubernetes cluster ID: %s has invalid api endpoint. Can not verify if cluster is in ready state", kubernetesCluster.getUuid()); + String msg = String.format("Kubernetes cluster ID: %s has invalid API endpoint. Can not verify if cluster is in ready state", kubernetesCluster.getUuid()); LOGGER.warn(msg, ex); throw new ManagementServerException(msg, ex); } - IPAddressVO publicIp = null; - List ips = ipAddressDao.listByAssociatedNetwork(kubernetesCluster.getNetworkId(), true); - if (ips == null || ips.isEmpty() || ips.get(0) == null) { - String msg = String.format("Failed to start Kubernetes cluster ID: %s, unable to retrieve associated public IP", kubernetesCluster.getUuid()); + Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(kubernetesCluster); + String publicIpAddress = publicIpSshPort.first(); + if (Strings.isNullOrEmpty(publicIpAddress)) { + String msg = String.format("Failed to start Kubernetes cluster ID: %s as no public IP found for the cluster" , kubernetesCluster.getUuid()); + LOGGER.warn(msg); + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + throw new ManagementServerException(msg); + } + + boolean k8sApiServerSetup = isKubernetesClusterServerRunning(kubernetesCluster, publicIpAddress, 10, 30000); + if (!k8sApiServerSetup) { + String msg = String.format("Failed to start Kubernetes cluster ID: %s in usable state", kubernetesCluster.getUuid()); LOGGER.error(msg); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + throw new ManagementServerException(msg); } - publicIp = ips.get(0); - int retryCounter = 0; - int maxRetries = 10; - boolean k8sApiServerSetup = false; - while (retryCounter < maxRetries) { - try { - String versionOutput = IOUtils.toString(new URL(String.format("https://%s:%d/version", publicIp.getAddress().addr(), 6443)), StandardCharsets.UTF_8); - if (!Strings.isNullOrEmpty(versionOutput)) { - LOGGER.debug(String.format("Kubernetes cluster ID: %s API has been successfully provisioned, %s", kubernetesCluster.getUuid(), versionOutput)); - k8sApiServerSetup = true; - break; - } - } catch (Exception e) { - LOGGER.warn(String.format("API endpoint for Kubernetes cluster ID: %s not available. Attempt: %d/%d", kubernetesCluster.getUuid(), retryCounter+1, maxRetries), e); + int sshPort = publicIpSshPort.second(); + KubernetesClusterDetailsVO kubeConfigDetail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "kubeConfigData"); + if (kubeConfigDetail == null || Strings.isNullOrEmpty(kubeConfigDetail.getValue())) { + boolean k8sKubeConfigCopied = false; + String kubeConfig = getKubernetesClusterConfig(kubernetesCluster, publicIpAddress, sshPort, 5); + if (!Strings.isNullOrEmpty(kubeConfig)) { + k8sKubeConfigCopied = true; } - try { - Thread.sleep(30000); - } catch (InterruptedException ie) { - LOGGER.error(String.format("Error while waiting for Kubernetes cluster ID: %s API endpoint to be available", kubernetesCluster.getUuid()), ie); + if (!k8sKubeConfigCopied) { + String msg = String.format("Failed to start Kubernetes cluster ID: %s in usable state as unable to retrieve kube-config for the cluster", kubernetesCluster.getUuid()); + LOGGER.error(msg); + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } - retryCounter++; + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "kubeConfigData", Base64.encodeBase64String(kubeConfig.getBytes(Charset.forName("UTF-8"))), false); } - if (!k8sApiServerSetup) { - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); - throw new ManagementServerException(String.format("Failed to setup Kubernetes cluster ID: %s is usable state", kubernetesCluster.getUuid())); + if (Strings.isNullOrEmpty(kubernetesCluster.getConsoleEndpoint())) { + boolean dashboardServiceRunning = isDashboardServiceRunning(kubernetesCluster, publicIpAddress, sshPort, 10, 20000); + if (!dashboardServiceRunning) { + String msg = String.format("Failed to start Kubernetes cluster ID: %s in usable state as unable to get Dashboard service running for the cluster", kubernetesCluster.getUuid()); + LOGGER.error(msg); + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); + } + KubernetesClusterVO cluster = kubernetesClusterDao.findById(kubernetesCluster.getId()); + cluster.setConsoleEndpoint("https://" + publicIpAddress + ":6443/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy#!/overview?namespace=_all"); + kubernetesClusterDao.update(cluster.getId(), cluster); } stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationSucceeded); @@ -681,11 +819,29 @@ private boolean startStoppedKubernetesCluster(long kubernetesClusterId) throws M // rule to forward public IP traffic to master VM private IP // Open up firewall ports 2222 to 2222+n for SSH access. Also create port-forwarding // rule to forward public IP traffic to all node VM private IP - private void setupKubernetesClusterNetworkRules(IPAddressVO publicIp, Account account, long kubernetesClusterId, + private void setupKubernetesClusterNetworkRules(KubernetesCluster kubernetesCluster, + Network network, Account account, List clusterVMIds) throws ManagementServerException { - - KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); - + if (!Network.GuestType.Isolated.equals(network.getGuestType())) { + LOGGER.debug(String.format("Network ID: %s for Kubernetes cluster ID: %s is not an isolated network, therefore, no need for network rules", network.getUuid(), kubernetesCluster.getUuid())); + return; + } + IpAddress publicIp = null; + List addresses = networkModel.listPublicIpsAssignedToGuestNtwk(network.getId(), true); + if (CollectionUtils.isEmpty(addresses)) { + LOGGER.error(String.format("No public IP addresses found for network ID: %s, Kubernetes cluster ID: %s", network.getUuid(), kubernetesCluster.getUuid())); + return; + } + for (IpAddress address : addresses) { + if (address.isSourceNat()) { + publicIp = address; + break; + } + } + if (publicIp == null) { + LOGGER.error(String.format("No source NAT IP addresses found for network ID: %s, Kubernetes cluster ID: %s", network.getUuid(), kubernetesCluster.getUuid())); + return; + } List sourceCidrList = new ArrayList(); sourceCidrList.add("0.0.0.0/0"); @@ -719,11 +875,10 @@ private void setupKubernetesClusterNetworkRules(IPAddressVO publicIp, Account ac LOGGER.debug(String.format("Provisioned firewall rule to open up port 6443 on %s for Kubernetes cluster ID: %s", publicIp.getAddress().addr(), kubernetesCluster.getUuid())); } catch (Exception e) { - LOGGER.warn("Failed to provision firewall rules for the Kubernetes cluster: " + kubernetesCluster.getName() - + " due to exception: " + getStackTrace(e)); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); - throw new ManagementServerException("Failed to provision firewall rules for the Kubernetes " + - "cluster: " + kubernetesCluster.getName()); + String msg = String.format("Failed to provision firewall rules for API access for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); + LOGGER.warn(msg, e); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); + throw new ManagementServerException(msg, e); } try { @@ -740,11 +895,12 @@ private void setupKubernetesClusterNetworkRules(IPAddressVO publicIp, Account ac Field startPortField = rule.getClass().getDeclaredField("publicStartPort"); startPortField.setAccessible(true); - startPortField.set(rule, new Integer(2222)); + startPortField.set(rule, 2222); Field endPortField = rule.getClass().getDeclaredField("publicEndPort"); endPortField.setAccessible(true); - endPortField.set(rule, new Integer(2222 + clusterVMIds.size() - 1)); // clusterVMIds contains all nodes including master + int endPort = 2222 + clusterVMIds.size() - 1; + endPortField.set(rule, endPort); // clusterVMIds contains all nodes including master Field cidrField = rule.getClass().getDeclaredField("cidrlist"); cidrField.setAccessible(true); @@ -753,16 +909,12 @@ private void setupKubernetesClusterNetworkRules(IPAddressVO publicIp, Account ac firewallService.createIngressFirewallRule(rule); firewallService.applyIngressFwRules(publicIp.getId(), account); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Provisioned firewall rule to open up port 2222 on " + publicIp.getAddress() + - " for cluster " + kubernetesCluster.getName()); - } + LOGGER.debug(String.format("Provisioned firewall rule to open up port 2222 to %d on %s for Kubernetes cluster ID: %s", endPort, publicIp.getAddress().addr(), kubernetesCluster.getUuid())); } catch (Exception e) { - LOGGER.warn("Failed to provision firewall rules for the Kubernetes cluster: " + kubernetesCluster.getName() - + " due to exception: " + getStackTrace(e)); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); - throw new ManagementServerException("Failed to provision firewall rules for the Kubernetes " + - "cluster: " + kubernetesCluster.getName()); + String msg = String.format("Failed to provision firewall rules for SSH access for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); + LOGGER.warn(msg, e); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); + throw new ManagementServerException(msg, e); } // Load balancer rule fo API access for master node VMs @@ -781,11 +933,10 @@ private void setupKubernetesClusterNetworkRules(IPAddressVO publicIp, Account ac } lbService.assignToLoadBalancer(lb.getId(), null, vmIdIpMap); } catch (Exception e) { - LOGGER.warn("Failed to provision load balancer rule for API access for the Kubernetes cluster: " + kubernetesCluster.getName() - + " due to exception: " + getStackTrace(e)); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); - throw new ManagementServerException("Failed to provision load balancer rule for API access for the Kubernetes " + - "cluster: " + kubernetesCluster.getName()); + String msg = String.format("Failed to provision load balancer rule for API access for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); + LOGGER.warn(msg, e); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); + throw new ManagementServerException(msg, e); } // Port forwarding rule fo SSH access on each node VM @@ -800,7 +951,6 @@ private void setupKubernetesClusterNetworkRules(IPAddressVO publicIp, Account ac final Ip vmIp = new Ip(vmNic.getIPv4Address()); final long vmIdFinal = vmId; final int srcPortFinal = 2222 + i; - try { PortForwardingRuleVO pfRule = Transaction.execute(new TransactionCallbackWithException() { @Override @@ -818,19 +968,12 @@ public PortForwardingRuleVO doInTransaction(TransactionStatus status) throws Net } }); rulesService.applyPortForwardingRules(publicIp.getId(), account); - - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Provisioning SSH port forwarding rule from port 2222 to 22 on " + publicIp.getAddress() + - " to the VM IP :" + vmIp + " in Kubernetes cluster " + kubernetesCluster.getName()); - } - } catch (RuntimeException rte) { - LOGGER.warn("Failed to activate SSH port forwarding rules for the Kubernetes cluster " + kubernetesCluster.getName() + " due to " + rte); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); - throw new ManagementServerException("Failed to activate SSH port forwarding rules for the cluster: " + kubernetesCluster.getName(), rte); + LOGGER.debug(String.format("Provisioned SSH port forwarding rule from port %d to 22 on %s to the VM IP: %s in Kubernetes cluster ID: %s", srcPortFinal, publicIp.getAddress().addr(), vmIp, kubernetesCluster.getUuid())); } catch (Exception e) { - LOGGER.warn("Failed to activate SSH port forwarding rules for the Kubernetes cluster " + kubernetesCluster.getName() + " due to " + e); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); - throw new ManagementServerException("Failed to activate SSH port forwarding rules for the cluster: " + kubernetesCluster.getName(), e); + String msg = String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); + LOGGER.warn(msg, e); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); + throw new ManagementServerException(msg, e); } } } @@ -838,9 +981,30 @@ public PortForwardingRuleVO doInTransaction(TransactionStatus status) throws Net // Open up firewall ports 2222 to 2222+n for SSH access. Also create port-forwarding // rule to forward public IP traffic to all node VM private IP. Existing node VMs before scaling // will already be having these rules - private void scaleKubernetesClusterNetworkRules(IPAddressVO publicIp, Account account, long kubernetesClusterId, - List clusterVMIds) throws ManagementServerException { - KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + private void scaleKubernetesClusterNetworkRules(KubernetesCluster kubernetesCluster, Network network, Account account, + List clusterVMIds, List removedVMIds) throws ManagementServerException { + if (!Network.GuestType.Isolated.equals(network.getGuestType())) { + LOGGER.debug(String.format("Network ID: %s for Kubernetes cluster ID: %s is not an isolated network, therefore, no need for network rules", network.getUuid(), kubernetesCluster.getUuid())); + return; + } + IpAddress publicIp = null; + List addresses = networkModel.listPublicIpsAssignedToGuestNtwk(network.getId(), true); + if (CollectionUtils.isEmpty(addresses)) { + String msg = String.format("No public IP addresses found for network ID: %s, Kubernetes cluster ID: %s", network.getUuid(), kubernetesCluster.getUuid()); + LOGGER.error(msg); + throw new ManagementServerException(msg); + } + for (IpAddress address : addresses) { + if (address.isSourceNat()) { + publicIp = address; + break; + } + } + if (publicIp == null) { + String msg = String.format("No source NAT IP addresses found for network ID: %s, Kubernetes cluster ID: %s", network.getUuid(), kubernetesCluster.getUuid()); + LOGGER.error(msg); + throw new ManagementServerException(msg); + } List sourceCidrList = new ArrayList(); sourceCidrList.add("0.0.0.0/0"); @@ -892,7 +1056,26 @@ private void scaleKubernetesClusterNetworkRules(IPAddressVO publicIp, Account ac throw new ManagementServerException(msg, e); } - if (clusterVMIds != null && !clusterVMIds.isEmpty()) { // Upscaling, add new port-forwarding rules + if (!CollectionUtils.isEmpty(removedVMIds)) { + try { + for (Long vmId : removedVMIds) { + List pfRules = portForwardingRulesDao.listByNetwork(kubernetesCluster.getNetworkId()); + for (PortForwardingRuleVO pfRule : pfRules) { + if (pfRule.getVirtualMachineId() == vmId) { + portForwardingRulesDao.remove(pfRule.getId()); + break; + } + } + } + rulesService.applyPortForwardingRules(publicIp.getId(), account); + } catch (Exception e) { + String msg = String.format("Failed to remove SSH port forwarding rules for removed VMs for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); + LOGGER.warn(msg, e); + throw new ManagementServerException(msg, e); + } + } + + if (!CollectionUtils.isEmpty(clusterVMIds)) { // Upscaling, add new port-forwarding rules // Apply port forwarding only to new VMs final long publicIpId = publicIp.getId(); final long networkId = kubernetesCluster.getNetworkId(); @@ -950,43 +1133,48 @@ private boolean validateNetwork(Network network, int clusterTotalNodeCount) { throw new InvalidParameterValueException(String.format("Network ID: %s does not support DHCP that is required for Kubernetes cluster", network.getUuid())); } - List addrs = networkModel.listPublicIpsAssignedToGuestNtwk(network.getId(), true); - IPAddressVO sourceNatIp = null; - if (addrs.isEmpty()) { - throw new InvalidParameterValueException(String.format("Network ID: %s does not have a public IP associated with it. To provision a Kubernetes Cluster, source NAT IP is required", network.getUuid())); - } else { - for (IpAddress addr : addrs) { - if (addr.isSourceNat()) { - sourceNatIp = ipAddressDao.findById(addr.getId()); - } + if (Network.GuestType.Isolated.equals(network.getGuestType())) { + if (Network.State.Allocated.equals(network.getState())) { // Allocated networks won't have IP and rules + return true; } - if (sourceNatIp == null) { - throw new InvalidParameterValueException(String.format("Network ID: %s does not have a source NAT IP associated with it. To provision a Kubernetes Cluster, source NAT IP is required", network.getUuid())); - } - } - List rules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(sourceNatIp.getId(), FirewallRule.Purpose.Firewall); - for (FirewallRuleVO rule : rules) { - Integer startPort = rule.getSourcePortStart(); - Integer endPort = rule.getSourcePortEnd(); - LOGGER.debug("Network rule : " + startPort + " " + endPort); - if (startPort <= 6443 && 6443 <= endPort) { - throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting firewall rules to provision Kubernetes cluster for API access", network.getUuid())); - } - if (startPort <= 2222 && 2222+clusterTotalNodeCount <= endPort) { - throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting firewall rules to provision Kubernetes cluster for node VM SSH access", network.getUuid())); + List addrs = networkModel.listPublicIpsAssignedToGuestNtwk(network.getId(), true); + IPAddressVO sourceNatIp = null; + if (addrs.isEmpty()) { + throw new InvalidParameterValueException(String.format("Network ID: %s does not have a public IP associated with it. To provision a Kubernetes Cluster, source NAT IP is required", network.getUuid())); + } else { + for (IpAddress addr : addrs) { + if (addr.isSourceNat()) { + sourceNatIp = ipAddressDao.findById(addr.getId()); + break; + } + } + if (sourceNatIp == null) { + throw new InvalidParameterValueException(String.format("Network ID: %s does not have a source NAT IP associated with it. To provision a Kubernetes Cluster, source NAT IP is required", network.getUuid())); + } } - } - - rules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(sourceNatIp.getId(), FirewallRule.Purpose.PortForwarding); - for (FirewallRuleVO rule : rules) { - Integer startPort = rule.getSourcePortStart(); - Integer endPort = rule.getSourcePortEnd(); - LOGGER.debug("Network rule : " + startPort + " " + endPort); - if (startPort <= 6443 && 6443 <= endPort) { - throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting port forwarding rules to provision Kubernetes cluster for API access", network.getUuid())); + List rules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(sourceNatIp.getId(), FirewallRule.Purpose.Firewall); + for (FirewallRuleVO rule : rules) { + Integer startPort = rule.getSourcePortStart(); + Integer endPort = rule.getSourcePortEnd(); + LOGGER.debug("Network rule : " + startPort + " " + endPort); + if (startPort <= 6443 && 6443 <= endPort) { + throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting firewall rules to provision Kubernetes cluster for API access", network.getUuid())); + } + if (startPort <= 2222 && 2222+clusterTotalNodeCount <= endPort) { + throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting firewall rules to provision Kubernetes cluster for node VM SSH access", network.getUuid())); + } } - if (startPort <= 2222 && 2222+clusterTotalNodeCount <= endPort) { - throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting port forwarding rules to provision Kubernetes cluster for node VM SSH access", network.getUuid())); + rules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(sourceNatIp.getId(), FirewallRule.Purpose.PortForwarding); + for (FirewallRuleVO rule : rules) { + Integer startPort = rule.getSourcePortStart(); + Integer endPort = rule.getSourcePortEnd(); + LOGGER.debug("Network rule : " + startPort + " " + endPort); + if (startPort <= 6443 && 6443 <= endPort) { + throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting port forwarding rules to provision Kubernetes cluster for API access", network.getUuid())); + } + if (startPort <= 2222 && 2222+clusterTotalNodeCount <= endPort) { + throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting port forwarding rules to provision Kubernetes cluster for node VM SSH access", network.getUuid())); + } } } return true; @@ -1064,9 +1252,7 @@ private DeployDestination plan(final long nodesCount, final DataCenter zone, fin break; } } - if (suitable_host_found) { - continue; - } else { + if (!suitable_host_found) { LOGGER.debug(String.format("Suitable hosts not found in datacenter ID: %s for node %d", zone.getUuid(), i)); break; } @@ -1089,62 +1275,45 @@ private DeployDestination plan(final KubernetesCluster kubernetesCluster, final return plan(kubernetesCluster.getTotalNodeCount(), zone, offering); } - private boolean isAddOnServiceRunning(Long clusterId, final String nameSpace, String svcName) { - KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(clusterId); - //FIXME: whole logic needs revamp. Assumption that management server has public network access is not practical - IPAddressVO publicIp = null; - List ips = ipAddressDao.listByAssociatedNetwork(kubernetesCluster.getNetworkId(), true); - publicIp = ips.get(0); - Runtime r = Runtime.getRuntime(); - int nodePort = 0; + private boolean isAddOnServiceRunning(KubernetesCluster kubernetesCluster, final String ipAddress, final int port, final String namespace, String serviceName) { try { - Boolean devel = Boolean.valueOf(globalConfigDao.getValue("developer")); - String keyFile = String.format("%s/.ssh/id_rsa", System.getProperty("user.home")); - if (devel) { - keyFile += ".cloud"; - } - File pkFile = new File(keyFile); String cmd = "sudo kubectl get pods --all-namespaces"; - if (!Strings.isNullOrEmpty(nameSpace)) { - cmd = String.format("sudo kubectl get pods --namespace=%s", nameSpace); + if (!Strings.isNullOrEmpty(namespace)) { + cmd = String.format("sudo kubectl get pods --namespace=%s", namespace); } - Pair result = SshHelper.sshExecute(publicIp.getAddress().addr(), 2222, "core", - pkFile, null, cmd, + Pair result = SshHelper.sshExecute(ipAddress, port, "core", + getManagementServerSshPublicKeyFile(), null, cmd, 10000, 10000, 10000); if (result.first() && !Strings.isNullOrEmpty(result.second())) { String[] lines = result.second().split("\n"); for (String line : lines) { - if (line.contains(svcName) && line.contains("Running")) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Service :" + svcName + " for the Kubernetes cluster " - + kubernetesCluster.getName() + " is running"); - } + if (line.contains(serviceName) && line.contains("Running")) { + LOGGER.debug(String.format("Service : %s in namespace: %s for the Kubernetes cluster ID: %s is running",serviceName, namespace, kubernetesCluster.getUuid())); return true; } } } } catch (Exception e) { - LOGGER.warn("KUBECTL: " + e); + LOGGER.warn(String.format("Unable to retrieve service: %s running status in namespace %s for Kubernetes cluster ID: %s", serviceName, namespace, kubernetesCluster.getUuid()), e); } return false; } - private boolean cleanupKubernetesClusterResources(Long kubernetesClusterId) throws ManagementServerException { - KubernetesClusterVO cluster = kubernetesClusterDao.findById(kubernetesClusterId); - if (!(cluster.getState().equals(KubernetesCluster.State.Running) - || cluster.getState().equals(KubernetesCluster.State.Stopped) - || cluster.getState().equals(KubernetesCluster.State.Alert) - || cluster.getState().equals(KubernetesCluster.State.Error) - || cluster.getState().equals(KubernetesCluster.State.Destroying))) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Cannot perform delete operation on cluster:" + cluster.getName() + " in state " + cluster.getState()); - } - throw new PermissionDeniedException("Cannot perform delete operation on cluster: " + cluster.getName() + " in state " + cluster.getState()); + protected boolean cleanupKubernetesClusterResources(Long kubernetesClusterId) throws ManagementServerException { + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + if (!(kubernetesCluster.getState().equals(KubernetesCluster.State.Running) + || kubernetesCluster.getState().equals(KubernetesCluster.State.Stopped) + || kubernetesCluster.getState().equals(KubernetesCluster.State.Alert) + || kubernetesCluster.getState().equals(KubernetesCluster.State.Error) + || kubernetesCluster.getState().equals(KubernetesCluster.State.Destroying))) { + String msg = String.format("Cannot perform delete operation on cluster ID: %s in state: %s",kubernetesCluster.getUuid(), kubernetesCluster.getState()); + LOGGER.warn(msg); + throw new PermissionDeniedException(msg); } stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.DestroyRequested); boolean failedVmDestroy = false; - List clusterVMs = kubernetesClusterVmMapDao.listByClusterId(cluster.getId()); + List clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); if ((clusterVMs != null) && !clusterVMs.isEmpty()) { for (KubernetesClusterVmMapVO clusterVM : clusterVMs) { long vmID = clusterVM.getVmId(); @@ -1157,26 +1326,23 @@ private boolean cleanupKubernetesClusterResources(Long kubernetesClusterId) thro try { UserVm vm = userVmService.destroyVm(vmID, true); if (!VirtualMachine.State.Expunging.equals(vm.getState())) { - LOGGER.warn(String.format("VM '%s' with uuid '%s' should have been expunging by now but is '%s'... retrying..." + LOGGER.warn(String.format("VM '%s' ID: %s should have been expunging by now but is '%s'... retrying..." , vm.getInstanceName() , vm.getUuid() , vm.getState().toString())); } vm = userVmService.expungeVm(vmID); if (!VirtualMachine.State.Expunging.equals(vm.getState())) { - LOGGER.error(String.format("VM '%s' is now in state '%s'. I will probably fail at deleting it's cluster." + LOGGER.error(String.format("VM '%s' ID: %s is now in state '%s'. Will probably fail at deleting it's Kubernetes cluster." , vm.getInstanceName() + , vm.getUuid() , vm.getState().toString())); } kubernetesClusterVmMapDao.expunge(clusterVM.getId()); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Destroyed VM: " + userVM.getInstanceName() + " as part of cluster: " + cluster.getName() + " destroy."); - } + LOGGER.debug(String.format("Destroyed VM ID: %s as part of Kubernetes cluster ID: %s cleanup", vm.getUuid(), kubernetesCluster.getUuid())); } catch (Exception e) { failedVmDestroy = true; - LOGGER.warn("Failed to destroy VM :" + userVM.getInstanceName() + " part of the cluster: " + cluster.getName() + - " due to " + e); - LOGGER.warn("Moving on with destroying remaining resources provisioned for the cluster: " + cluster.getName()); + LOGGER.warn(String.format("Failed to destroy VM ID: %s part of the Kubernetes cluster ID: %s cleanup. Moving on with destroying remaining resources provisioned for the Kubernetes cluster", userVM.getUuid(), kubernetesCluster.getUuid()), e); } } } @@ -1212,52 +1378,43 @@ private boolean cleanupKubernetesClusterResources(Long kubernetesClusterId) thro } NetworkVO network = null; try { - network = networkDao.findById(cluster.getNetworkId()); + network = networkDao.findById(kubernetesCluster.getNetworkId()); if (network != null && network.getRemoved() == null) { Account owner = accountManager.getAccount(network.getAccountId()); User callerUser = accountManager.getActiveUser(CallContext.current().getCallingUserId()); ReservationContext context = new ReservationContextImpl(null, null, callerUser, owner); - boolean networkDestroyed = networkMgr.destroyNetwork(cluster.getNetworkId(), context, true); + boolean networkDestroyed = networkMgr.destroyNetwork(kubernetesCluster.getNetworkId(), context, true); if (!networkDestroyed) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Failed to destroy network: " + cluster.getNetworkId() + - " as part of cluster: " + cluster.getName() + " destroy"); - } + String msg = String.format("Failed to destroy network ID: %s as part of Kubernetes cluster ID: %s cleanup", network.getUuid(), kubernetesCluster.getUuid()); + LOGGER.warn(msg); processFailedNetworkDelete(kubernetesClusterId); - throw new ManagementServerException("Failed to delete the network as part of Kubernetes cluster name:" + cluster.getName() + " clean up"); - } - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Destroyed network: " + network.getName() + " as part of cluster: " + cluster.getName() + " destroy"); + throw new ManagementServerException(msg); } + LOGGER.debug(String.format("Destroyed network: %s as part of Kubernetes cluster ID: %s cleanup", network.getUuid(), kubernetesCluster.getUuid())); } } catch (Exception e) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Failed to destroy network: " + cluster.getNetworkId() + - " as part of cluster: " + cluster.getName() + " destroy due to " + e); - } + String msg = String.format("Failed to destroy network ID: %s as part of Kubernetes cluster ID: %s cleanup", network.getUuid(), kubernetesCluster.getUuid()); + LOGGER.warn(msg, e); processFailedNetworkDelete(kubernetesClusterId); - throw new ManagementServerException("Failed to delete the network as part of Kubernetes cluster name:" + cluster.getName() + " clean up"); + throw new ManagementServerException(msg, e); } } } else { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("There are VM's that are not expunged in Kubernetes cluster " + cluster.getName()); - } + String msg = String.format("Failed to destroy one or more VMs as part of Kubernetes cluster ID: %s cleanup", kubernetesCluster.getUuid()); + LOGGER.debug(msg); processFailedNetworkDelete(kubernetesClusterId); - throw new ManagementServerException("Failed to destroy one or more VM's as part of Kubernetes cluster name:" + cluster.getName() + " clean up"); + throw new ManagementServerException(msg); } stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationSucceeded); - cluster = kubernetesClusterDao.findById(kubernetesClusterId); - cluster.setCheckForGc(false); - kubernetesClusterDao.update(cluster.getId(), cluster); + kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + kubernetesCluster.setCheckForGc(false); + kubernetesClusterDao.update(kubernetesCluster.getId(), kubernetesCluster); - kubernetesClusterDao.remove(cluster.getId()); + kubernetesClusterDao.remove(kubernetesCluster.getId()); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Kubernetes cluster name:" + cluster.getName() + " is successfully deleted"); - } + LOGGER.debug(String.format("Kubernetes cluster ID: %s is successfully deleted", kubernetesCluster.getUuid())); return true; } @@ -1269,7 +1426,7 @@ private void processFailedNetworkDelete(long kubernetesClusterId) { kubernetesClusterDao.update(cluster.getId(), cluster); } - private UserVm createKubernetesMaster(final KubernetesClusterVO kubernetesCluster, final List ips) throws ManagementServerException, + private UserVm createKubernetesMaster(final KubernetesClusterVO kubernetesCluster, final Pod pod, final Network network, final Account account, String serverIp) throws ManagementServerException, ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException { UserVm masterVm = null; DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); @@ -1278,7 +1435,28 @@ private UserVm createKubernetesMaster(final KubernetesClusterVO kubernetesCluste List networkIds = new ArrayList(); networkIds.add(kubernetesCluster.getNetworkId()); Account owner = accountDao.findById(kubernetesCluster.getAccountId()); - final String masterIp = ipAddressManager.acquireGuestIpAddress(networkDao.findById(kubernetesCluster.getNetworkId()), null); + String masterIp = null; + Map requestedIps = null; + if (Network.GuestType.Shared.equals(network.getGuestType())) { + List vlanIds = new ArrayList<>(); + List vlans = vlanDao.listVlansByNetworkId(network.getId()); + for (VlanVO vlan : vlans) { + vlanIds.add(vlan.getId()); + } + PublicIp ip = ipAddressManager.getAvailablePublicIpAddressFromVlans(zone.getId(), null, account, Vlan.VlanType.DirectAttached, vlanIds,network.getId(), null, false); + if (ip != null) { + masterIp = ip.getAddress().toString(); + } + if (Strings.isNullOrEmpty(serverIp)) { + serverIp = masterIp; + } + requestedIps = new HashMap<>(); + Ip ipAddress = ip.getAddress(); + boolean isIp6 = ipAddress.isIp6(); + requestedIps.put(network.getId(), new Network.IpAddresses(ipAddress.isIp4() ? ip.getAddress().addr() : null, null)); + } else { + masterIp = ipAddressManager.acquireGuestIpAddress(networkDao.findById(kubernetesCluster.getNetworkId()), null); + } Network.IpAddresses addrs = new Network.IpAddresses(masterIp, null); long rootDiskSize = kubernetesCluster.getNodeRootDiskSize(); Map customParameterMap = new HashMap(); @@ -1308,8 +1486,8 @@ private UserVm createKubernetesMaster(final KubernetesClusterVO kubernetesCluste final String clusterInitArgsKey = "{{ k8s_master.cluster.initargs }}"; final List addresses = new ArrayList<>(); addresses.add(masterIp); - for (final IPAddressVO ip : ips) { - addresses.add(ip.getAddress().addr()); + if (!serverIp.equals(masterIp)) { + addresses.add(serverIp); } final Certificate certificate = caManager.issueCertificate(null, Arrays.asList(hostName, "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.default.svc.cluster.local"), @@ -1333,29 +1511,28 @@ private UserVm createKubernetesMaster(final KubernetesClusterVO kubernetesCluste String initArgs = ""; if (haSupported) { initArgs = String.format("--control-plane-endpoint %s:6443 --upload-certs --certificate-key %s ", - ips.get(0).getAddress().addr(), + serverIp, generateClusterHACertificateKey(kubernetesCluster)); } - initArgs += String.format("--apiserver-cert-extra-sans=%s", ips.get(0).getAddress().addr()); + initArgs += String.format("--apiserver-cert-extra-sans=%s", serverIp); k8sMasterConfig = k8sMasterConfig.replace(clusterInitArgsKey, initArgs); } catch (Exception e) { - LOGGER.error("Failed to read Kubernetes master configuration file due to " + e); - throw new ManagementServerException("Failed to read Kubernetes master configuration file", e); + String msg = "Failed to read Kubernetes master configuration file"; + LOGGER.error(msg, e); + throw new ManagementServerException(msg, e); } - String base64UserData = Base64.encodeBase64String(k8sMasterConfig.getBytes(Charset.forName("UTF-8"))); + String base64UserData = Base64.encodeBase64String(k8sMasterConfig.getBytes(StandardCharsets.UTF_8)); masterVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, hostName, kubernetesCluster.getDescription(), null, null, null, null, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), - null, addrs, null, null, null, customParameterMap, null, null, null, null); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Created master VM: " + hostName + " in the Kubernetes cluster: " + kubernetesCluster.getName()); - } + requestedIps, addrs, null, null, null, customParameterMap, null, null, null, null); + LOGGER.debug(String.format("Created master VM ID: %s, %s in the Kubernetes cluster ID: %s", masterVm.getUuid(), hostName, kubernetesCluster.getUuid())); return masterVm; } private UserVm createKubernetesAdditionalMaster(final KubernetesClusterVO kubernetesCluster, final String joinIp, final int additionalMasterNodeInstance) throws ManagementServerException, ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException { - UserVm nodeVm = null; + UserVm additionalMasterVm = null; DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); ServiceOffering serviceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); VirtualMachineTemplate template = templateDao.findById(kubernetesCluster.getTemplateId()); @@ -1389,18 +1566,17 @@ private UserVm createKubernetesAdditionalMaster(final KubernetesClusterVO kubern k8sMasterConfig = k8sMasterConfig.replace(clusterTokenKey, generateClusterToken(kubernetesCluster)); k8sMasterConfig = k8sMasterConfig.replace(clusterHACertificateKey, generateClusterHACertificateKey(kubernetesCluster)); } catch (Exception e) { - LOGGER.warn("Failed to read node configuration file due to " + e); - throw new ManagementServerException("Failed to read cluster node configuration file.", e); + String msg = "Failed to read Kubernetes master configuration file"; + LOGGER.error(msg, e); + throw new ManagementServerException(msg, e); } - String base64UserData = Base64.encodeBase64String(k8sMasterConfig.getBytes(Charset.forName("UTF-8"))); - nodeVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, + String base64UserData = Base64.encodeBase64String(k8sMasterConfig.getBytes(StandardCharsets.UTF_8)); + additionalMasterVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, hostName, kubernetesCluster.getDescription(), null, null, null, null, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), null, addrs, null, null, null, customParameterMap, null, null, null, null); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Created cluster additional master node VM: " + hostName + " in the Kubernetes cluster: " + kubernetesCluster.getName()); - } - return nodeVm; + LOGGER.debug(String.format("Created master VM ID: %s, %s in the Kubernetes cluster ID: %s", additionalMasterVm.getUuid(), hostName, kubernetesCluster.getUuid())); + return additionalMasterVm; } private UserVm createKubernetesNode(KubernetesClusterVO kubernetesCluster, String joinIp, int nodeInstance) throws ManagementServerException, @@ -1478,23 +1654,22 @@ private UserVm createKubernetesNode(KubernetesClusterVO kubernetesCluster, Strin final String dockerAuthKey = "{{docker.secret}}"; final String dockerEmailKey = "{{docker.email}}"; final String usernamePasswordKey = dockerUserName + ":" + dockerPassword; - String base64Auth = Base64.encodeBase64String(usernamePasswordKey.getBytes(Charset.forName("UTF-8"))); + String base64Auth = Base64.encodeBase64String(usernamePasswordKey.getBytes(StandardCharsets.UTF_8)); k8sNodeConfig = k8sNodeConfig.replace(dockerUrlKey, "\"" + dockerRegistryUrl + "\""); k8sNodeConfig = k8sNodeConfig.replace(dockerAuthKey, "\"" + base64Auth + "\""); k8sNodeConfig = k8sNodeConfig.replace(dockerEmailKey, "\"" + dockerRegistryEmail + "\""); } } catch (Exception e) { - LOGGER.warn("Failed to read node configuration file due to " + e); - throw new ManagementServerException("Failed to read cluster node configuration file.", e); + String msg = "Failed to read Kubernetes node configuration file"; + LOGGER.error(msg, e); + throw new ManagementServerException(msg, e); } - String base64UserData = Base64.encodeBase64String(k8sNodeConfig.getBytes(Charset.forName("UTF-8"))); + String base64UserData = Base64.encodeBase64String(k8sNodeConfig.getBytes(StandardCharsets.UTF_8)); nodeVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, hostName, kubernetesCluster.getDescription(), null, null, null, null, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), null, addrs, null, null, null, customParameterMap, null, null, null, null); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Created cluster node VM: " + hostName + " in the Kubernetes cluster: " + kubernetesCluster.getName()); - } + LOGGER.debug(String.format("Created node VM ID: %s, %s in the Kubernetes cluster ID: %s", nodeVm.getUuid(), hostName, kubernetesCluster.getUuid())); return nodeVm; } @@ -1506,18 +1681,18 @@ private void startKubernetesVM(final UserVm vm, final KubernetesClusterVO kubern f.setAccessible(true); f.set(startVm, vm.getId()); userVmService.startVirtualMachine(startVm); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Started VM in the Kubernetes cluster: " + kubernetesCluster.getName()); - } + LOGGER.debug(String.format("Started VM ID: %s in the Kubernetes cluster ID: %s", vm.getUuid(), kubernetesCluster.getUuid())); } catch (Exception ex) { - LOGGER.warn("Failed to start VM in the Kubernetes cluster name:" + kubernetesCluster.getName() + " due to Exception: ", ex); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to start VM in the Kubernetes cluster name:" + kubernetesCluster.getName(), ex); + String msg = String.format("Failed to start VM in the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); + LOGGER.warn(msg, ex); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg, ex); } UserVm startVm = userVmDao.findById(vm.getId()); if (!startVm.getState().equals(VirtualMachine.State.Running)) { - LOGGER.warn("Failed to start VM instance."); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to start VM instance in Kubernetes cluster " + kubernetesCluster.getName()); + String msg = String.format("Failed to start VM in the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); + LOGGER.warn(msg); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } } @@ -1540,8 +1715,8 @@ private void attachIsoKubernetesVMs(KubernetesCluster kubernetesCluster, List clusterVMIds) throws ServerApiException { - for (int i = 0; i < clusterVMIds.size(); ++i) { - UserVm vm = userVmDao.findById(clusterVMIds.get(i)); + for (Long clusterVMId : clusterVMIds) { + UserVm vm = userVmDao.findById(clusterVMId); boolean result = false; try { result = templateService.detachIso(vm.getId()); @@ -1575,8 +1750,8 @@ private void stopClusterVM(final KubernetesClusterVmMapVO vmMapVO) throws Server try { userVmService.stopVirtualMachine(vmMapVO.getVmId(), false); } catch (ConcurrentOperationException ex) { - LOGGER.warn("Failed to stop Kubernetes cluster VM due to Exception: ", ex); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); + LOGGER.warn("Failed to stop Kubernetes cluster VM", ex); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to stop Kubernetes cluster VM", ex); } } @@ -1639,6 +1814,9 @@ protected boolean stateTransitTo(long kubernetesClusterId, KubernetesCluster.Eve @Override public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) throws InsufficientCapacityException, ResourceAllocationException, ManagementServerException { + if (!Boolean.parseBoolean(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesServiceEnabled.key()))) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Kubernetes Service plugin is disabled"); + } final String name = cmd.getName(); final String displayName = cmd.getDisplayName(); final Long zoneId = cmd.getZoneId(); @@ -1654,6 +1832,7 @@ public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) final String dockerRegistryUrl = cmd.getDockerRegistryUrl(); final String dockerRegistryEmail = cmd.getDockerRegistryEmail(); final Long nodeRootDiskSize = cmd.getNodeRootDiskSize(); + final String externalLoadBalancerIpAddress = cmd.getExternalLoadBalancerIpAddress(); if (name == null || name.isEmpty()) { throw new InvalidParameterValueException("Invalid name for the Kubernetes cluster name:" + name); @@ -1700,7 +1879,6 @@ public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) ServiceOffering serviceOffering = serviceOfferingDao.findById(serviceOfferingId); if (serviceOffering == null) { throw new InvalidParameterValueException("No service offering with ID: " + serviceOfferingId); - } else { } if (sshKeyPair != null && !sshKeyPair.isEmpty()) { @@ -1736,16 +1914,35 @@ public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) } } + if (!Strings.isNullOrEmpty(externalLoadBalancerIpAddress)) { + if (!NetUtils.isValidIp4(externalLoadBalancerIpAddress) && !NetUtils.isValidIp6(externalLoadBalancerIpAddress)) { + throw new InvalidParameterValueException("Invalid external load balancer IP address"); + } + if (network == null) { + throw new InvalidParameterValueException(String.format("%s parameter must be specified along with %s parameter", ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS, ApiConstants.NETWORK_ID)); + } + if (Network.GuestType.Shared.equals(network.getGuestType())) { + throw new InvalidParameterValueException(String.format("%s parameter must be specified along with %s type of network", ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS, Network.GuestType.Shared.toString())); + } + } + plan(masterNodeCount + clusterSize, zone, serviceOfferingDao.findById(serviceOfferingId)); if (network != null) { - if (kubernetesClusterDao.listByNetworkId(network.getId()).isEmpty()) { - if (!validateNetwork(network, (int)(masterNodeCount+clusterSize))) { - throw new InvalidParameterValueException(String.format("Network ID: %s is not suitable for Kubernetes cluster", network.getUuid())); + if (Network.GuestType.Isolated.equals(network.getGuestType())) { + if (kubernetesClusterDao.listByNetworkId(network.getId()).isEmpty()) { + if (!validateNetwork(network, (int) (masterNodeCount + clusterSize))) { + throw new InvalidParameterValueException(String.format("Network ID: %s is not suitable for Kubernetes cluster", network.getUuid())); + } + networkModel.checkNetworkPermissions(owner, network); + } else { + throw new InvalidParameterValueException(String.format("Network ID: %s is already under use by another Kubernetes cluster", network.getUuid())); + } + } else if (Network.GuestType.Shared.equals(network.getGuestType())) { + if (masterNodeCount > 1 && Strings.isNullOrEmpty(externalLoadBalancerIpAddress)) { + throw new InvalidParameterValueException(String.format("Multi-master, HA Kubernetes cluster with %s network ID: %s needs an external load balancer IP address. %s parameter can be used", + network.getGuestType().toString(), network.getUuid(), ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS)); } - networkModel.checkNetworkPermissions(owner, network); - } else { - throw new InvalidParameterValueException(String.format("Network ID: %s is already under use by another Kubernetes cluster", network.getUuid())); } } else { // user has not specified network in which cluster VM's to be provisioned, so create a network for Kubernetes cluster NetworkOfferingVO networkOffering = networkOfferingDao.findByUniqueName( @@ -1786,6 +1983,9 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { @Override public void doInTransactionWithoutResult(TransactionStatus status) { List details = new ArrayList<>(); + if (Network.GuestType.Shared.equals(defaultNetwork.getGuestType()) && !Strings.isNullOrEmpty(externalLoadBalancerIpAddress)) { + details.add(new KubernetesClusterDetailsVO(cluster.getId(), ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS, externalLoadBalancerIpAddress, true)); + } if (!Strings.isNullOrEmpty(dockerRegistryUserName)) { details.add(new KubernetesClusterDetailsVO(cluster.getId(), ApiConstants.DOCKER_REGISTRY_USER_NAME, dockerRegistryUserName, true)); } @@ -1811,7 +2011,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { } - // Start operation can be performed at two diffrent life stages of Kubernetes cluster. First when a freshly created cluster + // Start operation can be performed at two diffrent life stagevs of Kubernetes cluster. First when a freshly created cluster // in which case there are no resources provisisioned for the Kubernetes cluster. So during start all the resources // are provisioned from scratch. Second kind of start, happens on Stopped Kubernetes cluster, in which all resources // are provisioned (like volumes, nics, networks etc). It just that VM's are not in running state. So just @@ -1819,7 +2019,9 @@ public void doInTransactionWithoutResult(TransactionStatus status) { @Override public boolean startKubernetesCluster(long kubernetesClusterId, boolean onCreate) throws ManagementServerException, ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException { - + if (!Boolean.parseBoolean(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesServiceEnabled.key()))) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Kubernetes Service plugin is disabled"); + } if (onCreate) { // Start for Kubernetes cluster in 'Created' state return startKubernetesClusterOnCreate(kubernetesClusterId); @@ -1831,7 +2033,9 @@ public boolean startKubernetesCluster(long kubernetesClusterId, boolean onCreate @Override public boolean stopKubernetesCluster(long kubernetesClusterId) throws ManagementServerException { - + if (!Boolean.parseBoolean(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesServiceEnabled.key()))) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Kubernetes Service plugin is disabled"); + } final KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); if (kubernetesCluster == null) { throw new ManagementServerException("Failed to find Kubernetes cluster with given ID"); @@ -1883,6 +2087,9 @@ public boolean stopKubernetesCluster(long kubernetesClusterId) throws Management @Override public boolean deleteKubernetesCluster(Long kubernetesClusterId) throws ManagementServerException { + if (!Boolean.parseBoolean(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesServiceEnabled.key()))) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Kubernetes Service plugin is disabled"); + } KubernetesClusterVO cluster = kubernetesClusterDao.findById(kubernetesClusterId); if (cluster == null) { throw new InvalidParameterValueException("Invalid cluster id specified"); @@ -1895,6 +2102,9 @@ public boolean deleteKubernetesCluster(Long kubernetesClusterId) throws Manageme @Override public ListResponse listKubernetesClusters(ListKubernetesClustersCmd cmd) { + if (!Boolean.parseBoolean(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesServiceEnabled.key()))) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Kubernetes Service plugin is disabled"); + } final CallContext ctx = CallContext.current(); final Account caller = ctx.getCallingAccount(); final Long clusterId = cmd.getId(); @@ -1942,6 +2152,9 @@ public ListResponse listKubernetesClusters(ListKubern } public KubernetesClusterConfigResponse getKubernetesClusterConfig(GetKubernetesClusterConfigCmd cmd) { + if (!Boolean.parseBoolean(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesServiceEnabled.key()))) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Kubernetes Service plugin is disabled"); + } final Long clusterId = cmd.getId(); KubernetesCluster kubernetesCluster = kubernetesClusterDao.findById(clusterId); if (kubernetesCluster == null) { @@ -1963,6 +2176,9 @@ public KubernetesClusterConfigResponse getKubernetesClusterConfig(GetKubernetesC @Override public boolean scaleKubernetesCluster(ScaleKubernetesClusterCmd cmd) throws ManagementServerException, ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException { + if (!Boolean.parseBoolean(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesServiceEnabled.key()))) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Kubernetes Service plugin is disabled"); + } final Long kubernetesClusterId = cmd.getId(); final Long serviceOfferingId = cmd.getServiceOfferingId(); final Long clusterSize = cmd.getClusterSize(); @@ -2045,8 +2261,6 @@ public boolean scaleKubernetesCluster(ScaleKubernetesClusterCmd cmd) throws Mana throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, service offering for the Kubernetes cluster cannot be scaled down!", kubernetesCluster.getUuid())); } - // ToDo: Check capacity with new service offering at this point, how? - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.ScaleUpRequested); final long size = (clusterSize == null ? kubernetesCluster.getNodeCount() : clusterSize); @@ -2088,6 +2302,13 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { } if (clusterSizeScalingNeeded) { + Network network = networkDao.findById(kubernetesCluster.getNetworkId()); + if (network == null) { + String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, cluster network not found", kubernetesCluster.getUuid()); + LOGGER.error(msg); + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); + } // Check capacity and transition state final long newVmRequiredCount = clusterSize - originalNodeCount; final ServiceOffering clusterServiceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); @@ -2153,23 +2374,20 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } - IPAddressVO publicIp = null; - List ips = ipAddressDao.listByAssociatedNetwork(kubernetesCluster.getNetworkId(), true); - if (ips == null || ips.isEmpty() || ips.get(0) == null) { + + Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(kubernetesCluster); + String publicIpAddress = publicIpSshPort.first(); + int sshPort = publicIpSshPort.second(); + if (Strings.isNullOrEmpty(publicIpAddress)) { String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, unable to retrieve associated public IP", kubernetesCluster.getUuid()); LOGGER.error(msg); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } - publicIp = ips.get(0); - Boolean devel = Boolean.valueOf(globalConfigDao.getValue("developer")); - String keyFile = String.format("%s/.ssh/id_rsa", System.getProperty("user.home")); - if (devel) { - keyFile += ".cloud"; - } - File pkFile = new File(keyFile); + File pkFile = getManagementServerSshPublicKeyFile(); Account account = accountDao.findById(kubernetesCluster.getAccountId()); if (newVmRequiredCount < 0) { // downscale int i = vmList.size() - 1; + List removedVmIds = new ArrayList<>(); while (i > 1 && vmList.size() > clusterSize + 1) { // Reverse order as first VM will be k8s master KubernetesClusterVmMapVO vmMapVO = vmList.get(i); UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId()); @@ -2180,13 +2398,13 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { while (retryCounter < maxRetries) { retryCounter++; try { - Pair result = SshHelper.sshExecute(publicIp.getAddress().addr(), 2222, "core", + Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, "core", pkFile, null, String.format("sudo kubectl drain %s --ignore-daemonsets --delete-local-data", userVM.getHostName()), 10000, 10000, 30000); if (!result.first()) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Draining Kubernetes node unsuccessful"); } - result = SshHelper.sshExecute(publicIp.getAddress().addr(), 2222, "core", + result = SshHelper.sshExecute(publicIpAddress, sshPort, "core", pkFile, null, String.format("sudo kubectl delete node %s", userVM.getHostName()), 10000, 10000, 30000); if (!result.first()) { @@ -2207,15 +2425,8 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { } } - // Remove port-forwarding network rules - List pfRules = portForwardingRulesDao.listByNetwork(kubernetesCluster.getNetworkId()); - for (PortForwardingRuleVO pfRule : pfRules) { - if (pfRule.getVirtualMachineId() == userVM.getId()) { - portForwardingRulesDao.remove(pfRule.getId()); - break; - } - } - rulesService.applyPortForwardingRules(publicIp.getId(), account); + // For removing port-forwarding network rules + removedVmIds.add(userVM.getId()); // Expunge VM UserVm vm = userVmService.destroyVm(userVM.getId(), true); @@ -2242,7 +2453,7 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { // Scale network rules to update firewall rule try { - scaleKubernetesClusterNetworkRules(publicIp, account, kubernetesClusterId, null); + scaleKubernetesClusterNetworkRules(kubernetesCluster, network, account, null, removedVmIds); } catch (Exception e) { String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, unable to update network rules", kubernetesCluster.getUuid()); LOGGER.error(msg, e); @@ -2273,7 +2484,7 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { // Scale network rules to update firewall rule and add port-forwarding rules try { - scaleKubernetesClusterNetworkRules(publicIp, account, kubernetesClusterId, clusterVMIds); + scaleKubernetesClusterNetworkRules(kubernetesCluster, network, account, clusterVMIds, null); } catch (Exception e) { String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, unable to update network rules", kubernetesCluster.getUuid()); LOGGER.error(msg, e); @@ -2285,37 +2496,13 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { attachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); // Check if new nodes are added in k8s cluster - int retryCounter = 0; - int maxRetries = 15; // Max wait for 30 mins as online install can take time, same as while creating cluster - while (retryCounter < maxRetries) { - try { - Pair result = SshHelper.sshExecute(publicIp.getAddress().addr(), 2222, "core", - pkFile, null, "sudo kubectl get nodes -o json | jq \".items[].metadata.name\" | wc -l", - 20000, 10000, 30000); - if (result.first()) { - int nodesCount = Integer.parseInt(result.second().trim()); - if (nodesCount == kubernetesCluster.getTotalNodeCount()) { - LOGGER.debug(String.format("Scaling finished successfully for Kubernetes cluster ID: %s, new nodes are ready now", kubernetesCluster.getUuid())); - break; - } - } - } catch (Exception e) { - LOGGER.warn(String.format("Failed to retrieve node count for Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); - } - try { - LOGGER.debug(String.format("Waiting 30s for scaling to finish and new nodes to become ready for Kubernetes cluster ID: %s. %d/%d", kubernetesCluster.getUuid(), retryCounter+1, maxRetries)); - Thread.sleep(30000); - } catch (InterruptedException ex) { - LOGGER.warn(String.format("Failed to wait for 30s for scaling to finish and new nodes to become ready for Kubernetes cluster ID: %s. %d/%d", kubernetesCluster.getUuid(), retryCounter+1, maxRetries), ex); - } - retryCounter++; - } + boolean readyNodesCountVerified = validateKubernetesClusterReadyNodesCount(kubernetesCluster, publicIpAddress, sshPort, 15, 30000); // Detach binaries ISO from new VMs detachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); // Throw exception if nodes count for k8s cluster timed out - if (retryCounter >= maxRetries) { // Scaling failed + if (!readyNodesCountVerified) { // Scaling failed String msg = String.format("Scaling unsuccessful for Kubernetes cluster ID: %s as it does not have desired number of nodes in ready state", kubernetesCluster.getUuid()); LOGGER.warn(msg); stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); @@ -2330,6 +2517,9 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { @Override public boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws ManagementServerException { + if (!Boolean.parseBoolean(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesServiceEnabled.key()))) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Kubernetes Service plugin is disabled"); + } // Validate parameters final Long kubernetesClusterId = cmd.getId(); final Long upgradeVersionId = cmd.getKubernetesVersionId(); @@ -2368,14 +2558,14 @@ public boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws } // Get public IP - IPAddressVO publicIp = null; - List ips = ipAddressDao.listByAssociatedNetwork(kubernetesCluster.getNetworkId(), true); - if (CollectionUtils.isEmpty(ips)) { + Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(kubernetesCluster); + String publicIpAddress = publicIpSshPort.first(); + int sshPort = publicIpSshPort.second(); + if (Strings.isNullOrEmpty(publicIpAddress)) { String msg = String.format("Upgrade failed for Kubernetes cluster ID: %s, unable to retrieve associated public IP", kubernetesCluster.getUuid()); LOGGER.warn(msg); throw new ManagementServerException(msg); } - publicIp = ips.get(0); kubernetesCluster.setKubernetesVersionId(upgradeVersion.getId()); List clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); @@ -2389,12 +2579,7 @@ public boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws } Collections.sort(vmIds); - boolean devel = Boolean.parseBoolean(globalConfigDao.getValue("developer")); - String keyFile = String.format("%s/.ssh/id_rsa", System.getProperty("user.home")); - if (devel) { - keyFile += ".cloud"; - } - File pkFile = new File(keyFile); + File pkFile = getManagementServerSshPublicKeyFile(); File upgradeScriptFile = null; try { @@ -2420,37 +2605,45 @@ public boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws UserVm vm = userVmDao.findById(vmIds.get(i)); result = null; try { - result = SshHelper.sshExecute(publicIp.getAddress().addr(), 2222, "core", pkFile, null, + result = SshHelper.sshExecute(publicIpAddress, sshPort, "core", pkFile, null, String.format("sudo kubectl drain %s --ignore-daemonsets --delete-local-data", vm.getHostName()), 10000, 10000, 60000); } catch (Exception e) { String msg = String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to drain Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()); LOGGER.error(msg, e); + detachIsoKubernetesVMs(kubernetesCluster, vmIds); stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg, e); } if (!result.first()) { String msg = String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to drain Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()); - LOGGER.error(String.format("%s. Output:\n%s", msg, result.second())); + LOGGER.error(String.format("%s. Output:\n%s", msg, Strings.nullToEmpty(result.second()))); + detachIsoKubernetesVMs(kubernetesCluster, vmIds); stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } try { - SshHelper.scpTo(publicIp.getAddress().addr(), 2222 + i, "core", pkFile, null, + int nodeSshPort = sshPort == 22 ? sshPort : sshPort + i; + String nodeAddress = (i > 0 && sshPort == 22) ? vm.getPrivateIpAddress() : publicIpAddress; + SshHelper.scpTo(nodeAddress, nodeSshPort, "core", pkFile, null, "~/", upgradeScriptFile.getAbsolutePath(), "0755"); - - result = SshHelper.sshExecute(publicIp.getAddress().addr(), 2222 + i, "core", pkFile, null, - String.format("sudo ./%s %s", upgradeScriptFile.getName(), i == 0 ? upgradeVersion.getKubernetesVersion() : "''"), + String cmdStr = String.format("sudo ./%s %s %s %s", upgradeScriptFile.getName(), + upgradeVersion.getKubernetesVersion(), i == 0 ? "true" : "false", + KubernetesVersionManagerImpl.compareKubernetesVersion(upgradeVersion.getKubernetesVersion(), "1.15") < 0 ? "true" : "false"); + result = SshHelper.sshExecute(publicIpAddress, nodeSshPort, "core", pkFile, null, + cmdStr, 10000, 10000, 5 * 60 * 1000); } catch (Exception e) { String msg = String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to upgrade Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()); LOGGER.error(msg, e); + detachIsoKubernetesVMs(kubernetesCluster, vmIds); stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg, e); } if (!result.first()) { String msg = String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to upgrade Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()); - LOGGER.error(String.format("%s. Output:\n%s", msg, result.second())); + LOGGER.error(String.format("%s. Output:\n%s", msg, Strings.nullToEmpty(result.second()))); + detachIsoKubernetesVMs(kubernetesCluster, vmIds); stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } @@ -2458,12 +2651,13 @@ public boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws int maxRetries = 3; while (retryCounter < maxRetries) { try { - result = SshHelper.sshExecute(publicIp.getAddress().addr(), 2222, "core", pkFile, null, + result = SshHelper.sshExecute(publicIpAddress, sshPort, "core", pkFile, null, String.format("sudo kubectl uncordon %s", vm.getHostName()), 10000, 10000, 30000); } catch (Exception e) { String msg = String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to uncordon Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()); LOGGER.error(msg, e); + detachIsoKubernetesVMs(kubernetesCluster, vmIds); stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg, e); } @@ -2473,16 +2667,24 @@ public boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws try { Thread.sleep(30000); } catch (InterruptedException ie) { - LOGGER.warn(String.format("Error while waiting for uncordon Kubernetes cluster ID: %s node running on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()), ie); + LOGGER.warn(String.format("Error while waiting for uncordon Kubernetes cluster ID: %s node running on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()), ie); } retryCounter++; } if (!result.first()) { String msg = String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to uncordon Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()); - LOGGER.error(String.format("%s. Output:\n%s", msg, result.second())); + LOGGER.error(String.format("%s. Output:\n%s", msg, Strings.nullToEmpty(result.second()))); + detachIsoKubernetesVMs(kubernetesCluster, vmIds); stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } + if (i == 0) { // Wait for master to get in Ready state + try { + Thread.sleep(30000); + } catch (InterruptedException ie) { + LOGGER.warn(String.format("Error while waiting for master to become ready for Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), ie); + } + } } // Detach ISO @@ -2500,6 +2702,9 @@ public boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws @Override public List> getCommands() { List> cmdList = new ArrayList>(); + if (!Boolean.parseBoolean(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesServiceEnabled.key()))) { + return cmdList; + } cmdList.add(CreateKubernetesClusterCmd.class); cmdList.add(StartKubernetesClusterCmd.class); cmdList.add(StopKubernetesClusterCmd.class); @@ -2511,6 +2716,11 @@ public List> getCommands() { return cmdList; } + @Override + public KubernetesCluster findById(final Long id) { + return kubernetesClusterDao.findById(id); + } + // Garbage collector periodically run through the Kubernetes clusters marked for GC. For each Kubernetes cluster // marked for GC, attempt is made to destroy cluster. public class KubernetesClusterGarbageCollector extends ManagedContextRunnable { @@ -2563,6 +2773,7 @@ public void reallyRun() { be brought back to known good state or desired state. */ public class KubernetesClusterStatusScanner extends ManagedContextRunnable { + private boolean firstRun = true; @Override protected void runInContext() { GlobalLock gcLock = GlobalLock.getInternLock("KubernetesCluster.State.Scanner.Lock"); @@ -2584,15 +2795,13 @@ public void reallyRun() { // run through Kubernetes clusters in 'Running' state and ensure all the VM's are Running in the cluster List runningKubernetesClusters = kubernetesClusterDao.findKubernetesClustersInState(KubernetesCluster.State.Running); for (KubernetesCluster kubernetesCluster : runningKubernetesClusters) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Running Kubernetes cluster state scanner on Kubernetes cluster name:" + kubernetesCluster.getName()); - } + LOGGER.debug(String.format("Running Kubernetes cluster state scanner on Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); try { - if (!isClusterInDesiredState(kubernetesCluster, VirtualMachine.State.Running)) { + if (!isClusterVMsInDesiredState(kubernetesCluster, VirtualMachine.State.Running)) { stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.FaultsDetected); } } catch (Exception e) { - LOGGER.warn("Failed to run through VM states of Kubernetes cluster due to " + e); + LOGGER.warn(String.format("Failed to run Kubernetes cluster Running state scanner on Kubernetes cluster ID: %s status scanner", kubernetesCluster.getUuid()), e); } } @@ -2603,11 +2812,11 @@ public void reallyRun() { LOGGER.debug("Running Kubernetes cluster state scanner on Kubernetes cluster name:" + kubernetesCluster.getName() + " for state " + KubernetesCluster.State.Stopped); } try { - if (!isClusterInDesiredState(kubernetesCluster, VirtualMachine.State.Stopped)) { + if (!isClusterVMsInDesiredState(kubernetesCluster, VirtualMachine.State.Stopped)) { stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.FaultsDetected); } } catch (Exception e) { - LOGGER.warn("Failed to run through VM states of Kubernetes cluster due to " + e); + LOGGER.warn(String.format("Failed to run Kubernetes cluster Stopped state scanner on Kubernetes cluster ID: %s status scanner", kubernetesCluster.getUuid()), e); } } @@ -2618,41 +2827,55 @@ public void reallyRun() { LOGGER.debug("Running Kubernetes cluster state scanner on Kubernetes cluster name:" + kubernetesCluster.getName() + " for state " + KubernetesCluster.State.Alert); } try { - if (isClusterInDesiredState(kubernetesCluster, VirtualMachine.State.Running)) { + if (isClusterVMsInDesiredState(kubernetesCluster, VirtualMachine.State.Running) && + kubernetesCluster.getTotalNodeCount() == getKubernetesClusterReadyNodesCount(kubernetesCluster)) { // mark the cluster to be running stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.RecoveryRequested); stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); } } catch (Exception e) { - LOGGER.warn("Failed to run through VM states of Kubernetes cluster status scanner due to " + e); + LOGGER.warn(String.format("Failed to run Kubernetes cluster Alert state scanner on Kubernetes cluster ID: %s status scanner", kubernetesCluster.getUuid()), e); } } - // run through Kubernetes clusters in 'Starting' state and reconcile state as 'Running' or 'Error' if the VM's are running - List startingKubernetesClusters = kubernetesClusterDao.findKubernetesClustersInState(KubernetesCluster.State.Starting); - for (KubernetesCluster kubernetesCluster : startingKubernetesClusters) { - if (!Strings.isNullOrEmpty(kubernetesCluster.getEndpoint()) || - (new Date()).getTime() - kubernetesCluster.getCreated().getTime() < 10*60*1000) { - continue; + + if (firstRun) { + // run through Kubernetes clusters in 'Starting' state and reconcile state as 'Alert' or 'Error' if the VM's are running + List startingKubernetesClusters = kubernetesClusterDao.findKubernetesClustersInState(KubernetesCluster.State.Starting); + for (KubernetesCluster kubernetesCluster : startingKubernetesClusters) { + if ((new Date()).getTime() - kubernetesCluster.getCreated().getTime() < 10*60*1000) { + continue; + } + LOGGER.debug(String.format("Running Kubernetes cluster state scanner on Kubernetes cluster ID: %s for state: %s", kubernetesCluster.getUuid(), KubernetesCluster.State.Starting.toString())); + try { + if (isClusterVMsInDesiredState(kubernetesCluster, VirtualMachine.State.Running)) { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.FaultsDetected); + } else { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + } catch (Exception e) { + LOGGER.warn(String.format("Failed to run Kubernetes cluster Starting state scanner on Kubernetes cluster ID: %s status scanner", kubernetesCluster.getUuid()), e); + } } - LOGGER.debug(String.format("Running Kubernetes cluster state scanner on Kubernetes cluster ID: %s for state: %s", kubernetesCluster.getUuid(), KubernetesCluster.State.Starting.toString())); - try { - if (isClusterInDesiredState(kubernetesCluster, VirtualMachine.State.Running)) { - // mark the cluster to be running - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); + List destroyingKubernetesClusters = kubernetesClusterDao.findKubernetesClustersInState(KubernetesCluster.State.Destroying); + for (KubernetesCluster kubernetesCluster : destroyingKubernetesClusters) { + LOGGER.debug(String.format("Running Kubernetes cluster state scanner on Kubernetes cluster ID: %s for state: %s", kubernetesCluster.getUuid(), KubernetesCluster.State.Destroying.toString())); + try { + cleanupKubernetesClusterResources(kubernetesCluster.getId()); + } catch (Exception e) { + LOGGER.warn(String.format("Failed to run Kubernetes cluster Destroying state scanner on Kubernetes cluster ID: %s status scanner", kubernetesCluster.getUuid()), e); } - } catch (Exception e) { - LOGGER.warn(String.format("Failed to run through VM states of Kubernetes cluster ID: %s status scanner", kubernetesCluster.getUuid()), e); } } } catch (Exception e) { LOGGER.warn("Caught exception while running Kubernetes cluster state scanner", e); } + firstRun = false; } } // checks if Kubernetes cluster is in desired state - boolean isClusterInDesiredState(KubernetesCluster kubernetesCluster, VirtualMachine.State state) { + boolean isClusterVMsInDesiredState(KubernetesCluster kubernetesCluster, VirtualMachine.State state) { List clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); // check cluster is running at desired capacity include master nodes as well diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterService.java index 7693af65108a..dc313f887c61 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterService.java @@ -32,6 +32,7 @@ import com.cloud.utils.component.PluggableService; public interface KubernetesClusterService extends PluggableService { + static final String MIN_KUBERNETES_VERSION_HA_SUPPORT = "1.16"; KubernetesCluster findById(final Long id); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesServiceConfig.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesServiceConfig.java index 2c4e2330d7cd..86a717f7886b 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesServiceConfig.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesServiceConfig.java @@ -20,10 +20,9 @@ public enum KubernetesServiceConfig { + KubernetesServiceEnabled("Advanced", ManagementServer.class, Boolean.class, "cloud.kubernetes.service.enabled", "false", "Indicates whether Kubernetes Service plugin is enabled or not. Management server restart needed on change", null, null), KubernetesClusterTemplateName("Advanced", ManagementServer.class, String.class, "cloud.kubernetes.cluster.template.name", "Kubernetes-Service-Template", "Name of the template to be used for creating Kubernetes cluster nodes", null, null), - KubernetesClusterNetworkOffering("Advanced", ManagementServer.class, String.class, "cloud.kubernetes.cluster.network.offering", "DefaultNetworkOfferingforKubernetesService", "Name of the network offering that will be used to create isolated network in which Kubernetes cluster VMs will be launched.", null, null), - KubernetesClusterBinariesIsoName("Advanced", ManagementServer.class, String.class, "cloud.kubernetes.cluster.binaries.iso.name", "Kubernetes-Service-Binaries-ISO", "Name of the ISO that contains Kubernetes binaries and docker images for offline installation.", null, null); - + KubernetesClusterNetworkOffering("Advanced", ManagementServer.class, String.class, "cloud.kubernetes.cluster.network.offering", "DefaultNetworkOfferingforKubernetesService", "Name of the network offering that will be used to create isolated network in which Kubernetes cluster VMs will be launched.", null, null); private final String _category; private final Class _componentClass; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDaoImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDaoImpl.java index 4a0fd1897315..d31266675dac 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDaoImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDaoImpl.java @@ -106,7 +106,7 @@ public List listByNetworkId(long networkId) { @Override public List listAllByKubernetesVersion(long kubernetesVersionId) { - SearchCriteria sc = SameNetworkSearch.create(); + SearchCriteria sc = KubernetesVersionSearch.create(); sc.setParameters("kubernetesVersionId", kubernetesVersionId); return this.listBy(sc); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java index e9a8db5c03df..606d0ec4dec5 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java @@ -33,6 +33,7 @@ import org.apache.cloudstack.api.command.user.kubernetesversion.ListKubernetesSupportedVersionsCmd; import org.apache.cloudstack.api.response.KubernetesSupportedVersionResponse; import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.log4j.Logger; import com.cloud.api.ApiDBUtils; @@ -40,8 +41,9 @@ import com.cloud.dc.dao.DataCenterDao; import com.cloud.event.ActionEvent; import com.cloud.exception.InvalidParameterValueException; -import com.cloud.kubernetescluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetescluster.KubernetesClusterService; import com.cloud.kubernetescluster.KubernetesClusterVO; +import com.cloud.kubernetescluster.KubernetesServiceConfig; import com.cloud.kubernetescluster.dao.KubernetesClusterDao; import com.cloud.kubernetesversion.dao.KubernetesSupportedVersionDao; import com.cloud.storage.Storage; @@ -70,6 +72,8 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne private DataCenterDao dataCenterDao; @Inject private TemplateApiService templateService; + @Inject + protected ConfigurationDao globalConfigDao; private KubernetesSupportedVersionResponse createKubernetesSupportedVersionResponse(final KubernetesSupportedVersion kubernetesSupportedVersion) { KubernetesSupportedVersionResponse response = new KubernetesSupportedVersionResponse(); @@ -83,7 +87,7 @@ private KubernetesSupportedVersionResponse createKubernetesSupportedVersionRespo response.setZoneName(zone.getName()); } if (compareKubernetesVersion(kubernetesSupportedVersion.getKubernetesVersion(), - KubernetesClusterManagerImpl.MIN_KUBERNETES_VERSION_HA_SUPPORT)>=0) { + KubernetesClusterService.MIN_KUBERNETES_VERSION_HA_SUPPORT)>=0) { response.setSupportsHA(true); } else { response.setSupportsHA(false); @@ -149,6 +153,9 @@ public static boolean canUpgradeKubernetesVersion(String currentVersion, String @Override public ListResponse listKubernetesSupportedVersions(final ListKubernetesSupportedVersionsCmd cmd) { + if (!Boolean.parseBoolean(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesServiceEnabled.key()))) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Kubernetes Service plugin is disabled"); + } final Long versionId = cmd.getId(); final Long zoneId = cmd.getZoneId(); String minimumKubernetesVersion = cmd.getMinimumKubernetesVersion(); @@ -202,6 +209,9 @@ public ListResponse listKubernetesSupportedV @Override @ActionEvent(eventType = KubernetesVersionEventTypes.EVENT_KUBERNETES_VERSION_ADD, eventDescription = "Adding Kubernetes supported version") public KubernetesSupportedVersionResponse addKubernetesSupportedVersion(final AddKubernetesSupportedVersionCmd cmd) { + if (!Boolean.parseBoolean(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesServiceEnabled.key()))) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Kubernetes Service plugin is disabled"); + } final String name = cmd.getName(); final String kubernetesVersion = cmd.getKubernetesVersion(); final Long zoneId = cmd.getZoneId(); @@ -212,12 +222,14 @@ public KubernetesSupportedVersionResponse addKubernetesSupportedVersion(final Ad throw new InvalidParameterValueException("Name cannot be empty to add a new supported Kubernetes version"); } if (Strings.isNullOrEmpty(isoUrl) && (isoId == null || isoId <= 0)) { - throw new InvalidParameterValueException(String.format("Either %s or %s paramter must be passed to add a new supported Kubernetes version", "isourl", ApiConstants.ISO_ID)); + throw new InvalidParameterValueException(String.format("Either %s or %s parameter must be passed to add a new supported Kubernetes version", "isourl", ApiConstants.ISO_ID)); } - if (!Strings.isNullOrEmpty(isoUrl) && isoId != null && isoId > 0) { throw new InvalidParameterValueException(String.format("Both %s and %s parameters can not be passed simultaneously to add a new supported Kubernetes version", "isourl", ApiConstants.ISO_ID)); } + if (compareKubernetesVersion(kubernetesVersion, MIN_KUBERNETES_VERSION) < 0) { + throw new InvalidParameterValueException(String.format("New supported Kubernetes version cannot be added as %s is minimum version supported by Kubernetes Service", MIN_KUBERNETES_VERSION)); + } VMTemplateVO template = null; if (isoId != null) { @@ -281,6 +293,9 @@ public KubernetesSupportedVersionResponse addKubernetesSupportedVersion(final Ad @Override @ActionEvent(eventType = KubernetesVersionEventTypes.EVENT_KUBERNETES_VERSION_DELETE, eventDescription = "Deleting Kubernetes supported version", async = true) public boolean deleteKubernetesSupportedVersion(final DeleteKubernetesSupportedVersionCmd cmd) { + if (!Boolean.parseBoolean(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesServiceEnabled.key()))) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Kubernetes Service plugin is disabled"); + } final Long versionId = cmd.getId(); final boolean isDeleteIso = cmd.isDeleteIso(); KubernetesSupportedVersion version = kubernetesSupportedVersionDao.findById(versionId); @@ -289,7 +304,7 @@ public boolean deleteKubernetesSupportedVersion(final DeleteKubernetesSupportedV } List clusters = kubernetesClusterDao.listAllByKubernetesVersion(versionId); if (clusters.size() > 0) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Unable to delete Kubernetes version ID: %s. Exisiting clusters currently using the version.", version.getUuid())); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Unable to delete Kubernetes version ID: %s. Existing clusters currently using the version.", version.getUuid())); } VMTemplateVO template = templateDao.findById(version.getIsoId()); @@ -315,6 +330,9 @@ public boolean deleteKubernetesSupportedVersion(final DeleteKubernetesSupportedV @Override public List> getCommands() { List> cmdList = new ArrayList>(); + if (!Boolean.parseBoolean(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesServiceEnabled.key()))) { + return cmdList; + } cmdList.add(AddKubernetesSupportedVersionCmd.class); cmdList.add(ListKubernetesSupportedVersionsCmd.class); cmdList.add(DeleteKubernetesSupportedVersionCmd.class); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionService.java index 3e0b978c2fea..be11caa27cef 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionService.java @@ -26,6 +26,7 @@ import com.cloud.utils.component.PluggableService; public interface KubernetesVersionService extends PluggableService { + static final String MIN_KUBERNETES_VERSION = "1.11"; ListResponse listKubernetesSupportedVersions(ListKubernetesSupportedVersionsCmd cmd); KubernetesSupportedVersionResponse addKubernetesSupportedVersion(AddKubernetesSupportedVersionCmd cmd); boolean deleteKubernetesSupportedVersion(DeleteKubernetesSupportedVersionCmd cmd); diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/CreateKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/CreateKubernetesClusterCmd.java index 04ba93087bcb..9f2e5b6edbf1 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/CreateKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/CreateKubernetesClusterCmd.java @@ -117,6 +117,10 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { description = "number of Kubernetes cluster master nodes, default is 1") private Long masterNodes; + @Parameter(name=ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS, type = CommandType.STRING, + description = "external load balancer IP address while using shared network with Kubernetes HA cluster") + private String externalLoadBalancerIpAddress; + @Parameter(name=ApiConstants.SIZE, type = CommandType.LONG, required = true, description = "number of Kubernetes cluster worker nodes") private Long clusterSize; @@ -192,6 +196,10 @@ public Long getMasterNodes() { return masterNodes; } + public String getExternalLoadBalancerIpAddress() { + return externalLoadBalancerIpAddress; + } + public Long getClusterSize() { return clusterSize; } diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh index e92d1a94732e..a013003670ea 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh +++ b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh @@ -16,13 +16,21 @@ # specific language governing permissions and limitations # under the License. -echo "Params $#" -if [ $# -lt 1 ]; then - echo "Invalid input. Valid usage: ./upgrade-kubernetes.sh UPGRADE_VERSION" - echo "eg: ./upgrade-kubernetes.sh 1.16.3" +# Version 1.14 and below needs extra flags with kubeadm upgrade node +if [ $# -lt 2 ]; then + echo "Invalid input. Valid usage: ./upgrade-kubernetes.sh UPGRADE_VERSION IS_MASTER IS_OLD_VERSION" + echo "eg: ./upgrade-kubernetes.sh 1.16.3 true false" exit 1 fi UPGRADE_VERSION="${1}" +IS_MAIN_MASTER="" +if [ $# -gt 1 ]; then + IS_MAIN_MASTER="${2}" +fi +IS_OLD_VERSION="" +if [ $# -gt 2 ]; then + IS_OLD_VERSION="${3}" +fi export PATH=$PATH:/opt/bin @@ -86,8 +94,20 @@ if [ -d "$BINARIES_DIR" ]; then tar -f "${BINARIES_DIR}/cni/cni-plugins-amd64.tgz" -C /opt/cni/bin -xz tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-amd64.tar.gz" -C /opt/bin -xz - if [ "${UPGRADE_VERSION}" != '' ]; then + if [ "${IS_MAIN_MASTER}" == 'true' ]; then + set +e kubeadm upgrade apply ${UPGRADE_VERSION} -y + retval=$? + set -e + if [ $retval -ne 0 ]; then + kubeadm upgrade apply ${UPGRADE_VERSION} --ignore-preflight-errors=CoreDNSUnsupportedPlugins -y + fi + else + if [ "${IS_OLD_VERSION}" == 'true' ]; then + kubeadm upgrade node config --kubelet-version ${UPGRADE_VERSION} + else + kubeadm upgrade node + fi fi systemctl stop kubelet @@ -95,7 +115,7 @@ if [ -d "$BINARIES_DIR" ]; then chmod +x {kubelet,kubectl} systemctl restart kubelet - if [ "${UPGRADE_VERSION}" != '' ]; then + if [ "${IS_MAIN_MASTER}" == 'true' ]; then kubectl apply -f ${BINARIES_DIR}/network.yaml kubectl apply -f ${BINARIES_DIR}/dashboard.yaml fi diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 147c527b2739..f187840a9f6a 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -3502,6 +3502,8 @@ public Map listCapabilities(final ListCapabilitiesCmd cmd) { final boolean allowUserViewAllDomainAccounts = (QueryService.AllowUserViewAllDomainAccounts.valueIn(caller.getDomainId())); + final boolean kubernetesServiceEnabled = Boolean.parseBoolean(_configDao.getValue("cloud.kubernetes.service.enabled")); + // check if region-wide secondary storage is used boolean regionSecondaryEnabled = false; final List imgStores = _imgStoreDao.findRegionImageStores(); @@ -3521,7 +3523,8 @@ public Map listCapabilities(final ListCapabilitiesCmd cmd) { capabilities.put("KVMSnapshotEnabled", KVMSnapshotEnabled); capabilities.put("allowUserViewDestroyedVM", allowUserViewDestroyedVM); capabilities.put("allowUserExpungeRecoverVM", allowUserExpungeRecoverVM); - capabilities.put("allowUserViewAllDomainAccounts", allowUserViewAllDomainAccounts); + capabilities.put("allowUserViewAllDomainAccounts", allowUserViewAllDomainAccounts);; + capabilities.put("kubernetesServiceEnabled", kubernetesServiceEnabled); if (apiLimitEnabled) { capabilities.put("apiLimitInterval", apiLimitInterval); capabilities.put("apiLimitMax", apiLimitMax); diff --git a/ui/l10n/en.js b/ui/l10n/en.js index 4ce59e08599b..442cbb2ab51f 100644 --- a/ui/l10n/en.js +++ b/ui/l10n/en.js @@ -91,6 +91,7 @@ var dictionary = { "label.about":"About", "label.about.app":"About CloudStack", "label.accept.project.invitation":"Accept project invitation", +"label.access":"Access", "label.account":"Account", "label.accounts":"Accounts", "label.account.and.security.group":"Account, Security group", @@ -352,6 +353,8 @@ var dictionary = { "label.add.isolated.guest.network":"Add Isolated Guest Network", "label.add.isolated.guest.network.with.sourcenat":"Add Isolated Guest Network with SourceNat", "label.add.isolated.network":"Add Isolated Network", +"label.add.kubernetes.cluster":"Add Kubernetes Cluster", +"label.add.kubernetes.version":"Add Kubernetes Version", "label.add.l2.guest.network":"Add L2 Guest Network", "label.add.ldap.account":"Add LDAP account", "label.add.list.name":"ACL List Name", @@ -365,6 +368,7 @@ var dictionary = { "label.add.network.device":"Add Network Device", "label.add.network.offering":"Add network offering", "label.add.new.F5":"Add new F5", +"label.add.new.iso":"Add new ISO", "label.add.new.NetScaler":"Add new NetScaler", "label.add.new.PA":"Add new Palo Alto", "label.add.new.SRX":"Add new SRX", @@ -443,6 +447,7 @@ var dictionary = { "label.allocated":"Allocated", "label.allocation.state":"Allocation State", "label.allow":"Allow", +"label.all.zones":"All zones", "label.annotated.by":"Annotator", "label.annotation":"Annotation", "label.anti.affinity":"Anti-affinity", @@ -549,6 +554,8 @@ var dictionary = { "label.cloud.managed":"Cloud.com Managed", "label.cluster":"Cluster", "label.cluster.name":"Cluster Name", +"label.cluster.size":"Cluster size", +"label.cluster.size.worker.nodes":"Cluster size (Worker nodes)", "label.cluster.type":"Cluster Type", "label.clusters":"Clusters", "label.clvm":"CLVM", @@ -605,6 +612,7 @@ var dictionary = { "label.day":"Day", "label.day.of.month":"Day of Month", "label.day.of.week":"Day of Week", +"label.dashboard.endpoint":"Dashboard endpoint", "label.dc.name":"DC Name", "label.dead.peer.detection":"Dead Peer Detection", "label.decline.invitation":"Decline invitation", @@ -641,6 +649,8 @@ var dictionary = { "label.delete.events":"Delete events", "label.delete.gateway":"Delete gateway", "label.delete.internal.lb":"Delete Internal LB", +"label.delete.iso":"Delete ISO", +"label.delete.kubernetes.version":"Delete Kubernetes version", "label.delete.portable.ip.range":"Delete Portable IP Range", "label.delete.profile":"Delete Profile", "label.delete.project":"Delete project", @@ -658,6 +668,7 @@ var dictionary = { "label.destination.physical.network.id":"Destination physical network ID", "label.destination.zone":"Destination Zone", "label.destroy":"Destroy", +"label.destroy.kubernetes.cluster":"Destroy Kubernetes cluster", "label.destroy.router":"Destroy router", "label.destroy.vm.graceperiod":"Destroy VM Grace Period", "label.detaching.disk":"Detaching Disk", @@ -723,6 +734,7 @@ var dictionary = { "label.domain.suffix":"DNS Domain Suffix (i.e., xyz.com)", "label.done":"Done", "label.double.quotes.are.not.allowed":"Double quotes are not allowed", +"label.download.kubernetes.cluster.config":"Download Kubernetes cluster config", "label.download.progress":"Download Progress", "label.drag.new.position":"Drag to new position", "label.duration.in.sec":"Duration (in sec)", @@ -780,6 +792,7 @@ var dictionary = { "label.example":"Example", "label.expunge":"Expunge", "label.external.link":"External link", +'label.external.loadbalancer.ip.address': "External load balancer IP address", "label.extractable":"Extractable", "label.extractable.lower":"extractable", "label.f5":"F5", @@ -952,6 +965,9 @@ var dictionary = { "label.iscsi":"iSCSI", "label.iso":"ISO", "label.iso.boot":"ISO Boot", +"label.iso.id":"ISO ID", +"label.iso.name":"ISO name", +"label.iso.state":"ISO state", "label.isolated.networks":"Isolated networks", "label.isolation.method":"Isolation method", "label.isolation.mode":"Isolation Mode", @@ -963,6 +979,11 @@ var dictionary = { "label.key":"Key", "label.keyboard.language":"Keyboard language", "label.keyboard.type":"Keyboard type", +"label.kubernetes.cluster":"Kubernetes cluster", +"label.kubernetes.cluster.details":"Kubernetes cluster details", +"label.kubernetes.service":"Kubernetes Service", +"label.kubernetes.version":"Kubernetes version", +"label.kubernetes.version.details":"Kubernetes version details", "label.kvm.traffic.label":"KVM traffic label", "label.label":"Label", "label.lang.arabic":"Arabic", @@ -1030,6 +1051,7 @@ var dictionary = { "label.mac.address": "MAC Address", "label.management.servers":"Management Servers", "label.mac.address.changes":"MAC Address Changes", +"label.master.nodes":"Master nodes", "label.max.cpus":"Max. CPU cores", "label.max.guest.limit":"Max guest limit", "label.max.instances":"Max Instances", @@ -1232,6 +1254,7 @@ var dictionary = { "label.no.items":"No Available Items", "label.no.security.groups":"No Available Security Groups", "label.no.thanks":"No thanks", +"label.node.root.disk.size.gb":"Node root disk size (in GB)", "label.none":"None", "label.not.found":"Not Found", "label.notifications":"Notifications", @@ -1338,6 +1361,7 @@ var dictionary = { "label.private.key":"Private Key", "label.private.network":"Private network", "label.private.port":"Private Port", +"label.private.registry":"Private registry", "label.private.zone":"Private Zone", "label.privatekey":"PKCS#8 Private Key", "label.privatekey.name":"Private Key", @@ -1529,6 +1553,7 @@ var dictionary = { "label.save.and.continue":"Save and continue", "label.save.changes":"Save changes", "label.saving.processing":"Saving....", +"label.scale.kubernetes.cluster":"Scale Kubernetes cluster", "label.scale.up.policy":"SCALE UP POLICY", "label.scaledown.policy":"ScaleDown Policy", "label.scaleup.policy":"ScaleUp Policy", @@ -1564,6 +1589,7 @@ var dictionary = { "label.select.template":"Select Template", "label.select.tier":"Select Tier", "label.select.vm.for.static.nat":"Select VM for static NAT", +"label.semantic.version":"Semantic version", "label.sent":"Sent", "label.server":"Server", "label.service.capabilities":"Service Capabilities", @@ -1614,6 +1640,7 @@ var dictionary = { "label.sslcertificates":"SSL Certificates", "label.standard.us.keyboard":"Standard (US) keyboard", "label.start.IP":"Start IP", +"label.start.kuberentes.cluster":"Start Kubernetes cluster", "label.start.lb.vm":"Start LB VM", "label.start.port":"Start Port", "label.start.reserved.system.IP":"Start Reserved system IP", @@ -1654,6 +1681,7 @@ var dictionary = { "label.sticky.request-learn":"Request learn", "label.sticky.tablesize":"Table size", "label.stop":"Stop", +"label.stop.kuberentes.cluster":"Stop Kubernetes cluster", "label.stop.lb.vm":"Stop LB VM", "label.stopped.vms":"Stopped VMs", "label.storage":"Storage", @@ -1737,6 +1765,7 @@ var dictionary = { "label.update.ssl.cert":" SSL Certificate", "label.update.vmware.datacenter":"Update VMware datacenter", "label.updating":"Updating", +"label.upgrade.kubernetes.cluster":"Upgrade Kubernetes cluster", "label.upgrade.required":"Upgrade is required", "label.upgrade.router.newer.template":"Upgrade Router to Use Newer Template", "label.upload":"Upload", @@ -1763,6 +1792,7 @@ var dictionary = { "label.username.lower":"username", "label.users":"Users", "label.uuid":"UUID", +"label.versions":"Versions", "label.vSwitch.type":"vSwitch Type", "label.value":"Value", "label.vcdcname":"vCenter DC name", @@ -2058,8 +2088,10 @@ var dictionary = { "message.confirm.delete.ciscoASA1000v":"Please confirm you want to delete CiscoASA1000v", "message.confirm.delete.ciscovnmc.resource":"Please confirm you want to delete CiscoVNMC resource", "message.confirm.delete.internal.lb":"Please confirm you want to delete Internal LB", +"message.confirm.delete.kubernetes.version":"Please confirm that you want to delete this Kubernetes version.", "message.confirm.delete.secondary.staging.store":"Please confirm you want to delete Secondary Staging Store.", "message.confirm.delete.ucs.manager":"Please confirm that you want to delete UCS Manager", +"message.confirm.destroy.kubernetes.cluster":"Please confirm that you want to destroy this Kubernetes cluster.", "message.confirm.destroy.router":"Please confirm that you would like to destroy this router", "message.confirm.disable.host":"Please confirm that you want to disable the host", "message.confirm.disable.network.offering":"Are you sure you want to disable this network offering?", @@ -2092,7 +2124,9 @@ var dictionary = { "message.confirm.scale.up.router.vm":"Do you really want to scale up the Router VM ?", "message.confirm.scale.up.system.vm":"Do you really want to scale up the system VM ?", "message.confirm.shutdown.provider":"Please confirm that you would like to shutdown this provider", +"message.confirm.start.kubernetes.cluster":"Please confirm that you want to start this Kubernetes cluster.", "message.confirm.start.lb.vm":"Please confirm you want to start LB VM", +"message.confirm.stop.kubernetes.cluster":"Please confirm that you want to stop this Kubernetes cluster.", "message.confirm.stop.lb.vm":"Please confirm you want to stop LB VM", "message.confirm.upgrade.router.newer.template":"Please confirm that you want to upgrade router to use newer template", "message.confirm.upgrade.routers.account.newtemplate":"Please confirm that you want to upgrade all routers in this account to use newer template", diff --git a/ui/plugins/cks/cks.js b/ui/plugins/cks/cks.js index 9ec199cdbbea..86b729b20fac 100644 --- a/ui/plugins/cks/cks.js +++ b/ui/plugins/cks/cks.js @@ -49,9 +49,20 @@ cloudStack.plugins.cks = function(plugin) { plugin.ui.addSection({ id: 'cks', - title: 'Kubernetes Service', + title: 'label.kubernetes.service', preFilter: function(args) { - return true; + var pluginEnabled = false; + $.ajax({ + url: createURL('listCapabilities'), + async: false, + success: function(json) { + pluginEnabled = json.listcapabilitiesresponse.capability.kubernetesserviceenabled; + }, + error: function(XMLHttpResponse) { + pluginEnabled = false; + } + }); + return pluginEnabled; }, showOnNavigation: true, sectionSelect: { @@ -64,7 +75,7 @@ kubernetesclusters: { id: 'kubernetesclusters', type: 'select', - title: "Clusters", + title: "label.clusters", listView: { filters: { all: { @@ -137,9 +148,9 @@ // List view actions actions: { add: { - label: 'Add Kubernetes cluster', + label: 'label.add.kubernetes.cluster', createForm: { - title: 'Add Kubernetes cluster', + title: 'label.add.kubernetes.cluster', preFilter: function(args) { args.$form.find('.form-item[rel=masternodes]').find('input[name=masternodes]').val('2'); args.$form.find('.form-item[rel=size]').find('input[name=size]').val('1'); @@ -186,7 +197,7 @@ } }, kubernetesversion: { - label: 'Kubernetes version', + label: 'label.kubernetes.version', dependsOn: ['zone'], //docID: 'helpKubernetesClusterZone', validation: { @@ -268,7 +279,7 @@ } }, noderootdisksize: { - label: 'Node root disk size (in GB)', + label: 'label.node.root.disk.size.gb', //docID: 'helpKubernetesClusterNodeRootDiskSize', validation: { number: true @@ -305,13 +316,13 @@ } }, multimaster: { - label: "HA (Multi-master)", + label: "label.ha.enabled", dependsOn: 'kubernetesversion', isBoolean: true, isChecked: false, }, masternodes: { - label: 'Master nodes', + label: 'label.master.nodes', //docID: 'helpKubernetesClusterSize', validation: { required: true, @@ -320,8 +331,16 @@ dependsOn: "multimaster", isHidden: true, }, + externalloadbalanceripaddress: { + label: 'label.external.loadbalancer.ip.address', + validation: { + ipv4AndIpv6AddressValidator: true + }, + dependsOn: "multimaster", + isHidden: true, + }, size: { - label: 'Cluster size (Worker nodes)', + label: 'label.cluster.size.worker.nodes', //docID: 'helpKubernetesClusterSize', validation: { required: true, @@ -359,7 +378,7 @@ } }, supportPrivateRegistry: { - label: 'Private Registry', + label: 'label.private.registry', isBoolean: true, isChecked: false, }, @@ -419,6 +438,11 @@ var masterNodes = 1; if (args.data.multimaster === 'on') { masterNodes = args.data.masternodes; + if (args.data.externalloadbalanceripaddress != null && args.data.externalloadbalanceripaddress != "") { + $.extend(data, { + externalloadbalanceripaddress: args.data.externalloadbalanceripaddress + }); + } } $.extend(data, { masternodes: masterNodes @@ -516,11 +540,11 @@ }, detailView: { - name: 'Kubernetes cluster details', + name: 'label.kubernetes.cluster.details', isMaximized: true, actions: { start: { - label: 'Start Kubernetes Cluster', + label: 'label.start.kuberentes.cluster', action: function(args) { $.ajax({ url: createURL("startKubernetesCluster"), @@ -539,7 +563,7 @@ }, messages: { confirm: function(args) { - return 'Please confirm that you want to start this Kubernetes cluster.'; + return 'message.confirm.start.kubernetes.cluster'; }, notification: function(args) { return 'Started Kubernetes cluster.'; @@ -550,7 +574,7 @@ } }, stop: { - label: 'Stop Kubernetes Cluster', + label: 'label.stop.kuberentes.cluster', action: function(args) { $.ajax({ url: createURL("stopKubernetesCluster"), @@ -569,7 +593,7 @@ }, messages: { confirm: function(args) { - return 'Please confirm that you want to stop this Kubernetes cluster.'; + return 'message.confirm.stop.kubernetes.cluster'; }, notification: function(args) { return 'Stopped Kubernetes cluster.'; @@ -580,18 +604,18 @@ } }, destroy: { - label: 'Destroy Cluster', + label: 'label.destroy.kubernetes.cluster', compactLabel: 'label.destroy', createForm: { - title: 'Destroy Kubernetes Cluster', - desc: 'Destroy Kubernetes Cluster', + title: 'label.destroy.kubernetes.cluster', + desc: 'label.destroy.kubernetes.cluster', isWarning: true, fields: { } }, messages: { confirm: function(args) { - return 'Please confirm that you want to destroy this Kubernetes cluster.'; + return 'message.confirm.destroy.kubernetes.cluster'; }, notification: function(args) { return 'Destroyed Kubernetes cluster.'; @@ -599,8 +623,7 @@ }, action: function(args) { var data = { - id: args.context.kubernetesclusters[0].id, - expunge: true + id: args.context.kubernetesclusters[0].id }; $.ajax({ url: createURL('deleteKubernetesCluster'), @@ -624,10 +647,10 @@ } }, downloadKubernetesClusterKubeConfig: { - label: 'Download Kubernetes Cluster Config', + label: 'label.download.kubernetes.cluster.config', messages: { notification: function(args) { - return 'Download Kubernetes Cluster Config'; + return 'label.download.kubernetes.cluster.config'; } }, action: function(args) { @@ -658,14 +681,14 @@ } }, scaleKubernetesCluster: { - label: 'Scale Kubernetes Cluster', + label: 'label.scale.kubernetes.cluster', messages: { notification: function(args) { - return 'Scale Kubernetes Cluster'; + return 'label.scale.kubernetes.cluster'; } }, createForm: { - title: 'Scale Kubernetes Cluster', + title: 'label.scale.kubernetes.cluster', desc: '', preFilter: function(args) { var options = args.$form.find('.form-item[rel=serviceoffering]').find('option'); @@ -706,7 +729,7 @@ } }, size: { - label: 'Cluster size', + label: 'label.cluster.size', //docID: 'helpKubernetesClusterSize', validation: { required: true, @@ -743,19 +766,19 @@ } }, upgradeKubernetesCluster: { - label: 'Upgrade Kubernetes Cluster', + label: 'label.upgrade.kubernetes.cluster', messages: { notification: function(args) { - return 'Upgrade Kubernetes Cluster'; + return 'label.upgrade.kubernetes.cluster'; } }, createForm: { - title: 'Upgrade Kubernetes Cluster', + title: 'label.upgrade.kubernetes.cluster', desc: '', preFilter: function(args) {}, fields: { kubernetesversion: { - label: 'Kubernetes version', + label: 'label.kubernetes.version', //docID: 'helpKubernetesClusterZone', validation: { required: true @@ -835,13 +858,13 @@ label: 'label.zone.name' }, kubernetesversion: { - label: 'Kubernetes version' + label: 'label.kubernetes.version' }, masternodes : { - label: 'Master nodes' + label: 'label.master.nodes' }, size : { - label: 'Cluster Size' + label: 'label.cluster.size' }, cpunumber: { label: 'label.num.cpu.cores' @@ -861,12 +884,8 @@ keypair: { label: 'label.ssh.key.pair' }, - endpoint: { - label: 'API endpoint', - isCopyPaste: true - }, consoleendpoint: { - label: 'Dashboard endpoint', + label: 'label.dashboard.endpoint', isCopyPaste: true }, username: { @@ -898,7 +917,7 @@ } }, console : { - title: 'Access', + title: 'label.access', custom : function (args) { var showDashboard = function() { var state = args.context.kubernetesclusters[0].state; @@ -939,7 +958,7 @@ } }, clusterinstances: { - title: 'Instances', + title: 'label.instances', listView: { section: 'clusterinstances', fields: { @@ -1066,23 +1085,23 @@ kubernetesversions: { id: 'kubernetesversions', type: 'select', - title: "Versions", + title: "label.versions", listView: { fields: { name: { label: 'label.name' }, kubernetesversion: { - label: 'Kubernetes version' + label: 'label.kubernetes.version' }, zonename: { label: 'label.zone.name' }, isoname: { - label: 'ISO Name' + label: 'label.iso.name' }, isostate: { - label: 'ISO State' + label: 'label.iso.state' } }, advSearchFields: { @@ -1116,10 +1135,10 @@ // List view actions actions: { add: { - label: 'Add Kubernetes version', + label: 'label.add.kubernetes.version', preFilter: function(args) { return isAdmin(); }, createForm: { - title: 'Add Kubernetes version', + title: 'label.add.kubernetes.version', preFilter: cloudStack.preFilter.createTemplate, fields: { name: { @@ -1130,7 +1149,7 @@ } }, version: { - label: 'Semantic version', + label: 'label.semantic.version', //docID: 'Name of the cluster', validation: { required: true @@ -1163,7 +1182,7 @@ }); items.unshift({ id: -1, - description: "All Zones" + description: 'label.all.zones' }); args.response.success({ data: items @@ -1200,7 +1219,7 @@ }); items.unshift({ id: -1, - description: "Add new ISO" + description: 'label.add.new.iso' }); args.response.success({ data: items @@ -1247,7 +1266,7 @@ if (args.data.isoid < 0) { if (args.data.isourl == null || args.data.isourl == '') { cloudStack.dialog.notice({ - message: 'ISO URL is required to a new ISO' //_l('') + message: 'ISO URL is required to a new ISO' }); return; } @@ -1312,20 +1331,20 @@ }, detailView: { - name: 'Kubernetes version details', + name: 'label.kubernetes.version.details', isMaximized: true, actions: { destroy: { - label: 'Delete Version', + label: 'label.delete.kubernetes.version', compactLabel: 'label.delete', preFilter: function(args) { return isAdmin(); }, createForm: { - title: 'Delete Kubernetes Version', - desc: 'Delete Kubernetes Version', + title: 'label.delete.kubernetes.version', + desc: 'label.delete.kubernetes.version', isWarning: true, fields: { deleteiso: { - label: 'Delete ISO', + label: 'label.delete.iso', isBoolean: true, isChecked: false }, @@ -1333,7 +1352,7 @@ }, messages: { confirm: function(args) { - return 'Please confirm that you want to delete this Kubernetes version.'; + return 'message.confirm.delete.kubernetes.version'; }, notification: function(args) { return 'Deleted Kubernetes version.'; @@ -1385,13 +1404,13 @@ label: 'label.zone.name' }, isoid: { - label: 'ISO ID' + label: 'label.iso.id' }, isoname: { - label: 'ISO Name' + label: 'label.iso.name' }, isostate: { - label: 'ISO State' + label: 'label.iso.name' } }], From 051427de42d49872b8a3cdab1147af29135ee0a6 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 6 Dec 2019 13:04:48 +0530 Subject: [PATCH 016/134] increased cluster api server up timeout Signed-off-by: Abhishek Kumar --- .../cloud/kubernetescluster/KubernetesClusterManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java index debdffe6bad5..6dd06e9560a6 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java @@ -666,7 +666,7 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t setupKubernetesClusterNetworkRules(kubernetesCluster, network, account, clusterVMIds); attachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); - boolean k8sApiServerSetup = isKubernetesClusterServerRunning(kubernetesCluster, publicIpAddress,15, 30000); + boolean k8sApiServerSetup = isKubernetesClusterServerRunning(kubernetesCluster, publicIpAddress, 30, 30000); if (!k8sApiServerSetup) { String msg = String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to provision API endpoint for the cluster", kubernetesCluster.getUuid()); LOGGER.error(msg); From 93a21989d4d7c646fce4797bd1df25a16aadeb78 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 9 Dec 2019 01:41:04 +0530 Subject: [PATCH 017/134] moved binaries script to cloustack-common Signed-off-by: Abhishek Kumar --- .../scripts => scripts/util}/create-kubernetes-binaries-iso.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {plugins/integrations/kubernetes-service/scripts => scripts/util}/create-kubernetes-binaries-iso.sh (100%) mode change 100644 => 100755 diff --git a/plugins/integrations/kubernetes-service/scripts/create-kubernetes-binaries-iso.sh b/scripts/util/create-kubernetes-binaries-iso.sh old mode 100644 new mode 100755 similarity index 100% rename from plugins/integrations/kubernetes-service/scripts/create-kubernetes-binaries-iso.sh rename to scripts/util/create-kubernetes-binaries-iso.sh From 8806aa2f0d16f5c4ad82b1ea12b765806e411fa3 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 9 Dec 2019 01:41:35 +0530 Subject: [PATCH 018/134] dropped consoleendpoint column from kubernetes_cluster Signed-off-by: Abhishek Kumar --- .../src/main/resources/META-INF/db/schema-41300to41400.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql b/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql index cadf4d2172b2..014d02d9351f 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql @@ -73,7 +73,6 @@ CREATE TABLE IF NOT EXISTS `cloud`.`kubernetes_cluster` ( `memory` bigint unsigned NOT NULL COMMENT 'total memory', `node_root_disk_size` bigint(20) unsigned DEFAULT 0 COMMENT 'root disk size of root disk for each node', `endpoint` varchar(255) COMMENT 'url endpoint of the kubernetes cluster manager api access', - `console_endpoint` varchar(255) COMMENT 'url for the kubernetes cluster manager dashbaord', `created` datetime NOT NULL COMMENT 'date created', `removed` datetime COMMENT 'date removed if not null', `gc` tinyint unsigned NOT NULL DEFAULT 1 COMMENT 'gc this kubernetes cluster or not', From 16c3ce88a752e1a6f591cc7dce4a2c4b52985dbf Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 9 Dec 2019 01:42:43 +0530 Subject: [PATCH 019/134] reafctorings, fixes Signed-off-by: Abhishek Kumar --- .../kubernetescluster/KubernetesCluster.java | 1 - .../KubernetesClusterManagerImpl.java | 480 +++++++++++------- .../KubernetesClusterVO.java | 15 +- .../response/KubernetesClusterResponse.java | 6 +- 4 files changed, 295 insertions(+), 207 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesCluster.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesCluster.java index 7c76c052c570..b4d2c16522c5 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesCluster.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesCluster.java @@ -127,7 +127,6 @@ enum State { long getMemory(); long getNodeRootDiskSize(); String getEndpoint(); - String getConsoleEndpoint(); boolean isCheckForGc(); @Override State getState(); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java index 6dd06e9560a6..9ea17a6ffc1e 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java @@ -25,7 +25,9 @@ import java.lang.reflect.Field; import java.math.BigInteger; import java.net.InetAddress; +import java.net.InetSocketAddress; import java.net.MalformedURLException; +import java.net.Socket; import java.net.URL; import java.net.UnknownHostException; import java.nio.charset.Charset; @@ -34,6 +36,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -92,7 +95,6 @@ import com.cloud.dc.VlanVO; import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.DataCenterDao; -import com.cloud.dc.dao.HostPodDao; import com.cloud.dc.dao.VlanDao; import com.cloud.deploy.DeployDestination; import com.cloud.exception.ConcurrentOperationException; @@ -275,8 +277,10 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne protected LoadBalancingRulesService lbService; @Inject protected VlanDao vlanDao; - @Inject - protected HostPodDao hostPodDao; + + private static final String CLUSTER_NODE_VM_USER = "core"; + private static final int CLUSTER_API_PORT = 6443; + private static final int CLUSTER_NODES_DEFAULT_START_SSH_PORT = 2222; private static String getStackTrace(final Throwable throwable) { final StringWriter sw = new StringWriter(); @@ -378,7 +382,7 @@ private boolean isKubernetesClusterServerRunning(KubernetesCluster kubernetesClu boolean k8sApiServerSetup = false; while (retryCounter < retries) { try { - String versionOutput = IOUtils.toString(new URL(String.format("https://%s:%d/version", ipAddress, 6443)), StandardCharsets.UTF_8); + String versionOutput = IOUtils.toString(new URL(String.format("https://%s:%d/version", ipAddress, CLUSTER_API_PORT)), StandardCharsets.UTF_8); if (!Strings.isNullOrEmpty(versionOutput)) { LOGGER.debug(String.format("Kubernetes cluster ID: %s API has been successfully provisioned, %s", kubernetesCluster.getUuid(), versionOutput)); k8sApiServerSetup = true; @@ -402,7 +406,7 @@ private String getKubernetesClusterConfig(KubernetesCluster kubernetesCluster, S String kubeConfig = ""; while (retryCounter < retries) { try { - Pair result = SshHelper.sshExecute(ipAddress, port, "core", + Pair result = SshHelper.sshExecute(ipAddress, port, CLUSTER_NODE_VM_USER, getManagementServerSshPublicKeyFile(), null, "sudo cat /etc/kubernetes/admin.conf", 10000, 10000, 10000); @@ -418,13 +422,38 @@ private String getKubernetesClusterConfig(KubernetesCluster kubernetesCluster, S return kubeConfig; } - private boolean isDashboardServiceRunning(KubernetesCluster kubernetesCluster, String ipAddress, int port, int retries, long waitDuration) { + private boolean isKubernetesClusterAddOnServiceRunning(KubernetesCluster kubernetesCluster, final String ipAddress, final int port, final String namespace, String serviceName) { + try { + String cmd = "sudo kubectl get pods --all-namespaces"; + if (!Strings.isNullOrEmpty(namespace)) { + cmd = String.format("sudo kubectl get pods --namespace=%s", namespace); + } + Pair result = SshHelper.sshExecute(ipAddress, port, CLUSTER_NODE_VM_USER, + getManagementServerSshPublicKeyFile(), null, cmd, + 10000, 10000, 10000); + if (result.first() && !Strings.isNullOrEmpty(result.second())) { + String[] lines = result.second().split("\n"); + for (String line : + lines) { + if (line.contains(serviceName) && line.contains("Running")) { + LOGGER.debug(String.format("Service : %s in namespace: %s for the Kubernetes cluster ID: %s is running",serviceName, namespace, kubernetesCluster.getUuid())); + return true; + } + } + } + } catch (Exception e) { + LOGGER.warn(String.format("Unable to retrieve service: %s running status in namespace %s for Kubernetes cluster ID: %s", serviceName, namespace, kubernetesCluster.getUuid()), e); + } + return false; + } + + private boolean isKubernetesClusterDashboardServiceRunning(KubernetesCluster kubernetesCluster, String ipAddress, int port, int retries, long waitDuration) { boolean running = false; int retryCounter = 0; // Check if dashboard service is up running. while (retryCounter < retries) { LOGGER.debug(String.format("Checking dashboard service for the Kubernetes cluster ID: %s to come up. Attempt: %d/%d", kubernetesCluster.getUuid(), retryCounter+1, retries)); - if (isAddOnServiceRunning(kubernetesCluster, ipAddress, port, "kubernetes-dashboard", "kubernetes-dashboard")) { + if (isKubernetesClusterAddOnServiceRunning(kubernetesCluster, ipAddress, port, "kubernetes-dashboard", "kubernetes-dashboard")) { running = true; break; } @@ -439,7 +468,7 @@ private boolean isDashboardServiceRunning(KubernetesCluster kubernetesCluster, S } private Pair getKubernetesClusterServerIpSshPort(KubernetesCluster kubernetesCluster, UserVm masterVm) { - int port = 2222; + int port = CLUSTER_NODES_DEFAULT_START_SSH_PORT; KubernetesClusterDetailsVO detail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS); if (detail != null && !Strings.isNullOrEmpty(detail.getValue())) { return new Pair<>(detail.getValue(), port); @@ -493,7 +522,7 @@ private Pair getKubernetesClusterServerIpSshPort(KubernetesClus private int getKubernetesClusterReadyNodesCount(KubernetesCluster kubernetesCluster, String ipAddress, int port) throws Exception { Pair result = SshHelper.sshExecute(ipAddress, port, - "core", getManagementServerSshPublicKeyFile(), null, + CLUSTER_NODE_VM_USER, getManagementServerSshPublicKeyFile(), null, "sudo kubectl get nodes | awk '{if ($2 == \"Ready\") print $1}' | wc -l", 10000, 10000, 20000); if (result.first()) { @@ -502,6 +531,36 @@ private int getKubernetesClusterReadyNodesCount(KubernetesCluster kubernetesClus return 0; } + private boolean isKubernetesClusterNodeReady(KubernetesCluster kubernetesCluster, String ipAddress, int port, String nodeName) throws Exception { + Pair result = SshHelper.sshExecute(ipAddress, port, + CLUSTER_NODE_VM_USER, getManagementServerSshPublicKeyFile(), null, + String.format("sudo kubectl get nodes | awk '{if ($1 == \"%s\" && $2 == \"Ready\") print $1}'", nodeName), + 10000, 10000, 20000); + return result.first() && nodeName.equals(result.second().trim()); + } + + private boolean isKubernetesClusterNodeReady(KubernetesCluster kubernetesCluster, String ipAddress, int port, String nodeName, int retries, int waitDuration) { + int retryCounter = 0; + while (retryCounter < retries) { + boolean ready = false; + try { + ready = isKubernetesClusterNodeReady(kubernetesCluster, ipAddress, port, nodeName); + } catch (Exception e) { + LOGGER.warn(String.format("Failed to retrieve state of node: %s in Kubernetes cluster ID: %s", nodeName, kubernetesCluster.getUuid()), e); + } + if (ready) { + return true; + } + try { + Thread.sleep(waitDuration); + } catch (InterruptedException ie) { + LOGGER.error(String.format("Error while waiting for Kubernetes cluster ID: %s node: %s to become ready", kubernetesCluster.getUuid(), nodeName), ie); + } + retryCounter++; + } + return false; + } + private int getKubernetesClusterReadyNodesCount(KubernetesCluster kubernetesCluster) throws Exception { Pair ipSshPort = getKubernetesClusterServerIpSshPort(kubernetesCluster); String ipAddress = ipSshPort.first(); @@ -540,6 +599,66 @@ private boolean validateKubernetesClusterReadyNodesCount(KubernetesCluster kuber return false; } + private boolean removeKubernetesClusterNode(KubernetesCluster kubernetesCluster, String ipAddress, int port, UserVm userVm, int retries, int waitDuration) { + File pkFile = getManagementServerSshPublicKeyFile(); + int retryCounter = 0; + while (retryCounter < retries) { + retryCounter++; + try { + Pair result = SshHelper.sshExecute(ipAddress, port, CLUSTER_NODE_VM_USER, + pkFile, null, String.format("sudo kubectl drain %s --ignore-daemonsets --delete-local-data", userVm.getHostName()), + 10000, 10000, 60000); + if (!result.first()) { + LOGGER.warn(String.format("Draining node: %s on VM ID: %s in Kubernetes cluster ID: %s unsuccessful", userVm.getHostName(), userVm.getUuid(), kubernetesCluster.getUuid())); + } else { + result = SshHelper.sshExecute(ipAddress, port, CLUSTER_NODE_VM_USER, + pkFile, null, String.format("sudo kubectl delete node %s", userVm.getHostName()), + 10000, 10000, 30000); + if (result.first()) { + return true; + } else { + LOGGER.warn(String.format("Deleting node: %s on VM ID: %s in Kubernetes cluster ID: %s unsuccessful", userVm.getHostName(), userVm.getUuid(), kubernetesCluster.getUuid())); + } + } + break; + } catch (Exception e) { + String msg = String.format("Failed to remove Kubernetes cluster ID: %s node: %s on VM ID: %s", kubernetesCluster.getUuid(), userVm.getHostName(), userVm.getUuid()); + LOGGER.warn(msg, e); + } + try { + Thread.sleep(waitDuration); + } catch (InterruptedException ie) { + LOGGER.error(String.format("Error while waiting for Kubernetes cluster ID: %s node: %s on VM ID: %s removal", kubernetesCluster.getUuid(), userVm.getHostName(), userVm.getUuid()), ie); + } + retryCounter++; + } + return false; + } + + private boolean uncordonKubernetesClusterNode(KubernetesCluster kubernetesCluster, String ipAddress, int port, UserVm userVm, int retries, int waitDuration) { + int retryCounter = 0; + while (retryCounter < retries) { + Pair result = null; + try { + result = SshHelper.sshExecute(ipAddress, port, CLUSTER_NODE_VM_USER, getManagementServerSshPublicKeyFile(), null, + String.format("sudo kubectl uncordon %s", userVm.getHostName()), + 10000, 10000, 30000); + if (result.first()) { + break; + } + } catch (Exception e) { + LOGGER.warn(String.format("Failed to uncordon node: %s on VM ID: %s in Kubernetes cluster ID: %s", userVm.getHostName(), userVm.getUuid(), kubernetesCluster.getUuid()), e); + } + try { + Thread.sleep(waitDuration); + } catch (InterruptedException ie) { + LOGGER.warn(String.format("Error while waiting for uncordon Kubernetes cluster ID: %s node: %s on VM ID: %s", kubernetesCluster.getUuid(), userVm.getHostName(), userVm.getUuid()), ie); + } + retryCounter++; + } + return false; + } + // perform a cold start (which will provision resources as well) private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) throws ManagementServerException { @@ -559,7 +678,7 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Unable to find zone for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); } LOGGER.debug(String.format("Starting Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.StartRequested); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.StartRequested); Account account = accountDao.findById(kubernetesCluster.getAccountId()); DeployDestination dest = null; @@ -568,7 +687,7 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t } catch (InsufficientCapacityException e) { String msg = String.format("Provisioning the cluster failed due to insufficient capacity in the Kubernetes cluster: %s", kubernetesCluster.getUuid()); LOGGER.error(msg, e); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); throw new ManagementServerException(msg, e); } final ReservationContext context = new ReservationContextImpl(null, null, null, account); @@ -577,7 +696,7 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t if (network == null) { String msg = String.format("Network for Kubernetes cluster ID: %s not found", kubernetesCluster.getUuid()); LOGGER.warn(msg); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); throw new ManagementServerException(msg); } try { @@ -586,7 +705,7 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t } catch (Exception e) { String msg = String.format("Failed to start Kubernetes cluster ID: %s as unable to start associated network ID: %s" , kubernetesCluster.getUuid(), network.getUuid()); LOGGER.error(msg, e); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); throw new ManagementServerException(msg, e); } @@ -596,7 +715,7 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t (Network.GuestType.Isolated.equals(network.getGuestType()) || kubernetesCluster.getMasterNodeCount() > 1)) { // Shared network, single-master cluster won't have an IP yet String msg = String.format("Failed to start Kubernetes cluster ID: %s as no public IP found for the cluster" , kubernetesCluster.getUuid()); LOGGER.warn(msg); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); throw new ManagementServerException(msg); } @@ -613,7 +732,7 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t } catch (Exception e) { String msg = String.format("Provisioning the master VM failed in the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); LOGGER.warn(msg, e); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); throw new ManagementServerException(msg, e); } @@ -623,7 +742,7 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t if (Strings.isNullOrEmpty(publicIpAddress)) { String msg = String.format("Failed to start Kubernetes cluster ID: %s as no public IP found for the cluster", kubernetesCluster.getUuid()); LOGGER.warn(msg); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); throw new ManagementServerException(msg); } } @@ -640,7 +759,7 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t } catch (Exception e) { String msg = String.format("Provisioning additional master VM %d/%d failed in the Kubernetes cluster ID: %s", i+1, kubernetesCluster.getMasterNodeCount(), kubernetesCluster.getUuid()); LOGGER.warn(msg, e); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); throw new ManagementServerException(msg, e); } } @@ -657,7 +776,7 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t } catch (Exception e) { String msg = String.format("Provisioning node VM %d/%d failed in the Kubernetes cluster ID: %s", i, kubernetesCluster.getNodeCount(), kubernetesCluster.getUuid()); LOGGER.warn(msg, e); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); throw new ManagementServerException(msg, e); } } @@ -666,16 +785,46 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t setupKubernetesClusterNetworkRules(kubernetesCluster, network, account, clusterVMIds); attachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); - boolean k8sApiServerSetup = isKubernetesClusterServerRunning(kubernetesCluster, publicIpAddress, 30, 30000); + boolean masterVmRunning = false; + long startTime = System.currentTimeMillis(); + while (!masterVmRunning && System.currentTimeMillis() - startTime < 5 * 60 * 1000) { + try (Socket socket = new Socket()) { + socket.connect(new InetSocketAddress(publicIpAddress, publicIpSshPort.second()), 10000); + masterVmRunning = true; + } catch (IOException e) { + LOGGER.debug(String.format("Waiting for Kubernetes cluster ID: %s master node VMs to be accessible", kubernetesCluster.getUuid())); + try { + Thread.sleep(10000); + } catch (InterruptedException ex) { + LOGGER.warn(String.format("Error while waiting for Kubernetes cluster ID: %s master node VMs to be accessible", kubernetesCluster.getUuid()), ex); + } + } + } + if (!masterVmRunning) { + String msg = String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to access master node VMs of the cluster", kubernetesCluster.getUuid()); + if (kubernetesCluster.getMasterNodeCount() > 1 && Network.GuestType.Shared.equals(network.getGuestType())) { + msg = String.format("%s. Make sure external load-balancer has port forwarding rules for SSH access on ports %d-%d and API access on port %d", + msg, + CLUSTER_NODES_DEFAULT_START_SSH_PORT, + CLUSTER_NODES_DEFAULT_START_SSH_PORT + kubernetesCluster.getTotalNodeCount() - 1, + CLUSTER_API_PORT); + } + LOGGER.error(msg); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); + detachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); + } + + boolean k8sApiServerSetup = isKubernetesClusterServerRunning(kubernetesCluster, publicIpAddress, 20, 30000); if (!k8sApiServerSetup) { String msg = String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to provision API endpoint for the cluster", kubernetesCluster.getUuid()); LOGGER.error(msg); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.CreateFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); detachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); - kubernetesCluster.setEndpoint(String.format("https://%s:%d/", publicIpAddress, 6443)); + kubernetesCluster.setEndpoint(String.format("https://%s:%d/", publicIpAddress, CLUSTER_API_PORT)); kubernetesClusterDao.update(kubernetesCluster.getId(), kubernetesCluster); int sshPort = publicIpSshPort.second(); @@ -687,27 +836,25 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t if (!k8sKubeConfigCopied) { String msg = String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to retrieve kube-config for the cluster", kubernetesCluster.getUuid()); LOGGER.error(msg); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); detachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } - kubeConfig = kubeConfig.replace(String.format("server: https://%s:6443", k8sMasterVM.getPrivateIpAddress()), - String.format("server: https://%s:6443", publicIpAddress)); + kubeConfig = kubeConfig.replace(String.format("server: https://%s:%d", k8sMasterVM.getPrivateIpAddress(), CLUSTER_API_PORT), + String.format("server: https://%s:%d", publicIpAddress, CLUSTER_API_PORT)); kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "kubeConfigData", Base64.encodeBase64String(kubeConfig.getBytes(Charset.forName("UTF-8"))), false); - boolean dashboardServiceRunning = isDashboardServiceRunning(kubernetesCluster, publicIpAddress, sshPort, 10, 20000); + boolean dashboardServiceRunning = isKubernetesClusterDashboardServiceRunning(kubernetesCluster, publicIpAddress, sshPort, 10, 20000); if (!dashboardServiceRunning) { String msg = String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to get Dashboard service running for the cluster", kubernetesCluster.getUuid()); LOGGER.error(msg); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); detachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "dashboardServiceRunning", String.valueOf(dashboardServiceRunning), false); detachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); - kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); - kubernetesCluster.setConsoleEndpoint("https://" + publicIpAddress + ":6443/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy#!/overview?namespace=_all"); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationSucceeded); - kubernetesClusterDao.update(kubernetesCluster.getId(), kubernetesCluster); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); return true; } @@ -730,13 +877,13 @@ private boolean startStoppedKubernetesCluster(long kubernetesClusterId) throws M } LOGGER.debug(String.format("Starting Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.StartRequested); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.StartRequested); for (final KubernetesClusterVmMapVO vmMapVO : kubernetesClusterVmMapDao.listByClusterId(kubernetesClusterId)) { final UserVmVO vm = userVmDao.findById(vmMapVO.getVmId()); try { if (vm == null) { - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); throw new ManagementServerException("Failed to start all VMs in Kubernetes cluster ID: " + kubernetesClusterId); } startKubernetesVM(vm, kubernetesCluster); @@ -749,7 +896,7 @@ private boolean startStoppedKubernetesCluster(long kubernetesClusterId) throws M for (final KubernetesClusterVmMapVO vmMapVO : kubernetesClusterVmMapDao.listByClusterId(kubernetesClusterId)) { final UserVmVO vm = userVmDao.findById(vmMapVO.getVmId()); if (vm == null || !vm.getState().equals(VirtualMachine.State.Running)) { - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); throw new ManagementServerException("Failed to start all VMs in Kubernetes cluster ID: " + kubernetesClusterId); } } @@ -768,7 +915,7 @@ private boolean startStoppedKubernetesCluster(long kubernetesClusterId) throws M if (Strings.isNullOrEmpty(publicIpAddress)) { String msg = String.format("Failed to start Kubernetes cluster ID: %s as no public IP found for the cluster" , kubernetesCluster.getUuid()); LOGGER.warn(msg); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); throw new ManagementServerException(msg); } @@ -776,7 +923,7 @@ private boolean startStoppedKubernetesCluster(long kubernetesClusterId) throws M if (!k8sApiServerSetup) { String msg = String.format("Failed to start Kubernetes cluster ID: %s in usable state", kubernetesCluster.getUuid()); LOGGER.error(msg); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); throw new ManagementServerException(msg); } @@ -791,33 +938,31 @@ private boolean startStoppedKubernetesCluster(long kubernetesClusterId) throws M if (!k8sKubeConfigCopied) { String msg = String.format("Failed to start Kubernetes cluster ID: %s in usable state as unable to retrieve kube-config for the cluster", kubernetesCluster.getUuid()); LOGGER.error(msg); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "kubeConfigData", Base64.encodeBase64String(kubeConfig.getBytes(Charset.forName("UTF-8"))), false); } - - if (Strings.isNullOrEmpty(kubernetesCluster.getConsoleEndpoint())) { - boolean dashboardServiceRunning = isDashboardServiceRunning(kubernetesCluster, publicIpAddress, sshPort, 10, 20000); + KubernetesClusterDetailsVO dashboardServiceRunningDetail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "dashboardServiceRunning"); + if (kubeConfigDetail == null || !Boolean.parseBoolean(dashboardServiceRunningDetail.getValue())) { + boolean dashboardServiceRunning = isKubernetesClusterDashboardServiceRunning(kubernetesCluster, publicIpAddress, sshPort, 10, 20000); if (!dashboardServiceRunning) { String msg = String.format("Failed to start Kubernetes cluster ID: %s in usable state as unable to get Dashboard service running for the cluster", kubernetesCluster.getUuid()); LOGGER.error(msg); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } - KubernetesClusterVO cluster = kubernetesClusterDao.findById(kubernetesCluster.getId()); - cluster.setConsoleEndpoint("https://" + publicIpAddress + ":6443/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy#!/overview?namespace=_all"); - kubernetesClusterDao.update(cluster.getId(), cluster); + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "dashboardServiceRunning", String.valueOf(dashboardServiceRunning), false); } - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationSucceeded); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); LOGGER.debug(String.format("Kubernetes cluster ID: %s successfully started", kubernetesCluster.getUuid())); return true; } - // Open up firewall port 6443, secure port on which Kubernetes API server is running. Also create port-forwarding + // Open up firewall port CLUSTER_API_PORT, secure port on which Kubernetes API server is running. Also create port-forwarding // rule to forward public IP traffic to master VM private IP - // Open up firewall ports 2222 to 2222+n for SSH access. Also create port-forwarding + // Open up firewall ports NODES_DEFAULT_START_SSH_PORT to NODES_DEFAULT_START_SSH_PORT+n for SSH access. Also create port-forwarding // rule to forward public IP traffic to all node VM private IP private void setupKubernetesClusterNetworkRules(KubernetesCluster kubernetesCluster, Network network, Account account, @@ -859,11 +1004,11 @@ private void setupKubernetesClusterNetworkRules(KubernetesCluster kubernetesClus Field startPortField = rule.getClass().getDeclaredField("publicStartPort"); startPortField.setAccessible(true); - startPortField.set(rule, new Integer(6443)); + startPortField.set(rule, CLUSTER_API_PORT); Field endPortField = rule.getClass().getDeclaredField("publicEndPort"); endPortField.setAccessible(true); - endPortField.set(rule, new Integer(6443)); + endPortField.set(rule, CLUSTER_API_PORT); Field cidrField = rule.getClass().getDeclaredField("cidrlist"); cidrField.setAccessible(true); @@ -872,8 +1017,8 @@ private void setupKubernetesClusterNetworkRules(KubernetesCluster kubernetesClus firewallService.createIngressFirewallRule(rule); firewallService.applyIngressFwRules(publicIp.getId(), account); - LOGGER.debug(String.format("Provisioned firewall rule to open up port 6443 on %s for Kubernetes cluster ID: %s", - publicIp.getAddress().addr(), kubernetesCluster.getUuid())); + LOGGER.debug(String.format("Provisioned firewall rule to open up port %d on %s for Kubernetes cluster ID: %s", + CLUSTER_API_PORT, publicIp.getAddress().addr(), kubernetesCluster.getUuid())); } catch (Exception e) { String msg = String.format("Failed to provision firewall rules for API access for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); LOGGER.warn(msg, e); @@ -895,11 +1040,11 @@ private void setupKubernetesClusterNetworkRules(KubernetesCluster kubernetesClus Field startPortField = rule.getClass().getDeclaredField("publicStartPort"); startPortField.setAccessible(true); - startPortField.set(rule, 2222); + startPortField.set(rule, CLUSTER_NODES_DEFAULT_START_SSH_PORT); Field endPortField = rule.getClass().getDeclaredField("publicEndPort"); endPortField.setAccessible(true); - int endPort = 2222 + clusterVMIds.size() - 1; + int endPort = CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterVMIds.size() - 1; endPortField.set(rule, endPort); // clusterVMIds contains all nodes including master Field cidrField = rule.getClass().getDeclaredField("cidrlist"); @@ -909,7 +1054,7 @@ private void setupKubernetesClusterNetworkRules(KubernetesCluster kubernetesClus firewallService.createIngressFirewallRule(rule); firewallService.applyIngressFwRules(publicIp.getId(), account); - LOGGER.debug(String.format("Provisioned firewall rule to open up port 2222 to %d on %s for Kubernetes cluster ID: %s", endPort, publicIp.getAddress().addr(), kubernetesCluster.getUuid())); + LOGGER.debug(String.format("Provisioned firewall rule to open up port %d to %d on %s for Kubernetes cluster ID: %s", CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort, publicIp.getAddress().addr(), kubernetesCluster.getUuid())); } catch (Exception e) { String msg = String.format("Failed to provision firewall rules for SSH access for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); LOGGER.warn(msg, e); @@ -920,7 +1065,7 @@ private void setupKubernetesClusterNetworkRules(KubernetesCluster kubernetesClus // Load balancer rule fo API access for master node VMs try { LoadBalancer lb = lbService.createPublicLoadBalancerRule(null, "api-lb", "LB rule for API access", - 6443, 6443, 6443, 6443, + CLUSTER_API_PORT, CLUSTER_API_PORT, CLUSTER_API_PORT, CLUSTER_API_PORT, publicIp.getId(), NetUtils.TCP_PROTO, "roundrobin", kubernetesCluster.getNetworkId(), kubernetesCluster.getAccountId(), false, NetUtils.TCP_PROTO, true); @@ -950,7 +1095,7 @@ private void setupKubernetesClusterNetworkRules(KubernetesCluster kubernetesClus Nic vmNic = networkModel.getNicInNetwork(vmId, kubernetesCluster.getNetworkId()); final Ip vmIp = new Ip(vmNic.getIPv4Address()); final long vmIdFinal = vmId; - final int srcPortFinal = 2222 + i; + final int srcPortFinal = CLUSTER_NODES_DEFAULT_START_SSH_PORT + i; try { PortForwardingRuleVO pfRule = Transaction.execute(new TransactionCallbackWithException() { @Override @@ -978,7 +1123,7 @@ public PortForwardingRuleVO doInTransaction(TransactionStatus status) throws Net } } - // Open up firewall ports 2222 to 2222+n for SSH access. Also create port-forwarding + // Open up firewall ports NODES_DEFAULT_START_SSH_PORT to NODES_DEFAULT_START_SSH_PORT+n for SSH access. Also create port-forwarding // rule to forward public IP traffic to all node VM private IP. Existing node VMs before scaling // will already be having these rules private void scaleKubernetesClusterNetworkRules(KubernetesCluster kubernetesCluster, Network network, Account account, @@ -1009,10 +1154,10 @@ private void scaleKubernetesClusterNetworkRules(KubernetesCluster kubernetesClus List sourceCidrList = new ArrayList(); sourceCidrList.add("0.0.0.0/0"); boolean firewallRuleFound = false; - int existingFirewallRuleSourcePortEnd = 2222; + int existingFirewallRuleSourcePortEnd = CLUSTER_NODES_DEFAULT_START_SSH_PORT; List firewallRules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(publicIp.getId(), FirewallRule.Purpose.Firewall); for (FirewallRuleVO firewallRule : firewallRules) { - if (firewallRule.getSourcePortStart() == 2222) { + if (firewallRule.getSourcePortStart() == CLUSTER_NODES_DEFAULT_START_SSH_PORT) { firewallRuleFound = true; existingFirewallRuleSourcePortEnd = firewallRule.getSourcePortEnd(); firewallService.revokeIngressFwRule(firewallRule.getId(), true); @@ -1036,11 +1181,11 @@ private void scaleKubernetesClusterNetworkRules(KubernetesCluster kubernetesClus Field startPortField = rule.getClass().getDeclaredField("publicStartPort"); startPortField.setAccessible(true); - startPortField.set(rule, new Integer(2222)); + startPortField.set(rule, CLUSTER_NODES_DEFAULT_START_SSH_PORT); Field endPortField = rule.getClass().getDeclaredField("publicEndPort"); endPortField.setAccessible(true); - endPortField.set(rule, new Integer(2222 + (int)kubernetesCluster.getNodeCount())); + endPortField.set(rule, CLUSTER_NODES_DEFAULT_START_SSH_PORT + (int)kubernetesCluster.getNodeCount()); Field cidrField = rule.getClass().getDeclaredField("cidrlist"); cidrField.setAccessible(true); @@ -1049,7 +1194,8 @@ private void scaleKubernetesClusterNetworkRules(KubernetesCluster kubernetesClus firewallService.createIngressFirewallRule(rule); firewallService.applyIngressFwRules(publicIp.getId(), account); - LOGGER.debug(String.format("Provisioned firewall rule to open up port 2222 to %d on %s in Kubernetes cluster ID: %s", 2222 + (int)kubernetesCluster.getNodeCount(), publicIp.getAddress().addr(), kubernetesCluster.getName())); + LOGGER.debug(String.format("Provisioned firewall rule to open up port %d to %d on %s in Kubernetes cluster ID: %s", + CLUSTER_NODES_DEFAULT_START_SSH_PORT, CLUSTER_NODES_DEFAULT_START_SSH_PORT + (int)kubernetesCluster.getNodeCount(), publicIp.getAddress().addr(), kubernetesCluster.getName())); } catch (Exception e) { String msg = String.format("Failed to activate SSH firewall rules for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); LOGGER.warn(msg, e); @@ -1157,10 +1303,10 @@ private boolean validateNetwork(Network network, int clusterTotalNodeCount) { Integer startPort = rule.getSourcePortStart(); Integer endPort = rule.getSourcePortEnd(); LOGGER.debug("Network rule : " + startPort + " " + endPort); - if (startPort <= 6443 && 6443 <= endPort) { + if (startPort <= CLUSTER_API_PORT && CLUSTER_API_PORT <= endPort) { throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting firewall rules to provision Kubernetes cluster for API access", network.getUuid())); } - if (startPort <= 2222 && 2222+clusterTotalNodeCount <= endPort) { + if (startPort <= CLUSTER_NODES_DEFAULT_START_SSH_PORT && CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterTotalNodeCount <= endPort) { throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting firewall rules to provision Kubernetes cluster for node VM SSH access", network.getUuid())); } } @@ -1169,10 +1315,10 @@ private boolean validateNetwork(Network network, int clusterTotalNodeCount) { Integer startPort = rule.getSourcePortStart(); Integer endPort = rule.getSourcePortEnd(); LOGGER.debug("Network rule : " + startPort + " " + endPort); - if (startPort <= 6443 && 6443 <= endPort) { + if (startPort <= CLUSTER_API_PORT && CLUSTER_API_PORT <= endPort) { throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting port forwarding rules to provision Kubernetes cluster for API access", network.getUuid())); } - if (startPort <= 2222 && 2222+clusterTotalNodeCount <= endPort) { + if (startPort <= CLUSTER_NODES_DEFAULT_START_SSH_PORT && CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterTotalNodeCount <= endPort) { throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting port forwarding rules to provision Kubernetes cluster for node VM SSH access", network.getUuid())); } } @@ -1275,31 +1421,6 @@ private DeployDestination plan(final KubernetesCluster kubernetesCluster, final return plan(kubernetesCluster.getTotalNodeCount(), zone, offering); } - private boolean isAddOnServiceRunning(KubernetesCluster kubernetesCluster, final String ipAddress, final int port, final String namespace, String serviceName) { - try { - String cmd = "sudo kubectl get pods --all-namespaces"; - if (!Strings.isNullOrEmpty(namespace)) { - cmd = String.format("sudo kubectl get pods --namespace=%s", namespace); - } - Pair result = SshHelper.sshExecute(ipAddress, port, "core", - getManagementServerSshPublicKeyFile(), null, cmd, - 10000, 10000, 10000); - if (result.first() && !Strings.isNullOrEmpty(result.second())) { - String[] lines = result.second().split("\n"); - for (String line : - lines) { - if (line.contains(serviceName) && line.contains("Running")) { - LOGGER.debug(String.format("Service : %s in namespace: %s for the Kubernetes cluster ID: %s is running",serviceName, namespace, kubernetesCluster.getUuid())); - return true; - } - } - } - } catch (Exception e) { - LOGGER.warn(String.format("Unable to retrieve service: %s running status in namespace %s for Kubernetes cluster ID: %s", serviceName, namespace, kubernetesCluster.getUuid()), e); - } - return false; - } - protected boolean cleanupKubernetesClusterResources(Long kubernetesClusterId) throws ManagementServerException { KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); if (!(kubernetesCluster.getState().equals(KubernetesCluster.State.Running) @@ -1311,7 +1432,7 @@ protected boolean cleanupKubernetesClusterResources(Long kubernetesClusterId) th LOGGER.warn(msg); throw new PermissionDeniedException(msg); } - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.DestroyRequested); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.DestroyRequested); boolean failedVmDestroy = false; List clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); if ((clusterVMs != null) && !clusterVMs.isEmpty()) { @@ -1406,7 +1527,7 @@ protected boolean cleanupKubernetesClusterResources(Long kubernetesClusterId) th throw new ManagementServerException(msg); } - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationSucceeded); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); kubernetesCluster.setCheckForGc(false); @@ -1510,8 +1631,9 @@ private UserVm createKubernetesMaster(final KubernetesClusterVO kubernetesCluste k8sMasterConfig = k8sMasterConfig.replace(clusterToken, generateClusterToken(kubernetesCluster)); String initArgs = ""; if (haSupported) { - initArgs = String.format("--control-plane-endpoint %s:6443 --upload-certs --certificate-key %s ", + initArgs = String.format("--control-plane-endpoint %s:%d --upload-certs --certificate-key %s ", serverIp, + CLUSTER_API_PORT, generateClusterHACertificateKey(kubernetesCluster)); } initArgs += String.format("--apiserver-cert-extra-sans=%s", serverIp); @@ -1786,7 +1908,6 @@ public KubernetesClusterResponse createKubernetesClusterResponse(long kubernetes response.setEndpoint(kubernetesCluster.getEndpoint()); response.setNetworkId(ntwk.getUuid()); response.setAssociatedNetworkName(ntwk.getName()); - response.setConsoleEndpoint(kubernetesCluster.getConsoleEndpoint()); List vmIds = new ArrayList(); List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); if (vmList != null && !vmList.isEmpty()) { @@ -1973,7 +2094,7 @@ public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) public KubernetesClusterVO doInTransaction(TransactionStatus status) { KubernetesClusterVO newCluster = new KubernetesClusterVO(name, displayName, zoneId, clusterKubernetesVersion.getId(), serviceOfferingId, finalTemplate.getId(), defaultNetwork.getId(), owner.getDomainId(), - owner.getAccountId(), masterNodeCount, clusterSize, KubernetesCluster.State.Created, sshKeyPair, cores, memory, nodeRootDiskSize, "", ""); + owner.getAccountId(), masterNodeCount, clusterSize, KubernetesCluster.State.Created, sshKeyPair, cores, memory, nodeRootDiskSize, ""); kubernetesClusterDao.persist(newCluster); return newCluster; } @@ -2057,13 +2178,13 @@ public boolean stopKubernetesCluster(long kubernetesClusterId) throws Management LOGGER.debug(String.format("Stopping Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.StopRequested); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.StopRequested); for (final KubernetesClusterVmMapVO vmMapVO : kubernetesClusterVmMapDao.listByClusterId(kubernetesClusterId)) { final UserVmVO vm = userVmDao.findById(vmMapVO.getVmId()); try { if (vm == null) { - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); throw new ManagementServerException(String.format("Failed to find all VMs in Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); } stopClusterVM(vmMapVO); @@ -2076,12 +2197,12 @@ public boolean stopKubernetesCluster(long kubernetesClusterId) throws Management for (final KubernetesClusterVmMapVO vmMapVO : kubernetesClusterVmMapDao.listByClusterId(kubernetesClusterId)) { final UserVmVO vm = userVmDao.findById(vmMapVO.getVmId()); if (vm == null || !vm.getState().equals(VirtualMachine.State.Stopped)) { - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); throw new ManagementServerException(String.format("Failed to stop all VMs in Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); } } - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationSucceeded); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); return true; } @@ -2245,7 +2366,7 @@ public boolean scaleKubernetesCluster(ScaleKubernetesClusterCmd cmd) throws Mana if (serviceOfferingScalingNeeded) { List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); if (vmList == null || vmList.isEmpty() || vmList.size() - 1 < originalNodeCount) { - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, it is in unstable state as not enough existing VM instances found!", kubernetesCluster.getUuid())); } else { for (KubernetesClusterVmMapVO vmMapVO : vmList) { @@ -2261,9 +2382,9 @@ public boolean scaleKubernetesCluster(ScaleKubernetesClusterCmd cmd) throws Mana throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, service offering for the Kubernetes cluster cannot be scaled down!", kubernetesCluster.getUuid())); } - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.ScaleUpRequested); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.ScaleUpRequested); - final long size = (clusterSize == null ? kubernetesCluster.getNodeCount() : clusterSize); + final long size = (clusterSize == null ? kubernetesCluster.getTotalNodeCount() : kubernetesCluster.getMasterNodeCount() + clusterSize); final long cores = serviceOffering.getCpu() * size; final long memory = serviceOffering.getRamSize() * size; kubernetesCluster = Transaction.execute(new TransactionCallback() { @@ -2279,7 +2400,7 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { } }); if (kubernetesCluster == null) { - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, unable to update Kubernetes cluster!", kubernetesCluster.getUuid())); } kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); @@ -2291,11 +2412,11 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { try { result = userVmManager.upgradeVirtualMachine(userVM.getId(), serviceOffering.getId(), new HashMap()); } catch (Exception e) { - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, unable to scale cluster VM ID: %s! %s", kubernetesCluster.getUuid(), userVM.getUuid(), e.getMessage()), e); } if (!result) { - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, unable to scale cluster VM ID: %s!", kubernetesCluster.getUuid(), userVM.getUuid())); } } @@ -2306,7 +2427,7 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { if (network == null) { String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, cluster network not found", kubernetesCluster.getUuid()); LOGGER.error(msg); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } // Check capacity and transition state @@ -2315,12 +2436,12 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { if (clusterServiceOffering == null) { String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, cluster service offering not found", kubernetesCluster.getUuid()); LOGGER.error(msg); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } if (newVmRequiredCount > 0) { if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.ScaleUpRequested); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.ScaleUpRequested); } try { if (clusterState.equals(KubernetesCluster.State.Running)) { @@ -2331,19 +2452,19 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { } catch (InsufficientCapacityException e) { String msg = String.format("Scaling failed for Kubernetes cluster ID: %s in zone ID: %s, insufficient capacity", kubernetesCluster.getUuid(), zone.getUuid()); LOGGER.error(msg); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); throw new ServerApiException(ApiErrorCode.INSUFFICIENT_CAPACITY_ERROR, msg, e); } } else { if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.ScaleDownRequested); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.ScaleDownRequested); } } if (!serviceOfferingScalingNeeded) { // Else already updated // Update KubernetesClusterVO - final long cores = clusterServiceOffering.getCpu() * clusterSize; - final long memory = clusterServiceOffering.getRamSize() * clusterSize; + final long cores = clusterServiceOffering.getCpu() * (kubernetesCluster.getMasterNodeCount() + clusterSize); + final long memory = clusterServiceOffering.getRamSize() * (kubernetesCluster.getMasterNodeCount() + clusterSize); kubernetesCluster = Transaction.execute(new TransactionCallback() { @Override @@ -2359,7 +2480,7 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { if (kubernetesCluster == null) { String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, unable to update cluster", kubernetesCluster.getUuid()); LOGGER.warn(msg); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); @@ -2368,10 +2489,16 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { // Perform size scaling if (clusterState.equals(KubernetesCluster.State.Running)) { List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + vmList.sort(new Comparator() { + @Override + public int compare(KubernetesClusterVmMapVO kubernetesClusterVmMapVO, KubernetesClusterVmMapVO t1) { + return (int)((kubernetesClusterVmMapVO.getId() - t1.getId())/Math.abs(kubernetesClusterVmMapVO.getId() - t1.getId())); + } + }); if (vmList == null || vmList.isEmpty() || vmList.size() - 1 < originalNodeCount) { String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, t is in unstable state as not enough existing VM instances found", kubernetesCluster.getUuid()); LOGGER.error(msg); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } @@ -2388,41 +2515,16 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { if (newVmRequiredCount < 0) { // downscale int i = vmList.size() - 1; List removedVmIds = new ArrayList<>(); - while (i > 1 && vmList.size() > clusterSize + 1) { // Reverse order as first VM will be k8s master + while (i > kubernetesCluster.getMasterNodeCount() && vmList.size() > kubernetesCluster.getTotalNodeCount()) { // Reverse order as first VM will be k8s master KubernetesClusterVmMapVO vmMapVO = vmList.get(i); UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId()); // Gracefully remove-delete k8s node - int retryCounter = 0; - int maxRetries = 3; - while (retryCounter < maxRetries) { - retryCounter++; - try { - Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, "core", - pkFile, null, String.format("sudo kubectl drain %s --ignore-daemonsets --delete-local-data", userVM.getHostName()), - 10000, 10000, 30000); - if (!result.first()) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Draining Kubernetes node unsuccessful"); - } - result = SshHelper.sshExecute(publicIpAddress, sshPort, "core", - pkFile, null, String.format("sudo kubectl delete node %s", userVM.getHostName()), - 10000, 10000, 30000); - if (!result.first()) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Deleting Kubernetes node unsuccessful"); - } - break; - } catch (Exception e) { - if (retryCounter < maxRetries) { - try { - Thread.sleep(30000); - } catch (InterruptedException ie) {} - } else { - String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, failed to remove Kubernetes node", kubernetesCluster.getUuid()); - LOGGER.warn(msg, e); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); - } - } + if (!removeKubernetesClusterNode(kubernetesCluster, publicIpAddress, sshPort, userVM, 3, 30000)) { + String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, failed to remove Kubernetes node: %s running on VM ID: %s", kubernetesCluster.getUuid(), userVM.getHostName(), userVM.getUuid()); + LOGGER.warn(msg); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } // For removing port-forwarding network rules @@ -2431,7 +2533,7 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { // Expunge VM UserVm vm = userVmService.destroyVm(userVM.getId(), true); if (!VirtualMachine.State.Expunging.equals(vm.getState())) { - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, VM '%s' is now in state '%s'." , kubernetesCluster.getUuid() , vm.getInstanceName() @@ -2457,7 +2559,7 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { } catch (Exception e) { String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, unable to update network rules", kubernetesCluster.getUuid()); LOGGER.error(msg, e); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg, e); } } else { // upscale, same node count handled above @@ -2477,7 +2579,7 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { } catch (Exception e) { String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, unable to provision node VM in the cluster", kubernetesCluster.getUuid()); LOGGER.error(msg, e); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg, e); } } @@ -2488,7 +2590,7 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { } catch (Exception e) { String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, unable to update network rules", kubernetesCluster.getUuid()); LOGGER.error(msg, e); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg, e); } @@ -2496,22 +2598,22 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { attachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); // Check if new nodes are added in k8s cluster - boolean readyNodesCountVerified = validateKubernetesClusterReadyNodesCount(kubernetesCluster, publicIpAddress, sshPort, 15, 30000); + boolean readyNodesCountValid = validateKubernetesClusterReadyNodesCount(kubernetesCluster, publicIpAddress, sshPort, 20, 30000); // Detach binaries ISO from new VMs detachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); // Throw exception if nodes count for k8s cluster timed out - if (!readyNodesCountVerified) { // Scaling failed + if (!readyNodesCountValid) { // Scaling failed String msg = String.format("Scaling unsuccessful for Kubernetes cluster ID: %s as it does not have desired number of nodes in ready state", kubernetesCluster.getUuid()); LOGGER.warn(msg); - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } } } } - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationSucceeded); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); return true; } @@ -2605,7 +2707,7 @@ public boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws UserVm vm = userVmDao.findById(vmIds.get(i)); result = null; try { - result = SshHelper.sshExecute(publicIpAddress, sshPort, "core", pkFile, null, + result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, pkFile, null, String.format("sudo kubectl drain %s --ignore-daemonsets --delete-local-data", vm.getHostName()), 10000, 10000, 60000); } catch (Exception e) { @@ -2625,12 +2727,12 @@ public boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws try { int nodeSshPort = sshPort == 22 ? sshPort : sshPort + i; String nodeAddress = (i > 0 && sshPort == 22) ? vm.getPrivateIpAddress() : publicIpAddress; - SshHelper.scpTo(nodeAddress, nodeSshPort, "core", pkFile, null, + SshHelper.scpTo(nodeAddress, nodeSshPort, CLUSTER_NODE_VM_USER, pkFile, null, "~/", upgradeScriptFile.getAbsolutePath(), "0755"); String cmdStr = String.format("sudo ./%s %s %s %s", upgradeScriptFile.getName(), upgradeVersion.getKubernetesVersion(), i == 0 ? "true" : "false", KubernetesVersionManagerImpl.compareKubernetesVersion(upgradeVersion.getKubernetesVersion(), "1.15") < 0 ? "true" : "false"); - result = SshHelper.sshExecute(publicIpAddress, nodeSshPort, "core", pkFile, null, + result = SshHelper.sshExecute(publicIpAddress, nodeSshPort, CLUSTER_NODE_VM_USER, pkFile, null, cmdStr, 10000, 10000, 5 * 60 * 1000); } catch (Exception e) { @@ -2647,42 +2749,20 @@ public boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } - int retryCounter = 0; - int maxRetries = 3; - while (retryCounter < maxRetries) { - try { - result = SshHelper.sshExecute(publicIpAddress, sshPort, "core", pkFile, null, - String.format("sudo kubectl uncordon %s", vm.getHostName()), - 10000, 10000, 30000); - } catch (Exception e) { - String msg = String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to uncordon Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()); - LOGGER.error(msg, e); - detachIsoKubernetesVMs(kubernetesCluster, vmIds); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg, e); - } - if (result.first()) { - break; - } - try { - Thread.sleep(30000); - } catch (InterruptedException ie) { - LOGGER.warn(String.format("Error while waiting for uncordon Kubernetes cluster ID: %s node running on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()), ie); - } - retryCounter++; - } - if (!result.first()) { + if (!uncordonKubernetesClusterNode(kubernetesCluster, publicIpAddress, sshPort, vm, 3, 30000)) { String msg = String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to uncordon Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()); - LOGGER.error(String.format("%s. Output:\n%s", msg, Strings.nullToEmpty(result.second()))); + LOGGER.error(msg); detachIsoKubernetesVMs(kubernetesCluster, vmIds); stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } if (i == 0) { // Wait for master to get in Ready state - try { - Thread.sleep(30000); - } catch (InterruptedException ie) { - LOGGER.warn(String.format("Error while waiting for master to become ready for Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), ie); + if (!isKubernetesClusterNodeReady(kubernetesCluster, publicIpAddress, sshPort, vm.getHostName(), 5, 20000)) { + String msg = String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to get master Kubernetes node on VM ID: %s in ready state", kubernetesCluster.getUuid(), vm.getUuid()); + LOGGER.error(msg); + detachIsoKubernetesVMs(kubernetesCluster, vmIds); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } } } @@ -2822,13 +2902,39 @@ public void reallyRun() { // run through Kubernetes clusters in 'Alert' state and reconcile state as 'Running' if the VM's are running List alertKubernetesClusters = kubernetesClusterDao.findKubernetesClustersInState(KubernetesCluster.State.Alert); - for (KubernetesCluster kubernetesCluster : alertKubernetesClusters) { + for (KubernetesClusterVO kubernetesCluster : alertKubernetesClusters) { if (LOGGER.isDebugEnabled()) { LOGGER.debug("Running Kubernetes cluster state scanner on Kubernetes cluster name:" + kubernetesCluster.getName() + " for state " + KubernetesCluster.State.Alert); } try { if (isClusterVMsInDesiredState(kubernetesCluster, VirtualMachine.State.Running) && kubernetesCluster.getTotalNodeCount() == getKubernetesClusterReadyNodesCount(kubernetesCluster)) { + Pair sshIpPort = getKubernetesClusterServerIpSshPort(kubernetesCluster); + if (Strings.isNullOrEmpty(sshIpPort.first())) { + continue; + } + if (!isKubernetesClusterServerRunning(kubernetesCluster, sshIpPort.first(), 1, 0)) { + continue; + } + if (Strings.isNullOrEmpty(kubernetesCluster.getEndpoint())) { + kubernetesCluster.setEndpoint(String.format("https://%s:%d/", sshIpPort.first(), CLUSTER_API_PORT)); + } + KubernetesClusterDetailsVO kubeConfigDetail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "kubeConfigData"); + if (kubeConfigDetail == null || Strings.isNullOrEmpty(kubeConfigDetail.getValue())) { + String kubeConfig = getKubernetesClusterConfig(kubernetesCluster, sshIpPort.first(), sshIpPort.second(), 1); + if (Strings.isNullOrEmpty(kubeConfig)) { + continue; + } + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "kubeConfigData", Base64.encodeBase64String(kubeConfig.getBytes(Charset.forName("UTF-8"))), false); + } + KubernetesClusterDetailsVO dashboardServiceRunningDetail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "dashboardServiceRunning"); + if (kubeConfigDetail == null || !Boolean.parseBoolean(dashboardServiceRunningDetail.getValue())) { + boolean dashboardServiceRunning = isKubernetesClusterDashboardServiceRunning(kubernetesCluster, sshIpPort.first(), sshIpPort.second(), 1, 0); + if (!dashboardServiceRunning) { + continue; + } + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "dashboardServiceRunning", String.valueOf(dashboardServiceRunning), false); + } // mark the cluster to be running stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.RecoveryRequested); stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVO.java index 4e957a81d3cc..17688c4a03f9 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVO.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVO.java @@ -93,9 +93,6 @@ public class KubernetesClusterVO implements KubernetesCluster { @Column(name = "endpoint") private String endpoint; - @Column(name = "console_endpoint") - private String consoleEndpoint; - @Column(name = GenericDao.CREATED_COLUMN) private Date created; @@ -280,15 +277,6 @@ public void setKeyPair(String keyPair) { this.keyPair = keyPair; } - @Override - public String getConsoleEndpoint() { - return consoleEndpoint; - } - - public void setConsoleEndpoint(String consoleEndpoint) { - this.consoleEndpoint = consoleEndpoint; - } - @Override public boolean isDisplay() { return true; @@ -321,7 +309,7 @@ public KubernetesClusterVO() { public KubernetesClusterVO(String name, String description, long zoneId, long kubernetesVersionId, long serviceOfferingId, long templateId, long networkId, long domainId, long accountId, long masterNodeCount, long nodeCount, State state, - String keyPair, long cores, long memory, Long nodeRootDiskSize, String endpoint, String consoleEndpoint) { + String keyPair, long cores, long memory, Long nodeRootDiskSize, String endpoint) { this.uuid = UUID.randomUUID().toString(); this.name = name; this.description = description; @@ -342,7 +330,6 @@ public KubernetesClusterVO(String name, String description, long zoneId, long ku this.nodeRootDiskSize = nodeRootDiskSize; } this.endpoint = endpoint; - this.consoleEndpoint = consoleEndpoint; this.checkForGc = false; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java index 888a8ad64a7f..594d707bc0b8 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java @@ -73,7 +73,7 @@ public class KubernetesClusterResponse extends BaseResponse implements Controlle @Param(description = "the ID of the Kubernetes version for the cluster") private String kubernetesVersionId; - @SerializedName(ApiConstants.KUBERNETES_VERSION) + @SerializedName(ApiConstants.KUBERNETES_VERSION_NAME) @Param(description = "the name of the Kubernetes version for the cluster") private String kubernetesVersionName; @@ -225,10 +225,6 @@ public void setMemory(String memory) { public void setEndpoint(String endpoint) {this.endpoint = endpoint;} - public String getConsoleEndpoint() { return consoleEndpoint;} - - public void setConsoleEndpoint(String consoleEndpoint) {this.consoleEndpoint = consoleEndpoint;} - public String getId() { return this.id; } From 1aa3274fcaa263c66768ae1722ba24e23d8fde6b Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 9 Dec 2019 01:43:18 +0530 Subject: [PATCH 020/134] added version name constant Signed-off-by: Abhishek Kumar --- api/src/main/java/org/apache/cloudstack/api/ApiConstants.java | 1 + 1 file changed, 1 insertion(+) diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index ab47971c5469..f6073ef174e4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -752,6 +752,7 @@ public class ApiConstants { public static final String DOCKER_REGISTRY_EMAIL = "dockerregistryemail"; public static final String KUBERNETES_VERSION = "kubernetesversion"; public static final String KUBERNETES_VERSION_ID = "kubernetesversionid"; + public static final String KUBERNETES_VERSION_NAME = "kubernetesversionname"; public static final String MIN_KUBERNETES_VERSION = "minimumkubernetesversion"; public static final String MIN_KUBERNETES_VERSION_ID = "minimumkubernetesversionid"; public static final String NODE_ROOT_DISK_SIZE = "noderootdisksize"; From 0e57ec37e3458b438f796d1da62b044435dba567 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 9 Dec 2019 01:45:41 +0530 Subject: [PATCH 021/134] fixed cluster access tab Signed-off-by: Abhishek Kumar --- ui/plugins/cks/cks.js | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/ui/plugins/cks/cks.js b/ui/plugins/cks/cks.js index 86b729b20fac..ba4c4cb9e5bb 100644 --- a/ui/plugins/cks/cks.js +++ b/ui/plugins/cks/cks.js @@ -857,7 +857,7 @@ zonename: { label: 'label.zone.name' }, - kubernetesversion: { + kubernetesversionname: { label: 'label.kubernetes.version' }, masternodes : { @@ -884,10 +884,6 @@ keypair: { label: 'label.ssh.key.pair' }, - consoleendpoint: { - label: 'label.dashboard.endpoint', - isCopyPaste: true - }, username: { label: 'label.username', isCopyPaste: true @@ -916,10 +912,10 @@ }); } }, - console : { + clusteraccess: { title: 'label.access', custom : function (args) { - var showDashboard = function() { + var showAccess = function() { var state = args.context.kubernetesclusters[0].state; if (state == "Created" || state == "Starting") { // Starting @@ -932,29 +928,28 @@ if (state == "Running") { // Running var data = { - id: args.context.kubernetesclusters[0].id + id: args.context.kubernetesclusters[0].kubernetesversionid } + var version = ''; $.ajax({ - url: createURL("getKubernetesClusterConfig"), + url: createURL("listKubernetesSupportedVersions"), dataType: "json", data: data, - async: true, + async: false, success: function(json) { var jsonObj; - if (json.getkubernetesclusterconfigresponse.clusterconfig != null && - json.getkubernetesclusterconfigresponse.clusterconfig.configdata != null ) { - jsonObj = json.getkubernetesclusterconfigresponse.clusterconfig; - clusterKubeConfig = jsonObj.configdata ; + if (json.listkubernetessupportedversionsresponse.kubernetessupportedversion != null) { + version = json.listkubernetessupportedversionsresponse.kubernetessupportedversion[0].kubernetesversion; } } }); - return jQuery('

').html("Access Kubernetes cluster
Download Config File

Use kubectl
kubectl --kubeconfig /custom/path/kube.config {COMMAND}

List pods
kubectl --kubeconfig /custom/path/kube.config get pods --all-namespaces
Access dashboard web UI
Run proxy locally
kubectl --kubeconfig /custom/path/kube.config proxy
Open URL in browser
http://localhost:8001/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/"); + return jQuery('

').html("Access Kubernetes cluster

Download cluster's kubeocnfig file using action from Details tab.
Download kubectl tool for cluster's Kubernetes version from,
Linux: https://storage.googleapis.com/kubernetes-release/release/v" + version + "/bin/linux/amd64/kubectl
MacOS: https://storage.googleapis.com/kubernetes-release/release/v" + version + "/bin/darwin/amd64/kubectl
Windows: https://storage.googleapis.com/kubernetes-release/release/v" + version + "/bin/windows/amd64/kubectl.exe

Using kubectl and kubeconfig file to access cluster
kubectl --kubeconfig /custom/path/kube.config {COMMAND}

List pods
kubectl --kubeconfig /custom/path/kube.config get pods --all-namespaces
List nodes
kubectl --kubeconfig /custom/path/kube.config get nodes --all-namespaces
List services
kubectl --kubeconfig /custom/path/kube.config get services --all-namespaces

Access dashboard web UI
Run proxy locally
kubectl --kubeconfig /custom/path/kube.config proxy
Open URL in browser
http://localhost:8001/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/

Token for dashboard login can be retrieved using following command
kubectl --kubeconfig /custom/path/kube.config describe secret $(kubectl --kubeconfig /custom/path/kube.config get secrets -n kubernetes-dashboard | grep kubernetes-dashboard-token | awk '{print $1}') -n kubernetes-dashboard

More about accessing dashboard UI, https://kubernetes.io/docs/tasks/access-application-cluster/web-ui-dashboard/#accessing-the-dashboard-ui"); // return jQuery('

').html("Access Kubernetes cluster
Download Config File

How to do this
kubectl --kubeconfig /custom/path/kube.config get pods"); } return jQuery('

').html("Kubernetes cluster is not in a stable state, please check again in few minutes."); }; - return showDashboard(); + return showAccess(); } }, clusterinstances: { @@ -1283,11 +1278,9 @@ url: createURL('addKubernetesSupportedVersion'), data: data, success: function(json) { - var jid = json.addKubernetesSupportedVersion.jobid; + var version = json.addkubernetessupportedversionresponse.kubernetessupportedversion; args.response.success({ - _custom: { - jobId: jid - } + data: version }); }, error: function(XMLHttpResponse) { From 56e31d1fe87515de0218c0bb17b4352bf1f6d634 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 9 Dec 2019 10:45:19 +0530 Subject: [PATCH 022/134] refactoring Signed-off-by: Abhishek Kumar --- .../kubernetescluster/KubernetesClusterManagerImpl.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java index 9ea17a6ffc1e..632eb2c1388f 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java @@ -2888,9 +2888,8 @@ public void reallyRun() { // run through Kubernetes clusters in 'Stopped' state and ensure all the VM's are Stopped in the cluster List stoppedKubernetesClusters = kubernetesClusterDao.findKubernetesClustersInState(KubernetesCluster.State.Stopped); for (KubernetesCluster kubernetesCluster : stoppedKubernetesClusters) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Running Kubernetes cluster state scanner on Kubernetes cluster name:" + kubernetesCluster.getName() + " for state " + KubernetesCluster.State.Stopped); - } + + LOGGER.debug(String.format("Running Kubernetes cluster state scanner on Kubernetes cluster ID: %s for state: %s", kubernetesCluster.getUuid(), KubernetesCluster.State.Stopped.toString())); try { if (!isClusterVMsInDesiredState(kubernetesCluster, VirtualMachine.State.Stopped)) { stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.FaultsDetected); @@ -2903,9 +2902,7 @@ public void reallyRun() { // run through Kubernetes clusters in 'Alert' state and reconcile state as 'Running' if the VM's are running List alertKubernetesClusters = kubernetesClusterDao.findKubernetesClustersInState(KubernetesCluster.State.Alert); for (KubernetesClusterVO kubernetesCluster : alertKubernetesClusters) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Running Kubernetes cluster state scanner on Kubernetes cluster name:" + kubernetesCluster.getName() + " for state " + KubernetesCluster.State.Alert); - } + LOGGER.debug(String.format("Running Kubernetes cluster state scanner on Kubernetes cluster ID: %s for state: %s", kubernetesCluster.getUuid(), KubernetesCluster.State.Alert.toString())); try { if (isClusterVMsInDesiredState(kubernetesCluster, VirtualMachine.State.Running) && kubernetesCluster.getTotalNodeCount() == getKubernetesClusterReadyNodesCount(kubernetesCluster)) { From c738ca9ecb9d1bce98737745c128ec936ad0df19 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 9 Dec 2019 10:45:45 +0530 Subject: [PATCH 023/134] fixed upgrade icon Signed-off-by: Abhishek Kumar --- ui/plugins/cks/cks.css | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ui/plugins/cks/cks.css b/ui/plugins/cks/cks.css index bcab05922472..0ff01fd31d4c 100644 --- a/ui/plugins/cks/cks.css +++ b/ui/plugins/cks/cks.css @@ -29,3 +29,11 @@ .scaleKubernetesCluster:hover .icon { background-position: -263px -583px; } + +.upgradeKubernetesCluster .icon { + background-position: -264px -2px; +} + +.upgradeKubernetesCluster:hover .icon { + background-position: -263px -583px; +} From 76001c11cdff95e071a2e5979136654794ae1666 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 9 Dec 2019 14:05:41 +0530 Subject: [PATCH 024/134] refactorings Signed-off-by: Abhishek Kumar --- api/src/main/java/org/apache/cloudstack/api/ApiConstants.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index f6073ef174e4..c6498c1bb2f4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -208,7 +208,6 @@ public class ApiConstants { public static final String LUN = "lun"; public static final String LBID = "lbruleid"; public static final String MAC_ADDRESS = "macaddress"; - public static final String MASTER_NODES = "masternodes"; public static final String MAX = "max"; public static final String MAX_SNAPS = "maxsnaps"; public static final String MAX_CPU_NUMBER = "maxcpunumber"; @@ -753,6 +752,7 @@ public class ApiConstants { public static final String KUBERNETES_VERSION = "kubernetesversion"; public static final String KUBERNETES_VERSION_ID = "kubernetesversionid"; public static final String KUBERNETES_VERSION_NAME = "kubernetesversionname"; + public static final String MASTER_NODES = "masternodes"; public static final String MIN_KUBERNETES_VERSION = "minimumkubernetesversion"; public static final String MIN_KUBERNETES_VERSION_ID = "minimumkubernetesversionid"; public static final String NODE_ROOT_DISK_SIZE = "noderootdisksize"; From 1e3e9032a8977f7fc58645dc20495e4738f0d0e0 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 9 Dec 2019 14:06:02 +0530 Subject: [PATCH 025/134] nodeRootDiskSize value check Signed-off-by: Abhishek Kumar --- .../cloud/kubernetescluster/KubernetesClusterManagerImpl.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java index 632eb2c1388f..09e8b32b4dcc 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java @@ -2013,6 +2013,10 @@ public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) throw new ManagementServerException("Kubernetes service has not been configured properly to provision Kubernetes clusters"); } + if (nodeRootDiskSize <= 0) { + throw new ManagementServerException(String.format("Invalid value for %s", ApiConstants.NODE_ROOT_DISK_SIZE)); + } + VMTemplateVO template = templateDao.findByTemplateName(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesClusterTemplateName.key())); List listZoneTemplate = templateZoneDao.listByZoneTemplate(zone.getId(), template.getId()); if (listZoneTemplate == null || listZoneTemplate.isEmpty()) { From 436ea03fa28aa7cf07ab74478767a108af92e485 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 9 Dec 2019 14:07:17 +0530 Subject: [PATCH 026/134] fix for wrong parameter name for zoneid Signed-off-by: Abhishek Kumar --- .../kubernetesversion/AddKubernetesSupportedVersionCmd.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/AddKubernetesSupportedVersionCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/AddKubernetesSupportedVersionCmd.java index 86511fa4b314..4b33f9089def 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/AddKubernetesSupportedVersionCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/AddKubernetesSupportedVersionCmd.java @@ -67,7 +67,7 @@ public class AddKubernetesSupportedVersionCmd extends BaseCmd implements UserCmd description = "the semantic version of the Kubernetes") private String kubernetesVersion; - @Parameter(name = ApiConstants.ID, type = CommandType.UUID, + @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, description = "the ID of the zone in which Kubernetes supported version will be available") private Long zoneId; From 1a6feebd5d77b32fd40004850c9386e2f08cc50d Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 9 Dec 2019 14:25:45 +0530 Subject: [PATCH 027/134] added output path handling Signed-off-by: Abhishek Kumar --- scripts/util/create-kubernetes-binaries-iso.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/util/create-kubernetes-binaries-iso.sh b/scripts/util/create-kubernetes-binaries-iso.sh index 559f6b7df805..87315119adff 100755 --- a/scripts/util/create-kubernetes-binaries-iso.sh +++ b/scripts/util/create-kubernetes-binaries-iso.sh @@ -16,9 +16,9 @@ # specific language governing permissions and limitations # under the License. -if [ $# -lt 5 ]; then - echo "Invalid input. Valid usage: ./create-kubernetes-binaries-iso.sh KUBERNETES_VERSION CNI_VERSION CRICTL_VERSION WEAVENET_NETWORK_YAML_CONFIG DASHBOARD_YAML_CONFIG" - echo "eg: ./create-kubernetes-binaries-iso.sh 1.11.4 0.7.1 1.11.1 https://github.com/weaveworks/weave/releases/download/latest_release/weave-daemonset-k8s-1.11.yaml https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.0/src/deploy/recommended/kubernetes-dashboard.yaml" +if [ $# -lt 6 ]; then + echo "Invalid input. Valid usage: ./create-kubernetes-binaries-iso.sh OUTPUT_PATH KUBERNETES_VERSION CNI_VERSION CRICTL_VERSION WEAVENET_NETWORK_YAML_CONFIG DASHBOARD_YAML_CONFIG" + echo "eg: ./create-kubernetes-binaries-iso.sh ./ 1.11.4 0.7.1 1.11.1 https://github.com/weaveworks/weave/releases/download/latest_release/weave-daemonset-k8s-1.11.yaml https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.0/src/deploy/recommended/kubernetes-dashboard.yaml" exit 1 fi From a8e3e215ac98cadf409cceafe7cc3d8ff955bb54 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 9 Dec 2019 15:24:25 +0530 Subject: [PATCH 028/134] script fixes Signed-off-by: Abhishek Kumar --- scripts/util/create-kubernetes-binaries-iso.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/scripts/util/create-kubernetes-binaries-iso.sh b/scripts/util/create-kubernetes-binaries-iso.sh index 87315119adff..dfba6130aa8e 100755 --- a/scripts/util/create-kubernetes-binaries-iso.sh +++ b/scripts/util/create-kubernetes-binaries-iso.sh @@ -22,19 +22,20 @@ if [ $# -lt 6 ]; then exit 1 fi -RELEASE="v${1}" +RELEASE="v${2}" +output_dir="${1}" start_dir="$PWD" -iso_dir="${start_dir}/iso" +iso_dir="/tmp/iso" working_dir="${iso_dir}/" mkdir -p "${working_dir}" -CNI_VERSION="v${2}" +CNI_VERSION="v${3}" echo "Downloading CNI ${CNI_VERSION}..." cni_dir="${working_dir}/cni/" mkdir -p "${cni_dir}" curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" -o "${cni_dir}/cni-plugins-amd64.tgz" -CRICTL_VERSION="v${3}" +CRICTL_VERSION="v${4}" echo "Downloading CRI tools ${CRICTL_VERSION}..." crictl_dir="${working_dir}/cri-tools/" mkdir -p "${crictl_dir}" @@ -59,12 +60,12 @@ kubeadm_conf_file="${working_dir}/10-kubeadm.conf" touch "${kubeadm_conf_file}" curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > ${kubeadm_conf_file} -NETWORK_CONFIG_URL="${4}" +NETWORK_CONFIG_URL="${5}" echo "Downloading network config ${NETWORK_CONFIG_URL}" network_conf_file="${working_dir}/network.yaml" curl -sSL ${NETWORK_CONFIG_URL} -o ${network_conf_file} -DASHBORAD_CONFIG_URL="${5}" +DASHBORAD_CONFIG_URL="${6}" echo "Downloading dashboard config ${DASHBORAD_CONFIG_URL}" dashboard_conf_file="${working_dir}/dashboard.yaml" curl -sSL ${DASHBORAD_CONFIG_URL} -o ${dashboard_conf_file} @@ -91,6 +92,6 @@ if [ "${kubeadm_file_permissions}" -eq "" ]; then fi chmod ${kubeadm_file_permissions} "${working_dir}/k8s/kubeadm" -mkisofs -o "setup-${RELEASE}.iso" -J -R -l "${iso_dir}" +mkisofs -o "${output_dir}/setup-${RELEASE}.iso" -J -R -l "${iso_dir}" -rm -rf "${iso_dir}" \ No newline at end of file +rm -rf "${iso_dir}" From ffaef966d3a6e312e521cc9bf1528678ee750381 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 9 Dec 2019 16:24:28 +0530 Subject: [PATCH 029/134] remove acl for kubernetesversionid parameter Signed-off-by: Abhishek Kumar --- .../user/kubernetescluster/CreateKubernetesClusterCmd.java | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/CreateKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/CreateKubernetesClusterCmd.java index 9f2e5b6edbf1..b66572794caf 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/CreateKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/CreateKubernetesClusterCmd.java @@ -78,7 +78,6 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { description = "availability zone in which Kubernetes cluster to be launched") private Long zoneId; - @ACL(accessType = AccessType.UseEntry) @Parameter(name = ApiConstants.KUBERNETES_VERSION_ID, type = CommandType.UUID, entityType = KubernetesSupportedVersionResponse.class, required = true, description = "Kubernetes version with which cluster to be launched") private Long kubernetesVersionId; From 45dd50249faf95ef54a04f6340a5321502515da6 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 13 Dec 2019 12:08:08 +0530 Subject: [PATCH 030/134] fix for iso ready check, increased timeouts Signed-off-by: Abhishek Kumar --- .../apache/cloudstack/api/ApiConstants.java | 2 ++ .../KubernetesClusterManagerImpl.java | 23 +++++++++++++++---- .../KubernetesVersionManagerImpl.java | 8 +++++-- .../KubernetesSupportedVersionResponse.java | 4 ++-- ui/plugins/cks/cks.js | 2 +- 5 files changed, 30 insertions(+), 9 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index c6498c1bb2f4..7df52c430d2e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -749,6 +749,8 @@ public class ApiConstants { public static final String DOCKER_REGISTRY_PASSWORD = "dockerregistrypassword"; public static final String DOCKER_REGISTRY_URL = "dockerregistryurl"; public static final String DOCKER_REGISTRY_EMAIL = "dockerregistryemail"; + public static final String ISO_NAME = "isoname"; + public static final String ISO_STATE = "isostate"; public static final String KUBERNETES_VERSION = "kubernetesversion"; public static final String KUBERNETES_VERSION_ID = "kubernetesversionid"; public static final String KUBERNETES_VERSION_NAME = "kubernetesversionname"; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java index 09e8b32b4dcc..84ccb49b546f 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java @@ -74,6 +74,7 @@ import org.apache.cloudstack.ca.CAManager; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.framework.ca.Certificate; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.managed.context.ManagedContextRunnable; @@ -84,6 +85,8 @@ import org.apache.log4j.Logger; import com.cloud.api.ApiDBUtils; +import com.cloud.api.query.dao.TemplateJoinDao; +import com.cloud.api.query.vo.TemplateJoinVO; import com.cloud.capacity.CapacityManager; import com.cloud.dc.ClusterDetailsDao; import com.cloud.dc.ClusterDetailsVO; @@ -228,6 +231,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne @Inject protected VMTemplateZoneDao templateZoneDao; @Inject + protected TemplateJoinDao templateJoinDao; + @Inject protected AccountService accountService; @Inject protected AccountDao accountDao; @@ -787,7 +792,7 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t boolean masterVmRunning = false; long startTime = System.currentTimeMillis(); - while (!masterVmRunning && System.currentTimeMillis() - startTime < 5 * 60 * 1000) { + while (!masterVmRunning && System.currentTimeMillis() - startTime < 10 * 60 * 1000) { try (Socket socket = new Socket()) { socket.connect(new InetSocketAddress(publicIpAddress, publicIpSshPort.second()), 10000); masterVmRunning = true; @@ -1989,7 +1994,9 @@ public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) throw new InvalidParameterValueException(String.format("HA support is available only for Kubernetes version %s and above. Given version ID: %s is %s", MIN_KUBERNETES_VERSION_HA_SUPPORT, clusterKubernetesVersion.getUuid(), clusterKubernetesVersion.getKubernetesVersion())); } } catch (Exception e) { - LOGGER.error(String.format("Unable to compare Kubernetes version for given version ID: %s with %s", clusterKubernetesVersion.getUuid(), MIN_KUBERNETES_VERSION_HA_SUPPORT), e); + String msg = String.format("Unable to compare Kubernetes version for given version ID: %s with %s", clusterKubernetesVersion.getUuid(), MIN_KUBERNETES_VERSION_HA_SUPPORT); + LOGGER.error(msg, e); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg, e); } } @@ -1997,6 +2004,14 @@ public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) throw new InvalidParameterValueException(String.format("Kubernetes version ID: %s is not available for zone ID: %s", clusterKubernetesVersion.getUuid(), zone.getUuid())); } + TemplateJoinVO iso = templateJoinDao.findById(clusterKubernetesVersion.getIsoId()); + if (iso == null) { + throw new InvalidParameterValueException(String.format("Invalid ISO associated with version ID: %s", clusterKubernetesVersion.getUuid())); + } + if (!ObjectInDataStoreStateMachine.State.Ready.equals(iso.getState())) { + throw new InvalidParameterValueException(String.format("ISO associated with version ID: %s is not in Ready state", clusterKubernetesVersion.getUuid())); + } + ServiceOffering serviceOffering = serviceOfferingDao.findById(serviceOfferingId); if (serviceOffering == null) { throw new InvalidParameterValueException("No service offering with ID: " + serviceOfferingId); @@ -2300,7 +2315,7 @@ public KubernetesClusterConfigResponse getKubernetesClusterConfig(GetKubernetesC } @Override - public boolean scaleKubernetesCluster(ScaleKubernetesClusterCmd cmd) throws ManagementServerException, ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException { + public boolean scaleKubernetesCluster(ScaleKubernetesClusterCmd cmd) throws ResourceUnavailableException { if (!Boolean.parseBoolean(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesServiceEnabled.key()))) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Kubernetes Service plugin is disabled"); } @@ -2738,7 +2753,7 @@ public boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws KubernetesVersionManagerImpl.compareKubernetesVersion(upgradeVersion.getKubernetesVersion(), "1.15") < 0 ? "true" : "false"); result = SshHelper.sshExecute(publicIpAddress, nodeSshPort, CLUSTER_NODE_VM_USER, pkFile, null, cmdStr, - 10000, 10000, 5 * 60 * 1000); + 10000, 10000, 10 * 60 * 1000); } catch (Exception e) { String msg = String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to upgrade Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()); LOGGER.error(msg, e); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java index 606d0ec4dec5..d4f33ade7df9 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java @@ -37,6 +37,8 @@ import org.apache.log4j.Logger; import com.cloud.api.ApiDBUtils; +import com.cloud.api.query.dao.TemplateJoinDao; +import com.cloud.api.query.vo.TemplateJoinVO; import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; import com.cloud.event.ActionEvent; @@ -67,13 +69,15 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne @Inject private VMTemplateDao templateDao; @Inject + private TemplateJoinDao templateJoinDao; + @Inject private VMTemplateZoneDao templateZoneDao; @Inject private DataCenterDao dataCenterDao; @Inject private TemplateApiService templateService; @Inject - protected ConfigurationDao globalConfigDao; + private ConfigurationDao globalConfigDao; private KubernetesSupportedVersionResponse createKubernetesSupportedVersionResponse(final KubernetesSupportedVersion kubernetesSupportedVersion) { KubernetesSupportedVersionResponse response = new KubernetesSupportedVersionResponse(); @@ -92,7 +96,7 @@ private KubernetesSupportedVersionResponse createKubernetesSupportedVersionRespo } else { response.setSupportsHA(false); } - VMTemplateVO template = ApiDBUtils.findTemplateById(kubernetesSupportedVersion.getIsoId()); + TemplateJoinVO template = templateJoinDao.findById(kubernetesSupportedVersion.getIsoId()); response.setIsoId(template.getUuid()); response.setIsoName(template.getName()); response.setIsoState(template.getState().toString()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java index 26004899ebfb..1f7a2894b0b8 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java @@ -44,11 +44,11 @@ public class KubernetesSupportedVersionResponse extends BaseResponse { @Param(description = "the id of the binaries ISO for Kubernetes supported version") private String isoId; - @SerializedName("isoname") + @SerializedName(ApiConstants.ISO_NAME) @Param(description = "the name of the binaries ISO for Kubernetes supported version") private String isoName; - @SerializedName("isostate") + @SerializedName(ApiConstants.ISO_STATE) @Param(description = "the state of the binaries ISO for Kubernetes supported version") private String isoState; diff --git a/ui/plugins/cks/cks.js b/ui/plugins/cks/cks.js index ba4c4cb9e5bb..1ba4bd05b3f3 100644 --- a/ui/plugins/cks/cks.js +++ b/ui/plugins/cks/cks.js @@ -219,7 +219,7 @@ versionObjs = json.listkubernetessupportedversionsresponse.kubernetessupportedversion; if (versionObjs != null) { for (var i = 0; i < versionObjs.length; i++) { - if (versionObjs[i].isostate == 'Active') { + if (versionObjs[i].isostate == 'Ready') { items.push({ id: versionObjs[i].id, description: versionObjs[i].name From b969d46c296f6ad1434df58c34fb04e5a3c6de42 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 13 Dec 2019 12:19:07 +0530 Subject: [PATCH 031/134] refactoring Signed-off-by: Abhishek Kumar --- .../META-INF/db/schema-41300to41400.sql | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql b/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql index 014d02d9351f..dd149334c732 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql @@ -112,17 +112,17 @@ INSERT IGNORE INTO `cloud`.`network_offerings` (name, uuid, unique_name, display UPDATE `cloud`.`network_offerings` SET removed=NULL WHERE unique_name='DefaultNetworkOfferingforKubernetesService'; -SET @k8snetwork = (select id from network_offerings where name='DefaultNetworkOfferingforKubernetesService' and removed IS NULL); -INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@k8snetwork, 'Dhcp','VirtualRouter',now()); -INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@k8snetwork, 'Dns','VirtualRouter',now()); -INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@k8snetwork, 'Firewall','VirtualRouter',now()); -INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@k8snetwork, 'Gateway','VirtualRouter',now()); -INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@k8snetwork, 'Lb','VirtualRouter',now()); -INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@k8snetwork, 'PortForwarding','VirtualRouter',now()); -INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@k8snetwork, 'SourceNat','VirtualRouter',now()); -INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@k8snetwork, 'StaticNat','VirtualRouter',now()); -INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@k8snetwork, 'UserData','VirtualRouter',now()); -INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@k8snetwork, 'Vpn','VirtualRouter',now()); +SET @kubernetesnetwork = (select id from network_offerings where name='DefaultNetworkOfferingforKubernetesService' and removed IS NULL); +INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@kubernetesnetwork, 'Dhcp','VirtualRouter',now()); +INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@kubernetesnetwork, 'Dns','VirtualRouter',now()); +INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@kubernetesnetwork, 'Firewall','VirtualRouter',now()); +INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@kubernetesnetwork, 'Gateway','VirtualRouter',now()); +INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@kubernetesnetwork, 'Lb','VirtualRouter',now()); +INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@kubernetesnetwork, 'PortForwarding','VirtualRouter',now()); +INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@kubernetesnetwork, 'SourceNat','VirtualRouter',now()); +INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@kubernetesnetwork, 'StaticNat','VirtualRouter',now()); +INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@kubernetesnetwork, 'UserData','VirtualRouter',now()); +INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@kubernetesnetwork, 'Vpn','VirtualRouter',now()); INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', -'cloud.kubernetes.cluster.network.offering', 'DefaultNetworkOfferingforKubernetesService' , 'Network Offering used for CloudStack kubernetes service', 'DefaultNetworkOfferingforKubernetesService', NULL , NULL, 0); +'cloud.kubernetes.cluster.network.offering', 'DefaultNetworkOfferingforKubernetesService', 'Network Offering used for CloudStack kubernetes service', 'DefaultNetworkOfferingforKubernetesService', NULL , NULL, 0); From 7e4228cfa0966a0bd253b217e216d8b9e6339e5a Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 16 Dec 2019 15:25:38 +0530 Subject: [PATCH 032/134] fixes Signed-off-by: Abhishek Kumar --- .../KubernetesClusterManagerImpl.java | 22 ++++++++++++++----- ui/plugins/cks/cks.css | 4 ++-- ui/plugins/cks/cks.js | 2 +- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java index 84ccb49b546f..e387a7cd2085 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java @@ -649,7 +649,7 @@ private boolean uncordonKubernetesClusterNode(KubernetesCluster kubernetesCluste String.format("sudo kubectl uncordon %s", userVm.getHostName()), 10000, 10000, 30000); if (result.first()) { - break; + return true; } } catch (Exception e) { LOGGER.warn(String.format("Failed to uncordon node: %s on VM ID: %s in Kubernetes cluster ID: %s", userVm.getHostName(), userVm.getUuid(), kubernetesCluster.getUuid()), e); @@ -833,6 +833,19 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t kubernetesClusterDao.update(kubernetesCluster.getId(), kubernetesCluster); int sshPort = publicIpSshPort.second(); + boolean readyNodesCountValid = validateKubernetesClusterReadyNodesCount(kubernetesCluster, publicIpAddress, sshPort, 30, 30000); + + // Detach binaries ISO from new VMs + detachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); + + // Throw exception if nodes count for k8s cluster timed out + if (!readyNodesCountValid) { // Scaling failed + String msg = String.format("Failed to setup Kubernetes cluster ID: %s as it does not have desired number of nodes in ready state", kubernetesCluster.getUuid()); + LOGGER.warn(msg); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); + } + boolean k8sKubeConfigCopied = false; String kubeConfig = getKubernetesClusterConfig(kubernetesCluster, publicIpAddress, sshPort, 5); if (!Strings.isNullOrEmpty(kubeConfig)) { @@ -842,7 +855,6 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t String msg = String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to retrieve kube-config for the cluster", kubernetesCluster.getUuid()); LOGGER.error(msg); stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - detachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } kubeConfig = kubeConfig.replace(String.format("server: https://%s:%d", k8sMasterVM.getPrivateIpAddress(), CLUSTER_API_PORT), @@ -854,11 +866,9 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t String msg = String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to get Dashboard service running for the cluster", kubernetesCluster.getUuid()); LOGGER.error(msg); stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - detachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "dashboardServiceRunning", String.valueOf(dashboardServiceRunning), false); - detachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); return true; } @@ -2617,7 +2627,7 @@ public int compare(KubernetesClusterVmMapVO kubernetesClusterVmMapVO, Kubernetes attachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); // Check if new nodes are added in k8s cluster - boolean readyNodesCountValid = validateKubernetesClusterReadyNodesCount(kubernetesCluster, publicIpAddress, sshPort, 20, 30000); + boolean readyNodesCountValid = validateKubernetesClusterReadyNodesCount(kubernetesCluster, publicIpAddress, sshPort, 30, 30000); // Detach binaries ISO from new VMs detachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); @@ -2768,7 +2778,7 @@ public boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } - if (!uncordonKubernetesClusterNode(kubernetesCluster, publicIpAddress, sshPort, vm, 3, 30000)) { + if (!uncordonKubernetesClusterNode(kubernetesCluster, publicIpAddress, sshPort, vm, 5, 30000)) { String msg = String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to uncordon Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()); LOGGER.error(msg); detachIsoKubernetesVMs(kubernetesCluster, vmIds); diff --git a/ui/plugins/cks/cks.css b/ui/plugins/cks/cks.css index 0ff01fd31d4c..8a7591c07153 100644 --- a/ui/plugins/cks/cks.css +++ b/ui/plugins/cks/cks.css @@ -31,9 +31,9 @@ } .upgradeKubernetesCluster .icon { - background-position: -264px -2px; + background-position: -138px -65px; } .upgradeKubernetesCluster:hover .icon { - background-position: -263px -583px; + background-position: -138px -647px; } diff --git a/ui/plugins/cks/cks.js b/ui/plugins/cks/cks.js index 1ba4bd05b3f3..493ab7f6af9d 100644 --- a/ui/plugins/cks/cks.js +++ b/ui/plugins/cks/cks.js @@ -799,7 +799,7 @@ if (versionObjs != null) { for (var i = 0; i < versionObjs.length; i++) { if (versionObjs[i].id != args.context.kubernetesclusters[0].kubernetesversionid && - versionObjs[i].isostate == 'Active') { + versionObjs[i].isostate == 'Ready') { items.push({ id: versionObjs[i].id, description: versionObjs[i].name From 96365ce1c17071772113bbba1c9bf56ed3f5b37a Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 17 Dec 2019 09:51:16 +0530 Subject: [PATCH 033/134] added check for iso state for upgrade Signed-off-by: Abhishek Kumar --- .../KubernetesClusterManagerImpl.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java index e387a7cd2085..a23dfd3b9c9c 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java @@ -2681,13 +2681,21 @@ public boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws throw new InvalidParameterValueException(String.format("Invalid Kubernetes version associated with cluster ID: %s", kubernetesCluster.getUuid())); } - // Check upgradeVersion is either path upgrade or immediate minor upgrade + // Check upgradeVersion is either patch upgrade or immediate minor upgrade try { KubernetesVersionManagerImpl.canUpgradeKubernetesVersion(clusterVersion.getKubernetesVersion(), upgradeVersion.getKubernetesVersion()); } catch (IllegalArgumentException e) { throw new InvalidParameterValueException(e.getMessage()); } + TemplateJoinVO iso = templateJoinDao.findById(upgradeVersion.getIsoId()); + if (iso == null) { + throw new InvalidParameterValueException(String.format("Invalid ISO associated with version ID: %s", upgradeVersion.getUuid())); + } + if (!ObjectInDataStoreStateMachine.State.Ready.equals(iso.getState())) { + throw new InvalidParameterValueException(String.format("ISO associated with version ID: %s is not in Ready state", upgradeVersion.getUuid())); + } + // Get public IP Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(kubernetesCluster); String publicIpAddress = publicIpSshPort.first(); From 232a1f7acb1bd88d4552d762aca2e7b5dae4c20e Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 17 Dec 2019 10:06:07 +0530 Subject: [PATCH 034/134] added logs during upgrade Signed-off-by: Abhishek Kumar --- .../cloud/kubernetescluster/KubernetesClusterManagerImpl.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java index a23dfd3b9c9c..8fd8b81e054b 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java @@ -2743,6 +2743,8 @@ public boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws for (int i = 0; i < vmIds.size(); ++i) { UserVm vm = userVmDao.findById(vmIds.get(i)); result = null; + LOGGER.debug(String.format("Upgrading node on VM ID: %s in Kubernetes cluster ID: %s with Kubernetes version(%s) ID: %s", + vm.getUuid(), kubernetesCluster.getUuid(), upgradeVersion.getKubernetesVersion(), upgradeVersion.getUuid())); try { result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, pkFile, null, String.format("sudo kubectl drain %s --ignore-daemonsets --delete-local-data", vm.getHostName()), @@ -2802,6 +2804,8 @@ public boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); } } + LOGGER.debug(String.format("Successfully upgraded node on VM ID: %s in Kubernetes cluster ID: %s with Kubernetes version(%s) ID: %s", + vm.getUuid(), kubernetesCluster.getUuid(), upgradeVersion.getKubernetesVersion(), upgradeVersion.getUuid())); } // Detach ISO From 9957d8fe0d3594419df8bbdeb8e431705c21c7fd Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 2 Jan 2020 11:52:03 +0530 Subject: [PATCH 035/134] [wip] code review refactorings Signed-off-by: Abhishek Kumar --- .../apache/cloudstack/api/ApiConstants.java | 2 +- .../META-INF/db/schema-41300to41400.sql | 78 +- .../integrations/kubernetes-service/pom.xml | 27 +- .../KubernetesClusterManagerImpl.java | 1953 +++++++++-------- .../KubernetesClusterService.java | 27 +- .../KubernetesClusterVmMap.java | 1 - .../KubernetesServiceConfig.java | 74 - .../KubernetesSupportedVersion.java | 2 +- .../KubernetesSupportedVersionVO.java | 16 +- .../KubernetesVersionManagerImpl.java | 226 +- .../ListKubernetesSupportedVersionsCmd.java | 16 +- .../api/query/dao/NetworkOfferingJoinDao.java | 4 +- .../query/dao/NetworkOfferingJoinDaoImpl.java | 10 +- 13 files changed, 1226 insertions(+), 1210 deletions(-) delete mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesServiceConfig.java diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 7df52c430d2e..e310ad3072bd 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -755,7 +755,7 @@ public class ApiConstants { public static final String KUBERNETES_VERSION_ID = "kubernetesversionid"; public static final String KUBERNETES_VERSION_NAME = "kubernetesversionname"; public static final String MASTER_NODES = "masternodes"; - public static final String MIN_KUBERNETES_VERSION = "minimumkubernetesversion"; + public static final String MIN_SEMANTIC_VERSION = "minimumsemanticversion"; public static final String MIN_KUBERNETES_VERSION_ID = "minimumkubernetesversionid"; public static final String NODE_ROOT_DISK_SIZE = "noderootdisksize"; public static final String SUPPORTS_HA = "supportsha"; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql b/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql index dd149334c732..6101108deec6 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql @@ -35,18 +35,15 @@ UPDATE `cloud`.`guest_os` SET `category_id`='4' WHERE `id`=285 AND display_name= UPDATE `cloud`.`guest_os` SET `category_id`='4' WHERE `id`=286 AND display_name="Red Hat Enterprise Linux 8.0"; -- Kubernetes service -INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', -'cloud.kubernetes.service.enabled', 'false', 'Indicates whether Kubernetes Service plugin is enabled or not. Management server restart needed on change', 'false', NULL, NULL, 0); - CREATE TABLE IF NOT EXISTS `cloud`.`kubernetes_supported_version` ( - `id` bigint unsigned NOT NULL auto_increment COMMENT 'id', - `uuid` varchar(40) DEFAULT NULL COMMENT 'uuid', - `name` varchar(255) NOT NULL COMMENT 'kubernetes version name', - `kubernetes_version` varchar(32) NOT NULL COMMENT 'kubernetes semantic version', - `iso_id` bigint unsigned NOT NULL COMMENT 'kubernetes version binary ISO id', - `zone_id` bigint unsigned DEFAULT NULL COMMENT 'zone id in which kubernetes version is available', + `id` bigint unsigned NOT NULL auto_increment, + `uuid` varchar(40) DEFAULT NULL, + `name` varchar(255) NOT NULL COMMENT 'the name of this Kubernetes version', + `semantic_version` varchar(32) NOT NULL COMMENT 'the semantic version for this Kubernetes version', + `iso_id` bigint unsigned NOT NULL COMMENT 'the ID of the binaries ISO for this Kubernetes version', + `zone_id` bigint unsigned DEFAULT NULL COMMENT 'the ID of the zone for which this Kubernetes version is made available', `created` datetime NOT NULL COMMENT 'date created', - `removed` datetime COMMENT 'date removed if not null', + `removed` datetime COMMENT 'date removed or null, if still present', PRIMARY KEY(`id`), CONSTRAINT `fk_kubernetes_supported_version__iso_id` FOREIGN KEY `fk_kubernetes_supported_version__iso_id`(`iso_id`) REFERENCES `vm_template`(`id`) ON DELETE CASCADE, @@ -54,28 +51,28 @@ CREATE TABLE IF NOT EXISTS `cloud`.`kubernetes_supported_version` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8; CREATE TABLE IF NOT EXISTS `cloud`.`kubernetes_cluster` ( - `id` bigint unsigned NOT NULL auto_increment COMMENT 'id', + `id` bigint unsigned NOT NULL auto_increment, `uuid` varchar(40) DEFAULT NULL, `name` varchar(255) NOT NULL, - `description` varchar(4096) COMMENT 'display text for this kubernetes cluster', - `zone_id` bigint unsigned NOT NULL COMMENT 'zone id', - `kubernetes_version_id` bigint unsigned NOT NULL COMMENT 'kubernetes version id for the cluster', + `description` varchar(4096) COMMENT 'display text for this Kubernetes cluster', + `zone_id` bigint unsigned NOT NULL COMMENT 'the ID of the zone in which this Kubernetes cluster is deployed', + `kubernetes_version_id` bigint unsigned NOT NULL COMMENT 'the ID of the Kubernetes version of this Kubernetes cluster', `service_offering_id` bigint unsigned COMMENT 'service offering id for the cluster VM', - `template_id` bigint unsigned COMMENT 'vm_template.id', - `network_id` bigint unsigned COMMENT 'network this kubernetes cluster uses', - `master_node_count` bigint NOT NULL default '0', - `node_count` bigint NOT NULL default '0', - `account_id` bigint unsigned NOT NULL COMMENT 'owner of this cluster', - `domain_id` bigint unsigned NOT NULL COMMENT 'owner of this cluster', - `state` char(32) NOT NULL COMMENT 'current state of this cluster', + `template_id` bigint unsigned COMMENT 'the ID of the template used by this Kubernetes cluster', + `network_id` bigint unsigned COMMENT 'the ID of the network used by this Kubernetes cluster', + `master_node_count` bigint NOT NULL default '0' COMMENT 'the number of the master nodes deployed for this Kubernetes cluster', + `node_count` bigint NOT NULL default '0' COMMENT 'the number of the worker nodes deployed for this Kubernetes cluster', + `account_id` bigint unsigned NOT NULL COMMENT 'the ID of owner account of this Kubernetes cluster', + `domain_id` bigint unsigned NOT NULL COMMENT 'the ID of the domain of this cluster', + `state` char(32) NOT NULL COMMENT 'the current state of this Kubernetes cluster', `key_pair` varchar(40), - `cores` bigint unsigned NOT NULL COMMENT 'number of cores', - `memory` bigint unsigned NOT NULL COMMENT 'total memory', + `cores` bigint unsigned NOT NULL COMMENT 'total number of CPU cores used by this Kubernetes cluster', + `memory` bigint unsigned NOT NULL COMMENT 'total memory used by this Kubernetes cluster', `node_root_disk_size` bigint(20) unsigned DEFAULT 0 COMMENT 'root disk size of root disk for each node', - `endpoint` varchar(255) COMMENT 'url endpoint of the kubernetes cluster manager api access', + `endpoint` varchar(255) COMMENT 'url endpoint of the Kubernetes cluster manager api access', `created` datetime NOT NULL COMMENT 'date created', - `removed` datetime COMMENT 'date removed if not null', - `gc` tinyint unsigned NOT NULL DEFAULT 1 COMMENT 'gc this kubernetes cluster or not', + `removed` datetime COMMENT 'date removed or null, if still present', + `gc` tinyint unsigned NOT NULL DEFAULT 1 COMMENT 'gc this Kubernetes cluster or not', PRIMARY KEY(`id`), CONSTRAINT `fk_cluster__zone_id` FOREIGN KEY `fk_cluster__zone_id`(`zone_id`) REFERENCES `data_center` (`id`) ON DELETE CASCADE, @@ -86,29 +83,35 @@ CREATE TABLE IF NOT EXISTS `cloud`.`kubernetes_cluster` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8; CREATE TABLE IF NOT EXISTS `cloud`.`kubernetes_cluster_vm_map` ( - `id` bigint unsigned NOT NULL auto_increment COMMENT 'id', - `cluster_id` bigint unsigned NOT NULL COMMENT 'cluster id', - `vm_id` bigint unsigned NOT NULL COMMENT 'vm id', + `id` bigint unsigned NOT NULL auto_increment, + `cluster_id` bigint unsigned NOT NULL COMMENT 'the ID of the Kubernetes cluster', + `vm_id` bigint unsigned NOT NULL COMMENT 'the ID of the VM', PRIMARY KEY(`id`), CONSTRAINT `fk_kubernetes_cluster_vm_map__cluster_id` FOREIGN KEY `fk_kubernetes_cluster_vm_map__cluster_id`(`cluster_id`) REFERENCES `kubernetes_cluster`(`id`) ON DELETE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8; CREATE TABLE IF NOT EXISTS `cloud`.`kubernetes_cluster_details` ( - `id` bigint unsigned NOT NULL auto_increment COMMENT 'id', - `cluster_id` bigint unsigned NOT NULL COMMENT 'kubernetes cluster id', + `id` bigint unsigned NOT NULL auto_increment, + `cluster_id` bigint unsigned NOT NULL COMMENT 'the ID of the Kubernetes cluster', `name` varchar(255) NOT NULL, `value` varchar(10240) NOT NULL, - `display` tinyint(1) NOT NULL DEFAULT '1' COMMENT 'True if the detail can be displayed to the end user', + `display` tinyint(1) NOT NULL DEFAULT '1' COMMENT 'True if the detail can be displayed to the end user else false', PRIMARY KEY(`id`), CONSTRAINT `fk_kubernetes_cluster_details__cluster_id` FOREIGN KEY `fk_kubernetes_cluster_details__cluster_id`(`cluster_id`) REFERENCES `kubernetes_cluster`(`id`) ON DELETE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8; -INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', -'cloud.kubernetes.cluster.template.name', "Kubernetes-Service-Template", 'Name of the template to be used for creating Kubernetes cluster nodes', 'Kubernetes-Service-Template', NULL, NULL, 0); - -INSERT IGNORE INTO `cloud`.`network_offerings` (name, uuid, unique_name, display_text, nw_rate, mc_rate, traffic_type, tags, system_only, specify_vlan, service_offering_id, conserve_mode, created,availability, dedicated_lb_service, shared_source_nat_service, sort_key, redundant_router_service, state, guest_type, elastic_ip_service, eip_associate_public_ip, elastic_lb_service, specify_ip_ranges, inline,is_persistent,internal_lb, public_lb, egress_default_policy, concurrent_connections, keep_alive_enabled, supports_streched_l2, `default`, removed) VALUES ('DefaultNetworkOfferingforKubernetesService', UUID(), 'DefaultNetworkOfferingforKubernetesService', 'Network Offering used for CloudStack kubernetes service', NULL,NULL,'Guest',NULL,0,0,NULL,1,now(),'Required',1,0,0,0,'Enabled','Isolated',0,1,0,0,0,0,0,1,1,NULL,0,0,0,NULL); +INSERT IGNORE INTO `cloud`.`network_offerings` (name, uuid, unique_name, display_text, nw_rate, mc_rate, traffic_type, + tags, system_only, specify_vlan, service_offering_id, conserve_mode, created, availability, dedicated_lb_service, + shared_source_nat_service, sort_key, redundant_router_service, state, guest_type, elastic_ip_service, + eip_associate_public_ip, elastic_lb_service, specify_ip_ranges, inline, is_persistent, internal_lb, public_lb, + egress_default_policy, concurrent_connections, keep_alive_enabled, supports_streched_l2, `default`, removed) VALUES ( + 'DefaultNetworkOfferingforKubernetesService', UUID(), 'DefaultNetworkOfferingforKubernetesService', 'Network Offering used for CloudStack Kubernetes service', NULL,NULL,'Guest', + NULL, 0, 0, NULL, 1, now(),'Required', 1, + 0, 0, 0, 'Enabled', 'Isolated', 0, + 1, 0, 0, 0, 0, 0, 1, + 1, NULL, 0, 0, 0, NULL); UPDATE `cloud`.`network_offerings` SET removed=NULL WHERE unique_name='DefaultNetworkOfferingforKubernetesService'; @@ -123,6 +126,3 @@ INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, prov INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@kubernetesnetwork, 'StaticNat','VirtualRouter',now()); INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@kubernetesnetwork, 'UserData','VirtualRouter',now()); INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@kubernetesnetwork, 'Vpn','VirtualRouter',now()); - -INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', -'cloud.kubernetes.cluster.network.offering', 'DefaultNetworkOfferingforKubernetesService', 'Network Offering used for CloudStack kubernetes service', 'DefaultNetworkOfferingforKubernetesService', NULL , NULL, 0); diff --git a/plugins/integrations/kubernetes-service/pom.xml b/plugins/integrations/kubernetes-service/pom.xml index 800f9b5f9fa2..9fb2a4391a95 100644 --- a/plugins/integrations/kubernetes-service/pom.xml +++ b/plugins/integrations/kubernetes-service/pom.xml @@ -1,18 +1,21 @@ + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java index 8fd8b81e054b..4a461f2bde8e 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java @@ -31,7 +31,6 @@ import java.net.URL; import java.net.UnknownHostException; import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; import java.security.SecureRandom; import java.util.ArrayList; import java.util.Arrays; @@ -55,9 +54,7 @@ import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.SecurityChecker; import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseCmd; -import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.user.firewall.CreateFirewallRuleCmd; import org.apache.cloudstack.api.command.user.kubernetescluster.CreateKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetescluster.DeleteKubernetesClusterCmd; @@ -76,16 +73,20 @@ import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.framework.ca.Certificate; +import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.utils.security.CertUtils; import org.apache.commons.codec.binary.Base64; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.io.IOUtils; +import org.apache.log4j.Level; import org.apache.log4j.Logger; import com.cloud.api.ApiDBUtils; +import com.cloud.api.query.dao.NetworkOfferingJoinDao; import com.cloud.api.query.dao.TemplateJoinDao; +import com.cloud.api.query.vo.NetworkOfferingJoinVO; import com.cloud.api.query.vo.TemplateJoinVO; import com.cloud.capacity.CapacityManager; import com.cloud.dc.ClusterDetailsDao; @@ -101,6 +102,7 @@ import com.cloud.dc.dao.VlanDao; import com.cloud.deploy.DeployDestination; import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientAddressCapacityException; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InsufficientServerCapacityException; import com.cloud.exception.InvalidParameterValueException; @@ -109,6 +111,7 @@ import com.cloud.exception.PermissionDeniedException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; +import com.cloud.exception.VirtualMachineMigrationException; import com.cloud.host.Host.Type; import com.cloud.host.HostVO; import com.cloud.hypervisor.Hypervisor; @@ -129,7 +132,6 @@ import com.cloud.network.addr.PublicIp; import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.dao.IPAddressDao; -import com.cloud.network.dao.IPAddressVO; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; import com.cloud.network.dao.PhysicalNetworkDao; @@ -166,6 +168,7 @@ import com.cloud.user.dao.SSHKeyPairDao; import com.cloud.uservm.UserVm; import com.cloud.utils.Pair; +import com.cloud.utils.StringUtils; import com.cloud.utils.component.ComponentContext; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.concurrency.NamedThreadFactory; @@ -251,6 +254,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne @Inject protected NetworkOfferingDao networkOfferingDao; @Inject + protected NetworkOfferingJoinDao networkOfferingJoinDao; + @Inject protected NetworkService networkService; @Inject protected NetworkModel networkModel; @@ -298,11 +303,65 @@ private String readResourceFile(String resource) throws IOException { return IOUtils.toString(Objects.requireNonNull(Thread.currentThread().getContextClassLoader().getResourceAsStream(resource)), Charset.defaultCharset().name()); } - private boolean isKubernetesServiceConfigured(DataCenter zone) { + private void logMessage(final Level logLevel, final String message, final Exception e) { + if (logLevel == Level.DEBUG) { + LOGGER.debug(message); + } else if (logLevel == Level.ERROR) { + LOGGER.error(message); + } if (logLevel == Level.WARN) { + if (e != null) { + LOGGER.warn(message, e); + } else { + LOGGER.warn(message); + } + } else { + if (e != null) { + LOGGER.error(message, e); + } else { + LOGGER.error(message); + } + } + } + + private void logTransitStateAndThrow(final Level logLevel, final String message, final Long kubernetesClusterId, final KubernetesCluster.Event event, final Exception e) throws CloudRuntimeException { + logMessage(logLevel, message, e); + if (kubernetesClusterId != null && event != null) { + stateTransitTo(kubernetesClusterId, event); + } + if (e == null) { + throw new CloudRuntimeException(message); + } + throw new CloudRuntimeException(message, e); + } + + private void logTransitStateDetachIsoAndThrow(final Level logLevel, final String message, final KubernetesCluster kubernetesCluster, + final List clusterVMIds, final KubernetesCluster.Event event, final Exception e) throws CloudRuntimeException { + logMessage(logLevel, message, e); + stateTransitTo(kubernetesCluster.getId(), event); + detachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); + if (e == null) { + throw new CloudRuntimeException(message); + } + throw new CloudRuntimeException(message, e); + } + + private void logTransitStateAndThrow(final Level logLevel, final String message, final Long kubernetesClusterId, final KubernetesCluster.Event event) throws CloudRuntimeException { + logTransitStateAndThrow(logLevel, message, kubernetesClusterId, event, null); + } + + private void logAndThrow(final Level logLevel, final String message) throws CloudRuntimeException { + logTransitStateAndThrow(logLevel, message, null, null, null); + } + + private void logAndThrow(final Level logLevel, final String message, final Exception ex) throws CloudRuntimeException { + logTransitStateAndThrow(logLevel, message, null, null, ex); + } + + private boolean isKubernetesServiceTemplateConfigured(DataCenter zone) { // Check Kubernetes VM template for zone - String templateName = globalConfigDao.getValue(KubernetesServiceConfig.KubernetesClusterTemplateName.key()); + String templateName = KubernetesClusterTemplateName.value(); if (templateName == null || templateName.isEmpty()) { - LOGGER.warn(String.format("Global setting %s is empty. Template name need to be specified for Kubernetes service to function", KubernetesServiceConfig.KubernetesClusterTemplateName.key())); + LOGGER.warn(String.format("Global setting %s is empty. Template name need to be specified for Kubernetes service to function", KubernetesClusterTemplateName.key())); return false; } final VMTemplateVO template = templateDao.findByTemplateName(templateName); @@ -310,10 +369,14 @@ private boolean isKubernetesServiceConfigured(DataCenter zone) { LOGGER.warn(String.format("Unable to find the template %s to be used for provisioning Kubernetes cluster", templateName)); return false; } + return true; + } + + private boolean isKubernetesServiceNetworkOfferingConfigured(DataCenter zone) { // Check network offering - String networkOfferingName = globalConfigDao.getValue(KubernetesServiceConfig.KubernetesClusterNetworkOffering.key()); + String networkOfferingName = KubernetesClusterNetworkOffering.value(); if (networkOfferingName == null || networkOfferingName.isEmpty()) { - LOGGER.warn(String.format("Global setting %s is empty. Admin has not yet specified the network offering to be used for provisioning isolated network for the cluster", KubernetesServiceConfig.KubernetesClusterNetworkOffering.key())); + LOGGER.warn(String.format("Global setting %s is empty. Admin has not yet specified the network offering to be used for provisioning isolated network for the cluster", KubernetesClusterNetworkOffering.key())); return false; } NetworkOfferingVO networkOffering = networkOfferingDao.findByUniqueName(networkOfferingName); @@ -334,6 +397,18 @@ private boolean isKubernetesServiceConfigured(DataCenter zone) { LOGGER.warn(String.format("Network offering ID: %s has egress default policy turned off should be on to provision Kubernetes cluster", networkOffering.getUuid())); return false; } + boolean offeringAvailableForZone = false; + List networkOfferingJoinVOs = networkOfferingJoinDao.findByZoneId(zone.getId(), true); + for (NetworkOfferingJoinVO networkOfferingJoinVO : networkOfferingJoinVOs) { + if (networkOffering.getId() == networkOfferingJoinVO.getId()) { + offeringAvailableForZone = true; + break; + } + } + if (!offeringAvailableForZone) { + LOGGER.warn(String.format("Network offering ID: %s is not available for zone ID: %s", networkOffering.getUuid(), zone.getUuid())); + return false; + } long physicalNetworkId = networkModel.findPhysicalNetworkId(zone.getId(), networkOffering.getTags(), networkOffering.getTrafficType()); PhysicalNetwork physicalNetwork = physicalNetworkDao.findById(physicalNetworkId); if (physicalNetwork == null) { @@ -343,6 +418,16 @@ private boolean isKubernetesServiceConfigured(DataCenter zone) { return true; } + private boolean isKubernetesServiceConfigured(DataCenter zone) { + if (!isKubernetesServiceTemplateConfigured(zone)) { + return false; + } + if (!isKubernetesServiceNetworkOfferingConfigured(zone)) { + return false; + } + return true; + } + private File getManagementServerSshPublicKeyFile() { boolean devel = Boolean.parseBoolean(globalConfigDao.getValue("developer")); String keyFile = String.format("%s/.ssh/id_rsa", System.getProperty("user.home")); @@ -353,7 +438,6 @@ private File getManagementServerSshPublicKeyFile() { } private String generateClusterToken(KubernetesCluster kubernetesCluster) { - if (kubernetesCluster == null) return ""; String token = kubernetesCluster.getUuid(); token = token.replaceAll("-", ""); token = token.substring(0, 22); @@ -362,7 +446,6 @@ private String generateClusterToken(KubernetesCluster kubernetesCluster) { } private String generateClusterHACertificateKey(KubernetesCluster kubernetesCluster) { - if (kubernetesCluster == null) return ""; String uuid = kubernetesCluster.getUuid(); StringBuilder token = new StringBuilder(uuid.replaceAll("-", "")); while (token.length() < 64) { @@ -387,7 +470,7 @@ private boolean isKubernetesClusterServerRunning(KubernetesCluster kubernetesClu boolean k8sApiServerSetup = false; while (retryCounter < retries) { try { - String versionOutput = IOUtils.toString(new URL(String.format("https://%s:%d/version", ipAddress, CLUSTER_API_PORT)), StandardCharsets.UTF_8); + String versionOutput = IOUtils.toString(new URL(String.format("https://%s:%d/version", ipAddress, CLUSTER_API_PORT)), StringUtils.getPreferredCharset()); if (!Strings.isNullOrEmpty(versionOutput)) { LOGGER.debug(String.format("Kubernetes cluster ID: %s API has been successfully provisioned, %s", kubernetesCluster.getUuid(), versionOutput)); k8sApiServerSetup = true; @@ -418,6 +501,8 @@ private String getKubernetesClusterConfig(KubernetesCluster kubernetesCluster, S if (result.first() && !Strings.isNullOrEmpty(result.second())) { kubeConfig = result.second(); break; + } else { + LOGGER.debug(String.format("Failed to retrieve kube-config file for Kubernetes cluster ID: %s. Output: %s", kubernetesCluster.getUuid(), result.second())); } } catch (Exception e) { LOGGER.warn(String.format("Failed to retrieve kube-config file for Kubernetes cluster ID: %s. Attempt: %d/%d", kubernetesCluster.getUuid(), retryCounter+1, retries), e); @@ -459,6 +544,7 @@ private boolean isKubernetesClusterDashboardServiceRunning(KubernetesCluster kub while (retryCounter < retries) { LOGGER.debug(String.format("Checking dashboard service for the Kubernetes cluster ID: %s to come up. Attempt: %d/%d", kubernetesCluster.getUuid(), retryCounter+1, retries)); if (isKubernetesClusterAddOnServiceRunning(kubernetesCluster, ipAddress, port, "kubernetes-dashboard", "kubernetes-dashboard")) { + LOGGER.info(String.format("Dashboard service for the Kubernetes cluster ID: %s is in running state", kubernetesCluster.getUuid())); running = true; break; } @@ -472,6 +558,23 @@ private boolean isKubernetesClusterDashboardServiceRunning(KubernetesCluster kub return running; } + private UserVm fetchMasterVmIfMissing(final KubernetesCluster kubernetesCluster, final int port, final UserVm masterVm) { + if (masterVm != null) { + return masterVm; + } + List clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + if (CollectionUtils.isEmpty(clusterVMs)) { + LOGGER.warn(String.format("Unable to retrieve VMs for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + return null; + } + List vmIds = new ArrayList<>(); + for (KubernetesClusterVmMapVO vmMap : clusterVMs) { + vmIds.add(vmMap.getVmId()); + } + Collections.sort(vmIds); + return userVmDao.findById(vmIds.get(0)); + } + private Pair getKubernetesClusterServerIpSshPort(KubernetesCluster kubernetesCluster, UserVm masterVm) { int port = CLUSTER_NODES_DEFAULT_START_SSH_PORT; KubernetesClusterDetailsVO detail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS); @@ -498,19 +601,7 @@ private Pair getKubernetesClusterServerIpSshPort(KubernetesClus return new Pair<>(null, port); } else if (Network.GuestType.Shared.equals(network.getGuestType())) { port = 22; - if (masterVm == null) { - List clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); - if (CollectionUtils.isEmpty(clusterVMs)) { - LOGGER.warn(String.format("Unable to retrieve VMs for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); - return new Pair<>(null, port); - } - List vmIds = new ArrayList<>(); - for (KubernetesClusterVmMapVO vmMap : clusterVMs) { - vmIds.add(vmMap.getVmId()); - } - Collections.sort(vmIds); - masterVm = userVmDao.findById(vmIds.get(0)); - } + masterVm = fetchMasterVmIfMissing(kubernetesCluster, port, masterVm); if (masterVm == null) { LOGGER.warn(String.format("Unable to retrieve master VM for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); return new Pair<>(null, port); @@ -532,6 +623,8 @@ CLUSTER_NODE_VM_USER, getManagementServerSshPublicKeyFile(), null, 10000, 10000, 20000); if (result.first()) { return Integer.parseInt(result.second().trim().replace("\"", "")); + } else { + LOGGER.debug(String.format("Failed to retrieve ready nodes for Kubernetes cluster ID: %s. Output: %s", kubernetesCluster.getUuid(), result.second())); } return 0; } @@ -541,7 +634,11 @@ private boolean isKubernetesClusterNodeReady(KubernetesCluster kubernetesCluster CLUSTER_NODE_VM_USER, getManagementServerSshPublicKeyFile(), null, String.format("sudo kubectl get nodes | awk '{if ($1 == \"%s\" && $2 == \"Ready\") print $1}'", nodeName), 10000, 10000, 20000); - return result.first() && nodeName.equals(result.second().trim()); + if (result.first() && nodeName.equals(result.second().trim())) { + return true; + } + LOGGER.debug(String.format("Failed to retrieve status for node: %s in Kubernetes cluster ID: %s. Output: %s", nodeName, kubernetesCluster.getUuid(), result.second())); + return false; } private boolean isKubernetesClusterNodeReady(KubernetesCluster kubernetesCluster, String ipAddress, int port, String nodeName, int retries, int waitDuration) { @@ -664,39 +761,8 @@ private boolean uncordonKubernetesClusterNode(KubernetesCluster kubernetesCluste return false; } - // perform a cold start (which will provision resources as well) - private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) throws ManagementServerException { - - // Starting a Kubernetes cluster has below workflow - // - start the network - // - provision the master /node VM - // - provision node VM's (as many as cluster size) - // - update the book keeping data of the VM's provisioned for the cluster - // - setup networking (add Firewall and PF rules) - // - wait till Kubernetes API server on master VM to come up - // - wait till addon services (dashboard etc) to come up - // - update API and dashboard URL endpoints in Kubernetes cluster details - - KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); - final DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); - if (zone == null) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Unable to find zone for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); - } - LOGGER.debug(String.format("Starting Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.StartRequested); - Account account = accountDao.findById(kubernetesCluster.getAccountId()); - - DeployDestination dest = null; - try { - dest = plan(kubernetesCluster, zone); - } catch (InsufficientCapacityException e) { - String msg = String.format("Provisioning the cluster failed due to insufficient capacity in the Kubernetes cluster: %s", kubernetesCluster.getUuid()); - LOGGER.error(msg, e); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); - throw new ManagementServerException(msg, e); - } + private Network startKubernetesCLusterNetwork(final KubernetesCluster kubernetesCluster, final DeployDestination destination, final Account account) throws ManagementServerException { final ReservationContext context = new ReservationContextImpl(null, null, null, account); - Network network = networkDao.findById(kubernetesCluster.getNetworkId()); if (network == null) { String msg = String.format("Network for Kubernetes cluster ID: %s not found", kubernetesCluster.getUuid()); @@ -705,53 +771,36 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t throw new ManagementServerException(msg); } try { - networkMgr.startNetwork(network.getId(), dest, context); + networkMgr.startNetwork(network.getId(), destination, context); LOGGER.debug(String.format("Network ID: %s is started for the Kubernetes cluster ID: %s", network.getUuid(), kubernetesCluster.getUuid())); - } catch (Exception e) { + } catch (ConcurrentOperationException | ResourceUnavailableException |InsufficientCapacityException e) { String msg = String.format("Failed to start Kubernetes cluster ID: %s as unable to start associated network ID: %s" , kubernetesCluster.getUuid(), network.getUuid()); LOGGER.error(msg, e); stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); throw new ManagementServerException(msg, e); } + return network; + } - Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(kubernetesCluster); - String publicIpAddress = publicIpSshPort.first(); - if (Strings.isNullOrEmpty(publicIpAddress) && - (Network.GuestType.Isolated.equals(network.getGuestType()) || kubernetesCluster.getMasterNodeCount() > 1)) { // Shared network, single-master cluster won't have an IP yet - String msg = String.format("Failed to start Kubernetes cluster ID: %s as no public IP found for the cluster" , kubernetesCluster.getUuid()); - LOGGER.warn(msg); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); - throw new ManagementServerException(msg); - } - - List clusterVMIds = new ArrayList<>(); - + private UserVm provisionKubernetesClusterMasterVm(final KubernetesCluster kubernetesCluster, final DeployDestination destination, final Network network, final Account account, final String publicIpAddress) throws ManagementServerException { UserVm k8sMasterVM = null; try { - k8sMasterVM = createKubernetesMaster(kubernetesCluster, dest.getPod(), network, account, publicIpAddress); + k8sMasterVM = createKubernetesMaster(kubernetesCluster, destination.getPod(), network, account, publicIpAddress); addKubernetesClusterVm(kubernetesCluster.getId(), k8sMasterVM.getId()); startKubernetesVM(k8sMasterVM, kubernetesCluster); - clusterVMIds.add(k8sMasterVM.getId()); k8sMasterVM = userVmDao.findById(k8sMasterVM.getId()); LOGGER.debug(String.format("Provisioned the master VM ID: %s in to the Kubernetes cluster ID: %s", k8sMasterVM.getUuid(), kubernetesCluster.getUuid())); - } catch (Exception e) { + } catch (ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { String msg = String.format("Provisioning the master VM failed in the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); LOGGER.warn(msg, e); stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); throw new ManagementServerException(msg, e); } + return k8sMasterVM; + } - if (Strings.isNullOrEmpty(publicIpAddress)) { - publicIpSshPort = getKubernetesClusterServerIpSshPort(kubernetesCluster, k8sMasterVM); - publicIpAddress = publicIpSshPort.first(); - if (Strings.isNullOrEmpty(publicIpAddress)) { - String msg = String.format("Failed to start Kubernetes cluster ID: %s as no public IP found for the cluster", kubernetesCluster.getUuid()); - LOGGER.warn(msg); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); - throw new ManagementServerException(msg); - } - } - + private List provisionKubernetesClusterAdditionalMasterVms(final KubernetesCluster kubernetesCluster, final String publicIpAddress) throws ManagementServerException { + List additionalMasters = new ArrayList<>(); if (kubernetesCluster.getMasterNodeCount() > 1) { for (int i = 1; i < kubernetesCluster.getMasterNodeCount(); i++) { UserVm vm = null; @@ -759,9 +808,9 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t vm = createKubernetesAdditionalMaster(kubernetesCluster, publicIpAddress, i); addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId()); startKubernetesVM(vm, kubernetesCluster); - clusterVMIds.add(vm.getId()); + additionalMasters.add(vm); LOGGER.debug(String.format("Provisioned additional master VM ID: %s in to the Kubernetes cluster ID: %s", vm.getUuid(), kubernetesCluster.getUuid())); - } catch (Exception e) { + } catch (ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { String msg = String.format("Provisioning additional master VM %d/%d failed in the Kubernetes cluster ID: %s", i+1, kubernetesCluster.getMasterNodeCount(), kubernetesCluster.getUuid()); LOGGER.warn(msg, e); stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); @@ -769,32 +818,35 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t } } } + return additionalMasters; + } + private List provisionKubernetesClusterNodeVms(final KubernetesCluster kubernetesCluster, final String publicIpAddress) throws ManagementServerException { + List nodes = new ArrayList<>(); for (int i = 1; i <= kubernetesCluster.getNodeCount(); i++) { UserVm vm = null; try { vm = createKubernetesNode(kubernetesCluster, publicIpAddress, i); addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId()); startKubernetesVM(vm, kubernetesCluster); - clusterVMIds.add(vm.getId()); + nodes.add(vm); LOGGER.debug(String.format("Provisioned node master VM ID: %s in to the Kubernetes cluster ID: %s", vm.getUuid(), kubernetesCluster.getUuid())); - } catch (Exception e) { + } catch (ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { String msg = String.format("Provisioning node VM %d/%d failed in the Kubernetes cluster ID: %s", i, kubernetesCluster.getNodeCount(), kubernetesCluster.getUuid()); LOGGER.warn(msg, e); stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); throw new ManagementServerException(msg, e); } } - LOGGER.debug(String.format("Kubernetes cluster ID: %s VMs successfully provisioned", kubernetesCluster.getUuid())); - - setupKubernetesClusterNetworkRules(kubernetesCluster, network, account, clusterVMIds); - attachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); + return nodes; + } + private boolean isKubernetesClusterMasterVmRunning(final KubernetesCluster kubernetesCluster, final String ipAddress, final int port, final long timeout) { boolean masterVmRunning = false; long startTime = System.currentTimeMillis(); - while (!masterVmRunning && System.currentTimeMillis() - startTime < 10 * 60 * 1000) { + while (!masterVmRunning && System.currentTimeMillis() - startTime < timeout) { try (Socket socket = new Socket()) { - socket.connect(new InetSocketAddress(publicIpAddress, publicIpSshPort.second()), 10000); + socket.connect(new InetSocketAddress(ipAddress, port), 10000); masterVmRunning = true; } catch (IOException e) { LOGGER.debug(String.format("Waiting for Kubernetes cluster ID: %s master node VMs to be accessible", kubernetesCluster.getUuid())); @@ -805,7 +857,90 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t } } } - if (!masterVmRunning) { + return masterVmRunning; + } + + // Start cluster after creation (cluster will be started for first time therefore resources will be provisioned as well) + private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) throws ManagementServerException { + + // Starting a Kubernetes cluster has below workflow + // - start the network + // - provision the master / node VM + // - provision node VM's (as many as cluster size) + // - update the book keeping data of the VM's provisioned for the cluster + // - setup networking (add Firewall and PF rules) + // - wait till Kubernetes API server on master VM to come up + // - wait till addon services (dashboard etc) to come up + // - update API and dashboard URL endpoints in Kubernetes cluster details + + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + final DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); + if (zone == null) { + throw new CloudRuntimeException(String.format("Unable to find zone for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + } + LOGGER.debug(String.format("Starting Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.StartRequested); + Account account = accountDao.findById(kubernetesCluster.getAccountId()); + + DeployDestination dest = null; + try { + dest = plan(kubernetesCluster, zone); + } catch (InsufficientCapacityException e) { + String msg = String.format("Provisioning the cluster failed due to insufficient capacity in the Kubernetes cluster: %s", kubernetesCluster.getUuid()); + LOGGER.error(msg, e); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); + throw new ManagementServerException(msg, e); + } + + Network network = startKubernetesCLusterNetwork(kubernetesCluster, dest, account); + + Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(kubernetesCluster); + String publicIpAddress = publicIpSshPort.first(); + if (Strings.isNullOrEmpty(publicIpAddress) && + (Network.GuestType.Isolated.equals(network.getGuestType()) || kubernetesCluster.getMasterNodeCount() > 1)) { // Shared network, single-master cluster won't have an IP yet + String msg = String.format("Failed to start Kubernetes cluster ID: %s as no public IP found for the cluster" , kubernetesCluster.getUuid()); + LOGGER.warn(msg); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); + throw new ManagementServerException(msg); + } + + List clusterVMIds = new ArrayList<>(); + + UserVm k8sMasterVM = provisionKubernetesClusterMasterVm(kubernetesCluster, dest, network, account, publicIpAddress); + clusterVMIds.add(k8sMasterVM.getId()); + + if (Strings.isNullOrEmpty(publicIpAddress)) { + publicIpSshPort = getKubernetesClusterServerIpSshPort(kubernetesCluster, k8sMasterVM); + publicIpAddress = publicIpSshPort.first(); + if (Strings.isNullOrEmpty(publicIpAddress)) { + String msg = String.format("Failed to start Kubernetes cluster ID: %s as no public IP found for the cluster", kubernetesCluster.getUuid()); + LOGGER.warn(msg); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); + throw new ManagementServerException(msg); + } + } + + List additionalMasterVMs = provisionKubernetesClusterAdditionalMasterVms(kubernetesCluster, publicIpAddress); + for (UserVm vm : additionalMasterVMs){ + clusterVMIds.add(vm.getId()); + } + + List nodeVMs = provisionKubernetesClusterNodeVms(kubernetesCluster, publicIpAddress); + for (UserVm vm : nodeVMs){ + clusterVMIds.add(vm.getId()); + } + + LOGGER.debug(String.format("Kubernetes cluster ID: %s VMs successfully provisioned", kubernetesCluster.getUuid())); + + try { + setupKubernetesClusterNetworkRules(kubernetesCluster, network, account, clusterVMIds); + } catch (ManagementServerException e) { + logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster ID: %s, unable to setup network rules", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e); + } + + attachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); + + if (!isKubernetesClusterMasterVmRunning(kubernetesCluster, publicIpAddress, publicIpSshPort.second(), 10 * 60 * 1000)) { String msg = String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to access master node VMs of the cluster", kubernetesCluster.getUuid()); if (kubernetesCluster.getMasterNodeCount() > 1 && Network.GuestType.Shared.equals(network.getGuestType())) { msg = String.format("%s. Make sure external load-balancer has port forwarding rules for SSH access on ports %d-%d and API access on port %d", @@ -814,19 +949,12 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t CLUSTER_NODES_DEFAULT_START_SSH_PORT + kubernetesCluster.getTotalNodeCount() - 1, CLUSTER_API_PORT); } - LOGGER.error(msg); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); - detachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); + logTransitStateDetachIsoAndThrow(Level.ERROR, msg, kubernetesCluster, clusterVMIds, KubernetesCluster.Event.CreateFailed, null); } boolean k8sApiServerSetup = isKubernetesClusterServerRunning(kubernetesCluster, publicIpAddress, 20, 30000); if (!k8sApiServerSetup) { - String msg = String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to provision API endpoint for the cluster", kubernetesCluster.getUuid()); - LOGGER.error(msg); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); - detachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); + logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to provision API endpoint for the cluster", kubernetesCluster.getUuid()), kubernetesCluster, clusterVMIds, KubernetesCluster.Event.CreateFailed, null); } kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); kubernetesCluster.setEndpoint(String.format("https://%s:%d/", publicIpAddress, CLUSTER_API_PORT)); @@ -839,11 +967,8 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t detachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); // Throw exception if nodes count for k8s cluster timed out - if (!readyNodesCountValid) { // Scaling failed - String msg = String.format("Failed to setup Kubernetes cluster ID: %s as it does not have desired number of nodes in ready state", kubernetesCluster.getUuid()); - LOGGER.warn(msg); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); + if (!readyNodesCountValid) { + logTransitStateAndThrow(Level.WARN, String.format("Failed to setup Kubernetes cluster ID: %s as it does not have desired number of nodes in ready state", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); } boolean k8sKubeConfigCopied = false; @@ -852,29 +977,22 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t k8sKubeConfigCopied = true; } if (!k8sKubeConfigCopied) { - String msg = String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to retrieve kube-config for the cluster", kubernetesCluster.getUuid()); - LOGGER.error(msg); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); + logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to retrieve kube-config for the cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } kubeConfig = kubeConfig.replace(String.format("server: https://%s:%d", k8sMasterVM.getPrivateIpAddress(), CLUSTER_API_PORT), String.format("server: https://%s:%d", publicIpAddress, CLUSTER_API_PORT)); - kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "kubeConfigData", Base64.encodeBase64String(kubeConfig.getBytes(Charset.forName("UTF-8"))), false); + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "kubeConfigData", Base64.encodeBase64String(kubeConfig.getBytes(StringUtils.getPreferredCharset())), false); boolean dashboardServiceRunning = isKubernetesClusterDashboardServiceRunning(kubernetesCluster, publicIpAddress, sshPort, 10, 20000); if (!dashboardServiceRunning) { - String msg = String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to get Dashboard service running for the cluster", kubernetesCluster.getUuid()); - LOGGER.error(msg); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); + logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to get Dashboard service running for the cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(),KubernetesCluster.Event.OperationFailed); } kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "dashboardServiceRunning", String.valueOf(dashboardServiceRunning), false); stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); return true; } - private boolean startStoppedKubernetesCluster(long kubernetesClusterId) throws ManagementServerException, - ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException { + private boolean startStoppedKubernetesCluster(long kubernetesClusterId) throws ManagementServerException { final KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); if (kubernetesCluster == null) { throw new ManagementServerException("Invalid Kubernetes cluster ID"); @@ -902,8 +1020,8 @@ private boolean startStoppedKubernetesCluster(long kubernetesClusterId) throws M throw new ManagementServerException("Failed to start all VMs in Kubernetes cluster ID: " + kubernetesClusterId); } startKubernetesVM(vm, kubernetesCluster); - } catch (ServerApiException ex) { - LOGGER.warn("Failed to start VM in Kubernetes cluster ID:" + kubernetesClusterId + " due to " + ex); + } catch (CloudRuntimeException ex) { + LOGGER.warn(String.format("Failed to start VM in Kubernetes cluster ID: %s due to ", kubernetesCluster.getUuid()) + ex); // dont bail out here. proceed further to stop the reset of the VM's } } @@ -911,8 +1029,7 @@ private boolean startStoppedKubernetesCluster(long kubernetesClusterId) throws M for (final KubernetesClusterVmMapVO vmMapVO : kubernetesClusterVmMapDao.listByClusterId(kubernetesClusterId)) { final UserVmVO vm = userVmDao.findById(vmMapVO.getVmId()); if (vm == null || !vm.getState().equals(VirtualMachine.State.Running)) { - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ManagementServerException("Failed to start all VMs in Kubernetes cluster ID: " + kubernetesClusterId); + logTransitStateAndThrow(Level.ERROR, String.format("Failed to start all VMs in Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } } @@ -920,26 +1037,18 @@ private boolean startStoppedKubernetesCluster(long kubernetesClusterId) throws M try { address = InetAddress.getByName(new URL(kubernetesCluster.getEndpoint()).getHost()); } catch (MalformedURLException | UnknownHostException ex) { - String msg = String.format("Kubernetes cluster ID: %s has invalid API endpoint. Can not verify if cluster is in ready state", kubernetesCluster.getUuid()); - LOGGER.warn(msg, ex); - throw new ManagementServerException(msg, ex); + logTransitStateAndThrow(Level.ERROR, String.format("Kubernetes cluster ID: %s has invalid API endpoint. Can not verify if cluster is in ready state", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(kubernetesCluster); String publicIpAddress = publicIpSshPort.first(); if (Strings.isNullOrEmpty(publicIpAddress)) { - String msg = String.format("Failed to start Kubernetes cluster ID: %s as no public IP found for the cluster" , kubernetesCluster.getUuid()); - LOGGER.warn(msg); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ManagementServerException(msg); + logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s as no public IP found for the cluster" , kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } boolean k8sApiServerSetup = isKubernetesClusterServerRunning(kubernetesCluster, publicIpAddress, 10, 30000); if (!k8sApiServerSetup) { - String msg = String.format("Failed to start Kubernetes cluster ID: %s in usable state", kubernetesCluster.getUuid()); - LOGGER.error(msg); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ManagementServerException(msg); + logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s in usable state", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } int sshPort = publicIpSshPort.second(); @@ -951,21 +1060,15 @@ private boolean startStoppedKubernetesCluster(long kubernetesClusterId) throws M k8sKubeConfigCopied = true; } if (!k8sKubeConfigCopied) { - String msg = String.format("Failed to start Kubernetes cluster ID: %s in usable state as unable to retrieve kube-config for the cluster", kubernetesCluster.getUuid()); - LOGGER.error(msg); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); + logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s in usable state as unable to retrieve kube-config for the cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } - kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "kubeConfigData", Base64.encodeBase64String(kubeConfig.getBytes(Charset.forName("UTF-8"))), false); + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "kubeConfigData", Base64.encodeBase64String(kubeConfig.getBytes(StringUtils.getPreferredCharset())), false); } KubernetesClusterDetailsVO dashboardServiceRunningDetail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "dashboardServiceRunning"); if (kubeConfigDetail == null || !Boolean.parseBoolean(dashboardServiceRunningDetail.getValue())) { boolean dashboardServiceRunning = isKubernetesClusterDashboardServiceRunning(kubernetesCluster, publicIpAddress, sshPort, 10, 20000); if (!dashboardServiceRunning) { - String msg = String.format("Failed to start Kubernetes cluster ID: %s in usable state as unable to get Dashboard service running for the cluster", kubernetesCluster.getUuid()); - LOGGER.error(msg); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); + logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s in usable state as unable to get Dashboard service running for the cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "dashboardServiceRunning", String.valueOf(dashboardServiceRunning), false); } @@ -975,143 +1078,94 @@ private boolean startStoppedKubernetesCluster(long kubernetesClusterId) throws M return true; } - // Open up firewall port CLUSTER_API_PORT, secure port on which Kubernetes API server is running. Also create port-forwarding - // rule to forward public IP traffic to master VM private IP - // Open up firewall ports NODES_DEFAULT_START_SSH_PORT to NODES_DEFAULT_START_SSH_PORT+n for SSH access. Also create port-forwarding - // rule to forward public IP traffic to all node VM private IP - private void setupKubernetesClusterNetworkRules(KubernetesCluster kubernetesCluster, - Network network, Account account, - List clusterVMIds) throws ManagementServerException { - if (!Network.GuestType.Isolated.equals(network.getGuestType())) { - LOGGER.debug(String.format("Network ID: %s for Kubernetes cluster ID: %s is not an isolated network, therefore, no need for network rules", network.getUuid(), kubernetesCluster.getUuid())); - return; - } - IpAddress publicIp = null; + private IpAddress getSourceNatIp(Network network) { List addresses = networkModel.listPublicIpsAssignedToGuestNtwk(network.getId(), true); if (CollectionUtils.isEmpty(addresses)) { - LOGGER.error(String.format("No public IP addresses found for network ID: %s, Kubernetes cluster ID: %s", network.getUuid(), kubernetesCluster.getUuid())); - return; + return null; } for (IpAddress address : addresses) { if (address.isSourceNat()) { - publicIp = address; - break; + return address; } } - if (publicIp == null) { - LOGGER.error(String.format("No source NAT IP addresses found for network ID: %s, Kubernetes cluster ID: %s", network.getUuid(), kubernetesCluster.getUuid())); - return; + return null; + } + + private FirewallRule removeSshFirewallRule(IpAddress publicIp) { + FirewallRule rule = null; + List firewallRules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(publicIp.getId(), FirewallRule.Purpose.Firewall); + for (FirewallRuleVO firewallRule : firewallRules) { + if (firewallRule.getSourcePortStart() == CLUSTER_NODES_DEFAULT_START_SSH_PORT) { + rule = firewallRule; + firewallService.revokeIngressFwRule(firewallRule.getId(), true); + break; + } } + return rule; + } + + private void provisionFirewallRules(final IpAddress publicIp, final Account account, int startPort, int endPort) throws NoSuchFieldException, + IllegalAccessException, ResourceUnavailableException, NetworkRuleConflictException { List sourceCidrList = new ArrayList(); sourceCidrList.add("0.0.0.0/0"); - try { - CreateFirewallRuleCmd rule = new CreateFirewallRuleCmd(); - rule = ComponentContext.inject(rule); + CreateFirewallRuleCmd rule = new CreateFirewallRuleCmd(); + rule = ComponentContext.inject(rule); - Field addressField = rule.getClass().getDeclaredField("ipAddressId"); - addressField.setAccessible(true); - addressField.set(rule, publicIp.getId()); + Field addressField = rule.getClass().getDeclaredField("ipAddressId"); + addressField.setAccessible(true); + addressField.set(rule, publicIp.getId()); - Field protocolField = rule.getClass().getDeclaredField("protocol"); - protocolField.setAccessible(true); - protocolField.set(rule, "TCP"); + Field protocolField = rule.getClass().getDeclaredField("protocol"); + protocolField.setAccessible(true); + protocolField.set(rule, "TCP"); - Field startPortField = rule.getClass().getDeclaredField("publicStartPort"); - startPortField.setAccessible(true); - startPortField.set(rule, CLUSTER_API_PORT); + Field startPortField = rule.getClass().getDeclaredField("publicStartPort"); + startPortField.setAccessible(true); + startPortField.set(rule, startPort); - Field endPortField = rule.getClass().getDeclaredField("publicEndPort"); - endPortField.setAccessible(true); - endPortField.set(rule, CLUSTER_API_PORT); + Field endPortField = rule.getClass().getDeclaredField("publicEndPort"); + endPortField.setAccessible(true); + endPortField.set(rule, endPort); - Field cidrField = rule.getClass().getDeclaredField("cidrlist"); - cidrField.setAccessible(true); - cidrField.set(rule, sourceCidrList); + Field cidrField = rule.getClass().getDeclaredField("cidrlist"); + cidrField.setAccessible(true); + cidrField.set(rule, sourceCidrList); - firewallService.createIngressFirewallRule(rule); - firewallService.applyIngressFwRules(publicIp.getId(), account); + firewallService.createIngressFirewallRule(rule); + firewallService.applyIngressFwRules(publicIp.getId(), account); + } - LOGGER.debug(String.format("Provisioned firewall rule to open up port %d on %s for Kubernetes cluster ID: %s", - CLUSTER_API_PORT, publicIp.getAddress().addr(), kubernetesCluster.getUuid())); - } catch (Exception e) { - String msg = String.format("Failed to provision firewall rules for API access for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); - LOGGER.warn(msg, e); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); - throw new ManagementServerException(msg, e); + private void removePortForwardingRules(IpAddress publicIp, Network network, Account account, List removedVMIds) throws ResourceUnavailableException { + if (!CollectionUtils.isEmpty(removedVMIds)) { + for (Long vmId : removedVMIds) { + List pfRules = portForwardingRulesDao.listByNetwork(network.getId()); + for (PortForwardingRuleVO pfRule : pfRules) { + if (pfRule.getVirtualMachineId() == vmId) { + portForwardingRulesDao.remove(pfRule.getId()); + break; + } + } + } + rulesService.applyPortForwardingRules(publicIp.getId(), account); } + } - try { - CreateFirewallRuleCmd rule = new CreateFirewallRuleCmd(); - rule = ComponentContext.inject(rule); - - Field addressField = rule.getClass().getDeclaredField("ipAddressId"); - addressField.setAccessible(true); - addressField.set(rule, publicIp.getId()); + private void provisionSshPortForwardingRules(KubernetesCluster kubernetesCluster, IpAddress publicIp, Network network, Account account, List clusterVMIds, int firewallRuleSourcePortStart) throws ResourceUnavailableException, + NetworkRuleConflictException { + if (!CollectionUtils.isEmpty(clusterVMIds)) { // Upscaling, add new port-forwarding rules + // Apply port forwarding only to new VMs + final long publicIpId = publicIp.getId(); + final long networkId = network.getId(); + final long accountId = account.getId(); + final long domainId = account.getDomainId(); + for (int i = 0; i < clusterVMIds.size(); ++i) { + long vmId = clusterVMIds.get(i); + Nic vmNic = networkModel.getNicInNetwork(vmId, networkId); + final Ip vmIp = new Ip(vmNic.getIPv4Address()); + final long vmIdFinal = vmId; + final int srcPortFinal = firewallRuleSourcePortStart + i; - Field protocolField = rule.getClass().getDeclaredField("protocol"); - protocolField.setAccessible(true); - protocolField.set(rule, "TCP"); - - Field startPortField = rule.getClass().getDeclaredField("publicStartPort"); - startPortField.setAccessible(true); - startPortField.set(rule, CLUSTER_NODES_DEFAULT_START_SSH_PORT); - - Field endPortField = rule.getClass().getDeclaredField("publicEndPort"); - endPortField.setAccessible(true); - int endPort = CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterVMIds.size() - 1; - endPortField.set(rule, endPort); // clusterVMIds contains all nodes including master - - Field cidrField = rule.getClass().getDeclaredField("cidrlist"); - cidrField.setAccessible(true); - cidrField.set(rule, sourceCidrList); - - firewallService.createIngressFirewallRule(rule); - firewallService.applyIngressFwRules(publicIp.getId(), account); - - LOGGER.debug(String.format("Provisioned firewall rule to open up port %d to %d on %s for Kubernetes cluster ID: %s", CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort, publicIp.getAddress().addr(), kubernetesCluster.getUuid())); - } catch (Exception e) { - String msg = String.format("Failed to provision firewall rules for SSH access for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); - LOGGER.warn(msg, e); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); - throw new ManagementServerException(msg, e); - } - - // Load balancer rule fo API access for master node VMs - try { - LoadBalancer lb = lbService.createPublicLoadBalancerRule(null, "api-lb", "LB rule for API access", - CLUSTER_API_PORT, CLUSTER_API_PORT, CLUSTER_API_PORT, CLUSTER_API_PORT, - publicIp.getId(), NetUtils.TCP_PROTO, "roundrobin", kubernetesCluster.getNetworkId(), - kubernetesCluster.getAccountId(), false, NetUtils.TCP_PROTO, true); - - Map> vmIdIpMap = new HashMap<>(); - for (int i=0; i ips = new ArrayList<>(); - Nic masterVmNic = networkModel.getNicInNetwork(clusterVMIds.get(i), kubernetesCluster.getNetworkId()); - ips.add(masterVmNic.getIPv4Address()); - vmIdIpMap.put(clusterVMIds.get(i), ips); - } - lbService.assignToLoadBalancer(lb.getId(), null, vmIdIpMap); - } catch (Exception e) { - String msg = String.format("Failed to provision load balancer rule for API access for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); - LOGGER.warn(msg, e); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); - throw new ManagementServerException(msg, e); - } - - // Port forwarding rule fo SSH access on each node VM - final long publicIpId = publicIp.getId(); - final long networkId = kubernetesCluster.getNetworkId(); - final long accountId = account.getId(); - final long domainId = account.getDomainId(); - - for (int i = 0; i < clusterVMIds.size(); ++i) { - long vmId = clusterVMIds.get(i); - Nic vmNic = networkModel.getNicInNetwork(vmId, kubernetesCluster.getNetworkId()); - final Ip vmIp = new Ip(vmNic.getIPv4Address()); - final long vmIdFinal = vmId; - final int srcPortFinal = CLUSTER_NODES_DEFAULT_START_SSH_PORT + i; - try { PortForwardingRuleVO pfRule = Transaction.execute(new TransactionCallbackWithException() { @Override public PortForwardingRuleVO doInTransaction(TransactionStatus status) throws NetworkRuleConflictException { @@ -1128,190 +1182,126 @@ public PortForwardingRuleVO doInTransaction(TransactionStatus status) throws Net } }); rulesService.applyPortForwardingRules(publicIp.getId(), account); - LOGGER.debug(String.format("Provisioned SSH port forwarding rule from port %d to 22 on %s to the VM IP: %s in Kubernetes cluster ID: %s", srcPortFinal, publicIp.getAddress().addr(), vmIp, kubernetesCluster.getUuid())); - } catch (Exception e) { - String msg = String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); - LOGGER.warn(msg, e); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); - throw new ManagementServerException(msg, e); + LOGGER.debug(String.format("Provisioned SSH port forwarding rule from port %d to 22 on %s to the VM IP : %s in Kubernetes cluster ID: %s", srcPortFinal, publicIp.getAddress().addr(), vmIp.toString(), kubernetesCluster.getUuid())); } } } + private void provisionLoadBalancerRule(KubernetesCluster kubernetesCluster, IpAddress publicIp, Network network, Account account, List clusterVMIds, int port) throws NetworkRuleConflictException, + InsufficientAddressCapacityException { + LoadBalancer lb = lbService.createPublicLoadBalancerRule(null, "api-lb", "LB rule for API access", + port, port, port, port, + publicIp.getId(), NetUtils.TCP_PROTO, "roundrobin", network.getId(), + account.getId(), false, NetUtils.TCP_PROTO, true); + + Map> vmIdIpMap = new HashMap<>(); + for (int i=0; i ips = new ArrayList<>(); + Nic masterVmNic = networkModel.getNicInNetwork(clusterVMIds.get(i), kubernetesCluster.getNetworkId()); + ips.add(masterVmNic.getIPv4Address()); + vmIdIpMap.put(clusterVMIds.get(i), ips); + } + lbService.assignToLoadBalancer(lb.getId(), null, vmIdIpMap); + } + + // Open up firewall port CLUSTER_API_PORT, secure port on which Kubernetes API server is running. Also create port-forwarding + // rule to forward public IP traffic to master VM private IP // Open up firewall ports NODES_DEFAULT_START_SSH_PORT to NODES_DEFAULT_START_SSH_PORT+n for SSH access. Also create port-forwarding - // rule to forward public IP traffic to all node VM private IP. Existing node VMs before scaling - // will already be having these rules - private void scaleKubernetesClusterNetworkRules(KubernetesCluster kubernetesCluster, Network network, Account account, - List clusterVMIds, List removedVMIds) throws ManagementServerException { + // rule to forward public IP traffic to all node VM private IP + private void setupKubernetesClusterNetworkRules(KubernetesCluster kubernetesCluster, + Network network, Account account, + List clusterVMIds) throws ManagementServerException { if (!Network.GuestType.Isolated.equals(network.getGuestType())) { LOGGER.debug(String.format("Network ID: %s for Kubernetes cluster ID: %s is not an isolated network, therefore, no need for network rules", network.getUuid(), kubernetesCluster.getUuid())); return; } - IpAddress publicIp = null; - List addresses = networkModel.listPublicIpsAssignedToGuestNtwk(network.getId(), true); - if (CollectionUtils.isEmpty(addresses)) { - String msg = String.format("No public IP addresses found for network ID: %s, Kubernetes cluster ID: %s", network.getUuid(), kubernetesCluster.getUuid()); - LOGGER.error(msg); - throw new ManagementServerException(msg); - } - for (IpAddress address : addresses) { - if (address.isSourceNat()) { - publicIp = address; - break; - } - } + IpAddress publicIp = getSourceNatIp(network); if (publicIp == null) { - String msg = String.format("No source NAT IP addresses found for network ID: %s, Kubernetes cluster ID: %s", network.getUuid(), kubernetesCluster.getUuid()); - LOGGER.error(msg); - throw new ManagementServerException(msg); + throw new ManagementServerException(String.format("No source NAT IP addresses found for network ID: %s, Kubernetes cluster ID: %s", network.getUuid(), kubernetesCluster.getUuid())); } - List sourceCidrList = new ArrayList(); - sourceCidrList.add("0.0.0.0/0"); - boolean firewallRuleFound = false; - int existingFirewallRuleSourcePortEnd = CLUSTER_NODES_DEFAULT_START_SSH_PORT; - List firewallRules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(publicIp.getId(), FirewallRule.Purpose.Firewall); - for (FirewallRuleVO firewallRule : firewallRules) { - if (firewallRule.getSourcePortStart() == CLUSTER_NODES_DEFAULT_START_SSH_PORT) { - firewallRuleFound = true; - existingFirewallRuleSourcePortEnd = firewallRule.getSourcePortEnd(); - firewallService.revokeIngressFwRule(firewallRule.getId(), true); - break; - } - } - if (!firewallRuleFound) { - throw new ManagementServerException("Firewall rule for node SSH access can't be provisioned!"); - } try { - CreateFirewallRuleCmd rule = new CreateFirewallRuleCmd(); - rule = ComponentContext.inject(rule); - - Field addressField = rule.getClass().getDeclaredField("ipAddressId"); - addressField.setAccessible(true); - addressField.set(rule, publicIp.getId()); - - Field protocolField = rule.getClass().getDeclaredField("protocol"); - protocolField.setAccessible(true); - protocolField.set(rule, "TCP"); - - Field startPortField = rule.getClass().getDeclaredField("publicStartPort"); - startPortField.setAccessible(true); - startPortField.set(rule, CLUSTER_NODES_DEFAULT_START_SSH_PORT); - - Field endPortField = rule.getClass().getDeclaredField("publicEndPort"); - endPortField.setAccessible(true); - endPortField.set(rule, CLUSTER_NODES_DEFAULT_START_SSH_PORT + (int)kubernetesCluster.getNodeCount()); - - Field cidrField = rule.getClass().getDeclaredField("cidrlist"); - cidrField.setAccessible(true); - cidrField.set(rule, sourceCidrList); - - firewallService.createIngressFirewallRule(rule); - firewallService.applyIngressFwRules(publicIp.getId(), account); - - LOGGER.debug(String.format("Provisioned firewall rule to open up port %d to %d on %s in Kubernetes cluster ID: %s", - CLUSTER_NODES_DEFAULT_START_SSH_PORT, CLUSTER_NODES_DEFAULT_START_SSH_PORT + (int)kubernetesCluster.getNodeCount(), publicIp.getAddress().addr(), kubernetesCluster.getName())); - } catch (Exception e) { - String msg = String.format("Failed to activate SSH firewall rules for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); - LOGGER.warn(msg, e); - throw new ManagementServerException(msg, e); + provisionFirewallRules(publicIp, account, CLUSTER_API_PORT, CLUSTER_API_PORT); + LOGGER.debug(String.format("Provisioned firewall rule to open up port %d on %s for Kubernetes cluster ID: %s", + CLUSTER_API_PORT, publicIp.getAddress().addr(), kubernetesCluster.getUuid())); + } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) { + throw new ManagementServerException(String.format("Failed to provision firewall rules for API access for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); } - if (!CollectionUtils.isEmpty(removedVMIds)) { - try { - for (Long vmId : removedVMIds) { - List pfRules = portForwardingRulesDao.listByNetwork(kubernetesCluster.getNetworkId()); - for (PortForwardingRuleVO pfRule : pfRules) { - if (pfRule.getVirtualMachineId() == vmId) { - portForwardingRulesDao.remove(pfRule.getId()); - break; - } - } - } - rulesService.applyPortForwardingRules(publicIp.getId(), account); - } catch (Exception e) { - String msg = String.format("Failed to remove SSH port forwarding rules for removed VMs for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); - LOGGER.warn(msg, e); - throw new ManagementServerException(msg, e); - } + try { + int endPort = CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterVMIds.size() - 1; + provisionFirewallRules(publicIp, account, CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort); + LOGGER.debug(String.format("Provisioned firewall rule to open up port %d to %d on %s for Kubernetes cluster ID: %s", CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort, publicIp.getAddress().addr(), kubernetesCluster.getUuid())); + } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) { + throw new ManagementServerException(String.format("Failed to provision firewall rules for SSH access for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); } - if (!CollectionUtils.isEmpty(clusterVMIds)) { // Upscaling, add new port-forwarding rules - // Apply port forwarding only to new VMs - final long publicIpId = publicIp.getId(); - final long networkId = kubernetesCluster.getNetworkId(); - final long accountId = account.getId(); - final long domainId = account.getDomainId(); - for (int i = 0; i < clusterVMIds.size(); ++i) { - long vmId = clusterVMIds.get(i); - Nic vmNic = networkModel.getNicInNetwork(vmId, kubernetesCluster.getNetworkId()); - final Ip vmIp = new Ip(vmNic.getIPv4Address()); - final long vmIdFinal = vmId; - final int srcPortFinal = existingFirewallRuleSourcePortEnd + 1 + i; + // Load balancer rule fo API access for master node VMs + try { + provisionLoadBalancerRule(kubernetesCluster, publicIp, network, account, clusterVMIds, CLUSTER_API_PORT); + } catch (NetworkRuleConflictException | InsufficientAddressCapacityException e) { + throw new ManagementServerException(String.format("Failed to provision load balancer rule for API access for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); + } - try { - PortForwardingRuleVO pfRule = Transaction.execute(new TransactionCallbackWithException() { - @Override - public PortForwardingRuleVO doInTransaction(TransactionStatus status) throws NetworkRuleConflictException { - PortForwardingRuleVO newRule = - new PortForwardingRuleVO(null, publicIpId, - srcPortFinal, srcPortFinal, - vmIp, - 22, 22, - "tcp", networkId, accountId, domainId, vmIdFinal); - newRule.setDisplay(true); - newRule.setState(FirewallRule.State.Add); - newRule = portForwardingRulesDao.persist(newRule); - return newRule; - } - }); - rulesService.applyPortForwardingRules(publicIp.getId(), account); - LOGGER.debug(String.format("Provisioned SSH port forwarding rule from port %d to 22 on %s to the VM IP : %s in Kubernetes cluster ID: %s", srcPortFinal, publicIp.getAddress().addr(), vmIp.toString(), kubernetesCluster.getName())); - } catch (Exception e) { - String msg = String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); - LOGGER.warn(msg, e); - throw new ManagementServerException(msg, e); - } - } + // Port forwarding rule fo SSH access on each node VM + try { + provisionSshPortForwardingRules(kubernetesCluster, publicIp, network, account, clusterVMIds, CLUSTER_NODES_DEFAULT_START_SSH_PORT); + } catch (ResourceUnavailableException | NetworkRuleConflictException e) { + throw new ManagementServerException(String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); } } - private boolean validateNetwork(Network network, int clusterTotalNodeCount) { - NetworkOffering networkOffering = networkOfferingDao.findById(network.getNetworkOfferingId()); - if (networkOffering.isSystemOnly()) { - throw new InvalidParameterValueException(String.format("Network ID: %s is for system use only", network.getUuid())); + // Open up firewall ports NODES_DEFAULT_START_SSH_PORT to NODES_DEFAULT_START_SSH_PORT+n for SSH access. Also create port-forwarding + // rule to forward public IP traffic to all node VM private IP. Existing node VMs before scaling + // will already be having these rules + private void scaleKubernetesClusterNetworkRules(KubernetesCluster kubernetesCluster, Network network, Account account, + List clusterVMIds, List removedVMIds) throws ManagementServerException { + if (!Network.GuestType.Isolated.equals(network.getGuestType())) { + LOGGER.debug(String.format("Network ID: %s for Kubernetes cluster ID: %s is not an isolated network, therefore, no need for network rules", network.getUuid(), kubernetesCluster.getUuid())); + return; } - if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.UserData)) { - throw new InvalidParameterValueException(String.format("Network ID: %s does not support userdata that is required for Kubernetes cluster", network.getUuid())); + IpAddress publicIp = getSourceNatIp(network); + if (publicIp == null) { + throw new ManagementServerException(String.format("No source NAT IP addresses found for network ID: %s, Kubernetes cluster ID: %s", network.getUuid(), kubernetesCluster.getUuid())); } - if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.Firewall)) { - throw new InvalidParameterValueException(String.format("Network ID: %s does not support firewall that is required for Kubernetes cluster", network.getUuid())); + + // Remove existing SSH firewall rules + FirewallRule firewallRule = removeSshFirewallRule(publicIp); + if (firewallRule == null) { + throw new ManagementServerException("Firewall rule for node SSH access can't be provisioned!"); } - if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.PortForwarding)) { - throw new InvalidParameterValueException(String.format("Network ID: %s does not support port forwarding that is required for Kubernetes cluster", network.getUuid())); + int existingFirewallRuleSourcePortEnd = firewallRule.getSourcePortEnd(); + + // Provision new SSH firewall rules + try { + provisionFirewallRules(publicIp, account, CLUSTER_NODES_DEFAULT_START_SSH_PORT, CLUSTER_NODES_DEFAULT_START_SSH_PORT + (int)kubernetesCluster.getTotalNodeCount() - 1); + LOGGER.debug(String.format("Provisioned firewall rule to open up port %d to %d on %s in Kubernetes cluster ID: %s", + CLUSTER_NODES_DEFAULT_START_SSH_PORT, CLUSTER_NODES_DEFAULT_START_SSH_PORT + (int)kubernetesCluster.getTotalNodeCount() - 1, publicIp.getAddress().addr(), kubernetesCluster.getUuid())); + } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException e) { + throw new ManagementServerException(String.format("Failed to activate SSH firewall rules for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); } - if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.Dhcp)) { - throw new InvalidParameterValueException(String.format("Network ID: %s does not support DHCP that is required for Kubernetes cluster", network.getUuid())); + + try { + removePortForwardingRules(publicIp, network, account, removedVMIds); + } catch (ResourceUnavailableException e) { + throw new ManagementServerException(String.format("Failed to remove SSH port forwarding rules for removed VMs for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); } + try { + provisionSshPortForwardingRules(kubernetesCluster, publicIp, network, account, clusterVMIds, existingFirewallRuleSourcePortEnd + 1); + } catch (ResourceUnavailableException | NetworkRuleConflictException e) { + throw new ManagementServerException(String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); + } + } + + private boolean validateIsolatedNetwork(Network network, int clusterTotalNodeCount) { if (Network.GuestType.Isolated.equals(network.getGuestType())) { if (Network.State.Allocated.equals(network.getState())) { // Allocated networks won't have IP and rules return true; } - List addrs = networkModel.listPublicIpsAssignedToGuestNtwk(network.getId(), true); - IPAddressVO sourceNatIp = null; - if (addrs.isEmpty()) { - throw new InvalidParameterValueException(String.format("Network ID: %s does not have a public IP associated with it. To provision a Kubernetes Cluster, source NAT IP is required", network.getUuid())); - } else { - for (IpAddress addr : addrs) { - if (addr.isSourceNat()) { - sourceNatIp = ipAddressDao.findById(addr.getId()); - break; - } - } - if (sourceNatIp == null) { - throw new InvalidParameterValueException(String.format("Network ID: %s does not have a source NAT IP associated with it. To provision a Kubernetes Cluster, source NAT IP is required", network.getUuid())); - } + IpAddress sourceNatIp = getSourceNatIp(network); + if (sourceNatIp == null) { + throw new InvalidParameterValueException(String.format("Network ID: %s does not have a source NAT IP associated with it. To provision a Kubernetes Cluster, source NAT IP is required", network.getUuid())); } List rules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(sourceNatIp.getId(), FirewallRule.Purpose.Firewall); for (FirewallRuleVO rule : rules) { @@ -1341,6 +1331,27 @@ private boolean validateNetwork(Network network, int clusterTotalNodeCount) { return true; } + private boolean validateNetwork(Network network, int clusterTotalNodeCount) { + NetworkOffering networkOffering = networkOfferingDao.findById(network.getNetworkOfferingId()); + if (networkOffering.isSystemOnly()) { + throw new InvalidParameterValueException(String.format("Network ID: %s is for system use only", network.getUuid())); + } + if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.UserData)) { + throw new InvalidParameterValueException(String.format("Network ID: %s does not support userdata that is required for Kubernetes cluster", network.getUuid())); + } + if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.Firewall)) { + throw new InvalidParameterValueException(String.format("Network ID: %s does not support firewall that is required for Kubernetes cluster", network.getUuid())); + } + if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.PortForwarding)) { + throw new InvalidParameterValueException(String.format("Network ID: %s does not support port forwarding that is required for Kubernetes cluster", network.getUuid())); + } + if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.Dhcp)) { + throw new InvalidParameterValueException(String.format("Network ID: %s does not support DHCP that is required for Kubernetes cluster", network.getUuid())); + } + validateIsolatedNetwork(network, clusterTotalNodeCount); + return true; + } + private boolean validateServiceOffering(ServiceOffering serviceOffering) { if (serviceOffering.isDynamic()) { throw new InvalidParameterValueException(String.format("Custom service offerings are not supported for creating clusters, service offering ID: %s", serviceOffering.getUuid())); @@ -1562,8 +1573,8 @@ private void processFailedNetworkDelete(long kubernetesClusterId) { kubernetesClusterDao.update(cluster.getId(), cluster); } - private UserVm createKubernetesMaster(final KubernetesClusterVO kubernetesCluster, final Pod pod, final Network network, final Account account, String serverIp) throws ManagementServerException, - ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException { + private UserVm createKubernetesMaster(final KubernetesCluster kubernetesCluster, final Pod pod, final Network network, final Account account, String serverIp) throws ManagementServerException, + ResourceUnavailableException, InsufficientCapacityException { UserVm masterVm = null; DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); ServiceOffering serviceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); @@ -1604,7 +1615,7 @@ private UserVm createKubernetesMaster(final KubernetesClusterVO kubernetesCluste final KubernetesSupportedVersion version = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); if (version != null) { try { - if (KubernetesVersionManagerImpl.compareKubernetesVersion(version.getKubernetesVersion(), MIN_KUBERNETES_VERSION_HA_SUPPORT) >= 0) { + if (KubernetesVersionManagerImpl.compareKubernetesVersion(version.getSemanticVersion(), MIN_KUBERNETES_VERSION_HA_SUPPORT) >= 0) { haSupported = true; } } catch (Exception e) { @@ -1658,7 +1669,7 @@ private UserVm createKubernetesMaster(final KubernetesClusterVO kubernetesCluste LOGGER.error(msg, e); throw new ManagementServerException(msg, e); } - String base64UserData = Base64.encodeBase64String(k8sMasterConfig.getBytes(StandardCharsets.UTF_8)); + String base64UserData = Base64.encodeBase64String(k8sMasterConfig.getBytes(StringUtils.getPreferredCharset())); masterVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, hostName, kubernetesCluster.getDescription(), null, null, null, null, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), @@ -1667,8 +1678,8 @@ private UserVm createKubernetesMaster(final KubernetesClusterVO kubernetesCluste return masterVm; } - private UserVm createKubernetesAdditionalMaster(final KubernetesClusterVO kubernetesCluster, final String joinIp, final int additionalMasterNodeInstance) throws ManagementServerException, - ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException { + private UserVm createKubernetesAdditionalMaster(final KubernetesCluster kubernetesCluster, final String joinIp, final int additionalMasterNodeInstance) throws ManagementServerException, + ResourceUnavailableException, InsufficientCapacityException { UserVm additionalMasterVm = null; DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); ServiceOffering serviceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); @@ -1707,7 +1718,7 @@ private UserVm createKubernetesAdditionalMaster(final KubernetesClusterVO kubern LOGGER.error(msg, e); throw new ManagementServerException(msg, e); } - String base64UserData = Base64.encodeBase64String(k8sMasterConfig.getBytes(StandardCharsets.UTF_8)); + String base64UserData = Base64.encodeBase64String(k8sMasterConfig.getBytes(StringUtils.getPreferredCharset())); additionalMasterVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, hostName, kubernetesCluster.getDescription(), null, null, null, null, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), @@ -1716,8 +1727,8 @@ private UserVm createKubernetesAdditionalMaster(final KubernetesClusterVO kubern return additionalMasterVm; } - private UserVm createKubernetesNode(KubernetesClusterVO kubernetesCluster, String joinIp, int nodeInstance) throws ManagementServerException, - ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException { + private UserVm createKubernetesNode(KubernetesCluster kubernetesCluster, String joinIp, int nodeInstance) throws ManagementServerException, + ResourceUnavailableException, InsufficientCapacityException { UserVm nodeVm = null; DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); ServiceOffering serviceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); @@ -1791,7 +1802,7 @@ private UserVm createKubernetesNode(KubernetesClusterVO kubernetesCluster, Strin final String dockerAuthKey = "{{docker.secret}}"; final String dockerEmailKey = "{{docker.email}}"; final String usernamePasswordKey = dockerUserName + ":" + dockerPassword; - String base64Auth = Base64.encodeBase64String(usernamePasswordKey.getBytes(StandardCharsets.UTF_8)); + String base64Auth = Base64.encodeBase64String(usernamePasswordKey.getBytes(StringUtils.getPreferredCharset())); k8sNodeConfig = k8sNodeConfig.replace(dockerUrlKey, "\"" + dockerRegistryUrl + "\""); k8sNodeConfig = k8sNodeConfig.replace(dockerAuthKey, "\"" + base64Auth + "\""); k8sNodeConfig = k8sNodeConfig.replace(dockerEmailKey, "\"" + dockerRegistryEmail + "\""); @@ -1801,7 +1812,7 @@ private UserVm createKubernetesNode(KubernetesClusterVO kubernetesCluster, Strin LOGGER.error(msg, e); throw new ManagementServerException(msg, e); } - String base64UserData = Base64.encodeBase64String(k8sNodeConfig.getBytes(StandardCharsets.UTF_8)); + String base64UserData = Base64.encodeBase64String(k8sNodeConfig.getBytes(StringUtils.getPreferredCharset())); nodeVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, hostName, kubernetesCluster.getDescription(), null, null, null, null, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), @@ -1810,7 +1821,7 @@ private UserVm createKubernetesNode(KubernetesClusterVO kubernetesCluster, Strin return nodeVm; } - private void startKubernetesVM(final UserVm vm, final KubernetesClusterVO kubernetesCluster) throws ServerApiException { + private void startKubernetesVM(final UserVm vm, final KubernetesCluster kubernetesCluster) throws ConcurrentOperationException { try { StartVMCmd startVm = new StartVMCmd(); startVm = ComponentContext.inject(startVm); @@ -1820,53 +1831,42 @@ private void startKubernetesVM(final UserVm vm, final KubernetesClusterVO kubern userVmService.startVirtualMachine(startVm); LOGGER.debug(String.format("Started VM ID: %s in the Kubernetes cluster ID: %s", vm.getUuid(), kubernetesCluster.getUuid())); } catch (Exception ex) { - String msg = String.format("Failed to start VM in the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); - LOGGER.warn(msg, ex); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg, ex); + logAndThrow(Level.WARN, String.format("Failed to start VM in the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), ex); } UserVm startVm = userVmDao.findById(vm.getId()); if (!startVm.getState().equals(VirtualMachine.State.Running)) { - String msg = String.format("Failed to start VM in the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); - LOGGER.warn(msg); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); + logAndThrow(Level.WARN, String.format("Failed to start VM in the Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); } } - private void attachIsoKubernetesVMs(KubernetesCluster kubernetesCluster, List clusterVMIds) throws ServerApiException { + private void attachIsoKubernetesVMs(KubernetesCluster kubernetesCluster, List clusterVMIds) throws CloudRuntimeException { KubernetesSupportedVersion version = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); if (version == null) { - LOGGER.error(String .format("Unable to find Kubernetes version for cluster ID: %s", kubernetesCluster.getUuid())); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String .format("Unable to find Kubernetes version for cluster ID: %s", kubernetesCluster.getUuid())); + logAndThrow(Level.ERROR, String .format("Unable to find Kubernetes version for cluster ID: %s", kubernetesCluster.getUuid())); } VMTemplateVO iso = templateDao.findById(version.getIsoId()); if (iso == null) { - LOGGER.error(String.format("Unable to attach ISO to Kubernetes cluster ID: %s. Binaries ISO not found.", kubernetesCluster.getUuid())); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Unable to attach ISO to Kubernetes cluster ID: %s. Binaries ISO not found.", kubernetesCluster.getUuid())); + logAndThrow(Level.ERROR, String.format("Unable to attach ISO to Kubernetes cluster ID: %s. Binaries ISO not found.", kubernetesCluster.getUuid())); } if (!iso.getFormat().equals(Storage.ImageFormat.ISO)) { - LOGGER.error(String.format("Unable to attach ISO to Kubernetes cluster ID: %s. Invalid Binaries ISO.", kubernetesCluster.getUuid())); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Unable to attach ISO to Kubernetes cluster ID: %s. Invalid Binaries ISO.", kubernetesCluster.getUuid())); + logAndThrow(Level.ERROR, String.format("Unable to attach ISO to Kubernetes cluster ID: %s. Invalid Binaries ISO.", kubernetesCluster.getUuid())); } if (!iso.getState().equals(VirtualMachineTemplate.State.Active)) { - LOGGER.error(String.format("Unable to attach ISO to Kubernetes cluster ID: %s. Binaries ISO not active.", kubernetesCluster.getUuid())); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Unable to attach ISO to Kubernetes cluster ID: %s. Binaries ISO not active.", kubernetesCluster.getUuid())); + logAndThrow(Level.ERROR, String.format("Unable to attach ISO to Kubernetes cluster ID: %s. Binaries ISO not active.", kubernetesCluster.getUuid())); } for (Long clusterVMId : clusterVMIds) { UserVm vm = userVmDao.findById(clusterVMId); try { templateService.attachIso(iso.getId(), vm.getId()); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Attached binaries ISO for VM: %s in cluster: %s", vm.getUuid(), kubernetesCluster.getName())); - } + LOGGER.debug(String.format("Attached binaries ISO for VM: %s in cluster: %s", vm.getUuid(), kubernetesCluster.getName())); } catch (CloudRuntimeException ex) { - LOGGER.warn(String.format("Failed to attach binaries ISO for VM: %s in the Kubernetes cluster name: %s due to Exception: ", vm.getDisplayName(), kubernetesCluster.getName()), ex); - // throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to attach binaries ISO for VM: %s in the Kubernetes cluster name: %s", vm.getDisplayName(), kubernetesCluster.getName()), ex); + logAndThrow(Level.ERROR, String.format("Failed to attach binaries ISO for VM: %s in the Kubernetes cluster name: %s", vm.getDisplayName(), kubernetesCluster.getName()), ex); } } } - private void detachIsoKubernetesVMs(final KubernetesCluster kubernetesCluster, List clusterVMIds) throws ServerApiException { + private void detachIsoKubernetesVMs(final KubernetesCluster kubernetesCluster, List clusterVMIds) throws CloudRuntimeException { for (Long clusterVMId : clusterVMIds) { UserVm vm = userVmDao.findById(clusterVMId); boolean result = false; @@ -1883,12 +1883,11 @@ private void detachIsoKubernetesVMs(final KubernetesCluster kubernetesCluster, L } } - private void stopClusterVM(final KubernetesClusterVmMapVO vmMapVO) throws ServerApiException { + private void stopClusterVM(final KubernetesClusterVmMapVO vmMapVO) throws CloudRuntimeException { try { userVmService.stopVirtualMachine(vmMapVO.getVmId(), false); } catch (ConcurrentOperationException ex) { - LOGGER.warn("Failed to stop Kubernetes cluster VM", ex); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to stop Kubernetes cluster VM", ex); + logAndThrow(Level.WARN, "Failed to stop Kubernetes cluster VM", ex); } } @@ -1937,30 +1936,14 @@ public KubernetesClusterResponse createKubernetesClusterResponse(long kubernetes return response; } - protected boolean stateTransitTo(long kubernetesClusterId, KubernetesCluster.Event e) { - KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); - try { - return _stateMachine.transitTo(kubernetesCluster, e, null, kubernetesClusterDao); - } catch (NoTransitionException nte) { - LOGGER.warn(String.format("Failed to transition state of the Kubernetes cluster ID: %s in state %s on event %s", kubernetesCluster.getUuid(), kubernetesCluster.getState().toString(), e.toString()), nte); - return false; - } - } - - @Override - public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) - throws InsufficientCapacityException, ResourceAllocationException, ManagementServerException { - if (!Boolean.parseBoolean(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesServiceEnabled.key()))) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Kubernetes Service plugin is disabled"); - } + private void validateKubernetesClusterCreatePrameters(final CreateKubernetesClusterCmd cmd) throws ManagementServerException { final String name = cmd.getName(); - final String displayName = cmd.getDisplayName(); final Long zoneId = cmd.getZoneId(); final Long kubernetesVersionId = cmd.getKubernetesVersionId(); final Long serviceOfferingId = cmd.getServiceOfferingId(); final Account owner = accountService.getActiveAccountById(cmd.getEntityOwnerId()); final Long networkId = cmd.getNetworkId(); - final String sshKeyPair= cmd.getSSHKeyPairName(); + final String sshKeyPair = cmd.getSSHKeyPairName(); final Long masterNodeCount = cmd.getMasterNodes(); final Long clusterSize = cmd.getClusterSize(); final String dockerRegistryUserName = cmd.getDockerRegistryUserName(); @@ -1991,6 +1974,10 @@ public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) throw new PermissionDeniedException(String.format("Cannot perform this operation, zone ID: %s is currently disabled", zone.getUuid())); } + if (!isKubernetesServiceConfigured(zone)) { + throw new ManagementServerException("Kubernetes service has not been configured properly to provision Kubernetes clusters"); + } + final KubernetesSupportedVersion clusterKubernetesVersion = kubernetesSupportedVersionDao.findById(kubernetesVersionId); if (clusterKubernetesVersion == null) { throw new InvalidParameterValueException("Unable to find given Kubernetes version in supported versions"); @@ -2000,13 +1987,11 @@ public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) } if (masterNodeCount > 1 ) { try { - if (KubernetesVersionManagerImpl.compareKubernetesVersion(clusterKubernetesVersion.getKubernetesVersion(), MIN_KUBERNETES_VERSION_HA_SUPPORT) < 0) { - throw new InvalidParameterValueException(String.format("HA support is available only for Kubernetes version %s and above. Given version ID: %s is %s", MIN_KUBERNETES_VERSION_HA_SUPPORT, clusterKubernetesVersion.getUuid(), clusterKubernetesVersion.getKubernetesVersion())); + if (KubernetesVersionManagerImpl.compareKubernetesVersion(clusterKubernetesVersion.getSemanticVersion(), MIN_KUBERNETES_VERSION_HA_SUPPORT) < 0) { + throw new InvalidParameterValueException(String.format("HA support is available only for Kubernetes version %s and above. Given version ID: %s is %s", MIN_KUBERNETES_VERSION_HA_SUPPORT, clusterKubernetesVersion.getUuid(), clusterKubernetesVersion.getSemanticVersion())); } } catch (Exception e) { - String msg = String.format("Unable to compare Kubernetes version for given version ID: %s with %s", clusterKubernetesVersion.getUuid(), MIN_KUBERNETES_VERSION_HA_SUPPORT); - LOGGER.error(msg, e); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg, e); + logAndThrow(Level.WARN, String.format("Unable to compare Kubernetes version for given version ID: %s with %s", clusterKubernetesVersion.getUuid(), MIN_KUBERNETES_VERSION_HA_SUPPORT), e); } } @@ -2034,15 +2019,11 @@ public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) } } - if (!isKubernetesServiceConfigured(zone)) { - throw new ManagementServerException("Kubernetes service has not been configured properly to provision Kubernetes clusters"); - } - if (nodeRootDiskSize <= 0) { - throw new ManagementServerException(String.format("Invalid value for %s", ApiConstants.NODE_ROOT_DISK_SIZE)); + throw new InvalidParameterValueException(String.format("Invalid value for %s", ApiConstants.NODE_ROOT_DISK_SIZE)); } - VMTemplateVO template = templateDao.findByTemplateName(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesClusterTemplateName.key())); + VMTemplateVO template = templateDao.findByTemplateName(KubernetesClusterTemplateName.value()); List listZoneTemplate = templateZoneDao.listByZoneTemplate(zone.getId(), template.getId()); if (listZoneTemplate == null || listZoneTemplate.isEmpty()) { String msg = String.format("The template ID: %s is not available for use in zone ID: %s to provision Kubernetes cluster name: %s", template.getUuid(), zone.getUuid(), name); @@ -2075,13 +2056,16 @@ public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) throw new InvalidParameterValueException(String.format("%s parameter must be specified along with %s type of network", ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS, Network.GuestType.Shared.toString())); } } + } - plan(masterNodeCount + clusterSize, zone, serviceOfferingDao.findById(serviceOfferingId)); - - if (network != null) { + private Network getKubernetesClusterNetworkIfMissing(final String clusterName, final DataCenter zone, final Account owner, final int masterNodesCount, + final int nodesCount, final String externalLoadBalancerIpAddress, final Long networkId) throws ManagementServerException { + Network network = null; + if (networkId != null) { + network = networkDao.findById(networkId); if (Network.GuestType.Isolated.equals(network.getGuestType())) { if (kubernetesClusterDao.listByNetworkId(network.getId()).isEmpty()) { - if (!validateNetwork(network, (int) (masterNodeCount + clusterSize))) { + if (!validateNetwork(network, masterNodesCount + nodesCount)) { throw new InvalidParameterValueException(String.format("Network ID: %s is not suitable for Kubernetes cluster", network.getUuid())); } networkModel.checkNetworkPermissions(owner, network); @@ -2089,114 +2073,524 @@ public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) throw new InvalidParameterValueException(String.format("Network ID: %s is already under use by another Kubernetes cluster", network.getUuid())); } } else if (Network.GuestType.Shared.equals(network.getGuestType())) { - if (masterNodeCount > 1 && Strings.isNullOrEmpty(externalLoadBalancerIpAddress)) { + if (masterNodesCount > 1 && Strings.isNullOrEmpty(externalLoadBalancerIpAddress)) { throw new InvalidParameterValueException(String.format("Multi-master, HA Kubernetes cluster with %s network ID: %s needs an external load balancer IP address. %s parameter can be used", network.getGuestType().toString(), network.getUuid(), ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS)); } } } else { // user has not specified network in which cluster VM's to be provisioned, so create a network for Kubernetes cluster - NetworkOfferingVO networkOffering = networkOfferingDao.findByUniqueName( - globalConfigDao.getValue(KubernetesServiceConfig.KubernetesClusterNetworkOffering.key())); + NetworkOfferingVO networkOffering = networkOfferingDao.findByUniqueName(KubernetesClusterNetworkOffering.value()); long physicalNetworkId = networkModel.findPhysicalNetworkId(zone.getId(), networkOffering.getTags(), networkOffering.getTrafficType()); PhysicalNetwork physicalNetwork = physicalNetworkDao.findById(physicalNetworkId); - LOGGER.debug(String.format("Creating network for account ID: %s from the network offering ID: %s as part of Kubernetes cluster: %s deployment process", owner.getUuid(), networkOffering.getUuid(), name)); + LOGGER.debug(String.format("Creating network for account ID: %s from the network offering ID: %s as part of Kubernetes cluster: %s deployment process", owner.getUuid(), networkOffering.getUuid(), clusterName)); try { - network = networkMgr.createGuestNetwork(networkOffering.getId(), name + "-network", owner.getAccountName() + "-network", + network = networkMgr.createGuestNetwork(networkOffering.getId(), clusterName + "-network", owner.getAccountName() + "-network", null, null, null, false, null, owner, null, physicalNetwork, zone.getId(), ControlledEntity.ACLType.Account, null, null, null, null, true, null, null); - } catch (Exception e) { - String msg = String.format("Unable to create network for the Kubernetes cluster: %s", name); + } catch (ConcurrentOperationException | InsufficientCapacityException | ResourceAllocationException e) { + String msg = String.format("Unable to create network for the Kubernetes cluster: %s", clusterName); LOGGER.warn(msg, e); throw new ManagementServerException(msg, e); } } + return network; + } - final Network defaultNetwork = network; - final VMTemplateVO finalTemplate = template; - final long cores = serviceOffering.getCpu() * (masterNodeCount + clusterSize); - final long memory = serviceOffering.getRamSize() * (masterNodeCount + clusterSize); - - final KubernetesClusterVO cluster = Transaction.execute(new TransactionCallback() { - @Override - public KubernetesClusterVO doInTransaction(TransactionStatus status) { - KubernetesClusterVO newCluster = new KubernetesClusterVO(name, displayName, zoneId, clusterKubernetesVersion.getId(), - serviceOfferingId, finalTemplate.getId(), defaultNetwork.getId(), owner.getDomainId(), - owner.getAccountId(), masterNodeCount, clusterSize, KubernetesCluster.State.Created, sshKeyPair, cores, memory, nodeRootDiskSize, ""); - kubernetesClusterDao.persist(newCluster); - return newCluster; - } - }); - + private void addKubernetesClusterDetails(final KubernetesCluster kubernetesCluster, final Network network, final CreateKubernetesClusterCmd cmd) { + final String externalLoadBalancerIpAddress = cmd.getExternalLoadBalancerIpAddress(); + final String dockerRegistryUserName = cmd.getDockerRegistryUserName(); + final String dockerRegistryPassword = cmd.getDockerRegistryPassword(); + final String dockerRegistryUrl = cmd.getDockerRegistryUrl(); + final String dockerRegistryEmail = cmd.getDockerRegistryEmail(); + final boolean networkCleanup = cmd.getNetworkId() == null; Transaction.execute(new TransactionCallbackNoReturn() { @Override public void doInTransactionWithoutResult(TransactionStatus status) { List details = new ArrayList<>(); - if (Network.GuestType.Shared.equals(defaultNetwork.getGuestType()) && !Strings.isNullOrEmpty(externalLoadBalancerIpAddress)) { - details.add(new KubernetesClusterDetailsVO(cluster.getId(), ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS, externalLoadBalancerIpAddress, true)); + if (Network.GuestType.Shared.equals(network.getGuestType()) && !Strings.isNullOrEmpty(externalLoadBalancerIpAddress)) { + details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS, externalLoadBalancerIpAddress, true)); } if (!Strings.isNullOrEmpty(dockerRegistryUserName)) { - details.add(new KubernetesClusterDetailsVO(cluster.getId(), ApiConstants.DOCKER_REGISTRY_USER_NAME, dockerRegistryUserName, true)); + details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), ApiConstants.DOCKER_REGISTRY_USER_NAME, dockerRegistryUserName, true)); } if (!Strings.isNullOrEmpty(dockerRegistryPassword)) { - details.add(new KubernetesClusterDetailsVO(cluster.getId(), ApiConstants.DOCKER_REGISTRY_PASSWORD, dockerRegistryPassword, false)); + details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), ApiConstants.DOCKER_REGISTRY_PASSWORD, dockerRegistryPassword, false)); } if (!Strings.isNullOrEmpty(dockerRegistryUrl)) { - details.add(new KubernetesClusterDetailsVO(cluster.getId(), ApiConstants.DOCKER_REGISTRY_URL, dockerRegistryUrl, true)); + details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), ApiConstants.DOCKER_REGISTRY_URL, dockerRegistryUrl, true)); } if (!Strings.isNullOrEmpty(dockerRegistryEmail)) { - details.add(new KubernetesClusterDetailsVO(cluster.getId(), ApiConstants.DOCKER_REGISTRY_EMAIL, dockerRegistryEmail, true)); + details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), ApiConstants.DOCKER_REGISTRY_EMAIL, dockerRegistryEmail, true)); } - details.add(new KubernetesClusterDetailsVO(cluster.getId(), ApiConstants.USERNAME, "admin", true)); + details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), ApiConstants.USERNAME, "admin", true)); SecureRandom random = new SecureRandom(); String randomPassword = new BigInteger(130, random).toString(32); - details.add(new KubernetesClusterDetailsVO(cluster.getId(), ApiConstants.PASSWORD, randomPassword, false)); - details.add(new KubernetesClusterDetailsVO(cluster.getId(), "networkCleanup", String.valueOf(networkId == null), true)); + details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), ApiConstants.PASSWORD, randomPassword, false)); + details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), "networkCleanup", String.valueOf(networkCleanup), true)); kubernetesClusterDetailsDao.saveDetails(details); } }); - LOGGER.debug(String.format("Kubernetes cluster name: %s and ID: %s has been created", cluster.getName(), cluster.getUuid())); - return cluster; } - - // Start operation can be performed at two diffrent life stagevs of Kubernetes cluster. First when a freshly created cluster - // in which case there are no resources provisisioned for the Kubernetes cluster. So during start all the resources - // are provisioned from scratch. Second kind of start, happens on Stopped Kubernetes cluster, in which all resources - // are provisioned (like volumes, nics, networks etc). It just that VM's are not in running state. So just - // start the VM's (which can possibly implicitly start the network also). - @Override - public boolean startKubernetesCluster(long kubernetesClusterId, boolean onCreate) throws ManagementServerException, - ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException { - if (!Boolean.parseBoolean(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesServiceEnabled.key()))) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Kubernetes Service plugin is disabled"); - } - if (onCreate) { - // Start for Kubernetes cluster in 'Created' state - return startKubernetesClusterOnCreate(kubernetesClusterId); - } else { - // Start for Kubernetes cluster in 'Stopped' state. Resources are already provisioned, just need to be started - return startStoppedKubernetesCluster(kubernetesClusterId); + private void validateKubernetesClusterScaleParameters(ScaleKubernetesClusterCmd cmd) { + final Long kubernetesClusterId = cmd.getId(); + final Long serviceOfferingId = cmd.getServiceOfferingId(); + final Long clusterSize = cmd.getClusterSize(); + if (kubernetesClusterId == null || kubernetesClusterId < 1L) { + throw new InvalidParameterValueException("Invalid Kubernetes cluster ID"); } - } - - @Override - public boolean stopKubernetesCluster(long kubernetesClusterId) throws ManagementServerException { - if (!Boolean.parseBoolean(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesServiceEnabled.key()))) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Kubernetes Service plugin is disabled"); + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + if (kubernetesCluster == null || kubernetesCluster.getRemoved() != null) { + throw new InvalidParameterValueException("Invalid Kubernetes cluster ID"); } - final KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); - if (kubernetesCluster == null) { - throw new ManagementServerException("Failed to find Kubernetes cluster with given ID"); + final DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); + if (zone == null) { + logAndThrow(Level.WARN, String.format("Unable to find zone for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); } - if (kubernetesCluster.getRemoved() != null) { - throw new ManagementServerException(String.format("Kubernetes cluster ID: %s is already deleted", kubernetesCluster.getUuid())); + Account caller = CallContext.current().getCallingAccount(); + accountManager.checkAccess(caller, SecurityChecker.AccessType.OperateEntry, false, kubernetesCluster); + + if (serviceOfferingId == null && clusterSize == null) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be scaled, either a new service offering or a new cluster size must be passed", kubernetesCluster.getUuid())); } - if (kubernetesCluster.getState().equals(KubernetesCluster.State.Stopped)) { - LOGGER.debug(String.format("Kubernetes cluster ID: %s is already stopped", kubernetesCluster.getUuid())); + ServiceOffering serviceOffering = null; + if (serviceOfferingId != null) { + serviceOffering = serviceOfferingDao.findById(serviceOfferingId); + if (serviceOffering == null) { + throw new InvalidParameterValueException("Failed to find service offering ID: " + serviceOfferingId); + } else { + if (serviceOffering.isDynamic()) { + throw new InvalidParameterValueException(String.format("Custom service offerings are not supported for Kubernetes clusters. Kubernetes cluster ID: %s, service offering ID: %s", kubernetesCluster.getUuid(), serviceOffering.getUuid())); + } + if (serviceOffering.getCpu() < 2 || serviceOffering.getRamSize() < 2048) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be scaled with service offering ID: %s, Kubernetes cluster template(CoreOS) needs minimum 2 vCPUs and 2 GB RAM", kubernetesCluster.getUuid(), serviceOffering.getUuid())); + } + } + } + + if (!(kubernetesCluster.getState().equals(KubernetesCluster.State.Created) || + kubernetesCluster.getState().equals(KubernetesCluster.State.Running) || + kubernetesCluster.getState().equals(KubernetesCluster.State.Stopped))) { + throw new PermissionDeniedException(String.format("Kubernetes cluster ID: %s is in %s state", kubernetesCluster.getUuid(), kubernetesCluster.getState().toString())); + } + + if (clusterSize != null) { + if (kubernetesCluster.getState().equals(KubernetesCluster.State.Stopped)) { // Cannot scale stopped cluster currently for cluster size + throw new PermissionDeniedException(String.format("Kubernetes cluster ID: %s is in %s state", kubernetesCluster.getUuid(), kubernetesCluster.getState().toString())); + } + if (clusterSize < 1) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be scaled for size, %d", kubernetesCluster.getUuid(), clusterSize)); + } + } + } + + private void validateKubernetesClusterScaleOfferingParameters(final KubernetesCluster kubernetesCluster, final ServiceOffering existingServiceOffering, final ServiceOffering serviceOffering) { + final long originalNodeCount = kubernetesCluster.getTotalNodeCount(); + List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + if (vmList == null || vmList.isEmpty() || vmList.size() < originalNodeCount) { + logAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster ID: %s failed, it is in unstable state as not enough existing VM instances found!", kubernetesCluster.getUuid())); + } else { + for (KubernetesClusterVmMapVO vmMapVO : vmList) { + VMInstanceVO vmInstance = vmInstanceDao.findById(vmMapVO.getVmId()); + if (vmInstance != null && vmInstance.getState().equals(VirtualMachine.State.Running) && + vmInstance.getHypervisorType() != Hypervisor.HypervisorType.XenServer && + vmInstance.getHypervisorType() != Hypervisor.HypervisorType.VMware && + vmInstance.getHypervisorType() != Hypervisor.HypervisorType.Simulator) { + logAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster ID: %s failed, scaling Kubernetes cluster with running VMs on hypervisor %s is not supported!", kubernetesCluster.getUuid(), vmInstance.getHypervisorType())); + } + } + } + if (serviceOffering.getRamSize() < existingServiceOffering.getRamSize() || + serviceOffering.getCpu() * serviceOffering.getSpeed() < existingServiceOffering.getCpu() * existingServiceOffering.getSpeed()) { + logAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster ID: %s failed, service offering for the Kubernetes cluster cannot be scaled down!", kubernetesCluster.getUuid())); + } + } + + private void scaleKubernetesClusterOffering(final long kubernetesClusterId, final ServiceOffering serviceOffering, final Long clusterSize) { + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.ScaleUpRequested); + + final long size = (clusterSize == null ? kubernetesCluster.getTotalNodeCount() : kubernetesCluster.getMasterNodeCount() + clusterSize); + final long cores = serviceOffering.getCpu() * size; + final long memory = serviceOffering.getRamSize() * size; + KubernetesClusterVO updatedKubernetesCluster = updateKubernetesClusterEntry(kubernetesCluster.getId(), size, cores, memory, serviceOffering.getId()); + if (updatedKubernetesCluster == null) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, unable to update Kubernetes cluster!", updatedKubernetesCluster.getUuid()), kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + } + kubernetesCluster = updatedKubernetesCluster; + List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + final long tobeScaledVMCount = Math.min(vmList.size(), size); + for (long i = 0; i < tobeScaledVMCount; i++) { + KubernetesClusterVmMapVO vmMapVO = vmList.get((int) i); + UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId()); + boolean result = false; + try { + result = userVmManager.upgradeVirtualMachine(userVM.getId(), serviceOffering.getId(), new HashMap()); + } catch (ResourceUnavailableException | ManagementServerException | ConcurrentOperationException | VirtualMachineMigrationException e) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, unable to scale cluster VM ID: %s", kubernetesCluster.getUuid(), userVM.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); + } + if (!result) { + logTransitStateAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster ID: %s failed, unable to scale cluster VM ID: %s", kubernetesCluster.getUuid(), userVM.getUuid()),kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + } + } + + private void validateKubernetesClusterScaleSizeParameters(KubernetesClusterVO kubernetesCluster, final long originalClusterSize, final long clusterSize, final KubernetesCluster.State clusterState) { + Network network = networkDao.findById(kubernetesCluster.getNetworkId()); + if (network == null) { + String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, cluster network not found", kubernetesCluster.getUuid()); + if (KubernetesCluster.State.Scaling.equals(kubernetesCluster.getState())) { + logTransitStateAndThrow(Level.WARN, msg, kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } else { + logAndThrow(Level.WARN, msg); + } + } + // Check capacity and transition state + final long newVmRequiredCount = clusterSize - originalClusterSize; + final ServiceOffering clusterServiceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); + if (clusterServiceOffering == null) { + String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, cluster service offering not found", kubernetesCluster.getUuid()); + if (KubernetesCluster.State.Scaling.equals(kubernetesCluster.getState())) { + logTransitStateAndThrow(Level.WARN, msg, kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } else { + logAndThrow(Level.WARN, msg); + } + } + if (newVmRequiredCount > 0) { + final DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); + try { + if (clusterState.equals(KubernetesCluster.State.Running)) { + plan(newVmRequiredCount, zone, clusterServiceOffering); + } else { + plan(kubernetesCluster.getTotalNodeCount() + newVmRequiredCount, zone, clusterServiceOffering); + } + } catch (InsufficientCapacityException e) { + String msg = String.format("Scaling failed for Kubernetes cluster ID: %s in zone ID: %s, insufficient capacity", kubernetesCluster.getUuid(), zone.getUuid()); + if (KubernetesCluster.State.Scaling.equals(kubernetesCluster.getState())) { + logTransitStateAndThrow(Level.WARN, msg, kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } else { + logAndThrow(Level.WARN, msg); + } + } + } + } + + private void scaleKubernetesClusterSize(final long kubernetesClusterId, final long originalClusterSize, final long clusterSize) throws ResourceUnavailableException { + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + final Network network = networkDao.findById(kubernetesCluster.getNetworkId()); + final long newVmRequiredCount = clusterSize - originalClusterSize; + List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + vmList.sort(new Comparator() { + @Override + public int compare(KubernetesClusterVmMapVO kubernetesClusterVmMapVO, KubernetesClusterVmMapVO t1) { + return (int)((kubernetesClusterVmMapVO.getId() - t1.getId())/Math.abs(kubernetesClusterVmMapVO.getId() - t1.getId())); + } + }); + if (CollectionUtils.isEmpty(vmList) || vmList.size() - 1 < originalClusterSize) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, t is in unstable state as not enough existing VM instances found", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + + Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(kubernetesCluster); + String publicIpAddress = publicIpSshPort.first(); + int sshPort = publicIpSshPort.second(); + if (Strings.isNullOrEmpty(publicIpAddress)) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, unable to retrieve associated public IP", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + Account account = accountDao.findById(kubernetesCluster.getAccountId()); + if (newVmRequiredCount < 0) { // downscale + int i = vmList.size() - 1; + List removedVmIds = new ArrayList<>(); + while (i > kubernetesCluster.getMasterNodeCount() && vmList.size() > kubernetesCluster.getTotalNodeCount()) { // Reverse order as first VM will be k8s master + KubernetesClusterVmMapVO vmMapVO = vmList.get(i); + UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId()); + + // Gracefully remove-delete k8s node + if (!removeKubernetesClusterNode(kubernetesCluster, publicIpAddress, sshPort, userVM, 3, 30000)) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, failed to remove Kubernetes node: %s running on VM ID: %s", kubernetesCluster.getUuid(), userVM.getHostName(), userVM.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + + // For removing port-forwarding network rules + removedVmIds.add(userVM.getId()); + + // Expunge VM + UserVm vm = userVmService.destroyVm(userVM.getId(), true); + if (!VirtualMachine.State.Expunging.equals(vm.getState())) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, VM '%s' is now in state '%s'." + , kubernetesCluster.getUuid() + , vm.getInstanceName() + , vm.getState().toString()), + kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + vm = userVmService.expungeVm(userVM.getId()); + if (!VirtualMachine.State.Expunging.equals(vm.getState())) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, VM '%s' is now in state '%s'." + , kubernetesCluster.getUuid() + , vm.getInstanceName() + , vm.getState().toString()), + kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + + // Expunge cluster VMMapVO + kubernetesClusterVmMapDao.expunge(vmMapVO.getId()); + + i--; + } + + // Scale network rules to update firewall rule + try { + scaleKubernetesClusterNetworkRules(kubernetesCluster, network, account, null, removedVmIds); + } catch (ManagementServerException e) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, unable to update network rules", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); + } + } else { // upscale, same node count handled above + List clusterVMIds = new ArrayList<>(); + + // Create new node VMs + for (int i = (int) originalClusterSize + 1; i <= clusterSize; i++) { + UserVm vm = null; + try { + vm = createKubernetesNode(kubernetesCluster, publicIpAddress, i); + addKubernetesClusterVm(kubernetesCluster.getId(), vm .getId()); + startKubernetesVM(vm, kubernetesCluster); + clusterVMIds.add(vm.getId()); + LOGGER.debug(String.format("Provisioned a node VM in to the Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + } catch (ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, unable to provision node VM in the cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); + } + } + + // Scale network rules to update firewall rule and add port-forwarding rules + try { + scaleKubernetesClusterNetworkRules(kubernetesCluster, network, account, clusterVMIds, null); + } catch (ManagementServerException e) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, unable to update network rules", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); + } + + // Attach binaries ISO to new VMs + attachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); + + // Check if new nodes are added in k8s cluster + boolean readyNodesCountValid = validateKubernetesClusterReadyNodesCount(kubernetesCluster, publicIpAddress, sshPort, 30, 30000); + + // Detach binaries ISO from new VMs + detachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); + + // Throw exception if nodes count for k8s cluster timed out + if (!readyNodesCountValid) { // Scaling failed + logTransitStateAndThrow(Level.ERROR, String.format("Scaling unsuccessful for Kubernetes cluster ID: %s as it does not have desired number of nodes in ready state", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + } + } + + private void validateKubernetesClusterUpgradeParameters(UpgradeKubernetesClusterCmd cmd) { + // Validate parameters + final Long kubernetesClusterId = cmd.getId(); + final Long upgradeVersionId = cmd.getKubernetesVersionId(); + if (kubernetesClusterId == null || kubernetesClusterId < 1L) { + throw new InvalidParameterValueException("Invalid Kubernetes cluster ID"); + } + if (upgradeVersionId == null || upgradeVersionId < 1L) { + throw new InvalidParameterValueException("Invalid Kubernetes version ID"); + } + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + if (kubernetesCluster == null || kubernetesCluster.getRemoved() != null) { + throw new InvalidParameterValueException("Invalid Kubernetes cluster ID"); + } + if (!KubernetesCluster.State.Running.equals(kubernetesCluster.getState())) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s is not in running state", kubernetesCluster.getUuid())); + } + KubernetesSupportedVersionVO upgradeVersion = kubernetesSupportedVersionDao.findById(upgradeVersionId); + if (upgradeVersion == null || upgradeVersion.getRemoved() != null) { + throw new InvalidParameterValueException("Invalid Kubernetes version ID"); + } + KubernetesSupportedVersionVO clusterVersion = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); + if (clusterVersion == null || clusterVersion.getRemoved() != null) { + throw new InvalidParameterValueException(String.format("Invalid Kubernetes version associated with cluster ID: %s", + kubernetesCluster.getUuid())); + } + if (KubernetesVersionManagerImpl.compareKubernetesVersion( + upgradeVersion.getSemanticVersion(), clusterVersion.getSemanticVersion()) <= 0) { + throw new InvalidParameterValueException(String.format("Invalid Kubernetes version associated with cluster ID: %s", + kubernetesCluster.getUuid())); + } + // Check upgradeVersion is either patch upgrade or immediate minor upgrade + try { + KubernetesVersionManagerImpl.canUpgradeKubernetesVersion(clusterVersion.getSemanticVersion(), upgradeVersion.getSemanticVersion()); + } catch (IllegalArgumentException e) { + throw new InvalidParameterValueException(e.getMessage()); + } + + TemplateJoinVO iso = templateJoinDao.findById(upgradeVersion.getIsoId()); + if (iso == null) { + throw new InvalidParameterValueException(String.format("Invalid ISO associated with version ID: %s", upgradeVersion.getUuid())); + } + if (!ObjectInDataStoreStateMachine.State.Ready.equals(iso.getState())) { + throw new InvalidParameterValueException(String.format("ISO associated with version ID: %s is not in Ready state", upgradeVersion.getUuid())); + } + } + + private KubernetesClusterVO updateKubernetesClusterEntry(final long kubernetesClusterId, final long clusterSize, + final long cores, final long memory, final Long serviceOfferingId) { + return Transaction.execute(new TransactionCallback() { + @Override + public KubernetesClusterVO doInTransaction(TransactionStatus status) { + KubernetesClusterVO updatedCluster = kubernetesClusterDao.createForUpdate(kubernetesClusterId); + updatedCluster.setNodeCount(clusterSize); + updatedCluster.setCores(cores); + updatedCluster.setMemory(memory); + if (serviceOfferingId != null) { + updatedCluster.setServiceOfferingId(serviceOfferingId); + } + kubernetesClusterDao.persist(updatedCluster); + return updatedCluster; + } + }); + } + + private void upgradeKubernetesClusterNodes(final KubernetesCluster kubernetesCluster, final List vmIds, + final KubernetesSupportedVersion upgradeVersion, final String publicIpAddress, + final int sshPort, final File upgradeScriptFile) { + Pair result = null; + File pkFile = getManagementServerSshPublicKeyFile(); + for (int i = 0; i < vmIds.size(); ++i) { + UserVm vm = userVmDao.findById(vmIds.get(i)); + result = null; + LOGGER.debug(String.format("Upgrading node on VM ID: %s in Kubernetes cluster ID: %s with Kubernetes version(%s) ID: %s", + vm.getUuid(), kubernetesCluster.getUuid(), upgradeVersion.getSemanticVersion(), upgradeVersion.getUuid())); + try { + result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, pkFile, null, + String.format("sudo kubectl drain %s --ignore-daemonsets --delete-local-data", vm.getHostName()), + 10000, 10000, 60000); + } catch (Exception e) { + logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to drain Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()), kubernetesCluster, vmIds, KubernetesCluster.Event.OperationFailed, e); + } + if (!result.first()) { + logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to drain Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()), kubernetesCluster, vmIds, KubernetesCluster.Event.OperationFailed, null); + } + try { + int nodeSshPort = sshPort == 22 ? sshPort : sshPort + i; + String nodeAddress = (i > 0 && sshPort == 22) ? vm.getPrivateIpAddress() : publicIpAddress; + SshHelper.scpTo(nodeAddress, nodeSshPort, CLUSTER_NODE_VM_USER, pkFile, null, + "~/", upgradeScriptFile.getAbsolutePath(), "0755"); + String cmdStr = String.format("sudo ./%s %s %s %s", upgradeScriptFile.getName(), + upgradeVersion.getSemanticVersion(), i == 0 ? "true" : "false", + KubernetesVersionManagerImpl.compareKubernetesVersion(upgradeVersion.getSemanticVersion(), "1.15") < 0 ? "true" : "false"); + result = SshHelper.sshExecute(publicIpAddress, nodeSshPort, CLUSTER_NODE_VM_USER, pkFile, null, + cmdStr, + 10000, 10000, 10 * 60 * 1000); + } catch (Exception e) { + logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to upgrade Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()), kubernetesCluster, vmIds, KubernetesCluster.Event.OperationFailed, e); + } + if (!result.first()) { + logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to upgrade Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()), kubernetesCluster, vmIds, KubernetesCluster.Event.OperationFailed, null); + + } + if (!uncordonKubernetesClusterNode(kubernetesCluster, publicIpAddress, sshPort, vm, 5, 30000)) { + logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to uncordon Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()), kubernetesCluster, vmIds, KubernetesCluster.Event.OperationFailed, null); + } + if (i == 0) { // Wait for master to get in Ready state + if (!isKubernetesClusterNodeReady(kubernetesCluster, publicIpAddress, sshPort, vm.getHostName(), 5, 20000)) { + logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to get master Kubernetes node on VM ID: %s in ready state", kubernetesCluster.getUuid(), vm.getUuid()), kubernetesCluster, vmIds, KubernetesCluster.Event.OperationFailed, null); + } + } + LOGGER.debug(String.format("Successfully upgraded node on VM ID: %s in Kubernetes cluster ID: %s with Kubernetes version(%s) ID: %s", + vm.getUuid(), kubernetesCluster.getUuid(), upgradeVersion.getSemanticVersion(), upgradeVersion.getUuid())); + } + } + + protected boolean stateTransitTo(long kubernetesClusterId, KubernetesCluster.Event e) { + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + try { + return _stateMachine.transitTo(kubernetesCluster, e, null, kubernetesClusterDao); + } catch (NoTransitionException nte) { + LOGGER.warn(String.format("Failed to transition state of the Kubernetes cluster ID: %s in state %s on event %s", kubernetesCluster.getUuid(), kubernetesCluster.getState().toString(), e.toString()), nte); + return false; + } + } + + @Override + public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) + throws InsufficientCapacityException, ManagementServerException { + if (!KubernetesServiceEnabled.value()) { + logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); + } + + validateKubernetesClusterCreatePrameters(cmd); + + final DataCenter zone = dataCenterDao.findById(cmd.getZoneId()); + final long masterNodeCount = cmd.getMasterNodes(); + final long clusterSize = cmd.getClusterSize(); + final long totalNodeCount = masterNodeCount + clusterSize; + final ServiceOffering serviceOffering = serviceOfferingDao.findById(cmd.getServiceOfferingId()); + final Account owner = accountService.getActiveAccountById(cmd.getEntityOwnerId()); + final KubernetesSupportedVersion clusterKubernetesVersion = kubernetesSupportedVersionDao.findById(cmd.getKubernetesVersionId()); + + plan(totalNodeCount, zone, serviceOffering); + + final Network defaultNetwork = getKubernetesClusterNetworkIfMissing(cmd.getName(), zone, owner, (int)masterNodeCount, (int)clusterSize, cmd.getExternalLoadBalancerIpAddress(), cmd.getNetworkId()); + final VMTemplateVO finalTemplate = templateDao.findByTemplateName(KubernetesClusterTemplateName.value());; + final long cores = serviceOffering.getCpu() * (masterNodeCount + clusterSize); + final long memory = serviceOffering.getRamSize() * (masterNodeCount + clusterSize); + + final KubernetesClusterVO cluster = Transaction.execute(new TransactionCallback() { + @Override + public KubernetesClusterVO doInTransaction(TransactionStatus status) { + KubernetesClusterVO newCluster = new KubernetesClusterVO(cmd.getName(), cmd.getDisplayName(), zone.getId(), clusterKubernetesVersion.getId(), + serviceOffering.getId(), finalTemplate.getId(), defaultNetwork.getId(), owner.getDomainId(), + owner.getAccountId(), masterNodeCount, clusterSize, KubernetesCluster.State.Created, cmd.getSSHKeyPairName(), cores, memory, cmd.getNodeRootDiskSize(), ""); + kubernetesClusterDao.persist(newCluster); + return newCluster; + } + }); + + addKubernetesClusterDetails(cluster, defaultNetwork, cmd); + + LOGGER.debug(String.format("Kubernetes cluster name: %s and ID: %s has been created", cluster.getName(), cluster.getUuid())); + return cluster; + } + + + // Start operation can be performed at two diffrent life stagevs of Kubernetes cluster. First when a freshly created cluster + // in which case there are no resources provisisioned for the Kubernetes cluster. So during start all the resources + // are provisioned from scratch. Second kind of start, happens on Stopped Kubernetes cluster, in which all resources + // are provisioned (like volumes, nics, networks etc). It just that VM's are not in running state. So just + // start the VM's (which can possibly implicitly start the network also). + @Override + public boolean startKubernetesCluster(long kubernetesClusterId, boolean onCreate) throws ManagementServerException, + ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException { + if (!KubernetesServiceEnabled.value()) { + logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); + } + if (onCreate) { + // Start for Kubernetes cluster in 'Created' state + return startKubernetesClusterOnCreate(kubernetesClusterId); + } else { + // Start for Kubernetes cluster in 'Stopped' state. Resources are already provisioned, just need to be started + return startStoppedKubernetesCluster(kubernetesClusterId); + } + } + + @Override + public boolean stopKubernetesCluster(long kubernetesClusterId) throws ManagementServerException { + if (!KubernetesServiceEnabled.value()) { + logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); + } + final KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + if (kubernetesCluster == null) { + throw new InvalidParameterValueException("Failed to find Kubernetes cluster with given ID"); + } + + if (kubernetesCluster.getRemoved() != null) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s is already deleted", kubernetesCluster.getUuid())); + } + + if (kubernetesCluster.getState().equals(KubernetesCluster.State.Stopped)) { + LOGGER.debug(String.format("Kubernetes cluster ID: %s is already stopped", kubernetesCluster.getUuid())); return true; } @@ -2217,7 +2611,7 @@ public boolean stopKubernetesCluster(long kubernetesClusterId) throws Management throw new ManagementServerException(String.format("Failed to find all VMs in Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); } stopClusterVM(vmMapVO); - } catch (ServerApiException ex) { + } catch (CloudRuntimeException ex) { LOGGER.warn(String.format("Failed to stop VM in Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), ex); // dont bail out here. proceed further to stop the reset of the VM's } @@ -2226,8 +2620,7 @@ public boolean stopKubernetesCluster(long kubernetesClusterId) throws Management for (final KubernetesClusterVmMapVO vmMapVO : kubernetesClusterVmMapDao.listByClusterId(kubernetesClusterId)) { final UserVmVO vm = userVmDao.findById(vmMapVO.getVmId()); if (vm == null || !vm.getState().equals(VirtualMachine.State.Stopped)) { - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ManagementServerException(String.format("Failed to stop all VMs in Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + logTransitStateAndThrow(Level.ERROR, String.format("Failed to stop all VMs in Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } } @@ -2237,8 +2630,8 @@ public boolean stopKubernetesCluster(long kubernetesClusterId) throws Management @Override public boolean deleteKubernetesCluster(Long kubernetesClusterId) throws ManagementServerException { - if (!Boolean.parseBoolean(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesServiceEnabled.key()))) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Kubernetes Service plugin is disabled"); + if (!KubernetesServiceEnabled.value()) { + logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); } KubernetesClusterVO cluster = kubernetesClusterDao.findById(kubernetesClusterId); if (cluster == null) { @@ -2252,8 +2645,8 @@ public boolean deleteKubernetesCluster(Long kubernetesClusterId) throws Manageme @Override public ListResponse listKubernetesClusters(ListKubernetesClustersCmd cmd) { - if (!Boolean.parseBoolean(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesServiceEnabled.key()))) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Kubernetes Service plugin is disabled"); + if (!KubernetesServiceEnabled.value()) { + logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); } final CallContext ctx = CallContext.current(); final Account caller = ctx.getCallingAccount(); @@ -2302,8 +2695,8 @@ public ListResponse listKubernetesClusters(ListKubern } public KubernetesClusterConfigResponse getKubernetesClusterConfig(GetKubernetesClusterConfigCmd cmd) { - if (!Boolean.parseBoolean(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesServiceEnabled.key()))) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Kubernetes Service plugin is disabled"); + if (!KubernetesServiceEnabled.value()) { + logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); } final Long clusterId = cmd.getId(); KubernetesCluster kubernetesCluster = kubernetesClusterDao.findById(clusterId); @@ -2326,164 +2719,41 @@ public KubernetesClusterConfigResponse getKubernetesClusterConfig(GetKubernetesC @Override public boolean scaleKubernetesCluster(ScaleKubernetesClusterCmd cmd) throws ResourceUnavailableException { - if (!Boolean.parseBoolean(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesServiceEnabled.key()))) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Kubernetes Service plugin is disabled"); - } - final Long kubernetesClusterId = cmd.getId(); - final Long serviceOfferingId = cmd.getServiceOfferingId(); - final Long clusterSize = cmd.getClusterSize(); - if (kubernetesClusterId == null || kubernetesClusterId < 1L) { - throw new InvalidParameterValueException("Invalid Kubernetes cluster ID"); - } - KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); - if (kubernetesCluster == null || kubernetesCluster.getRemoved() != null) { - throw new InvalidParameterValueException("Invalid Kubernetes cluster ID"); - } - final DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); - if (zone == null) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Unable to find zone for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); - } - - Account caller = CallContext.current().getCallingAccount(); - accountManager.checkAccess(caller, SecurityChecker.AccessType.OperateEntry, false, kubernetesCluster); - - if (serviceOfferingId == null && clusterSize == null) { - throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be scaled, either a new service offering or a new cluster size must be passed", kubernetesCluster.getUuid())); - } - - ServiceOffering serviceOffering = null; - if (serviceOfferingId != null) { - serviceOffering = serviceOfferingDao.findById(serviceOfferingId); - if (serviceOffering == null) { - throw new InvalidParameterValueException("Failed to find service offering ID: " + serviceOfferingId); - } else { - if (serviceOffering.isDynamic()) { - throw new InvalidParameterValueException(String.format("Custom service offerings are not supported for Kubernetes clusters. Kubernetes cluster ID: %s, service offering ID: %s", kubernetesCluster.getUuid(), serviceOffering.getUuid())); - } - if (serviceOffering.getCpu() < 2 || serviceOffering.getRamSize() < 2048) { - throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be scaled with service offering ID: %s, Kubernetes cluster template(CoreOS) needs minimum 2 vCPUs and 2 GB RAM", kubernetesCluster.getUuid(), serviceOffering.getUuid())); - } - } + if (!KubernetesServiceEnabled.value()) { + logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); } - if (!(kubernetesCluster.getState().equals(KubernetesCluster.State.Created) || - kubernetesCluster.getState().equals(KubernetesCluster.State.Running) || - kubernetesCluster.getState().equals(KubernetesCluster.State.Stopped))) { - throw new PermissionDeniedException(String.format("Kubernetes cluster ID: %s is in %s state", kubernetesCluster.getUuid(), kubernetesCluster.getState().toString())); - } + validateKubernetesClusterScaleParameters(cmd); - if (clusterSize != null) { - if (kubernetesCluster.getState().equals(KubernetesCluster.State.Stopped)) { // Cannot scale stopped cluster currently for cluster size - throw new PermissionDeniedException(String.format("Kubernetes cluster ID: %s is in %s state", kubernetesCluster.getUuid(), kubernetesCluster.getState().toString())); - } - if (clusterSize < 1) { - throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be scaled for size, %d", kubernetesCluster.getUuid(), clusterSize)); - } - } + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(cmd.getId()); + final ServiceOffering serviceOffering = serviceOfferingDao.findById(cmd.getServiceOfferingId()); + final Long clusterSize = cmd.getClusterSize(); LOGGER.debug(String.format("Scaling Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); final KubernetesCluster.State clusterState = kubernetesCluster.getState(); - final long originalNodeCount = kubernetesCluster.getNodeCount(); + final long originalClusterSize = kubernetesCluster.getNodeCount(); final ServiceOffering existingServiceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); if (existingServiceOffering == null) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, service offering for the Kubernetes cluster not found!", kubernetesCluster.getUuid())); + logAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, service offering for the Kubernetes cluster not found!", kubernetesCluster.getUuid())); } final boolean serviceOfferingScalingNeeded = serviceOffering != null && serviceOffering.getId() != existingServiceOffering.getId(); - final boolean clusterSizeScalingNeeded = clusterSize != null && clusterSize != originalNodeCount; - if (serviceOfferingScalingNeeded) { - List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); - if (vmList == null || vmList.isEmpty() || vmList.size() - 1 < originalNodeCount) { - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, it is in unstable state as not enough existing VM instances found!", kubernetesCluster.getUuid())); - } else { - for (KubernetesClusterVmMapVO vmMapVO : vmList) { - VMInstanceVO vmInstance = vmInstanceDao.findById(vmMapVO.getVmId()); - if (vmInstance != null && vmInstance.getState().equals(VirtualMachine.State.Running) && vmInstance.getHypervisorType() != Hypervisor.HypervisorType.XenServer && vmInstance.getHypervisorType() != Hypervisor.HypervisorType.VMware && vmInstance.getHypervisorType() != Hypervisor.HypervisorType.Simulator) { - LOGGER.info("Scaling the VM dynamically is not supported for VMs running on Hypervisor " + vmInstance.getHypervisorType()); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, scaling Kubernetes cluster with running VMs on hypervisor %s is not supported!", kubernetesCluster.getUuid(), vmInstance.getHypervisorType())); - } - } - } - if (serviceOffering.getRamSize() < existingServiceOffering.getRamSize() || - serviceOffering.getCpu()*serviceOffering.getSpeed() < existingServiceOffering.getCpu()*existingServiceOffering.getSpeed()) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, service offering for the Kubernetes cluster cannot be scaled down!", kubernetesCluster.getUuid())); - } + final boolean clusterSizeScalingNeeded = clusterSize != null && clusterSize != originalClusterSize; - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.ScaleUpRequested); - - final long size = (clusterSize == null ? kubernetesCluster.getTotalNodeCount() : kubernetesCluster.getMasterNodeCount() + clusterSize); - final long cores = serviceOffering.getCpu() * size; - final long memory = serviceOffering.getRamSize() * size; - kubernetesCluster = Transaction.execute(new TransactionCallback() { - @Override - public KubernetesClusterVO doInTransaction(TransactionStatus status) { - KubernetesClusterVO updatedCluster = kubernetesClusterDao.createForUpdate(kubernetesClusterId); - updatedCluster.setNodeCount(size); - updatedCluster.setCores(cores); - updatedCluster.setMemory(memory); - updatedCluster.setServiceOfferingId(serviceOfferingId); - kubernetesClusterDao.persist(updatedCluster); - return updatedCluster; - } - }); - if (kubernetesCluster == null) { - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, unable to update Kubernetes cluster!", kubernetesCluster.getUuid())); - } - kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); - final long tobeScaledVMCount = Math.min(vmList.size(), size+1); - for (long i=0; i()); - } catch (Exception e) { - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, unable to scale cluster VM ID: %s! %s", kubernetesCluster.getUuid(), userVM.getUuid(), e.getMessage()), e); - } - if (!result) { - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, unable to scale cluster VM ID: %s!", kubernetesCluster.getUuid(), userVM.getUuid())); - } - } + if (serviceOfferingScalingNeeded) { + validateKubernetesClusterScaleOfferingParameters(kubernetesCluster, existingServiceOffering, serviceOffering); + scaleKubernetesClusterOffering(kubernetesCluster.getId(), serviceOffering, clusterSize); } if (clusterSizeScalingNeeded) { - Network network = networkDao.findById(kubernetesCluster.getNetworkId()); - if (network == null) { - String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, cluster network not found", kubernetesCluster.getUuid()); - LOGGER.error(msg); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); - } - // Check capacity and transition state - final long newVmRequiredCount = clusterSize - originalNodeCount; + validateKubernetesClusterScaleSizeParameters(kubernetesCluster, originalClusterSize, clusterSize, clusterState); + final long newVmRequiredCount = clusterSize - originalClusterSize; final ServiceOffering clusterServiceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); - if (clusterServiceOffering == null) { - String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, cluster service offering not found", kubernetesCluster.getUuid()); - LOGGER.error(msg); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); - } if (newVmRequiredCount > 0) { if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.ScaleUpRequested); } - try { - if (clusterState.equals(KubernetesCluster.State.Running)) { - plan(newVmRequiredCount, zone, clusterServiceOffering); - } else { - plan(kubernetesCluster.getTotalNodeCount() + newVmRequiredCount, zone, clusterServiceOffering); - } - } catch (InsufficientCapacityException e) { - String msg = String.format("Scaling failed for Kubernetes cluster ID: %s in zone ID: %s, insufficient capacity", kubernetesCluster.getUuid(), zone.getUuid()); - LOGGER.error(msg); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INSUFFICIENT_CAPACITY_ERROR, msg, e); - } } else { if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.ScaleDownRequested); @@ -2491,156 +2761,17 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { } if (!serviceOfferingScalingNeeded) { // Else already updated - // Update KubernetesClusterVO final long cores = clusterServiceOffering.getCpu() * (kubernetesCluster.getMasterNodeCount() + clusterSize); final long memory = clusterServiceOffering.getRamSize() * (kubernetesCluster.getMasterNodeCount() + clusterSize); - kubernetesCluster = Transaction.execute(new TransactionCallback() { - @Override - public KubernetesClusterVO doInTransaction(TransactionStatus status) { - KubernetesClusterVO updatedCluster = kubernetesClusterDao.createForUpdate(kubernetesClusterId); - updatedCluster.setNodeCount(clusterSize); - updatedCluster.setCores(cores); - updatedCluster.setMemory(memory); - kubernetesClusterDao.persist(updatedCluster); - return updatedCluster; - } - }); + kubernetesCluster = updateKubernetesClusterEntry(kubernetesCluster.getId(), clusterSize, cores, memory, null); if (kubernetesCluster == null) { - String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, unable to update cluster", kubernetesCluster.getUuid()); - LOGGER.warn(msg); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); + logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, unable to update cluster", kubernetesCluster.getUuid()), cmd.getId(), KubernetesCluster.Event.OperationFailed); } - kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); } // Perform size scaling - if (clusterState.equals(KubernetesCluster.State.Running)) { - List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); - vmList.sort(new Comparator() { - @Override - public int compare(KubernetesClusterVmMapVO kubernetesClusterVmMapVO, KubernetesClusterVmMapVO t1) { - return (int)((kubernetesClusterVmMapVO.getId() - t1.getId())/Math.abs(kubernetesClusterVmMapVO.getId() - t1.getId())); - } - }); - if (vmList == null || vmList.isEmpty() || vmList.size() - 1 < originalNodeCount) { - String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, t is in unstable state as not enough existing VM instances found", kubernetesCluster.getUuid()); - LOGGER.error(msg); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); - } - - Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(kubernetesCluster); - String publicIpAddress = publicIpSshPort.first(); - int sshPort = publicIpSshPort.second(); - if (Strings.isNullOrEmpty(publicIpAddress)) { - String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, unable to retrieve associated public IP", kubernetesCluster.getUuid()); - LOGGER.error(msg); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); - } - File pkFile = getManagementServerSshPublicKeyFile(); - Account account = accountDao.findById(kubernetesCluster.getAccountId()); - if (newVmRequiredCount < 0) { // downscale - int i = vmList.size() - 1; - List removedVmIds = new ArrayList<>(); - while (i > kubernetesCluster.getMasterNodeCount() && vmList.size() > kubernetesCluster.getTotalNodeCount()) { // Reverse order as first VM will be k8s master - KubernetesClusterVmMapVO vmMapVO = vmList.get(i); - UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId()); - - // Gracefully remove-delete k8s node - if (!removeKubernetesClusterNode(kubernetesCluster, publicIpAddress, sshPort, userVM, 3, 30000)) { - String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, failed to remove Kubernetes node: %s running on VM ID: %s", kubernetesCluster.getUuid(), userVM.getHostName(), userVM.getUuid()); - LOGGER.warn(msg); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); - } - - // For removing port-forwarding network rules - removedVmIds.add(userVM.getId()); - - // Expunge VM - UserVm vm = userVmService.destroyVm(userVM.getId(), true); - if (!VirtualMachine.State.Expunging.equals(vm.getState())) { - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, VM '%s' is now in state '%s'." - , kubernetesCluster.getUuid() - , vm.getInstanceName() - , vm.getState().toString())); - } - vm = userVmService.expungeVm(userVM.getId()); - if (!VirtualMachine.State.Expunging.equals(vm.getState())) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, VM '%s' is now in state '%s'." - , kubernetesCluster.getUuid() - , vm.getInstanceName() - , vm.getState().toString())); - } - - // Expunge cluster VMMapVO - kubernetesClusterVmMapDao.expunge(vmMapVO.getId()); - - i--; - } - - // Scale network rules to update firewall rule - try { - scaleKubernetesClusterNetworkRules(kubernetesCluster, network, account, null, removedVmIds); - } catch (Exception e) { - String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, unable to update network rules", kubernetesCluster.getUuid()); - LOGGER.error(msg, e); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg, e); - } - } else { // upscale, same node count handled above - UserVmVO masterVm = userVmDao.findById(vmList.get(0).getVmId()); - String masterIP = masterVm.getPrivateIpAddress(); - List clusterVMIds = new ArrayList<>(); - - // Create new node VMs - for (int i = (int) originalNodeCount + 1; i <= clusterSize; i++) { - UserVm vm = null; - try { - vm = createKubernetesNode(kubernetesCluster, masterIP, i); - addKubernetesClusterVm(kubernetesCluster.getId(), vm .getId()); - startKubernetesVM(vm, kubernetesCluster); - clusterVMIds.add(vm.getId()); - LOGGER.debug(String.format("Provisioned a node VM in to the Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); - } catch (Exception e) { - String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, unable to provision node VM in the cluster", kubernetesCluster.getUuid()); - LOGGER.error(msg, e); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg, e); - } - } - - // Scale network rules to update firewall rule and add port-forwarding rules - try { - scaleKubernetesClusterNetworkRules(kubernetesCluster, network, account, clusterVMIds, null); - } catch (Exception e) { - String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, unable to update network rules", kubernetesCluster.getUuid()); - LOGGER.error(msg, e); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg, e); - } - - // Attach binaries ISO to new VMs - attachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); - - // Check if new nodes are added in k8s cluster - boolean readyNodesCountValid = validateKubernetesClusterReadyNodesCount(kubernetesCluster, publicIpAddress, sshPort, 30, 30000); - - // Detach binaries ISO from new VMs - detachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); - - // Throw exception if nodes count for k8s cluster timed out - if (!readyNodesCountValid) { // Scaling failed - String msg = String.format("Scaling unsuccessful for Kubernetes cluster ID: %s as it does not have desired number of nodes in ready state", kubernetesCluster.getUuid()); - LOGGER.warn(msg); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); - } - } - } + scaleKubernetesClusterSize(cmd.getId(), originalClusterSize, clusterSize); } stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); return true; @@ -2648,62 +2779,19 @@ public int compare(KubernetesClusterVmMapVO kubernetesClusterVmMapVO, Kubernetes @Override public boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws ManagementServerException { - if (!Boolean.parseBoolean(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesServiceEnabled.key()))) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Kubernetes Service plugin is disabled"); - } - // Validate parameters - final Long kubernetesClusterId = cmd.getId(); - final Long upgradeVersionId = cmd.getKubernetesVersionId(); - if (kubernetesClusterId == null || kubernetesClusterId < 1L) { - throw new InvalidParameterValueException("Invalid Kubernetes cluster ID"); - } - if (upgradeVersionId == null || upgradeVersionId < 1L) { - throw new InvalidParameterValueException("Invalid Kubernetes version ID"); - } - KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); - if (kubernetesCluster == null || kubernetesCluster.getRemoved() != null) { - throw new InvalidParameterValueException("Invalid Kubernetes cluster ID"); - } - if (!KubernetesCluster.State.Running.equals(kubernetesCluster.getState())) { - throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s is not in running state", kubernetesCluster.getUuid())); - } - KubernetesSupportedVersionVO upgradeVersion = kubernetesSupportedVersionDao.findById(upgradeVersionId); - if (upgradeVersion == null || upgradeVersion.getRemoved() != null) { - throw new InvalidParameterValueException("Invalid Kubernetes version ID"); - } - KubernetesSupportedVersionVO clusterVersion = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); - if (clusterVersion == null || clusterVersion.getRemoved() != null) { - throw new InvalidParameterValueException(String.format("Invalid Kubernetes version associated with cluster ID: %s", - kubernetesCluster.getUuid())); - } - if (KubernetesVersionManagerImpl.compareKubernetesVersion( - upgradeVersion.getKubernetesVersion(), clusterVersion.getKubernetesVersion()) <= 0) { - throw new InvalidParameterValueException(String.format("Invalid Kubernetes version associated with cluster ID: %s", - kubernetesCluster.getUuid())); - } - // Check upgradeVersion is either patch upgrade or immediate minor upgrade - try { - KubernetesVersionManagerImpl.canUpgradeKubernetesVersion(clusterVersion.getKubernetesVersion(), upgradeVersion.getKubernetesVersion()); - } catch (IllegalArgumentException e) { - throw new InvalidParameterValueException(e.getMessage()); - } - - TemplateJoinVO iso = templateJoinDao.findById(upgradeVersion.getIsoId()); - if (iso == null) { - throw new InvalidParameterValueException(String.format("Invalid ISO associated with version ID: %s", upgradeVersion.getUuid())); - } - if (!ObjectInDataStoreStateMachine.State.Ready.equals(iso.getState())) { - throw new InvalidParameterValueException(String.format("ISO associated with version ID: %s is not in Ready state", upgradeVersion.getUuid())); + if (!KubernetesServiceEnabled.value()) { + logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); } + validateKubernetesClusterUpgradeParameters(cmd); + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(cmd.getId()); + final KubernetesSupportedVersion upgradeVersion = kubernetesSupportedVersionDao.findById(cmd.getKubernetesVersionId()); // Get public IP Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(kubernetesCluster); String publicIpAddress = publicIpSshPort.first(); int sshPort = publicIpSshPort.second(); if (Strings.isNullOrEmpty(publicIpAddress)) { - String msg = String.format("Upgrade failed for Kubernetes cluster ID: %s, unable to retrieve associated public IP", kubernetesCluster.getUuid()); - LOGGER.warn(msg); - throw new ManagementServerException(msg); + logAndThrow(Level.ERROR, String.format("Upgrade failed for Kubernetes cluster ID: %s, unable to retrieve associated public IP", kubernetesCluster.getUuid())); } kubernetesCluster.setKubernetesVersionId(upgradeVersion.getId()); @@ -2718,8 +2806,6 @@ public boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws } Collections.sort(vmIds); - File pkFile = getManagementServerSshPublicKeyFile(); - File upgradeScriptFile = null; try { String upgradeScriptData = readResourceFile("/script/upgrade-kubernetes.sh"); @@ -2728,9 +2814,7 @@ public boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws upgradeScriptFileWriter.write(upgradeScriptData); upgradeScriptFileWriter.close(); } catch (IOException e) { - String msg = String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to prepare upgrade script", kubernetesCluster.getUuid()); - LOGGER.error(msg, e); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg, e); + logAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to prepare upgrade script", kubernetesCluster.getUuid()), e); } stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.UpgradeRequested); @@ -2738,75 +2822,8 @@ public boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws // Attach ISO attachIsoKubernetesVMs(kubernetesCluster, vmIds); - // Upgrade master - Pair result = null; - for (int i = 0; i < vmIds.size(); ++i) { - UserVm vm = userVmDao.findById(vmIds.get(i)); - result = null; - LOGGER.debug(String.format("Upgrading node on VM ID: %s in Kubernetes cluster ID: %s with Kubernetes version(%s) ID: %s", - vm.getUuid(), kubernetesCluster.getUuid(), upgradeVersion.getKubernetesVersion(), upgradeVersion.getUuid())); - try { - result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, pkFile, null, - String.format("sudo kubectl drain %s --ignore-daemonsets --delete-local-data", vm.getHostName()), - 10000, 10000, 60000); - } catch (Exception e) { - String msg = String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to drain Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()); - LOGGER.error(msg, e); - detachIsoKubernetesVMs(kubernetesCluster, vmIds); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg, e); - } - if (!result.first()) { - String msg = String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to drain Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()); - LOGGER.error(String.format("%s. Output:\n%s", msg, Strings.nullToEmpty(result.second()))); - detachIsoKubernetesVMs(kubernetesCluster, vmIds); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); - } - try { - int nodeSshPort = sshPort == 22 ? sshPort : sshPort + i; - String nodeAddress = (i > 0 && sshPort == 22) ? vm.getPrivateIpAddress() : publicIpAddress; - SshHelper.scpTo(nodeAddress, nodeSshPort, CLUSTER_NODE_VM_USER, pkFile, null, - "~/", upgradeScriptFile.getAbsolutePath(), "0755"); - String cmdStr = String.format("sudo ./%s %s %s %s", upgradeScriptFile.getName(), - upgradeVersion.getKubernetesVersion(), i == 0 ? "true" : "false", - KubernetesVersionManagerImpl.compareKubernetesVersion(upgradeVersion.getKubernetesVersion(), "1.15") < 0 ? "true" : "false"); - result = SshHelper.sshExecute(publicIpAddress, nodeSshPort, CLUSTER_NODE_VM_USER, pkFile, null, - cmdStr, - 10000, 10000, 10 * 60 * 1000); - } catch (Exception e) { - String msg = String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to upgrade Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()); - LOGGER.error(msg, e); - detachIsoKubernetesVMs(kubernetesCluster, vmIds); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg, e); - } - if (!result.first()) { - String msg = String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to upgrade Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()); - LOGGER.error(String.format("%s. Output:\n%s", msg, Strings.nullToEmpty(result.second()))); - detachIsoKubernetesVMs(kubernetesCluster, vmIds); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); - } - if (!uncordonKubernetesClusterNode(kubernetesCluster, publicIpAddress, sshPort, vm, 5, 30000)) { - String msg = String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to uncordon Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()); - LOGGER.error(msg); - detachIsoKubernetesVMs(kubernetesCluster, vmIds); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); - } - if (i == 0) { // Wait for master to get in Ready state - if (!isKubernetesClusterNodeReady(kubernetesCluster, publicIpAddress, sshPort, vm.getHostName(), 5, 20000)) { - String msg = String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to get master Kubernetes node on VM ID: %s in ready state", kubernetesCluster.getUuid(), vm.getUuid()); - LOGGER.error(msg); - detachIsoKubernetesVMs(kubernetesCluster, vmIds); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg); - } - } - LOGGER.debug(String.format("Successfully upgraded node on VM ID: %s in Kubernetes cluster ID: %s with Kubernetes version(%s) ID: %s", - vm.getUuid(), kubernetesCluster.getUuid(), upgradeVersion.getKubernetesVersion(), upgradeVersion.getUuid())); - } + // Upgrade nodes + upgradeKubernetesClusterNodes(kubernetesCluster, vmIds, upgradeVersion, publicIpAddress, sshPort, upgradeScriptFile); // Detach ISO detachIsoKubernetesVMs(kubernetesCluster, vmIds); @@ -2823,7 +2840,7 @@ public boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws @Override public List> getCommands() { List> cmdList = new ArrayList>(); - if (!Boolean.parseBoolean(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesServiceEnabled.key()))) { + if (!KubernetesServiceEnabled.value()) { return cmdList; } cmdList.add(CreateKubernetesClusterCmd.class); @@ -3058,4 +3075,18 @@ public boolean configure(String name, Map params) throws Configu return true; } + + @Override + public String getConfigComponentName() { + return KubernetesClusterService.class.getSimpleName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[] { + KubernetesServiceEnabled, + KubernetesClusterTemplateName, + KubernetesClusterNetworkOffering + }; + } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterService.java index dc313f887c61..1d2beaf3a859 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterService.java @@ -24,23 +24,40 @@ import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse; import org.apache.cloudstack.api.response.KubernetesClusterResponse; import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.ManagementServerException; -import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.utils.component.PluggableService; -public interface KubernetesClusterService extends PluggableService { +public interface KubernetesClusterService extends PluggableService, Configurable { static final String MIN_KUBERNETES_VERSION_HA_SUPPORT = "1.16"; + static final ConfigKey KubernetesServiceEnabled = new ConfigKey("Advanced", Boolean.class, + "cloud.kubernetes.service.enabled", + "false", + "Indicates whether Kubernetes Service plugin is enabled or not. Management server restart needed on change", + false); + static final ConfigKey KubernetesClusterTemplateName = new ConfigKey("Advanced", String.class, + "cloud.kubernetes.cluster.template.name", + "Kubernetes-Service-Template", + "Name of the template to be used for creating Kubernetes cluster nodes", + true); + static final ConfigKey KubernetesClusterNetworkOffering = new ConfigKey("Advanced", String.class, + "cloud.kubernetes.cluster.network.offering", + "DefaultNetworkOfferingforKubernetesService", + "Name of the network offering that will be used to create isolated network in which Kubernetes cluster VMs will be launched", + false); + KubernetesCluster findById(final Long id); KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) throws InsufficientCapacityException, - ResourceAllocationException, ManagementServerException; + ManagementServerException; boolean startKubernetesCluster(long kubernetesClusterId, boolean onCreate) throws ManagementServerException, - ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException; + ResourceUnavailableException, InsufficientCapacityException; boolean stopKubernetesCluster(long kubernetesClusterId) throws ManagementServerException; @@ -53,7 +70,7 @@ boolean startKubernetesCluster(long kubernetesClusterId, boolean onCreate) throw KubernetesClusterResponse createKubernetesClusterResponse(long kubernetesClusterId); boolean scaleKubernetesCluster(ScaleKubernetesClusterCmd cmd) throws ManagementServerException, - ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException; + ResourceUnavailableException, InsufficientCapacityException; boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws ManagementServerException; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVmMap.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVmMap.java index d2d56f2c902a..f3bddecae75e 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVmMap.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVmMap.java @@ -18,7 +18,6 @@ /** * VirtualMachine describes the properties held by a virtual machine - * */ public interface KubernetesClusterVmMap { long getId(); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesServiceConfig.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesServiceConfig.java deleted file mode 100644 index 86a717f7886b..000000000000 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesServiceConfig.java +++ /dev/null @@ -1,74 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.kubernetescluster; - -import com.cloud.server.ManagementServer; - -public enum KubernetesServiceConfig { - - KubernetesServiceEnabled("Advanced", ManagementServer.class, Boolean.class, "cloud.kubernetes.service.enabled", "false", "Indicates whether Kubernetes Service plugin is enabled or not. Management server restart needed on change", null, null), - KubernetesClusterTemplateName("Advanced", ManagementServer.class, String.class, "cloud.kubernetes.cluster.template.name", "Kubernetes-Service-Template", "Name of the template to be used for creating Kubernetes cluster nodes", null, null), - KubernetesClusterNetworkOffering("Advanced", ManagementServer.class, String.class, "cloud.kubernetes.cluster.network.offering", "DefaultNetworkOfferingforKubernetesService", "Name of the network offering that will be used to create isolated network in which Kubernetes cluster VMs will be launched.", null, null); - - private final String _category; - private final Class _componentClass; - private final Class _type; - private final String _name; - private final String _defaultValue; - private final String _description; - private final String _range; - private final String _scope; - - private KubernetesServiceConfig(String category, Class componentClass, Class type, String name, String defaultValue, String description, String range, String scope) { - _category = category; - _componentClass = componentClass; - _type = type; - _name = name; - _defaultValue = defaultValue; - _description = description; - _range = range; - _scope = scope; - } - - public String getCategory() { - return _category; - } - - public String key() { - return _name; - } - - public String getDescription() { - return _description; - } - - public String getDefaultValue() { - return _defaultValue; - } - - public Class getType() { - return _type; - } - - public Class getComponentClass() { - return _componentClass; - } - - public String getScope() { - return _scope; - } -} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesSupportedVersion.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesSupportedVersion.java index 3706f43b6172..d09d6530add1 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesSupportedVersion.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesSupportedVersion.java @@ -27,7 +27,7 @@ public interface KubernetesSupportedVersion extends InternalIdentity, Identity { long getId(); String getName(); - String getKubernetesVersion(); + String getSemanticVersion(); long getIsoId(); Long getZoneId(); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesSupportedVersionVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesSupportedVersionVO.java index d857d9aab097..30c3c9b02751 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesSupportedVersionVO.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesSupportedVersionVO.java @@ -43,8 +43,8 @@ public class KubernetesSupportedVersionVO implements KubernetesSupportedVersion @Column(name = "name") private String name; - @Column(name = "kubernetes_version") - private String kubernetesVersion; + @Column(name = "semantic_version") + private String semanticVersion; @Column(name = "iso_id") private long isoId; @@ -62,10 +62,10 @@ public KubernetesSupportedVersionVO() { this.uuid = UUID.randomUUID().toString(); } - public KubernetesSupportedVersionVO(String name, String kubernetesVersion, long isoId, Long zoneId) { + public KubernetesSupportedVersionVO(String name, String semanticVersion, long isoId, Long zoneId) { this.uuid = UUID.randomUUID().toString(); this.name = name; - this.kubernetesVersion = kubernetesVersion; + this.semanticVersion = semanticVersion; this.isoId = isoId; this.zoneId = zoneId; } @@ -90,12 +90,12 @@ public void setName(String name) { } @Override - public String getKubernetesVersion() { - return kubernetesVersion; + public String getSemanticVersion() { + return semanticVersion; } - public void setKubernetesVersion(String kubernetesVersion) { - this.kubernetesVersion = kubernetesVersion; + public void setSemanticVersion(String semanticVersion) { + this.semanticVersion = semanticVersion; } @Override diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java index d4f33ade7df9..1661845cc4c3 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java @@ -24,8 +24,6 @@ import javax.inject.Inject; import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.ApiErrorCode; -import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.admin.kubernetesversion.AddKubernetesSupportedVersionCmd; import org.apache.cloudstack.api.command.admin.kubernetesversion.DeleteKubernetesSupportedVersionCmd; import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; @@ -43,9 +41,9 @@ import com.cloud.dc.dao.DataCenterDao; import com.cloud.event.ActionEvent; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ResourceAllocationException; import com.cloud.kubernetescluster.KubernetesClusterService; import com.cloud.kubernetescluster.KubernetesClusterVO; -import com.cloud.kubernetescluster.KubernetesServiceConfig; import com.cloud.kubernetescluster.dao.KubernetesClusterDao; import com.cloud.kubernetesversion.dao.KubernetesSupportedVersionDao; import com.cloud.storage.Storage; @@ -57,6 +55,7 @@ import com.cloud.template.VirtualMachineTemplate; import com.cloud.utils.component.ComponentContext; import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.exception.CloudRuntimeException; import com.google.common.base.Strings; public class KubernetesVersionManagerImpl extends ManagerBase implements KubernetesVersionService { @@ -84,13 +83,13 @@ private KubernetesSupportedVersionResponse createKubernetesSupportedVersionRespo response.setObjectName("kubernetessupportedversion"); response.setId(kubernetesSupportedVersion.getUuid()); response.setName(kubernetesSupportedVersion.getName()); - response.setKubernetesVersion(kubernetesSupportedVersion.getKubernetesVersion()); + response.setKubernetesVersion(kubernetesSupportedVersion.getSemanticVersion()); DataCenterVO zone = ApiDBUtils.findZoneById(kubernetesSupportedVersion.getZoneId()); if (zone != null) { response.setZoneId(zone.getUuid()); response.setZoneName(zone.getName()); } - if (compareKubernetesVersion(kubernetesSupportedVersion.getKubernetesVersion(), + if (compareKubernetesVersion(kubernetesSupportedVersion.getSemanticVersion(), KubernetesClusterService.MIN_KUBERNETES_VERSION_HA_SUPPORT)>=0) { response.setSupportsHA(true); } else { @@ -103,14 +102,35 @@ private KubernetesSupportedVersionResponse createKubernetesSupportedVersionRespo return response; } + private ListResponse createKubernetesSupportedVersionListResponse(List versions) { + List responseList = new ArrayList<>(); + for (KubernetesSupportedVersionVO version : versions) { + responseList.add(createKubernetesSupportedVersionResponse(version)); + } + ListResponse response = new ListResponse<>(); + response.setResponses(responseList); + return response; + } + + private static boolean isSemanticVersion(final String version) { + if(!version.matches("[0-9]+(\\.[0-9]+)*")) { + return false; + } + String[] parts = version.split("\\."); + if (parts.length < 3) { + return false; + } + return true; + } + public static int compareKubernetesVersion(String v1, String v2) throws IllegalArgumentException { if (Strings.isNullOrEmpty(v1) || Strings.isNullOrEmpty(v2)) { throw new IllegalArgumentException(String.format("Invalid version comparision with versions %s, %s", v1, v2)); } - if(!v1.matches("[0-9]+(\\.[0-9]+)*")) { + if(isSemanticVersion(v1)) { throw new IllegalArgumentException(String.format("Invalid version format, %s", v1)); } - if(!v2.matches("[0-9]+(\\.[0-9]+)*")) { + if(isSemanticVersion(v2)) { throw new IllegalArgumentException(String.format("Invalid version format, %s", v2)); } String[] thisParts = v1.split("\\."); @@ -133,48 +153,119 @@ public static boolean canUpgradeKubernetesVersion(String currentVersion, String if (Strings.isNullOrEmpty(currentVersion) || Strings.isNullOrEmpty(upgradeVersion)) { throw new IllegalArgumentException(String.format("Invalid version comparision with versions %s, %s", currentVersion, upgradeVersion)); } - if(!currentVersion.matches("[0-9]+(\\.[0-9]+)*")) { + if(isSemanticVersion(currentVersion)) { throw new IllegalArgumentException(String.format("Invalid version format, %s", currentVersion)); } - if(!upgradeVersion.matches("[0-9]+(\\.[0-9]+)*")) { + if(isSemanticVersion(upgradeVersion)) { throw new IllegalArgumentException(String.format("Invalid version format, %s", upgradeVersion)); } String[] thisParts = currentVersion.split("\\."); - if (thisParts.length < 3) { - throw new IllegalArgumentException(String.format("Invalid version format, %s", currentVersion)); - } String[] thatParts = upgradeVersion.split("\\."); - if (thatParts.length < 3) { - throw new IllegalArgumentException(String.format("Invalid version format, %s", upgradeVersion)); - } int majorVerDiff = Integer.parseInt(thatParts[0]) - Integer.parseInt(thisParts[0]); int minorVerDiff = Integer.parseInt(thatParts[1]) - Integer.parseInt(thisParts[1]); + // You only can upgrade from one MINOR version to the next MINOR version, or between PATCH versions of the same MINOR. + // That is, you cannot skip MINOR versions when you upgrade. + // For example, you can upgrade from 1.y to 1.y+1, but not from 1.y to 1.y+2 if (majorVerDiff != 0 || minorVerDiff != 1) { throw new IllegalArgumentException(String.format("Kubernetes clusters can be upgraded between next minor or patch version releases, current version: %s, upgrade version: %s", currentVersion, upgradeVersion)); } return true; } + private List filterKubernetesSupportedVersions(List versions, final String minimumSemanticVersion) { + if (!Strings.isNullOrEmpty(minimumSemanticVersion)) { + for (int i = versions.size() - 1; i >= 0; --i) { + KubernetesSupportedVersionVO version = versions.get(i); + try { + if (compareKubernetesVersion(minimumSemanticVersion, version.getSemanticVersion()) > 0) { + versions.remove(i); + } + } catch (Exception e) { + LOGGER.warn(String.format("Unable to compare Kubernetes version for supported version ID: %s with %s", version.getUuid(), minimumSemanticVersion)); + versions.remove(i); + } + } + } + return versions; + } + + private VirtualMachineTemplate registerKubernetesVersionIso(final String versionName, final String isoUrl, final String isoChecksum)throws IllegalAccessException, NoSuchFieldException, + IllegalArgumentException, ResourceAllocationException { + String isoName = String.format("%s-Kubernetes-Binaries-ISO", versionName); + RegisterIsoCmd registerIsoCmd = new RegisterIsoCmd(); + registerIsoCmd = ComponentContext.inject(registerIsoCmd); + Field f = registerIsoCmd.getClass().getDeclaredField("isoName"); + f.setAccessible(true); + f.set(registerIsoCmd, isoName); + f = registerIsoCmd.getClass().getDeclaredField("displayText"); + f.setAccessible(true); + f.set(registerIsoCmd, isoName); + f = registerIsoCmd.getClass().getDeclaredField("bootable"); + f.setAccessible(true); + f.set(registerIsoCmd, false); + f = registerIsoCmd.getClass().getDeclaredField("publicIso"); + f.setAccessible(true); + f.set(registerIsoCmd, true); + f = registerIsoCmd.getClass().getDeclaredField("url"); + f.setAccessible(true); + f.set(registerIsoCmd, isoUrl); + if (Strings.isNullOrEmpty(isoChecksum)) { + f = registerIsoCmd.getClass().getDeclaredField("checksum"); + f.setAccessible(true); + f.set(registerIsoCmd, isoChecksum); + } + return templateService.registerIso(registerIsoCmd); + } + + private void validateExistingTemplateForKubernetesVersionIso(VirtualMachineTemplate template, Long zoneId) { + if (!template.getFormat().equals(Storage.ImageFormat.ISO)) { + throw new InvalidParameterValueException(String.format("%s is not an ISO", template.getUuid())); + } + if (!template.isPublicTemplate()) { + throw new InvalidParameterValueException(String.format("ISO ID: %s is not public", template.getUuid())); + } + if (!template.isCrossZones() && zoneId == null) { + throw new InvalidParameterValueException(String.format("ISO ID: %s is not available across zones", template.getUuid())); + } + if (!template.isCrossZones() && zoneId != null) { + List templatesZoneVOs = templateZoneDao.listByZoneTemplate(zoneId, template.getId()); + if (templatesZoneVOs.isEmpty()) { + DataCenterVO zone = dataCenterDao.findById(zoneId); + throw new InvalidParameterValueException(String.format("ISO ID: %s is not available for zone ID: %s", template.getUuid(), zone.getUuid())); + } + } + } + + private void deleteKubernetesVersionIso(long templateId) throws IllegalAccessException, NoSuchFieldException, + IllegalArgumentException { + DeleteIsoCmd deleteIsoCmd = new DeleteIsoCmd(); + deleteIsoCmd = ComponentContext.inject(deleteIsoCmd); + Field f = deleteIsoCmd.getClass().getDeclaredField("id"); + f.setAccessible(true); + f.set(deleteIsoCmd, templateId); + templateService.deleteIso(deleteIsoCmd); + } + @Override public ListResponse listKubernetesSupportedVersions(final ListKubernetesSupportedVersionsCmd cmd) { - if (!Boolean.parseBoolean(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesServiceEnabled.key()))) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Kubernetes Service plugin is disabled"); + if (!KubernetesClusterService.KubernetesServiceEnabled.value()) { + throw new CloudRuntimeException("Kubernetes Service plugin is disabled"); } final Long versionId = cmd.getId(); final Long zoneId = cmd.getZoneId(); - String minimumKubernetesVersion = cmd.getMinimumKubernetesVersion(); + String minimumSemanticVersion = cmd.getMinimumSemanticVersion(); final Long minimumKubernetesVersionId = cmd.getMinimumKubernetesVersionId(); - if (!Strings.isNullOrEmpty(minimumKubernetesVersion) && minimumKubernetesVersionId != null) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Both parameters %s and %s can not be passed together", ApiConstants.MIN_KUBERNETES_VERSION, ApiConstants.MIN_KUBERNETES_VERSION_ID)); + if (!Strings.isNullOrEmpty(minimumSemanticVersion) && minimumKubernetesVersionId != null) { + throw new CloudRuntimeException(String.format("Both parameters %s and %s can not be passed together", ApiConstants.MIN_SEMANTIC_VERSION, ApiConstants.MIN_KUBERNETES_VERSION_ID)); } if (minimumKubernetesVersionId != null) { KubernetesSupportedVersionVO minVersion = kubernetesSupportedVersionDao.findById(minimumKubernetesVersionId); if (minVersion == null) { throw new InvalidParameterValueException(String.format("Invalid %s passed", ApiConstants.MIN_KUBERNETES_VERSION_ID)); } - minimumKubernetesVersion = minVersion.getKubernetesVersion(); + minimumSemanticVersion = minVersion.getSemanticVersion(); } - List responseList = new ArrayList<>(); + List versions = new ArrayList<>(); if (versionId != null) { KubernetesSupportedVersionVO version = kubernetesSupportedVersionDao.findById(versionId); @@ -189,32 +280,16 @@ public ListResponse listKubernetesSupportedV } } // Filter versions for minimum Kubernetes version - if (!Strings.isNullOrEmpty(minimumKubernetesVersion)) { - for (int i = versions.size() - 1; i >= 0; --i) { - KubernetesSupportedVersionVO version = versions.get(i); - try { - if (compareKubernetesVersion(minimumKubernetesVersion, version.getKubernetesVersion()) > 0) { - versions.remove(i); - } - } catch (Exception e) { - LOGGER.warn(String.format("Unable to compare Kubernetes version for supported version ID: %s with %s", version.getUuid(), minimumKubernetesVersion)); - versions.remove(i); - } - } - } - for (KubernetesSupportedVersionVO version : versions) { - responseList.add(createKubernetesSupportedVersionResponse(version)); - } - ListResponse response = new ListResponse<>(); - response.setResponses(responseList); - return response; + versions = filterKubernetesSupportedVersions(versions, minimumSemanticVersion); + + return createKubernetesSupportedVersionListResponse(versions); } @Override @ActionEvent(eventType = KubernetesVersionEventTypes.EVENT_KUBERNETES_VERSION_ADD, eventDescription = "Adding Kubernetes supported version") public KubernetesSupportedVersionResponse addKubernetesSupportedVersion(final AddKubernetesSupportedVersionCmd cmd) { - if (!Boolean.parseBoolean(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesServiceEnabled.key()))) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Kubernetes Service plugin is disabled"); + if (!KubernetesClusterService.KubernetesServiceEnabled.value()) { + throw new CloudRuntimeException("Kubernetes Service plugin is disabled"); } final String name = cmd.getName(); final String kubernetesVersion = cmd.getKubernetesVersion(); @@ -242,63 +317,27 @@ public KubernetesSupportedVersionResponse addKubernetesSupportedVersion(final Ad if (template == null) { // register new ISO VirtualMachineTemplate vmTemplate = null; try { - String isoName = String.format("%s-Kubernetes-Binaries-ISO", name); - RegisterIsoCmd registerIsoCmd = new RegisterIsoCmd(); - registerIsoCmd = ComponentContext.inject(registerIsoCmd); - Field f = registerIsoCmd.getClass().getDeclaredField("isoName"); - f.setAccessible(true); - f.set(registerIsoCmd, isoName); - f = registerIsoCmd.getClass().getDeclaredField("displayText"); - f.setAccessible(true); - f.set(registerIsoCmd, isoName); - f = registerIsoCmd.getClass().getDeclaredField("bootable"); - f.setAccessible(true); - f.set(registerIsoCmd, false); - f = registerIsoCmd.getClass().getDeclaredField("publicIso"); - f.setAccessible(true); - f.set(registerIsoCmd, true); - f = registerIsoCmd.getClass().getDeclaredField("url"); - f.setAccessible(true); - f.set(registerIsoCmd, isoUrl); - if (Strings.isNullOrEmpty(isoChecksum)) { - f = registerIsoCmd.getClass().getDeclaredField("checksum"); - f.setAccessible(true); - f.set(registerIsoCmd, isoChecksum); - } - vmTemplate = templateService.registerIso(registerIsoCmd); - } catch (Exception ex) { + vmTemplate = registerKubernetesVersionIso(name, isoUrl, isoChecksum); + } catch (IllegalAccessException | NoSuchFieldException | IllegalArgumentException | ResourceAllocationException ex) { LOGGER.error(String.format("Unable to register binaries ISO for supported kubernetes version, %s", name), ex); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Unable to register binaries ISO for supported kubernetes version, %s", name)); + throw new CloudRuntimeException(String.format("Unable to register binaries ISO for supported kubernetes version, %s", name)); } template = templateDao.findById(vmTemplate.getId()); } else { - if (!template.getFormat().equals(Storage.ImageFormat.ISO)) { - throw new InvalidParameterValueException(String.format("%s is not an ISO", template.getUuid())); - } - if (!template.isPublicTemplate()) { - throw new InvalidParameterValueException(String.format("ISO ID: %s is not public", template.getUuid())); - } - if (!template.isCrossZones() && zoneId == null) { - throw new InvalidParameterValueException(String.format("ISO ID: %s is not available across zones", template.getUuid())); - } - if (!template.isCrossZones() && zoneId != null) { - List templatesZoneVOs = templateZoneDao.listByZoneTemplate(zoneId, template.getId()); - if (templatesZoneVOs.isEmpty()) { - DataCenterVO zone = dataCenterDao.findById(zoneId); - throw new InvalidParameterValueException(String.format("ISO ID: %s is not available for zone ID: %s", template.getUuid(), zone.getUuid())); - } - } + validateExistingTemplateForKubernetesVersionIso(template, zoneId); } + KubernetesSupportedVersionVO supportedVersionVO = new KubernetesSupportedVersionVO(name, kubernetesVersion, template.getId(), zoneId); supportedVersionVO = kubernetesSupportedVersionDao.persist(supportedVersionVO); + return createKubernetesSupportedVersionResponse(supportedVersionVO); } @Override @ActionEvent(eventType = KubernetesVersionEventTypes.EVENT_KUBERNETES_VERSION_DELETE, eventDescription = "Deleting Kubernetes supported version", async = true) public boolean deleteKubernetesSupportedVersion(final DeleteKubernetesSupportedVersionCmd cmd) { - if (!Boolean.parseBoolean(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesServiceEnabled.key()))) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Kubernetes Service plugin is disabled"); + if (!KubernetesClusterService.KubernetesServiceEnabled.value()) { + throw new CloudRuntimeException("Kubernetes Service plugin is disabled"); } final Long versionId = cmd.getId(); final boolean isDeleteIso = cmd.isDeleteIso(); @@ -308,7 +347,7 @@ public boolean deleteKubernetesSupportedVersion(final DeleteKubernetesSupportedV } List clusters = kubernetesClusterDao.listAllByKubernetesVersion(versionId); if (clusters.size() > 0) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Unable to delete Kubernetes version ID: %s. Existing clusters currently using the version.", version.getUuid())); + throw new CloudRuntimeException(String.format("Unable to delete Kubernetes version ID: %s. Existing clusters currently using the version.", version.getUuid())); } VMTemplateVO template = templateDao.findById(version.getIsoId()); @@ -317,15 +356,10 @@ public boolean deleteKubernetesSupportedVersion(final DeleteKubernetesSupportedV } if (isDeleteIso && template != null) { // Delete ISO try { - DeleteIsoCmd deleteIsoCmd = new DeleteIsoCmd(); - deleteIsoCmd = ComponentContext.inject(deleteIsoCmd); - Field f = deleteIsoCmd.getClass().getDeclaredField("id"); - f.setAccessible(true); - f.set(deleteIsoCmd, template.getId()); - templateService.deleteIso(deleteIsoCmd); + deleteKubernetesVersionIso(template.getId()); } catch (Exception ex) { LOGGER.error(String.format("Unable to delete binaries ISO ID: %s associated with supported kubernetes version ID: %s", template.getUuid(), version.getUuid()), ex); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Unable to delete binaries ISO ID: %s associated with supported kubernetes version ID: %s", template.getUuid(), version.getUuid())); + throw new CloudRuntimeException(String.format("Unable to delete binaries ISO ID: %s associated with supported kubernetes version ID: %s", template.getUuid(), version.getUuid())); } } return kubernetesSupportedVersionDao.remove(version.getId()); @@ -334,7 +368,7 @@ public boolean deleteKubernetesSupportedVersion(final DeleteKubernetesSupportedV @Override public List> getCommands() { List> cmdList = new ArrayList>(); - if (!Boolean.parseBoolean(globalConfigDao.getValue(KubernetesServiceConfig.KubernetesServiceEnabled.key()))) { + if (!KubernetesClusterService.KubernetesServiceEnabled.value()) { return cmdList; } cmdList.add(AddKubernetesSupportedVersionCmd.class); diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetesversion/ListKubernetesSupportedVersionsCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetesversion/ListKubernetesSupportedVersionsCmd.java index d04018403c5d..c1550fd2d4ac 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetesversion/ListKubernetesSupportedVersionsCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetesversion/ListKubernetesSupportedVersionsCmd.java @@ -64,13 +64,13 @@ public class ListKubernetesSupportedVersionsCmd extends BaseListCmd { description = "the ID of the zone in which Kubernetes supported version will be available") private Long zoneId; - @Parameter(name = ApiConstants.MIN_KUBERNETES_VERSION, type = CommandType.STRING, - description = "the minimum Kubernetes version") - private String minimumKubernetesVersion; + @Parameter(name = ApiConstants.MIN_SEMANTIC_VERSION, type = CommandType.STRING, + description = "the minimum semantic version for the Kubernetes supported version to be listed") + private String minimumSemanticVersion; @Parameter(name = ApiConstants.MIN_KUBERNETES_VERSION_ID, type = CommandType.UUID, entityType = KubernetesSupportedVersionResponse.class, - description = "the ID of the minimum Kubernetes version") + description = "the ID of the minimum Kubernetes supported version") private Long minimumKubernetesVersionId; ///////////////////////////////////////////////////// @@ -84,12 +84,12 @@ public Long getZoneId() { return zoneId; } - public String getMinimumKubernetesVersion() { - if(!Strings.isNullOrEmpty(minimumKubernetesVersion) && - !minimumKubernetesVersion.matches("[0-9]+(\\.[0-9]+)*")) { + public String getMinimumSemanticVersion() { + if(!Strings.isNullOrEmpty(minimumSemanticVersion) && + !minimumSemanticVersion.matches("[0-9]+(\\.[0-9]+)*")) { throw new IllegalArgumentException("Invalid version format"); } - return minimumKubernetesVersion; + return minimumSemanticVersion; } public Long getMinimumKubernetesVersionId() { diff --git a/server/src/main/java/com/cloud/api/query/dao/NetworkOfferingJoinDao.java b/server/src/main/java/com/cloud/api/query/dao/NetworkOfferingJoinDao.java index 362cabb304f0..37eef84ebd0e 100644 --- a/server/src/main/java/com/cloud/api/query/dao/NetworkOfferingJoinDao.java +++ b/server/src/main/java/com/cloud/api/query/dao/NetworkOfferingJoinDao.java @@ -27,9 +27,9 @@ public interface NetworkOfferingJoinDao extends GenericDao { - List findByDomainId(long domainId); + List findByDomainId(long domainId, Boolean includeAllDomainOffering); - List findByZoneId(long zoneId); + List findByZoneId(long zoneId, Boolean includeAllZoneOffering); NetworkOfferingResponse newNetworkOfferingResponse(NetworkOffering nof); diff --git a/server/src/main/java/com/cloud/api/query/dao/NetworkOfferingJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/NetworkOfferingJoinDaoImpl.java index b53aef85d465..29ec66692dac 100644 --- a/server/src/main/java/com/cloud/api/query/dao/NetworkOfferingJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/NetworkOfferingJoinDaoImpl.java @@ -43,9 +43,12 @@ protected NetworkOfferingJoinDaoImpl() { } @Override - public List findByDomainId(long domainId) { + public List findByDomainId(long domainId, Boolean includeAllDomainOffering) { SearchBuilder sb = createSearchBuilder(); sb.and("domainId", sb.entity().getDomainId(), SearchCriteria.Op.FIND_IN_SET); + if (includeAllDomainOffering) { + sb.or("zId", sb.entity().getZoneId(), SearchCriteria.Op.NULL); + } sb.done(); SearchCriteria sc = sb.create(); @@ -54,9 +57,12 @@ public List findByDomainId(long domainId) { } @Override - public List findByZoneId(long zoneId) { + public List findByZoneId(long zoneId, Boolean includeAllZoneOffering) { SearchBuilder sb = createSearchBuilder(); sb.and("zoneId", sb.entity().getZoneId(), SearchCriteria.Op.FIND_IN_SET); + if (includeAllZoneOffering) { + sb.or("zId", sb.entity().getZoneId(), SearchCriteria.Op.NULL); + } sb.done(); SearchCriteria sc = sb.create(); From 0f4ea44dc0be94990f5c80797c4e345f2905dc28 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Sun, 5 Jan 2020 15:29:58 +0530 Subject: [PATCH 036/134] version related fixes Signed-off-by: Abhishek Kumar --- .../KubernetesClusterService.java | 2 +- .../KubernetesVersionManagerImpl.java | 24 ++++++++++++------- .../KubernetesVersionService.java | 2 +- ui/plugins/cks/cks.js | 13 ++++------ 4 files changed, 22 insertions(+), 19 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterService.java index 1d2beaf3a859..5081f6318725 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterService.java @@ -33,7 +33,7 @@ import com.cloud.utils.component.PluggableService; public interface KubernetesClusterService extends PluggableService, Configurable { - static final String MIN_KUBERNETES_VERSION_HA_SUPPORT = "1.16"; + static final String MIN_KUBERNETES_VERSION_HA_SUPPORT = "1.16.0"; static final ConfigKey KubernetesServiceEnabled = new ConfigKey("Advanced", Boolean.class, "cloud.kubernetes.service.enabled", diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java index 1661845cc4c3..86f3803c3efa 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java @@ -127,10 +127,10 @@ public static int compareKubernetesVersion(String v1, String v2) throws IllegalA if (Strings.isNullOrEmpty(v1) || Strings.isNullOrEmpty(v2)) { throw new IllegalArgumentException(String.format("Invalid version comparision with versions %s, %s", v1, v2)); } - if(isSemanticVersion(v1)) { + if(!isSemanticVersion(v1)) { throw new IllegalArgumentException(String.format("Invalid version format, %s", v1)); } - if(isSemanticVersion(v2)) { + if(!isSemanticVersion(v2)) { throw new IllegalArgumentException(String.format("Invalid version format, %s", v2)); } String[] thisParts = v1.split("\\."); @@ -153,10 +153,10 @@ public static boolean canUpgradeKubernetesVersion(String currentVersion, String if (Strings.isNullOrEmpty(currentVersion) || Strings.isNullOrEmpty(upgradeVersion)) { throw new IllegalArgumentException(String.format("Invalid version comparision with versions %s, %s", currentVersion, upgradeVersion)); } - if(isSemanticVersion(currentVersion)) { + if(!isSemanticVersion(currentVersion)) { throw new IllegalArgumentException(String.format("Invalid version format, %s", currentVersion)); } - if(isSemanticVersion(upgradeVersion)) { + if(!isSemanticVersion(upgradeVersion)) { throw new IllegalArgumentException(String.format("Invalid version format, %s", upgradeVersion)); } String[] thisParts = currentVersion.split("\\."); @@ -291,14 +291,14 @@ public KubernetesSupportedVersionResponse addKubernetesSupportedVersion(final Ad if (!KubernetesClusterService.KubernetesServiceEnabled.value()) { throw new CloudRuntimeException("Kubernetes Service plugin is disabled"); } - final String name = cmd.getName(); + String name = cmd.getName(); final String kubernetesVersion = cmd.getKubernetesVersion(); final Long zoneId = cmd.getZoneId(); final Long isoId = cmd.getIsoId(); final String isoUrl = cmd.getUrl(); final String isoChecksum = cmd.getChecksum(); - if (Strings.isNullOrEmpty(name)) { - throw new InvalidParameterValueException("Name cannot be empty to add a new supported Kubernetes version"); + if (compareKubernetesVersion(kubernetesVersion, MIN_KUBERNETES_VERSION) < 0) { + throw new InvalidParameterValueException(String.format("New supported Kubernetes version cannot be added as %s is minimum version supported by Kubernetes Service", MIN_KUBERNETES_VERSION)); } if (Strings.isNullOrEmpty(isoUrl) && (isoId == null || isoId <= 0)) { throw new InvalidParameterValueException(String.format("Either %s or %s parameter must be passed to add a new supported Kubernetes version", "isourl", ApiConstants.ISO_ID)); @@ -306,8 +306,14 @@ public KubernetesSupportedVersionResponse addKubernetesSupportedVersion(final Ad if (!Strings.isNullOrEmpty(isoUrl) && isoId != null && isoId > 0) { throw new InvalidParameterValueException(String.format("Both %s and %s parameters can not be passed simultaneously to add a new supported Kubernetes version", "isourl", ApiConstants.ISO_ID)); } - if (compareKubernetesVersion(kubernetesVersion, MIN_KUBERNETES_VERSION) < 0) { - throw new InvalidParameterValueException(String.format("New supported Kubernetes version cannot be added as %s is minimum version supported by Kubernetes Service", MIN_KUBERNETES_VERSION)); + if (zoneId != null && dataCenterDao.findById(zoneId) == null) { + throw new InvalidParameterValueException("Invalid zone specified"); + } + if (Strings.isNullOrEmpty(name)) { + name = String.format("v%s", kubernetesVersion); + if (zoneId != null) { + name = String.format("%s-%s", name, dataCenterDao.findById(zoneId).getName()); + } } VMTemplateVO template = null; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionService.java index be11caa27cef..6d4886c7608a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionService.java @@ -26,7 +26,7 @@ import com.cloud.utils.component.PluggableService; public interface KubernetesVersionService extends PluggableService { - static final String MIN_KUBERNETES_VERSION = "1.11"; + static final String MIN_KUBERNETES_VERSION = "1.11.0"; ListResponse listKubernetesSupportedVersions(ListKubernetesSupportedVersionsCmd cmd); KubernetesSupportedVersionResponse addKubernetesSupportedVersion(AddKubernetesSupportedVersionCmd cmd); boolean deleteKubernetesSupportedVersion(DeleteKubernetesSupportedVersionCmd cmd); diff --git a/ui/plugins/cks/cks.js b/ui/plugins/cks/cks.js index 493ab7f6af9d..8ecdc14a63aa 100644 --- a/ui/plugins/cks/cks.js +++ b/ui/plugins/cks/cks.js @@ -1136,13 +1136,6 @@ title: 'label.add.kubernetes.version', preFilter: cloudStack.preFilter.createTemplate, fields: { - name: { - label: 'label.name', - //docID: 'Name of the cluster', - validation: { - required: true - } - }, version: { label: 'label.semantic.version', //docID: 'Name of the cluster', @@ -1150,6 +1143,10 @@ required: true } }, + name: { + label: 'label.name', + //docID: 'Name of the cluster', + }, zone: { label: 'label.zone', //docID: 'helpKubernetesClusterZone', @@ -1253,7 +1250,7 @@ name: args.data.name, kubernetesversion: args.data.version, }; - if (args.data.zone > 0) { + if (args.data.zone != null && args.data.zone != -1) { $.extend(data, { zoneid: args.data.zone }); From 3dca9380d096ce1bd693cf682ac874948a394f0b Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 6 Jan 2020 02:53:30 +0530 Subject: [PATCH 037/134] refactorings Signed-off-by: Abhishek Kumar --- .../apache/cloudstack/api/ApiConstants.java | 2 +- .../KubernetesClusterManagerImpl.java | 8 ++++---- .../KubernetesVersionManagerImpl.java | 18 +++++++++--------- .../AddKubernetesSupportedVersionCmd.java | 14 +++++++------- .../KubernetesSupportedVersionResponse.java | 14 +++++++------- ui/plugins/cks/cks.js | 6 +++--- 6 files changed, 31 insertions(+), 31 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index e310ad3072bd..085f4f33be20 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -751,7 +751,7 @@ public class ApiConstants { public static final String DOCKER_REGISTRY_EMAIL = "dockerregistryemail"; public static final String ISO_NAME = "isoname"; public static final String ISO_STATE = "isostate"; - public static final String KUBERNETES_VERSION = "kubernetesversion"; + public static final String SEMANTIC_VERSION = "semanticversion"; public static final String KUBERNETES_VERSION_ID = "kubernetesversionid"; public static final String KUBERNETES_VERSION_NAME = "kubernetesversionname"; public static final String MASTER_NODES = "masternodes"; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java index 4a461f2bde8e..c8f0c5eaf749 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java @@ -1615,7 +1615,7 @@ private UserVm createKubernetesMaster(final KubernetesCluster kubernetesCluster, final KubernetesSupportedVersion version = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); if (version != null) { try { - if (KubernetesVersionManagerImpl.compareKubernetesVersion(version.getSemanticVersion(), MIN_KUBERNETES_VERSION_HA_SUPPORT) >= 0) { + if (KubernetesVersionManagerImpl.compareSemanticVersions(version.getSemanticVersion(), MIN_KUBERNETES_VERSION_HA_SUPPORT) >= 0) { haSupported = true; } } catch (Exception e) { @@ -1987,7 +1987,7 @@ private void validateKubernetesClusterCreatePrameters(final CreateKubernetesClus } if (masterNodeCount > 1 ) { try { - if (KubernetesVersionManagerImpl.compareKubernetesVersion(clusterKubernetesVersion.getSemanticVersion(), MIN_KUBERNETES_VERSION_HA_SUPPORT) < 0) { + if (KubernetesVersionManagerImpl.compareSemanticVersions(clusterKubernetesVersion.getSemanticVersion(), MIN_KUBERNETES_VERSION_HA_SUPPORT) < 0) { throw new InvalidParameterValueException(String.format("HA support is available only for Kubernetes version %s and above. Given version ID: %s is %s", MIN_KUBERNETES_VERSION_HA_SUPPORT, clusterKubernetesVersion.getUuid(), clusterKubernetesVersion.getSemanticVersion())); } } catch (Exception e) { @@ -2413,7 +2413,7 @@ private void validateKubernetesClusterUpgradeParameters(UpgradeKubernetesCluster throw new InvalidParameterValueException(String.format("Invalid Kubernetes version associated with cluster ID: %s", kubernetesCluster.getUuid())); } - if (KubernetesVersionManagerImpl.compareKubernetesVersion( + if (KubernetesVersionManagerImpl.compareSemanticVersions( upgradeVersion.getSemanticVersion(), clusterVersion.getSemanticVersion()) <= 0) { throw new InvalidParameterValueException(String.format("Invalid Kubernetes version associated with cluster ID: %s", kubernetesCluster.getUuid())); @@ -2479,7 +2479,7 @@ private void upgradeKubernetesClusterNodes(final KubernetesCluster kubernetesClu "~/", upgradeScriptFile.getAbsolutePath(), "0755"); String cmdStr = String.format("sudo ./%s %s %s %s", upgradeScriptFile.getName(), upgradeVersion.getSemanticVersion(), i == 0 ? "true" : "false", - KubernetesVersionManagerImpl.compareKubernetesVersion(upgradeVersion.getSemanticVersion(), "1.15") < 0 ? "true" : "false"); + KubernetesVersionManagerImpl.compareSemanticVersions(upgradeVersion.getSemanticVersion(), "1.15.0") < 0 ? "true" : "false"); result = SshHelper.sshExecute(publicIpAddress, nodeSshPort, CLUSTER_NODE_VM_USER, pkFile, null, cmdStr, 10000, 10000, 10 * 60 * 1000); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java index 86f3803c3efa..939d10ee966c 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java @@ -83,13 +83,13 @@ private KubernetesSupportedVersionResponse createKubernetesSupportedVersionRespo response.setObjectName("kubernetessupportedversion"); response.setId(kubernetesSupportedVersion.getUuid()); response.setName(kubernetesSupportedVersion.getName()); - response.setKubernetesVersion(kubernetesSupportedVersion.getSemanticVersion()); + response.setSemanticVersion(kubernetesSupportedVersion.getSemanticVersion()); DataCenterVO zone = ApiDBUtils.findZoneById(kubernetesSupportedVersion.getZoneId()); if (zone != null) { response.setZoneId(zone.getUuid()); response.setZoneName(zone.getName()); } - if (compareKubernetesVersion(kubernetesSupportedVersion.getSemanticVersion(), + if (compareSemanticVersions(kubernetesSupportedVersion.getSemanticVersion(), KubernetesClusterService.MIN_KUBERNETES_VERSION_HA_SUPPORT)>=0) { response.setSupportsHA(true); } else { @@ -123,7 +123,7 @@ private static boolean isSemanticVersion(final String version) { return true; } - public static int compareKubernetesVersion(String v1, String v2) throws IllegalArgumentException { + public static int compareSemanticVersions(String v1, String v2) throws IllegalArgumentException { if (Strings.isNullOrEmpty(v1) || Strings.isNullOrEmpty(v2)) { throw new IllegalArgumentException(String.format("Invalid version comparision with versions %s, %s", v1, v2)); } @@ -177,7 +177,7 @@ private List filterKubernetesSupportedVersions(Li for (int i = versions.size() - 1; i >= 0; --i) { KubernetesSupportedVersionVO version = versions.get(i); try { - if (compareKubernetesVersion(minimumSemanticVersion, version.getSemanticVersion()) > 0) { + if (compareSemanticVersions(minimumSemanticVersion, version.getSemanticVersion()) > 0) { versions.remove(i); } } catch (Exception e) { @@ -292,25 +292,25 @@ public KubernetesSupportedVersionResponse addKubernetesSupportedVersion(final Ad throw new CloudRuntimeException("Kubernetes Service plugin is disabled"); } String name = cmd.getName(); - final String kubernetesVersion = cmd.getKubernetesVersion(); + final String semanticVersion = cmd.getSemanticVersion(); final Long zoneId = cmd.getZoneId(); final Long isoId = cmd.getIsoId(); final String isoUrl = cmd.getUrl(); final String isoChecksum = cmd.getChecksum(); - if (compareKubernetesVersion(kubernetesVersion, MIN_KUBERNETES_VERSION) < 0) { + if (compareSemanticVersions(semanticVersion, MIN_KUBERNETES_VERSION) < 0) { throw new InvalidParameterValueException(String.format("New supported Kubernetes version cannot be added as %s is minimum version supported by Kubernetes Service", MIN_KUBERNETES_VERSION)); } if (Strings.isNullOrEmpty(isoUrl) && (isoId == null || isoId <= 0)) { throw new InvalidParameterValueException(String.format("Either %s or %s parameter must be passed to add a new supported Kubernetes version", "isourl", ApiConstants.ISO_ID)); } if (!Strings.isNullOrEmpty(isoUrl) && isoId != null && isoId > 0) { - throw new InvalidParameterValueException(String.format("Both %s and %s parameters can not be passed simultaneously to add a new supported Kubernetes version", "isourl", ApiConstants.ISO_ID)); + throw new InvalidParameterValueException(String.format("Both %s and %s parameters can not be passed simultaneously to add a new supported Kubernetes version", ApiConstants.URL, ApiConstants.ISO_ID)); } if (zoneId != null && dataCenterDao.findById(zoneId) == null) { throw new InvalidParameterValueException("Invalid zone specified"); } if (Strings.isNullOrEmpty(name)) { - name = String.format("v%s", kubernetesVersion); + name = String.format("v%s", semanticVersion); if (zoneId != null) { name = String.format("%s-%s", name, dataCenterDao.findById(zoneId).getName()); } @@ -333,7 +333,7 @@ public KubernetesSupportedVersionResponse addKubernetesSupportedVersion(final Ad validateExistingTemplateForKubernetesVersionIso(template, zoneId); } - KubernetesSupportedVersionVO supportedVersionVO = new KubernetesSupportedVersionVO(name, kubernetesVersion, template.getId(), zoneId); + KubernetesSupportedVersionVO supportedVersionVO = new KubernetesSupportedVersionVO(name, semanticVersion, template.getId(), zoneId); supportedVersionVO = kubernetesSupportedVersionDao.persist(supportedVersionVO); return createKubernetesSupportedVersionResponse(supportedVersionVO); diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/AddKubernetesSupportedVersionCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/AddKubernetesSupportedVersionCmd.java index 4b33f9089def..784ae37bcc72 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/AddKubernetesSupportedVersionCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/AddKubernetesSupportedVersionCmd.java @@ -63,9 +63,9 @@ public class AddKubernetesSupportedVersionCmd extends BaseCmd implements UserCmd description = "the name of the Kubernetes supported version") private String name; - @Parameter(name = ApiConstants.KUBERNETES_VERSION, type = CommandType.STRING, required = true, - description = "the semantic version of the Kubernetes") - private String kubernetesVersion; + @Parameter(name = ApiConstants.SEMANTIC_VERSION, type = CommandType.STRING, required = true, + description = "the semantic version of the Kubernetes version") + private String semanticVersion; @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, @@ -94,14 +94,14 @@ public String getName() { return name; } - public String getKubernetesVersion() { - if(Strings.isNullOrEmpty(kubernetesVersion)) { + public String getSemanticVersion() { + if(Strings.isNullOrEmpty(semanticVersion)) { throw new InvalidParameterValueException("Version can not be null"); } - if(!kubernetesVersion.matches("[0-9]+(\\.[0-9]+)*")) { + if(!semanticVersion.matches("[0-9]+(\\.[0-9]+)*")) { throw new IllegalArgumentException("Invalid version format. Semantic version needed"); } - return kubernetesVersion; + return semanticVersion; } public Long getZoneId() { diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java index 1f7a2894b0b8..f8ae5f87765b 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java @@ -36,9 +36,9 @@ public class KubernetesSupportedVersionResponse extends BaseResponse { @Param(description = "Name of the Kubernetes supported version") private String name; - @SerializedName(ApiConstants.KUBERNETES_VERSION) - @Param(description = "Semantic Kubernetes version") - private String kubernetesVersion; + @SerializedName(ApiConstants.SEMANTIC_VERSION) + @Param(description = "Kubernetes semantic version") + private String semanticVersion; @SerializedName(ApiConstants.ISO_ID) @Param(description = "the id of the binaries ISO for Kubernetes supported version") @@ -80,12 +80,12 @@ public void setName(String name) { this.name = name; } - public String getKubernetesVersion() { - return kubernetesVersion; + public String getSemanticVersion() { + return semanticVersion; } - public void setKubernetesVersion(String kubernetesVersion) { - this.kubernetesVersion = kubernetesVersion; + public void setSemanticVersion(String semanticVersion) { + this.semanticVersion = semanticVersion; } public String getIsoId() { diff --git a/ui/plugins/cks/cks.js b/ui/plugins/cks/cks.js index 8ecdc14a63aa..471d19083a0b 100644 --- a/ui/plugins/cks/cks.js +++ b/ui/plugins/cks/cks.js @@ -939,7 +939,7 @@ success: function(json) { var jsonObj; if (json.listkubernetessupportedversionsresponse.kubernetessupportedversion != null) { - version = json.listkubernetessupportedversionsresponse.kubernetessupportedversion[0].kubernetesversion; + version = json.listkubernetessupportedversionsresponse.kubernetessupportedversion[0].semanticversion; } } }); @@ -1086,7 +1086,7 @@ name: { label: 'label.name' }, - kubernetesversion: { + semanticversion: { label: 'label.kubernetes.version' }, zonename: { @@ -1248,7 +1248,7 @@ action: function(args) { var data = { name: args.data.name, - kubernetesversion: args.data.version, + semanticversion: args.data.version, }; if (args.data.zone != null && args.data.zone != -1) { $.extend(data, { From 716ea3d1212a2f919e7464e7c9cf11b8e27828bf Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 6 Jan 2020 02:55:11 +0530 Subject: [PATCH 038/134] added smoke test for kubernetes version Signed-off-by: Abhishek Kumar --- .../test_kubernetes_supported_versions.py | 230 ++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100644 plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_supported_versions.py diff --git a/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_supported_versions.py b/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_supported_versions.py new file mode 100644 index 000000000000..68bdb9ae5713 --- /dev/null +++ b/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_supported_versions.py @@ -0,0 +1,230 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" Tests for Kubernetes supported version """ + +#Import Local Modules +from marvin.cloudstackTestCase import cloudstackTestCase, unittest +from marvin.cloudstackAPI import (listKubernetesSupportedVersions, + addKubernetesSupportedVersion, + deleteKubernetesSupportedVersion) +from marvin.cloudstackException import CloudstackAPIException +from marvin.codes import FAILED +from marvin.lib.utils import (cleanup_resources, + random_gen) +from marvin.lib.common import get_zone +from nose.plugins.attrib import attr + +import time + +_multiprocess_shared_ = True + +class TestKubernetesSupportedVersion(cloudstackTestCase): + + @classmethod + def setUpClass(cls): + testClient = super(TestKubernetesSupportedVersion, cls).getClsTestClient() + cls.apiclient = testClient.getApiClient() + cls.services = testClient.getParsedTestDataConfig() + cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests()) + cls.kubernetes_version_iso_url = 'http://172.20.0.1/files/setup-1.16.3.iso' + cls._cleanup = [] + return + + @classmethod + def tearDownClass(cls): + try: + cleanup_resources(cls.apiclient, cls._cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + def setUp(self): + self.services = self.testClient.getParsedTestDataConfig() + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + self.cleanup = [] + return + + def tearDown(self): + try: + #Clean up, terminate the created templates + cleanup_resources(self.apiclient, self.cleanup) + + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + def test_01_add_delete_kubernetes_supported_version(self): + """Test to add a new Kubernetes supported version + + # Validate the following: + # 1. addKubernetesSupportedVersion should return valid info for new version + # 2. The Cloud Database contains the valid information when listKubernetesSupportedVersions is called + """ + + version = '1.16.3' + name = 'v' + version + '-' + random_gen() + + self.debug("Adding Kubernetes supported version with name: %s" % name) + + version_response = self.addKubernetesSupportedVersionUsingDetails(version, name, self.zone.id, self.kubernetes_version_iso_url) + + list_versions_response = self.listKubernetesSupportedVersionId(version_response.id) + + self.assertEqual( + list_versions_response.name, + name, + "Check KubernetesSupportedVersion name {}, {}".format(list_versions_response.name, name) + ) + + self.assertEqual( + list_versions_response.semanticversion, + version, + "Check KubernetesSupportedVersion version {}, {}".format(list_versions_response.semanticversion, version) + ) + self.assertEqual( + list_versions_response.zoneid, + self.zone.id, + "Check KubernetesSupportedVersion zone {}, {}".format(list_versions_response.zoneid, self.zone.id) + ) + + db_version_name = self.dbclient.execute("select name from kubernetes_supported_version where uuid = '%s';" % version_response.id)[0][0] + + self.assertEqual( + str(db_version_name), + name, + "Check KubernetesSupportedVersion name in DB {}, {}".format(db_version_name, name) + ) + + self.debug("Added Kubernetes supported version with ID: %s. Waiting for its ISO to be Ready" % version_response.id) + + self.checkKubernetesSupportedVersionIsoState(version_response.id) + + self.debug("Deleting Kubernetes supported version with ID: %s" % version_response.id) + + delete_response = self.deleteKubernetesSupportedVersionId(version_response.id, True) + + self.assertEqual( + delete_response.success, + True, + "Check KubernetesSupportedVersion deletion in DB {}, {}".format(delete_response.success, True) + ) + + db_version_removed = self.dbclient.execute("select removed from kubernetes_supported_version where uuid = '%s';" % version_response.id)[0][0] + + self.assertNotEqual( + db_version_removed, + None, + "KubernetesSupportedVersion not removed in DB" + ) + + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + def test_02_add_unsupported_kubernetes_supported_version(self): + """Test to trying to add a new unsupported Kubernetes supported version + + # Validate the following: + # 1. API should return an error + """ + + version = '1.1.1' + name = 'v' + version + '-' + random_gen() + try: + version_response = self.addKubernetesSupportedVersionUsingDetails(version, name, self.zone.id, self.kubernetes_version_iso_url) + self.fail("Kubernetes supported version below version 1.11.0 been added. Must be an error.") + except CloudstackAPIException as e: + self.debug("Unsupported version error check successful, API failure: %s" % e) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + def test_03_add_invalid_kubernetes_supported_version(self): + """Test to trying to add a new unsupported Kubernetes supported version + + # Validate the following: + # 1. API should return an error + """ + + version = 'invalid' + name = 'v' + version + '-' + random_gen() + try: + version_response = self.addKubernetesSupportedVersionUsingDetails(version, name, self.zone.id, self.kubernetes_version_iso_url) + self.fail("Kubernetes supported version below version 1.11.0 been added. Must be an error.") + except CloudstackAPIException as e: + self.debug("Unsupported version error check successful, API failure: %s" % e) + return + + def addKubernetesSupportedVersion(self, cmd): + version = self.apiclient.addKubernetesSupportedVersion(cmd) + if not version: + self.cleanup.append(version) + return version + + def addKubernetesSupportedVersionUsingDetails(self, version, name, zoneId, isoUrl): + addKubernetesSupportedVersionCmd = addKubernetesSupportedVersion.addKubernetesSupportedVersionCmd() + addKubernetesSupportedVersionCmd.semanticversion = version + addKubernetesSupportedVersionCmd.name = name + addKubernetesSupportedVersionCmd.zoneid = zoneId + addKubernetesSupportedVersionCmd.url = isoUrl + versionResponse = self.addKubernetesSupportedVersion(addKubernetesSupportedVersionCmd) + return versionResponse + + def listKubernetesSupportedVersions(self, cmd): + versions = self.apiclient.listKubernetesSupportedVersions(cmd) + return versions + + def listKubernetesSupportedVersionId(self, versionId): + listKubernetesSupportedVersionsCmd = listKubernetesSupportedVersions.listKubernetesSupportedVersionsCmd() + listKubernetesSupportedVersionsCmd.id = versionId + versionResponse = self.listKubernetesSupportedVersions(listKubernetesSupportedVersionsCmd) + return versionResponse[0] + + def listAllKubernetesSupportedVersions(self): + listKubernetesSupportedVersionsCmd = listKubernetesSupportedVersions.listKubernetesSupportedVersionsCmd() + versionResponse = self.listKubernetesSupportedVersionsCmd(listKubernetesSupportedVersionsCmd) + return versionResponse + + def deleteKubernetesSupportedVersion(self, cmd): + response = self.apiclient.deleteKubernetesSupportedVersion(cmd) + return response + + def deleteKubernetesSupportedVersionId(self, versionId, deleteIso): + deleteKubernetesSupportedVersionCmd = deleteKubernetesSupportedVersion.deleteKubernetesSupportedVersionCmd() + deleteKubernetesSupportedVersionCmd.id = versionId + deleteKubernetesSupportedVersionCmd.deleteiso = deleteIso + response = self.deleteKubernetesSupportedVersion(deleteKubernetesSupportedVersionCmd) + return response + + def checkKubernetesSupportedVersionIsoState(self, version_id, retries=15, interval=15): + """Check if Kubernetes supported version ISO is in Ready state""" + + while retries > -1: + time.sleep(interval) + list_versions_response = self.listKubernetesSupportedVersionId(version_id) + if not hasattr(list_versions_response, 'isostate') or not list_versions_response or not list_versions_response.isostate: + retries = retries - 1 + continue + if 'Creating' == list_versions_response.isostate: + retries = retries - 1 + elif 'Ready' == list_versions_response.isostate: + return + else: + raise Exception( + "Failed to download Kubernetes supported version ISO: status - %s" % + list_versions_response.isostate) + raise Exception("Kubernetes supported version Ready state timed out") From da8b2219a48c63445556d29d6971e325212aadc9 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 6 Jan 2020 13:59:34 +0530 Subject: [PATCH 039/134] made name optional param for addKubernetesSupportedVersion Signed-off-by: Abhishek Kumar --- .../kubernetesversion/AddKubernetesSupportedVersionCmd.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/AddKubernetesSupportedVersionCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/AddKubernetesSupportedVersionCmd.java index 784ae37bcc72..1568d00851c3 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/AddKubernetesSupportedVersionCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/AddKubernetesSupportedVersionCmd.java @@ -59,7 +59,7 @@ public class AddKubernetesSupportedVersionCmd extends BaseCmd implements UserCmd ///////////////////////////////////////////////////// //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// - @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "the name of the Kubernetes supported version") private String name; From 1853208d645727b77095e19b05e6de51d37f2d2d Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 8 Jan 2020 09:45:49 +0530 Subject: [PATCH 040/134] fixes Signed-off-by: Abhishek Kumar --- .../KubernetesClusterManagerImpl.java | 23 ++++---- .../KubernetesClusterService.java | 4 ++ .../KubernetesVersionManagerImpl.java | 15 ++---- .../test_kubernetes_supported_versions.py | 53 ++++++++----------- 4 files changed, 44 insertions(+), 51 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java index c8f0c5eaf749..f33bb373ccad 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java @@ -181,6 +181,7 @@ import com.cloud.utils.db.TransactionCallbackWithException; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.exception.ExecutionException; import com.cloud.utils.fsm.NoTransitionException; import com.cloud.utils.fsm.StateMachine2; import com.cloud.utils.net.Ip; @@ -1618,7 +1619,7 @@ private UserVm createKubernetesMaster(final KubernetesCluster kubernetesCluster, if (KubernetesVersionManagerImpl.compareSemanticVersions(version.getSemanticVersion(), MIN_KUBERNETES_VERSION_HA_SUPPORT) >= 0) { haSupported = true; } - } catch (Exception e) { + } catch (IllegalArgumentException e) { LOGGER.error(String.format("Unable to compare Kubernetes version for cluster version ID: %s with %s", version.getUuid(), MIN_KUBERNETES_VERSION_HA_SUPPORT), e); } } @@ -1664,7 +1665,7 @@ private UserVm createKubernetesMaster(final KubernetesCluster kubernetesCluster, } initArgs += String.format("--apiserver-cert-extra-sans=%s", serverIp); k8sMasterConfig = k8sMasterConfig.replace(clusterInitArgsKey, initArgs); - } catch (Exception e) { + } catch (IOException e) { String msg = "Failed to read Kubernetes master configuration file"; LOGGER.error(msg, e); throw new ManagementServerException(msg, e); @@ -1713,7 +1714,7 @@ private UserVm createKubernetesAdditionalMaster(final KubernetesCluster kubernet k8sMasterConfig = k8sMasterConfig.replace(joinIpKey, joinIp); k8sMasterConfig = k8sMasterConfig.replace(clusterTokenKey, generateClusterToken(kubernetesCluster)); k8sMasterConfig = k8sMasterConfig.replace(clusterHACertificateKey, generateClusterHACertificateKey(kubernetesCluster)); - } catch (Exception e) { + } catch (IOException e) { String msg = "Failed to read Kubernetes master configuration file"; LOGGER.error(msg, e); throw new ManagementServerException(msg, e); @@ -1807,7 +1808,7 @@ private UserVm createKubernetesNode(KubernetesCluster kubernetesCluster, String k8sNodeConfig = k8sNodeConfig.replace(dockerAuthKey, "\"" + base64Auth + "\""); k8sNodeConfig = k8sNodeConfig.replace(dockerEmailKey, "\"" + dockerRegistryEmail + "\""); } - } catch (Exception e) { + } catch (IOException e) { String msg = "Failed to read Kubernetes node configuration file"; LOGGER.error(msg, e); throw new ManagementServerException(msg, e); @@ -1830,7 +1831,8 @@ private void startKubernetesVM(final UserVm vm, final KubernetesCluster kubernet f.set(startVm, vm.getId()); userVmService.startVirtualMachine(startVm); LOGGER.debug(String.format("Started VM ID: %s in the Kubernetes cluster ID: %s", vm.getUuid(), kubernetesCluster.getUuid())); - } catch (Exception ex) { + } catch (IllegalAccessException | NoSuchFieldException | ExecutionException | + ResourceUnavailableException | ResourceAllocationException | InsufficientCapacityException ex) { logAndThrow(Level.WARN, String.format("Failed to start VM in the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), ex); } @@ -1990,7 +1992,7 @@ private void validateKubernetesClusterCreatePrameters(final CreateKubernetesClus if (KubernetesVersionManagerImpl.compareSemanticVersions(clusterKubernetesVersion.getSemanticVersion(), MIN_KUBERNETES_VERSION_HA_SUPPORT) < 0) { throw new InvalidParameterValueException(String.format("HA support is available only for Kubernetes version %s and above. Given version ID: %s is %s", MIN_KUBERNETES_VERSION_HA_SUPPORT, clusterKubernetesVersion.getUuid(), clusterKubernetesVersion.getSemanticVersion())); } - } catch (Exception e) { + } catch (IllegalArgumentException e) { logAndThrow(Level.WARN, String.format("Unable to compare Kubernetes version for given version ID: %s with %s", clusterKubernetesVersion.getUuid(), MIN_KUBERNETES_VERSION_HA_SUPPORT), e); } } @@ -2707,10 +2709,9 @@ public KubernetesClusterConfigResponse getKubernetesClusterConfig(GetKubernetesC response.setId(kubernetesCluster.getUuid()); response.setName(kubernetesCluster.getName()); String configData = ""; - try { - configData = new String(Base64.decodeBase64(kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "kubeConfigData").getValue())); - } catch (Exception e) { - LOGGER.warn(String.format("Failed to retrieve Kubernetes config for cluster ID: %s", kubernetesCluster.getUuid()), e); + KubernetesClusterDetailsVO clusterDetailsVO = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "kubeConfigData"); + if (clusterDetailsVO != null && !Strings.isNullOrEmpty(clusterDetailsVO.getValue())) { + configData = new String(Base64.decodeBase64(clusterDetailsVO.getValue())); } response.setConfigData(configData); response.setObjectName("clusterconfig"); @@ -2889,7 +2890,7 @@ public void reallyRun() { } else { LOGGER.warn(String.format("Garbage collection failed for Kubernetes cluster ID: %s, it will be attempted to garbage collected in next run", kubernetesCluster.getUuid())); } - } catch (Exception e) { + } catch (ManagementServerException e) { LOGGER.warn(String.format("Failed to destroy Kubernetes cluster ID: %s during GC", kubernetesCluster.getUuid()), e); // proceed further with rest of the Kubernetes cluster garbage collection } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterService.java index 5081f6318725..a29c73308365 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterService.java @@ -51,6 +51,10 @@ public interface KubernetesClusterService extends PluggableService, Configurable "Name of the network offering that will be used to create isolated network in which Kubernetes cluster VMs will be launched", false); + static boolean isKubernetesServiceEnabled() { + return KubernetesServiceEnabled.value(); + } + KubernetesCluster findById(final Long id); KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) throws InsufficientCapacityException, diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java index 939d10ee966c..0d69af476615 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java @@ -31,10 +31,8 @@ import org.apache.cloudstack.api.command.user.kubernetesversion.ListKubernetesSupportedVersionsCmd; import org.apache.cloudstack.api.response.KubernetesSupportedVersionResponse; import org.apache.cloudstack.api.response.ListResponse; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.log4j.Logger; -import com.cloud.api.ApiDBUtils; import com.cloud.api.query.dao.TemplateJoinDao; import com.cloud.api.query.vo.TemplateJoinVO; import com.cloud.dc.DataCenterVO; @@ -75,8 +73,6 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne private DataCenterDao dataCenterDao; @Inject private TemplateApiService templateService; - @Inject - private ConfigurationDao globalConfigDao; private KubernetesSupportedVersionResponse createKubernetesSupportedVersionResponse(final KubernetesSupportedVersion kubernetesSupportedVersion) { KubernetesSupportedVersionResponse response = new KubernetesSupportedVersionResponse(); @@ -84,7 +80,7 @@ private KubernetesSupportedVersionResponse createKubernetesSupportedVersionRespo response.setId(kubernetesSupportedVersion.getUuid()); response.setName(kubernetesSupportedVersion.getName()); response.setSemanticVersion(kubernetesSupportedVersion.getSemanticVersion()); - DataCenterVO zone = ApiDBUtils.findZoneById(kubernetesSupportedVersion.getZoneId()); + DataCenterVO zone = dataCenterDao.findById(kubernetesSupportedVersion.getZoneId()); if (zone != null) { response.setZoneId(zone.getUuid()); response.setZoneName(zone.getName()); @@ -180,7 +176,7 @@ private List filterKubernetesSupportedVersions(Li if (compareSemanticVersions(minimumSemanticVersion, version.getSemanticVersion()) > 0) { versions.remove(i); } - } catch (Exception e) { + } catch (IllegalArgumentException e) { LOGGER.warn(String.format("Unable to compare Kubernetes version for supported version ID: %s with %s", version.getUuid(), minimumSemanticVersion)); versions.remove(i); } @@ -209,7 +205,7 @@ private VirtualMachineTemplate registerKubernetesVersionIso(final String version f = registerIsoCmd.getClass().getDeclaredField("url"); f.setAccessible(true); f.set(registerIsoCmd, isoUrl); - if (Strings.isNullOrEmpty(isoChecksum)) { + if (!Strings.isNullOrEmpty(isoChecksum)) { f = registerIsoCmd.getClass().getDeclaredField("checksum"); f.setAccessible(true); f.set(registerIsoCmd, isoChecksum); @@ -248,7 +244,7 @@ private void deleteKubernetesVersionIso(long templateId) throws IllegalAccessExc @Override public ListResponse listKubernetesSupportedVersions(final ListKubernetesSupportedVersionsCmd cmd) { - if (!KubernetesClusterService.KubernetesServiceEnabled.value()) { + if (!KubernetesClusterService.isKubernetesServiceEnabled()) { throw new CloudRuntimeException("Kubernetes Service plugin is disabled"); } final Long versionId = cmd.getId(); @@ -265,7 +261,6 @@ public ListResponse listKubernetesSupportedV } minimumSemanticVersion = minVersion.getSemanticVersion(); } - List versions = new ArrayList<>(); if (versionId != null) { KubernetesSupportedVersionVO version = kubernetesSupportedVersionDao.findById(versionId); @@ -363,7 +358,7 @@ public boolean deleteKubernetesSupportedVersion(final DeleteKubernetesSupportedV if (isDeleteIso && template != null) { // Delete ISO try { deleteKubernetesVersionIso(template.getId()); - } catch (Exception ex) { + } catch (IllegalAccessException | NoSuchFieldException | IllegalArgumentException ex) { LOGGER.error(String.format("Unable to delete binaries ISO ID: %s associated with supported kubernetes version ID: %s", template.getUuid(), version.getUuid()), ex); throw new CloudRuntimeException(String.format("Unable to delete binaries ISO ID: %s associated with supported kubernetes version ID: %s", template.getUuid(), version.getUuid())); } diff --git a/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_supported_versions.py b/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_supported_versions.py index 68bdb9ae5713..595aafdda18f 100644 --- a/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_supported_versions.py +++ b/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_supported_versions.py @@ -82,9 +82,9 @@ def test_01_add_delete_kubernetes_supported_version(self): self.debug("Adding Kubernetes supported version with name: %s" % name) - version_response = self.addKubernetesSupportedVersionUsingDetails(version, name, self.zone.id, self.kubernetes_version_iso_url) + version_response = self.addKubernetesSupportedVersion(version, name, self.zone.id, self.kubernetes_version_iso_url) - list_versions_response = self.listKubernetesSupportedVersionId(version_response.id) + list_versions_response = self.listKubernetesSupportedVersion(version_response.id) self.assertEqual( list_versions_response.name, @@ -113,11 +113,11 @@ def test_01_add_delete_kubernetes_supported_version(self): self.debug("Added Kubernetes supported version with ID: %s. Waiting for its ISO to be Ready" % version_response.id) - self.checkKubernetesSupportedVersionIsoState(version_response.id) + self.waitForKubernetesSupportedVersionIsoReadyState(version_response.id) self.debug("Deleting Kubernetes supported version with ID: %s" % version_response.id) - delete_response = self.deleteKubernetesSupportedVersionId(version_response.id, True) + delete_response = self.deleteKubernetesSupportedVersion(version_response.id, True) self.assertEqual( delete_response.success, @@ -146,7 +146,10 @@ def test_02_add_unsupported_kubernetes_supported_version(self): version = '1.1.1' name = 'v' + version + '-' + random_gen() try: - version_response = self.addKubernetesSupportedVersionUsingDetails(version, name, self.zone.id, self.kubernetes_version_iso_url) + version_response = self.addKubernetesSupportedVersion(version, name, self.zone.id, self.kubernetes_version_iso_url) + self.debug("Unsupported CKS Kubernetes supported added with ID: %s. Deleting it and failing test." % version_response.id) + self.waitForKubernetesSupportedVersionIsoReadyState(version_response.id) + self.deleteKubernetesSupportedVersion(version_response.id, True) self.fail("Kubernetes supported version below version 1.11.0 been added. Must be an error.") except CloudstackAPIException as e: self.debug("Unsupported version error check successful, API failure: %s" % e) @@ -163,59 +166,49 @@ def test_03_add_invalid_kubernetes_supported_version(self): version = 'invalid' name = 'v' + version + '-' + random_gen() try: - version_response = self.addKubernetesSupportedVersionUsingDetails(version, name, self.zone.id, self.kubernetes_version_iso_url) - self.fail("Kubernetes supported version below version 1.11.0 been added. Must be an error.") + version_response = self.addKubernetesSupportedVersion(version, name, self.zone.id, self.kubernetes_version_iso_url) + self.debug("Invalid Kubernetes supported added with ID: %s. Deleting it and failing test." % version_response.id) + self.waitForKubernetesSupportedVersionIsoReadyState(version_response.id) + self.deleteKubernetesSupportedVersion(version_response.id, True) + self.fail("Invalid Kubernetes supported version has been added. Must be an error.") except CloudstackAPIException as e: self.debug("Unsupported version error check successful, API failure: %s" % e) return - def addKubernetesSupportedVersion(self, cmd): - version = self.apiclient.addKubernetesSupportedVersion(cmd) - if not version: - self.cleanup.append(version) - return version - - def addKubernetesSupportedVersionUsingDetails(self, version, name, zoneId, isoUrl): + def addKubernetesSupportedVersion(self, version, name, zoneId, isoUrl): addKubernetesSupportedVersionCmd = addKubernetesSupportedVersion.addKubernetesSupportedVersionCmd() addKubernetesSupportedVersionCmd.semanticversion = version addKubernetesSupportedVersionCmd.name = name addKubernetesSupportedVersionCmd.zoneid = zoneId addKubernetesSupportedVersionCmd.url = isoUrl - versionResponse = self.addKubernetesSupportedVersion(addKubernetesSupportedVersionCmd) + versionResponse = self.apiclient.addKubernetesSupportedVersion(addKubernetesSupportedVersionCmd) + if not versionResponse: + self.cleanup.append(versionResponse) return versionResponse - def listKubernetesSupportedVersions(self, cmd): - versions = self.apiclient.listKubernetesSupportedVersions(cmd) - return versions - - def listKubernetesSupportedVersionId(self, versionId): + def listKubernetesSupportedVersion(self, versionId): listKubernetesSupportedVersionsCmd = listKubernetesSupportedVersions.listKubernetesSupportedVersionsCmd() listKubernetesSupportedVersionsCmd.id = versionId - versionResponse = self.listKubernetesSupportedVersions(listKubernetesSupportedVersionsCmd) + versionResponse = self.apiclient.listKubernetesSupportedVersions(listKubernetesSupportedVersionsCmd) return versionResponse[0] - def listAllKubernetesSupportedVersions(self): - listKubernetesSupportedVersionsCmd = listKubernetesSupportedVersions.listKubernetesSupportedVersionsCmd() - versionResponse = self.listKubernetesSupportedVersionsCmd(listKubernetesSupportedVersionsCmd) - return versionResponse - def deleteKubernetesSupportedVersion(self, cmd): response = self.apiclient.deleteKubernetesSupportedVersion(cmd) return response - def deleteKubernetesSupportedVersionId(self, versionId, deleteIso): + def deleteKubernetesSupportedVersion(self, versionId, deleteIso): deleteKubernetesSupportedVersionCmd = deleteKubernetesSupportedVersion.deleteKubernetesSupportedVersionCmd() deleteKubernetesSupportedVersionCmd.id = versionId deleteKubernetesSupportedVersionCmd.deleteiso = deleteIso - response = self.deleteKubernetesSupportedVersion(deleteKubernetesSupportedVersionCmd) + response = self.apiclient.deleteKubernetesSupportedVersion(deleteKubernetesSupportedVersionCmd) return response - def checkKubernetesSupportedVersionIsoState(self, version_id, retries=15, interval=15): + def waitForKubernetesSupportedVersionIsoReadyState(self, version_id, retries=15, interval=15): """Check if Kubernetes supported version ISO is in Ready state""" while retries > -1: time.sleep(interval) - list_versions_response = self.listKubernetesSupportedVersionId(version_id) + list_versions_response = self.listKubernetesSupportedVersion(version_id) if not hasattr(list_versions_response, 'isostate') or not list_versions_response or not list_versions_response.isostate: retries = retries - 1 continue From 0977fe6c20bab5deb490688f95f5049c7d6957f3 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 8 Jan 2020 09:49:04 +0530 Subject: [PATCH 041/134] wip unit test added Signed-off-by: Abhishek Kumar --- .../KubernetesVersionServiceTest.java | 171 ++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetesversion/KubernetesVersionServiceTest.java diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetesversion/KubernetesVersionServiceTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetesversion/KubernetesVersionServiceTest.java new file mode 100644 index 000000000000..395998c2d8fd --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetesversion/KubernetesVersionServiceTest.java @@ -0,0 +1,171 @@ +package com.cloud.kubernetesversion; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; + +import com.cloud.api.query.dao.TemplateJoinDao; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.kubernetescluster.KubernetesClusterService; +import com.cloud.kubernetescluster.dao.KubernetesClusterDao; +import com.cloud.kubernetesversion.dao.KubernetesSupportedVersionDao; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.template.TemplateApiService; +import com.cloud.utils.component.ComponentContext; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({KubernetesClusterService.class, ComponentContext.class}) +public class KubernetesVersionServiceTest { + + @InjectMocks + private KubernetesVersionService kubernetesVersionService = new KubernetesVersionManagerImpl(); + + @Mock + private KubernetesSupportedVersionDao kubernetesSupportedVersionDao; + @Mock + private KubernetesClusterDao kubernetesClusterDao; + @Mock + private VMTemplateDao templateDao; + @Mock + private TemplateJoinDao templateJoinDao; + @Mock + private DataCenterDao dataCenterDao; + @Mock + private TemplateApiService templateService; + + @Before + public void setUp() throws Exception { +// MockitoAnnotations.initMocks(this); +// +// PowerMockito.mockStatic(KubernetesClusterService.class); +// PowerMockito.when(KubernetesClusterService.isKubernetesServiceEnabled()).thenReturn(true); +// +// DataCenterVO zone = Mockito.mock(DataCenterVO.class); +// when(zone.getId()).thenReturn(1L); +// when(dataCenterDao.findById(Mockito.anyLong())).thenReturn(zone); +// +// TemplateJoinVO templateJoinVO = Mockito.mock(TemplateJoinVO.class); +// when(templateJoinVO.getId()).thenReturn(1L); +// when(templateJoinVO.getUrl()).thenReturn("https://download.cloudstack.com"); +// when(templateJoinVO.getState()).thenReturn(ObjectInDataStoreStateMachine.State.Ready); +// when(templateJoinDao.findById(Mockito.anyLong())).thenReturn(templateJoinVO); +// +// KubernetesSupportedVersionVO versionVO = Mockito.mock(KubernetesSupportedVersionVO.class); +// when(versionVO.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); +// when(kubernetesSupportedVersionDao.persist(Mockito.any(KubernetesSupportedVersionVO.class))).thenReturn(versionVO); + } + + @After + public void tearDown() throws Exception { + } + + @Test + public void listKubernetesSupportedVersionsTest() { +// ListKubernetesSupportedVersionsCmd cmd = Mockito.mock(ListKubernetesSupportedVersionsCmd.class); +// List versionVOs = new ArrayList<>(); +// KubernetesSupportedVersionVO versionVO = Mockito.mock(KubernetesSupportedVersionVO.class); +// when(versionVO.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); +// versionVOs.add(versionVO); +// when(kubernetesSupportedVersionDao.listAll()).thenReturn(versionVOs); +// when(kubernetesSupportedVersionDao.listAllInZone(Mockito.anyLong())).thenReturn(versionVOs); +// when(kubernetesSupportedVersionDao.findById(Mockito.anyLong())).thenReturn(versionVO); +// kubernetesVersionService.listKubernetesSupportedVersions(cmd); + } + + @Test//(expected = InvalidParameterValueException.class) + public void addKubernetesSupportedVersionLowerUnsupportedTest() { +// AddKubernetesSupportedVersionCmd cmd = Mockito.mock(AddKubernetesSupportedVersionCmd.class); +// AccountVO account = new AccountVO("admin", 1L, "", Account.ACCOUNT_TYPE_ADMIN, "uuid"); +// UserVO user = new UserVO(1, "adminuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); +// CallContext.register(user, account); +// when(cmd.getSemanticVersion()).thenReturn("1.1.1"); +// kubernetesVersionService.addKubernetesSupportedVersion(cmd); + } + + @Test//(expected = InvalidParameterValueException.class) + public void addKubernetesSupportedVersionIsoIdUrlTest() { +// AddKubernetesSupportedVersionCmd cmd = Mockito.mock(AddKubernetesSupportedVersionCmd.class); +// AccountVO account = new AccountVO("admin", 1L, "", Account.ACCOUNT_TYPE_ADMIN, "uuid"); +// UserVO user = new UserVO(1, "adminuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); +// when(cmd.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); +// CallContext.register(user, account); +// when(cmd.getIsoId()).thenReturn(1L); +// when(cmd.getUrl()).thenReturn("url"); +// kubernetesVersionService.addKubernetesSupportedVersion(cmd); + } + + @Test + public void addKubernetesSupportedVersionIsoIdTest() { +// AddKubernetesSupportedVersionCmd cmd = Mockito.mock(AddKubernetesSupportedVersionCmd.class); +// AccountVO account = new AccountVO("admin", 1L, "", Account.ACCOUNT_TYPE_ADMIN, "uuid"); +// UserVO user = new UserVO(1, "adminuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); +// CallContext.register(user, account); +// when(cmd.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); +// when(cmd.getIsoId()).thenReturn(1L); +// when(cmd.getUrl()).thenReturn(null); +// VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); +// when(templateVO.getId()).thenReturn(1L); +// when(templateVO.getFormat()).thenReturn(Storage.ImageFormat.ISO); +// when(templateVO.isPublicTemplate()).thenReturn(true); +// when(templateVO.isCrossZones()).thenReturn(true); +// when(templateDao.findById(Mockito.anyLong())).thenReturn(templateVO); +// kubernetesVersionService.addKubernetesSupportedVersion(cmd); + } + + @Test + public void addKubernetesSupportedVersionIsoUrlTest() throws ResourceAllocationException, NoSuchFieldException { +// AddKubernetesSupportedVersionCmd cmd = Mockito.mock(AddKubernetesSupportedVersionCmd.class); +// AccountVO account = new AccountVO("admin", 1L, "", Account.ACCOUNT_TYPE_ADMIN, "uuid"); +// UserVO user = new UserVO(1, "adminuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); +// CallContext.register(user, account); +// when(cmd.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); +// when(cmd.getIsoId()).thenReturn(null); +// when(cmd.getUrl()).thenReturn("https://download.cloudstack.com"); +// when(cmd.getChecksum()).thenReturn(null); +// PowerMockito.mockStatic(ComponentContext.class); +// when(ComponentContext.inject(Mockito.any(RegisterIsoCmd.class))).thenReturn(new RegisterIsoCmd()); +// when(templateService.registerIso(Mockito.any(RegisterIsoCmd.class))).thenReturn(Mockito.mock(VirtualMachineTemplate.class)); +// VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); +// when(templateVO.getId()).thenReturn(1L); +// when(templateDao.findById(Mockito.anyLong())).thenReturn(templateVO); +// kubernetesVersionService.addKubernetesSupportedVersion(cmd); + } + + @Test//(expected = CloudRuntimeException.class) + public void deleteKubernetesSupportedVersionExistingClustersTest() { +// DeleteKubernetesSupportedVersionCmd cmd = Mockito.mock(DeleteKubernetesSupportedVersionCmd.class); +// AccountVO account = new AccountVO("admin", 1L, "", Account.ACCOUNT_TYPE_ADMIN, "uuid"); +// UserVO user = new UserVO(1, "adminuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); +// CallContext.register(user, account); +// when(cmd.isDeleteIso()).thenReturn(true); +// when(kubernetesSupportedVersionDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(KubernetesSupportedVersionVO.class)); +// List clusters = new ArrayList<>(); +// clusters.add(Mockito.mock(KubernetesClusterVO.class)); +// when(kubernetesClusterDao.listAllByKubernetesVersion(Mockito.anyLong())).thenReturn(clusters); +// kubernetesVersionService.deleteKubernetesSupportedVersion(cmd); + } + + @Test + public void deleteKubernetesSupportedVersionTest() { +// DeleteKubernetesSupportedVersionCmd cmd = Mockito.mock(DeleteKubernetesSupportedVersionCmd.class); +// AccountVO account = new AccountVO("admin", 1L, "", Account.ACCOUNT_TYPE_ADMIN, "uuid"); +// UserVO user = new UserVO(1, "adminuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); +// CallContext.register(user, account); +// when(cmd.isDeleteIso()).thenReturn(true); +// when(kubernetesSupportedVersionDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(KubernetesSupportedVersionVO.class)); +// List clusters = new ArrayList<>(); +// when(kubernetesClusterDao.listAllByKubernetesVersion(Mockito.anyLong())).thenReturn(clusters); +// when(templateDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(VMTemplateVO.class)); +// PowerMockito.mockStatic(ComponentContext.class); +// when(ComponentContext.inject(Mockito.any(DeleteIsoCmd.class))).thenReturn(new DeleteIsoCmd()); +// when(templateService.deleteIso(Mockito.any(DeleteIsoCmd.class))).thenReturn(true); +// when(kubernetesClusterDao.remove(Mockito.anyLong())).thenReturn(true); +// kubernetesVersionService.deleteKubernetesSupportedVersion(cmd); + } +} \ No newline at end of file From f9a558ad2a156b5487c9bd4cdd289eb5f2b794f8 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 8 Jan 2020 10:52:15 +0530 Subject: [PATCH 042/134] fixes Signed-off-by: Abhishek Kumar --- .../KubernetesVersionManagerImpl.java | 2 +- .../KubernetesVersionServiceTest.java | 231 ++++++++++-------- 2 files changed, 134 insertions(+), 99 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java index 0d69af476615..49dfa534ea8a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java @@ -244,7 +244,7 @@ private void deleteKubernetesVersionIso(long templateId) throws IllegalAccessExc @Override public ListResponse listKubernetesSupportedVersions(final ListKubernetesSupportedVersionsCmd cmd) { - if (!KubernetesClusterService.isKubernetesServiceEnabled()) { + if (!KubernetesClusterService.KubernetesServiceEnabled.value()) { throw new CloudRuntimeException("Kubernetes Service plugin is disabled"); } final Long versionId = cmd.getId(); diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetesversion/KubernetesVersionServiceTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetesversion/KubernetesVersionServiceTest.java index 395998c2d8fd..1d268072ddc0 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetesversion/KubernetesVersionServiceTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetesversion/KubernetesVersionServiceTest.java @@ -1,26 +1,56 @@ package com.cloud.kubernetesversion; +import static org.mockito.Mockito.when; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import org.apache.cloudstack.api.command.admin.kubernetesversion.AddKubernetesSupportedVersionCmd; +import org.apache.cloudstack.api.command.admin.kubernetesversion.DeleteKubernetesSupportedVersionCmd; +import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; +import org.apache.cloudstack.api.command.user.iso.RegisterIsoCmd; +import org.apache.cloudstack.api.command.user.kubernetesversion.ListKubernetesSupportedVersionsCmd; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; +import org.apache.cloudstack.framework.config.ConfigKey; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; import com.cloud.api.query.dao.TemplateJoinDao; +import com.cloud.api.query.vo.TemplateJoinVO; +import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.ResourceAllocationException; import com.cloud.kubernetescluster.KubernetesClusterService; +import com.cloud.kubernetescluster.KubernetesClusterVO; import com.cloud.kubernetescluster.dao.KubernetesClusterDao; import com.cloud.kubernetesversion.dao.KubernetesSupportedVersionDao; +import com.cloud.storage.Storage; +import com.cloud.storage.VMTemplateVO; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.template.TemplateApiService; +import com.cloud.template.VirtualMachineTemplate; +import com.cloud.user.Account; +import com.cloud.user.AccountVO; +import com.cloud.user.User; +import com.cloud.user.UserVO; import com.cloud.utils.component.ComponentContext; +import com.cloud.utils.exception.CloudRuntimeException; @RunWith(PowerMockRunner.class) -@PrepareForTest({KubernetesClusterService.class, ComponentContext.class}) +@PrepareForTest({ComponentContext.class}) public class KubernetesVersionServiceTest { @InjectMocks @@ -39,26 +69,31 @@ public class KubernetesVersionServiceTest { @Mock private TemplateApiService templateService; + private void overrideDefaultConfigValue(final ConfigKey configKey, final String name, final Object o) throws IllegalAccessException, NoSuchFieldException { + Field f = ConfigKey.class.getDeclaredField(name); + f.setAccessible(true); + f.set(configKey, o); + } + @Before public void setUp() throws Exception { -// MockitoAnnotations.initMocks(this); -// -// PowerMockito.mockStatic(KubernetesClusterService.class); -// PowerMockito.when(KubernetesClusterService.isKubernetesServiceEnabled()).thenReturn(true); -// -// DataCenterVO zone = Mockito.mock(DataCenterVO.class); -// when(zone.getId()).thenReturn(1L); -// when(dataCenterDao.findById(Mockito.anyLong())).thenReturn(zone); -// -// TemplateJoinVO templateJoinVO = Mockito.mock(TemplateJoinVO.class); -// when(templateJoinVO.getId()).thenReturn(1L); -// when(templateJoinVO.getUrl()).thenReturn("https://download.cloudstack.com"); -// when(templateJoinVO.getState()).thenReturn(ObjectInDataStoreStateMachine.State.Ready); -// when(templateJoinDao.findById(Mockito.anyLong())).thenReturn(templateJoinVO); -// -// KubernetesSupportedVersionVO versionVO = Mockito.mock(KubernetesSupportedVersionVO.class); -// when(versionVO.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); -// when(kubernetesSupportedVersionDao.persist(Mockito.any(KubernetesSupportedVersionVO.class))).thenReturn(versionVO); + MockitoAnnotations.initMocks(this); + + overrideDefaultConfigValue(KubernetesClusterService.KubernetesServiceEnabled, "_defaultValue", "true"); + + DataCenterVO zone = Mockito.mock(DataCenterVO.class); + when(zone.getId()).thenReturn(1L); + when(dataCenterDao.findById(Mockito.anyLong())).thenReturn(zone); + + TemplateJoinVO templateJoinVO = Mockito.mock(TemplateJoinVO.class); + when(templateJoinVO.getId()).thenReturn(1L); + when(templateJoinVO.getUrl()).thenReturn("https://download.cloudstack.com"); + when(templateJoinVO.getState()).thenReturn(ObjectInDataStoreStateMachine.State.Ready); + when(templateJoinDao.findById(Mockito.anyLong())).thenReturn(templateJoinVO); + + KubernetesSupportedVersionVO versionVO = Mockito.mock(KubernetesSupportedVersionVO.class); + when(versionVO.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); + when(kubernetesSupportedVersionDao.persist(Mockito.any(KubernetesSupportedVersionVO.class))).thenReturn(versionVO); } @After @@ -67,105 +102,105 @@ public void tearDown() throws Exception { @Test public void listKubernetesSupportedVersionsTest() { -// ListKubernetesSupportedVersionsCmd cmd = Mockito.mock(ListKubernetesSupportedVersionsCmd.class); -// List versionVOs = new ArrayList<>(); -// KubernetesSupportedVersionVO versionVO = Mockito.mock(KubernetesSupportedVersionVO.class); -// when(versionVO.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); -// versionVOs.add(versionVO); -// when(kubernetesSupportedVersionDao.listAll()).thenReturn(versionVOs); -// when(kubernetesSupportedVersionDao.listAllInZone(Mockito.anyLong())).thenReturn(versionVOs); -// when(kubernetesSupportedVersionDao.findById(Mockito.anyLong())).thenReturn(versionVO); -// kubernetesVersionService.listKubernetesSupportedVersions(cmd); + ListKubernetesSupportedVersionsCmd cmd = Mockito.mock(ListKubernetesSupportedVersionsCmd.class); + List versionVOs = new ArrayList<>(); + KubernetesSupportedVersionVO versionVO = Mockito.mock(KubernetesSupportedVersionVO.class); + when(versionVO.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); + versionVOs.add(versionVO); + when(kubernetesSupportedVersionDao.listAll()).thenReturn(versionVOs); + when(kubernetesSupportedVersionDao.listAllInZone(Mockito.anyLong())).thenReturn(versionVOs); + when(kubernetesSupportedVersionDao.findById(Mockito.anyLong())).thenReturn(versionVO); + kubernetesVersionService.listKubernetesSupportedVersions(cmd); } - @Test//(expected = InvalidParameterValueException.class) + @Test(expected = InvalidParameterValueException.class) public void addKubernetesSupportedVersionLowerUnsupportedTest() { -// AddKubernetesSupportedVersionCmd cmd = Mockito.mock(AddKubernetesSupportedVersionCmd.class); -// AccountVO account = new AccountVO("admin", 1L, "", Account.ACCOUNT_TYPE_ADMIN, "uuid"); -// UserVO user = new UserVO(1, "adminuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); -// CallContext.register(user, account); -// when(cmd.getSemanticVersion()).thenReturn("1.1.1"); -// kubernetesVersionService.addKubernetesSupportedVersion(cmd); + AddKubernetesSupportedVersionCmd cmd = Mockito.mock(AddKubernetesSupportedVersionCmd.class); + AccountVO account = new AccountVO("admin", 1L, "", Account.ACCOUNT_TYPE_ADMIN, "uuid"); + UserVO user = new UserVO(1, "adminuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); + CallContext.register(user, account); + when(cmd.getSemanticVersion()).thenReturn("1.1.1"); + kubernetesVersionService.addKubernetesSupportedVersion(cmd); } - @Test//(expected = InvalidParameterValueException.class) + @Test(expected = InvalidParameterValueException.class) public void addKubernetesSupportedVersionIsoIdUrlTest() { -// AddKubernetesSupportedVersionCmd cmd = Mockito.mock(AddKubernetesSupportedVersionCmd.class); -// AccountVO account = new AccountVO("admin", 1L, "", Account.ACCOUNT_TYPE_ADMIN, "uuid"); -// UserVO user = new UserVO(1, "adminuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); -// when(cmd.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); -// CallContext.register(user, account); -// when(cmd.getIsoId()).thenReturn(1L); -// when(cmd.getUrl()).thenReturn("url"); -// kubernetesVersionService.addKubernetesSupportedVersion(cmd); + AddKubernetesSupportedVersionCmd cmd = Mockito.mock(AddKubernetesSupportedVersionCmd.class); + AccountVO account = new AccountVO("admin", 1L, "", Account.ACCOUNT_TYPE_ADMIN, "uuid"); + UserVO user = new UserVO(1, "adminuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); + when(cmd.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); + CallContext.register(user, account); + when(cmd.getIsoId()).thenReturn(1L); + when(cmd.getUrl()).thenReturn("url"); + kubernetesVersionService.addKubernetesSupportedVersion(cmd); } @Test public void addKubernetesSupportedVersionIsoIdTest() { -// AddKubernetesSupportedVersionCmd cmd = Mockito.mock(AddKubernetesSupportedVersionCmd.class); -// AccountVO account = new AccountVO("admin", 1L, "", Account.ACCOUNT_TYPE_ADMIN, "uuid"); -// UserVO user = new UserVO(1, "adminuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); -// CallContext.register(user, account); -// when(cmd.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); -// when(cmd.getIsoId()).thenReturn(1L); -// when(cmd.getUrl()).thenReturn(null); -// VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); -// when(templateVO.getId()).thenReturn(1L); -// when(templateVO.getFormat()).thenReturn(Storage.ImageFormat.ISO); -// when(templateVO.isPublicTemplate()).thenReturn(true); -// when(templateVO.isCrossZones()).thenReturn(true); -// when(templateDao.findById(Mockito.anyLong())).thenReturn(templateVO); -// kubernetesVersionService.addKubernetesSupportedVersion(cmd); + AddKubernetesSupportedVersionCmd cmd = Mockito.mock(AddKubernetesSupportedVersionCmd.class); + AccountVO account = new AccountVO("admin", 1L, "", Account.ACCOUNT_TYPE_ADMIN, "uuid"); + UserVO user = new UserVO(1, "adminuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); + CallContext.register(user, account); + when(cmd.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); + when(cmd.getIsoId()).thenReturn(1L); + when(cmd.getUrl()).thenReturn(null); + VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); + when(templateVO.getId()).thenReturn(1L); + when(templateVO.getFormat()).thenReturn(Storage.ImageFormat.ISO); + when(templateVO.isPublicTemplate()).thenReturn(true); + when(templateVO.isCrossZones()).thenReturn(true); + when(templateDao.findById(Mockito.anyLong())).thenReturn(templateVO); + kubernetesVersionService.addKubernetesSupportedVersion(cmd); } @Test public void addKubernetesSupportedVersionIsoUrlTest() throws ResourceAllocationException, NoSuchFieldException { -// AddKubernetesSupportedVersionCmd cmd = Mockito.mock(AddKubernetesSupportedVersionCmd.class); -// AccountVO account = new AccountVO("admin", 1L, "", Account.ACCOUNT_TYPE_ADMIN, "uuid"); -// UserVO user = new UserVO(1, "adminuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); -// CallContext.register(user, account); -// when(cmd.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); -// when(cmd.getIsoId()).thenReturn(null); -// when(cmd.getUrl()).thenReturn("https://download.cloudstack.com"); -// when(cmd.getChecksum()).thenReturn(null); -// PowerMockito.mockStatic(ComponentContext.class); -// when(ComponentContext.inject(Mockito.any(RegisterIsoCmd.class))).thenReturn(new RegisterIsoCmd()); -// when(templateService.registerIso(Mockito.any(RegisterIsoCmd.class))).thenReturn(Mockito.mock(VirtualMachineTemplate.class)); -// VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); -// when(templateVO.getId()).thenReturn(1L); -// when(templateDao.findById(Mockito.anyLong())).thenReturn(templateVO); -// kubernetesVersionService.addKubernetesSupportedVersion(cmd); + AddKubernetesSupportedVersionCmd cmd = Mockito.mock(AddKubernetesSupportedVersionCmd.class); + AccountVO account = new AccountVO("admin", 1L, "", Account.ACCOUNT_TYPE_ADMIN, "uuid"); + UserVO user = new UserVO(1, "adminuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); + CallContext.register(user, account); + when(cmd.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); + when(cmd.getIsoId()).thenReturn(null); + when(cmd.getUrl()).thenReturn("https://download.cloudstack.com"); + when(cmd.getChecksum()).thenReturn(null); + PowerMockito.mockStatic(ComponentContext.class); + when(ComponentContext.inject(Mockito.any(RegisterIsoCmd.class))).thenReturn(new RegisterIsoCmd()); + when(templateService.registerIso(Mockito.any(RegisterIsoCmd.class))).thenReturn(Mockito.mock(VirtualMachineTemplate.class)); + VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); + when(templateVO.getId()).thenReturn(1L); + when(templateDao.findById(Mockito.anyLong())).thenReturn(templateVO); + kubernetesVersionService.addKubernetesSupportedVersion(cmd); } - @Test//(expected = CloudRuntimeException.class) + @Test(expected = CloudRuntimeException.class) public void deleteKubernetesSupportedVersionExistingClustersTest() { -// DeleteKubernetesSupportedVersionCmd cmd = Mockito.mock(DeleteKubernetesSupportedVersionCmd.class); -// AccountVO account = new AccountVO("admin", 1L, "", Account.ACCOUNT_TYPE_ADMIN, "uuid"); -// UserVO user = new UserVO(1, "adminuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); -// CallContext.register(user, account); -// when(cmd.isDeleteIso()).thenReturn(true); -// when(kubernetesSupportedVersionDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(KubernetesSupportedVersionVO.class)); -// List clusters = new ArrayList<>(); -// clusters.add(Mockito.mock(KubernetesClusterVO.class)); -// when(kubernetesClusterDao.listAllByKubernetesVersion(Mockito.anyLong())).thenReturn(clusters); -// kubernetesVersionService.deleteKubernetesSupportedVersion(cmd); + DeleteKubernetesSupportedVersionCmd cmd = Mockito.mock(DeleteKubernetesSupportedVersionCmd.class); + AccountVO account = new AccountVO("admin", 1L, "", Account.ACCOUNT_TYPE_ADMIN, "uuid"); + UserVO user = new UserVO(1, "adminuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); + CallContext.register(user, account); + when(cmd.isDeleteIso()).thenReturn(true); + when(kubernetesSupportedVersionDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(KubernetesSupportedVersionVO.class)); + List clusters = new ArrayList<>(); + clusters.add(Mockito.mock(KubernetesClusterVO.class)); + when(kubernetesClusterDao.listAllByKubernetesVersion(Mockito.anyLong())).thenReturn(clusters); + kubernetesVersionService.deleteKubernetesSupportedVersion(cmd); } @Test public void deleteKubernetesSupportedVersionTest() { -// DeleteKubernetesSupportedVersionCmd cmd = Mockito.mock(DeleteKubernetesSupportedVersionCmd.class); -// AccountVO account = new AccountVO("admin", 1L, "", Account.ACCOUNT_TYPE_ADMIN, "uuid"); -// UserVO user = new UserVO(1, "adminuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); -// CallContext.register(user, account); -// when(cmd.isDeleteIso()).thenReturn(true); -// when(kubernetesSupportedVersionDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(KubernetesSupportedVersionVO.class)); -// List clusters = new ArrayList<>(); -// when(kubernetesClusterDao.listAllByKubernetesVersion(Mockito.anyLong())).thenReturn(clusters); -// when(templateDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(VMTemplateVO.class)); -// PowerMockito.mockStatic(ComponentContext.class); -// when(ComponentContext.inject(Mockito.any(DeleteIsoCmd.class))).thenReturn(new DeleteIsoCmd()); -// when(templateService.deleteIso(Mockito.any(DeleteIsoCmd.class))).thenReturn(true); -// when(kubernetesClusterDao.remove(Mockito.anyLong())).thenReturn(true); -// kubernetesVersionService.deleteKubernetesSupportedVersion(cmd); + DeleteKubernetesSupportedVersionCmd cmd = Mockito.mock(DeleteKubernetesSupportedVersionCmd.class); + AccountVO account = new AccountVO("admin", 1L, "", Account.ACCOUNT_TYPE_ADMIN, "uuid"); + UserVO user = new UserVO(1, "adminuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); + CallContext.register(user, account); + when(cmd.isDeleteIso()).thenReturn(true); + when(kubernetesSupportedVersionDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(KubernetesSupportedVersionVO.class)); + List clusters = new ArrayList<>(); + when(kubernetesClusterDao.listAllByKubernetesVersion(Mockito.anyLong())).thenReturn(clusters); + when(templateDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(VMTemplateVO.class)); + PowerMockito.mockStatic(ComponentContext.class); + when(ComponentContext.inject(Mockito.any(DeleteIsoCmd.class))).thenReturn(new DeleteIsoCmd()); + when(templateService.deleteIso(Mockito.any(DeleteIsoCmd.class))).thenReturn(true); + when(kubernetesClusterDao.remove(Mockito.anyLong())).thenReturn(true); + kubernetesVersionService.deleteKubernetesSupportedVersion(cmd); } } \ No newline at end of file From 9602dddef31f039bfbf296ac97d429b674435abc Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 8 Jan 2020 10:53:04 +0530 Subject: [PATCH 043/134] removed unused method Signed-off-by: Abhishek Kumar --- .../com/cloud/kubernetescluster/KubernetesClusterService.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterService.java index a29c73308365..5081f6318725 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterService.java @@ -51,10 +51,6 @@ public interface KubernetesClusterService extends PluggableService, Configurable "Name of the network offering that will be used to create isolated network in which Kubernetes cluster VMs will be launched", false); - static boolean isKubernetesServiceEnabled() { - return KubernetesServiceEnabled.value(); - } - KubernetesCluster findById(final Long id); KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) throws InsufficientCapacityException, From 9f16bec3b6aef6df35d1a06aef8fe9b9d4a98d1e Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 8 Jan 2020 11:04:04 +0530 Subject: [PATCH 044/134] add, delete version implements AdminCmd Signed-off-by: Abhishek Kumar --- .../kubernetesversion/AddKubernetesSupportedVersionCmd.java | 4 ++-- .../DeleteKubernetesSupportedVersionCmd.java | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/AddKubernetesSupportedVersionCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/AddKubernetesSupportedVersionCmd.java index 1568d00851c3..d2c138810a99 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/AddKubernetesSupportedVersionCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/AddKubernetesSupportedVersionCmd.java @@ -26,7 +26,7 @@ import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ResponseObject; import org.apache.cloudstack.api.ServerApiException; -import org.apache.cloudstack.api.command.user.UserCmd; +import org.apache.cloudstack.api.command.admin.AdminCmd; import org.apache.cloudstack.api.response.KubernetesSupportedVersionResponse; import org.apache.cloudstack.api.response.TemplateResponse; import org.apache.cloudstack.api.response.ZoneResponse; @@ -49,7 +49,7 @@ responseView = ResponseObject.ResponseView.Restricted, entityType = {KubernetesSupportedVersion.class}, authorized = {RoleType.Admin}) -public class AddKubernetesSupportedVersionCmd extends BaseCmd implements UserCmd { +public class AddKubernetesSupportedVersionCmd extends BaseCmd implements AdminCmd { public static final Logger LOGGER = Logger.getLogger(AddKubernetesSupportedVersionCmd.class.getName()); public static final String APINAME = "addKubernetesSupportedVersion"; diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/DeleteKubernetesSupportedVersionCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/DeleteKubernetesSupportedVersionCmd.java index 90f00dff9645..1ae460a6d5ab 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/DeleteKubernetesSupportedVersionCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/DeleteKubernetesSupportedVersionCmd.java @@ -26,6 +26,7 @@ import org.apache.cloudstack.api.BaseAsyncCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.admin.AdminCmd; import org.apache.cloudstack.api.response.KubernetesSupportedVersionResponse; import org.apache.cloudstack.api.response.SuccessResponse; import org.apache.cloudstack.context.CallContext; @@ -45,7 +46,7 @@ responseObject = SuccessResponse.class, entityType = {KubernetesSupportedVersion.class}, authorized = {RoleType.Admin}) -public class DeleteKubernetesSupportedVersionCmd extends BaseAsyncCmd { +public class DeleteKubernetesSupportedVersionCmd extends BaseAsyncCmd implements AdminCmd { public static final Logger LOGGER = Logger.getLogger(DeleteKubernetesSupportedVersionCmd.class.getName()); public static final String APINAME = "deleteKubernetesSupportedVersion"; From 291e2b09ea012b17f9d99fcafb392341bd990c1a Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 8 Jan 2020 12:28:33 +0530 Subject: [PATCH 045/134] javadoc change Signed-off-by: Abhishek Kumar --- .../com/cloud/kubernetescluster/KubernetesClusterVmMap.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVmMap.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVmMap.java index f3bddecae75e..6a82fddffdf4 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVmMap.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVmMap.java @@ -17,7 +17,8 @@ package com.cloud.kubernetescluster; /** - * VirtualMachine describes the properties held by a virtual machine + * KubernetesClusterVmMap will store a map of ID of KubernetesCuster + * and ID of its VirtualMachine */ public interface KubernetesClusterVmMap { long getId(); From 8383439a162b0c9d66a2b87e37e4bd75c3ad47eb Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 8 Jan 2020 13:47:11 +0530 Subject: [PATCH 046/134] fixed exception message Signed-off-by: Abhishek Kumar --- .../cloud/kubernetescluster/KubernetesClusterManagerImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java index f33bb373ccad..bb361c74d981 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java @@ -2417,8 +2417,8 @@ private void validateKubernetesClusterUpgradeParameters(UpgradeKubernetesCluster } if (KubernetesVersionManagerImpl.compareSemanticVersions( upgradeVersion.getSemanticVersion(), clusterVersion.getSemanticVersion()) <= 0) { - throw new InvalidParameterValueException(String.format("Invalid Kubernetes version associated with cluster ID: %s", - kubernetesCluster.getUuid())); + throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s can not be upgraded from %s to %s", + kubernetesCluster.getUuid(), clusterVersion.getSemanticVersion(), upgradeVersion.getSemanticVersion())); } // Check upgradeVersion is either patch upgrade or immediate minor upgrade try { From f11ff1cc2b95e0ffc3c2ba0cdc5a5bffe36b4478 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 8 Jan 2020 13:47:50 +0530 Subject: [PATCH 047/134] added check for CKS enabled Signed-off-by: Abhishek Kumar --- .../test_kubernetes_supported_versions.py | 52 ++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_supported_versions.py b/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_supported_versions.py index 595aafdda18f..64eba0048701 100644 --- a/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_supported_versions.py +++ b/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_supported_versions.py @@ -18,14 +18,17 @@ #Import Local Modules from marvin.cloudstackTestCase import cloudstackTestCase, unittest -from marvin.cloudstackAPI import (listKubernetesSupportedVersions, +from marvin.cloudstackAPI import (listInfrastructure, + listKubernetesSupportedVersions, addKubernetesSupportedVersion, deleteKubernetesSupportedVersion) from marvin.cloudstackException import CloudstackAPIException from marvin.codes import FAILED +from marvin.lib.base import Configurations from marvin.lib.utils import (cleanup_resources, random_gen) from marvin.lib.common import get_zone +from marvin.sshClient import SshClient from nose.plugins.attrib import attr import time @@ -40,18 +43,65 @@ def setUpClass(cls): cls.apiclient = testClient.getApiClient() cls.services = testClient.getParsedTestDataConfig() cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests()) + cls.mgtSvrDetails = cls.config.__dict__["mgtSvr"][0].__dict__ cls.kubernetes_version_iso_url = 'http://172.20.0.1/files/setup-1.16.3.iso' + + cls.initial_configuration_cks_enabled = Configurations.list(cls.apiclient, + name="cloud.kubernetes.service.enabled")[0].value + if cls.initial_configuration_cks_enabled == False: + Configurations.update(cls.apiclient, + "cloud.kubernetes.service.enabled", + "true") + cls.restartServer() + cls._cleanup = [] return @classmethod def tearDownClass(cls): try: + # Restore CKS enabled + if cls.initial_configuration_cks_enabled == False: + Configurations.update(cls.apiclient, + "cloud.kubernetes.service.enabled", + "false") + cls.restartServer() cleanup_resources(cls.apiclient, cls._cleanup) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) return + @classmethod + def restartServer(cls): + """Restart management server""" + + sshClient = SshClient( + cls.mgtSvrDetails["mgtSvrIp"], + 22, + cls.mgtSvrDetails["user"], + cls.mgtSvrDetails["passwd"] + ) + command = "service cloudstack-management stop" + sshClient.execute(command) + + command = "service cloudstack-management start" + sshClient.execute(command) + + #Waits for management to come up in 5 mins, when it's up it will continue + timeout = time.time() + 300 + while time.time() < timeout: + if cls.isManagementUp() is True: return + time.sleep(5) + return cls.fail("Management server did not come up, failing") + + @classmethod + def isManagementUp(cls): + try: + cls.apiclient.listInfrastructure(listInfrastructure.listInfrastructureCmd()) + return True + except Exception: + return False + def setUp(self): self.services = self.testClient.getParsedTestDataConfig() self.apiclient = self.testClient.getApiClient() From 7632e5f035b5e0ac9ab53e72f86e76208ad2fcee Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 8 Jan 2020 13:49:47 +0530 Subject: [PATCH 048/134] added integration test for Kubernetes cluster Signed-off-by: Abhishek Kumar --- .../smoke/test_kubernetes_clusters.py | 632 ++++++++++++++++++ 1 file changed, 632 insertions(+) create mode 100644 plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_clusters.py diff --git a/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_clusters.py b/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_clusters.py new file mode 100644 index 000000000000..cd49f9f82a2a --- /dev/null +++ b/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_clusters.py @@ -0,0 +1,632 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" Tests for Kubernetes supported version """ + +#Import Local Modules +from marvin.cloudstackTestCase import cloudstackTestCase, unittest +from marvin.cloudstackAPI import (listInfrastructure, + listKubernetesSupportedVersions, + addKubernetesSupportedVersion, + deleteKubernetesSupportedVersion, + createKubernetesCluster, + deleteKubernetesCluster, + upgradeKubernetesCluster, + scaleKubernetesCluster) +from marvin.cloudstackException import CloudstackAPIException +from marvin.codes import FAILED +from marvin.lib.base import (Template, + ServiceOffering, + Configurations) +from marvin.lib.utils import (cleanup_resources, + random_gen) +from marvin.lib.common import (get_zone) +from marvin.sshClient import SshClient +from nose.plugins.attrib import attr + +import time + +_multiprocess_shared_ = True + +class TestKubernetesCluster(cloudstackTestCase): + + @classmethod + def setUpClass(cls): + testClient = super(TestKubernetesCluster, cls).getClsTestClient() + cls.apiclient = testClient.getApiClient() + cls.services = testClient.getParsedTestDataConfig() + cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests()) + cls.hypervisor = cls.testClient.getHypervisorInfo() + + cls.initial_configuration_cks_enabled = Configurations.list(cls.apiclient, + name="cloud.kubernetes.service.enabled")[0].value + if cls.initial_configuration_cks_enabled == False: + Configurations.update(cls.apiclient, + "cloud.kubernetes.service.enabled", + "true") + cls.restartServer() + + cls.kubernetes_version_ids = [] + cls.kuberetes_version_1 = cls.addKubernetesSupportedVersion('1.14.9', 'http://172.20.0.1/files/setup-1.14.9.iso') + cls.kubernetes_version_ids.append(cls.kuberetes_version_1.id) + cls.kuberetes_version_2 = cls.addKubernetesSupportedVersion('1.15.0', 'http://172.20.0.1/files/setup-1.15.0.iso') + cls.kubernetes_version_ids.append(cls.kuberetes_version_2.id) + cls.kuberetes_version_3 = cls.addKubernetesSupportedVersion('1.16.3', 'http://172.20.0.1/files/setup-1.16.3.iso') + cls.kubernetes_version_ids.append(cls.kuberetes_version_3.id) + + cks_offering_data = { + "name": "CKS Instance", + "displaytext": "CKS Instance", + "cpunumber": 2, + "cpuspeed": 1000, + "memory": 2048, + } + cls.cks_service_offering = ServiceOffering.create( + cls.apiclient, + cks_offering_data + ) + + cks_template_data = { + "name": "Kubernetes-Service-Template", + "displaytext": "Kubernetes-Service-Template", + "format": "qcow2", + "hypervisor": "kvm", + "ostype": "CoreOS", + "url": "http://172.20.0.1/files/coreos_production_cloudstack_image-kvm.qcow2.bz2", + "requireshvm": "True", + "ispublic": "True", + "isextractable": "True" + } + if cls.hypervisor.lower() == "vmware": + cks_template_data["url"] = "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-vmware.ova" + elif cls.hypervisor.lower() == "xenserver": + cks_template_data["url"] = "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-xen.vhd.bz2" + cls.cks_template = Template.register( + cls.apiclient, + cks_template_data, + zoneid=cls.zone.id, + hypervisor=cls.hypervisor + ) + cls.debug("Waiting for CKS template with ID %s to be ready" % cls.cks_template.id) + cls.waitForTemplateReadyState(cls.cks_template.id) + + cls.initial_configuration_cks_template_name = Configurations.list(cls.apiclient, + name="cloud.kubernetes.cluster.template.name")[0].value + Configurations.update(cls.apiclient, + "cloud.kubernetes.cluster.template.name", + cls.cks_template.name) + + cls._cleanup = [ + cls.cks_service_offering, + cls.cks_template + ] + return + + @classmethod + def tearDownClass(cls): + version_delete_failed = False + # Delete added Kubernetes supported version + for version_id in cls.kubernetes_version_ids: + try: + cls.deleteKubernetesSupportedVersion(version_id) + except Exception as e: + version_delete_failed = True + cls.debug("Error: Exception during cleanup for added Kubernetes supported versions: %s" % e) + try: + # Restore original CKS template + Configurations.update(cls.apiclient, + "cloud.kubernetes.cluster.template.name", + cls.initial_configuration_cks_template_name) + # Restore CKS enabled + if cls.initial_configuration_cks_enabled == False: + Configurations.update(cls.apiclient, + "cloud.kubernetes.service.enabled", + "false") + cls.restartServer() + + cleanup_resources(cls.apiclient, cls._cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + if version_delete_failed == True: + raise Exception("Warning: Exception during cleanup, unable to delete Kubernetes supported versions") + return + + @classmethod + def restartServer(cls): + """Restart management server""" + + sshClient = SshClient( + cls.mgtSvrDetails["mgtSvrIp"], + 22, + cls.mgtSvrDetails["user"], + cls.mgtSvrDetails["passwd"] + ) + command = "service cloudstack-management stop" + sshClient.execute(command) + + command = "service cloudstack-management start" + sshClient.execute(command) + + #Waits for management to come up in 5 mins, when it's up it will continue + timeout = time.time() + 300 + while time.time() < timeout: + if cls.isManagementUp() is True: return + time.sleep(5) + return cls.fail("Management server did not come up, failing") + + @classmethod + def isManagementUp(cls): + try: + cls.apiclient.listInfrastructure(listInfrastructure.listInfrastructureCmd()) + return True + except Exception: + return False + + @classmethod + def waitForTemplateReadyState(cls, template_id, retries=15, interval=15): + """Check if template download will finish""" + while retries > -1: + time.sleep(interval) + template_response = Template.list( + cls.apiclient, + id=template_id, + zoneid=cls.zone.id, + templatefilter='self' + ) + + if isinstance(template_response, list): + template = template_response[0] + if not hasattr(template, 'status') or not template or not template.status: + retries = retries - 1 + continue + if 'Failed' == template.status: + raise Exception("Failed to download template: status - %s" % template.status) + elif template.status == 'Download Complete' and template.isready: + return + retries = retries - 1 + raise Exception("Template download timed out") + + @classmethod + def waitForKubernetesSupportedVersionIsoReadyState(cls, version_id, retries=15, interval=15): + """Check if Kubernetes supported version ISO is in Ready state""" + + while retries > -1: + time.sleep(interval) + list_versions_response = cls.listKubernetesSupportedVersion(version_id) + if not hasattr(list_versions_response, 'isostate') or not list_versions_response or not list_versions_response.isostate: + retries = retries - 1 + continue + if 'Creating' == list_versions_response.isostate: + retries = retries - 1 + elif 'Ready' == list_versions_response.isostate: + return + elif 'Failed' == list_versions_response.isostate: + raise Exception( "Failed to download template: status - %s" % template.status) + else: + raise Exception( + "Failed to download Kubernetes supported version ISO: status - %s" % + list_versions_response.isostate) + raise Exception("Kubernetes supported version Ready state timed out") + + @classmethod + def listKubernetesSupportedVersion(cls, version_id): + listKubernetesSupportedVersionsCmd = listKubernetesSupportedVersions.listKubernetesSupportedVersionsCmd() + listKubernetesSupportedVersionsCmd.id = version_id + versionResponse = cls.apiclient.listKubernetesSupportedVersions(listKubernetesSupportedVersionsCmd) + return versionResponse[0] + + @classmethod + def addKubernetesSupportedVersion(cls, semantic_version, iso_url): + addKubernetesSupportedVersionCmd = addKubernetesSupportedVersion.addKubernetesSupportedVersionCmd() + addKubernetesSupportedVersionCmd.semanticversion = semantic_version + addKubernetesSupportedVersionCmd.url = iso_url + kubernetes_version = cls.apiclient.addKubernetesSupportedVersion(addKubernetesSupportedVersionCmd) + cls.debug("Waiting for Kubernetes version with ID %s to be ready" % kubernetes_version.id) + cls.waitForKubernetesSupportedVersionIsoReadyState(kubernetes_version.id) + kubernetes_version = cls.listKubernetesSupportedVersion(kubernetes_version.id) + return kubernetes_version + + @classmethod + def deleteKubernetesSupportedVersion(cls, version_id): + deleteKubernetesSupportedVersionCmd = deleteKubernetesSupportedVersion.deleteKubernetesSupportedVersionCmd() + deleteKubernetesSupportedVersionCmd.id = version_id + deleteKubernetesSupportedVersionCmd.deleteiso = True + cls.apiclient.deleteKubernetesSupportedVersion(deleteKubernetesSupportedVersionCmd) + + def setUp(self): + self.services = self.testClient.getParsedTestDataConfig() + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + self.cleanup = [] + return + + def tearDown(self): + try: + #Clean up, terminate the created templates + cleanup_resources(self.apiclient, self.cleanup) + + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + def test_01_deploy_kubernetes_cluster(self): + """Test to deploy a new Kubernetes cluster + + # Validate the following: + # 1. createKubernetesCluster should return valid info for new cluster + # 2. The Cloud Database contains the valid information + """ + if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: + self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) + name = 'testcluster-' + random_gen() + self.debug("Creating for Kubernetes cluster with name %s" % name) + + cluster_response = self.createKubernetesCluster(name, self.kuberetes_version_2.id) + + self.verifyKubernetesCluster(cluster_response, name, self.kuberetes_version_2.id) + + self.debug("Kubernetes cluster with ID: %s successfully deployed, now deleting it" % cluster_response.id) + + self.deleteAndVerifyKubernetesCluster(cluster_response.id) + + self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) + + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + def test_02_deploy_kubernetes_ha_cluster(self): + """Test to deploy a new Kubernetes cluster + + # Validate the following: + # 1. createKubernetesCluster should return valid info for new cluster + # 2. The Cloud Database contains the valid information + """ + if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: + self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) + name = 'testcluster-' + random_gen() + self.debug("Creating for Kubernetes cluster with name %s" % name) + + cluster_response = self.createKubernetesCluster(name, self.kuberetes_version_3.id, 1, 2) + + self.verifyKubernetesCluster(cluster_response, name, self.kuberetes_version_3.id, 1, 2) + + self.debug("Kubernetes cluster with ID: %s successfully deployed, now deleting it" % cluster_response.id) + + self.deleteAndVerifyKubernetesCluster(cluster_response.id) + + self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) + + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + def test_03_deploy_invalid_kubernetes_ha_cluster(self): + """Test to deploy a new Kubernetes cluster + + # Validate the following: + # 1. createKubernetesCluster should return valid info for new cluster + # 2. The Cloud Database contains the valid information + """ + if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: + self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) + name = 'testcluster-' + random_gen() + self.debug("Creating for Kubernetes cluster with name %s" % name) + + try: + cluster_response = self.createKubernetesCluster(name, self.kuberetes_version_2.id, 1, 2) + self.debug("Invslid CKS Kubernetes HA cluster deployed with ID: %s. Deleting it and failing test." % cluster_response.id) + deleteAndVerifyKubernetesCluster(cluster_response.id) + self.fail("HA Kubernetes cluster deployed with Kubernetes supported version below version 1.16.0. Must be an error.") + except CloudstackAPIException as e: + self.debug("HA Kubernetes cluster with invalid Kubernetes supported version check successful, API failure: %s" % e) + + return + + # @attr(tags=["advanced", "smoke"], required_hardware="true") + # def test_04_deploy_and_upgrade_kubernetes_cluster(self): + # """Test to deploy a new Kubernetes cluster and upgrade it to newer version + + # # Validate the following: + # # 1. createKubernetesCluster should return valid info for new cluster + # # 2. The Cloud Database contains the valid information + # # 3. upgradeKubernetesCluster should return valid info for the cluster + # """ + # if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: + # self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) + # name = 'testcluster-' + random_gen() + # self.debug("Creating for Kubernetes cluster with name %s" % name) + + # cluster_response = self.createKubernetesCluster(name, self.kuberetes_version_2.id) + + # self.verifyKubernetesCluster(cluster_response, name, self.kuberetes_version_2.id) + + # self.debug("Kubernetes cluster with ID: %s successfully deployed, now upgrading it" % cluster_response.id) + + # time.sleep(20) + + # cluster_response = self.upgradeKubernetesCluster(cluster_response.id, self.kuberetes_version_3.id) + + # self.verifyKubernetesClusterUpgrade(cluster_response, self.kuberetes_version_3.id) + + # self.debug("Kubernetes cluster with ID: %s successfully upgraded, now deleting it" % cluster_response.id) + + # self.deleteAndVerifyKubernetesCluster(cluster_response.id) + + # self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) + + # return + + # def test_05_deploy_and_upgrade_kubernetes_ha_cluster(self): + # """Test to deploy a new HA Kubernetes cluster and upgrade it to newer version + + # # Validate the following: + # # 1. createKubernetesCluster should return valid info for new cluster + # # 2. The Cloud Database contains the valid information + # # 3. upgradeKubernetesCluster should return valid info for the cluster + # """ + # if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: + # self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) + # name = 'testcluster-' + random_gen() + # self.debug("Creating for Kubernetes cluster with name %s" % name) + + # cluster_response = self.createKubernetesCluster(name, self.kuberetes_version_2.id, 1, 2) + + # self.verifyKubernetesCluster(cluster_response, name, self.kuberetes_version_2.id, 1, 2) + + # self.debug("Kubernetes cluster with ID: %s successfully deployed, now upgrading it" % cluster_response.id) + + # cluster_response = self.upgradeKubernetesCluster(cluster_response.id, self.kuberetes_version_3.id) + + # self.verifyKubernetesClusterUpgrade(cluster_response, self.kuberetes_version_3.id) + + # self.debug("Kubernetes cluster with ID: %s successfully upgraded, now deleting it" % cluster_response.id) + + # self.deleteAndVerifyKubernetesCluster(cluster_response.id) + + # self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) + + # return + + # @attr(tags=["advanced", "smoke"], required_hardware="true") + # def test_06_deploy_and_invalid_upgrade_kubernetes_cluster(self): + # """Test to deploy a new Kubernetes cluster and check for failure while tying to upgrade it to a lower version + + # # Validate the following: + # # 1. createKubernetesCluster should return valid info for new cluster + # # 2. The Cloud Database contains the valid information + # # 3. upgradeKubernetesCluster should fail + # """ + # if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: + # self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) + # name = 'testcluster-' + random_gen() + # self.debug("Creating for Kubernetes cluster with name %s" % name) + + # cluster_response = self.createKubernetesCluster(name, self.kuberetes_version_2.id) + + # self.verifyKubernetesCluster(cluster_response, name, self.kuberetes_version_2.id) + + # self.debug("Kubernetes cluster with ID: %s successfully deployed, now scaling it" % cluster_response.id) + + # try: + # cluster_response = self.upgradeKubernetesCluster(cluster_response.id, self.kuberetes_version_1.id) + # self.debug("Invalid CKS Kubernetes HA cluster deployed with ID: %s. Deleting it and failing test." % kuberetes_version_1.id) + # self.deleteAndVerifyKubernetesCluster(cluster_response.id) + # self.fail("Kubernetes cluster upgraded to a lower Kubernetes supported version. Must be an error.") + # except CloudstackAPIException as e: + # self.debug("Upgrading Kubernetes cluster with invalid Kubernetes supported version check successful, API failure: %s" % e) + + # self.debug("Deleting Kubernetes cluster with ID: %s" % cluster_response.id) + + # self.deleteAndVerifyKubernetesCluster(cluster_response.id) + + # self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) + + # return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + def test_07_deploy_and_scale_up_kubernetes_cluster(self): + """Test to deploy a new Kubernetes cluster and check for failure while tying to upgrade it to a lower version + + # Validate the following: + # 1. createKubernetesCluster should return valid info for new cluster + # 2. The Cloud Database contains the valid information + # 3. scaleKubernetesCluster should return valid info for the cluster and it should be scaled up + """ + if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: + self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) + name = 'testcluster-' + random_gen() + self.debug("Creating for Kubernetes cluster with name %s" % name) + + cluster_response = self.createKubernetesCluster(name, self.kuberetes_version_2.id) + + self.verifyKubernetesCluster(cluster_response, name, self.kuberetes_version_2.id) + + self.debug("Kubernetes cluster with ID: %s successfully deployed, now upscaling it" % cluster_response.id) + + cluster_response = self.scaleKubernetesCluster(cluster_response.id, 2) + + self.verifyKubernetesClusterScale(cluster_response, 2) + + self.debug("Kubernetes cluster with ID: %s successfully upscaled, now deleting it" % cluster_response.id) + + self.deleteAndVerifyKubernetesCluster(cluster_response.id) + + self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) + + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + def test_08_deploy_and_scale_down_kubernetes_cluster(self): + """Test to deploy a new Kubernetes cluster and check for failure while tying to upgrade it to a lower version + + # Validate the following: + # 1. createKubernetesCluster should return valid info for new cluster + # 2. The Cloud Database contains the valid information + # 3. scaleKubernetesCluster should return valid info for the cluster and it should be scaled down + """ + name = 'testcluster-' + random_gen() + self.debug("Creating for Kubernetes cluster with name %s" % name) + + cluster_response = self.createKubernetesCluster(name, self.kuberetes_version_2.id, 2) + + self.verifyKubernetesCluster(cluster_response, name, self.kuberetes_version_2.id, 2) + + self.debug("Kubernetes cluster with ID: %s successfully deployed, now downscaling it" % cluster_response.id) + + cluster_response = self.scaleKubernetesCluster(cluster_response.id, 1) + + self.verifyKubernetesClusterScale(cluster_response) + + self.debug("Kubernetes cluster with ID: %s successfully downscaled, now deleting it" % cluster_response.id) + + self.deleteAndVerifyKubernetesCluster(cluster_response.id) + + self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) + + return + + def listKubernetesCluster(self, clusterId): + listKubernetesClustersCmd = listKubernetesClusters.listKubernetesClustersCmd() + listKubernetesClustersCmd.id = clusterId + clusterResponse = self.apiclient.listKubernetesClusters(listKubernetesClustersCmd) + return clusterResponse[0] + + def createKubernetesCluster(self, name, version_id, size=1, master_nodes=1): + createKubernetesClusterCmd = createKubernetesCluster.createKubernetesClusterCmd() + createKubernetesClusterCmd.name = name + createKubernetesClusterCmd.kubernetesversionid = version_id + createKubernetesClusterCmd.size = size + createKubernetesClusterCmd.masternodes = master_nodes + createKubernetesClusterCmd.serviceofferingid = self.cks_service_offering.id + createKubernetesClusterCmd.zoneid = self.zone.id + createKubernetesClusterCmd.noderootdisksize = 10 + clusterResponse = self.apiclient.createKubernetesCluster(createKubernetesClusterCmd) + if not clusterResponse: + self.cleanup.append(clusterResponse) + return clusterResponse + + def deleteKubernetesCluster(self, clusterId): + deleteKubernetesClusterCmd = deleteKubernetesCluster.deleteKubernetesClusterCmd() + deleteKubernetesClusterCmd.id = clusterId + response = self.apiclient.deleteKubernetesCluster(deleteKubernetesClusterCmd) + return response + + def upgradeKubernetesCluster(self, clusterId, version_id): + upgradeKubernetesClusterCmd = upgradeKubernetesCluster.upgradeKubernetesClusterCmd() + upgradeKubernetesClusterCmd.id = clusterId + upgradeKubernetesClusterCmd.kubernetesversionid = version_id + response = self.apiclient.upgradeKubernetesCluster(upgradeKubernetesClusterCmd) + return response + + def scaleKubernetesCluster(self, clusterId, size): + scaleKubernetesClusterCmd = scaleKubernetesCluster.scaleKubernetesClusterCmd() + scaleKubernetesClusterCmd.id = clusterId + scaleKubernetesClusterCmd.size = size + response = self.apiclient.scaleKubernetesCluster(scaleKubernetesClusterCmd) + return response + + def verifyKubernetesCluster(self, cluster_response, name, version_id, size=1, master_nodes=1): + """Check if Kubernetes cluster is valid""" + + self.verifyKubernetesClusterState(cluster_response) + + self.assertEqual( + cluster_response.name, + name, + "Check KubernetesCluster name {}, {}".format(cluster_response.name, name) + ) + + self.verifyKubernetesClusterVersion(cluster_response, version_id) + + self.assertEqual( + cluster_response.zoneid, + self.zone.id, + "Check KubernetesCluster zone {}, {}".format(cluster_response.zoneid, self.zone.id) + ) + + self.verifyKubernetesClusterSize(cluster_response, size, master_nodes) + + db_cluster_name = self.dbclient.execute("select name from kubernetes_cluster where uuid = '%s';" % cluster_response.id)[0][0] + + self.assertEqual( + str(db_cluster_name), + name, + "Check KubernetesCluster name in DB {}, {}".format(db_cluster_name, name) + ) + + def verifyKubernetesClusterState(self, cluster_response): + """Check if Kubernetes cluster state is Running""" + + self.assertEqual( + cluster_response.state, + 'Running', + "Check KubernetesCluster state {}, {}".format(cluster_response.state, 'Running') + ) + + def verifyKubernetesClusterVersion(self, cluster_response, version_id): + """Check if Kubernetes cluster node sizes are valid""" + + self.assertEqual( + cluster_response.kubernetesversionid, + version_id, + "Check KubernetesCluster version {}, {}".format(cluster_response.kubernetesversionid, version_id) + ) + + def verifyKubernetesClusterSize(self, cluster_response, size=1, master_nodes=1): + """Check if Kubernetes cluster node sizes are valid""" + + self.assertEqual( + cluster_response.size, + size, + "Check KubernetesCluster size {}, {}".format(cluster_response.size, size) + ) + + self.assertEqual( + cluster_response.masternodes, + master_nodes, + "Check KubernetesCluster master nodes {}, {}".format(cluster_response.masternodes, master_nodes) + ) + + def verifyKubernetesClusterUpgrade(self, cluster_response, version_id): + """Check if Kubernetes cluster state and version are valid after upgrade""" + + self.verifyKubernetesClusterState(cluster_response) + self.verifyKubernetesClusterVersion(cluster_response, version_id) + + def verifyKubernetesClusterScale(self, cluster_response, size=1, master_nodes=1): + """Check if Kubernetes cluster state and node sizes are valid after upgrade""" + + self.verifyKubernetesClusterState(cluster_response) + self.verifyKubernetesClusterSize(cluster_response, size, master_nodes) + + def deleteAndVerifyKubernetesCluster(self, clusterId): + """Delete Kubernetes cluster and check if it is really deleted""" + + delete_response = self.deleteKubernetesCluster(clusterId) + + self.assertEqual( + delete_response.success, + True, + "Check KubernetesCluster deletion in DB {}, {}".format(delete_response.success, True) + ) + + db_cluster_removed = self.dbclient.execute("select removed from kubernetes_cluster where uuid = '%s';" % clusterId)[0][0] + + self.assertNotEqual( + db_cluster_removed, + None, + "KubernetesCluster not removed in DB" + ) From d97abee4d7d22d5f61bb7eaee3ea6be2855cd3b0 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 8 Jan 2020 14:34:07 +0530 Subject: [PATCH 049/134] fixed node name case for kubectl Signed-off-by: Abhishek Kumar --- .../KubernetesClusterManagerImpl.java | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java index bb361c74d981..2ee562d73028 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java @@ -633,7 +633,7 @@ CLUSTER_NODE_VM_USER, getManagementServerSshPublicKeyFile(), null, private boolean isKubernetesClusterNodeReady(KubernetesCluster kubernetesCluster, String ipAddress, int port, String nodeName) throws Exception { Pair result = SshHelper.sshExecute(ipAddress, port, CLUSTER_NODE_VM_USER, getManagementServerSshPublicKeyFile(), null, - String.format("sudo kubectl get nodes | awk '{if ($1 == \"%s\" && $2 == \"Ready\") print $1}'", nodeName), + String.format("sudo kubectl get nodes | awk '{if ($1 == \"%s\" && $2 == \"Ready\") print $1}'", nodeName.toLowerCase()), 10000, 10000, 20000); if (result.first() && nodeName.equals(result.second().trim())) { return true; @@ -705,33 +705,37 @@ private boolean validateKubernetesClusterReadyNodesCount(KubernetesCluster kuber private boolean removeKubernetesClusterNode(KubernetesCluster kubernetesCluster, String ipAddress, int port, UserVm userVm, int retries, int waitDuration) { File pkFile = getManagementServerSshPublicKeyFile(); int retryCounter = 0; + String hostName = userVm.getHostName(); + if (!Strings.isNullOrEmpty(hostName)) { + hostName = hostName.toLowerCase(); + } while (retryCounter < retries) { retryCounter++; try { Pair result = SshHelper.sshExecute(ipAddress, port, CLUSTER_NODE_VM_USER, - pkFile, null, String.format("sudo kubectl drain %s --ignore-daemonsets --delete-local-data", userVm.getHostName()), + pkFile, null, String.format("sudo kubectl drain %s --ignore-daemonsets --delete-local-data", hostName), 10000, 10000, 60000); if (!result.first()) { - LOGGER.warn(String.format("Draining node: %s on VM ID: %s in Kubernetes cluster ID: %s unsuccessful", userVm.getHostName(), userVm.getUuid(), kubernetesCluster.getUuid())); + LOGGER.warn(String.format("Draining node: %s on VM ID: %s in Kubernetes cluster ID: %s unsuccessful", hostName, userVm.getUuid(), kubernetesCluster.getUuid())); } else { result = SshHelper.sshExecute(ipAddress, port, CLUSTER_NODE_VM_USER, - pkFile, null, String.format("sudo kubectl delete node %s", userVm.getHostName()), + pkFile, null, String.format("sudo kubectl delete node %s", hostName), 10000, 10000, 30000); if (result.first()) { return true; } else { - LOGGER.warn(String.format("Deleting node: %s on VM ID: %s in Kubernetes cluster ID: %s unsuccessful", userVm.getHostName(), userVm.getUuid(), kubernetesCluster.getUuid())); + LOGGER.warn(String.format("Deleting node: %s on VM ID: %s in Kubernetes cluster ID: %s unsuccessful", hostName, userVm.getUuid(), kubernetesCluster.getUuid())); } } break; } catch (Exception e) { - String msg = String.format("Failed to remove Kubernetes cluster ID: %s node: %s on VM ID: %s", kubernetesCluster.getUuid(), userVm.getHostName(), userVm.getUuid()); + String msg = String.format("Failed to remove Kubernetes cluster ID: %s node: %s on VM ID: %s", kubernetesCluster.getUuid(), hostName, userVm.getUuid()); LOGGER.warn(msg, e); } try { Thread.sleep(waitDuration); } catch (InterruptedException ie) { - LOGGER.error(String.format("Error while waiting for Kubernetes cluster ID: %s node: %s on VM ID: %s removal", kubernetesCluster.getUuid(), userVm.getHostName(), userVm.getUuid()), ie); + LOGGER.error(String.format("Error while waiting for Kubernetes cluster ID: %s node: %s on VM ID: %s removal", kubernetesCluster.getUuid(), hostName, userVm.getUuid()), ie); } retryCounter++; } @@ -740,22 +744,26 @@ private boolean removeKubernetesClusterNode(KubernetesCluster kubernetesCluster, private boolean uncordonKubernetesClusterNode(KubernetesCluster kubernetesCluster, String ipAddress, int port, UserVm userVm, int retries, int waitDuration) { int retryCounter = 0; + String hostName = userVm.getHostName(); + if (!Strings.isNullOrEmpty(hostName)) { + hostName = hostName.toLowerCase(); + } while (retryCounter < retries) { Pair result = null; try { result = SshHelper.sshExecute(ipAddress, port, CLUSTER_NODE_VM_USER, getManagementServerSshPublicKeyFile(), null, - String.format("sudo kubectl uncordon %s", userVm.getHostName()), + String.format("sudo kubectl uncordon %s", hostName), 10000, 10000, 30000); if (result.first()) { return true; } } catch (Exception e) { - LOGGER.warn(String.format("Failed to uncordon node: %s on VM ID: %s in Kubernetes cluster ID: %s", userVm.getHostName(), userVm.getUuid(), kubernetesCluster.getUuid()), e); + LOGGER.warn(String.format("Failed to uncordon node: %s on VM ID: %s in Kubernetes cluster ID: %s", hostName, userVm.getUuid(), kubernetesCluster.getUuid()), e); } try { Thread.sleep(waitDuration); } catch (InterruptedException ie) { - LOGGER.warn(String.format("Error while waiting for uncordon Kubernetes cluster ID: %s node: %s on VM ID: %s", kubernetesCluster.getUuid(), userVm.getHostName(), userVm.getUuid()), ie); + LOGGER.warn(String.format("Error while waiting for uncordon Kubernetes cluster ID: %s node: %s on VM ID: %s", kubernetesCluster.getUuid(), hostName, userVm.getUuid()), ie); } retryCounter++; } @@ -2461,12 +2469,16 @@ private void upgradeKubernetesClusterNodes(final KubernetesCluster kubernetesClu File pkFile = getManagementServerSshPublicKeyFile(); for (int i = 0; i < vmIds.size(); ++i) { UserVm vm = userVmDao.findById(vmIds.get(i)); + String hostName = vm.getHostName(); + if (!Strings.isNullOrEmpty(hostName)) { + hostName = hostName.toLowerCase(); + } result = null; LOGGER.debug(String.format("Upgrading node on VM ID: %s in Kubernetes cluster ID: %s with Kubernetes version(%s) ID: %s", vm.getUuid(), kubernetesCluster.getUuid(), upgradeVersion.getSemanticVersion(), upgradeVersion.getUuid())); try { result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, pkFile, null, - String.format("sudo kubectl drain %s --ignore-daemonsets --delete-local-data", vm.getHostName()), + String.format("sudo kubectl drain %s --ignore-daemonsets --delete-local-data", hostName), 10000, 10000, 60000); } catch (Exception e) { logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to drain Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()), kubernetesCluster, vmIds, KubernetesCluster.Event.OperationFailed, e); @@ -2496,7 +2508,7 @@ private void upgradeKubernetesClusterNodes(final KubernetesCluster kubernetesClu logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to uncordon Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()), kubernetesCluster, vmIds, KubernetesCluster.Event.OperationFailed, null); } if (i == 0) { // Wait for master to get in Ready state - if (!isKubernetesClusterNodeReady(kubernetesCluster, publicIpAddress, sshPort, vm.getHostName(), 5, 20000)) { + if (!isKubernetesClusterNodeReady(kubernetesCluster, publicIpAddress, sshPort, hostName, 5, 20000)) { logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to get master Kubernetes node on VM ID: %s in ready state", kubernetesCluster.getUuid(), vm.getUuid()), kubernetesCluster, vmIds, KubernetesCluster.Event.OperationFailed, null); } } From 6fcad49410b0ecc4753bc56a8e2ca77644e872dc Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 8 Jan 2020 15:54:07 +0530 Subject: [PATCH 050/134] fixed failing upgrade cluster test Signed-off-by: Abhishek Kumar --- .../smoke/test_kubernetes_clusters.py | 84 +++++++++---------- 1 file changed, 41 insertions(+), 43 deletions(-) diff --git a/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_clusters.py b/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_clusters.py index cd49f9f82a2a..23797c8c9f04 100644 --- a/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_clusters.py +++ b/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_clusters.py @@ -335,70 +335,68 @@ def test_03_deploy_invalid_kubernetes_ha_cluster(self): return - # @attr(tags=["advanced", "smoke"], required_hardware="true") - # def test_04_deploy_and_upgrade_kubernetes_cluster(self): - # """Test to deploy a new Kubernetes cluster and upgrade it to newer version - - # # Validate the following: - # # 1. createKubernetesCluster should return valid info for new cluster - # # 2. The Cloud Database contains the valid information - # # 3. upgradeKubernetesCluster should return valid info for the cluster - # """ - # if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: - # self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) - # name = 'testcluster-' + random_gen() - # self.debug("Creating for Kubernetes cluster with name %s" % name) + @attr(tags=["advanced", "smoke"], required_hardware="true") + def test_04_deploy_and_upgrade_kubernetes_cluster(self): + """Test to deploy a new Kubernetes cluster and upgrade it to newer version - # cluster_response = self.createKubernetesCluster(name, self.kuberetes_version_2.id) + # Validate the following: + # 1. createKubernetesCluster should return valid info for new cluster + # 2. The Cloud Database contains the valid information + # 3. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: + self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) + name = 'testcluster-' + random_gen() + self.debug("Creating for Kubernetes cluster with name %s" % name) - # self.verifyKubernetesCluster(cluster_response, name, self.kuberetes_version_2.id) + cluster_response = self.createKubernetesCluster(name, self.kuberetes_version_2.id) - # self.debug("Kubernetes cluster with ID: %s successfully deployed, now upgrading it" % cluster_response.id) + self.verifyKubernetesCluster(cluster_response, name, self.kuberetes_version_2.id) - # time.sleep(20) + self.debug("Kubernetes cluster with ID: %s successfully deployed, now upgrading it" % cluster_response.id) - # cluster_response = self.upgradeKubernetesCluster(cluster_response.id, self.kuberetes_version_3.id) + cluster_response = self.upgradeKubernetesCluster(cluster_response.id, self.kuberetes_version_3.id) - # self.verifyKubernetesClusterUpgrade(cluster_response, self.kuberetes_version_3.id) + self.verifyKubernetesClusterUpgrade(cluster_response, self.kuberetes_version_3.id) - # self.debug("Kubernetes cluster with ID: %s successfully upgraded, now deleting it" % cluster_response.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded, now deleting it" % cluster_response.id) - # self.deleteAndVerifyKubernetesCluster(cluster_response.id) + self.deleteAndVerifyKubernetesCluster(cluster_response.id) - # self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) + self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) - # return + return - # def test_05_deploy_and_upgrade_kubernetes_ha_cluster(self): - # """Test to deploy a new HA Kubernetes cluster and upgrade it to newer version + def test_05_deploy_and_upgrade_kubernetes_ha_cluster(self): + """Test to deploy a new HA Kubernetes cluster and upgrade it to newer version - # # Validate the following: - # # 1. createKubernetesCluster should return valid info for new cluster - # # 2. The Cloud Database contains the valid information - # # 3. upgradeKubernetesCluster should return valid info for the cluster - # """ - # if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: - # self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) - # name = 'testcluster-' + random_gen() - # self.debug("Creating for Kubernetes cluster with name %s" % name) + # Validate the following: + # 1. createKubernetesCluster should return valid info for new cluster + # 2. The Cloud Database contains the valid information + # 3. upgradeKubernetesCluster should return valid info for the cluster + """ + if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: + self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) + name = 'testcluster-' + random_gen() + self.debug("Creating for Kubernetes cluster with name %s" % name) - # cluster_response = self.createKubernetesCluster(name, self.kuberetes_version_2.id, 1, 2) + cluster_response = self.createKubernetesCluster(name, self.kuberetes_version_2.id, 1, 2) - # self.verifyKubernetesCluster(cluster_response, name, self.kuberetes_version_2.id, 1, 2) + self.verifyKubernetesCluster(cluster_response, name, self.kuberetes_version_2.id, 1, 2) - # self.debug("Kubernetes cluster with ID: %s successfully deployed, now upgrading it" % cluster_response.id) + self.debug("Kubernetes cluster with ID: %s successfully deployed, now upgrading it" % cluster_response.id) - # cluster_response = self.upgradeKubernetesCluster(cluster_response.id, self.kuberetes_version_3.id) + cluster_response = self.upgradeKubernetesCluster(cluster_response.id, self.kuberetes_version_3.id) - # self.verifyKubernetesClusterUpgrade(cluster_response, self.kuberetes_version_3.id) + self.verifyKubernetesClusterUpgrade(cluster_response, self.kuberetes_version_3.id) - # self.debug("Kubernetes cluster with ID: %s successfully upgraded, now deleting it" % cluster_response.id) + self.debug("Kubernetes cluster with ID: %s successfully upgraded, now deleting it" % cluster_response.id) - # self.deleteAndVerifyKubernetesCluster(cluster_response.id) + self.deleteAndVerifyKubernetesCluster(cluster_response.id) - # self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) + self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) - # return + return # @attr(tags=["advanced", "smoke"], required_hardware="true") # def test_06_deploy_and_invalid_upgrade_kubernetes_cluster(self): From 7accba3988de0b4dc815d1df5e72a779e2f5b170 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 8 Jan 2020 15:55:38 +0530 Subject: [PATCH 051/134] logging refactoring Signed-off-by: Abhishek Kumar --- .../KubernetesClusterManagerImpl.java | 272 +++++++++++++----- 1 file changed, 197 insertions(+), 75 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java index 2ee562d73028..8ff239440de0 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java @@ -306,7 +306,9 @@ private String readResourceFile(String resource) throws IOException { private void logMessage(final Level logLevel, final String message, final Exception e) { if (logLevel == Level.DEBUG) { - LOGGER.debug(message); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(message); + } } else if (logLevel == Level.ERROR) { LOGGER.error(message); } if (logLevel == Level.WARN) { @@ -473,7 +475,9 @@ private boolean isKubernetesClusterServerRunning(KubernetesCluster kubernetesClu try { String versionOutput = IOUtils.toString(new URL(String.format("https://%s:%d/version", ipAddress, CLUSTER_API_PORT)), StringUtils.getPreferredCharset()); if (!Strings.isNullOrEmpty(versionOutput)) { - LOGGER.debug(String.format("Kubernetes cluster ID: %s API has been successfully provisioned, %s", kubernetesCluster.getUuid(), versionOutput)); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Kubernetes cluster ID: %s API has been successfully provisioned, %s", kubernetesCluster.getUuid(), versionOutput)); + } k8sApiServerSetup = true; break; } @@ -503,7 +507,9 @@ private String getKubernetesClusterConfig(KubernetesCluster kubernetesCluster, S kubeConfig = result.second(); break; } else { - LOGGER.debug(String.format("Failed to retrieve kube-config file for Kubernetes cluster ID: %s. Output: %s", kubernetesCluster.getUuid(), result.second())); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Failed to retrieve kube-config file for Kubernetes cluster ID: %s. Output: %s", kubernetesCluster.getUuid(), result.second())); + } } } catch (Exception e) { LOGGER.warn(String.format("Failed to retrieve kube-config file for Kubernetes cluster ID: %s. Attempt: %d/%d", kubernetesCluster.getUuid(), retryCounter+1, retries), e); @@ -527,7 +533,9 @@ private boolean isKubernetesClusterAddOnServiceRunning(KubernetesCluster kuberne for (String line : lines) { if (line.contains(serviceName) && line.contains("Running")) { - LOGGER.debug(String.format("Service : %s in namespace: %s for the Kubernetes cluster ID: %s is running",serviceName, namespace, kubernetesCluster.getUuid())); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Service : %s in namespace: %s for the Kubernetes cluster ID: %s is running", serviceName, namespace, kubernetesCluster.getUuid())); + } return true; } } @@ -543,9 +551,13 @@ private boolean isKubernetesClusterDashboardServiceRunning(KubernetesCluster kub int retryCounter = 0; // Check if dashboard service is up running. while (retryCounter < retries) { - LOGGER.debug(String.format("Checking dashboard service for the Kubernetes cluster ID: %s to come up. Attempt: %d/%d", kubernetesCluster.getUuid(), retryCounter+1, retries)); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Checking dashboard service for the Kubernetes cluster ID: %s to come up. Attempt: %d/%d", kubernetesCluster.getUuid(), retryCounter + 1, retries)); + } if (isKubernetesClusterAddOnServiceRunning(kubernetesCluster, ipAddress, port, "kubernetes-dashboard", "kubernetes-dashboard")) { - LOGGER.info(String.format("Dashboard service for the Kubernetes cluster ID: %s is in running state", kubernetesCluster.getUuid())); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Dashboard service for the Kubernetes cluster ID: %s is in running state", kubernetesCluster.getUuid())); + } running = true; break; } @@ -625,7 +637,9 @@ CLUSTER_NODE_VM_USER, getManagementServerSshPublicKeyFile(), null, if (result.first()) { return Integer.parseInt(result.second().trim().replace("\"", "")); } else { - LOGGER.debug(String.format("Failed to retrieve ready nodes for Kubernetes cluster ID: %s. Output: %s", kubernetesCluster.getUuid(), result.second())); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Failed to retrieve ready nodes for Kubernetes cluster ID: %s. Output: %s", kubernetesCluster.getUuid(), result.second())); + } } return 0; } @@ -638,7 +652,9 @@ CLUSTER_NODE_VM_USER, getManagementServerSshPublicKeyFile(), null, if (result.first() && nodeName.equals(result.second().trim())) { return true; } - LOGGER.debug(String.format("Failed to retrieve status for node: %s in Kubernetes cluster ID: %s. Output: %s", nodeName, kubernetesCluster.getUuid(), result.second())); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Failed to retrieve status for node: %s in Kubernetes cluster ID: %s. Output: %s", nodeName, kubernetesCluster.getUuid(), result.second())); + } return false; } @@ -679,15 +695,20 @@ private int getKubernetesClusterReadyNodesCount(KubernetesCluster kubernetesClus private boolean validateKubernetesClusterReadyNodesCount(KubernetesCluster kubernetesCluster, String ipAddress, int port, int retries, long waitDuration) { int retryCounter = 0; while (retryCounter < retries) { - // "sudo kubectl get nodes -o json | jq \".items[].metadata.name\" | wc -l" - LOGGER.debug(String.format("Checking ready nodes for the Kubernetes cluster ID: %s with total %d provisioned nodes. Attempt: %d/%d", kubernetesCluster.getUuid(), kubernetesCluster.getTotalNodeCount(), retryCounter+1, retries)); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Checking ready nodes for the Kubernetes cluster ID: %s with total %d provisioned nodes. Attempt: %d/%d", kubernetesCluster.getUuid(), kubernetesCluster.getTotalNodeCount(), retryCounter + 1, retries)); + } try { int nodesCount = getKubernetesClusterReadyNodesCount(kubernetesCluster, ipAddress, port); if (nodesCount == kubernetesCluster.getTotalNodeCount()) { - LOGGER.debug(String.format("Kubernetes cluster ID: %s has %d ready now", kubernetesCluster.getUuid(), kubernetesCluster.getTotalNodeCount())); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Kubernetes cluster ID: %s has %d ready nodes now", kubernetesCluster.getUuid(), kubernetesCluster.getTotalNodeCount())); + } return true; } else { - LOGGER.debug(String.format("Kubernetes cluster ID: %s has total %d provisioned nodes while %d ready now", kubernetesCluster.getUuid(), kubernetesCluster.getTotalNodeCount(), nodesCount)); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Kubernetes cluster ID: %s has total %d provisioned nodes while %d ready now", kubernetesCluster.getUuid(), kubernetesCluster.getTotalNodeCount(), nodesCount)); + } } } catch (Exception e) { LOGGER.warn(String.format("Failed to retrieve ready node count for Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); @@ -781,7 +802,9 @@ private Network startKubernetesCLusterNetwork(final KubernetesCluster kubernetes } try { networkMgr.startNetwork(network.getId(), destination, context); - LOGGER.debug(String.format("Network ID: %s is started for the Kubernetes cluster ID: %s", network.getUuid(), kubernetesCluster.getUuid())); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Network ID: %s is started for the Kubernetes cluster ID: %s", network.getUuid(), kubernetesCluster.getUuid())); + } } catch (ConcurrentOperationException | ResourceUnavailableException |InsufficientCapacityException e) { String msg = String.format("Failed to start Kubernetes cluster ID: %s as unable to start associated network ID: %s" , kubernetesCluster.getUuid(), network.getUuid()); LOGGER.error(msg, e); @@ -798,7 +821,9 @@ private UserVm provisionKubernetesClusterMasterVm(final KubernetesCluster kubern addKubernetesClusterVm(kubernetesCluster.getId(), k8sMasterVM.getId()); startKubernetesVM(k8sMasterVM, kubernetesCluster); k8sMasterVM = userVmDao.findById(k8sMasterVM.getId()); - LOGGER.debug(String.format("Provisioned the master VM ID: %s in to the Kubernetes cluster ID: %s", k8sMasterVM.getUuid(), kubernetesCluster.getUuid())); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Provisioned the master VM ID: %s in to the Kubernetes cluster ID: %s", k8sMasterVM.getUuid(), kubernetesCluster.getUuid())); + } } catch (ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { String msg = String.format("Provisioning the master VM failed in the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); LOGGER.warn(msg, e); @@ -818,7 +843,9 @@ private List provisionKubernetesClusterAdditionalMasterVms(final Kuberne addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId()); startKubernetesVM(vm, kubernetesCluster); additionalMasters.add(vm); - LOGGER.debug(String.format("Provisioned additional master VM ID: %s in to the Kubernetes cluster ID: %s", vm.getUuid(), kubernetesCluster.getUuid())); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Provisioned additional master VM ID: %s in to the Kubernetes cluster ID: %s", vm.getUuid(), kubernetesCluster.getUuid())); + } } catch (ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { String msg = String.format("Provisioning additional master VM %d/%d failed in the Kubernetes cluster ID: %s", i+1, kubernetesCluster.getMasterNodeCount(), kubernetesCluster.getUuid()); LOGGER.warn(msg, e); @@ -839,7 +866,9 @@ private List provisionKubernetesClusterNodeVms(final KubernetesCluster k addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId()); startKubernetesVM(vm, kubernetesCluster); nodes.add(vm); - LOGGER.debug(String.format("Provisioned node master VM ID: %s in to the Kubernetes cluster ID: %s", vm.getUuid(), kubernetesCluster.getUuid())); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Provisioned node master VM ID: %s in to the Kubernetes cluster ID: %s", vm.getUuid(), kubernetesCluster.getUuid())); + } } catch (ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { String msg = String.format("Provisioning node VM %d/%d failed in the Kubernetes cluster ID: %s", i, kubernetesCluster.getNodeCount(), kubernetesCluster.getUuid()); LOGGER.warn(msg, e); @@ -858,7 +887,9 @@ private boolean isKubernetesClusterMasterVmRunning(final KubernetesCluster kuber socket.connect(new InetSocketAddress(ipAddress, port), 10000); masterVmRunning = true; } catch (IOException e) { - LOGGER.debug(String.format("Waiting for Kubernetes cluster ID: %s master node VMs to be accessible", kubernetesCluster.getUuid())); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Waiting for Kubernetes cluster ID: %s master node VMs to be accessible", kubernetesCluster.getUuid())); + } try { Thread.sleep(10000); } catch (InterruptedException ex) { @@ -887,7 +918,9 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t if (zone == null) { throw new CloudRuntimeException(String.format("Unable to find zone for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); } - LOGGER.debug(String.format("Starting Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Starting Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + } stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.StartRequested); Account account = accountDao.findById(kubernetesCluster.getAccountId()); @@ -939,8 +972,9 @@ private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) t clusterVMIds.add(vm.getId()); } - LOGGER.debug(String.format("Kubernetes cluster ID: %s VMs successfully provisioned", kubernetesCluster.getUuid())); - + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Kubernetes cluster ID: %s VMs successfully provisioned", kubernetesCluster.getUuid())); + } try { setupKubernetesClusterNetworkRules(kubernetesCluster, network, account, clusterVMIds); } catch (ManagementServerException e) { @@ -1010,14 +1044,20 @@ private boolean startStoppedKubernetesCluster(long kubernetesClusterId) throws M throw new ManagementServerException(String.format("Kubernetes cluster ID: %s is already deleted", kubernetesCluster.getUuid())); } if (kubernetesCluster.getState().equals(KubernetesCluster.State.Running)) { - LOGGER.debug(String.format("Kubernetes cluster ID: %s is in running state", kubernetesCluster.getUuid())); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Kubernetes cluster ID: %s is in running state", kubernetesCluster.getUuid())); + } return true; } if (kubernetesCluster.getState().equals(KubernetesCluster.State.Starting)) { - LOGGER.debug(String.format("Kubernetes cluster ID: %s is already in starting state", kubernetesCluster.getUuid())); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Kubernetes cluster ID: %s is already in starting state", kubernetesCluster.getUuid())); + } return true; } - LOGGER.debug(String.format("Starting Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Starting Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + } stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.StartRequested); @@ -1083,7 +1123,9 @@ private boolean startStoppedKubernetesCluster(long kubernetesClusterId) throws M } stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); - LOGGER.debug(String.format("Kubernetes cluster ID: %s successfully started", kubernetesCluster.getUuid())); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Kubernetes cluster ID: %s successfully started", kubernetesCluster.getUuid())); + } return true; } @@ -1191,7 +1233,9 @@ public PortForwardingRuleVO doInTransaction(TransactionStatus status) throws Net } }); rulesService.applyPortForwardingRules(publicIp.getId(), account); - LOGGER.debug(String.format("Provisioned SSH port forwarding rule from port %d to 22 on %s to the VM IP : %s in Kubernetes cluster ID: %s", srcPortFinal, publicIp.getAddress().addr(), vmIp.toString(), kubernetesCluster.getUuid())); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Provisioned SSH port forwarding rule from port %d to 22 on %s to the VM IP : %s in Kubernetes cluster ID: %s", srcPortFinal, publicIp.getAddress().addr(), vmIp.toString(), kubernetesCluster.getUuid())); + } } } } @@ -1221,7 +1265,9 @@ private void setupKubernetesClusterNetworkRules(KubernetesCluster kubernetesClus Network network, Account account, List clusterVMIds) throws ManagementServerException { if (!Network.GuestType.Isolated.equals(network.getGuestType())) { - LOGGER.debug(String.format("Network ID: %s for Kubernetes cluster ID: %s is not an isolated network, therefore, no need for network rules", network.getUuid(), kubernetesCluster.getUuid())); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Network ID: %s for Kubernetes cluster ID: %s is not an isolated network, therefore, no need for network rules", network.getUuid(), kubernetesCluster.getUuid())); + } return; } IpAddress publicIp = getSourceNatIp(network); @@ -1231,8 +1277,10 @@ private void setupKubernetesClusterNetworkRules(KubernetesCluster kubernetesClus try { provisionFirewallRules(publicIp, account, CLUSTER_API_PORT, CLUSTER_API_PORT); - LOGGER.debug(String.format("Provisioned firewall rule to open up port %d on %s for Kubernetes cluster ID: %s", - CLUSTER_API_PORT, publicIp.getAddress().addr(), kubernetesCluster.getUuid())); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Provisioned firewall rule to open up port %d on %s for Kubernetes cluster ID: %s", + CLUSTER_API_PORT, publicIp.getAddress().addr(), kubernetesCluster.getUuid())); + } } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) { throw new ManagementServerException(String.format("Failed to provision firewall rules for API access for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); } @@ -1240,7 +1288,9 @@ private void setupKubernetesClusterNetworkRules(KubernetesCluster kubernetesClus try { int endPort = CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterVMIds.size() - 1; provisionFirewallRules(publicIp, account, CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort); - LOGGER.debug(String.format("Provisioned firewall rule to open up port %d to %d on %s for Kubernetes cluster ID: %s", CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort, publicIp.getAddress().addr(), kubernetesCluster.getUuid())); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Provisioned firewall rule to open up port %d to %d on %s for Kubernetes cluster ID: %s", CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort, publicIp.getAddress().addr(), kubernetesCluster.getUuid())); + } } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) { throw new ManagementServerException(String.format("Failed to provision firewall rules for SSH access for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); } @@ -1266,7 +1316,9 @@ private void setupKubernetesClusterNetworkRules(KubernetesCluster kubernetesClus private void scaleKubernetesClusterNetworkRules(KubernetesCluster kubernetesCluster, Network network, Account account, List clusterVMIds, List removedVMIds) throws ManagementServerException { if (!Network.GuestType.Isolated.equals(network.getGuestType())) { - LOGGER.debug(String.format("Network ID: %s for Kubernetes cluster ID: %s is not an isolated network, therefore, no need for network rules", network.getUuid(), kubernetesCluster.getUuid())); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Network ID: %s for Kubernetes cluster ID: %s is not an isolated network, therefore, no need for network rules", network.getUuid(), kubernetesCluster.getUuid())); + } return; } IpAddress publicIp = getSourceNatIp(network); @@ -1284,8 +1336,10 @@ private void scaleKubernetesClusterNetworkRules(KubernetesCluster kubernetesClus // Provision new SSH firewall rules try { provisionFirewallRules(publicIp, account, CLUSTER_NODES_DEFAULT_START_SSH_PORT, CLUSTER_NODES_DEFAULT_START_SSH_PORT + (int)kubernetesCluster.getTotalNodeCount() - 1); - LOGGER.debug(String.format("Provisioned firewall rule to open up port %d to %d on %s in Kubernetes cluster ID: %s", - CLUSTER_NODES_DEFAULT_START_SSH_PORT, CLUSTER_NODES_DEFAULT_START_SSH_PORT + (int)kubernetesCluster.getTotalNodeCount() - 1, publicIp.getAddress().addr(), kubernetesCluster.getUuid())); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Provisioned firewall rule to open up port %d to %d on %s in Kubernetes cluster ID: %s", + CLUSTER_NODES_DEFAULT_START_SSH_PORT, CLUSTER_NODES_DEFAULT_START_SSH_PORT + (int) kubernetesCluster.getTotalNodeCount() - 1, publicIp.getAddress().addr(), kubernetesCluster.getUuid())); + } } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException e) { throw new ManagementServerException(String.format("Failed to activate SSH firewall rules for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); } @@ -1316,7 +1370,9 @@ private boolean validateIsolatedNetwork(Network network, int clusterTotalNodeCou for (FirewallRuleVO rule : rules) { Integer startPort = rule.getSourcePortStart(); Integer endPort = rule.getSourcePortEnd(); - LOGGER.debug("Network rule : " + startPort + " " + endPort); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Network rule : " + startPort + " " + endPort); + } if (startPort <= CLUSTER_API_PORT && CLUSTER_API_PORT <= endPort) { throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting firewall rules to provision Kubernetes cluster for API access", network.getUuid())); } @@ -1328,7 +1384,9 @@ private boolean validateIsolatedNetwork(Network network, int clusterTotalNodeCou for (FirewallRuleVO rule : rules) { Integer startPort = rule.getSourcePortStart(); Integer endPort = rule.getSourcePortEnd(); - LOGGER.debug("Network rule : " + startPort + " " + endPort); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Network rule : " + startPort + " " + endPort); + } if (startPort <= CLUSTER_API_PORT && CLUSTER_API_PORT <= endPort) { throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting port forwarding rules to provision Kubernetes cluster for API access", network.getUuid())); } @@ -1425,21 +1483,29 @@ private DeployDestination plan(final long nodesCount, final DataCenter zone, fin ClusterDetailsVO cluster_detail_ram = clusterDetailsDao.findDetail(cluster.getId(), "memoryOvercommitRatio"); Float cpuOvercommitRatio = Float.parseFloat(cluster_detail_cpu.getValue()); Float memoryOvercommitRatio = Float.parseFloat(cluster_detail_ram.getValue()); - LOGGER.debug(String.format("Checking host ID: %s for capacity already reserved %d", h.getUuid(), reserved)); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Checking host ID: %s for capacity already reserved %d", h.getUuid(), reserved)); + } if (capacityManager.checkIfHostHasCapacity(h.getId(), cpu_requested * reserved, ram_requested * reserved, false, cpuOvercommitRatio, memoryOvercommitRatio, true)) { - LOGGER.debug(String.format("Found host ID: %s for with enough capacity, CPU=%d RAM=%d", h.getUuid(), cpu_requested * reserved, ram_requested * reserved)); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Found host ID: %s for with enough capacity, CPU=%d RAM=%d", h.getUuid(), cpu_requested * reserved, ram_requested * reserved)); + } hostEntry.setValue(new Pair(h, reserved)); suitable_host_found = true; break; } } if (!suitable_host_found) { - LOGGER.debug(String.format("Suitable hosts not found in datacenter ID: %s for node %d", zone.getUuid(), i)); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Suitable hosts not found in datacenter ID: %s for node %d", zone.getUuid(), i)); + } break; } } if (suitable_host_found) { - LOGGER.debug(String.format("Suitable hosts found in datacenter ID: %s, creating deployment destination", zone.getUuid())); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Suitable hosts found in datacenter ID: %s, creating deployment destination", zone.getUuid())); + } return new DeployDestination(zone, null, null, null); } String msg = String.format("Cannot find enough capacity for Kubernetes cluster(requested cpu=%1$s memory=%2$s)", @@ -1451,8 +1517,9 @@ private DeployDestination plan(final long nodesCount, final DataCenter zone, fin private DeployDestination plan(final KubernetesCluster kubernetesCluster, final DataCenter zone) throws InsufficientServerCapacityException { ServiceOffering offering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); - LOGGER.debug(String.format("Checking deployment destination for Kubernetes cluster ID: %s in zone ID: %s", kubernetesCluster.getUuid(), zone.getUuid())); - + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Checking deployment destination for Kubernetes cluster ID: %s in zone ID: %s", kubernetesCluster.getUuid(), zone.getUuid())); + } return plan(kubernetesCluster.getTotalNodeCount(), zone, offering); } @@ -1495,7 +1562,9 @@ protected boolean cleanupKubernetesClusterResources(Long kubernetesClusterId) th , vm.getState().toString())); } kubernetesClusterVmMapDao.expunge(clusterVM.getId()); - LOGGER.debug(String.format("Destroyed VM ID: %s as part of Kubernetes cluster ID: %s cleanup", vm.getUuid(), kubernetesCluster.getUuid())); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Destroyed VM ID: %s as part of Kubernetes cluster ID: %s cleanup", vm.getUuid(), kubernetesCluster.getUuid())); + } } catch (Exception e) { failedVmDestroy = true; LOGGER.warn(String.format("Failed to destroy VM ID: %s part of the Kubernetes cluster ID: %s cleanup. Moving on with destroying remaining resources provisioned for the Kubernetes cluster", userVM.getUuid(), kubernetesCluster.getUuid()), e); @@ -1546,7 +1615,9 @@ protected boolean cleanupKubernetesClusterResources(Long kubernetesClusterId) th processFailedNetworkDelete(kubernetesClusterId); throw new ManagementServerException(msg); } - LOGGER.debug(String.format("Destroyed network: %s as part of Kubernetes cluster ID: %s cleanup", network.getUuid(), kubernetesCluster.getUuid())); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Destroyed network: %s as part of Kubernetes cluster ID: %s cleanup", network.getUuid(), kubernetesCluster.getUuid())); + } } } catch (Exception e) { String msg = String.format("Failed to destroy network ID: %s as part of Kubernetes cluster ID: %s cleanup", network.getUuid(), kubernetesCluster.getUuid()); @@ -1557,7 +1628,9 @@ protected boolean cleanupKubernetesClusterResources(Long kubernetesClusterId) th } } else { String msg = String.format("Failed to destroy one or more VMs as part of Kubernetes cluster ID: %s cleanup", kubernetesCluster.getUuid()); - LOGGER.debug(msg); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(msg); + } processFailedNetworkDelete(kubernetesClusterId); throw new ManagementServerException(msg); } @@ -1570,7 +1643,9 @@ protected boolean cleanupKubernetesClusterResources(Long kubernetesClusterId) th kubernetesClusterDao.remove(kubernetesCluster.getId()); - LOGGER.debug(String.format("Kubernetes cluster ID: %s is successfully deleted", kubernetesCluster.getUuid())); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Kubernetes cluster ID: %s is successfully deleted", kubernetesCluster.getUuid())); + } return true; } @@ -1683,7 +1758,9 @@ private UserVm createKubernetesMaster(final KubernetesCluster kubernetesCluster, hostName, kubernetesCluster.getDescription(), null, null, null, null, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), requestedIps, addrs, null, null, null, customParameterMap, null, null, null, null); - LOGGER.debug(String.format("Created master VM ID: %s, %s in the Kubernetes cluster ID: %s", masterVm.getUuid(), hostName, kubernetesCluster.getUuid())); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Created master VM ID: %s, %s in the Kubernetes cluster ID: %s", masterVm.getUuid(), hostName, kubernetesCluster.getUuid())); + } return masterVm; } @@ -1732,7 +1809,9 @@ private UserVm createKubernetesAdditionalMaster(final KubernetesCluster kubernet hostName, kubernetesCluster.getDescription(), null, null, null, null, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), null, addrs, null, null, null, customParameterMap, null, null, null, null); - LOGGER.debug(String.format("Created master VM ID: %s, %s in the Kubernetes cluster ID: %s", additionalMasterVm.getUuid(), hostName, kubernetesCluster.getUuid())); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Created master VM ID: %s, %s in the Kubernetes cluster ID: %s", additionalMasterVm.getUuid(), hostName, kubernetesCluster.getUuid())); + } return additionalMasterVm; } @@ -1826,7 +1905,9 @@ private UserVm createKubernetesNode(KubernetesCluster kubernetesCluster, String hostName, kubernetesCluster.getDescription(), null, null, null, null, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), null, addrs, null, null, null, customParameterMap, null, null, null, null); - LOGGER.debug(String.format("Created node VM ID: %s, %s in the Kubernetes cluster ID: %s", nodeVm.getUuid(), hostName, kubernetesCluster.getUuid())); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Created node VM ID: %s, %s in the Kubernetes cluster ID: %s", nodeVm.getUuid(), hostName, kubernetesCluster.getUuid())); + } return nodeVm; } @@ -1838,7 +1919,9 @@ private void startKubernetesVM(final UserVm vm, final KubernetesCluster kubernet f.setAccessible(true); f.set(startVm, vm.getId()); userVmService.startVirtualMachine(startVm); - LOGGER.debug(String.format("Started VM ID: %s in the Kubernetes cluster ID: %s", vm.getUuid(), kubernetesCluster.getUuid())); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Started VM ID: %s in the Kubernetes cluster ID: %s", vm.getUuid(), kubernetesCluster.getUuid())); + } } catch (IllegalAccessException | NoSuchFieldException | ExecutionException | ResourceUnavailableException | ResourceAllocationException | InsufficientCapacityException ex) { logAndThrow(Level.WARN, String.format("Failed to start VM in the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), ex); @@ -1869,7 +1952,9 @@ private void attachIsoKubernetesVMs(KubernetesCluster kubernetesCluster, List kubernetesClusters = kubernetesClusterDao.findKubernetesClustersToGarbageCollect(); for (KubernetesCluster kubernetesCluster : kubernetesClusters) { - LOGGER.debug(String.format("Running Kubernetes cluster garbage collector on Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Running Kubernetes cluster garbage collector on Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + } try { if (cleanupKubernetesClusterResources(kubernetesCluster.getId())) { - LOGGER.debug(String.format("Garbage collection complete for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Garbage collection complete for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + } } else { LOGGER.warn(String.format("Garbage collection failed for Kubernetes cluster ID: %s, it will be attempted to garbage collected in next run", kubernetesCluster.getUuid())); } @@ -2946,7 +3055,9 @@ public void reallyRun() { // run through Kubernetes clusters in 'Running' state and ensure all the VM's are Running in the cluster List runningKubernetesClusters = kubernetesClusterDao.findKubernetesClustersInState(KubernetesCluster.State.Running); for (KubernetesCluster kubernetesCluster : runningKubernetesClusters) { - LOGGER.debug(String.format("Running Kubernetes cluster state scanner on Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Running Kubernetes cluster state scanner on Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + } try { if (!isClusterVMsInDesiredState(kubernetesCluster, VirtualMachine.State.Running)) { stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.FaultsDetected); @@ -2959,8 +3070,9 @@ public void reallyRun() { // run through Kubernetes clusters in 'Stopped' state and ensure all the VM's are Stopped in the cluster List stoppedKubernetesClusters = kubernetesClusterDao.findKubernetesClustersInState(KubernetesCluster.State.Stopped); for (KubernetesCluster kubernetesCluster : stoppedKubernetesClusters) { - - LOGGER.debug(String.format("Running Kubernetes cluster state scanner on Kubernetes cluster ID: %s for state: %s", kubernetesCluster.getUuid(), KubernetesCluster.State.Stopped.toString())); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Running Kubernetes cluster state scanner on Kubernetes cluster ID: %s for state: %s", kubernetesCluster.getUuid(), KubernetesCluster.State.Stopped.toString())); + } try { if (!isClusterVMsInDesiredState(kubernetesCluster, VirtualMachine.State.Stopped)) { stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.FaultsDetected); @@ -2973,7 +3085,9 @@ public void reallyRun() { // run through Kubernetes clusters in 'Alert' state and reconcile state as 'Running' if the VM's are running List alertKubernetesClusters = kubernetesClusterDao.findKubernetesClustersInState(KubernetesCluster.State.Alert); for (KubernetesClusterVO kubernetesCluster : alertKubernetesClusters) { - LOGGER.debug(String.format("Running Kubernetes cluster state scanner on Kubernetes cluster ID: %s for state: %s", kubernetesCluster.getUuid(), KubernetesCluster.State.Alert.toString())); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Running Kubernetes cluster state scanner on Kubernetes cluster ID: %s for state: %s", kubernetesCluster.getUuid(), KubernetesCluster.State.Alert.toString())); + } try { if (isClusterVMsInDesiredState(kubernetesCluster, VirtualMachine.State.Running) && kubernetesCluster.getTotalNodeCount() == getKubernetesClusterReadyNodesCount(kubernetesCluster)) { @@ -3020,7 +3134,9 @@ public void reallyRun() { if ((new Date()).getTime() - kubernetesCluster.getCreated().getTime() < 10*60*1000) { continue; } - LOGGER.debug(String.format("Running Kubernetes cluster state scanner on Kubernetes cluster ID: %s for state: %s", kubernetesCluster.getUuid(), KubernetesCluster.State.Starting.toString())); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Running Kubernetes cluster state scanner on Kubernetes cluster ID: %s for state: %s", kubernetesCluster.getUuid(), KubernetesCluster.State.Starting.toString())); + } try { if (isClusterVMsInDesiredState(kubernetesCluster, VirtualMachine.State.Running)) { stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.FaultsDetected); @@ -3033,7 +3149,9 @@ public void reallyRun() { } List destroyingKubernetesClusters = kubernetesClusterDao.findKubernetesClustersInState(KubernetesCluster.State.Destroying); for (KubernetesCluster kubernetesCluster : destroyingKubernetesClusters) { - LOGGER.debug(String.format("Running Kubernetes cluster state scanner on Kubernetes cluster ID: %s for state: %s", kubernetesCluster.getUuid(), KubernetesCluster.State.Destroying.toString())); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Running Kubernetes cluster state scanner on Kubernetes cluster ID: %s for state: %s", kubernetesCluster.getUuid(), KubernetesCluster.State.Destroying.toString())); + } try { cleanupKubernetesClusterResources(kubernetesCluster.getId()); } catch (Exception e) { @@ -3054,16 +3172,20 @@ boolean isClusterVMsInDesiredState(KubernetesCluster kubernetesCluster, VirtualM // check cluster is running at desired capacity include master nodes as well if (clusterVMs.size() < kubernetesCluster.getTotalNodeCount()) { - LOGGER.debug(String.format("Found only %d VMs in the Kubernetes cluster ID: %s while expected %d VMs to be in state: %s", - clusterVMs.size(), kubernetesCluster.getUuid(), kubernetesCluster.getTotalNodeCount(), state.toString())); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Found only %d VMs in the Kubernetes cluster ID: %s while expected %d VMs to be in state: %s", + clusterVMs.size(), kubernetesCluster.getUuid(), kubernetesCluster.getTotalNodeCount(), state.toString())); + } return false; } // check if all the VM's are in same state for (KubernetesClusterVmMapVO clusterVm : clusterVMs) { VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(clusterVm.getVmId()); if (vm.getState() != state) { - LOGGER.debug(String.format("Found VM ID: %s in the Kubernetes cluster ID: %s in state: %s while expected to be in state: %s. So moving the cluster to Alert state for reconciliation", - vm.getUuid(), kubernetesCluster.getUuid(), vm.getState().toString(), state.toString())); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Found VM ID: %s in the Kubernetes cluster ID: %s in state: %s while expected to be in state: %s. So moving the cluster to Alert state for reconciliation", + vm.getUuid(), kubernetesCluster.getUuid(), vm.getState().toString(), state.toString())); + } return false; } } From 1a2aa5688c310a6dbffeb6ce31074ae573a322ba Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 8 Jan 2020 17:03:39 +0530 Subject: [PATCH 052/134] version comparison changes Signed-off-by: Abhishek Kumar --- .../KubernetesClusterManagerImpl.java | 5 ----- .../KubernetesVersionManagerImpl.java | 10 ++-------- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java index 8ff239440de0..2958d5edf108 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java @@ -2514,11 +2514,6 @@ private void validateKubernetesClusterUpgradeParameters(UpgradeKubernetesCluster throw new InvalidParameterValueException(String.format("Invalid Kubernetes version associated with cluster ID: %s", kubernetesCluster.getUuid())); } - if (KubernetesVersionManagerImpl.compareSemanticVersions( - upgradeVersion.getSemanticVersion(), clusterVersion.getSemanticVersion()) <= 0) { - throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s can not be upgraded from %s to %s", - kubernetesCluster.getUuid(), clusterVersion.getSemanticVersion(), upgradeVersion.getSemanticVersion())); - } // Check upgradeVersion is either patch upgrade or immediate minor upgrade try { KubernetesVersionManagerImpl.canUpgradeKubernetesVersion(clusterVersion.getSemanticVersion(), upgradeVersion.getSemanticVersion()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java index 49dfa534ea8a..5f4e82827d8d 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java @@ -146,14 +146,8 @@ public static int compareSemanticVersions(String v1, String v2) throws IllegalAr } public static boolean canUpgradeKubernetesVersion(String currentVersion, String upgradeVersion) throws IllegalArgumentException { - if (Strings.isNullOrEmpty(currentVersion) || Strings.isNullOrEmpty(upgradeVersion)) { - throw new IllegalArgumentException(String.format("Invalid version comparision with versions %s, %s", currentVersion, upgradeVersion)); - } - if(!isSemanticVersion(currentVersion)) { - throw new IllegalArgumentException(String.format("Invalid version format, %s", currentVersion)); - } - if(!isSemanticVersion(upgradeVersion)) { - throw new IllegalArgumentException(String.format("Invalid version format, %s", upgradeVersion)); + if (compareSemanticVersions(currentVersion, upgradeVersion) <= 0) { + throw new IllegalArgumentException(String.format("Kubernetes clusters can not be downgraded, current version: %s, upgrade version: %s", currentVersion, upgradeVersion)); } String[] thisParts = currentVersion.split("\\."); String[] thatParts = upgradeVersion.split("\\."); From 8bbac076e626e39fbec9d78e069d1857ba659aed Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 8 Jan 2020 17:04:06 +0530 Subject: [PATCH 053/134] fixed upgrade negative test Signed-off-by: Abhishek Kumar --- .../smoke/test_kubernetes_clusters.py | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_clusters.py b/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_clusters.py index 23797c8c9f04..6df314a8e17c 100644 --- a/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_clusters.py +++ b/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_clusters.py @@ -398,41 +398,41 @@ def test_05_deploy_and_upgrade_kubernetes_ha_cluster(self): return - # @attr(tags=["advanced", "smoke"], required_hardware="true") - # def test_06_deploy_and_invalid_upgrade_kubernetes_cluster(self): - # """Test to deploy a new Kubernetes cluster and check for failure while tying to upgrade it to a lower version + @attr(tags=["advanced", "smoke"], required_hardware="true") + def test_06_deploy_and_invalid_upgrade_kubernetes_cluster(self): + """Test to deploy a new Kubernetes cluster and check for failure while tying to upgrade it to a lower version - # # Validate the following: - # # 1. createKubernetesCluster should return valid info for new cluster - # # 2. The Cloud Database contains the valid information - # # 3. upgradeKubernetesCluster should fail - # """ - # if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: - # self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) - # name = 'testcluster-' + random_gen() - # self.debug("Creating for Kubernetes cluster with name %s" % name) + # Validate the following: + # 1. createKubernetesCluster should return valid info for new cluster + # 2. The Cloud Database contains the valid information + # 3. upgradeKubernetesCluster should fail + """ + if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: + self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) + name = 'testcluster-' + random_gen() + self.debug("Creating for Kubernetes cluster with name %s" % name) - # cluster_response = self.createKubernetesCluster(name, self.kuberetes_version_2.id) + cluster_response = self.createKubernetesCluster(name, self.kuberetes_version_2.id) - # self.verifyKubernetesCluster(cluster_response, name, self.kuberetes_version_2.id) + self.verifyKubernetesCluster(cluster_response, name, self.kuberetes_version_2.id) - # self.debug("Kubernetes cluster with ID: %s successfully deployed, now scaling it" % cluster_response.id) + self.debug("Kubernetes cluster with ID: %s successfully deployed, now scaling it" % cluster_response.id) - # try: - # cluster_response = self.upgradeKubernetesCluster(cluster_response.id, self.kuberetes_version_1.id) - # self.debug("Invalid CKS Kubernetes HA cluster deployed with ID: %s. Deleting it and failing test." % kuberetes_version_1.id) - # self.deleteAndVerifyKubernetesCluster(cluster_response.id) - # self.fail("Kubernetes cluster upgraded to a lower Kubernetes supported version. Must be an error.") - # except CloudstackAPIException as e: - # self.debug("Upgrading Kubernetes cluster with invalid Kubernetes supported version check successful, API failure: %s" % e) + try: + cluster_response = self.upgradeKubernetesCluster(cluster_response.id, self.kuberetes_version_1.id) + self.debug("Invalid CKS Kubernetes HA cluster deployed with ID: %s. Deleting it and failing test." % kuberetes_version_1.id) + self.deleteAndVerifyKubernetesCluster(cluster_response.id) + self.fail("Kubernetes cluster upgraded to a lower Kubernetes supported version. Must be an error.") + except Exception as e: + self.debug("Upgrading Kubernetes cluster with invalid Kubernetes supported version check successful, API failure: %s" % e) - # self.debug("Deleting Kubernetes cluster with ID: %s" % cluster_response.id) + self.debug("Deleting Kubernetes cluster with ID: %s" % cluster_response.id) - # self.deleteAndVerifyKubernetesCluster(cluster_response.id) + self.deleteAndVerifyKubernetesCluster(cluster_response.id) - # self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) + self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) - # return + return @attr(tags=["advanced", "smoke"], required_hardware="true") def test_07_deploy_and_scale_up_kubernetes_cluster(self): From 51a6aa786f4d0712f6713adc1767b1682120c60f Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 9 Jan 2020 11:16:04 +0530 Subject: [PATCH 054/134] include null domainid fix Signed-off-by: Abhishek Kumar --- .../com/cloud/api/query/dao/NetworkOfferingJoinDaoImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/api/query/dao/NetworkOfferingJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/NetworkOfferingJoinDaoImpl.java index 29ec66692dac..0c258d1966a4 100644 --- a/server/src/main/java/com/cloud/api/query/dao/NetworkOfferingJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/NetworkOfferingJoinDaoImpl.java @@ -47,7 +47,7 @@ public List findByDomainId(long domainId, Boolean include SearchBuilder sb = createSearchBuilder(); sb.and("domainId", sb.entity().getDomainId(), SearchCriteria.Op.FIND_IN_SET); if (includeAllDomainOffering) { - sb.or("zId", sb.entity().getZoneId(), SearchCriteria.Op.NULL); + sb.or("dId", sb.entity().getDomainId(), SearchCriteria.Op.NULL); } sb.done(); From 1a907cf235f779f4228e14764bdab7a7186b2c78 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 9 Jan 2020 11:36:53 +0530 Subject: [PATCH 055/134] added javadoc Signed-off-by: Abhishek Kumar --- .../api/query/dao/NetworkOfferingJoinDao.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/server/src/main/java/com/cloud/api/query/dao/NetworkOfferingJoinDao.java b/server/src/main/java/com/cloud/api/query/dao/NetworkOfferingJoinDao.java index 37eef84ebd0e..767b9acf5d41 100644 --- a/server/src/main/java/com/cloud/api/query/dao/NetworkOfferingJoinDao.java +++ b/server/src/main/java/com/cloud/api/query/dao/NetworkOfferingJoinDao.java @@ -27,8 +27,26 @@ public interface NetworkOfferingJoinDao extends GenericDao { + /** + * Returns list of network offerings for a given domain + * NetworkOfferingJoinVO can have multiple domains set. Method will search for + * given domainId in list of domains for the offering. + * @param long domainId + * @param Boolean includeAllDomainOffering (if set to true offerings for which domain + * is not set will also be returned) + * @return List List of network offerings + */ List findByDomainId(long domainId, Boolean includeAllDomainOffering); + /** + * Returns list of network offerings for a given zone + * NetworkOfferingJoinVO can have multiple zones set. Method will search for + * given zoneId in list of zones for the offering. + * @param long zoneId + * @param Boolean includeAllZoneOffering (if set to true offerings for which zone + * is not set will also be returned) + * @return List List of network offerings + */ List findByZoneId(long zoneId, Boolean includeAllZoneOffering); NetworkOfferingResponse newNetworkOfferingResponse(NetworkOffering nof); From ab427c151549274cbe6030b24c3ef0f5c8456918 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 9 Jan 2020 11:56:51 +0530 Subject: [PATCH 056/134] added javadoc Signed-off-by: Abhishek Kumar --- .../KubernetesClusterManagerImpl.java | 49 +++++++++++++++---- 1 file changed, 40 insertions(+), 9 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java index 2958d5edf108..52c38268d3a2 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java @@ -1202,10 +1202,23 @@ private void removePortForwardingRules(IpAddress publicIp, Network network, Acco } } + /** + * To provision SSH port forwarding rules for the given Kubernetes cluster + * for its given virtual machines + * @param kubernetesCluster + * @param publicIp + * @param network + * @param account + * @param List clusterVMIds (when empty then method must be called while + * down-scaling of the KubernetesCluster therefore no new rules + * to be added) + * @param firewallRuleSourcePortStart + * @throws ResourceUnavailableException + * @throws NetworkRuleConflictException + */ private void provisionSshPortForwardingRules(KubernetesCluster kubernetesCluster, IpAddress publicIp, Network network, Account account, List clusterVMIds, int firewallRuleSourcePortStart) throws ResourceUnavailableException, NetworkRuleConflictException { - if (!CollectionUtils.isEmpty(clusterVMIds)) { // Upscaling, add new port-forwarding rules - // Apply port forwarding only to new VMs + if (!CollectionUtils.isEmpty(clusterVMIds)) { final long publicIpId = publicIp.getId(); final long networkId = network.getId(); final long accountId = account.getId(); @@ -1257,10 +1270,19 @@ private void provisionLoadBalancerRule(KubernetesCluster kubernetesCluster, IpAd lbService.assignToLoadBalancer(lb.getId(), null, vmIdIpMap); } - // Open up firewall port CLUSTER_API_PORT, secure port on which Kubernetes API server is running. Also create port-forwarding - // rule to forward public IP traffic to master VM private IP - // Open up firewall ports NODES_DEFAULT_START_SSH_PORT to NODES_DEFAULT_START_SSH_PORT+n for SSH access. Also create port-forwarding - // rule to forward public IP traffic to all node VM private IP + /** + * Setup network rules for Kubernetes cluster + * Open up firewall port CLUSTER_API_PORT, secure port on which Kubernetes + * API server is running. Also create load balancing rule to forward public + * IP traffic to master VMs' private IP. + * Open up firewall ports NODES_DEFAULT_START_SSH_PORT to NODES_DEFAULT_START_SSH_PORT+n + * for SSH access. Also create port-forwarding rule to forward public IP traffic to all + * @param kubernetesCluster + * @param network + * @param account + * @param clusterVMIds + * @throws ManagementServerException + */ private void setupKubernetesClusterNetworkRules(KubernetesCluster kubernetesCluster, Network network, Account account, List clusterVMIds) throws ManagementServerException { @@ -1310,9 +1332,18 @@ private void setupKubernetesClusterNetworkRules(KubernetesCluster kubernetesClus } } - // Open up firewall ports NODES_DEFAULT_START_SSH_PORT to NODES_DEFAULT_START_SSH_PORT+n for SSH access. Also create port-forwarding - // rule to forward public IP traffic to all node VM private IP. Existing node VMs before scaling - // will already be having these rules + /** + * Scale network rules for an existing Kubernetes cluster while scaling it + * Open up firewall for SSH access from port NODES_DEFAULT_START_SSH_PORT to NODES_DEFAULT_START_SSH_PORT+n. + * Also remove port forwarding rules for removed virtual machines and create port-forwarding rule + * to forward public IP traffic to all node VMs' private IP. + * @param kubernetesCluster + * @param network + * @param account + * @param clusterVMIds + * @param removedVMIds + * @throws ManagementServerException + */ private void scaleKubernetesClusterNetworkRules(KubernetesCluster kubernetesCluster, Network network, Account account, List clusterVMIds, List removedVMIds) throws ManagementServerException { if (!Network.GuestType.Isolated.equals(network.getGuestType())) { From 3e3840b43f7e6bed079c4a2475465c9afe0e316f Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 10 Jan 2020 19:35:18 +0530 Subject: [PATCH 057/134] refactoring Moved cluster opertaions(start, stop, scale, upgrade, delete) related code to individual worker classes. Refactored packages com.cloud.kubernetescluster and com.cloud.kubernetesversion to com.cloud.kubernetes.cluster and com.cloud.kubernetes.version. Refactored out generic cluster functions to separate util class. Signed-off-by: Abhishek Kumar --- .../cluster}/KubernetesCluster.java | 8 +- .../cluster}/KubernetesClusterDetailsVO.java | 2 +- .../cluster}/KubernetesClusterEventTypes.java | 2 +- .../cluster/KubernetesClusterManagerImpl.java | 1339 +++++++ .../cluster}/KubernetesClusterService.java | 24 +- .../cluster}/KubernetesClusterVO.java | 2 +- .../cluster}/KubernetesClusterVmMap.java | 2 +- .../cluster}/KubernetesClusterVmMapVO.java | 2 +- .../KubernetesClusterActionWorker.java | 348 ++ .../KubernetesClusterDestroyWorker.java | 205 ++ ...esClusterResourceModifierActionWorker.java | 407 +++ .../KubernetesClusterScaleWorker.java | 457 +++ .../KubernetesClusterStartWorker.java | 607 +++ .../KubernetesClusterStopWorker.java | 61 + .../KubernetesClusterUpgradeWorker.java | 157 + .../cluster}/dao/KubernetesClusterDao.java | 6 +- .../dao/KubernetesClusterDaoImpl.java | 9 +- .../dao/KubernetesClusterDetailsDao.java | 4 +- .../dao/KubernetesClusterDetailsDaoImpl.java | 4 +- .../dao/KubernetesClusterVmMapDao.java | 4 +- .../dao/KubernetesClusterVmMapDaoImpl.java | 4 +- .../cluster/utils/KubernetesClusterUtil.java | 306 ++ .../version}/KubernetesSupportedVersion.java | 2 +- .../KubernetesSupportedVersionVO.java | 2 +- .../version}/KubernetesVersionEventTypes.java | 2 +- .../KubernetesVersionManagerImpl.java | 16 +- .../version}/KubernetesVersionService.java | 13 +- .../dao/KubernetesSupportedVersionDao.java | 4 +- .../KubernetesSupportedVersionDaoImpl.java | 7 +- .../KubernetesClusterManagerImpl.java | 3253 ----------------- .../AddKubernetesSupportedVersionCmd.java | 18 +- .../DeleteKubernetesSupportedVersionCmd.java | 18 +- .../cluster}/CreateKubernetesClusterCmd.java | 20 +- .../cluster}/DeleteKubernetesClusterCmd.java | 30 +- .../GetKubernetesClusterConfigCmd.java | 17 +- .../cluster}/ListKubernetesClustersCmd.java | 17 +- .../cluster}/ScaleKubernetesClusterCmd.java | 35 +- .../cluster}/StartKubernetesClusterCmd.java | 17 +- .../cluster}/StopKubernetesClusterCmd.java | 30 +- .../cluster}/UpgradeKubernetesClusterCmd.java | 35 +- .../ListKubernetesSupportedVersionsCmd.java | 4 +- .../response/KubernetesClusterResponse.java | 2 +- .../KubernetesSupportedVersionResponse.java | 2 +- .../spring-kubernetes-service-context.xml | 12 +- .../KubernetesVersionServiceTest.java | 16 +- 45 files changed, 4076 insertions(+), 3456 deletions(-) rename plugins/integrations/kubernetes-service/src/main/java/com/cloud/{kubernetescluster => kubernetes/cluster}/KubernetesCluster.java (96%) rename plugins/integrations/kubernetes-service/src/main/java/com/cloud/{kubernetescluster => kubernetes/cluster}/KubernetesClusterDetailsVO.java (98%) rename plugins/integrations/kubernetes-service/src/main/java/com/cloud/{kubernetescluster => kubernetes/cluster}/KubernetesClusterEventTypes.java (97%) create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java rename plugins/integrations/kubernetes-service/src/main/java/com/cloud/{kubernetescluster => kubernetes/cluster}/KubernetesClusterService.java (82%) rename plugins/integrations/kubernetes-service/src/main/java/com/cloud/{kubernetescluster => kubernetes/cluster}/KubernetesClusterVO.java (99%) rename plugins/integrations/kubernetes-service/src/main/java/com/cloud/{kubernetescluster => kubernetes/cluster}/KubernetesClusterVmMap.java (96%) rename plugins/integrations/kubernetes-service/src/main/java/com/cloud/{kubernetescluster => kubernetes/cluster}/KubernetesClusterVmMapVO.java (98%) create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStopWorker.java create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java rename plugins/integrations/kubernetes-service/src/main/java/com/cloud/{kubernetescluster => kubernetes/cluster}/dao/KubernetesClusterDao.java (90%) rename plugins/integrations/kubernetes-service/src/main/java/com/cloud/{kubernetescluster => kubernetes/cluster}/dao/KubernetesClusterDaoImpl.java (94%) rename plugins/integrations/kubernetes-service/src/main/java/com/cloud/{kubernetescluster => kubernetes/cluster}/dao/KubernetesClusterDetailsDao.java (90%) rename plugins/integrations/kubernetes-service/src/main/java/com/cloud/{kubernetescluster => kubernetes/cluster}/dao/KubernetesClusterDetailsDaoImpl.java (92%) rename plugins/integrations/kubernetes-service/src/main/java/com/cloud/{kubernetescluster => kubernetes/cluster}/dao/KubernetesClusterVmMapDao.java (90%) rename plugins/integrations/kubernetes-service/src/main/java/com/cloud/{kubernetescluster => kubernetes/cluster}/dao/KubernetesClusterVmMapDaoImpl.java (94%) create mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java rename plugins/integrations/kubernetes-service/src/main/java/com/cloud/{kubernetesversion => kubernetes/version}/KubernetesSupportedVersion.java (96%) rename plugins/integrations/kubernetes-service/src/main/java/com/cloud/{kubernetesversion => kubernetes/version}/KubernetesSupportedVersionVO.java (98%) rename plugins/integrations/kubernetes-service/src/main/java/com/cloud/{kubernetesversion => kubernetes/version}/KubernetesVersionEventTypes.java (96%) rename plugins/integrations/kubernetes-service/src/main/java/com/cloud/{kubernetesversion => kubernetes/version}/KubernetesVersionManagerImpl.java (97%) rename plugins/integrations/kubernetes-service/src/main/java/com/cloud/{kubernetesversion => kubernetes/version}/KubernetesVersionService.java (72%) rename plugins/integrations/kubernetes-service/src/main/java/com/cloud/{kubernetesversion => kubernetes/version}/dao/KubernetesSupportedVersionDao.java (90%) rename plugins/integrations/kubernetes-service/src/main/java/com/cloud/{kubernetesversion => kubernetes/version}/dao/KubernetesSupportedVersionDaoImpl.java (90%) delete mode 100644 plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java rename plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/{kubernetesversion => kubernetes/version}/AddKubernetesSupportedVersionCmd.java (89%) rename plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/{kubernetesversion => kubernetes/version}/DeleteKubernetesSupportedVersionCmd.java (89%) rename plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/{kubernetescluster => kubernetes/cluster}/CreateKubernetesClusterCmd.java (94%) rename plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/{kubernetescluster => kubernetes/cluster}/DeleteKubernetesClusterCmd.java (78%) rename plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/{kubernetescluster => kubernetes/cluster}/GetKubernetesClusterConfigCmd.java (83%) rename plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/{kubernetescluster => kubernetes/cluster}/ListKubernetesClustersCmd.java (84%) rename plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/{kubernetescluster => kubernetes/cluster}/ScaleKubernetesClusterCmd.java (79%) rename plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/{kubernetescluster => kubernetes/cluster}/StartKubernetesClusterCmd.java (87%) rename plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/{kubernetescluster => kubernetes/cluster}/StopKubernetesClusterCmd.java (77%) rename plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/{kubernetescluster => kubernetes/cluster}/UpgradeKubernetesClusterCmd.java (79%) rename plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/{kubernetesversion => kubernetes/version}/ListKubernetesSupportedVersionsCmd.java (97%) rename plugins/integrations/kubernetes-service/src/test/java/com/cloud/{kubernetesversion => kubernetes/version}/KubernetesVersionServiceTest.java (94%) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesCluster.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java similarity index 96% rename from plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesCluster.java rename to plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java index b4d2c16522c5..27712d4e79f5 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesCluster.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.cloud.kubernetescluster; +package com.cloud.kubernetes.cluster; import java.util.Date; @@ -26,7 +26,8 @@ import com.cloud.utils.fsm.StateMachine2; /** - * KubernetesCluster describes the properties of Kubernetes cluster + * KubernetesCluster describes the properties of a Kubernetes cluster + * StateMachine maintains its states. * */ public interface KubernetesCluster extends ControlledEntity, com.cloud.utils.fsm.StateObject, Identity, InternalIdentity, Displayable { @@ -97,9 +98,8 @@ enum State { s_fsm.addTransition(State.Alert, Event.DestroyRequested, State.Destroying); s_fsm.addTransition(State.Error, Event.DestroyRequested, State.Destroying); - s_fsm.addTransition(State.Destroying, Event.DestroyRequested, State.Destroying); s_fsm.addTransition(State.Destroying, Event.OperationSucceeded, State.Destroyed); - s_fsm.addTransition(State.Destroying, Event.OperationFailed, State.Destroying); + s_fsm.addTransition(State.Destroying, Event.OperationFailed, State.Error); } String _description; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterDetailsVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterDetailsVO.java similarity index 98% rename from plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterDetailsVO.java rename to plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterDetailsVO.java index 29bdd48f65d7..30b286463763 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterDetailsVO.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterDetailsVO.java @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.cloud.kubernetescluster; +package com.cloud.kubernetes.cluster; import javax.persistence.Column; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterEventTypes.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java similarity index 97% rename from plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterEventTypes.java rename to plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java index a13740697540..a947e4273be0 100755 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterEventTypes.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.cloud.kubernetescluster; +package com.cloud.kubernetes.cluster; public class KubernetesClusterEventTypes { public static final String EVENT_KUBERNETES_CLUSTER_CREATE = "KUBERNETES.CLUSTER.CREATE"; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java new file mode 100644 index 000000000000..e57e2bb71520 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -0,0 +1,1339 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster; + +import java.math.BigInteger; +import java.net.MalformedURLException; +import java.net.URL; +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; + +import org.apache.cloudstack.acl.ControlledEntity; +import org.apache.cloudstack.acl.SecurityChecker; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.CreateKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.DeleteKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.GetKubernetesClusterConfigCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.ListKubernetesClustersCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.ScaleKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.StartKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.StopKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.UpgradeKubernetesClusterCmd; +import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse; +import org.apache.cloudstack.api.response.KubernetesClusterResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.collections.CollectionUtils; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; + +import com.cloud.api.ApiDBUtils; +import com.cloud.api.query.dao.NetworkOfferingJoinDao; +import com.cloud.api.query.dao.TemplateJoinDao; +import com.cloud.api.query.vo.NetworkOfferingJoinVO; +import com.cloud.api.query.vo.TemplateJoinVO; +import com.cloud.capacity.CapacityManager; +import com.cloud.dc.ClusterDetailsDao; +import com.cloud.dc.ClusterDetailsVO; +import com.cloud.dc.ClusterVO; +import com.cloud.dc.DataCenter; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.ClusterDao; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.deploy.DeployDestination; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.InsufficientServerCapacityException; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ManagementServerException; +import com.cloud.exception.PermissionDeniedException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.host.Host.Type; +import com.cloud.host.HostVO; +import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterActionWorker; +import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterDestroyWorker; +import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterScaleWorker; +import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterStartWorker; +import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterStopWorker; +import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterUpgradeWorker; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; +import com.cloud.kubernetes.version.KubernetesSupportedVersion; +import com.cloud.kubernetes.version.KubernetesSupportedVersionVO; +import com.cloud.kubernetes.version.KubernetesVersionManagerImpl; +import com.cloud.kubernetes.version.dao.KubernetesSupportedVersionDao; +import com.cloud.network.IpAddress; +import com.cloud.network.Network; +import com.cloud.network.Network.Service; +import com.cloud.network.NetworkModel; +import com.cloud.network.NetworkService; +import com.cloud.network.PhysicalNetwork; +import com.cloud.network.dao.FirewallRulesDao; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; +import com.cloud.network.dao.PhysicalNetworkDao; +import com.cloud.network.rules.FirewallRule; +import com.cloud.network.rules.FirewallRuleVO; +import com.cloud.offering.NetworkOffering; +import com.cloud.offering.ServiceOffering; +import com.cloud.offerings.NetworkOfferingVO; +import com.cloud.offerings.dao.NetworkOfferingDao; +import com.cloud.offerings.dao.NetworkOfferingServiceMapDao; +import com.cloud.org.Grouping; +import com.cloud.resource.ResourceManager; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VMTemplateZoneVO; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VMTemplateZoneDao; +import com.cloud.template.TemplateApiService; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.AccountService; +import com.cloud.user.SSHKeyPairVO; +import com.cloud.user.dao.SSHKeyPairDao; +import com.cloud.utils.Pair; +import com.cloud.utils.component.ComponentContext; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.concurrency.NamedThreadFactory; +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GlobalLock; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.db.TransactionCallbackNoReturn; +import com.cloud.utils.db.TransactionStatus; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.fsm.NoTransitionException; +import com.cloud.utils.fsm.StateMachine2; +import com.cloud.utils.net.NetUtils; +import com.cloud.vm.UserVmVO; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.dao.UserVmDao; +import com.cloud.vm.dao.VMInstanceDao; +import com.google.common.base.Strings; + +public class KubernetesClusterManagerImpl extends ManagerBase implements KubernetesClusterService { + + private static final Logger LOGGER = Logger.getLogger(KubernetesClusterManagerImpl.class); + + protected StateMachine2 _stateMachine = KubernetesCluster.State.getStateMachine(); + + ScheduledExecutorService _gcExecutor; + ScheduledExecutorService _stateScanner; + + @Inject + public KubernetesClusterDao kubernetesClusterDao; + @Inject + public KubernetesClusterVmMapDao kubernetesClusterVmMapDao; + @Inject + public KubernetesClusterDetailsDao kubernetesClusterDetailsDao; + @Inject + public KubernetesSupportedVersionDao kubernetesSupportedVersionDao; + @Inject + protected SSHKeyPairDao sshKeyPairDao; + @Inject + protected DataCenterDao dataCenterDao; + @Inject + protected ClusterDao clusterDao; + @Inject + protected ClusterDetailsDao clusterDetailsDao; + @Inject + protected ServiceOfferingDao serviceOfferingDao; + @Inject + protected VMTemplateDao templateDao; + @Inject + protected TemplateApiService templateService; + @Inject + protected VMTemplateZoneDao templateZoneDao; + @Inject + protected TemplateJoinDao templateJoinDao; + @Inject + protected AccountService accountService; + @Inject + protected AccountManager accountManager; + @Inject + protected VMInstanceDao vmInstanceDao; + @Inject + protected UserVmDao userVmDao; + @Inject + protected NetworkOfferingDao networkOfferingDao; + @Inject + protected NetworkOfferingJoinDao networkOfferingJoinDao; + @Inject + protected NetworkService networkService; + @Inject + protected NetworkModel networkModel; + @Inject + protected PhysicalNetworkDao physicalNetworkDao; + @Inject + protected NetworkOrchestrationService networkMgr; + @Inject + protected NetworkDao networkDao; + @Inject + protected NetworkOfferingServiceMapDao networkOfferingServiceMapDao; + @Inject + protected CapacityManager capacityManager; + @Inject + protected ResourceManager resourceManager; + @Inject + protected FirewallRulesDao firewallRulesDao; + + private void logMessage(final Level logLevel, final String message, final Exception e) { + if (logLevel == Level.DEBUG) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(message); + } + } else if (logLevel == Level.ERROR) { + LOGGER.error(message); + } if (logLevel == Level.WARN) { + if (e != null) { + LOGGER.warn(message, e); + } else { + LOGGER.warn(message); + } + } else { + if (e != null) { + LOGGER.error(message, e); + } else { + LOGGER.error(message); + } + } + } + + private void logTransitStateAndThrow(final Level logLevel, final String message, final Long kubernetesClusterId, final KubernetesCluster.Event event, final Exception e) throws CloudRuntimeException { + logMessage(logLevel, message, e); + if (kubernetesClusterId != null && event != null) { + stateTransitTo(kubernetesClusterId, event); + } + if (e == null) { + throw new CloudRuntimeException(message); + } + throw new CloudRuntimeException(message, e); + } + + private void logAndThrow(final Level logLevel, final String message) throws CloudRuntimeException { + logTransitStateAndThrow(logLevel, message, null, null, null); + } + + private void logAndThrow(final Level logLevel, final String message, final Exception ex) throws CloudRuntimeException { + logTransitStateAndThrow(logLevel, message, null, null, ex); + } + + private boolean isKubernetesServiceTemplateConfigured(DataCenter zone) { + // Check Kubernetes VM template for zone + String templateName = KubernetesClusterTemplateName.value(); + if (templateName == null || templateName.isEmpty()) { + LOGGER.warn(String.format("Global setting %s is empty. Template name need to be specified for Kubernetes service to function", KubernetesClusterTemplateName.key())); + return false; + } + final VMTemplateVO template = templateDao.findByTemplateName(templateName); + if (template == null) { + LOGGER.warn(String.format("Unable to find the template %s to be used for provisioning Kubernetes cluster", templateName)); + return false; + } + return true; + } + + private boolean isKubernetesServiceNetworkOfferingConfigured(DataCenter zone) { + // Check network offering + String networkOfferingName = KubernetesClusterNetworkOffering.value(); + if (networkOfferingName == null || networkOfferingName.isEmpty()) { + LOGGER.warn(String.format("Global setting %s is empty. Admin has not yet specified the network offering to be used for provisioning isolated network for the cluster", KubernetesClusterNetworkOffering.key())); + return false; + } + NetworkOfferingVO networkOffering = networkOfferingDao.findByUniqueName(networkOfferingName); + if (networkOffering == null) { + LOGGER.warn(String.format("Unable to find the network offering %s to be used for provisioning Kubernetes cluster", networkOfferingName)); + return false; + } + if (networkOffering.getState() == NetworkOffering.State.Disabled) { + LOGGER.warn(String.format("Network offering ID: %s is not enabled", networkOffering.getUuid())); + return false; + } + List services = networkOfferingServiceMapDao.listServicesForNetworkOffering(networkOffering.getId()); + if (services == null || services.isEmpty() || !services.contains("SourceNat")) { + LOGGER.warn(String.format("Network offering ID: %s does not have necessary services to provision Kubernetes cluster", networkOffering.getUuid())); + return false; + } + if (!networkOffering.isEgressDefaultPolicy()) { + LOGGER.warn(String.format("Network offering ID: %s has egress default policy turned off should be on to provision Kubernetes cluster", networkOffering.getUuid())); + return false; + } + boolean offeringAvailableForZone = false; + List networkOfferingJoinVOs = networkOfferingJoinDao.findByZoneId(zone.getId(), true); + for (NetworkOfferingJoinVO networkOfferingJoinVO : networkOfferingJoinVOs) { + if (networkOffering.getId() == networkOfferingJoinVO.getId()) { + offeringAvailableForZone = true; + break; + } + } + if (!offeringAvailableForZone) { + LOGGER.warn(String.format("Network offering ID: %s is not available for zone ID: %s", networkOffering.getUuid(), zone.getUuid())); + return false; + } + long physicalNetworkId = networkModel.findPhysicalNetworkId(zone.getId(), networkOffering.getTags(), networkOffering.getTrafficType()); + PhysicalNetwork physicalNetwork = physicalNetworkDao.findById(physicalNetworkId); + if (physicalNetwork == null) { + LOGGER.warn(String.format("Unable to find physical network with tag: %s", networkOffering.getTags())); + return false; + } + return true; + } + + private boolean isKubernetesServiceConfigured(DataCenter zone) { + if (!isKubernetesServiceTemplateConfigured(zone)) { + return false; + } + if (!isKubernetesServiceNetworkOfferingConfigured(zone)) { + return false; + } + return true; + } + + private IpAddress getSourceNatIp(Network network) { + List addresses = networkModel.listPublicIpsAssignedToGuestNtwk(network.getId(), true); + if (CollectionUtils.isEmpty(addresses)) { + return null; + } + for (IpAddress address : addresses) { + if (address.isSourceNat()) { + return address; + } + } + return null; + } + + private boolean validateIsolatedNetwork(Network network, int clusterTotalNodeCount) { + if (Network.GuestType.Isolated.equals(network.getGuestType())) { + if (Network.State.Allocated.equals(network.getState())) { // Allocated networks won't have IP and rules + return true; + } + IpAddress sourceNatIp = getSourceNatIp(network); + if (sourceNatIp == null) { + throw new InvalidParameterValueException(String.format("Network ID: %s does not have a source NAT IP associated with it. To provision a Kubernetes Cluster, source NAT IP is required", network.getUuid())); + } + List rules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(sourceNatIp.getId(), FirewallRule.Purpose.Firewall); + for (FirewallRuleVO rule : rules) { + Integer startPort = rule.getSourcePortStart(); + Integer endPort = rule.getSourcePortEnd(); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Network rule : " + startPort + " " + endPort); + } + if (startPort <= KubernetesClusterActionWorker.CLUSTER_API_PORT && KubernetesClusterActionWorker.CLUSTER_API_PORT <= endPort) { + throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting firewall rules to provision Kubernetes cluster for API access", network.getUuid())); + } + if (startPort <= KubernetesClusterActionWorker.CLUSTER_NODES_DEFAULT_START_SSH_PORT && KubernetesClusterActionWorker.CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterTotalNodeCount <= endPort) { + throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting firewall rules to provision Kubernetes cluster for node VM SSH access", network.getUuid())); + } + } + rules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(sourceNatIp.getId(), FirewallRule.Purpose.PortForwarding); + for (FirewallRuleVO rule : rules) { + Integer startPort = rule.getSourcePortStart(); + Integer endPort = rule.getSourcePortEnd(); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Network rule : " + startPort + " " + endPort); + } + if (startPort <= KubernetesClusterActionWorker.CLUSTER_API_PORT && KubernetesClusterActionWorker.CLUSTER_API_PORT <= endPort) { + throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting port forwarding rules to provision Kubernetes cluster for API access", network.getUuid())); + } + if (startPort <= KubernetesClusterActionWorker.CLUSTER_NODES_DEFAULT_START_SSH_PORT && KubernetesClusterActionWorker.CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterTotalNodeCount <= endPort) { + throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting port forwarding rules to provision Kubernetes cluster for node VM SSH access", network.getUuid())); + } + } + } + return true; + } + + private boolean validateNetwork(Network network, int clusterTotalNodeCount) { + NetworkOffering networkOffering = networkOfferingDao.findById(network.getNetworkOfferingId()); + if (networkOffering.isSystemOnly()) { + throw new InvalidParameterValueException(String.format("Network ID: %s is for system use only", network.getUuid())); + } + if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.UserData)) { + throw new InvalidParameterValueException(String.format("Network ID: %s does not support userdata that is required for Kubernetes cluster", network.getUuid())); + } + if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.Firewall)) { + throw new InvalidParameterValueException(String.format("Network ID: %s does not support firewall that is required for Kubernetes cluster", network.getUuid())); + } + if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.PortForwarding)) { + throw new InvalidParameterValueException(String.format("Network ID: %s does not support port forwarding that is required for Kubernetes cluster", network.getUuid())); + } + if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.Dhcp)) { + throw new InvalidParameterValueException(String.format("Network ID: %s does not support DHCP that is required for Kubernetes cluster", network.getUuid())); + } + validateIsolatedNetwork(network, clusterTotalNodeCount); + return true; + } + + private boolean validateServiceOffering(ServiceOffering serviceOffering) { + if (serviceOffering.isDynamic()) { + throw new InvalidParameterValueException(String.format("Custom service offerings are not supported for creating clusters, service offering ID: %s", serviceOffering.getUuid())); + } + if (serviceOffering.getCpu() < 2 || serviceOffering.getRamSize() < 2048) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster cannot be created with service offering ID: %s, Kubernetes cluster template(CoreOS) needs minimum 2 vCPUs and 2 GB RAM", serviceOffering.getUuid())); + } + return true; + } + + private void validateDockerRegistryParams(final String dockerRegistryUserName, + final String dockerRegistryPassword, + final String dockerRegistryUrl, + final String dockerRegistryEmail) { + // if no params related to docker registry specified then nothing to validate so return true + if ((dockerRegistryUserName == null || dockerRegistryUserName.isEmpty()) && + (dockerRegistryPassword == null || dockerRegistryPassword.isEmpty()) && + (dockerRegistryUrl == null || dockerRegistryUrl.isEmpty()) && + (dockerRegistryEmail == null || dockerRegistryEmail.isEmpty())) { + return; + } + + // all params related to docker registry must be specified or nothing + if (!((dockerRegistryUserName != null && !dockerRegistryUserName.isEmpty()) && + (dockerRegistryPassword != null && !dockerRegistryPassword.isEmpty()) && + (dockerRegistryUrl != null && !dockerRegistryUrl.isEmpty()) && + (dockerRegistryEmail != null && !dockerRegistryEmail.isEmpty()))) { + throw new InvalidParameterValueException("All the docker private registry parameters (username, password, url, email) required are specified"); + } + + try { + URL url = new URL(dockerRegistryUrl); + } catch (MalformedURLException e) { + throw new InvalidParameterValueException("Invalid docker registry url specified"); + } + + Pattern VALID_EMAIL_ADDRESS_REGEX = Pattern.compile("^[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,6}$", Pattern.CASE_INSENSITIVE); + Matcher matcher = VALID_EMAIL_ADDRESS_REGEX.matcher(dockerRegistryEmail); + if (!matcher.find()) { + throw new InvalidParameterValueException("Invalid docker registry email specified"); + } + } + + private DeployDestination plan(final long nodesCount, final DataCenter zone, final ServiceOffering offering) throws InsufficientServerCapacityException { + final int cpu_requested = offering.getCpu() * offering.getSpeed(); + final long ram_requested = offering.getRamSize() * 1024L * 1024L; + List hosts = resourceManager.listAllHostsInOneZoneByType(Type.Routing, zone.getId()); + final Map> hosts_with_resevered_capacity = new ConcurrentHashMap>(); + for (HostVO h : hosts) { + hosts_with_resevered_capacity.put(h.getUuid(), new Pair(h, 0)); + } + boolean suitable_host_found = false; + for (int i = 1; i <= nodesCount + 1; i++) { + suitable_host_found = false; + for (Map.Entry> hostEntry : hosts_with_resevered_capacity.entrySet()) { + Pair hp = hostEntry.getValue(); + HostVO h = hp.first(); + int reserved = hp.second(); + reserved++; + ClusterVO cluster = clusterDao.findById(h.getClusterId()); + ClusterDetailsVO cluster_detail_cpu = clusterDetailsDao.findDetail(cluster.getId(), "cpuOvercommitRatio"); + ClusterDetailsVO cluster_detail_ram = clusterDetailsDao.findDetail(cluster.getId(), "memoryOvercommitRatio"); + Float cpuOvercommitRatio = Float.parseFloat(cluster_detail_cpu.getValue()); + Float memoryOvercommitRatio = Float.parseFloat(cluster_detail_ram.getValue()); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Checking host ID: %s for capacity already reserved %d", h.getUuid(), reserved)); + } + if (capacityManager.checkIfHostHasCapacity(h.getId(), cpu_requested * reserved, ram_requested * reserved, false, cpuOvercommitRatio, memoryOvercommitRatio, true)) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Found host ID: %s for with enough capacity, CPU=%d RAM=%d", h.getUuid(), cpu_requested * reserved, ram_requested * reserved)); + } + hostEntry.setValue(new Pair(h, reserved)); + suitable_host_found = true; + break; + } + } + if (!suitable_host_found) { + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Suitable hosts not found in datacenter ID: %s for node %d", zone.getUuid(), i)); + } + break; + } + } + if (suitable_host_found) { + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Suitable hosts found in datacenter ID: %s, creating deployment destination", zone.getUuid())); + } + return new DeployDestination(zone, null, null, null); + } + String msg = String.format("Cannot find enough capacity for Kubernetes cluster(requested cpu=%1$s memory=%2$s)", + cpu_requested * nodesCount, ram_requested * nodesCount); + LOGGER.warn(msg); + throw new InsufficientServerCapacityException(msg, DataCenter.class, zone.getId()); + } + + @Override + public KubernetesClusterResponse createKubernetesClusterResponse(long kubernetesClusterId) { + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + KubernetesClusterResponse response = new KubernetesClusterResponse(); + response.setObjectName(KubernetesCluster.class.getSimpleName().toLowerCase()); + response.setId(kubernetesCluster.getUuid()); + response.setName(kubernetesCluster.getName()); + response.setDescription(kubernetesCluster.getDescription()); + DataCenterVO zone = ApiDBUtils.findZoneById(kubernetesCluster.getZoneId()); + response.setZoneId(zone.getUuid()); + response.setZoneName(zone.getName()); + response.setMasterNodes(kubernetesCluster.getMasterNodeCount()); + response.setClusterSize(kubernetesCluster.getNodeCount()); + VMTemplateVO template = ApiDBUtils.findTemplateById(kubernetesCluster.getTemplateId()); + response.setTemplateId(template.getUuid()); + ServiceOfferingVO offering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); + response.setServiceOfferingId(offering.getUuid()); + response.setServiceOfferingName(offering.getName()); + KubernetesSupportedVersionVO version = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); + if (version != null) { + response.setKubernetesVersionId(version.getUuid()); + response.setKubernetesVersionName(version.getName()); + } + response.setKeypair(kubernetesCluster.getKeyPair()); + response.setState(kubernetesCluster.getState().toString()); + response.setCores(String.valueOf(kubernetesCluster.getCores())); + response.setMemory(String.valueOf(kubernetesCluster.getMemory())); + NetworkVO ntwk = networkDao.findByIdIncludingRemoved(kubernetesCluster.getNetworkId()); + response.setEndpoint(kubernetesCluster.getEndpoint()); + response.setNetworkId(ntwk.getUuid()); + response.setAssociatedNetworkName(ntwk.getName()); + List vmIds = new ArrayList(); + List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + if (vmList != null && !vmList.isEmpty()) { + for (KubernetesClusterVmMapVO vmMapVO : vmList) { + UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId()); + if (userVM != null) { + vmIds.add(userVM.getUuid()); + } + } + } + response.setVirtualMachineIds(vmIds); + return response; + } + + private void validateKubernetesClusterCreateParameters(final CreateKubernetesClusterCmd cmd) throws ManagementServerException { + final String name = cmd.getName(); + final Long zoneId = cmd.getZoneId(); + final Long kubernetesVersionId = cmd.getKubernetesVersionId(); + final Long serviceOfferingId = cmd.getServiceOfferingId(); + final Account owner = accountService.getActiveAccountById(cmd.getEntityOwnerId()); + final Long networkId = cmd.getNetworkId(); + final String sshKeyPair = cmd.getSSHKeyPairName(); + final Long masterNodeCount = cmd.getMasterNodes(); + final Long clusterSize = cmd.getClusterSize(); + final String dockerRegistryUserName = cmd.getDockerRegistryUserName(); + final String dockerRegistryPassword = cmd.getDockerRegistryPassword(); + final String dockerRegistryUrl = cmd.getDockerRegistryUrl(); + final String dockerRegistryEmail = cmd.getDockerRegistryEmail(); + final Long nodeRootDiskSize = cmd.getNodeRootDiskSize(); + final String externalLoadBalancerIpAddress = cmd.getExternalLoadBalancerIpAddress(); + + if (name == null || name.isEmpty()) { + throw new InvalidParameterValueException("Invalid name for the Kubernetes cluster name:" + name); + } + + if (masterNodeCount < 1 || masterNodeCount > 100) { + throw new InvalidParameterValueException("Invalid cluster master nodes count: " + masterNodeCount); + } + + if (clusterSize < 1 || clusterSize > 100) { + throw new InvalidParameterValueException("Invalid cluster size: " + clusterSize); + } + + DataCenter zone = dataCenterDao.findById(zoneId); + if (zone == null) { + throw new InvalidParameterValueException("Unable to find zone by ID: " + zoneId); + } + + if (Grouping.AllocationState.Disabled == zone.getAllocationState()) { + throw new PermissionDeniedException(String.format("Cannot perform this operation, zone ID: %s is currently disabled", zone.getUuid())); + } + + if (!isKubernetesServiceConfigured(zone)) { + throw new CloudRuntimeException("Kubernetes service has not been configured properly to provision Kubernetes clusters"); + } + + final KubernetesSupportedVersion clusterKubernetesVersion = kubernetesSupportedVersionDao.findById(kubernetesVersionId); + if (clusterKubernetesVersion == null) { + throw new InvalidParameterValueException("Unable to find given Kubernetes version in supported versions"); + } + if (clusterKubernetesVersion.getZoneId() != null && !clusterKubernetesVersion.getZoneId().equals(zone.getId())) { + throw new InvalidParameterValueException(String.format("Kubernetes version ID: %s is not available for zone ID: %s", clusterKubernetesVersion.getUuid(), zone.getUuid())); + } + if (masterNodeCount > 1 ) { + try { + if (KubernetesVersionManagerImpl.compareSemanticVersions(clusterKubernetesVersion.getSemanticVersion(), MIN_KUBERNETES_VERSION_HA_SUPPORT) < 0) { + throw new InvalidParameterValueException(String.format("HA support is available only for Kubernetes version %s and above. Given version ID: %s is %s", MIN_KUBERNETES_VERSION_HA_SUPPORT, clusterKubernetesVersion.getUuid(), clusterKubernetesVersion.getSemanticVersion())); + } + } catch (IllegalArgumentException e) { + logAndThrow(Level.WARN, String.format("Unable to compare Kubernetes version for given version ID: %s with %s", clusterKubernetesVersion.getUuid(), MIN_KUBERNETES_VERSION_HA_SUPPORT), e); + } + } + + if (clusterKubernetesVersion.getZoneId() != null && clusterKubernetesVersion.getZoneId() != zone.getId()) { + throw new InvalidParameterValueException(String.format("Kubernetes version ID: %s is not available for zone ID: %s", clusterKubernetesVersion.getUuid(), zone.getUuid())); + } + + TemplateJoinVO iso = templateJoinDao.findById(clusterKubernetesVersion.getIsoId()); + if (iso == null) { + throw new InvalidParameterValueException(String.format("Invalid ISO associated with version ID: %s", clusterKubernetesVersion.getUuid())); + } + if (!ObjectInDataStoreStateMachine.State.Ready.equals(iso.getState())) { + throw new InvalidParameterValueException(String.format("ISO associated with version ID: %s is not in Ready state", clusterKubernetesVersion.getUuid())); + } + + ServiceOffering serviceOffering = serviceOfferingDao.findById(serviceOfferingId); + if (serviceOffering == null) { + throw new InvalidParameterValueException("No service offering with ID: " + serviceOfferingId); + } + + if (sshKeyPair != null && !sshKeyPair.isEmpty()) { + SSHKeyPairVO sshKeyPairVO = sshKeyPairDao.findByName(owner.getAccountId(), owner.getDomainId(), sshKeyPair); + if (sshKeyPairVO == null) { + throw new InvalidParameterValueException(String.format("Given SSH key pair with name: %s was not found for the account %s", sshKeyPair, owner.getAccountName())); + } + } + + if (nodeRootDiskSize != null && nodeRootDiskSize <= 0) { + throw new InvalidParameterValueException(String.format("Invalid value for %s", ApiConstants.NODE_ROOT_DISK_SIZE)); + } + + VMTemplateVO template = templateDao.findByTemplateName(KubernetesClusterTemplateName.value()); + List listZoneTemplate = templateZoneDao.listByZoneTemplate(zone.getId(), template.getId()); + if (listZoneTemplate == null || listZoneTemplate.isEmpty()) { + logAndThrow(Level.WARN, String.format("The template ID: %s is not available for use in zone ID: %s to provision Kubernetes cluster name: %s", template.getUuid(), zone.getUuid(), name)); + } + + if (!validateServiceOffering(serviceOfferingDao.findById(serviceOfferingId))) { + throw new InvalidParameterValueException("Given service offering ID: %s is not suitable for Kubernetes cluster"); + } + + validateDockerRegistryParams(dockerRegistryUserName, dockerRegistryPassword, dockerRegistryUrl, dockerRegistryEmail); + + Network network = null; + if (networkId != null) { + network = networkService.getNetwork(networkId); + if (network == null) { + throw new InvalidParameterValueException("Unable to find network with given ID"); + } + } + + if (!Strings.isNullOrEmpty(externalLoadBalancerIpAddress)) { + if (!NetUtils.isValidIp4(externalLoadBalancerIpAddress) && !NetUtils.isValidIp6(externalLoadBalancerIpAddress)) { + throw new InvalidParameterValueException("Invalid external load balancer IP address"); + } + if (network == null) { + throw new InvalidParameterValueException(String.format("%s parameter must be specified along with %s parameter", ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS, ApiConstants.NETWORK_ID)); + } + if (Network.GuestType.Shared.equals(network.getGuestType())) { + throw new InvalidParameterValueException(String.format("%s parameter must be specified along with %s type of network", ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS, Network.GuestType.Shared.toString())); + } + } + } + + private Network getKubernetesClusterNetworkIfMissing(final String clusterName, final DataCenter zone, final Account owner, final int masterNodesCount, + final int nodesCount, final String externalLoadBalancerIpAddress, final Long networkId) throws ManagementServerException { + Network network = null; + if (networkId != null) { + network = networkDao.findById(networkId); + if (Network.GuestType.Isolated.equals(network.getGuestType())) { + if (kubernetesClusterDao.listByNetworkId(network.getId()).isEmpty()) { + if (!validateNetwork(network, masterNodesCount + nodesCount)) { + throw new InvalidParameterValueException(String.format("Network ID: %s is not suitable for Kubernetes cluster", network.getUuid())); + } + networkModel.checkNetworkPermissions(owner, network); + } else { + throw new InvalidParameterValueException(String.format("Network ID: %s is already under use by another Kubernetes cluster", network.getUuid())); + } + } else if (Network.GuestType.Shared.equals(network.getGuestType())) { + if (masterNodesCount > 1 && Strings.isNullOrEmpty(externalLoadBalancerIpAddress)) { + throw new InvalidParameterValueException(String.format("Multi-master, HA Kubernetes cluster with %s network ID: %s needs an external load balancer IP address. %s parameter can be used", + network.getGuestType().toString(), network.getUuid(), ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS)); + } + } + } else { // user has not specified network in which cluster VM's to be provisioned, so create a network for Kubernetes cluster + NetworkOfferingVO networkOffering = networkOfferingDao.findByUniqueName(KubernetesClusterNetworkOffering.value()); + + long physicalNetworkId = networkModel.findPhysicalNetworkId(zone.getId(), networkOffering.getTags(), networkOffering.getTrafficType()); + PhysicalNetwork physicalNetwork = physicalNetworkDao.findById(physicalNetworkId); + + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Creating network for account ID: %s from the network offering ID: %s as part of Kubernetes cluster: %s deployment process", owner.getUuid(), networkOffering.getUuid(), clusterName)); + } + + try { + network = networkMgr.createGuestNetwork(networkOffering.getId(), clusterName + "-network", owner.getAccountName() + "-network", + null, null, null, false, null, owner, null, physicalNetwork, zone.getId(), ControlledEntity.ACLType.Account, null, null, null, null, true, null, null); + } catch (ConcurrentOperationException | InsufficientCapacityException | ResourceAllocationException e) { + logAndThrow(Level.ERROR, String.format("Unable to create network for the Kubernetes cluster: %s", clusterName)); + } + } + return network; + } + + private void addKubernetesClusterDetails(final KubernetesCluster kubernetesCluster, final Network network, final CreateKubernetesClusterCmd cmd) { + final String externalLoadBalancerIpAddress = cmd.getExternalLoadBalancerIpAddress(); + final String dockerRegistryUserName = cmd.getDockerRegistryUserName(); + final String dockerRegistryPassword = cmd.getDockerRegistryPassword(); + final String dockerRegistryUrl = cmd.getDockerRegistryUrl(); + final String dockerRegistryEmail = cmd.getDockerRegistryEmail(); + final boolean networkCleanup = cmd.getNetworkId() == null; + Transaction.execute(new TransactionCallbackNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + List details = new ArrayList<>(); + if (Network.GuestType.Shared.equals(network.getGuestType()) && !Strings.isNullOrEmpty(externalLoadBalancerIpAddress)) { + details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS, externalLoadBalancerIpAddress, true)); + } + if (!Strings.isNullOrEmpty(dockerRegistryUserName)) { + details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), ApiConstants.DOCKER_REGISTRY_USER_NAME, dockerRegistryUserName, true)); + } + if (!Strings.isNullOrEmpty(dockerRegistryPassword)) { + details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), ApiConstants.DOCKER_REGISTRY_PASSWORD, dockerRegistryPassword, false)); + } + if (!Strings.isNullOrEmpty(dockerRegistryUrl)) { + details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), ApiConstants.DOCKER_REGISTRY_URL, dockerRegistryUrl, true)); + } + if (!Strings.isNullOrEmpty(dockerRegistryEmail)) { + details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), ApiConstants.DOCKER_REGISTRY_EMAIL, dockerRegistryEmail, true)); + } + details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), ApiConstants.USERNAME, "admin", true)); + SecureRandom random = new SecureRandom(); + String randomPassword = new BigInteger(130, random).toString(32); + details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), ApiConstants.PASSWORD, randomPassword, false)); + details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), "networkCleanup", String.valueOf(networkCleanup), true)); + kubernetesClusterDetailsDao.saveDetails(details); + } + }); + } + + private void validateKubernetesClusterScaleParameters(ScaleKubernetesClusterCmd cmd) { + final Long kubernetesClusterId = cmd.getId(); + final Long serviceOfferingId = cmd.getServiceOfferingId(); + final Long clusterSize = cmd.getClusterSize(); + if (kubernetesClusterId == null || kubernetesClusterId < 1L) { + throw new InvalidParameterValueException("Invalid Kubernetes cluster ID"); + } + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + if (kubernetesCluster == null || kubernetesCluster.getRemoved() != null) { + throw new InvalidParameterValueException("Invalid Kubernetes cluster ID"); + } + final DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); + if (zone == null) { + logAndThrow(Level.WARN, String.format("Unable to find zone for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + } + + Account caller = CallContext.current().getCallingAccount(); + accountManager.checkAccess(caller, SecurityChecker.AccessType.OperateEntry, false, kubernetesCluster); + + if (serviceOfferingId == null && clusterSize == null) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be scaled, either a new service offering or a new cluster size must be passed", kubernetesCluster.getUuid())); + } + + ServiceOffering serviceOffering = null; + if (serviceOfferingId != null) { + serviceOffering = serviceOfferingDao.findById(serviceOfferingId); + if (serviceOffering == null) { + throw new InvalidParameterValueException("Failed to find service offering ID: " + serviceOfferingId); + } else { + if (serviceOffering.isDynamic()) { + throw new InvalidParameterValueException(String.format("Custom service offerings are not supported for Kubernetes clusters. Kubernetes cluster ID: %s, service offering ID: %s", kubernetesCluster.getUuid(), serviceOffering.getUuid())); + } + if (serviceOffering.getCpu() < 2 || serviceOffering.getRamSize() < 2048) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be scaled with service offering ID: %s, Kubernetes cluster template(CoreOS) needs minimum 2 vCPUs and 2 GB RAM", kubernetesCluster.getUuid(), serviceOffering.getUuid())); + } + } + } + + if (!(kubernetesCluster.getState().equals(KubernetesCluster.State.Created) || + kubernetesCluster.getState().equals(KubernetesCluster.State.Running) || + kubernetesCluster.getState().equals(KubernetesCluster.State.Stopped))) { + throw new PermissionDeniedException(String.format("Kubernetes cluster ID: %s is in %s state", kubernetesCluster.getUuid(), kubernetesCluster.getState().toString())); + } + + if (clusterSize != null) { + if (kubernetesCluster.getState().equals(KubernetesCluster.State.Stopped)) { // Cannot scale stopped cluster currently for cluster size + throw new PermissionDeniedException(String.format("Kubernetes cluster ID: %s is in %s state", kubernetesCluster.getUuid(), kubernetesCluster.getState().toString())); + } + if (clusterSize < 1) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be scaled for size, %d", kubernetesCluster.getUuid(), clusterSize)); + } + } + } + + private void validateKubernetesClusterUpgradeParameters(UpgradeKubernetesClusterCmd cmd) { + // Validate parameters + final Long kubernetesClusterId = cmd.getId(); + final Long upgradeVersionId = cmd.getKubernetesVersionId(); + if (kubernetesClusterId == null || kubernetesClusterId < 1L) { + throw new InvalidParameterValueException("Invalid Kubernetes cluster ID"); + } + if (upgradeVersionId == null || upgradeVersionId < 1L) { + throw new InvalidParameterValueException("Invalid Kubernetes version ID"); + } + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + if (kubernetesCluster == null || kubernetesCluster.getRemoved() != null) { + throw new InvalidParameterValueException("Invalid Kubernetes cluster ID"); + } + accountManager.checkAccess(CallContext.current().getCallingAccount(), SecurityChecker.AccessType.OperateEntry, false, kubernetesCluster); + if (!KubernetesCluster.State.Running.equals(kubernetesCluster.getState())) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s is not in running state", kubernetesCluster.getUuid())); + } + KubernetesSupportedVersionVO upgradeVersion = kubernetesSupportedVersionDao.findById(upgradeVersionId); + if (upgradeVersion == null || upgradeVersion.getRemoved() != null) { + throw new InvalidParameterValueException("Invalid Kubernetes version ID"); + } + KubernetesSupportedVersionVO clusterVersion = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); + if (clusterVersion == null || clusterVersion.getRemoved() != null) { + throw new InvalidParameterValueException(String.format("Invalid Kubernetes version associated with cluster ID: %s", + kubernetesCluster.getUuid())); + } + // Check upgradeVersion is either patch upgrade or immediate minor upgrade + try { + KubernetesVersionManagerImpl.canUpgradeKubernetesVersion(clusterVersion.getSemanticVersion(), upgradeVersion.getSemanticVersion()); + } catch (IllegalArgumentException e) { + throw new InvalidParameterValueException(e.getMessage()); + } + + TemplateJoinVO iso = templateJoinDao.findById(upgradeVersion.getIsoId()); + if (iso == null) { + throw new InvalidParameterValueException(String.format("Invalid ISO associated with version ID: %s", upgradeVersion.getUuid())); + } + if (!ObjectInDataStoreStateMachine.State.Ready.equals(iso.getState())) { + throw new InvalidParameterValueException(String.format("ISO associated with version ID: %s is not in Ready state", upgradeVersion.getUuid())); + } + } + + protected boolean stateTransitTo(long kubernetesClusterId, KubernetesCluster.Event e) { + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + try { + return _stateMachine.transitTo(kubernetesCluster, e, null, kubernetesClusterDao); + } catch (NoTransitionException nte) { + LOGGER.warn(String.format("Failed to transition state of the Kubernetes cluster ID: %s in state %s on event %s", kubernetesCluster.getUuid(), kubernetesCluster.getState().toString(), e.toString()), nte); + return false; + } + } + + @Override + public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) + throws InsufficientCapacityException, ManagementServerException { + if (!KubernetesServiceEnabled.value()) { + logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); + } + + validateKubernetesClusterCreateParameters(cmd); + + final DataCenter zone = dataCenterDao.findById(cmd.getZoneId()); + final long masterNodeCount = cmd.getMasterNodes(); + final long clusterSize = cmd.getClusterSize(); + final long totalNodeCount = masterNodeCount + clusterSize; + final ServiceOffering serviceOffering = serviceOfferingDao.findById(cmd.getServiceOfferingId()); + final Account owner = accountService.getActiveAccountById(cmd.getEntityOwnerId()); + final KubernetesSupportedVersion clusterKubernetesVersion = kubernetesSupportedVersionDao.findById(cmd.getKubernetesVersionId()); + + plan(totalNodeCount, zone, serviceOffering); + + final Network defaultNetwork = getKubernetesClusterNetworkIfMissing(cmd.getName(), zone, owner, (int)masterNodeCount, (int)clusterSize, cmd.getExternalLoadBalancerIpAddress(), cmd.getNetworkId()); + final VMTemplateVO finalTemplate = templateDao.findByTemplateName(KubernetesClusterTemplateName.value());; + final long cores = serviceOffering.getCpu() * (masterNodeCount + clusterSize); + final long memory = serviceOffering.getRamSize() * (masterNodeCount + clusterSize); + + final KubernetesClusterVO cluster = Transaction.execute(new TransactionCallback() { + @Override + public KubernetesClusterVO doInTransaction(TransactionStatus status) { + KubernetesClusterVO newCluster = new KubernetesClusterVO(cmd.getName(), cmd.getDisplayName(), zone.getId(), clusterKubernetesVersion.getId(), + serviceOffering.getId(), finalTemplate.getId(), defaultNetwork.getId(), owner.getDomainId(), + owner.getAccountId(), masterNodeCount, clusterSize, KubernetesCluster.State.Created, cmd.getSSHKeyPairName(), cores, memory, cmd.getNodeRootDiskSize(), ""); + kubernetesClusterDao.persist(newCluster); + return newCluster; + } + }); + + addKubernetesClusterDetails(cluster, defaultNetwork, cmd); + + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Kubernetes cluster name: %s and ID: %s has been created", cluster.getName(), cluster.getUuid())); + } + return cluster; + } + + /** + * Start operation can be performed at two different life stages of Kubernetes cluster. First when a freshly created cluster + * in which case there are no resources provisioned for the Kubernetes cluster. So during start all the resources + * are provisioned from scratch. Second kind of start, happens on Stopped Kubernetes cluster, in which all resources + * are provisioned (like volumes, nics, networks etc). It just that VM's are not in running state. So just + * start the VM's (which can possibly implicitly start the network also). + * @param kubernetesClusterId + * @param onCreate + * @return + * @throws ManagementServerException + * @throws ResourceAllocationException + * @throws ResourceUnavailableException + * @throws InsufficientCapacityException + */ + + @Override + public boolean startKubernetesCluster(long kubernetesClusterId, boolean onCreate) throws ManagementServerException, + ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException { + if (!KubernetesServiceEnabled.value()) { + logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); + } + final KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + if (kubernetesCluster == null) { + throw new InvalidParameterValueException("Failed to find Kubernetes cluster with given ID"); + } + if (kubernetesCluster.getRemoved() != null) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s is already deleted", kubernetesCluster.getUuid())); + } + accountManager.checkAccess(CallContext.current().getCallingAccount(), SecurityChecker.AccessType.OperateEntry, false, kubernetesCluster); + if (kubernetesCluster.getState().equals(KubernetesCluster.State.Running)) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Kubernetes cluster ID: %s is in running state", kubernetesCluster.getUuid())); + } + return true; + } + if (kubernetesCluster.getState().equals(KubernetesCluster.State.Starting)) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Kubernetes cluster ID: %s is already in starting state", kubernetesCluster.getUuid())); + } + return true; + } + final DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); + if (zone == null) { + throw new CloudRuntimeException(String.format("Unable to find zone for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + } + KubernetesClusterStartWorker startWorker = + new KubernetesClusterStartWorker(kubernetesCluster, this); + startWorker = ComponentContext.inject(startWorker); + if (onCreate) { + // Start for Kubernetes cluster in 'Created' state + return startWorker.startKubernetesClusterOnCreate(); + } else { + // Start for Kubernetes cluster in 'Stopped' state. Resources are already provisioned, just need to be started + return startWorker.startStoppedKubernetesCluster(); + } + } + + @Override + public boolean stopKubernetesCluster(long kubernetesClusterId) throws CloudRuntimeException { + if (!KubernetesServiceEnabled.value()) { + logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); + } + final KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + if (kubernetesCluster == null) { + throw new InvalidParameterValueException("Failed to find Kubernetes cluster with given ID"); + } + if (kubernetesCluster.getRemoved() != null) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s is already deleted", kubernetesCluster.getUuid())); + } + accountManager.checkAccess(CallContext.current().getCallingAccount(), SecurityChecker.AccessType.OperateEntry, false, kubernetesCluster); + if (kubernetesCluster.getState().equals(KubernetesCluster.State.Stopped)) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Kubernetes cluster ID: %s is already stopped", kubernetesCluster.getUuid())); + } + return true; + } + if (kubernetesCluster.getState().equals(KubernetesCluster.State.Stopping)) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Kubernetes cluster ID: %s is getting stopped", kubernetesCluster.getUuid())); + } + return true; + } + KubernetesClusterStopWorker stopWorker = new KubernetesClusterStopWorker(kubernetesCluster, this); + stopWorker = ComponentContext.inject(stopWorker); + return stopWorker.stop(); + } + + @Override + public boolean deleteKubernetesCluster(Long kubernetesClusterId) throws ManagementServerException { + if (!KubernetesServiceEnabled.value()) { + logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); + } + KubernetesClusterVO cluster = kubernetesClusterDao.findById(kubernetesClusterId); + if (cluster == null) { + throw new InvalidParameterValueException("Invalid cluster id specified"); + } + accountManager.checkAccess(CallContext.current().getCallingAccount(), SecurityChecker.AccessType.OperateEntry, false, cluster); + KubernetesClusterDestroyWorker destroyWorker = new KubernetesClusterDestroyWorker(cluster, this); + destroyWorker = ComponentContext.inject(destroyWorker); + return destroyWorker.destroy(); + } + + @Override + public ListResponse listKubernetesClusters(ListKubernetesClustersCmd cmd) { + if (!KubernetesServiceEnabled.value()) { + logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); + } + final CallContext ctx = CallContext.current(); + final Account caller = ctx.getCallingAccount(); + final Long clusterId = cmd.getId(); + final String state = cmd.getState(); + final String name = cmd.getName(); + + List responsesList = new ArrayList(); + if (state != null && !state.isEmpty()) { + if (!KubernetesCluster.State.Running.toString().equals(state) && + !KubernetesCluster.State.Stopped.toString().equals(state) && + !KubernetesCluster.State.Destroyed.toString().equals(state)) { + throw new InvalidParameterValueException("Invalid value for Kubernetes cluster state specified"); + } + } + if (clusterId != null) { + KubernetesClusterVO cluster = kubernetesClusterDao.findById(clusterId); + if (cluster == null) { + throw new InvalidParameterValueException("Invalid Kubernetes cluster ID specified"); + } + accountManager.checkAccess(caller, SecurityChecker.AccessType.ListEntry, false, cluster); + responsesList.add(createKubernetesClusterResponse(clusterId)); + } else { + SearchCriteria sc = kubernetesClusterDao.createSearchCriteria(); + Filter searchFilter = new Filter(KubernetesClusterVO.class, "id", true, cmd.getStartIndex(), cmd.getPageSizeVal()); + if (state != null && !state.isEmpty()) { + sc.addAnd("state", SearchCriteria.Op.EQ, state); + } + if (accountManager.isNormalUser(caller.getId())) { + sc.addAnd("accountId", SearchCriteria.Op.EQ, caller.getAccountId()); + } else if (accountManager.isDomainAdmin(caller.getId())) { + sc.addAnd("domainId", SearchCriteria.Op.EQ, caller.getDomainId()); + } + if (name != null && !name.isEmpty()) { + sc.addAnd("name", SearchCriteria.Op.LIKE, name); + } + List kubernetesClusters = kubernetesClusterDao.search(sc, searchFilter); + for (KubernetesClusterVO cluster : kubernetesClusters) { + KubernetesClusterResponse clusterResponse = createKubernetesClusterResponse(cluster.getId()); + responsesList.add(clusterResponse); + } + } + ListResponse response = new ListResponse(); + response.setResponses(responsesList); + return response; + } + + public KubernetesClusterConfigResponse getKubernetesClusterConfig(GetKubernetesClusterConfigCmd cmd) { + if (!KubernetesServiceEnabled.value()) { + logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); + } + final Long clusterId = cmd.getId(); + KubernetesCluster kubernetesCluster = kubernetesClusterDao.findById(clusterId); + if (kubernetesCluster == null) { + throw new InvalidParameterValueException("Invalid Kubernetes cluster ID specified"); + } + KubernetesClusterConfigResponse response = new KubernetesClusterConfigResponse(); + response.setId(kubernetesCluster.getUuid()); + response.setName(kubernetesCluster.getName()); + String configData = ""; + KubernetesClusterDetailsVO clusterDetailsVO = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "kubeConfigData"); + if (clusterDetailsVO != null && !Strings.isNullOrEmpty(clusterDetailsVO.getValue())) { + configData = new String(Base64.decodeBase64(clusterDetailsVO.getValue())); + } + response.setConfigData(configData); + response.setObjectName("clusterconfig"); + return response; + } + + @Override + public boolean scaleKubernetesCluster(ScaleKubernetesClusterCmd cmd) throws CloudRuntimeException { + if (!KubernetesServiceEnabled.value()) { + logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); + } + validateKubernetesClusterScaleParameters(cmd); + KubernetesClusterScaleWorker scaleWorker = + new KubernetesClusterScaleWorker(kubernetesClusterDao.findById(cmd.getId()), + serviceOfferingDao.findById(cmd.getServiceOfferingId()), cmd.getClusterSize(), this); + scaleWorker = ComponentContext.inject(scaleWorker); + return scaleWorker.scaleCluster(); + } + + @Override + public boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws CloudRuntimeException { + if (!KubernetesServiceEnabled.value()) { + logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); + } + validateKubernetesClusterUpgradeParameters(cmd); + KubernetesClusterUpgradeWorker upgradeWorker = + new KubernetesClusterUpgradeWorker(kubernetesClusterDao.findById(cmd.getId()), + kubernetesSupportedVersionDao.findById(cmd.getKubernetesVersionId()), this); + upgradeWorker = ComponentContext.inject(upgradeWorker); + return upgradeWorker.upgradeCluster(); + } + + @Override + public List> getCommands() { + List> cmdList = new ArrayList>(); + if (!KubernetesServiceEnabled.value()) { + return cmdList; + } + cmdList.add(CreateKubernetesClusterCmd.class); + cmdList.add(StartKubernetesClusterCmd.class); + cmdList.add(StopKubernetesClusterCmd.class); + cmdList.add(DeleteKubernetesClusterCmd.class); + cmdList.add(ListKubernetesClustersCmd.class); + cmdList.add(GetKubernetesClusterConfigCmd.class); + cmdList.add(ScaleKubernetesClusterCmd.class); + cmdList.add(UpgradeKubernetesClusterCmd.class); + return cmdList; + } + + @Override + public KubernetesCluster findById(final Long id) { + return kubernetesClusterDao.findById(id); + } + + // Garbage collector periodically run through the Kubernetes clusters marked for GC. For each Kubernetes cluster + // marked for GC, attempt is made to destroy cluster. + public class KubernetesClusterGarbageCollector extends ManagedContextRunnable { + @Override + protected void runInContext() { + GlobalLock gcLock = GlobalLock.getInternLock("KubernetesCluster.GC.Lock"); + try { + if (gcLock.lock(3)) { + try { + reallyRun(); + } finally { + gcLock.unlock(); + } + } + } finally { + gcLock.releaseRef(); + } + } + + public void reallyRun() { + try { + List kubernetesClusters = kubernetesClusterDao.findKubernetesClustersToGarbageCollect(); + for (KubernetesCluster kubernetesCluster : kubernetesClusters) { + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Running Kubernetes cluster garbage collector on Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + } + try { + KubernetesClusterDestroyWorker destroyWorker = new KubernetesClusterDestroyWorker(kubernetesCluster, KubernetesClusterManagerImpl.this); + destroyWorker = ComponentContext.inject(destroyWorker); + if (destroyWorker.destroy()) { + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Garbage collection complete for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + } + } else { + LOGGER.warn(String.format("Garbage collection failed for Kubernetes cluster ID: %s, it will be attempted to garbage collected in next run", kubernetesCluster.getUuid())); + } + } catch (ManagementServerException e) { + LOGGER.warn(String.format("Failed to destroy Kubernetes cluster ID: %s during GC", kubernetesCluster.getUuid()), e); + // proceed further with rest of the Kubernetes cluster garbage collection + } + } + } catch (Exception e) { + LOGGER.warn("Caught exception while running Kubernetes cluster gc: ", e); + } + } + } + + /* Kubernetes cluster scanner checks if the Kubernetes cluster is in desired state. If it detects Kubernetes cluster + is not in desired state, it will trigger an event and marks the Kubernetes cluster to be 'Alert' state. For e.g a + Kubernetes cluster in 'Running' state should mean all the cluster of node VM's in the custer should be running and + number of the node VM's should be of cluster size, and the master node VM's is running. It is possible due to + out of band changes by user or hosts going down, we may end up one or more VM's in stopped state. in which case + scanner detects these changes and marks the cluster in 'Alert' state. Similarly cluster in 'Stopped' state means + all the cluster VM's are in stopped state any mismatch in states should get picked up by Kubernetes cluster and + mark the Kubernetes cluster to be 'Alert' state. Through recovery API, or reconciliation clusters in 'Alert' will + be brought back to known good state or desired state. + */ + public class KubernetesClusterStatusScanner extends ManagedContextRunnable { + private boolean firstRun = true; + @Override + protected void runInContext() { + GlobalLock gcLock = GlobalLock.getInternLock("KubernetesCluster.State.Scanner.Lock"); + try { + if (gcLock.lock(3)) { + try { + reallyRun(); + } finally { + gcLock.unlock(); + } + } + } finally { + gcLock.releaseRef(); + } + } + + public void reallyRun() { + try { + // run through Kubernetes clusters in 'Running' state and ensure all the VM's are Running in the cluster + List runningKubernetesClusters = kubernetesClusterDao.findKubernetesClustersInState(KubernetesCluster.State.Running); + for (KubernetesCluster kubernetesCluster : runningKubernetesClusters) { + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Running Kubernetes cluster state scanner on Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + } + try { + if (!isClusterVMsInDesiredState(kubernetesCluster, VirtualMachine.State.Running)) { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.FaultsDetected); + } + } catch (Exception e) { + LOGGER.warn(String.format("Failed to run Kubernetes cluster Running state scanner on Kubernetes cluster ID: %s status scanner", kubernetesCluster.getUuid()), e); + } + } + + // run through Kubernetes clusters in 'Stopped' state and ensure all the VM's are Stopped in the cluster + List stoppedKubernetesClusters = kubernetesClusterDao.findKubernetesClustersInState(KubernetesCluster.State.Stopped); + for (KubernetesCluster kubernetesCluster : stoppedKubernetesClusters) { + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Running Kubernetes cluster state scanner on Kubernetes cluster ID: %s for state: %s", kubernetesCluster.getUuid(), KubernetesCluster.State.Stopped.toString())); + } + try { + if (!isClusterVMsInDesiredState(kubernetesCluster, VirtualMachine.State.Stopped)) { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.FaultsDetected); + } + } catch (Exception e) { + LOGGER.warn(String.format("Failed to run Kubernetes cluster Stopped state scanner on Kubernetes cluster ID: %s status scanner", kubernetesCluster.getUuid()), e); + } + } + + // run through Kubernetes clusters in 'Alert' state and reconcile state as 'Running' if the VM's are running + List alertKubernetesClusters = kubernetesClusterDao.findKubernetesClustersInState(KubernetesCluster.State.Alert); + for (KubernetesClusterVO kubernetesCluster : alertKubernetesClusters) { + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Running Kubernetes cluster state scanner on Kubernetes cluster ID: %s for state: %s", kubernetesCluster.getUuid(), KubernetesCluster.State.Alert.toString())); + } + try { + if (isClusterVMsInDesiredState(kubernetesCluster, VirtualMachine.State.Running)) { + + KubernetesClusterStartWorker startWorker = + new KubernetesClusterStartWorker(kubernetesCluster, KubernetesClusterManagerImpl.this); + startWorker = ComponentContext.inject(startWorker); + startWorker.reconcileAlertCluster(); + } + } catch (Exception e) { + LOGGER.warn(String.format("Failed to run Kubernetes cluster Alert state scanner on Kubernetes cluster ID: %s status scanner", kubernetesCluster.getUuid()), e); + } + } + + + if (firstRun) { + // run through Kubernetes clusters in 'Starting' state and reconcile state as 'Alert' or 'Error' if the VM's are running + List startingKubernetesClusters = kubernetesClusterDao.findKubernetesClustersInState(KubernetesCluster.State.Starting); + for (KubernetesCluster kubernetesCluster : startingKubernetesClusters) { + if ((new Date()).getTime() - kubernetesCluster.getCreated().getTime() < 10*60*1000) { + continue; + } + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Running Kubernetes cluster state scanner on Kubernetes cluster ID: %s for state: %s", kubernetesCluster.getUuid(), KubernetesCluster.State.Starting.toString())); + } + try { + if (isClusterVMsInDesiredState(kubernetesCluster, VirtualMachine.State.Running)) { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.FaultsDetected); + } else { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + } catch (Exception e) { + LOGGER.warn(String.format("Failed to run Kubernetes cluster Starting state scanner on Kubernetes cluster ID: %s status scanner", kubernetesCluster.getUuid()), e); + } + } + List destroyingKubernetesClusters = kubernetesClusterDao.findKubernetesClustersInState(KubernetesCluster.State.Destroying); + for (KubernetesCluster kubernetesCluster : destroyingKubernetesClusters) { + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Running Kubernetes cluster state scanner on Kubernetes cluster ID: %s for state: %s", kubernetesCluster.getUuid(), KubernetesCluster.State.Destroying.toString())); + } + try { + KubernetesClusterDestroyWorker destroyWorker = new KubernetesClusterDestroyWorker(kubernetesCluster, KubernetesClusterManagerImpl.this); + destroyWorker = ComponentContext.inject(destroyWorker); + destroyWorker.destroy(); + } catch (Exception e) { + LOGGER.warn(String.format("Failed to run Kubernetes cluster Destroying state scanner on Kubernetes cluster ID: %s status scanner", kubernetesCluster.getUuid()), e); + } + } + } + } catch (Exception e) { + LOGGER.warn("Caught exception while running Kubernetes cluster state scanner", e); + } + firstRun = false; + } + } + + // checks if Kubernetes cluster is in desired state + boolean isClusterVMsInDesiredState(KubernetesCluster kubernetesCluster, VirtualMachine.State state) { + List clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + + // check cluster is running at desired capacity include master nodes as well + if (clusterVMs.size() < kubernetesCluster.getTotalNodeCount()) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Found only %d VMs in the Kubernetes cluster ID: %s while expected %d VMs to be in state: %s", + clusterVMs.size(), kubernetesCluster.getUuid(), kubernetesCluster.getTotalNodeCount(), state.toString())); + } + return false; + } + // check if all the VM's are in same state + for (KubernetesClusterVmMapVO clusterVm : clusterVMs) { + VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(clusterVm.getVmId()); + if (vm.getState() != state) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Found VM ID: %s in the Kubernetes cluster ID: %s in state: %s while expected to be in state: %s. So moving the cluster to Alert state for reconciliation", + vm.getUuid(), kubernetesCluster.getUuid(), vm.getState().toString(), state.toString())); + } + return false; + } + } + + return true; + } + + @Override + public boolean start() { + _gcExecutor.scheduleWithFixedDelay(new KubernetesClusterGarbageCollector(), 300, 300, TimeUnit.SECONDS); + _stateScanner.scheduleWithFixedDelay(new KubernetesClusterStatusScanner(), 300, 30, TimeUnit.SECONDS); + + return true; + } + + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + _name = name; + _configParams = params; + _gcExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("Kubernetes-Cluster-Scavenger")); + _stateScanner = Executors.newScheduledThreadPool(1, new NamedThreadFactory("Kubernetes-Cluster-State-Scanner")); + + return true; + } + + @Override + public String getConfigComponentName() { + return KubernetesClusterService.class.getSimpleName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[] { + KubernetesServiceEnabled, + KubernetesClusterTemplateName, + KubernetesClusterNetworkOffering + }; + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java similarity index 82% rename from plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterService.java rename to plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java index 5081f6318725..b873ec8c28fd 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java @@ -14,13 +14,13 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.cloud.kubernetescluster; +package com.cloud.kubernetes.cluster; -import org.apache.cloudstack.api.command.user.kubernetescluster.CreateKubernetesClusterCmd; -import org.apache.cloudstack.api.command.user.kubernetescluster.GetKubernetesClusterConfigCmd; -import org.apache.cloudstack.api.command.user.kubernetescluster.ListKubernetesClustersCmd; -import org.apache.cloudstack.api.command.user.kubernetescluster.ScaleKubernetesClusterCmd; -import org.apache.cloudstack.api.command.user.kubernetescluster.UpgradeKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.CreateKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.GetKubernetesClusterConfigCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.ListKubernetesClustersCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.ScaleKubernetesClusterCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.UpgradeKubernetesClusterCmd; import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse; import org.apache.cloudstack.api.response.KubernetesClusterResponse; import org.apache.cloudstack.api.response.ListResponse; @@ -31,6 +31,7 @@ import com.cloud.exception.ManagementServerException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.utils.component.PluggableService; +import com.cloud.utils.exception.CloudRuntimeException; public interface KubernetesClusterService extends PluggableService, Configurable { static final String MIN_KUBERNETES_VERSION_HA_SUPPORT = "1.16.0"; @@ -54,12 +55,12 @@ public interface KubernetesClusterService extends PluggableService, Configurable KubernetesCluster findById(final Long id); KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) throws InsufficientCapacityException, - ManagementServerException; + ManagementServerException, CloudRuntimeException; boolean startKubernetesCluster(long kubernetesClusterId, boolean onCreate) throws ManagementServerException, - ResourceUnavailableException, InsufficientCapacityException; + ResourceUnavailableException, InsufficientCapacityException, CloudRuntimeException; - boolean stopKubernetesCluster(long kubernetesClusterId) throws ManagementServerException; + boolean stopKubernetesCluster(long kubernetesClusterId) throws CloudRuntimeException; boolean deleteKubernetesCluster(Long kubernetesClusterId) throws ManagementServerException; @@ -69,8 +70,7 @@ boolean startKubernetesCluster(long kubernetesClusterId, boolean onCreate) throw KubernetesClusterResponse createKubernetesClusterResponse(long kubernetesClusterId); - boolean scaleKubernetesCluster(ScaleKubernetesClusterCmd cmd) throws ManagementServerException, - ResourceUnavailableException, InsufficientCapacityException; + boolean scaleKubernetesCluster(ScaleKubernetesClusterCmd cmd) throws CloudRuntimeException; - boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws ManagementServerException; + boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws CloudRuntimeException; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java similarity index 99% rename from plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVO.java rename to plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java index 17688c4a03f9..08e598336d0d 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVO.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.cloud.kubernetescluster; +package com.cloud.kubernetes.cluster; import java.util.Date; import java.util.UUID; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVmMap.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMap.java similarity index 96% rename from plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVmMap.java rename to plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMap.java index 6a82fddffdf4..8e5d6d4beda4 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVmMap.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMap.java @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.cloud.kubernetescluster; +package com.cloud.kubernetes.cluster; /** * KubernetesClusterVmMap will store a map of ID of KubernetesCuster diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVmMapVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMapVO.java similarity index 98% rename from plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVmMapVO.java rename to plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMapVO.java index 64d0bba3fc86..edb06e79534a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterVmMapVO.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMapVO.java @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.cloud.kubernetescluster; +package com.cloud.kubernetes.cluster; import javax.persistence.Column; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java new file mode 100644 index 000000000000..5004d0e6656f --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -0,0 +1,348 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.kubernetes.cluster.actionworkers; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; + +import javax.inject.Inject; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.ca.CAManager; +import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.io.IOUtils; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; + +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.dc.dao.VlanDao; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO; +import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterVO; +import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterDetailsDao; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; +import com.cloud.kubernetes.version.KubernetesSupportedVersion; +import com.cloud.kubernetes.version.dao.KubernetesSupportedVersionDao; +import com.cloud.network.IpAddress; +import com.cloud.network.IpAddressManager; +import com.cloud.network.Network; +import com.cloud.network.NetworkModel; +import com.cloud.network.dao.NetworkDao; +import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.storage.Storage; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.template.TemplateApiService; +import com.cloud.template.VirtualMachineTemplate; +import com.cloud.user.Account; +import com.cloud.user.dao.AccountDao; +import com.cloud.user.dao.SSHKeyPairDao; +import com.cloud.uservm.UserVm; +import com.cloud.utils.Pair; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.db.TransactionStatus; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.fsm.NoTransitionException; +import com.cloud.utils.fsm.StateMachine2; +import com.cloud.vm.UserVmService; +import com.cloud.vm.dao.UserVmDao; +import com.google.common.base.Strings; + +public class KubernetesClusterActionWorker { + + public static final String CLUSTER_NODE_VM_USER = "core"; + public static final int CLUSTER_API_PORT = 6443; + public static final int CLUSTER_NODES_DEFAULT_START_SSH_PORT = 2222; + + protected static final Logger LOGGER = Logger.getLogger(KubernetesClusterActionWorker.class); + + protected StateMachine2 _stateMachine = KubernetesCluster.State.getStateMachine(); + + @Inject + protected CAManager caManager; + @Inject + protected ConfigurationDao configurationDao; + @Inject + protected DataCenterDao dataCenterDao; + @Inject + protected AccountDao accountDao; + @Inject + protected IpAddressManager ipAddressManager; + @Inject + protected NetworkOrchestrationService networkMgr; + @Inject + protected NetworkDao networkDao; + @Inject + protected NetworkModel networkModel; + @Inject + protected ServiceOfferingDao serviceOfferingDao; + @Inject + protected SSHKeyPairDao sshKeyPairDao; + @Inject + protected VMTemplateDao templateDao; + @Inject + protected TemplateApiService templateService; + @Inject + protected UserVmDao userVmDao; + @Inject + protected UserVmService userVmService; + @Inject + protected VlanDao vlanDao; + + protected KubernetesClusterDao kubernetesClusterDao; + protected KubernetesClusterVmMapDao kubernetesClusterVmMapDao; + protected KubernetesClusterDetailsDao kubernetesClusterDetailsDao; + protected KubernetesSupportedVersionDao kubernetesSupportedVersionDao; + + protected KubernetesCluster kubernetesCluster; + protected Account owner; + protected File sshKeyFile; + protected String publicIpAddress; + protected int sshPort; + + protected KubernetesClusterActionWorker(final KubernetesCluster kubernetesCluster, final KubernetesClusterManagerImpl clusterManager) { + this.kubernetesCluster = kubernetesCluster; + this.kubernetesClusterDao = clusterManager.kubernetesClusterDao; + this.kubernetesClusterDetailsDao = clusterManager.kubernetesClusterDetailsDao; + this.kubernetesClusterVmMapDao = clusterManager.kubernetesClusterVmMapDao; + this.kubernetesSupportedVersionDao = clusterManager.kubernetesSupportedVersionDao; + } + + protected void init() { + this.owner = accountDao.findById(kubernetesCluster.getAccountId()); + this.sshKeyFile = getManagementServerSshPublicKeyFile(); + } + + protected String readResourceFile(String resource) throws IOException { + return IOUtils.toString(Objects.requireNonNull(Thread.currentThread().getContextClassLoader().getResourceAsStream(resource)), Charset.defaultCharset().name()); + } + + protected void logMessage(final Level logLevel, final String message, final Exception e) { + if (logLevel == Level.DEBUG) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(message); + } + } else if (logLevel == Level.ERROR) { + LOGGER.error(message); + } if (logLevel == Level.WARN) { + if (e != null) { + LOGGER.warn(message, e); + } else { + LOGGER.warn(message); + } + } else { + if (e != null) { + LOGGER.error(message, e); + } else { + LOGGER.error(message); + } + } + } + + protected void logTransitStateDetachIsoAndThrow(final Level logLevel, final String message, final KubernetesCluster kubernetesCluster, + final List clusterVMs, final KubernetesCluster.Event event, final Exception e) throws CloudRuntimeException { + logMessage(logLevel, message, e); + stateTransitTo(kubernetesCluster.getId(), event); + detachIsoKubernetesVMs(clusterVMs); + if (e == null) { + throw new CloudRuntimeException(message); + } + throw new CloudRuntimeException(message, e); + } + + protected void logTransitStateAndThrow(final Level logLevel, final String message, final Long kubernetesClusterId, final KubernetesCluster.Event event, final Exception e) throws CloudRuntimeException { + logMessage(logLevel, message, e); + if (kubernetesClusterId != null && event != null) { + stateTransitTo(kubernetesClusterId, event); + } + if (e == null) { + throw new CloudRuntimeException(message); + } + throw new CloudRuntimeException(message, e); + } + + protected void logTransitStateAndThrow(final Level logLevel, final String message, final Long kubernetesClusterId, final KubernetesCluster.Event event) throws CloudRuntimeException { + logTransitStateAndThrow(logLevel, message, kubernetesClusterId, event, null); + } + + protected void logAndThrow(final Level logLevel, final String message) throws CloudRuntimeException { + logTransitStateAndThrow(logLevel, message, null, null, null); + } + + protected void logAndThrow(final Level logLevel, final String message, final Exception ex) throws CloudRuntimeException { + logTransitStateAndThrow(logLevel, message, null, null, ex); + } + + protected File getManagementServerSshPublicKeyFile() { + boolean devel = Boolean.parseBoolean(configurationDao.getValue("developer")); + String keyFile = String.format("%s/.ssh/id_rsa", System.getProperty("user.home")); + if (devel) { + keyFile += ".cloud"; + } + return new File(keyFile); + } + + protected KubernetesClusterVmMapVO addKubernetesClusterVm(final long kubernetesClusterId, final long vmId) { + return Transaction.execute(new TransactionCallback() { + @Override + public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { + KubernetesClusterVmMapVO newClusterVmMap = new KubernetesClusterVmMapVO(kubernetesClusterId, vmId); + kubernetesClusterVmMapDao.persist(newClusterVmMap); + return newClusterVmMap; + } + }); + } + + private UserVm fetchMasterVmIfMissing(final UserVm masterVm) { + if (masterVm != null) { + return masterVm; + } + List clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + if (CollectionUtils.isEmpty(clusterVMs)) { + LOGGER.warn(String.format("Unable to retrieve VMs for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + return null; + } + List vmIds = new ArrayList<>(); + for (KubernetesClusterVmMapVO vmMap : clusterVMs) { + vmIds.add(vmMap.getVmId()); + } + Collections.sort(vmIds); + return userVmDao.findById(vmIds.get(0)); + } + + protected Pair getKubernetesClusterServerIpSshPort(UserVm masterVm) { + int port = CLUSTER_NODES_DEFAULT_START_SSH_PORT; + KubernetesClusterDetailsVO detail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS); + if (detail != null && !Strings.isNullOrEmpty(detail.getValue())) { + return new Pair<>(detail.getValue(), port); + } + Network network = networkDao.findById(kubernetesCluster.getNetworkId()); + if (network == null) { + LOGGER.warn(String.format("Network for Kubernetes cluster ID: %s cannot be found", kubernetesCluster.getUuid())); + return new Pair<>(null, port); + } + if (Network.GuestType.Isolated.equals(network.getGuestType())) { + List addresses = networkModel.listPublicIpsAssignedToGuestNtwk(network.getId(), true); + if (CollectionUtils.isEmpty(addresses)) { + LOGGER.warn(String.format("No public IP addresses found for network ID: %s, Kubernetes cluster ID: %s", network.getUuid(), kubernetesCluster.getUuid())); + return new Pair<>(null, port); + } + for (IpAddress address : addresses) { + if (address.isSourceNat()) { + return new Pair<>(address.getAddress().addr(), port); + } + } + LOGGER.warn(String.format("No source NAT IP addresses found for network ID: %s, Kubernetes cluster ID: %s", network.getUuid(), kubernetesCluster.getUuid())); + return new Pair<>(null, port); + } else if (Network.GuestType.Shared.equals(network.getGuestType())) { + port = 22; + masterVm = fetchMasterVmIfMissing(masterVm); + if (masterVm == null) { + LOGGER.warn(String.format("Unable to retrieve master VM for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + return new Pair<>(null, port); + } + return new Pair<>(masterVm.getPrivateIpAddress(), port); + } + LOGGER.warn(String.format("Unable to retrieve server IP address for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + return new Pair<>(null, port); + } + + protected void attachIsoKubernetesVMs(List clusterVMs) throws CloudRuntimeException { + KubernetesSupportedVersion version = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); + if (version == null) { + logAndThrow(Level.ERROR, String .format("Unable to find Kubernetes version for cluster ID: %s", kubernetesCluster.getUuid())); + } + VMTemplateVO iso = templateDao.findById(version.getIsoId()); + if (iso == null) { + logAndThrow(Level.ERROR, String.format("Unable to attach ISO to Kubernetes cluster ID: %s. Binaries ISO not found.", kubernetesCluster.getUuid())); + } + if (!iso.getFormat().equals(Storage.ImageFormat.ISO)) { + logAndThrow(Level.ERROR, String.format("Unable to attach ISO to Kubernetes cluster ID: %s. Invalid Binaries ISO.", kubernetesCluster.getUuid())); + } + if (!iso.getState().equals(VirtualMachineTemplate.State.Active)) { + logAndThrow(Level.ERROR, String.format("Unable to attach ISO to Kubernetes cluster ID: %s. Binaries ISO not active.", kubernetesCluster.getUuid())); + } + for (UserVm vm : clusterVMs) { + try { + templateService.attachIso(iso.getId(), vm.getId()); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Attached binaries ISO for VM: %s in cluster: %s", vm.getUuid(), kubernetesCluster.getName())); + } + } catch (CloudRuntimeException ex) { + logAndThrow(Level.ERROR, String.format("Failed to attach binaries ISO for VM: %s in the Kubernetes cluster name: %s", vm.getDisplayName(), kubernetesCluster.getName()), ex); + } + } + } + + protected void detachIsoKubernetesVMs(List clusterVMs) throws CloudRuntimeException { + for (UserVm vm : clusterVMs) { + boolean result = false; + try { + result = templateService.detachIso(vm.getId()); + } catch (CloudRuntimeException ex) { + LOGGER.warn(String.format("Failed to detach binaries ISO from VM ID: %s in the Kubernetes cluster ID: %s ", vm.getUuid(), kubernetesCluster.getUuid()), ex); + } + if (result) { + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Detached Kubernetes binaries from VM ID: %s in the Kubernetes cluster ID: %s", vm.getUuid(), kubernetesCluster.getUuid())); + } + continue; + } + LOGGER.warn(String.format("Failed to detach binaries ISO from VM ID: %s in the Kubernetes cluster ID: %s ", vm.getUuid(), kubernetesCluster.getUuid())); + } + } + + protected List getKubernetesClusterVMs() { + List vmList = new ArrayList<>(); + List clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + if (!CollectionUtils.isEmpty(clusterVMs)) { + clusterVMs.sort(new Comparator() { + @Override + public int compare(KubernetesClusterVmMapVO t1, KubernetesClusterVmMapVO t2) { + return (int)((t1.getId() - t2.getId())/Math.abs(t1.getId() - t2.getId())); + } + }); + for (KubernetesClusterVmMapVO vmMap : clusterVMs) { + vmList.add(userVmDao.findById(vmMap.getVmId())); + } + } + return vmList; + } + + protected boolean stateTransitTo(long kubernetesClusterId, KubernetesCluster.Event e) { + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + try { + return _stateMachine.transitTo(kubernetesCluster, e, null, kubernetesClusterDao); + } catch (NoTransitionException nte) { + LOGGER.warn(String.format("Failed to transition state of the Kubernetes cluster ID: %s in state %s on event %s", kubernetesCluster.getUuid(), kubernetesCluster.getState().toString(), e.toString()), nte); + return false; + } + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java new file mode 100644 index 000000000000..ecf47ef17870 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java @@ -0,0 +1,205 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.kubernetes.cluster.actionworkers; + +import java.util.List; + +import javax.inject.Inject; + +import org.apache.cloudstack.context.CallContext; + +import com.cloud.exception.ManagementServerException; +import com.cloud.exception.PermissionDeniedException; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO; +import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterVO; +import com.cloud.kubernetes.cluster.KubernetesClusterVmMap; +import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; +import com.cloud.network.dao.NetworkVO; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.User; +import com.cloud.uservm.UserVm; +import com.cloud.vm.ReservationContext; +import com.cloud.vm.ReservationContextImpl; +import com.cloud.vm.UserVmVO; +import com.cloud.vm.VirtualMachine; + +public class KubernetesClusterDestroyWorker extends KubernetesClusterResourceModifierActionWorker { + + @Inject + protected AccountManager accountManager; + + private List clusterVMs; + + public KubernetesClusterDestroyWorker(final KubernetesCluster kubernetesCluster, final KubernetesClusterManagerImpl clusterManager) { + super(kubernetesCluster, clusterManager); + } + + private void validateClusterSate() { + if (!(kubernetesCluster.getState().equals(KubernetesCluster.State.Running) + || kubernetesCluster.getState().equals(KubernetesCluster.State.Stopped) + || kubernetesCluster.getState().equals(KubernetesCluster.State.Alert) + || kubernetesCluster.getState().equals(KubernetesCluster.State.Error) + || kubernetesCluster.getState().equals(KubernetesCluster.State.Destroying))) { + String msg = String.format("Cannot perform delete operation on cluster ID: %s in state: %s",kubernetesCluster.getUuid(), kubernetesCluster.getState()); + LOGGER.warn(msg); + throw new PermissionDeniedException(msg); + } + } + + private boolean destroyClusterVMs() { + boolean vmDestroyed = true; + if ((clusterVMs != null) && !clusterVMs.isEmpty()) { + for (KubernetesClusterVmMapVO clusterVM : clusterVMs) { + long vmID = clusterVM.getVmId(); + + // delete only if VM exists and is not removed + UserVmVO userVM = userVmDao.findById(vmID); + if (userVM == null || userVM.isRemoved()) { + continue; + } + try { + UserVm vm = userVmService.destroyVm(vmID, true); + if (!VirtualMachine.State.Expunging.equals(vm.getState())) { + LOGGER.warn(String.format("VM '%s' ID: %s should have been expunging by now but is '%s'... retrying..." + , vm.getInstanceName() + , vm.getUuid() + , vm.getState().toString())); + } + vm = userVmService.expungeVm(vmID); + if (!VirtualMachine.State.Expunging.equals(vm.getState())) { + LOGGER.error(String.format("VM '%s' ID: %s is now in state '%s'. Will probably fail at deleting it's Kubernetes cluster." + , vm.getInstanceName() + , vm.getUuid() + , vm.getState().toString())); + } + kubernetesClusterVmMapDao.expunge(clusterVM.getId()); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Destroyed VM ID: %s as part of Kubernetes cluster ID: %s cleanup", vm.getUuid(), kubernetesCluster.getUuid())); + } + } catch (Exception e) { + vmDestroyed = false; + LOGGER.warn(String.format("Failed to destroy VM ID: %s part of the Kubernetes cluster ID: %s cleanup. Moving on with destroying remaining resources provisioned for the Kubernetes cluster", userVM.getUuid(), kubernetesCluster.getUuid()), e); + } + } + } + return vmDestroyed; + } + + private void processFailedNetworkDelete(long kubernetesClusterId) { + stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + KubernetesClusterVO cluster = kubernetesClusterDao.findById(kubernetesClusterId); + cluster.setCheckForGc(true); + kubernetesClusterDao.update(cluster.getId(), cluster); + } + + private boolean updateKubernetesClusterEntryForGC() { + KubernetesClusterVO kubernetesClusterVO = kubernetesClusterDao.findById(kubernetesCluster.getId()); + kubernetesClusterVO.setCheckForGc(false); + return kubernetesClusterDao.update(kubernetesCluster.getId(), kubernetesClusterVO); + } + + private void destroyKubernetesClusterNetwork() throws ManagementServerException { + NetworkVO network = networkDao.findById(kubernetesCluster.getNetworkId()); + if (network != null && network.getRemoved() == null) { + Account owner = accountManager.getAccount(network.getAccountId()); + User callerUser = accountManager.getActiveUser(CallContext.current().getCallingUserId()); + ReservationContext context = new ReservationContextImpl(null, null, callerUser, owner); + boolean networkDestroyed = networkMgr.destroyNetwork(kubernetesCluster.getNetworkId(), context, true); + if (!networkDestroyed) { + String msg = String.format("Failed to destroy network ID: %s as part of Kubernetes cluster ID: %s cleanup", network.getUuid(), kubernetesCluster.getUuid()); + LOGGER.warn(msg); + processFailedNetworkDelete(kubernetesCluster.getId()); + throw new ManagementServerException(msg); + } + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Destroyed network: %s as part of Kubernetes cluster ID: %s cleanup", network.getUuid(), kubernetesCluster.getUuid())); + } + } + } + + private void validateClusterVMsDestroyed() { + if(clusterVMs!=null && !clusterVMs.isEmpty()) { // Wait for few seconds to get all VMs really expunged + final int maxRetries = 3; + int retryCounter = 0; + while (retryCounter < maxRetries) { + boolean allVMsRemoved = true; + for (KubernetesClusterVmMap clusterVM : clusterVMs) { + UserVmVO userVM = userVmDao.findById(clusterVM.getVmId()); + if (userVM != null && !userVM.isRemoved()) { + allVMsRemoved = false; + break; + } + } + if (allVMsRemoved) { + break; + } + try { + Thread.sleep(10000); + } catch (InterruptedException ie) {} + retryCounter++; + } + } + } + + public boolean destroy() throws ManagementServerException, PermissionDeniedException { + init(); + validateClusterSate(); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Destroying Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + } + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.DestroyRequested); + this.clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + boolean vmsDestroyed = destroyClusterVMs(); + boolean cleanupNetwork = true; + final KubernetesClusterDetailsVO clusterDetails = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "networkCleanup"); + if (clusterDetails != null) { + cleanupNetwork = Boolean.parseBoolean(clusterDetails.getValue()); + } + // if there are VM's that were not expunged, we can not delete the network + if (vmsDestroyed) { + if (cleanupNetwork) { + validateClusterVMsDestroyed(); + try { + destroyKubernetesClusterNetwork(); + } catch (Exception e) { + String msg = String.format("Failed to destroy network of Kubernetes cluster ID: %s cleanup", kubernetesCluster.getUuid()); + LOGGER.warn(msg, e); + processFailedNetworkDelete(kubernetesCluster.getId()); + throw new ManagementServerException(msg, e); + } + } + } else { + String msg = String.format("Failed to destroy one or more VMs as part of Kubernetes cluster ID: %s cleanup", kubernetesCluster.getUuid()); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(msg); + } + processFailedNetworkDelete(kubernetesCluster.getId()); + throw new ManagementServerException(msg); + } + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); + updateKubernetesClusterEntryForGC(); + kubernetesClusterDao.remove(kubernetesCluster.getId()); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Kubernetes cluster ID: %s is successfully deleted", kubernetesCluster.getUuid())); + } + return true; + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java new file mode 100644 index 000000000000..267127b7e865 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -0,0 +1,407 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.kubernetes.cluster.actionworkers; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import javax.inject.Inject; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.command.user.firewall.CreateFirewallRuleCmd; +import org.apache.cloudstack.api.command.user.vm.StartVMCmd; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.collections.CollectionUtils; +import org.apache.log4j.Level; + +import com.cloud.capacity.CapacityManager; +import com.cloud.dc.ClusterDetailsDao; +import com.cloud.dc.ClusterDetailsVO; +import com.cloud.dc.ClusterVO; +import com.cloud.dc.DataCenter; +import com.cloud.dc.dao.ClusterDao; +import com.cloud.deploy.DeployDestination; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.InsufficientServerCapacityException; +import com.cloud.exception.ManagementServerException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.host.Host; +import com.cloud.host.HostVO; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO; +import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil; +import com.cloud.network.IpAddress; +import com.cloud.network.Network; +import com.cloud.network.dao.FirewallRulesDao; +import com.cloud.network.firewall.FirewallService; +import com.cloud.network.lb.LoadBalancingRulesService; +import com.cloud.network.rules.FirewallRule; +import com.cloud.network.rules.PortForwardingRuleVO; +import com.cloud.network.rules.RulesService; +import com.cloud.network.rules.dao.PortForwardingRulesDao; +import com.cloud.offering.ServiceOffering; +import com.cloud.resource.ResourceManager; +import com.cloud.template.VirtualMachineTemplate; +import com.cloud.user.Account; +import com.cloud.user.SSHKeyPairVO; +import com.cloud.uservm.UserVm; +import com.cloud.utils.Pair; +import com.cloud.utils.StringUtils; +import com.cloud.utils.component.ComponentContext; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallbackWithException; +import com.cloud.utils.db.TransactionStatus; +import com.cloud.utils.exception.ExecutionException; +import com.cloud.utils.net.Ip; +import com.cloud.vm.Nic; +import com.cloud.vm.VirtualMachine; +import com.google.common.base.Strings; + +public class KubernetesClusterResourceModifierActionWorker extends KubernetesClusterActionWorker { + + @Inject + protected CapacityManager capacityManager; + @Inject + protected ClusterDao clusterDao; + @Inject + protected ClusterDetailsDao clusterDetailsDao; + @Inject + protected FirewallRulesDao firewallRulesDao; + @Inject + protected FirewallService firewallService; + @Inject + protected LoadBalancingRulesService lbService; + @Inject + protected RulesService rulesService; + @Inject + protected PortForwardingRulesDao portForwardingRulesDao; + @Inject + protected ResourceManager resourceManager; + + protected KubernetesClusterResourceModifierActionWorker(final KubernetesCluster kubernetesCluster, final KubernetesClusterManagerImpl clusterManager) { + super(kubernetesCluster, clusterManager); + } + + private String getKubernetesNodeConfig(String joinIp) throws IOException { + String k8sNodeConfig = readResourceFile("/conf/k8s-node.yml"); + final String sshPubKey = "{{ k8s.ssh.pub.key }}"; + final String joinIpKey = "{{ k8s_master.join_ip }}"; + final String clusterTokenKey = "{{ k8s_master.cluster.token }}"; + String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\""; + String sshKeyPair = kubernetesCluster.getKeyPair(); + if (!Strings.isNullOrEmpty(sshKeyPair)) { + SSHKeyPairVO sshkp = sshKeyPairDao.findByName(owner.getAccountId(), owner.getDomainId(), sshKeyPair); + if (sshkp != null) { + pubKey += "\n - \"" + sshkp.getPublicKey() + "\""; + } + } + k8sNodeConfig = k8sNodeConfig.replace(sshPubKey, pubKey); + k8sNodeConfig = k8sNodeConfig.replace(joinIpKey, joinIp); + k8sNodeConfig = k8sNodeConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); + /* genarate /.docker/config.json file on the nodes only if Kubernetes cluster is created to + * use docker private registry */ + String dockerUserName = null; + String dockerPassword = null; + String dockerRegistryUrl = null; + String dockerRegistryEmail = null; + List details = kubernetesClusterDetailsDao.listDetails(kubernetesCluster.getId()); + for (KubernetesClusterDetailsVO detail : details) { + if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_USER_NAME)) { + dockerUserName = detail.getValue(); + } + if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_PASSWORD)) { + dockerPassword = detail.getValue(); + } + if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_URL)) { + dockerRegistryUrl = detail.getValue(); + } + if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_EMAIL)) { + dockerRegistryEmail = detail.getValue(); + } + } + if (!Strings.isNullOrEmpty(dockerUserName) && !Strings.isNullOrEmpty(dockerPassword)) { + // do write file for /.docker/config.json through the code instead of k8s-node.yml as we can no make a section + // optional or conditionally applied + String dockerConfigString = "write-files:\n" + + " - path: /.docker/config.json\n" + + " owner: core:core\n" + + " permissions: '0644'\n" + + " content: |\n" + + " {\n" + + " \"auths\": {\n" + + " {{docker.url}}: {\n" + + " \"auth\": {{docker.secret}},\n" + + " \"email\": {{docker.email}}\n" + + " }\n" + + " }\n" + + " }"; + k8sNodeConfig = k8sNodeConfig.replace("write-files:", dockerConfigString); + final String dockerUrlKey = "{{docker.url}}"; + final String dockerAuthKey = "{{docker.secret}}"; + final String dockerEmailKey = "{{docker.email}}"; + final String usernamePasswordKey = dockerUserName + ":" + dockerPassword; + String base64Auth = Base64.encodeBase64String(usernamePasswordKey.getBytes(StringUtils.getPreferredCharset())); + k8sNodeConfig = k8sNodeConfig.replace(dockerUrlKey, "\"" + dockerRegistryUrl + "\""); + k8sNodeConfig = k8sNodeConfig.replace(dockerAuthKey, "\"" + base64Auth + "\""); + k8sNodeConfig = k8sNodeConfig.replace(dockerEmailKey, "\"" + dockerRegistryEmail + "\""); + } + return k8sNodeConfig; + } + + protected DeployDestination plan(final long nodesCount, final DataCenter zone, final ServiceOffering offering) throws InsufficientServerCapacityException { + final int cpu_requested = offering.getCpu() * offering.getSpeed(); + final long ram_requested = offering.getRamSize() * 1024L * 1024L; + List hosts = resourceManager.listAllHostsInOneZoneByType(Host.Type.Routing, zone.getId()); + final Map> hosts_with_resevered_capacity = new ConcurrentHashMap>(); + for (HostVO h : hosts) { + hosts_with_resevered_capacity.put(h.getUuid(), new Pair(h, 0)); + } + boolean suitable_host_found = false; + for (int i = 1; i <= nodesCount + 1; i++) { + suitable_host_found = false; + for (Map.Entry> hostEntry : hosts_with_resevered_capacity.entrySet()) { + Pair hp = hostEntry.getValue(); + HostVO h = hp.first(); + int reserved = hp.second(); + reserved++; + ClusterVO cluster = clusterDao.findById(h.getClusterId()); + ClusterDetailsVO cluster_detail_cpu = clusterDetailsDao.findDetail(cluster.getId(), "cpuOvercommitRatio"); + ClusterDetailsVO cluster_detail_ram = clusterDetailsDao.findDetail(cluster.getId(), "memoryOvercommitRatio"); + Float cpuOvercommitRatio = Float.parseFloat(cluster_detail_cpu.getValue()); + Float memoryOvercommitRatio = Float.parseFloat(cluster_detail_ram.getValue()); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Checking host ID: %s for capacity already reserved %d", h.getUuid(), reserved)); + } + if (capacityManager.checkIfHostHasCapacity(h.getId(), cpu_requested * reserved, ram_requested * reserved, false, cpuOvercommitRatio, memoryOvercommitRatio, true)) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Found host ID: %s for with enough capacity, CPU=%d RAM=%d", h.getUuid(), cpu_requested * reserved, ram_requested * reserved)); + } + hostEntry.setValue(new Pair(h, reserved)); + suitable_host_found = true; + break; + } + } + if (!suitable_host_found) { + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Suitable hosts not found in datacenter ID: %s for node %d", zone.getUuid(), i)); + } + break; + } + } + if (suitable_host_found) { + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Suitable hosts found in datacenter ID: %s, creating deployment destination", zone.getUuid())); + } + return new DeployDestination(zone, null, null, null); + } + String msg = String.format("Cannot find enough capacity for Kubernetes cluster(requested cpu=%1$s memory=%2$s)", + cpu_requested * nodesCount, ram_requested * nodesCount); + LOGGER.warn(msg); + throw new InsufficientServerCapacityException(msg, DataCenter.class, zone.getId()); + } + + protected DeployDestination plan() throws InsufficientServerCapacityException { + ServiceOffering offering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); + DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Checking deployment destination for Kubernetes cluster ID: %s in zone ID: %s", kubernetesCluster.getUuid(), zone.getUuid())); + } + return plan(kubernetesCluster.getTotalNodeCount(), zone, offering); + } + + protected void startKubernetesVM(final UserVm vm) throws ConcurrentOperationException { + try { + StartVMCmd startVm = new StartVMCmd(); + startVm = ComponentContext.inject(startVm); + Field f = startVm.getClass().getDeclaredField("id"); + f.setAccessible(true); + f.set(startVm, vm.getId()); + userVmService.startVirtualMachine(startVm); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Started VM ID: %s in the Kubernetes cluster ID: %s", vm.getUuid(), kubernetesCluster.getUuid())); + } + } catch (IllegalAccessException | NoSuchFieldException | ExecutionException | + ResourceUnavailableException | ResourceAllocationException | InsufficientCapacityException ex) { + logAndThrow(Level.WARN, String.format("Failed to start VM in the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), ex); + } + + UserVm startVm = userVmDao.findById(vm.getId()); + if (!startVm.getState().equals(VirtualMachine.State.Running)) { + logAndThrow(Level.WARN, String.format("Failed to start VM in the Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + } + } + + protected List provisionKubernetesClusterNodeVms(final long nodeCount, final int offset, final String publicIpAddress) throws ManagementServerException, + ResourceUnavailableException, InsufficientCapacityException { + List nodes = new ArrayList<>(); + for (int i = offset + 1; i <= nodeCount; i++) { + UserVm vm = createKubernetesNode(publicIpAddress, i); + addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId()); + startKubernetesVM(vm); + nodes.add(vm); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Provisioned node master VM ID: %s in to the Kubernetes cluster ID: %s", vm.getUuid(), kubernetesCluster.getUuid())); + } + } + return nodes; + } + + protected List provisionKubernetesClusterNodeVms(final long nodeCount, final String publicIpAddress) throws ManagementServerException, + ResourceUnavailableException, InsufficientCapacityException { + return provisionKubernetesClusterNodeVms(nodeCount, 0, publicIpAddress); + } + + protected UserVm createKubernetesNode(String joinIp, int nodeInstance) throws ManagementServerException, + ResourceUnavailableException, InsufficientCapacityException { + UserVm nodeVm = null; + DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); + ServiceOffering serviceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); + VirtualMachineTemplate template = templateDao.findById(kubernetesCluster.getTemplateId()); + List networkIds = new ArrayList(); + networkIds.add(kubernetesCluster.getNetworkId()); + Account owner = accountDao.findById(kubernetesCluster.getAccountId()); + Network.IpAddresses addrs = new Network.IpAddresses(null, null); + long rootDiskSize = kubernetesCluster.getNodeRootDiskSize(); + Map customParameterMap = new HashMap(); + if (rootDiskSize > 0) { + customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); + } + String hostName = String.format("%s-k8s-node-%s", kubernetesCluster.getName(), nodeInstance); + String k8sNodeConfig = null; + try { + k8sNodeConfig = getKubernetesNodeConfig(joinIp); + } catch (IOException e) { + logAndThrow(Level.ERROR, "Failed to read Kubernetes node configuration file", e); + } + String base64UserData = Base64.encodeBase64String(k8sNodeConfig.getBytes(StringUtils.getPreferredCharset())); + nodeVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, + hostName, kubernetesCluster.getDescription(), null, null, null, + null, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), + null, addrs, null, null, null, customParameterMap, null, null, null, null); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Created node VM ID: %s, %s in the Kubernetes cluster ID: %s", nodeVm.getUuid(), hostName, kubernetesCluster.getUuid())); + } + return nodeVm; + } + + protected IpAddress getSourceNatIp(Network network) { + List addresses = networkModel.listPublicIpsAssignedToGuestNtwk(network.getId(), true); + if (CollectionUtils.isEmpty(addresses)) { + return null; + } + for (IpAddress address : addresses) { + if (address.isSourceNat()) { + return address; + } + } + return null; + } + + protected void provisionFirewallRules(final IpAddress publicIp, final Account account, int startPort, int endPort) throws NoSuchFieldException, + IllegalAccessException, ResourceUnavailableException, NetworkRuleConflictException { + List sourceCidrList = new ArrayList(); + sourceCidrList.add("0.0.0.0/0"); + + CreateFirewallRuleCmd rule = new CreateFirewallRuleCmd(); + rule = ComponentContext.inject(rule); + + Field addressField = rule.getClass().getDeclaredField("ipAddressId"); + addressField.setAccessible(true); + addressField.set(rule, publicIp.getId()); + + Field protocolField = rule.getClass().getDeclaredField("protocol"); + protocolField.setAccessible(true); + protocolField.set(rule, "TCP"); + + Field startPortField = rule.getClass().getDeclaredField("publicStartPort"); + startPortField.setAccessible(true); + startPortField.set(rule, startPort); + + Field endPortField = rule.getClass().getDeclaredField("publicEndPort"); + endPortField.setAccessible(true); + endPortField.set(rule, endPort); + + Field cidrField = rule.getClass().getDeclaredField("cidrlist"); + cidrField.setAccessible(true); + cidrField.set(rule, sourceCidrList); + + firewallService.createIngressFirewallRule(rule); + firewallService.applyIngressFwRules(publicIp.getId(), account); + } + + /** + * To provision SSH port forwarding rules for the given Kubernetes cluster + * for its given virtual machines + * @param publicIp + * @param network + * @param account + * @param List clusterVMIds (when empty then method must be called while + * down-scaling of the KubernetesCluster therefore no new rules + * to be added) + * @param firewallRuleSourcePortStart + * @throws ResourceUnavailableException + * @throws NetworkRuleConflictException + */ + protected void provisionSshPortForwardingRules(IpAddress publicIp, Network network, Account account, + List clusterVMIds, int firewallRuleSourcePortStart) throws ResourceUnavailableException, + NetworkRuleConflictException { + if (!CollectionUtils.isEmpty(clusterVMIds)) { + final long publicIpId = publicIp.getId(); + final long networkId = network.getId(); + final long accountId = account.getId(); + final long domainId = account.getDomainId(); + for (int i = 0; i < clusterVMIds.size(); ++i) { + long vmId = clusterVMIds.get(i); + Nic vmNic = networkModel.getNicInNetwork(vmId, networkId); + final Ip vmIp = new Ip(vmNic.getIPv4Address()); + final long vmIdFinal = vmId; + final int srcPortFinal = firewallRuleSourcePortStart + i; + + PortForwardingRuleVO pfRule = Transaction.execute(new TransactionCallbackWithException() { + @Override + public PortForwardingRuleVO doInTransaction(TransactionStatus status) throws NetworkRuleConflictException { + PortForwardingRuleVO newRule = + new PortForwardingRuleVO(null, publicIpId, + srcPortFinal, srcPortFinal, + vmIp, + 22, 22, + "tcp", networkId, accountId, domainId, vmIdFinal); + newRule.setDisplay(true); + newRule.setState(FirewallRule.State.Add); + newRule = portForwardingRulesDao.persist(newRule); + return newRule; + } + }); + rulesService.applyPortForwardingRules(publicIp.getId(), account); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Provisioned SSH port forwarding rule from port %d to 22 on %s to the VM IP : %s in Kubernetes cluster ID: %s", srcPortFinal, publicIp.getAddress().addr(), vmIp.toString(), kubernetesCluster.getUuid())); + } + } + } + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java new file mode 100644 index 000000000000..6ad054713512 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -0,0 +1,457 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.kubernetes.cluster.actionworkers; + +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import javax.inject.Inject; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.log4j.Level; + +import com.cloud.dc.DataCenter; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.ManagementServerException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.exception.VirtualMachineMigrationException; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterVO; +import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil; +import com.cloud.network.IpAddress; +import com.cloud.network.Network; +import com.cloud.network.rules.FirewallRule; +import com.cloud.network.rules.FirewallRuleVO; +import com.cloud.network.rules.PortForwardingRuleVO; +import com.cloud.offering.ServiceOffering; +import com.cloud.user.Account; +import com.cloud.uservm.UserVm; +import com.cloud.utils.Pair; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.db.TransactionStatus; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.ssh.SshHelper; +import com.cloud.vm.UserVmManager; +import com.cloud.vm.UserVmVO; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.dao.VMInstanceDao; +import com.google.common.base.Strings; + +public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModifierActionWorker { + + @Inject + protected VMInstanceDao vmInstanceDao; + @Inject + protected UserVmManager userVmManager; + + private ServiceOffering serviceOffering; + private Long clusterSize; + + public KubernetesClusterScaleWorker(final KubernetesCluster kubernetesCluster, + final ServiceOffering serviceOffering, + final Long clusterSize, + final KubernetesClusterManagerImpl clusterManager) { + super(kubernetesCluster, clusterManager); + this.serviceOffering = serviceOffering; + this.clusterSize = clusterSize; + } + + private FirewallRule removeSshFirewallRule(IpAddress publicIp) { + FirewallRule rule = null; + List firewallRules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(publicIp.getId(), FirewallRule.Purpose.Firewall); + for (FirewallRuleVO firewallRule : firewallRules) { + if (firewallRule.getSourcePortStart() == CLUSTER_NODES_DEFAULT_START_SSH_PORT) { + rule = firewallRule; + firewallService.revokeIngressFwRule(firewallRule.getId(), true); + break; + } + } + return rule; + } + + private void removePortForwardingRules(IpAddress publicIp, Network network, Account account, List removedVMIds) throws ResourceUnavailableException { + if (!CollectionUtils.isEmpty(removedVMIds)) { + for (Long vmId : removedVMIds) { + List pfRules = portForwardingRulesDao.listByNetwork(network.getId()); + for (PortForwardingRuleVO pfRule : pfRules) { + if (pfRule.getVirtualMachineId() == vmId) { + portForwardingRulesDao.remove(pfRule.getId()); + break; + } + } + } + rulesService.applyPortForwardingRules(publicIp.getId(), account); + } + } + + /** + * Scale network rules for an existing Kubernetes cluster while scaling it + * Open up firewall for SSH access from port NODES_DEFAULT_START_SSH_PORT to NODES_DEFAULT_START_SSH_PORT+n. + * Also remove port forwarding rules for removed virtual machines and create port-forwarding rule + * to forward public IP traffic to all node VMs' private IP. + * @param network + * @param account + * @param clusterVMIds + * @param removedVMIds + * @throws ManagementServerException + */ + private void scaleKubernetesClusterNetworkRules(Network network, Account account, + List clusterVMIds, List removedVMIds) throws ManagementServerException { + if (!Network.GuestType.Isolated.equals(network.getGuestType())) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Network ID: %s for Kubernetes cluster ID: %s is not an isolated network, therefore, no need for network rules", network.getUuid(), kubernetesCluster.getUuid())); + } + return; + } + IpAddress publicIp = getSourceNatIp(network); + if (publicIp == null) { + throw new ManagementServerException(String.format("No source NAT IP addresses found for network ID: %s, Kubernetes cluster ID: %s", network.getUuid(), kubernetesCluster.getUuid())); + } + + // Remove existing SSH firewall rules + FirewallRule firewallRule = removeSshFirewallRule(publicIp); + if (firewallRule == null) { + throw new ManagementServerException("Firewall rule for node SSH access can't be provisioned!"); + } + int existingFirewallRuleSourcePortEnd = firewallRule.getSourcePortEnd(); + + // Provision new SSH firewall rules + try { + provisionFirewallRules(publicIp, account, CLUSTER_NODES_DEFAULT_START_SSH_PORT, CLUSTER_NODES_DEFAULT_START_SSH_PORT + (int)kubernetesCluster.getTotalNodeCount() - 1); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Provisioned firewall rule to open up port %d to %d on %s in Kubernetes cluster ID: %s", + CLUSTER_NODES_DEFAULT_START_SSH_PORT, CLUSTER_NODES_DEFAULT_START_SSH_PORT + (int) kubernetesCluster.getTotalNodeCount() - 1, publicIp.getAddress().addr(), kubernetesCluster.getUuid())); + } + } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException e) { + throw new ManagementServerException(String.format("Failed to activate SSH firewall rules for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); + } + + try { + removePortForwardingRules(publicIp, network, account, removedVMIds); + } catch (ResourceUnavailableException e) { + throw new ManagementServerException(String.format("Failed to remove SSH port forwarding rules for removed VMs for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); + } + + try { + provisionSshPortForwardingRules(publicIp, network, account, clusterVMIds, existingFirewallRuleSourcePortEnd + 1); + } catch (ResourceUnavailableException | NetworkRuleConflictException e) { + throw new ManagementServerException(String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); + } + } + + private KubernetesClusterVO updateKubernetesClusterEntry(final long kubernetesClusterId, final long clusterSize, + final long cores, final long memory, final Long serviceOfferingId) { + return Transaction.execute(new TransactionCallback() { + @Override + public KubernetesClusterVO doInTransaction(TransactionStatus status) { + KubernetesClusterVO updatedCluster = kubernetesClusterDao.createForUpdate(kubernetesClusterId); + updatedCluster.setNodeCount(clusterSize); + updatedCluster.setCores(cores); + updatedCluster.setMemory(memory); + if (serviceOfferingId != null) { + updatedCluster.setServiceOfferingId(serviceOfferingId); + } + kubernetesClusterDao.persist(updatedCluster); + return updatedCluster; + } + }); + } + + private boolean removeKubernetesClusterNode(String ipAddress, int port, UserVm userVm, int retries, int waitDuration) { + File pkFile = getManagementServerSshPublicKeyFile(); + int retryCounter = 0; + String hostName = userVm.getHostName(); + if (!Strings.isNullOrEmpty(hostName)) { + hostName = hostName.toLowerCase(); + } + while (retryCounter < retries) { + retryCounter++; + try { + Pair result = SshHelper.sshExecute(ipAddress, port, CLUSTER_NODE_VM_USER, + pkFile, null, String.format("sudo kubectl drain %s --ignore-daemonsets --delete-local-data", hostName), + 10000, 10000, 60000); + if (!result.first()) { + LOGGER.warn(String.format("Draining node: %s on VM ID: %s in Kubernetes cluster ID: %s unsuccessful", hostName, userVm.getUuid(), kubernetesCluster.getUuid())); + } else { + result = SshHelper.sshExecute(ipAddress, port, CLUSTER_NODE_VM_USER, + pkFile, null, String.format("sudo kubectl delete node %s", hostName), + 10000, 10000, 30000); + if (result.first()) { + return true; + } else { + LOGGER.warn(String.format("Deleting node: %s on VM ID: %s in Kubernetes cluster ID: %s unsuccessful", hostName, userVm.getUuid(), kubernetesCluster.getUuid())); + } + } + break; + } catch (Exception e) { + String msg = String.format("Failed to remove Kubernetes cluster ID: %s node: %s on VM ID: %s", kubernetesCluster.getUuid(), hostName, userVm.getUuid()); + LOGGER.warn(msg, e); + } + try { + Thread.sleep(waitDuration); + } catch (InterruptedException ie) { + LOGGER.error(String.format("Error while waiting for Kubernetes cluster ID: %s node: %s on VM ID: %s removal", kubernetesCluster.getUuid(), hostName, userVm.getUuid()), ie); + } + retryCounter++; + } + return false; + } + + private void scaleKubernetesClusterOffering(final long kubernetesClusterId, final ServiceOffering serviceOffering, final Long clusterSize) { + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.ScaleUpRequested); + + final long size = (clusterSize == null ? kubernetesCluster.getTotalNodeCount() : kubernetesCluster.getMasterNodeCount() + clusterSize); + final long cores = serviceOffering.getCpu() * size; + final long memory = serviceOffering.getRamSize() * size; + KubernetesClusterVO updatedKubernetesCluster = updateKubernetesClusterEntry(kubernetesCluster.getId(), size, cores, memory, serviceOffering.getId()); + if (updatedKubernetesCluster == null) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, unable to update Kubernetes cluster!", updatedKubernetesCluster.getUuid()), kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + } + kubernetesCluster = updatedKubernetesCluster; + List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + final long tobeScaledVMCount = Math.min(vmList.size(), size); + for (long i = 0; i < tobeScaledVMCount; i++) { + KubernetesClusterVmMapVO vmMapVO = vmList.get((int) i); + UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId()); + boolean result = false; + try { + result = userVmManager.upgradeVirtualMachine(userVM.getId(), serviceOffering.getId(), new HashMap()); + } catch (ResourceUnavailableException | ManagementServerException | ConcurrentOperationException | VirtualMachineMigrationException e) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, unable to scale cluster VM ID: %s", kubernetesCluster.getUuid(), userVM.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); + } + if (!result) { + logTransitStateAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster ID: %s failed, unable to scale cluster VM ID: %s", kubernetesCluster.getUuid(), userVM.getUuid()),kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + } + } + + private void scaleDownKubernetesClusterSize(final List originalVmList, final Network network) throws CloudRuntimeException { + int i = originalVmList.size() - 1; + List removedVmIds = new ArrayList<>(); + while (i > kubernetesCluster.getMasterNodeCount() && originalVmList.size() > kubernetesCluster.getTotalNodeCount()) { // Reverse order as first VM will be k8s master + KubernetesClusterVmMapVO vmMapVO = originalVmList.get(i); + UserVm userVM = userVmDao.findById(vmMapVO.getId()); + if (!removeKubernetesClusterNode(publicIpAddress, sshPort, userVM, 3, 30000)) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, failed to remove Kubernetes node: %s running on VM ID: %s", kubernetesCluster.getUuid(), userVM.getHostName(), userVM.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + // For removing port-forwarding network rules + removedVmIds.add(userVM.getId()); + try { + UserVm vm = userVmService.destroyVm(userVM.getId(), true); + if (!VirtualMachine.State.Expunging.equals(vm.getState())) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, VM '%s' is now in state '%s'." + , kubernetesCluster.getUuid() + , vm.getInstanceName() + , vm.getState().toString()), + kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + vm = userVmService.expungeVm(userVM.getId()); + if (!VirtualMachine.State.Expunging.equals(vm.getState())) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, VM '%s' is now in state '%s'." + , kubernetesCluster.getUuid() + , vm.getInstanceName() + , vm.getState().toString()), + kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + } catch (ResourceUnavailableException e) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, unable to remove VM ID: %s" + , kubernetesCluster.getUuid() , userVM.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); + } + kubernetesClusterVmMapDao.expunge(vmMapVO.getId()); + i--; + } + // Scale network rules to update firewall rule + try { + scaleKubernetesClusterNetworkRules(network, owner, null, removedVmIds); + } catch (ManagementServerException e) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, unable to update network rules", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); + } + } + + private void scaleUpKubernetesClusterSize(final List originalVmList, final long newVmCount, final Network network) throws CloudRuntimeException { + List clusterVMs = new ArrayList<>(); + List clusterVMIds = new ArrayList<>(); + try { + clusterVMs = provisionKubernetesClusterNodeVms((int) newVmCount + originalVmList.size(), originalVmList.size(), publicIpAddress); + } catch (ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, unable to provision node VM in the cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); + } + for (UserVm vm : clusterVMs) { + clusterVMIds.add(vm.getId()); + } + try { + scaleKubernetesClusterNetworkRules(network, owner, clusterVMIds, null); + } catch (ManagementServerException e) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, unable to update network rules", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); + } + attachIsoKubernetesVMs(clusterVMs); + boolean readyNodesCountValid = KubernetesClusterUtil.validateKubernetesClusterReadyNodesCount(kubernetesCluster, publicIpAddress, sshPort, + CLUSTER_NODE_VM_USER, sshKeyFile, 30, 30000); + detachIsoKubernetesVMs(clusterVMs); + if (!readyNodesCountValid) { // Scaling failed + logTransitStateAndThrow(Level.ERROR, String.format("Scaling unsuccessful for Kubernetes cluster ID: %s as it does not have desired number of nodes in ready state", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + } + + private void scaleKubernetesClusterSize(final long originalClusterSize) throws CloudRuntimeException { + KubernetesClusterVO kubernetesClusterVO = kubernetesClusterDao.findById(kubernetesCluster.getId()); + final Network network = networkDao.findById(kubernetesClusterVO.getNetworkId()); + final long newVmRequiredCount = clusterSize - originalClusterSize; + List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesClusterVO.getId()); + if (CollectionUtils.isEmpty(vmList) || vmList.size() - 1 < originalClusterSize) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, t is in unstable state as not enough existing VM instances found", kubernetesClusterVO.getUuid()), kubernetesClusterVO.getId(), KubernetesCluster.Event.OperationFailed); + } + + Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(null); + String publicIpAddress = publicIpSshPort.first(); + int sshPort = publicIpSshPort.second(); + if (Strings.isNullOrEmpty(publicIpAddress)) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, unable to retrieve associated public IP", kubernetesClusterVO.getUuid()), kubernetesClusterVO.getId(), KubernetesCluster.Event.OperationFailed); + } + Account account = accountDao.findById(kubernetesClusterVO.getAccountId()); + if (newVmRequiredCount < 0) { // downscale + scaleDownKubernetesClusterSize(vmList, network); + } else { // upscale, same node count handled above + scaleUpKubernetesClusterSize(vmList, newVmRequiredCount, network); + } + } + + private void validateKubernetesClusterScaleOfferingParameters(final ServiceOffering existingServiceOffering, final ServiceOffering serviceOffering) throws CloudRuntimeException { + final long originalNodeCount = kubernetesCluster.getTotalNodeCount(); + List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + if (vmList == null || vmList.isEmpty() || vmList.size() < originalNodeCount) { + logAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster ID: %s failed, it is in unstable state as not enough existing VM instances found!", kubernetesCluster.getUuid())); + } else { + for (KubernetesClusterVmMapVO vmMapVO : vmList) { + VMInstanceVO vmInstance = vmInstanceDao.findById(vmMapVO.getVmId()); + if (vmInstance != null && vmInstance.getState().equals(VirtualMachine.State.Running) && + vmInstance.getHypervisorType() != Hypervisor.HypervisorType.XenServer && + vmInstance.getHypervisorType() != Hypervisor.HypervisorType.VMware && + vmInstance.getHypervisorType() != Hypervisor.HypervisorType.Simulator) { + logAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster ID: %s failed, scaling Kubernetes cluster with running VMs on hypervisor %s is not supported!", kubernetesCluster.getUuid(), vmInstance.getHypervisorType())); + } + } + } + if (serviceOffering.getRamSize() < existingServiceOffering.getRamSize() || + serviceOffering.getCpu() * serviceOffering.getSpeed() < existingServiceOffering.getCpu() * existingServiceOffering.getSpeed()) { + logAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster ID: %s failed, service offering for the Kubernetes cluster cannot be scaled down!", kubernetesCluster.getUuid())); + } + } + + private void validateKubernetesClusterScaleSizeParameters(final long originalClusterSize, final long clusterSize, final KubernetesCluster.State clusterState) throws CloudRuntimeException { + Network network = networkDao.findById(kubernetesCluster.getNetworkId()); + if (network == null) { + String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, cluster network not found", kubernetesCluster.getUuid()); + if (KubernetesCluster.State.Scaling.equals(kubernetesCluster.getState())) { + logTransitStateAndThrow(Level.WARN, msg, kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } else { + logAndThrow(Level.WARN, msg); + } + } + // Check capacity and transition state + final long newVmRequiredCount = clusterSize - originalClusterSize; + final ServiceOffering clusterServiceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); + if (clusterServiceOffering == null) { + String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, cluster service offering not found", kubernetesCluster.getUuid()); + if (KubernetesCluster.State.Scaling.equals(kubernetesCluster.getState())) { + logTransitStateAndThrow(Level.WARN, msg, kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } else { + logAndThrow(Level.WARN, msg); + } + } + if (newVmRequiredCount > 0) { + final DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); + try { + if (clusterState.equals(KubernetesCluster.State.Running)) { + plan(newVmRequiredCount, zone, clusterServiceOffering); + } else { + plan(kubernetesCluster.getTotalNodeCount() + newVmRequiredCount, zone, clusterServiceOffering); + } + } catch (InsufficientCapacityException e) { + String msg = String.format("Scaling failed for Kubernetes cluster ID: %s in zone ID: %s, insufficient capacity", kubernetesCluster.getUuid(), zone.getUuid()); + if (KubernetesCluster.State.Scaling.equals(kubernetesCluster.getState())) { + logTransitStateAndThrow(Level.WARN, msg, kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } else { + logAndThrow(Level.WARN, msg); + } + } + } + } + + public boolean scaleCluster() throws CloudRuntimeException { + init(); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Scaling Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + } + + final KubernetesCluster.State clusterState = kubernetesCluster.getState(); + final long originalClusterSize = kubernetesCluster.getNodeCount(); + + final ServiceOffering existingServiceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); + if (existingServiceOffering == null) { + logAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, service offering for the Kubernetes cluster not found!", kubernetesCluster.getUuid())); + } + final boolean serviceOfferingScalingNeeded = serviceOffering != null && serviceOffering.getId() != existingServiceOffering.getId(); + final boolean clusterSizeScalingNeeded = clusterSize != null && clusterSize != originalClusterSize; + + if (serviceOfferingScalingNeeded) { + validateKubernetesClusterScaleOfferingParameters(existingServiceOffering, serviceOffering); + scaleKubernetesClusterOffering(kubernetesCluster.getId(), serviceOffering, clusterSize); + } + + if (clusterSizeScalingNeeded) { + validateKubernetesClusterScaleSizeParameters(originalClusterSize, clusterSize, clusterState); + final long newVmRequiredCount = clusterSize - originalClusterSize; + final ServiceOffering clusterServiceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); + if (newVmRequiredCount > 0) { + if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.ScaleUpRequested); + } + } else { + if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.ScaleDownRequested); + } + } + + if (!serviceOfferingScalingNeeded) { // Else already updated + final long cores = clusterServiceOffering.getCpu() * (kubernetesCluster.getMasterNodeCount() + clusterSize); + final long memory = clusterServiceOffering.getRamSize() * (kubernetesCluster.getMasterNodeCount() + clusterSize); + + if (updateKubernetesClusterEntry(kubernetesCluster.getId(), clusterSize, cores, memory, null) == null) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, unable to update cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + } + + // Perform size scaling + scaleKubernetesClusterSize(originalClusterSize); + } + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); + return true; + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java new file mode 100644 index 000000000000..2dbf6fd3c436 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -0,0 +1,607 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.kubernetes.cluster.actionworkers; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.framework.ca.Certificate; +import org.apache.cloudstack.utils.security.CertUtils; +import org.apache.commons.codec.binary.Base64; +import org.apache.log4j.Level; + +import com.cloud.dc.DataCenter; +import com.cloud.dc.Vlan; +import com.cloud.dc.VlanVO; +import com.cloud.deploy.DeployDestination; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientAddressCapacityException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.ManagementServerException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO; +import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.kubernetes.cluster.KubernetesClusterVO; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil; +import com.cloud.kubernetes.version.KubernetesSupportedVersion; +import com.cloud.kubernetes.version.KubernetesVersionManagerImpl; +import com.cloud.network.IpAddress; +import com.cloud.network.Network; +import com.cloud.network.addr.PublicIp; +import com.cloud.network.rules.LoadBalancer; +import com.cloud.offering.ServiceOffering; +import com.cloud.template.VirtualMachineTemplate; +import com.cloud.user.Account; +import com.cloud.user.SSHKeyPairVO; +import com.cloud.uservm.UserVm; +import com.cloud.utils.Pair; +import com.cloud.utils.StringUtils; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.Ip; +import com.cloud.utils.net.NetUtils; +import com.cloud.vm.Nic; +import com.cloud.vm.ReservationContext; +import com.cloud.vm.ReservationContextImpl; +import com.cloud.vm.VirtualMachine; +import com.google.common.base.Strings; + +public class KubernetesClusterStartWorker extends KubernetesClusterResourceModifierActionWorker { + + public KubernetesClusterStartWorker(final KubernetesCluster kubernetesCluster, final KubernetesClusterManagerImpl clusterManager) { + super(kubernetesCluster, clusterManager); + } + + private Pair> getKubernetesMasterIpAddresses(final DataCenter zone, final Network network, final Account account) throws InsufficientAddressCapacityException { + String masterIp = null; + Map requestedIps = null; + if (Network.GuestType.Shared.equals(network.getGuestType())) { + List vlanIds = new ArrayList<>(); + List vlans = vlanDao.listVlansByNetworkId(network.getId()); + for (VlanVO vlan : vlans) { + vlanIds.add(vlan.getId()); + } + PublicIp ip = ipAddressManager.getAvailablePublicIpAddressFromVlans(zone.getId(), null, account, Vlan.VlanType.DirectAttached, vlanIds,network.getId(), null, false); + if (ip != null) { + masterIp = ip.getAddress().toString(); + } + requestedIps = new HashMap<>(); + Ip ipAddress = ip.getAddress(); + boolean isIp6 = ipAddress.isIp6(); + requestedIps.put(network.getId(), new Network.IpAddresses(ipAddress.isIp4() ? ip.getAddress().addr() : null, null)); + } else { + masterIp = ipAddressManager.acquireGuestIpAddress(networkDao.findById(kubernetesCluster.getNetworkId()), null); + } + return new Pair<>(masterIp, requestedIps); + } + + private boolean isKubernetesVersionSupportsHA() { + boolean haSupported = false; + final KubernetesSupportedVersion version = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); + if (version != null) { + try { + if (KubernetesVersionManagerImpl.compareSemanticVersions(version.getSemanticVersion(), KubernetesClusterService.MIN_KUBERNETES_VERSION_HA_SUPPORT) >= 0) { + haSupported = true; + } + } catch (IllegalArgumentException e) { + LOGGER.error(String.format("Unable to compare Kubernetes version for cluster version ID: %s with %s", version.getUuid(), KubernetesClusterService.MIN_KUBERNETES_VERSION_HA_SUPPORT), e); + } + } + return haSupported; + } + + private String getKubernetesMasterConfig(final String masterIp, final String serverIp, final Account owner, + final String hostName, final boolean haSupported) throws IOException { + String k8sMasterConfig = readResourceFile("/conf/k8s-master.yml"); + final String apiServerCert = "{{ k8s_master.apiserver.crt }}"; + final String apiServerKey = "{{ k8s_master.apiserver.key }}"; + final String caCert = "{{ k8s_master.ca.crt }}"; + final String sshPubKey = "{{ k8s.ssh.pub.key }}"; + final String clusterToken = "{{ k8s_master.cluster.token }}"; + final String clusterInitArgsKey = "{{ k8s_master.cluster.initargs }}"; + final List addresses = new ArrayList<>(); + addresses.add(masterIp); + if (!serverIp.equals(masterIp)) { + addresses.add(serverIp); + } + final Certificate certificate = caManager.issueCertificate(null, Arrays.asList(hostName, "kubernetes", + "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.default.svc.cluster.local"), + addresses, 3650, null); + final String tlsClientCert = CertUtils.x509CertificateToPem(certificate.getClientCertificate()); + final String tlsPrivateKey = CertUtils.privateKeyToPem(certificate.getPrivateKey()); + final String tlsCaCert = CertUtils.x509CertificatesToPem(certificate.getCaCertificates()); + k8sMasterConfig = k8sMasterConfig.replace(apiServerCert, tlsClientCert.replace("\n", "\n ")); + k8sMasterConfig = k8sMasterConfig.replace(apiServerKey, tlsPrivateKey.replace("\n", "\n ")); + k8sMasterConfig = k8sMasterConfig.replace(caCert, tlsCaCert.replace("\n", "\n ")); + String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\""; + String sshKeyPair = kubernetesCluster.getKeyPair(); + if (!Strings.isNullOrEmpty(sshKeyPair)) { + SSHKeyPairVO sshkp = sshKeyPairDao.findByName(owner.getAccountId(), owner.getDomainId(), sshKeyPair); + if (sshkp != null) { + pubKey += "\n - \"" + sshkp.getPublicKey() + "\""; + } + } + k8sMasterConfig = k8sMasterConfig.replace(sshPubKey, pubKey); + k8sMasterConfig = k8sMasterConfig.replace(clusterToken, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); + String initArgs = ""; + if (haSupported) { + initArgs = String.format("--control-plane-endpoint %s:%d --upload-certs --certificate-key %s ", + serverIp, + CLUSTER_API_PORT, + KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster)); + } + initArgs += String.format("--apiserver-cert-extra-sans=%s", serverIp); + k8sMasterConfig = k8sMasterConfig.replace(clusterInitArgsKey, initArgs); + return k8sMasterConfig; + } + + private UserVm createKubernetesMaster(final Network network, final Account account, String serverIp) throws ManagementServerException, + ResourceUnavailableException, InsufficientCapacityException { + UserVm masterVm = null; + DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); + ServiceOffering serviceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); + VirtualMachineTemplate template = templateDao.findById(kubernetesCluster.getTemplateId()); + List networkIds = new ArrayList(); + networkIds.add(kubernetesCluster.getNetworkId()); + Pair> ipAddresses = getKubernetesMasterIpAddresses(zone, network, account); + String masterIp = ipAddresses.first(); + Map requestedIps = ipAddresses.second(); + if (Network.GuestType.Shared.equals(network.getGuestType()) && Strings.isNullOrEmpty(serverIp)) { + serverIp = masterIp; + } + Network.IpAddresses addrs = new Network.IpAddresses(masterIp, null); + long rootDiskSize = kubernetesCluster.getNodeRootDiskSize(); + Map customParameterMap = new HashMap(); + if (rootDiskSize > 0) { + customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); + } + String hostName = kubernetesCluster.getName() + "-k8s-master"; + boolean haSupported = isKubernetesVersionSupportsHA(); + String k8sMasterConfig = null; + try { + k8sMasterConfig = getKubernetesMasterConfig(masterIp, serverIp, account, hostName, haSupported); + } catch (IOException e) { + logAndThrow(Level.ERROR, "Failed to read Kubernetes master configuration file", e); + } + String base64UserData = Base64.encodeBase64String(k8sMasterConfig.getBytes(StringUtils.getPreferredCharset())); + masterVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, account, + hostName, kubernetesCluster.getDescription(), null, null, null, + null, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), + requestedIps, addrs, null, null, null, customParameterMap, null, null, null, null); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Created master VM ID: %s, %s in the Kubernetes cluster ID: %s", masterVm.getUuid(), hostName, kubernetesCluster.getUuid())); + } + return masterVm; + } + + private String getKubernetesAdditionalMasterConfig(final String joinIp) throws IOException { + String k8sMasterConfig = readResourceFile("/conf/k8s-master-add.yml"); + final String joinIpKey = "{{ k8s_master.join_ip }}"; + final String clusterTokenKey = "{{ k8s_master.cluster.token }}"; + final String sshPubKey = "{{ k8s.ssh.pub.key }}"; + final String clusterHACertificateKey = "{{ k8s_master.cluster.ha.certificate.key }}"; + String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\""; + String sshKeyPair = kubernetesCluster.getKeyPair(); + if (!Strings.isNullOrEmpty(sshKeyPair)) { + SSHKeyPairVO sshkp = sshKeyPairDao.findByName(owner.getAccountId(), owner.getDomainId(), sshKeyPair); + if (sshkp != null) { + pubKey += "\n - \"" + sshkp.getPublicKey() + "\""; + } + } + k8sMasterConfig = k8sMasterConfig.replace(sshPubKey, pubKey); + k8sMasterConfig = k8sMasterConfig.replace(joinIpKey, joinIp); + k8sMasterConfig = k8sMasterConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); + k8sMasterConfig = k8sMasterConfig.replace(clusterHACertificateKey, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster)); + return k8sMasterConfig; + } + + private UserVm createKubernetesAdditionalMaster(final String joinIp, final int additionalMasterNodeInstance) throws ManagementServerException, + ResourceUnavailableException, InsufficientCapacityException { + UserVm additionalMasterVm = null; + DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); + ServiceOffering serviceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); + VirtualMachineTemplate template = templateDao.findById(kubernetesCluster.getTemplateId()); + List networkIds = new ArrayList(); + networkIds.add(kubernetesCluster.getNetworkId()); + Network.IpAddresses addrs = new Network.IpAddresses(null, null); + long rootDiskSize = kubernetesCluster.getNodeRootDiskSize(); + Map customParameterMap = new HashMap(); + if (rootDiskSize > 0) { + customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); + } + String hostName = String.format("%s-k8s-master-%s", kubernetesCluster.getName(), additionalMasterNodeInstance); + String k8sMasterConfig = null; + try { + k8sMasterConfig = getKubernetesAdditionalMasterConfig(joinIp); + } catch (IOException e) { + logAndThrow(Level.ERROR, "Failed to read Kubernetes master configuration file", e); + } + String base64UserData = Base64.encodeBase64String(k8sMasterConfig.getBytes(StringUtils.getPreferredCharset())); + additionalMasterVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, + hostName, kubernetesCluster.getDescription(), null, null, null, + null, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), + null, addrs, null, null, null, customParameterMap, null, null, null, null); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Created master VM ID: %s, %s in the Kubernetes cluster ID: %s", additionalMasterVm.getUuid(), hostName, kubernetesCluster.getUuid())); + } + return additionalMasterVm; + } + + private UserVm provisionKubernetesClusterMasterVm(final Network network, final Account account, final String publicIpAddress) throws CloudRuntimeException { + UserVm k8sMasterVM = null; + try { + k8sMasterVM = createKubernetesMaster(network, account, publicIpAddress); + addKubernetesClusterVm(kubernetesCluster.getId(), k8sMasterVM.getId()); + startKubernetesVM(k8sMasterVM); + k8sMasterVM = userVmDao.findById(k8sMasterVM.getId()); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Provisioned the master VM ID: %s in to the Kubernetes cluster ID: %s", k8sMasterVM.getUuid(), kubernetesCluster.getUuid())); + } + } catch (ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { + logTransitStateAndThrow(Level.ERROR, String.format("Provisioning the master VM failed in the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e); + } + return k8sMasterVM; + } + + private List provisionKubernetesClusterAdditionalMasterVms(final String publicIpAddress) throws ManagementServerException { + List additionalMasters = new ArrayList<>(); + if (kubernetesCluster.getMasterNodeCount() > 1) { + for (int i = 1; i < kubernetesCluster.getMasterNodeCount(); i++) { + UserVm vm = null; + try { + vm = createKubernetesAdditionalMaster(publicIpAddress, i); + addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId()); + startKubernetesVM(vm); + additionalMasters.add(vm); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Provisioned additional master VM ID: %s in to the Kubernetes cluster ID: %s", vm.getUuid(), kubernetesCluster.getUuid())); + } + } catch (ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { + logTransitStateAndThrow(Level.ERROR, String.format("Provisioning additional master VM %d/%d failed in the Kubernetes cluster ID: %s", i+1, kubernetesCluster.getMasterNodeCount(), kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e); + } + } + } + return additionalMasters; + } + + private Network startKubernetesClusterNetwork(final DeployDestination destination, final Account account) throws ManagementServerException { + final ReservationContext context = new ReservationContextImpl(null, null, null, account); + Network network = networkDao.findById(kubernetesCluster.getNetworkId()); + if (network == null) { + String msg = String.format("Network for Kubernetes cluster ID: %s not found", kubernetesCluster.getUuid()); + LOGGER.warn(msg); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); + throw new ManagementServerException(msg); + } + try { + networkMgr.startNetwork(network.getId(), destination, context); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Network ID: %s is started for the Kubernetes cluster ID: %s", network.getUuid(), kubernetesCluster.getUuid())); + } + } catch (ConcurrentOperationException | ResourceUnavailableException |InsufficientCapacityException e) { + String msg = String.format("Failed to start Kubernetes cluster ID: %s as unable to start associated network ID: %s" , kubernetesCluster.getUuid(), network.getUuid()); + LOGGER.error(msg, e); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); + throw new ManagementServerException(msg, e); + } + return network; + } + + private void provisionLoadBalancerRule(IpAddress publicIp, Network network, + Account account, List clusterVMIds, int port) throws NetworkRuleConflictException, + InsufficientAddressCapacityException { + LoadBalancer lb = lbService.createPublicLoadBalancerRule(null, "api-lb", "LB rule for API access", + port, port, port, port, + publicIp.getId(), NetUtils.TCP_PROTO, "roundrobin", network.getId(), + account.getId(), false, NetUtils.TCP_PROTO, true); + + Map> vmIdIpMap = new HashMap<>(); + for (int i = 0; i < kubernetesCluster.getMasterNodeCount(); ++i) { + List ips = new ArrayList<>(); + Nic masterVmNic = networkModel.getNicInNetwork(clusterVMIds.get(i), kubernetesCluster.getNetworkId()); + ips.add(masterVmNic.getIPv4Address()); + vmIdIpMap.put(clusterVMIds.get(i), ips); + } + lbService.assignToLoadBalancer(lb.getId(), null, vmIdIpMap); + } + + /** + * Setup network rules for Kubernetes cluster + * Open up firewall port CLUSTER_API_PORT, secure port on which Kubernetes + * API server is running. Also create load balancing rule to forward public + * IP traffic to master VMs' private IP. + * Open up firewall ports NODES_DEFAULT_START_SSH_PORT to NODES_DEFAULT_START_SSH_PORT+n + * for SSH access. Also create port-forwarding rule to forward public IP traffic to all + * @param network + * @param account + * @param clusterVMs + * @throws ManagementServerException + */ + private void setupKubernetesClusterNetworkRules(Network network, Account account, + List clusterVMs) throws ManagementServerException { + if (!Network.GuestType.Isolated.equals(network.getGuestType())) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Network ID: %s for Kubernetes cluster ID: %s is not an isolated network, therefore, no need for network rules", network.getUuid(), kubernetesCluster.getUuid())); + } + return; + } + List clusterVMIds = new ArrayList<>(); + for (UserVm vm : clusterVMs) { + clusterVMIds.add(vm.getId()); + } + IpAddress publicIp = getSourceNatIp(network); + if (publicIp == null) { + throw new ManagementServerException(String.format("No source NAT IP addresses found for network ID: %s, Kubernetes cluster ID: %s", network.getUuid(), kubernetesCluster.getUuid())); + } + + try { + provisionFirewallRules(publicIp, account, CLUSTER_API_PORT, CLUSTER_API_PORT); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Provisioned firewall rule to open up port %d on %s for Kubernetes cluster ID: %s", + CLUSTER_API_PORT, publicIp.getAddress().addr(), kubernetesCluster.getUuid())); + } + } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) { + throw new ManagementServerException(String.format("Failed to provision firewall rules for API access for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); + } + + try { + int endPort = CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterVMs.size() - 1; + provisionFirewallRules(publicIp, account, CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Provisioned firewall rule to open up port %d to %d on %s for Kubernetes cluster ID: %s", CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort, publicIp.getAddress().addr(), kubernetesCluster.getUuid())); + } + } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) { + throw new ManagementServerException(String.format("Failed to provision firewall rules for SSH access for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); + } + + // Load balancer rule fo API access for master node VMs + try { + provisionLoadBalancerRule(publicIp, network, account, clusterVMIds, CLUSTER_API_PORT); + } catch (NetworkRuleConflictException | InsufficientAddressCapacityException e) { + throw new ManagementServerException(String.format("Failed to provision load balancer rule for API access for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); + } + + // Port forwarding rule fo SSH access on each node VM + try { + provisionSshPortForwardingRules(publicIp, network, account, clusterVMIds, CLUSTER_NODES_DEFAULT_START_SSH_PORT); + } catch (ResourceUnavailableException | NetworkRuleConflictException e) { + throw new ManagementServerException(String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); + } + } + + private void startKubernetesClusterVMs() { + List clusterVms = getKubernetesClusterVMs(); + for (final UserVm vm : clusterVms) { + if (vm == null) { + logTransitStateAndThrow(Level.ERROR, String.format("Failed to start all VMs in Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + try { + startKubernetesVM(vm); + } catch (CloudRuntimeException ex) { + LOGGER.warn(String.format("Failed to start VM ID: %s in Kubernetes cluster ID: %s due to ", vm.getUuid(), kubernetesCluster.getUuid()) + ex); + // dont bail out here. proceed further to stop the reset of the VM's + } + } + for (final UserVm vm : clusterVms) { + if (vm == null || !vm.getState().equals(VirtualMachine.State.Running)) { + logTransitStateAndThrow(Level.ERROR, String.format("Failed to start all VMs in Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + } + } + + private boolean isKubernetesClusterKubeConfigAvailable(final String masterVMPrivateIpAddress) { + if (Strings.isNullOrEmpty(masterVMPrivateIpAddress)) { + KubernetesClusterDetailsVO kubeConfigDetail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "kubeConfigData"); + if (kubeConfigDetail != null && !Strings.isNullOrEmpty(kubeConfigDetail.getValue())) { + return true; + } + } + String kubeConfig = KubernetesClusterUtil.getKubernetesClusterConfig(kubernetesCluster, publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, 5); + if (!Strings.isNullOrEmpty(kubeConfig)) { + if (!Strings.isNullOrEmpty(masterVMPrivateIpAddress)) { + kubeConfig = kubeConfig.replace(String.format("server: https://%s:%d", masterVMPrivateIpAddress, CLUSTER_API_PORT), + String.format("server: https://%s:%d", publicIpAddress, CLUSTER_API_PORT)); + } + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "kubeConfigData", Base64.encodeBase64String(kubeConfig.getBytes(StringUtils.getPreferredCharset())), false); + return true; + } + return false; + } + + private boolean isKubernetesClusterDashboardServiceRunning(boolean onCreate) { + if (!onCreate) { + KubernetesClusterDetailsVO dashboardServiceRunningDetail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "dashboardServiceRunning"); + if (dashboardServiceRunningDetail != null && Boolean.parseBoolean(dashboardServiceRunningDetail.getValue())) { + return true; + } + } + if (KubernetesClusterUtil.isKubernetesClusterDashboardServiceRunning(kubernetesCluster, publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, 10, 20000)) { + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "dashboardServiceRunning", String.valueOf(true), false); + return true; + } + return false; + } + + private void updateKubernetesClusterEntryEndpoint() { + KubernetesClusterVO kubernetesClusterVO = kubernetesClusterDao.findById(kubernetesCluster.getId()); + kubernetesClusterVO.setEndpoint(String.format("https://%s:%d/", publicIpAddress, CLUSTER_API_PORT)); + kubernetesClusterDao.update(kubernetesCluster.getId(), kubernetesClusterVO); + } + + public boolean startKubernetesClusterOnCreate() throws ManagementServerException { + init(); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Starting Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + } + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.StartRequested); + DeployDestination dest = null; + try { + dest = plan(); + } catch (InsufficientCapacityException e) { + logTransitStateAndThrow(Level.ERROR, String.format("Provisioning the cluster failed due to insufficient capacity in the Kubernetes cluster: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e); + } + Network network = startKubernetesClusterNetwork(dest, owner); + Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(null); + publicIpAddress = publicIpSshPort.first(); + if (Strings.isNullOrEmpty(publicIpAddress) && + (Network.GuestType.Isolated.equals(network.getGuestType()) || kubernetesCluster.getMasterNodeCount() > 1)) { // Shared network, single-master cluster won't have an IP yet + logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s as no public IP found for the cluster" , kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); + } + List clusterVMs = new ArrayList<>(); + UserVm k8sMasterVM = provisionKubernetesClusterMasterVm(network, owner, publicIpAddress); + clusterVMs.add(k8sMasterVM); + if (Strings.isNullOrEmpty(publicIpAddress)) { + publicIpSshPort = getKubernetesClusterServerIpSshPort(k8sMasterVM); + publicIpAddress = publicIpSshPort.first(); + if (Strings.isNullOrEmpty(publicIpAddress)) { + logTransitStateAndThrow(Level.WARN, String.format("Failed to start Kubernetes cluster ID: %s as no public IP found for the cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); + } + } + List additionalMasterVMs = provisionKubernetesClusterAdditionalMasterVms(publicIpAddress); + clusterVMs.addAll(additionalMasterVMs); + try { + List nodeVMs = provisionKubernetesClusterNodeVms(kubernetesCluster.getNodeCount(), publicIpAddress); + clusterVMs.addAll(nodeVMs); + } catch (ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { + logTransitStateAndThrow(Level.ERROR, String.format("Provisioning node VM failed in the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e); + } + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Kubernetes cluster ID: %s VMs successfully provisioned", kubernetesCluster.getUuid())); + } + try { + setupKubernetesClusterNetworkRules(network, owner, clusterVMs); + } catch (ManagementServerException e) { + logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster ID: %s, unable to setup network rules", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e); + } + attachIsoKubernetesVMs(clusterVMs); + if (!KubernetesClusterUtil.isKubernetesClusterMasterVmRunning(kubernetesCluster, publicIpAddress, publicIpSshPort.second(), 10 * 60 * 1000)) { + String msg = String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to access master node VMs of the cluster", kubernetesCluster.getUuid()); + if (kubernetesCluster.getMasterNodeCount() > 1 && Network.GuestType.Shared.equals(network.getGuestType())) { + msg = String.format("%s. Make sure external load-balancer has port forwarding rules for SSH access on ports %d-%d and API access on port %d", + msg, + CLUSTER_NODES_DEFAULT_START_SSH_PORT, + CLUSTER_NODES_DEFAULT_START_SSH_PORT + kubernetesCluster.getTotalNodeCount() - 1, + CLUSTER_API_PORT); + } + logTransitStateDetachIsoAndThrow(Level.ERROR, msg, kubernetesCluster, clusterVMs, KubernetesCluster.Event.CreateFailed, null); + } + boolean k8sApiServerSetup = KubernetesClusterUtil.isKubernetesClusterServerRunning(kubernetesCluster, publicIpAddress, CLUSTER_API_PORT, 20, 30000); + if (!k8sApiServerSetup) { + logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to provision API endpoint for the cluster", kubernetesCluster.getUuid()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.CreateFailed, null); + } + sshPort = publicIpSshPort.second(); + updateKubernetesClusterEntryEndpoint(); + boolean readyNodesCountValid = KubernetesClusterUtil.validateKubernetesClusterReadyNodesCount(kubernetesCluster, publicIpAddress, sshPort, + CLUSTER_NODE_VM_USER, sshKeyFile, 30, 30000); + detachIsoKubernetesVMs(clusterVMs); + if (!readyNodesCountValid) { + logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster ID: %s as it does not have desired number of nodes in ready state", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); + } + if (!isKubernetesClusterKubeConfigAvailable(k8sMasterVM.getPrivateIpAddress())) { + logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to retrieve kube-config for the cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + if (!isKubernetesClusterDashboardServiceRunning(true)) { + logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to get Dashboard service running for the cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(),KubernetesCluster.Event.OperationFailed); + } + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); + return true; + } + + public boolean startStoppedKubernetesCluster() throws CloudRuntimeException { + init(); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Starting Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + } + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.StartRequested); + startKubernetesClusterVMs(); + InetAddress address = null; + try { + address = InetAddress.getByName(new URL(kubernetesCluster.getEndpoint()).getHost()); + } catch (MalformedURLException | UnknownHostException ex) { + logTransitStateAndThrow(Level.ERROR, String.format("Kubernetes cluster ID: %s has invalid API endpoint. Can not verify if cluster is in ready state", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(null); + String publicIpAddress = publicIpSshPort.first(); + if (Strings.isNullOrEmpty(publicIpAddress)) { + logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s as no public IP found for the cluster" , kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + if (!KubernetesClusterUtil.isKubernetesClusterServerRunning(kubernetesCluster, publicIpAddress, CLUSTER_API_PORT, 10, 30000)) { + logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s in usable state", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + sshPort = publicIpSshPort.second(); + if (!isKubernetesClusterKubeConfigAvailable(null)) { + logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s in usable state as unable to retrieve kube-config for the cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + if (!isKubernetesClusterDashboardServiceRunning(false)) { + logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s in usable state as unable to get Dashboard service running for the cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Kubernetes cluster ID: %s successfully started", kubernetesCluster.getUuid())); + } + return true; + } + + public boolean reconcileAlertCluster() { + init(); + Pair sshIpPort = getKubernetesClusterServerIpSshPort(null); + publicIpAddress = sshIpPort.first(); + sshPort = sshIpPort.second(); + if (Strings.isNullOrEmpty(publicIpAddress)) { + return false; + } + long actualNodeCount = 0; + try { + actualNodeCount = KubernetesClusterUtil.getKubernetesClusterReadyNodesCount(kubernetesCluster, publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile); + } catch (Exception e) { + return false; + } + if (kubernetesCluster.getTotalNodeCount() != actualNodeCount) { + return false; + } + if (Strings.isNullOrEmpty(sshIpPort.first())) { + return false; + } + if (!KubernetesClusterUtil.isKubernetesClusterServerRunning(kubernetesCluster, sshIpPort.first(), + KubernetesClusterActionWorker.CLUSTER_API_PORT, 1, 0)) { + return false; + } + updateKubernetesClusterEntryEndpoint(); + if (!isKubernetesClusterKubeConfigAvailable(null)) { + return false; + } + if (!isKubernetesClusterDashboardServiceRunning(false)) { + return false; + } + // mark the cluster to be running + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.RecoveryRequested); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); + return true; + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStopWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStopWorker.java new file mode 100644 index 000000000000..ff03a0198b59 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStopWorker.java @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.kubernetes.cluster.actionworkers; + +import java.util.List; + +import org.apache.log4j.Level; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.uservm.UserVm; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VirtualMachine; + +public class KubernetesClusterStopWorker extends KubernetesClusterActionWorker { + public KubernetesClusterStopWorker(final KubernetesCluster kubernetesCluster, final KubernetesClusterManagerImpl clusterManager) { + super(kubernetesCluster, clusterManager); + } + + public boolean stop() throws CloudRuntimeException { + init(); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Stopping Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + } + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.StopRequested); + List clusterVMs = getKubernetesClusterVMs(); + for (UserVm vm : clusterVMs) { + if (vm == null) { + logTransitStateAndThrow(Level.ERROR, String.format("Failed to find all VMs in Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + try { + userVmService.stopVirtualMachine(vm.getId(), false); + } catch (ConcurrentOperationException ex) { + LOGGER.warn(String.format("Failed to stop VM ID: %s in Kubernetes cluster ID: %s", vm.getUuid(), kubernetesCluster.getUuid()), ex); + } + } + for (final UserVm vm : clusterVMs) { + if (vm == null || !vm.getState().equals(VirtualMachine.State.Stopped)) { + logTransitStateAndThrow(Level.ERROR, String.format("Failed to stop all VMs in Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + } + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); + return true; + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java new file mode 100644 index 000000000000..8b325e32cfe8 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java @@ -0,0 +1,157 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.kubernetes.cluster.actionworkers; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.log4j.Level; + +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterVO; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil; +import com.cloud.kubernetes.version.KubernetesSupportedVersion; +import com.cloud.kubernetes.version.KubernetesVersionManagerImpl; +import com.cloud.uservm.UserVm; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.ssh.SshHelper; +import com.google.common.base.Strings; + +public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorker { + + private List clusterVMs = new ArrayList<>(); + private KubernetesSupportedVersion upgradeVersion; + private File upgradeScriptFile; + + public KubernetesClusterUpgradeWorker(final KubernetesCluster kubernetesCluster, + final KubernetesSupportedVersion upgradeVersion, + final KubernetesClusterManagerImpl clusterManager) { + super(kubernetesCluster, clusterManager); + this.upgradeVersion = upgradeVersion; + } + + private void retrieveUpgradeScriptFile() { + try { + String upgradeScriptData = readResourceFile("/script/upgrade-kubernetes.sh"); + upgradeScriptFile = File.createTempFile("upgrade-kuberntes", ".sh"); + BufferedWriter upgradeScriptFileWriter = new BufferedWriter(new FileWriter(upgradeScriptFile)); + upgradeScriptFileWriter.write(upgradeScriptData); + upgradeScriptFileWriter.close(); + } catch (IOException e) { + logAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to prepare upgrade script", kubernetesCluster.getUuid()), e); + } + } + + private Pair runInstallScriptOnVM(final UserVm vm, final int index) throws Exception { + int nodeSshPort = sshPort == 22 ? sshPort : sshPort + index; + String nodeAddress = (index > 0 && sshPort == 22) ? vm.getPrivateIpAddress() : publicIpAddress; + SshHelper.scpTo(nodeAddress, nodeSshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null, + "~/", upgradeScriptFile.getAbsolutePath(), "0755"); + String cmdStr = String.format("sudo ./%s %s %s %s", upgradeScriptFile.getName(), + upgradeVersion.getSemanticVersion(), index == 0 ? "true" : "false", + KubernetesVersionManagerImpl.compareSemanticVersions(upgradeVersion.getSemanticVersion(), "1.15.0") < 0 ? "true" : "false"); + return SshHelper.sshExecute(publicIpAddress, nodeSshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null, + cmdStr, + 10000, 10000, 10 * 60 * 1000); + } + + private void upgradeKubernetesClusterNodes() { + Pair result = null; + for (int i = 0; i < clusterVMs.size(); ++i) { + UserVm vm = clusterVMs.get(i); + String hostName = vm.getHostName(); + if (!Strings.isNullOrEmpty(hostName)) { + hostName = hostName.toLowerCase(); + } + result = null; + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Upgrading node on VM ID: %s in Kubernetes cluster ID: %s with Kubernetes version(%s) ID: %s", + vm.getUuid(), kubernetesCluster.getUuid(), upgradeVersion.getSemanticVersion(), upgradeVersion.getUuid())); + } + try { + result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null, + String.format("sudo kubectl drain %s --ignore-daemonsets --delete-local-data", hostName), + 10000, 10000, 60000); + } catch (Exception e) { + logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to drain Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, e); + } + if (!result.first()) { + logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to drain Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); + } + try { + result = runInstallScriptOnVM(vm, i); + } catch (Exception e) { + logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to upgrade Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, e); + } + if (!result.first()) { + logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to upgrade Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); + + } + if (!KubernetesClusterUtil.uncordonKubernetesClusterNode(kubernetesCluster, publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, getManagementServerSshPublicKeyFile(), vm, 5, 30000)) { + logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to uncordon Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); + } + if (i == 0) { // Wait for master to get in Ready state + if (!KubernetesClusterUtil.isKubernetesClusterNodeReady(kubernetesCluster, publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, getManagementServerSshPublicKeyFile(), hostName, 5, 20000)) { + logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to get master Kubernetes node on VM ID: %s in ready state", kubernetesCluster.getUuid(), vm.getUuid()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); + } + } + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Successfully upgraded node on VM ID: %s in Kubernetes cluster ID: %s with Kubernetes version(%s) ID: %s", + vm.getUuid(), kubernetesCluster.getUuid(), upgradeVersion.getSemanticVersion(), upgradeVersion.getUuid())); + } + } + } + + public boolean upgradeCluster() throws CloudRuntimeException { + init(); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Upgrading Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + } + Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(null); + publicIpAddress = publicIpSshPort.first(); + sshPort = publicIpSshPort.second(); + if (Strings.isNullOrEmpty(publicIpAddress)) { + logAndThrow(Level.ERROR, String.format("Upgrade failed for Kubernetes cluster ID: %s, unable to retrieve associated public IP", kubernetesCluster.getUuid())); + } + clusterVMs = getKubernetesClusterVMs(); + if (CollectionUtils.isEmpty(clusterVMs)) { + logAndThrow(Level.ERROR, String.format("Upgrade failed for Kubernetes cluster ID: %s, unable to retrieve VMs for cluster", kubernetesCluster.getUuid())); + } + retrieveUpgradeScriptFile(); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.UpgradeRequested); + attachIsoKubernetesVMs(clusterVMs); + upgradeKubernetesClusterNodes(); + detachIsoKubernetesVMs(clusterVMs); + KubernetesClusterVO kubernetesClusterVO = kubernetesClusterDao.findById(kubernetesCluster.getId()); + kubernetesClusterVO.setKubernetesVersionId(upgradeVersion.getId()); + boolean updated = kubernetesClusterDao.update(kubernetesCluster.getId(), kubernetesClusterVO); + if (!updated) { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } else { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); + } + return updated; + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDao.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDao.java similarity index 90% rename from plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDao.java rename to plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDao.java index 3192c94cd67a..fe673234ec88 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDao.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDao.java @@ -14,12 +14,12 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.cloud.kubernetescluster.dao; +package com.cloud.kubernetes.cluster.dao; import java.util.List; -import com.cloud.kubernetescluster.KubernetesCluster; -import com.cloud.kubernetescluster.KubernetesClusterVO; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterVO; import com.cloud.utils.db.GenericDao; import com.cloud.utils.fsm.StateDao; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDaoImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDaoImpl.java similarity index 94% rename from plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDaoImpl.java rename to plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDaoImpl.java index d31266675dac..5d750bc199b6 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDaoImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDaoImpl.java @@ -14,15 +14,14 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.cloud.kubernetescluster.dao; +package com.cloud.kubernetes.cluster.dao; import java.util.List; import org.springframework.stereotype.Component; -import com.cloud.kubernetescluster.KubernetesCluster; -import com.cloud.kubernetescluster.KubernetesCluster.Event; -import com.cloud.kubernetescluster.KubernetesClusterVO; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterVO; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; @@ -83,7 +82,7 @@ public List findKubernetesClustersInState(KubernetesCluster } @Override - public boolean updateState(KubernetesCluster.State currentState, Event event, KubernetesCluster.State nextState, + public boolean updateState(KubernetesCluster.State currentState, KubernetesCluster.Event event, KubernetesCluster.State nextState, KubernetesCluster vo, Object data) { // TODO: ensure this update is correct TransactionLegacy txn = TransactionLegacy.currentTxn(); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDetailsDao.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDetailsDao.java similarity index 90% rename from plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDetailsDao.java rename to plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDetailsDao.java index 109114f578ea..52990ebf1b84 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDetailsDao.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDetailsDao.java @@ -14,12 +14,12 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.cloud.kubernetescluster.dao; +package com.cloud.kubernetes.cluster.dao; import org.apache.cloudstack.resourcedetail.ResourceDetailsDao; -import com.cloud.kubernetescluster.KubernetesClusterDetailsVO; +import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO; import com.cloud.utils.db.GenericDao; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDetailsDaoImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDetailsDaoImpl.java similarity index 92% rename from plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDetailsDaoImpl.java rename to plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDetailsDaoImpl.java index fe751009e27b..66ef2adbc91c 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterDetailsDaoImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDetailsDaoImpl.java @@ -14,12 +14,12 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.cloud.kubernetescluster.dao; +package com.cloud.kubernetes.cluster.dao; import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; import org.springframework.stereotype.Component; -import com.cloud.kubernetescluster.KubernetesClusterDetailsVO; +import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO; @Component diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterVmMapDao.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java similarity index 90% rename from plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterVmMapDao.java rename to plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java index 4b8143e91840..8b08dd37d553 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterVmMapDao.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java @@ -14,9 +14,9 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.cloud.kubernetescluster.dao; +package com.cloud.kubernetes.cluster.dao; -import com.cloud.kubernetescluster.KubernetesClusterVmMapVO; +import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; import com.cloud.utils.db.GenericDao; import java.util.List; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterVmMapDaoImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java similarity index 94% rename from plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterVmMapDaoImpl.java rename to plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java index f32474cf22f5..0b86b2c1a622 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/dao/KubernetesClusterVmMapDaoImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java @@ -14,13 +14,13 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.cloud.kubernetescluster.dao; +package com.cloud.kubernetes.cluster.dao; import java.util.List; import org.springframework.stereotype.Component; -import com.cloud.kubernetescluster.KubernetesClusterVmMapVO; +import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java new file mode 100644 index 000000000000..d238449ff28a --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java @@ -0,0 +1,306 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.kubernetes.cluster.utils; + +import java.io.File; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.net.URL; + +import org.apache.commons.io.IOUtils; +import org.apache.log4j.Logger; + +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.uservm.UserVm; +import com.cloud.utils.Pair; +import com.cloud.utils.StringUtils; +import com.cloud.utils.ssh.SshHelper; +import com.google.common.base.Strings; + +public class KubernetesClusterUtil { + + protected static final Logger LOGGER = Logger.getLogger(KubernetesClusterUtil.class); + + public static boolean isKubernetesClusterNodeReady(KubernetesCluster kubernetesCluster, String ipAddress, int port, + String user, File sshKeyFile, String nodeName) throws Exception { + Pair result = SshHelper.sshExecute(ipAddress, port, + user, sshKeyFile, null, + String.format("sudo kubectl get nodes | awk '{if ($1 == \"%s\" && $2 == \"Ready\") print $1}'", nodeName.toLowerCase()), + 10000, 10000, 20000); + if (result.first() && nodeName.equals(result.second().trim())) { + return true; + } + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Failed to retrieve status for node: %s in Kubernetes cluster ID: %s. Output: %s", nodeName, kubernetesCluster.getUuid(), result.second())); + } + return false; + } + + public static boolean isKubernetesClusterNodeReady(KubernetesCluster kubernetesCluster, String ipAddress, int port, + String user, File sshKeyFile, String nodeName, int retries, + int waitDuration) { + int retryCounter = 0; + while (retryCounter < retries) { + boolean ready = false; + try { + ready = isKubernetesClusterNodeReady(kubernetesCluster, ipAddress, port, user, sshKeyFile, nodeName); + } catch (Exception e) { + LOGGER.warn(String.format("Failed to retrieve state of node: %s in Kubernetes cluster ID: %s", nodeName, kubernetesCluster.getUuid()), e); + } + if (ready) { + return true; + } + try { + Thread.sleep(waitDuration); + } catch (InterruptedException ie) { + LOGGER.error(String.format("Error while waiting for Kubernetes cluster ID: %s node: %s to become ready", kubernetesCluster.getUuid(), nodeName), ie); + } + retryCounter++; + } + return false; + } + + public static boolean uncordonKubernetesClusterNode(KubernetesCluster kubernetesCluster, String ipAddress, int port, + String user, File sshKeyFile, UserVm userVm, int retries, int waitDuration) { + int retryCounter = 0; + String hostName = userVm.getHostName(); + if (!Strings.isNullOrEmpty(hostName)) { + hostName = hostName.toLowerCase(); + } + while (retryCounter < retries) { + Pair result = null; + try { + result = SshHelper.sshExecute(ipAddress, port, user, sshKeyFile, null, + String.format("sudo kubectl uncordon %s", hostName), + 10000, 10000, 30000); + if (result.first()) { + return true; + } + } catch (Exception e) { + LOGGER.warn(String.format("Failed to uncordon node: %s on VM ID: %s in Kubernetes cluster ID: %s", hostName, userVm.getUuid(), kubernetesCluster.getUuid()), e); + } + try { + Thread.sleep(waitDuration); + } catch (InterruptedException ie) { + LOGGER.warn(String.format("Error while waiting for uncordon Kubernetes cluster ID: %s node: %s on VM ID: %s", kubernetesCluster.getUuid(), hostName, userVm.getUuid()), ie); + } + retryCounter++; + } + return false; + } + + public static boolean isKubernetesClusterAddOnServiceRunning(KubernetesCluster kubernetesCluster, final String ipAddress, + final int port, final String user, final File sshKeyFile, + final String namespace, String serviceName) { + try { + String cmd = "sudo kubectl get pods --all-namespaces"; + if (!Strings.isNullOrEmpty(namespace)) { + cmd = String.format("sudo kubectl get pods --namespace=%s", namespace); + } + Pair result = SshHelper.sshExecute(ipAddress, port, user, + sshKeyFile, null, cmd, + 10000, 10000, 10000); + if (result.first() && !Strings.isNullOrEmpty(result.second())) { + String[] lines = result.second().split("\n"); + for (String line : + lines) { + if (line.contains(serviceName) && line.contains("Running")) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Service : %s in namespace: %s for the Kubernetes cluster ID: %s is running", serviceName, namespace, kubernetesCluster.getUuid())); + } + return true; + } + } + } + } catch (Exception e) { + LOGGER.warn(String.format("Unable to retrieve service: %s running status in namespace %s for Kubernetes cluster ID: %s", serviceName, namespace, kubernetesCluster.getUuid()), e); + } + return false; + } + + public static boolean isKubernetesClusterDashboardServiceRunning(KubernetesCluster kubernetesCluster, String ipAddress, + final int port, final String user, final File sshKeyFile, + int retries, long waitDuration) { + boolean running = false; + int retryCounter = 0; + // Check if dashboard service is up running. + while (retryCounter < retries) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Checking dashboard service for the Kubernetes cluster ID: %s to come up. Attempt: %d/%d", kubernetesCluster.getUuid(), retryCounter + 1, retries)); + } + if (isKubernetesClusterAddOnServiceRunning(kubernetesCluster, ipAddress, port, user, sshKeyFile, "kubernetes-dashboard", "kubernetes-dashboard")) { + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Dashboard service for the Kubernetes cluster ID: %s is in running state", kubernetesCluster.getUuid())); + } + running = true; + break; + } + try { + Thread.sleep(waitDuration); + } catch (InterruptedException ex) { + LOGGER.error(String.format("Error while waiting for Kubernetes cluster: %s API dashboard service to be available", kubernetesCluster.getUuid()), ex); + } + retryCounter++; + } + return running; + } + + public static String getKubernetesClusterConfig(KubernetesCluster kubernetesCluster, String ipAddress, int port, + final String user, final File sshKeyFile,int retries) { + int retryCounter = 0; + String kubeConfig = ""; + while (retryCounter < retries) { + try { + Pair result = SshHelper.sshExecute(ipAddress, port, user, + sshKeyFile, null, "sudo cat /etc/kubernetes/admin.conf", + 10000, 10000, 10000); + + if (result.first() && !Strings.isNullOrEmpty(result.second())) { + kubeConfig = result.second(); + break; + } else { + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Failed to retrieve kube-config file for Kubernetes cluster ID: %s. Output: %s", kubernetesCluster.getUuid(), result.second())); + } + } + } catch (Exception e) { + LOGGER.warn(String.format("Failed to retrieve kube-config file for Kubernetes cluster ID: %s. Attempt: %d/%d", kubernetesCluster.getUuid(), retryCounter+1, retries), e); + } + retryCounter++; + } + return kubeConfig; + } + + public static int getKubernetesClusterReadyNodesCount(final KubernetesCluster kubernetesCluster, final String ipAddress, + final int port, final String user, final File sshKeyFile) throws Exception { + Pair result = SshHelper.sshExecute(ipAddress, port, + user, sshKeyFile, null, + "sudo kubectl get nodes | awk '{if ($2 == \"Ready\") print $1}' | wc -l", + 10000, 10000, 20000); + if (result.first()) { + return Integer.parseInt(result.second().trim().replace("\"", "")); + } else { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Failed to retrieve ready nodes for Kubernetes cluster ID: %s. Output: %s", kubernetesCluster.getUuid(), result.second())); + } + } + return 0; + } + + public static boolean isKubernetesClusterServerRunning(KubernetesCluster kubernetesCluster, String ipAddress, + int port,int retries, long waitDuration) { + int retryCounter = 0; + boolean k8sApiServerSetup = false; + while (retryCounter < retries) { + try { + String versionOutput = IOUtils.toString(new URL(String.format("https://%s:%d/version", ipAddress, port)), StringUtils.getPreferredCharset()); + if (!Strings.isNullOrEmpty(versionOutput)) { + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Kubernetes cluster ID: %s API has been successfully provisioned, %s", kubernetesCluster.getUuid(), versionOutput)); + } + k8sApiServerSetup = true; + break; + } + } catch (Exception e) { + LOGGER.warn(String.format("API endpoint for Kubernetes cluster ID: %s not available. Attempt: %d/%d", kubernetesCluster.getUuid(), retryCounter+1, retries), e); + } + try { + Thread.sleep(waitDuration); + } catch (InterruptedException ie) { + LOGGER.error(String.format("Error while waiting for Kubernetes cluster ID: %s API endpoint to be available", kubernetesCluster.getUuid()), ie); + } + retryCounter++; + } + return k8sApiServerSetup; + } + + public static boolean isKubernetesClusterMasterVmRunning(final KubernetesCluster kubernetesCluster, + final String ipAddress, final int port, final long timeout) { + boolean masterVmRunning = false; + long startTime = System.currentTimeMillis(); + while (!masterVmRunning && System.currentTimeMillis() - startTime < timeout) { + try (Socket socket = new Socket()) { + socket.connect(new InetSocketAddress(ipAddress, port), 10000); + masterVmRunning = true; + } catch (IOException e) { + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Waiting for Kubernetes cluster ID: %s master node VMs to be accessible", kubernetesCluster.getUuid())); + } + try { + Thread.sleep(10000); + } catch (InterruptedException ex) { + LOGGER.warn(String.format("Error while waiting for Kubernetes cluster ID: %s master node VMs to be accessible", kubernetesCluster.getUuid()), ex); + } + } + } + return masterVmRunning; + } + + public static boolean validateKubernetesClusterReadyNodesCount(final KubernetesCluster kubernetesCluster, + final String ipAddress, int port, + final String user, final File sshKeyFile, + int retries, long waitDuration) { + int retryCounter = 0; + while (retryCounter < retries) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Checking ready nodes for the Kubernetes cluster ID: %s with total %d provisioned nodes. Attempt: %d/%d", kubernetesCluster.getUuid(), kubernetesCluster.getTotalNodeCount(), retryCounter + 1, retries)); + } + try { + int nodesCount = KubernetesClusterUtil.getKubernetesClusterReadyNodesCount(kubernetesCluster, ipAddress, port, + user, sshKeyFile);; + if (nodesCount == kubernetesCluster.getTotalNodeCount()) { + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Kubernetes cluster ID: %s has %d ready nodes now", kubernetesCluster.getUuid(), kubernetesCluster.getTotalNodeCount())); + } + return true; + } else { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format("Kubernetes cluster ID: %s has total %d provisioned nodes while %d ready now", kubernetesCluster.getUuid(), kubernetesCluster.getTotalNodeCount(), nodesCount)); + } + } + } catch (Exception e) { + LOGGER.warn(String.format("Failed to retrieve ready node count for Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); + } + try { + Thread.sleep(waitDuration); + } catch (InterruptedException ex) { + LOGGER.warn(String.format("Error while waiting during Kubernetes cluster ID: %s ready node check. %d/%d", kubernetesCluster.getUuid(), retryCounter+1, retries), ex); + } + retryCounter++; + } + return false; + } + + public static String generateClusterToken(final KubernetesCluster kubernetesCluster) { + String token = kubernetesCluster.getUuid(); + token = token.replaceAll("-", ""); + token = token.substring(0, 22); + token = token.substring(0, 6) + "." + token.substring(6); + return token; + } + + public static String generateClusterHACertificateKey(final KubernetesCluster kubernetesCluster) { + String uuid = kubernetesCluster.getUuid(); + StringBuilder token = new StringBuilder(uuid.replaceAll("-", "")); + while (token.length() < 64) { + token.append(token); + } + return token.toString().substring(0, 64); + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesSupportedVersion.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesSupportedVersion.java similarity index 96% rename from plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesSupportedVersion.java rename to plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesSupportedVersion.java index d09d6530add1..3a2e885d9eb6 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesSupportedVersion.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesSupportedVersion.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package com.cloud.kubernetesversion; +package com.cloud.kubernetes.version; import org.apache.cloudstack.api.Identity; import org.apache.cloudstack.api.InternalIdentity; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesSupportedVersionVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesSupportedVersionVO.java similarity index 98% rename from plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesSupportedVersionVO.java rename to plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesSupportedVersionVO.java index 30c3c9b02751..536e25b0a20b 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesSupportedVersionVO.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesSupportedVersionVO.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package com.cloud.kubernetesversion; +package com.cloud.kubernetes.version; import java.util.Date; import java.util.UUID; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionEventTypes.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionEventTypes.java similarity index 96% rename from plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionEventTypes.java rename to plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionEventTypes.java index 6f067e736035..ad169a6d4129 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionEventTypes.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionEventTypes.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package com.cloud.kubernetesversion; +package com.cloud.kubernetes.version; public class KubernetesVersionEventTypes { public static final String EVENT_KUBERNETES_VERSION_ADD = "KUBERNETES.VERSION.ADD"; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java similarity index 97% rename from plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java rename to plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java index 5f4e82827d8d..6d6f797414c7 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package com.cloud.kubernetesversion; +package com.cloud.kubernetes.version; import java.lang.reflect.Field; import java.util.ArrayList; @@ -24,11 +24,11 @@ import javax.inject.Inject; import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.command.admin.kubernetesversion.AddKubernetesSupportedVersionCmd; -import org.apache.cloudstack.api.command.admin.kubernetesversion.DeleteKubernetesSupportedVersionCmd; +import org.apache.cloudstack.api.command.admin.kubernetes.version.AddKubernetesSupportedVersionCmd; +import org.apache.cloudstack.api.command.admin.kubernetes.version.DeleteKubernetesSupportedVersionCmd; import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; import org.apache.cloudstack.api.command.user.iso.RegisterIsoCmd; -import org.apache.cloudstack.api.command.user.kubernetesversion.ListKubernetesSupportedVersionsCmd; +import org.apache.cloudstack.api.command.user.kubernetes.version.ListKubernetesSupportedVersionsCmd; import org.apache.cloudstack.api.response.KubernetesSupportedVersionResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.log4j.Logger; @@ -40,10 +40,10 @@ import com.cloud.event.ActionEvent; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.ResourceAllocationException; -import com.cloud.kubernetescluster.KubernetesClusterService; -import com.cloud.kubernetescluster.KubernetesClusterVO; -import com.cloud.kubernetescluster.dao.KubernetesClusterDao; -import com.cloud.kubernetesversion.dao.KubernetesSupportedVersionDao; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.kubernetes.cluster.KubernetesClusterVO; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; +import com.cloud.kubernetes.version.dao.KubernetesSupportedVersionDao; import com.cloud.storage.Storage; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.VMTemplateZoneVO; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionService.java similarity index 72% rename from plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionService.java rename to plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionService.java index 6d4886c7608a..cb84b37024ab 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/KubernetesVersionService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionService.java @@ -15,19 +15,20 @@ // specific language governing permissions and limitations // under the License. -package com.cloud.kubernetesversion; +package com.cloud.kubernetes.version; -import org.apache.cloudstack.api.command.admin.kubernetesversion.AddKubernetesSupportedVersionCmd; -import org.apache.cloudstack.api.command.admin.kubernetesversion.DeleteKubernetesSupportedVersionCmd; -import org.apache.cloudstack.api.command.user.kubernetesversion.ListKubernetesSupportedVersionsCmd; +import org.apache.cloudstack.api.command.admin.kubernetes.version.AddKubernetesSupportedVersionCmd; +import org.apache.cloudstack.api.command.admin.kubernetes.version.DeleteKubernetesSupportedVersionCmd; +import org.apache.cloudstack.api.command.user.kubernetes.version.ListKubernetesSupportedVersionsCmd; import org.apache.cloudstack.api.response.KubernetesSupportedVersionResponse; import org.apache.cloudstack.api.response.ListResponse; import com.cloud.utils.component.PluggableService; +import com.cloud.utils.exception.CloudRuntimeException; public interface KubernetesVersionService extends PluggableService { static final String MIN_KUBERNETES_VERSION = "1.11.0"; ListResponse listKubernetesSupportedVersions(ListKubernetesSupportedVersionsCmd cmd); - KubernetesSupportedVersionResponse addKubernetesSupportedVersion(AddKubernetesSupportedVersionCmd cmd); - boolean deleteKubernetesSupportedVersion(DeleteKubernetesSupportedVersionCmd cmd); + KubernetesSupportedVersionResponse addKubernetesSupportedVersion(AddKubernetesSupportedVersionCmd cmd) throws CloudRuntimeException; + boolean deleteKubernetesSupportedVersion(DeleteKubernetesSupportedVersionCmd cmd) throws CloudRuntimeException; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/dao/KubernetesSupportedVersionDao.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/dao/KubernetesSupportedVersionDao.java similarity index 90% rename from plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/dao/KubernetesSupportedVersionDao.java rename to plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/dao/KubernetesSupportedVersionDao.java index 551c3e30673c..69de862985ba 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/dao/KubernetesSupportedVersionDao.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/dao/KubernetesSupportedVersionDao.java @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -package com.cloud.kubernetesversion.dao; +package com.cloud.kubernetes.version.dao; import java.util.List; -import com.cloud.kubernetesversion.KubernetesSupportedVersionVO; +import com.cloud.kubernetes.version.KubernetesSupportedVersionVO; import com.cloud.utils.db.GenericDao; public interface KubernetesSupportedVersionDao extends GenericDao { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/dao/KubernetesSupportedVersionDaoImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/dao/KubernetesSupportedVersionDaoImpl.java similarity index 90% rename from plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/dao/KubernetesSupportedVersionDaoImpl.java rename to plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/dao/KubernetesSupportedVersionDaoImpl.java index ab866a6ea58f..5dd6eff199ae 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetesversion/dao/KubernetesSupportedVersionDaoImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/dao/KubernetesSupportedVersionDaoImpl.java @@ -15,14 +15,17 @@ // specific language governing permissions and limitations // under the License. -package com.cloud.kubernetesversion.dao; +package com.cloud.kubernetes.version.dao; import java.util.List; -import com.cloud.kubernetesversion.KubernetesSupportedVersionVO; +import org.springframework.stereotype.Component; + +import com.cloud.kubernetes.version.KubernetesSupportedVersionVO; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchCriteria; +@Component public class KubernetesSupportedVersionDaoImpl extends GenericDaoBase implements KubernetesSupportedVersionDao { public KubernetesSupportedVersionDaoImpl() { } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java deleted file mode 100644 index 52c38268d3a2..000000000000 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetescluster/KubernetesClusterManagerImpl.java +++ /dev/null @@ -1,3253 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.kubernetescluster; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.lang.reflect.Field; -import java.math.BigInteger; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.MalformedURLException; -import java.net.Socket; -import java.net.URL; -import java.net.UnknownHostException; -import java.nio.charset.Charset; -import java.security.SecureRandom; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import javax.inject.Inject; -import javax.naming.ConfigurationException; - -import org.apache.cloudstack.acl.ControlledEntity; -import org.apache.cloudstack.acl.SecurityChecker; -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.BaseCmd; -import org.apache.cloudstack.api.command.user.firewall.CreateFirewallRuleCmd; -import org.apache.cloudstack.api.command.user.kubernetescluster.CreateKubernetesClusterCmd; -import org.apache.cloudstack.api.command.user.kubernetescluster.DeleteKubernetesClusterCmd; -import org.apache.cloudstack.api.command.user.kubernetescluster.GetKubernetesClusterConfigCmd; -import org.apache.cloudstack.api.command.user.kubernetescluster.ListKubernetesClustersCmd; -import org.apache.cloudstack.api.command.user.kubernetescluster.ScaleKubernetesClusterCmd; -import org.apache.cloudstack.api.command.user.kubernetescluster.StartKubernetesClusterCmd; -import org.apache.cloudstack.api.command.user.kubernetescluster.StopKubernetesClusterCmd; -import org.apache.cloudstack.api.command.user.kubernetescluster.UpgradeKubernetesClusterCmd; -import org.apache.cloudstack.api.command.user.vm.StartVMCmd; -import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse; -import org.apache.cloudstack.api.response.KubernetesClusterResponse; -import org.apache.cloudstack.api.response.ListResponse; -import org.apache.cloudstack.ca.CAManager; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; -import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; -import org.apache.cloudstack.framework.ca.Certificate; -import org.apache.cloudstack.framework.config.ConfigKey; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.managed.context.ManagedContextRunnable; -import org.apache.cloudstack.utils.security.CertUtils; -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.io.IOUtils; -import org.apache.log4j.Level; -import org.apache.log4j.Logger; - -import com.cloud.api.ApiDBUtils; -import com.cloud.api.query.dao.NetworkOfferingJoinDao; -import com.cloud.api.query.dao.TemplateJoinDao; -import com.cloud.api.query.vo.NetworkOfferingJoinVO; -import com.cloud.api.query.vo.TemplateJoinVO; -import com.cloud.capacity.CapacityManager; -import com.cloud.dc.ClusterDetailsDao; -import com.cloud.dc.ClusterDetailsVO; -import com.cloud.dc.ClusterVO; -import com.cloud.dc.DataCenter; -import com.cloud.dc.DataCenterVO; -import com.cloud.dc.Pod; -import com.cloud.dc.Vlan; -import com.cloud.dc.VlanVO; -import com.cloud.dc.dao.ClusterDao; -import com.cloud.dc.dao.DataCenterDao; -import com.cloud.dc.dao.VlanDao; -import com.cloud.deploy.DeployDestination; -import com.cloud.exception.ConcurrentOperationException; -import com.cloud.exception.InsufficientAddressCapacityException; -import com.cloud.exception.InsufficientCapacityException; -import com.cloud.exception.InsufficientServerCapacityException; -import com.cloud.exception.InvalidParameterValueException; -import com.cloud.exception.ManagementServerException; -import com.cloud.exception.NetworkRuleConflictException; -import com.cloud.exception.PermissionDeniedException; -import com.cloud.exception.ResourceAllocationException; -import com.cloud.exception.ResourceUnavailableException; -import com.cloud.exception.VirtualMachineMigrationException; -import com.cloud.host.Host.Type; -import com.cloud.host.HostVO; -import com.cloud.hypervisor.Hypervisor; -import com.cloud.kubernetescluster.dao.KubernetesClusterDao; -import com.cloud.kubernetescluster.dao.KubernetesClusterDetailsDao; -import com.cloud.kubernetescluster.dao.KubernetesClusterVmMapDao; -import com.cloud.kubernetesversion.KubernetesSupportedVersion; -import com.cloud.kubernetesversion.KubernetesSupportedVersionVO; -import com.cloud.kubernetesversion.KubernetesVersionManagerImpl; -import com.cloud.kubernetesversion.dao.KubernetesSupportedVersionDao; -import com.cloud.network.IpAddress; -import com.cloud.network.IpAddressManager; -import com.cloud.network.Network; -import com.cloud.network.Network.Service; -import com.cloud.network.NetworkModel; -import com.cloud.network.NetworkService; -import com.cloud.network.PhysicalNetwork; -import com.cloud.network.addr.PublicIp; -import com.cloud.network.dao.FirewallRulesDao; -import com.cloud.network.dao.IPAddressDao; -import com.cloud.network.dao.NetworkDao; -import com.cloud.network.dao.NetworkVO; -import com.cloud.network.dao.PhysicalNetworkDao; -import com.cloud.network.firewall.FirewallService; -import com.cloud.network.lb.LoadBalancingRulesService; -import com.cloud.network.rules.FirewallRule; -import com.cloud.network.rules.FirewallRuleVO; -import com.cloud.network.rules.LoadBalancer; -import com.cloud.network.rules.PortForwardingRuleVO; -import com.cloud.network.rules.RulesService; -import com.cloud.network.rules.dao.PortForwardingRulesDao; -import com.cloud.offering.NetworkOffering; -import com.cloud.offering.ServiceOffering; -import com.cloud.offerings.NetworkOfferingVO; -import com.cloud.offerings.dao.NetworkOfferingDao; -import com.cloud.offerings.dao.NetworkOfferingServiceMapDao; -import com.cloud.org.Grouping; -import com.cloud.resource.ResourceManager; -import com.cloud.service.ServiceOfferingVO; -import com.cloud.service.dao.ServiceOfferingDao; -import com.cloud.storage.Storage; -import com.cloud.storage.VMTemplateVO; -import com.cloud.storage.VMTemplateZoneVO; -import com.cloud.storage.dao.VMTemplateDao; -import com.cloud.storage.dao.VMTemplateZoneDao; -import com.cloud.template.TemplateApiService; -import com.cloud.template.VirtualMachineTemplate; -import com.cloud.user.Account; -import com.cloud.user.AccountManager; -import com.cloud.user.AccountService; -import com.cloud.user.SSHKeyPairVO; -import com.cloud.user.User; -import com.cloud.user.dao.AccountDao; -import com.cloud.user.dao.SSHKeyPairDao; -import com.cloud.uservm.UserVm; -import com.cloud.utils.Pair; -import com.cloud.utils.StringUtils; -import com.cloud.utils.component.ComponentContext; -import com.cloud.utils.component.ManagerBase; -import com.cloud.utils.concurrency.NamedThreadFactory; -import com.cloud.utils.db.Filter; -import com.cloud.utils.db.GlobalLock; -import com.cloud.utils.db.SearchCriteria; -import com.cloud.utils.db.Transaction; -import com.cloud.utils.db.TransactionCallback; -import com.cloud.utils.db.TransactionCallbackNoReturn; -import com.cloud.utils.db.TransactionCallbackWithException; -import com.cloud.utils.db.TransactionStatus; -import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.utils.exception.ExecutionException; -import com.cloud.utils.fsm.NoTransitionException; -import com.cloud.utils.fsm.StateMachine2; -import com.cloud.utils.net.Ip; -import com.cloud.utils.net.NetUtils; -import com.cloud.utils.ssh.SshHelper; -import com.cloud.vm.Nic; -import com.cloud.vm.ReservationContext; -import com.cloud.vm.ReservationContextImpl; -import com.cloud.vm.UserVmManager; -import com.cloud.vm.UserVmService; -import com.cloud.vm.UserVmVO; -import com.cloud.vm.VMInstanceVO; -import com.cloud.vm.VirtualMachine; -import com.cloud.vm.dao.UserVmDao; -import com.cloud.vm.dao.VMInstanceDao; -import com.google.common.base.Strings; - -public class KubernetesClusterManagerImpl extends ManagerBase implements KubernetesClusterService { - - private static final Logger LOGGER = Logger.getLogger(KubernetesClusterManagerImpl.class); - - protected StateMachine2 _stateMachine = KubernetesCluster.State.getStateMachine(); - - ScheduledExecutorService _gcExecutor; - ScheduledExecutorService _stateScanner; - - @Inject - protected KubernetesClusterDao kubernetesClusterDao; - @Inject - protected KubernetesClusterVmMapDao kubernetesClusterVmMapDao; - @Inject - protected KubernetesClusterDetailsDao kubernetesClusterDetailsDao; - @Inject - protected KubernetesSupportedVersionDao kubernetesSupportedVersionDao; - @Inject - protected CAManager caManager; - @Inject - protected SSHKeyPairDao sshKeyPairDao; - @Inject - protected DataCenterDao dataCenterDao; - @Inject - protected ClusterDao clusterDao; - @Inject - protected ClusterDetailsDao clusterDetailsDao; - @Inject - protected ServiceOfferingDao serviceOfferingDao; - @Inject - protected VMTemplateDao templateDao; - @Inject - protected TemplateApiService templateService; - @Inject - protected VMTemplateZoneDao templateZoneDao; - @Inject - protected TemplateJoinDao templateJoinDao; - @Inject - protected AccountService accountService; - @Inject - protected AccountDao accountDao; - @Inject - protected AccountManager accountManager; - @Inject - protected VMInstanceDao vmInstanceDao; - @Inject - protected UserVmDao userVmDao; - @Inject - protected UserVmService userVmService; - @Inject - protected UserVmManager userVmManager; - @Inject - protected ConfigurationDao globalConfigDao; - @Inject - protected NetworkOfferingDao networkOfferingDao; - @Inject - protected NetworkOfferingJoinDao networkOfferingJoinDao; - @Inject - protected NetworkService networkService; - @Inject - protected NetworkModel networkModel; - @Inject - protected PhysicalNetworkDao physicalNetworkDao; - @Inject - protected NetworkOrchestrationService networkMgr; - @Inject - protected NetworkDao networkDao; - @Inject - protected IPAddressDao ipAddressDao; - @Inject - protected PortForwardingRulesDao portForwardingRulesDao; - @Inject - protected FirewallService firewallService; - @Inject - protected RulesService rulesService; - @Inject - protected NetworkOfferingServiceMapDao networkOfferingServiceMapDao; - @Inject - protected CapacityManager capacityManager; - @Inject - protected ResourceManager resourceManager; - @Inject - protected FirewallRulesDao firewallRulesDao; - @Inject - protected IpAddressManager ipAddressManager; - @Inject - protected LoadBalancingRulesService lbService; - @Inject - protected VlanDao vlanDao; - - private static final String CLUSTER_NODE_VM_USER = "core"; - private static final int CLUSTER_API_PORT = 6443; - private static final int CLUSTER_NODES_DEFAULT_START_SSH_PORT = 2222; - - private static String getStackTrace(final Throwable throwable) { - final StringWriter sw = new StringWriter(); - final PrintWriter pw = new PrintWriter(sw, true); - throwable.printStackTrace(pw); - return sw.getBuffer().toString(); - } - - private String readResourceFile(String resource) throws IOException { - return IOUtils.toString(Objects.requireNonNull(Thread.currentThread().getContextClassLoader().getResourceAsStream(resource)), Charset.defaultCharset().name()); - } - - private void logMessage(final Level logLevel, final String message, final Exception e) { - if (logLevel == Level.DEBUG) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug(message); - } - } else if (logLevel == Level.ERROR) { - LOGGER.error(message); - } if (logLevel == Level.WARN) { - if (e != null) { - LOGGER.warn(message, e); - } else { - LOGGER.warn(message); - } - } else { - if (e != null) { - LOGGER.error(message, e); - } else { - LOGGER.error(message); - } - } - } - - private void logTransitStateAndThrow(final Level logLevel, final String message, final Long kubernetesClusterId, final KubernetesCluster.Event event, final Exception e) throws CloudRuntimeException { - logMessage(logLevel, message, e); - if (kubernetesClusterId != null && event != null) { - stateTransitTo(kubernetesClusterId, event); - } - if (e == null) { - throw new CloudRuntimeException(message); - } - throw new CloudRuntimeException(message, e); - } - - private void logTransitStateDetachIsoAndThrow(final Level logLevel, final String message, final KubernetesCluster kubernetesCluster, - final List clusterVMIds, final KubernetesCluster.Event event, final Exception e) throws CloudRuntimeException { - logMessage(logLevel, message, e); - stateTransitTo(kubernetesCluster.getId(), event); - detachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); - if (e == null) { - throw new CloudRuntimeException(message); - } - throw new CloudRuntimeException(message, e); - } - - private void logTransitStateAndThrow(final Level logLevel, final String message, final Long kubernetesClusterId, final KubernetesCluster.Event event) throws CloudRuntimeException { - logTransitStateAndThrow(logLevel, message, kubernetesClusterId, event, null); - } - - private void logAndThrow(final Level logLevel, final String message) throws CloudRuntimeException { - logTransitStateAndThrow(logLevel, message, null, null, null); - } - - private void logAndThrow(final Level logLevel, final String message, final Exception ex) throws CloudRuntimeException { - logTransitStateAndThrow(logLevel, message, null, null, ex); - } - - private boolean isKubernetesServiceTemplateConfigured(DataCenter zone) { - // Check Kubernetes VM template for zone - String templateName = KubernetesClusterTemplateName.value(); - if (templateName == null || templateName.isEmpty()) { - LOGGER.warn(String.format("Global setting %s is empty. Template name need to be specified for Kubernetes service to function", KubernetesClusterTemplateName.key())); - return false; - } - final VMTemplateVO template = templateDao.findByTemplateName(templateName); - if (template == null) { - LOGGER.warn(String.format("Unable to find the template %s to be used for provisioning Kubernetes cluster", templateName)); - return false; - } - return true; - } - - private boolean isKubernetesServiceNetworkOfferingConfigured(DataCenter zone) { - // Check network offering - String networkOfferingName = KubernetesClusterNetworkOffering.value(); - if (networkOfferingName == null || networkOfferingName.isEmpty()) { - LOGGER.warn(String.format("Global setting %s is empty. Admin has not yet specified the network offering to be used for provisioning isolated network for the cluster", KubernetesClusterNetworkOffering.key())); - return false; - } - NetworkOfferingVO networkOffering = networkOfferingDao.findByUniqueName(networkOfferingName); - if (networkOffering == null) { - LOGGER.warn(String.format("Unable to find the network offering %s to be used for provisioning Kubernetes cluster", networkOfferingName)); - return false; - } - if (networkOffering.getState() == NetworkOffering.State.Disabled) { - LOGGER.warn(String.format("Network offering ID: %s is not enabled", networkOffering.getUuid())); - return false; - } - List services = networkOfferingServiceMapDao.listServicesForNetworkOffering(networkOffering.getId()); - if (services == null || services.isEmpty() || !services.contains("SourceNat")) { - LOGGER.warn(String.format("Network offering ID: %s does not have necessary services to provision Kubernetes cluster", networkOffering.getUuid())); - return false; - } - if (!networkOffering.isEgressDefaultPolicy()) { - LOGGER.warn(String.format("Network offering ID: %s has egress default policy turned off should be on to provision Kubernetes cluster", networkOffering.getUuid())); - return false; - } - boolean offeringAvailableForZone = false; - List networkOfferingJoinVOs = networkOfferingJoinDao.findByZoneId(zone.getId(), true); - for (NetworkOfferingJoinVO networkOfferingJoinVO : networkOfferingJoinVOs) { - if (networkOffering.getId() == networkOfferingJoinVO.getId()) { - offeringAvailableForZone = true; - break; - } - } - if (!offeringAvailableForZone) { - LOGGER.warn(String.format("Network offering ID: %s is not available for zone ID: %s", networkOffering.getUuid(), zone.getUuid())); - return false; - } - long physicalNetworkId = networkModel.findPhysicalNetworkId(zone.getId(), networkOffering.getTags(), networkOffering.getTrafficType()); - PhysicalNetwork physicalNetwork = physicalNetworkDao.findById(physicalNetworkId); - if (physicalNetwork == null) { - LOGGER.warn(String.format("Unable to find physical network with tag: %s", networkOffering.getTags())); - return false; - } - return true; - } - - private boolean isKubernetesServiceConfigured(DataCenter zone) { - if (!isKubernetesServiceTemplateConfigured(zone)) { - return false; - } - if (!isKubernetesServiceNetworkOfferingConfigured(zone)) { - return false; - } - return true; - } - - private File getManagementServerSshPublicKeyFile() { - boolean devel = Boolean.parseBoolean(globalConfigDao.getValue("developer")); - String keyFile = String.format("%s/.ssh/id_rsa", System.getProperty("user.home")); - if (devel) { - keyFile += ".cloud"; - } - return new File(keyFile); - } - - private String generateClusterToken(KubernetesCluster kubernetesCluster) { - String token = kubernetesCluster.getUuid(); - token = token.replaceAll("-", ""); - token = token.substring(0, 22); - token = token.substring(0, 6) + "." + token.substring(6); - return token; - } - - private String generateClusterHACertificateKey(KubernetesCluster kubernetesCluster) { - String uuid = kubernetesCluster.getUuid(); - StringBuilder token = new StringBuilder(uuid.replaceAll("-", "")); - while (token.length() < 64) { - token.append(token); - } - return token.toString().substring(0, 64); - } - - private KubernetesClusterVmMapVO addKubernetesClusterVm(final long kubernetesClusterId, final long vmId) { - return Transaction.execute(new TransactionCallback() { - @Override - public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { - KubernetesClusterVmMapVO newClusterVmMap = new KubernetesClusterVmMapVO(kubernetesClusterId, vmId); - kubernetesClusterVmMapDao.persist(newClusterVmMap); - return newClusterVmMap; - } - }); - } - - private boolean isKubernetesClusterServerRunning(KubernetesCluster kubernetesCluster, String ipAddress, int retries, long waitDuration) { - int retryCounter = 0; - boolean k8sApiServerSetup = false; - while (retryCounter < retries) { - try { - String versionOutput = IOUtils.toString(new URL(String.format("https://%s:%d/version", ipAddress, CLUSTER_API_PORT)), StringUtils.getPreferredCharset()); - if (!Strings.isNullOrEmpty(versionOutput)) { - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Kubernetes cluster ID: %s API has been successfully provisioned, %s", kubernetesCluster.getUuid(), versionOutput)); - } - k8sApiServerSetup = true; - break; - } - } catch (Exception e) { - LOGGER.warn(String.format("API endpoint for Kubernetes cluster ID: %s not available. Attempt: %d/%d", kubernetesCluster.getUuid(), retryCounter+1, retries), e); - } - try { - Thread.sleep(waitDuration); - } catch (InterruptedException ie) { - LOGGER.error(String.format("Error while waiting for Kubernetes cluster ID: %s API endpoint to be available", kubernetesCluster.getUuid()), ie); - } - retryCounter++; - } - return k8sApiServerSetup; - } - - private String getKubernetesClusterConfig(KubernetesCluster kubernetesCluster, String ipAddress, int port, int retries) { - int retryCounter = 0; - String kubeConfig = ""; - while (retryCounter < retries) { - try { - Pair result = SshHelper.sshExecute(ipAddress, port, CLUSTER_NODE_VM_USER, - getManagementServerSshPublicKeyFile(), null, "sudo cat /etc/kubernetes/admin.conf", - 10000, 10000, 10000); - - if (result.first() && !Strings.isNullOrEmpty(result.second())) { - kubeConfig = result.second(); - break; - } else { - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Failed to retrieve kube-config file for Kubernetes cluster ID: %s. Output: %s", kubernetesCluster.getUuid(), result.second())); - } - } - } catch (Exception e) { - LOGGER.warn(String.format("Failed to retrieve kube-config file for Kubernetes cluster ID: %s. Attempt: %d/%d", kubernetesCluster.getUuid(), retryCounter+1, retries), e); - } - retryCounter++; - } - return kubeConfig; - } - - private boolean isKubernetesClusterAddOnServiceRunning(KubernetesCluster kubernetesCluster, final String ipAddress, final int port, final String namespace, String serviceName) { - try { - String cmd = "sudo kubectl get pods --all-namespaces"; - if (!Strings.isNullOrEmpty(namespace)) { - cmd = String.format("sudo kubectl get pods --namespace=%s", namespace); - } - Pair result = SshHelper.sshExecute(ipAddress, port, CLUSTER_NODE_VM_USER, - getManagementServerSshPublicKeyFile(), null, cmd, - 10000, 10000, 10000); - if (result.first() && !Strings.isNullOrEmpty(result.second())) { - String[] lines = result.second().split("\n"); - for (String line : - lines) { - if (line.contains(serviceName) && line.contains("Running")) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Service : %s in namespace: %s for the Kubernetes cluster ID: %s is running", serviceName, namespace, kubernetesCluster.getUuid())); - } - return true; - } - } - } - } catch (Exception e) { - LOGGER.warn(String.format("Unable to retrieve service: %s running status in namespace %s for Kubernetes cluster ID: %s", serviceName, namespace, kubernetesCluster.getUuid()), e); - } - return false; - } - - private boolean isKubernetesClusterDashboardServiceRunning(KubernetesCluster kubernetesCluster, String ipAddress, int port, int retries, long waitDuration) { - boolean running = false; - int retryCounter = 0; - // Check if dashboard service is up running. - while (retryCounter < retries) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Checking dashboard service for the Kubernetes cluster ID: %s to come up. Attempt: %d/%d", kubernetesCluster.getUuid(), retryCounter + 1, retries)); - } - if (isKubernetesClusterAddOnServiceRunning(kubernetesCluster, ipAddress, port, "kubernetes-dashboard", "kubernetes-dashboard")) { - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Dashboard service for the Kubernetes cluster ID: %s is in running state", kubernetesCluster.getUuid())); - } - running = true; - break; - } - try { - Thread.sleep(waitDuration); - } catch (InterruptedException ex) { - LOGGER.error(String.format("Error while waiting for Kubernetes cluster: %s API dashboard service to be available", kubernetesCluster.getUuid()), ex); - } - retryCounter++; - } - return running; - } - - private UserVm fetchMasterVmIfMissing(final KubernetesCluster kubernetesCluster, final int port, final UserVm masterVm) { - if (masterVm != null) { - return masterVm; - } - List clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); - if (CollectionUtils.isEmpty(clusterVMs)) { - LOGGER.warn(String.format("Unable to retrieve VMs for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); - return null; - } - List vmIds = new ArrayList<>(); - for (KubernetesClusterVmMapVO vmMap : clusterVMs) { - vmIds.add(vmMap.getVmId()); - } - Collections.sort(vmIds); - return userVmDao.findById(vmIds.get(0)); - } - - private Pair getKubernetesClusterServerIpSshPort(KubernetesCluster kubernetesCluster, UserVm masterVm) { - int port = CLUSTER_NODES_DEFAULT_START_SSH_PORT; - KubernetesClusterDetailsVO detail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS); - if (detail != null && !Strings.isNullOrEmpty(detail.getValue())) { - return new Pair<>(detail.getValue(), port); - } - Network network = networkDao.findById(kubernetesCluster.getNetworkId()); - if (network == null) { - LOGGER.warn(String.format("Network for Kubernetes cluster ID: %s cannot be found", kubernetesCluster.getUuid())); - return new Pair<>(null, port); - } - if (Network.GuestType.Isolated.equals(network.getGuestType())) { - List addresses = networkModel.listPublicIpsAssignedToGuestNtwk(network.getId(), true); - if (CollectionUtils.isEmpty(addresses)) { - LOGGER.warn(String.format("No public IP addresses found for network ID: %s, Kubernetes cluster ID: %s", network.getUuid(), kubernetesCluster.getUuid())); - return new Pair<>(null, port); - } - for (IpAddress address : addresses) { - if (address.isSourceNat()) { - return new Pair<>(address.getAddress().addr(), port); - } - } - LOGGER.warn(String.format("No source NAT IP addresses found for network ID: %s, Kubernetes cluster ID: %s", network.getUuid(), kubernetesCluster.getUuid())); - return new Pair<>(null, port); - } else if (Network.GuestType.Shared.equals(network.getGuestType())) { - port = 22; - masterVm = fetchMasterVmIfMissing(kubernetesCluster, port, masterVm); - if (masterVm == null) { - LOGGER.warn(String.format("Unable to retrieve master VM for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); - return new Pair<>(null, port); - } - return new Pair<>(masterVm.getPrivateIpAddress(), port); - } - LOGGER.warn(String.format("Unable to retrieve server IP address for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); - return new Pair<>(null, port); - } - - private Pair getKubernetesClusterServerIpSshPort(KubernetesCluster kubernetesCluster) { - return getKubernetesClusterServerIpSshPort(kubernetesCluster, null); - } - - private int getKubernetesClusterReadyNodesCount(KubernetesCluster kubernetesCluster, String ipAddress, int port) throws Exception { - Pair result = SshHelper.sshExecute(ipAddress, port, - CLUSTER_NODE_VM_USER, getManagementServerSshPublicKeyFile(), null, - "sudo kubectl get nodes | awk '{if ($2 == \"Ready\") print $1}' | wc -l", - 10000, 10000, 20000); - if (result.first()) { - return Integer.parseInt(result.second().trim().replace("\"", "")); - } else { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Failed to retrieve ready nodes for Kubernetes cluster ID: %s. Output: %s", kubernetesCluster.getUuid(), result.second())); - } - } - return 0; - } - - private boolean isKubernetesClusterNodeReady(KubernetesCluster kubernetesCluster, String ipAddress, int port, String nodeName) throws Exception { - Pair result = SshHelper.sshExecute(ipAddress, port, - CLUSTER_NODE_VM_USER, getManagementServerSshPublicKeyFile(), null, - String.format("sudo kubectl get nodes | awk '{if ($1 == \"%s\" && $2 == \"Ready\") print $1}'", nodeName.toLowerCase()), - 10000, 10000, 20000); - if (result.first() && nodeName.equals(result.second().trim())) { - return true; - } - if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Failed to retrieve status for node: %s in Kubernetes cluster ID: %s. Output: %s", nodeName, kubernetesCluster.getUuid(), result.second())); - } - return false; - } - - private boolean isKubernetesClusterNodeReady(KubernetesCluster kubernetesCluster, String ipAddress, int port, String nodeName, int retries, int waitDuration) { - int retryCounter = 0; - while (retryCounter < retries) { - boolean ready = false; - try { - ready = isKubernetesClusterNodeReady(kubernetesCluster, ipAddress, port, nodeName); - } catch (Exception e) { - LOGGER.warn(String.format("Failed to retrieve state of node: %s in Kubernetes cluster ID: %s", nodeName, kubernetesCluster.getUuid()), e); - } - if (ready) { - return true; - } - try { - Thread.sleep(waitDuration); - } catch (InterruptedException ie) { - LOGGER.error(String.format("Error while waiting for Kubernetes cluster ID: %s node: %s to become ready", kubernetesCluster.getUuid(), nodeName), ie); - } - retryCounter++; - } - return false; - } - - private int getKubernetesClusterReadyNodesCount(KubernetesCluster kubernetesCluster) throws Exception { - Pair ipSshPort = getKubernetesClusterServerIpSshPort(kubernetesCluster); - String ipAddress = ipSshPort.first(); - int sshPort = ipSshPort.second(); - if (Strings.isNullOrEmpty(ipAddress)) { - String msg = String.format("No public IP found for Kubernetes cluster ID: %s" , kubernetesCluster.getUuid()); - LOGGER.warn(msg); - throw new ManagementServerException(msg); - } - return getKubernetesClusterReadyNodesCount(kubernetesCluster, ipAddress, sshPort); - } - - private boolean validateKubernetesClusterReadyNodesCount(KubernetesCluster kubernetesCluster, String ipAddress, int port, int retries, long waitDuration) { - int retryCounter = 0; - while (retryCounter < retries) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Checking ready nodes for the Kubernetes cluster ID: %s with total %d provisioned nodes. Attempt: %d/%d", kubernetesCluster.getUuid(), kubernetesCluster.getTotalNodeCount(), retryCounter + 1, retries)); - } - try { - int nodesCount = getKubernetesClusterReadyNodesCount(kubernetesCluster, ipAddress, port); - if (nodesCount == kubernetesCluster.getTotalNodeCount()) { - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Kubernetes cluster ID: %s has %d ready nodes now", kubernetesCluster.getUuid(), kubernetesCluster.getTotalNodeCount())); - } - return true; - } else { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Kubernetes cluster ID: %s has total %d provisioned nodes while %d ready now", kubernetesCluster.getUuid(), kubernetesCluster.getTotalNodeCount(), nodesCount)); - } - } - } catch (Exception e) { - LOGGER.warn(String.format("Failed to retrieve ready node count for Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); - } - try { - Thread.sleep(waitDuration); - } catch (InterruptedException ex) { - LOGGER.warn(String.format("Error while waiting during Kubernetes cluster ID: %s ready node check. %d/%d", kubernetesCluster.getUuid(), retryCounter+1, retries), ex); - } - retryCounter++; - } - return false; - } - - private boolean removeKubernetesClusterNode(KubernetesCluster kubernetesCluster, String ipAddress, int port, UserVm userVm, int retries, int waitDuration) { - File pkFile = getManagementServerSshPublicKeyFile(); - int retryCounter = 0; - String hostName = userVm.getHostName(); - if (!Strings.isNullOrEmpty(hostName)) { - hostName = hostName.toLowerCase(); - } - while (retryCounter < retries) { - retryCounter++; - try { - Pair result = SshHelper.sshExecute(ipAddress, port, CLUSTER_NODE_VM_USER, - pkFile, null, String.format("sudo kubectl drain %s --ignore-daemonsets --delete-local-data", hostName), - 10000, 10000, 60000); - if (!result.first()) { - LOGGER.warn(String.format("Draining node: %s on VM ID: %s in Kubernetes cluster ID: %s unsuccessful", hostName, userVm.getUuid(), kubernetesCluster.getUuid())); - } else { - result = SshHelper.sshExecute(ipAddress, port, CLUSTER_NODE_VM_USER, - pkFile, null, String.format("sudo kubectl delete node %s", hostName), - 10000, 10000, 30000); - if (result.first()) { - return true; - } else { - LOGGER.warn(String.format("Deleting node: %s on VM ID: %s in Kubernetes cluster ID: %s unsuccessful", hostName, userVm.getUuid(), kubernetesCluster.getUuid())); - } - } - break; - } catch (Exception e) { - String msg = String.format("Failed to remove Kubernetes cluster ID: %s node: %s on VM ID: %s", kubernetesCluster.getUuid(), hostName, userVm.getUuid()); - LOGGER.warn(msg, e); - } - try { - Thread.sleep(waitDuration); - } catch (InterruptedException ie) { - LOGGER.error(String.format("Error while waiting for Kubernetes cluster ID: %s node: %s on VM ID: %s removal", kubernetesCluster.getUuid(), hostName, userVm.getUuid()), ie); - } - retryCounter++; - } - return false; - } - - private boolean uncordonKubernetesClusterNode(KubernetesCluster kubernetesCluster, String ipAddress, int port, UserVm userVm, int retries, int waitDuration) { - int retryCounter = 0; - String hostName = userVm.getHostName(); - if (!Strings.isNullOrEmpty(hostName)) { - hostName = hostName.toLowerCase(); - } - while (retryCounter < retries) { - Pair result = null; - try { - result = SshHelper.sshExecute(ipAddress, port, CLUSTER_NODE_VM_USER, getManagementServerSshPublicKeyFile(), null, - String.format("sudo kubectl uncordon %s", hostName), - 10000, 10000, 30000); - if (result.first()) { - return true; - } - } catch (Exception e) { - LOGGER.warn(String.format("Failed to uncordon node: %s on VM ID: %s in Kubernetes cluster ID: %s", hostName, userVm.getUuid(), kubernetesCluster.getUuid()), e); - } - try { - Thread.sleep(waitDuration); - } catch (InterruptedException ie) { - LOGGER.warn(String.format("Error while waiting for uncordon Kubernetes cluster ID: %s node: %s on VM ID: %s", kubernetesCluster.getUuid(), hostName, userVm.getUuid()), ie); - } - retryCounter++; - } - return false; - } - - private Network startKubernetesCLusterNetwork(final KubernetesCluster kubernetesCluster, final DeployDestination destination, final Account account) throws ManagementServerException { - final ReservationContext context = new ReservationContextImpl(null, null, null, account); - Network network = networkDao.findById(kubernetesCluster.getNetworkId()); - if (network == null) { - String msg = String.format("Network for Kubernetes cluster ID: %s not found", kubernetesCluster.getUuid()); - LOGGER.warn(msg); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); - throw new ManagementServerException(msg); - } - try { - networkMgr.startNetwork(network.getId(), destination, context); - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Network ID: %s is started for the Kubernetes cluster ID: %s", network.getUuid(), kubernetesCluster.getUuid())); - } - } catch (ConcurrentOperationException | ResourceUnavailableException |InsufficientCapacityException e) { - String msg = String.format("Failed to start Kubernetes cluster ID: %s as unable to start associated network ID: %s" , kubernetesCluster.getUuid(), network.getUuid()); - LOGGER.error(msg, e); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); - throw new ManagementServerException(msg, e); - } - return network; - } - - private UserVm provisionKubernetesClusterMasterVm(final KubernetesCluster kubernetesCluster, final DeployDestination destination, final Network network, final Account account, final String publicIpAddress) throws ManagementServerException { - UserVm k8sMasterVM = null; - try { - k8sMasterVM = createKubernetesMaster(kubernetesCluster, destination.getPod(), network, account, publicIpAddress); - addKubernetesClusterVm(kubernetesCluster.getId(), k8sMasterVM.getId()); - startKubernetesVM(k8sMasterVM, kubernetesCluster); - k8sMasterVM = userVmDao.findById(k8sMasterVM.getId()); - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Provisioned the master VM ID: %s in to the Kubernetes cluster ID: %s", k8sMasterVM.getUuid(), kubernetesCluster.getUuid())); - } - } catch (ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { - String msg = String.format("Provisioning the master VM failed in the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); - LOGGER.warn(msg, e); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); - throw new ManagementServerException(msg, e); - } - return k8sMasterVM; - } - - private List provisionKubernetesClusterAdditionalMasterVms(final KubernetesCluster kubernetesCluster, final String publicIpAddress) throws ManagementServerException { - List additionalMasters = new ArrayList<>(); - if (kubernetesCluster.getMasterNodeCount() > 1) { - for (int i = 1; i < kubernetesCluster.getMasterNodeCount(); i++) { - UserVm vm = null; - try { - vm = createKubernetesAdditionalMaster(kubernetesCluster, publicIpAddress, i); - addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId()); - startKubernetesVM(vm, kubernetesCluster); - additionalMasters.add(vm); - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Provisioned additional master VM ID: %s in to the Kubernetes cluster ID: %s", vm.getUuid(), kubernetesCluster.getUuid())); - } - } catch (ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { - String msg = String.format("Provisioning additional master VM %d/%d failed in the Kubernetes cluster ID: %s", i+1, kubernetesCluster.getMasterNodeCount(), kubernetesCluster.getUuid()); - LOGGER.warn(msg, e); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); - throw new ManagementServerException(msg, e); - } - } - } - return additionalMasters; - } - - private List provisionKubernetesClusterNodeVms(final KubernetesCluster kubernetesCluster, final String publicIpAddress) throws ManagementServerException { - List nodes = new ArrayList<>(); - for (int i = 1; i <= kubernetesCluster.getNodeCount(); i++) { - UserVm vm = null; - try { - vm = createKubernetesNode(kubernetesCluster, publicIpAddress, i); - addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId()); - startKubernetesVM(vm, kubernetesCluster); - nodes.add(vm); - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Provisioned node master VM ID: %s in to the Kubernetes cluster ID: %s", vm.getUuid(), kubernetesCluster.getUuid())); - } - } catch (ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { - String msg = String.format("Provisioning node VM %d/%d failed in the Kubernetes cluster ID: %s", i, kubernetesCluster.getNodeCount(), kubernetesCluster.getUuid()); - LOGGER.warn(msg, e); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); - throw new ManagementServerException(msg, e); - } - } - return nodes; - } - - private boolean isKubernetesClusterMasterVmRunning(final KubernetesCluster kubernetesCluster, final String ipAddress, final int port, final long timeout) { - boolean masterVmRunning = false; - long startTime = System.currentTimeMillis(); - while (!masterVmRunning && System.currentTimeMillis() - startTime < timeout) { - try (Socket socket = new Socket()) { - socket.connect(new InetSocketAddress(ipAddress, port), 10000); - masterVmRunning = true; - } catch (IOException e) { - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Waiting for Kubernetes cluster ID: %s master node VMs to be accessible", kubernetesCluster.getUuid())); - } - try { - Thread.sleep(10000); - } catch (InterruptedException ex) { - LOGGER.warn(String.format("Error while waiting for Kubernetes cluster ID: %s master node VMs to be accessible", kubernetesCluster.getUuid()), ex); - } - } - } - return masterVmRunning; - } - - // Start cluster after creation (cluster will be started for first time therefore resources will be provisioned as well) - private boolean startKubernetesClusterOnCreate(final long kubernetesClusterId) throws ManagementServerException { - - // Starting a Kubernetes cluster has below workflow - // - start the network - // - provision the master / node VM - // - provision node VM's (as many as cluster size) - // - update the book keeping data of the VM's provisioned for the cluster - // - setup networking (add Firewall and PF rules) - // - wait till Kubernetes API server on master VM to come up - // - wait till addon services (dashboard etc) to come up - // - update API and dashboard URL endpoints in Kubernetes cluster details - - KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); - final DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); - if (zone == null) { - throw new CloudRuntimeException(String.format("Unable to find zone for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); - } - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Starting Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); - } - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.StartRequested); - Account account = accountDao.findById(kubernetesCluster.getAccountId()); - - DeployDestination dest = null; - try { - dest = plan(kubernetesCluster, zone); - } catch (InsufficientCapacityException e) { - String msg = String.format("Provisioning the cluster failed due to insufficient capacity in the Kubernetes cluster: %s", kubernetesCluster.getUuid()); - LOGGER.error(msg, e); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); - throw new ManagementServerException(msg, e); - } - - Network network = startKubernetesCLusterNetwork(kubernetesCluster, dest, account); - - Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(kubernetesCluster); - String publicIpAddress = publicIpSshPort.first(); - if (Strings.isNullOrEmpty(publicIpAddress) && - (Network.GuestType.Isolated.equals(network.getGuestType()) || kubernetesCluster.getMasterNodeCount() > 1)) { // Shared network, single-master cluster won't have an IP yet - String msg = String.format("Failed to start Kubernetes cluster ID: %s as no public IP found for the cluster" , kubernetesCluster.getUuid()); - LOGGER.warn(msg); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); - throw new ManagementServerException(msg); - } - - List clusterVMIds = new ArrayList<>(); - - UserVm k8sMasterVM = provisionKubernetesClusterMasterVm(kubernetesCluster, dest, network, account, publicIpAddress); - clusterVMIds.add(k8sMasterVM.getId()); - - if (Strings.isNullOrEmpty(publicIpAddress)) { - publicIpSshPort = getKubernetesClusterServerIpSshPort(kubernetesCluster, k8sMasterVM); - publicIpAddress = publicIpSshPort.first(); - if (Strings.isNullOrEmpty(publicIpAddress)) { - String msg = String.format("Failed to start Kubernetes cluster ID: %s as no public IP found for the cluster", kubernetesCluster.getUuid()); - LOGGER.warn(msg); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); - throw new ManagementServerException(msg); - } - } - - List additionalMasterVMs = provisionKubernetesClusterAdditionalMasterVms(kubernetesCluster, publicIpAddress); - for (UserVm vm : additionalMasterVMs){ - clusterVMIds.add(vm.getId()); - } - - List nodeVMs = provisionKubernetesClusterNodeVms(kubernetesCluster, publicIpAddress); - for (UserVm vm : nodeVMs){ - clusterVMIds.add(vm.getId()); - } - - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Kubernetes cluster ID: %s VMs successfully provisioned", kubernetesCluster.getUuid())); - } - try { - setupKubernetesClusterNetworkRules(kubernetesCluster, network, account, clusterVMIds); - } catch (ManagementServerException e) { - logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster ID: %s, unable to setup network rules", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e); - } - - attachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); - - if (!isKubernetesClusterMasterVmRunning(kubernetesCluster, publicIpAddress, publicIpSshPort.second(), 10 * 60 * 1000)) { - String msg = String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to access master node VMs of the cluster", kubernetesCluster.getUuid()); - if (kubernetesCluster.getMasterNodeCount() > 1 && Network.GuestType.Shared.equals(network.getGuestType())) { - msg = String.format("%s. Make sure external load-balancer has port forwarding rules for SSH access on ports %d-%d and API access on port %d", - msg, - CLUSTER_NODES_DEFAULT_START_SSH_PORT, - CLUSTER_NODES_DEFAULT_START_SSH_PORT + kubernetesCluster.getTotalNodeCount() - 1, - CLUSTER_API_PORT); - } - logTransitStateDetachIsoAndThrow(Level.ERROR, msg, kubernetesCluster, clusterVMIds, KubernetesCluster.Event.CreateFailed, null); - } - - boolean k8sApiServerSetup = isKubernetesClusterServerRunning(kubernetesCluster, publicIpAddress, 20, 30000); - if (!k8sApiServerSetup) { - logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to provision API endpoint for the cluster", kubernetesCluster.getUuid()), kubernetesCluster, clusterVMIds, KubernetesCluster.Event.CreateFailed, null); - } - kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); - kubernetesCluster.setEndpoint(String.format("https://%s:%d/", publicIpAddress, CLUSTER_API_PORT)); - kubernetesClusterDao.update(kubernetesCluster.getId(), kubernetesCluster); - - int sshPort = publicIpSshPort.second(); - boolean readyNodesCountValid = validateKubernetesClusterReadyNodesCount(kubernetesCluster, publicIpAddress, sshPort, 30, 30000); - - // Detach binaries ISO from new VMs - detachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); - - // Throw exception if nodes count for k8s cluster timed out - if (!readyNodesCountValid) { - logTransitStateAndThrow(Level.WARN, String.format("Failed to setup Kubernetes cluster ID: %s as it does not have desired number of nodes in ready state", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); - } - - boolean k8sKubeConfigCopied = false; - String kubeConfig = getKubernetesClusterConfig(kubernetesCluster, publicIpAddress, sshPort, 5); - if (!Strings.isNullOrEmpty(kubeConfig)) { - k8sKubeConfigCopied = true; - } - if (!k8sKubeConfigCopied) { - logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to retrieve kube-config for the cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } - kubeConfig = kubeConfig.replace(String.format("server: https://%s:%d", k8sMasterVM.getPrivateIpAddress(), CLUSTER_API_PORT), - String.format("server: https://%s:%d", publicIpAddress, CLUSTER_API_PORT)); - kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "kubeConfigData", Base64.encodeBase64String(kubeConfig.getBytes(StringUtils.getPreferredCharset())), false); - - boolean dashboardServiceRunning = isKubernetesClusterDashboardServiceRunning(kubernetesCluster, publicIpAddress, sshPort, 10, 20000); - if (!dashboardServiceRunning) { - logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to get Dashboard service running for the cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(),KubernetesCluster.Event.OperationFailed); - } - kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "dashboardServiceRunning", String.valueOf(dashboardServiceRunning), false); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); - return true; - } - - private boolean startStoppedKubernetesCluster(long kubernetesClusterId) throws ManagementServerException { - final KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); - if (kubernetesCluster == null) { - throw new ManagementServerException("Invalid Kubernetes cluster ID"); - } - if (kubernetesCluster.getRemoved() != null) { - throw new ManagementServerException(String.format("Kubernetes cluster ID: %s is already deleted", kubernetesCluster.getUuid())); - } - if (kubernetesCluster.getState().equals(KubernetesCluster.State.Running)) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Kubernetes cluster ID: %s is in running state", kubernetesCluster.getUuid())); - } - return true; - } - if (kubernetesCluster.getState().equals(KubernetesCluster.State.Starting)) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Kubernetes cluster ID: %s is already in starting state", kubernetesCluster.getUuid())); - } - return true; - } - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Starting Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); - } - - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.StartRequested); - - for (final KubernetesClusterVmMapVO vmMapVO : kubernetesClusterVmMapDao.listByClusterId(kubernetesClusterId)) { - final UserVmVO vm = userVmDao.findById(vmMapVO.getVmId()); - try { - if (vm == null) { - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ManagementServerException("Failed to start all VMs in Kubernetes cluster ID: " + kubernetesClusterId); - } - startKubernetesVM(vm, kubernetesCluster); - } catch (CloudRuntimeException ex) { - LOGGER.warn(String.format("Failed to start VM in Kubernetes cluster ID: %s due to ", kubernetesCluster.getUuid()) + ex); - // dont bail out here. proceed further to stop the reset of the VM's - } - } - - for (final KubernetesClusterVmMapVO vmMapVO : kubernetesClusterVmMapDao.listByClusterId(kubernetesClusterId)) { - final UserVmVO vm = userVmDao.findById(vmMapVO.getVmId()); - if (vm == null || !vm.getState().equals(VirtualMachine.State.Running)) { - logTransitStateAndThrow(Level.ERROR, String.format("Failed to start all VMs in Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } - } - - InetAddress address = null; - try { - address = InetAddress.getByName(new URL(kubernetesCluster.getEndpoint()).getHost()); - } catch (MalformedURLException | UnknownHostException ex) { - logTransitStateAndThrow(Level.ERROR, String.format("Kubernetes cluster ID: %s has invalid API endpoint. Can not verify if cluster is in ready state", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } - - Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(kubernetesCluster); - String publicIpAddress = publicIpSshPort.first(); - if (Strings.isNullOrEmpty(publicIpAddress)) { - logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s as no public IP found for the cluster" , kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } - - boolean k8sApiServerSetup = isKubernetesClusterServerRunning(kubernetesCluster, publicIpAddress, 10, 30000); - if (!k8sApiServerSetup) { - logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s in usable state", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } - - int sshPort = publicIpSshPort.second(); - KubernetesClusterDetailsVO kubeConfigDetail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "kubeConfigData"); - if (kubeConfigDetail == null || Strings.isNullOrEmpty(kubeConfigDetail.getValue())) { - boolean k8sKubeConfigCopied = false; - String kubeConfig = getKubernetesClusterConfig(kubernetesCluster, publicIpAddress, sshPort, 5); - if (!Strings.isNullOrEmpty(kubeConfig)) { - k8sKubeConfigCopied = true; - } - if (!k8sKubeConfigCopied) { - logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s in usable state as unable to retrieve kube-config for the cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } - kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "kubeConfigData", Base64.encodeBase64String(kubeConfig.getBytes(StringUtils.getPreferredCharset())), false); - } - KubernetesClusterDetailsVO dashboardServiceRunningDetail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "dashboardServiceRunning"); - if (kubeConfigDetail == null || !Boolean.parseBoolean(dashboardServiceRunningDetail.getValue())) { - boolean dashboardServiceRunning = isKubernetesClusterDashboardServiceRunning(kubernetesCluster, publicIpAddress, sshPort, 10, 20000); - if (!dashboardServiceRunning) { - logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s in usable state as unable to get Dashboard service running for the cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } - kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "dashboardServiceRunning", String.valueOf(dashboardServiceRunning), false); - } - - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Kubernetes cluster ID: %s successfully started", kubernetesCluster.getUuid())); - } - return true; - } - - private IpAddress getSourceNatIp(Network network) { - List addresses = networkModel.listPublicIpsAssignedToGuestNtwk(network.getId(), true); - if (CollectionUtils.isEmpty(addresses)) { - return null; - } - for (IpAddress address : addresses) { - if (address.isSourceNat()) { - return address; - } - } - return null; - } - - private FirewallRule removeSshFirewallRule(IpAddress publicIp) { - FirewallRule rule = null; - List firewallRules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(publicIp.getId(), FirewallRule.Purpose.Firewall); - for (FirewallRuleVO firewallRule : firewallRules) { - if (firewallRule.getSourcePortStart() == CLUSTER_NODES_DEFAULT_START_SSH_PORT) { - rule = firewallRule; - firewallService.revokeIngressFwRule(firewallRule.getId(), true); - break; - } - } - return rule; - } - - private void provisionFirewallRules(final IpAddress publicIp, final Account account, int startPort, int endPort) throws NoSuchFieldException, - IllegalAccessException, ResourceUnavailableException, NetworkRuleConflictException { - List sourceCidrList = new ArrayList(); - sourceCidrList.add("0.0.0.0/0"); - - CreateFirewallRuleCmd rule = new CreateFirewallRuleCmd(); - rule = ComponentContext.inject(rule); - - Field addressField = rule.getClass().getDeclaredField("ipAddressId"); - addressField.setAccessible(true); - addressField.set(rule, publicIp.getId()); - - Field protocolField = rule.getClass().getDeclaredField("protocol"); - protocolField.setAccessible(true); - protocolField.set(rule, "TCP"); - - Field startPortField = rule.getClass().getDeclaredField("publicStartPort"); - startPortField.setAccessible(true); - startPortField.set(rule, startPort); - - Field endPortField = rule.getClass().getDeclaredField("publicEndPort"); - endPortField.setAccessible(true); - endPortField.set(rule, endPort); - - Field cidrField = rule.getClass().getDeclaredField("cidrlist"); - cidrField.setAccessible(true); - cidrField.set(rule, sourceCidrList); - - firewallService.createIngressFirewallRule(rule); - firewallService.applyIngressFwRules(publicIp.getId(), account); - } - - private void removePortForwardingRules(IpAddress publicIp, Network network, Account account, List removedVMIds) throws ResourceUnavailableException { - if (!CollectionUtils.isEmpty(removedVMIds)) { - for (Long vmId : removedVMIds) { - List pfRules = portForwardingRulesDao.listByNetwork(network.getId()); - for (PortForwardingRuleVO pfRule : pfRules) { - if (pfRule.getVirtualMachineId() == vmId) { - portForwardingRulesDao.remove(pfRule.getId()); - break; - } - } - } - rulesService.applyPortForwardingRules(publicIp.getId(), account); - } - } - - /** - * To provision SSH port forwarding rules for the given Kubernetes cluster - * for its given virtual machines - * @param kubernetesCluster - * @param publicIp - * @param network - * @param account - * @param List clusterVMIds (when empty then method must be called while - * down-scaling of the KubernetesCluster therefore no new rules - * to be added) - * @param firewallRuleSourcePortStart - * @throws ResourceUnavailableException - * @throws NetworkRuleConflictException - */ - private void provisionSshPortForwardingRules(KubernetesCluster kubernetesCluster, IpAddress publicIp, Network network, Account account, List clusterVMIds, int firewallRuleSourcePortStart) throws ResourceUnavailableException, - NetworkRuleConflictException { - if (!CollectionUtils.isEmpty(clusterVMIds)) { - final long publicIpId = publicIp.getId(); - final long networkId = network.getId(); - final long accountId = account.getId(); - final long domainId = account.getDomainId(); - for (int i = 0; i < clusterVMIds.size(); ++i) { - long vmId = clusterVMIds.get(i); - Nic vmNic = networkModel.getNicInNetwork(vmId, networkId); - final Ip vmIp = new Ip(vmNic.getIPv4Address()); - final long vmIdFinal = vmId; - final int srcPortFinal = firewallRuleSourcePortStart + i; - - PortForwardingRuleVO pfRule = Transaction.execute(new TransactionCallbackWithException() { - @Override - public PortForwardingRuleVO doInTransaction(TransactionStatus status) throws NetworkRuleConflictException { - PortForwardingRuleVO newRule = - new PortForwardingRuleVO(null, publicIpId, - srcPortFinal, srcPortFinal, - vmIp, - 22, 22, - "tcp", networkId, accountId, domainId, vmIdFinal); - newRule.setDisplay(true); - newRule.setState(FirewallRule.State.Add); - newRule = portForwardingRulesDao.persist(newRule); - return newRule; - } - }); - rulesService.applyPortForwardingRules(publicIp.getId(), account); - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Provisioned SSH port forwarding rule from port %d to 22 on %s to the VM IP : %s in Kubernetes cluster ID: %s", srcPortFinal, publicIp.getAddress().addr(), vmIp.toString(), kubernetesCluster.getUuid())); - } - } - } - } - - private void provisionLoadBalancerRule(KubernetesCluster kubernetesCluster, IpAddress publicIp, Network network, Account account, List clusterVMIds, int port) throws NetworkRuleConflictException, - InsufficientAddressCapacityException { - LoadBalancer lb = lbService.createPublicLoadBalancerRule(null, "api-lb", "LB rule for API access", - port, port, port, port, - publicIp.getId(), NetUtils.TCP_PROTO, "roundrobin", network.getId(), - account.getId(), false, NetUtils.TCP_PROTO, true); - - Map> vmIdIpMap = new HashMap<>(); - for (int i=0; i ips = new ArrayList<>(); - Nic masterVmNic = networkModel.getNicInNetwork(clusterVMIds.get(i), kubernetesCluster.getNetworkId()); - ips.add(masterVmNic.getIPv4Address()); - vmIdIpMap.put(clusterVMIds.get(i), ips); - } - lbService.assignToLoadBalancer(lb.getId(), null, vmIdIpMap); - } - - /** - * Setup network rules for Kubernetes cluster - * Open up firewall port CLUSTER_API_PORT, secure port on which Kubernetes - * API server is running. Also create load balancing rule to forward public - * IP traffic to master VMs' private IP. - * Open up firewall ports NODES_DEFAULT_START_SSH_PORT to NODES_DEFAULT_START_SSH_PORT+n - * for SSH access. Also create port-forwarding rule to forward public IP traffic to all - * @param kubernetesCluster - * @param network - * @param account - * @param clusterVMIds - * @throws ManagementServerException - */ - private void setupKubernetesClusterNetworkRules(KubernetesCluster kubernetesCluster, - Network network, Account account, - List clusterVMIds) throws ManagementServerException { - if (!Network.GuestType.Isolated.equals(network.getGuestType())) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Network ID: %s for Kubernetes cluster ID: %s is not an isolated network, therefore, no need for network rules", network.getUuid(), kubernetesCluster.getUuid())); - } - return; - } - IpAddress publicIp = getSourceNatIp(network); - if (publicIp == null) { - throw new ManagementServerException(String.format("No source NAT IP addresses found for network ID: %s, Kubernetes cluster ID: %s", network.getUuid(), kubernetesCluster.getUuid())); - } - - try { - provisionFirewallRules(publicIp, account, CLUSTER_API_PORT, CLUSTER_API_PORT); - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Provisioned firewall rule to open up port %d on %s for Kubernetes cluster ID: %s", - CLUSTER_API_PORT, publicIp.getAddress().addr(), kubernetesCluster.getUuid())); - } - } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) { - throw new ManagementServerException(String.format("Failed to provision firewall rules for API access for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); - } - - try { - int endPort = CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterVMIds.size() - 1; - provisionFirewallRules(publicIp, account, CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort); - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Provisioned firewall rule to open up port %d to %d on %s for Kubernetes cluster ID: %s", CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort, publicIp.getAddress().addr(), kubernetesCluster.getUuid())); - } - } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) { - throw new ManagementServerException(String.format("Failed to provision firewall rules for SSH access for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); - } - - // Load balancer rule fo API access for master node VMs - try { - provisionLoadBalancerRule(kubernetesCluster, publicIp, network, account, clusterVMIds, CLUSTER_API_PORT); - } catch (NetworkRuleConflictException | InsufficientAddressCapacityException e) { - throw new ManagementServerException(String.format("Failed to provision load balancer rule for API access for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); - } - - // Port forwarding rule fo SSH access on each node VM - try { - provisionSshPortForwardingRules(kubernetesCluster, publicIp, network, account, clusterVMIds, CLUSTER_NODES_DEFAULT_START_SSH_PORT); - } catch (ResourceUnavailableException | NetworkRuleConflictException e) { - throw new ManagementServerException(String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); - } - } - - /** - * Scale network rules for an existing Kubernetes cluster while scaling it - * Open up firewall for SSH access from port NODES_DEFAULT_START_SSH_PORT to NODES_DEFAULT_START_SSH_PORT+n. - * Also remove port forwarding rules for removed virtual machines and create port-forwarding rule - * to forward public IP traffic to all node VMs' private IP. - * @param kubernetesCluster - * @param network - * @param account - * @param clusterVMIds - * @param removedVMIds - * @throws ManagementServerException - */ - private void scaleKubernetesClusterNetworkRules(KubernetesCluster kubernetesCluster, Network network, Account account, - List clusterVMIds, List removedVMIds) throws ManagementServerException { - if (!Network.GuestType.Isolated.equals(network.getGuestType())) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Network ID: %s for Kubernetes cluster ID: %s is not an isolated network, therefore, no need for network rules", network.getUuid(), kubernetesCluster.getUuid())); - } - return; - } - IpAddress publicIp = getSourceNatIp(network); - if (publicIp == null) { - throw new ManagementServerException(String.format("No source NAT IP addresses found for network ID: %s, Kubernetes cluster ID: %s", network.getUuid(), kubernetesCluster.getUuid())); - } - - // Remove existing SSH firewall rules - FirewallRule firewallRule = removeSshFirewallRule(publicIp); - if (firewallRule == null) { - throw new ManagementServerException("Firewall rule for node SSH access can't be provisioned!"); - } - int existingFirewallRuleSourcePortEnd = firewallRule.getSourcePortEnd(); - - // Provision new SSH firewall rules - try { - provisionFirewallRules(publicIp, account, CLUSTER_NODES_DEFAULT_START_SSH_PORT, CLUSTER_NODES_DEFAULT_START_SSH_PORT + (int)kubernetesCluster.getTotalNodeCount() - 1); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Provisioned firewall rule to open up port %d to %d on %s in Kubernetes cluster ID: %s", - CLUSTER_NODES_DEFAULT_START_SSH_PORT, CLUSTER_NODES_DEFAULT_START_SSH_PORT + (int) kubernetesCluster.getTotalNodeCount() - 1, publicIp.getAddress().addr(), kubernetesCluster.getUuid())); - } - } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException e) { - throw new ManagementServerException(String.format("Failed to activate SSH firewall rules for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); - } - - try { - removePortForwardingRules(publicIp, network, account, removedVMIds); - } catch (ResourceUnavailableException e) { - throw new ManagementServerException(String.format("Failed to remove SSH port forwarding rules for removed VMs for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); - } - - try { - provisionSshPortForwardingRules(kubernetesCluster, publicIp, network, account, clusterVMIds, existingFirewallRuleSourcePortEnd + 1); - } catch (ResourceUnavailableException | NetworkRuleConflictException e) { - throw new ManagementServerException(String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); - } - } - - private boolean validateIsolatedNetwork(Network network, int clusterTotalNodeCount) { - if (Network.GuestType.Isolated.equals(network.getGuestType())) { - if (Network.State.Allocated.equals(network.getState())) { // Allocated networks won't have IP and rules - return true; - } - IpAddress sourceNatIp = getSourceNatIp(network); - if (sourceNatIp == null) { - throw new InvalidParameterValueException(String.format("Network ID: %s does not have a source NAT IP associated with it. To provision a Kubernetes Cluster, source NAT IP is required", network.getUuid())); - } - List rules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(sourceNatIp.getId(), FirewallRule.Purpose.Firewall); - for (FirewallRuleVO rule : rules) { - Integer startPort = rule.getSourcePortStart(); - Integer endPort = rule.getSourcePortEnd(); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Network rule : " + startPort + " " + endPort); - } - if (startPort <= CLUSTER_API_PORT && CLUSTER_API_PORT <= endPort) { - throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting firewall rules to provision Kubernetes cluster for API access", network.getUuid())); - } - if (startPort <= CLUSTER_NODES_DEFAULT_START_SSH_PORT && CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterTotalNodeCount <= endPort) { - throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting firewall rules to provision Kubernetes cluster for node VM SSH access", network.getUuid())); - } - } - rules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(sourceNatIp.getId(), FirewallRule.Purpose.PortForwarding); - for (FirewallRuleVO rule : rules) { - Integer startPort = rule.getSourcePortStart(); - Integer endPort = rule.getSourcePortEnd(); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Network rule : " + startPort + " " + endPort); - } - if (startPort <= CLUSTER_API_PORT && CLUSTER_API_PORT <= endPort) { - throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting port forwarding rules to provision Kubernetes cluster for API access", network.getUuid())); - } - if (startPort <= CLUSTER_NODES_DEFAULT_START_SSH_PORT && CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterTotalNodeCount <= endPort) { - throw new InvalidParameterValueException(String.format("Network ID: %s has conflicting port forwarding rules to provision Kubernetes cluster for node VM SSH access", network.getUuid())); - } - } - } - return true; - } - - private boolean validateNetwork(Network network, int clusterTotalNodeCount) { - NetworkOffering networkOffering = networkOfferingDao.findById(network.getNetworkOfferingId()); - if (networkOffering.isSystemOnly()) { - throw new InvalidParameterValueException(String.format("Network ID: %s is for system use only", network.getUuid())); - } - if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.UserData)) { - throw new InvalidParameterValueException(String.format("Network ID: %s does not support userdata that is required for Kubernetes cluster", network.getUuid())); - } - if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.Firewall)) { - throw new InvalidParameterValueException(String.format("Network ID: %s does not support firewall that is required for Kubernetes cluster", network.getUuid())); - } - if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.PortForwarding)) { - throw new InvalidParameterValueException(String.format("Network ID: %s does not support port forwarding that is required for Kubernetes cluster", network.getUuid())); - } - if (!networkModel.areServicesSupportedInNetwork(network.getId(), Service.Dhcp)) { - throw new InvalidParameterValueException(String.format("Network ID: %s does not support DHCP that is required for Kubernetes cluster", network.getUuid())); - } - validateIsolatedNetwork(network, clusterTotalNodeCount); - return true; - } - - private boolean validateServiceOffering(ServiceOffering serviceOffering) { - if (serviceOffering.isDynamic()) { - throw new InvalidParameterValueException(String.format("Custom service offerings are not supported for creating clusters, service offering ID: %s", serviceOffering.getUuid())); - } - if (serviceOffering.getCpu() < 2 || serviceOffering.getRamSize() < 2048) { - throw new InvalidParameterValueException(String.format("Kubernetes cluster cannot be created with service offering ID: %s, Kubernetes cluster template(CoreOS) needs minimum 2 vCPUs and 2 GB RAM", serviceOffering.getUuid())); - } - return true; - } - - private void validateDockerRegistryParams(final String dockerRegistryUserName, - final String dockerRegistryPassword, - final String dockerRegistryUrl, - final String dockerRegistryEmail) { - // if no params related to docker registry specified then nothing to validate so return true - if ((dockerRegistryUserName == null || dockerRegistryUserName.isEmpty()) && - (dockerRegistryPassword == null || dockerRegistryPassword.isEmpty()) && - (dockerRegistryUrl == null || dockerRegistryUrl.isEmpty()) && - (dockerRegistryEmail == null || dockerRegistryEmail.isEmpty())) { - return; - } - - // all params related to docker registry must be specified or nothing - if (!((dockerRegistryUserName != null && !dockerRegistryUserName.isEmpty()) && - (dockerRegistryPassword != null && !dockerRegistryPassword.isEmpty()) && - (dockerRegistryUrl != null && !dockerRegistryUrl.isEmpty()) && - (dockerRegistryEmail != null && !dockerRegistryEmail.isEmpty()))) { - throw new InvalidParameterValueException("All the docker private registry parameters (username, password, url, email) required are specified"); - } - - try { - URL url = new URL(dockerRegistryUrl); - } catch (MalformedURLException e) { - throw new InvalidParameterValueException("Invalid docker registry url specified"); - } - - Pattern VALID_EMAIL_ADDRESS_REGEX = Pattern.compile("^[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,6}$", Pattern.CASE_INSENSITIVE); - Matcher matcher = VALID_EMAIL_ADDRESS_REGEX.matcher(dockerRegistryEmail); - if (!matcher.find()) { - throw new InvalidParameterValueException("Invalid docker registry email specified"); - } - } - - private DeployDestination plan(final long nodesCount, final DataCenter zone, final ServiceOffering offering) throws InsufficientServerCapacityException { - final int cpu_requested = offering.getCpu() * offering.getSpeed(); - final long ram_requested = offering.getRamSize() * 1024L * 1024L; - List hosts = resourceManager.listAllHostsInOneZoneByType(Type.Routing, zone.getId()); - final Map> hosts_with_resevered_capacity = new ConcurrentHashMap>(); - for (HostVO h : hosts) { - hosts_with_resevered_capacity.put(h.getUuid(), new Pair(h, 0)); - } - boolean suitable_host_found = false; - for (int i = 1; i <= nodesCount + 1; i++) { - suitable_host_found = false; - for (Map.Entry> hostEntry : hosts_with_resevered_capacity.entrySet()) { - Pair hp = hostEntry.getValue(); - HostVO h = hp.first(); - int reserved = hp.second(); - reserved++; - ClusterVO cluster = clusterDao.findById(h.getClusterId()); - ClusterDetailsVO cluster_detail_cpu = clusterDetailsDao.findDetail(cluster.getId(), "cpuOvercommitRatio"); - ClusterDetailsVO cluster_detail_ram = clusterDetailsDao.findDetail(cluster.getId(), "memoryOvercommitRatio"); - Float cpuOvercommitRatio = Float.parseFloat(cluster_detail_cpu.getValue()); - Float memoryOvercommitRatio = Float.parseFloat(cluster_detail_ram.getValue()); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Checking host ID: %s for capacity already reserved %d", h.getUuid(), reserved)); - } - if (capacityManager.checkIfHostHasCapacity(h.getId(), cpu_requested * reserved, ram_requested * reserved, false, cpuOvercommitRatio, memoryOvercommitRatio, true)) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Found host ID: %s for with enough capacity, CPU=%d RAM=%d", h.getUuid(), cpu_requested * reserved, ram_requested * reserved)); - } - hostEntry.setValue(new Pair(h, reserved)); - suitable_host_found = true; - break; - } - } - if (!suitable_host_found) { - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Suitable hosts not found in datacenter ID: %s for node %d", zone.getUuid(), i)); - } - break; - } - } - if (suitable_host_found) { - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Suitable hosts found in datacenter ID: %s, creating deployment destination", zone.getUuid())); - } - return new DeployDestination(zone, null, null, null); - } - String msg = String.format("Cannot find enough capacity for Kubernetes cluster(requested cpu=%1$s memory=%2$s)", - cpu_requested * nodesCount, ram_requested * nodesCount); - LOGGER.warn(msg); - throw new InsufficientServerCapacityException(msg, DataCenter.class, zone.getId()); - } - - private DeployDestination plan(final KubernetesCluster kubernetesCluster, final DataCenter zone) throws InsufficientServerCapacityException { - ServiceOffering offering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); - - if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Checking deployment destination for Kubernetes cluster ID: %s in zone ID: %s", kubernetesCluster.getUuid(), zone.getUuid())); - } - return plan(kubernetesCluster.getTotalNodeCount(), zone, offering); - } - - protected boolean cleanupKubernetesClusterResources(Long kubernetesClusterId) throws ManagementServerException { - KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); - if (!(kubernetesCluster.getState().equals(KubernetesCluster.State.Running) - || kubernetesCluster.getState().equals(KubernetesCluster.State.Stopped) - || kubernetesCluster.getState().equals(KubernetesCluster.State.Alert) - || kubernetesCluster.getState().equals(KubernetesCluster.State.Error) - || kubernetesCluster.getState().equals(KubernetesCluster.State.Destroying))) { - String msg = String.format("Cannot perform delete operation on cluster ID: %s in state: %s",kubernetesCluster.getUuid(), kubernetesCluster.getState()); - LOGGER.warn(msg); - throw new PermissionDeniedException(msg); - } - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.DestroyRequested); - boolean failedVmDestroy = false; - List clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); - if ((clusterVMs != null) && !clusterVMs.isEmpty()) { - for (KubernetesClusterVmMapVO clusterVM : clusterVMs) { - long vmID = clusterVM.getVmId(); - - // delete only if VM exists and is not removed - UserVmVO userVM = userVmDao.findById(vmID); - if (userVM == null || userVM.isRemoved()) { - continue; - } - try { - UserVm vm = userVmService.destroyVm(vmID, true); - if (!VirtualMachine.State.Expunging.equals(vm.getState())) { - LOGGER.warn(String.format("VM '%s' ID: %s should have been expunging by now but is '%s'... retrying..." - , vm.getInstanceName() - , vm.getUuid() - , vm.getState().toString())); - } - vm = userVmService.expungeVm(vmID); - if (!VirtualMachine.State.Expunging.equals(vm.getState())) { - LOGGER.error(String.format("VM '%s' ID: %s is now in state '%s'. Will probably fail at deleting it's Kubernetes cluster." - , vm.getInstanceName() - , vm.getUuid() - , vm.getState().toString())); - } - kubernetesClusterVmMapDao.expunge(clusterVM.getId()); - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Destroyed VM ID: %s as part of Kubernetes cluster ID: %s cleanup", vm.getUuid(), kubernetesCluster.getUuid())); - } - } catch (Exception e) { - failedVmDestroy = true; - LOGGER.warn(String.format("Failed to destroy VM ID: %s part of the Kubernetes cluster ID: %s cleanup. Moving on with destroying remaining resources provisioned for the Kubernetes cluster", userVM.getUuid(), kubernetesCluster.getUuid()), e); - } - } - } - boolean cleanupNetwork = true; - try { - final KubernetesClusterDetailsVO clusterDetails = kubernetesClusterDetailsDao.findDetail(kubernetesClusterId, "networkCleanup"); - cleanupNetwork = Boolean.parseBoolean(clusterDetails.getValue()); - } catch (Exception e) {} - - // if there are VM's that were not expunged, we can not delete the network - if (!failedVmDestroy) { - if (cleanupNetwork) { - if(clusterVMs!=null && !clusterVMs.isEmpty()) { // Wait for few seconds to get all VMs really expunged - final int maxRetries = 3; - int retryCounter = 0; - while (retryCounter < maxRetries) { - boolean allVMsRemoved = true; - for (KubernetesClusterVmMap clusterVM : clusterVMs) { - UserVmVO userVM = userVmDao.findById(clusterVM.getVmId()); - if (userVM != null && !userVM.isRemoved()) { - allVMsRemoved = false; - break; - } - } - if (allVMsRemoved) { - break; - } - try { - Thread.sleep(10000); - } catch (InterruptedException ie) {} - retryCounter++; - } - } - NetworkVO network = null; - try { - network = networkDao.findById(kubernetesCluster.getNetworkId()); - if (network != null && network.getRemoved() == null) { - Account owner = accountManager.getAccount(network.getAccountId()); - User callerUser = accountManager.getActiveUser(CallContext.current().getCallingUserId()); - ReservationContext context = new ReservationContextImpl(null, null, callerUser, owner); - boolean networkDestroyed = networkMgr.destroyNetwork(kubernetesCluster.getNetworkId(), context, true); - if (!networkDestroyed) { - String msg = String.format("Failed to destroy network ID: %s as part of Kubernetes cluster ID: %s cleanup", network.getUuid(), kubernetesCluster.getUuid()); - LOGGER.warn(msg); - processFailedNetworkDelete(kubernetesClusterId); - throw new ManagementServerException(msg); - } - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Destroyed network: %s as part of Kubernetes cluster ID: %s cleanup", network.getUuid(), kubernetesCluster.getUuid())); - } - } - } catch (Exception e) { - String msg = String.format("Failed to destroy network ID: %s as part of Kubernetes cluster ID: %s cleanup", network.getUuid(), kubernetesCluster.getUuid()); - LOGGER.warn(msg, e); - processFailedNetworkDelete(kubernetesClusterId); - throw new ManagementServerException(msg, e); - } - } - } else { - String msg = String.format("Failed to destroy one or more VMs as part of Kubernetes cluster ID: %s cleanup", kubernetesCluster.getUuid()); - if (LOGGER.isInfoEnabled()) { - LOGGER.info(msg); - } - processFailedNetworkDelete(kubernetesClusterId); - throw new ManagementServerException(msg); - } - - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); - - kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); - kubernetesCluster.setCheckForGc(false); - kubernetesClusterDao.update(kubernetesCluster.getId(), kubernetesCluster); - - kubernetesClusterDao.remove(kubernetesCluster.getId()); - - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Kubernetes cluster ID: %s is successfully deleted", kubernetesCluster.getUuid())); - } - - return true; - } - - private void processFailedNetworkDelete(long kubernetesClusterId) { - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); - KubernetesClusterVO cluster = kubernetesClusterDao.findById(kubernetesClusterId); - cluster.setCheckForGc(true); - kubernetesClusterDao.update(cluster.getId(), cluster); - } - - private UserVm createKubernetesMaster(final KubernetesCluster kubernetesCluster, final Pod pod, final Network network, final Account account, String serverIp) throws ManagementServerException, - ResourceUnavailableException, InsufficientCapacityException { - UserVm masterVm = null; - DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); - ServiceOffering serviceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); - VirtualMachineTemplate template = templateDao.findById(kubernetesCluster.getTemplateId()); - List networkIds = new ArrayList(); - networkIds.add(kubernetesCluster.getNetworkId()); - Account owner = accountDao.findById(kubernetesCluster.getAccountId()); - String masterIp = null; - Map requestedIps = null; - if (Network.GuestType.Shared.equals(network.getGuestType())) { - List vlanIds = new ArrayList<>(); - List vlans = vlanDao.listVlansByNetworkId(network.getId()); - for (VlanVO vlan : vlans) { - vlanIds.add(vlan.getId()); - } - PublicIp ip = ipAddressManager.getAvailablePublicIpAddressFromVlans(zone.getId(), null, account, Vlan.VlanType.DirectAttached, vlanIds,network.getId(), null, false); - if (ip != null) { - masterIp = ip.getAddress().toString(); - } - if (Strings.isNullOrEmpty(serverIp)) { - serverIp = masterIp; - } - requestedIps = new HashMap<>(); - Ip ipAddress = ip.getAddress(); - boolean isIp6 = ipAddress.isIp6(); - requestedIps.put(network.getId(), new Network.IpAddresses(ipAddress.isIp4() ? ip.getAddress().addr() : null, null)); - } else { - masterIp = ipAddressManager.acquireGuestIpAddress(networkDao.findById(kubernetesCluster.getNetworkId()), null); - } - Network.IpAddresses addrs = new Network.IpAddresses(masterIp, null); - long rootDiskSize = kubernetesCluster.getNodeRootDiskSize(); - Map customParameterMap = new HashMap(); - if (rootDiskSize > 0) { - customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); - } - String hostName = kubernetesCluster.getName() + "-k8s-master"; - boolean haSupported = false; - final KubernetesSupportedVersion version = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); - if (version != null) { - try { - if (KubernetesVersionManagerImpl.compareSemanticVersions(version.getSemanticVersion(), MIN_KUBERNETES_VERSION_HA_SUPPORT) >= 0) { - haSupported = true; - } - } catch (IllegalArgumentException e) { - LOGGER.error(String.format("Unable to compare Kubernetes version for cluster version ID: %s with %s", version.getUuid(), MIN_KUBERNETES_VERSION_HA_SUPPORT), e); - } - } - String k8sMasterConfig = null; - try { - k8sMasterConfig = readResourceFile("/conf/k8s-master.yml"); - final String apiServerCert = "{{ k8s_master.apiserver.crt }}"; - final String apiServerKey = "{{ k8s_master.apiserver.key }}"; - final String caCert = "{{ k8s_master.ca.crt }}"; - final String sshPubKey = "{{ k8s.ssh.pub.key }}"; - final String clusterToken = "{{ k8s_master.cluster.token }}"; - final String clusterInitArgsKey = "{{ k8s_master.cluster.initargs }}"; - final List addresses = new ArrayList<>(); - addresses.add(masterIp); - if (!serverIp.equals(masterIp)) { - addresses.add(serverIp); - } - final Certificate certificate = caManager.issueCertificate(null, Arrays.asList(hostName, "kubernetes", - "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.default.svc.cluster.local"), - addresses, 3650, null); - final String tlsClientCert = CertUtils.x509CertificateToPem(certificate.getClientCertificate()); - final String tlsPrivateKey = CertUtils.privateKeyToPem(certificate.getPrivateKey()); - final String tlsCaCert = CertUtils.x509CertificatesToPem(certificate.getCaCertificates()); - k8sMasterConfig = k8sMasterConfig.replace(apiServerCert, tlsClientCert.replace("\n", "\n ")); - k8sMasterConfig = k8sMasterConfig.replace(apiServerKey, tlsPrivateKey.replace("\n", "\n ")); - k8sMasterConfig = k8sMasterConfig.replace(caCert, tlsCaCert.replace("\n", "\n ")); - String pubKey = "- \"" + globalConfigDao.getValue("ssh.publickey") + "\""; - String sshKeyPair = kubernetesCluster.getKeyPair(); - if (!Strings.isNullOrEmpty(sshKeyPair)) { - SSHKeyPairVO sshkp = sshKeyPairDao.findByName(owner.getAccountId(), owner.getDomainId(), sshKeyPair); - if (sshkp != null) { - pubKey += "\n - \"" + sshkp.getPublicKey() + "\""; - } - } - k8sMasterConfig = k8sMasterConfig.replace(sshPubKey, pubKey); - k8sMasterConfig = k8sMasterConfig.replace(clusterToken, generateClusterToken(kubernetesCluster)); - String initArgs = ""; - if (haSupported) { - initArgs = String.format("--control-plane-endpoint %s:%d --upload-certs --certificate-key %s ", - serverIp, - CLUSTER_API_PORT, - generateClusterHACertificateKey(kubernetesCluster)); - } - initArgs += String.format("--apiserver-cert-extra-sans=%s", serverIp); - k8sMasterConfig = k8sMasterConfig.replace(clusterInitArgsKey, initArgs); - } catch (IOException e) { - String msg = "Failed to read Kubernetes master configuration file"; - LOGGER.error(msg, e); - throw new ManagementServerException(msg, e); - } - String base64UserData = Base64.encodeBase64String(k8sMasterConfig.getBytes(StringUtils.getPreferredCharset())); - masterVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, - hostName, kubernetesCluster.getDescription(), null, null, null, - null, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), - requestedIps, addrs, null, null, null, customParameterMap, null, null, null, null); - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Created master VM ID: %s, %s in the Kubernetes cluster ID: %s", masterVm.getUuid(), hostName, kubernetesCluster.getUuid())); - } - return masterVm; - } - - private UserVm createKubernetesAdditionalMaster(final KubernetesCluster kubernetesCluster, final String joinIp, final int additionalMasterNodeInstance) throws ManagementServerException, - ResourceUnavailableException, InsufficientCapacityException { - UserVm additionalMasterVm = null; - DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); - ServiceOffering serviceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); - VirtualMachineTemplate template = templateDao.findById(kubernetesCluster.getTemplateId()); - List networkIds = new ArrayList(); - networkIds.add(kubernetesCluster.getNetworkId()); - Account owner = accountDao.findById(kubernetesCluster.getAccountId()); - Network.IpAddresses addrs = new Network.IpAddresses(null, null); - long rootDiskSize = kubernetesCluster.getNodeRootDiskSize(); - Map customParameterMap = new HashMap(); - if (rootDiskSize > 0) { - customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); - } - String hostName = String.format("%s-k8s-master-%s", kubernetesCluster.getName(), additionalMasterNodeInstance); - String k8sMasterConfig = null; - try { - k8sMasterConfig = readResourceFile("/conf/k8s-master-add.yml"); - final String joinIpKey = "{{ k8s_master.join_ip }}"; - final String clusterTokenKey = "{{ k8s_master.cluster.token }}"; - final String sshPubKey = "{{ k8s.ssh.pub.key }}"; - final String clusterHACertificateKey = "{{ k8s_master.cluster.ha.certificate.key }}"; - String pubKey = "- \"" + globalConfigDao.getValue("ssh.publickey") + "\""; - String sshKeyPair = kubernetesCluster.getKeyPair(); - if (!Strings.isNullOrEmpty(sshKeyPair)) { - SSHKeyPairVO sshkp = sshKeyPairDao.findByName(owner.getAccountId(), owner.getDomainId(), sshKeyPair); - if (sshkp != null) { - pubKey += "\n - \"" + sshkp.getPublicKey() + "\""; - } - } - k8sMasterConfig = k8sMasterConfig.replace(sshPubKey, pubKey); - k8sMasterConfig = k8sMasterConfig.replace(joinIpKey, joinIp); - k8sMasterConfig = k8sMasterConfig.replace(clusterTokenKey, generateClusterToken(kubernetesCluster)); - k8sMasterConfig = k8sMasterConfig.replace(clusterHACertificateKey, generateClusterHACertificateKey(kubernetesCluster)); - } catch (IOException e) { - String msg = "Failed to read Kubernetes master configuration file"; - LOGGER.error(msg, e); - throw new ManagementServerException(msg, e); - } - String base64UserData = Base64.encodeBase64String(k8sMasterConfig.getBytes(StringUtils.getPreferredCharset())); - additionalMasterVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, - hostName, kubernetesCluster.getDescription(), null, null, null, - null, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), - null, addrs, null, null, null, customParameterMap, null, null, null, null); - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Created master VM ID: %s, %s in the Kubernetes cluster ID: %s", additionalMasterVm.getUuid(), hostName, kubernetesCluster.getUuid())); - } - return additionalMasterVm; - } - - private UserVm createKubernetesNode(KubernetesCluster kubernetesCluster, String joinIp, int nodeInstance) throws ManagementServerException, - ResourceUnavailableException, InsufficientCapacityException { - UserVm nodeVm = null; - DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); - ServiceOffering serviceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); - VirtualMachineTemplate template = templateDao.findById(kubernetesCluster.getTemplateId()); - List networkIds = new ArrayList(); - networkIds.add(kubernetesCluster.getNetworkId()); - Account owner = accountDao.findById(kubernetesCluster.getAccountId()); - Network.IpAddresses addrs = new Network.IpAddresses(null, null); - long rootDiskSize = kubernetesCluster.getNodeRootDiskSize(); - Map customParameterMap = new HashMap(); - if (rootDiskSize > 0) { - customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); - } - String hostName = String.format("%s-k8s-node-%s", kubernetesCluster.getName(), nodeInstance); - String k8sNodeConfig = null; - try { - k8sNodeConfig = readResourceFile("/conf/k8s-node.yml"); - final String sshPubKey = "{{ k8s.ssh.pub.key }}"; - final String joinIpKey = "{{ k8s_master.join_ip }}"; - final String clusterTokenKey = "{{ k8s_master.cluster.token }}"; - String pubKey = "- \"" + globalConfigDao.getValue("ssh.publickey") + "\""; - String sshKeyPair = kubernetesCluster.getKeyPair(); - if (!Strings.isNullOrEmpty(sshKeyPair)) { - SSHKeyPairVO sshkp = sshKeyPairDao.findByName(owner.getAccountId(), owner.getDomainId(), sshKeyPair); - if (sshkp != null) { - pubKey += "\n - \"" + sshkp.getPublicKey() + "\""; - } - } - k8sNodeConfig = k8sNodeConfig.replace(sshPubKey, pubKey); - k8sNodeConfig = k8sNodeConfig.replace(joinIpKey, joinIp); - k8sNodeConfig = k8sNodeConfig.replace(clusterTokenKey, generateClusterToken(kubernetesCluster)); - /* genarate /.docker/config.json file on the nodes only if Kubernetes cluster is created to - * use docker private registry */ - String dockerUserName = null; - String dockerPassword = null; - String dockerRegistryUrl = null; - String dockerRegistryEmail = null; - List details = kubernetesClusterDetailsDao.listDetails(kubernetesCluster.getId()); - for (KubernetesClusterDetailsVO detail : details) { - if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_USER_NAME)) { - dockerUserName = detail.getValue(); - } - if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_PASSWORD)) { - dockerPassword = detail.getValue(); - } - if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_URL)) { - dockerRegistryUrl = detail.getValue(); - } - if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_EMAIL)) { - dockerRegistryEmail = detail.getValue(); - } - } - if (!Strings.isNullOrEmpty(dockerUserName) && !Strings.isNullOrEmpty(dockerPassword)) { - // do write file for /.docker/config.json through the code instead of k8s-node.yml as we can no make a section - // optional or conditionally applied - String dockerConfigString = "write-files:\n" + - " - path: /.docker/config.json\n" + - " owner: core:core\n" + - " permissions: '0644'\n" + - " content: |\n" + - " {\n" + - " \"auths\": {\n" + - " {{docker.url}}: {\n" + - " \"auth\": {{docker.secret}},\n" + - " \"email\": {{docker.email}}\n" + - " }\n" + - " }\n" + - " }"; - k8sNodeConfig = k8sNodeConfig.replace("write-files:", dockerConfigString); - final String dockerUrlKey = "{{docker.url}}"; - final String dockerAuthKey = "{{docker.secret}}"; - final String dockerEmailKey = "{{docker.email}}"; - final String usernamePasswordKey = dockerUserName + ":" + dockerPassword; - String base64Auth = Base64.encodeBase64String(usernamePasswordKey.getBytes(StringUtils.getPreferredCharset())); - k8sNodeConfig = k8sNodeConfig.replace(dockerUrlKey, "\"" + dockerRegistryUrl + "\""); - k8sNodeConfig = k8sNodeConfig.replace(dockerAuthKey, "\"" + base64Auth + "\""); - k8sNodeConfig = k8sNodeConfig.replace(dockerEmailKey, "\"" + dockerRegistryEmail + "\""); - } - } catch (IOException e) { - String msg = "Failed to read Kubernetes node configuration file"; - LOGGER.error(msg, e); - throw new ManagementServerException(msg, e); - } - String base64UserData = Base64.encodeBase64String(k8sNodeConfig.getBytes(StringUtils.getPreferredCharset())); - nodeVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, - hostName, kubernetesCluster.getDescription(), null, null, null, - null, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), - null, addrs, null, null, null, customParameterMap, null, null, null, null); - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Created node VM ID: %s, %s in the Kubernetes cluster ID: %s", nodeVm.getUuid(), hostName, kubernetesCluster.getUuid())); - } - return nodeVm; - } - - private void startKubernetesVM(final UserVm vm, final KubernetesCluster kubernetesCluster) throws ConcurrentOperationException { - try { - StartVMCmd startVm = new StartVMCmd(); - startVm = ComponentContext.inject(startVm); - Field f = startVm.getClass().getDeclaredField("id"); - f.setAccessible(true); - f.set(startVm, vm.getId()); - userVmService.startVirtualMachine(startVm); - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Started VM ID: %s in the Kubernetes cluster ID: %s", vm.getUuid(), kubernetesCluster.getUuid())); - } - } catch (IllegalAccessException | NoSuchFieldException | ExecutionException | - ResourceUnavailableException | ResourceAllocationException | InsufficientCapacityException ex) { - logAndThrow(Level.WARN, String.format("Failed to start VM in the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), ex); - } - - UserVm startVm = userVmDao.findById(vm.getId()); - if (!startVm.getState().equals(VirtualMachine.State.Running)) { - logAndThrow(Level.WARN, String.format("Failed to start VM in the Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); - } - } - - private void attachIsoKubernetesVMs(KubernetesCluster kubernetesCluster, List clusterVMIds) throws CloudRuntimeException { - KubernetesSupportedVersion version = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); - if (version == null) { - logAndThrow(Level.ERROR, String .format("Unable to find Kubernetes version for cluster ID: %s", kubernetesCluster.getUuid())); - } - VMTemplateVO iso = templateDao.findById(version.getIsoId()); - if (iso == null) { - logAndThrow(Level.ERROR, String.format("Unable to attach ISO to Kubernetes cluster ID: %s. Binaries ISO not found.", kubernetesCluster.getUuid())); - } - if (!iso.getFormat().equals(Storage.ImageFormat.ISO)) { - logAndThrow(Level.ERROR, String.format("Unable to attach ISO to Kubernetes cluster ID: %s. Invalid Binaries ISO.", kubernetesCluster.getUuid())); - } - if (!iso.getState().equals(VirtualMachineTemplate.State.Active)) { - logAndThrow(Level.ERROR, String.format("Unable to attach ISO to Kubernetes cluster ID: %s. Binaries ISO not active.", kubernetesCluster.getUuid())); - } - for (Long clusterVMId : clusterVMIds) { - UserVm vm = userVmDao.findById(clusterVMId); - try { - templateService.attachIso(iso.getId(), vm.getId()); - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Attached binaries ISO for VM: %s in cluster: %s", vm.getUuid(), kubernetesCluster.getName())); - } - } catch (CloudRuntimeException ex) { - logAndThrow(Level.ERROR, String.format("Failed to attach binaries ISO for VM: %s in the Kubernetes cluster name: %s", vm.getDisplayName(), kubernetesCluster.getName()), ex); - } - } - } - - private void detachIsoKubernetesVMs(final KubernetesCluster kubernetesCluster, List clusterVMIds) throws CloudRuntimeException { - for (Long clusterVMId : clusterVMIds) { - UserVm vm = userVmDao.findById(clusterVMId); - boolean result = false; - try { - result = templateService.detachIso(vm.getId()); - } catch (CloudRuntimeException ex) { - LOGGER.warn(String.format("Failed to detach binaries ISO from VM ID: %s in the Kubernetes cluster ID: %s ", vm.getUuid(), kubernetesCluster.getUuid()), ex); - } - if (result) { - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Detached Kubernetes binaries from VM ID: %s in the Kubernetes cluster ID: %s", vm.getUuid(), kubernetesCluster.getUuid())); - } - continue; - } - LOGGER.warn(String.format("Failed to detach binaries ISO from VM ID: %s in the Kubernetes cluster ID: %s ", vm.getUuid(), kubernetesCluster.getUuid())); - } - } - - private void stopClusterVM(final KubernetesClusterVmMapVO vmMapVO) throws CloudRuntimeException { - try { - userVmService.stopVirtualMachine(vmMapVO.getVmId(), false); - } catch (ConcurrentOperationException ex) { - logAndThrow(Level.WARN, "Failed to stop Kubernetes cluster VM", ex); - } - } - - @Override - public KubernetesClusterResponse createKubernetesClusterResponse(long kubernetesClusterId) { - KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); - KubernetesClusterResponse response = new KubernetesClusterResponse(); - response.setObjectName(KubernetesCluster.class.getSimpleName().toLowerCase()); - response.setId(kubernetesCluster.getUuid()); - response.setName(kubernetesCluster.getName()); - response.setDescription(kubernetesCluster.getDescription()); - DataCenterVO zone = ApiDBUtils.findZoneById(kubernetesCluster.getZoneId()); - response.setZoneId(zone.getUuid()); - response.setZoneName(zone.getName()); - response.setMasterNodes(kubernetesCluster.getMasterNodeCount()); - response.setClusterSize(kubernetesCluster.getNodeCount()); - VMTemplateVO template = ApiDBUtils.findTemplateById(kubernetesCluster.getTemplateId()); - response.setTemplateId(template.getUuid()); - ServiceOfferingVO offering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); - response.setServiceOfferingId(offering.getUuid()); - response.setServiceOfferingName(offering.getName()); - KubernetesSupportedVersionVO version = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); - if (version != null) { - response.setKubernetesVersionId(version.getUuid()); - response.setKubernetesVersionName(version.getName()); - } - response.setKeypair(kubernetesCluster.getKeyPair()); - response.setState(kubernetesCluster.getState().toString()); - response.setCores(String.valueOf(kubernetesCluster.getCores())); - response.setMemory(String.valueOf(kubernetesCluster.getMemory())); - NetworkVO ntwk = networkDao.findByIdIncludingRemoved(kubernetesCluster.getNetworkId()); - response.setEndpoint(kubernetesCluster.getEndpoint()); - response.setNetworkId(ntwk.getUuid()); - response.setAssociatedNetworkName(ntwk.getName()); - List vmIds = new ArrayList(); - List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); - if (vmList != null && !vmList.isEmpty()) { - for (KubernetesClusterVmMapVO vmMapVO : vmList) { - UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId()); - if (userVM != null) { - vmIds.add(userVM.getUuid()); - } - } - } - response.setVirtualMachineIds(vmIds); - return response; - } - - private void validateKubernetesClusterCreatePrameters(final CreateKubernetesClusterCmd cmd) throws ManagementServerException { - final String name = cmd.getName(); - final Long zoneId = cmd.getZoneId(); - final Long kubernetesVersionId = cmd.getKubernetesVersionId(); - final Long serviceOfferingId = cmd.getServiceOfferingId(); - final Account owner = accountService.getActiveAccountById(cmd.getEntityOwnerId()); - final Long networkId = cmd.getNetworkId(); - final String sshKeyPair = cmd.getSSHKeyPairName(); - final Long masterNodeCount = cmd.getMasterNodes(); - final Long clusterSize = cmd.getClusterSize(); - final String dockerRegistryUserName = cmd.getDockerRegistryUserName(); - final String dockerRegistryPassword = cmd.getDockerRegistryPassword(); - final String dockerRegistryUrl = cmd.getDockerRegistryUrl(); - final String dockerRegistryEmail = cmd.getDockerRegistryEmail(); - final Long nodeRootDiskSize = cmd.getNodeRootDiskSize(); - final String externalLoadBalancerIpAddress = cmd.getExternalLoadBalancerIpAddress(); - - if (name == null || name.isEmpty()) { - throw new InvalidParameterValueException("Invalid name for the Kubernetes cluster name:" + name); - } - - if (masterNodeCount < 1 || masterNodeCount > 100) { - throw new InvalidParameterValueException("Invalid cluster master nodes count: " + masterNodeCount); - } - - if (clusterSize < 1 || clusterSize > 100) { - throw new InvalidParameterValueException("Invalid cluster size: " + clusterSize); - } - - DataCenter zone = dataCenterDao.findById(zoneId); - if (zone == null) { - throw new InvalidParameterValueException("Unable to find zone by ID: " + zoneId); - } - - if (Grouping.AllocationState.Disabled == zone.getAllocationState()) { - throw new PermissionDeniedException(String.format("Cannot perform this operation, zone ID: %s is currently disabled", zone.getUuid())); - } - - if (!isKubernetesServiceConfigured(zone)) { - throw new ManagementServerException("Kubernetes service has not been configured properly to provision Kubernetes clusters"); - } - - final KubernetesSupportedVersion clusterKubernetesVersion = kubernetesSupportedVersionDao.findById(kubernetesVersionId); - if (clusterKubernetesVersion == null) { - throw new InvalidParameterValueException("Unable to find given Kubernetes version in supported versions"); - } - if (clusterKubernetesVersion.getZoneId() != null && !clusterKubernetesVersion.getZoneId().equals(zone.getId())) { - throw new InvalidParameterValueException(String.format("Kubernetes version ID: %s is not available for zone ID: %s", clusterKubernetesVersion.getUuid(), zone.getUuid())); - } - if (masterNodeCount > 1 ) { - try { - if (KubernetesVersionManagerImpl.compareSemanticVersions(clusterKubernetesVersion.getSemanticVersion(), MIN_KUBERNETES_VERSION_HA_SUPPORT) < 0) { - throw new InvalidParameterValueException(String.format("HA support is available only for Kubernetes version %s and above. Given version ID: %s is %s", MIN_KUBERNETES_VERSION_HA_SUPPORT, clusterKubernetesVersion.getUuid(), clusterKubernetesVersion.getSemanticVersion())); - } - } catch (IllegalArgumentException e) { - logAndThrow(Level.WARN, String.format("Unable to compare Kubernetes version for given version ID: %s with %s", clusterKubernetesVersion.getUuid(), MIN_KUBERNETES_VERSION_HA_SUPPORT), e); - } - } - - if (clusterKubernetesVersion.getZoneId() != null && clusterKubernetesVersion.getZoneId() != zone.getId()) { - throw new InvalidParameterValueException(String.format("Kubernetes version ID: %s is not available for zone ID: %s", clusterKubernetesVersion.getUuid(), zone.getUuid())); - } - - TemplateJoinVO iso = templateJoinDao.findById(clusterKubernetesVersion.getIsoId()); - if (iso == null) { - throw new InvalidParameterValueException(String.format("Invalid ISO associated with version ID: %s", clusterKubernetesVersion.getUuid())); - } - if (!ObjectInDataStoreStateMachine.State.Ready.equals(iso.getState())) { - throw new InvalidParameterValueException(String.format("ISO associated with version ID: %s is not in Ready state", clusterKubernetesVersion.getUuid())); - } - - ServiceOffering serviceOffering = serviceOfferingDao.findById(serviceOfferingId); - if (serviceOffering == null) { - throw new InvalidParameterValueException("No service offering with ID: " + serviceOfferingId); - } - - if (sshKeyPair != null && !sshKeyPair.isEmpty()) { - SSHKeyPairVO sshKeyPairVO = sshKeyPairDao.findByName(owner.getAccountId(), owner.getDomainId(), sshKeyPair); - if (sshKeyPairVO == null) { - throw new InvalidParameterValueException(String.format("Given SSH key pair with name: %s was not found for the account %s", sshKeyPair, owner.getAccountName())); - } - } - - if (nodeRootDiskSize <= 0) { - throw new InvalidParameterValueException(String.format("Invalid value for %s", ApiConstants.NODE_ROOT_DISK_SIZE)); - } - - VMTemplateVO template = templateDao.findByTemplateName(KubernetesClusterTemplateName.value()); - List listZoneTemplate = templateZoneDao.listByZoneTemplate(zone.getId(), template.getId()); - if (listZoneTemplate == null || listZoneTemplate.isEmpty()) { - String msg = String.format("The template ID: %s is not available for use in zone ID: %s to provision Kubernetes cluster name: %s", template.getUuid(), zone.getUuid(), name); - LOGGER.warn(msg); - throw new ManagementServerException(msg); - } - - if (!validateServiceOffering(serviceOfferingDao.findById(serviceOfferingId))) { - throw new InvalidParameterValueException("Given service offering ID: %s is not suitable for Kubernetes cluster"); - } - - validateDockerRegistryParams(dockerRegistryUserName, dockerRegistryPassword, dockerRegistryUrl, dockerRegistryEmail); - - Network network = null; - if (networkId != null) { - network = networkService.getNetwork(networkId); - if (network == null) { - throw new InvalidParameterValueException("Unable to find network with given ID"); - } - } - - if (!Strings.isNullOrEmpty(externalLoadBalancerIpAddress)) { - if (!NetUtils.isValidIp4(externalLoadBalancerIpAddress) && !NetUtils.isValidIp6(externalLoadBalancerIpAddress)) { - throw new InvalidParameterValueException("Invalid external load balancer IP address"); - } - if (network == null) { - throw new InvalidParameterValueException(String.format("%s parameter must be specified along with %s parameter", ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS, ApiConstants.NETWORK_ID)); - } - if (Network.GuestType.Shared.equals(network.getGuestType())) { - throw new InvalidParameterValueException(String.format("%s parameter must be specified along with %s type of network", ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS, Network.GuestType.Shared.toString())); - } - } - } - - private Network getKubernetesClusterNetworkIfMissing(final String clusterName, final DataCenter zone, final Account owner, final int masterNodesCount, - final int nodesCount, final String externalLoadBalancerIpAddress, final Long networkId) throws ManagementServerException { - Network network = null; - if (networkId != null) { - network = networkDao.findById(networkId); - if (Network.GuestType.Isolated.equals(network.getGuestType())) { - if (kubernetesClusterDao.listByNetworkId(network.getId()).isEmpty()) { - if (!validateNetwork(network, masterNodesCount + nodesCount)) { - throw new InvalidParameterValueException(String.format("Network ID: %s is not suitable for Kubernetes cluster", network.getUuid())); - } - networkModel.checkNetworkPermissions(owner, network); - } else { - throw new InvalidParameterValueException(String.format("Network ID: %s is already under use by another Kubernetes cluster", network.getUuid())); - } - } else if (Network.GuestType.Shared.equals(network.getGuestType())) { - if (masterNodesCount > 1 && Strings.isNullOrEmpty(externalLoadBalancerIpAddress)) { - throw new InvalidParameterValueException(String.format("Multi-master, HA Kubernetes cluster with %s network ID: %s needs an external load balancer IP address. %s parameter can be used", - network.getGuestType().toString(), network.getUuid(), ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS)); - } - } - } else { // user has not specified network in which cluster VM's to be provisioned, so create a network for Kubernetes cluster - NetworkOfferingVO networkOffering = networkOfferingDao.findByUniqueName(KubernetesClusterNetworkOffering.value()); - - long physicalNetworkId = networkModel.findPhysicalNetworkId(zone.getId(), networkOffering.getTags(), networkOffering.getTrafficType()); - PhysicalNetwork physicalNetwork = physicalNetworkDao.findById(physicalNetworkId); - - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Creating network for account ID: %s from the network offering ID: %s as part of Kubernetes cluster: %s deployment process", owner.getUuid(), networkOffering.getUuid(), clusterName)); - } - - try { - network = networkMgr.createGuestNetwork(networkOffering.getId(), clusterName + "-network", owner.getAccountName() + "-network", - null, null, null, false, null, owner, null, physicalNetwork, zone.getId(), ControlledEntity.ACLType.Account, null, null, null, null, true, null, null); - } catch (ConcurrentOperationException | InsufficientCapacityException | ResourceAllocationException e) { - String msg = String.format("Unable to create network for the Kubernetes cluster: %s", clusterName); - LOGGER.warn(msg, e); - throw new ManagementServerException(msg, e); - } - } - return network; - } - - private void addKubernetesClusterDetails(final KubernetesCluster kubernetesCluster, final Network network, final CreateKubernetesClusterCmd cmd) { - final String externalLoadBalancerIpAddress = cmd.getExternalLoadBalancerIpAddress(); - final String dockerRegistryUserName = cmd.getDockerRegistryUserName(); - final String dockerRegistryPassword = cmd.getDockerRegistryPassword(); - final String dockerRegistryUrl = cmd.getDockerRegistryUrl(); - final String dockerRegistryEmail = cmd.getDockerRegistryEmail(); - final boolean networkCleanup = cmd.getNetworkId() == null; - Transaction.execute(new TransactionCallbackNoReturn() { - @Override - public void doInTransactionWithoutResult(TransactionStatus status) { - List details = new ArrayList<>(); - if (Network.GuestType.Shared.equals(network.getGuestType()) && !Strings.isNullOrEmpty(externalLoadBalancerIpAddress)) { - details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS, externalLoadBalancerIpAddress, true)); - } - if (!Strings.isNullOrEmpty(dockerRegistryUserName)) { - details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), ApiConstants.DOCKER_REGISTRY_USER_NAME, dockerRegistryUserName, true)); - } - if (!Strings.isNullOrEmpty(dockerRegistryPassword)) { - details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), ApiConstants.DOCKER_REGISTRY_PASSWORD, dockerRegistryPassword, false)); - } - if (!Strings.isNullOrEmpty(dockerRegistryUrl)) { - details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), ApiConstants.DOCKER_REGISTRY_URL, dockerRegistryUrl, true)); - } - if (!Strings.isNullOrEmpty(dockerRegistryEmail)) { - details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), ApiConstants.DOCKER_REGISTRY_EMAIL, dockerRegistryEmail, true)); - } - details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), ApiConstants.USERNAME, "admin", true)); - SecureRandom random = new SecureRandom(); - String randomPassword = new BigInteger(130, random).toString(32); - details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), ApiConstants.PASSWORD, randomPassword, false)); - details.add(new KubernetesClusterDetailsVO(kubernetesCluster.getId(), "networkCleanup", String.valueOf(networkCleanup), true)); - kubernetesClusterDetailsDao.saveDetails(details); - } - }); - } - - private void validateKubernetesClusterScaleParameters(ScaleKubernetesClusterCmd cmd) { - final Long kubernetesClusterId = cmd.getId(); - final Long serviceOfferingId = cmd.getServiceOfferingId(); - final Long clusterSize = cmd.getClusterSize(); - if (kubernetesClusterId == null || kubernetesClusterId < 1L) { - throw new InvalidParameterValueException("Invalid Kubernetes cluster ID"); - } - KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); - if (kubernetesCluster == null || kubernetesCluster.getRemoved() != null) { - throw new InvalidParameterValueException("Invalid Kubernetes cluster ID"); - } - final DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); - if (zone == null) { - logAndThrow(Level.WARN, String.format("Unable to find zone for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); - } - - Account caller = CallContext.current().getCallingAccount(); - accountManager.checkAccess(caller, SecurityChecker.AccessType.OperateEntry, false, kubernetesCluster); - - if (serviceOfferingId == null && clusterSize == null) { - throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be scaled, either a new service offering or a new cluster size must be passed", kubernetesCluster.getUuid())); - } - - ServiceOffering serviceOffering = null; - if (serviceOfferingId != null) { - serviceOffering = serviceOfferingDao.findById(serviceOfferingId); - if (serviceOffering == null) { - throw new InvalidParameterValueException("Failed to find service offering ID: " + serviceOfferingId); - } else { - if (serviceOffering.isDynamic()) { - throw new InvalidParameterValueException(String.format("Custom service offerings are not supported for Kubernetes clusters. Kubernetes cluster ID: %s, service offering ID: %s", kubernetesCluster.getUuid(), serviceOffering.getUuid())); - } - if (serviceOffering.getCpu() < 2 || serviceOffering.getRamSize() < 2048) { - throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be scaled with service offering ID: %s, Kubernetes cluster template(CoreOS) needs minimum 2 vCPUs and 2 GB RAM", kubernetesCluster.getUuid(), serviceOffering.getUuid())); - } - } - } - - if (!(kubernetesCluster.getState().equals(KubernetesCluster.State.Created) || - kubernetesCluster.getState().equals(KubernetesCluster.State.Running) || - kubernetesCluster.getState().equals(KubernetesCluster.State.Stopped))) { - throw new PermissionDeniedException(String.format("Kubernetes cluster ID: %s is in %s state", kubernetesCluster.getUuid(), kubernetesCluster.getState().toString())); - } - - if (clusterSize != null) { - if (kubernetesCluster.getState().equals(KubernetesCluster.State.Stopped)) { // Cannot scale stopped cluster currently for cluster size - throw new PermissionDeniedException(String.format("Kubernetes cluster ID: %s is in %s state", kubernetesCluster.getUuid(), kubernetesCluster.getState().toString())); - } - if (clusterSize < 1) { - throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be scaled for size, %d", kubernetesCluster.getUuid(), clusterSize)); - } - } - } - - private void validateKubernetesClusterScaleOfferingParameters(final KubernetesCluster kubernetesCluster, final ServiceOffering existingServiceOffering, final ServiceOffering serviceOffering) { - final long originalNodeCount = kubernetesCluster.getTotalNodeCount(); - List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); - if (vmList == null || vmList.isEmpty() || vmList.size() < originalNodeCount) { - logAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster ID: %s failed, it is in unstable state as not enough existing VM instances found!", kubernetesCluster.getUuid())); - } else { - for (KubernetesClusterVmMapVO vmMapVO : vmList) { - VMInstanceVO vmInstance = vmInstanceDao.findById(vmMapVO.getVmId()); - if (vmInstance != null && vmInstance.getState().equals(VirtualMachine.State.Running) && - vmInstance.getHypervisorType() != Hypervisor.HypervisorType.XenServer && - vmInstance.getHypervisorType() != Hypervisor.HypervisorType.VMware && - vmInstance.getHypervisorType() != Hypervisor.HypervisorType.Simulator) { - logAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster ID: %s failed, scaling Kubernetes cluster with running VMs on hypervisor %s is not supported!", kubernetesCluster.getUuid(), vmInstance.getHypervisorType())); - } - } - } - if (serviceOffering.getRamSize() < existingServiceOffering.getRamSize() || - serviceOffering.getCpu() * serviceOffering.getSpeed() < existingServiceOffering.getCpu() * existingServiceOffering.getSpeed()) { - logAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster ID: %s failed, service offering for the Kubernetes cluster cannot be scaled down!", kubernetesCluster.getUuid())); - } - } - - private void scaleKubernetesClusterOffering(final long kubernetesClusterId, final ServiceOffering serviceOffering, final Long clusterSize) { - KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); - - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.ScaleUpRequested); - - final long size = (clusterSize == null ? kubernetesCluster.getTotalNodeCount() : kubernetesCluster.getMasterNodeCount() + clusterSize); - final long cores = serviceOffering.getCpu() * size; - final long memory = serviceOffering.getRamSize() * size; - KubernetesClusterVO updatedKubernetesCluster = updateKubernetesClusterEntry(kubernetesCluster.getId(), size, cores, memory, serviceOffering.getId()); - if (updatedKubernetesCluster == null) { - logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, unable to update Kubernetes cluster!", updatedKubernetesCluster.getUuid()), kubernetesClusterId, KubernetesCluster.Event.OperationFailed); - } - kubernetesCluster = updatedKubernetesCluster; - List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); - final long tobeScaledVMCount = Math.min(vmList.size(), size); - for (long i = 0; i < tobeScaledVMCount; i++) { - KubernetesClusterVmMapVO vmMapVO = vmList.get((int) i); - UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId()); - boolean result = false; - try { - result = userVmManager.upgradeVirtualMachine(userVM.getId(), serviceOffering.getId(), new HashMap()); - } catch (ResourceUnavailableException | ManagementServerException | ConcurrentOperationException | VirtualMachineMigrationException e) { - logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, unable to scale cluster VM ID: %s", kubernetesCluster.getUuid(), userVM.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); - } - if (!result) { - logTransitStateAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster ID: %s failed, unable to scale cluster VM ID: %s", kubernetesCluster.getUuid(), userVM.getUuid()),kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } - } - } - - private void validateKubernetesClusterScaleSizeParameters(KubernetesClusterVO kubernetesCluster, final long originalClusterSize, final long clusterSize, final KubernetesCluster.State clusterState) { - Network network = networkDao.findById(kubernetesCluster.getNetworkId()); - if (network == null) { - String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, cluster network not found", kubernetesCluster.getUuid()); - if (KubernetesCluster.State.Scaling.equals(kubernetesCluster.getState())) { - logTransitStateAndThrow(Level.WARN, msg, kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } else { - logAndThrow(Level.WARN, msg); - } - } - // Check capacity and transition state - final long newVmRequiredCount = clusterSize - originalClusterSize; - final ServiceOffering clusterServiceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); - if (clusterServiceOffering == null) { - String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, cluster service offering not found", kubernetesCluster.getUuid()); - if (KubernetesCluster.State.Scaling.equals(kubernetesCluster.getState())) { - logTransitStateAndThrow(Level.WARN, msg, kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } else { - logAndThrow(Level.WARN, msg); - } - } - if (newVmRequiredCount > 0) { - final DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); - try { - if (clusterState.equals(KubernetesCluster.State.Running)) { - plan(newVmRequiredCount, zone, clusterServiceOffering); - } else { - plan(kubernetesCluster.getTotalNodeCount() + newVmRequiredCount, zone, clusterServiceOffering); - } - } catch (InsufficientCapacityException e) { - String msg = String.format("Scaling failed for Kubernetes cluster ID: %s in zone ID: %s, insufficient capacity", kubernetesCluster.getUuid(), zone.getUuid()); - if (KubernetesCluster.State.Scaling.equals(kubernetesCluster.getState())) { - logTransitStateAndThrow(Level.WARN, msg, kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } else { - logAndThrow(Level.WARN, msg); - } - } - } - } - - private void scaleKubernetesClusterSize(final long kubernetesClusterId, final long originalClusterSize, final long clusterSize) throws ResourceUnavailableException { - KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); - final Network network = networkDao.findById(kubernetesCluster.getNetworkId()); - final long newVmRequiredCount = clusterSize - originalClusterSize; - List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); - vmList.sort(new Comparator() { - @Override - public int compare(KubernetesClusterVmMapVO kubernetesClusterVmMapVO, KubernetesClusterVmMapVO t1) { - return (int)((kubernetesClusterVmMapVO.getId() - t1.getId())/Math.abs(kubernetesClusterVmMapVO.getId() - t1.getId())); - } - }); - if (CollectionUtils.isEmpty(vmList) || vmList.size() - 1 < originalClusterSize) { - logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, t is in unstable state as not enough existing VM instances found", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } - - Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(kubernetesCluster); - String publicIpAddress = publicIpSshPort.first(); - int sshPort = publicIpSshPort.second(); - if (Strings.isNullOrEmpty(publicIpAddress)) { - logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, unable to retrieve associated public IP", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } - Account account = accountDao.findById(kubernetesCluster.getAccountId()); - if (newVmRequiredCount < 0) { // downscale - int i = vmList.size() - 1; - List removedVmIds = new ArrayList<>(); - while (i > kubernetesCluster.getMasterNodeCount() && vmList.size() > kubernetesCluster.getTotalNodeCount()) { // Reverse order as first VM will be k8s master - KubernetesClusterVmMapVO vmMapVO = vmList.get(i); - UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId()); - - // Gracefully remove-delete k8s node - if (!removeKubernetesClusterNode(kubernetesCluster, publicIpAddress, sshPort, userVM, 3, 30000)) { - logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, failed to remove Kubernetes node: %s running on VM ID: %s", kubernetesCluster.getUuid(), userVM.getHostName(), userVM.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } - - // For removing port-forwarding network rules - removedVmIds.add(userVM.getId()); - - // Expunge VM - UserVm vm = userVmService.destroyVm(userVM.getId(), true); - if (!VirtualMachine.State.Expunging.equals(vm.getState())) { - logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, VM '%s' is now in state '%s'." - , kubernetesCluster.getUuid() - , vm.getInstanceName() - , vm.getState().toString()), - kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } - vm = userVmService.expungeVm(userVM.getId()); - if (!VirtualMachine.State.Expunging.equals(vm.getState())) { - logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, VM '%s' is now in state '%s'." - , kubernetesCluster.getUuid() - , vm.getInstanceName() - , vm.getState().toString()), - kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } - - // Expunge cluster VMMapVO - kubernetesClusterVmMapDao.expunge(vmMapVO.getId()); - - i--; - } - - // Scale network rules to update firewall rule - try { - scaleKubernetesClusterNetworkRules(kubernetesCluster, network, account, null, removedVmIds); - } catch (ManagementServerException e) { - logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, unable to update network rules", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); - } - } else { // upscale, same node count handled above - List clusterVMIds = new ArrayList<>(); - - // Create new node VMs - for (int i = (int) originalClusterSize + 1; i <= clusterSize; i++) { - UserVm vm = null; - try { - vm = createKubernetesNode(kubernetesCluster, publicIpAddress, i); - addKubernetesClusterVm(kubernetesCluster.getId(), vm .getId()); - startKubernetesVM(vm, kubernetesCluster); - clusterVMIds.add(vm.getId()); - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Provisioned a node VM in to the Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); - } - } catch (ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { - logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, unable to provision node VM in the cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); - } - } - - // Scale network rules to update firewall rule and add port-forwarding rules - try { - scaleKubernetesClusterNetworkRules(kubernetesCluster, network, account, clusterVMIds, null); - } catch (ManagementServerException e) { - logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, unable to update network rules", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); - } - - // Attach binaries ISO to new VMs - attachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); - - // Check if new nodes are added in k8s cluster - boolean readyNodesCountValid = validateKubernetesClusterReadyNodesCount(kubernetesCluster, publicIpAddress, sshPort, 30, 30000); - - // Detach binaries ISO from new VMs - detachIsoKubernetesVMs(kubernetesCluster, clusterVMIds); - - // Throw exception if nodes count for k8s cluster timed out - if (!readyNodesCountValid) { // Scaling failed - logTransitStateAndThrow(Level.ERROR, String.format("Scaling unsuccessful for Kubernetes cluster ID: %s as it does not have desired number of nodes in ready state", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } - } - } - - private void validateKubernetesClusterUpgradeParameters(UpgradeKubernetesClusterCmd cmd) { - // Validate parameters - final Long kubernetesClusterId = cmd.getId(); - final Long upgradeVersionId = cmd.getKubernetesVersionId(); - if (kubernetesClusterId == null || kubernetesClusterId < 1L) { - throw new InvalidParameterValueException("Invalid Kubernetes cluster ID"); - } - if (upgradeVersionId == null || upgradeVersionId < 1L) { - throw new InvalidParameterValueException("Invalid Kubernetes version ID"); - } - KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); - if (kubernetesCluster == null || kubernetesCluster.getRemoved() != null) { - throw new InvalidParameterValueException("Invalid Kubernetes cluster ID"); - } - if (!KubernetesCluster.State.Running.equals(kubernetesCluster.getState())) { - throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s is not in running state", kubernetesCluster.getUuid())); - } - KubernetesSupportedVersionVO upgradeVersion = kubernetesSupportedVersionDao.findById(upgradeVersionId); - if (upgradeVersion == null || upgradeVersion.getRemoved() != null) { - throw new InvalidParameterValueException("Invalid Kubernetes version ID"); - } - KubernetesSupportedVersionVO clusterVersion = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); - if (clusterVersion == null || clusterVersion.getRemoved() != null) { - throw new InvalidParameterValueException(String.format("Invalid Kubernetes version associated with cluster ID: %s", - kubernetesCluster.getUuid())); - } - // Check upgradeVersion is either patch upgrade or immediate minor upgrade - try { - KubernetesVersionManagerImpl.canUpgradeKubernetesVersion(clusterVersion.getSemanticVersion(), upgradeVersion.getSemanticVersion()); - } catch (IllegalArgumentException e) { - throw new InvalidParameterValueException(e.getMessage()); - } - - TemplateJoinVO iso = templateJoinDao.findById(upgradeVersion.getIsoId()); - if (iso == null) { - throw new InvalidParameterValueException(String.format("Invalid ISO associated with version ID: %s", upgradeVersion.getUuid())); - } - if (!ObjectInDataStoreStateMachine.State.Ready.equals(iso.getState())) { - throw new InvalidParameterValueException(String.format("ISO associated with version ID: %s is not in Ready state", upgradeVersion.getUuid())); - } - } - - private KubernetesClusterVO updateKubernetesClusterEntry(final long kubernetesClusterId, final long clusterSize, - final long cores, final long memory, final Long serviceOfferingId) { - return Transaction.execute(new TransactionCallback() { - @Override - public KubernetesClusterVO doInTransaction(TransactionStatus status) { - KubernetesClusterVO updatedCluster = kubernetesClusterDao.createForUpdate(kubernetesClusterId); - updatedCluster.setNodeCount(clusterSize); - updatedCluster.setCores(cores); - updatedCluster.setMemory(memory); - if (serviceOfferingId != null) { - updatedCluster.setServiceOfferingId(serviceOfferingId); - } - kubernetesClusterDao.persist(updatedCluster); - return updatedCluster; - } - }); - } - - private void upgradeKubernetesClusterNodes(final KubernetesCluster kubernetesCluster, final List vmIds, - final KubernetesSupportedVersion upgradeVersion, final String publicIpAddress, - final int sshPort, final File upgradeScriptFile) { - Pair result = null; - File pkFile = getManagementServerSshPublicKeyFile(); - for (int i = 0; i < vmIds.size(); ++i) { - UserVm vm = userVmDao.findById(vmIds.get(i)); - String hostName = vm.getHostName(); - if (!Strings.isNullOrEmpty(hostName)) { - hostName = hostName.toLowerCase(); - } - result = null; - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Upgrading node on VM ID: %s in Kubernetes cluster ID: %s with Kubernetes version(%s) ID: %s", - vm.getUuid(), kubernetesCluster.getUuid(), upgradeVersion.getSemanticVersion(), upgradeVersion.getUuid())); - } - try { - result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, pkFile, null, - String.format("sudo kubectl drain %s --ignore-daemonsets --delete-local-data", hostName), - 10000, 10000, 60000); - } catch (Exception e) { - logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to drain Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()), kubernetesCluster, vmIds, KubernetesCluster.Event.OperationFailed, e); - } - if (!result.first()) { - logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to drain Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()), kubernetesCluster, vmIds, KubernetesCluster.Event.OperationFailed, null); - } - try { - int nodeSshPort = sshPort == 22 ? sshPort : sshPort + i; - String nodeAddress = (i > 0 && sshPort == 22) ? vm.getPrivateIpAddress() : publicIpAddress; - SshHelper.scpTo(nodeAddress, nodeSshPort, CLUSTER_NODE_VM_USER, pkFile, null, - "~/", upgradeScriptFile.getAbsolutePath(), "0755"); - String cmdStr = String.format("sudo ./%s %s %s %s", upgradeScriptFile.getName(), - upgradeVersion.getSemanticVersion(), i == 0 ? "true" : "false", - KubernetesVersionManagerImpl.compareSemanticVersions(upgradeVersion.getSemanticVersion(), "1.15.0") < 0 ? "true" : "false"); - result = SshHelper.sshExecute(publicIpAddress, nodeSshPort, CLUSTER_NODE_VM_USER, pkFile, null, - cmdStr, - 10000, 10000, 10 * 60 * 1000); - } catch (Exception e) { - logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to upgrade Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()), kubernetesCluster, vmIds, KubernetesCluster.Event.OperationFailed, e); - } - if (!result.first()) { - logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to upgrade Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()), kubernetesCluster, vmIds, KubernetesCluster.Event.OperationFailed, null); - - } - if (!uncordonKubernetesClusterNode(kubernetesCluster, publicIpAddress, sshPort, vm, 5, 30000)) { - logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to uncordon Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()), kubernetesCluster, vmIds, KubernetesCluster.Event.OperationFailed, null); - } - if (i == 0) { // Wait for master to get in Ready state - if (!isKubernetesClusterNodeReady(kubernetesCluster, publicIpAddress, sshPort, hostName, 5, 20000)) { - logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to get master Kubernetes node on VM ID: %s in ready state", kubernetesCluster.getUuid(), vm.getUuid()), kubernetesCluster, vmIds, KubernetesCluster.Event.OperationFailed, null); - } - } - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Successfully upgraded node on VM ID: %s in Kubernetes cluster ID: %s with Kubernetes version(%s) ID: %s", - vm.getUuid(), kubernetesCluster.getUuid(), upgradeVersion.getSemanticVersion(), upgradeVersion.getUuid())); - } - } - } - - protected boolean stateTransitTo(long kubernetesClusterId, KubernetesCluster.Event e) { - KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); - try { - return _stateMachine.transitTo(kubernetesCluster, e, null, kubernetesClusterDao); - } catch (NoTransitionException nte) { - LOGGER.warn(String.format("Failed to transition state of the Kubernetes cluster ID: %s in state %s on event %s", kubernetesCluster.getUuid(), kubernetesCluster.getState().toString(), e.toString()), nte); - return false; - } - } - - @Override - public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) - throws InsufficientCapacityException, ManagementServerException { - if (!KubernetesServiceEnabled.value()) { - logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); - } - - validateKubernetesClusterCreatePrameters(cmd); - - final DataCenter zone = dataCenterDao.findById(cmd.getZoneId()); - final long masterNodeCount = cmd.getMasterNodes(); - final long clusterSize = cmd.getClusterSize(); - final long totalNodeCount = masterNodeCount + clusterSize; - final ServiceOffering serviceOffering = serviceOfferingDao.findById(cmd.getServiceOfferingId()); - final Account owner = accountService.getActiveAccountById(cmd.getEntityOwnerId()); - final KubernetesSupportedVersion clusterKubernetesVersion = kubernetesSupportedVersionDao.findById(cmd.getKubernetesVersionId()); - - plan(totalNodeCount, zone, serviceOffering); - - final Network defaultNetwork = getKubernetesClusterNetworkIfMissing(cmd.getName(), zone, owner, (int)masterNodeCount, (int)clusterSize, cmd.getExternalLoadBalancerIpAddress(), cmd.getNetworkId()); - final VMTemplateVO finalTemplate = templateDao.findByTemplateName(KubernetesClusterTemplateName.value());; - final long cores = serviceOffering.getCpu() * (masterNodeCount + clusterSize); - final long memory = serviceOffering.getRamSize() * (masterNodeCount + clusterSize); - - final KubernetesClusterVO cluster = Transaction.execute(new TransactionCallback() { - @Override - public KubernetesClusterVO doInTransaction(TransactionStatus status) { - KubernetesClusterVO newCluster = new KubernetesClusterVO(cmd.getName(), cmd.getDisplayName(), zone.getId(), clusterKubernetesVersion.getId(), - serviceOffering.getId(), finalTemplate.getId(), defaultNetwork.getId(), owner.getDomainId(), - owner.getAccountId(), masterNodeCount, clusterSize, KubernetesCluster.State.Created, cmd.getSSHKeyPairName(), cores, memory, cmd.getNodeRootDiskSize(), ""); - kubernetesClusterDao.persist(newCluster); - return newCluster; - } - }); - - addKubernetesClusterDetails(cluster, defaultNetwork, cmd); - - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Kubernetes cluster name: %s and ID: %s has been created", cluster.getName(), cluster.getUuid())); - } - return cluster; - } - - - // Start operation can be performed at two diffrent life stagevs of Kubernetes cluster. First when a freshly created cluster - // in which case there are no resources provisisioned for the Kubernetes cluster. So during start all the resources - // are provisioned from scratch. Second kind of start, happens on Stopped Kubernetes cluster, in which all resources - // are provisioned (like volumes, nics, networks etc). It just that VM's are not in running state. So just - // start the VM's (which can possibly implicitly start the network also). - @Override - public boolean startKubernetesCluster(long kubernetesClusterId, boolean onCreate) throws ManagementServerException, - ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException { - if (!KubernetesServiceEnabled.value()) { - logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); - } - if (onCreate) { - // Start for Kubernetes cluster in 'Created' state - return startKubernetesClusterOnCreate(kubernetesClusterId); - } else { - // Start for Kubernetes cluster in 'Stopped' state. Resources are already provisioned, just need to be started - return startStoppedKubernetesCluster(kubernetesClusterId); - } - } - - @Override - public boolean stopKubernetesCluster(long kubernetesClusterId) throws ManagementServerException { - if (!KubernetesServiceEnabled.value()) { - logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); - } - final KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); - if (kubernetesCluster == null) { - throw new InvalidParameterValueException("Failed to find Kubernetes cluster with given ID"); - } - - if (kubernetesCluster.getRemoved() != null) { - throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s is already deleted", kubernetesCluster.getUuid())); - } - - if (kubernetesCluster.getState().equals(KubernetesCluster.State.Stopped)) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Kubernetes cluster ID: %s is already stopped", kubernetesCluster.getUuid())); - } - return true; - } - - if (kubernetesCluster.getState().equals(KubernetesCluster.State.Stopping)) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Kubernetes cluster ID: %s is getting stopped", kubernetesCluster.getUuid())); - } - return true; - } - - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Stopping Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); - } - - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.StopRequested); - - for (final KubernetesClusterVmMapVO vmMapVO : kubernetesClusterVmMapDao.listByClusterId(kubernetesClusterId)) { - final UserVmVO vm = userVmDao.findById(vmMapVO.getVmId()); - try { - if (vm == null) { - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - throw new ManagementServerException(String.format("Failed to find all VMs in Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); - } - stopClusterVM(vmMapVO); - } catch (CloudRuntimeException ex) { - LOGGER.warn(String.format("Failed to stop VM in Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), ex); - // dont bail out here. proceed further to stop the reset of the VM's - } - } - - for (final KubernetesClusterVmMapVO vmMapVO : kubernetesClusterVmMapDao.listByClusterId(kubernetesClusterId)) { - final UserVmVO vm = userVmDao.findById(vmMapVO.getVmId()); - if (vm == null || !vm.getState().equals(VirtualMachine.State.Stopped)) { - logTransitStateAndThrow(Level.ERROR, String.format("Failed to stop all VMs in Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } - } - - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); - return true; - } - - @Override - public boolean deleteKubernetesCluster(Long kubernetesClusterId) throws ManagementServerException { - if (!KubernetesServiceEnabled.value()) { - logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); - } - KubernetesClusterVO cluster = kubernetesClusterDao.findById(kubernetesClusterId); - if (cluster == null) { - throw new InvalidParameterValueException("Invalid cluster id specified"); - } - CallContext ctx = CallContext.current(); - Account caller = ctx.getCallingAccount(); - accountManager.checkAccess(caller, SecurityChecker.AccessType.OperateEntry, false, cluster); - return cleanupKubernetesClusterResources(kubernetesClusterId); - } - - @Override - public ListResponse listKubernetesClusters(ListKubernetesClustersCmd cmd) { - if (!KubernetesServiceEnabled.value()) { - logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); - } - final CallContext ctx = CallContext.current(); - final Account caller = ctx.getCallingAccount(); - final Long clusterId = cmd.getId(); - final String state = cmd.getState(); - final String name = cmd.getName(); - - List responsesList = new ArrayList(); - if (state != null && !state.isEmpty()) { - if (!KubernetesCluster.State.Running.toString().equals(state) && - !KubernetesCluster.State.Stopped.toString().equals(state) && - !KubernetesCluster.State.Destroyed.toString().equals(state)) { - throw new InvalidParameterValueException("Invalid value for Kubernetes cluster state specified"); - } - } - if (clusterId != null) { - KubernetesClusterVO cluster = kubernetesClusterDao.findById(clusterId); - if (cluster == null) { - throw new InvalidParameterValueException("Invalid Kubernetes cluster ID specified"); - } - accountManager.checkAccess(caller, SecurityChecker.AccessType.ListEntry, false, cluster); - responsesList.add(createKubernetesClusterResponse(clusterId)); - } else { - SearchCriteria sc = kubernetesClusterDao.createSearchCriteria(); - Filter searchFilter = new Filter(KubernetesClusterVO.class, "id", true, cmd.getStartIndex(), cmd.getPageSizeVal()); - if (state != null && !state.isEmpty()) { - sc.addAnd("state", SearchCriteria.Op.EQ, state); - } - if (accountManager.isNormalUser(caller.getId())) { - sc.addAnd("accountId", SearchCriteria.Op.EQ, caller.getAccountId()); - } else if (accountManager.isDomainAdmin(caller.getId())) { - sc.addAnd("domainId", SearchCriteria.Op.EQ, caller.getDomainId()); - } - if (name != null && !name.isEmpty()) { - sc.addAnd("name", SearchCriteria.Op.LIKE, name); - } - List kubernetesClusters = kubernetesClusterDao.search(sc, searchFilter); - for (KubernetesClusterVO cluster : kubernetesClusters) { - KubernetesClusterResponse clusterResponse = createKubernetesClusterResponse(cluster.getId()); - responsesList.add(clusterResponse); - } - } - ListResponse response = new ListResponse(); - response.setResponses(responsesList); - return response; - } - - public KubernetesClusterConfigResponse getKubernetesClusterConfig(GetKubernetesClusterConfigCmd cmd) { - if (!KubernetesServiceEnabled.value()) { - logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); - } - final Long clusterId = cmd.getId(); - KubernetesCluster kubernetesCluster = kubernetesClusterDao.findById(clusterId); - if (kubernetesCluster == null) { - throw new InvalidParameterValueException("Invalid Kubernetes cluster ID specified"); - } - KubernetesClusterConfigResponse response = new KubernetesClusterConfigResponse(); - response.setId(kubernetesCluster.getUuid()); - response.setName(kubernetesCluster.getName()); - String configData = ""; - KubernetesClusterDetailsVO clusterDetailsVO = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "kubeConfigData"); - if (clusterDetailsVO != null && !Strings.isNullOrEmpty(clusterDetailsVO.getValue())) { - configData = new String(Base64.decodeBase64(clusterDetailsVO.getValue())); - } - response.setConfigData(configData); - response.setObjectName("clusterconfig"); - return response; - } - - @Override - public boolean scaleKubernetesCluster(ScaleKubernetesClusterCmd cmd) throws ResourceUnavailableException { - if (!KubernetesServiceEnabled.value()) { - logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); - } - - validateKubernetesClusterScaleParameters(cmd); - - KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(cmd.getId()); - final ServiceOffering serviceOffering = serviceOfferingDao.findById(cmd.getServiceOfferingId()); - final Long clusterSize = cmd.getClusterSize(); - - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Scaling Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); - } - - final KubernetesCluster.State clusterState = kubernetesCluster.getState(); - final long originalClusterSize = kubernetesCluster.getNodeCount(); - - final ServiceOffering existingServiceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); - if (existingServiceOffering == null) { - logAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, service offering for the Kubernetes cluster not found!", kubernetesCluster.getUuid())); - } - final boolean serviceOfferingScalingNeeded = serviceOffering != null && serviceOffering.getId() != existingServiceOffering.getId(); - final boolean clusterSizeScalingNeeded = clusterSize != null && clusterSize != originalClusterSize; - - if (serviceOfferingScalingNeeded) { - validateKubernetesClusterScaleOfferingParameters(kubernetesCluster, existingServiceOffering, serviceOffering); - scaleKubernetesClusterOffering(kubernetesCluster.getId(), serviceOffering, clusterSize); - } - - if (clusterSizeScalingNeeded) { - validateKubernetesClusterScaleSizeParameters(kubernetesCluster, originalClusterSize, clusterSize, clusterState); - final long newVmRequiredCount = clusterSize - originalClusterSize; - final ServiceOffering clusterServiceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); - if (newVmRequiredCount > 0) { - if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.ScaleUpRequested); - } - } else { - if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.ScaleDownRequested); - } - } - - if (!serviceOfferingScalingNeeded) { // Else already updated - final long cores = clusterServiceOffering.getCpu() * (kubernetesCluster.getMasterNodeCount() + clusterSize); - final long memory = clusterServiceOffering.getRamSize() * (kubernetesCluster.getMasterNodeCount() + clusterSize); - - kubernetesCluster = updateKubernetesClusterEntry(kubernetesCluster.getId(), clusterSize, cores, memory, null); - if (kubernetesCluster == null) { - logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, unable to update cluster", kubernetesCluster.getUuid()), cmd.getId(), KubernetesCluster.Event.OperationFailed); - } - } - - // Perform size scaling - scaleKubernetesClusterSize(cmd.getId(), originalClusterSize, clusterSize); - } - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); - return true; - } - - @Override - public boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws ManagementServerException { - if (!KubernetesServiceEnabled.value()) { - logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); - } - validateKubernetesClusterUpgradeParameters(cmd); - KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(cmd.getId()); - final KubernetesSupportedVersion upgradeVersion = kubernetesSupportedVersionDao.findById(cmd.getKubernetesVersionId()); - - // Get public IP - Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(kubernetesCluster); - String publicIpAddress = publicIpSshPort.first(); - int sshPort = publicIpSshPort.second(); - if (Strings.isNullOrEmpty(publicIpAddress)) { - logAndThrow(Level.ERROR, String.format("Upgrade failed for Kubernetes cluster ID: %s, unable to retrieve associated public IP", kubernetesCluster.getUuid())); - } - - kubernetesCluster.setKubernetesVersionId(upgradeVersion.getId()); - List clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); - if (CollectionUtils.isEmpty(clusterVMs)) { - String msg = String.format("Upgrade failed for Kubernetes cluster ID: %s, unable to retrieve VMs for cluster", kubernetesCluster.getUuid()); - throw new ManagementServerException(msg); - } - List vmIds = new ArrayList<>(); - for (KubernetesClusterVmMapVO vmMap : clusterVMs) { - vmIds.add(vmMap.getVmId()); - } - Collections.sort(vmIds); - - File upgradeScriptFile = null; - try { - String upgradeScriptData = readResourceFile("/script/upgrade-kubernetes.sh"); - upgradeScriptFile = File.createTempFile("upgrade-kuberntes", ".sh"); - BufferedWriter upgradeScriptFileWriter = new BufferedWriter(new FileWriter(upgradeScriptFile)); - upgradeScriptFileWriter.write(upgradeScriptData); - upgradeScriptFileWriter.close(); - } catch (IOException e) { - logAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to prepare upgrade script", kubernetesCluster.getUuid()), e); - } - - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.UpgradeRequested); - - // Attach ISO - attachIsoKubernetesVMs(kubernetesCluster, vmIds); - - // Upgrade nodes - upgradeKubernetesClusterNodes(kubernetesCluster, vmIds, upgradeVersion, publicIpAddress, sshPort, upgradeScriptFile); - - // Detach ISO - detachIsoKubernetesVMs(kubernetesCluster, vmIds); - - boolean updated = kubernetesClusterDao.update(kubernetesCluster.getId(), kubernetesCluster); - if (!updated) { - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } else { - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); - } - return updated; - } - - @Override - public List> getCommands() { - List> cmdList = new ArrayList>(); - if (!KubernetesServiceEnabled.value()) { - return cmdList; - } - cmdList.add(CreateKubernetesClusterCmd.class); - cmdList.add(StartKubernetesClusterCmd.class); - cmdList.add(StopKubernetesClusterCmd.class); - cmdList.add(DeleteKubernetesClusterCmd.class); - cmdList.add(ListKubernetesClustersCmd.class); - cmdList.add(GetKubernetesClusterConfigCmd.class); - cmdList.add(ScaleKubernetesClusterCmd.class); - cmdList.add(UpgradeKubernetesClusterCmd.class); - return cmdList; - } - - @Override - public KubernetesCluster findById(final Long id) { - return kubernetesClusterDao.findById(id); - } - - // Garbage collector periodically run through the Kubernetes clusters marked for GC. For each Kubernetes cluster - // marked for GC, attempt is made to destroy cluster. - public class KubernetesClusterGarbageCollector extends ManagedContextRunnable { - @Override - protected void runInContext() { - GlobalLock gcLock = GlobalLock.getInternLock("KubernetesCluster.GC.Lock"); - try { - if (gcLock.lock(3)) { - try { - reallyRun(); - } finally { - gcLock.unlock(); - } - } - } finally { - gcLock.releaseRef(); - } - } - - public void reallyRun() { - try { - List kubernetesClusters = kubernetesClusterDao.findKubernetesClustersToGarbageCollect(); - for (KubernetesCluster kubernetesCluster : kubernetesClusters) { - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Running Kubernetes cluster garbage collector on Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); - } - try { - if (cleanupKubernetesClusterResources(kubernetesCluster.getId())) { - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Garbage collection complete for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); - } - } else { - LOGGER.warn(String.format("Garbage collection failed for Kubernetes cluster ID: %s, it will be attempted to garbage collected in next run", kubernetesCluster.getUuid())); - } - } catch (ManagementServerException e) { - LOGGER.warn(String.format("Failed to destroy Kubernetes cluster ID: %s during GC", kubernetesCluster.getUuid()), e); - // proceed further with rest of the Kubernetes cluster garbage collection - } - } - } catch (Exception e) { - LOGGER.warn("Caught exception while running Kubernetes cluster gc: ", e); - } - } - } - - /* Kubernetes cluster scanner checks if the Kubernetes cluster is in desired state. If it detects Kubernetes cluster - is not in desired state, it will trigger an event and marks the Kubernetes cluster to be 'Alert' state. For e.g a - Kubernetes cluster in 'Running' state should mean all the cluster of node VM's in the custer should be running and - number of the node VM's should be of cluster size, and the master node VM's is running. It is possible due to - out of band changes by user or hosts going down, we may end up one or more VM's in stopped state. in which case - scanner detects these changes and marks the cluster in 'Alert' state. Similarly cluster in 'Stopped' state means - all the cluster VM's are in stopped state any mismatch in states should get picked up by Kubernetes cluster and - mark the Kubernetes cluster to be 'Alert' state. Through recovery API, or reconciliation clusters in 'Alert' will - be brought back to known good state or desired state. - */ - public class KubernetesClusterStatusScanner extends ManagedContextRunnable { - private boolean firstRun = true; - @Override - protected void runInContext() { - GlobalLock gcLock = GlobalLock.getInternLock("KubernetesCluster.State.Scanner.Lock"); - try { - if (gcLock.lock(3)) { - try { - reallyRun(); - } finally { - gcLock.unlock(); - } - } - } finally { - gcLock.releaseRef(); - } - } - - public void reallyRun() { - try { - // run through Kubernetes clusters in 'Running' state and ensure all the VM's are Running in the cluster - List runningKubernetesClusters = kubernetesClusterDao.findKubernetesClustersInState(KubernetesCluster.State.Running); - for (KubernetesCluster kubernetesCluster : runningKubernetesClusters) { - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Running Kubernetes cluster state scanner on Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); - } - try { - if (!isClusterVMsInDesiredState(kubernetesCluster, VirtualMachine.State.Running)) { - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.FaultsDetected); - } - } catch (Exception e) { - LOGGER.warn(String.format("Failed to run Kubernetes cluster Running state scanner on Kubernetes cluster ID: %s status scanner", kubernetesCluster.getUuid()), e); - } - } - - // run through Kubernetes clusters in 'Stopped' state and ensure all the VM's are Stopped in the cluster - List stoppedKubernetesClusters = kubernetesClusterDao.findKubernetesClustersInState(KubernetesCluster.State.Stopped); - for (KubernetesCluster kubernetesCluster : stoppedKubernetesClusters) { - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Running Kubernetes cluster state scanner on Kubernetes cluster ID: %s for state: %s", kubernetesCluster.getUuid(), KubernetesCluster.State.Stopped.toString())); - } - try { - if (!isClusterVMsInDesiredState(kubernetesCluster, VirtualMachine.State.Stopped)) { - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.FaultsDetected); - } - } catch (Exception e) { - LOGGER.warn(String.format("Failed to run Kubernetes cluster Stopped state scanner on Kubernetes cluster ID: %s status scanner", kubernetesCluster.getUuid()), e); - } - } - - // run through Kubernetes clusters in 'Alert' state and reconcile state as 'Running' if the VM's are running - List alertKubernetesClusters = kubernetesClusterDao.findKubernetesClustersInState(KubernetesCluster.State.Alert); - for (KubernetesClusterVO kubernetesCluster : alertKubernetesClusters) { - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Running Kubernetes cluster state scanner on Kubernetes cluster ID: %s for state: %s", kubernetesCluster.getUuid(), KubernetesCluster.State.Alert.toString())); - } - try { - if (isClusterVMsInDesiredState(kubernetesCluster, VirtualMachine.State.Running) && - kubernetesCluster.getTotalNodeCount() == getKubernetesClusterReadyNodesCount(kubernetesCluster)) { - Pair sshIpPort = getKubernetesClusterServerIpSshPort(kubernetesCluster); - if (Strings.isNullOrEmpty(sshIpPort.first())) { - continue; - } - if (!isKubernetesClusterServerRunning(kubernetesCluster, sshIpPort.first(), 1, 0)) { - continue; - } - if (Strings.isNullOrEmpty(kubernetesCluster.getEndpoint())) { - kubernetesCluster.setEndpoint(String.format("https://%s:%d/", sshIpPort.first(), CLUSTER_API_PORT)); - } - KubernetesClusterDetailsVO kubeConfigDetail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "kubeConfigData"); - if (kubeConfigDetail == null || Strings.isNullOrEmpty(kubeConfigDetail.getValue())) { - String kubeConfig = getKubernetesClusterConfig(kubernetesCluster, sshIpPort.first(), sshIpPort.second(), 1); - if (Strings.isNullOrEmpty(kubeConfig)) { - continue; - } - kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "kubeConfigData", Base64.encodeBase64String(kubeConfig.getBytes(Charset.forName("UTF-8"))), false); - } - KubernetesClusterDetailsVO dashboardServiceRunningDetail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "dashboardServiceRunning"); - if (kubeConfigDetail == null || !Boolean.parseBoolean(dashboardServiceRunningDetail.getValue())) { - boolean dashboardServiceRunning = isKubernetesClusterDashboardServiceRunning(kubernetesCluster, sshIpPort.first(), sshIpPort.second(), 1, 0); - if (!dashboardServiceRunning) { - continue; - } - kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "dashboardServiceRunning", String.valueOf(dashboardServiceRunning), false); - } - // mark the cluster to be running - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.RecoveryRequested); - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); - } - } catch (Exception e) { - LOGGER.warn(String.format("Failed to run Kubernetes cluster Alert state scanner on Kubernetes cluster ID: %s status scanner", kubernetesCluster.getUuid()), e); - } - } - - - if (firstRun) { - // run through Kubernetes clusters in 'Starting' state and reconcile state as 'Alert' or 'Error' if the VM's are running - List startingKubernetesClusters = kubernetesClusterDao.findKubernetesClustersInState(KubernetesCluster.State.Starting); - for (KubernetesCluster kubernetesCluster : startingKubernetesClusters) { - if ((new Date()).getTime() - kubernetesCluster.getCreated().getTime() < 10*60*1000) { - continue; - } - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Running Kubernetes cluster state scanner on Kubernetes cluster ID: %s for state: %s", kubernetesCluster.getUuid(), KubernetesCluster.State.Starting.toString())); - } - try { - if (isClusterVMsInDesiredState(kubernetesCluster, VirtualMachine.State.Running)) { - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.FaultsDetected); - } else { - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } - } catch (Exception e) { - LOGGER.warn(String.format("Failed to run Kubernetes cluster Starting state scanner on Kubernetes cluster ID: %s status scanner", kubernetesCluster.getUuid()), e); - } - } - List destroyingKubernetesClusters = kubernetesClusterDao.findKubernetesClustersInState(KubernetesCluster.State.Destroying); - for (KubernetesCluster kubernetesCluster : destroyingKubernetesClusters) { - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Running Kubernetes cluster state scanner on Kubernetes cluster ID: %s for state: %s", kubernetesCluster.getUuid(), KubernetesCluster.State.Destroying.toString())); - } - try { - cleanupKubernetesClusterResources(kubernetesCluster.getId()); - } catch (Exception e) { - LOGGER.warn(String.format("Failed to run Kubernetes cluster Destroying state scanner on Kubernetes cluster ID: %s status scanner", kubernetesCluster.getUuid()), e); - } - } - } - } catch (Exception e) { - LOGGER.warn("Caught exception while running Kubernetes cluster state scanner", e); - } - firstRun = false; - } - } - - // checks if Kubernetes cluster is in desired state - boolean isClusterVMsInDesiredState(KubernetesCluster kubernetesCluster, VirtualMachine.State state) { - List clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); - - // check cluster is running at desired capacity include master nodes as well - if (clusterVMs.size() < kubernetesCluster.getTotalNodeCount()) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Found only %d VMs in the Kubernetes cluster ID: %s while expected %d VMs to be in state: %s", - clusterVMs.size(), kubernetesCluster.getUuid(), kubernetesCluster.getTotalNodeCount(), state.toString())); - } - return false; - } - // check if all the VM's are in same state - for (KubernetesClusterVmMapVO clusterVm : clusterVMs) { - VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(clusterVm.getVmId()); - if (vm.getState() != state) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Found VM ID: %s in the Kubernetes cluster ID: %s in state: %s while expected to be in state: %s. So moving the cluster to Alert state for reconciliation", - vm.getUuid(), kubernetesCluster.getUuid(), vm.getState().toString(), state.toString())); - } - return false; - } - } - - return true; - } - - @Override - public boolean start() { - _gcExecutor.scheduleWithFixedDelay(new KubernetesClusterGarbageCollector(), 300, 300, TimeUnit.SECONDS); - _stateScanner.scheduleWithFixedDelay(new KubernetesClusterStatusScanner(), 300, 30, TimeUnit.SECONDS); - - return true; - } - - @Override - public boolean configure(String name, Map params) throws ConfigurationException { - _name = name; - _configParams = params; - _gcExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("Kubernetes-Cluster-Scavenger")); - _stateScanner = Executors.newScheduledThreadPool(1, new NamedThreadFactory("Kubernetes-Cluster-State-Scanner")); - - return true; - } - - @Override - public String getConfigComponentName() { - return KubernetesClusterService.class.getSimpleName(); - } - - @Override - public ConfigKey[] getConfigKeys() { - return new ConfigKey[] { - KubernetesServiceEnabled, - KubernetesClusterTemplateName, - KubernetesClusterNetworkOffering - }; - } -} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/AddKubernetesSupportedVersionCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java similarity index 89% rename from plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/AddKubernetesSupportedVersionCmd.java rename to plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java index d2c138810a99..11e1a4574cb8 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/AddKubernetesSupportedVersionCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java @@ -15,13 +15,14 @@ // specific language governing permissions and limitations // under the License. -package org.apache.cloudstack.api.command.admin.kubernetesversion; +package org.apache.cloudstack.api.command.admin.kubernetes.version; import javax.inject.Inject; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ResponseObject; @@ -39,8 +40,9 @@ import com.cloud.exception.NetworkRuleConflictException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; -import com.cloud.kubernetesversion.KubernetesSupportedVersion; -import com.cloud.kubernetesversion.KubernetesVersionService; +import com.cloud.kubernetes.version.KubernetesSupportedVersion; +import com.cloud.kubernetes.version.KubernetesVersionService; +import com.cloud.utils.exception.CloudRuntimeException; import com.google.common.base.Strings; @APICommand(name = AddKubernetesSupportedVersionCmd.APINAME, @@ -135,8 +137,12 @@ public long getEntityOwnerId() { ///////////////////////////////////////////////////// @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { - KubernetesSupportedVersionResponse response = kubernetesVersionService.addKubernetesSupportedVersion(this); - response.setResponseName(getCommandName()); - setResponseObject(response); + try { + KubernetesSupportedVersionResponse response = kubernetesVersionService.addKubernetesSupportedVersion(this); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (CloudRuntimeException ex) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); + } } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/DeleteKubernetesSupportedVersionCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/DeleteKubernetesSupportedVersionCmd.java similarity index 89% rename from plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/DeleteKubernetesSupportedVersionCmd.java rename to plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/DeleteKubernetesSupportedVersionCmd.java index 1ae460a6d5ab..cccc625f5914 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetesversion/DeleteKubernetesSupportedVersionCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/DeleteKubernetesSupportedVersionCmd.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.cloudstack.api.command.admin.kubernetesversion; +package org.apache.cloudstack.api.command.admin.kubernetes.version; import javax.inject.Inject; @@ -37,9 +37,10 @@ import com.cloud.exception.NetworkRuleConflictException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; -import com.cloud.kubernetesversion.KubernetesSupportedVersion; -import com.cloud.kubernetesversion.KubernetesVersionEventTypes; -import com.cloud.kubernetesversion.KubernetesVersionService; +import com.cloud.kubernetes.version.KubernetesSupportedVersion; +import com.cloud.kubernetes.version.KubernetesVersionEventTypes; +import com.cloud.kubernetes.version.KubernetesVersionService; +import com.cloud.utils.exception.CloudRuntimeException; @APICommand(name = DeleteKubernetesSupportedVersionCmd.APINAME, description = "Deletes a Kubernetes cluster", @@ -101,12 +102,13 @@ public String getEventDescription() { ///////////////////////////////////////////////////// @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { - boolean result = kubernetesVersionService.deleteKubernetesSupportedVersion(this); - if (result) { + try { + boolean result = kubernetesVersionService.deleteKubernetesSupportedVersion(this); SuccessResponse response = new SuccessResponse(getCommandName()); + response.setSuccess(result); setResponseObject(response); - } else { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to delete supported Kubernetes version"); + } catch (CloudRuntimeException ex) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); } } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/CreateKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java similarity index 94% rename from plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/CreateKubernetesClusterCmd.java rename to plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java index b66572794caf..83420213c2d8 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/CreateKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package org.apache.cloudstack.api.command.user.kubernetescluster; +package org.apache.cloudstack.api.command.user.kubernetes.cluster; import javax.inject.Inject; @@ -44,9 +44,10 @@ import com.cloud.exception.ManagementServerException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; -import com.cloud.kubernetescluster.KubernetesCluster; -import com.cloud.kubernetescluster.KubernetesClusterEventTypes; -import com.cloud.kubernetescluster.KubernetesClusterService; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.utils.exception.CloudRuntimeException; @APICommand(name = CreateKubernetesClusterCmd.APINAME, description = "Creates a Kubernetes cluster", @@ -293,15 +294,16 @@ public void execute() { public void create() throws ResourceAllocationException { try { KubernetesCluster cluster = kubernetesClusterService.createKubernetesCluster(this); - if (cluster != null) { - setEntityId(cluster.getId()); - setEntityUuid(cluster.getUuid()); - } else { + if (cluster == null) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create Kubernetes cluster"); } - } catch (ConcurrentOperationException | InsufficientCapacityException | ManagementServerException me) { + setEntityId(cluster.getId()); + setEntityUuid(cluster.getUuid()); + } catch (ConcurrentOperationException | InsufficientCapacityException | ManagementServerException me ) { LOGGER.error("Exception: ", me); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, me.getMessage()); + } catch (CloudRuntimeException e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); } } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/DeleteKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/DeleteKubernetesClusterCmd.java similarity index 78% rename from plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/DeleteKubernetesClusterCmd.java rename to plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/DeleteKubernetesClusterCmd.java index 280923a530eb..a336eb8e1b1e 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/DeleteKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/DeleteKubernetesClusterCmd.java @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package org.apache.cloudstack.api.command.user.kubernetescluster; +package org.apache.cloudstack.api.command.user.kubernetes.cluster; import javax.inject.Inject; @@ -30,14 +30,16 @@ import org.apache.cloudstack.context.CallContext; import org.apache.log4j.Logger; -import com.cloud.kubernetescluster.KubernetesClusterEventTypes; -import com.cloud.kubernetescluster.KubernetesCluster; -import com.cloud.kubernetescluster.KubernetesClusterService; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.ManagementServerException; import com.cloud.exception.NetworkRuleConflictException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.utils.exception.CloudRuntimeException; @APICommand(name = DeleteKubernetesClusterCmd.APINAME, description = "Deletes a Kubernetes cluster", @@ -73,30 +75,18 @@ public Long getId() { ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// - public KubernetesCluster validateRequest() { - if (getId() == null || getId() < 1L) { - throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Invalid Kubernetes cluster ID provided"); - } - final KubernetesCluster kubernetesCluster = kubernetesClusterService.findById(getId()); - if (kubernetesCluster == null) { - throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Given Kubernetes cluster was not found"); - } - return kubernetesCluster; - } - @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { - final KubernetesCluster kubernetesCluster = validateRequest(); try { - kubernetesClusterService.deleteKubernetesCluster(id); + boolean result = kubernetesClusterService.deleteKubernetesCluster(id); SuccessResponse response = new SuccessResponse(getCommandName()); + response.setSuccess(result); setResponseObject(response); - } catch (Exception e) { - LOGGER.warn("Failed to delete vm Kubernetes cluster due to " + e); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to delete Kubernetes cluster ID: %s. %s", kubernetesCluster.getUuid(), e.getMessage()), e); + } catch (ManagementServerException | CloudRuntimeException e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/GetKubernetesClusterConfigCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/GetKubernetesClusterConfigCmd.java similarity index 83% rename from plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/GetKubernetesClusterConfigCmd.java rename to plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/GetKubernetesClusterConfigCmd.java index 2039c04473ba..d2b8c4019b96 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/GetKubernetesClusterConfigCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/GetKubernetesClusterConfigCmd.java @@ -14,23 +14,26 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package org.apache.cloudstack.api.command.user.kubernetescluster; +package org.apache.cloudstack.api.command.user.kubernetes.cluster; import javax.inject.Inject; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse; import org.apache.cloudstack.api.response.KubernetesClusterResponse; import org.apache.cloudstack.context.CallContext; import org.apache.log4j.Logger; -import com.cloud.kubernetescluster.KubernetesClusterService; +import com.cloud.kubernetes.cluster.KubernetesClusterService; import com.cloud.user.Account; +import com.cloud.utils.exception.CloudRuntimeException; @APICommand(name = GetKubernetesClusterConfigCmd.APINAME, @@ -84,8 +87,12 @@ public String getCommandName() { @Override public void execute() { - KubernetesClusterConfigResponse response = kubernetesClusterService.getKubernetesClusterConfig(this); - response.setResponseName(getCommandName()); - setResponseObject(response); + try { + KubernetesClusterConfigResponse response = kubernetesClusterService.getKubernetesClusterConfig(this); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (CloudRuntimeException e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); + } } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/ListKubernetesClustersCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ListKubernetesClustersCmd.java similarity index 84% rename from plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/ListKubernetesClustersCmd.java rename to plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ListKubernetesClustersCmd.java index 73833e19c2be..58296f9cccc4 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/ListKubernetesClustersCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ListKubernetesClustersCmd.java @@ -14,11 +14,13 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package org.apache.cloudstack.api.command.user.kubernetescluster; +package org.apache.cloudstack.api.command.user.kubernetes.cluster; import javax.inject.Inject; import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.KubernetesClusterResponse; import org.apache.log4j.Logger; @@ -28,7 +30,8 @@ import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.response.ListResponse; -import com.cloud.kubernetescluster.KubernetesClusterService; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.utils.exception.CloudRuntimeException; @APICommand(name = ListKubernetesClustersCmd.APINAME, description = "Lists Kubernetes clusters", @@ -86,8 +89,12 @@ public String getCommandName() { @Override public void execute() { - ListResponse response = kubernetesClusterService.listKubernetesClusters(this); - response.setResponseName(getCommandName()); - setResponseObject(response); + try { + ListResponse response = kubernetesClusterService.listKubernetesClusters(this); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (CloudRuntimeException e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); + } } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/ScaleKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java similarity index 79% rename from plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/ScaleKubernetesClusterCmd.java rename to plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java index 49642adb9852..33c44678a8c6 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/ScaleKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package org.apache.cloudstack.api.command.user.kubernetescluster; +package org.apache.cloudstack.api.command.user.kubernetes.cluster; import javax.inject.Inject; @@ -33,15 +33,15 @@ import org.apache.cloudstack.context.CallContext; import org.apache.log4j.Logger; -import com.cloud.kubernetescluster.KubernetesClusterEventTypes; -import com.cloud.kubernetescluster.KubernetesCluster; -import com.cloud.kubernetescluster.KubernetesClusterService; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; -import com.cloud.exception.ManagementServerException; import com.cloud.exception.NetworkRuleConflictException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.utils.exception.CloudRuntimeException; @APICommand(name = ScaleKubernetesClusterCmd.APINAME, description = "Scales a created or running Kubernetes cluster", @@ -116,29 +116,18 @@ public long getEntityOwnerId() { /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// - public KubernetesCluster validateRequest() { - if (getId() == null || getId() < 1L) { - throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Invalid Kubernetes cluster ID provided"); - } - final KubernetesCluster kubernetesCluster = kubernetesClusterService.findById(getId()); - if (kubernetesCluster == null) { - throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Given Kubernetes cluster was not found"); - } - return kubernetesCluster; - } - @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { - final KubernetesCluster kubernetesCluster = validateRequest(); try { - kubernetesClusterService.scaleKubernetesCluster(this); - final KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(kubernetesCluster.getId()); + boolean scaled = kubernetesClusterService.scaleKubernetesCluster(this); + if (!scaled) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to scale Kubernetes cluster ID: %d", getId())); + } + final KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(getId()); response.setResponseName(getCommandName()); setResponseObject(response); - } catch (InsufficientCapacityException | ResourceUnavailableException | ManagementServerException ex) { - String msg = String.format("Failed to scale Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); - LOGGER.error(msg, ex); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg, ex); + } catch (CloudRuntimeException ex) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); } } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/StartKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java similarity index 87% rename from plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/StartKubernetesClusterCmd.java rename to plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java index 5d12a1163f96..230a03ca6680 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/StartKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package org.apache.cloudstack.api.command.user.kubernetescluster; +package org.apache.cloudstack.api.command.user.kubernetes.cluster; import javax.inject.Inject; @@ -36,9 +36,10 @@ import com.cloud.exception.NetworkRuleConflictException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; -import com.cloud.kubernetescluster.KubernetesCluster; -import com.cloud.kubernetescluster.KubernetesClusterEventTypes; -import com.cloud.kubernetescluster.KubernetesClusterService; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.utils.exception.CloudRuntimeException; @APICommand(name = StartKubernetesClusterCmd.APINAME, description = "Starts a stopped Kubernetes cluster", responseObject = KubernetesClusterResponse.class, @@ -110,14 +111,18 @@ public KubernetesCluster validateRequest() { public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { final KubernetesCluster kubernetesCluster = validateRequest(); try { - kubernetesClusterService.startKubernetesCluster(kubernetesCluster.getId(), false); + if (!kubernetesClusterService.startKubernetesCluster(kubernetesCluster.getId(), false)) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to start Kubernetes cluster ID: %d", getId())); + } final KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(kubernetesCluster.getId()); response.setResponseName(getCommandName()); setResponseObject(response); } catch (InsufficientCapacityException | ResourceUnavailableException | ManagementServerException ex) { - String msg = String.format("Failed to start Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); + String msg = String.format("Failed to start Kubernetes cluster ID: %d", getId()); LOGGER.error(msg, ex); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg, ex); + } catch (CloudRuntimeException ex) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/StopKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StopKubernetesClusterCmd.java similarity index 77% rename from plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/StopKubernetesClusterCmd.java rename to plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StopKubernetesClusterCmd.java index 85bc1023e1d0..671d14bbe8e4 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/StopKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StopKubernetesClusterCmd.java @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package org.apache.cloudstack.api.command.user.kubernetescluster; +package org.apache.cloudstack.api.command.user.kubernetes.cluster; import javax.inject.Inject; @@ -33,13 +33,13 @@ import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; -import com.cloud.exception.ManagementServerException; import com.cloud.exception.NetworkRuleConflictException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; -import com.cloud.kubernetescluster.KubernetesCluster; -import com.cloud.kubernetescluster.KubernetesClusterEventTypes; -import com.cloud.kubernetescluster.KubernetesClusterService; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.utils.exception.CloudRuntimeException; @APICommand(name = StopKubernetesClusterCmd.APINAME, description = "Stops a running Kubernetes cluster", responseObject = SuccessResponse.class, @@ -96,29 +96,15 @@ public long getEntityOwnerId() { /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// - public KubernetesCluster validateRequest() { - if (getId() == null || getId() < 1L) { - throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Invalid Kubernetes cluster ID provided"); - } - final KubernetesCluster kubernetesCluster = kubernetesClusterService.findById(getId()); - if (kubernetesCluster == null) { - throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Given Kubernetes cluster was not found"); - } - return kubernetesCluster; - } - @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { - final KubernetesCluster kubernetesCluster = validateRequest(); try { - final boolean result = kubernetesClusterService.stopKubernetesCluster(getId()); + boolean result = kubernetesClusterService.stopKubernetesCluster(getId()); final SuccessResponse response = new SuccessResponse(getCommandName()); response.setSuccess(result); setResponseObject(response); - } catch (ManagementServerException ex) { - LOGGER.warn("Failed to stop Kubernetes cluster:" + kubernetesCluster.getUuid() + " due to " + ex.getMessage()); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, - "Failed to stop Kubernetes cluster:" + kubernetesCluster.getUuid(), ex); + } catch (CloudRuntimeException ex) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/UpgradeKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/UpgradeKubernetesClusterCmd.java similarity index 79% rename from plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/UpgradeKubernetesClusterCmd.java rename to plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/UpgradeKubernetesClusterCmd.java index de198f5cc2ec..f0130883006a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetescluster/UpgradeKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/UpgradeKubernetesClusterCmd.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.cloudstack.api.command.user.kubernetescluster; +package org.apache.cloudstack.api.command.user.kubernetes.cluster; import javax.inject.Inject; @@ -34,13 +34,13 @@ import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; -import com.cloud.exception.ManagementServerException; import com.cloud.exception.NetworkRuleConflictException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; -import com.cloud.kubernetescluster.KubernetesCluster; -import com.cloud.kubernetescluster.KubernetesClusterEventTypes; -import com.cloud.kubernetescluster.KubernetesClusterService; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.utils.exception.CloudRuntimeException; @APICommand(name = UpgradeKubernetesClusterCmd.APINAME, description = "Upgrades a running Kubernetes cluster", responseObject = KubernetesClusterResponse.class, @@ -106,29 +106,18 @@ public long getEntityOwnerId() { /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// - public KubernetesCluster validateRequest() { - if (getId() == null || getId() < 1L) { - throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Invalid Kubernetes cluster ID provided"); - } - final KubernetesCluster kubernetesCluster = kubernetesClusterService.findById(getId()); - if (kubernetesCluster == null) { - throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Given Kubernetes cluster was not found"); - } - return kubernetesCluster; - } - @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { - final KubernetesCluster kubernetesCluster = validateRequest(); try { - kubernetesClusterService.upgradeKubernetesCluster(this); - final KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(kubernetesCluster.getId()); + boolean upgraded = kubernetesClusterService.upgradeKubernetesCluster(this); + if (!upgraded) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %d", getId())); + } + final KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(getId()); response.setResponseName(getCommandName()); setResponseObject(response); - } catch (ManagementServerException ex) { - String msg = String.format("Failed to upgrade Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); - LOGGER.error(msg, ex); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg, ex); + } catch (CloudRuntimeException ex) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); } } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetesversion/ListKubernetesSupportedVersionsCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/version/ListKubernetesSupportedVersionsCmd.java similarity index 97% rename from plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetesversion/ListKubernetesSupportedVersionsCmd.java rename to plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/version/ListKubernetesSupportedVersionsCmd.java index c1550fd2d4ac..2837c1dfc524 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetesversion/ListKubernetesSupportedVersionsCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/version/ListKubernetesSupportedVersionsCmd.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.cloudstack.api.command.user.kubernetesversion; +package org.apache.cloudstack.api.command.user.kubernetes.version; import javax.inject.Inject; @@ -36,7 +36,7 @@ import com.cloud.exception.NetworkRuleConflictException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; -import com.cloud.kubernetesversion.KubernetesVersionService; +import com.cloud.kubernetes.version.KubernetesVersionService; import com.google.common.base.Strings; @APICommand(name = ListKubernetesSupportedVersionsCmd.APINAME, diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java index 594d707bc0b8..4b22fb4a4636 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java @@ -22,7 +22,7 @@ import org.apache.cloudstack.api.BaseResponse; import org.apache.cloudstack.api.EntityReference; -import com.cloud.kubernetescluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.serializer.Param; import com.google.gson.annotations.SerializedName; diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java index f8ae5f87765b..2126d551c4db 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java @@ -21,7 +21,7 @@ import org.apache.cloudstack.api.BaseResponse; import org.apache.cloudstack.api.EntityReference; -import com.cloud.kubernetesversion.KubernetesSupportedVersion; +import com.cloud.kubernetes.version.KubernetesSupportedVersion; import com.cloud.serializer.Param; import com.google.gson.annotations.SerializedName; diff --git a/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml b/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml index f56c04c35450..26ef4733b86a 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml +++ b/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml @@ -24,11 +24,11 @@ http://www.springframework.org/schema/context/spring-context-3.0.xsd" > - - - - - - + + + + + + diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetesversion/KubernetesVersionServiceTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionServiceTest.java similarity index 94% rename from plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetesversion/KubernetesVersionServiceTest.java rename to plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionServiceTest.java index 1d268072ddc0..3d38cb12ab98 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetesversion/KubernetesVersionServiceTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionServiceTest.java @@ -1,4 +1,4 @@ -package com.cloud.kubernetesversion; +package com.cloud.kubernetes.version; import static org.mockito.Mockito.when; @@ -7,11 +7,11 @@ import java.util.List; import java.util.UUID; -import org.apache.cloudstack.api.command.admin.kubernetesversion.AddKubernetesSupportedVersionCmd; -import org.apache.cloudstack.api.command.admin.kubernetesversion.DeleteKubernetesSupportedVersionCmd; +import org.apache.cloudstack.api.command.admin.kubernetes.version.AddKubernetesSupportedVersionCmd; +import org.apache.cloudstack.api.command.admin.kubernetes.version.DeleteKubernetesSupportedVersionCmd; import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; import org.apache.cloudstack.api.command.user.iso.RegisterIsoCmd; -import org.apache.cloudstack.api.command.user.kubernetesversion.ListKubernetesSupportedVersionsCmd; +import org.apache.cloudstack.api.command.user.kubernetes.version.ListKubernetesSupportedVersionsCmd; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.framework.config.ConfigKey; @@ -33,10 +33,10 @@ import com.cloud.dc.dao.DataCenterDao; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.ResourceAllocationException; -import com.cloud.kubernetescluster.KubernetesClusterService; -import com.cloud.kubernetescluster.KubernetesClusterVO; -import com.cloud.kubernetescluster.dao.KubernetesClusterDao; -import com.cloud.kubernetesversion.dao.KubernetesSupportedVersionDao; +import com.cloud.kubernetes.version.dao.KubernetesSupportedVersionDao; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.kubernetes.cluster.KubernetesClusterVO; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; import com.cloud.storage.Storage; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.dao.VMTemplateDao; From e7c453a0aaa624fe90f7684e9aa558a7ee2b69df Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Sat, 11 Jan 2020 01:30:59 +0530 Subject: [PATCH 058/134] fixes for upgrade failure Signed-off-by: Abhishek Kumar --- .../actionworkers/KubernetesClusterActionWorker.java | 11 +++++++++-- .../actionworkers/KubernetesClusterUpgradeWorker.java | 2 +- .../version/KubernetesVersionManagerImpl.java | 5 ++++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 5004d0e6656f..19dfc01e548b 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -274,8 +274,11 @@ protected Pair getKubernetesClusterServerIpSshPort(UserVm maste return new Pair<>(null, port); } - protected void attachIsoKubernetesVMs(List clusterVMs) throws CloudRuntimeException { - KubernetesSupportedVersion version = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); + protected void attachIsoKubernetesVMs(List clusterVMs, final KubernetesSupportedVersion kubernetesSupportedVersion) throws CloudRuntimeException { + KubernetesSupportedVersion version = kubernetesSupportedVersion; + if (kubernetesSupportedVersion == null) { + version = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); + } if (version == null) { logAndThrow(Level.ERROR, String .format("Unable to find Kubernetes version for cluster ID: %s", kubernetesCluster.getUuid())); } @@ -301,6 +304,10 @@ protected void attachIsoKubernetesVMs(List clusterVMs) throws CloudRunti } } + protected void attachIsoKubernetesVMs(List clusterVMs) { + attachIsoKubernetesVMs(clusterVMs, null); + } + protected void detachIsoKubernetesVMs(List clusterVMs) throws CloudRuntimeException { for (UserVm vm : clusterVMs) { boolean result = false; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java index 8b325e32cfe8..ec397303cd83 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java @@ -141,7 +141,7 @@ public boolean upgradeCluster() throws CloudRuntimeException { } retrieveUpgradeScriptFile(); stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.UpgradeRequested); - attachIsoKubernetesVMs(clusterVMs); + attachIsoKubernetesVMs(clusterVMs, upgradeVersion); upgradeKubernetesClusterNodes(); detachIsoKubernetesVMs(clusterVMs); KubernetesClusterVO kubernetesClusterVO = kubernetesClusterDao.findById(kubernetesCluster.getId()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java index 6d6f797414c7..81a50771bc57 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java @@ -146,7 +146,10 @@ public static int compareSemanticVersions(String v1, String v2) throws IllegalAr } public static boolean canUpgradeKubernetesVersion(String currentVersion, String upgradeVersion) throws IllegalArgumentException { - if (compareSemanticVersions(currentVersion, upgradeVersion) <= 0) { + int versionDiff = compareSemanticVersions(upgradeVersion, currentVersion); + if (versionDiff == 0) { + throw new IllegalArgumentException(String.format("Kubernetes clusters can not be upgraded, current version: %s, upgrade version: %s", currentVersion, upgradeVersion)); + } else if (versionDiff < 0) { throw new IllegalArgumentException(String.format("Kubernetes clusters can not be downgraded, current version: %s, upgrade version: %s", currentVersion, upgradeVersion)); } String[] thisParts = currentVersion.split("\\."); From b9e8e3954d237b84c3b49699acc4cf9cf0285f08 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Sat, 11 Jan 2020 01:32:08 +0530 Subject: [PATCH 059/134] cluster test improvement Signed-off-by: Abhishek Kumar --- .../smoke/test_kubernetes_clusters.py | 34 ++++++++++++++----- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_clusters.py b/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_clusters.py index 6df314a8e17c..e1d78d593080 100644 --- a/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_clusters.py +++ b/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_clusters.py @@ -68,12 +68,13 @@ def setUpClass(cls): cls.kubernetes_version_ids.append(cls.kuberetes_version_3.id) cks_offering_data = { - "name": "CKS Instance", + "name": "CKS-Instance", "displaytext": "CKS Instance", "cpunumber": 2, "cpuspeed": 1000, "memory": 2048, } + cks_offering_data["name"] = cks_offering_data["name"] + '-' + random_gen() cls.cks_service_offering = ServiceOffering.create( cls.apiclient, cks_offering_data @@ -85,7 +86,7 @@ def setUpClass(cls): "format": "qcow2", "hypervisor": "kvm", "ostype": "CoreOS", - "url": "http://172.20.0.1/files/coreos_production_cloudstack_image-kvm.qcow2.bz2", + "url": "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-kvm.qcow2.bz2", "requireshvm": "True", "ispublic": "True", "isextractable": "True" @@ -232,6 +233,7 @@ def listKubernetesSupportedVersion(cls, version_id): def addKubernetesSupportedVersion(cls, semantic_version, iso_url): addKubernetesSupportedVersionCmd = addKubernetesSupportedVersion.addKubernetesSupportedVersionCmd() addKubernetesSupportedVersionCmd.semanticversion = semantic_version + addKubernetesSupportedVersionCmd.name = 'v' + semantic_version + '-' + random_gen() addKubernetesSupportedVersionCmd.url = iso_url kubernetes_version = cls.apiclient.addKubernetesSupportedVersion(addKubernetesSupportedVersionCmd) cls.debug("Waiting for Kubernetes version with ID %s to be ready" % kubernetes_version.id) @@ -328,7 +330,7 @@ def test_03_deploy_invalid_kubernetes_ha_cluster(self): try: cluster_response = self.createKubernetesCluster(name, self.kuberetes_version_2.id, 1, 2) self.debug("Invslid CKS Kubernetes HA cluster deployed with ID: %s. Deleting it and failing test." % cluster_response.id) - deleteAndVerifyKubernetesCluster(cluster_response.id) + self.deleteKubernetesCluster(cluster_response.id) self.fail("HA Kubernetes cluster deployed with Kubernetes supported version below version 1.16.0. Must be an error.") except CloudstackAPIException as e: self.debug("HA Kubernetes cluster with invalid Kubernetes supported version check successful, API failure: %s" % e) @@ -355,7 +357,11 @@ def test_04_deploy_and_upgrade_kubernetes_cluster(self): self.debug("Kubernetes cluster with ID: %s successfully deployed, now upgrading it" % cluster_response.id) - cluster_response = self.upgradeKubernetesCluster(cluster_response.id, self.kuberetes_version_3.id) + try: + cluster_response = self.upgradeKubernetesCluster(cluster_response.id, self.kuberetes_version_3.id) + except Exception as e: + self.deleteKubernetesCluster(cluster_response.id) + self.fail("Failed to upgrade Kubernetes cluster due to: %s" % e) self.verifyKubernetesClusterUpgrade(cluster_response, self.kuberetes_version_3.id) @@ -386,7 +392,11 @@ def test_05_deploy_and_upgrade_kubernetes_ha_cluster(self): self.debug("Kubernetes cluster with ID: %s successfully deployed, now upgrading it" % cluster_response.id) - cluster_response = self.upgradeKubernetesCluster(cluster_response.id, self.kuberetes_version_3.id) + try: + cluster_response = self.upgradeKubernetesCluster(cluster_response.id, self.kuberetes_version_3.id) + except Exception as e: + self.deleteKubernetesCluster(cluster_response.id) + self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) self.verifyKubernetesClusterUpgrade(cluster_response, self.kuberetes_version_3.id) @@ -421,7 +431,7 @@ def test_06_deploy_and_invalid_upgrade_kubernetes_cluster(self): try: cluster_response = self.upgradeKubernetesCluster(cluster_response.id, self.kuberetes_version_1.id) self.debug("Invalid CKS Kubernetes HA cluster deployed with ID: %s. Deleting it and failing test." % kuberetes_version_1.id) - self.deleteAndVerifyKubernetesCluster(cluster_response.id) + self.deleteKubernetesCluster(cluster_response.id) self.fail("Kubernetes cluster upgraded to a lower Kubernetes supported version. Must be an error.") except Exception as e: self.debug("Upgrading Kubernetes cluster with invalid Kubernetes supported version check successful, API failure: %s" % e) @@ -454,7 +464,11 @@ def test_07_deploy_and_scale_up_kubernetes_cluster(self): self.debug("Kubernetes cluster with ID: %s successfully deployed, now upscaling it" % cluster_response.id) - cluster_response = self.scaleKubernetesCluster(cluster_response.id, 2) + try: + cluster_response = self.scaleKubernetesCluster(cluster_response.id, 2) + except Exception as e: + self.deleteKubernetesCluster(cluster_response.id) + self.fail("Failed to upscale Kubernetes cluster due to: %s" % e) self.verifyKubernetesClusterScale(cluster_response, 2) @@ -484,7 +498,11 @@ def test_08_deploy_and_scale_down_kubernetes_cluster(self): self.debug("Kubernetes cluster with ID: %s successfully deployed, now downscaling it" % cluster_response.id) - cluster_response = self.scaleKubernetesCluster(cluster_response.id, 1) + try: + cluster_response = self.scaleKubernetesCluster(cluster_response.id, 1) + except Exception as e: + self.deleteKubernetesCluster(cluster_response.id) + self.fail("Failed to downscale Kubernetes cluster due to: %s" % e) self.verifyKubernetesClusterScale(cluster_response) From b920cf71a8a1c61e55ad8162caf3cd1611029753 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 13 Jan 2020 02:14:49 +0530 Subject: [PATCH 060/134] cluster start stop fixes Signed-off-by: Abhishek Kumar --- .../com/cloud/kubernetes/cluster/KubernetesCluster.java | 1 + .../kubernetes/cluster/KubernetesClusterManagerImpl.java | 6 ++++-- .../cluster/actionworkers/KubernetesClusterStartWorker.java | 3 ++- .../cluster/actionworkers/KubernetesClusterStopWorker.java | 3 ++- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java index 27712d4e79f5..123fd5f576cc 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java @@ -73,6 +73,7 @@ enum State { s_fsm.addTransition(State.Starting, Event.StopRequested, State.Stopping); s_fsm.addTransition(State.Running, Event.StopRequested, State.Stopping); + s_fsm.addTransition(State.Alert, Event.StopRequested, State.Stopping); s_fsm.addTransition(State.Stopping, Event.OperationSucceeded, State.Stopped); s_fsm.addTransition(State.Stopping, Event.OperationFailed, State.Alert); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index e57e2bb71520..b65abb8d57eb 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -1217,7 +1217,7 @@ public void reallyRun() { } } - // run through Kubernetes clusters in 'Alert' state and reconcile state as 'Running' if the VM's are running + // run through Kubernetes clusters in 'Alert' state and reconcile state as 'Running' if the VM's are running or 'Stopped' if VM's are stopped List alertKubernetesClusters = kubernetesClusterDao.findKubernetesClustersInState(KubernetesCluster.State.Alert); for (KubernetesClusterVO kubernetesCluster : alertKubernetesClusters) { if (LOGGER.isInfoEnabled()) { @@ -1225,11 +1225,13 @@ public void reallyRun() { } try { if (isClusterVMsInDesiredState(kubernetesCluster, VirtualMachine.State.Running)) { - KubernetesClusterStartWorker startWorker = new KubernetesClusterStartWorker(kubernetesCluster, KubernetesClusterManagerImpl.this); startWorker = ComponentContext.inject(startWorker); startWorker.reconcileAlertCluster(); + } else if (isClusterVMsInDesiredState(kubernetesCluster, VirtualMachine.State.Stopped)) { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.StopRequested); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); } } catch (Exception e) { LOGGER.warn(String.format("Failed to run Kubernetes cluster Alert state scanner on Kubernetes cluster ID: %s status scanner", kubernetesCluster.getUuid()), e); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index 2dbf6fd3c436..66fd09b11eaf 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -408,7 +408,8 @@ private void startKubernetesClusterVMs() { // dont bail out here. proceed further to stop the reset of the VM's } } - for (final UserVm vm : clusterVms) { + for (final UserVm userVm : clusterVms) { + UserVm vm = userVmDao.findById(userVm.getId()); if (vm == null || !vm.getState().equals(VirtualMachine.State.Running)) { logTransitStateAndThrow(Level.ERROR, String.format("Failed to start all VMs in Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStopWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStopWorker.java index ff03a0198b59..a8e1a2c4d841 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStopWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStopWorker.java @@ -50,7 +50,8 @@ public boolean stop() throws CloudRuntimeException { LOGGER.warn(String.format("Failed to stop VM ID: %s in Kubernetes cluster ID: %s", vm.getUuid(), kubernetesCluster.getUuid()), ex); } } - for (final UserVm vm : clusterVMs) { + for (final UserVm userVm : clusterVMs) { + UserVm vm = userVmDao.findById(userVm.getId()); if (vm == null || !vm.getState().equals(VirtualMachine.State.Stopped)) { logTransitStateAndThrow(Level.ERROR, String.format("Failed to stop all VMs in Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } From 1aa449cfbefa6c93d93383b33b75a6015d236149 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 13 Jan 2020 03:45:35 +0530 Subject: [PATCH 061/134] scale cluster fixes Signed-off-by: Abhishek Kumar --- .../cluster/KubernetesClusterManagerImpl.java | 6 + .../KubernetesClusterActionWorker.java | 17 +- ...esClusterResourceModifierActionWorker.java | 2 +- .../KubernetesClusterScaleWorker.java | 306 +++++++++--------- .../cluster/utils/KubernetesClusterUtil.java | 2 +- .../smoke/test_kubernetes_clusters.py | 33 +- 6 files changed, 178 insertions(+), 188 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index b65abb8d57eb..1bcca56fe522 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -773,6 +773,12 @@ private void validateKubernetesClusterScaleParameters(ScaleKubernetesClusterCmd throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be scaled with service offering ID: %s, Kubernetes cluster template(CoreOS) needs minimum 2 vCPUs and 2 GB RAM", kubernetesCluster.getUuid(), serviceOffering.getUuid())); } } + final ServiceOffering existingServiceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); + if (serviceOffering.getRamSize() < existingServiceOffering.getRamSize() || + serviceOffering.getCpu() * serviceOffering.getSpeed() < existingServiceOffering.getCpu() * existingServiceOffering.getSpeed()) { + logAndThrow(Level.WARN, String.format("Kubernetes cluster cannot be scaled down for service offering. Service offering ID: %s offers lesser resources as compared to service offering ID: %s of Kubernetes cluster ID: %s", + serviceOffering.getUuid(), existingServiceOffering.getUuid(), kubernetesCluster.getUuid())); + } } if (!(kubernetesCluster.getState().equals(KubernetesCluster.State.Created) || diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 19dfc01e548b..e1c8c26a18cd 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -22,7 +22,6 @@ import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Collections; -import java.util.Comparator; import java.util.List; import java.util.Objects; @@ -326,16 +325,18 @@ protected void detachIsoKubernetesVMs(List clusterVMs) throws CloudRunti } } + protected List getKubernetesClusterVMMaps() { + List clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + if (!CollectionUtils.isEmpty(clusterVMs)) { + clusterVMs.sort((t1, t2) -> (int)((t1.getId() - t2.getId())/Math.abs(t1.getId() - t2.getId()))); + } + return clusterVMs; + } + protected List getKubernetesClusterVMs() { List vmList = new ArrayList<>(); - List clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + List clusterVMs = getKubernetesClusterVMMaps(); if (!CollectionUtils.isEmpty(clusterVMs)) { - clusterVMs.sort(new Comparator() { - @Override - public int compare(KubernetesClusterVmMapVO t1, KubernetesClusterVmMapVO t2) { - return (int)((t1.getId() - t2.getId())/Math.abs(t1.getId() - t2.getId())); - } - }); for (KubernetesClusterVmMapVO vmMap : clusterVMs) { vmList.add(userVmDao.findById(vmMap.getVmId())); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 267127b7e865..a48ab298936f 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -265,7 +265,7 @@ protected List provisionKubernetesClusterNodeVms(final long nodeCount, f startKubernetesVM(vm); nodes.add(vm); if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Provisioned node master VM ID: %s in to the Kubernetes cluster ID: %s", vm.getUuid(), kubernetesCluster.getUuid())); + LOGGER.info(String.format("Provisioned node VM ID: %s in to the Kubernetes cluster ID: %s", vm.getUuid(), kubernetesCluster.getUuid())); } } return nodes; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index 6ad054713512..cea98ad3a849 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -51,7 +51,6 @@ import com.cloud.utils.Pair; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallback; -import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.ssh.SshHelper; import com.cloud.vm.UserVmManager; @@ -70,6 +69,8 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif private ServiceOffering serviceOffering; private Long clusterSize; + private KubernetesCluster.State originalState; + private Network network; public KubernetesClusterScaleWorker(final KubernetesCluster kubernetesCluster, final ServiceOffering serviceOffering, @@ -78,6 +79,24 @@ public KubernetesClusterScaleWorker(final KubernetesCluster kubernetesCluster, super(kubernetesCluster, clusterManager); this.serviceOffering = serviceOffering; this.clusterSize = clusterSize; + this.originalState = kubernetesCluster.getState(); + } + + protected void init() { + super.init(); + this.network = networkDao.findById(kubernetesCluster.getNetworkId()); + } + + private void logTransitStateToFailedIfNeededAndThrow(final Level logLevel, final String message, final Exception e) throws CloudRuntimeException { + if (kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { + logTransitStateAndThrow(logLevel, message, kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); + } else { + logAndThrow(logLevel, message, e); + } + } + + private void logTransitStateToFailedIfNeededAndThrow(final Level logLevel, final String message) throws CloudRuntimeException { + logTransitStateToFailedIfNeededAndThrow(logLevel, message, null); } private FirewallRule removeSshFirewallRule(IpAddress publicIp) { @@ -113,14 +132,11 @@ private void removePortForwardingRules(IpAddress publicIp, Network network, Acco * Open up firewall for SSH access from port NODES_DEFAULT_START_SSH_PORT to NODES_DEFAULT_START_SSH_PORT+n. * Also remove port forwarding rules for removed virtual machines and create port-forwarding rule * to forward public IP traffic to all node VMs' private IP. - * @param network - * @param account * @param clusterVMIds * @param removedVMIds * @throws ManagementServerException */ - private void scaleKubernetesClusterNetworkRules(Network network, Account account, - List clusterVMIds, List removedVMIds) throws ManagementServerException { + private void scaleKubernetesClusterNetworkRules(List clusterVMIds, List removedVMIds) throws ManagementServerException { if (!Network.GuestType.Isolated.equals(network.getGuestType())) { if (LOGGER.isDebugEnabled()) { LOGGER.debug(String.format("Network ID: %s for Kubernetes cluster ID: %s is not an isolated network, therefore, no need for network rules", network.getUuid(), kubernetesCluster.getUuid())); @@ -141,7 +157,7 @@ private void scaleKubernetesClusterNetworkRules(Network network, Account account // Provision new SSH firewall rules try { - provisionFirewallRules(publicIp, account, CLUSTER_NODES_DEFAULT_START_SSH_PORT, CLUSTER_NODES_DEFAULT_START_SSH_PORT + (int)kubernetesCluster.getTotalNodeCount() - 1); + provisionFirewallRules(publicIp, owner, CLUSTER_NODES_DEFAULT_START_SSH_PORT, CLUSTER_NODES_DEFAULT_START_SSH_PORT + (int)kubernetesCluster.getTotalNodeCount() - 1); if (LOGGER.isDebugEnabled()) { LOGGER.debug(String.format("Provisioned firewall rule to open up port %d to %d on %s in Kubernetes cluster ID: %s", CLUSTER_NODES_DEFAULT_START_SSH_PORT, CLUSTER_NODES_DEFAULT_START_SSH_PORT + (int) kubernetesCluster.getTotalNodeCount() - 1, publicIp.getAddress().addr(), kubernetesCluster.getUuid())); @@ -151,36 +167,50 @@ private void scaleKubernetesClusterNetworkRules(Network network, Account account } try { - removePortForwardingRules(publicIp, network, account, removedVMIds); + removePortForwardingRules(publicIp, network, owner, removedVMIds); } catch (ResourceUnavailableException e) { throw new ManagementServerException(String.format("Failed to remove SSH port forwarding rules for removed VMs for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); } try { - provisionSshPortForwardingRules(publicIp, network, account, clusterVMIds, existingFirewallRuleSourcePortEnd + 1); + provisionSshPortForwardingRules(publicIp, network, owner, clusterVMIds, existingFirewallRuleSourcePortEnd + 1); } catch (ResourceUnavailableException | NetworkRuleConflictException e) { throw new ManagementServerException(String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); } } - private KubernetesClusterVO updateKubernetesClusterEntry(final long kubernetesClusterId, final long clusterSize, - final long cores, final long memory, final Long serviceOfferingId) { - return Transaction.execute(new TransactionCallback() { - @Override - public KubernetesClusterVO doInTransaction(TransactionStatus status) { - KubernetesClusterVO updatedCluster = kubernetesClusterDao.createForUpdate(kubernetesClusterId); - updatedCluster.setNodeCount(clusterSize); - updatedCluster.setCores(cores); - updatedCluster.setMemory(memory); - if (serviceOfferingId != null) { - updatedCluster.setServiceOfferingId(serviceOfferingId); - } - kubernetesClusterDao.persist(updatedCluster); - return updatedCluster; + private KubernetesClusterVO updateKubernetesClusterEntry(final long cores, final long memory, + final Long size, final Long serviceOfferingId) { + return Transaction.execute((TransactionCallback) status -> { + KubernetesClusterVO updatedCluster = kubernetesClusterDao.createForUpdate(kubernetesCluster.getId()); + updatedCluster.setCores(cores); + updatedCluster.setMemory(memory); + if (size != null) { + updatedCluster.setNodeCount(size); + } + if (serviceOfferingId != null) { + updatedCluster.setServiceOfferingId(serviceOfferingId); } + kubernetesClusterDao.persist(updatedCluster); + return updatedCluster; }); } + private KubernetesClusterVO updateKubernetesClusterEntry(final Long newSize, final ServiceOffering newServiceOffering) throws CloudRuntimeException { + final ServiceOffering serviceOffering = newServiceOffering == null ? + serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()) : newServiceOffering; + final Long serviceOfferingId = newServiceOffering == null ? null : serviceOffering.getId(); + final long size = newSize == null ? kubernetesCluster.getNodeCount() : newSize; + final long cores = serviceOffering.getCpu() * size; + final long memory = serviceOffering.getRamSize() * size; + KubernetesClusterVO kubernetesClusterVO = updateKubernetesClusterEntry(cores, memory, size, serviceOfferingId); + if (kubernetesClusterVO == null) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, unable to update Kubernetes cluster", + kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + return kubernetesClusterVO; + } + private boolean removeKubernetesClusterNode(String ipAddress, int port, UserVm userVm, int retries, int waitDuration) { File pkFile = getManagementServerSshPublicKeyFile(); int retryCounter = 0; @@ -221,21 +251,68 @@ private boolean removeKubernetesClusterNode(String ipAddress, int port, UserVm u return false; } - private void scaleKubernetesClusterOffering(final long kubernetesClusterId, final ServiceOffering serviceOffering, final Long clusterSize) { - KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + private void validateKubernetesClusterScaleOfferingParameters() throws CloudRuntimeException { + if (KubernetesCluster.State.Created.equals(originalState)) { + return; + } + final long originalNodeCount = kubernetesCluster.getTotalNodeCount(); + List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + if (vmList == null || vmList.isEmpty() || vmList.size() < originalNodeCount) { + logTransitStateToFailedIfNeededAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster ID: %s failed, it is in unstable state as not enough existing VM instances found!", kubernetesCluster.getUuid())); + } else { + for (KubernetesClusterVmMapVO vmMapVO : vmList) { + VMInstanceVO vmInstance = vmInstanceDao.findById(vmMapVO.getVmId()); + if (vmInstance != null && vmInstance.getState().equals(VirtualMachine.State.Running) && + vmInstance.getHypervisorType() != Hypervisor.HypervisorType.XenServer && + vmInstance.getHypervisorType() != Hypervisor.HypervisorType.VMware && + vmInstance.getHypervisorType() != Hypervisor.HypervisorType.Simulator) { + logTransitStateToFailedIfNeededAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster ID: %s failed, scaling Kubernetes cluster with running VMs on hypervisor %s is not supported!", kubernetesCluster.getUuid(), vmInstance.getHypervisorType())); + } + } + } + } - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.ScaleUpRequested); + private void validateKubernetesClusterScaleSizeParameters() throws CloudRuntimeException { + final long originalClusterSize = kubernetesCluster.getNodeCount(); + if (network == null) { + logTransitStateToFailedIfNeededAndThrow(Level.WARN, String.format("Scaling failed for Kubernetes cluster ID: %s, cluster network not found", kubernetesCluster.getUuid())); + } + // Check capacity and transition state + final long newVmRequiredCount = clusterSize - originalClusterSize; + final ServiceOffering clusterServiceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); + if (clusterServiceOffering == null) { + logTransitStateToFailedIfNeededAndThrow(Level.WARN, String.format("Scaling failed for Kubernetes cluster ID: %s, cluster service offering not found", kubernetesCluster.getUuid())); + } + if (newVmRequiredCount > 0) { + final DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); + try { + if (originalState.equals(KubernetesCluster.State.Running)) { + plan(newVmRequiredCount, zone, clusterServiceOffering); + } else { + plan(kubernetesCluster.getTotalNodeCount() + newVmRequiredCount, zone, clusterServiceOffering); + } + } catch (InsufficientCapacityException e) { + logTransitStateToFailedIfNeededAndThrow(Level.WARN, String.format("Scaling failed for Kubernetes cluster ID: %s in zone ID: %s, insufficient capacity", kubernetesCluster.getUuid(), zone.getUuid())); + } + } + List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + if (CollectionUtils.isEmpty(vmList) || vmList.size() < kubernetesCluster.getTotalNodeCount()) { + logTransitStateToFailedIfNeededAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, it is in unstable state as not enough existing VM instances found", kubernetesCluster.getUuid())); + } + } - final long size = (clusterSize == null ? kubernetesCluster.getTotalNodeCount() : kubernetesCluster.getMasterNodeCount() + clusterSize); - final long cores = serviceOffering.getCpu() * size; - final long memory = serviceOffering.getRamSize() * size; - KubernetesClusterVO updatedKubernetesCluster = updateKubernetesClusterEntry(kubernetesCluster.getId(), size, cores, memory, serviceOffering.getId()); - if (updatedKubernetesCluster == null) { - logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, unable to update Kubernetes cluster!", updatedKubernetesCluster.getUuid()), kubernetesClusterId, KubernetesCluster.Event.OperationFailed); + private void scaleKubernetesClusterOffering() throws CloudRuntimeException { + validateKubernetesClusterScaleOfferingParameters(); + if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.ScaleUpRequested); + } + if (KubernetesCluster.State.Created.equals(originalState)) { + kubernetesCluster = updateKubernetesClusterEntry(null, serviceOffering); + return; } - kubernetesCluster = updatedKubernetesCluster; + final long size = kubernetesCluster.getTotalNodeCount(); List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); - final long tobeScaledVMCount = Math.min(vmList.size(), size); + final long tobeScaledVMCount = Math.min(vmList.size(), size); for (long i = 0; i < tobeScaledVMCount; i++) { KubernetesClusterVmMapVO vmMapVO = vmList.get((int) i); UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId()); @@ -249,14 +326,18 @@ private void scaleKubernetesClusterOffering(final long kubernetesClusterId, fina logTransitStateAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster ID: %s failed, unable to scale cluster VM ID: %s", kubernetesCluster.getUuid(), userVM.getUuid()),kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } } + kubernetesCluster = updateKubernetesClusterEntry(null, serviceOffering); } - private void scaleDownKubernetesClusterSize(final List originalVmList, final Network network) throws CloudRuntimeException { + private void scaleDownKubernetesClusterSize(final List originalVmList) throws CloudRuntimeException { + if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.ScaleDownRequested); + } int i = originalVmList.size() - 1; List removedVmIds = new ArrayList<>(); - while (i > kubernetesCluster.getMasterNodeCount() && originalVmList.size() > kubernetesCluster.getTotalNodeCount()) { // Reverse order as first VM will be k8s master + while (i > kubernetesCluster.getMasterNodeCount()) { KubernetesClusterVmMapVO vmMapVO = originalVmList.get(i); - UserVm userVM = userVmDao.findById(vmMapVO.getId()); + UserVm userVM = userVmDao.findById(vmMapVO.getVmId()); if (!removeKubernetesClusterNode(publicIpAddress, sshPort, userVM, 3, 30000)) { logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, failed to remove Kubernetes node: %s running on VM ID: %s", kubernetesCluster.getUuid(), userVM.getHostName(), userVM.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } @@ -288,120 +369,66 @@ private void scaleDownKubernetesClusterSize(final List } // Scale network rules to update firewall rule try { - scaleKubernetesClusterNetworkRules(network, owner, null, removedVmIds); + scaleKubernetesClusterNetworkRules(null, removedVmIds); } catch (ManagementServerException e) { logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, unable to update network rules", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); } } - private void scaleUpKubernetesClusterSize(final List originalVmList, final long newVmCount, final Network network) throws CloudRuntimeException { + private void scaleUpKubernetesClusterSize(final List originalVmList, final long newVmCount) throws CloudRuntimeException { + if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.ScaleUpRequested); + } List clusterVMs = new ArrayList<>(); List clusterVMIds = new ArrayList<>(); try { clusterVMs = provisionKubernetesClusterNodeVms((int) newVmCount + originalVmList.size(), originalVmList.size(), publicIpAddress); } catch (ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { - logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, unable to provision node VM in the cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); + logTransitStateToFailedIfNeededAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, unable to provision node VM in the cluster", kubernetesCluster.getUuid()), e); } for (UserVm vm : clusterVMs) { clusterVMIds.add(vm.getId()); } try { - scaleKubernetesClusterNetworkRules(network, owner, clusterVMIds, null); + scaleKubernetesClusterNetworkRules(clusterVMIds, null); } catch (ManagementServerException e) { - logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, unable to update network rules", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); + logTransitStateToFailedIfNeededAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, unable to update network rules", kubernetesCluster.getUuid()), e); } attachIsoKubernetesVMs(clusterVMs); - boolean readyNodesCountValid = KubernetesClusterUtil.validateKubernetesClusterReadyNodesCount(kubernetesCluster, publicIpAddress, sshPort, + KubernetesClusterVO kubernetesClusterVO = kubernetesClusterDao.findById(kubernetesCluster.getId()); + kubernetesClusterVO.setNodeCount(clusterSize); + boolean readyNodesCountValid = KubernetesClusterUtil.validateKubernetesClusterReadyNodesCount(kubernetesClusterVO, publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, 30, 30000); detachIsoKubernetesVMs(clusterVMs); if (!readyNodesCountValid) { // Scaling failed - logTransitStateAndThrow(Level.ERROR, String.format("Scaling unsuccessful for Kubernetes cluster ID: %s as it does not have desired number of nodes in ready state", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + logTransitStateToFailedIfNeededAndThrow(Level.ERROR, String.format("Scaling unsuccessful for Kubernetes cluster ID: %s as it does not have desired number of nodes in ready state", kubernetesCluster.getUuid())); } } - private void scaleKubernetesClusterSize(final long originalClusterSize) throws CloudRuntimeException { - KubernetesClusterVO kubernetesClusterVO = kubernetesClusterDao.findById(kubernetesCluster.getId()); - final Network network = networkDao.findById(kubernetesClusterVO.getNetworkId()); + private void scaleKubernetesClusterSize() throws CloudRuntimeException { + validateKubernetesClusterScaleSizeParameters(); + final long originalClusterSize = kubernetesCluster.getNodeCount(); final long newVmRequiredCount = clusterSize - originalClusterSize; - List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesClusterVO.getId()); - if (CollectionUtils.isEmpty(vmList) || vmList.size() - 1 < originalClusterSize) { - logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, t is in unstable state as not enough existing VM instances found", kubernetesClusterVO.getUuid()), kubernetesClusterVO.getId(), KubernetesCluster.Event.OperationFailed); + if (KubernetesCluster.State.Created.equals(originalState)) { + if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { + stateTransitTo(kubernetesCluster.getId(), newVmRequiredCount > 0 ? KubernetesCluster.Event.ScaleUpRequested : KubernetesCluster.Event.ScaleDownRequested); + } + kubernetesCluster = updateKubernetesClusterEntry(null, serviceOffering); + return; } - Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(null); - String publicIpAddress = publicIpSshPort.first(); - int sshPort = publicIpSshPort.second(); + publicIpAddress = publicIpSshPort.first(); + sshPort = publicIpSshPort.second(); if (Strings.isNullOrEmpty(publicIpAddress)) { - logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, unable to retrieve associated public IP", kubernetesClusterVO.getUuid()), kubernetesClusterVO.getId(), KubernetesCluster.Event.OperationFailed); + logTransitStateToFailedIfNeededAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, unable to retrieve associated public IP", kubernetesCluster.getUuid())); } - Account account = accountDao.findById(kubernetesClusterVO.getAccountId()); + List vmList = getKubernetesClusterVMMaps(); if (newVmRequiredCount < 0) { // downscale - scaleDownKubernetesClusterSize(vmList, network); + scaleDownKubernetesClusterSize(vmList); } else { // upscale, same node count handled above - scaleUpKubernetesClusterSize(vmList, newVmRequiredCount, network); - } - } - - private void validateKubernetesClusterScaleOfferingParameters(final ServiceOffering existingServiceOffering, final ServiceOffering serviceOffering) throws CloudRuntimeException { - final long originalNodeCount = kubernetesCluster.getTotalNodeCount(); - List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); - if (vmList == null || vmList.isEmpty() || vmList.size() < originalNodeCount) { - logAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster ID: %s failed, it is in unstable state as not enough existing VM instances found!", kubernetesCluster.getUuid())); - } else { - for (KubernetesClusterVmMapVO vmMapVO : vmList) { - VMInstanceVO vmInstance = vmInstanceDao.findById(vmMapVO.getVmId()); - if (vmInstance != null && vmInstance.getState().equals(VirtualMachine.State.Running) && - vmInstance.getHypervisorType() != Hypervisor.HypervisorType.XenServer && - vmInstance.getHypervisorType() != Hypervisor.HypervisorType.VMware && - vmInstance.getHypervisorType() != Hypervisor.HypervisorType.Simulator) { - logAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster ID: %s failed, scaling Kubernetes cluster with running VMs on hypervisor %s is not supported!", kubernetesCluster.getUuid(), vmInstance.getHypervisorType())); - } - } - } - if (serviceOffering.getRamSize() < existingServiceOffering.getRamSize() || - serviceOffering.getCpu() * serviceOffering.getSpeed() < existingServiceOffering.getCpu() * existingServiceOffering.getSpeed()) { - logAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster ID: %s failed, service offering for the Kubernetes cluster cannot be scaled down!", kubernetesCluster.getUuid())); - } - } - - private void validateKubernetesClusterScaleSizeParameters(final long originalClusterSize, final long clusterSize, final KubernetesCluster.State clusterState) throws CloudRuntimeException { - Network network = networkDao.findById(kubernetesCluster.getNetworkId()); - if (network == null) { - String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, cluster network not found", kubernetesCluster.getUuid()); - if (KubernetesCluster.State.Scaling.equals(kubernetesCluster.getState())) { - logTransitStateAndThrow(Level.WARN, msg, kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } else { - logAndThrow(Level.WARN, msg); - } - } - // Check capacity and transition state - final long newVmRequiredCount = clusterSize - originalClusterSize; - final ServiceOffering clusterServiceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); - if (clusterServiceOffering == null) { - String msg = String.format("Scaling failed for Kubernetes cluster ID: %s, cluster service offering not found", kubernetesCluster.getUuid()); - if (KubernetesCluster.State.Scaling.equals(kubernetesCluster.getState())) { - logTransitStateAndThrow(Level.WARN, msg, kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } else { - logAndThrow(Level.WARN, msg); - } - } - if (newVmRequiredCount > 0) { - final DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); - try { - if (clusterState.equals(KubernetesCluster.State.Running)) { - plan(newVmRequiredCount, zone, clusterServiceOffering); - } else { - plan(kubernetesCluster.getTotalNodeCount() + newVmRequiredCount, zone, clusterServiceOffering); - } - } catch (InsufficientCapacityException e) { - String msg = String.format("Scaling failed for Kubernetes cluster ID: %s in zone ID: %s, insufficient capacity", kubernetesCluster.getUuid(), zone.getUuid()); - if (KubernetesCluster.State.Scaling.equals(kubernetesCluster.getState())) { - logTransitStateAndThrow(Level.WARN, msg, kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } else { - logAndThrow(Level.WARN, msg); - } - } + scaleUpKubernetesClusterSize(vmList, newVmRequiredCount); } + kubernetesCluster = updateKubernetesClusterEntry(clusterSize, null); } public boolean scaleCluster() throws CloudRuntimeException { @@ -409,47 +436,26 @@ public boolean scaleCluster() throws CloudRuntimeException { if (LOGGER.isInfoEnabled()) { LOGGER.info(String.format("Scaling Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); } - - final KubernetesCluster.State clusterState = kubernetesCluster.getState(); final long originalClusterSize = kubernetesCluster.getNodeCount(); - final ServiceOffering existingServiceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); if (existingServiceOffering == null) { logAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, service offering for the Kubernetes cluster not found!", kubernetesCluster.getUuid())); } final boolean serviceOfferingScalingNeeded = serviceOffering != null && serviceOffering.getId() != existingServiceOffering.getId(); final boolean clusterSizeScalingNeeded = clusterSize != null && clusterSize != originalClusterSize; - - if (serviceOfferingScalingNeeded) { - validateKubernetesClusterScaleOfferingParameters(existingServiceOffering, serviceOffering); - scaleKubernetesClusterOffering(kubernetesCluster.getId(), serviceOffering, clusterSize); - } - - if (clusterSizeScalingNeeded) { - validateKubernetesClusterScaleSizeParameters(originalClusterSize, clusterSize, clusterState); - final long newVmRequiredCount = clusterSize - originalClusterSize; - final ServiceOffering clusterServiceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); - if (newVmRequiredCount > 0) { - if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.ScaleUpRequested); - } + final long newVMRequired = clusterSize == null ? 0 : clusterSize - originalClusterSize; + if (serviceOfferingScalingNeeded && clusterSizeScalingNeeded) { + if (newVMRequired > 0) { + scaleKubernetesClusterOffering(); + scaleKubernetesClusterSize(); } else { - if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.ScaleDownRequested); - } + scaleKubernetesClusterSize(); + scaleKubernetesClusterOffering(); } - - if (!serviceOfferingScalingNeeded) { // Else already updated - final long cores = clusterServiceOffering.getCpu() * (kubernetesCluster.getMasterNodeCount() + clusterSize); - final long memory = clusterServiceOffering.getRamSize() * (kubernetesCluster.getMasterNodeCount() + clusterSize); - - if (updateKubernetesClusterEntry(kubernetesCluster.getId(), clusterSize, cores, memory, null) == null) { - logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, unable to update cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } - } - - // Perform size scaling - scaleKubernetesClusterSize(originalClusterSize); + } else if (serviceOfferingScalingNeeded) { + scaleKubernetesClusterOffering(); + } else if (clusterSizeScalingNeeded) { + scaleKubernetesClusterSize(); } stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); return true; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java index d238449ff28a..e23a3b8878c8 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java @@ -263,7 +263,7 @@ public static boolean validateKubernetesClusterReadyNodesCount(final KubernetesC } try { int nodesCount = KubernetesClusterUtil.getKubernetesClusterReadyNodesCount(kubernetesCluster, ipAddress, port, - user, sshKeyFile);; + user, sshKeyFile); if (nodesCount == kubernetesCluster.getTotalNodeCount()) { if (LOGGER.isInfoEnabled()) { LOGGER.info(String.format("Kubernetes cluster ID: %s has %d ready nodes now", kubernetesCluster.getUuid(), kubernetesCluster.getTotalNodeCount())); diff --git a/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_clusters.py b/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_clusters.py index e1d78d593080..d5302ecb90f7 100644 --- a/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_clusters.py +++ b/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_clusters.py @@ -445,13 +445,14 @@ def test_06_deploy_and_invalid_upgrade_kubernetes_cluster(self): return @attr(tags=["advanced", "smoke"], required_hardware="true") - def test_07_deploy_and_scale_up_kubernetes_cluster(self): - """Test to deploy a new Kubernetes cluster and check for failure while tying to upgrade it to a lower version + def test_07_deploy_and_scale_kubernetes_cluster(self): + """Test to deploy a new Kubernetes cluster and check for failure while tying to scale it # Validate the following: # 1. createKubernetesCluster should return valid info for new cluster # 2. The Cloud Database contains the valid information - # 3. scaleKubernetesCluster should return valid info for the cluster and it should be scaled up + # 3. scaleKubernetesCluster should return valid info for the cluster when it is scaled up + # 4. scaleKubernetesCluster should return valid info for the cluster when it is scaled down """ if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) @@ -472,31 +473,7 @@ def test_07_deploy_and_scale_up_kubernetes_cluster(self): self.verifyKubernetesClusterScale(cluster_response, 2) - self.debug("Kubernetes cluster with ID: %s successfully upscaled, now deleting it" % cluster_response.id) - - self.deleteAndVerifyKubernetesCluster(cluster_response.id) - - self.debug("Kubernetes cluster with ID: %s successfully deleted" % cluster_response.id) - - return - - @attr(tags=["advanced", "smoke"], required_hardware="true") - def test_08_deploy_and_scale_down_kubernetes_cluster(self): - """Test to deploy a new Kubernetes cluster and check for failure while tying to upgrade it to a lower version - - # Validate the following: - # 1. createKubernetesCluster should return valid info for new cluster - # 2. The Cloud Database contains the valid information - # 3. scaleKubernetesCluster should return valid info for the cluster and it should be scaled down - """ - name = 'testcluster-' + random_gen() - self.debug("Creating for Kubernetes cluster with name %s" % name) - - cluster_response = self.createKubernetesCluster(name, self.kuberetes_version_2.id, 2) - - self.verifyKubernetesCluster(cluster_response, name, self.kuberetes_version_2.id, 2) - - self.debug("Kubernetes cluster with ID: %s successfully deployed, now downscaling it" % cluster_response.id) + self.debug("Kubernetes cluster with ID: %s successfully upscaled, now downscaling it" % cluster_response.id) try: cluster_response = self.scaleKubernetesCluster(cluster_response.id, 1) From b46dcc6cde3441010f14dd9c0f59910db00ab966 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 13 Jan 2020 11:45:59 +0530 Subject: [PATCH 062/134] deploy test changes Signed-off-by: Abhishek Kumar --- .../smoke/test_kubernetes_clusters.py | 69 ++++++++++++++----- 1 file changed, 50 insertions(+), 19 deletions(-) diff --git a/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_clusters.py b/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_clusters.py index d5302ecb90f7..38b823832b51 100644 --- a/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_clusters.py +++ b/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_clusters.py @@ -23,6 +23,7 @@ addKubernetesSupportedVersion, deleteKubernetesSupportedVersion, createKubernetesCluster, + stopKubernetesCluster, deleteKubernetesCluster, upgradeKubernetesCluster, scaleKubernetesCluster) @@ -271,6 +272,7 @@ def test_01_deploy_kubernetes_cluster(self): # Validate the following: # 1. createKubernetesCluster should return valid info for new cluster # 2. The Cloud Database contains the valid information + # 3. stopKubernetesCluster should stop the cluster """ if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) @@ -281,7 +283,11 @@ def test_01_deploy_kubernetes_cluster(self): self.verifyKubernetesCluster(cluster_response, name, self.kuberetes_version_2.id) - self.debug("Kubernetes cluster with ID: %s successfully deployed, now deleting it" % cluster_response.id) + self.debug("Kubernetes cluster with ID: %s successfully deployed, now stopping it" % cluster_response.id) + + self.stopAndVerifyKubernetesCluster(cluster_response.id) + + self.debug("Kubernetes cluster with ID: %s successfully stopped, now deleting it" % cluster_response.id) self.deleteAndVerifyKubernetesCluster(cluster_response.id) @@ -491,9 +497,9 @@ def test_07_deploy_and_scale_kubernetes_cluster(self): return - def listKubernetesCluster(self, clusterId): + def listKubernetesCluster(self, cluster_id): listKubernetesClustersCmd = listKubernetesClusters.listKubernetesClustersCmd() - listKubernetesClustersCmd.id = clusterId + listKubernetesClustersCmd.id = cluster_id clusterResponse = self.apiclient.listKubernetesClusters(listKubernetesClustersCmd) return clusterResponse[0] @@ -511,22 +517,28 @@ def createKubernetesCluster(self, name, version_id, size=1, master_nodes=1): self.cleanup.append(clusterResponse) return clusterResponse - def deleteKubernetesCluster(self, clusterId): + def stopKubernetesCluster(self, cluster_id): + stopKubernetesClusterCmd = stopKubernetesCluster.stopKubernetesClusterCmd() + stopKubernetesClusterCmd.id = cluster_id + response = self.apiclient.stopKubernetesCluster(stopKubernetesClusterCmd) + return response + + def deleteKubernetesCluster(self, cluster_id): deleteKubernetesClusterCmd = deleteKubernetesCluster.deleteKubernetesClusterCmd() - deleteKubernetesClusterCmd.id = clusterId + deleteKubernetesClusterCmd.id = cluster_id response = self.apiclient.deleteKubernetesCluster(deleteKubernetesClusterCmd) return response - def upgradeKubernetesCluster(self, clusterId, version_id): + def upgradeKubernetesCluster(self, cluster_id, version_id): upgradeKubernetesClusterCmd = upgradeKubernetesCluster.upgradeKubernetesClusterCmd() - upgradeKubernetesClusterCmd.id = clusterId + upgradeKubernetesClusterCmd.id = cluster_id upgradeKubernetesClusterCmd.kubernetesversionid = version_id response = self.apiclient.upgradeKubernetesCluster(upgradeKubernetesClusterCmd) return response - def scaleKubernetesCluster(self, clusterId, size): + def scaleKubernetesCluster(self, cluster_id, size): scaleKubernetesClusterCmd = scaleKubernetesCluster.scaleKubernetesClusterCmd() - scaleKubernetesClusterCmd.id = clusterId + scaleKubernetesClusterCmd.id = cluster_id scaleKubernetesClusterCmd.size = size response = self.apiclient.scaleKubernetesCluster(scaleKubernetesClusterCmd) return response @@ -534,7 +546,7 @@ def scaleKubernetesCluster(self, clusterId, size): def verifyKubernetesCluster(self, cluster_response, name, version_id, size=1, master_nodes=1): """Check if Kubernetes cluster is valid""" - self.verifyKubernetesClusterState(cluster_response) + self.verifyKubernetesClusterState(cluster_response, 'Running') self.assertEqual( cluster_response.name, @@ -560,13 +572,13 @@ def verifyKubernetesCluster(self, cluster_response, name, version_id, size=1, ma "Check KubernetesCluster name in DB {}, {}".format(db_cluster_name, name) ) - def verifyKubernetesClusterState(self, cluster_response): + def verifyKubernetesClusterState(self, cluster_response, state): """Check if Kubernetes cluster state is Running""" self.assertEqual( cluster_response.state, 'Running', - "Check KubernetesCluster state {}, {}".format(cluster_response.state, 'Running') + "Check KubernetesCluster state {}, {}".format(cluster_response.state, state) ) def verifyKubernetesClusterVersion(self, cluster_response, version_id): @@ -596,30 +608,49 @@ def verifyKubernetesClusterSize(self, cluster_response, size=1, master_nodes=1): def verifyKubernetesClusterUpgrade(self, cluster_response, version_id): """Check if Kubernetes cluster state and version are valid after upgrade""" - self.verifyKubernetesClusterState(cluster_response) + self.verifyKubernetesClusterState(cluster_response, 'Running') self.verifyKubernetesClusterVersion(cluster_response, version_id) def verifyKubernetesClusterScale(self, cluster_response, size=1, master_nodes=1): """Check if Kubernetes cluster state and node sizes are valid after upgrade""" - self.verifyKubernetesClusterState(cluster_response) + self.verifyKubernetesClusterState(cluster_response, 'Running') self.verifyKubernetesClusterSize(cluster_response, size, master_nodes) - def deleteAndVerifyKubernetesCluster(self, clusterId): + def stopAndVerifyKubernetesCluster(self, cluster_id): + """Stop Kubernetes cluster and check if it is really stopped""" + + stop_response = self.stopKubernetesCluster(cluster_id) + + self.assertEqual( + stop_response.success, + True, + "Check KubernetesCluster stop response {}, {}".format(stop_response.success, True) + ) + + db_cluster_state = self.dbclient.execute("select state from kubernetes_cluster where uuid = '%s';" % cluster_id)[0][0] + + self.assertEqual( + db_cluster_state, + 'Stopped', + "KubernetesCluster not stopped in DB, {}".format(db_cluster_state) + ) + + def deleteAndVerifyKubernetesCluster(self, cluster_id): """Delete Kubernetes cluster and check if it is really deleted""" - delete_response = self.deleteKubernetesCluster(clusterId) + delete_response = self.deleteKubernetesCluster(cluster_id) self.assertEqual( delete_response.success, True, - "Check KubernetesCluster deletion in DB {}, {}".format(delete_response.success, True) + "Check KubernetesCluster delete response {}, {}".format(delete_response.success, True) ) - db_cluster_removed = self.dbclient.execute("select removed from kubernetes_cluster where uuid = '%s';" % clusterId)[0][0] + db_cluster_removed = self.dbclient.execute("select removed from kubernetes_cluster where uuid = '%s';" % cluster_id)[0][0] self.assertNotEqual( db_cluster_removed, None, - "KubernetesCluster not removed in DB" + "KubernetesCluster not removed in DB, {}".format(db_cluster_removed) ) From f8e326bdd545288c5c16af6f0ca04fe7d9e8647b Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 14 Jan 2020 00:35:40 +0530 Subject: [PATCH 063/134] refactorings Signed-off-by: Abhishek Kumar --- .../cluster/KubernetesClusterManagerImpl.java | 29 ++-- .../cluster/KubernetesClusterService.java | 11 +- .../cluster/KubernetesClusterVmMap.java | 7 +- .../KubernetesClusterActionWorker.java | 4 +- .../KubernetesClusterDestroyWorker.java | 15 +- ...esClusterResourceModifierActionWorker.java | 2 +- .../KubernetesClusterScaleWorker.java | 8 +- .../KubernetesClusterStartWorker.java | 49 +++--- .../cluster/utils/KubernetesClusterUtil.java | 47 ++++-- .../version/KubernetesVersionManagerImpl.java | 139 ++++++++++-------- .../AddKubernetesSupportedVersionCmd.java | 9 +- .../DeleteKubernetesSupportedVersionCmd.java | 11 +- .../cluster/CreateKubernetesClusterCmd.java | 24 +-- .../cluster/DeleteKubernetesClusterCmd.java | 16 +- .../GetKubernetesClusterConfigCmd.java | 2 +- .../cluster/ListKubernetesClustersCmd.java | 2 +- .../cluster/ScaleKubernetesClusterCmd.java | 9 +- .../cluster/StartKubernetesClusterCmd.java | 11 +- .../cluster/StopKubernetesClusterCmd.java | 11 +- .../cluster/UpgradeKubernetesClusterCmd.java | 9 +- .../ListKubernetesSupportedVersionsCmd.java | 6 +- .../kubernetes-service/module.properties | 25 ++-- .../spring-kubernetes-service-context.xml | 27 ++-- ui/plugins/cks/cks.css | 32 ++-- 24 files changed, 246 insertions(+), 259 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 1bcca56fe522..c31d43cc3c04 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -76,10 +76,8 @@ import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InsufficientServerCapacityException; import com.cloud.exception.InvalidParameterValueException; -import com.cloud.exception.ManagementServerException; import com.cloud.exception.PermissionDeniedException; import com.cloud.exception.ResourceAllocationException; -import com.cloud.exception.ResourceUnavailableException; import com.cloud.host.Host.Type; import com.cloud.host.HostVO; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterActionWorker; @@ -541,7 +539,7 @@ public KubernetesClusterResponse createKubernetesClusterResponse(long kubernetes return response; } - private void validateKubernetesClusterCreateParameters(final CreateKubernetesClusterCmd cmd) throws ManagementServerException { + private void validateKubernetesClusterCreateParameters(final CreateKubernetesClusterCmd cmd) throws CloudRuntimeException { final String name = cmd.getName(); final Long zoneId = cmd.getZoneId(); final Long kubernetesVersionId = cmd.getKubernetesVersionId(); @@ -662,7 +660,7 @@ private void validateKubernetesClusterCreateParameters(final CreateKubernetesClu } private Network getKubernetesClusterNetworkIfMissing(final String clusterName, final DataCenter zone, final Account owner, final int masterNodesCount, - final int nodesCount, final String externalLoadBalancerIpAddress, final Long networkId) throws ManagementServerException { + final int nodesCount, final String externalLoadBalancerIpAddress, final Long networkId) throws CloudRuntimeException { Network network = null; if (networkId != null) { network = networkDao.findById(networkId); @@ -851,8 +849,7 @@ protected boolean stateTransitTo(long kubernetesClusterId, KubernetesCluster.Eve } @Override - public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) - throws InsufficientCapacityException, ManagementServerException { + public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) throws CloudRuntimeException { if (!KubernetesServiceEnabled.value()) { logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); } @@ -867,7 +864,11 @@ public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) final Account owner = accountService.getActiveAccountById(cmd.getEntityOwnerId()); final KubernetesSupportedVersion clusterKubernetesVersion = kubernetesSupportedVersionDao.findById(cmd.getKubernetesVersionId()); - plan(totalNodeCount, zone, serviceOffering); + try { + plan(totalNodeCount, zone, serviceOffering); + } catch (InsufficientCapacityException e) { + logAndThrow(Level.ERROR, String.format("Creating Kubernetes cluster failed due to insufficient capacity for %d cluster nodes in zone ID: %s with service offering ID: %s", totalNodeCount, zone.getUuid(), serviceOffering.getUuid())); + } final Network defaultNetwork = getKubernetesClusterNetworkIfMissing(cmd.getName(), zone, owner, (int)masterNodeCount, (int)clusterSize, cmd.getExternalLoadBalancerIpAddress(), cmd.getNetworkId()); final VMTemplateVO finalTemplate = templateDao.findByTemplateName(KubernetesClusterTemplateName.value());; @@ -902,15 +903,11 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { * @param kubernetesClusterId * @param onCreate * @return - * @throws ManagementServerException - * @throws ResourceAllocationException - * @throws ResourceUnavailableException - * @throws InsufficientCapacityException + * @throws CloudRuntimeException */ @Override - public boolean startKubernetesCluster(long kubernetesClusterId, boolean onCreate) throws ManagementServerException, - ResourceAllocationException, ResourceUnavailableException, InsufficientCapacityException { + public boolean startKubernetesCluster(long kubernetesClusterId, boolean onCreate) throws CloudRuntimeException { if (!KubernetesServiceEnabled.value()) { logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); } @@ -936,7 +933,7 @@ public boolean startKubernetesCluster(long kubernetesClusterId, boolean onCreate } final DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); if (zone == null) { - throw new CloudRuntimeException(String.format("Unable to find zone for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + logAndThrow(Level.WARN, String.format("Unable to find zone for Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); } KubernetesClusterStartWorker startWorker = new KubernetesClusterStartWorker(kubernetesCluster, this); @@ -981,7 +978,7 @@ public boolean stopKubernetesCluster(long kubernetesClusterId) throws CloudRunti } @Override - public boolean deleteKubernetesCluster(Long kubernetesClusterId) throws ManagementServerException { + public boolean deleteKubernetesCluster(Long kubernetesClusterId) throws CloudRuntimeException { if (!KubernetesServiceEnabled.value()) { logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); } @@ -1152,7 +1149,7 @@ public void reallyRun() { } else { LOGGER.warn(String.format("Garbage collection failed for Kubernetes cluster ID: %s, it will be attempted to garbage collected in next run", kubernetesCluster.getUuid())); } - } catch (ManagementServerException e) { + } catch (CloudRuntimeException e) { LOGGER.warn(String.format("Failed to destroy Kubernetes cluster ID: %s during GC", kubernetesCluster.getUuid()), e); // proceed further with rest of the Kubernetes cluster garbage collection } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java index b873ec8c28fd..b58b4c3226f0 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java @@ -27,9 +27,6 @@ import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; -import com.cloud.exception.InsufficientCapacityException; -import com.cloud.exception.ManagementServerException; -import com.cloud.exception.ResourceUnavailableException; import com.cloud.utils.component.PluggableService; import com.cloud.utils.exception.CloudRuntimeException; @@ -54,15 +51,13 @@ public interface KubernetesClusterService extends PluggableService, Configurable KubernetesCluster findById(final Long id); - KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) throws InsufficientCapacityException, - ManagementServerException, CloudRuntimeException; + KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) throws CloudRuntimeException; - boolean startKubernetesCluster(long kubernetesClusterId, boolean onCreate) throws ManagementServerException, - ResourceUnavailableException, InsufficientCapacityException, CloudRuntimeException; + boolean startKubernetesCluster(long kubernetesClusterId, boolean onCreate) throws CloudRuntimeException; boolean stopKubernetesCluster(long kubernetesClusterId) throws CloudRuntimeException; - boolean deleteKubernetesCluster(Long kubernetesClusterId) throws ManagementServerException; + boolean deleteKubernetesCluster(Long kubernetesClusterId) throws CloudRuntimeException; ListResponse listKubernetesClusters(ListKubernetesClustersCmd cmd); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMap.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMap.java index 8e5d6d4beda4..c7399202348f 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMap.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMap.java @@ -17,8 +17,11 @@ package com.cloud.kubernetes.cluster; /** - * KubernetesClusterVmMap will store a map of ID of KubernetesCuster - * and ID of its VirtualMachine + * KubernetesClusterVmMap will describe mapping of ID of KubernetesCuster + * and ID of its VirtualMachine. A KubernetesCluster can have multiple VMs + * deployed for it therefore a list of KubernetesClusterVmMap are associated + * with a KubernetesCluster. + * A particular VM can be deployed only for a single KubernetesCluster. */ public interface KubernetesClusterVmMap { long getId(); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index e1c8c26a18cd..364a671fae8f 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -19,7 +19,6 @@ import java.io.File; import java.io.IOException; -import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -64,6 +63,7 @@ import com.cloud.user.dao.SSHKeyPairDao; import com.cloud.uservm.UserVm; import com.cloud.utils.Pair; +import com.cloud.utils.StringUtils; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallback; import com.cloud.utils.db.TransactionStatus; @@ -140,7 +140,7 @@ protected void init() { } protected String readResourceFile(String resource) throws IOException { - return IOUtils.toString(Objects.requireNonNull(Thread.currentThread().getContextClassLoader().getResourceAsStream(resource)), Charset.defaultCharset().name()); + return IOUtils.toString(Objects.requireNonNull(Thread.currentThread().getContextClassLoader().getResourceAsStream(resource)), StringUtils.getPreferredCharset()); } protected void logMessage(final Level logLevel, final String message, final Exception e) { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java index ecf47ef17870..a2f1db9bd12d 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java @@ -36,6 +36,7 @@ import com.cloud.user.AccountManager; import com.cloud.user.User; import com.cloud.uservm.UserVm; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.ReservationContext; import com.cloud.vm.ReservationContextImpl; import com.cloud.vm.UserVmVO; @@ -103,11 +104,11 @@ private boolean destroyClusterVMs() { return vmDestroyed; } - private void processFailedNetworkDelete(long kubernetesClusterId) { + private void processFailedNetworkDelete(final long kubernetesClusterId) { stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); - KubernetesClusterVO cluster = kubernetesClusterDao.findById(kubernetesClusterId); - cluster.setCheckForGc(true); - kubernetesClusterDao.update(cluster.getId(), cluster); + KubernetesClusterVO kubernetesClusterVO = kubernetesClusterDao.findById(kubernetesClusterId); + kubernetesClusterVO.setCheckForGc(true); + kubernetesClusterDao.update(kubernetesClusterId, kubernetesClusterVO); } private boolean updateKubernetesClusterEntryForGC() { @@ -159,7 +160,7 @@ private void validateClusterVMsDestroyed() { } } - public boolean destroy() throws ManagementServerException, PermissionDeniedException { + public boolean destroy() throws CloudRuntimeException { init(); validateClusterSate(); if (LOGGER.isInfoEnabled()) { @@ -183,7 +184,7 @@ public boolean destroy() throws ManagementServerException, PermissionDeniedExcep String msg = String.format("Failed to destroy network of Kubernetes cluster ID: %s cleanup", kubernetesCluster.getUuid()); LOGGER.warn(msg, e); processFailedNetworkDelete(kubernetesCluster.getId()); - throw new ManagementServerException(msg, e); + throw new CloudRuntimeException(msg, e); } } } else { @@ -192,7 +193,7 @@ public boolean destroy() throws ManagementServerException, PermissionDeniedExcep LOGGER.info(msg); } processFailedNetworkDelete(kubernetesCluster.getId()); - throw new ManagementServerException(msg); + throw new CloudRuntimeException(msg); } stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); updateKubernetesClusterEntryForGC(); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index a48ab298936f..fa225243fc05 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -107,7 +107,7 @@ protected KubernetesClusterResourceModifierActionWorker(final KubernetesCluster super(kubernetesCluster, clusterManager); } - private String getKubernetesNodeConfig(String joinIp) throws IOException { + private String getKubernetesNodeConfig(final String joinIp) throws IOException { String k8sNodeConfig = readResourceFile("/conf/k8s-node.yml"); final String sshPubKey = "{{ k8s.ssh.pub.key }}"; final String joinIpKey = "{{ k8s_master.join_ip }}"; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index cea98ad3a849..fd30328bc957 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -99,7 +99,7 @@ private void logTransitStateToFailedIfNeededAndThrow(final Level logLevel, final logTransitStateToFailedIfNeededAndThrow(logLevel, message, null); } - private FirewallRule removeSshFirewallRule(IpAddress publicIp) { + private FirewallRule removeSshFirewallRule(final IpAddress publicIp) { FirewallRule rule = null; List firewallRules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(publicIp.getId(), FirewallRule.Purpose.Firewall); for (FirewallRuleVO firewallRule : firewallRules) { @@ -112,7 +112,7 @@ private FirewallRule removeSshFirewallRule(IpAddress publicIp) { return rule; } - private void removePortForwardingRules(IpAddress publicIp, Network network, Account account, List removedVMIds) throws ResourceUnavailableException { + private void removePortForwardingRules(final IpAddress publicIp, final Network network, final Account account, final List removedVMIds) throws ResourceUnavailableException { if (!CollectionUtils.isEmpty(removedVMIds)) { for (Long vmId : removedVMIds) { List pfRules = portForwardingRulesDao.listByNetwork(network.getId()); @@ -136,7 +136,7 @@ private void removePortForwardingRules(IpAddress publicIp, Network network, Acco * @param removedVMIds * @throws ManagementServerException */ - private void scaleKubernetesClusterNetworkRules(List clusterVMIds, List removedVMIds) throws ManagementServerException { + private void scaleKubernetesClusterNetworkRules(final List clusterVMIds, final List removedVMIds) throws ManagementServerException { if (!Network.GuestType.Isolated.equals(network.getGuestType())) { if (LOGGER.isDebugEnabled()) { LOGGER.debug(String.format("Network ID: %s for Kubernetes cluster ID: %s is not an isolated network, therefore, no need for network rules", network.getUuid(), kubernetesCluster.getUuid())); @@ -211,7 +211,7 @@ private KubernetesClusterVO updateKubernetesClusterEntry(final Long newSize, fin return kubernetesClusterVO; } - private boolean removeKubernetesClusterNode(String ipAddress, int port, UserVm userVm, int retries, int waitDuration) { + private boolean removeKubernetesClusterNode(final String ipAddress, final int port, final UserVm userVm, final int retries, final int waitDuration) { File pkFile = getManagementServerSshPublicKeyFile(); int retryCounter = 0; String hostName = userVm.getHostName(); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index 66fd09b11eaf..6c477fa87b67 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -116,7 +116,7 @@ private boolean isKubernetesVersionSupportsHA() { return haSupported; } - private String getKubernetesMasterConfig(final String masterIp, final String serverIp, final Account owner, + private String getKubernetesMasterConfig(final String masterIp, final String serverIp, final String hostName, final boolean haSupported) throws IOException { String k8sMasterConfig = readResourceFile("/conf/k8s-master.yml"); final String apiServerCert = "{{ k8s_master.apiserver.crt }}"; @@ -161,7 +161,7 @@ private String getKubernetesMasterConfig(final String masterIp, final String ser return k8sMasterConfig; } - private UserVm createKubernetesMaster(final Network network, final Account account, String serverIp) throws ManagementServerException, + private UserVm createKubernetesMaster(final Network network, String serverIp) throws ManagementServerException, ResourceUnavailableException, InsufficientCapacityException { UserVm masterVm = null; DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); @@ -169,7 +169,7 @@ private UserVm createKubernetesMaster(final Network network, final Account accou VirtualMachineTemplate template = templateDao.findById(kubernetesCluster.getTemplateId()); List networkIds = new ArrayList(); networkIds.add(kubernetesCluster.getNetworkId()); - Pair> ipAddresses = getKubernetesMasterIpAddresses(zone, network, account); + Pair> ipAddresses = getKubernetesMasterIpAddresses(zone, network, owner); String masterIp = ipAddresses.first(); Map requestedIps = ipAddresses.second(); if (Network.GuestType.Shared.equals(network.getGuestType()) && Strings.isNullOrEmpty(serverIp)) { @@ -185,12 +185,12 @@ private UserVm createKubernetesMaster(final Network network, final Account accou boolean haSupported = isKubernetesVersionSupportsHA(); String k8sMasterConfig = null; try { - k8sMasterConfig = getKubernetesMasterConfig(masterIp, serverIp, account, hostName, haSupported); + k8sMasterConfig = getKubernetesMasterConfig(masterIp, serverIp, hostName, haSupported); } catch (IOException e) { logAndThrow(Level.ERROR, "Failed to read Kubernetes master configuration file", e); } String base64UserData = Base64.encodeBase64String(k8sMasterConfig.getBytes(StringUtils.getPreferredCharset())); - masterVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, account, + masterVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, hostName, kubernetesCluster.getDescription(), null, null, null, null, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), requestedIps, addrs, null, null, null, customParameterMap, null, null, null, null); @@ -253,10 +253,10 @@ private UserVm createKubernetesAdditionalMaster(final String joinIp, final int a return additionalMasterVm; } - private UserVm provisionKubernetesClusterMasterVm(final Network network, final Account account, final String publicIpAddress) throws CloudRuntimeException { + private UserVm provisionKubernetesClusterMasterVm(final Network network, final String publicIpAddress) throws CloudRuntimeException { UserVm k8sMasterVM = null; try { - k8sMasterVM = createKubernetesMaster(network, account, publicIpAddress); + k8sMasterVM = createKubernetesMaster(network, publicIpAddress); addKubernetesClusterVm(kubernetesCluster.getId(), k8sMasterVM.getId()); startKubernetesVM(k8sMasterVM); k8sMasterVM = userVmDao.findById(k8sMasterVM.getId()); @@ -269,7 +269,7 @@ private UserVm provisionKubernetesClusterMasterVm(final Network network, final A return k8sMasterVM; } - private List provisionKubernetesClusterAdditionalMasterVms(final String publicIpAddress) throws ManagementServerException { + private List provisionKubernetesClusterAdditionalMasterVms(final String publicIpAddress) throws CloudRuntimeException { List additionalMasters = new ArrayList<>(); if (kubernetesCluster.getMasterNodeCount() > 1) { for (int i = 1; i < kubernetesCluster.getMasterNodeCount(); i++) { @@ -290,8 +290,8 @@ private List provisionKubernetesClusterAdditionalMasterVms(final String return additionalMasters; } - private Network startKubernetesClusterNetwork(final DeployDestination destination, final Account account) throws ManagementServerException { - final ReservationContext context = new ReservationContextImpl(null, null, null, account); + private Network startKubernetesClusterNetwork(final DeployDestination destination) throws ManagementServerException { + final ReservationContext context = new ReservationContextImpl(null, null, null, owner); Network network = networkDao.findById(kubernetesCluster.getNetworkId()); if (network == null) { String msg = String.format("Network for Kubernetes cluster ID: %s not found", kubernetesCluster.getUuid()); @@ -313,8 +313,8 @@ private Network startKubernetesClusterNetwork(final DeployDestination destinatio return network; } - private void provisionLoadBalancerRule(IpAddress publicIp, Network network, - Account account, List clusterVMIds, int port) throws NetworkRuleConflictException, + private void provisionLoadBalancerRule(final IpAddress publicIp, final Network network, + final Account account, final List clusterVMIds, final int port) throws NetworkRuleConflictException, InsufficientAddressCapacityException { LoadBalancer lb = lbService.createPublicLoadBalancerRule(null, "api-lb", "LB rule for API access", port, port, port, port, @@ -339,12 +339,10 @@ private void provisionLoadBalancerRule(IpAddress publicIp, Network network, * Open up firewall ports NODES_DEFAULT_START_SSH_PORT to NODES_DEFAULT_START_SSH_PORT+n * for SSH access. Also create port-forwarding rule to forward public IP traffic to all * @param network - * @param account * @param clusterVMs * @throws ManagementServerException */ - private void setupKubernetesClusterNetworkRules(Network network, Account account, - List clusterVMs) throws ManagementServerException { + private void setupKubernetesClusterNetworkRules(Network network, List clusterVMs) throws ManagementServerException { if (!Network.GuestType.Isolated.equals(network.getGuestType())) { if (LOGGER.isDebugEnabled()) { LOGGER.debug(String.format("Network ID: %s for Kubernetes cluster ID: %s is not an isolated network, therefore, no need for network rules", network.getUuid(), kubernetesCluster.getUuid())); @@ -361,7 +359,7 @@ private void setupKubernetesClusterNetworkRules(Network network, Account account } try { - provisionFirewallRules(publicIp, account, CLUSTER_API_PORT, CLUSTER_API_PORT); + provisionFirewallRules(publicIp, owner, CLUSTER_API_PORT, CLUSTER_API_PORT); if (LOGGER.isInfoEnabled()) { LOGGER.info(String.format("Provisioned firewall rule to open up port %d on %s for Kubernetes cluster ID: %s", CLUSTER_API_PORT, publicIp.getAddress().addr(), kubernetesCluster.getUuid())); @@ -372,7 +370,7 @@ private void setupKubernetesClusterNetworkRules(Network network, Account account try { int endPort = CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterVMs.size() - 1; - provisionFirewallRules(publicIp, account, CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort); + provisionFirewallRules(publicIp, owner, CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort); if (LOGGER.isInfoEnabled()) { LOGGER.info(String.format("Provisioned firewall rule to open up port %d to %d on %s for Kubernetes cluster ID: %s", CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort, publicIp.getAddress().addr(), kubernetesCluster.getUuid())); } @@ -382,14 +380,14 @@ private void setupKubernetesClusterNetworkRules(Network network, Account account // Load balancer rule fo API access for master node VMs try { - provisionLoadBalancerRule(publicIp, network, account, clusterVMIds, CLUSTER_API_PORT); + provisionLoadBalancerRule(publicIp, network, owner, clusterVMIds, CLUSTER_API_PORT); } catch (NetworkRuleConflictException | InsufficientAddressCapacityException e) { throw new ManagementServerException(String.format("Failed to provision load balancer rule for API access for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); } // Port forwarding rule fo SSH access on each node VM try { - provisionSshPortForwardingRules(publicIp, network, account, clusterVMIds, CLUSTER_NODES_DEFAULT_START_SSH_PORT); + provisionSshPortForwardingRules(publicIp, network, owner, clusterVMIds, CLUSTER_NODES_DEFAULT_START_SSH_PORT); } catch (ResourceUnavailableException | NetworkRuleConflictException e) { throw new ManagementServerException(String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); } @@ -455,7 +453,7 @@ private void updateKubernetesClusterEntryEndpoint() { kubernetesClusterDao.update(kubernetesCluster.getId(), kubernetesClusterVO); } - public boolean startKubernetesClusterOnCreate() throws ManagementServerException { + public boolean startKubernetesClusterOnCreate() { init(); if (LOGGER.isInfoEnabled()) { LOGGER.info(String.format("Starting Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); @@ -467,7 +465,12 @@ public boolean startKubernetesClusterOnCreate() throws ManagementServerException } catch (InsufficientCapacityException e) { logTransitStateAndThrow(Level.ERROR, String.format("Provisioning the cluster failed due to insufficient capacity in the Kubernetes cluster: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e); } - Network network = startKubernetesClusterNetwork(dest, owner); + Network network = null; + try { + network = startKubernetesClusterNetwork(dest); + } catch (ManagementServerException e) { + logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s as its network cannot be started", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e); + } Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(null); publicIpAddress = publicIpSshPort.first(); if (Strings.isNullOrEmpty(publicIpAddress) && @@ -475,7 +478,7 @@ public boolean startKubernetesClusterOnCreate() throws ManagementServerException logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s as no public IP found for the cluster" , kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); } List clusterVMs = new ArrayList<>(); - UserVm k8sMasterVM = provisionKubernetesClusterMasterVm(network, owner, publicIpAddress); + UserVm k8sMasterVM = provisionKubernetesClusterMasterVm(network, publicIpAddress); clusterVMs.add(k8sMasterVM); if (Strings.isNullOrEmpty(publicIpAddress)) { publicIpSshPort = getKubernetesClusterServerIpSshPort(k8sMasterVM); @@ -496,7 +499,7 @@ public boolean startKubernetesClusterOnCreate() throws ManagementServerException LOGGER.info(String.format("Kubernetes cluster ID: %s VMs successfully provisioned", kubernetesCluster.getUuid())); } try { - setupKubernetesClusterNetworkRules(network, owner, clusterVMs); + setupKubernetesClusterNetworkRules(network, clusterVMs); } catch (ManagementServerException e) { logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster ID: %s, unable to setup network rules", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java index e23a3b8878c8..1affcdb08acb 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java @@ -37,7 +37,7 @@ public class KubernetesClusterUtil { protected static final Logger LOGGER = Logger.getLogger(KubernetesClusterUtil.class); - public static boolean isKubernetesClusterNodeReady(KubernetesCluster kubernetesCluster, String ipAddress, int port, + public static boolean isKubernetesClusterNodeReady(final KubernetesCluster kubernetesCluster, String ipAddress, int port, String user, File sshKeyFile, String nodeName) throws Exception { Pair result = SshHelper.sshExecute(ipAddress, port, user, sshKeyFile, null, @@ -52,9 +52,9 @@ public static boolean isKubernetesClusterNodeReady(KubernetesCluster kubernetesC return false; } - public static boolean isKubernetesClusterNodeReady(KubernetesCluster kubernetesCluster, String ipAddress, int port, - String user, File sshKeyFile, String nodeName, int retries, - int waitDuration) { + public static boolean isKubernetesClusterNodeReady(final KubernetesCluster kubernetesCluster, final String ipAddress, final int port, + final String user, final File sshKeyFile, final String nodeName, final int retries, + final int waitDuration) { int retryCounter = 0; while (retryCounter < retries) { boolean ready = false; @@ -76,8 +76,23 @@ public static boolean isKubernetesClusterNodeReady(KubernetesCluster kubernetesC return false; } - public static boolean uncordonKubernetesClusterNode(KubernetesCluster kubernetesCluster, String ipAddress, int port, - String user, File sshKeyFile, UserVm userVm, int retries, int waitDuration) { + /** + * Mark a given node in a given Kubernetes cluster as schedulable. + * kubectl uncordon command will be called through SSH using IP address and port of the host virtual machine or load balancer. + * Multiple retries with a given delay can be used. + * uncordon is required when a particular node in Kubernetes cluster is drained (usually during upgrade) + * @param kubernetesCluster + * @param ipAddress + * @param port + * @param user + * @param sshKeyFile + * @param userVm + * @param retries + * @param waitDuration + * @return + */ + public static boolean uncordonKubernetesClusterNode(final KubernetesCluster kubernetesCluster, final String ipAddress, final int port, + final String user, final File sshKeyFile, final UserVm userVm, final int retries, final int waitDuration) { int retryCounter = 0; String hostName = userVm.getHostName(); if (!Strings.isNullOrEmpty(hostName)) { @@ -105,7 +120,7 @@ public static boolean uncordonKubernetesClusterNode(KubernetesCluster kubernetes return false; } - public static boolean isKubernetesClusterAddOnServiceRunning(KubernetesCluster kubernetesCluster, final String ipAddress, + public static boolean isKubernetesClusterAddOnServiceRunning(final KubernetesCluster kubernetesCluster, final String ipAddress, final int port, final String user, final File sshKeyFile, final String namespace, String serviceName) { try { @@ -134,7 +149,7 @@ public static boolean isKubernetesClusterAddOnServiceRunning(KubernetesCluster k return false; } - public static boolean isKubernetesClusterDashboardServiceRunning(KubernetesCluster kubernetesCluster, String ipAddress, + public static boolean isKubernetesClusterDashboardServiceRunning(final KubernetesCluster kubernetesCluster, String ipAddress, final int port, final String user, final File sshKeyFile, int retries, long waitDuration) { boolean running = false; @@ -161,8 +176,8 @@ public static boolean isKubernetesClusterDashboardServiceRunning(KubernetesClust return running; } - public static String getKubernetesClusterConfig(KubernetesCluster kubernetesCluster, String ipAddress, int port, - final String user, final File sshKeyFile,int retries) { + public static String getKubernetesClusterConfig(final KubernetesCluster kubernetesCluster, final String ipAddress, final int port, + final String user, final File sshKeyFile, final int retries) { int retryCounter = 0; String kubeConfig = ""; while (retryCounter < retries) { @@ -203,8 +218,8 @@ public static int getKubernetesClusterReadyNodesCount(final KubernetesCluster ku return 0; } - public static boolean isKubernetesClusterServerRunning(KubernetesCluster kubernetesCluster, String ipAddress, - int port,int retries, long waitDuration) { + public static boolean isKubernetesClusterServerRunning(final KubernetesCluster kubernetesCluster, final String ipAddress, + final int port, final int retries, final long waitDuration) { int retryCounter = 0; boolean k8sApiServerSetup = false; while (retryCounter < retries) { @@ -230,8 +245,8 @@ public static boolean isKubernetesClusterServerRunning(KubernetesCluster kuberne return k8sApiServerSetup; } - public static boolean isKubernetesClusterMasterVmRunning(final KubernetesCluster kubernetesCluster, - final String ipAddress, final int port, final long timeout) { + public static boolean isKubernetesClusterMasterVmRunning(final KubernetesCluster kubernetesCluster, final String ipAddress, + final int port, final long timeout) { boolean masterVmRunning = false; long startTime = System.currentTimeMillis(); while (!masterVmRunning && System.currentTimeMillis() - startTime < timeout) { @@ -253,9 +268,9 @@ public static boolean isKubernetesClusterMasterVmRunning(final KubernetesCluster } public static boolean validateKubernetesClusterReadyNodesCount(final KubernetesCluster kubernetesCluster, - final String ipAddress, int port, + final String ipAddress, final int port, final String user, final File sshKeyFile, - int retries, long waitDuration) { + final int retries, final long waitDuration) { int retryCounter = 0; while (retryCounter < retries) { if (LOGGER.isDebugEnabled()) { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java index 81a50771bc57..65c37529684a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java @@ -119,52 +119,6 @@ private static boolean isSemanticVersion(final String version) { return true; } - public static int compareSemanticVersions(String v1, String v2) throws IllegalArgumentException { - if (Strings.isNullOrEmpty(v1) || Strings.isNullOrEmpty(v2)) { - throw new IllegalArgumentException(String.format("Invalid version comparision with versions %s, %s", v1, v2)); - } - if(!isSemanticVersion(v1)) { - throw new IllegalArgumentException(String.format("Invalid version format, %s", v1)); - } - if(!isSemanticVersion(v2)) { - throw new IllegalArgumentException(String.format("Invalid version format, %s", v2)); - } - String[] thisParts = v1.split("\\."); - String[] thatParts = v2.split("\\."); - int length = Math.max(thisParts.length, thatParts.length); - for(int i = 0; i < length; i++) { - int thisPart = i < thisParts.length ? - Integer.parseInt(thisParts[i]) : 0; - int thatPart = i < thatParts.length ? - Integer.parseInt(thatParts[i]) : 0; - if(thisPart < thatPart) - return -1; - if(thisPart > thatPart) - return 1; - } - return 0; - } - - public static boolean canUpgradeKubernetesVersion(String currentVersion, String upgradeVersion) throws IllegalArgumentException { - int versionDiff = compareSemanticVersions(upgradeVersion, currentVersion); - if (versionDiff == 0) { - throw new IllegalArgumentException(String.format("Kubernetes clusters can not be upgraded, current version: %s, upgrade version: %s", currentVersion, upgradeVersion)); - } else if (versionDiff < 0) { - throw new IllegalArgumentException(String.format("Kubernetes clusters can not be downgraded, current version: %s, upgrade version: %s", currentVersion, upgradeVersion)); - } - String[] thisParts = currentVersion.split("\\."); - String[] thatParts = upgradeVersion.split("\\."); - int majorVerDiff = Integer.parseInt(thatParts[0]) - Integer.parseInt(thisParts[0]); - int minorVerDiff = Integer.parseInt(thatParts[1]) - Integer.parseInt(thisParts[1]); - // You only can upgrade from one MINOR version to the next MINOR version, or between PATCH versions of the same MINOR. - // That is, you cannot skip MINOR versions when you upgrade. - // For example, you can upgrade from 1.y to 1.y+1, but not from 1.y to 1.y+2 - if (majorVerDiff != 0 || minorVerDiff != 1) { - throw new IllegalArgumentException(String.format("Kubernetes clusters can be upgraded between next minor or patch version releases, current version: %s, upgrade version: %s", currentVersion, upgradeVersion)); - } - return true; - } - private List filterKubernetesSupportedVersions(List versions, final String minimumSemanticVersion) { if (!Strings.isNullOrEmpty(minimumSemanticVersion)) { for (int i = versions.size() - 1; i >= 0; --i) { @@ -210,7 +164,7 @@ private VirtualMachineTemplate registerKubernetesVersionIso(final String version return templateService.registerIso(registerIsoCmd); } - private void validateExistingTemplateForKubernetesVersionIso(VirtualMachineTemplate template, Long zoneId) { + private void validateExistingTemplateForKubernetesVersionIso(final VirtualMachineTemplate template, final Long zoneId) { if (!template.getFormat().equals(Storage.ImageFormat.ISO)) { throw new InvalidParameterValueException(String.format("%s is not an ISO", template.getUuid())); } @@ -229,6 +183,25 @@ private void validateExistingTemplateForKubernetesVersionIso(VirtualMachineTempl } } + private VMTemplateVO registerKubernetesVersionIsoIfNeeded(final Long isoId, final Long zoneId, final String name, final String isoUrl, final String isoChecksum) throws CloudRuntimeException { + VMTemplateVO templateVO = null; + if (isoId != null) { + templateVO = templateDao.findById(isoId); + } + if (templateVO == null) { + try { + VirtualMachineTemplate vmTemplate = registerKubernetesVersionIso(name, isoUrl, isoChecksum); + templateVO = templateDao.findById(vmTemplate.getId()); + } catch (IllegalAccessException | NoSuchFieldException | IllegalArgumentException | ResourceAllocationException ex) { + LOGGER.error(String.format("Unable to register binaries ISO for supported kubernetes version, %s", name), ex); + throw new CloudRuntimeException(String.format("Unable to register binaries ISO for supported kubernetes version, %s", name)); + } + } else { + validateExistingTemplateForKubernetesVersionIso(templateVO, zoneId); + } + return templateVO; + } + private void deleteKubernetesVersionIso(long templateId) throws IllegalAccessException, NoSuchFieldException, IllegalArgumentException { DeleteIsoCmd deleteIsoCmd = new DeleteIsoCmd(); @@ -239,6 +212,60 @@ private void deleteKubernetesVersionIso(long templateId) throws IllegalAccessExc templateService.deleteIso(deleteIsoCmd); } + public static int compareSemanticVersions(String v1, String v2) throws IllegalArgumentException { + if (Strings.isNullOrEmpty(v1) || Strings.isNullOrEmpty(v2)) { + throw new IllegalArgumentException(String.format("Invalid version comparision with versions %s, %s", v1, v2)); + } + if(!isSemanticVersion(v1)) { + throw new IllegalArgumentException(String.format("Invalid version format, %s", v1)); + } + if(!isSemanticVersion(v2)) { + throw new IllegalArgumentException(String.format("Invalid version format, %s", v2)); + } + String[] thisParts = v1.split("\\."); + String[] thatParts = v2.split("\\."); + int length = Math.max(thisParts.length, thatParts.length); + for(int i = 0; i < length; i++) { + int thisPart = i < thisParts.length ? + Integer.parseInt(thisParts[i]) : 0; + int thatPart = i < thatParts.length ? + Integer.parseInt(thatParts[i]) : 0; + if(thisPart < thatPart) + return -1; + if(thisPart > thatPart) + return 1; + } + return 0; + } + + /** + * Returns a boolean value whether Kubernetes cluster upgrade can be carried from a given currentVersion to upgradeVersion + * Kubernetes clusters can only be upgraded from one MINOR version to the next MINOR version, or between PATCH versions of the same MINOR. + * That is, MINOR versions cannot be skipped during upgrade. + * For example, you can upgrade from 1.y to 1.y+1, but not from 1.y to 1.y+2 + * @param currentVersion + * @param upgradeVersion + * @return + * @throws IllegalArgumentException + */ + public static boolean canUpgradeKubernetesVersion(final String currentVersion, final String upgradeVersion) throws IllegalArgumentException { + int versionDiff = compareSemanticVersions(upgradeVersion, currentVersion); + if (versionDiff == 0) { + throw new IllegalArgumentException(String.format("Kubernetes clusters can not be upgraded, current version: %s, upgrade version: %s", currentVersion, upgradeVersion)); + } else if (versionDiff < 0) { + throw new IllegalArgumentException(String.format("Kubernetes clusters can not be downgraded, current version: %s, upgrade version: %s", currentVersion, upgradeVersion)); + } + String[] thisParts = currentVersion.split("\\."); + String[] thatParts = upgradeVersion.split("\\."); + int majorVerDiff = Integer.parseInt(thatParts[0]) - Integer.parseInt(thisParts[0]); + int minorVerDiff = Integer.parseInt(thatParts[1]) - Integer.parseInt(thisParts[1]); + + if (majorVerDiff != 0 || minorVerDiff != 1) { + throw new IllegalArgumentException(String.format("Kubernetes clusters can be upgraded between next minor or patch version releases, current version: %s, upgrade version: %s", currentVersion, upgradeVersion)); + } + return true; + } + @Override public ListResponse listKubernetesSupportedVersions(final ListKubernetesSupportedVersionsCmd cmd) { if (!KubernetesClusterService.KubernetesServiceEnabled.value()) { @@ -271,7 +298,6 @@ public ListResponse listKubernetesSupportedV versions = kubernetesSupportedVersionDao.listAllInZone(zoneId); } } - // Filter versions for minimum Kubernetes version versions = filterKubernetesSupportedVersions(versions, minimumSemanticVersion); return createKubernetesSupportedVersionListResponse(versions); @@ -308,22 +334,7 @@ public KubernetesSupportedVersionResponse addKubernetesSupportedVersion(final Ad } } - VMTemplateVO template = null; - if (isoId != null) { - template = templateDao.findById(isoId); - } - if (template == null) { // register new ISO - VirtualMachineTemplate vmTemplate = null; - try { - vmTemplate = registerKubernetesVersionIso(name, isoUrl, isoChecksum); - } catch (IllegalAccessException | NoSuchFieldException | IllegalArgumentException | ResourceAllocationException ex) { - LOGGER.error(String.format("Unable to register binaries ISO for supported kubernetes version, %s", name), ex); - throw new CloudRuntimeException(String.format("Unable to register binaries ISO for supported kubernetes version, %s", name)); - } - template = templateDao.findById(vmTemplate.getId()); - } else { - validateExistingTemplateForKubernetesVersionIso(template, zoneId); - } + VMTemplateVO template = registerKubernetesVersionIsoIfNeeded(isoId, zoneId, name, isoUrl, isoChecksum); KubernetesSupportedVersionVO supportedVersionVO = new KubernetesSupportedVersionVO(name, semanticVersion, template.getId(), zoneId); supportedVersionVO = kubernetesSupportedVersionDao.persist(supportedVersionVO); diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java index 11e1a4574cb8..d0236c890d67 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java @@ -35,11 +35,7 @@ import org.apache.log4j.Logger; import com.cloud.exception.ConcurrentOperationException; -import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InvalidParameterValueException; -import com.cloud.exception.NetworkRuleConflictException; -import com.cloud.exception.ResourceAllocationException; -import com.cloud.exception.ResourceUnavailableException; import com.cloud.kubernetes.version.KubernetesSupportedVersion; import com.cloud.kubernetes.version.KubernetesVersionService; import com.cloud.utils.exception.CloudRuntimeException; @@ -136,9 +132,12 @@ public long getEntityOwnerId() { /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @Override - public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + public void execute() throws ServerApiException, ConcurrentOperationException { try { KubernetesSupportedVersionResponse response = kubernetesVersionService.addKubernetesSupportedVersion(this); + if (response == null) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add Kubernetes supported version"); + } response.setResponseName(getCommandName()); setResponseObject(response); } catch (CloudRuntimeException ex) { diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/DeleteKubernetesSupportedVersionCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/DeleteKubernetesSupportedVersionCmd.java index cccc625f5914..16526a10f9c7 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/DeleteKubernetesSupportedVersionCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/DeleteKubernetesSupportedVersionCmd.java @@ -33,10 +33,6 @@ import org.apache.log4j.Logger; import com.cloud.exception.ConcurrentOperationException; -import com.cloud.exception.InsufficientCapacityException; -import com.cloud.exception.NetworkRuleConflictException; -import com.cloud.exception.ResourceAllocationException; -import com.cloud.exception.ResourceUnavailableException; import com.cloud.kubernetes.version.KubernetesSupportedVersion; import com.cloud.kubernetes.version.KubernetesVersionEventTypes; import com.cloud.kubernetes.version.KubernetesVersionService; @@ -101,11 +97,12 @@ public String getEventDescription() { /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @Override - public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + public void execute() throws ServerApiException, ConcurrentOperationException { try { - boolean result = kubernetesVersionService.deleteKubernetesSupportedVersion(this); + if (kubernetesVersionService.deleteKubernetesSupportedVersion(this)) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to delete Kubernetes supported version ID: %d", getId())); + } SuccessResponse response = new SuccessResponse(getCommandName()); - response.setSuccess(result); setResponseObject(response); } catch (CloudRuntimeException ex) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java index 83420213c2d8..6f77301e987c 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java @@ -39,11 +39,6 @@ import org.apache.cloudstack.context.CallContext; import org.apache.log4j.Logger; -import com.cloud.exception.ConcurrentOperationException; -import com.cloud.exception.InsufficientCapacityException; -import com.cloud.exception.ManagementServerException; -import com.cloud.exception.ResourceAllocationException; -import com.cloud.exception.ResourceUnavailableException; import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; import com.cloud.kubernetes.cluster.KubernetesClusterService; @@ -275,23 +270,19 @@ public ApiCommandJobType getInstanceType() { @Override public void execute() { try { - kubernetesClusterService.startKubernetesCluster(getEntityId(), true); + if (!kubernetesClusterService.startKubernetesCluster(getEntityId(), true)) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to start Kubernetes cluster"); + } KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(getEntityId()); response.setResponseName(getCommandName()); setResponseObject(response); - } catch (InsufficientCapacityException | ResourceUnavailableException | ResourceAllocationException ex) { - LOGGER.warn("Failed to deploy Kubernetes cluster:" + getEntityUuid() + " due to " + ex.getMessage()); - throw new ServerApiException(ApiErrorCode.RESOURCE_ALLOCATION_ERROR, - "Failed to deploy Kubernetes cluster:" + getEntityUuid(), ex); - } catch (ManagementServerException ex) { - LOGGER.warn("Failed to deploy Kubernetes cluster:" + getEntityUuid() + " due to " + ex.getMessage()); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, - "Failed to deploy Kubernetes cluster:" + getEntityUuid(), ex); + } catch (CloudRuntimeException e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); } } @Override - public void create() throws ResourceAllocationException { + public void create() throws CloudRuntimeException { try { KubernetesCluster cluster = kubernetesClusterService.createKubernetesCluster(this); if (cluster == null) { @@ -299,9 +290,6 @@ public void create() throws ResourceAllocationException { } setEntityId(cluster.getId()); setEntityUuid(cluster.getUuid()); - } catch (ConcurrentOperationException | InsufficientCapacityException | ManagementServerException me ) { - LOGGER.error("Exception: ", me); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, me.getMessage()); } catch (CloudRuntimeException e) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/DeleteKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/DeleteKubernetesClusterCmd.java index a336eb8e1b1e..9617b7d0aab4 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/DeleteKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/DeleteKubernetesClusterCmd.java @@ -31,11 +31,6 @@ import org.apache.log4j.Logger; import com.cloud.exception.ConcurrentOperationException; -import com.cloud.exception.InsufficientCapacityException; -import com.cloud.exception.ManagementServerException; -import com.cloud.exception.NetworkRuleConflictException; -import com.cloud.exception.ResourceAllocationException; -import com.cloud.exception.ResourceUnavailableException; import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; import com.cloud.kubernetes.cluster.KubernetesClusterService; @@ -77,15 +72,14 @@ public Long getId() { ///////////////////////////////////////////////////// @Override - public void execute() throws ResourceUnavailableException, InsufficientCapacityException, - ServerApiException, ConcurrentOperationException, ResourceAllocationException, - NetworkRuleConflictException { + public void execute() throws ServerApiException, ConcurrentOperationException { try { - boolean result = kubernetesClusterService.deleteKubernetesCluster(id); + if (kubernetesClusterService.deleteKubernetesCluster(id)) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to delete Kubernetes cluster ID: %d", getId())); + } SuccessResponse response = new SuccessResponse(getCommandName()); - response.setSuccess(result); setResponseObject(response); - } catch (ManagementServerException | CloudRuntimeException e) { + } catch (CloudRuntimeException e) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/GetKubernetesClusterConfigCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/GetKubernetesClusterConfigCmd.java index d2b8c4019b96..c88f0eb17df7 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/GetKubernetesClusterConfigCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/GetKubernetesClusterConfigCmd.java @@ -86,7 +86,7 @@ public String getCommandName() { } @Override - public void execute() { + public void execute() throws ServerApiException { try { KubernetesClusterConfigResponse response = kubernetesClusterService.getKubernetesClusterConfig(this); response.setResponseName(getCommandName()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ListKubernetesClustersCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ListKubernetesClustersCmd.java index 58296f9cccc4..c02893a8bac6 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ListKubernetesClustersCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ListKubernetesClustersCmd.java @@ -88,7 +88,7 @@ public String getCommandName() { } @Override - public void execute() { + public void execute() throws ServerApiException { try { ListResponse response = kubernetesClusterService.listKubernetesClusters(this); response.setResponseName(getCommandName()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java index 33c44678a8c6..b83dbc705c2c 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java @@ -34,10 +34,6 @@ import org.apache.log4j.Logger; import com.cloud.exception.ConcurrentOperationException; -import com.cloud.exception.InsufficientCapacityException; -import com.cloud.exception.NetworkRuleConflictException; -import com.cloud.exception.ResourceAllocationException; -import com.cloud.exception.ResourceUnavailableException; import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; import com.cloud.kubernetes.cluster.KubernetesClusterService; @@ -117,10 +113,9 @@ public long getEntityOwnerId() { ///////////////////////////////////////////////////// @Override - public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + public void execute() throws ServerApiException, ConcurrentOperationException { try { - boolean scaled = kubernetesClusterService.scaleKubernetesCluster(this); - if (!scaled) { + if (!kubernetesClusterService.scaleKubernetesCluster(this)) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to scale Kubernetes cluster ID: %d", getId())); } final KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(getId()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java index 230a03ca6680..1ce2fe09c10f 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java @@ -31,11 +31,6 @@ import org.apache.log4j.Logger; import com.cloud.exception.ConcurrentOperationException; -import com.cloud.exception.InsufficientCapacityException; -import com.cloud.exception.ManagementServerException; -import com.cloud.exception.NetworkRuleConflictException; -import com.cloud.exception.ResourceAllocationException; -import com.cloud.exception.ResourceUnavailableException; import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; import com.cloud.kubernetes.cluster.KubernetesClusterService; @@ -108,7 +103,7 @@ public KubernetesCluster validateRequest() { } @Override - public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + public void execute() throws ServerApiException, ConcurrentOperationException { final KubernetesCluster kubernetesCluster = validateRequest(); try { if (!kubernetesClusterService.startKubernetesCluster(kubernetesCluster.getId(), false)) { @@ -117,10 +112,6 @@ public void execute() throws ResourceUnavailableException, InsufficientCapacityE final KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(kubernetesCluster.getId()); response.setResponseName(getCommandName()); setResponseObject(response); - } catch (InsufficientCapacityException | ResourceUnavailableException | ManagementServerException ex) { - String msg = String.format("Failed to start Kubernetes cluster ID: %d", getId()); - LOGGER.error(msg, ex); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, msg, ex); } catch (CloudRuntimeException ex) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StopKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StopKubernetesClusterCmd.java index 671d14bbe8e4..ba2649f863e0 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StopKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StopKubernetesClusterCmd.java @@ -32,10 +32,6 @@ import org.apache.log4j.Logger; import com.cloud.exception.ConcurrentOperationException; -import com.cloud.exception.InsufficientCapacityException; -import com.cloud.exception.NetworkRuleConflictException; -import com.cloud.exception.ResourceAllocationException; -import com.cloud.exception.ResourceUnavailableException; import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; import com.cloud.kubernetes.cluster.KubernetesClusterService; @@ -97,11 +93,12 @@ public long getEntityOwnerId() { ///////////////////////////////////////////////////// @Override - public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + public void execute() throws ServerApiException, ConcurrentOperationException { try { - boolean result = kubernetesClusterService.stopKubernetesCluster(getId()); + if (!kubernetesClusterService.stopKubernetesCluster(getId())) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to start Kubernetes cluster ID: %d", getId())); + } final SuccessResponse response = new SuccessResponse(getCommandName()); - response.setSuccess(result); setResponseObject(response); } catch (CloudRuntimeException ex) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/UpgradeKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/UpgradeKubernetesClusterCmd.java index f0130883006a..2c99b005ff46 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/UpgradeKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/UpgradeKubernetesClusterCmd.java @@ -33,10 +33,6 @@ import org.apache.log4j.Logger; import com.cloud.exception.ConcurrentOperationException; -import com.cloud.exception.InsufficientCapacityException; -import com.cloud.exception.NetworkRuleConflictException; -import com.cloud.exception.ResourceAllocationException; -import com.cloud.exception.ResourceUnavailableException; import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; import com.cloud.kubernetes.cluster.KubernetesClusterService; @@ -107,10 +103,9 @@ public long getEntityOwnerId() { ///////////////////////////////////////////////////// @Override - public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + public void execute() throws ServerApiException, ConcurrentOperationException { try { - boolean upgraded = kubernetesClusterService.upgradeKubernetesCluster(this); - if (!upgraded) { + if (!kubernetesClusterService.upgradeKubernetesCluster(this)) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %d", getId())); } final KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(getId()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/version/ListKubernetesSupportedVersionsCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/version/ListKubernetesSupportedVersionsCmd.java index 2837c1dfc524..efa029a1de53 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/version/ListKubernetesSupportedVersionsCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/version/ListKubernetesSupportedVersionsCmd.java @@ -32,10 +32,6 @@ import org.apache.log4j.Logger; import com.cloud.exception.ConcurrentOperationException; -import com.cloud.exception.InsufficientCapacityException; -import com.cloud.exception.NetworkRuleConflictException; -import com.cloud.exception.ResourceAllocationException; -import com.cloud.exception.ResourceUnavailableException; import com.cloud.kubernetes.version.KubernetesVersionService; import com.google.common.base.Strings; @@ -105,7 +101,7 @@ public String getCommandName() { /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @Override - public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + public void execute() throws ServerApiException, ConcurrentOperationException { ListResponse response = kubernetesVersionService.listKubernetesSupportedVersions(this); response.setResponseName(getCommandName()); setResponseObject(response); diff --git a/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/module.properties b/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/module.properties index 4fe375e1e3c2..e6f02da65862 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/module.properties +++ b/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/module.properties @@ -1,15 +1,18 @@ -# Copyright 2016 ShapeBlue Ltd +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 # -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. name=kubernetes-service parent=compute diff --git a/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml b/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml index 26ef4733b86a..12f2a46a8ac3 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml +++ b/plugins/integrations/kubernetes-service/src/main/resources/META-INF/cloudstack/kubernetes-service/spring-kubernetes-service-context.xml @@ -1,18 +1,21 @@ + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> Date: Tue, 14 Jan 2020 11:50:57 +0530 Subject: [PATCH 064/134] mark failed destroy for GC Signed-off-by: Abhishek Kumar --- .../kubernetes/cluster/KubernetesCluster.java | 1 - .../KubernetesClusterDestroyWorker.java | 16 +++------------- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java index 123fd5f576cc..aef304a03d36 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java @@ -100,7 +100,6 @@ enum State { s_fsm.addTransition(State.Error, Event.DestroyRequested, State.Destroying); s_fsm.addTransition(State.Destroying, Event.OperationSucceeded, State.Destroyed); - s_fsm.addTransition(State.Destroying, Event.OperationFailed, State.Error); } String _description; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java index a2f1db9bd12d..eb98f719da64 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java @@ -104,13 +104,6 @@ private boolean destroyClusterVMs() { return vmDestroyed; } - private void processFailedNetworkDelete(final long kubernetesClusterId) { - stateTransitTo(kubernetesClusterId, KubernetesCluster.Event.OperationFailed); - KubernetesClusterVO kubernetesClusterVO = kubernetesClusterDao.findById(kubernetesClusterId); - kubernetesClusterVO.setCheckForGc(true); - kubernetesClusterDao.update(kubernetesClusterId, kubernetesClusterVO); - } - private boolean updateKubernetesClusterEntryForGC() { KubernetesClusterVO kubernetesClusterVO = kubernetesClusterDao.findById(kubernetesCluster.getId()); kubernetesClusterVO.setCheckForGc(false); @@ -127,7 +120,6 @@ private void destroyKubernetesClusterNetwork() throws ManagementServerException if (!networkDestroyed) { String msg = String.format("Failed to destroy network ID: %s as part of Kubernetes cluster ID: %s cleanup", network.getUuid(), kubernetesCluster.getUuid()); LOGGER.warn(msg); - processFailedNetworkDelete(kubernetesCluster.getId()); throw new ManagementServerException(msg); } if (LOGGER.isInfoEnabled()) { @@ -183,16 +175,14 @@ public boolean destroy() throws CloudRuntimeException { } catch (Exception e) { String msg = String.format("Failed to destroy network of Kubernetes cluster ID: %s cleanup", kubernetesCluster.getUuid()); LOGGER.warn(msg, e); - processFailedNetworkDelete(kubernetesCluster.getId()); + updateKubernetesClusterEntryForGC(); throw new CloudRuntimeException(msg, e); } } } else { String msg = String.format("Failed to destroy one or more VMs as part of Kubernetes cluster ID: %s cleanup", kubernetesCluster.getUuid()); - if (LOGGER.isInfoEnabled()) { - LOGGER.info(msg); - } - processFailedNetworkDelete(kubernetesCluster.getId()); + LOGGER.warn(msg); + updateKubernetesClusterEntryForGC(); throw new CloudRuntimeException(msg); } stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); From a2b0a0e9e3c389c40a528eeaf0ae30abb66e7f0c Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 14 Jan 2020 11:51:38 +0530 Subject: [PATCH 065/134] refactorings Signed-off-by: Abhishek Kumar --- .../cluster/KubernetesClusterManagerImpl.java | 8 +------- .../KubernetesClusterActionWorker.java | 20 ++++++++++++++----- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index c31d43cc3c04..8c1a8a7f1214 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -213,13 +213,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne protected FirewallRulesDao firewallRulesDao; private void logMessage(final Level logLevel, final String message, final Exception e) { - if (logLevel == Level.DEBUG) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug(message); - } - } else if (logLevel == Level.ERROR) { - LOGGER.error(message); - } if (logLevel == Level.WARN) { + if (logLevel == Level.WARN) { if (e != null) { LOGGER.warn(message, e); } else { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 364a671fae8f..fc5a2e06b71c 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -144,13 +144,23 @@ protected String readResourceFile(String resource) throws IOException { } protected void logMessage(final Level logLevel, final String message, final Exception e) { - if (logLevel == Level.DEBUG) { + if (logLevel == Level.INFO) { + if (LOGGER.isInfoEnabled()) { + if (e != null) { + LOGGER.info(message, e); + } else { + LOGGER.info(message); + } + } + } else if (logLevel == Level.DEBUG) { if (LOGGER.isDebugEnabled()) { - LOGGER.debug(message); + if (e != null) { + LOGGER.debug(message, e); + } else { + LOGGER.debug(message); + } } - } else if (logLevel == Level.ERROR) { - LOGGER.error(message); - } if (logLevel == Level.WARN) { + } else if (logLevel == Level.WARN) { if (e != null) { LOGGER.warn(message, e); } else { From e344a14ec870690d6b8875f38efef5964c928049 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 14 Jan 2020 12:47:16 +0530 Subject: [PATCH 066/134] added centos install commands Signed-off-by: Abhishek Kumar --- scripts/util/create-kubernetes-binaries-iso.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/scripts/util/create-kubernetes-binaries-iso.sh b/scripts/util/create-kubernetes-binaries-iso.sh index dfba6130aa8e..bf97f0662f7c 100755 --- a/scripts/util/create-kubernetes-binaries-iso.sh +++ b/scripts/util/create-kubernetes-binaries-iso.sh @@ -74,8 +74,16 @@ echo "Fetching k8s docker images..." docker -v if [ $? -ne 0 ]; then echo "Installing docker..." - sudo apt update && sudo apt install docker.io -y - sudo systemctl enable docker && sudo systemctl start docker + if [ -f /etc/redhat-release ]; then + sudo yum -y remove docker-common docker container-selinux docker-selinux docker-engine + sudo yum -y install lvm2 device-mapper device-mapper-persistent-data device-mapper-event device-mapper-libs device-mapper-event-libs + sudo yum install -y http://mirror.centos.org/centos/7/extras/x86_64/Packages/container-selinux-2.107-3.el7.noarch.rpm + sudo wget https://download.docker.com/linux/centos/docker-ce.repo -O /etc/yum.repos.d/docker-ce.repo && sudo yum -y install docker-ce + sudo systemctl enable docker && sudo systemctl start docker + elif [ -f /etc/lsb-release ]; then + sudo apt update && sudo apt install docker.io -y + sudo systemctl enable docker && sudo systemctl start docker + fi fi mkdir -p "${working_dir}/docker" output=`${k8s_dir}/kubeadm config images list` @@ -84,6 +92,7 @@ while read -r line; do sudo docker pull "$line" image_name=`echo "$line" | grep -oE "[^/]+$"` sudo docker save "$line" > "${working_dir}/docker/$image_name.tar" + sudo docker image rm "$line" done <<< "$output" echo "Restore kubeadm permissions..." From 6948e152c52b5be8169d43ae1d593c7a7155345a Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 14 Jan 2020 13:09:30 +0530 Subject: [PATCH 067/134] scale cluster improvement Signed-off-by: Abhishek Kumar --- .../user/kubernetes/cluster/ScaleKubernetesClusterCmd.java | 2 +- ui/plugins/cks/cks.js | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java index b83dbc705c2c..90ccfa419175 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java @@ -40,7 +40,7 @@ import com.cloud.utils.exception.CloudRuntimeException; @APICommand(name = ScaleKubernetesClusterCmd.APINAME, - description = "Scales a created or running Kubernetes cluster", + description = "Scales a created, running or stopped Kubernetes cluster", responseObject = KubernetesClusterResponse.class, responseView = ResponseObject.ResponseView.Restricted, entityType = {KubernetesCluster.class}, diff --git a/ui/plugins/cks/cks.js b/ui/plugins/cks/cks.js index 471d19083a0b..ca56a2c2781a 100644 --- a/ui/plugins/cks/cks.js +++ b/ui/plugins/cks/cks.js @@ -697,6 +697,7 @@ $(option).attr('selected','selected'); } }); + args.$form.find('.form-item[rel=size]').find('input[name=size]').val(args.context.kubernetesclusters[0].size); }, fields: { serviceoffering: { From 24f0ba8c3a65836a395816b1d3c95b5ba62c1b73 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 14 Jan 2020 13:12:51 +0530 Subject: [PATCH 068/134] fixed delete condition check Signed-off-by: Abhishek Kumar --- .../kubernetes/version/DeleteKubernetesSupportedVersionCmd.java | 2 +- .../user/kubernetes/cluster/DeleteKubernetesClusterCmd.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/DeleteKubernetesSupportedVersionCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/DeleteKubernetesSupportedVersionCmd.java index 16526a10f9c7..1e024bc6e4a7 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/DeleteKubernetesSupportedVersionCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/DeleteKubernetesSupportedVersionCmd.java @@ -99,7 +99,7 @@ public String getEventDescription() { @Override public void execute() throws ServerApiException, ConcurrentOperationException { try { - if (kubernetesVersionService.deleteKubernetesSupportedVersion(this)) { + if (!kubernetesVersionService.deleteKubernetesSupportedVersion(this)) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to delete Kubernetes supported version ID: %d", getId())); } SuccessResponse response = new SuccessResponse(getCommandName()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/DeleteKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/DeleteKubernetesClusterCmd.java index 9617b7d0aab4..4f32138758ea 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/DeleteKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/DeleteKubernetesClusterCmd.java @@ -74,7 +74,7 @@ public Long getId() { @Override public void execute() throws ServerApiException, ConcurrentOperationException { try { - if (kubernetesClusterService.deleteKubernetesCluster(id)) { + if (!kubernetesClusterService.deleteKubernetesCluster(id)) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to delete Kubernetes cluster ID: %d", getId())); } SuccessResponse response = new SuccessResponse(getCommandName()); From 797ce7abdd8c3974fddb6e2aa136a18020dc8e15 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 14 Jan 2020 13:21:09 +0530 Subject: [PATCH 069/134] fixed PATH updation Signed-off-by: Abhishek Kumar --- .../src/main/resources/conf/k8s-master-add.yml | 10 +++++++--- .../src/main/resources/conf/k8s-master.yml | 10 +++++++--- .../src/main/resources/conf/k8s-node.yml | 10 +++++++--- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml index 0d3de7d3e1e7..052b651a2673 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml @@ -16,8 +16,6 @@ write-files: exit 0 fi - export PATH=$PATH:/opt/bin - ISO_MOUNT_DIR=/mnt/k8sdisk BINARIES_DIR=${ISO_MOUNT_DIR}/ ATTEMPT_ONLINE_INSTALL=false @@ -63,6 +61,10 @@ write-files: offline_attempts=$[$offline_attempts + 1] done + if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin + fi + if [ -d "$BINARIES_DIR" ]; then ### Binaries available offline ### echo "Installing binaries from ${BINARIES_DIR}" @@ -168,7 +170,9 @@ write-files: modprobe ip_vs_wrr modprobe ip_vs_sh modprobe nf_conntrack_ipv4 - export PATH=$PATH:/opt/bin + if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin + fi kubeadm join {{ k8s_master.join_ip }}:6443 --token {{ k8s_master.cluster.token }} --control-plane --certificate-key {{ k8s_master.cluster.ha.certificate.key }} --discovery-token-unsafe-skip-ca-verification sudo touch /home/core/success diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml index 6696179f6147..ee48add2dc08 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml @@ -36,8 +36,6 @@ write-files: exit 0 fi - export PATH=$PATH:/opt/bin - ISO_MOUNT_DIR=/mnt/k8sdisk BINARIES_DIR=${ISO_MOUNT_DIR}/ ATTEMPT_ONLINE_INSTALL=false @@ -83,6 +81,10 @@ write-files: offline_attempts=$[$offline_attempts + 1] done + if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin + fi + if [ -d "$BINARIES_DIR" ]; then ### Binaries available offline ### echo "Installing binaries from ${BINARIES_DIR}" @@ -204,7 +206,9 @@ write-files: echo "setup-kube-system is running!" exit 1 fi - export PATH=$PATH:/opt/bin + if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin + fi export KUBECONFIG=/etc/kubernetes/admin.conf mkdir -p /root/.kube diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index 4f4ac73b11a2..c22167453364 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -16,8 +16,6 @@ write-files: exit 0 fi - export PATH=$PATH:/opt/bin - ISO_MOUNT_DIR=/mnt/k8sdisk BINARIES_DIR=${ISO_MOUNT_DIR}/ ATTEMPT_ONLINE_INSTALL=false @@ -63,6 +61,10 @@ write-files: offline_attempts=$[$offline_attempts + 1] done + if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin + fi + if [ -d "$BINARIES_DIR" ]; then ### Binaries available offline ### echo "Installing binaries from ${BINARIES_DIR}" @@ -168,7 +170,9 @@ write-files: modprobe ip_vs_wrr modprobe ip_vs_sh modprobe nf_conntrack_ipv4 - export PATH=$PATH:/opt/bin + if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin + fi kubeadm join {{ k8s_master.join_ip }}:6443 --token {{ k8s_master.cluster.token }} --discovery-token-unsafe-skip-ca-verification sudo touch /home/core/success From fd7de01a03674886a0a175dad5bff98099ec5596 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 14 Jan 2020 14:09:18 +0530 Subject: [PATCH 070/134] refactored long method Signed-off-by: Abhishek Kumar --- .../cluster/KubernetesClusterServiceTest.java | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterServiceTest.java diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterServiceTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterServiceTest.java new file mode 100644 index 000000000000..8ebdc1f4acf9 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterServiceTest.java @@ -0,0 +1,56 @@ +package com.cloud.kubernetes.cluster; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class KubernetesClusterServiceTest { + + @Before + public void setUp() throws Exception { + } + + @After + public void tearDown() throws Exception { + } + + @Test + public void findById() { + } + + @Test + public void createKubernetesCluster() { + } + + @Test + public void startKubernetesCluster() { + } + + @Test + public void stopKubernetesCluster() { + } + + @Test + public void deleteKubernetesCluster() { + } + + @Test + public void listKubernetesClusters() { + } + + @Test + public void getKubernetesClusterConfig() { + } + + @Test + public void createKubernetesClusterResponse() { + } + + @Test + public void scaleKubernetesCluster() { + } + + @Test + public void upgradeKubernetesCluster() { + } +} \ No newline at end of file From e1ef5314e3820d615a94338b37917e13faece262 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 14 Jan 2020 14:23:23 +0530 Subject: [PATCH 071/134] refactored long IpAddressManager method Signed-off-by: Abhishek Kumar --- .../cluster/KubernetesClusterServiceTest.java | 56 -------- .../cloud/network/IpAddressManagerImpl.java | 125 ++++++++++-------- 2 files changed, 68 insertions(+), 113 deletions(-) delete mode 100644 plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterServiceTest.java diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterServiceTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterServiceTest.java deleted file mode 100644 index 8ebdc1f4acf9..000000000000 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterServiceTest.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.cloud.kubernetes.cluster; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -public class KubernetesClusterServiceTest { - - @Before - public void setUp() throws Exception { - } - - @After - public void tearDown() throws Exception { - } - - @Test - public void findById() { - } - - @Test - public void createKubernetesCluster() { - } - - @Test - public void startKubernetesCluster() { - } - - @Test - public void stopKubernetesCluster() { - } - - @Test - public void deleteKubernetesCluster() { - } - - @Test - public void listKubernetesClusters() { - } - - @Test - public void getKubernetesClusterConfig() { - } - - @Test - public void createKubernetesClusterResponse() { - } - - @Test - public void scaleKubernetesCluster() { - } - - @Test - public void upgradeKubernetesCluster() { - } -} \ No newline at end of file diff --git a/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java b/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java index 69b020031c1f..d840e724ca73 100644 --- a/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java +++ b/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java @@ -300,6 +300,72 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage private Random rand = new Random(System.currentTimeMillis()); + @DB + private IPAddressVO assignAndAllocateIpAddressEntry(final Account owner, final VlanType vlanUse, final Long guestNetworkId, + final boolean sourceNat, final boolean allocate, final boolean isSystem, + final Long vpcId, final Boolean displayIp, final boolean fetchFromDedicatedRange, + final List addressVOS) throws CloudRuntimeException { + return Transaction.execute((TransactionCallbackWithException) status -> { + IPAddressVO finalAddress = null; + if (!fetchFromDedicatedRange && VlanType.VirtualNetwork.equals(vlanUse)) { + // Check that the maximum number of public IPs for the given accountId will not be exceeded + try { + _resourceLimitMgr.checkResourceLimit(owner, ResourceType.public_ip); + } catch (ResourceAllocationException ex) { + s_logger.warn("Failed to allocate resource of type " + ex.getResourceType() + " for account " + owner); + throw new AccountLimitException("Maximum number of public IP addresses for account: " + owner.getAccountName() + " has been exceeded."); + } + } + + for (final IPAddressVO possibleAddr : addressVOS) { + if (possibleAddr.getState() != State.Free) { + continue; + } + final IPAddressVO addressVO = possibleAddr; + addressVO.setSourceNat(sourceNat); + addressVO.setAllocatedTime(new Date()); + addressVO.setAllocatedInDomainId(owner.getDomainId()); + addressVO.setAllocatedToAccountId(owner.getId()); + addressVO.setSystem(isSystem); + + if (displayIp != null) { + addressVO.setDisplay(displayIp); + } + + if (vlanUse != VlanType.DirectAttached) { + addressVO.setAssociatedWithNetworkId(guestNetworkId); + addressVO.setVpcId(vpcId); + } + if (_ipAddressDao.lockRow(possibleAddr.getId(), true) != null) { + final IPAddressVO userIp = _ipAddressDao.findById(addressVO.getId()); + if (userIp.getState() == State.Free) { + addressVO.setState(State.Allocating); + if (_ipAddressDao.update(addressVO.getId(), addressVO)) { + finalAddress = addressVO; + break; + } + } + } + } + + if (finalAddress == null) { + s_logger.error("Failed to fetch any free public IP address"); + throw new CloudRuntimeException("Failed to fetch any free public IP address"); + } + + if (allocate) { + markPublicIpAsAllocated(finalAddress); + } + + final State expectedAddressState = allocate ? State.Allocated : State.Allocating; + if (finalAddress.getState() != expectedAddressState) { + s_logger.error("Failed to fetch new public IP and get in expected state=" + expectedAddressState); + throw new CloudRuntimeException("Failed to fetch new public IP with expected state " + expectedAddressState); + } + return finalAddress; + }); + } + @Override public boolean configure(String name, Map params) { // populate providers @@ -821,65 +887,10 @@ public IPAddressVO doInTransaction(TransactionStatus status) throws Insufficient } assert(addrs.size() == 1) : "Return size is incorrect: " + addrs.size(); - IPAddressVO finalAddr = null; if (assign) { - if (!fetchFromDedicatedRange && VlanType.VirtualNetwork.equals(vlanUse)) { - // Check that the maximum number of public IPs for the given accountId will not be exceeded - try { - _resourceLimitMgr.checkResourceLimit(owner, ResourceType.public_ip); - } catch (ResourceAllocationException ex) { - s_logger.warn("Failed to allocate resource of type " + ex.getResourceType() + " for account " + owner); - throw new AccountLimitException("Maximum number of public IP addresses for account: " + owner.getAccountName() + " has been exceeded."); - } - } - - for (final IPAddressVO possibleAddr : addrs) { - if (possibleAddr.getState() != IpAddress.State.Free) { - continue; - } - final IPAddressVO addr = possibleAddr; - addr.setSourceNat(sourceNat); - addr.setAllocatedTime(new Date()); - addr.setAllocatedInDomainId(owner.getDomainId()); - addr.setAllocatedToAccountId(owner.getId()); - addr.setSystem(isSystem); - - if (displayIp != null) { - addr.setDisplay(displayIp); - } - - if (vlanUse != VlanType.DirectAttached) { - addr.setAssociatedWithNetworkId(guestNetworkId); - addr.setVpcId(vpcId); - } - if (_ipAddressDao.lockRow(possibleAddr.getId(), true) != null) { - final IPAddressVO userIp = _ipAddressDao.findById(addr.getId()); - if (userIp.getState() == IpAddress.State.Free) { - addr.setState(IpAddress.State.Allocating); - if (_ipAddressDao.update(addr.getId(), addr)) { - finalAddr = addr; - break; - } - } - } - } - - if (finalAddr == null) { - s_logger.error("Failed to fetch any free public IP address"); - throw new CloudRuntimeException("Failed to fetch any free public IP address"); - } - - if (allocate) { - markPublicIpAsAllocated(finalAddr); - } - - final State expectedAddressState = allocate ? State.Allocated : State.Allocating; - if (finalAddr.getState() != expectedAddressState) { - s_logger.error("Failed to fetch new public IP and get in expected state=" + expectedAddressState); - throw new CloudRuntimeException("Failed to fetch new public IP with expected state " + expectedAddressState); - } - + finalAddr = assignAndAllocateIpAddressEntry(owner, vlanUse, guestNetworkId, sourceNat, allocate, + isSystem,vpcId, displayIp, fetchFromDedicatedRange, addrs); } else { finalAddr = addrs.get(0); } From ad347c3a82efd27fef8aa5fba78237a1c23969ee Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 14 Jan 2020 15:56:55 +0530 Subject: [PATCH 072/134] changed scaling timeout Signed-off-by: Abhishek Kumar --- .../cluster/actionworkers/KubernetesClusterScaleWorker.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index fd30328bc957..93c165c66ef1 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -398,7 +398,7 @@ private void scaleUpKubernetesClusterSize(final List o KubernetesClusterVO kubernetesClusterVO = kubernetesClusterDao.findById(kubernetesCluster.getId()); kubernetesClusterVO.setNodeCount(clusterSize); boolean readyNodesCountValid = KubernetesClusterUtil.validateKubernetesClusterReadyNodesCount(kubernetesClusterVO, publicIpAddress, sshPort, - CLUSTER_NODE_VM_USER, sshKeyFile, 30, 30000); + CLUSTER_NODE_VM_USER, sshKeyFile, 20, 60000); detachIsoKubernetesVMs(clusterVMs); if (!readyNodesCountValid) { // Scaling failed logTransitStateToFailedIfNeededAndThrow(Level.ERROR, String.format("Scaling unsuccessful for Kubernetes cluster ID: %s as it does not have desired number of nodes in ready state", kubernetesCluster.getUuid())); From c1c26969b324923a08d4d4c5ee09c3a2ec8836a7 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 14 Jan 2020 16:03:30 +0530 Subject: [PATCH 073/134] fixed license issue Signed-off-by: Abhishek Kumar --- .../version/KubernetesVersionServiceTest.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionServiceTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionServiceTest.java index 3d38cb12ab98..e63bff065bc0 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionServiceTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionServiceTest.java @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package com.cloud.kubernetes.version; import static org.mockito.Mockito.when; From 85e08add9e92dc1163dc4cacfbad1b9c9a224a7d Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 14 Jan 2020 16:35:00 +0530 Subject: [PATCH 074/134] moved integration test to root test dir Signed-off-by: Abhishek Kumar --- .../integration/smoke/test_kubernetes_clusters.py | 6 +++--- .../integration/smoke/test_kubernetes_supported_versions.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) rename {plugins/integrations/kubernetes-service/test => test}/integration/smoke/test_kubernetes_clusters.py (99%) rename {plugins/integrations/kubernetes-service/test => test}/integration/smoke/test_kubernetes_supported_versions.py (99%) diff --git a/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py similarity index 99% rename from plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_clusters.py rename to test/integration/smoke/test_kubernetes_clusters.py index 38b823832b51..850f3483a91a 100644 --- a/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -61,11 +61,11 @@ def setUpClass(cls): cls.restartServer() cls.kubernetes_version_ids = [] - cls.kuberetes_version_1 = cls.addKubernetesSupportedVersion('1.14.9', 'http://172.20.0.1/files/setup-1.14.9.iso') + cls.kuberetes_version_1 = cls.addKubernetesSupportedVersion('1.14.9', 'http://staging.yadav.xyz/cks/binaries-iso/setup-1.14.9.iso') cls.kubernetes_version_ids.append(cls.kuberetes_version_1.id) - cls.kuberetes_version_2 = cls.addKubernetesSupportedVersion('1.15.0', 'http://172.20.0.1/files/setup-1.15.0.iso') + cls.kuberetes_version_2 = cls.addKubernetesSupportedVersion('1.15.0', 'http://staging.yadav.xyz/cks/binaries-iso/setup-1.15.0.iso') cls.kubernetes_version_ids.append(cls.kuberetes_version_2.id) - cls.kuberetes_version_3 = cls.addKubernetesSupportedVersion('1.16.3', 'http://172.20.0.1/files/setup-1.16.3.iso') + cls.kuberetes_version_3 = cls.addKubernetesSupportedVersion('1.16.3', 'http://staging.yadav.xyz/cks/binaries-iso/setup-1.16.3.iso') cls.kubernetes_version_ids.append(cls.kuberetes_version_3.id) cks_offering_data = { diff --git a/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_supported_versions.py b/test/integration/smoke/test_kubernetes_supported_versions.py similarity index 99% rename from plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_supported_versions.py rename to test/integration/smoke/test_kubernetes_supported_versions.py index 64eba0048701..fa49072ab357 100644 --- a/plugins/integrations/kubernetes-service/test/integration/smoke/test_kubernetes_supported_versions.py +++ b/test/integration/smoke/test_kubernetes_supported_versions.py @@ -44,7 +44,7 @@ def setUpClass(cls): cls.services = testClient.getParsedTestDataConfig() cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests()) cls.mgtSvrDetails = cls.config.__dict__["mgtSvr"][0].__dict__ - cls.kubernetes_version_iso_url = 'http://172.20.0.1/files/setup-1.16.3.iso' + cls.kubernetes_version_iso_url = 'http://staging.yadav.xyz/cks/binaries-iso/setup-1.16.3.iso' cls.initial_configuration_cks_enabled = Configurations.list(cls.apiclient, name="cloud.kubernetes.service.enabled")[0].value From 2ec7edd10ad9ecf975f9dd334c7a8ad258c331fb Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 15 Jan 2020 09:18:26 +0530 Subject: [PATCH 075/134] fix for management server restart if plugin disabled Signed-off-by: Abhishek Kumar --- test/integration/smoke/test_kubernetes_clusters.py | 4 ++-- test/integration/smoke/test_kubernetes_supported_versions.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index 850f3483a91a..cd861bd69cb9 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -54,7 +54,7 @@ def setUpClass(cls): cls.initial_configuration_cks_enabled = Configurations.list(cls.apiclient, name="cloud.kubernetes.service.enabled")[0].value - if cls.initial_configuration_cks_enabled == False: + if cls.initial_configuration_cks_enabled not in ["true", True]: Configurations.update(cls.apiclient, "cloud.kubernetes.service.enabled", "true") @@ -133,7 +133,7 @@ def tearDownClass(cls): "cloud.kubernetes.cluster.template.name", cls.initial_configuration_cks_template_name) # Restore CKS enabled - if cls.initial_configuration_cks_enabled == False: + if cls.initial_configuration_cks_enabled not in ["true", True]: Configurations.update(cls.apiclient, "cloud.kubernetes.service.enabled", "false") diff --git a/test/integration/smoke/test_kubernetes_supported_versions.py b/test/integration/smoke/test_kubernetes_supported_versions.py index fa49072ab357..d3f5a9ce53c4 100644 --- a/test/integration/smoke/test_kubernetes_supported_versions.py +++ b/test/integration/smoke/test_kubernetes_supported_versions.py @@ -48,7 +48,8 @@ def setUpClass(cls): cls.initial_configuration_cks_enabled = Configurations.list(cls.apiclient, name="cloud.kubernetes.service.enabled")[0].value - if cls.initial_configuration_cks_enabled == False: + if cls.initial_configuration_cks_enabled not in ["true", True]: + self.debug("Enabling CloudStack Kubernetes Service plugin and restarting management server") Configurations.update(cls.apiclient, "cloud.kubernetes.service.enabled", "true") @@ -61,7 +62,7 @@ def setUpClass(cls): def tearDownClass(cls): try: # Restore CKS enabled - if cls.initial_configuration_cks_enabled == False: + if cls.initial_configuration_cks_enabled not in ["true", True]: Configurations.update(cls.apiclient, "cloud.kubernetes.service.enabled", "false") From 165a27d4db2998fd77ee2b67e73895da3e8da970 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 16 Jan 2020 01:52:05 +0530 Subject: [PATCH 076/134] fixed MS restart and template, iso ready timeouts Signed-off-by: Abhishek Kumar --- test/integration/smoke/test_kubernetes_clusters.py | 14 +++++++++----- .../smoke/test_kubernetes_supported_versions.py | 12 +++++++----- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index cd861bd69cb9..5400fcee6c79 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -46,15 +46,17 @@ class TestKubernetesCluster(cloudstackTestCase): @classmethod def setUpClass(cls): - testClient = super(TestKubernetesCluster, cls).getClsTestClient() - cls.apiclient = testClient.getApiClient() - cls.services = testClient.getParsedTestDataConfig() + cls.testClient = super(TestKubernetesCluster, cls).getClsTestClient() + cls.apiclient = cls.testClient.getApiClient() + cls.services = cls.testClient.getParsedTestDataConfig() cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests()) cls.hypervisor = cls.testClient.getHypervisorInfo() + cls.mgtSvrDetails = cls.config.__dict__["mgtSvr"][0].__dict__ cls.initial_configuration_cks_enabled = Configurations.list(cls.apiclient, name="cloud.kubernetes.service.enabled")[0].value if cls.initial_configuration_cks_enabled not in ["true", True]: + cls.debug("Enabling CloudStack Kubernetes Service plugin and restarting management server") Configurations.update(cls.apiclient, "cloud.kubernetes.service.enabled", "true") @@ -134,6 +136,7 @@ def tearDownClass(cls): cls.initial_configuration_cks_template_name) # Restore CKS enabled if cls.initial_configuration_cks_enabled not in ["true", True]: + cls.debug("Restoring Kubernetes Service enabled value") Configurations.update(cls.apiclient, "cloud.kubernetes.service.enabled", "false") @@ -150,6 +153,7 @@ def tearDownClass(cls): def restartServer(cls): """Restart management server""" + cls.debug("Restarting management server") sshClient = SshClient( cls.mgtSvrDetails["mgtSvrIp"], 22, @@ -178,7 +182,7 @@ def isManagementUp(cls): return False @classmethod - def waitForTemplateReadyState(cls, template_id, retries=15, interval=15): + def waitForTemplateReadyState(cls, template_id, retries=30, interval=30): """Check if template download will finish""" while retries > -1: time.sleep(interval) @@ -202,7 +206,7 @@ def waitForTemplateReadyState(cls, template_id, retries=15, interval=15): raise Exception("Template download timed out") @classmethod - def waitForKubernetesSupportedVersionIsoReadyState(cls, version_id, retries=15, interval=15): + def waitForKubernetesSupportedVersionIsoReadyState(cls, version_id, retries=20, interval=30): """Check if Kubernetes supported version ISO is in Ready state""" while retries > -1: diff --git a/test/integration/smoke/test_kubernetes_supported_versions.py b/test/integration/smoke/test_kubernetes_supported_versions.py index d3f5a9ce53c4..0835ced251df 100644 --- a/test/integration/smoke/test_kubernetes_supported_versions.py +++ b/test/integration/smoke/test_kubernetes_supported_versions.py @@ -39,9 +39,9 @@ class TestKubernetesSupportedVersion(cloudstackTestCase): @classmethod def setUpClass(cls): - testClient = super(TestKubernetesSupportedVersion, cls).getClsTestClient() - cls.apiclient = testClient.getApiClient() - cls.services = testClient.getParsedTestDataConfig() + cls.testClient = super(TestKubernetesSupportedVersion, cls).getClsTestClient() + cls.apiclient = cls.testClient.getApiClient() + cls.services = cls.testClient.getParsedTestDataConfig() cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests()) cls.mgtSvrDetails = cls.config.__dict__["mgtSvr"][0].__dict__ cls.kubernetes_version_iso_url = 'http://staging.yadav.xyz/cks/binaries-iso/setup-1.16.3.iso' @@ -49,7 +49,7 @@ def setUpClass(cls): cls.initial_configuration_cks_enabled = Configurations.list(cls.apiclient, name="cloud.kubernetes.service.enabled")[0].value if cls.initial_configuration_cks_enabled not in ["true", True]: - self.debug("Enabling CloudStack Kubernetes Service plugin and restarting management server") + cls.debug("Enabling CloudStack Kubernetes Service plugin and restarting management server") Configurations.update(cls.apiclient, "cloud.kubernetes.service.enabled", "true") @@ -63,6 +63,7 @@ def tearDownClass(cls): try: # Restore CKS enabled if cls.initial_configuration_cks_enabled not in ["true", True]: + cls.debug("Restoring Kubernetes Service enabled value") Configurations.update(cls.apiclient, "cloud.kubernetes.service.enabled", "false") @@ -76,6 +77,7 @@ def tearDownClass(cls): def restartServer(cls): """Restart management server""" + cls.debug("Restarting management server") sshClient = SshClient( cls.mgtSvrDetails["mgtSvrIp"], 22, @@ -254,7 +256,7 @@ def deleteKubernetesSupportedVersion(self, versionId, deleteIso): response = self.apiclient.deleteKubernetesSupportedVersion(deleteKubernetesSupportedVersionCmd) return response - def waitForKubernetesSupportedVersionIsoReadyState(self, version_id, retries=15, interval=15): + def waitForKubernetesSupportedVersionIsoReadyState(self, version_id, retries=20, interval=30): """Check if Kubernetes supported version ISO is in Ready state""" while retries > -1: From d4d7885740ae7f8811007298c8f6822b9c568e34 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 16 Jan 2020 03:01:45 +0530 Subject: [PATCH 077/134] fail on exception Signed-off-by: Abhishek Kumar --- .../smoke/test_kubernetes_clusters.py | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index 5400fcee6c79..1f42bea59611 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -63,12 +63,21 @@ def setUpClass(cls): cls.restartServer() cls.kubernetes_version_ids = [] - cls.kuberetes_version_1 = cls.addKubernetesSupportedVersion('1.14.9', 'http://staging.yadav.xyz/cks/binaries-iso/setup-1.14.9.iso') - cls.kubernetes_version_ids.append(cls.kuberetes_version_1.id) - cls.kuberetes_version_2 = cls.addKubernetesSupportedVersion('1.15.0', 'http://staging.yadav.xyz/cks/binaries-iso/setup-1.15.0.iso') - cls.kubernetes_version_ids.append(cls.kuberetes_version_2.id) - cls.kuberetes_version_3 = cls.addKubernetesSupportedVersion('1.16.3', 'http://staging.yadav.xyz/cks/binaries-iso/setup-1.16.3.iso') - cls.kubernetes_version_ids.append(cls.kuberetes_version_3.id) + try: + cls.kuberetes_version_1 = cls.addKubernetesSupportedVersion('1.14.9', 'http://staging.yadav.xyz/cks/binaries-iso/setup-1.14.9.iso') + cls.kubernetes_version_ids.append(cls.kuberetes_version_1.id) + except Exception as e: + cls.fail("Failed to get Kubernetes version ISO in ready state, http://staging.yadav.xyz/cks/binaries-iso/setup-1.14.9.iso, %s" % e) + try: + cls.kuberetes_version_2 = cls.addKubernetesSupportedVersion('1.15.0', 'http://staging.yadav.xyz/cks/binaries-iso/setup-1.15.0.iso') + cls.kubernetes_version_ids.append(cls.kuberetes_version_2.id) + except Exception as e: + cls.fail("Failed to get Kubernetes version ISO in ready state, http://staging.yadav.xyz/cks/binaries-iso/setup-1.15.0.iso, %s" % e) + try: + cls.kuberetes_version_3 = cls.addKubernetesSupportedVersion('1.16.3', 'http://staging.yadav.xyz/cks/binaries-iso/setup-1.16.3.iso') + cls.kubernetes_version_ids.append(cls.kuberetes_version_3.id) + except Exception as e: + cls.fail("Failed to get Kubernetes version ISO in ready state, http://staging.yadav.xyz/cks/binaries-iso/setup-1.16.3.is, %s" % e) cks_offering_data = { "name": "CKS-Instance", @@ -105,7 +114,10 @@ def setUpClass(cls): hypervisor=cls.hypervisor ) cls.debug("Waiting for CKS template with ID %s to be ready" % cls.cks_template.id) - cls.waitForTemplateReadyState(cls.cks_template.id) + try: + cls.waitForTemplateReadyState(cls.cks_template.id) + except Exception as e: + cls.fail("Failed to get CKS template in ready state, {}, {}".format(cks_template_data.url, e)) cls.initial_configuration_cks_template_name = Configurations.list(cls.apiclient, name="cloud.kubernetes.cluster.template.name")[0].value From 8685251ac5c520f5b83bf2f0d02f8ebc394609c8 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 17 Jan 2020 17:42:15 +0530 Subject: [PATCH 078/134] fix for GC Signed-off-by: Abhishek Kumar --- .../actionworkers/KubernetesClusterDestroyWorker.java | 11 ++++++++--- .../cluster/dao/KubernetesClusterDaoImpl.java | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java index eb98f719da64..52de1d7c91c5 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java @@ -22,6 +22,7 @@ import javax.inject.Inject; import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Level; import com.cloud.exception.ManagementServerException; import com.cloud.exception.PermissionDeniedException; @@ -106,7 +107,7 @@ private boolean destroyClusterVMs() { private boolean updateKubernetesClusterEntryForGC() { KubernetesClusterVO kubernetesClusterVO = kubernetesClusterDao.findById(kubernetesCluster.getId()); - kubernetesClusterVO.setCheckForGc(false); + kubernetesClusterVO.setCheckForGc(true); return kubernetesClusterDao.update(kubernetesCluster.getId(), kubernetesClusterVO); } @@ -186,8 +187,12 @@ public boolean destroy() throws CloudRuntimeException { throw new CloudRuntimeException(msg); } stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); - updateKubernetesClusterEntryForGC(); - kubernetesClusterDao.remove(kubernetesCluster.getId()); + boolean deleted = kubernetesClusterDao.remove(kubernetesCluster.getId()); + if (!deleted) { + logMessage(Level.WARN, String.format("Failed to delete Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), null); + updateKubernetesClusterEntryForGC(); + return false; + } if (LOGGER.isInfoEnabled()) { LOGGER.info(String.format("Kubernetes cluster ID: %s is successfully deleted", kubernetesCluster.getUuid())); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDaoImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDaoImpl.java index 5d750bc199b6..003286c860bf 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDaoImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDaoImpl.java @@ -43,7 +43,7 @@ public KubernetesClusterDaoImpl() { GarbageCollectedSearch = createSearchBuilder(); GarbageCollectedSearch.and("gc", GarbageCollectedSearch.entity().isCheckForGc(), SearchCriteria.Op.EQ); - GarbageCollectedSearch.and("state", GarbageCollectedSearch.entity().getState(), SearchCriteria.Op.NEQ); + GarbageCollectedSearch.and("state", GarbageCollectedSearch.entity().getState(), SearchCriteria.Op.EQ); GarbageCollectedSearch.done(); StateSearch = createSearchBuilder(); From aed8bb0bf25632ea2dcfd655f7ffde03b4d52836 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 17 Jan 2020 17:43:42 +0530 Subject: [PATCH 079/134] fix for patch version upgrade Signed-off-by: Abhishek Kumar --- .../cloud/kubernetes/version/KubernetesVersionManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java index 65c37529684a..d161f36e4a78 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java @@ -260,7 +260,7 @@ public static boolean canUpgradeKubernetesVersion(final String currentVersion, f int majorVerDiff = Integer.parseInt(thatParts[0]) - Integer.parseInt(thisParts[0]); int minorVerDiff = Integer.parseInt(thatParts[1]) - Integer.parseInt(thisParts[1]); - if (majorVerDiff != 0 || minorVerDiff != 1) { + if (majorVerDiff != 0 || minorVerDiff > 1) { throw new IllegalArgumentException(String.format("Kubernetes clusters can be upgraded between next minor or patch version releases, current version: %s, upgrade version: %s", currentVersion, upgradeVersion)); } return true; From deb58303cbefa0ee550631994afac79fb0f1c026 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 17 Jan 2020 17:44:41 +0530 Subject: [PATCH 080/134] fix for version ISO missing check Signed-off-by: Abhishek Kumar --- .../version/KubernetesVersionManagerImpl.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java index d161f36e4a78..c5615c7301de 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java @@ -92,9 +92,11 @@ private KubernetesSupportedVersionResponse createKubernetesSupportedVersionRespo response.setSupportsHA(false); } TemplateJoinVO template = templateJoinDao.findById(kubernetesSupportedVersion.getIsoId()); - response.setIsoId(template.getUuid()); - response.setIsoName(template.getName()); - response.setIsoState(template.getState().toString()); + if (template != null) { + response.setIsoId(template.getUuid()); + response.setIsoName(template.getName()); + response.setIsoState(template.getState().toString()); + } return response; } @@ -359,11 +361,11 @@ public boolean deleteKubernetesSupportedVersion(final DeleteKubernetesSupportedV throw new CloudRuntimeException(String.format("Unable to delete Kubernetes version ID: %s. Existing clusters currently using the version.", version.getUuid())); } - VMTemplateVO template = templateDao.findById(version.getIsoId()); + VMTemplateVO template = templateDao.findByIdIncludingRemoved(version.getIsoId()); if (template == null) { LOGGER.warn(String.format("Unable to find ISO associated with supported Kubernetes version ID: %s", version.getUuid())); } - if (isDeleteIso && template != null) { // Delete ISO + if (isDeleteIso && template != null && template.getRemoved() == null) { // Delete ISO try { deleteKubernetesVersionIso(template.getId()); } catch (IllegalAccessException | NoSuchFieldException | IllegalArgumentException ex) { From 762178351e367db132d9afc7aea6d63226ea9fbc Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 17 Jan 2020 17:47:03 +0530 Subject: [PATCH 081/134] test setup, improvements Signed-off-by: Abhishek Kumar --- .../smoke/test_kubernetes_clusters.py | 169 +++++++++++------- 1 file changed, 108 insertions(+), 61 deletions(-) diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index 1f42bea59611..e1e341c8ca62 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -53,6 +53,8 @@ def setUpClass(cls): cls.hypervisor = cls.testClient.getHypervisorInfo() cls.mgtSvrDetails = cls.config.__dict__["mgtSvr"][0].__dict__ + cls.setup_failed = False + cls.initial_configuration_cks_enabled = Configurations.list(cls.apiclient, name="cloud.kubernetes.service.enabled")[0].value if cls.initial_configuration_cks_enabled not in ["true", True]: @@ -62,35 +64,39 @@ def setUpClass(cls): "true") cls.restartServer() - cls.kubernetes_version_ids = [] - try: - cls.kuberetes_version_1 = cls.addKubernetesSupportedVersion('1.14.9', 'http://staging.yadav.xyz/cks/binaries-iso/setup-1.14.9.iso') - cls.kubernetes_version_ids.append(cls.kuberetes_version_1.id) - except Exception as e: - cls.fail("Failed to get Kubernetes version ISO in ready state, http://staging.yadav.xyz/cks/binaries-iso/setup-1.14.9.iso, %s" % e) - try: - cls.kuberetes_version_2 = cls.addKubernetesSupportedVersion('1.15.0', 'http://staging.yadav.xyz/cks/binaries-iso/setup-1.15.0.iso') - cls.kubernetes_version_ids.append(cls.kuberetes_version_2.id) - except Exception as e: - cls.fail("Failed to get Kubernetes version ISO in ready state, http://staging.yadav.xyz/cks/binaries-iso/setup-1.15.0.iso, %s" % e) - try: - cls.kuberetes_version_3 = cls.addKubernetesSupportedVersion('1.16.3', 'http://staging.yadav.xyz/cks/binaries-iso/setup-1.16.3.iso') - cls.kubernetes_version_ids.append(cls.kuberetes_version_3.id) - except Exception as e: - cls.fail("Failed to get Kubernetes version ISO in ready state, http://staging.yadav.xyz/cks/binaries-iso/setup-1.16.3.is, %s" % e) + cls.cks_template = None + cls.initial_configuration_cks_template_name = None + cls.cks_service_offering = None - cks_offering_data = { - "name": "CKS-Instance", - "displaytext": "CKS Instance", - "cpunumber": 2, - "cpuspeed": 1000, - "memory": 2048, - } - cks_offering_data["name"] = cks_offering_data["name"] + '-' + random_gen() - cls.cks_service_offering = ServiceOffering.create( - cls.apiclient, - cks_offering_data - ) + cls.kubernetes_version_ids = [] + if cls.setup_failed == False: + try: + cls.kuberetes_version_1 = cls.addKubernetesSupportedVersion('1.14.9', 'http://staging.yadav.xyz/cks/binaries-iso/setup-1.14.9.iso') + cls.kubernetes_version_ids.append(cls.kuberetes_version_1.id) + except Exception as e: + cls.setup_failed = True + cls.debug("Failed to get Kubernetes version ISO in ready state, http://staging.yadav.xyz/cks/binaries-iso/setup-1.14.9.iso, %s" % e) + if cls.setup_failed == False: + try: + cls.kuberetes_version_2 = cls.addKubernetesSupportedVersion('1.15.0', 'http://staging.yadav.xyz/cks/binaries-iso/setup-1.15.0.iso') + cls.kubernetes_version_ids.append(cls.kuberetes_version_2.id) + except Exception as e: + cls.setup_failed = True + cls.debug("Failed to get Kubernetes version ISO in ready state, http://staging.yadav.xyz/cks/binaries-iso/setup-1.15.0.iso, %s" % e) + if cls.setup_failed == False: + try: + cls.kuberetes_version_3 = cls.addKubernetesSupportedVersion('1.16.0', 'http://staging.yadav.xyz/cks/binaries-iso/setup-1.16.0.iso') + cls.kubernetes_version_ids.append(cls.kuberetes_version_3.id) + except Exception as e: + cls.setup_failed = True + cls.debug("Failed to get Kubernetes version ISO in ready state, http://staging.yadav.xyz/cks/binaries-iso/setup-1.16.0.is, %s" % e) + if cls.setup_failed == False: + try: + cls.kuberetes_version_4 = cls.addKubernetesSupportedVersion('1.16.3', 'http://staging.yadav.xyz/cks/binaries-iso/setup-1.16.3.iso') + cls.kubernetes_version_ids.append(cls.kuberetes_version_4.id) + except Exception as e: + cls.setup_failed = True + cls.debug("Failed to get Kubernetes version ISO in ready state, http://staging.yadav.xyz/cks/binaries-iso/setup-1.16.3.is, %s" % e) cks_template_data = { "name": "Kubernetes-Service-Template", @@ -98,37 +104,55 @@ def setUpClass(cls): "format": "qcow2", "hypervisor": "kvm", "ostype": "CoreOS", - "url": "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-kvm.qcow2.bz2", + "url": "http://staging.yadav.xyz/cks/templates/coreos_production_cloudstack_image-kvm.qcow2.bz2", "requireshvm": "True", "ispublic": "True", "isextractable": "True" } + # "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-kvm.qcow2.bz2" if cls.hypervisor.lower() == "vmware": - cks_template_data["url"] = "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-vmware.ova" + cks_template_data["url"] = "http://staging.yadav.xyz/cks/templates/coreos_production_cloudstack_image-vmware.ova" # "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-vmware.ova" elif cls.hypervisor.lower() == "xenserver": - cks_template_data["url"] = "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-xen.vhd.bz2" - cls.cks_template = Template.register( - cls.apiclient, - cks_template_data, - zoneid=cls.zone.id, - hypervisor=cls.hypervisor - ) - cls.debug("Waiting for CKS template with ID %s to be ready" % cls.cks_template.id) - try: - cls.waitForTemplateReadyState(cls.cks_template.id) - except Exception as e: - cls.fail("Failed to get CKS template in ready state, {}, {}".format(cks_template_data.url, e)) - - cls.initial_configuration_cks_template_name = Configurations.list(cls.apiclient, - name="cloud.kubernetes.cluster.template.name")[0].value - Configurations.update(cls.apiclient, - "cloud.kubernetes.cluster.template.name", - cls.cks_template.name) - - cls._cleanup = [ - cls.cks_service_offering, - cls.cks_template - ] + cks_template_data["url"] = "http://staging.yadav.xyz/cks/templates/coreos_production_cloudstack_image-xen.vhd.bz2" # "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-xen.vhd.bz2" + if cls.setup_failed == False: + cls.cks_template = Template.register( + cls.apiclient, + cks_template_data, + zoneid=cls.zone.id, + hypervisor=cls.hypervisor + ) + cls.debug("Waiting for CKS template with ID %s to be ready" % cls.cks_template.id) + try: + cls.waitForTemplateReadyState(cls.cks_template.id) + except Exception as e: + cls.setup_failed = True + cls.debug("Failed to get CKS template in ready state, {}, {}".format(cks_template_data["url"], e)) + + cls.initial_configuration_cks_template_name = Configurations.list(cls.apiclient, + name="cloud.kubernetes.cluster.template.name")[0].value + Configurations.update(cls.apiclient, + "cloud.kubernetes.cluster.template.name", + cls.cks_template.name) + + cks_offering_data = { + "name": "CKS-Instance", + "displaytext": "CKS Instance", + "cpunumber": 2, + "cpuspeed": 1000, + "memory": 2048, + } + cks_offering_data["name"] = cks_offering_data["name"] + '-' + random_gen() + if cls.setup_failed == False: + cls.cks_service_offering = ServiceOffering.create( + cls.apiclient, + cks_offering_data + ) + + cls._cleanup = [] + if cls.cks_template != None: + cls._cleanup.append(cls.cks_template) + if cls.cks_service_offering != None: + cls._cleanup.append(cls.cks_service_offering) return @classmethod @@ -143,9 +167,14 @@ def tearDownClass(cls): cls.debug("Error: Exception during cleanup for added Kubernetes supported versions: %s" % e) try: # Restore original CKS template - Configurations.update(cls.apiclient, - "cloud.kubernetes.cluster.template.name", - cls.initial_configuration_cks_template_name) + if cls.initial_configuration_cks_template_name != None: + Configurations.update(cls.apiclient, + "cloud.kubernetes.cluster.template.name", + cls.initial_configuration_cks_template_name) + # Delete created CKS template + if cls.setup_failed == False and cls.cks_template != None: + cls.cks_template.delete(cls.apiclient, + cls.zone.id) # Restore CKS enabled if cls.initial_configuration_cks_enabled not in ["true", True]: cls.debug("Restoring Kubernetes Service enabled value") @@ -183,7 +212,9 @@ def restartServer(cls): while time.time() < timeout: if cls.isManagementUp() is True: return time.sleep(5) - return cls.fail("Management server did not come up, failing") + cls.setup_failed = True + cls.debug("Management server did not come up, failing") + return @classmethod def isManagementUp(cls): @@ -292,6 +323,8 @@ def test_01_deploy_kubernetes_cluster(self): """ if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) + if self.setup_failed == True: + self.skipTest("Setup incomplete") name = 'testcluster-' + random_gen() self.debug("Creating for Kubernetes cluster with name %s" % name) @@ -321,6 +354,8 @@ def test_02_deploy_kubernetes_ha_cluster(self): """ if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) + if self.setup_failed == True: + self.skipTest("Setup incomplete") name = 'testcluster-' + random_gen() self.debug("Creating for Kubernetes cluster with name %s" % name) @@ -346,6 +381,8 @@ def test_03_deploy_invalid_kubernetes_ha_cluster(self): """ if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) + if self.setup_failed == True: + self.skipTest("Setup incomplete") name = 'testcluster-' + random_gen() self.debug("Creating for Kubernetes cluster with name %s" % name) @@ -370,6 +407,8 @@ def test_04_deploy_and_upgrade_kubernetes_cluster(self): """ if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) + if self.setup_failed == True: + self.skipTest("Setup incomplete") name = 'testcluster-' + random_gen() self.debug("Creating for Kubernetes cluster with name %s" % name) @@ -395,6 +434,8 @@ def test_04_deploy_and_upgrade_kubernetes_cluster(self): return + + @attr(tags=["advanced", "smoke"], required_hardware="true") def test_05_deploy_and_upgrade_kubernetes_ha_cluster(self): """Test to deploy a new HA Kubernetes cluster and upgrade it to newer version @@ -405,22 +446,24 @@ def test_05_deploy_and_upgrade_kubernetes_ha_cluster(self): """ if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) + if self.setup_failed == True: + self.skipTest("Setup incomplete") name = 'testcluster-' + random_gen() self.debug("Creating for Kubernetes cluster with name %s" % name) - cluster_response = self.createKubernetesCluster(name, self.kuberetes_version_2.id, 1, 2) + cluster_response = self.createKubernetesCluster(name, self.kuberetes_version_3.id, 1, 2) - self.verifyKubernetesCluster(cluster_response, name, self.kuberetes_version_2.id, 1, 2) + self.verifyKubernetesCluster(cluster_response, name, self.kuberetes_version_3.id, 1, 2) self.debug("Kubernetes cluster with ID: %s successfully deployed, now upgrading it" % cluster_response.id) try: - cluster_response = self.upgradeKubernetesCluster(cluster_response.id, self.kuberetes_version_3.id) + cluster_response = self.upgradeKubernetesCluster(cluster_response.id, self.kuberetes_version_4.id) except Exception as e: self.deleteKubernetesCluster(cluster_response.id) self.fail("Failed to upgrade Kubernetes HA cluster due to: %s" % e) - self.verifyKubernetesClusterUpgrade(cluster_response, self.kuberetes_version_3.id) + self.verifyKubernetesClusterUpgrade(cluster_response, self.kuberetes_version_4.id) self.debug("Kubernetes cluster with ID: %s successfully upgraded, now deleting it" % cluster_response.id) @@ -441,6 +484,8 @@ def test_06_deploy_and_invalid_upgrade_kubernetes_cluster(self): """ if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) + if self.setup_failed == True: + self.skipTest("Setup incomplete") name = 'testcluster-' + random_gen() self.debug("Creating for Kubernetes cluster with name %s" % name) @@ -478,6 +523,8 @@ def test_07_deploy_and_scale_kubernetes_cluster(self): """ if self.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: self.skipTest("CKS not supported for hypervisor: %s" % self.hypervisor.lower()) + if self.setup_failed == True: + self.skipTest("Setup incomplete") name = 'testcluster-' + random_gen() self.debug("Creating for Kubernetes cluster with name %s" % name) From 587b582be92f9e3c05c7916501bea1e9c280a904 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 17 Jan 2020 18:42:05 +0530 Subject: [PATCH 082/134] kubernetes version iso will be system and alwys deleted Signed-off-by: Abhishek Kumar --- .../version/KubernetesVersionManagerImpl.java | 78 ++++++----------- .../AddKubernetesSupportedVersionCmd.java | 10 --- .../DeleteKubernetesSupportedVersionCmd.java | 7 -- .../version/KubernetesVersionServiceTest.java | 34 ++------ ui/plugins/cks/cks.js | 85 ++----------------- 5 files changed, 41 insertions(+), 173 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java index c5615c7301de..f865804ee9db 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java @@ -44,13 +44,12 @@ import com.cloud.kubernetes.cluster.KubernetesClusterVO; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; import com.cloud.kubernetes.version.dao.KubernetesSupportedVersionDao; -import com.cloud.storage.Storage; import com.cloud.storage.VMTemplateVO; -import com.cloud.storage.VMTemplateZoneVO; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateZoneDao; import com.cloud.template.TemplateApiService; import com.cloud.template.VirtualMachineTemplate; +import com.cloud.user.AccountManager; import com.cloud.utils.component.ComponentContext; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.exception.CloudRuntimeException; @@ -64,6 +63,8 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne @Inject private KubernetesClusterDao kubernetesClusterDao; @Inject + private AccountManager accountManager; + @Inject private VMTemplateDao templateDao; @Inject private TemplateJoinDao templateJoinDao; @@ -138,7 +139,7 @@ private List filterKubernetesSupportedVersions(Li return versions; } - private VirtualMachineTemplate registerKubernetesVersionIso(final String versionName, final String isoUrl, final String isoChecksum)throws IllegalAccessException, NoSuchFieldException, + private VirtualMachineTemplate registerKubernetesVersionIso(final Long zoneId, final String versionName, final String isoUrl, final String isoChecksum)throws IllegalAccessException, NoSuchFieldException, IllegalArgumentException, ResourceAllocationException { String isoName = String.format("%s-Kubernetes-Binaries-ISO", versionName); RegisterIsoCmd registerIsoCmd = new RegisterIsoCmd(); @@ -146,6 +147,11 @@ private VirtualMachineTemplate registerKubernetesVersionIso(final String version Field f = registerIsoCmd.getClass().getDeclaredField("isoName"); f.setAccessible(true); f.set(registerIsoCmd, isoName); + if (zoneId != null) { + f = registerIsoCmd.getClass().getDeclaredField("zoneId"); + f.setAccessible(true); + f.set(registerIsoCmd, zoneId); + } f = registerIsoCmd.getClass().getDeclaredField("displayText"); f.setAccessible(true); f.set(registerIsoCmd, isoName); @@ -163,47 +169,15 @@ private VirtualMachineTemplate registerKubernetesVersionIso(final String version f.setAccessible(true); f.set(registerIsoCmd, isoChecksum); } + f = registerIsoCmd.getClass().getDeclaredField("accountName"); + f.setAccessible(true); + f.set(registerIsoCmd, accountManager.getSystemAccount().getAccountName()); + f = registerIsoCmd.getClass().getDeclaredField("domainId"); + f.setAccessible(true); + f.set(registerIsoCmd, accountManager.getSystemAccount().getDomainId()); return templateService.registerIso(registerIsoCmd); } - private void validateExistingTemplateForKubernetesVersionIso(final VirtualMachineTemplate template, final Long zoneId) { - if (!template.getFormat().equals(Storage.ImageFormat.ISO)) { - throw new InvalidParameterValueException(String.format("%s is not an ISO", template.getUuid())); - } - if (!template.isPublicTemplate()) { - throw new InvalidParameterValueException(String.format("ISO ID: %s is not public", template.getUuid())); - } - if (!template.isCrossZones() && zoneId == null) { - throw new InvalidParameterValueException(String.format("ISO ID: %s is not available across zones", template.getUuid())); - } - if (!template.isCrossZones() && zoneId != null) { - List templatesZoneVOs = templateZoneDao.listByZoneTemplate(zoneId, template.getId()); - if (templatesZoneVOs.isEmpty()) { - DataCenterVO zone = dataCenterDao.findById(zoneId); - throw new InvalidParameterValueException(String.format("ISO ID: %s is not available for zone ID: %s", template.getUuid(), zone.getUuid())); - } - } - } - - private VMTemplateVO registerKubernetesVersionIsoIfNeeded(final Long isoId, final Long zoneId, final String name, final String isoUrl, final String isoChecksum) throws CloudRuntimeException { - VMTemplateVO templateVO = null; - if (isoId != null) { - templateVO = templateDao.findById(isoId); - } - if (templateVO == null) { - try { - VirtualMachineTemplate vmTemplate = registerKubernetesVersionIso(name, isoUrl, isoChecksum); - templateVO = templateDao.findById(vmTemplate.getId()); - } catch (IllegalAccessException | NoSuchFieldException | IllegalArgumentException | ResourceAllocationException ex) { - LOGGER.error(String.format("Unable to register binaries ISO for supported kubernetes version, %s", name), ex); - throw new CloudRuntimeException(String.format("Unable to register binaries ISO for supported kubernetes version, %s", name)); - } - } else { - validateExistingTemplateForKubernetesVersionIso(templateVO, zoneId); - } - return templateVO; - } - private void deleteKubernetesVersionIso(long templateId) throws IllegalAccessException, NoSuchFieldException, IllegalArgumentException { DeleteIsoCmd deleteIsoCmd = new DeleteIsoCmd(); @@ -314,21 +288,17 @@ public KubernetesSupportedVersionResponse addKubernetesSupportedVersion(final Ad String name = cmd.getName(); final String semanticVersion = cmd.getSemanticVersion(); final Long zoneId = cmd.getZoneId(); - final Long isoId = cmd.getIsoId(); final String isoUrl = cmd.getUrl(); final String isoChecksum = cmd.getChecksum(); if (compareSemanticVersions(semanticVersion, MIN_KUBERNETES_VERSION) < 0) { throw new InvalidParameterValueException(String.format("New supported Kubernetes version cannot be added as %s is minimum version supported by Kubernetes Service", MIN_KUBERNETES_VERSION)); } - if (Strings.isNullOrEmpty(isoUrl) && (isoId == null || isoId <= 0)) { - throw new InvalidParameterValueException(String.format("Either %s or %s parameter must be passed to add a new supported Kubernetes version", "isourl", ApiConstants.ISO_ID)); - } - if (!Strings.isNullOrEmpty(isoUrl) && isoId != null && isoId > 0) { - throw new InvalidParameterValueException(String.format("Both %s and %s parameters can not be passed simultaneously to add a new supported Kubernetes version", ApiConstants.URL, ApiConstants.ISO_ID)); - } if (zoneId != null && dataCenterDao.findById(zoneId) == null) { throw new InvalidParameterValueException("Invalid zone specified"); } + if (Strings.isNullOrEmpty(isoUrl)) { + throw new InvalidParameterValueException(String.format("Invalid URL for ISO specified, %s", isoUrl)); + } if (Strings.isNullOrEmpty(name)) { name = String.format("v%s", semanticVersion); if (zoneId != null) { @@ -336,7 +306,14 @@ public KubernetesSupportedVersionResponse addKubernetesSupportedVersion(final Ad } } - VMTemplateVO template = registerKubernetesVersionIsoIfNeeded(isoId, zoneId, name, isoUrl, isoChecksum); + VMTemplateVO template = null; + try { + VirtualMachineTemplate vmTemplate = registerKubernetesVersionIso(zoneId, name, isoUrl, isoChecksum); + template = templateDao.findById(vmTemplate.getId()); + } catch (IllegalAccessException | NoSuchFieldException | IllegalArgumentException | ResourceAllocationException ex) { + LOGGER.error(String.format("Unable to register binaries ISO for supported kubernetes version, %s, with url: %s", name, isoUrl), ex); + throw new CloudRuntimeException(String.format("Unable to register binaries ISO for supported kubernetes version, %s, with url: %s", name, isoUrl)); + } KubernetesSupportedVersionVO supportedVersionVO = new KubernetesSupportedVersionVO(name, semanticVersion, template.getId(), zoneId); supportedVersionVO = kubernetesSupportedVersionDao.persist(supportedVersionVO); @@ -351,7 +328,6 @@ public boolean deleteKubernetesSupportedVersion(final DeleteKubernetesSupportedV throw new CloudRuntimeException("Kubernetes Service plugin is disabled"); } final Long versionId = cmd.getId(); - final boolean isDeleteIso = cmd.isDeleteIso(); KubernetesSupportedVersion version = kubernetesSupportedVersionDao.findById(versionId); if (version == null) { throw new InvalidParameterValueException("Invalid Kubernetes version id specified"); @@ -365,7 +341,7 @@ public boolean deleteKubernetesSupportedVersion(final DeleteKubernetesSupportedV if (template == null) { LOGGER.warn(String.format("Unable to find ISO associated with supported Kubernetes version ID: %s", version.getUuid())); } - if (isDeleteIso && template != null && template.getRemoved() == null) { // Delete ISO + if (template != null && template.getRemoved() == null) { // Delete ISO try { deleteKubernetesVersionIso(template.getId()); } catch (IllegalAccessException | NoSuchFieldException | IllegalArgumentException ex) { diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java index d0236c890d67..923211328650 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java @@ -29,7 +29,6 @@ import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.admin.AdminCmd; import org.apache.cloudstack.api.response.KubernetesSupportedVersionResponse; -import org.apache.cloudstack.api.response.TemplateResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.context.CallContext; import org.apache.log4j.Logger; @@ -70,11 +69,6 @@ public class AddKubernetesSupportedVersionCmd extends BaseCmd implements AdminCm description = "the ID of the zone in which Kubernetes supported version will be available") private Long zoneId; - @Parameter(name = ApiConstants.ISO_ID, type = CommandType.UUID, - entityType = TemplateResponse.class, - description = "the ID of the binaries ISO for Kubernetes supported version") - private Long isoId; - @Parameter(name = ApiConstants.URL, type = CommandType.STRING, description = "the URL of the binaries ISO for Kubernetes supported version") private String url; @@ -106,10 +100,6 @@ public Long getZoneId() { return zoneId; } - public Long getIsoId() { - return isoId; - } - public String getUrl() { return url; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/DeleteKubernetesSupportedVersionCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/DeleteKubernetesSupportedVersionCmd.java index 1e024bc6e4a7..02489147c65a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/DeleteKubernetesSupportedVersionCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/DeleteKubernetesSupportedVersionCmd.java @@ -58,9 +58,6 @@ public class DeleteKubernetesSupportedVersionCmd extends BaseAsyncCmd implements description = "the ID of the Kubernetes supported version", required = true) private Long id; - @Parameter(name = ApiConstants.DELETE_ISO, type = CommandType.BOOLEAN, - description = "true if ISO associated with the Kubernetes version to be deleted else false. Default is false") - private Boolean deleteIso; ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// @@ -69,10 +66,6 @@ public Long getId() { return id; } - public Boolean isDeleteIso() { - return deleteIso == null ? false : deleteIso; - } - @Override public String getCommandName() { return APINAME.toLowerCase() + "response"; diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionServiceTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionServiceTest.java index e63bff065bc0..f1cb85f6797e 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionServiceTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionServiceTest.java @@ -50,16 +50,16 @@ import com.cloud.dc.dao.DataCenterDao; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.ResourceAllocationException; -import com.cloud.kubernetes.version.dao.KubernetesSupportedVersionDao; import com.cloud.kubernetes.cluster.KubernetesClusterService; import com.cloud.kubernetes.cluster.KubernetesClusterVO; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; -import com.cloud.storage.Storage; +import com.cloud.kubernetes.version.dao.KubernetesSupportedVersionDao; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.template.TemplateApiService; import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.Account; +import com.cloud.user.AccountManager; import com.cloud.user.AccountVO; import com.cloud.user.User; import com.cloud.user.UserVO; @@ -78,6 +78,8 @@ public class KubernetesVersionServiceTest { @Mock private KubernetesClusterDao kubernetesClusterDao; @Mock + private AccountManager accountManager; + @Mock private VMTemplateDao templateDao; @Mock private TemplateJoinDao templateJoinDao; @@ -141,32 +143,13 @@ public void addKubernetesSupportedVersionLowerUnsupportedTest() { } @Test(expected = InvalidParameterValueException.class) - public void addKubernetesSupportedVersionIsoIdUrlTest() { + public void addKubernetesSupportedVersionEmptyUrlTest() { AddKubernetesSupportedVersionCmd cmd = Mockito.mock(AddKubernetesSupportedVersionCmd.class); AccountVO account = new AccountVO("admin", 1L, "", Account.ACCOUNT_TYPE_ADMIN, "uuid"); UserVO user = new UserVO(1, "adminuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); when(cmd.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); CallContext.register(user, account); - when(cmd.getIsoId()).thenReturn(1L); - when(cmd.getUrl()).thenReturn("url"); - kubernetesVersionService.addKubernetesSupportedVersion(cmd); - } - - @Test - public void addKubernetesSupportedVersionIsoIdTest() { - AddKubernetesSupportedVersionCmd cmd = Mockito.mock(AddKubernetesSupportedVersionCmd.class); - AccountVO account = new AccountVO("admin", 1L, "", Account.ACCOUNT_TYPE_ADMIN, "uuid"); - UserVO user = new UserVO(1, "adminuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); - CallContext.register(user, account); - when(cmd.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); - when(cmd.getIsoId()).thenReturn(1L); - when(cmd.getUrl()).thenReturn(null); - VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); - when(templateVO.getId()).thenReturn(1L); - when(templateVO.getFormat()).thenReturn(Storage.ImageFormat.ISO); - when(templateVO.isPublicTemplate()).thenReturn(true); - when(templateVO.isCrossZones()).thenReturn(true); - when(templateDao.findById(Mockito.anyLong())).thenReturn(templateVO); + when(cmd.getUrl()).thenReturn(""); kubernetesVersionService.addKubernetesSupportedVersion(cmd); } @@ -177,9 +160,10 @@ public void addKubernetesSupportedVersionIsoUrlTest() throws ResourceAllocationE UserVO user = new UserVO(1, "adminuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); CallContext.register(user, account); when(cmd.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); - when(cmd.getIsoId()).thenReturn(null); when(cmd.getUrl()).thenReturn("https://download.cloudstack.com"); when(cmd.getChecksum()).thenReturn(null); + Account systemAccount = new AccountVO("system", 1L, "", Account.ACCOUNT_TYPE_ADMIN, "uuid"); + when(accountManager.getSystemAccount()).thenReturn(systemAccount); PowerMockito.mockStatic(ComponentContext.class); when(ComponentContext.inject(Mockito.any(RegisterIsoCmd.class))).thenReturn(new RegisterIsoCmd()); when(templateService.registerIso(Mockito.any(RegisterIsoCmd.class))).thenReturn(Mockito.mock(VirtualMachineTemplate.class)); @@ -195,7 +179,6 @@ public void deleteKubernetesSupportedVersionExistingClustersTest() { AccountVO account = new AccountVO("admin", 1L, "", Account.ACCOUNT_TYPE_ADMIN, "uuid"); UserVO user = new UserVO(1, "adminuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); CallContext.register(user, account); - when(cmd.isDeleteIso()).thenReturn(true); when(kubernetesSupportedVersionDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(KubernetesSupportedVersionVO.class)); List clusters = new ArrayList<>(); clusters.add(Mockito.mock(KubernetesClusterVO.class)); @@ -209,7 +192,6 @@ public void deleteKubernetesSupportedVersionTest() { AccountVO account = new AccountVO("admin", 1L, "", Account.ACCOUNT_TYPE_ADMIN, "uuid"); UserVO user = new UserVO(1, "adminuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); CallContext.register(user, account); - when(cmd.isDeleteIso()).thenReturn(true); when(kubernetesSupportedVersionDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(KubernetesSupportedVersionVO.class)); List clusters = new ArrayList<>(); when(kubernetesClusterDao.listAllByKubernetesVersion(Mockito.anyLong())).thenReturn(clusters); diff --git a/ui/plugins/cks/cks.js b/ui/plugins/cks/cks.js index ca56a2c2781a..8b7a23363ca4 100644 --- a/ui/plugins/cks/cks.js +++ b/ui/plugins/cks/cks.js @@ -1184,64 +1184,16 @@ }); } }, - isoid: { - label: 'label.iso', - //docID: 'helpKubernetesClusterZone', - validation: { - required: true - }, - select: function(args) { - $.ajax({ - url: createURL("listIsos&ispublic=true&bootable=false"), - data: { zoneid: args.zone }, - dataType: "json", - async: true, - success: function(json) { - var items = []; - var isoObjs = json.listisosresponse.iso; - if (isoObjs != null) { - for (var i = 0; i < isoObjs.length; i++) { - items.push({ - id: isoObjs[i].id, - description: isoObjs[i].name - }); - } - } - items.sort(function(a, b) { - return a.description.localeCompare(b.description); - }); - items.unshift({ - id: -1, - description: 'label.add.new.iso' - }); - args.response.success({ - data: items - }); - } - }); - - args.$select.change(function() { - var $form = $(this).closest('form'); - var currentIsoId = $(this).val(); - if (currentIsoId < 0) { - $form.find('.form-item[rel=isourl]').css('display', 'inline-block'); - $form.find('.form-item[rel=isochecksum]').css('display', 'inline-block'); - } else { - $form.find('.form-item[rel=isourl]').hide(); - $form.find('.form-item[rel=isochecksum]').hide(); - } - }); - } - }, isourl: { label: 'label.url', //docID: 'Name of the cluster', - isHidden: true + validation: { + required: true + } }, isochecksum: { label: 'label.checksum', //docID: 'Name of the cluster', - isHidden: true }, } }, @@ -1250,28 +1202,14 @@ var data = { name: args.data.name, semanticversion: args.data.version, + url: args.data.isourl, + checksum: args.data.isochecksum }; if (args.data.zone != null && args.data.zone != -1) { $.extend(data, { zoneid: args.data.zone }); } - if (args.data.isoid < 0) { - if (args.data.isourl == null || args.data.isourl == '') { - cloudStack.dialog.notice({ - message: 'ISO URL is required to a new ISO' - }); - return; - } - $.extend(data, { - url: args.data.isourl, - checksum: args.data.isochecksum - }); - } else { - $.extend(data, { - isoid: args.data.isoid - }); - } $.ajax({ url: createURL('addKubernetesSupportedVersion'), data: data, @@ -1333,13 +1271,7 @@ title: 'label.delete.kubernetes.version', desc: 'label.delete.kubernetes.version', isWarning: true, - fields: { - deleteiso: { - label: 'label.delete.iso', - isBoolean: true, - isChecked: false - }, - } + fields: {} }, messages: { confirm: function(args) { @@ -1353,11 +1285,6 @@ var data = { id: args.context.kubernetesversions[0].id }; - if (args.data.deleteiso === 'on') { - $.extend(data, { - deleteiso: true - }); - } $.ajax({ url: createURL('deleteKubernetesSupportedVersion'), data: data, From a4f6445ad4ebb03299f406d62fbd027f7603ae27 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 20 Jan 2020 01:25:14 +0530 Subject: [PATCH 083/134] replaced accessing api cmd field names with getters Signed-off-by: Abhishek Kumar --- .../api/command/user/iso/DeleteIsoCmd.java | 4 ++ .../api/command/user/iso/RegisterIsoCmd.java | 32 +++++++++++++++ .../version/KubernetesVersionManagerImpl.java | 40 +++++-------------- 3 files changed, 45 insertions(+), 31 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/DeleteIsoCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/DeleteIsoCmd.java index 103e9227c248..b38a24f8ad7f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/DeleteIsoCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/DeleteIsoCmd.java @@ -61,6 +61,10 @@ public Long getId() { return id; } + public void setId(Long id) { + this.id = id; + } + public Long getZoneId() { return zoneId; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java index a06b54f1a01a..e6a62925afab 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java @@ -127,10 +127,18 @@ public Boolean isBootable() { return bootable; } + public void setBootable(Boolean bootable) { + this.bootable = bootable; + } + public String getDisplayText() { return displayText; } + public void setDisplayText(String displayText) { + this.displayText = displayText; + } + public Boolean isFeatured() { return featured; } @@ -147,6 +155,10 @@ public String getIsoName() { return isoName; } + public void setIsoName(String isoName) { + this.isoName = isoName; + } + public Long getOsTypeId() { return osTypeId; } @@ -155,22 +167,42 @@ public String getUrl() { return url; } + public void setUrl(String url) { + this.url = url; + } + public Long getZoneId() { return zoneId; } + public void setZoneId(Long zoneId) { + this.zoneId = zoneId; + } + public Long getDomainId() { return domainId; } + public void setDomainId(Long domainId) { + this.domainId = domainId; + } + public String getAccountName() { return accountName; } + public void setAccountName(String accountName) { + this.accountName = accountName; + } + public String getChecksum() { return checksum; } + public void setChecksum(String checksum) { + this.checksum = checksum; + } + public String getImageStoreUuid() { return imageStoreUuid; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java index f865804ee9db..36d5fe314d04 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java @@ -17,7 +17,6 @@ package com.cloud.kubernetes.version; -import java.lang.reflect.Field; import java.util.ArrayList; import java.util.List; @@ -144,37 +143,18 @@ private VirtualMachineTemplate registerKubernetesVersionIso(final Long zoneId, f String isoName = String.format("%s-Kubernetes-Binaries-ISO", versionName); RegisterIsoCmd registerIsoCmd = new RegisterIsoCmd(); registerIsoCmd = ComponentContext.inject(registerIsoCmd); - Field f = registerIsoCmd.getClass().getDeclaredField("isoName"); - f.setAccessible(true); - f.set(registerIsoCmd, isoName); + registerIsoCmd.setIsoName(isoName); if (zoneId != null) { - f = registerIsoCmd.getClass().getDeclaredField("zoneId"); - f.setAccessible(true); - f.set(registerIsoCmd, zoneId); + registerIsoCmd.setZoneId(zoneId); } - f = registerIsoCmd.getClass().getDeclaredField("displayText"); - f.setAccessible(true); - f.set(registerIsoCmd, isoName); - f = registerIsoCmd.getClass().getDeclaredField("bootable"); - f.setAccessible(true); - f.set(registerIsoCmd, false); - f = registerIsoCmd.getClass().getDeclaredField("publicIso"); - f.setAccessible(true); - f.set(registerIsoCmd, true); - f = registerIsoCmd.getClass().getDeclaredField("url"); - f.setAccessible(true); - f.set(registerIsoCmd, isoUrl); + registerIsoCmd.setDisplayText(isoName); + registerIsoCmd.setBootable(false); + registerIsoCmd.setUrl(isoUrl); if (!Strings.isNullOrEmpty(isoChecksum)) { - f = registerIsoCmd.getClass().getDeclaredField("checksum"); - f.setAccessible(true); - f.set(registerIsoCmd, isoChecksum); + registerIsoCmd.setChecksum(isoChecksum); } - f = registerIsoCmd.getClass().getDeclaredField("accountName"); - f.setAccessible(true); - f.set(registerIsoCmd, accountManager.getSystemAccount().getAccountName()); - f = registerIsoCmd.getClass().getDeclaredField("domainId"); - f.setAccessible(true); - f.set(registerIsoCmd, accountManager.getSystemAccount().getDomainId()); + registerIsoCmd.setAccountName(accountManager.getSystemAccount().getAccountName()); + registerIsoCmd.setDomainId(accountManager.getSystemAccount().getDomainId()); return templateService.registerIso(registerIsoCmd); } @@ -182,9 +162,7 @@ private void deleteKubernetesVersionIso(long templateId) throws IllegalAccessExc IllegalArgumentException { DeleteIsoCmd deleteIsoCmd = new DeleteIsoCmd(); deleteIsoCmd = ComponentContext.inject(deleteIsoCmd); - Field f = deleteIsoCmd.getClass().getDeclaredField("id"); - f.setAccessible(true); - f.set(deleteIsoCmd, templateId); + deleteIsoCmd.setId(templateId); templateService.deleteIso(deleteIsoCmd); } From 175358cf6037936757ddf875cc1753c49147826a Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 20 Jan 2020 01:26:23 +0530 Subject: [PATCH 084/134] increased timeouts value Signed-off-by: Abhishek Kumar --- .../actionworkers/KubernetesClusterStartWorker.java | 7 +++---- .../actionworkers/KubernetesClusterUpgradeWorker.java | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index 6c477fa87b67..f2a28fef8436 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -504,7 +504,7 @@ public boolean startKubernetesClusterOnCreate() { logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster ID: %s, unable to setup network rules", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e); } attachIsoKubernetesVMs(clusterVMs); - if (!KubernetesClusterUtil.isKubernetesClusterMasterVmRunning(kubernetesCluster, publicIpAddress, publicIpSshPort.second(), 10 * 60 * 1000)) { + if (!KubernetesClusterUtil.isKubernetesClusterMasterVmRunning(kubernetesCluster, publicIpAddress, publicIpSshPort.second(), 20 * 60 * 1000)) { String msg = String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to access master node VMs of the cluster", kubernetesCluster.getUuid()); if (kubernetesCluster.getMasterNodeCount() > 1 && Network.GuestType.Shared.equals(network.getGuestType())) { msg = String.format("%s. Make sure external load-balancer has port forwarding rules for SSH access on ports %d-%d and API access on port %d", @@ -544,9 +544,8 @@ public boolean startStoppedKubernetesCluster() throws CloudRuntimeException { } stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.StartRequested); startKubernetesClusterVMs(); - InetAddress address = null; try { - address = InetAddress.getByName(new URL(kubernetesCluster.getEndpoint()).getHost()); + InetAddress address = InetAddress.getByName(new URL(kubernetesCluster.getEndpoint()).getHost()); } catch (MalformedURLException | UnknownHostException ex) { logTransitStateAndThrow(Level.ERROR, String.format("Kubernetes cluster ID: %s has invalid API endpoint. Can not verify if cluster is in ready state", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } @@ -555,7 +554,7 @@ public boolean startStoppedKubernetesCluster() throws CloudRuntimeException { if (Strings.isNullOrEmpty(publicIpAddress)) { logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s as no public IP found for the cluster" , kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } - if (!KubernetesClusterUtil.isKubernetesClusterServerRunning(kubernetesCluster, publicIpAddress, CLUSTER_API_PORT, 10, 30000)) { + if (!KubernetesClusterUtil.isKubernetesClusterServerRunning(kubernetesCluster, publicIpAddress, CLUSTER_API_PORT, 20, 30000)) { logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s in usable state", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } sshPort = publicIpSshPort.second(); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java index ec397303cd83..ec2bd2b072d3 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java @@ -113,7 +113,7 @@ private void upgradeKubernetesClusterNodes() { logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to uncordon Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); } if (i == 0) { // Wait for master to get in Ready state - if (!KubernetesClusterUtil.isKubernetesClusterNodeReady(kubernetesCluster, publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, getManagementServerSshPublicKeyFile(), hostName, 5, 20000)) { + if (!KubernetesClusterUtil.isKubernetesClusterNodeReady(kubernetesCluster, publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, getManagementServerSshPublicKeyFile(), hostName, 10, 30000)) { logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to get master Kubernetes node on VM ID: %s in ready state", kubernetesCluster.getUuid(), vm.getUuid()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); } } From 004af01a13e5af45bf0ad99f0151702ad6bb01ce Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 21 Jan 2020 16:37:01 +0530 Subject: [PATCH 085/134] changes for global setting timeout for cluster action Signed-off-by: Abhishek Kumar --- .../cluster/KubernetesClusterManagerImpl.java | 5 +- .../cluster/KubernetesClusterService.java | 15 +++++ .../KubernetesClusterScaleWorker.java | 11 +++- .../KubernetesClusterStartWorker.java | 33 +++++----- .../KubernetesClusterUpgradeWorker.java | 14 ++++- .../cluster/utils/KubernetesClusterUtil.java | 60 ++++++++----------- 6 files changed, 83 insertions(+), 55 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 8c1a8a7f1214..eccc331193e6 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -1332,7 +1332,10 @@ public ConfigKey[] getConfigKeys() { return new ConfigKey[] { KubernetesServiceEnabled, KubernetesClusterTemplateName, - KubernetesClusterNetworkOffering + KubernetesClusterNetworkOffering, + KubernetesClusterStartTimeout, + KubernetesClusterScaleTimeout, + KubernetesClusterUpgradeTimeout }; } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java index b58b4c3226f0..00b9eedad309 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java @@ -48,6 +48,21 @@ public interface KubernetesClusterService extends PluggableService, Configurable "DefaultNetworkOfferingforKubernetesService", "Name of the network offering that will be used to create isolated network in which Kubernetes cluster VMs will be launched", false); + static final ConfigKey KubernetesClusterStartTimeout = new ConfigKey("Advanced", Long.class, + "cloud.kubernetes.cluster.start.timeout", + "3600", + "Timeout interval (in seconds) in which start operation for a Kubernetes cluster should be completed", + true); + static final ConfigKey KubernetesClusterScaleTimeout = new ConfigKey("Advanced", Long.class, + "cloud.kubernetes.cluster.scale.timeout", + "3600", + "Timeout interval (in seconds) in which scale operation for a Kubernetes cluster should be completed", + true); + static final ConfigKey KubernetesClusterUpgradeTimeout = new ConfigKey("Advanced", Long.class, + "cloud.kubernetes.cluster.upgrade.timeout", + "3600", + "Timeout interval (in seconds) in which upgrade operation for a Kubernetes cluster should be completed. Not strictly obeyed while upgrade is in progress on a node", + true); KubernetesCluster findById(final Long id); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index 93c165c66ef1..a3ce570e4588 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -37,6 +37,7 @@ import com.cloud.hypervisor.Hypervisor; import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterService; import com.cloud.kubernetes.cluster.KubernetesClusterVO; import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil; @@ -71,6 +72,7 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif private Long clusterSize; private KubernetesCluster.State originalState; private Network network; + private long scaleTimeoutTime; public KubernetesClusterScaleWorker(final KubernetesCluster kubernetesCluster, final ServiceOffering serviceOffering, @@ -325,6 +327,9 @@ private void scaleKubernetesClusterOffering() throws CloudRuntimeException { if (!result) { logTransitStateAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster ID: %s failed, unable to scale cluster VM ID: %s", kubernetesCluster.getUuid(), userVM.getUuid()),kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } + if (System.currentTimeMillis() > scaleTimeoutTime) { + logTransitStateAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster ID: %s failed, scaling action timed out", kubernetesCluster.getUuid()),kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } } kubernetesCluster = updateKubernetesClusterEntry(null, serviceOffering); } @@ -365,6 +370,9 @@ private void scaleDownKubernetesClusterSize(final List , kubernetesCluster.getUuid() , userVM.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); } kubernetesClusterVmMapDao.expunge(vmMapVO.getId()); + if (System.currentTimeMillis() > scaleTimeoutTime) { + logTransitStateAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster ID: %s failed, scaling action timed out", kubernetesCluster.getUuid()),kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } i--; } // Scale network rules to update firewall rule @@ -398,7 +406,7 @@ private void scaleUpKubernetesClusterSize(final List o KubernetesClusterVO kubernetesClusterVO = kubernetesClusterDao.findById(kubernetesCluster.getId()); kubernetesClusterVO.setNodeCount(clusterSize); boolean readyNodesCountValid = KubernetesClusterUtil.validateKubernetesClusterReadyNodesCount(kubernetesClusterVO, publicIpAddress, sshPort, - CLUSTER_NODE_VM_USER, sshKeyFile, 20, 60000); + CLUSTER_NODE_VM_USER, sshKeyFile, scaleTimeoutTime, 15000); detachIsoKubernetesVMs(clusterVMs); if (!readyNodesCountValid) { // Scaling failed logTransitStateToFailedIfNeededAndThrow(Level.ERROR, String.format("Scaling unsuccessful for Kubernetes cluster ID: %s as it does not have desired number of nodes in ready state", kubernetesCluster.getUuid())); @@ -436,6 +444,7 @@ public boolean scaleCluster() throws CloudRuntimeException { if (LOGGER.isInfoEnabled()) { LOGGER.info(String.format("Scaling Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); } + scaleTimeoutTime = System.currentTimeMillis() + KubernetesClusterService.KubernetesClusterScaleTimeout.value() * 1000; final long originalClusterSize = kubernetesCluster.getNodeCount(); final ServiceOffering existingServiceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); if (existingServiceOffering == null) { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index f2a28fef8436..1dfff5ef7ad9 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -414,14 +414,14 @@ private void startKubernetesClusterVMs() { } } - private boolean isKubernetesClusterKubeConfigAvailable(final String masterVMPrivateIpAddress) { + private boolean isKubernetesClusterKubeConfigAvailable(final String masterVMPrivateIpAddress, final long timeoutTime) { if (Strings.isNullOrEmpty(masterVMPrivateIpAddress)) { KubernetesClusterDetailsVO kubeConfigDetail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "kubeConfigData"); if (kubeConfigDetail != null && !Strings.isNullOrEmpty(kubeConfigDetail.getValue())) { return true; } } - String kubeConfig = KubernetesClusterUtil.getKubernetesClusterConfig(kubernetesCluster, publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, 5); + String kubeConfig = KubernetesClusterUtil.getKubernetesClusterConfig(kubernetesCluster, publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, timeoutTime); if (!Strings.isNullOrEmpty(kubeConfig)) { if (!Strings.isNullOrEmpty(masterVMPrivateIpAddress)) { kubeConfig = kubeConfig.replace(String.format("server: https://%s:%d", masterVMPrivateIpAddress, CLUSTER_API_PORT), @@ -433,14 +433,14 @@ private boolean isKubernetesClusterKubeConfigAvailable(final String masterVMPriv return false; } - private boolean isKubernetesClusterDashboardServiceRunning(boolean onCreate) { + private boolean isKubernetesClusterDashboardServiceRunning(final boolean onCreate, final Long timeoutTime) { if (!onCreate) { KubernetesClusterDetailsVO dashboardServiceRunningDetail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "dashboardServiceRunning"); if (dashboardServiceRunningDetail != null && Boolean.parseBoolean(dashboardServiceRunningDetail.getValue())) { return true; } } - if (KubernetesClusterUtil.isKubernetesClusterDashboardServiceRunning(kubernetesCluster, publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, 10, 20000)) { + if (KubernetesClusterUtil.isKubernetesClusterDashboardServiceRunning(kubernetesCluster, publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, timeoutTime, 15000)) { kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), "dashboardServiceRunning", String.valueOf(true), false); return true; } @@ -458,6 +458,7 @@ public boolean startKubernetesClusterOnCreate() { if (LOGGER.isInfoEnabled()) { LOGGER.info(String.format("Starting Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); } + final long startTimeoutTime = System.currentTimeMillis() + KubernetesClusterService.KubernetesClusterStartTimeout.value() * 1000; stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.StartRequested); DeployDestination dest = null; try { @@ -504,7 +505,7 @@ public boolean startKubernetesClusterOnCreate() { logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster ID: %s, unable to setup network rules", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e); } attachIsoKubernetesVMs(clusterVMs); - if (!KubernetesClusterUtil.isKubernetesClusterMasterVmRunning(kubernetesCluster, publicIpAddress, publicIpSshPort.second(), 20 * 60 * 1000)) { + if (!KubernetesClusterUtil.isKubernetesClusterMasterVmRunning(kubernetesCluster, publicIpAddress, publicIpSshPort.second(), startTimeoutTime)) { String msg = String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to access master node VMs of the cluster", kubernetesCluster.getUuid()); if (kubernetesCluster.getMasterNodeCount() > 1 && Network.GuestType.Shared.equals(network.getGuestType())) { msg = String.format("%s. Make sure external load-balancer has port forwarding rules for SSH access on ports %d-%d and API access on port %d", @@ -515,22 +516,22 @@ public boolean startKubernetesClusterOnCreate() { } logTransitStateDetachIsoAndThrow(Level.ERROR, msg, kubernetesCluster, clusterVMs, KubernetesCluster.Event.CreateFailed, null); } - boolean k8sApiServerSetup = KubernetesClusterUtil.isKubernetesClusterServerRunning(kubernetesCluster, publicIpAddress, CLUSTER_API_PORT, 20, 30000); + boolean k8sApiServerSetup = KubernetesClusterUtil.isKubernetesClusterServerRunning(kubernetesCluster, publicIpAddress, CLUSTER_API_PORT, startTimeoutTime, 15000); if (!k8sApiServerSetup) { logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to provision API endpoint for the cluster", kubernetesCluster.getUuid()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.CreateFailed, null); } sshPort = publicIpSshPort.second(); updateKubernetesClusterEntryEndpoint(); boolean readyNodesCountValid = KubernetesClusterUtil.validateKubernetesClusterReadyNodesCount(kubernetesCluster, publicIpAddress, sshPort, - CLUSTER_NODE_VM_USER, sshKeyFile, 30, 30000); + CLUSTER_NODE_VM_USER, sshKeyFile, startTimeoutTime, 15000); detachIsoKubernetesVMs(clusterVMs); if (!readyNodesCountValid) { logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster ID: %s as it does not have desired number of nodes in ready state", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); } - if (!isKubernetesClusterKubeConfigAvailable(k8sMasterVM.getPrivateIpAddress())) { + if (!isKubernetesClusterKubeConfigAvailable(publicIpAddress, startTimeoutTime)) { logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to retrieve kube-config for the cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } - if (!isKubernetesClusterDashboardServiceRunning(true)) { + if (!isKubernetesClusterDashboardServiceRunning(true, startTimeoutTime)) { logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to get Dashboard service running for the cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(),KubernetesCluster.Event.OperationFailed); } stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); @@ -542,6 +543,7 @@ public boolean startStoppedKubernetesCluster() throws CloudRuntimeException { if (LOGGER.isInfoEnabled()) { LOGGER.info(String.format("Starting Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); } + final long startTimeoutTime = System.currentTimeMillis() + KubernetesClusterService.KubernetesClusterStartTimeout.value() * 1000; stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.StartRequested); startKubernetesClusterVMs(); try { @@ -554,14 +556,14 @@ public boolean startStoppedKubernetesCluster() throws CloudRuntimeException { if (Strings.isNullOrEmpty(publicIpAddress)) { logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s as no public IP found for the cluster" , kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } - if (!KubernetesClusterUtil.isKubernetesClusterServerRunning(kubernetesCluster, publicIpAddress, CLUSTER_API_PORT, 20, 30000)) { + if (!KubernetesClusterUtil.isKubernetesClusterServerRunning(kubernetesCluster, publicIpAddress, CLUSTER_API_PORT, startTimeoutTime, 15000)) { logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s in usable state", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } sshPort = publicIpSshPort.second(); - if (!isKubernetesClusterKubeConfigAvailable(null)) { + if (!isKubernetesClusterKubeConfigAvailable(null, startTimeoutTime)) { logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s in usable state as unable to retrieve kube-config for the cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } - if (!isKubernetesClusterDashboardServiceRunning(false)) { + if (!isKubernetesClusterDashboardServiceRunning(false, startTimeoutTime)) { logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s in usable state as unable to get Dashboard service running for the cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); @@ -573,6 +575,7 @@ public boolean startStoppedKubernetesCluster() throws CloudRuntimeException { public boolean reconcileAlertCluster() { init(); + final long startTimeoutTime = System.currentTimeMillis() + 3 * 60 * 1000; Pair sshIpPort = getKubernetesClusterServerIpSshPort(null); publicIpAddress = sshIpPort.first(); sshPort = sshIpPort.second(); @@ -592,14 +595,14 @@ public boolean reconcileAlertCluster() { return false; } if (!KubernetesClusterUtil.isKubernetesClusterServerRunning(kubernetesCluster, sshIpPort.first(), - KubernetesClusterActionWorker.CLUSTER_API_PORT, 1, 0)) { + KubernetesClusterActionWorker.CLUSTER_API_PORT, startTimeoutTime, 0)) { return false; } updateKubernetesClusterEntryEndpoint(); - if (!isKubernetesClusterKubeConfigAvailable(null)) { + if (!isKubernetesClusterKubeConfigAvailable(null, startTimeoutTime)) { return false; } - if (!isKubernetesClusterDashboardServiceRunning(false)) { + if (!isKubernetesClusterDashboardServiceRunning(false, startTimeoutTime)) { return false; } // mark the cluster to be running diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java index ec2bd2b072d3..1093ea149b8c 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java @@ -29,6 +29,7 @@ import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterService; import com.cloud.kubernetes.cluster.KubernetesClusterVO; import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil; import com.cloud.kubernetes.version.KubernetesSupportedVersion; @@ -44,6 +45,7 @@ public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorke private List clusterVMs = new ArrayList<>(); private KubernetesSupportedVersion upgradeVersion; private File upgradeScriptFile; + private long upgradeTimeoutTime; public KubernetesClusterUpgradeWorker(final KubernetesCluster kubernetesCluster, final KubernetesSupportedVersion upgradeVersion, @@ -100,6 +102,9 @@ private void upgradeKubernetesClusterNodes() { if (!result.first()) { logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to drain Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); } + if (System.currentTimeMillis() > upgradeTimeoutTime) { + logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, upgrade action timed out", kubernetesCluster.getUuid()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); + } try { result = runInstallScriptOnVM(vm, i); } catch (Exception e) { @@ -107,13 +112,15 @@ private void upgradeKubernetesClusterNodes() { } if (!result.first()) { logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to upgrade Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); - } - if (!KubernetesClusterUtil.uncordonKubernetesClusterNode(kubernetesCluster, publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, getManagementServerSshPublicKeyFile(), vm, 5, 30000)) { + if (System.currentTimeMillis() > upgradeTimeoutTime) { + logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, upgrade action timed out", kubernetesCluster.getUuid()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); + } + if (!KubernetesClusterUtil.uncordonKubernetesClusterNode(kubernetesCluster, publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, getManagementServerSshPublicKeyFile(), vm, upgradeTimeoutTime, 15000)) { logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to uncordon Kubernetes node on VM ID: %s", kubernetesCluster.getUuid(), vm.getUuid()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); } if (i == 0) { // Wait for master to get in Ready state - if (!KubernetesClusterUtil.isKubernetesClusterNodeReady(kubernetesCluster, publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, getManagementServerSshPublicKeyFile(), hostName, 10, 30000)) { + if (!KubernetesClusterUtil.isKubernetesClusterNodeReady(kubernetesCluster, publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, getManagementServerSshPublicKeyFile(), hostName, upgradeTimeoutTime, 15000)) { logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster ID: %s, unable to get master Kubernetes node on VM ID: %s in ready state", kubernetesCluster.getUuid(), vm.getUuid()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); } } @@ -129,6 +136,7 @@ public boolean upgradeCluster() throws CloudRuntimeException { if (LOGGER.isInfoEnabled()) { LOGGER.info(String.format("Upgrading Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); } + upgradeTimeoutTime = System.currentTimeMillis() + KubernetesClusterService.KubernetesClusterUpgradeTimeout.value() * 1000; Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(null); publicIpAddress = publicIpSshPort.first(); sshPort = publicIpSshPort.second(); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java index 1affcdb08acb..68cd91601939 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java @@ -53,10 +53,9 @@ public static boolean isKubernetesClusterNodeReady(final KubernetesCluster kuber } public static boolean isKubernetesClusterNodeReady(final KubernetesCluster kubernetesCluster, final String ipAddress, final int port, - final String user, final File sshKeyFile, final String nodeName, final int retries, - final int waitDuration) { - int retryCounter = 0; - while (retryCounter < retries) { + final String user, final File sshKeyFile, final String nodeName, + final long timeoutTime, final int waitDuration) { + while (System.currentTimeMillis() < timeoutTime) { boolean ready = false; try { ready = isKubernetesClusterNodeReady(kubernetesCluster, ipAddress, port, user, sshKeyFile, nodeName); @@ -71,7 +70,6 @@ public static boolean isKubernetesClusterNodeReady(final KubernetesCluster kuber } catch (InterruptedException ie) { LOGGER.error(String.format("Error while waiting for Kubernetes cluster ID: %s node: %s to become ready", kubernetesCluster.getUuid(), nodeName), ie); } - retryCounter++; } return false; } @@ -87,18 +85,20 @@ public static boolean isKubernetesClusterNodeReady(final KubernetesCluster kuber * @param user * @param sshKeyFile * @param userVm - * @param retries + * @param timeoutTime * @param waitDuration * @return */ - public static boolean uncordonKubernetesClusterNode(final KubernetesCluster kubernetesCluster, final String ipAddress, final int port, - final String user, final File sshKeyFile, final UserVm userVm, final int retries, final int waitDuration) { - int retryCounter = 0; + public static boolean uncordonKubernetesClusterNode(final KubernetesCluster kubernetesCluster, + final String ipAddress, final int port, + final String user, final File sshKeyFile, + final UserVm userVm, final long timeoutTime, + final int waitDuration) { String hostName = userVm.getHostName(); if (!Strings.isNullOrEmpty(hostName)) { hostName = hostName.toLowerCase(); } - while (retryCounter < retries) { + while (System.currentTimeMillis() < timeoutTime) { Pair result = null; try { result = SshHelper.sshExecute(ipAddress, port, user, sshKeyFile, null, @@ -115,7 +115,6 @@ public static boolean uncordonKubernetesClusterNode(final KubernetesCluster kube } catch (InterruptedException ie) { LOGGER.warn(String.format("Error while waiting for uncordon Kubernetes cluster ID: %s node: %s on VM ID: %s", kubernetesCluster.getUuid(), hostName, userVm.getUuid()), ie); } - retryCounter++; } return false; } @@ -151,13 +150,12 @@ public static boolean isKubernetesClusterAddOnServiceRunning(final KubernetesClu public static boolean isKubernetesClusterDashboardServiceRunning(final KubernetesCluster kubernetesCluster, String ipAddress, final int port, final String user, final File sshKeyFile, - int retries, long waitDuration) { + final long timeoutTime, final long waitDuration) { boolean running = false; - int retryCounter = 0; // Check if dashboard service is up running. - while (retryCounter < retries) { + while (System.currentTimeMillis() < timeoutTime) { if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Checking dashboard service for the Kubernetes cluster ID: %s to come up. Attempt: %d/%d", kubernetesCluster.getUuid(), retryCounter + 1, retries)); + LOGGER.debug(String.format("Checking dashboard service for the Kubernetes cluster ID: %s to come up", kubernetesCluster.getUuid())); } if (isKubernetesClusterAddOnServiceRunning(kubernetesCluster, ipAddress, port, user, sshKeyFile, "kubernetes-dashboard", "kubernetes-dashboard")) { if (LOGGER.isInfoEnabled()) { @@ -171,16 +169,14 @@ public static boolean isKubernetesClusterDashboardServiceRunning(final Kubernete } catch (InterruptedException ex) { LOGGER.error(String.format("Error while waiting for Kubernetes cluster: %s API dashboard service to be available", kubernetesCluster.getUuid()), ex); } - retryCounter++; } return running; } public static String getKubernetesClusterConfig(final KubernetesCluster kubernetesCluster, final String ipAddress, final int port, - final String user, final File sshKeyFile, final int retries) { - int retryCounter = 0; + final String user, final File sshKeyFile, final long timeoutTime) { String kubeConfig = ""; - while (retryCounter < retries) { + while (System.currentTimeMillis() < timeoutTime) { try { Pair result = SshHelper.sshExecute(ipAddress, port, user, sshKeyFile, null, "sudo cat /etc/kubernetes/admin.conf", @@ -195,9 +191,8 @@ public static String getKubernetesClusterConfig(final KubernetesCluster kubernet } } } catch (Exception e) { - LOGGER.warn(String.format("Failed to retrieve kube-config file for Kubernetes cluster ID: %s. Attempt: %d/%d", kubernetesCluster.getUuid(), retryCounter+1, retries), e); + LOGGER.warn(String.format("Failed to retrieve kube-config file for Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); } - retryCounter++; } return kubeConfig; } @@ -219,10 +214,9 @@ public static int getKubernetesClusterReadyNodesCount(final KubernetesCluster ku } public static boolean isKubernetesClusterServerRunning(final KubernetesCluster kubernetesCluster, final String ipAddress, - final int port, final int retries, final long waitDuration) { - int retryCounter = 0; + final int port, final long timeoutTime, final long waitDuration) { boolean k8sApiServerSetup = false; - while (retryCounter < retries) { + while (System.currentTimeMillis() < timeoutTime) { try { String versionOutput = IOUtils.toString(new URL(String.format("https://%s:%d/version", ipAddress, port)), StringUtils.getPreferredCharset()); if (!Strings.isNullOrEmpty(versionOutput)) { @@ -233,23 +227,21 @@ public static boolean isKubernetesClusterServerRunning(final KubernetesCluster k break; } } catch (Exception e) { - LOGGER.warn(String.format("API endpoint for Kubernetes cluster ID: %s not available. Attempt: %d/%d", kubernetesCluster.getUuid(), retryCounter+1, retries), e); + LOGGER.warn(String.format("API endpoint for Kubernetes cluster ID: %s not available", kubernetesCluster.getUuid()), e); } try { Thread.sleep(waitDuration); } catch (InterruptedException ie) { LOGGER.error(String.format("Error while waiting for Kubernetes cluster ID: %s API endpoint to be available", kubernetesCluster.getUuid()), ie); } - retryCounter++; } return k8sApiServerSetup; } public static boolean isKubernetesClusterMasterVmRunning(final KubernetesCluster kubernetesCluster, final String ipAddress, - final int port, final long timeout) { + final int port, final long timeoutTime) { boolean masterVmRunning = false; - long startTime = System.currentTimeMillis(); - while (!masterVmRunning && System.currentTimeMillis() - startTime < timeout) { + while (!masterVmRunning && System.currentTimeMillis() < timeoutTime) { try (Socket socket = new Socket()) { socket.connect(new InetSocketAddress(ipAddress, port), 10000); masterVmRunning = true; @@ -270,11 +262,10 @@ public static boolean isKubernetesClusterMasterVmRunning(final KubernetesCluster public static boolean validateKubernetesClusterReadyNodesCount(final KubernetesCluster kubernetesCluster, final String ipAddress, final int port, final String user, final File sshKeyFile, - final int retries, final long waitDuration) { - int retryCounter = 0; - while (retryCounter < retries) { + final long timeoutTime, final long waitDuration) { + while (System.currentTimeMillis() < timeoutTime) { if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Checking ready nodes for the Kubernetes cluster ID: %s with total %d provisioned nodes. Attempt: %d/%d", kubernetesCluster.getUuid(), kubernetesCluster.getTotalNodeCount(), retryCounter + 1, retries)); + LOGGER.debug(String.format("Checking ready nodes for the Kubernetes cluster ID: %s with total %d provisioned nodes", kubernetesCluster.getUuid(), kubernetesCluster.getTotalNodeCount())); } try { int nodesCount = KubernetesClusterUtil.getKubernetesClusterReadyNodesCount(kubernetesCluster, ipAddress, port, @@ -295,9 +286,8 @@ public static boolean validateKubernetesClusterReadyNodesCount(final KubernetesC try { Thread.sleep(waitDuration); } catch (InterruptedException ex) { - LOGGER.warn(String.format("Error while waiting during Kubernetes cluster ID: %s ready node check. %d/%d", kubernetesCluster.getUuid(), retryCounter+1, retries), ex); + LOGGER.warn(String.format("Error while waiting during Kubernetes cluster ID: %s ready node check", kubernetesCluster.getUuid()), ex); } - retryCounter++; } return false; } From 741cf7a43023debd6213c1002716387577d5474d Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 22 Jan 2020 16:08:51 +0530 Subject: [PATCH 086/134] fix for making version ISO public Signed-off-by: Abhishek Kumar --- .../cloudstack/api/command/user/iso/RegisterIsoCmd.java | 4 ++++ .../kubernetes/version/KubernetesVersionManagerImpl.java | 1 + 2 files changed, 5 insertions(+) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java index e6a62925afab..1c1a767aa3bc 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java @@ -147,6 +147,10 @@ public Boolean isPublic() { return publicIso; } + public void setPublic(Boolean publicIso) { + this.publicIso = publicIso; + } + public Boolean isExtractable() { return extractable; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java index 36d5fe314d04..aaf24fdcd1b5 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java @@ -144,6 +144,7 @@ private VirtualMachineTemplate registerKubernetesVersionIso(final Long zoneId, f RegisterIsoCmd registerIsoCmd = new RegisterIsoCmd(); registerIsoCmd = ComponentContext.inject(registerIsoCmd); registerIsoCmd.setIsoName(isoName); + registerIsoCmd.setPublic(true); if (zoneId != null) { registerIsoCmd.setZoneId(zoneId); } From c138142f7a0acbc9d21373e7cea5d692e993ce5c Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 22 Jan 2020 16:09:59 +0530 Subject: [PATCH 087/134] fix for transitioning state on attach iso fail Signed-off-by: Abhishek Kumar --- .../KubernetesClusterActionWorker.java | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index fc5a2e06b71c..39c7263ea259 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -288,18 +288,23 @@ protected void attachIsoKubernetesVMs(List clusterVMs, final KubernetesS if (kubernetesSupportedVersion == null) { version = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); } + KubernetesCluster.Event failedEvent = KubernetesCluster.Event.OperationFailed; + KubernetesCluster cluster = kubernetesClusterDao.findById(kubernetesCluster.getId()); + if (cluster != null && cluster.getState() == KubernetesCluster.State.Starting) { + failedEvent = KubernetesCluster.Event.CreateFailed; + } if (version == null) { - logAndThrow(Level.ERROR, String .format("Unable to find Kubernetes version for cluster ID: %s", kubernetesCluster.getUuid())); + logTransitStateAndThrow(Level.ERROR, String .format("Unable to find Kubernetes version for cluster ID: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(), failedEvent); } VMTemplateVO iso = templateDao.findById(version.getIsoId()); if (iso == null) { - logAndThrow(Level.ERROR, String.format("Unable to attach ISO to Kubernetes cluster ID: %s. Binaries ISO not found.", kubernetesCluster.getUuid())); + logTransitStateAndThrow(Level.ERROR, String.format("Unable to attach ISO to Kubernetes cluster ID: %s. Binaries ISO not found.", kubernetesCluster.getUuid()), kubernetesCluster.getId(), failedEvent); } if (!iso.getFormat().equals(Storage.ImageFormat.ISO)) { - logAndThrow(Level.ERROR, String.format("Unable to attach ISO to Kubernetes cluster ID: %s. Invalid Binaries ISO.", kubernetesCluster.getUuid())); + logTransitStateAndThrow(Level.ERROR, String.format("Unable to attach ISO to Kubernetes cluster ID: %s. Invalid Binaries ISO.", kubernetesCluster.getUuid()), kubernetesCluster.getId(), failedEvent); } if (!iso.getState().equals(VirtualMachineTemplate.State.Active)) { - logAndThrow(Level.ERROR, String.format("Unable to attach ISO to Kubernetes cluster ID: %s. Binaries ISO not active.", kubernetesCluster.getUuid())); + logTransitStateAndThrow(Level.ERROR, String.format("Unable to attach ISO to Kubernetes cluster ID: %s. Binaries ISO not active.", kubernetesCluster.getUuid()), kubernetesCluster.getId(), failedEvent); } for (UserVm vm : clusterVMs) { try { @@ -308,16 +313,16 @@ protected void attachIsoKubernetesVMs(List clusterVMs, final KubernetesS LOGGER.info(String.format("Attached binaries ISO for VM: %s in cluster: %s", vm.getUuid(), kubernetesCluster.getName())); } } catch (CloudRuntimeException ex) { - logAndThrow(Level.ERROR, String.format("Failed to attach binaries ISO for VM: %s in the Kubernetes cluster name: %s", vm.getDisplayName(), kubernetesCluster.getName()), ex); + logTransitStateAndThrow(Level.ERROR, String.format("Failed to attach binaries ISO for VM: %s in the Kubernetes cluster name: %s", vm.getDisplayName(), kubernetesCluster.getName()), kubernetesCluster.getId(), failedEvent, ex); } } } - protected void attachIsoKubernetesVMs(List clusterVMs) { + protected void attachIsoKubernetesVMs(List clusterVMs) throws CloudRuntimeException { attachIsoKubernetesVMs(clusterVMs, null); } - protected void detachIsoKubernetesVMs(List clusterVMs) throws CloudRuntimeException { + protected void detachIsoKubernetesVMs(List clusterVMs) { for (UserVm vm : clusterVMs) { boolean result = false; try { From b69e7c0c05870961026de9c4636226434e599538 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 22 Jan 2020 16:11:12 +0530 Subject: [PATCH 088/134] fix for expunge error for user Signed-off-by: Abhishek Kumar --- .../KubernetesClusterDestroyWorker.java | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java index 52de1d7c91c5..731db7d771a0 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java @@ -77,21 +77,30 @@ private boolean destroyClusterVMs() { if (userVM == null || userVM.isRemoved()) { continue; } + boolean isAdmin = accountManager.isAdmin(CallContext.current().getCallingAccountId()); try { - UserVm vm = userVmService.destroyVm(vmID, true); - if (!VirtualMachine.State.Expunging.equals(vm.getState())) { + UserVm vm = userVmService.destroyVm(vmID, isAdmin); + if (isAdmin && !VirtualMachine.State.Expunging.equals(vm.getState())) { LOGGER.warn(String.format("VM '%s' ID: %s should have been expunging by now but is '%s'... retrying..." , vm.getInstanceName() , vm.getUuid() , vm.getState().toString())); } - vm = userVmService.expungeVm(vmID); - if (!VirtualMachine.State.Expunging.equals(vm.getState())) { - LOGGER.error(String.format("VM '%s' ID: %s is now in state '%s'. Will probably fail at deleting it's Kubernetes cluster." + if (!isAdmin && !VirtualMachine.State.Destroyed.equals(vm.getState())) { + LOGGER.warn(String.format("VM '%s' ID: %s should have been destroyed by now but is '%s'... retrying..." , vm.getInstanceName() , vm.getUuid() , vm.getState().toString())); } + if (isAdmin) { + vm = userVmService.expungeVm(vmID); + if (!VirtualMachine.State.Expunging.equals(vm.getState())) { + LOGGER.error(String.format("VM '%s' ID: %s is now in state '%s'. Will probably fail at deleting it's Kubernetes cluster." + , vm.getInstanceName() + , vm.getUuid() + , vm.getState().toString())); + } + } kubernetesClusterVmMapDao.expunge(clusterVM.getId()); if (LOGGER.isInfoEnabled()) { LOGGER.info(String.format("Destroyed VM ID: %s as part of Kubernetes cluster ID: %s cleanup", vm.getUuid(), kubernetesCluster.getUuid())); From 094ef104d1c0ed1ed90be9e5fd2b2aa5fb442e1b Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 22 Jan 2020 17:53:25 +0530 Subject: [PATCH 089/134] fix for scaling issues VM name issue Network rules - port forwarding fix Expunge error Signed-off-by: Abhishek Kumar --- .../KubernetesClusterScaleWorker.java | 47 ++++++++++++------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index a3ce570e4588..5764895c5888 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -24,6 +24,7 @@ import javax.inject.Inject; +import org.apache.cloudstack.context.CallContext; import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Level; @@ -48,6 +49,7 @@ import com.cloud.network.rules.PortForwardingRuleVO; import com.cloud.offering.ServiceOffering; import com.cloud.user.Account; +import com.cloud.user.AccountManager; import com.cloud.uservm.UserVm; import com.cloud.utils.Pair; import com.cloud.utils.db.Transaction; @@ -63,6 +65,8 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModifierActionWorker { + @Inject + protected AccountManager accountManager; @Inject protected VMInstanceDao vmInstanceDao; @Inject @@ -90,7 +94,8 @@ protected void init() { } private void logTransitStateToFailedIfNeededAndThrow(final Level logLevel, final String message, final Exception e) throws CloudRuntimeException { - if (kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { + KubernetesCluster cluster = kubernetesClusterDao.findById(kubernetesCluster.getId()); + if (cluster != null && KubernetesCluster.State.Scaling.equals(kubernetesCluster.getState())) { logTransitStateAndThrow(logLevel, message, kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); } else { logAndThrow(logLevel, message, e); @@ -156,13 +161,13 @@ private void scaleKubernetesClusterNetworkRules(final List clusterVMIds, f throw new ManagementServerException("Firewall rule for node SSH access can't be provisioned!"); } int existingFirewallRuleSourcePortEnd = firewallRule.getSourcePortEnd(); - + final int scaledTotalNodeCount = clusterSize == null ? (int)kubernetesCluster.getTotalNodeCount() : (int)(clusterSize + kubernetesCluster.getMasterNodeCount()); // Provision new SSH firewall rules try { - provisionFirewallRules(publicIp, owner, CLUSTER_NODES_DEFAULT_START_SSH_PORT, CLUSTER_NODES_DEFAULT_START_SSH_PORT + (int)kubernetesCluster.getTotalNodeCount() - 1); + provisionFirewallRules(publicIp, owner, CLUSTER_NODES_DEFAULT_START_SSH_PORT, CLUSTER_NODES_DEFAULT_START_SSH_PORT + scaledTotalNodeCount - 1); if (LOGGER.isDebugEnabled()) { LOGGER.debug(String.format("Provisioned firewall rule to open up port %d to %d on %s in Kubernetes cluster ID: %s", - CLUSTER_NODES_DEFAULT_START_SSH_PORT, CLUSTER_NODES_DEFAULT_START_SSH_PORT + (int) kubernetesCluster.getTotalNodeCount() - 1, publicIp.getAddress().addr(), kubernetesCluster.getUuid())); + CLUSTER_NODES_DEFAULT_START_SSH_PORT, CLUSTER_NODES_DEFAULT_START_SSH_PORT + scaledTotalNodeCount - 1, publicIp.getAddress().addr(), kubernetesCluster.getUuid())); } } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException e) { throw new ManagementServerException(String.format("Failed to activate SSH firewall rules for the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), e); @@ -202,10 +207,10 @@ private KubernetesClusterVO updateKubernetesClusterEntry(final Long newSize, fin final ServiceOffering serviceOffering = newServiceOffering == null ? serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()) : newServiceOffering; final Long serviceOfferingId = newServiceOffering == null ? null : serviceOffering.getId(); - final long size = newSize == null ? kubernetesCluster.getNodeCount() : newSize; + final long size = newSize == null ? kubernetesCluster.getTotalNodeCount() : (newSize + kubernetesCluster.getMasterNodeCount()); final long cores = serviceOffering.getCpu() * size; final long memory = serviceOffering.getRamSize() * size; - KubernetesClusterVO kubernetesClusterVO = updateKubernetesClusterEntry(cores, memory, size, serviceOfferingId); + KubernetesClusterVO kubernetesClusterVO = updateKubernetesClusterEntry(cores, memory, newSize, serviceOfferingId); if (kubernetesClusterVO == null) { logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, unable to update Kubernetes cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); @@ -334,12 +339,14 @@ private void scaleKubernetesClusterOffering() throws CloudRuntimeException { kubernetesCluster = updateKubernetesClusterEntry(null, serviceOffering); } - private void scaleDownKubernetesClusterSize(final List originalVmList) throws CloudRuntimeException { + private void scaleDownKubernetesClusterSize() throws CloudRuntimeException { if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.ScaleDownRequested); } + final List originalVmList = getKubernetesClusterVMMaps(); int i = originalVmList.size() - 1; List removedVmIds = new ArrayList<>(); + boolean isCallerAdmin = accountManager.isAdmin(CallContext.current().getCallingAccountId()); while (i > kubernetesCluster.getMasterNodeCount()) { KubernetesClusterVmMapVO vmMapVO = originalVmList.get(i); UserVm userVM = userVmDao.findById(vmMapVO.getVmId()); @@ -349,22 +356,31 @@ private void scaleDownKubernetesClusterSize(final List // For removing port-forwarding network rules removedVmIds.add(userVM.getId()); try { - UserVm vm = userVmService.destroyVm(userVM.getId(), true); - if (!VirtualMachine.State.Expunging.equals(vm.getState())) { + UserVm vm = userVmService.destroyVm(userVM.getId(), isCallerAdmin); + if (isCallerAdmin && !VirtualMachine.State.Expunging.equals(vm.getState())) { logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, VM '%s' is now in state '%s'." , kubernetesCluster.getUuid() , vm.getInstanceName() , vm.getState().toString()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } - vm = userVmService.expungeVm(userVM.getId()); - if (!VirtualMachine.State.Expunging.equals(vm.getState())) { + if (!isCallerAdmin && !VirtualMachine.State.Destroyed.equals(vm.getState())) { logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, VM '%s' is now in state '%s'." , kubernetesCluster.getUuid() , vm.getInstanceName() , vm.getState().toString()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } + if (isCallerAdmin) { + vm = userVmService.expungeVm(userVM.getId()); + if (!VirtualMachine.State.Expunging.equals(vm.getState())) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, VM '%s' is now in state '%s'." + , kubernetesCluster.getUuid() + , vm.getInstanceName() + , vm.getState().toString()), + kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } + } } catch (ResourceUnavailableException e) { logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, unable to remove VM ID: %s" , kubernetesCluster.getUuid() , userVM.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); @@ -383,14 +399,14 @@ private void scaleDownKubernetesClusterSize(final List } } - private void scaleUpKubernetesClusterSize(final List originalVmList, final long newVmCount) throws CloudRuntimeException { + private void scaleUpKubernetesClusterSize(final long newVmCount) throws CloudRuntimeException { if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.ScaleUpRequested); } List clusterVMs = new ArrayList<>(); List clusterVMIds = new ArrayList<>(); try { - clusterVMs = provisionKubernetesClusterNodeVms((int) newVmCount + originalVmList.size(), originalVmList.size(), publicIpAddress); + clusterVMs = provisionKubernetesClusterNodeVms((int)(newVmCount + kubernetesCluster.getNodeCount()), (int)kubernetesCluster.getNodeCount(), publicIpAddress); } catch (ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { logTransitStateToFailedIfNeededAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, unable to provision node VM in the cluster", kubernetesCluster.getUuid()), e); } @@ -430,11 +446,10 @@ private void scaleKubernetesClusterSize() throws CloudRuntimeException { if (Strings.isNullOrEmpty(publicIpAddress)) { logTransitStateToFailedIfNeededAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, unable to retrieve associated public IP", kubernetesCluster.getUuid())); } - List vmList = getKubernetesClusterVMMaps(); if (newVmRequiredCount < 0) { // downscale - scaleDownKubernetesClusterSize(vmList); + scaleDownKubernetesClusterSize(); } else { // upscale, same node count handled above - scaleUpKubernetesClusterSize(vmList, newVmRequiredCount); + scaleUpKubernetesClusterSize(newVmRequiredCount); } kubernetesCluster = updateKubernetesClusterEntry(clusterSize, null); } From 90d41255b47e6b58d0a60bed11136939dac021f9 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 23 Jan 2020 16:30:37 +0530 Subject: [PATCH 090/134] changed kubeconfig file name fixed typo in access tab instructions Signed-off-by: Abhishek Kumar --- ui/plugins/cks/cks.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ui/plugins/cks/cks.js b/ui/plugins/cks/cks.js index 8b7a23363ca4..bfbb803cf9a1 100644 --- a/ui/plugins/cks/cks.js +++ b/ui/plugins/cks/cks.js @@ -34,7 +34,7 @@ var clusterKubeConfig = ""; var downloadClusterKubeConfig = function() { var blob = new Blob([clusterKubeConfig], {type: 'text/plain'}); - var filename = "admin.conf"; + var filename = "kube.conf"; if(window.navigator.msSaveOrOpenBlob) { window.navigator.msSaveBlob(blob, filename); } else{ @@ -944,8 +944,7 @@ } } }); - return jQuery('

').html("Access Kubernetes cluster

Download cluster's kubeocnfig file using action from Details tab.
Download kubectl tool for cluster's Kubernetes version from,
Linux: https://storage.googleapis.com/kubernetes-release/release/v" + version + "/bin/linux/amd64/kubectl
MacOS: https://storage.googleapis.com/kubernetes-release/release/v" + version + "/bin/darwin/amd64/kubectl
Windows: https://storage.googleapis.com/kubernetes-release/release/v" + version + "/bin/windows/amd64/kubectl.exe

Using kubectl and kubeconfig file to access cluster
kubectl --kubeconfig /custom/path/kube.config {COMMAND}

List pods
kubectl --kubeconfig /custom/path/kube.config get pods --all-namespaces
List nodes
kubectl --kubeconfig /custom/path/kube.config get nodes --all-namespaces
List services
kubectl --kubeconfig /custom/path/kube.config get services --all-namespaces

Access dashboard web UI
Run proxy locally
kubectl --kubeconfig /custom/path/kube.config proxy
Open URL in browser
http://localhost:8001/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/

Token for dashboard login can be retrieved using following command
kubectl --kubeconfig /custom/path/kube.config describe secret $(kubectl --kubeconfig /custom/path/kube.config get secrets -n kubernetes-dashboard | grep kubernetes-dashboard-token | awk '{print $1}') -n kubernetes-dashboard

More about accessing dashboard UI, https://kubernetes.io/docs/tasks/access-application-cluster/web-ui-dashboard/#accessing-the-dashboard-ui"); - // return jQuery('

').html("Access Kubernetes cluster
Download Config File

How to do this
kubectl --kubeconfig /custom/path/kube.config get pods"); + return jQuery('

').html("Access Kubernetes cluster

Download cluster's kubeconfig file using action from Details tab.
Download kubectl tool for cluster's Kubernetes version from,
Linux: https://storage.googleapis.com/kubernetes-release/release/v" + version + "/bin/linux/amd64/kubectl
MacOS: https://storage.googleapis.com/kubernetes-release/release/v" + version + "/bin/darwin/amd64/kubectl
Windows: https://storage.googleapis.com/kubernetes-release/release/v" + version + "/bin/windows/amd64/kubectl.exe

Using kubectl and kubeconfig file to access cluster
kubectl --kubeconfig /custom/path/kube.conf {COMMAND}

List pods
kubectl --kubeconfig /custom/path/kube.conf get pods --all-namespaces
List nodes
kubectl --kubeconfig /custom/path/kube.conf get nodes --all-namespaces
List services
kubectl --kubeconfig /custom/path/kube.conf get services --all-namespaces

Access dashboard web UI
Run proxy locally
kubectl --kubeconfig /custom/path/kube.conf proxy
Open URL in browser
http://localhost:8001/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/

Token for dashboard login can be retrieved using following command
kubectl --kubeconfig /custom/path/kube.conf describe secret $(kubectl --kubeconfig /custom/path/kube.conf get secrets -n kubernetes-dashboard | grep kubernetes-dashboard-token | awk '{print $1}') -n kubernetes-dashboard

More about accessing dashboard UI, https://kubernetes.io/docs/tasks/access-application-cluster/web-ui-dashboard/#accessing-the-dashboard-ui"); } return jQuery('

').html("Kubernetes cluster is not in a stable state, please check again in few minutes."); From 852976f424cee73545edc09aac0f01ddae9ba2c4 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 28 Jan 2020 12:05:55 +0530 Subject: [PATCH 091/134] added licenses in config files Signed-off-by: Abhishek Kumar --- .../src/main/resources/conf/k8s-master-add.yml | 16 ++++++++++++++++ .../src/main/resources/conf/k8s-master.yml | 16 ++++++++++++++++ .../src/main/resources/conf/k8s-node.yml | 16 ++++++++++++++++ 3 files changed, 48 insertions(+) diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml index 052b651a2673..a518cbc3264d 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml @@ -1,4 +1,20 @@ #cloud-config +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- ssh_authorized_keys: diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml index ee48add2dc08..0face90ee0f3 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml @@ -1,4 +1,20 @@ #cloud-config +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- ssh_authorized_keys: diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index c22167453364..467cfcc9236d 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -1,4 +1,20 @@ #cloud-config +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. --- ssh_authorized_keys: From 0a9be5634f6a9495c3752903d8d6f1572506a5fa Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 29 Jan 2020 13:39:50 +0530 Subject: [PATCH 092/134] test changes Master node VM naming fix Cleanup network rules on destroy use instance name for display name as well Signed-off-by: Abhishek Kumar --- .../KubernetesClusterDestroyWorker.java | 54 +++++++++++++++- ...esClusterResourceModifierActionWorker.java | 63 ++++++++++++++++++- .../KubernetesClusterScaleWorker.java | 33 +--------- .../KubernetesClusterStartWorker.java | 9 ++- .../cluster/CreateKubernetesClusterCmd.java | 2 +- 5 files changed, 122 insertions(+), 39 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java index 731db7d771a0..75bc4a3e2845 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java @@ -17,22 +17,28 @@ package com.cloud.kubernetes.cluster.actionworkers; +import java.util.ArrayList; import java.util.List; import javax.inject.Inject; import org.apache.cloudstack.context.CallContext; +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Level; import com.cloud.exception.ManagementServerException; import com.cloud.exception.PermissionDeniedException; +import com.cloud.exception.ResourceUnavailableException; import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; import com.cloud.kubernetes.cluster.KubernetesClusterVO; import com.cloud.kubernetes.cluster.KubernetesClusterVmMap; import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; +import com.cloud.network.IpAddress; +import com.cloud.network.Network; import com.cloud.network.dao.NetworkVO; +import com.cloud.network.rules.FirewallRule; import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.User; @@ -68,7 +74,7 @@ private void validateClusterSate() { private boolean destroyClusterVMs() { boolean vmDestroyed = true; - if ((clusterVMs != null) && !clusterVMs.isEmpty()) { + if (!CollectionUtils.isEmpty(clusterVMs)) { for (KubernetesClusterVmMapVO clusterVM : clusterVMs) { long vmID = clusterVM.getVmId(); @@ -138,6 +144,41 @@ private void destroyKubernetesClusterNetwork() throws ManagementServerException } } + private void deleteKubernetesClusterNetworkRules() throws ManagementServerException { + NetworkVO network = networkDao.findById(kubernetesCluster.getNetworkId()); + if (!Network.GuestType.Isolated.equals(network.getGuestType())) { + return; + } + List removedVmIds = new ArrayList<>(); + if (!CollectionUtils.isEmpty(clusterVMs)) { + for (KubernetesClusterVmMapVO clusterVM : clusterVMs) { + removedVmIds.add(clusterVM.getVmId()); + } + } + IpAddress publicIp = getSourceNatIp(network); + if (publicIp == null) { + throw new ManagementServerException(String.format("No source NAT IP addresses found for network ID: %s", network.getUuid())); + } + try { + removeLoadBalancingRule(publicIp, network, owner, CLUSTER_API_PORT); + } catch (ResourceUnavailableException e) { + throw new ManagementServerException(String.format("Failed to KubernetesCluster load balancing rule for network ID: %s", network.getUuid())); + } + FirewallRule firewallRule = removeApiFirewallRule(publicIp); + if (firewallRule == null) { + throw new ManagementServerException("Firewall rule for API access can't be removed"); + } + firewallRule = removeSshFirewallRule(publicIp); + if (firewallRule == null) { + throw new ManagementServerException("Firewall rule for SSH access can't be removed"); + } + try { + removePortForwardingRules(publicIp, network, owner, removedVmIds); + } catch (ResourceUnavailableException e) { + throw new ManagementServerException(String.format("Failed to KubernetesCluster port forwarding rules for network ID: %s", network.getUuid())); + } + } + private void validateClusterVMsDestroyed() { if(clusterVMs!=null && !clusterVMs.isEmpty()) { // Wait for few seconds to get all VMs really expunged final int maxRetries = 3; @@ -182,12 +223,21 @@ public boolean destroy() throws CloudRuntimeException { validateClusterVMsDestroyed(); try { destroyKubernetesClusterNetwork(); - } catch (Exception e) { + } catch (ManagementServerException e) { String msg = String.format("Failed to destroy network of Kubernetes cluster ID: %s cleanup", kubernetesCluster.getUuid()); LOGGER.warn(msg, e); updateKubernetesClusterEntryForGC(); throw new CloudRuntimeException(msg, e); } + } else { + try { + deleteKubernetesClusterNetworkRules(); + } catch (ManagementServerException e) { + String msg = String.format("Failed to remove network rules of Kubernetes cluster ID: %s", kubernetesCluster.getUuid()); + LOGGER.warn(msg, e); + updateKubernetesClusterEntryForGC(); + throw new CloudRuntimeException(msg, e); + } } } else { String msg = String.format("Failed to destroy one or more VMs as part of Kubernetes cluster ID: %s cleanup", kubernetesCluster.getUuid()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index fa225243fc05..bd044ec219a3 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -58,9 +58,12 @@ import com.cloud.network.IpAddress; import com.cloud.network.Network; import com.cloud.network.dao.FirewallRulesDao; +import com.cloud.network.dao.LoadBalancerDao; +import com.cloud.network.dao.LoadBalancerVO; import com.cloud.network.firewall.FirewallService; import com.cloud.network.lb.LoadBalancingRulesService; import com.cloud.network.rules.FirewallRule; +import com.cloud.network.rules.FirewallRuleVO; import com.cloud.network.rules.PortForwardingRuleVO; import com.cloud.network.rules.RulesService; import com.cloud.network.rules.dao.PortForwardingRulesDao; @@ -102,6 +105,8 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu protected PortForwardingRulesDao portForwardingRulesDao; @Inject protected ResourceManager resourceManager; + @Inject + protected LoadBalancerDao loadBalancerDao; protected KubernetesClusterResourceModifierActionWorker(final KubernetesCluster kubernetesCluster, final KubernetesClusterManagerImpl clusterManager) { super(kubernetesCluster, clusterManager); @@ -300,7 +305,7 @@ protected UserVm createKubernetesNode(String joinIp, int nodeInstance) throws Ma } String base64UserData = Base64.encodeBase64String(k8sNodeConfig.getBytes(StringUtils.getPreferredCharset())); nodeVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, - hostName, kubernetesCluster.getDescription(), null, null, null, + hostName, hostName, null, null, null, null, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), null, addrs, null, null, null, customParameterMap, null, null, null, null); if (LOGGER.isInfoEnabled()) { @@ -404,4 +409,60 @@ public PortForwardingRuleVO doInTransaction(TransactionStatus status) throws Net } } } + + protected FirewallRule removeApiFirewallRule(final IpAddress publicIp) { + FirewallRule rule = null; + List firewallRules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(publicIp.getId(), FirewallRule.Purpose.Firewall); + for (FirewallRuleVO firewallRule : firewallRules) { + if (firewallRule.getSourcePortStart() == CLUSTER_API_PORT && + firewallRule.getSourcePortEnd() == CLUSTER_API_PORT) { + rule = firewallRule; + firewallService.revokeIngressFwRule(firewallRule.getId(), true); + break; + } + } + return rule; + } + + protected FirewallRule removeSshFirewallRule(final IpAddress publicIp) { + FirewallRule rule = null; + List firewallRules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(publicIp.getId(), FirewallRule.Purpose.Firewall); + for (FirewallRuleVO firewallRule : firewallRules) { + if (firewallRule.getSourcePortStart() == CLUSTER_NODES_DEFAULT_START_SSH_PORT) { + rule = firewallRule; + firewallService.revokeIngressFwRule(firewallRule.getId(), true); + break; + } + } + return rule; + } + + protected void removePortForwardingRules(final IpAddress publicIp, final Network network, final Account account, final List removedVMIds) throws ResourceUnavailableException { + if (!CollectionUtils.isEmpty(removedVMIds)) { + for (Long vmId : removedVMIds) { + List pfRules = portForwardingRulesDao.listByNetwork(network.getId()); + for (PortForwardingRuleVO pfRule : pfRules) { + if (pfRule.getVirtualMachineId() == vmId) { + portForwardingRulesDao.remove(pfRule.getId()); + break; + } + } + } + rulesService.applyPortForwardingRules(publicIp.getId(), account); + } + } + + protected void removeLoadBalancingRule(final IpAddress publicIp, final Network network, + final Account account, final int port) throws ResourceUnavailableException { + List rules = loadBalancerDao.listByIpAddress(publicIp.getId()); + for (LoadBalancerVO rule : rules) { + if (rule.getNetworkId() == network.getId() && + rule.getAccountId() == account.getId() && + rule.getSourcePortStart() == port && + rule.getSourcePortEnd() == port) { + lbService.deleteLoadBalancerRule(rule.getId(), true); + break; + } + } + } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index 5764895c5888..cee48ead71cf 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -45,10 +45,7 @@ import com.cloud.network.IpAddress; import com.cloud.network.Network; import com.cloud.network.rules.FirewallRule; -import com.cloud.network.rules.FirewallRuleVO; -import com.cloud.network.rules.PortForwardingRuleVO; import com.cloud.offering.ServiceOffering; -import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.uservm.UserVm; import com.cloud.utils.Pair; @@ -106,34 +103,6 @@ private void logTransitStateToFailedIfNeededAndThrow(final Level logLevel, final logTransitStateToFailedIfNeededAndThrow(logLevel, message, null); } - private FirewallRule removeSshFirewallRule(final IpAddress publicIp) { - FirewallRule rule = null; - List firewallRules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(publicIp.getId(), FirewallRule.Purpose.Firewall); - for (FirewallRuleVO firewallRule : firewallRules) { - if (firewallRule.getSourcePortStart() == CLUSTER_NODES_DEFAULT_START_SSH_PORT) { - rule = firewallRule; - firewallService.revokeIngressFwRule(firewallRule.getId(), true); - break; - } - } - return rule; - } - - private void removePortForwardingRules(final IpAddress publicIp, final Network network, final Account account, final List removedVMIds) throws ResourceUnavailableException { - if (!CollectionUtils.isEmpty(removedVMIds)) { - for (Long vmId : removedVMIds) { - List pfRules = portForwardingRulesDao.listByNetwork(network.getId()); - for (PortForwardingRuleVO pfRule : pfRules) { - if (pfRule.getVirtualMachineId() == vmId) { - portForwardingRulesDao.remove(pfRule.getId()); - break; - } - } - } - rulesService.applyPortForwardingRules(publicIp.getId(), account); - } - } - /** * Scale network rules for an existing Kubernetes cluster while scaling it * Open up firewall for SSH access from port NODES_DEFAULT_START_SSH_PORT to NODES_DEFAULT_START_SSH_PORT+n. @@ -158,7 +127,7 @@ private void scaleKubernetesClusterNetworkRules(final List clusterVMIds, f // Remove existing SSH firewall rules FirewallRule firewallRule = removeSshFirewallRule(publicIp); if (firewallRule == null) { - throw new ManagementServerException("Firewall rule for node SSH access can't be provisioned!"); + throw new ManagementServerException("Firewall rule for node SSH access can't be provisioned"); } int existingFirewallRuleSourcePortEnd = firewallRule.getSourcePortEnd(); final int scaledTotalNodeCount = clusterSize == null ? (int)kubernetesCluster.getTotalNodeCount() : (int)(clusterSize + kubernetesCluster.getMasterNodeCount()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index 1dfff5ef7ad9..df5464740060 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -182,6 +182,9 @@ private UserVm createKubernetesMaster(final Network network, String serverIp) th customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); } String hostName = kubernetesCluster.getName() + "-k8s-master"; + if (kubernetesCluster.getMasterNodeCount() > 1) { + hostName += "-1"; + } boolean haSupported = isKubernetesVersionSupportsHA(); String k8sMasterConfig = null; try { @@ -191,7 +194,7 @@ private UserVm createKubernetesMaster(final Network network, String serverIp) th } String base64UserData = Base64.encodeBase64String(k8sMasterConfig.getBytes(StringUtils.getPreferredCharset())); masterVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, - hostName, kubernetesCluster.getDescription(), null, null, null, + hostName, hostName, null, null, null, null, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), requestedIps, addrs, null, null, null, customParameterMap, null, null, null, null); if (LOGGER.isInfoEnabled()) { @@ -235,7 +238,7 @@ private UserVm createKubernetesAdditionalMaster(final String joinIp, final int a if (rootDiskSize > 0) { customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); } - String hostName = String.format("%s-k8s-master-%s", kubernetesCluster.getName(), additionalMasterNodeInstance); + String hostName = String.format("%s-k8s-master-%d", kubernetesCluster.getName(), additionalMasterNodeInstance + 1); String k8sMasterConfig = null; try { k8sMasterConfig = getKubernetesAdditionalMasterConfig(joinIp); @@ -244,7 +247,7 @@ private UserVm createKubernetesAdditionalMaster(final String joinIp, final int a } String base64UserData = Base64.encodeBase64String(k8sMasterConfig.getBytes(StringUtils.getPreferredCharset())); additionalMasterVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, - hostName, kubernetesCluster.getDescription(), null, null, null, + hostName, hostName, null, null, null, null, BaseCmd.HTTPMethod.POST, base64UserData, kubernetesCluster.getKeyPair(), null, addrs, null, null, null, customParameterMap, null, null, null, null); if (LOGGER.isInfoEnabled()) { diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java index 6f77301e987c..4e98ac27dfe1 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java @@ -63,7 +63,7 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// - @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, description = "name for the Kubernetes cluster") + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, description = "name for the Kubernetes cluster") private String name; @Parameter(name = ApiConstants.DESCRIPTION, type = CommandType.STRING, description = "description for the Kubernetes cluster") From 446f214e8eb2fb9ff44189e9205fc995bac19c7f Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 29 Jan 2020 14:25:45 +0530 Subject: [PATCH 093/134] made description required for cluster creation Signed-off-by: Abhishek Kumar --- .../user/kubernetes/cluster/CreateKubernetesClusterCmd.java | 2 +- ui/plugins/cks/cks.js | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java index 4e98ac27dfe1..32b07c4c36a4 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java @@ -66,7 +66,7 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, description = "name for the Kubernetes cluster") private String name; - @Parameter(name = ApiConstants.DESCRIPTION, type = CommandType.STRING, description = "description for the Kubernetes cluster") + @Parameter(name = ApiConstants.DESCRIPTION, type = CommandType.STRING, required = true, description = "description for the Kubernetes cluster") private String description; @ACL(accessType = AccessType.UseEntry) diff --git a/ui/plugins/cks/cks.js b/ui/plugins/cks/cks.js index bfbb803cf9a1..b6c722dd123e 100644 --- a/ui/plugins/cks/cks.js +++ b/ui/plugins/cks/cks.js @@ -166,6 +166,9 @@ description: { label: 'label.description', //docID: 'helpKubernetesClusterDesc', + validation: { + required: true + } }, zone: { label: 'label.zone', From 3d6b7f16fe9d53a41f39db9e0d8f531747d66bda Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 29 Jan 2020 17:25:59 +0530 Subject: [PATCH 094/134] changes for destroying cluster vm Signed-off-by: Abhishek Kumar --- .../KubernetesClusterDestroyWorker.java | 46 +++++++++++-------- .../KubernetesClusterScaleWorker.java | 22 ++------- 2 files changed, 33 insertions(+), 35 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java index 75bc4a3e2845..c210c1bf75dc 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java @@ -26,6 +26,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Level; +import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.ManagementServerException; import com.cloud.exception.PermissionDeniedException; import com.cloud.exception.ResourceUnavailableException; @@ -83,37 +84,46 @@ private boolean destroyClusterVMs() { if (userVM == null || userVM.isRemoved()) { continue; } - boolean isAdmin = accountManager.isAdmin(CallContext.current().getCallingAccountId()); try { - UserVm vm = userVmService.destroyVm(vmID, isAdmin); - if (isAdmin && !VirtualMachine.State.Expunging.equals(vm.getState())) { - LOGGER.warn(String.format("VM '%s' ID: %s should have been expunging by now but is '%s'... retrying..." + UserVm vm = userVmService.destroyVm(vmID, true); + if (!VirtualMachine.State.Expunging.equals(vm.getState())) { + LOGGER.warn(String.format("VM '%s' ID: %s should have been expunging by now but is '%s'. Retrying..." , vm.getInstanceName() , vm.getUuid() , vm.getState().toString())); - } - if (!isAdmin && !VirtualMachine.State.Destroyed.equals(vm.getState())) { - LOGGER.warn(String.format("VM '%s' ID: %s should have been destroyed by now but is '%s'... retrying..." + vm = userVmService.expungeVm(vmID); + LOGGER.warn(String.format("VM '%s' ID: %s is in state: %s, Kubernetes cluster will probably fail" , vm.getInstanceName() , vm.getUuid() , vm.getState().toString())); } - if (isAdmin) { - vm = userVmService.expungeVm(vmID); - if (!VirtualMachine.State.Expunging.equals(vm.getState())) { - LOGGER.error(String.format("VM '%s' ID: %s is now in state '%s'. Will probably fail at deleting it's Kubernetes cluster." - , vm.getInstanceName() - , vm.getUuid() - , vm.getState().toString())); - } - } kubernetesClusterVmMapDao.expunge(clusterVM.getId()); if (LOGGER.isInfoEnabled()) { LOGGER.info(String.format("Destroyed VM ID: %s as part of Kubernetes cluster ID: %s cleanup", vm.getUuid(), kubernetesCluster.getUuid())); } - } catch (Exception e) { - vmDestroyed = false; + } catch (ResourceUnavailableException | ConcurrentOperationException e) { LOGGER.warn(String.format("Failed to destroy VM ID: %s part of the Kubernetes cluster ID: %s cleanup. Moving on with destroying remaining resources provisioned for the Kubernetes cluster", userVM.getUuid(), kubernetesCluster.getUuid()), e); + return false; + } + } + final long startTime = System.currentTimeMillis(); + while (System.currentTimeMillis() - startTime < 10 * 60 * 1000) { + vmDestroyed = true; + for (KubernetesClusterVmMapVO clusterVM : clusterVMs) { + UserVmVO userVM = userVmDao.findById(clusterVM.getVmId()); + if (userVM != null && !userVM.isRemoved()) { + LOGGER.info(String.format("Waiting for Kubernetes cluster ID: %s VMs to get expunged", kubernetesCluster.getUuid())); + vmDestroyed = false; + break; + } + } + if (vmDestroyed) { + break; + } + try { + Thread.sleep(15 * 1000); + } catch (InterruptedException ie) { + LOGGER.warn(String.format("Error while waiting for Kubernetes cluster ID: %s VMs to get expunged", kubernetesCluster.getUuid()), ie); } } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index cee48ead71cf..a5b575c1371a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -24,7 +24,6 @@ import javax.inject.Inject; -import org.apache.cloudstack.context.CallContext; import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Level; @@ -315,7 +314,6 @@ private void scaleDownKubernetesClusterSize() throws CloudRuntimeException { final List originalVmList = getKubernetesClusterVMMaps(); int i = originalVmList.size() - 1; List removedVmIds = new ArrayList<>(); - boolean isCallerAdmin = accountManager.isAdmin(CallContext.current().getCallingAccountId()); while (i > kubernetesCluster.getMasterNodeCount()) { KubernetesClusterVmMapVO vmMapVO = originalVmList.get(i); UserVm userVM = userVmDao.findById(vmMapVO.getVmId()); @@ -325,22 +323,12 @@ private void scaleDownKubernetesClusterSize() throws CloudRuntimeException { // For removing port-forwarding network rules removedVmIds.add(userVM.getId()); try { - UserVm vm = userVmService.destroyVm(userVM.getId(), isCallerAdmin); - if (isCallerAdmin && !VirtualMachine.State.Expunging.equals(vm.getState())) { - logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, VM '%s' is now in state '%s'." - , kubernetesCluster.getUuid() + UserVm vm = userVmService.destroyVm(userVM.getId(), true); + if (!VirtualMachine.State.Expunging.equals(vm.getState())) { + logMessage(Level.WARN, String.format("VM '%s' is in state '%s' while destroying it during scaling Kubernetes cluster ID: %s. Retrying..." , vm.getInstanceName() - , vm.getState().toString()), - kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } - if (!isCallerAdmin && !VirtualMachine.State.Destroyed.equals(vm.getState())) { - logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, VM '%s' is now in state '%s'." - , kubernetesCluster.getUuid() - , vm.getInstanceName() - , vm.getState().toString()), - kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } - if (isCallerAdmin) { + , vm.getState().toString() + , kubernetesCluster.getUuid()), null); vm = userVmService.expungeVm(userVM.getId()); if (!VirtualMachine.State.Expunging.equals(vm.getState())) { logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, VM '%s' is now in state '%s'." From 583e46c98da2a976def42034137d9bdbfd9df2fe Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 30 Jan 2020 15:23:51 +0530 Subject: [PATCH 095/134] added null pointer check Signed-off-by: Abhishek Kumar --- .../cluster/actionworkers/KubernetesClusterDestroyWorker.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java index c210c1bf75dc..1182ac1a7cf2 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java @@ -156,7 +156,7 @@ private void destroyKubernetesClusterNetwork() throws ManagementServerException private void deleteKubernetesClusterNetworkRules() throws ManagementServerException { NetworkVO network = networkDao.findById(kubernetesCluster.getNetworkId()); - if (!Network.GuestType.Isolated.equals(network.getGuestType())) { + if (network == null || !Network.GuestType.Isolated.equals(network.getGuestType())) { return; } List removedVmIds = new ArrayList<>(); From 53ac5eca4d4a519e6c89b73ac19bc6659cbdc261 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 30 Jan 2020 15:24:16 +0530 Subject: [PATCH 096/134] admin should see full view Signed-off-by: Abhishek Kumar --- .../kubernetes/version/AddKubernetesSupportedVersionCmd.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java index 923211328650..54ffd6156387 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java @@ -43,7 +43,7 @@ @APICommand(name = AddKubernetesSupportedVersionCmd.APINAME, description = "Add a supported Kubernetes version", responseObject = KubernetesSupportedVersionResponse.class, - responseView = ResponseObject.ResponseView.Restricted, + responseView = ResponseObject.ResponseView.Full, entityType = {KubernetesSupportedVersion.class}, authorized = {RoleType.Admin}) public class AddKubernetesSupportedVersionCmd extends BaseCmd implements AdminCmd { From 23c2a812aa52d14561dec4507d33cd430646623d Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 30 Jan 2020 15:24:35 +0530 Subject: [PATCH 097/134] refactorings Signed-off-by: Abhishek Kumar --- ui/plugins/cks/cks.js | 7 ------- 1 file changed, 7 deletions(-) diff --git a/ui/plugins/cks/cks.js b/ui/plugins/cks/cks.js index b6c722dd123e..a3a1a1ee87bd 100644 --- a/ui/plugins/cks/cks.js +++ b/ui/plugins/cks/cks.js @@ -214,9 +214,6 @@ data: filterData, dataType: "json", async: true, - url: createURL("listKubernetesSupportedVersions"), - dataType: "json", - async: true, success: function(json) { var items = []; versionObjs = json.listkubernetessupportedversionsresponse.kubernetessupportedversion; @@ -1227,14 +1224,10 @@ } }); }, - messages: { notification: function(args) { return 'Kubernetes Supported Version Add'; } - }, - notification: { - poll: pollAsyncJobResult } } }, From 895632b28d476016d4f9f7162c49c88e704aeb3c Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 30 Jan 2020 15:20:01 +0530 Subject: [PATCH 098/134] changes for k8s version allow admin to disable and enable versions versions can have minimum cpu and memory associated with them Signed-off-by: Abhishek Kumar --- .../META-INF/db/schema-41300to41400.sql | 3 + .../cluster/KubernetesClusterManagerImpl.java | 50 +++++- .../cluster/KubernetesClusterService.java | 2 + .../version/KubernetesSupportedVersion.java | 16 ++ .../version/KubernetesSupportedVersionVO.java | 46 ++++++ .../version/KubernetesVersionEventTypes.java | 1 + .../version/KubernetesVersionManagerImpl.java | 45 ++++- .../version/KubernetesVersionService.java | 2 + .../AddKubernetesSupportedVersionCmd.java | 16 ++ .../UpdateKubernetesSupportedVersionCmd.java | 103 ++++++++++++ .../KubernetesSupportedVersionResponse.java | 36 ++++ .../version/KubernetesVersionServiceTest.java | 48 ++++++ ui/l10n/en.js | 1 + ui/plugins/cks/cks.js | 154 ++++++++++++++++-- 14 files changed, 505 insertions(+), 18 deletions(-) create mode 100644 plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/UpdateKubernetesSupportedVersionCmd.java diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql b/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql index c9a423cb3303..eebb7bfa9be0 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql @@ -45,6 +45,9 @@ CREATE TABLE IF NOT EXISTS `cloud`.`kubernetes_supported_version` ( `semantic_version` varchar(32) NOT NULL COMMENT 'the semantic version for this Kubernetes version', `iso_id` bigint unsigned NOT NULL COMMENT 'the ID of the binaries ISO for this Kubernetes version', `zone_id` bigint unsigned DEFAULT NULL COMMENT 'the ID of the zone for which this Kubernetes version is made available', + `state` char(32) DEFAULT NULL COMMENT 'the enabled or disabled state for this Kubernetes version', + `min_cpu` int(10) unsigned DEFAULT NULL COMMENT 'the minimum CPU needed by cluster nodes for using this Kubernetes version', + `min_ram_size` bigint(20) unsigned DEFAULT NULL COMMENT 'the minimum RAM in MB needed by cluster nodes for this Kubernetes version', `created` datetime NOT NULL COMMENT 'date created', `removed` datetime COMMENT 'date removed or null, if still present', diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index eccc331193e6..a2294bdcb34b 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -393,12 +393,18 @@ private boolean validateNetwork(Network network, int clusterTotalNodeCount) { return true; } - private boolean validateServiceOffering(ServiceOffering serviceOffering) { + private boolean validateServiceOffering(final ServiceOffering serviceOffering, final KubernetesSupportedVersion version) { if (serviceOffering.isDynamic()) { throw new InvalidParameterValueException(String.format("Custom service offerings are not supported for creating clusters, service offering ID: %s", serviceOffering.getUuid())); } - if (serviceOffering.getCpu() < 2 || serviceOffering.getRamSize() < 2048) { - throw new InvalidParameterValueException(String.format("Kubernetes cluster cannot be created with service offering ID: %s, Kubernetes cluster template(CoreOS) needs minimum 2 vCPUs and 2 GB RAM", serviceOffering.getUuid())); + if ((version.getMinimumCpu() == null || version.getMinimumRamSize() == null) && serviceOffering.getCpu() < MIN_KUBERNETES_CLUSTER_NODE_CPU || serviceOffering.getRamSize() < MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster cannot be created with service offering ID: %s, Kubernetes cluster template(CoreOS) needs minimum %d vCPUs and %d MB RAM", serviceOffering.getUuid(), MIN_KUBERNETES_CLUSTER_NODE_CPU, MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE)); + } + if (version.getMinimumCpu() != null && serviceOffering.getCpu() < version.getMinimumCpu()) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster cannot be created with service offering ID: %s, Kubernetes version ID: %s needs minimum %d vCPUs", serviceOffering.getUuid(), version.getUuid(), version.getMinimumCpu())); + } + if (version.getMinimumRamSize() != null && serviceOffering.getRamSize() < version.getMinimumRamSize()) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster cannot be created with service offering ID: %s, associated Kubernetes version ID: %s needs minimum %d MB RAM", serviceOffering.getUuid(), version.getUuid(), version.getMinimumRamSize())); } return true; } @@ -579,6 +585,9 @@ private void validateKubernetesClusterCreateParameters(final CreateKubernetesClu if (clusterKubernetesVersion == null) { throw new InvalidParameterValueException("Unable to find given Kubernetes version in supported versions"); } + if (!KubernetesSupportedVersion.State.Enabled.equals(clusterKubernetesVersion.getState())) { + throw new InvalidParameterValueException(String.format("Kubernetes version ID: %s is in %s state", clusterKubernetesVersion.getUuid(), clusterKubernetesVersion.getState())); + } if (clusterKubernetesVersion.getZoneId() != null && !clusterKubernetesVersion.getZoneId().equals(zone.getId())) { throw new InvalidParameterValueException(String.format("Kubernetes version ID: %s is not available for zone ID: %s", clusterKubernetesVersion.getUuid(), zone.getUuid())); } @@ -626,7 +635,7 @@ private void validateKubernetesClusterCreateParameters(final CreateKubernetesClu logAndThrow(Level.WARN, String.format("The template ID: %s is not available for use in zone ID: %s to provision Kubernetes cluster name: %s", template.getUuid(), zone.getUuid(), name)); } - if (!validateServiceOffering(serviceOfferingDao.findById(serviceOfferingId))) { + if (!validateServiceOffering(serviceOffering, clusterKubernetesVersion)) { throw new InvalidParameterValueException("Given service offering ID: %s is not suitable for Kubernetes cluster"); } @@ -752,6 +761,11 @@ private void validateKubernetesClusterScaleParameters(ScaleKubernetesClusterCmd throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be scaled, either a new service offering or a new cluster size must be passed", kubernetesCluster.getUuid())); } + final KubernetesSupportedVersion clusterVersion = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); + if (clusterVersion == null) { + throw new CloudRuntimeException(String.format("Invalid Kubernetes version associated with Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + } + ServiceOffering serviceOffering = null; if (serviceOfferingId != null) { serviceOffering = serviceOfferingDao.findById(serviceOfferingId); @@ -761,8 +775,17 @@ private void validateKubernetesClusterScaleParameters(ScaleKubernetesClusterCmd if (serviceOffering.isDynamic()) { throw new InvalidParameterValueException(String.format("Custom service offerings are not supported for Kubernetes clusters. Kubernetes cluster ID: %s, service offering ID: %s", kubernetesCluster.getUuid(), serviceOffering.getUuid())); } - if (serviceOffering.getCpu() < 2 || serviceOffering.getRamSize() < 2048) { - throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be scaled with service offering ID: %s, Kubernetes cluster template(CoreOS) needs minimum 2 vCPUs and 2 GB RAM", kubernetesCluster.getUuid(), serviceOffering.getUuid())); + if ((clusterVersion.getMinimumCpu() == null || clusterVersion.getMinimumRamSize() == null) && serviceOffering.getCpu() < MIN_KUBERNETES_CLUSTER_NODE_CPU || serviceOffering.getRamSize() < MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be scaled with service offering ID: %s, Kubernetes cluster template(CoreOS) needs minimum %d vCPUs and %d MB RAM", + kubernetesCluster.getUuid(), serviceOffering.getUuid(), MIN_KUBERNETES_CLUSTER_NODE_CPU, MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE)); + } + if (clusterVersion.getMinimumCpu() != null && serviceOffering.getCpu() < clusterVersion.getMinimumCpu()) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be scaled with service offering ID: %s, associated Kubernetes version ID: %s needs minimum %d vCPUs", + kubernetesCluster.getUuid(), serviceOffering.getUuid(), clusterVersion.getUuid(), clusterVersion.getMinimumCpu())); + } + if (clusterVersion.getMinimumRamSize() != null && serviceOffering.getRamSize() < clusterVersion.getMinimumRamSize()) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be scaled with service offering ID: %s, associated Kubernetes version ID: %s needs minimum %d MB RAM", + kubernetesCluster.getUuid(), serviceOffering.getUuid(), clusterVersion.getUuid(), clusterVersion.getMinimumRamSize())); } } final ServiceOffering existingServiceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); @@ -811,11 +834,26 @@ private void validateKubernetesClusterUpgradeParameters(UpgradeKubernetesCluster if (upgradeVersion == null || upgradeVersion.getRemoved() != null) { throw new InvalidParameterValueException("Invalid Kubernetes version ID"); } + if (!KubernetesSupportedVersion.State.Enabled.equals(upgradeVersion.getState())) { + throw new InvalidParameterValueException(String.format("Kubernetes version ID: %s for upgrade is in %s state", upgradeVersion.getUuid(), upgradeVersion.getState())); + } KubernetesSupportedVersionVO clusterVersion = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); if (clusterVersion == null || clusterVersion.getRemoved() != null) { throw new InvalidParameterValueException(String.format("Invalid Kubernetes version associated with cluster ID: %s", kubernetesCluster.getUuid())); } + final ServiceOffering serviceOffering = serviceOfferingDao.findByIdIncludingRemoved(kubernetesCluster.getServiceOfferingId()); + if (serviceOffering == null) { + throw new CloudRuntimeException(String.format("Invalid service offering associated with Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + } + if (upgradeVersion.getMinimumCpu() != null && serviceOffering.getCpu() < upgradeVersion.getMinimumCpu()) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be upgraded with Kubernetes version ID: %s which needs minimum %d vCPUs while associated service offering ID: %s offers only %d vCPUs", + kubernetesCluster.getUuid(), upgradeVersion.getUuid(), upgradeVersion.getMinimumCpu(), serviceOffering.getUuid(), serviceOffering.getCpu())); + } + if (upgradeVersion.getMinimumRamSize() != null && serviceOffering.getRamSize() < upgradeVersion.getMinimumRamSize()) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be upgraded with Kubernetes version ID: %s which needs minimum %d MB RAM while associated service offering ID: %s offers only %d MB RAM", + kubernetesCluster.getUuid(), upgradeVersion.getUuid(), upgradeVersion.getMinimumRamSize(), serviceOffering.getUuid(), serviceOffering.getRamSize())); + } // Check upgradeVersion is either patch upgrade or immediate minor upgrade try { KubernetesVersionManagerImpl.canUpgradeKubernetesVersion(clusterVersion.getSemanticVersion(), upgradeVersion.getSemanticVersion()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java index 00b9eedad309..d1c8c9af9a41 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java @@ -32,6 +32,8 @@ public interface KubernetesClusterService extends PluggableService, Configurable { static final String MIN_KUBERNETES_VERSION_HA_SUPPORT = "1.16.0"; + static final int MIN_KUBERNETES_CLUSTER_NODE_CPU = 2; + static final int MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE = 2048; static final ConfigKey KubernetesServiceEnabled = new ConfigKey("Advanced", Boolean.class, "cloud.kubernetes.service.enabled", diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesSupportedVersion.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesSupportedVersion.java index 3a2e885d9eb6..efbf13a2c5d9 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesSupportedVersion.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesSupportedVersion.java @@ -25,9 +25,25 @@ * */ public interface KubernetesSupportedVersion extends InternalIdentity, Identity { + + public enum State { + Disabled, Enabled + } + long getId(); String getName(); String getSemanticVersion(); long getIsoId(); Long getZoneId(); + State getState(); + + /** + * @return minimum # of cpu. + */ + Integer getMinimumCpu(); + + /** + * @return minimum ram size in megabytes + */ + Integer getMinimumRamSize(); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesSupportedVersionVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesSupportedVersionVO.java index 536e25b0a20b..65de4d748e76 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesSupportedVersionVO.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesSupportedVersionVO.java @@ -22,6 +22,8 @@ import javax.persistence.Column; import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; import javax.persistence.GeneratedValue; import javax.persistence.GenerationType; import javax.persistence.Id; @@ -52,6 +54,16 @@ public class KubernetesSupportedVersionVO implements KubernetesSupportedVersion @Column(name = "zone_id") private Long zoneId; + @Column(name = "state") + @Enumerated(value = EnumType.STRING) + State state = State.Enabled; + + @Column(name = "min_cpu") + private Integer minimumCpu; + + @Column(name = "min_ram_size") + private Integer minimumRamSize; + @Column(name = GenericDao.CREATED_COLUMN) Date created; @@ -63,11 +75,18 @@ public KubernetesSupportedVersionVO() { } public KubernetesSupportedVersionVO(String name, String semanticVersion, long isoId, Long zoneId) { + this(name, semanticVersion, isoId, zoneId, null, null); + } + + public KubernetesSupportedVersionVO(String name, String semanticVersion, long isoId, Long zoneId, + Integer minimumCpu, Integer minimumRamSize) { this.uuid = UUID.randomUUID().toString(); this.name = name; this.semanticVersion = semanticVersion; this.isoId = isoId; this.zoneId = zoneId; + this.minimumCpu = minimumCpu; + this.minimumRamSize = minimumRamSize; } @Override @@ -116,6 +135,33 @@ public void setZoneId(Long zoneId) { this.zoneId = zoneId; } + @Override + public State getState() { + return this.state; + } + + public void setState(State state) { + this.state = state; + } + + @Override + public Integer getMinimumCpu() { + return minimumCpu; + } + + public void setMinimumCpu(Integer minimumCpu) { + this.minimumCpu = minimumCpu; + } + + @Override + public Integer getMinimumRamSize() { + return minimumRamSize; + } + + public void setMinimumRamSize(Integer minimumRamSize) { + this.minimumRamSize = minimumRamSize; + } + public Date getCreated() { return created; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionEventTypes.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionEventTypes.java index ad169a6d4129..4c979ba89799 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionEventTypes.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionEventTypes.java @@ -20,4 +20,5 @@ public class KubernetesVersionEventTypes { public static final String EVENT_KUBERNETES_VERSION_ADD = "KUBERNETES.VERSION.ADD"; public static final String EVENT_KUBERNETES_VERSION_DELETE = "KUBERNETES.VERSION.DELETE"; + public static final String EVENT_KUBERNETES_VERSION_UPDATE = "KUBERNETES.VERSION.UPDATE"; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java index aaf24fdcd1b5..133212f5eeb2 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java @@ -25,6 +25,7 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.command.admin.kubernetes.version.AddKubernetesSupportedVersionCmd; import org.apache.cloudstack.api.command.admin.kubernetes.version.DeleteKubernetesSupportedVersionCmd; +import org.apache.cloudstack.api.command.admin.kubernetes.version.UpdateKubernetesSupportedVersionCmd; import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; import org.apache.cloudstack.api.command.user.iso.RegisterIsoCmd; import org.apache.cloudstack.api.command.user.kubernetes.version.ListKubernetesSupportedVersionsCmd; @@ -80,6 +81,11 @@ private KubernetesSupportedVersionResponse createKubernetesSupportedVersionRespo response.setId(kubernetesSupportedVersion.getUuid()); response.setName(kubernetesSupportedVersion.getName()); response.setSemanticVersion(kubernetesSupportedVersion.getSemanticVersion()); + if (kubernetesSupportedVersion.getState() != null) { + response.setState(kubernetesSupportedVersion.getState().toString()); + } + response.setMinimumCpu(kubernetesSupportedVersion.getMinimumCpu()); + response.setMinimumRamSize(kubernetesSupportedVersion.getMinimumRamSize()); DataCenterVO zone = dataCenterDao.findById(kubernetesSupportedVersion.getZoneId()); if (zone != null) { response.setZoneId(zone.getUuid()); @@ -269,6 +275,14 @@ public KubernetesSupportedVersionResponse addKubernetesSupportedVersion(final Ad final Long zoneId = cmd.getZoneId(); final String isoUrl = cmd.getUrl(); final String isoChecksum = cmd.getChecksum(); + final Integer minimumCpu = cmd.getMinimumCpu(); + final Integer minimumRamSize = cmd.getMinimumRamSize(); + if (minimumCpu != null && minimumCpu < KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_CPU) { + throw new InvalidParameterValueException(String.format("Invalid value for %s parameter", ApiConstants.MIN_CPU_NUMBER)); + } + if (minimumRamSize != null && minimumRamSize < KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE) { + throw new InvalidParameterValueException(String.format("Invalid value for %s parameter", ApiConstants.MIN_MEMORY)); + } if (compareSemanticVersions(semanticVersion, MIN_KUBERNETES_VERSION) < 0) { throw new InvalidParameterValueException(String.format("New supported Kubernetes version cannot be added as %s is minimum version supported by Kubernetes Service", MIN_KUBERNETES_VERSION)); } @@ -294,7 +308,7 @@ public KubernetesSupportedVersionResponse addKubernetesSupportedVersion(final Ad throw new CloudRuntimeException(String.format("Unable to register binaries ISO for supported kubernetes version, %s, with url: %s", name, isoUrl)); } - KubernetesSupportedVersionVO supportedVersionVO = new KubernetesSupportedVersionVO(name, semanticVersion, template.getId(), zoneId); + KubernetesSupportedVersionVO supportedVersionVO = new KubernetesSupportedVersionVO(name, semanticVersion, template.getId(), zoneId, minimumCpu, minimumRamSize); supportedVersionVO = kubernetesSupportedVersionDao.persist(supportedVersionVO); return createKubernetesSupportedVersionResponse(supportedVersionVO); @@ -331,6 +345,34 @@ public boolean deleteKubernetesSupportedVersion(final DeleteKubernetesSupportedV return kubernetesSupportedVersionDao.remove(version.getId()); } + @Override + @ActionEvent(eventType = KubernetesVersionEventTypes.EVENT_KUBERNETES_VERSION_UPDATE, eventDescription = "Updating Kubernetes supported version") + public KubernetesSupportedVersionResponse updateKubernetesSupportedVersion(final UpdateKubernetesSupportedVersionCmd cmd) { + if (!KubernetesClusterService.KubernetesServiceEnabled.value()) { + throw new CloudRuntimeException("Kubernetes Service plugin is disabled"); + } + final Long versionId = cmd.getId(); + KubernetesSupportedVersion.State state = null; + KubernetesSupportedVersionVO version = kubernetesSupportedVersionDao.findById(versionId); + if (version == null) { + throw new InvalidParameterValueException("Invalid Kubernetes version id specified"); + } + try { + state = KubernetesSupportedVersion.State.valueOf(cmd.getState()); + } catch (IllegalArgumentException iae) { + throw new InvalidParameterValueException(String.format("Invalid value for %s parameter", ApiConstants.STATE)); + } + if (!state.equals(version.getState())) { + version = kubernetesSupportedVersionDao.createForUpdate(version.getId()); + version.setState(state); + if (!kubernetesSupportedVersionDao.update(version.getId(), version)) { + throw new CloudRuntimeException(String.format("Failed to update Kubernetes supported version ID: %s", version.getUuid())); + } + version = kubernetesSupportedVersionDao.findById(versionId); + } + return createKubernetesSupportedVersionResponse(version); + } + @Override public List> getCommands() { List> cmdList = new ArrayList>(); @@ -340,6 +382,7 @@ public List> getCommands() { cmdList.add(AddKubernetesSupportedVersionCmd.class); cmdList.add(ListKubernetesSupportedVersionsCmd.class); cmdList.add(DeleteKubernetesSupportedVersionCmd.class); + cmdList.add(UpdateKubernetesSupportedVersionCmd.class); return cmdList; } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionService.java index cb84b37024ab..8e4cd0325563 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionService.java @@ -19,6 +19,7 @@ import org.apache.cloudstack.api.command.admin.kubernetes.version.AddKubernetesSupportedVersionCmd; import org.apache.cloudstack.api.command.admin.kubernetes.version.DeleteKubernetesSupportedVersionCmd; +import org.apache.cloudstack.api.command.admin.kubernetes.version.UpdateKubernetesSupportedVersionCmd; import org.apache.cloudstack.api.command.user.kubernetes.version.ListKubernetesSupportedVersionsCmd; import org.apache.cloudstack.api.response.KubernetesSupportedVersionResponse; import org.apache.cloudstack.api.response.ListResponse; @@ -31,4 +32,5 @@ public interface KubernetesVersionService extends PluggableService { ListResponse listKubernetesSupportedVersions(ListKubernetesSupportedVersionsCmd cmd); KubernetesSupportedVersionResponse addKubernetesSupportedVersion(AddKubernetesSupportedVersionCmd cmd) throws CloudRuntimeException; boolean deleteKubernetesSupportedVersion(DeleteKubernetesSupportedVersionCmd cmd) throws CloudRuntimeException; + KubernetesSupportedVersionResponse updateKubernetesSupportedVersion(UpdateKubernetesSupportedVersionCmd cmd) throws CloudRuntimeException; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java index 54ffd6156387..928102f27aa0 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java @@ -77,6 +77,14 @@ public class AddKubernetesSupportedVersionCmd extends BaseCmd implements AdminCm description = "the checksum value of the binaries ISO. " + ApiConstants.CHECKSUM_PARAMETER_PREFIX_DESCRIPTION) private String checksum; + @Parameter(name = ApiConstants.MIN_CPU_NUMBER, type = CommandType.INTEGER, + description = "the minimum number of CPUs to be set with the Kubernetes version") + private Integer minimumCpu; + + @Parameter(name = ApiConstants.MIN_MEMORY, type = CommandType.INTEGER, + description = "the minimum RAM size in MB to be set with the Kubernetes version") + private Integer minimumRamSize; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -108,6 +116,14 @@ public String getChecksum() { return checksum; } + public Integer getMinimumCpu() { + return minimumCpu; + } + + public Integer getMinimumRamSize() { + return minimumRamSize; + } + @Override public String getCommandName() { return APINAME.toLowerCase() + "response"; diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/UpdateKubernetesSupportedVersionCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/UpdateKubernetesSupportedVersionCmd.java new file mode 100644 index 000000000000..a182c5dc464b --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/UpdateKubernetesSupportedVersionCmd.java @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.kubernetes.version; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.response.KubernetesSupportedVersionResponse; +import org.apache.log4j.Logger; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.kubernetes.version.KubernetesSupportedVersion; +import com.cloud.kubernetes.version.KubernetesVersionService; +import com.cloud.utils.exception.CloudRuntimeException; + +@APICommand(name = UpdateKubernetesSupportedVersionCmd.APINAME, + description = "Update a supported Kubernetes version", + responseObject = KubernetesSupportedVersionResponse.class, + responseView = ResponseObject.ResponseView.Full, + entityType = {KubernetesSupportedVersion.class}, + authorized = {RoleType.Admin}) +public class UpdateKubernetesSupportedVersionCmd extends BaseCmd implements AdminCmd { + public static final Logger LOGGER = Logger.getLogger(UpdateKubernetesSupportedVersionCmd.class.getName()); + public static final String APINAME = "updateKubernetesSupportedVersion"; + + @Inject + private KubernetesVersionService kubernetesVersionService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @Parameter(name = ApiConstants.ID, type = BaseCmd.CommandType.UUID, + entityType = KubernetesSupportedVersionResponse.class, + description = "the ID of the Kubernetes supported version", + required = true) + private Long id; + + @Parameter(name = ApiConstants.STATE, type = CommandType.STRING, + description = "the enabled or disabled state of the host", + required = true) + private String state; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + public Long getId() { + return id; + } + + public String getState() { + return state; + } + + @Override + public String getCommandName() { + return APINAME.toLowerCase() + "response"; + } + + @Override + public long getEntityOwnerId() { + return 0; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + @Override + public void execute() throws ServerApiException, ConcurrentOperationException { + try { + KubernetesSupportedVersionResponse response = kubernetesVersionService.updateKubernetesSupportedVersion(this); + if (response == null) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to update Kubernetes supported version"); + } + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (CloudRuntimeException ex) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); + } + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java index 2126d551c4db..4deb50d4a0b5 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java @@ -64,6 +64,18 @@ public class KubernetesSupportedVersionResponse extends BaseResponse { @Param(description = "whether Kubernetes supported version supports HA, multi-master") private Boolean supportsHA; + @SerializedName(ApiConstants.STATE) + @Param(description = "the enabled or disabled state of the Kubernetes supported version") + private String state; + + @SerializedName(ApiConstants.MIN_CPU_NUMBER) + @Param(description = "the minimum number of CPUs needed for the Kubernetes supported version") + private Integer minimumCpu; + + @SerializedName(ApiConstants.MIN_MEMORY) + @Param(description = "the minimum RAM size in MB needed for the Kubernetes supported version") + private Integer minimumRamSize; + public String getId() { return id; } @@ -135,4 +147,28 @@ public Boolean isSupportsHA() { public void setSupportsHA(Boolean supportsHA) { this.supportsHA = supportsHA; } + + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + + public Integer getMinimumCpu() { + return minimumCpu; + } + + public void setMinimumCpu(Integer minimumCpu) { + this.minimumCpu = minimumCpu; + } + + public Integer getMinimumRamSize() { + return minimumRamSize; + } + + public void setMinimumRamSize(Integer minimumRamSize) { + this.minimumRamSize = minimumRamSize; + } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionServiceTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionServiceTest.java index f1cb85f6797e..6878c4cd29b9 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionServiceTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionServiceTest.java @@ -26,6 +26,7 @@ import org.apache.cloudstack.api.command.admin.kubernetes.version.AddKubernetesSupportedVersionCmd; import org.apache.cloudstack.api.command.admin.kubernetes.version.DeleteKubernetesSupportedVersionCmd; +import org.apache.cloudstack.api.command.admin.kubernetes.version.UpdateKubernetesSupportedVersionCmd; import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; import org.apache.cloudstack.api.command.user.iso.RegisterIsoCmd; import org.apache.cloudstack.api.command.user.kubernetes.version.ListKubernetesSupportedVersionsCmd; @@ -135,6 +136,8 @@ public void listKubernetesSupportedVersionsTest() { @Test(expected = InvalidParameterValueException.class) public void addKubernetesSupportedVersionLowerUnsupportedTest() { AddKubernetesSupportedVersionCmd cmd = Mockito.mock(AddKubernetesSupportedVersionCmd.class); + when(cmd.getMinimumCpu()).thenReturn(KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_CPU); + when(cmd.getMinimumRamSize()).thenReturn(KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE); AccountVO account = new AccountVO("admin", 1L, "", Account.ACCOUNT_TYPE_ADMIN, "uuid"); UserVO user = new UserVO(1, "adminuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); CallContext.register(user, account); @@ -142,9 +145,35 @@ public void addKubernetesSupportedVersionLowerUnsupportedTest() { kubernetesVersionService.addKubernetesSupportedVersion(cmd); } + @Test(expected = InvalidParameterValueException.class) + public void addKubernetesSupportedVersionInvalidCpuTest() { + AddKubernetesSupportedVersionCmd cmd = Mockito.mock(AddKubernetesSupportedVersionCmd.class); + when(cmd.getMinimumCpu()).thenReturn(KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_CPU-1); + when(cmd.getMinimumRamSize()).thenReturn(KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE); + AccountVO account = new AccountVO("admin", 1L, "", Account.ACCOUNT_TYPE_ADMIN, "uuid"); + UserVO user = new UserVO(1, "adminuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); + when(cmd.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); + CallContext.register(user, account); + kubernetesVersionService.addKubernetesSupportedVersion(cmd); + } + + @Test(expected = InvalidParameterValueException.class) + public void addKubernetesSupportedVersionInvalidRamSizeTest() { + AddKubernetesSupportedVersionCmd cmd = Mockito.mock(AddKubernetesSupportedVersionCmd.class); + when(cmd.getMinimumCpu()).thenReturn(KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_CPU); + when(cmd.getMinimumRamSize()).thenReturn(KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE-10); + AccountVO account = new AccountVO("admin", 1L, "", Account.ACCOUNT_TYPE_ADMIN, "uuid"); + UserVO user = new UserVO(1, "adminuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); + when(cmd.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); + CallContext.register(user, account); + kubernetesVersionService.addKubernetesSupportedVersion(cmd); + } + @Test(expected = InvalidParameterValueException.class) public void addKubernetesSupportedVersionEmptyUrlTest() { AddKubernetesSupportedVersionCmd cmd = Mockito.mock(AddKubernetesSupportedVersionCmd.class); + when(cmd.getMinimumCpu()).thenReturn(KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_CPU); + when(cmd.getMinimumRamSize()).thenReturn(KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE); AccountVO account = new AccountVO("admin", 1L, "", Account.ACCOUNT_TYPE_ADMIN, "uuid"); UserVO user = new UserVO(1, "adminuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); when(cmd.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); @@ -162,6 +191,8 @@ public void addKubernetesSupportedVersionIsoUrlTest() throws ResourceAllocationE when(cmd.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); when(cmd.getUrl()).thenReturn("https://download.cloudstack.com"); when(cmd.getChecksum()).thenReturn(null); + when(cmd.getMinimumCpu()).thenReturn(KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_CPU); + when(cmd.getMinimumRamSize()).thenReturn(KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE); Account systemAccount = new AccountVO("system", 1L, "", Account.ACCOUNT_TYPE_ADMIN, "uuid"); when(accountManager.getSystemAccount()).thenReturn(systemAccount); PowerMockito.mockStatic(ComponentContext.class); @@ -202,4 +233,21 @@ public void deleteKubernetesSupportedVersionTest() { when(kubernetesClusterDao.remove(Mockito.anyLong())).thenReturn(true); kubernetesVersionService.deleteKubernetesSupportedVersion(cmd); } + + @Test + public void updateKubernetesSupportedVersionTest() { + UpdateKubernetesSupportedVersionCmd cmd = Mockito.mock(UpdateKubernetesSupportedVersionCmd.class); + when(cmd.getState()).thenReturn(KubernetesSupportedVersion.State.Disabled.toString()); + AccountVO account = new AccountVO("admin", 1L, "", Account.ACCOUNT_TYPE_ADMIN, "uuid"); + UserVO user = new UserVO(1, "adminuser", "password", "firstname", "lastName", "email", "timezone", UUID.randomUUID().toString(), User.Source.UNKNOWN); + CallContext.register(user, account); + when(kubernetesSupportedVersionDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(KubernetesSupportedVersionVO.class)); + KubernetesSupportedVersionVO version = Mockito.mock(KubernetesSupportedVersionVO.class); + when(kubernetesSupportedVersionDao.createForUpdate(Mockito.anyLong())).thenReturn(version); + when(kubernetesSupportedVersionDao.update(Mockito.anyLong(), Mockito.any(KubernetesSupportedVersionVO.class))).thenReturn(true); + when(version.getState()).thenReturn(KubernetesSupportedVersion.State.Disabled); + when(version.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); + when(kubernetesSupportedVersionDao.findById(Mockito.anyLong())).thenReturn(version); + kubernetesVersionService.updateKubernetesSupportedVersion(cmd); + } } \ No newline at end of file diff --git a/ui/l10n/en.js b/ui/l10n/en.js index 442cbb2ab51f..b125850da3dd 100644 --- a/ui/l10n/en.js +++ b/ui/l10n/en.js @@ -1760,6 +1760,7 @@ var dictionary = { "label.unhealthy.threshold":"Unhealthy Threshold", "label.unlimited":"Unlimited", "label.untagged":"Untagged", +"label.update.kubernetes.version":"Update Kubernetes Version", "label.update.project.resources":"Update project resources", "label.update.ssl":" SSL Certificate", "label.update.ssl.cert":" SSL Certificate", diff --git a/ui/plugins/cks/cks.js b/ui/plugins/cks/cks.js index a3a1a1ee87bd..13fa9bdf782a 100644 --- a/ui/plugins/cks/cks.js +++ b/ui/plugins/cks/cks.js @@ -46,6 +46,8 @@ document.body.removeChild(elem); } }; + var minCpu = 0; + var minRamSize = 0; cloudStack.plugins.cks = function(plugin) { plugin.ui.addSection({ id: 'cks', @@ -219,7 +221,7 @@ versionObjs = json.listkubernetessupportedversionsresponse.kubernetessupportedversion; if (versionObjs != null) { for (var i = 0; i < versionObjs.length; i++) { - if (versionObjs[i].isostate == 'Ready') { + if (versionObjs[i].state == 'Enabled' && versionObjs[i].isostate == 'Ready') { items.push({ id: versionObjs[i].id, description: versionObjs[i].name @@ -241,8 +243,19 @@ var currentVersionId = $(this).val(); if (currentVersionId != null && versionObjs != null) { for (var i = 0; i < versionObjs.length; i++) { - if (currentVersionId == versionObjs[i].id && versionObjs[i].supportsha === true) { - $form.find('.form-item[rel=multimaster]').css('display', 'inline-block'); + if (currentVersionId == versionObjs[i].id) { + if (versionObjs[i].supportsha === true) { + $form.find('.form-item[rel=multimaster]').css('display', 'inline-block'); + } + minCpu = 0; + if (versionObjs[i].mincpunumber != null && versionObjs[i].mincpunumber != undefined) { + minCpu = versionObjs[i].mincpunumber; + } + minRamSize = 0; + if (versionObjs[i].minmemory != null && versionObjs[i].minmemory != undefined) { + minRamSize = versionObjs[i].minmemory; + } + break; } } } @@ -251,6 +264,7 @@ }, serviceoffering: { label: 'label.menu.service.offerings', + dependsOn: ['kubernetesversion'], //docID: 'helpKubernetesClusterServiceOffering', validation: { required: true @@ -265,10 +279,13 @@ var items = json.listserviceofferingsresponse.serviceoffering; if (items != null) { for (var i = 0; i < items.length; i++) { - offeringObjs.push({ - id: items[i].id, - description: items[i].name - }); + if (items[i].iscustomized == false && + items[i].cpunumber >= minCpu && items[i].memory >= minRamSize) { + offeringObjs.push({ + id: items[i].id, + description: items[i].name + }); + } } } args.response.success({ @@ -791,16 +808,13 @@ data: filterData, dataType: "json", async: true, - url: createURL("listKubernetesSupportedVersions"), - dataType: "json", - async: true, success: function(json) { var items = []; var versionObjs = json.listkubernetessupportedversionsresponse.kubernetessupportedversion; if (versionObjs != null) { for (var i = 0; i < versionObjs.length; i++) { if (versionObjs[i].id != args.context.kubernetesclusters[0].kubernetesversionid && - versionObjs[i].isostate == 'Ready') { + versionObjs[i].state == 'Enabled' && versionObjs[i].isostate == 'Ready') { items.push({ id: versionObjs[i].id, description: versionObjs[i].name @@ -1097,6 +1111,19 @@ }, isostate: { label: 'label.iso.state' + }, + mincpunumber: { + label: 'label.min.cpu.cores' + }, + minmemory: { + label: 'label.memory.minimum.mb' + }, + state: { + label: 'label.state', + indicator: { + 'Enabled': 'on', + 'Disabled': 'off' + } } }, advSearchFields: { @@ -1194,6 +1221,18 @@ label: 'label.checksum', //docID: 'Name of the cluster', }, + mincpunumber: { + label: 'label.min.cpu.cores', + validation: { + number: true + }, + }, + minmemory: { + label: 'label.memory.minimum.mb', + validation: { + number: true + } + } } }, @@ -1209,6 +1248,16 @@ zoneid: args.data.zone }); } + if (args.data.mincpunumber != null && args.data.mincpunumber != "" && args.data.mincpunumber > 0) { + $.extend(data, { + mincpunumber: args.data.mincpunumber + }); + } + if (args.data.minmemory != null && args.data.minmemory != "" && args.data.minmemory > 0) { + $.extend(data, { + minmemory: args.data.minmemory + }); + } $.ajax({ url: createURL('addKubernetesSupportedVersion'), data: data, @@ -1258,6 +1307,89 @@ name: 'label.kubernetes.version.details', isMaximized: true, actions: { + update: { + label: 'label.edit', + messages: { + notification: function(args) { + return 'label.update.kubernetes.version'; + } + }, + createForm: { + title: 'label.update.kubernetes.version', + desc: '', + preFilter: function(args) { + var formVersion = args.context.kubernetesversions[0]; + $.ajax({ + url: createURL('listKubernetesSupportedVersions'), + data: { + id: args.context.kubernetesversions[0].id + }, + dataType: "json", + async: false, + success: function (json) { + if (json.listkubernetessupportedversionsresponse.kubernetessupportedversion != null && + json.listkubernetessupportedversionsresponse.kubernetessupportedversion.length > 0) { + formVersion = json.listkubernetessupportedversionsresponse.kubernetessupportedversion[0]; + } + } + }); + if (formVersion.state != null) { + var options = args.$form.find('.form-item[rel=state]').find('option'); + $.each(options, function(optionIndex, option) { + if ($(option).val() === formVersion.state) { + $(option).attr('selected','selected'); + } + }); + } + }, + fields: { + state: { + label: 'label.state', + //docID: 'helpKubernetesClusterZone', + validation: { + required: true + }, + select: function(args) { + var items = []; + items.push({ + id: 'Enabled', + description: 'state.Enabled' + }, { + id: 'Disabled', + description: 'state.Disabled' + }); + args.response.success({ + data: items + }); + } + }, + } + }, + action: function(args) { + var data = { + id: args.context.kubernetesversions[0].id, + state: args.data.state + }; + $.ajax({ + url: createURL('updateKubernetesSupportedVersion'), + data: data, + dataType: "json", + success: function (json) { + var jsonObj; + if (json.updatekubernetessupportedversionresponse.kubernetessupportedversion != null) { + jsonObj = json.updatekubernetessupportedversionresponse.kubernetessupportedversion; + } + args.response.success({ + data: jsonObj + }); + }, + error: function(XMLHttpResponse) { + var errorMsg = parseXMLHttpResponse(XMLHttpResponse); + args.response.error(errorMsg); + } + }); //end ajax + } + }, destroy: { label: 'label.delete.kubernetes.version', compactLabel: 'label.delete', From 86d536363c6e232ff12df2f656a1442beb3b3067 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 30 Jan 2020 17:28:46 +0530 Subject: [PATCH 099/134] fixed param description error Signed-off-by: Abhishek Kumar --- .../kubernetes/version/UpdateKubernetesSupportedVersionCmd.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/UpdateKubernetesSupportedVersionCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/UpdateKubernetesSupportedVersionCmd.java index a182c5dc464b..bf888c54921e 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/UpdateKubernetesSupportedVersionCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/UpdateKubernetesSupportedVersionCmd.java @@ -59,7 +59,7 @@ public class UpdateKubernetesSupportedVersionCmd extends BaseCmd implements Admi private Long id; @Parameter(name = ApiConstants.STATE, type = CommandType.STRING, - description = "the enabled or disabled state of the host", + description = "the enabled or disabled state of the Kubernetes supported version", required = true) private String state; From 10b6c34dabc20f0a89fc710d3c1b33180166594d Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 31 Jan 2020 16:26:04 +0530 Subject: [PATCH 100/134] changes for vm start failures Signed-off-by: Abhishek Kumar --- ...esClusterResourceModifierActionWorker.java | 11 ++-- .../KubernetesClusterStartWorker.java | 62 +++++++++++-------- 2 files changed, 43 insertions(+), 30 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index bd044ec219a3..d8f28b9b6442 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -42,7 +42,6 @@ import com.cloud.dc.DataCenter; import com.cloud.dc.dao.ClusterDao; import com.cloud.deploy.DeployDestination; -import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InsufficientServerCapacityException; import com.cloud.exception.ManagementServerException; @@ -239,7 +238,7 @@ protected DeployDestination plan() throws InsufficientServerCapacityException { return plan(kubernetesCluster.getTotalNodeCount(), zone, offering); } - protected void startKubernetesVM(final UserVm vm) throws ConcurrentOperationException { + protected void startKubernetesVM(final UserVm vm) throws ManagementServerException { try { StartVMCmd startVm = new StartVMCmd(); startVm = ComponentContext.inject(startVm); @@ -252,12 +251,12 @@ protected void startKubernetesVM(final UserVm vm) throws ConcurrentOperationExce } } catch (IllegalAccessException | NoSuchFieldException | ExecutionException | ResourceUnavailableException | ResourceAllocationException | InsufficientCapacityException ex) { - logAndThrow(Level.WARN, String.format("Failed to start VM in the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), ex); + throw new ManagementServerException(String.format("Failed to start VM in the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), ex); } UserVm startVm = userVmDao.findById(vm.getId()); if (!startVm.getState().equals(VirtualMachine.State.Running)) { - logAndThrow(Level.WARN, String.format("Failed to start VM in the Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); + throw new ManagementServerException(String.format("Failed to start VM in the Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); } } @@ -268,6 +267,10 @@ protected List provisionKubernetesClusterNodeVms(final long nodeCount, f UserVm vm = createKubernetesNode(publicIpAddress, i); addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId()); startKubernetesVM(vm); + vm = userVmDao.findById(vm.getId()); + if (vm == null) { + throw new ManagementServerException(String.format("Failed to provision worker VM for Kubernetes cluster ID: %s" , kubernetesCluster.getUuid())); + } nodes.add(vm); if (LOGGER.isInfoEnabled()) { LOGGER.info(String.format("Provisioned node VM ID: %s in to the Kubernetes cluster ID: %s", vm.getUuid(), kubernetesCluster.getUuid())); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index df5464740060..29c7d3f167a4 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -256,37 +256,38 @@ private UserVm createKubernetesAdditionalMaster(final String joinIp, final int a return additionalMasterVm; } - private UserVm provisionKubernetesClusterMasterVm(final Network network, final String publicIpAddress) throws CloudRuntimeException { + private UserVm provisionKubernetesClusterMasterVm(final Network network, final String publicIpAddress) throws + ManagementServerException, InsufficientCapacityException, ResourceUnavailableException { UserVm k8sMasterVM = null; - try { - k8sMasterVM = createKubernetesMaster(network, publicIpAddress); - addKubernetesClusterVm(kubernetesCluster.getId(), k8sMasterVM.getId()); - startKubernetesVM(k8sMasterVM); - k8sMasterVM = userVmDao.findById(k8sMasterVM.getId()); - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Provisioned the master VM ID: %s in to the Kubernetes cluster ID: %s", k8sMasterVM.getUuid(), kubernetesCluster.getUuid())); - } - } catch (ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { - logTransitStateAndThrow(Level.ERROR, String.format("Provisioning the master VM failed in the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e); + k8sMasterVM = createKubernetesMaster(network, publicIpAddress); + addKubernetesClusterVm(kubernetesCluster.getId(), k8sMasterVM.getId()); + startKubernetesVM(k8sMasterVM); + k8sMasterVM = userVmDao.findById(k8sMasterVM.getId()); + if (k8sMasterVM == null) { + throw new ManagementServerException(String.format("Failed to provision master VM for Kubernetes cluster ID: %s" , kubernetesCluster.getUuid())); + } + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Provisioned the master VM ID: %s in to the Kubernetes cluster ID: %s", k8sMasterVM.getUuid(), kubernetesCluster.getUuid())); } return k8sMasterVM; } - private List provisionKubernetesClusterAdditionalMasterVms(final String publicIpAddress) throws CloudRuntimeException { + private List provisionKubernetesClusterAdditionalMasterVms(final String publicIpAddress) throws + InsufficientCapacityException, ManagementServerException, ResourceUnavailableException { List additionalMasters = new ArrayList<>(); if (kubernetesCluster.getMasterNodeCount() > 1) { for (int i = 1; i < kubernetesCluster.getMasterNodeCount(); i++) { UserVm vm = null; - try { - vm = createKubernetesAdditionalMaster(publicIpAddress, i); - addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId()); - startKubernetesVM(vm); - additionalMasters.add(vm); - if (LOGGER.isInfoEnabled()) { - LOGGER.info(String.format("Provisioned additional master VM ID: %s in to the Kubernetes cluster ID: %s", vm.getUuid(), kubernetesCluster.getUuid())); - } - } catch (ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { - logTransitStateAndThrow(Level.ERROR, String.format("Provisioning additional master VM %d/%d failed in the Kubernetes cluster ID: %s", i+1, kubernetesCluster.getMasterNodeCount(), kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e); + vm = createKubernetesAdditionalMaster(publicIpAddress, i); + addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId()); + startKubernetesVM(vm); + vm = userVmDao.findById(vm.getId()); + if (vm == null) { + throw new ManagementServerException(String.format("Failed to provision additional master VM for Kubernetes cluster ID: %s" , kubernetesCluster.getUuid())); + } + additionalMasters.add(vm); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("Provisioned additional master VM ID: %s in to the Kubernetes cluster ID: %s", vm.getUuid(), kubernetesCluster.getUuid())); } } } @@ -404,7 +405,7 @@ private void startKubernetesClusterVMs() { } try { startKubernetesVM(vm); - } catch (CloudRuntimeException ex) { + } catch (ManagementServerException ex) { LOGGER.warn(String.format("Failed to start VM ID: %s in Kubernetes cluster ID: %s due to ", vm.getUuid(), kubernetesCluster.getUuid()) + ex); // dont bail out here. proceed further to stop the reset of the VM's } @@ -482,7 +483,12 @@ public boolean startKubernetesClusterOnCreate() { logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s as no public IP found for the cluster" , kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); } List clusterVMs = new ArrayList<>(); - UserVm k8sMasterVM = provisionKubernetesClusterMasterVm(network, publicIpAddress); + UserVm k8sMasterVM = null; + try { + k8sMasterVM = provisionKubernetesClusterMasterVm(network, publicIpAddress); + } catch (ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { + logTransitStateAndThrow(Level.ERROR, String.format("Provisioning the master VM failed in the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e); + } clusterVMs.add(k8sMasterVM); if (Strings.isNullOrEmpty(publicIpAddress)) { publicIpSshPort = getKubernetesClusterServerIpSshPort(k8sMasterVM); @@ -491,8 +497,12 @@ public boolean startKubernetesClusterOnCreate() { logTransitStateAndThrow(Level.WARN, String.format("Failed to start Kubernetes cluster ID: %s as no public IP found for the cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); } } - List additionalMasterVMs = provisionKubernetesClusterAdditionalMasterVms(publicIpAddress); - clusterVMs.addAll(additionalMasterVMs); + try { + List additionalMasterVMs = provisionKubernetesClusterAdditionalMasterVms(publicIpAddress); + clusterVMs.addAll(additionalMasterVMs); + } catch (ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { + logTransitStateAndThrow(Level.ERROR, String.format("Provisioning additional master VM failed in the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e); + } try { List nodeVMs = provisionKubernetesClusterNodeVms(kubernetesCluster.getNodeCount(), publicIpAddress); clusterVMs.addAll(nodeVMs); From f1d8e22e1672f896f5f546ae480bcfc7e03be7cd Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 31 Jan 2020 17:20:54 +0530 Subject: [PATCH 101/134] fix for create vm exception Signed-off-by: Abhishek Kumar --- .../cluster/actionworkers/KubernetesClusterScaleWorker.java | 2 +- .../cluster/actionworkers/KubernetesClusterStartWorker.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index a5b575c1371a..35e5b2726c38 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -364,7 +364,7 @@ private void scaleUpKubernetesClusterSize(final long newVmCount) throws CloudRun List clusterVMIds = new ArrayList<>(); try { clusterVMs = provisionKubernetesClusterNodeVms((int)(newVmCount + kubernetesCluster.getNodeCount()), (int)kubernetesCluster.getNodeCount(), publicIpAddress); - } catch (ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { + } catch (CloudRuntimeException | ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { logTransitStateToFailedIfNeededAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, unable to provision node VM in the cluster", kubernetesCluster.getUuid()), e); } for (UserVm vm : clusterVMs) { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index 29c7d3f167a4..0dfcbcd64b9b 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -486,7 +486,7 @@ public boolean startKubernetesClusterOnCreate() { UserVm k8sMasterVM = null; try { k8sMasterVM = provisionKubernetesClusterMasterVm(network, publicIpAddress); - } catch (ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { + } catch (CloudRuntimeException | ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { logTransitStateAndThrow(Level.ERROR, String.format("Provisioning the master VM failed in the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e); } clusterVMs.add(k8sMasterVM); @@ -500,13 +500,13 @@ public boolean startKubernetesClusterOnCreate() { try { List additionalMasterVMs = provisionKubernetesClusterAdditionalMasterVms(publicIpAddress); clusterVMs.addAll(additionalMasterVMs); - } catch (ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { + } catch (CloudRuntimeException | ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { logTransitStateAndThrow(Level.ERROR, String.format("Provisioning additional master VM failed in the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e); } try { List nodeVMs = provisionKubernetesClusterNodeVms(kubernetesCluster.getNodeCount(), publicIpAddress); clusterVMs.addAll(nodeVMs); - } catch (ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { + } catch (CloudRuntimeException | ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { logTransitStateAndThrow(Level.ERROR, String.format("Provisioning node VM failed in the Kubernetes cluster ID: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e); } if (LOGGER.isInfoEnabled()) { From d670c55ca45116ea7eeff07279a1f62cbca514ec Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 3 Feb 2020 19:00:35 +0530 Subject: [PATCH 102/134] changes for multiple hypervisor specific template global settings Signed-off-by: Abhishek Kumar --- .../cluster/KubernetesClusterManagerImpl.java | 100 ++++++++++++++---- .../cluster/KubernetesClusterService.java | 21 +++- 2 files changed, 100 insertions(+), 21 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index a2294bdcb34b..20954339a947 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -80,6 +80,7 @@ import com.cloud.exception.ResourceAllocationException; import com.cloud.host.Host.Type; import com.cloud.host.HostVO; +import com.cloud.hypervisor.Hypervisor; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterActionWorker; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterDestroyWorker; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterScaleWorker; @@ -110,6 +111,7 @@ import com.cloud.offerings.NetworkOfferingVO; import com.cloud.offerings.dao.NetworkOfferingDao; import com.cloud.offerings.dao.NetworkOfferingServiceMapDao; +import com.cloud.org.Cluster; import com.cloud.org.Grouping; import com.cloud.resource.ResourceManager; import com.cloud.service.ServiceOfferingVO; @@ -249,15 +251,55 @@ private void logAndThrow(final Level logLevel, final String message, final Excep private boolean isKubernetesServiceTemplateConfigured(DataCenter zone) { // Check Kubernetes VM template for zone - String templateName = KubernetesClusterTemplateName.value(); - if (templateName == null || templateName.isEmpty()) { - LOGGER.warn(String.format("Global setting %s is empty. Template name need to be specified for Kubernetes service to function", KubernetesClusterTemplateName.key())); - return false; + boolean isHyperVAvailable = false; + boolean isKVMAvailable = false; + boolean isVMwareAvailable = false; + boolean isXenserverAvailable = false; + List clusters = clusterDao.listByZoneId(zone.getId()); + for (ClusterVO clusterVO : clusters) { + if (Hypervisor.HypervisorType.Hyperv.equals(clusterVO.getHypervisorType())) { + isHyperVAvailable = true; + } + if (Hypervisor.HypervisorType.KVM.equals(clusterVO.getHypervisorType())) { + isKVMAvailable = true; + } + if (Hypervisor.HypervisorType.VMware.equals(clusterVO.getHypervisorType())) { + isVMwareAvailable = true; + } + if (Hypervisor.HypervisorType.XenServer.equals(clusterVO.getHypervisorType())) { + isXenserverAvailable = true; + } } - final VMTemplateVO template = templateDao.findByTemplateName(templateName); - if (template == null) { - LOGGER.warn(String.format("Unable to find the template %s to be used for provisioning Kubernetes cluster", templateName)); - return false; + List> templatePairs = new ArrayList<>(); + if (isHyperVAvailable) { + templatePairs.add(new Pair<>(KubernetesClusterHyperVTemplateName.key(), KubernetesClusterHyperVTemplateName.value())); + } + if (isKVMAvailable) { + templatePairs.add(new Pair<>(KubernetesClusterKVMTemplateName.key(), KubernetesClusterKVMTemplateName.value())); + } + if (isVMwareAvailable) { + templatePairs.add(new Pair<>(KubernetesClusterVMwareTemplateName.key(), KubernetesClusterVMwareTemplateName.value())); + } + if (isXenserverAvailable) { + templatePairs.add(new Pair<>(KubernetesClusterXenserverTemplateName.key(), KubernetesClusterXenserverTemplateName.value())); + } + for (Pair templatePair : templatePairs) { + String templateKey = templatePair.first(); + String templateName = templatePair.second(); + if (Strings.isNullOrEmpty(templateName)) { + LOGGER.warn(String.format("Global setting %s is empty. Template name need to be specified for Kubernetes service to function", templateKey)); + return false; + } + final VMTemplateVO template = templateDao.findByTemplateName(templateName); + if (template == null) { + LOGGER.warn(String.format("Unable to find the template %s to be used for provisioning Kubernetes cluster nodes", templateName)); + return false; + } + List listZoneTemplate = templateZoneDao.listByZoneTemplate(zone.getId(), template.getId()); + if (listZoneTemplate == null || listZoneTemplate.isEmpty()) { + LOGGER.warn(String.format("The template ID: %s, name: %s is not available for use in zone ID: %s provisioning Kubernetes cluster nodes", template.getUuid(), templateName, zone.getUuid())); + return false; + } } return true; } @@ -331,6 +373,25 @@ private IpAddress getSourceNatIp(Network network) { return null; } + private VMTemplateVO getKubernetesServiceTemplate(Hypervisor.HypervisorType hypervisorType) { + String tempalteName = null; + switch (hypervisorType) { + case Hyperv: + tempalteName = KubernetesClusterHyperVTemplateName.value(); + break; + case KVM: + tempalteName = KubernetesClusterKVMTemplateName.value(); + break; + case VMware: + tempalteName = KubernetesClusterVMwareTemplateName.value(); + break; + case XenServer: + tempalteName = KubernetesClusterXenserverTemplateName.value(); + break; + } + return templateDao.findByTemplateName(tempalteName); + } + private boolean validateIsolatedNetwork(Network network, int clusterTotalNodeCount) { if (Network.GuestType.Isolated.equals(network.getGuestType())) { if (Network.State.Allocated.equals(network.getState())) { // Allocated networks won't have IP and rules @@ -451,6 +512,7 @@ private DeployDestination plan(final long nodesCount, final DataCenter zone, fin hosts_with_resevered_capacity.put(h.getUuid(), new Pair(h, 0)); } boolean suitable_host_found = false; + Cluster planCluster = null; for (int i = 1; i <= nodesCount + 1; i++) { suitable_host_found = false; for (Map.Entry> hostEntry : hosts_with_resevered_capacity.entrySet()) { @@ -472,6 +534,7 @@ private DeployDestination plan(final long nodesCount, final DataCenter zone, fin } hostEntry.setValue(new Pair(h, reserved)); suitable_host_found = true; + planCluster = cluster; break; } } @@ -486,7 +549,7 @@ private DeployDestination plan(final long nodesCount, final DataCenter zone, fin if (LOGGER.isInfoEnabled()) { LOGGER.info(String.format("Suitable hosts found in datacenter ID: %s, creating deployment destination", zone.getUuid())); } - return new DeployDestination(zone, null, null, null); + return new DeployDestination(zone, null, planCluster, null); } String msg = String.format("Cannot find enough capacity for Kubernetes cluster(requested cpu=%1$s memory=%2$s)", cpu_requested * nodesCount, ram_requested * nodesCount); @@ -629,12 +692,6 @@ private void validateKubernetesClusterCreateParameters(final CreateKubernetesClu throw new InvalidParameterValueException(String.format("Invalid value for %s", ApiConstants.NODE_ROOT_DISK_SIZE)); } - VMTemplateVO template = templateDao.findByTemplateName(KubernetesClusterTemplateName.value()); - List listZoneTemplate = templateZoneDao.listByZoneTemplate(zone.getId(), template.getId()); - if (listZoneTemplate == null || listZoneTemplate.isEmpty()) { - logAndThrow(Level.WARN, String.format("The template ID: %s is not available for use in zone ID: %s to provision Kubernetes cluster name: %s", template.getUuid(), zone.getUuid(), name)); - } - if (!validateServiceOffering(serviceOffering, clusterKubernetesVersion)) { throw new InvalidParameterValueException("Given service offering ID: %s is not suitable for Kubernetes cluster"); } @@ -896,14 +953,18 @@ public KubernetesCluster createKubernetesCluster(CreateKubernetesClusterCmd cmd) final Account owner = accountService.getActiveAccountById(cmd.getEntityOwnerId()); final KubernetesSupportedVersion clusterKubernetesVersion = kubernetesSupportedVersionDao.findById(cmd.getKubernetesVersionId()); + DeployDestination deployDestination = null; try { - plan(totalNodeCount, zone, serviceOffering); + deployDestination = plan(totalNodeCount, zone, serviceOffering); } catch (InsufficientCapacityException e) { logAndThrow(Level.ERROR, String.format("Creating Kubernetes cluster failed due to insufficient capacity for %d cluster nodes in zone ID: %s with service offering ID: %s", totalNodeCount, zone.getUuid(), serviceOffering.getUuid())); } + if (deployDestination == null || deployDestination.getCluster() == null) { + logAndThrow(Level.ERROR, String.format("Creating Kubernetes cluster failed due to error while finding suitable deployment plan for cluster in zone ID: %s", zone.getUuid())); + } final Network defaultNetwork = getKubernetesClusterNetworkIfMissing(cmd.getName(), zone, owner, (int)masterNodeCount, (int)clusterSize, cmd.getExternalLoadBalancerIpAddress(), cmd.getNetworkId()); - final VMTemplateVO finalTemplate = templateDao.findByTemplateName(KubernetesClusterTemplateName.value());; + final VMTemplateVO finalTemplate = getKubernetesServiceTemplate(deployDestination.getCluster().getHypervisorType()); final long cores = serviceOffering.getCpu() * (masterNodeCount + clusterSize); final long memory = serviceOffering.getRamSize() * (masterNodeCount + clusterSize); @@ -1369,7 +1430,10 @@ public String getConfigComponentName() { public ConfigKey[] getConfigKeys() { return new ConfigKey[] { KubernetesServiceEnabled, - KubernetesClusterTemplateName, + KubernetesClusterHyperVTemplateName, + KubernetesClusterKVMTemplateName, + KubernetesClusterVMwareTemplateName, + KubernetesClusterXenserverTemplateName, KubernetesClusterNetworkOffering, KubernetesClusterStartTimeout, KubernetesClusterScaleTimeout, diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java index d1c8c9af9a41..88639e4cc7e6 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java @@ -40,10 +40,25 @@ public interface KubernetesClusterService extends PluggableService, Configurable "false", "Indicates whether Kubernetes Service plugin is enabled or not. Management server restart needed on change", false); - static final ConfigKey KubernetesClusterTemplateName = new ConfigKey("Advanced", String.class, - "cloud.kubernetes.cluster.template.name", + static final ConfigKey KubernetesClusterHyperVTemplateName = new ConfigKey("Advanced", String.class, + "cloud.kubernetes.cluster.template.name.hyperv", "Kubernetes-Service-Template", - "Name of the template to be used for creating Kubernetes cluster nodes", + "Name of the template to be used for creating Kubernetes cluster nodes on HyperV", + true); + static final ConfigKey KubernetesClusterKVMTemplateName = new ConfigKey("Advanced", String.class, + "cloud.kubernetes.cluster.template.name.kvm", + "Kubernetes-Service-Template", + "Name of the template to be used for creating Kubernetes cluster nodes on KVM", + true); + static final ConfigKey KubernetesClusterVMwareTemplateName = new ConfigKey("Advanced", String.class, + "cloud.kubernetes.cluster.template.name.vmware", + "Kubernetes-Service-Template", + "Name of the template to be used for creating Kubernetes cluster nodes on VMware", + true); + static final ConfigKey KubernetesClusterXenserverTemplateName = new ConfigKey("Advanced", String.class, + "cloud.kubernetes.cluster.template.name.xenserver", + "Kubernetes-Service-Template", + "Name of the template to be used for creating Kubernetes cluster nodes on Xenserver", true); static final ConfigKey KubernetesClusterNetworkOffering = new ConfigKey("Advanced", String.class, "cloud.kubernetes.cluster.network.offering", From e6bb497565529f3bf70c97346f619e451a234e7a Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 5 Feb 2020 01:09:27 +0530 Subject: [PATCH 103/134] fix for cluster node VM name Signed-off-by: Abhishek Kumar --- ...esClusterResourceModifierActionWorker.java | 25 ++++++++++++++++++- .../KubernetesClusterStartWorker.java | 4 +-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index d8f28b9b6442..6ca480faa925 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -80,6 +80,7 @@ import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.ExecutionException; import com.cloud.utils.net.Ip; +import com.cloud.utils.net.NetUtils; import com.cloud.vm.Nic; import com.cloud.vm.VirtualMachine; import com.google.common.base.Strings; @@ -107,10 +108,17 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu @Inject protected LoadBalancerDao loadBalancerDao; + protected String kubernetesClusterNodeNamePrefix; + protected KubernetesClusterResourceModifierActionWorker(final KubernetesCluster kubernetesCluster, final KubernetesClusterManagerImpl clusterManager) { super(kubernetesCluster, clusterManager); } + protected void init() { + super.init(); + kubernetesClusterNodeNamePrefix = getKubernetesClusterNodeNamePrefix(); + } + private String getKubernetesNodeConfig(final String joinIp) throws IOException { String k8sNodeConfig = readResourceFile("/conf/k8s-node.yml"); final String sshPubKey = "{{ k8s.ssh.pub.key }}"; @@ -299,7 +307,7 @@ protected UserVm createKubernetesNode(String joinIp, int nodeInstance) throws Ma if (rootDiskSize > 0) { customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); } - String hostName = String.format("%s-k8s-node-%s", kubernetesCluster.getName(), nodeInstance); + String hostName = String.format("%s-node-%s", kubernetesClusterNodeNamePrefix, nodeInstance); String k8sNodeConfig = null; try { k8sNodeConfig = getKubernetesNodeConfig(joinIp); @@ -468,4 +476,19 @@ protected void removeLoadBalancingRule(final IpAddress publicIp, final Network n } } } + + protected String getKubernetesClusterNodeNamePrefix() { + String prefix = kubernetesCluster.getName(); + if (!NetUtils.verifyDomainNameLabel(prefix, true)) { + prefix = prefix.replaceAll("[^a-zA-Z0-9-]", ""); + if (prefix.length() == 0) { + prefix = kubernetesCluster.getUuid(); + } + prefix = "k8s-" + prefix; + } + if (prefix.length() > 40) { + prefix = prefix.substring(0, 40); + } + return prefix; + } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index 0dfcbcd64b9b..b8f0de371b61 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -181,7 +181,7 @@ private UserVm createKubernetesMaster(final Network network, String serverIp) th if (rootDiskSize > 0) { customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); } - String hostName = kubernetesCluster.getName() + "-k8s-master"; + String hostName = kubernetesClusterNodeNamePrefix + "-master"; if (kubernetesCluster.getMasterNodeCount() > 1) { hostName += "-1"; } @@ -238,7 +238,7 @@ private UserVm createKubernetesAdditionalMaster(final String joinIp, final int a if (rootDiskSize > 0) { customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); } - String hostName = String.format("%s-k8s-master-%d", kubernetesCluster.getName(), additionalMasterNodeInstance + 1); + String hostName = String.format("%s-master-%d", kubernetesClusterNodeNamePrefix, additionalMasterNodeInstance + 1); String k8sMasterConfig = null; try { k8sMasterConfig = getKubernetesAdditionalMasterConfig(joinIp); From d11e261050386de4f171845a1445ef2894a6dc02 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 6 Feb 2020 02:57:34 +0530 Subject: [PATCH 104/134] k8s config changes, iso eject Signed-off-by: Abhishek Kumar --- .../main/resources/conf/k8s-master-add.yml | 10 +++++++-- .../src/main/resources/conf/k8s-master.yml | 22 +++++++++++++------ .../src/main/resources/conf/k8s-node.yml | 10 +++++++-- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml index a518cbc3264d..84dbd8a093e1 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml @@ -34,14 +34,16 @@ write-files: ISO_MOUNT_DIR=/mnt/k8sdisk BINARIES_DIR=${ISO_MOUNT_DIR}/ + K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/ ATTEMPT_ONLINE_INSTALL=false setup_complete=false - OFFLINE_INSTALL_ATTEMPT_SLEEP=5 - MAX_OFFLINE_INSTALL_ATTEMPTS=36 + OFFLINE_INSTALL_ATTEMPT_SLEEP=15 + MAX_OFFLINE_INSTALL_ATTEMPTS=40 offline_attempts=1 MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 crucial_cmd_attempts=1 + iso_drive_path="" while true; do if (( "$offline_attempts" > "$MAX_OFFLINE_INSTALL_ATTEMPTS" )); then echo "Warning: Offline install timed out!" @@ -62,6 +64,7 @@ write-files: set -e if [ $retval -eq 0 ]; then if [ -d "$BINARIES_DIR" ]; then + iso_drive_path="${line}" break else umount "${line}" && rmdir "${ISO_MOUNT_DIR}" @@ -122,6 +125,9 @@ write-files: setup_complete=true fi umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" + if [ "$iso_drive_path" != "" ]; then + eject "${iso_drive_path}" + fi fi if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then ### Binaries not available offline ### diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml index 0face90ee0f3..c9578c12df91 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml @@ -54,14 +54,16 @@ write-files: ISO_MOUNT_DIR=/mnt/k8sdisk BINARIES_DIR=${ISO_MOUNT_DIR}/ + K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/ ATTEMPT_ONLINE_INSTALL=false setup_complete=false - OFFLINE_INSTALL_ATTEMPT_SLEEP=5 - MAX_OFFLINE_INSTALL_ATTEMPTS=36 + OFFLINE_INSTALL_ATTEMPT_SLEEP=15 + MAX_OFFLINE_INSTALL_ATTEMPTS=40 offline_attempts=1 MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 crucial_cmd_attempts=1 + iso_drive_path="" while true; do if (( "$offline_attempts" > "$MAX_OFFLINE_INSTALL_ATTEMPTS" )); then echo "Warning: Offline install timed out!" @@ -82,6 +84,7 @@ write-files: set -e if [ $retval -eq 0 ]; then if [ -d "$BINARIES_DIR" ]; then + iso_drive_path="${line}" break else umount "${line}" && rmdir "${ISO_MOUNT_DIR}" @@ -141,7 +144,12 @@ write-files: done <<< "$output" setup_complete=true fi + mkdir -p "${K8S_CONFIG_SCRIPTS_COPY_DIR}" + cp ${BINARIES_DIR}/*.yaml "${K8S_CONFIG_SCRIPTS_COPY_DIR}" umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" + if [ "$iso_drive_path" != "" ]; then + eject "${iso_drive_path}" + fi fi if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then ### Binaries not available offline ### @@ -215,8 +223,7 @@ write-files: exit 0 fi - ISO_MOUNT_DIR=/mnt/k8sdisk - BINARIES_DIR=${ISO_MOUNT_DIR}/ + K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/ if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then echo "setup-kube-system is running!" @@ -232,11 +239,12 @@ write-files: chown $(id -u):$(id -g) /root/.kube/config echo export PATH=\$PATH:/opt/bin >> /root/.bashrc - if [ -d "$BINARIES_DIR" ]; then + if [ -d "$K8S_CONFIG_SCRIPTS_COPY_DIR" ]; then ### Network, dashboard configs available offline ### echo "Offline configs are available!" - kubectl apply -f ${BINARIES_DIR}/network.yaml - kubectl apply -f ${BINARIES_DIR}/dashboard.yaml + kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/network.yaml + kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/dashboard.yaml + rm -rf "${K8S_CONFIG_SCRIPTS_COPY_DIR}" else kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')" kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-beta6/aio/deploy/recommended.yaml diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index 467cfcc9236d..63af4c833a1b 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -34,14 +34,16 @@ write-files: ISO_MOUNT_DIR=/mnt/k8sdisk BINARIES_DIR=${ISO_MOUNT_DIR}/ + K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/ ATTEMPT_ONLINE_INSTALL=false setup_complete=false - OFFLINE_INSTALL_ATTEMPT_SLEEP=5 - MAX_OFFLINE_INSTALL_ATTEMPTS=36 + OFFLINE_INSTALL_ATTEMPT_SLEEP=15 + MAX_OFFLINE_INSTALL_ATTEMPTS=40 offline_attempts=1 MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 crucial_cmd_attempts=1 + iso_drive_path="" while true; do if (( "$offline_attempts" > "$MAX_OFFLINE_INSTALL_ATTEMPTS" )); then echo "Warning: Offline install timed out!" @@ -62,6 +64,7 @@ write-files: set -e if [ $retval -eq 0 ]; then if [ -d "$BINARIES_DIR" ]; then + iso_drive_path="${line}" break else umount "${line}" && rmdir "${ISO_MOUNT_DIR}" @@ -122,6 +125,9 @@ write-files: setup_complete=true fi umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" + if [ "$iso_drive_path" != "" ]; then + eject "${iso_drive_path}" + fi fi if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then ### Binaries not available offline ### From b63f5f62eeada7486883987e905d3763a8b36a98 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 6 Feb 2020 03:39:54 +0530 Subject: [PATCH 105/134] destroy improvement Signed-off-by: Abhishek Kumar --- .../KubernetesClusterDestroyWorker.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java index 1182ac1a7cf2..06b3a9513e4e 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java @@ -30,6 +30,7 @@ import com.cloud.exception.ManagementServerException; import com.cloud.exception.PermissionDeniedException; import com.cloud.exception.ResourceUnavailableException; +import com.cloud.hypervisor.Hypervisor; import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; @@ -91,11 +92,16 @@ private boolean destroyClusterVMs() { , vm.getInstanceName() , vm.getUuid() , vm.getState().toString())); + } + if (!VirtualMachine.State.Expunging.equals(vm.getState()) || + Hypervisor.HypervisorType.VMware.equals(vm.getHypervisorType())) { vm = userVmService.expungeVm(vmID); - LOGGER.warn(String.format("VM '%s' ID: %s is in state: %s, Kubernetes cluster will probably fail" - , vm.getInstanceName() - , vm.getUuid() - , vm.getState().toString())); + if (!VirtualMachine.State.Expunging.equals(vm.getState())) { + LOGGER.warn(String.format("VM '%s' ID: %s is in state: %s, Kubernetes cluster will probably fail" + , vm.getInstanceName() + , vm.getUuid() + , vm.getState().toString())); + } } kubernetesClusterVmMapDao.expunge(clusterVM.getId()); if (LOGGER.isInfoEnabled()) { From e0f9c2089d567b3b451be81578a0caf156233800 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 6 Feb 2020 03:47:59 +0530 Subject: [PATCH 106/134] upgrade script changes Signed-off-by: Abhishek Kumar --- .../src/main/resources/script/upgrade-kubernetes.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh index a013003670ea..96520d35d121 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh +++ b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh @@ -40,6 +40,7 @@ BINARIES_DIR=${ISO_MOUNT_DIR}/ OFFLINE_INSTALL_ATTEMPT_SLEEP=5 MAX_OFFLINE_INSTALL_ATTEMPTS=10 offline_attempts=1 +iso_drive_path="" while true; do if (( "$offline_attempts" > "$MAX_OFFLINE_INSTALL_ATTEMPTS" )); then echo "Warning: Offline install timed out!" @@ -60,6 +61,7 @@ while true; do set -e if [ $retval -eq 0 ]; then if [ -d "$BINARIES_DIR" ]; then + iso_drive_path="${line}" break else umount "${line}" && rmdir "${ISO_MOUNT_DIR}" @@ -121,4 +123,7 @@ if [ -d "$BINARIES_DIR" ]; then fi umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" -fi \ No newline at end of file + if [ "$iso_drive_path" != "" ]; then + eject "${iso_drive_path}" + fi +fi From ba08fb8e3fa3ad4d7b6c7896e898f4e18d8c0424 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 12 Feb 2020 12:11:54 +0530 Subject: [PATCH 107/134] changes for making mincpu and minram required for version Signed-off-by: Abhishek Kumar --- .../META-INF/db/schema-41300to41400.sql | 4 ++-- .../cluster/KubernetesClusterManagerImpl.java | 19 ++++++++----------- .../cluster/KubernetesClusterVO.java | 2 +- .../version/KubernetesSupportedVersion.java | 4 ++-- .../version/KubernetesSupportedVersionVO.java | 18 +++++++----------- .../version/KubernetesVersionManagerImpl.java | 4 ++-- .../AddKubernetesSupportedVersionCmd.java | 4 ++-- ui/plugins/cks/cks.js | 2 ++ 8 files changed, 26 insertions(+), 31 deletions(-) diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql b/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql index 26c03fdbb9cc..6c26d743d790 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql @@ -61,8 +61,8 @@ CREATE TABLE IF NOT EXISTS `cloud`.`kubernetes_supported_version` ( `iso_id` bigint unsigned NOT NULL COMMENT 'the ID of the binaries ISO for this Kubernetes version', `zone_id` bigint unsigned DEFAULT NULL COMMENT 'the ID of the zone for which this Kubernetes version is made available', `state` char(32) DEFAULT NULL COMMENT 'the enabled or disabled state for this Kubernetes version', - `min_cpu` int(10) unsigned DEFAULT NULL COMMENT 'the minimum CPU needed by cluster nodes for using this Kubernetes version', - `min_ram_size` bigint(20) unsigned DEFAULT NULL COMMENT 'the minimum RAM in MB needed by cluster nodes for this Kubernetes version', + `min_cpu` int(10) unsigned NOT NULL COMMENT 'the minimum CPU needed by cluster nodes for using this Kubernetes version', + `min_ram_size` bigint(20) unsigned NOT NULL COMMENT 'the minimum RAM in MB needed by cluster nodes for this Kubernetes version', `created` datetime NOT NULL COMMENT 'date created', `removed` datetime COMMENT 'date removed or null, if still present', diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 20954339a947..777b1f6b6345 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -120,7 +120,6 @@ import com.cloud.storage.VMTemplateZoneVO; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateZoneDao; -import com.cloud.template.TemplateApiService; import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.AccountService; @@ -178,8 +177,6 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne @Inject protected VMTemplateDao templateDao; @Inject - protected TemplateApiService templateService; - @Inject protected VMTemplateZoneDao templateZoneDao; @Inject protected TemplateJoinDao templateJoinDao; @@ -458,13 +455,13 @@ private boolean validateServiceOffering(final ServiceOffering serviceOffering, f if (serviceOffering.isDynamic()) { throw new InvalidParameterValueException(String.format("Custom service offerings are not supported for creating clusters, service offering ID: %s", serviceOffering.getUuid())); } - if ((version.getMinimumCpu() == null || version.getMinimumRamSize() == null) && serviceOffering.getCpu() < MIN_KUBERNETES_CLUSTER_NODE_CPU || serviceOffering.getRamSize() < MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE) { + if (serviceOffering.getCpu() < MIN_KUBERNETES_CLUSTER_NODE_CPU || serviceOffering.getRamSize() < MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE) { throw new InvalidParameterValueException(String.format("Kubernetes cluster cannot be created with service offering ID: %s, Kubernetes cluster template(CoreOS) needs minimum %d vCPUs and %d MB RAM", serviceOffering.getUuid(), MIN_KUBERNETES_CLUSTER_NODE_CPU, MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE)); } - if (version.getMinimumCpu() != null && serviceOffering.getCpu() < version.getMinimumCpu()) { + if (serviceOffering.getCpu() < version.getMinimumCpu()) { throw new InvalidParameterValueException(String.format("Kubernetes cluster cannot be created with service offering ID: %s, Kubernetes version ID: %s needs minimum %d vCPUs", serviceOffering.getUuid(), version.getUuid(), version.getMinimumCpu())); } - if (version.getMinimumRamSize() != null && serviceOffering.getRamSize() < version.getMinimumRamSize()) { + if (serviceOffering.getRamSize() < version.getMinimumRamSize()) { throw new InvalidParameterValueException(String.format("Kubernetes cluster cannot be created with service offering ID: %s, associated Kubernetes version ID: %s needs minimum %d MB RAM", serviceOffering.getUuid(), version.getUuid(), version.getMinimumRamSize())); } return true; @@ -832,15 +829,15 @@ private void validateKubernetesClusterScaleParameters(ScaleKubernetesClusterCmd if (serviceOffering.isDynamic()) { throw new InvalidParameterValueException(String.format("Custom service offerings are not supported for Kubernetes clusters. Kubernetes cluster ID: %s, service offering ID: %s", kubernetesCluster.getUuid(), serviceOffering.getUuid())); } - if ((clusterVersion.getMinimumCpu() == null || clusterVersion.getMinimumRamSize() == null) && serviceOffering.getCpu() < MIN_KUBERNETES_CLUSTER_NODE_CPU || serviceOffering.getRamSize() < MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE) { + if (serviceOffering.getCpu() < MIN_KUBERNETES_CLUSTER_NODE_CPU || serviceOffering.getRamSize() < MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE) { throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be scaled with service offering ID: %s, Kubernetes cluster template(CoreOS) needs minimum %d vCPUs and %d MB RAM", kubernetesCluster.getUuid(), serviceOffering.getUuid(), MIN_KUBERNETES_CLUSTER_NODE_CPU, MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE)); } - if (clusterVersion.getMinimumCpu() != null && serviceOffering.getCpu() < clusterVersion.getMinimumCpu()) { + if (serviceOffering.getCpu() < clusterVersion.getMinimumCpu()) { throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be scaled with service offering ID: %s, associated Kubernetes version ID: %s needs minimum %d vCPUs", kubernetesCluster.getUuid(), serviceOffering.getUuid(), clusterVersion.getUuid(), clusterVersion.getMinimumCpu())); } - if (clusterVersion.getMinimumRamSize() != null && serviceOffering.getRamSize() < clusterVersion.getMinimumRamSize()) { + if (serviceOffering.getRamSize() < clusterVersion.getMinimumRamSize()) { throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be scaled with service offering ID: %s, associated Kubernetes version ID: %s needs minimum %d MB RAM", kubernetesCluster.getUuid(), serviceOffering.getUuid(), clusterVersion.getUuid(), clusterVersion.getMinimumRamSize())); } @@ -903,11 +900,11 @@ private void validateKubernetesClusterUpgradeParameters(UpgradeKubernetesCluster if (serviceOffering == null) { throw new CloudRuntimeException(String.format("Invalid service offering associated with Kubernetes cluster ID: %s", kubernetesCluster.getUuid())); } - if (upgradeVersion.getMinimumCpu() != null && serviceOffering.getCpu() < upgradeVersion.getMinimumCpu()) { + if (serviceOffering.getCpu() < upgradeVersion.getMinimumCpu()) { throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be upgraded with Kubernetes version ID: %s which needs minimum %d vCPUs while associated service offering ID: %s offers only %d vCPUs", kubernetesCluster.getUuid(), upgradeVersion.getUuid(), upgradeVersion.getMinimumCpu(), serviceOffering.getUuid(), serviceOffering.getCpu())); } - if (upgradeVersion.getMinimumRamSize() != null && serviceOffering.getRamSize() < upgradeVersion.getMinimumRamSize()) { + if (serviceOffering.getRamSize() < upgradeVersion.getMinimumRamSize()) { throw new InvalidParameterValueException(String.format("Kubernetes cluster ID: %s cannot be upgraded with Kubernetes version ID: %s which needs minimum %d MB RAM while associated service offering ID: %s offers only %d MB RAM", kubernetesCluster.getUuid(), upgradeVersion.getUuid(), upgradeVersion.getMinimumRamSize(), serviceOffering.getUuid(), serviceOffering.getRamSize())); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java index 08e598336d0d..9ff0be335f37 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java @@ -304,7 +304,7 @@ public Date getCreated() { } public KubernetesClusterVO() { - + this.uuid = UUID.randomUUID().toString(); } public KubernetesClusterVO(String name, String description, long zoneId, long kubernetesVersionId, long serviceOfferingId, long templateId, diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesSupportedVersion.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesSupportedVersion.java index efbf13a2c5d9..0cb430acfe17 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesSupportedVersion.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesSupportedVersion.java @@ -40,10 +40,10 @@ public enum State { /** * @return minimum # of cpu. */ - Integer getMinimumCpu(); + int getMinimumCpu(); /** * @return minimum ram size in megabytes */ - Integer getMinimumRamSize(); + int getMinimumRamSize(); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesSupportedVersionVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesSupportedVersionVO.java index 65de4d748e76..3f66f943b69f 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesSupportedVersionVO.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesSupportedVersionVO.java @@ -59,10 +59,10 @@ public class KubernetesSupportedVersionVO implements KubernetesSupportedVersion State state = State.Enabled; @Column(name = "min_cpu") - private Integer minimumCpu; + private int minimumCpu; @Column(name = "min_ram_size") - private Integer minimumRamSize; + private int minimumRamSize; @Column(name = GenericDao.CREATED_COLUMN) Date created; @@ -74,12 +74,8 @@ public KubernetesSupportedVersionVO() { this.uuid = UUID.randomUUID().toString(); } - public KubernetesSupportedVersionVO(String name, String semanticVersion, long isoId, Long zoneId) { - this(name, semanticVersion, isoId, zoneId, null, null); - } - public KubernetesSupportedVersionVO(String name, String semanticVersion, long isoId, Long zoneId, - Integer minimumCpu, Integer minimumRamSize) { + int minimumCpu, int minimumRamSize) { this.uuid = UUID.randomUUID().toString(); this.name = name; this.semanticVersion = semanticVersion; @@ -145,20 +141,20 @@ public void setState(State state) { } @Override - public Integer getMinimumCpu() { + public int getMinimumCpu() { return minimumCpu; } - public void setMinimumCpu(Integer minimumCpu) { + public void setMinimumCpu(int minimumCpu) { this.minimumCpu = minimumCpu; } @Override - public Integer getMinimumRamSize() { + public int getMinimumRamSize() { return minimumRamSize; } - public void setMinimumRamSize(Integer minimumRamSize) { + public void setMinimumRamSize(int minimumRamSize) { this.minimumRamSize = minimumRamSize; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java index 133212f5eeb2..4eefc3f12ecc 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java @@ -277,10 +277,10 @@ public KubernetesSupportedVersionResponse addKubernetesSupportedVersion(final Ad final String isoChecksum = cmd.getChecksum(); final Integer minimumCpu = cmd.getMinimumCpu(); final Integer minimumRamSize = cmd.getMinimumRamSize(); - if (minimumCpu != null && minimumCpu < KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_CPU) { + if (minimumCpu == null || minimumCpu < KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_CPU) { throw new InvalidParameterValueException(String.format("Invalid value for %s parameter", ApiConstants.MIN_CPU_NUMBER)); } - if (minimumRamSize != null && minimumRamSize < KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE) { + if (minimumRamSize == null || minimumRamSize < KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE) { throw new InvalidParameterValueException(String.format("Invalid value for %s parameter", ApiConstants.MIN_MEMORY)); } if (compareSemanticVersions(semanticVersion, MIN_KUBERNETES_VERSION) < 0) { diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java index 928102f27aa0..a85e6ee064ac 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/admin/kubernetes/version/AddKubernetesSupportedVersionCmd.java @@ -77,11 +77,11 @@ public class AddKubernetesSupportedVersionCmd extends BaseCmd implements AdminCm description = "the checksum value of the binaries ISO. " + ApiConstants.CHECKSUM_PARAMETER_PREFIX_DESCRIPTION) private String checksum; - @Parameter(name = ApiConstants.MIN_CPU_NUMBER, type = CommandType.INTEGER, + @Parameter(name = ApiConstants.MIN_CPU_NUMBER, type = CommandType.INTEGER, required = true, description = "the minimum number of CPUs to be set with the Kubernetes version") private Integer minimumCpu; - @Parameter(name = ApiConstants.MIN_MEMORY, type = CommandType.INTEGER, + @Parameter(name = ApiConstants.MIN_MEMORY, type = CommandType.INTEGER, required = true, description = "the minimum RAM size in MB to be set with the Kubernetes version") private Integer minimumRamSize; diff --git a/ui/plugins/cks/cks.js b/ui/plugins/cks/cks.js index 13fa9bdf782a..2ff789202ffa 100644 --- a/ui/plugins/cks/cks.js +++ b/ui/plugins/cks/cks.js @@ -1224,12 +1224,14 @@ mincpunumber: { label: 'label.min.cpu.cores', validation: { + required: true, number: true }, }, minmemory: { label: 'label.memory.minimum.mb', validation: { + required: true, number: true } } From 1a759ab9ccdf845e19a001cf2c89e40133bb34bd Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 12 Feb 2020 12:12:46 +0530 Subject: [PATCH 108/134] hypervisor specific CKS template name Signed-off-by: Abhishek Kumar --- .../kubernetes/cluster/KubernetesClusterService.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java index 88639e4cc7e6..217b661c36e1 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java @@ -42,22 +42,22 @@ public interface KubernetesClusterService extends PluggableService, Configurable false); static final ConfigKey KubernetesClusterHyperVTemplateName = new ConfigKey("Advanced", String.class, "cloud.kubernetes.cluster.template.name.hyperv", - "Kubernetes-Service-Template", + "Kubernetes-Service-Template-HyperV", "Name of the template to be used for creating Kubernetes cluster nodes on HyperV", true); static final ConfigKey KubernetesClusterKVMTemplateName = new ConfigKey("Advanced", String.class, "cloud.kubernetes.cluster.template.name.kvm", - "Kubernetes-Service-Template", + "Kubernetes-Service-Template-KVM", "Name of the template to be used for creating Kubernetes cluster nodes on KVM", true); static final ConfigKey KubernetesClusterVMwareTemplateName = new ConfigKey("Advanced", String.class, "cloud.kubernetes.cluster.template.name.vmware", - "Kubernetes-Service-Template", + "Kubernetes-Service-Template-VMware", "Name of the template to be used for creating Kubernetes cluster nodes on VMware", true); static final ConfigKey KubernetesClusterXenserverTemplateName = new ConfigKey("Advanced", String.class, "cloud.kubernetes.cluster.template.name.xenserver", - "Kubernetes-Service-Template", + "Kubernetes-Service-Template-Xenserver", "Name of the template to be used for creating Kubernetes cluster nodes on Xenserver", true); static final ConfigKey KubernetesClusterNetworkOffering = new ConfigKey("Advanced", String.class, From b6e0ac9a8024bcd101ffc9ba9a43f647a6c6f549 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 12 Feb 2020 12:31:11 +0530 Subject: [PATCH 109/134] added k8s cluster experimental features Signed-off-by: Abhishek Kumar --- .../api/command/user/config/ListCapabilitiesCmd.java | 1 + .../api/response/CapabilitiesResponse.java | 8 ++++++++ .../cluster/KubernetesClusterManagerImpl.java | 8 +++++++- .../kubernetes/cluster/KubernetesClusterService.java | 5 +++++ .../java/com/cloud/server/ManagementServerImpl.java | 4 +++- ui/plugins/cks/cks.js | 12 ++++++++++++ 6 files changed, 36 insertions(+), 2 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/config/ListCapabilitiesCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/config/ListCapabilitiesCmd.java index 948097a15ffe..f4bc28fd3ec2 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/config/ListCapabilitiesCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/config/ListCapabilitiesCmd.java @@ -61,6 +61,7 @@ public void execute() { response.setAllowUserExpungeRecoverVM((Boolean)capabilities.get("allowUserExpungeRecoverVM")); response.setAllowUserViewAllDomainAccounts((Boolean)capabilities.get("allowUserViewAllDomainAccounts")); response.setKubernetesServiceEnabled((Boolean)capabilities.get("kubernetesServiceEnabled")); + response.setKubernetesClusterExperimentalFeaturesEnabled((Boolean)capabilities.get("kubernetesClusterExperimentalFeaturesEnabled")); if (capabilities.containsKey("apiLimitInterval")) { response.setApiLimitInterval((Integer)capabilities.get("apiLimitInterval")); } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/CapabilitiesResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/CapabilitiesResponse.java index 6646437fd64f..ce92f92308e3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/CapabilitiesResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/CapabilitiesResponse.java @@ -92,6 +92,10 @@ public class CapabilitiesResponse extends BaseResponse { @Param(description = "true if Kubernetes Service plugin is enabled, false otherwise") private boolean kubernetesServiceEnabled; + @SerializedName("kubernetesclusterexperimentalfeaturesenabled") + @Param(description = "true if experimental features for Kubernetes cluster such as Docker private registry are enabled, false otherwise") + private boolean kubernetesClusterExperimentalFeaturesEnabled; + public void setSecurityGroupsEnabled(boolean securityGroupsEnabled) { this.securityGroupsEnabled = securityGroupsEnabled; } @@ -159,4 +163,8 @@ public void setAllowUserViewAllDomainAccounts(boolean allowUserViewAllDomainAcco public void setKubernetesServiceEnabled(boolean kubernetesServiceEnabled) { this.kubernetesServiceEnabled = kubernetesServiceEnabled; } + + public void setKubernetesClusterExperimentalFeaturesEnabled(boolean kubernetesClusterExperimentalFeaturesEnabled) { + this.kubernetesClusterExperimentalFeaturesEnabled = kubernetesClusterExperimentalFeaturesEnabled; + } } \ No newline at end of file diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 777b1f6b6345..37ae5f1a98ce 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -714,6 +714,11 @@ private void validateKubernetesClusterCreateParameters(final CreateKubernetesClu throw new InvalidParameterValueException(String.format("%s parameter must be specified along with %s type of network", ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS, Network.GuestType.Shared.toString())); } } + + if (!KubernetesClusterExperimentalFeaturesEnabled.value() && (!Strings.isNullOrEmpty(dockerRegistryUrl) || + !Strings.isNullOrEmpty(dockerRegistryUserName) || !Strings.isNullOrEmpty(dockerRegistryEmail) || !Strings.isNullOrEmpty(dockerRegistryPassword))) { + throw new CloudRuntimeException(String.format("Private registry for the Kubernetes cluster is an experimental feature. Use %s configuration for enabling experimental features", KubernetesClusterExperimentalFeaturesEnabled.key())); + } } private Network getKubernetesClusterNetworkIfMissing(final String clusterName, final DataCenter zone, final Account owner, final int masterNodesCount, @@ -1434,7 +1439,8 @@ public ConfigKey[] getConfigKeys() { KubernetesClusterNetworkOffering, KubernetesClusterStartTimeout, KubernetesClusterScaleTimeout, - KubernetesClusterUpgradeTimeout + KubernetesClusterUpgradeTimeout, + KubernetesClusterExperimentalFeaturesEnabled }; } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java index 217b661c36e1..db5ab91b3d11 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java @@ -80,6 +80,11 @@ public interface KubernetesClusterService extends PluggableService, Configurable "3600", "Timeout interval (in seconds) in which upgrade operation for a Kubernetes cluster should be completed. Not strictly obeyed while upgrade is in progress on a node", true); + static final ConfigKey KubernetesClusterExperimentalFeaturesEnabled = new ConfigKey("Advanced", Boolean.class, + "cloud.kubernetes.cluster.experimental.features.enabled", + "false", + "Indicates whether experimental feature for Kubernetes cluster such as Docker private registry are enabled or not", + true); KubernetesCluster findById(final Long id); diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index e492938df797..29da4f7b35d4 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -3505,6 +3505,7 @@ public Map listCapabilities(final ListCapabilitiesCmd cmd) { final boolean allowUserViewAllDomainAccounts = (QueryService.AllowUserViewAllDomainAccounts.valueIn(caller.getDomainId())); final boolean kubernetesServiceEnabled = Boolean.parseBoolean(_configDao.getValue("cloud.kubernetes.service.enabled")); + final boolean kubernetesClusterExperimentalFeaturesEnabled = Boolean.parseBoolean(_configDao.getValue("cloud.kubernetes.cluster.experimental.features.enabled")); // check if region-wide secondary storage is used boolean regionSecondaryEnabled = false; @@ -3525,8 +3526,9 @@ public Map listCapabilities(final ListCapabilitiesCmd cmd) { capabilities.put("KVMSnapshotEnabled", KVMSnapshotEnabled); capabilities.put("allowUserViewDestroyedVM", allowUserViewDestroyedVM); capabilities.put("allowUserExpungeRecoverVM", allowUserExpungeRecoverVM); - capabilities.put("allowUserViewAllDomainAccounts", allowUserViewAllDomainAccounts);; + capabilities.put("allowUserViewAllDomainAccounts", allowUserViewAllDomainAccounts); capabilities.put("kubernetesServiceEnabled", kubernetesServiceEnabled); + capabilities.put("kubernetesClusterExperimentalFeaturesEnabled", kubernetesClusterExperimentalFeaturesEnabled); if (apiLimitEnabled) { capabilities.put("apiLimitInterval", apiLimitInterval); capabilities.put("apiLimitMax", apiLimitMax); diff --git a/ui/plugins/cks/cks.js b/ui/plugins/cks/cks.js index 2ff789202ffa..dbf44fbeae3c 100644 --- a/ui/plugins/cks/cks.js +++ b/ui/plugins/cks/cks.js @@ -156,6 +156,17 @@ preFilter: function(args) { args.$form.find('.form-item[rel=masternodes]').find('input[name=masternodes]').val('2'); args.$form.find('.form-item[rel=size]').find('input[name=size]').val('1'); + var experimentalFeaturesEnabled = false; + $.ajax({ + url: createURL('listCapabilities'), + async: false, + success: function(json) { + experimentalFeaturesEnabled = json.listcapabilitiesresponse.capability.kubernetesclusterexperimentalfeaturesenabled; + } + }); + if (experimentalFeaturesEnabled == true) { + args.$form.find('.form-item[rel=supportPrivateRegistry]').css('display', 'inline-block'); + } }, fields: { name: { @@ -398,6 +409,7 @@ label: 'label.private.registry', isBoolean: true, isChecked: false, + isHidden: true }, username: { label: 'label.username', From c32dcfa15b3c95c4b4262c285c5ad824c49aca32 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 12 Feb 2020 13:44:10 +0530 Subject: [PATCH 110/134] master merge fix Signed-off-by: Abhishek Kumar --- .../cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 37ae5f1a98ce..21d1b6862e4a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -753,7 +753,7 @@ private Network getKubernetesClusterNetworkIfMissing(final String clusterName, f try { network = networkMgr.createGuestNetwork(networkOffering.getId(), clusterName + "-network", owner.getAccountName() + "-network", - null, null, null, false, null, owner, null, physicalNetwork, zone.getId(), ControlledEntity.ACLType.Account, null, null, null, null, true, null, null); + null, null, null, false, null, owner, null, physicalNetwork, zone.getId(), ControlledEntity.ACLType.Account, null, null, null, null, true, null, null, null); } catch (ConcurrentOperationException | InsufficientCapacityException | ResourceAllocationException e) { logAndThrow(Level.ERROR, String.format("Unable to create network for the Kubernetes cluster: %s", clusterName)); } From ba6c6f8283122758f73becafc4ce339da024743f Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 12 Feb 2020 16:37:46 +0530 Subject: [PATCH 111/134] test changes for required params version API Signed-off-by: Abhishek Kumar --- test/integration/smoke/test_kubernetes_clusters.py | 4 +++- test/integration/smoke/test_kubernetes_supported_versions.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index e1e341c8ca62..6cdc8194c165 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -282,7 +282,9 @@ def addKubernetesSupportedVersion(cls, semantic_version, iso_url): addKubernetesSupportedVersionCmd = addKubernetesSupportedVersion.addKubernetesSupportedVersionCmd() addKubernetesSupportedVersionCmd.semanticversion = semantic_version addKubernetesSupportedVersionCmd.name = 'v' + semantic_version + '-' + random_gen() - addKubernetesSupportedVersionCmd.url = iso_url + addKubernetesSupportedVersionCmd.url = iso_url + addKubernetesSupportedVersionCmd.mincpunumber = 2 + addKubernetesSupportedVersionCmd.minmemory = 2048 kubernetes_version = cls.apiclient.addKubernetesSupportedVersion(addKubernetesSupportedVersionCmd) cls.debug("Waiting for Kubernetes version with ID %s to be ready" % kubernetes_version.id) cls.waitForKubernetesSupportedVersionIsoReadyState(kubernetes_version.id) diff --git a/test/integration/smoke/test_kubernetes_supported_versions.py b/test/integration/smoke/test_kubernetes_supported_versions.py index 0835ced251df..3d699e4b6766 100644 --- a/test/integration/smoke/test_kubernetes_supported_versions.py +++ b/test/integration/smoke/test_kubernetes_supported_versions.py @@ -233,7 +233,9 @@ def addKubernetesSupportedVersion(self, version, name, zoneId, isoUrl): addKubernetesSupportedVersionCmd.semanticversion = version addKubernetesSupportedVersionCmd.name = name addKubernetesSupportedVersionCmd.zoneid = zoneId - addKubernetesSupportedVersionCmd.url = isoUrl + addKubernetesSupportedVersionCmd.url = isoUrl + addKubernetesSupportedVersionCmd.mincpunumber = 2 + addKubernetesSupportedVersionCmd.minmemory = 2048 versionResponse = self.apiclient.addKubernetesSupportedVersion(addKubernetesSupportedVersionCmd) if not versionResponse: self.cleanup.append(versionResponse) From f8eb6fdf0d741116b4969b49cfb3e29dd8bee1e2 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 12 Feb 2020 18:39:16 +0530 Subject: [PATCH 112/134] test fixes Signed-off-by: Abhishek Kumar --- test/integration/smoke/test_kubernetes_clusters.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index 6cdc8194c165..8e365a39fb0b 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -52,6 +52,7 @@ def setUpClass(cls): cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests()) cls.hypervisor = cls.testClient.getHypervisorInfo() cls.mgtSvrDetails = cls.config.__dict__["mgtSvr"][0].__dict__ + cls.cks_template_name_key = "cloud.kubernetes.cluster.template.name." + cls.hypervisor.lower() cls.setup_failed = False @@ -129,9 +130,9 @@ def setUpClass(cls): cls.debug("Failed to get CKS template in ready state, {}, {}".format(cks_template_data["url"], e)) cls.initial_configuration_cks_template_name = Configurations.list(cls.apiclient, - name="cloud.kubernetes.cluster.template.name")[0].value + name=cls.cks_template_name_key)[0].value Configurations.update(cls.apiclient, - "cloud.kubernetes.cluster.template.name", + cls.cks_template_name_key, cls.cks_template.name) cks_offering_data = { @@ -169,7 +170,7 @@ def tearDownClass(cls): # Restore original CKS template if cls.initial_configuration_cks_template_name != None: Configurations.update(cls.apiclient, - "cloud.kubernetes.cluster.template.name", + cls.cks_template_name_key, cls.initial_configuration_cks_template_name) # Delete created CKS template if cls.setup_failed == False and cls.cks_template != None: @@ -571,6 +572,7 @@ def listKubernetesCluster(self, cluster_id): def createKubernetesCluster(self, name, version_id, size=1, master_nodes=1): createKubernetesClusterCmd = createKubernetesCluster.createKubernetesClusterCmd() createKubernetesClusterCmd.name = name + createKubernetesClusterCmd.description = name + "-description" createKubernetesClusterCmd.kubernetesversionid = version_id createKubernetesClusterCmd.size = size createKubernetesClusterCmd.masternodes = master_nodes From fd0cd9cb1c8f648072a20fbfba757038a2793eb7 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 19 Feb 2020 12:05:16 +0530 Subject: [PATCH 113/134] scale failure fix Signed-off-by: Abhishek Kumar --- .../cluster/actionworkers/KubernetesClusterScaleWorker.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index 35e5b2726c38..a3a71c1e74c5 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -91,7 +91,7 @@ protected void init() { private void logTransitStateToFailedIfNeededAndThrow(final Level logLevel, final String message, final Exception e) throws CloudRuntimeException { KubernetesCluster cluster = kubernetesClusterDao.findById(kubernetesCluster.getId()); - if (cluster != null && KubernetesCluster.State.Scaling.equals(kubernetesCluster.getState())) { + if (cluster != null && KubernetesCluster.State.Scaling.equals(cluster.getState())) { logTransitStateAndThrow(logLevel, message, kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); } else { logAndThrow(logLevel, message, e); From 4ead5d0c26e1c607577733ca5a13fd4f68f7939b Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 19 Feb 2020 12:45:37 +0530 Subject: [PATCH 114/134] cluster node duplicate name fix Signed-off-by: Abhishek Kumar --- ...rnetesClusterResourceModifierActionWorker.java | 15 ++++++++++++++- .../KubernetesClusterStartWorker.java | 3 ++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 6ca480faa925..78b0d81ce6f1 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -83,6 +83,7 @@ import com.cloud.utils.net.NetUtils; import com.cloud.vm.Nic; import com.cloud.vm.VirtualMachine; +import com.cloud.vm.dao.VMInstanceDao; import com.google.common.base.Strings; public class KubernetesClusterResourceModifierActionWorker extends KubernetesClusterActionWorker { @@ -107,6 +108,8 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu protected ResourceManager resourceManager; @Inject protected LoadBalancerDao loadBalancerDao; + @Inject + protected VMInstanceDao vmInstanceDao; protected String kubernetesClusterNodeNamePrefix; @@ -307,7 +310,7 @@ protected UserVm createKubernetesNode(String joinIp, int nodeInstance) throws Ma if (rootDiskSize > 0) { customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); } - String hostName = String.format("%s-node-%s", kubernetesClusterNodeNamePrefix, nodeInstance); + String hostName = getKubernetesClusterNodeAvailableName(String.format("%s-node-%s", kubernetesClusterNodeNamePrefix, nodeInstance)); String k8sNodeConfig = null; try { k8sNodeConfig = getKubernetesNodeConfig(joinIp); @@ -491,4 +494,14 @@ protected String getKubernetesClusterNodeNamePrefix() { } return prefix; } + + protected String getKubernetesClusterNodeAvailableName(final String hostName) { + String name = hostName; + int suffix = 1; + while (vmInstanceDao.findVMByHostName(name) != null) { + name = String.format("%s-%d", hostName, suffix); + suffix++; + } + return name; + } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index b8f0de371b61..875d73a9507a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -185,6 +185,7 @@ private UserVm createKubernetesMaster(final Network network, String serverIp) th if (kubernetesCluster.getMasterNodeCount() > 1) { hostName += "-1"; } + hostName = getKubernetesClusterNodeAvailableName(hostName); boolean haSupported = isKubernetesVersionSupportsHA(); String k8sMasterConfig = null; try { @@ -238,7 +239,7 @@ private UserVm createKubernetesAdditionalMaster(final String joinIp, final int a if (rootDiskSize > 0) { customParameterMap.put("rootdisksize", String.valueOf(rootDiskSize)); } - String hostName = String.format("%s-master-%d", kubernetesClusterNodeNamePrefix, additionalMasterNodeInstance + 1); + String hostName = getKubernetesClusterNodeAvailableName(String.format("%s-master-%d", kubernetesClusterNodeNamePrefix, additionalMasterNodeInstance + 1)); String k8sMasterConfig = null; try { k8sMasterConfig = getKubernetesAdditionalMasterConfig(joinIp); From b7ead47824db8858f4cb95a4857488abf29abb35 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 19 Feb 2020 13:18:03 +0530 Subject: [PATCH 115/134] fix for supported service offerings for scale Signed-off-by: Abhishek Kumar --- ui/plugins/cks/cks.js | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/ui/plugins/cks/cks.js b/ui/plugins/cks/cks.js index dbf44fbeae3c..7c0b814e5a5a 100644 --- a/ui/plugins/cks/cks.js +++ b/ui/plugins/cks/cks.js @@ -736,6 +736,25 @@ required: true }, select: function(args) { + $.ajax({ + url: createURL("listKubernetesSupportedVersions"), + data: {id: args.context.kubernetesclusters[0].kubernetesversionid}, + dataType: "json", + async: false, + success: function(json) { + var versionObjs = json.listkubernetessupportedversionsresponse.kubernetessupportedversion; + if (versionObjs != null && versionObjs.length > 0) { + minCpu = 0; + if (versionObjs[0].mincpunumber != null && versionObjs[0].mincpunumber != undefined) { + minCpu = versionObjs[0].mincpunumber; + } + minRamSize = 0; + if (versionObjs[0].minmemory != null && versionObjs[0].minmemory != undefined) { + minRamSize = versionObjs[0].minmemory; + } + } + } + }); $.ajax({ url: createURL("listServiceOfferings"), dataType: "json", @@ -745,10 +764,13 @@ var items = json.listserviceofferingsresponse.serviceoffering; if (items != null) { for (var i = 0; i < items.length; i++) { - offeringObjs.push({ - id: items[i].id, - description: items[i].name - }); + if (items[i].iscustomized == false && + items[i].cpunumber >= minCpu && items[i].memory >= minRamSize) { + offeringObjs.push({ + id: items[i].id, + description: items[i].name + }); + } } } args.response.success({ From 7a17108bd3e758efce80056f0cee7a27fa709ca9 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 19 Feb 2020 13:32:51 +0530 Subject: [PATCH 116/134] vmware cluster node expunge fix Signed-off-by: Abhishek Kumar --- .../cluster/actionworkers/KubernetesClusterScaleWorker.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index a3a71c1e74c5..27cbb54a26ac 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -329,6 +329,9 @@ private void scaleDownKubernetesClusterSize() throws CloudRuntimeException { , vm.getInstanceName() , vm.getState().toString() , kubernetesCluster.getUuid()), null); + } + if (!VirtualMachine.State.Expunging.equals(vm.getState()) || + Hypervisor.HypervisorType.VMware.equals(vm.getHypervisorType())) { vm = userVmService.expungeVm(userVM.getId()); if (!VirtualMachine.State.Expunging.equals(vm.getState())) { logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, VM '%s' is now in state '%s'." From 2f925848f15284d947a55ef5d55255ec6143f0c1 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 19 Feb 2020 18:59:46 +0530 Subject: [PATCH 117/134] cluster node vm expunge fix Signed-off-by: Abhishek Kumar --- .../KubernetesClusterDestroyWorker.java | 39 ++----------------- ...esClusterResourceModifierActionWorker.java | 3 ++ .../KubernetesClusterScaleWorker.java | 30 ++++---------- 3 files changed, 13 insertions(+), 59 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java index 06b3a9513e4e..faf8ee19c73c 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java @@ -30,7 +30,6 @@ import com.cloud.exception.ManagementServerException; import com.cloud.exception.PermissionDeniedException; import com.cloud.exception.ResourceUnavailableException; -import com.cloud.hypervisor.Hypervisor; import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; @@ -49,7 +48,6 @@ import com.cloud.vm.ReservationContext; import com.cloud.vm.ReservationContextImpl; import com.cloud.vm.UserVmVO; -import com.cloud.vm.VirtualMachine; public class KubernetesClusterDestroyWorker extends KubernetesClusterResourceModifierActionWorker { @@ -87,21 +85,10 @@ private boolean destroyClusterVMs() { } try { UserVm vm = userVmService.destroyVm(vmID, true); - if (!VirtualMachine.State.Expunging.equals(vm.getState())) { - LOGGER.warn(String.format("VM '%s' ID: %s should have been expunging by now but is '%s'. Retrying..." + if (!userVmManager.expunge(userVM, CallContext.current().getCallingUserId(), CallContext.current().getCallingAccount())) { + LOGGER.warn(String.format("Unable to expunge VM '%s' ID: %s, destroying Kubernetes cluster will probably fail" , vm.getInstanceName() - , vm.getUuid() - , vm.getState().toString())); - } - if (!VirtualMachine.State.Expunging.equals(vm.getState()) || - Hypervisor.HypervisorType.VMware.equals(vm.getHypervisorType())) { - vm = userVmService.expungeVm(vmID); - if (!VirtualMachine.State.Expunging.equals(vm.getState())) { - LOGGER.warn(String.format("VM '%s' ID: %s is in state: %s, Kubernetes cluster will probably fail" - , vm.getInstanceName() - , vm.getUuid() - , vm.getState().toString())); - } + , vm.getUuid())); } kubernetesClusterVmMapDao.expunge(clusterVM.getId()); if (LOGGER.isInfoEnabled()) { @@ -112,26 +99,6 @@ private boolean destroyClusterVMs() { return false; } } - final long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() - startTime < 10 * 60 * 1000) { - vmDestroyed = true; - for (KubernetesClusterVmMapVO clusterVM : clusterVMs) { - UserVmVO userVM = userVmDao.findById(clusterVM.getVmId()); - if (userVM != null && !userVM.isRemoved()) { - LOGGER.info(String.format("Waiting for Kubernetes cluster ID: %s VMs to get expunged", kubernetesCluster.getUuid())); - vmDestroyed = false; - break; - } - } - if (vmDestroyed) { - break; - } - try { - Thread.sleep(15 * 1000); - } catch (InterruptedException ie) { - LOGGER.warn(String.format("Error while waiting for Kubernetes cluster ID: %s VMs to get expunged", kubernetesCluster.getUuid()), ie); - } - } } return vmDestroyed; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 78b0d81ce6f1..dedfbb2d7380 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -82,6 +82,7 @@ import com.cloud.utils.net.Ip; import com.cloud.utils.net.NetUtils; import com.cloud.vm.Nic; +import com.cloud.vm.UserVmManager; import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.VMInstanceDao; import com.google.common.base.Strings; @@ -110,6 +111,8 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu protected LoadBalancerDao loadBalancerDao; @Inject protected VMInstanceDao vmInstanceDao; + @Inject + protected UserVmManager userVmManager; protected String kubernetesClusterNodeNamePrefix; diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index 27cbb54a26ac..e9a36d36c042 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -24,6 +24,7 @@ import javax.inject.Inject; +import org.apache.cloudstack.context.CallContext; import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Level; @@ -45,14 +46,12 @@ import com.cloud.network.Network; import com.cloud.network.rules.FirewallRule; import com.cloud.offering.ServiceOffering; -import com.cloud.user.AccountManager; import com.cloud.uservm.UserVm; import com.cloud.utils.Pair; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallback; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.ssh.SshHelper; -import com.cloud.vm.UserVmManager; import com.cloud.vm.UserVmVO; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; @@ -61,12 +60,8 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModifierActionWorker { - @Inject - protected AccountManager accountManager; @Inject protected VMInstanceDao vmInstanceDao; - @Inject - protected UserVmManager userVmManager; private ServiceOffering serviceOffering; private Long clusterSize; @@ -316,7 +311,7 @@ private void scaleDownKubernetesClusterSize() throws CloudRuntimeException { List removedVmIds = new ArrayList<>(); while (i > kubernetesCluster.getMasterNodeCount()) { KubernetesClusterVmMapVO vmMapVO = originalVmList.get(i); - UserVm userVM = userVmDao.findById(vmMapVO.getVmId()); + UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId()); if (!removeKubernetesClusterNode(publicIpAddress, sshPort, userVM, 3, 30000)) { logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, failed to remove Kubernetes node: %s running on VM ID: %s", kubernetesCluster.getUuid(), userVM.getHostName(), userVM.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } @@ -324,22 +319,11 @@ private void scaleDownKubernetesClusterSize() throws CloudRuntimeException { removedVmIds.add(userVM.getId()); try { UserVm vm = userVmService.destroyVm(userVM.getId(), true); - if (!VirtualMachine.State.Expunging.equals(vm.getState())) { - logMessage(Level.WARN, String.format("VM '%s' is in state '%s' while destroying it during scaling Kubernetes cluster ID: %s. Retrying..." - , vm.getInstanceName() - , vm.getState().toString() - , kubernetesCluster.getUuid()), null); - } - if (!VirtualMachine.State.Expunging.equals(vm.getState()) || - Hypervisor.HypervisorType.VMware.equals(vm.getHypervisorType())) { - vm = userVmService.expungeVm(userVM.getId()); - if (!VirtualMachine.State.Expunging.equals(vm.getState())) { - logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, VM '%s' is now in state '%s'." - , kubernetesCluster.getUuid() - , vm.getInstanceName() - , vm.getState().toString()), - kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); - } + if (!userVmManager.expunge(userVM, CallContext.current().getCallingUserId(), CallContext.current().getCallingAccount())) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, unable to expunge VM '%s'." + , kubernetesCluster.getUuid() + , vm.getInstanceName()), + kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } } catch (ResourceUnavailableException e) { logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster ID: %s failed, unable to remove VM ID: %s" From 81fbfc8ed784404c432a45bb17425e03984dabad Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 20 Feb 2020 12:52:11 +0530 Subject: [PATCH 118/134] display access tab for more states Signed-off-by: Abhishek Kumar --- ui/plugins/cks/cks.js | 48 ++++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/ui/plugins/cks/cks.js b/ui/plugins/cks/cks.js index 7c0b814e5a5a..2c3fd90341a3 100644 --- a/ui/plugins/cks/cks.js +++ b/ui/plugins/cks/cks.js @@ -966,36 +966,32 @@ custom : function (args) { var showAccess = function() { var state = args.context.kubernetesclusters[0].state; - - if (state == "Created" || state == "Starting") { // Starting + if (state == "Created") { // Created return jQuery('

').html("Kubernetes cluster setup is under progress, please check again in few minutes."); + } else if (state == "Error") { // Error + return jQuery('

').html("Kubernetes cluster is in error state, it cannot be accessed."); + } else if (state == "Destroying") { // Destroying + return jQuery('

').html("Kubernetes cluster is in destroying state, it cannot be accessed."); + } else if (state == "Destroyed") { // Destroyed + return jQuery('

').html("Kubernetes cluster is already destroyed, it cannot be accessed."); } - - if (state == "Stopped" || state == "Stopping") { // Starting - return jQuery('

').html("Kubernetes cluster setup is under progress, please check again in few minutes."); + var data = { + id: args.context.kubernetesclusters[0].kubernetesversionid } - - if (state == "Running") { // Running - var data = { - id: args.context.kubernetesclusters[0].kubernetesversionid - } - var version = ''; - $.ajax({ - url: createURL("listKubernetesSupportedVersions"), - dataType: "json", - data: data, - async: false, - success: function(json) { - var jsonObj; - if (json.listkubernetessupportedversionsresponse.kubernetessupportedversion != null) { - version = json.listkubernetessupportedversionsresponse.kubernetessupportedversion[0].semanticversion; - } + var version = ''; + $.ajax({ + url: createURL("listKubernetesSupportedVersions"), + dataType: "json", + data: data, + async: false, + success: function(json) { + var jsonObj; + if (json.listkubernetessupportedversionsresponse.kubernetessupportedversion != null) { + version = json.listkubernetessupportedversionsresponse.kubernetessupportedversion[0].semanticversion; } - }); - return jQuery('

').html("Access Kubernetes cluster

Download cluster's kubeconfig file using action from Details tab.
Download kubectl tool for cluster's Kubernetes version from,
Linux: https://storage.googleapis.com/kubernetes-release/release/v" + version + "/bin/linux/amd64/kubectl
MacOS: https://storage.googleapis.com/kubernetes-release/release/v" + version + "/bin/darwin/amd64/kubectl
Windows: https://storage.googleapis.com/kubernetes-release/release/v" + version + "/bin/windows/amd64/kubectl.exe

Using kubectl and kubeconfig file to access cluster
kubectl --kubeconfig /custom/path/kube.conf {COMMAND}

List pods
kubectl --kubeconfig /custom/path/kube.conf get pods --all-namespaces
List nodes
kubectl --kubeconfig /custom/path/kube.conf get nodes --all-namespaces
List services
kubectl --kubeconfig /custom/path/kube.conf get services --all-namespaces

Access dashboard web UI
Run proxy locally
kubectl --kubeconfig /custom/path/kube.conf proxy
Open URL in browser
http://localhost:8001/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/

Token for dashboard login can be retrieved using following command
kubectl --kubeconfig /custom/path/kube.conf describe secret $(kubectl --kubeconfig /custom/path/kube.conf get secrets -n kubernetes-dashboard | grep kubernetes-dashboard-token | awk '{print $1}') -n kubernetes-dashboard

More about accessing dashboard UI, https://kubernetes.io/docs/tasks/access-application-cluster/web-ui-dashboard/#accessing-the-dashboard-ui"); - } - - return jQuery('

').html("Kubernetes cluster is not in a stable state, please check again in few minutes."); + } + }); + return jQuery('

').html("Access Kubernetes cluster

Download cluster's kubeconfig file using action from Details tab.
Download kubectl tool for cluster's Kubernetes version from,
Linux: https://storage.googleapis.com/kubernetes-release/release/v" + version + "/bin/linux/amd64/kubectl
MacOS: https://storage.googleapis.com/kubernetes-release/release/v" + version + "/bin/darwin/amd64/kubectl
Windows: https://storage.googleapis.com/kubernetes-release/release/v" + version + "/bin/windows/amd64/kubectl.exe

Using kubectl and kubeconfig file to access cluster
kubectl --kubeconfig /custom/path/kube.conf {COMMAND}

List pods
kubectl --kubeconfig /custom/path/kube.conf get pods --all-namespaces
List nodes
kubectl --kubeconfig /custom/path/kube.conf get nodes --all-namespaces
List services
kubectl --kubeconfig /custom/path/kube.conf get services --all-namespaces

Access dashboard web UI
Run proxy locally
kubectl --kubeconfig /custom/path/kube.conf proxy
Open URL in browser
http://localhost:8001/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/

Token for dashboard login can be retrieved using following command
kubectl --kubeconfig /custom/path/kube.conf describe secret $(kubectl --kubeconfig /custom/path/kube.conf get secrets -n kubernetes-dashboard | grep kubernetes-dashboard-token | awk '{print $1}') -n kubernetes-dashboard

More about accessing dashboard UI, https://kubernetes.io/docs/tasks/access-application-cluster/web-ui-dashboard/#accessing-the-dashboard-ui"); }; return showAccess(); } From 4950b3b2b3f9b2be8e9803431b28315e5ac7d9a6 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 20 Feb 2020 13:23:09 +0530 Subject: [PATCH 119/134] hide node vm internal name for nod-admins Signed-off-by: Abhishek Kumar --- ui/plugins/cks/cks.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ui/plugins/cks/cks.js b/ui/plugins/cks/cks.js index 2c3fd90341a3..43ffb7b07af0 100644 --- a/ui/plugins/cks/cks.js +++ b/ui/plugins/cks/cks.js @@ -1000,6 +1000,13 @@ title: 'label.instances', listView: { section: 'clusterinstances', + preFilter: function(args) { + var hiddenFields = []; + if (!isAdmin()) { + hiddenFields.push('instancename'); + } + return hiddenFields; + }, fields: { name: { label: 'label.name', From c28fec3e975c3ee2f28f68cd119b917f91b8b37e Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 20 Feb 2020 13:31:31 +0530 Subject: [PATCH 120/134] download config action improvement Signed-off-by: Abhishek Kumar --- .../cluster/KubernetesClusterManagerImpl.java | 6 ++++++ ui/plugins/cks/cks.js | 12 +++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 21d1b6862e4a..3c22e169c26a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -1154,6 +1154,12 @@ public KubernetesClusterConfigResponse getKubernetesClusterConfig(GetKubernetesC KubernetesClusterDetailsVO clusterDetailsVO = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "kubeConfigData"); if (clusterDetailsVO != null && !Strings.isNullOrEmpty(clusterDetailsVO.getValue())) { configData = new String(Base64.decodeBase64(clusterDetailsVO.getValue())); + } else { + if (KubernetesCluster.State.Starting.equals(kubernetesCluster.getState())) { + throw new CloudRuntimeException(String.format("Setup is in progress for Kubernetes cluster ID: %s, config not available at this moment", kubernetesCluster.getUuid())); + } else { + throw new CloudRuntimeException((String.format("Config not found for Kubernetes cluster ID: %s", kubernetesCluster.getUuid()))); + } } response.setConfigData(configData); response.setObjectName("clusterconfig"); diff --git a/ui/plugins/cks/cks.js b/ui/plugins/cks/cks.js index 43ffb7b07af0..df850a7c1993 100644 --- a/ui/plugins/cks/cks.js +++ b/ui/plugins/cks/cks.js @@ -696,12 +696,18 @@ if (json.getkubernetesclusterconfigresponse.clusterconfig != null && json.getkubernetesclusterconfigresponse.clusterconfig.configdata != null ) { jsonObj = json.getkubernetesclusterconfigresponse.clusterconfig; - clusterKubeConfig = jsonObj.configdata ; + clusterKubeConfig = jsonObj.configdata; + downloadClusterKubeConfig(); + args.response.success({}); + } else { + args.response.error("Unable to retrieve Kubernetes cluster config"); } + }, + error: function(XMLHttpResponse) { + var errorMsg = parseXMLHttpResponse(XMLHttpResponse); + args.response.error(errorMsg); } }); - downloadClusterKubeConfig(); - args.response.success({}); }, notification: { poll: function(args) { From d967a74391ecdfeed8b2eaec370d92b7f91fd755 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 21 Feb 2020 05:20:19 +0530 Subject: [PATCH 121/134] fix for project in k8s clusters Signed-off-by: Abhishek Kumar --- .../cluster/KubernetesClusterManagerImpl.java | 77 ++++--- .../cluster/ListKubernetesClustersCmd.java | 14 +- .../response/KubernetesClusterResponse.java | 203 ++++++++++-------- ui/plugins/cks/cks.js | 36 +++- 4 files changed, 194 insertions(+), 136 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 3c22e169c26a..cb7961cfeda7 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -72,6 +72,7 @@ import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.DataCenterDao; import com.cloud.deploy.DeployDestination; +import com.cloud.domain.Domain; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InsufficientServerCapacityException; @@ -113,6 +114,7 @@ import com.cloud.offerings.dao.NetworkOfferingServiceMapDao; import com.cloud.org.Cluster; import com.cloud.org.Grouping; +import com.cloud.projects.Project; import com.cloud.resource.ResourceManager; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; @@ -126,11 +128,13 @@ import com.cloud.user.SSHKeyPairVO; import com.cloud.user.dao.SSHKeyPairDao; import com.cloud.utils.Pair; +import com.cloud.utils.Ternary; import com.cloud.utils.component.ComponentContext; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.db.Filter; import com.cloud.utils.db.GlobalLock; +import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallback; @@ -577,6 +581,17 @@ public KubernetesClusterResponse createKubernetesClusterResponse(long kubernetes response.setKubernetesVersionId(version.getUuid()); response.setKubernetesVersionName(version.getName()); } + Account account = ApiDBUtils.findAccountById(kubernetesCluster.getAccountId()); + if (account.getType() == Account.ACCOUNT_TYPE_PROJECT) { + Project project = ApiDBUtils.findProjectByProjectAccountId(account.getId()); + response.setProjectId(project.getUuid()); + response.setProjectName(project.getName()); + } else { + response.setAccountName(account.getAccountName()); + } + Domain domain = ApiDBUtils.findDomainById(kubernetesCluster.getDomainId()); + response.setDomainId(domain.getUuid()); + response.setDomainName(domain.getName()); response.setKeypair(kubernetesCluster.getKeyPair()); response.setState(kubernetesCluster.getState().toString()); response.setCores(String.valueOf(kubernetesCluster.getCores())); @@ -1097,41 +1112,39 @@ public ListResponse listKubernetesClusters(ListKubern final Long clusterId = cmd.getId(); final String state = cmd.getState(); final String name = cmd.getName(); - + final String keyword = cmd.getKeyword(); List responsesList = new ArrayList(); - if (state != null && !state.isEmpty()) { - if (!KubernetesCluster.State.Running.toString().equals(state) && - !KubernetesCluster.State.Stopped.toString().equals(state) && - !KubernetesCluster.State.Destroyed.toString().equals(state)) { - throw new InvalidParameterValueException("Invalid value for Kubernetes cluster state specified"); - } + List permittedAccounts = new ArrayList(); + Ternary domainIdRecursiveListProject = new Ternary(cmd.getDomainId(), cmd.isRecursive(), null); + accountManager.buildACLSearchParameters(caller, clusterId, cmd.getAccountName(), cmd.getProjectId(), permittedAccounts, domainIdRecursiveListProject, cmd.listAll(), false); + Long domainId = domainIdRecursiveListProject.first(); + Boolean isRecursive = domainIdRecursiveListProject.second(); + Project.ListProjectResourcesCriteria listProjectResourcesCriteria = domainIdRecursiveListProject.third(); + Filter searchFilter = new Filter(KubernetesClusterVO.class, "id", true, cmd.getStartIndex(), cmd.getPageSizeVal()); + SearchBuilder sb = kubernetesClusterDao.createSearchBuilder(); + accountManager.buildACLSearchBuilder(sb, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria); + sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); + sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ); + sb.and("keyword", sb.entity().getName(), SearchCriteria.Op.LIKE); + sb.and("state", sb.entity().getState(), SearchCriteria.Op.IN); + SearchCriteria sc = sb.create(); + accountManager.buildACLSearchCriteria(sc, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria); + if (state != null) { + sc.setParameters("state", state); + } + if(keyword != null){ + sc.setParameters("keyword", "%" + keyword + "%"); } if (clusterId != null) { - KubernetesClusterVO cluster = kubernetesClusterDao.findById(clusterId); - if (cluster == null) { - throw new InvalidParameterValueException("Invalid Kubernetes cluster ID specified"); - } - accountManager.checkAccess(caller, SecurityChecker.AccessType.ListEntry, false, cluster); - responsesList.add(createKubernetesClusterResponse(clusterId)); - } else { - SearchCriteria sc = kubernetesClusterDao.createSearchCriteria(); - Filter searchFilter = new Filter(KubernetesClusterVO.class, "id", true, cmd.getStartIndex(), cmd.getPageSizeVal()); - if (state != null && !state.isEmpty()) { - sc.addAnd("state", SearchCriteria.Op.EQ, state); - } - if (accountManager.isNormalUser(caller.getId())) { - sc.addAnd("accountId", SearchCriteria.Op.EQ, caller.getAccountId()); - } else if (accountManager.isDomainAdmin(caller.getId())) { - sc.addAnd("domainId", SearchCriteria.Op.EQ, caller.getDomainId()); - } - if (name != null && !name.isEmpty()) { - sc.addAnd("name", SearchCriteria.Op.LIKE, name); - } - List kubernetesClusters = kubernetesClusterDao.search(sc, searchFilter); - for (KubernetesClusterVO cluster : kubernetesClusters) { - KubernetesClusterResponse clusterResponse = createKubernetesClusterResponse(cluster.getId()); - responsesList.add(clusterResponse); - } + sc.setParameters("id", clusterId); + } + if (name != null) { + sc.setParameters("name", name); + } + List kubernetesClusters = kubernetesClusterDao.search(sc, searchFilter); + for (KubernetesClusterVO cluster : kubernetesClusters) { + KubernetesClusterResponse clusterResponse = createKubernetesClusterResponse(cluster.getId()); + responsesList.add(clusterResponse); } ListResponse response = new ListResponse(); response.setResponses(responsesList); diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ListKubernetesClustersCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ListKubernetesClustersCmd.java index c02893a8bac6..ef960d532ee6 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ListKubernetesClustersCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ListKubernetesClustersCmd.java @@ -19,17 +19,17 @@ import javax.inject.Inject; import org.apache.cloudstack.acl.RoleType; -import org.apache.cloudstack.api.ApiErrorCode; -import org.apache.cloudstack.api.ServerApiException; -import org.apache.cloudstack.api.response.KubernetesClusterResponse; -import org.apache.log4j.Logger; - import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseListProjectAndAccountResourcesCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ResponseObject.ResponseView; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.KubernetesClusterResponse; import org.apache.cloudstack.api.response.ListResponse; +import org.apache.log4j.Logger; + import com.cloud.kubernetes.cluster.KubernetesClusterService; import com.cloud.utils.exception.CloudRuntimeException; @@ -40,7 +40,7 @@ requestHasSensitiveInfo = false, responseHasSensitiveInfo = true, authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) -public class ListKubernetesClustersCmd extends BaseListCmd { +public class ListKubernetesClustersCmd extends BaseListProjectAndAccountResourcesCmd { public static final Logger LOGGER = Logger.getLogger(ListKubernetesClustersCmd.class.getName()); public static final String APINAME = "listKubernetesClusters"; diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java index 4b22fb4a4636..2c6fc8191e58 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java @@ -34,92 +34,107 @@ public class KubernetesClusterResponse extends BaseResponse implements Controlle private String id; @SerializedName(ApiConstants.NAME) - @Param(description = "Name of the Kubernetes cluster") + @Param(description = "the name of the Kubernetes cluster") private String name; @SerializedName(ApiConstants.DESCRIPTION) - @Param(description = "Description of the Kubernetes cluster") + @Param(description = "the description of the Kubernetes cluster") private String description; @SerializedName(ApiConstants.ZONE_ID) - @Param(description = "zone id") + @Param(description = "the name of the zone of the Kubernetes cluster") private String zoneId; @SerializedName(ApiConstants.ZONE_NAME) - @Param(description = "zone name") + @Param(description = "the name of the zone of the Kubernetes cluster") private String zoneName; @SerializedName(ApiConstants.SERVICE_OFFERING_ID) - @Param(description = "Service Offering id") + @Param(description = "the ID of the service offering of the Kubernetes cluster") private String serviceOfferingId; @SerializedName("serviceofferingname") - @Param(description = "the name of the service offering of the virtual machine") + @Param(description = "the name of the service offering of the Kubernetes cluster") private String serviceOfferingName; @SerializedName(ApiConstants.TEMPLATE_ID) - @Param(description = "template id") + @Param(description = "the ID of the template of the Kubernetes cluster") private String templateId; @SerializedName(ApiConstants.NETWORK_ID) - @Param(description = "network id details") + @Param(description = "the ID of the network of the Kubernetes cluster") private String networkId; @SerializedName(ApiConstants.ASSOCIATED_NETWORK_NAME) - @Param(description = "the name of the Network associated with the IP address") + @Param(description = "the name of the network of the Kubernetes cluster") private String associatedNetworkName; @SerializedName(ApiConstants.KUBERNETES_VERSION_ID) - @Param(description = "the ID of the Kubernetes version for the cluster") + @Param(description = "the ID of the Kubernetes version for the Kubernetes cluster") private String kubernetesVersionId; @SerializedName(ApiConstants.KUBERNETES_VERSION_NAME) - @Param(description = "the name of the Kubernetes version for the cluster") + @Param(description = "the name of the Kubernetes version for the Kubernetes cluster") private String kubernetesVersionName; + @SerializedName(ApiConstants.ACCOUNT) + @Param(description = "the account associated with the Kubernetes cluster") + private String accountName; + + @SerializedName(ApiConstants.PROJECT_ID) + @Param(description = "the project id of the Kubernetes cluster") + private String projectId; + + @SerializedName(ApiConstants.PROJECT) + @Param(description = "the project name of the Kubernetes cluster") + private String projectName; + + @SerializedName(ApiConstants.DOMAIN_ID) + @Param(description = "the ID of the domain in which the Kubernetes cluster exists") + private String domainId; + + @SerializedName(ApiConstants.DOMAIN) + @Param(description = "the name of the domain in which the Kubernetes cluster exists") + private String domainName; + @SerializedName(ApiConstants.SSH_KEYPAIR) @Param(description = "keypair details") private String keypair; @SerializedName(ApiConstants.MASTER_NODES) - @Param(description = "master nodes count") + @Param(description = "the master nodes count for the Kubernetes cluster") private Long masterNodes; @SerializedName(ApiConstants.SIZE) - @Param(description = "cluster size") + @Param(description = "the size (worker nodes count) of the Kubernetes cluster") private Long clusterSize; @SerializedName(ApiConstants.STATE) - @Param(description = "state of the cluster") + @Param(description = "the state of the Kubernetes cluster") private String state; @SerializedName(ApiConstants.CPU_NUMBER) - @Param(description = "cluster cpu cores") + @Param(description = "the cpu cores of the Kubernetes cluster") private String cores; @SerializedName(ApiConstants.MEMORY) - @Param(description = "cluster size") + @Param(description = "the memory the Kubernetes cluster") private String memory; @SerializedName(ApiConstants.END_POINT) - @Param(description = "URL end point for the cluster") + @Param(description = "URL end point for the Kubernetes cluster") private String endpoint; @SerializedName(ApiConstants.CONSOLE_END_POINT) - @Param(description = "URL end point for the cluster UI") + @Param(description = "URL end point for the Kubernetes cluster dashboard UI") private String consoleEndpoint; @SerializedName(ApiConstants.VIRTUAL_MACHINE_IDS) - @Param(description = "the list of virtualmachine ids associated with this Kubernetes cluster") + @Param(description = "the list of virtualmachine IDs associated with this Kubernetes cluster") private List virtualMachineIds; - @SerializedName(ApiConstants.USERNAME) - @Param(description = "Username with which Kubernetes cluster is setup") - private String username; - - @SerializedName(ApiConstants.PASSWORD) - @Param(description = "Password with which Kubernetes cluster is setup") - private String password; + public KubernetesClusterResponse() { + } public String getName() { return name; @@ -177,6 +192,59 @@ public void setNetworkId(String networkId) { this.networkId = networkId; } + public String getAssociatedNetworkName() { + return associatedNetworkName; + } + + public void setAssociatedNetworkName(String associatedNetworkName) { + this.associatedNetworkName = associatedNetworkName; + } + + public String getKubernetesVersionId() { + return kubernetesVersionId; + } + + public void setKubernetesVersionId(String kubernetesVersionId) { + this.kubernetesVersionId = kubernetesVersionId; + } + + public String getKubernetesVersionName() { + return kubernetesVersionName; + } + + public void setKubernetesVersionName(String kubernetesVersionName) { + this.kubernetesVersionName = kubernetesVersionName; + } + + public String getProjectId() { + return projectId; + } + + @Override + public void setAccountName(String accountName) { + this.accountName = accountName; + } + + @Override + public void setProjectId(String projectId) { + this.projectId = projectId; + } + + @Override + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + @Override + public void setDomainId(String domainId) { + this.domainId = domainId; + } + + @Override + public void setDomainName(String domainName) { + this.domainName = domainName; + } + public String getKeypair() { return keypair; } @@ -217,13 +285,21 @@ public void setMemory(String memory) { this.memory = memory; } - public String getState() { return state;} + public String getState() { + return state; + } - public void setState(String state) {this.state = state;} + public void setState(String state) { + this.state = state; + } - public String getEndpoint() { return endpoint;} + public String getEndpoint() { + return endpoint; + } - public void setEndpoint(String endpoint) {this.endpoint = endpoint;} + public void setEndpoint(String endpoint) { + this.endpoint = endpoint; + } public String getId() { return this.id; @@ -241,74 +317,13 @@ public void setServiceOfferingName(String serviceOfferingName) { this.serviceOfferingName = serviceOfferingName; } - public void setVirtualMachineIds(List virtualMachineIds) { this.virtualMachineIds = virtualMachineIds;}; - - public void setUsername(String userName) { this.username = userName;} - - public String getPassword() { - return password; + public void setVirtualMachineIds(List virtualMachineIds) { + this.virtualMachineIds = virtualMachineIds; } - public void setPassword(String password) { this.password = password;} - - public String getUsername() { - return username; - } + ; public List getVirtualMachineIds() { return virtualMachineIds; } - - public String getAssociatedNetworkName() { - return associatedNetworkName; - } - - public void setAssociatedNetworkName(String associatedNetworkName) { - this.associatedNetworkName = associatedNetworkName; - } - - public String getKubernetesVersionId() { - return kubernetesVersionId; - } - - public void setKubernetesVersionId(String kubernetesVersionId) { - this.kubernetesVersionId = kubernetesVersionId; - } - - public String getKubernetesVersionName() { - return kubernetesVersionName; - } - - public void setKubernetesVersionName(String kubernetesVersionName) { - this.kubernetesVersionName = kubernetesVersionName; - } - - public KubernetesClusterResponse() { - - } - - @Override - public void setAccountName(String accountName) { - - } - - @Override - public void setProjectId(String projectId) { - - } - - @Override - public void setProjectName(String projectName) { - - } - - @Override - public void setDomainId(String domainId) { - - } - - @Override - public void setDomainName(String domainName) { - - } } diff --git a/ui/plugins/cks/cks.js b/ui/plugins/cks/cks.js index df850a7c1993..119518a70b02 100644 --- a/ui/plugins/cks/cks.js +++ b/ui/plugins/cks/cks.js @@ -1062,6 +1062,13 @@ ids: vmlist.join() }); + if (items && items.length > 0 && items[0].projectid != null && + items[0].projectid != undefined && items[0].projectid.length > 0) { + $.extend(data, { + projectid: items[0].projectid + }); + } + if (data.ids.length == 0) { args.response.success({ data: [] @@ -1099,9 +1106,21 @@ firewall: { title: 'label.firewall', custom: function(args) { + var data = { + id: args.context.kubernetesclusters[0].networkid, + listAll: true + } + if (args.context.kubernetesclusters[0].projectid != null && + args.context.kubernetesclusters[0].projectid != undefined && + args.context.kubernetesclusters[0].projectid.length > 0) { + $.extend(data, { + projectid: args.context.kubernetesclusters[0].projectid + }); + $.extend(args.context, {"projectid": args.context.kubernetesclusters[0].projectid}); + } $.ajax({ url: createURL('listNetworks'), - data: {id: args.context.kubernetesclusters[0].networkid, listAll: true}, + data: data, async: false, dataType: "json", success: function(json) { @@ -1109,10 +1128,21 @@ $.extend(args.context, {"networks": [network]}); } }); - + data = { + associatedNetworkId: args.context.kubernetesclusters[0].networkid, + listAll: true, + forvirtualnetwork: true + } + if (args.context.kubernetesclusters[0].projectid != null && + args.context.kubernetesclusters[0].projectid != undefined && + args.context.kubernetesclusters[0].projectid.length > 0) { + $.extend(data, { + projectid: args.context.kubernetesclusters[0].projectid + }); + } $.ajax({ url: createURL('listPublicIpAddresses'), - data: {associatedNetworkId: args.context.kubernetesclusters[0].networkid, listAll: true, forvirtualnetwork: true}, + data: data, async: false, dataType: "json", success: function(json) { From 54ec1cfdea5b29405b2df680e16437dc001a1f07 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Sat, 22 Feb 2020 11:15:04 +0530 Subject: [PATCH 122/134] kube config fix Signed-off-by: Abhishek Kumar --- .../KubernetesClusterActionWorker.java | 9 +++++++++ .../KubernetesClusterStartWorker.java | 17 +++++++++-------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 39c7263ea259..aad9a225a44e 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -246,6 +246,15 @@ private UserVm fetchMasterVmIfMissing(final UserVm masterVm) { return userVmDao.findById(vmIds.get(0)); } + protected String getMasterVmPrivateIp() { + String ip = null; + UserVm vm = fetchMasterVmIfMissing(null); + if (vm != null) { + ip = vm.getPrivateIpAddress(); + } + return ip; + } + protected Pair getKubernetesClusterServerIpSshPort(UserVm masterVm) { int port = CLUSTER_NODES_DEFAULT_START_SSH_PORT; KubernetesClusterDetailsVO detail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), ApiConstants.EXTERNAL_LOAD_BALANCER_IP_ADDRESS); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index 875d73a9507a..07f463e1c1d5 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -419,8 +419,8 @@ private void startKubernetesClusterVMs() { } } - private boolean isKubernetesClusterKubeConfigAvailable(final String masterVMPrivateIpAddress, final long timeoutTime) { - if (Strings.isNullOrEmpty(masterVMPrivateIpAddress)) { + private boolean isKubernetesClusterKubeConfigAvailable(final long timeoutTime) { + if (Strings.isNullOrEmpty(publicIpAddress)) { KubernetesClusterDetailsVO kubeConfigDetail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "kubeConfigData"); if (kubeConfigDetail != null && !Strings.isNullOrEmpty(kubeConfigDetail.getValue())) { return true; @@ -428,6 +428,7 @@ private boolean isKubernetesClusterKubeConfigAvailable(final String masterVMPriv } String kubeConfig = KubernetesClusterUtil.getKubernetesClusterConfig(kubernetesCluster, publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, timeoutTime); if (!Strings.isNullOrEmpty(kubeConfig)) { + final String masterVMPrivateIpAddress = getMasterVmPrivateIp(); if (!Strings.isNullOrEmpty(masterVMPrivateIpAddress)) { kubeConfig = kubeConfig.replace(String.format("server: https://%s:%d", masterVMPrivateIpAddress, CLUSTER_API_PORT), String.format("server: https://%s:%d", publicIpAddress, CLUSTER_API_PORT)); @@ -542,7 +543,7 @@ public boolean startKubernetesClusterOnCreate() { if (!readyNodesCountValid) { logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster ID: %s as it does not have desired number of nodes in ready state", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); } - if (!isKubernetesClusterKubeConfigAvailable(publicIpAddress, startTimeoutTime)) { + if (!isKubernetesClusterKubeConfigAvailable(startTimeoutTime)) { logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster ID: %s in usable state as unable to retrieve kube-config for the cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } if (!isKubernetesClusterDashboardServiceRunning(true, startTimeoutTime)) { @@ -565,16 +566,16 @@ public boolean startStoppedKubernetesCluster() throws CloudRuntimeException { } catch (MalformedURLException | UnknownHostException ex) { logTransitStateAndThrow(Level.ERROR, String.format("Kubernetes cluster ID: %s has invalid API endpoint. Can not verify if cluster is in ready state", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } - Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(null); - String publicIpAddress = publicIpSshPort.first(); + Pair sshIpPort = getKubernetesClusterServerIpSshPort(null); + publicIpAddress = sshIpPort.first(); + sshPort = sshIpPort.second(); if (Strings.isNullOrEmpty(publicIpAddress)) { logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s as no public IP found for the cluster" , kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } if (!KubernetesClusterUtil.isKubernetesClusterServerRunning(kubernetesCluster, publicIpAddress, CLUSTER_API_PORT, startTimeoutTime, 15000)) { logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s in usable state", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } - sshPort = publicIpSshPort.second(); - if (!isKubernetesClusterKubeConfigAvailable(null, startTimeoutTime)) { + if (!isKubernetesClusterKubeConfigAvailable(startTimeoutTime)) { logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster ID: %s in usable state as unable to retrieve kube-config for the cluster", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } if (!isKubernetesClusterDashboardServiceRunning(false, startTimeoutTime)) { @@ -613,7 +614,7 @@ public boolean reconcileAlertCluster() { return false; } updateKubernetesClusterEntryEndpoint(); - if (!isKubernetesClusterKubeConfigAvailable(null, startTimeoutTime)) { + if (!isKubernetesClusterKubeConfigAvailable(startTimeoutTime)) { return false; } if (!isKubernetesClusterDashboardServiceRunning(false, startTimeoutTime)) { From 00f4bf4b537d2532d77beb10eec9917cd8d86e84 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Sat, 22 Feb 2020 11:16:09 +0530 Subject: [PATCH 123/134] smoke test template format fix Signed-off-by: Abhishek Kumar --- test/integration/smoke/test_kubernetes_clusters.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index 8e365a39fb0b..45e3babbbc5f 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -106,15 +106,18 @@ def setUpClass(cls): "hypervisor": "kvm", "ostype": "CoreOS", "url": "http://staging.yadav.xyz/cks/templates/coreos_production_cloudstack_image-kvm.qcow2.bz2", - "requireshvm": "True", "ispublic": "True", "isextractable": "True" } # "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-kvm.qcow2.bz2" if cls.hypervisor.lower() == "vmware": cks_template_data["url"] = "http://staging.yadav.xyz/cks/templates/coreos_production_cloudstack_image-vmware.ova" # "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-vmware.ova" + cks_template_data["format"] = "OVA" elif cls.hypervisor.lower() == "xenserver": cks_template_data["url"] = "http://staging.yadav.xyz/cks/templates/coreos_production_cloudstack_image-xen.vhd.bz2" # "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-xen.vhd.bz2" + cks_template_data["format"] = "VHD" + elif cls.hypervisor.lower() == "kvm": + cks_template_data["requireshvm"] = "True" if cls.setup_failed == False: cls.cks_template = Template.register( cls.apiclient, From 0c86833ad54fbb6fa60c01f15c81c1851fa44e42 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Sat, 22 Feb 2020 11:19:02 +0530 Subject: [PATCH 124/134] fix missing firewall removal Signed-off-by: Abhishek Kumar --- .../cluster/actionworkers/KubernetesClusterDestroyWorker.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java index faf8ee19c73c..8d7f42730aae 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java @@ -149,11 +149,11 @@ private void deleteKubernetesClusterNetworkRules() throws ManagementServerExcept } FirewallRule firewallRule = removeApiFirewallRule(publicIp); if (firewallRule == null) { - throw new ManagementServerException("Firewall rule for API access can't be removed"); + logMessage(Level.WARN, "Firewall rule for API access can't be removed", null); } firewallRule = removeSshFirewallRule(publicIp); if (firewallRule == null) { - throw new ManagementServerException("Firewall rule for SSH access can't be removed"); + logMessage(Level.WARN, "Firewall rule for SSH access can't be removed", null); } try { removePortForwardingRules(publicIp, network, owner, removedVmIds); From 5c09ad1ababbe7022a6ec02b8385e05362c30c52 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Sat, 22 Feb 2020 13:00:36 +0530 Subject: [PATCH 125/134] fix upgrade version same semantic version in ui Signed-off-by: Abhishek Kumar --- ui/plugins/cks/cks.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ui/plugins/cks/cks.js b/ui/plugins/cks/cks.js index 119518a70b02..40e20c5da45b 100644 --- a/ui/plugins/cks/cks.js +++ b/ui/plugins/cks/cks.js @@ -852,8 +852,16 @@ var items = []; var versionObjs = json.listkubernetessupportedversionsresponse.kubernetessupportedversion; if (versionObjs != null) { + var clusterVersion = null; + for (var j = 0; j < versionObjs.length; j++) { + if (versionObjs[j].id != args.context.kubernetesclusters[0].kubernetesversionid) { + clusterVersion = versionObjs[j]; + break; + } + } for (var i = 0; i < versionObjs.length; i++) { if (versionObjs[i].id != args.context.kubernetesclusters[0].kubernetesversionid && + (clusterVersion == null || (clusterVersion != null && versionObjs[i].semanticversion != clusterVersion.semanticversion)) && versionObjs[i].state == 'Enabled' && versionObjs[i].isostate == 'Ready') { items.push({ id: versionObjs[i].id, From 26d0a2ff831fe6e49670bb70792a7f6fcbbea7f9 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Sun, 23 Feb 2020 18:43:32 +0530 Subject: [PATCH 126/134] fix test vmware template settings Signed-off-by: Abhishek Kumar --- test/integration/smoke/test_kubernetes_clusters.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index 45e3babbbc5f..021a7490ff2b 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -110,9 +110,11 @@ def setUpClass(cls): "isextractable": "True" } # "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-kvm.qcow2.bz2" + cks_template_data_details = [] if cls.hypervisor.lower() == "vmware": cks_template_data["url"] = "http://staging.yadav.xyz/cks/templates/coreos_production_cloudstack_image-vmware.ova" # "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-vmware.ova" cks_template_data["format"] = "OVA" + cks_template_data_details = [{"keyboard":"us","nicAdapter":"Vmxnet3","rootDiskController":"pvscsi"}] elif cls.hypervisor.lower() == "xenserver": cks_template_data["url"] = "http://staging.yadav.xyz/cks/templates/coreos_production_cloudstack_image-xen.vhd.bz2" # "http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-xen.vhd.bz2" cks_template_data["format"] = "VHD" @@ -123,7 +125,8 @@ def setUpClass(cls): cls.apiclient, cks_template_data, zoneid=cls.zone.id, - hypervisor=cls.hypervisor + hypervisor=cls.hypervisor, + details=cks_template_data_details ) cls.debug("Waiting for CKS template with ID %s to be ready" % cls.cks_template.id) try: From 1bde9d6914f020916dc278f9d47587d12b1333dd Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 24 Feb 2020 01:23:34 +0530 Subject: [PATCH 127/134] eject iso from os conditionally Signed-off-by: Abhishek Kumar --- ...ernetesClusterResourceModifierActionWorker.java | 7 +++++-- .../KubernetesClusterStartWorker.java | 14 ++++++++++---- .../KubernetesClusterUpgradeWorker.java | 10 +++++++--- .../src/main/resources/conf/k8s-master-add.yml | 3 ++- .../src/main/resources/conf/k8s-master.yml | 3 ++- .../src/main/resources/conf/k8s-node.yml | 3 ++- .../main/resources/script/upgrade-kubernetes.sh | 12 ++++++++---- 7 files changed, 36 insertions(+), 16 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index dedfbb2d7380..5d256140f345 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -50,6 +50,7 @@ import com.cloud.exception.ResourceUnavailableException; import com.cloud.host.Host; import com.cloud.host.HostVO; +import com.cloud.hypervisor.Hypervisor; import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; @@ -125,11 +126,12 @@ protected void init() { kubernetesClusterNodeNamePrefix = getKubernetesClusterNodeNamePrefix(); } - private String getKubernetesNodeConfig(final String joinIp) throws IOException { + private String getKubernetesNodeConfig(final String joinIp, final boolean ejectIso) throws IOException { String k8sNodeConfig = readResourceFile("/conf/k8s-node.yml"); final String sshPubKey = "{{ k8s.ssh.pub.key }}"; final String joinIpKey = "{{ k8s_master.join_ip }}"; final String clusterTokenKey = "{{ k8s_master.cluster.token }}"; + final String ejectIsoKey = "{{ k8s.eject.iso }}"; String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\""; String sshKeyPair = kubernetesCluster.getKeyPair(); if (!Strings.isNullOrEmpty(sshKeyPair)) { @@ -141,6 +143,7 @@ private String getKubernetesNodeConfig(final String joinIp) throws IOException { k8sNodeConfig = k8sNodeConfig.replace(sshPubKey, pubKey); k8sNodeConfig = k8sNodeConfig.replace(joinIpKey, joinIp); k8sNodeConfig = k8sNodeConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); + k8sNodeConfig = k8sNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); /* genarate /.docker/config.json file on the nodes only if Kubernetes cluster is created to * use docker private registry */ String dockerUserName = null; @@ -316,7 +319,7 @@ protected UserVm createKubernetesNode(String joinIp, int nodeInstance) throws Ma String hostName = getKubernetesClusterNodeAvailableName(String.format("%s-node-%s", kubernetesClusterNodeNamePrefix, nodeInstance)); String k8sNodeConfig = null; try { - k8sNodeConfig = getKubernetesNodeConfig(joinIp); + k8sNodeConfig = getKubernetesNodeConfig(joinIp, Hypervisor.HypervisorType.VMware.equals(template.getHypervisorType())); } catch (IOException e) { logAndThrow(Level.ERROR, "Failed to read Kubernetes node configuration file", e); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index 07f463e1c1d5..3c2a9ebe9286 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -44,6 +44,7 @@ import com.cloud.exception.ManagementServerException; import com.cloud.exception.NetworkRuleConflictException; import com.cloud.exception.ResourceUnavailableException; +import com.cloud.hypervisor.Hypervisor; import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; @@ -117,7 +118,8 @@ private boolean isKubernetesVersionSupportsHA() { } private String getKubernetesMasterConfig(final String masterIp, final String serverIp, - final String hostName, final boolean haSupported) throws IOException { + final String hostName, final boolean haSupported, + final boolean ejectIso) throws IOException { String k8sMasterConfig = readResourceFile("/conf/k8s-master.yml"); final String apiServerCert = "{{ k8s_master.apiserver.crt }}"; final String apiServerKey = "{{ k8s_master.apiserver.key }}"; @@ -125,6 +127,7 @@ private String getKubernetesMasterConfig(final String masterIp, final String ser final String sshPubKey = "{{ k8s.ssh.pub.key }}"; final String clusterToken = "{{ k8s_master.cluster.token }}"; final String clusterInitArgsKey = "{{ k8s_master.cluster.initargs }}"; + final String ejectIsoKey = "{{ k8s.eject.iso }}"; final List addresses = new ArrayList<>(); addresses.add(masterIp); if (!serverIp.equals(masterIp)) { @@ -158,6 +161,7 @@ private String getKubernetesMasterConfig(final String masterIp, final String ser } initArgs += String.format("--apiserver-cert-extra-sans=%s", serverIp); k8sMasterConfig = k8sMasterConfig.replace(clusterInitArgsKey, initArgs); + k8sMasterConfig = k8sMasterConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); return k8sMasterConfig; } @@ -189,7 +193,7 @@ private UserVm createKubernetesMaster(final Network network, String serverIp) th boolean haSupported = isKubernetesVersionSupportsHA(); String k8sMasterConfig = null; try { - k8sMasterConfig = getKubernetesMasterConfig(masterIp, serverIp, hostName, haSupported); + k8sMasterConfig = getKubernetesMasterConfig(masterIp, serverIp, hostName, haSupported, Hypervisor.HypervisorType.VMware.equals(template.getHypervisorType())); } catch (IOException e) { logAndThrow(Level.ERROR, "Failed to read Kubernetes master configuration file", e); } @@ -204,12 +208,13 @@ private UserVm createKubernetesMaster(final Network network, String serverIp) th return masterVm; } - private String getKubernetesAdditionalMasterConfig(final String joinIp) throws IOException { + private String getKubernetesAdditionalMasterConfig(final String joinIp, final boolean ejectIso) throws IOException { String k8sMasterConfig = readResourceFile("/conf/k8s-master-add.yml"); final String joinIpKey = "{{ k8s_master.join_ip }}"; final String clusterTokenKey = "{{ k8s_master.cluster.token }}"; final String sshPubKey = "{{ k8s.ssh.pub.key }}"; final String clusterHACertificateKey = "{{ k8s_master.cluster.ha.certificate.key }}"; + final String ejectIsoKey = "{{ k8s.eject.iso }}"; String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\""; String sshKeyPair = kubernetesCluster.getKeyPair(); if (!Strings.isNullOrEmpty(sshKeyPair)) { @@ -222,6 +227,7 @@ private String getKubernetesAdditionalMasterConfig(final String joinIp) throws I k8sMasterConfig = k8sMasterConfig.replace(joinIpKey, joinIp); k8sMasterConfig = k8sMasterConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); k8sMasterConfig = k8sMasterConfig.replace(clusterHACertificateKey, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster)); + k8sMasterConfig = k8sMasterConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); return k8sMasterConfig; } @@ -242,7 +248,7 @@ private UserVm createKubernetesAdditionalMaster(final String joinIp, final int a String hostName = getKubernetesClusterNodeAvailableName(String.format("%s-master-%d", kubernetesClusterNodeNamePrefix, additionalMasterNodeInstance + 1)); String k8sMasterConfig = null; try { - k8sMasterConfig = getKubernetesAdditionalMasterConfig(joinIp); + k8sMasterConfig = getKubernetesAdditionalMasterConfig(joinIp, Hypervisor.HypervisorType.VMware.equals(template.getHypervisorType())); } catch (IOException e) { logAndThrow(Level.ERROR, "Failed to read Kubernetes master configuration file", e); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java index 1093ea149b8c..eb9058d765db 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java @@ -27,6 +27,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Level; +import com.cloud.hypervisor.Hypervisor; import com.cloud.kubernetes.cluster.KubernetesCluster; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; import com.cloud.kubernetes.cluster.KubernetesClusterService; @@ -71,9 +72,12 @@ private Pair runInstallScriptOnVM(final UserVm vm, final int in String nodeAddress = (index > 0 && sshPort == 22) ? vm.getPrivateIpAddress() : publicIpAddress; SshHelper.scpTo(nodeAddress, nodeSshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null, "~/", upgradeScriptFile.getAbsolutePath(), "0755"); - String cmdStr = String.format("sudo ./%s %s %s %s", upgradeScriptFile.getName(), - upgradeVersion.getSemanticVersion(), index == 0 ? "true" : "false", - KubernetesVersionManagerImpl.compareSemanticVersions(upgradeVersion.getSemanticVersion(), "1.15.0") < 0 ? "true" : "false"); + String cmdStr = String.format("sudo ./%s %s %s %s %s", + upgradeScriptFile.getName(), + upgradeVersion.getSemanticVersion(), + index == 0 ? "true" : "false", + KubernetesVersionManagerImpl.compareSemanticVersions(upgradeVersion.getSemanticVersion(), "1.15.0") < 0 ? "true" : "false", + Hypervisor.HypervisorType.VMware.equals(vm.getHypervisorType())); return SshHelper.sshExecute(publicIpAddress, nodeSshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null, cmdStr, 10000, 10000, 10 * 60 * 1000); diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml index 84dbd8a093e1..8e632c5c6530 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml @@ -42,6 +42,7 @@ write-files: MAX_OFFLINE_INSTALL_ATTEMPTS=40 offline_attempts=1 MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 + EJECT_ISO_FROM_OS={{ k8s.eject.iso }} crucial_cmd_attempts=1 iso_drive_path="" while true; do @@ -125,7 +126,7 @@ write-files: setup_complete=true fi umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" - if [ "$iso_drive_path" != "" ]; then + if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then eject "${iso_drive_path}" fi fi diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml index c9578c12df91..01335a1ba980 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml @@ -62,6 +62,7 @@ write-files: MAX_OFFLINE_INSTALL_ATTEMPTS=40 offline_attempts=1 MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 + EJECT_ISO_FROM_OS={{ k8s.eject.iso }} crucial_cmd_attempts=1 iso_drive_path="" while true; do @@ -147,7 +148,7 @@ write-files: mkdir -p "${K8S_CONFIG_SCRIPTS_COPY_DIR}" cp ${BINARIES_DIR}/*.yaml "${K8S_CONFIG_SCRIPTS_COPY_DIR}" umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" - if [ "$iso_drive_path" != "" ]; then + if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then eject "${iso_drive_path}" fi fi diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index 63af4c833a1b..666fe1632c2c 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -42,6 +42,7 @@ write-files: MAX_OFFLINE_INSTALL_ATTEMPTS=40 offline_attempts=1 MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 + EJECT_ISO_FROM_OS={{ k8s.eject.iso }} crucial_cmd_attempts=1 iso_drive_path="" while true; do @@ -125,7 +126,7 @@ write-files: setup_complete=true fi umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" - if [ "$iso_drive_path" != "" ]; then + if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then eject "${iso_drive_path}" fi fi diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh index 96520d35d121..ea36d7ee8970 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh +++ b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh @@ -17,9 +17,9 @@ # under the License. # Version 1.14 and below needs extra flags with kubeadm upgrade node -if [ $# -lt 2 ]; then - echo "Invalid input. Valid usage: ./upgrade-kubernetes.sh UPGRADE_VERSION IS_MASTER IS_OLD_VERSION" - echo "eg: ./upgrade-kubernetes.sh 1.16.3 true false" +if [ $# -lt 4 ]; then + echo "Invalid input. Valid usage: ./upgrade-kubernetes.sh UPGRADE_VERSION IS_MASTER IS_OLD_VERSION IS_EJECT_ISO" + echo "eg: ./upgrade-kubernetes.sh 1.16.3 true false false" exit 1 fi UPGRADE_VERSION="${1}" @@ -31,6 +31,10 @@ IS_OLD_VERSION="" if [ $# -gt 2 ]; then IS_OLD_VERSION="${3}" fi +EJECT_ISO_FROM_OS=false +if [ $# -gt 3 ]; then + EJECT_ISO_FROM_OS="${4}" +fi export PATH=$PATH:/opt/bin @@ -123,7 +127,7 @@ if [ -d "$BINARIES_DIR" ]; then fi umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}" - if [ "$iso_drive_path" != "" ]; then + if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then eject "${iso_drive_path}" fi fi From a55c01852fe94f6ddc7964b97191c66a60921292 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 24 Feb 2020 22:11:54 +0530 Subject: [PATCH 128/134] fix wrong check for smae semantic version filtering Signed-off-by: Abhishek Kumar --- ui/plugins/cks/cks.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/plugins/cks/cks.js b/ui/plugins/cks/cks.js index 40e20c5da45b..e74e118f3e60 100644 --- a/ui/plugins/cks/cks.js +++ b/ui/plugins/cks/cks.js @@ -854,7 +854,7 @@ if (versionObjs != null) { var clusterVersion = null; for (var j = 0; j < versionObjs.length; j++) { - if (versionObjs[j].id != args.context.kubernetesclusters[0].kubernetesversionid) { + if (versionObjs[j].id == args.context.kubernetesclusters[0].kubernetesversionid) { clusterVersion = versionObjs[j]; break; } From 69a30d5b977d49f982c2d521dba9b6b76f1f5291 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 24 Feb 2020 22:12:27 +0530 Subject: [PATCH 129/134] remove redundant fields in details tab Signed-off-by: Abhishek Kumar --- ui/plugins/cks/cks.js | 8 -------- 1 file changed, 8 deletions(-) diff --git a/ui/plugins/cks/cks.js b/ui/plugins/cks/cks.js index e74e118f3e60..c353c24d0939 100644 --- a/ui/plugins/cks/cks.js +++ b/ui/plugins/cks/cks.js @@ -946,14 +946,6 @@ }, keypair: { label: 'label.ssh.key.pair' - }, - username: { - label: 'label.username', - isCopyPaste: true - }, - password: { - label: 'label.password', - isCopyPaste: true } }], From a58b0926c6d83d4b8e7183ad80cc2992e56e28f5 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 26 Feb 2020 15:13:01 +0530 Subject: [PATCH 130/134] move default CKS network creation in java code Signed-off-by: Abhishek Kumar --- .../META-INF/db/schema-41300to41400.sql | 25 ------------- .../cluster/KubernetesClusterManagerImpl.java | 36 +++++++++++++++++-- 2 files changed, 34 insertions(+), 27 deletions(-) diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql b/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql index a07c5bf94c3c..728155283cc1 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41300to41400.sql @@ -125,28 +125,3 @@ CREATE TABLE IF NOT EXISTS `cloud`.`kubernetes_cluster_details` ( PRIMARY KEY(`id`), CONSTRAINT `fk_kubernetes_cluster_details__cluster_id` FOREIGN KEY `fk_kubernetes_cluster_details__cluster_id`(`cluster_id`) REFERENCES `kubernetes_cluster`(`id`) ON DELETE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -INSERT IGNORE INTO `cloud`.`network_offerings` (name, uuid, unique_name, display_text, nw_rate, mc_rate, traffic_type, - tags, system_only, specify_vlan, service_offering_id, conserve_mode, created, availability, dedicated_lb_service, - shared_source_nat_service, sort_key, redundant_router_service, state, guest_type, elastic_ip_service, - eip_associate_public_ip, elastic_lb_service, specify_ip_ranges, inline, is_persistent, internal_lb, public_lb, - egress_default_policy, concurrent_connections, keep_alive_enabled, supports_streched_l2, `default`, removed) VALUES ( - 'DefaultNetworkOfferingforKubernetesService', UUID(), 'DefaultNetworkOfferingforKubernetesService', 'Network Offering used for CloudStack Kubernetes service', NULL,NULL,'Guest', - NULL, 0, 0, NULL, 1, now(),'Required', 1, - 0, 0, 0, 'Enabled', 'Isolated', 0, - 1, 0, 0, 0, 0, 0, 1, - 1, NULL, 0, 0, 0, NULL); - -UPDATE `cloud`.`network_offerings` SET removed=NULL WHERE unique_name='DefaultNetworkOfferingforKubernetesService'; - -SET @kubernetesnetwork = (select id from network_offerings where name='DefaultNetworkOfferingforKubernetesService' and removed IS NULL); -INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@kubernetesnetwork, 'Dhcp','VirtualRouter',now()); -INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@kubernetesnetwork, 'Dns','VirtualRouter',now()); -INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@kubernetesnetwork, 'Firewall','VirtualRouter',now()); -INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@kubernetesnetwork, 'Gateway','VirtualRouter',now()); -INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@kubernetesnetwork, 'Lb','VirtualRouter',now()); -INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@kubernetesnetwork, 'PortForwarding','VirtualRouter',now()); -INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@kubernetesnetwork, 'SourceNat','VirtualRouter',now()); -INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@kubernetesnetwork, 'StaticNat','VirtualRouter',now()); -INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@kubernetesnetwork, 'UserData','VirtualRouter',now()); -INSERT IGNORE INTO ntwk_offering_service_map (network_offering_id, service, provider, created) VALUES (@kubernetesnetwork, 'Vpn','VirtualRouter',now()); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index cb7961cfeda7..11dc6a90e355 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -22,6 +22,7 @@ import java.security.SecureRandom; import java.util.ArrayList; import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -100,6 +101,7 @@ import com.cloud.network.Network.Service; import com.cloud.network.NetworkModel; import com.cloud.network.NetworkService; +import com.cloud.network.Networks; import com.cloud.network.PhysicalNetwork; import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.dao.NetworkDao; @@ -109,6 +111,7 @@ import com.cloud.network.rules.FirewallRuleVO; import com.cloud.offering.NetworkOffering; import com.cloud.offering.ServiceOffering; +import com.cloud.offerings.NetworkOfferingServiceMapVO; import com.cloud.offerings.NetworkOfferingVO; import com.cloud.offerings.dao.NetworkOfferingDao; import com.cloud.offerings.dao.NetworkOfferingServiceMapDao; @@ -154,6 +157,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements KubernetesClusterService { private static final Logger LOGGER = Logger.getLogger(KubernetesClusterManagerImpl.class); + private static final String DEFAULT_NETWORK_OFFERING_FOR_KUBERNETES_SERVICE_NAME = "DefaultNetworkOfferingforKubernetesService"; protected StateMachine2 _stateMachine = KubernetesCluster.State.getStateMachine(); @@ -197,6 +201,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne @Inject protected NetworkOfferingJoinDao networkOfferingJoinDao; @Inject + protected NetworkOfferingServiceMapDao networkOfferingServiceMapDao; + @Inject protected NetworkService networkService; @Inject protected NetworkModel networkModel; @@ -207,8 +213,6 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne @Inject protected NetworkDao networkDao; @Inject - protected NetworkOfferingServiceMapDao networkOfferingServiceMapDao; - @Inject protected CapacityManager capacityManager; @Inject protected ResourceManager resourceManager; @@ -1426,6 +1430,34 @@ boolean isClusterVMsInDesiredState(KubernetesCluster kubernetesCluster, VirtualM @Override public boolean start() { + final Map defaultKubernetesServiceNetworkOfferingProviders = new HashMap(); + defaultKubernetesServiceNetworkOfferingProviders.put(Service.Dhcp, Network.Provider.VirtualRouter); + defaultKubernetesServiceNetworkOfferingProviders.put(Service.Dns, Network.Provider.VirtualRouter); + defaultKubernetesServiceNetworkOfferingProviders.put(Service.UserData, Network.Provider.VirtualRouter); + defaultKubernetesServiceNetworkOfferingProviders.put(Service.Firewall, Network.Provider.VirtualRouter); + defaultKubernetesServiceNetworkOfferingProviders.put(Service.Gateway, Network.Provider.VirtualRouter); + defaultKubernetesServiceNetworkOfferingProviders.put(Service.Lb, Network.Provider.VirtualRouter); + defaultKubernetesServiceNetworkOfferingProviders.put(Service.SourceNat, Network.Provider.VirtualRouter); + defaultKubernetesServiceNetworkOfferingProviders.put(Service.StaticNat, Network.Provider.VirtualRouter); + defaultKubernetesServiceNetworkOfferingProviders.put(Service.PortForwarding, Network.Provider.VirtualRouter); + defaultKubernetesServiceNetworkOfferingProviders.put(Service.Vpn, Network.Provider.VirtualRouter); + + NetworkOfferingVO defaultKubernetesServiceNetworkOffering = + new NetworkOfferingVO(DEFAULT_NETWORK_OFFERING_FOR_KUBERNETES_SERVICE_NAME, + "Network Offering used for CloudStack Kubernetes service", Networks.TrafficType.Guest, false, false, null, null, true, NetworkOffering.Availability.Required, null, + Network.GuestType.Isolated, true, false, false, false, true, false); + + defaultKubernetesServiceNetworkOffering.setState(NetworkOffering.State.Enabled); + defaultKubernetesServiceNetworkOffering = networkOfferingDao.persistDefaultNetworkOffering(defaultKubernetesServiceNetworkOffering); + + for (Service service : defaultKubernetesServiceNetworkOfferingProviders.keySet()) { + NetworkOfferingServiceMapVO offService = + new NetworkOfferingServiceMapVO(defaultKubernetesServiceNetworkOffering.getId(), service, + defaultKubernetesServiceNetworkOfferingProviders.get(service)); + networkOfferingServiceMapDao.persist(offService); + LOGGER.trace("Added service for the network offering: " + offService); + } + _gcExecutor.scheduleWithFixedDelay(new KubernetesClusterGarbageCollector(), 300, 300, TimeUnit.SECONDS); _stateScanner.scheduleWithFixedDelay(new KubernetesClusterStatusScanner(), 300, 30, TimeUnit.SECONDS); From f1830e0e0a26380d671df390fa322d6794f201f0 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 26 Feb 2020 17:52:29 +0530 Subject: [PATCH 131/134] enable egress policy Signed-off-by: Abhishek Kumar --- .../kubernetes/cluster/KubernetesClusterManagerImpl.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 11dc6a90e355..358fa034a8ba 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -1444,9 +1444,12 @@ public boolean start() { NetworkOfferingVO defaultKubernetesServiceNetworkOffering = new NetworkOfferingVO(DEFAULT_NETWORK_OFFERING_FOR_KUBERNETES_SERVICE_NAME, - "Network Offering used for CloudStack Kubernetes service", Networks.TrafficType.Guest, false, false, null, null, true, NetworkOffering.Availability.Required, null, - Network.GuestType.Isolated, true, false, false, false, true, false); - + "Network Offering used for CloudStack Kubernetes service", Networks.TrafficType.Guest, + false, false, null, null, true, + NetworkOffering.Availability.Required, null, Network.GuestType.Isolated, true, + true, false, false, false, false, + false, false, false, true, true, false, + false, true, false, false); defaultKubernetesServiceNetworkOffering.setState(NetworkOffering.State.Enabled); defaultKubernetesServiceNetworkOffering = networkOfferingDao.persistDefaultNetworkOffering(defaultKubernetesServiceNetworkOffering); From a755b26f0991152cf5b2ea0629005ee336d02d47 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 28 Feb 2020 13:29:03 +0530 Subject: [PATCH 132/134] scaling improvements cluster with scaling failure remains in Alert state if cluster VO node count differs from cluster node VMs count Signed-off-by: Abhishek Kumar --- .../cluster/actionworkers/KubernetesClusterScaleWorker.java | 2 +- .../cluster/actionworkers/KubernetesClusterStartWorker.java | 6 ++++++ .../src/main/resources/conf/k8s-master-add.yml | 2 +- .../src/main/resources/conf/k8s-master.yml | 2 +- .../kubernetes-service/src/main/resources/conf/k8s-node.yml | 2 +- 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index e9a36d36c042..762fb375c1a3 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -354,6 +354,7 @@ private void scaleUpKubernetesClusterSize(final long newVmCount) throws CloudRun } catch (CloudRuntimeException | ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { logTransitStateToFailedIfNeededAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, unable to provision node VM in the cluster", kubernetesCluster.getUuid()), e); } + attachIsoKubernetesVMs(clusterVMs); for (UserVm vm : clusterVMs) { clusterVMIds.add(vm.getId()); } @@ -362,7 +363,6 @@ private void scaleUpKubernetesClusterSize(final long newVmCount) throws CloudRun } catch (ManagementServerException e) { logTransitStateToFailedIfNeededAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster ID: %s, unable to update network rules", kubernetesCluster.getUuid()), e); } - attachIsoKubernetesVMs(clusterVMs); KubernetesClusterVO kubernetesClusterVO = kubernetesClusterDao.findById(kubernetesCluster.getId()); kubernetesClusterVO.setNodeCount(clusterSize); boolean readyNodesCountValid = KubernetesClusterUtil.validateKubernetesClusterReadyNodesCount(kubernetesClusterVO, publicIpAddress, sshPort, diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index 3c2a9ebe9286..d4525630196c 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -32,6 +32,7 @@ import org.apache.cloudstack.framework.ca.Certificate; import org.apache.cloudstack.utils.security.CertUtils; import org.apache.commons.codec.binary.Base64; +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Level; import com.cloud.dc.DataCenter; @@ -50,6 +51,7 @@ import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; import com.cloud.kubernetes.cluster.KubernetesClusterService; import com.cloud.kubernetes.cluster.KubernetesClusterVO; +import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil; import com.cloud.kubernetes.version.KubernetesSupportedVersion; import com.cloud.kubernetes.version.KubernetesVersionManagerImpl; @@ -597,6 +599,10 @@ public boolean startStoppedKubernetesCluster() throws CloudRuntimeException { public boolean reconcileAlertCluster() { init(); final long startTimeoutTime = System.currentTimeMillis() + 3 * 60 * 1000; + List vmMapVOList = getKubernetesClusterVMMaps(); + if (CollectionUtils.isEmpty(vmMapVOList) || vmMapVOList.size() != kubernetesCluster.getTotalNodeCount()) { + return false; + } Pair sshIpPort = getKubernetesClusterServerIpSshPort(null); publicIpAddress = sshIpPort.first(); sshPort = sshIpPort.second(); diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml index 8e632c5c6530..787ea97491ce 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master-add.yml @@ -39,7 +39,7 @@ write-files: setup_complete=false OFFLINE_INSTALL_ATTEMPT_SLEEP=15 - MAX_OFFLINE_INSTALL_ATTEMPTS=40 + MAX_OFFLINE_INSTALL_ATTEMPTS=100 offline_attempts=1 MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 EJECT_ISO_FROM_OS={{ k8s.eject.iso }} diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml index 01335a1ba980..14828578ed8c 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-master.yml @@ -59,7 +59,7 @@ write-files: setup_complete=false OFFLINE_INSTALL_ATTEMPT_SLEEP=15 - MAX_OFFLINE_INSTALL_ATTEMPTS=40 + MAX_OFFLINE_INSTALL_ATTEMPTS=100 offline_attempts=1 MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 EJECT_ISO_FROM_OS={{ k8s.eject.iso }} diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index 666fe1632c2c..d2f5454a669d 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -38,7 +38,7 @@ write-files: ATTEMPT_ONLINE_INSTALL=false setup_complete=false - OFFLINE_INSTALL_ATTEMPT_SLEEP=15 + OFFLINE_INSTALL_ATTEMPT_SLEEP=30 MAX_OFFLINE_INSTALL_ATTEMPTS=40 offline_attempts=1 MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 From 30e71eec0647a24a95ff5263905d7ffbcf6550df Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 28 Feb 2020 14:45:53 +0530 Subject: [PATCH 133/134] down scaling fix Signed-off-by: Abhishek Kumar --- .../cluster/actionworkers/KubernetesClusterScaleWorker.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index 762fb375c1a3..985279363bc4 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -309,7 +309,7 @@ private void scaleDownKubernetesClusterSize() throws CloudRuntimeException { final List originalVmList = getKubernetesClusterVMMaps(); int i = originalVmList.size() - 1; List removedVmIds = new ArrayList<>(); - while (i > kubernetesCluster.getMasterNodeCount()) { + while (i > kubernetesCluster.getMasterNodeCount() + clusterSize) { KubernetesClusterVmMapVO vmMapVO = originalVmList.get(i); UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId()); if (!removeKubernetesClusterNode(publicIpAddress, sshPort, userVM, 3, 30000)) { From bf38e620f4b8326287c2ebcec102280736047fd9 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 28 Feb 2020 16:14:18 +0530 Subject: [PATCH 134/134] scale down Signed-off-by: Abhishek Kumar --- .../cluster/actionworkers/KubernetesClusterScaleWorker.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index 985279363bc4..0d6a028c9a80 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -309,7 +309,7 @@ private void scaleDownKubernetesClusterSize() throws CloudRuntimeException { final List originalVmList = getKubernetesClusterVMMaps(); int i = originalVmList.size() - 1; List removedVmIds = new ArrayList<>(); - while (i > kubernetesCluster.getMasterNodeCount() + clusterSize) { + while (i >= kubernetesCluster.getMasterNodeCount() + clusterSize) { KubernetesClusterVmMapVO vmMapVO = originalVmList.get(i); UserVmVO userVM = userVmDao.findById(vmMapVO.getVmId()); if (!removeKubernetesClusterNode(publicIpAddress, sshPort, userVM, 3, 30000)) {