From efdea0667d86d4de23e1fdb5b7d0a261786b58fc Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 15 Sep 2025 14:04:55 +0530 Subject: [PATCH 01/12] systemvm-template: support on-demand download during setup and registration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bundling all hypervisor SystemVM templates in release packages simplifies installs but inflates build time and artifact size. This change enables downloading templates on demand when they’re not found after package installation. The download path is wired into both cloud-setup-management and the existing SystemVM template registration flow. For connected or mirrored environments, a repository URL prefix can be provided to support air-gapped setups: pass --systemvm-templates-repository to cloud-setup-management, or set system.vm.templates.download.repository= in server.properties for post-setup registration. If templates are already present (bundled or preseeded), behavior is unchanged and no download is attempted. Signed-off-by: Abhishek Kumar --- client/bindir/cloud-setup-management.in | 124 ++++++++++++++++++ client/conf/server.properties.in | 5 + .../upgrade/SystemVmTemplateRegistration.java | 20 ++- .../SystemVmTemplateRegistrationTest.java | 2 +- engine/schema/templateConfig.sh | 25 ++-- .../ExternalPathPayloadProvisioner.java | 28 +--- .../utils/ServerPropertiesUtil.java | 58 ++++++++ 7 files changed, 225 insertions(+), 37 deletions(-) create mode 100644 utils/src/main/java/org/apache/cloudstack/utils/ServerPropertiesUtil.java diff --git a/client/bindir/cloud-setup-management.in b/client/bindir/cloud-setup-management.in index 84c87ae2e442..6993f5dab793 100755 --- a/client/bindir/cloud-setup-management.in +++ b/client/bindir/cloud-setup-management.in @@ -36,6 +36,100 @@ from cloudutils.cloudException import CloudRuntimeException, CloudInternalExcept from cloudutils.globalEnv import globalEnv from cloudutils.serviceConfigServer import cloudManagementConfig from optparse import OptionParser +import urllib.request +import configparser +import hashlib + +SYSTEMVM_TEMPLATES_PATH = "/usr/share/cloudstack-management/templates/systemvm" +SYSTEMVM_TEMPLATES_METADATA_FILE = SYSTEMVM_TEMPLATES_PATH + "/metadata.ini" + +def verify_sha512_checksum(file_path, expected_checksum): + sha512 = hashlib.sha512() + try: + with open(file_path, "rb") as f: + for chunk in iter(lambda: f.read(8192), b""): + sha512.update(chunk) + return sha512.hexdigest().lower() == expected_checksum.lower() + except Exception as e: + print(f"Failed to verify checksum for {file_path}: {e}") + return False + +def download_file(url, dest_path, chunk_size=8 * 1024 * 1024): + """ + Downloads a file from the given URL to the specified destination path in chunks. + """ + try: + with urllib.request.urlopen(url) as response: + total_size = response.length if response.length else None + downloaded = 0 + try: + with open(dest_path, 'wb') as out_file: + while True: + chunk = response.read(chunk_size) + if not chunk: + break + out_file.write(chunk) + downloaded += len(chunk) + if total_size: + print(f"Downloaded {downloaded / (1024 * 1024):.2f}MB of {total_size / (1024 * 1024):.2f}MB", end='\r') + except PermissionError as pe: + print(f"Permission denied: {dest_path}") + raise + print(f"\nDownloaded file from {url} to {dest_path}") + except Exception as e: + print(f"Failed to download file: {e}") + raise + +def download_template_if_needed(template, url, filename, checksum): + dest_path = os.path.join(SYSTEMVM_TEMPLATES_PATH, filename) + if os.path.exists(dest_path): + if checksum and verify_sha512_checksum(dest_path, checksum): + print(f"{template} System VM template already exists at {dest_path} with valid checksum, skipping download.") + return + else: + print(f"{template} System VM template at {dest_path} has invalid or missing checksum, re-downloading...") + else: + print(f"Downloading {template} System VM template from {url} to {dest_path}...") + try: + download_file(url, dest_path) + except Exception as e: + print(f"ERROR: Failed to download {template} System VM template: {e}") + +def collect_template_metadata(selected_templates, options): + template_metadata_list = [] + if not os.path.exists(SYSTEMVM_TEMPLATES_METADATA_FILE): + print(f"ERROR: System VM templates metadata file not found at {SYSTEMVM_TEMPLATES_METADATA_FILE}, cannot download templates.") + sys.exit(1) + config = configparser.ConfigParser() + config.read(SYSTEMVM_TEMPLATES_METADATA_FILE) + template_repo_url = None + if options.systemvm_templates_repository: + if "default" in config and "downloadrepository" in config["default"]: + template_repo_url = config["default"]["downloadrepository"].strip() + if not template_repo_url: + print("ERROR: downloadrepository value is empty in metadata file, cannot use --systemvm-template-repository option.") + sys.exit(1) + for template in selected_templates: + if template in config: + url = config[template].get("downloadurl") + filename = config[template].get("filename") + checksum = config[template].get("checksum") + if url and filename: + if template_repo_url: + url = url.replace(template_repo_url, options.systemvm_templates_repository) + template_metadata_list.append({ + "template": template, + "url": url, + "filename": filename, + "checksum": checksum + }) + else: + print(f"ERROR: URL or filename not found for {template} System VM template in metadata.") + sys.exit(1) + else: + print(f"ERROR: No metadata found for {template} System VM template.") + sys.exit(1) + return template_metadata_list if __name__ == '__main__': initLoging("@MSLOGDIR@/setupManagement.log") @@ -45,6 +139,16 @@ if __name__ == '__main__': parser.add_option("--https", action="store_true", dest="https", help="Enable HTTPs connection of management server") parser.add_option("--tomcat7", action="store_true", dest="tomcat7", help="Depreciated option, don't use it") parser.add_option("--no-start", action="store_true", dest="nostart", help="Do not start management server after successful configuration") + parser.add_option( + "--systemvm-templates", + dest="systemvm_templates", + help="Specify System VM templates to download: all, kvm-aarch64, kvm-x86_64, xenserver, vmware or comma-separated list of hypervisor combinations (e.g., kvm-x86_64,xenserver). Default is kvm-x86_64.", + ) + parser.add_option( + "--systemvm-templates-repository", + dest="systemvm_templates_repository", + help="Specify the URL to download System VM templates from." + ) (options, args) = parser.parse_args() if options.https: glbEnv.svrMode = "HttpsServer" @@ -53,6 +157,22 @@ if __name__ == '__main__': if options.nostart: glbEnv.noStart = True + available_templates = ["kvm-aarch64", "kvm-x86_64", "xenserver", "vmware"] + templates_arg = options.systemvm_templates + + selected_templates = ["kvm-x86_64"] + if templates_arg: + templates_list = [t.strip().lower() for t in templates_arg.split(",")] + if "all" in templates_list: + selected_templates = available_templates + else: + selected_templates = [t for t in templates_list if t in available_templates] + print(f"Selected systemvm templates to download: {', '.join(selected_templates) if selected_templates else 'None'}") + + template_metadata_list = [] + if selected_templates: + template_metadata_list = collect_template_metadata(selected_templates, options) + glbEnv.mode = "Server" print("Starting to configure CloudStack Management Server:") @@ -68,9 +188,13 @@ if __name__ == '__main__': print("CloudStack Management Server setup is Done!") print("Please ensure ports 8080, 8250, 8443, and 9090 are opened and not firewalled for the management server and not in use by other processes on this host.") except (CloudRuntimeException, CloudInternalException) as e: + print(e) print("Try to restore your system:") try: syscfg.restore() except: pass + + for meta in template_metadata_list: + download_template_if_needed(meta["template"], meta["url"], meta["filename"], meta["checksum"]) diff --git a/client/conf/server.properties.in b/client/conf/server.properties.in index 5958486b4dff..56d9925ab923 100644 --- a/client/conf/server.properties.in +++ b/client/conf/server.properties.in @@ -62,3 +62,8 @@ extensions.deployment.mode=@EXTENSIONSDEPLOYMENTMODE@ # Thread pool configuration #threads.min=10 #threads.max=500 + +# The URL prefix for the system VM templates repository. When downloading system VM templates, the server replaces the +# `downloadrepository` key value from the metadata file in template URLs. If not specified, the original template URL +# will be for download. +# system.vm.templates.download.repository=http://download.cloudstack.org/systemvm/4.20/ diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index 9b1420f22c37..593478c3417d 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -108,7 +108,10 @@ public class SystemVmTemplateRegistration { private static Integer LINUX_12_ID = 363; private static final Integer SCRIPT_TIMEOUT = 1800000; private static final Integer LOCK_WAIT_TIMEOUT = 1200; + protected static final String TEMPLATES_DOWNLOAD_REPOSITORY_KEY = "downloadurl"; + protected static final String TEMPLATES_CUSTOM_DOWNLOAD_REPOSITORY_KEY = "system.vm.templates.download.repository"; protected static final List DOWNLOADABLE_TEMPLATE_ARCH_TYPES = Arrays.asList( + CPU.CPUArch.amd64, CPU.CPUArch.arm64 ); @@ -820,6 +823,14 @@ public static String parseMetadataFile() { LOGGER.error(errMsg); throw new CloudRuntimeException(errMsg); } + Ini.Section defaultSection = ini.get("default"); + boolean updateCustomDownloadRepository = false; + String defaultDownloadRepository = defaultSection.get(TEMPLATES_DOWNLOAD_REPOSITORY_KEY); + String customDownloadRepository = System.getProperty(TEMPLATES_CUSTOM_DOWNLOAD_REPOSITORY_KEY); + if (StringUtils.isNotBlank(customDownloadRepository) && StringUtils.isNotBlank(defaultDownloadRepository)) { + LOGGER.debug("Updating custom download repository: {}", customDownloadRepository); + updateCustomDownloadRepository = true; + } for (Pair hypervisorType : hypervisorList) { String key = getHypervisorArchKey(hypervisorType.first(), hypervisorType.second()); Ini.Section section = ini.get(key); @@ -828,16 +839,21 @@ public static String parseMetadataFile() { key, metadataFilePath); continue; } + String url = section.get("downloadurl"); + if (StringUtils.isNotBlank(url) && updateCustomDownloadRepository) { + url = url.replaceFirst(defaultDownloadRepository.trim(), + customDownloadRepository.trim()); + LOGGER.info("Updated download URL for {} to {}", key, url); + } NewTemplateMap.put(key, new MetadataTemplateDetails( hypervisorType.first(), section.get("templatename"), section.get("filename"), - section.get("downloadurl"), + url, section.get("checksum"), hypervisorType.second(), section.get("guestos"))); } - Ini.Section defaultSection = ini.get("default"); return defaultSection.get("version").trim(); } diff --git a/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java b/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java index dceb8e07b07d..b9b0cbcf0011 100644 --- a/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java +++ b/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java @@ -344,7 +344,7 @@ public void testIsTemplateFileChecksumDifferent_mismatch() { @Test(expected = CloudRuntimeException.class) public void testValidateTemplates_metadataTemplateFailure() { List> list = new ArrayList<>(); - list.add(new Pair<>(Hypervisor.HypervisorType.KVM, CPU.CPUArch.amd64)); + list.add(new Pair<>(Hypervisor.HypervisorType.VMware, CPU.CPUArch.arm64)); systemVmTemplateRegistration.validateTemplates(list); } diff --git a/engine/schema/templateConfig.sh b/engine/schema/templateConfig.sh index d6d1809c24d4..b899624d5557 100644 --- a/engine/schema/templateConfig.sh +++ b/engine/schema/templateConfig.sh @@ -27,6 +27,7 @@ function getTemplateVersion() { export CS_VERSION="${subversion1}"."${subversion2}" export CS_MINOR_VERSION="${minorversion}" export VERSION="${CS_VERSION}.${CS_MINOR_VERSION}" + export CS_SYSTEMTEMPLATE_REPO="https://download.cloudstack.org/systemvm/${CS_VERSION}/" } function getGenericName() { @@ -63,7 +64,7 @@ function getChecksum() { function createMetadataFile() { local fileData=$(cat $SOURCEFILE) - echo -e "["default"]\nversion = $VERSION.${securityversion}\n" >> $METADATAFILE + echo -e "["default"]\nversion = $VERSION.${securityversion}\ndownloadrepository = $CS_SYSTEMTEMPLATE_REPO\n" >> $METADATAFILE for template in "${templates[@]}" do section="${template%%:*}" @@ -82,13 +83,21 @@ function createMetadataFile() { declare -a templates getTemplateVersion $1 -templates=( "kvm-x86_64:https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-x86_64-kvm.qcow2.bz2" - "kvm-aarch64:https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-aarch64-kvm.qcow2.bz2" - "vmware:https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-x86_64-vmware.ova" - "xenserver:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-xen.vhd.bz2" - "hyperv:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-hyperv.vhd.zip" - "lxc:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-kvm.qcow2.bz2" - "ovm3:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-ovm.raw.bz2" ) +declare -A template_specs=( + [kvm-x86_64]="x86_64-kvm.qcow2.bz2" + [kvm-aarch64]="aarch64-kvm.qcow2.bz2" + [vmware]="x86_64-vmware.ova" + [xenserver]="x86_64-xen.vhd.bz2" + [hyperv4]="x86_64-hyperv.vhd.zip" + [lxc]="x86_64-kvm.qcow2.bz2" + [ovm3]="x86_64-ovm.raw.bz2" +) + +templates=() +for key in "${!template_specs[@]}"; do + url="${CS_SYSTEMTEMPLATE_REPO}/systemvmtemplate-$VERSION-${template_specs[$key]}" + templates+=("$key:$url") +done PARENTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )/dist/systemvm-templates/" mkdir -p $PARENTPATH diff --git a/plugins/hypervisors/external/src/main/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisioner.java b/plugins/hypervisors/external/src/main/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisioner.java index 5a1632ce9777..b456c8a495a8 100644 --- a/plugins/hypervisors/external/src/main/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisioner.java +++ b/plugins/hypervisors/external/src/main/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisioner.java @@ -20,7 +20,6 @@ import java.io.BufferedReader; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; import java.nio.file.Files; @@ -51,6 +50,7 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.extension.Extension; import org.apache.cloudstack.framework.extensions.manager.ExtensionsManager; +import org.apache.cloudstack.utils.ServerPropertiesUtil; import org.apache.cloudstack.utils.security.DigestHelper; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; @@ -77,7 +77,6 @@ import com.cloud.serializer.GsonHelper; import com.cloud.utils.FileUtil; import com.cloud.utils.Pair; -import com.cloud.utils.PropertiesUtil; import com.cloud.utils.StringUtils; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.component.PluggableService; @@ -208,29 +207,6 @@ protected void createOrCheckExtensionsDataDirectory() throws ConfigurationExcept logger.info("Extensions data directory path: {}", extensionsDataDirectory); } - private String getServerProperty(String name) { - Properties props = propertiesRef.get(); - if (props == null) { - File propsFile = PropertiesUtil.findConfigFile(PROPERTIES_FILE); - if (propsFile == null) { - logger.error("{} file not found", PROPERTIES_FILE); - return null; - } - Properties tempProps = new Properties(); - try (FileInputStream is = new FileInputStream(propsFile)) { - tempProps.load(is); - } catch (IOException e) { - logger.error("Error loading {}: {}", PROPERTIES_FILE, e.getMessage(), e); - return null; - } - if (!propertiesRef.compareAndSet(null, tempProps)) { - tempProps = propertiesRef.get(); - } - props = tempProps; - } - return props.getProperty(name); - } - @Override public boolean configure(String name, Map params) throws ConfigurationException { super.configure(name, params); @@ -242,7 +218,7 @@ public boolean configure(String name, Map params) throws Configu } private void initializeExtensionDirectories() { - String deploymentMode = getServerProperty(EXTENSIONS_DEPLOYMENT_MODE_NAME); + String deploymentMode = ServerPropertiesUtil.getProperty(EXTENSIONS_DEPLOYMENT_MODE_NAME); if ("developer".equals(deploymentMode)) { extensionsDirectory = EXTENSIONS_DIRECTORY_DEV; extensionsDataDirectory = EXTENSIONS_DATA_DIRECTORY_DEV; diff --git a/utils/src/main/java/org/apache/cloudstack/utils/ServerPropertiesUtil.java b/utils/src/main/java/org/apache/cloudstack/utils/ServerPropertiesUtil.java new file mode 100644 index 000000000000..e1cb230cad11 --- /dev/null +++ b/utils/src/main/java/org/apache/cloudstack/utils/ServerPropertiesUtil.java @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.utils; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicReference; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.cloud.utils.PropertiesUtil; + +public class ServerPropertiesUtil { + private static final Logger logger = LoggerFactory.getLogger(ServerPropertiesUtil.class); + private static final String PROPERTIES_FILE = "server.properties"; + private static final AtomicReference propertiesRef = new AtomicReference<>(); + + public static String getProperty(String name) { + Properties props = propertiesRef.get(); + if (props != null) { + return props.getProperty(name); + } + File propsFile = PropertiesUtil.findConfigFile(PROPERTIES_FILE); + if (propsFile == null) { + logger.error("{} file not found", PROPERTIES_FILE); + return null; + } + Properties tempProps = new Properties(); + try (FileInputStream is = new FileInputStream(propsFile)) { + tempProps.load(is); + } catch (IOException e) { + logger.error("Error loading {}: {}", PROPERTIES_FILE, e.getMessage(), e); + return null; + } + if (!propertiesRef.compareAndSet(null, tempProps)) { + tempProps = propertiesRef.get(); + } + return tempProps.getProperty(name); + } +} From 9649e71262b0d3869117681d32dc06d358d96afd Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 9 Oct 2025 16:05:27 +0530 Subject: [PATCH 02/12] refactor Signed-off-by: Abhishek Kumar --- .../java/com/cloud/upgrade/SystemVmTemplateRegistration.java | 3 ++- .../external/provisioner/ExternalPathPayloadProvisioner.java | 2 +- .../cloudstack/utils/{ => server}/ServerPropertiesUtil.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) rename utils/src/main/java/org/apache/cloudstack/utils/{ => server}/ServerPropertiesUtil.java (98%) diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index 593478c3417d..9d3166de5167 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -51,6 +51,7 @@ import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.utils.security.DigestHelper; +import org.apache.cloudstack.utils.server.ServerPropertiesUtil; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -826,7 +827,7 @@ public static String parseMetadataFile() { Ini.Section defaultSection = ini.get("default"); boolean updateCustomDownloadRepository = false; String defaultDownloadRepository = defaultSection.get(TEMPLATES_DOWNLOAD_REPOSITORY_KEY); - String customDownloadRepository = System.getProperty(TEMPLATES_CUSTOM_DOWNLOAD_REPOSITORY_KEY); + String customDownloadRepository = ServerPropertiesUtil.getProperty(TEMPLATES_CUSTOM_DOWNLOAD_REPOSITORY_KEY); if (StringUtils.isNotBlank(customDownloadRepository) && StringUtils.isNotBlank(defaultDownloadRepository)) { LOGGER.debug("Updating custom download repository: {}", customDownloadRepository); updateCustomDownloadRepository = true; diff --git a/plugins/hypervisors/external/src/main/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisioner.java b/plugins/hypervisors/external/src/main/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisioner.java index 443e88230cd1..9c511b3bfd6a 100644 --- a/plugins/hypervisors/external/src/main/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisioner.java +++ b/plugins/hypervisors/external/src/main/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisioner.java @@ -50,7 +50,7 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.extension.Extension; import org.apache.cloudstack.framework.extensions.manager.ExtensionsManager; -import org.apache.cloudstack.utils.ServerPropertiesUtil; +import org.apache.cloudstack.utils.server.ServerPropertiesUtil; import org.apache.cloudstack.utils.security.DigestHelper; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; diff --git a/utils/src/main/java/org/apache/cloudstack/utils/ServerPropertiesUtil.java b/utils/src/main/java/org/apache/cloudstack/utils/server/ServerPropertiesUtil.java similarity index 98% rename from utils/src/main/java/org/apache/cloudstack/utils/ServerPropertiesUtil.java rename to utils/src/main/java/org/apache/cloudstack/utils/server/ServerPropertiesUtil.java index e1cb230cad11..af1fbc28605b 100644 --- a/utils/src/main/java/org/apache/cloudstack/utils/ServerPropertiesUtil.java +++ b/utils/src/main/java/org/apache/cloudstack/utils/server/ServerPropertiesUtil.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.cloudstack.utils; +package org.apache.cloudstack.utils.server; import java.io.File; import java.io.FileInputStream; From 94417df7bd135156feddc5b2d6d5d17cee4f99f3 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 13 Oct 2025 14:30:00 +0530 Subject: [PATCH 03/12] add tests, fix comment Signed-off-by: Abhishek Kumar --- client/bindir/cloud-setup-management.in | 1 - .../ExternalPathPayloadProvisioner.java | 28 +++++- .../utils/server/ServerPropertiesUtil.java | 4 +- .../server/ServerPropertiesUtilTest.java | 95 +++++++++++++++++++ 4 files changed, 123 insertions(+), 5 deletions(-) create mode 100644 utils/src/test/java/org/apache/cloudstack/utils/server/ServerPropertiesUtilTest.java diff --git a/client/bindir/cloud-setup-management.in b/client/bindir/cloud-setup-management.in index 6993f5dab793..6f790f02b444 100755 --- a/client/bindir/cloud-setup-management.in +++ b/client/bindir/cloud-setup-management.in @@ -188,7 +188,6 @@ if __name__ == '__main__': print("CloudStack Management Server setup is Done!") print("Please ensure ports 8080, 8250, 8443, and 9090 are opened and not firewalled for the management server and not in use by other processes on this host.") except (CloudRuntimeException, CloudInternalException) as e: - print(e) print("Try to restore your system:") try: diff --git a/plugins/hypervisors/external/src/main/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisioner.java b/plugins/hypervisors/external/src/main/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisioner.java index a6a334458768..92205b13c6ff 100644 --- a/plugins/hypervisors/external/src/main/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisioner.java +++ b/plugins/hypervisors/external/src/main/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisioner.java @@ -20,6 +20,7 @@ import java.io.BufferedReader; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; import java.nio.file.Files; @@ -50,7 +51,6 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.extension.Extension; import org.apache.cloudstack.framework.extensions.manager.ExtensionsManager; -import org.apache.cloudstack.utils.server.ServerPropertiesUtil; import org.apache.cloudstack.utils.security.DigestHelper; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; @@ -80,6 +80,7 @@ import com.cloud.serializer.GsonHelper; import com.cloud.utils.FileUtil; import com.cloud.utils.Pair; +import com.cloud.utils.PropertiesUtil; import com.cloud.utils.StringUtils; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.component.PluggableService; @@ -226,6 +227,29 @@ protected String getSanitizedJsonStringForLog(String json) { return json.replaceAll("(\"password\"\\s*:\\s*\")([^\"]*)(\")", "$1****$3"); } + private String getServerProperty(String name) { + Properties props = propertiesRef.get(); + if (props == null) { + File propsFile = PropertiesUtil.findConfigFile(PROPERTIES_FILE); + if (propsFile == null) { + logger.error("{} file not found", PROPERTIES_FILE); + return null; + } + Properties tempProps = new Properties(); + try (FileInputStream is = new FileInputStream(propsFile)) { + tempProps.load(is); + } catch (IOException e) { + logger.error("Error loading {}: {}", PROPERTIES_FILE, e.getMessage(), e); + return null; + } + if (!propertiesRef.compareAndSet(null, tempProps)) { + tempProps = propertiesRef.get(); + } + props = tempProps; + } + return props.getProperty(name); + } + @Override public boolean configure(String name, Map params) throws ConfigurationException { super.configure(name, params); @@ -237,7 +261,7 @@ public boolean configure(String name, Map params) throws Configu } private void initializeExtensionDirectories() { - String deploymentMode = ServerPropertiesUtil.getProperty(EXTENSIONS_DEPLOYMENT_MODE_NAME); + String deploymentMode = getServerProperty(EXTENSIONS_DEPLOYMENT_MODE_NAME); if ("developer".equals(deploymentMode)) { extensionsDirectory = EXTENSIONS_DIRECTORY_DEV; extensionsDataDirectory = EXTENSIONS_DATA_DIRECTORY_DEV; diff --git a/utils/src/main/java/org/apache/cloudstack/utils/server/ServerPropertiesUtil.java b/utils/src/main/java/org/apache/cloudstack/utils/server/ServerPropertiesUtil.java index af1fbc28605b..14d24dbb6410 100644 --- a/utils/src/main/java/org/apache/cloudstack/utils/server/ServerPropertiesUtil.java +++ b/utils/src/main/java/org/apache/cloudstack/utils/server/ServerPropertiesUtil.java @@ -30,8 +30,8 @@ public class ServerPropertiesUtil { private static final Logger logger = LoggerFactory.getLogger(ServerPropertiesUtil.class); - private static final String PROPERTIES_FILE = "server.properties"; - private static final AtomicReference propertiesRef = new AtomicReference<>(); + protected static final String PROPERTIES_FILE = "server.properties"; + protected static final AtomicReference propertiesRef = new AtomicReference<>(); public static String getProperty(String name) { Properties props = propertiesRef.get(); diff --git a/utils/src/test/java/org/apache/cloudstack/utils/server/ServerPropertiesUtilTest.java b/utils/src/test/java/org/apache/cloudstack/utils/server/ServerPropertiesUtilTest.java new file mode 100644 index 000000000000..2eece43e47b1 --- /dev/null +++ b/utils/src/test/java/org/apache/cloudstack/utils/server/ServerPropertiesUtilTest.java @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.utils.server; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Properties; + +import org.junit.After; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.MockedStatic; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.utils.PropertiesUtil; + +@RunWith(MockitoJUnitRunner.class) +public class ServerPropertiesUtilTest { + + @After + public void clearCache() { + ServerPropertiesUtil.propertiesRef.set(null); + } + + @Test + public void returnsPropertyValueWhenPropertiesAreLoaded() { + Properties mockProperties = mock(Properties.class); + when(mockProperties.getProperty("key")).thenReturn("value"); + ServerPropertiesUtil.propertiesRef.set(mockProperties); + String result = ServerPropertiesUtil.getProperty("key"); + assertEquals("value", result); + } + + @Test + public void returnsNullWhenPropertyDoesNotExist() { + Properties mockProperties = mock(Properties.class); + ServerPropertiesUtil.propertiesRef.set(mockProperties); + assertNull(ServerPropertiesUtil.getProperty("nonexistentKey")); + } + + @Test + public void loadsPropertiesFromFileWhenNotCached() throws Exception { + File tempFile = Files.createTempFile("server", ".properties").toFile(); + tempFile.deleteOnExit(); + Files.writeString(tempFile.toPath(), "key=value\n"); + try (MockedStatic mocked = mockStatic(PropertiesUtil.class)) { + mocked.when(() -> PropertiesUtil.findConfigFile(ServerPropertiesUtil.PROPERTIES_FILE)) + .thenReturn(tempFile); + assertEquals("value", ServerPropertiesUtil.getProperty("key")); + } + } + + @Test + public void returnsNullWhenPropertiesFileNotFound() { + try (MockedStatic mocked = mockStatic(PropertiesUtil.class)) { + mocked.when(() -> PropertiesUtil.findConfigFile(ServerPropertiesUtil.PROPERTIES_FILE)) + .thenReturn(null); + assertNull(ServerPropertiesUtil.getProperty("key")); + } + } + + @Test + public void returnsNullWhenIOExceptionOccurs() throws IOException { + File tempFile = Files.createTempFile("bad", ".properties").toFile(); + tempFile.deleteOnExit(); + Files.writeString(tempFile.toPath(), "\u0000\u0000\u0000"); + try (MockedStatic mocked = mockStatic(PropertiesUtil.class)) { + mocked.when(() -> PropertiesUtil.findConfigFile(ServerPropertiesUtil.PROPERTIES_FILE)) + .thenReturn(tempFile); + assertNull(ServerPropertiesUtil.getProperty("key")); + } + } +} From 2fd18bdd85e1c56586b96bcaa61f01ad34ffcbb7 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 27 Oct 2025 13:24:40 +0530 Subject: [PATCH 04/12] dont use fixed version Signed-off-by: Abhishek Kumar --- client/conf/server.properties.in | 2 +- engine/schema/templateConfig.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/conf/server.properties.in b/client/conf/server.properties.in index 56d9925ab923..10452c9b6572 100644 --- a/client/conf/server.properties.in +++ b/client/conf/server.properties.in @@ -66,4 +66,4 @@ extensions.deployment.mode=@EXTENSIONSDEPLOYMENTMODE@ # The URL prefix for the system VM templates repository. When downloading system VM templates, the server replaces the # `downloadrepository` key value from the metadata file in template URLs. If not specified, the original template URL # will be for download. -# system.vm.templates.download.repository=http://download.cloudstack.org/systemvm/4.20/ +# system.vm.templates.download.repository=http://download.cloudstack.org/systemvm/ diff --git a/engine/schema/templateConfig.sh b/engine/schema/templateConfig.sh index b899624d5557..b1178a01d09e 100755 --- a/engine/schema/templateConfig.sh +++ b/engine/schema/templateConfig.sh @@ -27,7 +27,7 @@ function getTemplateVersion() { export CS_VERSION="${subversion1}"."${subversion2}" export CS_MINOR_VERSION="${minorversion}" export VERSION="${CS_VERSION}.${CS_MINOR_VERSION}" - export CS_SYSTEMTEMPLATE_REPO="https://download.cloudstack.org/systemvm/${CS_VERSION}/" + export CS_SYSTEMTEMPLATE_REPO="https://download.cloudstack.org/systemvm/" } function getGenericName() { @@ -95,7 +95,7 @@ declare -A template_specs=( templates=() for key in "${!template_specs[@]}"; do - url="${CS_SYSTEMTEMPLATE_REPO}/systemvmtemplate-$VERSION-${template_specs[$key]}" + url="${CS_SYSTEMTEMPLATE_REPO}/${CS_VERSION}/systemvmtemplate-$VERSION-${template_specs[$key]}" templates+=("$key:$url") done From 29cd1843552db8aaf48ee3af9ed0778396973c12 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 28 Oct 2025 14:49:08 +0530 Subject: [PATCH 05/12] minor refactor Signed-off-by: Abhishek Kumar --- .../com/cloud/upgrade/SystemVmTemplateRegistration.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index 9d3166de5167..b6491db0f4aa 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -825,13 +825,10 @@ public static String parseMetadataFile() { throw new CloudRuntimeException(errMsg); } Ini.Section defaultSection = ini.get("default"); - boolean updateCustomDownloadRepository = false; String defaultDownloadRepository = defaultSection.get(TEMPLATES_DOWNLOAD_REPOSITORY_KEY); String customDownloadRepository = ServerPropertiesUtil.getProperty(TEMPLATES_CUSTOM_DOWNLOAD_REPOSITORY_KEY); - if (StringUtils.isNotBlank(customDownloadRepository) && StringUtils.isNotBlank(defaultDownloadRepository)) { - LOGGER.debug("Updating custom download repository: {}", customDownloadRepository); - updateCustomDownloadRepository = true; - } + boolean updateCustomDownloadRepository = StringUtils.isNotBlank(customDownloadRepository) && + StringUtils.isNotBlank(defaultDownloadRepository); for (Pair hypervisorType : hypervisorList) { String key = getHypervisorArchKey(hypervisorType.first(), hypervisorType.second()); Ini.Section section = ini.get(key); @@ -844,7 +841,7 @@ public static String parseMetadataFile() { if (StringUtils.isNotBlank(url) && updateCustomDownloadRepository) { url = url.replaceFirst(defaultDownloadRepository.trim(), customDownloadRepository.trim()); - LOGGER.info("Updated download URL for {} to {}", key, url); + LOGGER.debug("Updated download URL for {} using custom repository to {}", key, url); } NewTemplateMap.put(key, new MetadataTemplateDetails( hypervisorType.first(), From 6eac66d6271188dddc48d9a8a5e62f42ece15bef Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 10 Nov 2025 19:16:17 +0530 Subject: [PATCH 06/12] address review Signed-off-by: Abhishek Kumar --- client/bindir/cloud-setup-management.in | 20 ++++++++++++++++++- .../upgrade/SystemVmTemplateRegistration.java | 5 +++-- engine/schema/templateConfig.sh | 2 +- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/client/bindir/cloud-setup-management.in b/client/bindir/cloud-setup-management.in index 6f790f02b444..b4fe76cc8d8a 100755 --- a/client/bindir/cloud-setup-management.in +++ b/client/bindir/cloud-setup-management.in @@ -92,6 +92,12 @@ def download_template_if_needed(template, url, filename, checksum): print(f"Downloading {template} System VM template from {url} to {dest_path}...") try: download_file(url, dest_path) + #After download, verify checksum if provided + if checksum: + if verify_sha512_checksum(dest_path, checksum): + print(f"{template} System VM template downloaded and verified successfully.") + else: + print(f"ERROR: Checksum verification failed for {template} System VM template after download.") except Exception as e: print(f"ERROR: Failed to download {template} System VM template: {e}") @@ -164,9 +170,21 @@ if __name__ == '__main__': if templates_arg: templates_list = [t.strip().lower() for t in templates_arg.split(",")] if "all" in templates_list: + if len(templates_list) > 1: + print("WARNING: 'all' specified for System VM templates, ignoring other specified templates.") selected_templates = available_templates else: - selected_templates = [t for t in templates_list if t in available_templates] + invalid_templates = [] + for t in templates_list: + if t in available_templates: + if t not in selected_templates: + selected_templates.append(t) + else: + if t not in invalid_templates: + invalid_templates.append(t) + if invalid_templates: + print(f"ERROR: Invalid System VM template names provided: {', '.join(invalid_templates)}") + sys.exit(1) print(f"Selected systemvm templates to download: {', '.join(selected_templates) if selected_templates else 'None'}") template_metadata_list = [] diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index b6491db0f4aa..4a57b7bd820e 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -109,7 +109,8 @@ public class SystemVmTemplateRegistration { private static Integer LINUX_12_ID = 363; private static final Integer SCRIPT_TIMEOUT = 1800000; private static final Integer LOCK_WAIT_TIMEOUT = 1200; - protected static final String TEMPLATES_DOWNLOAD_REPOSITORY_KEY = "downloadurl"; + protected static final String TEMPLATE_DOWNLOAD_URL_KEY = "downloadurl"; + protected static final String TEMPLATES_DOWNLOAD_REPOSITORY_KEY = "downloadrepository"; protected static final String TEMPLATES_CUSTOM_DOWNLOAD_REPOSITORY_KEY = "system.vm.templates.download.repository"; protected static final List DOWNLOADABLE_TEMPLATE_ARCH_TYPES = Arrays.asList( CPU.CPUArch.amd64, @@ -837,7 +838,7 @@ public static String parseMetadataFile() { key, metadataFilePath); continue; } - String url = section.get("downloadurl"); + String url = section.get(TEMPLATE_DOWNLOAD_URL_KEY); if (StringUtils.isNotBlank(url) && updateCustomDownloadRepository) { url = url.replaceFirst(defaultDownloadRepository.trim(), customDownloadRepository.trim()); diff --git a/engine/schema/templateConfig.sh b/engine/schema/templateConfig.sh index b1178a01d09e..dcf3ecd382a1 100755 --- a/engine/schema/templateConfig.sh +++ b/engine/schema/templateConfig.sh @@ -88,7 +88,7 @@ declare -A template_specs=( [kvm-aarch64]="aarch64-kvm.qcow2.bz2" [vmware]="x86_64-vmware.ova" [xenserver]="x86_64-xen.vhd.bz2" - [hyperv4]="x86_64-hyperv.vhd.zip" + [hyperv]="x86_64-hyperv.vhd.zip" [lxc]="x86_64-kvm.qcow2.bz2" [ovm3]="x86_64-ovm.raw.bz2" ) From 4bd5458750cf15d2193114c9b414d0e76c115e6e Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 13 Nov 2025 10:00:35 +0530 Subject: [PATCH 07/12] Update client/conf/server.properties.in --- client/conf/server.properties.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/conf/server.properties.in b/client/conf/server.properties.in index 10452c9b6572..7c5e3f925b08 100644 --- a/client/conf/server.properties.in +++ b/client/conf/server.properties.in @@ -65,5 +65,5 @@ extensions.deployment.mode=@EXTENSIONSDEPLOYMENTMODE@ # The URL prefix for the system VM templates repository. When downloading system VM templates, the server replaces the # `downloadrepository` key value from the metadata file in template URLs. If not specified, the original template URL -# will be for download. +# will be used for download. # system.vm.templates.download.repository=http://download.cloudstack.org/systemvm/ From 8171d19094f70e9c0ca1f534675774a91973aef8 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 9 Dec 2025 14:25:28 +0530 Subject: [PATCH 08/12] improve check for already registered template Signed-off-by: Abhishek Kumar --- .../com/cloud/storage/dao/VMTemplateDao.java | 5 +- .../cloud/storage/dao/VMTemplateDaoImpl.java | 37 +++++++- .../upgrade/SystemVmTemplateRegistration.java | 69 +++++++------- .../storage/dao/VMTemplateDaoImplTest.java | 87 +++++++++++++++++- .../SystemVmTemplateRegistrationTest.java | 90 +++++++++++++++++-- .../com/cloud/storage/StorageManagerImpl.java | 11 +-- 6 files changed, 253 insertions(+), 46 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java index 6785c3653290..4c9f906b68a9 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java @@ -94,7 +94,7 @@ public interface VMTemplateDao extends GenericDao, StateDao< List listByParentTemplatetId(long parentTemplatetId); - VMTemplateVO findLatestTemplateByName(String name, CPU.CPUArch arch); + VMTemplateVO findLatestTemplateByName(String name, HypervisorType hypervisorType, CPU.CPUArch arch); List findTemplatesLinkedToUserdata(long userdataId); @@ -103,4 +103,7 @@ public interface VMTemplateDao extends GenericDao, StateDao< List listIdsByTemplateTag(String tag); List listIdsByExtensionId(long extensionId); + + VMTemplateVO findActiveSystemTemplateByHypervisorArchAndUrlPath(HypervisorType hypervisorType, + CPU.CPUArch arch, String urlPathSuffix); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java index 08b82cbb45bc..305dfcbcf936 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java @@ -246,13 +246,17 @@ public List listReadyTemplates() { @Override - public VMTemplateVO findLatestTemplateByName(String name, CPU.CPUArch arch) { + public VMTemplateVO findLatestTemplateByName(String name, HypervisorType hypervisorType, CPU.CPUArch arch) { SearchBuilder sb = createSearchBuilder(); sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ); + sb.and("hypervisorType", sb.entity().getHypervisorType(), SearchCriteria.Op.EQ); sb.and("arch", sb.entity().getArch(), SearchCriteria.Op.EQ); sb.done(); SearchCriteria sc = sb.create(); sc.setParameters("name", name); + if (hypervisorType != null) { + sc.setParameters("hypervisorType", hypervisorType); + } if (arch != null) { sc.setParameters("arch", arch); } @@ -857,6 +861,37 @@ public List listIdsByExtensionId(long extensionId) { return customSearch(sc, null); } + @Override + public VMTemplateVO findActiveSystemTemplateByHypervisorArchAndUrlPath(HypervisorType hypervisorType, + CPU.CPUArch arch, String urlPathSuffix) { + if (StringUtils.isBlank(urlPathSuffix)) { + return null; + } + SearchBuilder sb = createSearchBuilder(); + sb.and("templateType", sb.entity().getTemplateType(), SearchCriteria.Op.EQ); + sb.and("hypervisorType", sb.entity().getHypervisorType(), SearchCriteria.Op.EQ); + sb.and("arch", sb.entity().getArch(), SearchCriteria.Op.EQ); + sb.and("urlPathSuffix", sb.entity().getUrl(), SearchCriteria.Op.LIKE); + sb.and("state", sb.entity().getState(), SearchCriteria.Op.EQ); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("templateType", TemplateType.SYSTEM); + if (hypervisorType != null) { + sc.setParameters("hypervisorType", hypervisorType); + } + if (arch != null) { + sc.setParameters("arch", arch); + } + sc.setParameters("urlPathSuffix", "%" + urlPathSuffix); + sc.setParameters("state", VirtualMachineTemplate.State.Active); + Filter filter = new Filter(VMTemplateVO.class, "id", false, null, 1L); + List templates = listBy(sc, filter); + if (CollectionUtils.isNotEmpty(templates)) { + return templates.get(0); + } + return null; + } + @Override public boolean updateState( com.cloud.template.VirtualMachineTemplate.State currentState, diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index 4a57b7bd820e..7d307e0acd23 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -417,8 +417,15 @@ protected static MetadataTemplateDetails getMetadataTemplateDetails(Hypervisor.H return NewTemplateMap.get(getHypervisorArchKey(hypervisorType, arch)); } - public VMTemplateVO getRegisteredTemplate(String templateName, CPU.CPUArch arch) { - return vmTemplateDao.findLatestTemplateByName(templateName, arch); + public VMTemplateVO getRegisteredTemplate(String templateName, Hypervisor.HypervisorType hypervisorType, + CPU.CPUArch arch, String url) { + VMTemplateVO registeredTemplate = vmTemplateDao.findLatestTemplateByName(templateName, hypervisorType, arch); + if (registeredTemplate == null && StringUtils.isNotBlank(url)) { + String urlPath = url.substring(url.lastIndexOf("/") + 1); + registeredTemplate = vmTemplateDao.findActiveSystemTemplateByHypervisorArchAndUrlPath(hypervisorType, arch, + urlPath); + } + return registeredTemplate; } private static boolean isRunningInTest() { @@ -940,7 +947,8 @@ protected void registerTemplatesForZone(long zoneId, String filePath) { if (templateDetails == null) { continue; } - VMTemplateVO templateVO = getRegisteredTemplate(templateDetails.getName(), templateDetails.getArch()); + VMTemplateVO templateVO = getRegisteredTemplate(templateDetails.getName(), + templateDetails.getHypervisorType(), templateDetails.getArch(), templateDetails.getUrl()); if (templateVO != null) { TemplateDataStoreVO templateDataStoreVO = templateDataStoreDao.findByStoreTemplate(storeUrlAndId.second(), templateVO.getId()); @@ -999,7 +1007,7 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { } } - private void updateRegisteredTemplateDetails(Long templateId, MetadataTemplateDetails templateDetails) { + protected void updateRegisteredTemplateDetails(Long templateId, MetadataTemplateDetails templateDetails) { VMTemplateVO templateVO = vmTemplateDao.findById(templateId); templateVO.setTemplateType(Storage.TemplateType.SYSTEM); GuestOSVO guestOS = guestOSDao.findOneByDisplayName(templateDetails.getGuestOs()); @@ -1039,37 +1047,36 @@ private void updateTemplateUrlChecksumAndGuestOsId(VMTemplateVO templateVO, Meta protected boolean registerOrUpdateSystemVmTemplate(MetadataTemplateDetails templateDetails, List> hypervisorsInUse) { LOGGER.debug("Updating System VM template for {}", templateDetails.getHypervisorArchLog()); - VMTemplateVO registeredTemplate = getRegisteredTemplate(templateDetails.getName(), templateDetails.getArch()); - // change template type to SYSTEM + VMTemplateVO registeredTemplate = getRegisteredTemplate(templateDetails.getName(), + templateDetails.getHypervisorType(), templateDetails.getArch(), templateDetails.getUrl()); if (registeredTemplate != null) { updateRegisteredTemplateDetails(registeredTemplate.getId(), templateDetails); - } else { - boolean isHypervisorArchMatchMetadata = hypervisorsInUse.stream() - .anyMatch(p -> p.first().equals(templateDetails.getHypervisorType()) - && Objects.equals(p.second(), templateDetails.getArch())); - if (isHypervisorArchMatchMetadata) { - try { - registerTemplates(hypervisorsInUse); - return true; - } catch (final Exception e) { - throw new CloudRuntimeException(String.format("Failed to register %s templates for hypervisors: [%s]. " + - "Cannot upgrade system VMs", - getSystemVmTemplateVersion(), - StringUtils.join(hypervisorsInUse.stream() - .map(x -> getHypervisorArchKey(x.first(), x.second())) - .collect(Collectors.toList()), ",")), e); - } - } else { - LOGGER.warn("Cannot upgrade {} system VM template for {} as it is not used, not failing upgrade", - getSystemVmTemplateVersion(), templateDetails.getHypervisorArchLog()); - VMTemplateVO templateVO = vmTemplateDao.findLatestTemplateByTypeAndHypervisorAndArch( - templateDetails.getHypervisorType(), templateDetails.getArch(), Storage.TemplateType.SYSTEM); - if (templateVO != null) { - updateTemplateUrlChecksumAndGuestOsId(templateVO, templateDetails); - } + return false; + } + boolean isHypervisorArchMatchMetadata = hypervisorsInUse.stream() + .anyMatch(p -> p.first().equals(templateDetails.getHypervisorType()) + && Objects.equals(p.second(), templateDetails.getArch())); + if (!isHypervisorArchMatchMetadata) { + LOGGER.warn("Skipping upgrading {} system VM template for {} as it is not used, not failing upgrade", + getSystemVmTemplateVersion(), templateDetails.getHypervisorArchLog()); + VMTemplateVO templateVO = vmTemplateDao.findLatestTemplateByTypeAndHypervisorAndArch( + templateDetails.getHypervisorType(), templateDetails.getArch(), Storage.TemplateType.SYSTEM); + if (templateVO != null) { + updateTemplateUrlChecksumAndGuestOsId(templateVO, templateDetails); } + return false; + } + try { + registerTemplates(hypervisorsInUse); + return true; + } catch (final Exception e) { + throw new CloudRuntimeException(String.format("Failed to register %s templates for hypervisors: [%s]. " + + "Cannot upgrade system VMs", + getSystemVmTemplateVersion(), + StringUtils.join(hypervisorsInUse.stream() + .map(x -> getHypervisorArchKey(x.first(), x.second())) + .collect(Collectors.toList()), ",")), e); } - return false; } public void updateSystemVmTemplates(final Connection conn) { diff --git a/engine/schema/src/test/java/com/cloud/storage/dao/VMTemplateDaoImplTest.java b/engine/schema/src/test/java/com/cloud/storage/dao/VMTemplateDaoImplTest.java index 3c8e4c046ae7..5cff77869be8 100644 --- a/engine/schema/src/test/java/com/cloud/storage/dao/VMTemplateDaoImplTest.java +++ b/engine/schema/src/test/java/com/cloud/storage/dao/VMTemplateDaoImplTest.java @@ -76,7 +76,8 @@ public void testFindLatestTemplateByName_ReturnsTemplate() { VMTemplateVO expectedTemplate = new VMTemplateVO(); List returnedList = Collections.singletonList(expectedTemplate); doReturn(returnedList).when(templateDao).listBy(any(SearchCriteria.class), any(Filter.class)); - VMTemplateVO result = templateDao.findLatestTemplateByName("test", CPU.CPUArch.getDefault()); + VMTemplateVO result = templateDao.findLatestTemplateByName("test", Hypervisor.HypervisorType.KVM, + CPU.CPUArch.getDefault()); assertNotNull("Expected a non-null template", result); assertEquals("Expected the returned template to be the first element", expectedTemplate, result); } @@ -85,7 +86,8 @@ public void testFindLatestTemplateByName_ReturnsTemplate() { public void testFindLatestTemplateByName_ReturnsNullWhenNoTemplateFound() { List emptyList = Collections.emptyList(); doReturn(emptyList).when(templateDao).listBy(any(SearchCriteria.class), any(Filter.class)); - VMTemplateVO result = templateDao.findLatestTemplateByName("test", CPU.CPUArch.getDefault()); + VMTemplateVO result = templateDao.findLatestTemplateByName("test", Hypervisor.HypervisorType.VMware, + CPU.CPUArch.getDefault()); assertNull("Expected null when no templates are found", result); } @@ -94,7 +96,8 @@ public void testFindLatestTemplateByName_NullArch() { VMTemplateVO expectedTemplate = new VMTemplateVO(); List returnedList = Collections.singletonList(expectedTemplate); doReturn(returnedList).when(templateDao).listBy(any(SearchCriteria.class), any(Filter.class)); - VMTemplateVO result = templateDao.findLatestTemplateByName("test", null); + VMTemplateVO result = templateDao.findLatestTemplateByName("test", Hypervisor.HypervisorType.XenServer, + null); assertNotNull("Expected a non-null template even if arch is null", result); assertEquals("Expected the returned template to be the first element", expectedTemplate, result); } @@ -337,4 +340,82 @@ public void testFindSystemVMReadyTemplate() { VMTemplateVO readyTemplate = templateDao.findSystemVMReadyTemplate(zoneId, Hypervisor.HypervisorType.KVM, CPU.CPUArch.arm64.getType()); Assert.assertEquals(CPU.CPUArch.arm64, readyTemplate.getArch()); } + + @Test + public void findActiveSystemTemplateByHypervisorArchAndUrlPath_ReturnsTemplate() { + VMTemplateVO expectedTemplate = mock(VMTemplateVO.class); + SearchBuilder sb = mock(SearchBuilder.class); + when(sb.entity()).thenReturn(expectedTemplate); + SearchCriteriasc = mock(SearchCriteria.class); + when(sb.create()).thenReturn(sc); + when(templateDao.createSearchBuilder()).thenReturn(sb); + List templates = Collections.singletonList(expectedTemplate); + doReturn(templates).when(templateDao).listBy(any(SearchCriteria.class), any(Filter.class)); + + VMTemplateVO result = templateDao.findActiveSystemTemplateByHypervisorArchAndUrlPath( + Hypervisor.HypervisorType.KVM, CPU.CPUArch.amd64, "testPath"); + + assertNotNull(result); + assertEquals(expectedTemplate, result); + } + + @Test + public void findActiveSystemTemplateByHypervisorArchAndUrlPath_ReturnsNullWhenNoTemplatesFound() { + VMTemplateVO template = mock(VMTemplateVO.class); + SearchBuilder sb = mock(SearchBuilder.class); + when(sb.entity()).thenReturn(template); + SearchCriteriasc = mock(SearchCriteria.class); + when(sb.create()).thenReturn(sc); + when(templateDao.createSearchBuilder()).thenReturn(sb); + doReturn(Collections.emptyList()).when(templateDao).listBy(any(SearchCriteria.class), any(Filter.class)); + + VMTemplateVO result = templateDao.findActiveSystemTemplateByHypervisorArchAndUrlPath( + Hypervisor.HypervisorType.KVM, CPU.CPUArch.amd64, "testPath"); + + assertNull(result); + } + + @Test + public void findActiveSystemTemplateByHypervisorArchAndUrlPath_NullHypervisor() { + VMTemplateVO expectedTemplate = mock(VMTemplateVO.class); + SearchBuilder sb = mock(SearchBuilder.class); + when(sb.entity()).thenReturn(expectedTemplate); + SearchCriteriasc = mock(SearchCriteria.class); + when(sb.create()).thenReturn(sc); + when(templateDao.createSearchBuilder()).thenReturn(sb); + List templates = Collections.singletonList(expectedTemplate); + doReturn(templates).when(templateDao).listBy(any(SearchCriteria.class), any(Filter.class)); + + VMTemplateVO result = templateDao.findActiveSystemTemplateByHypervisorArchAndUrlPath( + null, CPU.CPUArch.amd64, "testPath"); + + assertNotNull(result); + assertEquals(expectedTemplate, result); + } + + @Test + public void findActiveSystemTemplateByHypervisorArchAndUrlPath_NullArch() { + VMTemplateVO expectedTemplate = mock(VMTemplateVO.class); + SearchBuilder sb = mock(SearchBuilder.class); + when(sb.entity()).thenReturn(expectedTemplate); + SearchCriteriasc = mock(SearchCriteria.class); + when(sb.create()).thenReturn(sc); + when(templateDao.createSearchBuilder()).thenReturn(sb); + List templates = Collections.singletonList(expectedTemplate); + doReturn(templates).when(templateDao).listBy(any(SearchCriteria.class), any(Filter.class)); + + VMTemplateVO result = templateDao.findActiveSystemTemplateByHypervisorArchAndUrlPath( + Hypervisor.HypervisorType.KVM, null, "testPath"); + + assertNotNull(result); + assertEquals(expectedTemplate, result); + } + + @Test + public void findActiveSystemTemplateByHypervisorArchAndUrlPath_EmptyUrlPathSuffix() { + VMTemplateVO result = templateDao.findActiveSystemTemplateByHypervisorArchAndUrlPath( + Hypervisor.HypervisorType.KVM, CPU.CPUArch.amd64, ""); + + assertNull(result); + } } diff --git a/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java b/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java index b9b0cbcf0011..030276e1765a 100644 --- a/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java +++ b/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java @@ -25,10 +25,13 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -55,6 +58,7 @@ import com.cloud.cpu.CPU; import com.cloud.dc.dao.ClusterDao; import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.Storage; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.utils.HttpUtils; @@ -363,6 +367,7 @@ public void testValidateTemplates_fileFailure() { systemVmTemplateRegistration.validateTemplates(list); } + @Test public void testValidateTemplates_downloadableFileNotFound() { CPU.CPUArch arch = SystemVmTemplateRegistration.DOWNLOADABLE_TEMPLATE_ARCH_TYPES.get(0); List> list = new ArrayList<>(); @@ -396,6 +401,8 @@ public void testRegisterTemplatesForZone() { String filePath = "dummyFilePath"; String nfsVersion = "nfs3"; Pair storeUrlAndId = new Pair<>("nfs://dummy", 100L); + String name = "existing"; + String url = "url"; doReturn(storeUrlAndId).when(systemVmTemplateRegistration).getNfsStoreInZone(zoneId); doReturn(nfsVersion).when(systemVmTemplateRegistration).getNfsVersion(storeUrlAndId.second()); try (MockedStatic mockedStatic = Mockito.mockStatic( @@ -407,13 +414,11 @@ public void testRegisterTemplatesForZone() { doReturn(hypervisorArchList).when(clusterDao).listDistinctHypervisorsAndArchExcludingExternalType(zoneId); SystemVmTemplateRegistration.MetadataTemplateDetails details = Mockito.mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class); - String name = "existing"; - Mockito.when(details.getArch()).thenReturn(CPU.CPUArch.getDefault()); - Mockito.when(details.getName()).thenReturn(name); + when(details.getArch()).thenReturn(CPU.CPUArch.getDefault()); + when(details.getName()).thenReturn(name); + when(details.getUrl()).thenReturn(url); mockedStatic.when(() -> SystemVmTemplateRegistration.getMetadataTemplateDetails(Mockito.any(), Mockito.any())).thenReturn(details); - when(systemVmTemplateRegistration.getRegisteredTemplate(name, arch)) - .thenReturn(null); doNothing().when(systemVmTemplateRegistration).registerTemplateForNonExistingEntries( hypervisorType, arch, name, storeUrlAndId, filePath); @@ -424,4 +429,79 @@ public void testRegisterTemplatesForZone() { arch, name, storeUrlAndId, filePath); } } + + @Test + public void registerOrUpdateSystemVmTemplate_UpdatesRegisteredTemplate() { + SystemVmTemplateRegistration.MetadataTemplateDetails templateDetails = Mockito.mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class); + when(templateDetails.getName()).thenReturn("templateName"); + when(templateDetails.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + when(templateDetails.getArch()).thenReturn(CPU.CPUArch.amd64); + when(templateDetails.getUrl()).thenReturn("http://example.com/template"); + VMTemplateVO registeredTemplate = Mockito.mock(VMTemplateVO.class); + when(registeredTemplate.getId()).thenReturn(1L); + doReturn(registeredTemplate).when(systemVmTemplateRegistration).getRegisteredTemplate( + "templateName", Hypervisor.HypervisorType.KVM, CPU.CPUArch.amd64, "http://example.com/template"); + doNothing().when(systemVmTemplateRegistration).updateRegisteredTemplateDetails(1L, templateDetails); + + boolean result = systemVmTemplateRegistration.registerOrUpdateSystemVmTemplate(templateDetails, new ArrayList<>()); + + assertFalse(result); + verify(systemVmTemplateRegistration).updateRegisteredTemplateDetails(eq(1L), eq(templateDetails)); + } + + @Test + public void registerOrUpdateSystemVmTemplate_SkipsUnusedHypervisorArch() { + SystemVmTemplateRegistration.MetadataTemplateDetails templateDetails = Mockito.mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class); + when(templateDetails.getName()).thenReturn("templateName"); + when(templateDetails.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + when(templateDetails.getArch()).thenReturn(CPU.CPUArch.amd64); + when(templateDetails.getUrl()).thenReturn("http://example.com/template"); + doReturn(null).when(systemVmTemplateRegistration).getRegisteredTemplate( + "templateName", Hypervisor.HypervisorType.KVM, CPU.CPUArch.amd64, "http://example.com/template"); + doReturn(null).when(vmTemplateDao).findLatestTemplateByTypeAndHypervisorAndArch( + Hypervisor.HypervisorType.KVM, CPU.CPUArch.amd64, Storage.TemplateType.SYSTEM); + + boolean result = systemVmTemplateRegistration.registerOrUpdateSystemVmTemplate(templateDetails, new ArrayList<>()); + + assertFalse(result); + verify(systemVmTemplateRegistration, never()).registerTemplates(anyList()); + } + + @Test + public void registerOrUpdateSystemVmTemplate_RegistersNewTemplate() { + SystemVmTemplateRegistration.MetadataTemplateDetails templateDetails = Mockito.mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class); + when(templateDetails.getName()).thenReturn("templateName"); + when(templateDetails.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + when(templateDetails.getArch()).thenReturn(CPU.CPUArch.amd64); + when(templateDetails.getUrl()).thenReturn("http://example.com/template"); + doReturn(null).when(systemVmTemplateRegistration).getRegisteredTemplate( + "templateName", Hypervisor.HypervisorType.KVM, CPU.CPUArch.amd64, "http://example.com/template"); + List> hypervisorsInUse = new ArrayList<>(); + hypervisorsInUse.add(new Pair<>(Hypervisor.HypervisorType.KVM, CPU.CPUArch.amd64)); + doNothing().when(systemVmTemplateRegistration).registerTemplates(hypervisorsInUse); + + boolean result = systemVmTemplateRegistration.registerOrUpdateSystemVmTemplate(templateDetails, hypervisorsInUse); + + assertTrue(result); + verify(systemVmTemplateRegistration).registerTemplates(eq(hypervisorsInUse)); + } + + @Test + public void registerOrUpdateSystemVmTemplate_ThrowsExceptionOnRegistrationFailure() { + SystemVmTemplateRegistration.MetadataTemplateDetails templateDetails = Mockito.mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class); + when(templateDetails.getName()).thenReturn("templateName"); + when(templateDetails.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + when(templateDetails.getArch()).thenReturn(CPU.CPUArch.amd64); + when(templateDetails.getUrl()).thenReturn("http://example.com/template"); + doReturn(null).when(systemVmTemplateRegistration).getRegisteredTemplate( + "templateName", Hypervisor.HypervisorType.KVM, CPU.CPUArch.amd64, "http://example.com/template"); + List> hypervisorsInUse = new ArrayList<>(); + hypervisorsInUse.add(new Pair<>(Hypervisor.HypervisorType.KVM, CPU.CPUArch.amd64)); + doThrow(new CloudRuntimeException("Registration failed")).when(systemVmTemplateRegistration).registerTemplates(hypervisorsInUse); + + CloudRuntimeException exception = assertThrows(CloudRuntimeException.class, + () -> systemVmTemplateRegistration.registerOrUpdateSystemVmTemplate(templateDetails, hypervisorsInUse)); + + assertTrue(exception.getMessage().contains("Failed to register")); + } } diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index df0283ae2d61..45071cc5cb0e 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -56,10 +56,6 @@ import javax.inject.Inject; -import com.cloud.dc.HostPodVO; -import com.cloud.dc.dao.HostPodDao; -import com.cloud.resource.ResourceManager; -import com.cloud.storage.dao.StoragePoolAndAccessGroupMapDao; import org.apache.cloudstack.annotation.AnnotationService; import org.apache.cloudstack.annotation.dao.AnnotationDao; import org.apache.cloudstack.api.ApiConstants; @@ -189,9 +185,11 @@ import com.cloud.cpu.CPU; import com.cloud.dc.ClusterVO; import com.cloud.dc.DataCenterVO; +import com.cloud.dc.HostPodVO; import com.cloud.dc.VsphereStoragePolicyVO; import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.DataCenterDao; +import com.cloud.dc.dao.HostPodDao; import com.cloud.dc.dao.VsphereStoragePolicyDao; import com.cloud.event.ActionEvent; import com.cloud.event.EventTypes; @@ -218,6 +216,7 @@ import com.cloud.offering.ServiceOffering; import com.cloud.org.Grouping; import com.cloud.org.Grouping.AllocationState; +import com.cloud.resource.ResourceManager; import com.cloud.resource.ResourceState; import com.cloud.server.ConfigurationServer; import com.cloud.server.ManagementServer; @@ -230,6 +229,7 @@ import com.cloud.storage.dao.BucketDao; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.SnapshotDao; +import com.cloud.storage.dao.StoragePoolAndAccessGroupMapDao; import com.cloud.storage.dao.StoragePoolHostDao; import com.cloud.storage.dao.StoragePoolTagsDao; import com.cloud.storage.dao.StoragePoolWorkDao; @@ -4008,7 +4008,8 @@ protected void registerSystemVmTemplateForHypervisorArch(final HypervisorType hy return; } String templateName = getValidTemplateName(zoneId, hypervisorType); - VMTemplateVO registeredTemplate = systemVmTemplateRegistration.getRegisteredTemplate(templateName, arch); + VMTemplateVO registeredTemplate = systemVmTemplateRegistration.getRegisteredTemplate(templateName, + hypervisorType, arch, url); TemplateDataStoreVO templateDataStoreVO = null; if (registeredTemplate != null) { templateDataStoreVO = _templateStoreDao.findByStoreTemplate(store.getId(), registeredTemplate.getId()); From 8eab6fda6bdc10636ead0e397d0fac8d1dea9baf Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 10 Dec 2025 20:10:06 +0530 Subject: [PATCH 09/12] improve logs Signed-off-by: Abhishek Kumar --- .../upgrade/SystemVmTemplateRegistration.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index 7d307e0acd23..1c0dc2126cc9 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -422,9 +422,13 @@ public VMTemplateVO getRegisteredTemplate(String templateName, Hypervisor.Hyperv VMTemplateVO registeredTemplate = vmTemplateDao.findLatestTemplateByName(templateName, hypervisorType, arch); if (registeredTemplate == null && StringUtils.isNotBlank(url)) { String urlPath = url.substring(url.lastIndexOf("/") + 1); + LOGGER.debug("No template found by name, falling back to search existing SYSTEM template by " + + "urlPath: {}, hypervisor: {}, arch:{}", urlPath, hypervisorType, arch); registeredTemplate = vmTemplateDao.findActiveSystemTemplateByHypervisorArchAndUrlPath(hypervisorType, arch, urlPath); } + LOGGER.debug("Found existing registered template for hypervisor: {}, arch: {}: {}", hypervisorType, + arch, registeredTemplate); return registeredTemplate; } @@ -1046,10 +1050,15 @@ private void updateTemplateUrlChecksumAndGuestOsId(VMTemplateVO templateVO, Meta protected boolean registerOrUpdateSystemVmTemplate(MetadataTemplateDetails templateDetails, List> hypervisorsInUse) { - LOGGER.debug("Updating System VM template for {}", templateDetails.getHypervisorArchLog()); + String systemVmTemplateLog = String.format("%s system VM template for %s", getSystemVmTemplateVersion(), + templateDetails.getHypervisorArchLog()); + LOGGER.debug("Registering or updating {}", systemVmTemplateLog, + templateDetails.getHypervisorArchLog()); VMTemplateVO registeredTemplate = getRegisteredTemplate(templateDetails.getName(), templateDetails.getHypervisorType(), templateDetails.getArch(), templateDetails.getUrl()); if (registeredTemplate != null) { + LOGGER.info("{} is already registered, updating details for: {}", + systemVmTemplateLog, templateDetails.getHypervisorArchLog(), registeredTemplate); updateRegisteredTemplateDetails(registeredTemplate.getId(), templateDetails); return false; } @@ -1057,7 +1066,7 @@ protected boolean registerOrUpdateSystemVmTemplate(MetadataTemplateDetails templ .anyMatch(p -> p.first().equals(templateDetails.getHypervisorType()) && Objects.equals(p.second(), templateDetails.getArch())); if (!isHypervisorArchMatchMetadata) { - LOGGER.warn("Skipping upgrading {} system VM template for {} as it is not used, not failing upgrade", + LOGGER.warn("Skipping upgrading {} as it is not used, not failing upgrade", getSystemVmTemplateVersion(), templateDetails.getHypervisorArchLog()); VMTemplateVO templateVO = vmTemplateDao.findLatestTemplateByTypeAndHypervisorAndArch( templateDetails.getHypervisorType(), templateDetails.getArch(), Storage.TemplateType.SYSTEM); @@ -1163,7 +1172,7 @@ public String getChecksum() { } public CPU.CPUArch getArch() { - return arch; + return arch != null ? arch : CPU.CPUArch.getDefault(); } public String getGuestOs() { From c74763c385af2bf576498167d210456dcdc0d0db Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 11 Dec 2025 22:27:40 +0530 Subject: [PATCH 10/12] fix and refactor Signed-off-by: Abhishek Kumar --- .../dc/dao/DataCenterDetailsDaoImpl.java | 3 +- .../upgrade/SystemVmTemplateRegistration.java | 871 ++++++++++-------- .../SystemVmTemplateRegistrationTest.java | 195 ++-- .../com/cloud/storage/StorageManagerImpl.java | 83 +- 4 files changed, 651 insertions(+), 501 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/DataCenterDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/dc/dao/DataCenterDetailsDaoImpl.java index bb03a96d02ee..aec54e20d989 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/DataCenterDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/DataCenterDetailsDaoImpl.java @@ -31,7 +31,8 @@ public class DataCenterDetailsDaoImpl extends ResourceDetailsDaoBase DetailSearch; - DataCenterDetailsDaoImpl() { + public DataCenterDetailsDaoImpl() { + super(); DetailSearch = createSearchBuilder(); DetailSearch.and("zoneId", DetailSearch.entity().getResourceId(), SearchCriteria.Op.EQ); DetailSearch.and("name", DetailSearch.entity().getName(), SearchCriteria.Op.EQ); diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index 1c0dc2126cc9..3597cba5e612 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -28,7 +28,6 @@ import java.sql.Date; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Locale; @@ -63,6 +62,8 @@ import com.cloud.dc.dao.ClusterDaoImpl; import com.cloud.dc.dao.DataCenterDao; import com.cloud.dc.dao.DataCenterDaoImpl; +import com.cloud.dc.dao.DataCenterDetailsDao; +import com.cloud.dc.dao.DataCenterDetailsDaoImpl; import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.DataStoreRole; import com.cloud.storage.GuestOSVO; @@ -116,6 +117,8 @@ public class SystemVmTemplateRegistration { CPU.CPUArch.amd64, CPU.CPUArch.arm64 ); + protected static final String MINIMUM_SYSTEM_VM_VERSION_KEY = "minreq.sysvmtemplate.version"; + protected static final String DEFAULT_SYSTEM_VM_GUEST_OS_NAME = "Debian GNU/Linux 12 (64-bit)"; public static String CS_MAJOR_VERSION = null; public static String CS_TINY_VERSION = null; @@ -139,7 +142,9 @@ public class SystemVmTemplateRegistration { @Inject ConfigurationDao configurationDao; @Inject - private GuestOSDao guestOSDao; + DataCenterDetailsDao dataCenterDetailsDao; + @Inject + GuestOSDao guestOSDao; private String systemVmTemplateVersion; @@ -147,6 +152,7 @@ public class SystemVmTemplateRegistration { public SystemVmTemplateRegistration() { dataCenterDao = new DataCenterDaoImpl(); + dataCenterDetailsDao = new DataCenterDetailsDaoImpl(); vmTemplateDao = new VMTemplateDaoImpl(); vmTemplateZoneDao = new VMTemplateZoneDaoImpl(); templateDataStoreDao = new BasicTemplateDataStoreDaoImpl(); @@ -167,26 +173,7 @@ public SystemVmTemplateRegistration(String systemVmTemplateVersion) { this.systemVmTemplateVersion = systemVmTemplateVersion; } - public static String getMountCommand(String nfsVersion, String device, String dir) { - String cmd = MOUNT_COMMAND_BASE; - if (StringUtils.isNotBlank(nfsVersion)) { - cmd = String.format("%s -o vers=%s", cmd, nfsVersion); - } - return String.format("%s %s %s", cmd, device, dir); - } - - public String getSystemVmTemplateVersion() { - if (StringUtils.isEmpty(systemVmTemplateVersion)) { - return String.format("%s.%s", CS_MAJOR_VERSION, CS_TINY_VERSION); - } - return systemVmTemplateVersion; - } - - public File getTempDownloadDir() { - return tempDownloadDir; - } - - private static class SystemVMTemplateDetails { + protected static class SystemVMTemplateDetails { Long id; String uuid; String name; @@ -317,19 +304,19 @@ public void setUpdated(Date updated) { } } - public static final List> hypervisorList = Arrays.asList( + protected static final List> AVAILABLE_SYSTEM_TEMPLATES_HYPERVISOR_ARCH_LIST = Arrays.asList( new Pair<>(Hypervisor.HypervisorType.KVM, CPU.CPUArch.amd64), new Pair<>(Hypervisor.HypervisorType.KVM, CPU.CPUArch.arm64), - new Pair<>(Hypervisor.HypervisorType.VMware, null), - new Pair<>(Hypervisor.HypervisorType.XenServer, null), - new Pair<>(Hypervisor.HypervisorType.Hyperv, null), - new Pair<>(Hypervisor.HypervisorType.LXC, null), - new Pair<>(Hypervisor.HypervisorType.Ovm3, null) + new Pair<>(Hypervisor.HypervisorType.VMware, CPU.CPUArch.amd64), + new Pair<>(Hypervisor.HypervisorType.XenServer, CPU.CPUArch.amd64), + new Pair<>(Hypervisor.HypervisorType.Hyperv, CPU.CPUArch.amd64), + new Pair<>(Hypervisor.HypervisorType.LXC, CPU.CPUArch.amd64), + new Pair<>(Hypervisor.HypervisorType.Ovm3, CPU.CPUArch.amd64) ); - public static final Map NewTemplateMap = new HashMap<>(); + protected static final List METADATA_TEMPLATE_LIST = new ArrayList<>(); - public static final Map RouterTemplateConfigurationNames = new HashMap<>() { + protected static final Map ROUTER_TEMPLATE_CONFIGURATION_NAMES = new HashMap<>() { { put(Hypervisor.HypervisorType.KVM, "router.template.kvm"); put(Hypervisor.HypervisorType.VMware, "router.template.vmware"); @@ -340,7 +327,7 @@ public void setUpdated(Date updated) { } }; - public static Map hypervisorGuestOsMap = new HashMap<>() { + protected static Map hypervisorGuestOsMap = new HashMap<>() { { put(Hypervisor.HypervisorType.KVM, LINUX_12_ID); put(Hypervisor.HypervisorType.XenServer, OTHER_LINUX_ID); @@ -351,7 +338,7 @@ public void setUpdated(Date updated) { } }; - public static final Map hypervisorImageFormat = new HashMap() { + protected static final Map hypervisorImageFormat = new HashMap() { { put(Hypervisor.HypervisorType.KVM, ImageFormat.QCOW2); put(Hypervisor.HypervisorType.XenServer, ImageFormat.VHD); @@ -362,80 +349,16 @@ public void setUpdated(Date updated) { } }; - public boolean validateIfSeeded(TemplateDataStoreVO templDataStoreVO, String url, String path, String nfsVersion) { - String filePath = null; - try { - filePath = Files.createTempDirectory(TEMPORARY_SECONDARY_STORE).toString(); - if (filePath == null) { - throw new CloudRuntimeException("Failed to create temporary directory to mount secondary store"); - } - mountStore(url, filePath, nfsVersion); - int lastIdx = path.lastIndexOf(File.separator); - String partialDirPath = path.substring(0, lastIdx); - String templatePath = filePath + File.separator + partialDirPath; - File templateProps = new File(templatePath + "/template.properties"); - if (templateProps.exists()) { - Pair templateSizes = readTemplatePropertiesSizes(templatePath + "/template.properties"); - updateSeededTemplateDetails(templDataStoreVO.getTemplateId(), templDataStoreVO.getDataStoreId(), - templateSizes.first(), templateSizes.second()); - LOGGER.info("SystemVM template already seeded, skipping registration"); - return true; - } - LOGGER.info("SystemVM template not seeded"); - return false; - } catch (Exception e) { - LOGGER.error("Failed to verify if the template is seeded", e); - throw new CloudRuntimeException("Failed to verify if the template is seeded", e); - } finally { - unmountStore(filePath); - try { - Files.delete(Path.of(filePath)); - } catch (IOException e) { - LOGGER.error(String.format("Failed to delete temporary directory: %s", filePath)); - } - } + private static boolean isRunningInTest() { + return "true".equalsIgnoreCase(System.getProperty("test.mode")); } private static String getHypervisorArchLog(Hypervisor.HypervisorType hypervisorType, CPU.CPUArch arch) { StringBuilder sb = new StringBuilder("hypervisor: ").append(hypervisorType.name()); - if (Hypervisor.HypervisorType.KVM.equals(hypervisorType)) { - sb.append(", arch: ").append(arch == null ? CPU.CPUArch.amd64.getType() : arch.getType()); - } + sb.append(", arch: ").append(arch == null ? CPU.CPUArch.amd64.getType() : arch.getType()); return sb.toString(); } - protected static String getHypervisorArchKey(Hypervisor.HypervisorType hypervisorType, CPU.CPUArch arch) { - if (Hypervisor.HypervisorType.KVM.equals(hypervisorType)) { - return String.format("%s-%s", hypervisorType.name().toLowerCase(), - arch == null ? CPU.CPUArch.amd64.getType() : arch.getType()); - } - return hypervisorType.name().toLowerCase(); - } - - protected static MetadataTemplateDetails getMetadataTemplateDetails(Hypervisor.HypervisorType hypervisorType, - CPU.CPUArch arch) { - return NewTemplateMap.get(getHypervisorArchKey(hypervisorType, arch)); - } - - public VMTemplateVO getRegisteredTemplate(String templateName, Hypervisor.HypervisorType hypervisorType, - CPU.CPUArch arch, String url) { - VMTemplateVO registeredTemplate = vmTemplateDao.findLatestTemplateByName(templateName, hypervisorType, arch); - if (registeredTemplate == null && StringUtils.isNotBlank(url)) { - String urlPath = url.substring(url.lastIndexOf("/") + 1); - LOGGER.debug("No template found by name, falling back to search existing SYSTEM template by " + - "urlPath: {}, hypervisor: {}, arch:{}", urlPath, hypervisorType, arch); - registeredTemplate = vmTemplateDao.findActiveSystemTemplateByHypervisorArchAndUrlPath(hypervisorType, arch, - urlPath); - } - LOGGER.debug("Found existing registered template for hypervisor: {}, arch: {}: {}", hypervisorType, - arch, registeredTemplate); - return registeredTemplate; - } - - private static boolean isRunningInTest() { - return "true".equalsIgnoreCase(System.getProperty("test.mode")); - } - /** * Attempts to determine the templates directory path by locating the metadata file. *

@@ -476,7 +399,137 @@ private static String fetchTemplatesPath() { throw new CloudRuntimeException(errMsg); } - private List getEligibleZoneIds() { + private static void cleanupStore(Long templateId, String filePath) { + String destTempFolder = filePath + PARTIAL_TEMPLATE_FOLDER + String.valueOf(templateId); + try { + Files.deleteIfExists(Paths.get(destTempFolder)); + } catch (IOException e) { + LOGGER.error("Failed to cleanup mounted store at: {}", filePath, e); + } + } + + private static Pair readTemplatePropertiesSizes(String path) { + File tmpFile = new File(path); + Long size = null; + Long physicalSize = 0L; + try (FileReader fr = new FileReader(tmpFile); BufferedReader brf = new BufferedReader(fr);) { + String line = null; + while ((line = brf.readLine()) != null) { + if (line.startsWith("size=")) { + physicalSize = Long.parseLong(line.split("=")[1]); + } else if (line.startsWith("virtualsize=")) { + size = Long.parseLong(line.split("=")[1]); + } + if (size == null) { + size = physicalSize; + } + } + } catch (IOException ex) { + LOGGER.warn("Failed to read from template.properties", ex); + } + return new Pair<>(size, physicalSize); + } + + private static void readTemplateProperties(String path, SystemVMTemplateDetails details) { + Pair templateSizes = readTemplatePropertiesSizes(path); + details.setSize(templateSizes.first()); + details.setPhysicalSize(templateSizes.second()); + } + + protected static MetadataTemplateDetails getMetadataTemplateDetails(Hypervisor.HypervisorType hypervisorType, + CPU.CPUArch arch) { + return METADATA_TEMPLATE_LIST + .stream() + .filter(x -> Objects.equals(x.getHypervisorType(), hypervisorType) && + Objects.equals(x.getArch(), arch)) + .findFirst() + .orElse(null); + } + + protected static String getMetadataFilePath() { + return METADATA_FILE; + } + + protected static Ini.Section getMetadataSectionForHypervisorAndArch(Ini ini, + Hypervisor.HypervisorType hypervisorType, CPU.CPUArch arch) { + String key = String.format("%s-%s", hypervisorType.name().toLowerCase(), + arch.getType().toLowerCase()); + Ini.Section section = ini.get(key); + if (section == null && !Hypervisor.HypervisorType.KVM.equals(hypervisorType)) { + key = String.format("%s", hypervisorType.name().toLowerCase()); + section = ini.get(key); + } + return section; + } + + protected static String getMountCommand(String nfsVersion, String device, String dir) { + String cmd = MOUNT_COMMAND_BASE; + if (StringUtils.isNotBlank(nfsVersion)) { + cmd = String.format("%s -o vers=%s", cmd, nfsVersion); + } + return String.format("%s %s %s", cmd, device, dir); + } + + /** + * This method parses the metadata file consisting of the system VM templates information + * @return the version of the system VM template that is to be used. This is done in order + * to fallback on the latest available version of the system VM template when there doesn't + * exist a template corresponding to the current code version. + */ + public static String parseMetadataFile() { + String metadataFilePath = getMetadataFilePath(); + String errMsg = String.format("Failed to parse system VM template metadata file: %s", metadataFilePath); + final Ini ini = new Ini(); + try (FileReader reader = new FileReader(metadataFilePath)) { + ini.load(reader); + } catch (IOException e) { + LOGGER.error(errMsg, e); + throw new CloudRuntimeException(errMsg, e); + } + if (!ini.containsKey("default")) { + errMsg = String.format("%s as unable to default section", errMsg); + LOGGER.error(errMsg); + throw new CloudRuntimeException(errMsg); + } + Ini.Section defaultSection = ini.get("default"); + String defaultDownloadRepository = defaultSection.get(TEMPLATES_DOWNLOAD_REPOSITORY_KEY); + String customDownloadRepository = ServerPropertiesUtil.getProperty(TEMPLATES_CUSTOM_DOWNLOAD_REPOSITORY_KEY); + boolean updateCustomDownloadRepository = StringUtils.isNotBlank(customDownloadRepository) && + StringUtils.isNotBlank(defaultDownloadRepository); + for (Pair hypervisorTypeArchPair : AVAILABLE_SYSTEM_TEMPLATES_HYPERVISOR_ARCH_LIST) { + String key = String.format("%s-%s", hypervisorTypeArchPair.first().name().toLowerCase(), + hypervisorTypeArchPair.second().getType().toLowerCase()); + Ini.Section section = getMetadataSectionForHypervisorAndArch(ini, hypervisorTypeArchPair.first(), + hypervisorTypeArchPair.second()); + if (section == null) { + LOGGER.error("Failed to find details for {} in template metadata file: {}", + getHypervisorArchLog(hypervisorTypeArchPair.first(), hypervisorTypeArchPair.second()), + metadataFilePath); + continue; + } + String url = section.get(TEMPLATE_DOWNLOAD_URL_KEY); + if (StringUtils.isNotBlank(url) && updateCustomDownloadRepository) { + url = url.replaceFirst(defaultDownloadRepository.trim(), + customDownloadRepository.trim()); + LOGGER.debug("Updated download URL for {} using custom repository to {}", key, url); + } + METADATA_TEMPLATE_LIST.add(new MetadataTemplateDetails( + hypervisorTypeArchPair.first(), + section.get("templatename"), + section.get("filename"), + url, + section.get("checksum"), + hypervisorTypeArchPair.second(), + section.get("guestos"))); + } + return defaultSection.get("version").trim(); + } + + protected File getTempDownloadDir() { + return tempDownloadDir; + } + + protected List getEligibleZoneIds() { List zoneIds = new ArrayList<>(); List stores = imageStoreDao.findByProtocol("nfs"); for (ImageStoreVO store : stores) { @@ -500,20 +553,11 @@ protected Pair getNfsStoreInZone(Long zoneId) { return new Pair<>(url, storeId); } - public static void mountStore(String storeUrl, String path, String nfsVersion) { - try { - if (storeUrl == null) { - return; - } - URI uri = new URI(UriUtils.encodeURIComponent(storeUrl)); - String host = uri.getHost(); - String mountPath = uri.getPath(); - Script.runSimpleBashScript(getMountCommand(nfsVersion, host + ":" + mountPath, path)); - } catch (Exception e) { - String msg = "NFS Store URL is not in the correct format"; - LOGGER.error(msg, e); - throw new CloudRuntimeException(msg, e); + protected String getSystemVmTemplateVersion() { + if (StringUtils.isEmpty(systemVmTemplateVersion)) { + return String.format("%s.%s", CS_MAJOR_VERSION, CS_TINY_VERSION); } + return systemVmTemplateVersion; } private VMTemplateVO createTemplateObjectInDB(SystemVMTemplateDetails details) { @@ -562,28 +606,32 @@ private void createCrossZonesTemplateZoneRefEntries(Long templateId) { for (DataCenterVO dc : dcs) { VMTemplateZoneVO templateZoneVO = createOrUpdateTemplateZoneEntry(dc.getId(), templateId); if (templateZoneVO == null) { - throw new CloudRuntimeException(String.format("Failed to create template_zone_ref record for the systemVM template (id: %s) and zone: %s", templateId, dc)); + throw new CloudRuntimeException(String.format("Failed to create template-zone record for the system " + + "VM template (ID : %d) and zone: %s", templateId, dc)); } } } private void createTemplateStoreRefEntry(SystemVMTemplateDetails details) { - TemplateDataStoreVO templateDataStoreVO = new TemplateDataStoreVO(details.storeId, details.getId(), details.getCreated(), 0, - VMTemplateStorageResourceAssoc.Status.NOT_DOWNLOADED, null, null, null, details.getInstallPath(), details.getUrl()); + TemplateDataStoreVO templateDataStoreVO = new TemplateDataStoreVO(details.getStoreId(), details.getId(), + details.getCreated(), 0, VMTemplateStorageResourceAssoc.Status.NOT_DOWNLOADED, + null, null, null, details.getInstallPath(), details.getUrl()); templateDataStoreVO.setDataStoreRole(DataStoreRole.Image); templateDataStoreVO = templateDataStoreDao.persist(templateDataStoreVO); if (templateDataStoreVO == null) { - throw new CloudRuntimeException(String.format("Failed to create template_store_ref record for the systemVM template for hypervisor: %s", details.getHypervisorType().name())); + throw new CloudRuntimeException(String.format("Failed to create template-store record for the system VM " + + "template (ID : %d) and store (ID: %d)", details.getId(), details.getStoreId())); } } - public void updateTemplateDetails(SystemVMTemplateDetails details) { + protected void updateTemplateDetails(SystemVMTemplateDetails details) { VMTemplateVO template = vmTemplateDao.findById(details.getId()); template.setSize(details.getSize()); template.setState(VirtualMachineTemplate.State.Active); vmTemplateDao.update(template.getId(), template); - TemplateDataStoreVO templateDataStoreVO = templateDataStoreDao.findByStoreTemplate(details.getStoreId(), template.getId()); + TemplateDataStoreVO templateDataStoreVO = templateDataStoreDao.findByStoreTemplate(details.getStoreId(), + template.getId()); templateDataStoreVO.setSize(details.getSize()); templateDataStoreVO.setPhysicalSize(details.getPhysicalSize()); templateDataStoreVO.setDownloadPercent(100); @@ -592,11 +640,11 @@ public void updateTemplateDetails(SystemVMTemplateDetails details) { templateDataStoreVO.setState(ObjectInDataStoreStateMachine.State.Ready); boolean updated = templateDataStoreDao.update(templateDataStoreVO.getId(), templateDataStoreVO); if (!updated) { - throw new CloudRuntimeException("Failed to update template_store_ref entry for registered systemVM template"); + throw new CloudRuntimeException("Failed to update template-store record for registered system VM template"); } } - public void updateSeededTemplateDetails(long templateId, long storeId, long size, long physicalSize) { + protected void updateSeededTemplateDetails(long templateId, long storeId, long size, long physicalSize) { VMTemplateVO template = vmTemplateDao.findById(templateId); template.setSize(size); vmTemplateDao.update(template.getId(), template); @@ -607,108 +655,77 @@ public void updateSeededTemplateDetails(long templateId, long storeId, long size templateDataStoreVO.setLastUpdated(new Date(DateUtil.currentGMTTime().getTime())); boolean updated = templateDataStoreDao.update(templateDataStoreVO.getId(), templateDataStoreVO); if (!updated) { - throw new CloudRuntimeException("Failed to update template_store_ref entry for seeded systemVM template"); + throw new CloudRuntimeException("Failed to update template-store record for seeded system VM template"); } } - public void updateSystemVMEntries(Long templateId, Hypervisor.HypervisorType hypervisorType) { + protected void updateSystemVMEntries(Long templateId, Hypervisor.HypervisorType hypervisorType) { vmInstanceDao.updateSystemVmTemplateId(templateId, hypervisorType); } - private void updateSystemVmTemplateGuestOsId() { - String systemVmGuestOsName = "Debian GNU/Linux 12 (64-bit)"; // default + private void updateHypervisorGuestOsMap() { try { - GuestOSVO guestOS = guestOSDao.findOneByDisplayName(systemVmGuestOsName); - if (guestOS != null) { - LOGGER.debug("Updating SystemVM Template Guest OS [{}] id", systemVmGuestOsName); - SystemVmTemplateRegistration.LINUX_12_ID = Math.toIntExact(guestOS.getId()); - hypervisorGuestOsMap.put(Hypervisor.HypervisorType.KVM, LINUX_12_ID); - hypervisorGuestOsMap.put(Hypervisor.HypervisorType.Hyperv, LINUX_12_ID); - hypervisorGuestOsMap.put(Hypervisor.HypervisorType.LXC, LINUX_12_ID); - hypervisorGuestOsMap.put(Hypervisor.HypervisorType.Ovm3, LINUX_12_ID); + GuestOSVO guestOS = guestOSDao.findOneByDisplayName(DEFAULT_SYSTEM_VM_GUEST_OS_NAME); + if (guestOS == null) { + LOGGER.warn("Couldn't find Guest OS by name [{}] to update system VM Template guest OS ID", + DEFAULT_SYSTEM_VM_GUEST_OS_NAME); + return; } + LOGGER.debug("Updating system VM template guest OS [{}] ID", DEFAULT_SYSTEM_VM_GUEST_OS_NAME); + SystemVmTemplateRegistration.LINUX_12_ID = Math.toIntExact(guestOS.getId()); + hypervisorGuestOsMap.put(Hypervisor.HypervisorType.KVM, LINUX_12_ID); + hypervisorGuestOsMap.put(Hypervisor.HypervisorType.Hyperv, LINUX_12_ID); + hypervisorGuestOsMap.put(Hypervisor.HypervisorType.LXC, LINUX_12_ID); + hypervisorGuestOsMap.put(Hypervisor.HypervisorType.Ovm3, LINUX_12_ID); } catch (Exception e) { - LOGGER.warn("Couldn't update SystemVM Template Guest OS id, due to {}", e.getMessage()); + LOGGER.warn("Couldn't update System VM template guest OS ID, due to {}", e.getMessage()); } } - public void updateConfigurationParams(Map configParams) { - for (Map.Entry config : configParams.entrySet()) { - boolean updated = configurationDao.update(config.getKey(), config.getValue()); - if (!updated) { - throw new CloudRuntimeException(String.format("Failed to update configuration parameter %s", config.getKey())); - } + protected void updateConfigurationParams(Hypervisor.HypervisorType hypervisorType, String templateName, Long zoneId) { + String configName = ROUTER_TEMPLATE_CONFIGURATION_NAMES.get(hypervisorType); + boolean updated = configurationDao.update(configName, templateName); + if (!updated) { + throw new CloudRuntimeException(String.format("Failed to update configuration parameter %s", configName)); } - } - - private static Pair readTemplatePropertiesSizes(String path) { - File tmpFile = new File(path); - Long size = null; - Long physicalSize = 0L; - try (FileReader fr = new FileReader(tmpFile); BufferedReader brf = new BufferedReader(fr);) { - String line = null; - while ((line = brf.readLine()) != null) { - if (line.startsWith("size=")) { - physicalSize = Long.parseLong(line.split("=")[1]); - } else if (line.startsWith("virtualsize=")) { - size = Long.parseLong(line.split("=")[1]); - } - if (size == null) { - size = physicalSize; - } - } - } catch (IOException ex) { - LOGGER.warn("Failed to read from template.properties", ex); + if (zoneId != null) { + dataCenterDetailsDao.removeDetail(zoneId, configName); + } + updated = configurationDao.update(MINIMUM_SYSTEM_VM_VERSION_KEY, getSystemVmTemplateVersion()); + if (!updated) { + throw new CloudRuntimeException(String.format("Failed to update configuration parameter %s", configName)); + } + if (zoneId != null) { + dataCenterDetailsDao.removeDetail(zoneId, MINIMUM_SYSTEM_VM_VERSION_KEY); } - return new Pair<>(size, physicalSize); - } - - public static void readTemplateProperties(String path, SystemVMTemplateDetails details) { - Pair templateSizes = readTemplatePropertiesSizes(path); - details.setSize(templateSizes.first()); - details.setPhysicalSize(templateSizes.second()); } - private void updateTemplateTablesOnFailure(long templateId) { + private void updateTemplateEntriesOnFailure(long templateId) { VMTemplateVO template = vmTemplateDao.createForUpdate(templateId); template.setState(VirtualMachineTemplate.State.Inactive); vmTemplateDao.update(template.getId(), template); vmTemplateDao.remove(templateId); - TemplateDataStoreVO templateDataStoreVO = templateDataStoreDao.findByTemplate(template.getId(), DataStoreRole.Image); - templateDataStoreDao.remove(templateDataStoreVO.getId()); - } - - public static void unmountStore(String filePath) { - try { - LOGGER.info("Unmounting store"); - String umountCmd = String.format(UMOUNT_COMMAND, filePath); - Script.runSimpleBashScript(umountCmd); - try { - Files.deleteIfExists(Paths.get(filePath)); - } catch (IOException e) { - LOGGER.error(String.format("Failed to cleanup mounted store at: %s", filePath), e); - } - } catch (Exception e) { - String msg = String.format("Failed to unmount store mounted at %s", filePath); - LOGGER.error(msg, e); - throw new CloudRuntimeException(msg, e); + TemplateDataStoreVO templateDataStoreVO = templateDataStoreDao.findByTemplate(template.getId(), + DataStoreRole.Image); + if (templateDataStoreVO == null) { + return; } + templateDataStoreDao.remove(templateDataStoreVO.getId()); } - private void setupTemplate(String templateName, Hypervisor.HypervisorType hypervisor, CPU.CPUArch arch, - String destTempFolder) throws CloudRuntimeException { + private void setupTemplateOnStore(String templateName, MetadataTemplateDetails templateDetails, + String destTempFolder) throws CloudRuntimeException { String setupTmpltScript = Script.findScript(storageScriptsDir, "setup-sysvm-tmplt"); if (setupTmpltScript == null) { - throw new CloudRuntimeException("Unable to find the createtmplt.sh"); + throw new CloudRuntimeException("Unable to find the setup-sysvm-tmplt script"); } Script scr = new Script(setupTmpltScript, SCRIPT_TIMEOUT, LOGGER); scr.add("-u", templateName); - MetadataTemplateDetails templateDetails = NewTemplateMap.get(getHypervisorArchKey(hypervisor, arch)); String filePath = StringUtils.isNotBlank(templateDetails.getDownloadedFilePath()) ? templateDetails.getDownloadedFilePath() : templateDetails.getDefaultFilePath(); scr.add("-f", filePath); - scr.add("-h", hypervisor.name().toLowerCase(Locale.ROOT)); + scr.add("-h", templateDetails.getHypervisorType().name().toLowerCase(Locale.ROOT)); scr.add("-d", destTempFolder); String result = scr.execute(); if (result != null) { @@ -718,17 +735,18 @@ private void setupTemplate(String templateName, Hypervisor.HypervisorType hyperv } } - private Long performTemplateRegistrationOperations(Hypervisor.HypervisorType hypervisor, - String name, CPU.CPUArch arch, String url, String checksum, ImageFormat format, long guestOsId, + protected Long performTemplateRegistrationOperations(String name, MetadataTemplateDetails templateDetails, + String url, String checksum, ImageFormat format, long guestOsId, Long storeId, Long templateId, String filePath, TemplateDataStoreVO templateDataStoreVO) { String templateName = UUID.randomUUID().toString(); Date created = new Date(DateUtil.currentGMTTime().getTime()); - SystemVMTemplateDetails details = new SystemVMTemplateDetails(templateName, name, created, - url, checksum, format, (int) guestOsId, hypervisor, arch, storeId); + SystemVMTemplateDetails details = new SystemVMTemplateDetails(templateName, name, created, url, checksum, + format, (int) guestOsId, templateDetails.getHypervisorType(), templateDetails.getArch(), storeId); if (templateId == null) { VMTemplateVO template = createTemplateObjectInDB(details); if (template == null) { - throw new CloudRuntimeException(String.format("Failed to register template for hypervisor: %s", hypervisor.name())); + throw new CloudRuntimeException(String.format("Failed to register template for hypervisor: %s", + templateDetails.getHypervisorType().name())); } templateId = template.getId(); } @@ -737,163 +755,124 @@ private Long performTemplateRegistrationOperations(Hypervisor.HypervisorType hyp details.setId(templateId); String destTempFolderName = String.valueOf(templateId); String destTempFolder = filePath + PARTIAL_TEMPLATE_FOLDER + destTempFolderName; - details.setInstallPath(PARTIAL_TEMPLATE_FOLDER + destTempFolderName + File.separator + templateName + "." + hypervisorImageFormat.get(hypervisor).getFileExtension()); + details.setInstallPath(PARTIAL_TEMPLATE_FOLDER + destTempFolderName + File.separator + templateName + "." + hypervisorImageFormat.get(templateDetails.getHypervisorType()).getFileExtension()); if (templateDataStoreVO == null) { createTemplateStoreRefEntry(details); } - setupTemplate(templateName, hypervisor, arch, destTempFolder); + setupTemplateOnStore(templateName, templateDetails, destTempFolder); readTemplateProperties(destTempFolder + "/template.properties", details); details.setUpdated(new Date(DateUtil.currentGMTTime().getTime())); updateTemplateDetails(details); return templateId; } - public void registerTemplate(Hypervisor.HypervisorType hypervisor, String name, Long storeId, - VMTemplateVO templateVO, TemplateDataStoreVO templateDataStoreVO, String filePath) { + /** + * Add an existing system VM template to a secondary image store and update related DB entries. + * + * @param templateVO the existing VM template (must not be null) + * @param templateDetails the metadata details of the template to be added + * @param templateDataStoreVO optional existing template-store mapping; may be null + * @param zoneId zone id where the operation is performed + * @param storeId target image store id + * @param filePath temporary mount path for the store + * @throws CloudRuntimeException on failure; the method attempts rollback/cleanup + */ + protected void addExistingTemplateToStore(VMTemplateVO templateVO, MetadataTemplateDetails templateDetails, + TemplateDataStoreVO templateDataStoreVO, long zoneId, Long storeId, String filePath) { try { - performTemplateRegistrationOperations(hypervisor, name, templateVO.getArch(), templateVO.getUrl(), + performTemplateRegistrationOperations(templateVO.getName(), templateDetails, templateVO.getUrl(), templateVO.getChecksum(), templateVO.getFormat(), templateVO.getGuestOSId(), storeId, templateVO.getId(), filePath, templateDataStoreVO); } catch (Exception e) { - String errMsg = String.format("Failed to register template for hypervisor: %s", hypervisor); + String errMsg = String.format("Failed to add %s to store ID: %d, zone ID: %d", templateVO, storeId, zoneId); LOGGER.error(errMsg, e); - updateTemplateTablesOnFailure(templateVO.getId()); cleanupStore(templateVO.getId(), filePath); throw new CloudRuntimeException(errMsg, e); } } - public void registerTemplateForNonExistingEntries(Hypervisor.HypervisorType hypervisor, CPU.CPUArch arch, - String name, Pair storeUrlAndId, String filePath) { + /** + * Registers a new system VM template for the given hypervisor/arch when no existing template is present. + * + * @param name the name of the new template + * @param templateDetails the metadata details of the template to be registered + * @param zoneId the zone id for which the new template should be seeded + * @param storeId the store id on which the new template will be seeded + * @param filePath temporary mount path for the store + * @throws CloudRuntimeException on failure; the method attempts rollback/cleanup + */ + protected void registerNewTemplate(String name, MetadataTemplateDetails templateDetails, long zoneId, Long storeId, + String filePath) { Long templateId = null; + Hypervisor.HypervisorType hypervisor = templateDetails.getHypervisorType(); try { - MetadataTemplateDetails templateDetails = getMetadataTemplateDetails(hypervisor, arch); - templateId = performTemplateRegistrationOperations(hypervisor, name, - templateDetails.getArch(), templateDetails.getUrl(), + templateId = performTemplateRegistrationOperations(name, templateDetails, templateDetails.getUrl(), templateDetails.getChecksum(), hypervisorImageFormat.get(hypervisor), - hypervisorGuestOsMap.get(hypervisor), storeUrlAndId.second(), null, filePath, null); - Map configParams = new HashMap<>(); - configParams.put(RouterTemplateConfigurationNames.get(hypervisor), templateDetails.getName()); - configParams.put("minreq.sysvmtemplate.version", getSystemVmTemplateVersion()); - updateConfigurationParams(configParams); + hypervisorGuestOsMap.get(hypervisor), storeId, null, filePath, null); + updateConfigurationParams(hypervisor, name, zoneId); updateSystemVMEntries(templateId, hypervisor); } catch (Exception e) { - String errMsg = String.format("Failed to register template for hypervisor: %s", hypervisor); + String errMsg = String.format("Failed to register template for %s", templateDetails.getHypervisorType()); LOGGER.error(errMsg, e); if (templateId != null) { - updateTemplateTablesOnFailure(templateId); + updateTemplateEntriesOnFailure(templateId); cleanupStore(templateId, filePath); } throw new CloudRuntimeException(errMsg, e); } } - protected void validateTemplateFileForHypervisorAndArch(Hypervisor.HypervisorType hypervisor, CPU.CPUArch arch) { + /** + * Validate presence and integrity of metadata and local template file for the given hypervisor/arch. + * + * @param hypervisor target hypervisor type + * @param arch target CPU architecture + * @return validated MetadataTemplateDetails + * @throws CloudRuntimeException if template is not available, missing, or checksum validation fails + */ + protected MetadataTemplateDetails getValidatedTemplateDetailsForHypervisorAndArch( + Hypervisor.HypervisorType hypervisor, CPU.CPUArch arch) { + if (!AVAILABLE_SYSTEM_TEMPLATES_HYPERVISOR_ARCH_LIST.contains(new Pair<>(hypervisor, arch))) { + throw new CloudRuntimeException("No system VM template available for the given hypervisor and arch"); + } MetadataTemplateDetails templateDetails = getMetadataTemplateDetails(hypervisor, arch); + if (templateDetails == null) { + throw new CloudRuntimeException("No template details found for the given hypervisor and arch"); + } File templateFile = getTemplateFile(templateDetails); if (templateFile == null) { throw new CloudRuntimeException("Failed to find local template file"); } - if (isTemplateFileChecksumDifferent(templateDetails, templateFile)) { + if (templateDetails.isFileChecksumDifferent(templateFile)) { throw new CloudRuntimeException("Checksum failed for local template file"); } - } - - public void validateAndRegisterTemplate(Hypervisor.HypervisorType hypervisor, String name, Long storeId, - VMTemplateVO templateVO, TemplateDataStoreVO templateDataStoreVO, String filePath) { - validateTemplateFileForHypervisorAndArch(hypervisor, templateVO.getArch()); - registerTemplate(hypervisor, name, storeId, templateVO, templateDataStoreVO, filePath); - } - - public void validateAndRegisterTemplateForNonExistingEntries(Hypervisor.HypervisorType hypervisor, - CPU.CPUArch arch, String name, Pair storeUrlAndId, String filePath) { - validateTemplateFileForHypervisorAndArch(hypervisor, arch); - registerTemplateForNonExistingEntries(hypervisor, arch, name, storeUrlAndId, filePath); - } - - protected static String getMetadataFilePath() { - return METADATA_FILE; + return templateDetails; } /** - * This method parses the metadata file consisting of the systemVM templates information - * @return the version of the systemvm template that is to be used. This is done in order - * to fallback on the latest available version of the systemVM template when there doesn't - * exist a template corresponding to the current code version. + * Return the local template file. Downloads it if not present locally and url is present. + * + * @param templateDetails template metadata; may set `downloadedFilePath` + * @return the template {@code File} on disk, or {@code null} if not found/downloaded */ - public static String parseMetadataFile() { - String metadataFilePath = getMetadataFilePath(); - String errMsg = String.format("Failed to parse systemVM template metadata file: %s", metadataFilePath); - final Ini ini = new Ini(); - try (FileReader reader = new FileReader(metadataFilePath)) { - ini.load(reader); - } catch (IOException e) { - LOGGER.error(errMsg, e); - throw new CloudRuntimeException(errMsg, e); - } - if (!ini.containsKey("default")) { - errMsg = String.format("%s as unable to default section", errMsg); - LOGGER.error(errMsg); - throw new CloudRuntimeException(errMsg); - } - Ini.Section defaultSection = ini.get("default"); - String defaultDownloadRepository = defaultSection.get(TEMPLATES_DOWNLOAD_REPOSITORY_KEY); - String customDownloadRepository = ServerPropertiesUtil.getProperty(TEMPLATES_CUSTOM_DOWNLOAD_REPOSITORY_KEY); - boolean updateCustomDownloadRepository = StringUtils.isNotBlank(customDownloadRepository) && - StringUtils.isNotBlank(defaultDownloadRepository); - for (Pair hypervisorType : hypervisorList) { - String key = getHypervisorArchKey(hypervisorType.first(), hypervisorType.second()); - Ini.Section section = ini.get(key); - if (section == null) { - LOGGER.error("Failed to find details for {} in template metadata file: {}", - key, metadataFilePath); - continue; - } - String url = section.get(TEMPLATE_DOWNLOAD_URL_KEY); - if (StringUtils.isNotBlank(url) && updateCustomDownloadRepository) { - url = url.replaceFirst(defaultDownloadRepository.trim(), - customDownloadRepository.trim()); - LOGGER.debug("Updated download URL for {} using custom repository to {}", key, url); - } - NewTemplateMap.put(key, new MetadataTemplateDetails( - hypervisorType.first(), - section.get("templatename"), - section.get("filename"), - url, - section.get("checksum"), - hypervisorType.second(), - section.get("guestos"))); - } - return defaultSection.get("version").trim(); - } - - - private static void cleanupStore(Long templateId, String filePath) { - String destTempFolder = filePath + PARTIAL_TEMPLATE_FOLDER + String.valueOf(templateId); - try { - Files.deleteIfExists(Paths.get(destTempFolder)); - } catch (IOException e) { - LOGGER.error(String.format("Failed to cleanup mounted store at: %s", filePath), e); - } - } - protected File getTemplateFile(MetadataTemplateDetails templateDetails) { File templateFile = new File(templateDetails.getDefaultFilePath()); if (templateFile.exists()) { return templateFile; } LOGGER.debug("{} is not present", templateFile.getAbsolutePath()); - if (DOWNLOADABLE_TEMPLATE_ARCH_TYPES.contains(templateDetails.getArch()) && - StringUtils.isNotBlank(templateDetails.getUrl())) { + if (StringUtils.isNotBlank(templateDetails.getUrl())) { LOGGER.debug("Downloading the template file {} for {}", templateDetails.getUrl(), templateDetails.getHypervisorArchLog()); Path path = Path.of(TEMPLATES_PATH); if (!Files.isWritable(path)) { - templateFile = new File(tempDownloadDir, templateDetails.getFilename()); + templateFile = new File(getTempDownloadDir(), templateDetails.getFilename()); } if (!templateFile.exists() && !HttpUtils.downloadFileWithProgress(templateDetails.getUrl(), templateFile.getAbsolutePath(), LOGGER)) { + LOGGER.error("Failed to download template for {} using url: {}", + templateDetails.getHypervisorArchLog(), templateDetails.getUrl()); return null; } templateDetails.setDownloadedFilePath(templateFile.getAbsolutePath()); @@ -901,32 +880,28 @@ protected File getTemplateFile(MetadataTemplateDetails templateDetails) { return templateFile; } - protected boolean isTemplateFileChecksumDifferent(MetadataTemplateDetails templateDetails, File templateFile) { - String templateChecksum = DigestHelper.calculateChecksum(templateFile); - if (!templateChecksum.equals(templateDetails.getChecksum())) { - LOGGER.error("Checksum {} for file {} does not match checksum {} from metadata", - templateChecksum, templateFile, templateDetails.getChecksum()); - return true; - } - return false; - } - - protected void validateTemplates(List> hypervisorsArchInUse) { + /** + * Validate that templates for the provided hypervisor/architecture pairs which are in use and are valid. + * + * If a template is missing or validation fails for any required pair, a + * {@link CloudRuntimeException} is thrown to abort the upgrade. If system VM template for a hypervisor/arch is + * not considered available then validation is skipped for that pair. + * + * @param hypervisorArchList list of hypervisor/architecture pairs to validate + */ + protected void validateTemplates(List> hypervisorArchList) { boolean templatesFound = true; - for (Pair hypervisorArch : hypervisorsArchInUse) { - MetadataTemplateDetails matchedTemplate = getMetadataTemplateDetails(hypervisorArch.first(), - hypervisorArch.second()); - if (matchedTemplate == null) { - templatesFound = false; - break; - } - File tempFile = getTemplateFile(matchedTemplate); - if (tempFile == null) { - LOGGER.warn("Failed to download template for {}, moving ahead", - matchedTemplate.getHypervisorArchLog()); + for (Pair hypervisorArch : hypervisorArchList) { + if (!AVAILABLE_SYSTEM_TEMPLATES_HYPERVISOR_ARCH_LIST.contains(hypervisorArch)) { + LOGGER.info("No system VM template available for {}. Skipping validation.", + getHypervisorArchLog(hypervisorArch.first(), hypervisorArch.second())); continue; } - if (isTemplateFileChecksumDifferent(matchedTemplate, tempFile)) { + try { + getValidatedTemplateDetailsForHypervisorAndArch(hypervisorArch.first(), hypervisorArch.second()); + } catch (CloudRuntimeException e) { + LOGGER.error("Validation failed for {}: {}", + getHypervisorArchLog(hypervisorArch.first(), hypervisorArch.second()), e.getMessage()); templatesFound = false; break; } @@ -938,10 +913,10 @@ protected void validateTemplates(List storeUrlAndId = getNfsStoreInZone(zoneId); String nfsVersion = getNfsVersion(storeUrlAndId.second()); - mountStore(storeUrlAndId.first(), filePath, nfsVersion); + mountStore(storeUrlAndId.first(), storeMountPath, nfsVersion); List> hypervisorArchList = clusterDao.listDistinctHypervisorsAndArchExcludingExternalType(zoneId); for (Pair hypervisorArch : hypervisorArchList) { @@ -962,22 +937,22 @@ protected void registerTemplatesForZone(long zoneId, String filePath) { continue; } } - registerTemplate(hypervisorType, templateDetails.getName(), storeUrlAndId.second(), templateVO, - templateDataStoreVO, filePath); - updateRegisteredTemplateDetails(templateVO.getId(), templateDetails); + addExistingTemplateToStore(templateVO, templateDetails, templateDataStoreVO, zoneId, + storeUrlAndId.second(), storeMountPath); + updateRegisteredTemplateDetails(templateVO.getId(), templateDetails, zoneId); continue; } - registerTemplateForNonExistingEntries(hypervisorType, templateDetails.getArch(), templateDetails.getName(), - storeUrlAndId, filePath); + registerNewTemplate(templateDetails.getName(), templateDetails, zoneId, storeUrlAndId.second(), + storeMountPath); } } - public void registerTemplates(List> hypervisorsArchInUse) { + protected void registerTemplates(List> hypervisorsArchInUse) { GlobalLock lock = GlobalLock.getInternLock("UpgradeDatabase-Lock"); try { LOGGER.info("Grabbing lock to register templates."); if (!lock.lock(LOCK_WAIT_TIMEOUT)) { - throw new CloudRuntimeException("Unable to acquire lock to register SystemVM template."); + throw new CloudRuntimeException("Unable to acquire lock to register system VM template."); } try { validateTemplates(hypervisorsArchInUse); @@ -997,13 +972,13 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { unmountStore(filePath); } catch (Exception e) { unmountStore(filePath); - throw new CloudRuntimeException("Failed to register systemVM template. Upgrade Failed"); + throw new CloudRuntimeException("Failed to register system VM template. Upgrade Failed"); } } } }); } catch (Exception e) { - throw new CloudRuntimeException("Failed to register systemVM template. Upgrade Failed"); + throw new CloudRuntimeException("Failed to register system VM template. Upgrade Failed"); } } finally { lock.unlock(); @@ -1011,7 +986,8 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { } } - protected void updateRegisteredTemplateDetails(Long templateId, MetadataTemplateDetails templateDetails) { + protected void updateRegisteredTemplateDetails(Long templateId, MetadataTemplateDetails templateDetails, + Long zoneId) { VMTemplateVO templateVO = vmTemplateDao.findById(templateId); templateVO.setTemplateType(Storage.TemplateType.SYSTEM); GuestOSVO guestOS = guestOSDao.findOneByDisplayName(templateDetails.getGuestOs()); @@ -1020,20 +996,18 @@ protected void updateRegisteredTemplateDetails(Long templateId, MetadataTemplate } boolean updated = vmTemplateDao.update(templateVO.getId(), templateVO); if (!updated) { - String errMsg = String.format("updateSystemVmTemplates:Exception while updating template with id %s to be marked as 'system'", templateId); + String errMsg = String.format("Exception while updating template with id %s to be marked as 'system'", templateId); LOGGER.error(errMsg); throw new CloudRuntimeException(errMsg); } Hypervisor.HypervisorType hypervisorType = templateDetails.getHypervisorType(); updateSystemVMEntries(templateId, hypervisorType); // Change value of global configuration parameter router.template.* for the corresponding hypervisor and minreq.sysvmtemplate.version for the ACS version - Map configParams = new HashMap<>(); - configParams.put(RouterTemplateConfigurationNames.get(hypervisorType), templateDetails.getName()); - configParams.put("minreq.sysvmtemplate.version", getSystemVmTemplateVersion()); - updateConfigurationParams(configParams); + updateConfigurationParams(hypervisorType, templateDetails.getName(), zoneId); } - private void updateTemplateUrlChecksumAndGuestOsId(VMTemplateVO templateVO, MetadataTemplateDetails templateDetails) { + protected void updateTemplateUrlChecksumAndGuestOsId(VMTemplateVO templateVO, + MetadataTemplateDetails templateDetails) { templateVO.setUrl(templateDetails.getUrl()); templateVO.setChecksum(templateDetails.getChecksum()); GuestOSVO guestOS = guestOSDao.findOneByDisplayName(templateDetails.getGuestOs()); @@ -1042,14 +1016,21 @@ private void updateTemplateUrlChecksumAndGuestOsId(VMTemplateVO templateVO, Meta } boolean updated = vmTemplateDao.update(templateVO.getId(), templateVO); if (!updated) { - String errMsg = String.format("updateSystemVmTemplates:Exception while updating 'url' and 'checksum' for hypervisor type %s", templateDetails.getHypervisorType()); + String errMsg = String.format("Exception while updating 'url' and 'checksum' for hypervisor type %s", + templateDetails.getHypervisorType()); LOGGER.error(errMsg); throw new CloudRuntimeException(errMsg); } } - protected boolean registerOrUpdateSystemVmTemplate(MetadataTemplateDetails templateDetails, - List> hypervisorsInUse) { + /** + * Updates or registers the system VM template for the given hypervisor/arch if not already present. + * Returns true if a new template was registered. + * If there is an existing system VM template for the given hypervisor/arch, its details are updated. + * If no existing template is found, new templates are registered for the valid hypervisor/arch which are in use. + */ + protected boolean updateOrRegisterSystemVmTemplate(MetadataTemplateDetails templateDetails, + List> hypervisorArchInUse) { String systemVmTemplateLog = String.format("%s system VM template for %s", getSystemVmTemplateVersion(), templateDetails.getHypervisorArchLog()); LOGGER.debug("Registering or updating {}", systemVmTemplateLog, @@ -1059,10 +1040,10 @@ protected boolean registerOrUpdateSystemVmTemplate(MetadataTemplateDetails templ if (registeredTemplate != null) { LOGGER.info("{} is already registered, updating details for: {}", systemVmTemplateLog, templateDetails.getHypervisorArchLog(), registeredTemplate); - updateRegisteredTemplateDetails(registeredTemplate.getId(), templateDetails); + updateRegisteredTemplateDetails(registeredTemplate.getId(), templateDetails, null); return false; } - boolean isHypervisorArchMatchMetadata = hypervisorsInUse.stream() + boolean isHypervisorArchMatchMetadata = hypervisorArchInUse.stream() .anyMatch(p -> p.first().equals(templateDetails.getHypervisorType()) && Objects.equals(p.second(), templateDetails.getArch())); if (!isHypervisorArchMatchMetadata) { @@ -1076,21 +1057,185 @@ protected boolean registerOrUpdateSystemVmTemplate(MetadataTemplateDetails templ return false; } try { - registerTemplates(hypervisorsInUse); + registerTemplates(hypervisorArchInUse); return true; } catch (final Exception e) { throw new CloudRuntimeException(String.format("Failed to register %s templates for hypervisors: [%s]. " + "Cannot upgrade system VMs", getSystemVmTemplateVersion(), - StringUtils.join(hypervisorsInUse.stream() - .map(x -> getHypervisorArchKey(x.first(), x.second())) + StringUtils.join(hypervisorArchInUse.stream() + .map(x -> String.format("%s-%s", x.first().name(), x.second().name())) .collect(Collectors.toList()), ",")), e); } } + /** + * Return NFS version for the store: store-specific config if present + * or global config if absent. Returns null if not set. + */ + protected String getNfsVersion(long storeId) { + final String configKey = "secstorage.nfs.version"; + final Map storeDetails = imageStoreDetailsDao.getDetails(storeId); + if (storeDetails != null && storeDetails.containsKey(configKey)) { + return storeDetails.get(configKey); + } + ConfigurationVO globalNfsVersion = configurationDao.findByName(configKey); + if (globalNfsVersion != null) { + return globalNfsVersion.getValue(); + } + return null; + } + + public static void mountStore(String storeUrl, String path, String nfsVersion) { + try { + if (storeUrl == null) { + return; + } + URI uri = new URI(UriUtils.encodeURIComponent(storeUrl)); + String host = uri.getHost(); + String mountPath = uri.getPath(); + Script.runSimpleBashScript(getMountCommand(nfsVersion, host + ":" + mountPath, path)); + } catch (Exception e) { + String msg = "NFS Store URL is not in the correct format"; + LOGGER.error(msg, e); + throw new CloudRuntimeException(msg, e); + } + } + + public static void unmountStore(String filePath) { + try { + LOGGER.info("Unmounting store"); + String umountCmd = String.format(UMOUNT_COMMAND, filePath); + Script.runSimpleBashScript(umountCmd); + try { + Files.deleteIfExists(Paths.get(filePath)); + } catch (IOException e) { + LOGGER.error(String.format("Failed to cleanup mounted store at: %s", filePath), e); + } + } catch (Exception e) { + String msg = String.format("Failed to unmount store mounted at %s", filePath); + LOGGER.error(msg, e); + throw new CloudRuntimeException(msg, e); + } + } + + /** + * Validate metadata for the given template's hypervisor/arch and add the existing template + * to the specified secondary store. On success, database entries are created/updated. + * + * @param templateVO template to add + * @param templateDataStoreVO existing template-store mapping; may be null + * @param zoneId zone id where the operation is performed + * @param storeId target image store id + * @param filePath temporary mount path for the store + * @throws CloudRuntimeException on failure; the method attempts rollback/cleanup + */ + public void validateAndAddTemplateToStore(VMTemplateVO templateVO, TemplateDataStoreVO templateDataStoreVO, + long zoneId, long storeId, String filePath) { + MetadataTemplateDetails templateDetails = getValidatedTemplateDetailsForHypervisorAndArch( + templateVO.getHypervisorType(), templateVO.getArch()); + addExistingTemplateToStore(templateVO, templateDetails, templateDataStoreVO, zoneId, storeId, filePath); + } + + /** + * Validate metadata for the given hypervisor/arch and register a new system VM template + * on the specified store and zone. Creates DB entries and seeds the template on the store. + * + * @param hypervisor hypervisor type + * @param arch cpu architecture + * @param name template name to register + * @param zoneId zone id where the operation is performed + * @param storeId target image store id + * @param filePath temporary mount path for the store + * @throws CloudRuntimeException on failure; the method attempts rollback/cleanup + */ + public void validateAndRegisterNewTemplate(Hypervisor.HypervisorType hypervisor, CPU.CPUArch arch, String name, + long zoneId, long storeId, String filePath) { + MetadataTemplateDetails templateDetails = getValidatedTemplateDetailsForHypervisorAndArch(hypervisor, arch); + registerNewTemplate(name, templateDetails, zoneId, storeId, filePath); + } + + /** + * Check whether the template at the given `path` on NFS `url` is already seeded. + * If found, updates DB with sizes and returns true; otherwise returns false. + * + * @throws CloudRuntimeException on any error + */ + public boolean validateIfSeeded(TemplateDataStoreVO templDataStoreVO, String url, String path, String nfsVersion) { + String filePath = null; + try { + filePath = Files.createTempDirectory(TEMPORARY_SECONDARY_STORE).toString(); + if (filePath == null) { + throw new CloudRuntimeException("Failed to create temporary directory to mount secondary store"); + } + mountStore(url, filePath, nfsVersion); + int lastIdx = path.lastIndexOf(File.separator); + String partialDirPath = path.substring(0, lastIdx); + String templatePath = filePath + File.separator + partialDirPath; + File templateProps = new File(templatePath + "/template.properties"); + if (templateProps.exists()) { + Pair templateSizes = readTemplatePropertiesSizes(templatePath + "/template.properties"); + updateSeededTemplateDetails(templDataStoreVO.getTemplateId(), templDataStoreVO.getDataStoreId(), + templateSizes.first(), templateSizes.second()); + LOGGER.info("System VM template already seeded, skipping registration"); + return true; + } + LOGGER.info("System VM template not seeded"); + return false; + } catch (Exception e) { + LOGGER.error("Failed to verify if the template is seeded", e); + throw new CloudRuntimeException("Failed to verify if the template is seeded", e); + } finally { + unmountStore(filePath); + try { + Files.delete(Path.of(filePath)); + } catch (IOException e) { + LOGGER.error("Failed to delete temporary directory: {}", filePath); + } + } + } + + /** + * Finds a registered system VM template matching the provided criteria. + * + *

The method first attempts to locate the latest template by {@code templateName}, + * {@code hypervisorType} and {@code arch}. If none is found and a non-blank {@code url} + * is provided, it falls back to searching for an active system template by the + * URL path segment (the substring after the last '/' in the URL).

+ * + * @param templateName the template name to search for + * @param hypervisorType the hypervisor type + * @param arch the CPU architecture + * @param url optional download URL used as a fallback; may be {@code null} or blank + * @return the matching {@code VMTemplateVO} if found; {@code null} otherwise + */ + public VMTemplateVO getRegisteredTemplate(String templateName, Hypervisor.HypervisorType hypervisorType, + CPU.CPUArch arch, String url) { + VMTemplateVO registeredTemplate = vmTemplateDao.findLatestTemplateByName(templateName, hypervisorType, arch); + if (registeredTemplate == null && StringUtils.isNotBlank(url)) { + String urlPath = url.substring(url.lastIndexOf("/") + 1); + LOGGER.debug("No template found by name, falling back to search existing SYSTEM template by " + + "urlPath: {}, hypervisor: {}, arch:{}", urlPath, hypervisorType, arch); + registeredTemplate = vmTemplateDao.findActiveSystemTemplateByHypervisorArchAndUrlPath(hypervisorType, arch, + urlPath); + } + LOGGER.debug("Found existing registered template for hypervisor: {}, arch: {}: {}", hypervisorType, + arch, registeredTemplate); + return registeredTemplate; + } + + /** + * Update or register system VM templates based on metadata. + * + * Runs the registration logic inside a database transaction: obtains the + * set of hypervisors/architectures in use, iterates over metadata entries + * and attempts to register or update each template. + * + * @param conn retained for compatibility with callers (not used directly) + */ public void updateSystemVmTemplates(final Connection conn) { - LOGGER.debug("Updating System Vm template IDs"); - updateSystemVmTemplateGuestOsId(); + LOGGER.debug("Updating System VM templates"); + updateHypervisorGuestOsMap(); Transaction.execute(new TransactionCallbackNoReturn() { @Override public void doInTransactionWithoutResult(final TransactionStatus status) { @@ -1100,10 +1245,9 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { } catch (final Exception e) { throw new CloudRuntimeException("Exception while getting hypervisor types from clusters", e); } - Collection templateEntries = NewTemplateMap.values(); - for (MetadataTemplateDetails templateDetails : templateEntries) { + for (MetadataTemplateDetails templateDetails : METADATA_TEMPLATE_LIST) { try { - if (registerOrUpdateSystemVmTemplate(templateDetails, hypervisorsInUse)) { + if (updateOrRegisterSystemVmTemplate(templateDetails, hypervisorsInUse)) { break; } } catch (final Exception e) { @@ -1112,24 +1256,11 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { throw new CloudRuntimeException(errMsg, e); } } - LOGGER.debug("Updating System Vm Template IDs Complete"); + LOGGER.debug("Updating System VM Templates Complete"); } }); } - public String getNfsVersion(long storeId) { - final String configKey = "secstorage.nfs.version"; - final Map storeDetails = imageStoreDetailsDao.getDetails(storeId); - if (storeDetails != null && storeDetails.containsKey(configKey)) { - return storeDetails.get(configKey); - } - ConfigurationVO globalNfsVersion = configurationDao.findByName(configKey); - if (globalNfsVersion != null) { - return globalNfsVersion.getValue(); - } - return null; - } - protected static class MetadataTemplateDetails { private final Hypervisor.HypervisorType hypervisorType; private final String name; @@ -1172,7 +1303,7 @@ public String getChecksum() { } public CPU.CPUArch getArch() { - return arch != null ? arch : CPU.CPUArch.getDefault(); + return arch; } public String getGuestOs() { @@ -1191,6 +1322,16 @@ public String getDefaultFilePath() { return TEMPLATES_PATH + filename; } + public boolean isFileChecksumDifferent(File file) { + String fileChecksum = DigestHelper.calculateChecksum(file); + if (!fileChecksum.equals(getChecksum())) { + LOGGER.error("Checksum {} for file {} does not match checksum {} from metadata", + fileChecksum, file, getChecksum()); + return true; + } + return false; + } + public String getHypervisorArchLog() { return SystemVmTemplateRegistration.getHypervisorArchLog(hypervisorType, arch); } diff --git a/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java b/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java index 030276e1765a..aa8c3bee5834 100644 --- a/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java +++ b/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java @@ -31,6 +31,7 @@ import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -46,6 +47,7 @@ import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.utils.security.DigestHelper; import org.apache.commons.lang3.StringUtils; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; @@ -80,6 +82,11 @@ public class SystemVmTemplateRegistrationTest { @InjectMocks SystemVmTemplateRegistration systemVmTemplateRegistration = new SystemVmTemplateRegistration(); + @Before + public void setup() { + SystemVmTemplateRegistration.METADATA_TEMPLATE_LIST.clear(); + } + private void setupMetadataFile(MockedStatic mockedStatic, String content) { try { String location = "metadata.ini"; @@ -102,7 +109,7 @@ public void test_parseMetadataFile_noFile() { setupMetadataFile(mockedStatic, null); CloudRuntimeException exception = assertThrows(CloudRuntimeException.class, SystemVmTemplateRegistration::parseMetadataFile); - assertTrue(exception.getMessage().contains("Failed to parse systemVM template metadata file")); + assertTrue(exception.getMessage().startsWith("Failed to parse system VM template metadata file")); } } @@ -113,7 +120,7 @@ public void test_parseMetadataFile_invalidContent() { setupMetadataFile(mockedStatic, "abc"); CloudRuntimeException exception = assertThrows(CloudRuntimeException.class, SystemVmTemplateRegistration::parseMetadataFile); - assertTrue(exception.getMessage().contains("Failed to parse systemVM template metadata file")); + assertTrue(exception.getMessage().startsWith("Failed to parse system VM template metadata file")); } } @@ -145,21 +152,25 @@ public void test_parseMetadataFile_success() { String version = SystemVmTemplateRegistration.parseMetadataFile(); assertEquals("x.y.z.0", version); } - assertNull(SystemVmTemplateRegistration.NewTemplateMap.get("xenserver")); + assertNull(SystemVmTemplateRegistration.getMetadataTemplateDetails(Hypervisor.HypervisorType.XenServer, + CPU.CPUArch.getDefault())); SystemVmTemplateRegistration.MetadataTemplateDetails templateDetails = - SystemVmTemplateRegistration.NewTemplateMap.get("kvm-x86_64"); + SystemVmTemplateRegistration.getMetadataTemplateDetails(Hypervisor.HypervisorType.KVM, + CPU.CPUArch.amd64); assertNotNull(templateDetails); assertEquals(CPU.CPUArch.amd64, templateDetails.getArch()); assertEquals(Hypervisor.HypervisorType.KVM, templateDetails.getHypervisorType()); templateDetails = - SystemVmTemplateRegistration.NewTemplateMap.get("kvm-aarch64"); + SystemVmTemplateRegistration.getMetadataTemplateDetails(Hypervisor.HypervisorType.KVM, + CPU.CPUArch.arm64); assertNotNull(templateDetails); assertEquals(CPU.CPUArch.arm64, templateDetails.getArch()); assertEquals(Hypervisor.HypervisorType.KVM, templateDetails.getHypervisorType()); templateDetails = - SystemVmTemplateRegistration.NewTemplateMap.get("vmware"); + SystemVmTemplateRegistration.getMetadataTemplateDetails(Hypervisor.HypervisorType.VMware, + CPU.CPUArch.getDefault()); assertNotNull(templateDetails); - assertNull(templateDetails.getArch()); + assertEquals(CPU.CPUArch.getDefault(), templateDetails.getArch()); assertEquals(Hypervisor.HypervisorType.VMware, templateDetails.getHypervisorType()); } @@ -197,11 +208,10 @@ public void testValidateTemplateFile_fileNotFound() { SystemVmTemplateRegistration.MetadataTemplateDetails details = new SystemVmTemplateRegistration.MetadataTemplateDetails(Hypervisor.HypervisorType.KVM, "name", "file", "url", "checksum", CPU.CPUArch.amd64, "guestos"); - SystemVmTemplateRegistration.NewTemplateMap.put(SystemVmTemplateRegistration.getHypervisorArchKey( - details.getHypervisorType(), details.getArch()), details); + SystemVmTemplateRegistration.METADATA_TEMPLATE_LIST.add(details); doReturn(null).when(systemVmTemplateRegistration).getTemplateFile(details); try { - systemVmTemplateRegistration.validateTemplateFileForHypervisorAndArch(details.getHypervisorType(), + systemVmTemplateRegistration.getValidatedTemplateDetailsForHypervisorAndArch(details.getHypervisorType(), details.getArch()); fail("Expected CloudRuntimeException due to missing template file"); } catch (CloudRuntimeException e) { @@ -215,12 +225,11 @@ public void testValidateTemplateFile_checksumMismatch() { new SystemVmTemplateRegistration.MetadataTemplateDetails(Hypervisor.HypervisorType.KVM, "name", "file", "url", "checksum", CPU.CPUArch.amd64, "guestos"); File dummyFile = new File("dummy.txt"); - SystemVmTemplateRegistration.NewTemplateMap.put(SystemVmTemplateRegistration.getHypervisorArchKey( - details.getHypervisorType(), details.getArch()), details); + SystemVmTemplateRegistration.METADATA_TEMPLATE_LIST.add(details); doReturn(dummyFile).when(systemVmTemplateRegistration).getTemplateFile(details); - doReturn(true).when(systemVmTemplateRegistration).isTemplateFileChecksumDifferent(details, dummyFile); - try { - systemVmTemplateRegistration.validateTemplateFileForHypervisorAndArch(details.getHypervisorType(), + try (MockedStatic digestMock = Mockito.mockStatic(DigestHelper.class)) { + digestMock.when(() -> DigestHelper.calculateChecksum(dummyFile)).thenReturn("differentChecksum"); + systemVmTemplateRegistration.getValidatedTemplateDetailsForHypervisorAndArch(details.getHypervisorType(), details.getArch()); fail("Expected CloudRuntimeException due to checksum failure"); } catch (CloudRuntimeException e) { @@ -234,42 +243,55 @@ public void testValidateTemplateFile_success() { new SystemVmTemplateRegistration.MetadataTemplateDetails(Hypervisor.HypervisorType.KVM, "name", "file", "url", "checksum", CPU.CPUArch.amd64, "guestos"); File dummyFile = new File("dummy.txt"); - SystemVmTemplateRegistration.NewTemplateMap.put(SystemVmTemplateRegistration.getHypervisorArchKey( - details.getHypervisorType(), details.getArch()), details); + SystemVmTemplateRegistration.METADATA_TEMPLATE_LIST.add(details); doReturn(dummyFile).when(systemVmTemplateRegistration).getTemplateFile(details); - doReturn(false).when(systemVmTemplateRegistration).isTemplateFileChecksumDifferent(details, dummyFile); - systemVmTemplateRegistration.validateTemplateFileForHypervisorAndArch(details.getHypervisorType(), - details.getArch()); + try (MockedStatic digestMock = Mockito.mockStatic(DigestHelper.class)) { + digestMock.when(() -> DigestHelper.calculateChecksum(dummyFile)).thenReturn("checksum"); + systemVmTemplateRegistration.getValidatedTemplateDetailsForHypervisorAndArch(details.getHypervisorType(), + details.getArch()); + } } @Test - public void testValidateAndRegisterTemplate() { + public void testValidateAndAddExistingTemplateToStore() { + long zoneId = 1L; Hypervisor.HypervisorType hypervisor = Hypervisor.HypervisorType.KVM; - String name = "TestTemplate"; - Long storeId = 123L; VMTemplateVO templateVO = new VMTemplateVO(); - templateVO.setArch(CPU.CPUArch.x86); + templateVO.setHypervisorType(hypervisor); + templateVO.setArch(CPU.CPUArch.getDefault()); TemplateDataStoreVO templateDataStoreVO = new TemplateDataStoreVO(); + Long storeId = 123L; String filePath = "/dummy/path"; - doNothing().when(systemVmTemplateRegistration).validateTemplateFileForHypervisorAndArch(hypervisor, templateVO.getArch()); - doNothing().when(systemVmTemplateRegistration).registerTemplate(hypervisor, name, storeId, templateVO, templateDataStoreVO, filePath); - systemVmTemplateRegistration.validateAndRegisterTemplate(hypervisor, name, storeId, templateVO, templateDataStoreVO, filePath); - verify(systemVmTemplateRegistration).validateTemplateFileForHypervisorAndArch(eq(hypervisor), eq(templateVO.getArch())); - verify(systemVmTemplateRegistration).registerTemplate(eq(hypervisor), eq(name), eq(storeId), eq(templateVO), eq(templateDataStoreVO), eq(filePath)); + SystemVmTemplateRegistration.MetadataTemplateDetails details = + mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class); + doReturn(details).when(systemVmTemplateRegistration) + .getValidatedTemplateDetailsForHypervisorAndArch(hypervisor, templateVO.getArch()); + doNothing().when(systemVmTemplateRegistration).addExistingTemplateToStore(templateVO, details, + templateDataStoreVO, zoneId, storeId, filePath); + systemVmTemplateRegistration.validateAndAddTemplateToStore(templateVO, templateDataStoreVO, zoneId, storeId, + filePath); + verify(systemVmTemplateRegistration) + .getValidatedTemplateDetailsForHypervisorAndArch(hypervisor, templateVO.getArch()); + verify(systemVmTemplateRegistration).addExistingTemplateToStore(templateVO, details, templateDataStoreVO, + zoneId, storeId, filePath); } @Test - public void testValidateAndRegisterTemplateForNonExistingEntries() { + public void testValidateAndAddExistingTemplateToStoreForNonExistingEntries() { + long zoneId = 1L; Hypervisor.HypervisorType hypervisor = Hypervisor.HypervisorType.KVM; CPU.CPUArch arch = CPU.CPUArch.amd64; String name = "TestTemplateNonExisting"; - Pair storeUrlAndId = new Pair<>("nfs://dummy", 456L); + long storeId = 123L; String filePath = "/dummy/path/nonexisting"; - doNothing().when(systemVmTemplateRegistration).validateTemplateFileForHypervisorAndArch(hypervisor, arch); - doNothing().when(systemVmTemplateRegistration).registerTemplateForNonExistingEntries(hypervisor, arch, name, storeUrlAndId, filePath); - systemVmTemplateRegistration.validateAndRegisterTemplateForNonExistingEntries(hypervisor, arch, name, storeUrlAndId, filePath); - verify(systemVmTemplateRegistration).validateTemplateFileForHypervisorAndArch(eq(hypervisor), eq(arch)); - verify(systemVmTemplateRegistration).registerTemplateForNonExistingEntries(eq(hypervisor), eq(arch), eq(name), eq(storeUrlAndId), eq(filePath)); + SystemVmTemplateRegistration.MetadataTemplateDetails details = + mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class); + doReturn(details).when(systemVmTemplateRegistration) + .getValidatedTemplateDetailsForHypervisorAndArch(hypervisor, arch); + doNothing().when(systemVmTemplateRegistration).registerNewTemplate(name, details, zoneId, storeId, filePath); + systemVmTemplateRegistration.validateAndRegisterNewTemplate(hypervisor, arch, name, zoneId, storeId, filePath); + verify(systemVmTemplateRegistration) .getValidatedTemplateDetailsForHypervisorAndArch(hypervisor, arch); + verify(systemVmTemplateRegistration).registerNewTemplate(name, details, zoneId, storeId, filePath); } @Test @@ -320,83 +342,67 @@ public void testGetTemplateFile_fileDoesNotExist_downloadSucceeds() { } @Test - public void testIsTemplateFileChecksumDifferent_noMismatch() { - SystemVmTemplateRegistration.MetadataTemplateDetails details = - Mockito.mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class); - when(details.getChecksum()).thenReturn("dummyChecksum"); - File file = new File("dummy.txt"); - try (MockedStatic digestMock = Mockito.mockStatic(DigestHelper.class)) { - digestMock.when(() -> DigestHelper.calculateChecksum(file)).thenReturn("dummyChecksum"); - boolean result = systemVmTemplateRegistration.isTemplateFileChecksumDifferent(details, file); - assertFalse(result); - } - } - - @Test - public void testIsTemplateFileChecksumDifferent_mismatch() { - SystemVmTemplateRegistration.MetadataTemplateDetails details = - Mockito.mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class); - when(details.getChecksum()).thenReturn("expectedChecksum"); - File file = new File("dummy.txt"); - try (MockedStatic digestMock = Mockito.mockStatic(DigestHelper.class)) { - digestMock.when(() -> DigestHelper.calculateChecksum(file)).thenReturn("actualChecksum"); - boolean result = systemVmTemplateRegistration.isTemplateFileChecksumDifferent(details, file); - assertTrue(result); - } - } - - @Test(expected = CloudRuntimeException.class) - public void testValidateTemplates_metadataTemplateFailure() { + public void testValidateTemplates_metadataTemplateSkip() { + Hypervisor.HypervisorType hypervisorType = Hypervisor.HypervisorType.VMware; + CPU.CPUArch arch = CPU.CPUArch.arm64; List> list = new ArrayList<>(); - list.add(new Pair<>(Hypervisor.HypervisorType.VMware, CPU.CPUArch.arm64)); + list.add(new Pair<>(hypervisorType, arch)); systemVmTemplateRegistration.validateTemplates(list); + verify(systemVmTemplateRegistration, never()).getValidatedTemplateDetailsForHypervisorAndArch(hypervisorType, + arch); } @Test(expected = CloudRuntimeException.class) public void testValidateTemplates_fileFailure() { + Hypervisor.HypervisorType hypervisorType = Hypervisor.HypervisorType.KVM; + CPU.CPUArch arch = CPU.CPUArch.amd64; List> list = new ArrayList<>(); - list.add(new Pair<>(Hypervisor.HypervisorType.KVM, CPU.CPUArch.amd64)); - + list.add(new Pair<>(hypervisorType, arch)); SystemVmTemplateRegistration.MetadataTemplateDetails details = Mockito.mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class); - SystemVmTemplateRegistration.NewTemplateMap.put(SystemVmTemplateRegistration.getHypervisorArchKey( - Hypervisor.HypervisorType.KVM, CPU.CPUArch.amd64), details); + when(details.getHypervisorType()).thenReturn(hypervisorType); + when(details.getArch()).thenReturn(arch); File mockFile = Mockito.mock(File.class); + when(details.isFileChecksumDifferent(mockFile)).thenReturn(true); + SystemVmTemplateRegistration.METADATA_TEMPLATE_LIST.add(details); doReturn(mockFile).when(systemVmTemplateRegistration).getTemplateFile(details); - doReturn(true).when(systemVmTemplateRegistration).isTemplateFileChecksumDifferent(details, mockFile); systemVmTemplateRegistration.validateTemplates(list); } - @Test + @Test(expected = CloudRuntimeException.class) public void testValidateTemplates_downloadableFileNotFound() { CPU.CPUArch arch = SystemVmTemplateRegistration.DOWNLOADABLE_TEMPLATE_ARCH_TYPES.get(0); List> list = new ArrayList<>(); - list.add(new Pair<>(Hypervisor.HypervisorType.KVM, arch)); + Hypervisor.HypervisorType hypervisorType = Hypervisor.HypervisorType.KVM; + list.add(new Pair<>(hypervisorType, arch)); SystemVmTemplateRegistration.MetadataTemplateDetails details = Mockito.mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class); - SystemVmTemplateRegistration.NewTemplateMap.put(SystemVmTemplateRegistration.getHypervisorArchKey( - Hypervisor.HypervisorType.KVM, arch), details); + when(details.getHypervisorType()).thenReturn(hypervisorType); + when(details.getArch()).thenReturn(arch); + SystemVmTemplateRegistration.METADATA_TEMPLATE_LIST.add(details); doReturn(null).when(systemVmTemplateRegistration).getTemplateFile(details); systemVmTemplateRegistration.validateTemplates(list); } @Test public void testValidateTemplates_success() { + Hypervisor.HypervisorType hypervisorType = Hypervisor.HypervisorType.KVM; + CPU.CPUArch arch = CPU.CPUArch.amd64; List> list = new ArrayList<>(); - list.add(new Pair<>(Hypervisor.HypervisorType.KVM, CPU.CPUArch.amd64)); - + list.add(new Pair<>(hypervisorType, arch)); SystemVmTemplateRegistration.MetadataTemplateDetails details = Mockito.mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class); - SystemVmTemplateRegistration.NewTemplateMap.put(SystemVmTemplateRegistration.getHypervisorArchKey( - Hypervisor.HypervisorType.KVM, CPU.CPUArch.amd64), details); + when(details.getHypervisorType()).thenReturn(hypervisorType); + when(details.getArch()).thenReturn(arch); File mockFile = Mockito.mock(File.class); + when(details.isFileChecksumDifferent(mockFile)).thenReturn(false); + SystemVmTemplateRegistration.METADATA_TEMPLATE_LIST.add(details); doReturn(mockFile).when(systemVmTemplateRegistration).getTemplateFile(details); - doReturn(false).when(systemVmTemplateRegistration).isTemplateFileChecksumDifferent(details, mockFile); systemVmTemplateRegistration.validateTemplates(list); } @Test - public void testRegisterTemplatesForZone() { + public void testAddExistingTemplatesForZoneToStore() { long zoneId = 1L; String filePath = "dummyFilePath"; String nfsVersion = "nfs3"; @@ -419,19 +425,18 @@ public void testRegisterTemplatesForZone() { when(details.getUrl()).thenReturn(url); mockedStatic.when(() -> SystemVmTemplateRegistration.getMetadataTemplateDetails(Mockito.any(), Mockito.any())).thenReturn(details); - doNothing().when(systemVmTemplateRegistration).registerTemplateForNonExistingEntries( - hypervisorType, arch, - name, storeUrlAndId, filePath); + doNothing().when(systemVmTemplateRegistration).registerNewTemplate(name, details, zoneId, + storeUrlAndId.second(), filePath); systemVmTemplateRegistration.registerTemplatesForZone(zoneId, filePath); mockedStatic.verify(() -> SystemVmTemplateRegistration.mountStore(storeUrlAndId.first(), filePath, nfsVersion)); - verify(systemVmTemplateRegistration).registerTemplateForNonExistingEntries(hypervisorType, - arch, name, storeUrlAndId, filePath); + verify(systemVmTemplateRegistration).registerNewTemplate(name, details, zoneId, + storeUrlAndId.second(), filePath); } } @Test - public void registerOrUpdateSystemVmTemplate_UpdatesRegisteredTemplate() { + public void updateOrRegisterSystemVmTemplate_UpdatesRegisteredTemplate() { SystemVmTemplateRegistration.MetadataTemplateDetails templateDetails = Mockito.mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class); when(templateDetails.getName()).thenReturn("templateName"); when(templateDetails.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); @@ -441,16 +446,18 @@ public void registerOrUpdateSystemVmTemplate_UpdatesRegisteredTemplate() { when(registeredTemplate.getId()).thenReturn(1L); doReturn(registeredTemplate).when(systemVmTemplateRegistration).getRegisteredTemplate( "templateName", Hypervisor.HypervisorType.KVM, CPU.CPUArch.amd64, "http://example.com/template"); - doNothing().when(systemVmTemplateRegistration).updateRegisteredTemplateDetails(1L, templateDetails); + doNothing().when(systemVmTemplateRegistration).updateRegisteredTemplateDetails(1L, templateDetails, + null); - boolean result = systemVmTemplateRegistration.registerOrUpdateSystemVmTemplate(templateDetails, new ArrayList<>()); + boolean result = systemVmTemplateRegistration.updateOrRegisterSystemVmTemplate(templateDetails, + new ArrayList<>()); assertFalse(result); - verify(systemVmTemplateRegistration).updateRegisteredTemplateDetails(eq(1L), eq(templateDetails)); + verify(systemVmTemplateRegistration).updateRegisteredTemplateDetails(1L, templateDetails, null); } @Test - public void registerOrUpdateSystemVmTemplate_SkipsUnusedHypervisorArch() { + public void updateOrRegisterSystemVmTemplate_SkipsUnusedHypervisorArch() { SystemVmTemplateRegistration.MetadataTemplateDetails templateDetails = Mockito.mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class); when(templateDetails.getName()).thenReturn("templateName"); when(templateDetails.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); @@ -461,14 +468,14 @@ public void registerOrUpdateSystemVmTemplate_SkipsUnusedHypervisorArch() { doReturn(null).when(vmTemplateDao).findLatestTemplateByTypeAndHypervisorAndArch( Hypervisor.HypervisorType.KVM, CPU.CPUArch.amd64, Storage.TemplateType.SYSTEM); - boolean result = systemVmTemplateRegistration.registerOrUpdateSystemVmTemplate(templateDetails, new ArrayList<>()); + boolean result = systemVmTemplateRegistration.updateOrRegisterSystemVmTemplate(templateDetails, new ArrayList<>()); assertFalse(result); verify(systemVmTemplateRegistration, never()).registerTemplates(anyList()); } @Test - public void registerOrUpdateSystemVmTemplate_RegistersNewTemplate() { + public void updateOrRegisterSystemVmTemplate_RegistersNewTemplate() { SystemVmTemplateRegistration.MetadataTemplateDetails templateDetails = Mockito.mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class); when(templateDetails.getName()).thenReturn("templateName"); when(templateDetails.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); @@ -480,14 +487,14 @@ public void registerOrUpdateSystemVmTemplate_RegistersNewTemplate() { hypervisorsInUse.add(new Pair<>(Hypervisor.HypervisorType.KVM, CPU.CPUArch.amd64)); doNothing().when(systemVmTemplateRegistration).registerTemplates(hypervisorsInUse); - boolean result = systemVmTemplateRegistration.registerOrUpdateSystemVmTemplate(templateDetails, hypervisorsInUse); + boolean result = systemVmTemplateRegistration.updateOrRegisterSystemVmTemplate(templateDetails, hypervisorsInUse); assertTrue(result); verify(systemVmTemplateRegistration).registerTemplates(eq(hypervisorsInUse)); } @Test - public void registerOrUpdateSystemVmTemplate_ThrowsExceptionOnRegistrationFailure() { + public void updateOrRegisterSystemVmTemplate_ThrowsExceptionOnRegistrationFailure() { SystemVmTemplateRegistration.MetadataTemplateDetails templateDetails = Mockito.mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class); when(templateDetails.getName()).thenReturn("templateName"); when(templateDetails.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); @@ -500,7 +507,7 @@ public void registerOrUpdateSystemVmTemplate_ThrowsExceptionOnRegistrationFailur doThrow(new CloudRuntimeException("Registration failed")).when(systemVmTemplateRegistration).registerTemplates(hypervisorsInUse); CloudRuntimeException exception = assertThrows(CloudRuntimeException.class, - () -> systemVmTemplateRegistration.registerOrUpdateSystemVmTemplate(templateDetails, hypervisorsInUse)); + () -> systemVmTemplateRegistration.updateOrRegisterSystemVmTemplate(templateDetails, hypervisorsInUse)); assertTrue(exception.getMessage().contains("Failed to register")); } diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 45071cc5cb0e..31a056fa49ef 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -4025,56 +4025,57 @@ protected void registerSystemVmTemplateForHypervisorArch(final HypervisorType hy } } SystemVmTemplateRegistration.mountStore(storeUrlAndId.first(), filePath, nfsVersion); - if (templateDataStoreVO != null) { - systemVmTemplateRegistration.validateAndRegisterTemplate(hypervisorType, templateName, - storeUrlAndId.second(), registeredTemplate, templateDataStoreVO, filePath); + if (registeredTemplate != null) { + systemVmTemplateRegistration.validateAndAddTemplateToStore(registeredTemplate, templateDataStoreVO, zoneId, + storeUrlAndId.second(), filePath); } else { - systemVmTemplateRegistration.validateAndRegisterTemplateForNonExistingEntries(hypervisorType, arch, - templateName, storeUrlAndId, filePath); + systemVmTemplateRegistration.validateAndRegisterNewTemplate(hypervisorType, arch, templateName, zoneId, + storeUrlAndId.second(), filePath); } } private void registerSystemVmTemplateOnFirstNfsStore(Long zoneId, String providerName, String url, DataStore store) { - if (DataStoreProvider.NFS_IMAGE.equals(providerName) && zoneId != null) { - Transaction.execute(new TransactionCallbackNoReturn() { - @Override - public void doInTransactionWithoutResult(final TransactionStatus status) { - List stores = _imageStoreDao.listAllStoresInZoneExceptId(zoneId, providerName, - DataStoreRole.Image, store.getId()); - if (CollectionUtils.isEmpty(stores)) { - List> hypervisorTypes = - _clusterDao.listDistinctHypervisorsAndArchExcludingExternalType(zoneId); - TransactionLegacy txn = TransactionLegacy.open("AutomaticTemplateRegister"); - SystemVmTemplateRegistration systemVmTemplateRegistration = new SystemVmTemplateRegistration(); - String filePath = null; - try { - filePath = Files.createTempDirectory(SystemVmTemplateRegistration.TEMPORARY_SECONDARY_STORE).toString(); - if (filePath == null) { - throw new CloudRuntimeException("Failed to create temporary file path to mount the store"); - } - Pair storeUrlAndId = new Pair<>(url, store.getId()); - String nfsVersion = imageStoreDetailsUtil.getNfsVersion(store.getId()); - for (Pair hypervisorArchType : hypervisorTypes) { - try { - registerSystemVmTemplateForHypervisorArch(hypervisorArchType.first(), - hypervisorArchType.second(), zoneId, url, store, - systemVmTemplateRegistration, filePath, storeUrlAndId, nfsVersion); - } catch (CloudRuntimeException e) { - SystemVmTemplateRegistration.unmountStore(filePath); - logger.error("Failed to register system VM template for hypervisor: {} {}", - hypervisorArchType.first().name(), hypervisorArchType.second().name(), e); - } + if (zoneId == null || !DataStoreProvider.NFS_IMAGE.equals(providerName)) { + logger.debug("Skipping system VM template registration as either zoneId is null or {} " + + "provider is not NFS", store); + return; + } + Transaction.execute(new TransactionCallbackNoReturn() { + @Override + public void doInTransactionWithoutResult(final TransactionStatus status) { + List stores = _imageStoreDao.listAllStoresInZoneExceptId(zoneId, providerName, + DataStoreRole.Image, store.getId()); + if (CollectionUtils.isEmpty(stores)) { + List> hypervisorArchTypes = + _clusterDao.listDistinctHypervisorsAndArchExcludingExternalType(zoneId); + TransactionLegacy txn = TransactionLegacy.open("AutomaticTemplateRegister"); + SystemVmTemplateRegistration systemVmTemplateRegistration = new SystemVmTemplateRegistration(); + String filePath = null; + try { + filePath = Files.createTempDirectory(SystemVmTemplateRegistration.TEMPORARY_SECONDARY_STORE) + .toString(); + Pair storeUrlAndId = new Pair<>(url, store.getId()); + String nfsVersion = imageStoreDetailsUtil.getNfsVersion(store.getId()); + for (Pair hypervisorArchType : hypervisorArchTypes) { + try { + registerSystemVmTemplateForHypervisorArch(hypervisorArchType.first(), + hypervisorArchType.second(), zoneId, url, store, + systemVmTemplateRegistration, filePath, storeUrlAndId, nfsVersion); + } catch (CloudRuntimeException e) { + SystemVmTemplateRegistration.unmountStore(filePath); + logger.error("Failed to register system VM template for hypervisor: {} {}", + hypervisorArchType.first().name(), hypervisorArchType.second().name(), e); } - } catch (Exception e) { - logger.error("Failed to register systemVM template(s) due to: ", e); - } finally { - SystemVmTemplateRegistration.unmountStore(filePath); - txn.close(); } + } catch (Exception e) { + logger.error("Failed to register systemVM template(s) due to: ", e); + } finally { + SystemVmTemplateRegistration.unmountStore(filePath); + txn.close(); } } - }); - } + } + }); } @Override public ImageStore migrateToObjectStore(String name, String url, String providerName, Map details) throws DiscoveryException, InvalidParameterValueException { From b0dd56fd1768f71d2731d02474a315f585d43781 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 12 Dec 2025 13:24:33 +0530 Subject: [PATCH 11/12] add tests Signed-off-by: Abhishek Kumar --- .../upgrade/SystemVmTemplateRegistration.java | 170 ++- .../SystemVmTemplateRegistrationTest.java | 1127 ++++++++++++++++- 2 files changed, 1229 insertions(+), 68 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index 3597cba5e612..a0ad2f98c65c 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -105,9 +105,9 @@ public class SystemVmTemplateRegistration { private static final String METADATA_FILE = TEMPLATES_PATH + METADATA_FILE_NAME; public static final String TEMPORARY_SECONDARY_STORE = "tmp"; private static final String PARTIAL_TEMPLATE_FOLDER = String.format("/template/tmpl/%d/", Account.ACCOUNT_ID_SYSTEM); - private static final String storageScriptsDir = "scripts/storage/secondary"; + protected static final String STORAGE_SCRIPTS_DIR = "scripts/storage/secondary"; private static final Integer OTHER_LINUX_ID = 99; - private static Integer LINUX_12_ID = 363; + protected static Integer LINUX_12_ID = 363; private static final Integer SCRIPT_TIMEOUT = 1800000; private static final Integer LOCK_WAIT_TIMEOUT = 1200; protected static final String TEMPLATE_DOWNLOAD_URL_KEY = "downloadurl"; @@ -327,6 +327,17 @@ public void setUpdated(Date updated) { } }; + protected static final Map HYPERVISOR_IMAGE_FORMAT_MAP = new HashMap<>() { + { + put(Hypervisor.HypervisorType.KVM, ImageFormat.QCOW2); + put(Hypervisor.HypervisorType.XenServer, ImageFormat.VHD); + put(Hypervisor.HypervisorType.VMware, ImageFormat.OVA); + put(Hypervisor.HypervisorType.Hyperv, ImageFormat.VHD); + put(Hypervisor.HypervisorType.LXC, ImageFormat.QCOW2); + put(Hypervisor.HypervisorType.Ovm3, ImageFormat.RAW); + } + }; + protected static Map hypervisorGuestOsMap = new HashMap<>() { { put(Hypervisor.HypervisorType.KVM, LINUX_12_ID); @@ -338,17 +349,6 @@ public void setUpdated(Date updated) { } }; - protected static final Map hypervisorImageFormat = new HashMap() { - { - put(Hypervisor.HypervisorType.KVM, ImageFormat.QCOW2); - put(Hypervisor.HypervisorType.XenServer, ImageFormat.VHD); - put(Hypervisor.HypervisorType.VMware, ImageFormat.OVA); - put(Hypervisor.HypervisorType.Hyperv, ImageFormat.VHD); - put(Hypervisor.HypervisorType.LXC, ImageFormat.QCOW2); - put(Hypervisor.HypervisorType.Ovm3, ImageFormat.RAW); - } - }; - private static boolean isRunningInTest() { return "true".equalsIgnoreCase(System.getProperty("test.mode")); } @@ -399,7 +399,7 @@ private static String fetchTemplatesPath() { throw new CloudRuntimeException(errMsg); } - private static void cleanupStore(Long templateId, String filePath) { + protected static void cleanupStore(Long templateId, String filePath) { String destTempFolder = filePath + PARTIAL_TEMPLATE_FOLDER + String.valueOf(templateId); try { Files.deleteIfExists(Paths.get(destTempFolder)); @@ -408,7 +408,7 @@ private static void cleanupStore(Long templateId, String filePath) { } } - private static Pair readTemplatePropertiesSizes(String path) { + protected static Pair readTemplatePropertiesSizes(String path) { File tmpFile = new File(path); Long size = null; Long physicalSize = 0L; @@ -430,14 +430,8 @@ private static Pair readTemplatePropertiesSizes(String path) { return new Pair<>(size, physicalSize); } - private static void readTemplateProperties(String path, SystemVMTemplateDetails details) { - Pair templateSizes = readTemplatePropertiesSizes(path); - details.setSize(templateSizes.first()); - details.setPhysicalSize(templateSizes.second()); - } - protected static MetadataTemplateDetails getMetadataTemplateDetails(Hypervisor.HypervisorType hypervisorType, - CPU.CPUArch arch) { + CPU.CPUArch arch) { return METADATA_TEMPLATE_LIST .stream() .filter(x -> Objects.equals(x.getHypervisorType(), hypervisorType) && @@ -525,10 +519,49 @@ public static String parseMetadataFile() { return defaultSection.get("version").trim(); } + public static void mountStore(String storeUrl, String path, String nfsVersion) { + try { + if (storeUrl == null) { + return; + } + URI uri = new URI(UriUtils.encodeURIComponent(storeUrl)); + String host = uri.getHost(); + String mountPath = uri.getPath(); + Script.runSimpleBashScript(getMountCommand(nfsVersion, host + ":" + mountPath, path)); + } catch (Exception e) { + String msg = "NFS Store URL is not in the correct format"; + LOGGER.error(msg, e); + throw new CloudRuntimeException(msg, e); + } + } + + public static void unmountStore(String filePath) { + try { + LOGGER.info("Unmounting store"); + String umountCmd = String.format(UMOUNT_COMMAND, filePath); + Script.runSimpleBashScript(umountCmd); + try { + Files.deleteIfExists(Paths.get(filePath)); + } catch (IOException e) { + LOGGER.error(String.format("Failed to cleanup mounted store at: %s", filePath), e); + } + } catch (Exception e) { + String msg = String.format("Failed to unmount store mounted at %s", filePath); + LOGGER.error(msg, e); + throw new CloudRuntimeException(msg, e); + } + } + protected File getTempDownloadDir() { return tempDownloadDir; } + protected void readTemplateProperties(String path, SystemVMTemplateDetails details) { + Pair templateSizes = readTemplatePropertiesSizes(path); + details.setSize(templateSizes.first()); + details.setPhysicalSize(templateSizes.second()); + } + protected List getEligibleZoneIds() { List zoneIds = new ArrayList<>(); List stores = imageStoreDao.findByProtocol("nfs"); @@ -587,7 +620,7 @@ private VMTemplateVO createTemplateObjectInDB(SystemVMTemplateDetails details) { return template; } - private VMTemplateZoneVO createOrUpdateTemplateZoneEntry(long zoneId, long templateId) { + protected VMTemplateZoneVO createOrUpdateTemplateZoneEntry(long zoneId, long templateId) { VMTemplateZoneVO templateZoneVO = vmTemplateZoneDao.findByZoneTemplate(zoneId, templateId); if (templateZoneVO == null) { templateZoneVO = new VMTemplateZoneVO(zoneId, templateId, new java.util.Date()); @@ -601,7 +634,7 @@ private VMTemplateZoneVO createOrUpdateTemplateZoneEntry(long zoneId, long templ return templateZoneVO; } - private void createCrossZonesTemplateZoneRefEntries(Long templateId) { + protected void createCrossZonesTemplateZoneRefEntries(Long templateId) { List dcs = dataCenterDao.listAll(); for (DataCenterVO dc : dcs) { VMTemplateZoneVO templateZoneVO = createOrUpdateTemplateZoneEntry(dc.getId(), templateId); @@ -612,7 +645,7 @@ private void createCrossZonesTemplateZoneRefEntries(Long templateId) { } } - private void createTemplateStoreRefEntry(SystemVMTemplateDetails details) { + protected void createTemplateStoreRefEntry(SystemVMTemplateDetails details) { TemplateDataStoreVO templateDataStoreVO = new TemplateDataStoreVO(details.getStoreId(), details.getId(), details.getCreated(), 0, VMTemplateStorageResourceAssoc.Status.NOT_DOWNLOADED, null, null, null, details.getInstallPath(), details.getUrl()); @@ -663,7 +696,7 @@ protected void updateSystemVMEntries(Long templateId, Hypervisor.HypervisorType vmInstanceDao.updateSystemVmTemplateId(templateId, hypervisorType); } - private void updateHypervisorGuestOsMap() { + protected void updateHypervisorGuestOsMap() { try { GuestOSVO guestOS = guestOSDao.findOneByDisplayName(DEFAULT_SYSTEM_VM_GUEST_OS_NAME); if (guestOS == null) { @@ -700,7 +733,7 @@ protected void updateConfigurationParams(Hypervisor.HypervisorType hypervisorTyp } } - private void updateTemplateEntriesOnFailure(long templateId) { + protected void updateTemplateEntriesOnFailure(long templateId) { VMTemplateVO template = vmTemplateDao.createForUpdate(templateId); template.setState(VirtualMachineTemplate.State.Inactive); vmTemplateDao.update(template.getId(), template); @@ -713,9 +746,9 @@ private void updateTemplateEntriesOnFailure(long templateId) { templateDataStoreDao.remove(templateDataStoreVO.getId()); } - private void setupTemplateOnStore(String templateName, MetadataTemplateDetails templateDetails, + protected void setupTemplateOnStore(String templateName, MetadataTemplateDetails templateDetails, String destTempFolder) throws CloudRuntimeException { - String setupTmpltScript = Script.findScript(storageScriptsDir, "setup-sysvm-tmplt"); + String setupTmpltScript = Script.findScript(STORAGE_SCRIPTS_DIR, "setup-sysvm-tmplt"); if (setupTmpltScript == null) { throw new CloudRuntimeException("Unable to find the setup-sysvm-tmplt script"); } @@ -735,9 +768,24 @@ private void setupTemplateOnStore(String templateName, MetadataTemplateDetails t } } + /** + * Register or update a system VM template record and seed it on the target store. + * + * @param name display name of the template + * @param templateDetails metadata for the template + * @param url download URL of the template + * @param checksum expected checksum of the template file + * @param format image format of the template + * @param guestOsId guest OS id + * @param storeId target image store id + * @param templateId existing template id if present, otherwise {@code null} + * @param filePath temporary mount path for the store + * @param templateDataStoreVO existing template-store mapping; may be {@code null} + * @return the id of the template that was created or updated + */ protected Long performTemplateRegistrationOperations(String name, MetadataTemplateDetails templateDetails, - String url, String checksum, ImageFormat format, long guestOsId, - Long storeId, Long templateId, String filePath, TemplateDataStoreVO templateDataStoreVO) { + String url, String checksum, ImageFormat format, long guestOsId, Long storeId, Long templateId, + String filePath, TemplateDataStoreVO templateDataStoreVO) { String templateName = UUID.randomUUID().toString(); Date created = new Date(DateUtil.currentGMTTime().getTime()); SystemVMTemplateDetails details = new SystemVMTemplateDetails(templateName, name, created, url, checksum, @@ -755,7 +803,9 @@ protected Long performTemplateRegistrationOperations(String name, MetadataTempla details.setId(templateId); String destTempFolderName = String.valueOf(templateId); String destTempFolder = filePath + PARTIAL_TEMPLATE_FOLDER + destTempFolderName; - details.setInstallPath(PARTIAL_TEMPLATE_FOLDER + destTempFolderName + File.separator + templateName + "." + hypervisorImageFormat.get(templateDetails.getHypervisorType()).getFileExtension()); + details.setInstallPath(String.format("%s%s%s%s.%s", PARTIAL_TEMPLATE_FOLDER, destTempFolderName, + File.separator, templateName, + HYPERVISOR_IMAGE_FORMAT_MAP.get(templateDetails.getHypervisorType()).getFileExtension())); if (templateDataStoreVO == null) { createTemplateStoreRefEntry(details); } @@ -807,7 +857,7 @@ protected void registerNewTemplate(String name, MetadataTemplateDetails template Hypervisor.HypervisorType hypervisor = templateDetails.getHypervisorType(); try { templateId = performTemplateRegistrationOperations(name, templateDetails, templateDetails.getUrl(), - templateDetails.getChecksum(), hypervisorImageFormat.get(hypervisor), + templateDetails.getChecksum(), HYPERVISOR_IMAGE_FORMAT_MAP.get(hypervisor), hypervisorGuestOsMap.get(hypervisor), storeId, null, filePath, null); updateConfigurationParams(hypervisor, name, zoneId); updateSystemVMEntries(templateId, hypervisor); @@ -913,6 +963,16 @@ protected void validateTemplates(List storeUrlAndId = getNfsStoreInZone(zoneId); String nfsVersion = getNfsVersion(storeUrlAndId.second()); @@ -986,6 +1046,16 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { } } + /** + * Update the DB record for an existing template to mark it as a system template, + * set the guest OS (if resolvable), and propagate the change to system VM entries + * and related configuration for the template's hypervisor. + * + * @param templateId id of the template to update + * @param templateDetails metadata used to update the template record + * @param zoneId zone id whose per-zone details (if any) should be cleared; may be null + * @throws CloudRuntimeException if updating the template record fails + */ protected void updateRegisteredTemplateDetails(Long templateId, MetadataTemplateDetails templateDetails, Long zoneId) { VMTemplateVO templateVO = vmTemplateDao.findById(templateId); @@ -1002,7 +1072,6 @@ protected void updateRegisteredTemplateDetails(Long templateId, MetadataTemplate } Hypervisor.HypervisorType hypervisorType = templateDetails.getHypervisorType(); updateSystemVMEntries(templateId, hypervisorType); - // Change value of global configuration parameter router.template.* for the corresponding hypervisor and minreq.sysvmtemplate.version for the ACS version updateConfigurationParams(hypervisorType, templateDetails.getName(), zoneId); } @@ -1086,39 +1155,6 @@ protected String getNfsVersion(long storeId) { return null; } - public static void mountStore(String storeUrl, String path, String nfsVersion) { - try { - if (storeUrl == null) { - return; - } - URI uri = new URI(UriUtils.encodeURIComponent(storeUrl)); - String host = uri.getHost(); - String mountPath = uri.getPath(); - Script.runSimpleBashScript(getMountCommand(nfsVersion, host + ":" + mountPath, path)); - } catch (Exception e) { - String msg = "NFS Store URL is not in the correct format"; - LOGGER.error(msg, e); - throw new CloudRuntimeException(msg, e); - } - } - - public static void unmountStore(String filePath) { - try { - LOGGER.info("Unmounting store"); - String umountCmd = String.format(UMOUNT_COMMAND, filePath); - Script.runSimpleBashScript(umountCmd); - try { - Files.deleteIfExists(Paths.get(filePath)); - } catch (IOException e) { - LOGGER.error(String.format("Failed to cleanup mounted store at: %s", filePath), e); - } - } catch (Exception e) { - String msg = String.format("Failed to unmount store mounted at %s", filePath); - LOGGER.error(msg, e); - throw new CloudRuntimeException(msg, e); - } - } - /** * Validate metadata for the given template's hypervisor/arch and add the existing template * to the specified secondary store. On success, database entries are created/updated. diff --git a/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java b/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java index aa8c3bee5834..6e6f9cc805a6 100644 --- a/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java +++ b/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java @@ -17,6 +17,7 @@ package com.cloud.upgrade; +import static com.cloud.upgrade.SystemVmTemplateRegistration.DEFAULT_SYSTEM_VM_GUEST_OS_NAME; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -25,7 +26,9 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doNothing; @@ -38,12 +41,22 @@ import static org.mockito.Mockito.when; import java.io.File; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Date; import java.util.List; +import java.util.Map; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.config.impl.ConfigurationVO; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.utils.security.DigestHelper; import org.apache.commons.lang3.StringUtils; @@ -58,16 +71,27 @@ import org.mockito.junit.MockitoJUnitRunner; import com.cloud.cpu.CPU; +import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.ClusterDao; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.dc.dao.DataCenterDetailsDao; import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.GuestOSVO; import com.cloud.storage.Storage; +import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VMTemplateZoneVO; +import com.cloud.storage.dao.GuestOSDao; import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VMTemplateZoneDao; +import com.cloud.template.VirtualMachineTemplate; import com.cloud.utils.HttpUtils; import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; +import com.cloud.vm.dao.VMInstanceDao; @RunWith(MockitoJUnitRunner.class) public class SystemVmTemplateRegistrationTest { @@ -78,6 +102,33 @@ public class SystemVmTemplateRegistrationTest { @Mock VMTemplateDao vmTemplateDao; + @Mock + GuestOSDao guestOSDao; + + @Mock + TemplateDataStoreDao templateDataStoreDao; + + @Mock + ConfigurationDao configurationDao; + + @Mock + DataCenterDao dataCenterDao; + + @Mock + DataCenterDetailsDao dataCenterDetailsDao; + + @Mock + VMTemplateZoneDao vmTemplateZoneDao; + + @Mock + ImageStoreDao imageStoreDao; + + @Mock + ImageStoreDetailsDao imageStoreDetailsDao; + + @Mock + VMInstanceDao vmInstanceDao; + @Spy @InjectMocks SystemVmTemplateRegistration systemVmTemplateRegistration = new SystemVmTemplateRegistration(); @@ -290,7 +341,7 @@ public void testValidateAndAddExistingTemplateToStoreForNonExistingEntries() { .getValidatedTemplateDetailsForHypervisorAndArch(hypervisor, arch); doNothing().when(systemVmTemplateRegistration).registerNewTemplate(name, details, zoneId, storeId, filePath); systemVmTemplateRegistration.validateAndRegisterNewTemplate(hypervisor, arch, name, zoneId, storeId, filePath); - verify(systemVmTemplateRegistration) .getValidatedTemplateDetailsForHypervisorAndArch(hypervisor, arch); + verify(systemVmTemplateRegistration).getValidatedTemplateDetailsForHypervisorAndArch(hypervisor, arch); verify(systemVmTemplateRegistration).registerNewTemplate(name, details, zoneId, storeId, filePath); } @@ -511,4 +562,1078 @@ public void updateOrRegisterSystemVmTemplate_ThrowsExceptionOnRegistrationFailur assertTrue(exception.getMessage().contains("Failed to register")); } + + @Test + public void updateRegisteredTemplateDetails_UpdatesTemplateSuccessfully() { + Long templateId = 1L; + Long zoneId = 2L; + SystemVmTemplateRegistration.MetadataTemplateDetails templateDetails = + Mockito.mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class); + VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); + GuestOSVO guestOS = Mockito.mock(GuestOSVO.class); + + when(templateDetails.getGuestOs()).thenReturn("Debian"); + when(templateDetails.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + when(templateDetails.getName()).thenReturn("templateName"); + when(vmTemplateDao.findById(templateId)).thenReturn(templateVO); + when(guestOSDao.findOneByDisplayName("Debian")).thenReturn(guestOS); + when(guestOS.getId()).thenReturn(10L); + when(vmTemplateDao.update(templateVO.getId(), templateVO)).thenReturn(true); + doNothing().when(systemVmTemplateRegistration).updateSystemVMEntries(templateId, Hypervisor.HypervisorType.KVM); + doNothing().when(systemVmTemplateRegistration).updateConfigurationParams(Hypervisor.HypervisorType.KVM, + "templateName", zoneId); + + systemVmTemplateRegistration.updateRegisteredTemplateDetails(templateId, templateDetails, zoneId); + + verify(templateVO).setTemplateType(Storage.TemplateType.SYSTEM); + verify(templateVO).setGuestOSId(10); + verify(vmTemplateDao).update(templateVO.getId(), templateVO); + verify(systemVmTemplateRegistration).updateSystemVMEntries(templateId, Hypervisor.HypervisorType.KVM); + verify(systemVmTemplateRegistration).updateConfigurationParams(Hypervisor.HypervisorType.KVM, + "templateName", zoneId); + } + + @Test + public void updateRegisteredTemplateDetails_ThrowsExceptionWhenUpdateFails() { + Long templateId = 1L; + Long zoneId = 2L; + SystemVmTemplateRegistration.MetadataTemplateDetails templateDetails = + Mockito.mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class); + VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); + + when(templateDetails.getGuestOs()).thenReturn("Debian"); + when(vmTemplateDao.findById(templateId)).thenReturn(templateVO); + when(vmTemplateDao.update(templateVO.getId(), templateVO)).thenReturn(false); + + CloudRuntimeException exception = assertThrows(CloudRuntimeException.class, + () -> systemVmTemplateRegistration.updateRegisteredTemplateDetails(templateId, templateDetails, zoneId)); + + assertTrue(exception.getMessage().contains("Exception while updating template with id")); + verify(systemVmTemplateRegistration, never()).updateSystemVMEntries(anyLong(), any()); + verify(systemVmTemplateRegistration, never()).updateConfigurationParams(any(), any(), any()); + } + + @Test + public void updateRegisteredTemplateDetails_SkipsGuestOSUpdateWhenNotFound() { + Long templateId = 1L; + Long zoneId = 2L; + SystemVmTemplateRegistration.MetadataTemplateDetails templateDetails = + Mockito.mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class); + VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); + + when(templateDetails.getGuestOs()).thenReturn("NonExistentOS"); + when(templateDetails.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + when(templateDetails.getName()).thenReturn("templateName"); + when(vmTemplateDao.findById(templateId)).thenReturn(templateVO); + when(guestOSDao.findOneByDisplayName("NonExistentOS")).thenReturn(null); + when(vmTemplateDao.update(templateVO.getId(), templateVO)).thenReturn(true); + doNothing().when(systemVmTemplateRegistration).updateSystemVMEntries(templateId, Hypervisor.HypervisorType.KVM); + doNothing().when(systemVmTemplateRegistration).updateConfigurationParams(Hypervisor.HypervisorType.KVM, + "templateName", zoneId); + + systemVmTemplateRegistration.updateRegisteredTemplateDetails(templateId, templateDetails, zoneId); + + verify(templateVO, never()).setGuestOSId(anyInt()); + verify(vmTemplateDao).update(templateVO.getId(), templateVO); + verify(systemVmTemplateRegistration).updateSystemVMEntries(templateId, Hypervisor.HypervisorType.KVM); + verify(systemVmTemplateRegistration).updateConfigurationParams(Hypervisor.HypervisorType.KVM, + "templateName", zoneId); + } + + @Test + public void registerTemplatesForZone_SuccessfullyRegistersNewTemplate() { + long zoneId = 1L; + String storeMountPath = "/mnt/nfs"; + Pair storeUrlAndId = new Pair<>("nfs://dummy", 100L); + String nfsVersion = "nfs3"; + List> hypervisorArchList = new ArrayList<>(); + Hypervisor.HypervisorType hypervisorType = Hypervisor.HypervisorType.KVM; + CPU.CPUArch arch = CPU.CPUArch.amd64; + hypervisorArchList.add(new Pair<>(hypervisorType, arch)); + SystemVmTemplateRegistration.MetadataTemplateDetails templateDetails = + Mockito.mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class); + when(templateDetails.getHypervisorType()).thenReturn(hypervisorType); + when(templateDetails.getArch()).thenReturn(arch); + String name = "TestTemplate"; + String url = "http://example.com/template"; + when(templateDetails.getName()).thenReturn(name); + when(templateDetails.getUrl()).thenReturn(url); + doReturn(storeUrlAndId).when(systemVmTemplateRegistration).getNfsStoreInZone(zoneId); + doReturn(nfsVersion).when(systemVmTemplateRegistration).getNfsVersion(storeUrlAndId.second()); + doReturn(null).when(systemVmTemplateRegistration).getRegisteredTemplate( + name, hypervisorType, arch, url); + doNothing().when(systemVmTemplateRegistration).registerNewTemplate( + name, templateDetails, zoneId, storeUrlAndId.second(), storeMountPath); + doReturn(hypervisorArchList).when(clusterDao).listDistinctHypervisorsAndArchExcludingExternalType(zoneId); + try (MockedStatic mockedStatic = + Mockito.mockStatic(SystemVmTemplateRegistration.class)) { + mockedStatic.when(() -> SystemVmTemplateRegistration.getMetadataTemplateDetails( + hypervisorType, arch)).thenReturn(templateDetails); + + systemVmTemplateRegistration.registerTemplatesForZone(zoneId, storeMountPath); + + mockedStatic.verify(() -> SystemVmTemplateRegistration.mountStore( + eq(storeUrlAndId.first()), eq(storeMountPath), eq(nfsVersion)), times(1)); + verify(systemVmTemplateRegistration).registerNewTemplate( + templateDetails.getName(), templateDetails, zoneId, storeUrlAndId.second(), storeMountPath); + } + } + + @Test + public void registerTemplatesForZone_SkipsWhenTemplateDetailsNotFound() { + long zoneId = 1L; + String storeMountPath = "/mnt/nfs"; + Pair storeUrlAndId = new Pair<>("nfs://dummy", 100L); + String nfsVersion = "nfs3"; + List> hypervisorArchList = new ArrayList<>(); + Hypervisor.HypervisorType hypervisorType = Hypervisor.HypervisorType.KVM; + CPU.CPUArch arch = CPU.CPUArch.amd64; + hypervisorArchList.add(new Pair<>(hypervisorType, arch)); + doReturn(storeUrlAndId).when(systemVmTemplateRegistration).getNfsStoreInZone(zoneId); + doReturn(nfsVersion).when(systemVmTemplateRegistration).getNfsVersion(storeUrlAndId.second()); + doReturn(hypervisorArchList).when(clusterDao).listDistinctHypervisorsAndArchExcludingExternalType(zoneId); + + try (MockedStatic mockedStatic = + Mockito.mockStatic(SystemVmTemplateRegistration.class)) { + mockedStatic.when(() -> SystemVmTemplateRegistration.getMetadataTemplateDetails( + hypervisorType, arch)).thenReturn(null); + + systemVmTemplateRegistration.registerTemplatesForZone(zoneId, storeMountPath); + + mockedStatic.verify(() -> SystemVmTemplateRegistration.mountStore( + eq(storeUrlAndId.first()), eq(storeMountPath), eq(nfsVersion)), times(1)); + verify(systemVmTemplateRegistration, never()).registerNewTemplate(any(), any(), anyLong(), anyLong(), any()); + } + } + + @Test + public void registerTemplatesForZone_AddsExistingTemplateToStore() { + long zoneId = 1L; + String storeMountPath = "/mnt/nfs"; + Pair storeUrlAndId = new Pair<>("nfs://dummy", 100L); + String nfsVersion = "nfs3"; + List> hypervisorArchList = new ArrayList<>(); + Hypervisor.HypervisorType hypervisorType = Hypervisor.HypervisorType.KVM; + CPU.CPUArch arch = CPU.CPUArch.amd64; + hypervisorArchList.add(new Pair<>(hypervisorType, arch)); + SystemVmTemplateRegistration.MetadataTemplateDetails templateDetails = + Mockito.mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class); + when(templateDetails.getHypervisorType()).thenReturn(hypervisorType); + when(templateDetails.getArch()).thenReturn(arch); + String name = "TestTemplate"; + String url = "http://example.com/template"; + when(templateDetails.getName()).thenReturn(name); + when(templateDetails.getUrl()).thenReturn(url); + VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); + long templateId = 100L; + when(templateVO.getId()).thenReturn(templateId); + TemplateDataStoreVO templateDataStoreVO = Mockito.mock(TemplateDataStoreVO.class); + String installPath = "/template/install/path"; + when(templateDataStoreVO.getInstallPath()).thenReturn(installPath); + + doReturn(storeUrlAndId).when(systemVmTemplateRegistration).getNfsStoreInZone(zoneId); + doReturn(nfsVersion).when(systemVmTemplateRegistration).getNfsVersion(storeUrlAndId.second()); + doReturn(hypervisorArchList).when(clusterDao).listDistinctHypervisorsAndArchExcludingExternalType(zoneId); + doReturn(templateVO).when(systemVmTemplateRegistration).getRegisteredTemplate(name, hypervisorType, arch, url); + doReturn(templateDataStoreVO).when(templateDataStoreDao) + .findByStoreTemplate(storeUrlAndId.second(), templateId); + doReturn(false).when(systemVmTemplateRegistration).validateIfSeeded( + templateDataStoreVO, storeUrlAndId.first(), installPath, nfsVersion); + doNothing().when(systemVmTemplateRegistration).addExistingTemplateToStore( + templateVO, templateDetails, templateDataStoreVO, zoneId, storeUrlAndId.second(), storeMountPath); + doNothing().when(systemVmTemplateRegistration).updateRegisteredTemplateDetails( + templateId, templateDetails, zoneId); + + try (MockedStatic mockedStatic = + Mockito.mockStatic(SystemVmTemplateRegistration.class)) { + mockedStatic.when(() -> SystemVmTemplateRegistration.getMetadataTemplateDetails( + hypervisorType, arch)).thenReturn(templateDetails); + + systemVmTemplateRegistration.registerTemplatesForZone(zoneId, storeMountPath); + + verify(systemVmTemplateRegistration).addExistingTemplateToStore( + templateVO, templateDetails, templateDataStoreVO, zoneId, storeUrlAndId.second(), storeMountPath); + verify(systemVmTemplateRegistration).updateRegisteredTemplateDetails(templateId, templateDetails, zoneId); + } + } + + @Test + public void performTemplateRegistrationOperations_CreatesNewTemplateWhenNotExists() { + String name = "TestTemplate"; + String url = "http://example.com/template"; + String checksum = "abc123"; + Storage.ImageFormat format = Storage.ImageFormat.QCOW2; + long guestOsId = 1L; + Long storeId = 100L; + Long templateId = null; + String filePath = "/mnt/nfs"; + TemplateDataStoreVO templateDataStoreVO = null; + SystemVmTemplateRegistration.MetadataTemplateDetails templateDetails = Mockito.mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class); + + when(templateDetails.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + when(templateDetails.getArch()).thenReturn(CPU.CPUArch.amd64); + doReturn(new VMTemplateVO()).when(vmTemplateDao).persist(any()); + doNothing().when(systemVmTemplateRegistration).createCrossZonesTemplateZoneRefEntries(anyLong()); + doNothing().when(systemVmTemplateRegistration).createTemplateStoreRefEntry(any()); + doNothing().when(systemVmTemplateRegistration).setupTemplateOnStore(anyString(), any(), anyString()); + doNothing().when(systemVmTemplateRegistration).readTemplateProperties(anyString(), any()); + doNothing().when(systemVmTemplateRegistration).updateTemplateDetails(any()); + + Long result = systemVmTemplateRegistration.performTemplateRegistrationOperations(name, templateDetails, url, checksum, format, guestOsId, storeId, templateId, filePath, templateDataStoreVO); + + assertNotNull(result); + verify(vmTemplateDao).persist(any()); + verify(systemVmTemplateRegistration).createCrossZonesTemplateZoneRefEntries(anyLong()); + verify(systemVmTemplateRegistration).createTemplateStoreRefEntry(any()); + verify(systemVmTemplateRegistration).setupTemplateOnStore(anyString(), any(), anyString()); + verify(systemVmTemplateRegistration).updateTemplateDetails(any()); + } + + @Test + public void performTemplateRegistrationOperations_UpdatesExistingTemplate() { + String name = "TestTemplate"; + String url = "http://example.com/template"; + String checksum = "abc123"; + Storage.ImageFormat format = Storage.ImageFormat.QCOW2; + long guestOsId = 1L; + Long storeId = 100L; + Long templateId = 1L; + String filePath = "/mnt/nfs"; + TemplateDataStoreVO templateDataStoreVO = Mockito.mock(TemplateDataStoreVO.class); + SystemVmTemplateRegistration.MetadataTemplateDetails templateDetails = Mockito.mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class); + + when(templateDetails.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + when(templateDetails.getArch()).thenReturn(CPU.CPUArch.amd64); + doNothing().when(systemVmTemplateRegistration).createCrossZonesTemplateZoneRefEntries(anyLong()); + doNothing().when(systemVmTemplateRegistration).setupTemplateOnStore(anyString(), any(), anyString()); + doNothing().when(systemVmTemplateRegistration).readTemplateProperties(anyString(), any()); + doNothing().when(systemVmTemplateRegistration).updateTemplateDetails(any()); + + Long result = systemVmTemplateRegistration.performTemplateRegistrationOperations(name, templateDetails, url, checksum, format, guestOsId, storeId, templateId, filePath, templateDataStoreVO); + + assertNotNull(result); + assertEquals(templateId, result); + verify(vmTemplateDao, never()).persist(any()); + verify(systemVmTemplateRegistration).createCrossZonesTemplateZoneRefEntries(anyLong()); + verify(systemVmTemplateRegistration, never()).createTemplateStoreRefEntry(any()); + verify(systemVmTemplateRegistration).setupTemplateOnStore(anyString(), any(), anyString()); + verify(systemVmTemplateRegistration).updateTemplateDetails(any()); + } + + @Test + public void performTemplateRegistrationOperations_ThrowsExceptionWhenTemplateCreationFails() { + String name = "TestTemplate"; + String url = "http://example.com/template"; + String checksum = "abc123"; + Storage.ImageFormat format = Storage.ImageFormat.QCOW2; + long guestOsId = 1L; + Long storeId = 100L; + Long templateId = null; + String filePath = "/mnt/nfs"; + TemplateDataStoreVO templateDataStoreVO = null; + SystemVmTemplateRegistration.MetadataTemplateDetails templateDetails = Mockito.mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class); + + when(templateDetails.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + when(templateDetails.getArch()).thenReturn(CPU.CPUArch.amd64); + doReturn(null).when(vmTemplateDao).persist(any()); + + assertThrows(CloudRuntimeException.class, () -> { + systemVmTemplateRegistration.performTemplateRegistrationOperations(name, templateDetails, url, checksum, format, guestOsId, storeId, templateId, filePath, templateDataStoreVO); + }); + + verify(vmTemplateDao).persist(any()); + verify(systemVmTemplateRegistration, never()).createCrossZonesTemplateZoneRefEntries(anyLong()); + verify(systemVmTemplateRegistration, never()).createTemplateStoreRefEntry(any()); + verify(systemVmTemplateRegistration, never()).setupTemplateOnStore(anyString(), any(), anyString()); + verify(systemVmTemplateRegistration, never()).updateTemplateDetails(any()); + } + + @Test + public void setupTemplateOnStore_ThrowsExceptionWhenScriptNotFound() { + String templateName = "templateName"; + String destTempFolder = "/tmp/folder"; + SystemVmTemplateRegistration.MetadataTemplateDetails templateDetails = + Mockito.mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class); + + try (MockedStatic