storage: post process locally uploaded multi-disk ova template (#3215)

Problem: When a multi-disk OVA template is uploaded, only the root disk is recognized and VMs deployed using such template only get the root disk provisioned.
Root Cause: The template processor for multi-disk OVA was not used in the template upload processor.
Solution: Added support for local multi-disk OVA template upload. After a multi-disk OVA template is
uploaded, the mechanism that worked on multi-disk OVA templates registered using URL is now also used to discovers and creates data-disk templates in cloud.vm_template table and on the secondary storage.

To enable SSL on SSVMs :
• Upload the certificates like you usually do via the API or UI->Infrastructure tab
• Set the global settings secstorage.encrypt.copy, secstorage.ssl.cert.domain to appropriate values
along with the CPVM ones
• Restart management server (no need to destroy/restart SSVM (or the ssvm agent))

Test cases:
- Upload template and check it creates multi-disk folders on secondary 
storage and entries in cloud.vm_template table
- Upload template and kill/shutdown management server. Then restart MS
to check if template sync works
- Copy template across zone of an uploaded template

Signed-off-by: Rohit Yadav rohit.yadav@shapeblue.com

Author: yadvr

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java

@@ -47,6 +47,8 @@ public TemplateInfo getTemplate() {
 
     AsyncCallFuture<TemplateApiResult> createTemplateFromVolumeAsync(VolumeInfo volume, TemplateInfo template, DataStore store);
 
+    boolean createOvaDataDiskTemplates(TemplateInfo parentTemplate);
+
     AsyncCallFuture<TemplateApiResult> deleteTemplateAsync(TemplateInfo template);
 
     AsyncCallFuture<TemplateApiResult> copyTemplate(TemplateInfo srcTemplate, DataStore destStore);

engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java

@@ -409,8 +409,15 @@ public void handleTemplateSync(DataStore store) {
                                     _templateDao.update(tmplt.getId(), tmlpt);
 
                                     if (tmplt.getState() == VirtualMachineTemplate.State.NotUploaded || tmplt.getState() == VirtualMachineTemplate.State.UploadInProgress) {
+                                        VirtualMachineTemplate.Event event = VirtualMachineTemplate.Event.OperationSucceeded;
+                                        // For multi-disk OVA, check and create data disk templates
+                                        if (tmplt.getFormat().equals(ImageFormat.OVA)) {
+                                            if (!createOvaDataDiskTemplates(_templateFactory.getTemplate(tmlpt.getId(), store))) {
+                                                event = VirtualMachineTemplate.Event.OperationFailed;
+                                            }
+                                        }
                                         try {
-                                            stateMachine.transitTo(tmplt, VirtualMachineTemplate.Event.OperationSucceeded, null, _templateDao);
+                                            stateMachine.transitTo(tmplt, event, null, _templateDao);
                                         } catch (NoTransitionException e) {
                                             s_logger.error("Unexpected state transition exception for template " + tmplt.getName() + ". Details: " + e.getMessage());
                                         }
@@ -701,7 +708,7 @@ protected Void createTemplateCallback(AsyncCallbackDispatcher<TemplateServiceImp
             return null;
         }
 
-        // Check if OVA contains additional data disks. If yes, create Datadisk templates for each of the additional datadisk present in the OVA
+        // For multi-disk OVA, check and create data disk templates
         if (template.getFormat().equals(ImageFormat.OVA)) {
             if (!createOvaDataDiskTemplates(template)) {
                 template.processEvent(ObjectInDataStoreStateMachine.Event.OperationFailed);
@@ -729,8 +736,8 @@ protected Void createTemplateCallback(AsyncCallbackDispatcher<TemplateServiceImp
         return null;
     }
 
-
-    protected boolean createOvaDataDiskTemplates(TemplateInfo parentTemplate) {
+    @Override
+    public boolean createOvaDataDiskTemplates(TemplateInfo parentTemplate) {
         try {
             // Get Datadisk template (if any) for OVA
             List<DatadiskTO> dataDiskTemplates = new ArrayList<DatadiskTO>();

engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java

@@ -90,6 +90,7 @@ public ObjectInDataStoreManagerImpl() {
         stateMachines.addTransition(State.Allocated, Event.CreateOnlyRequested, State.Creating);
         stateMachines.addTransition(State.Allocated, Event.DestroyRequested, State.Destroying);
         stateMachines.addTransition(State.Allocated, Event.OperationFailed, State.Failed);
+        stateMachines.addTransition(State.Allocated, Event.OperationSuccessed, State.Ready);
         stateMachines.addTransition(State.Creating, Event.OperationFailed, State.Allocated);
         stateMachines.addTransition(State.Creating, Event.OperationSuccessed, State.Ready);
         stateMachines.addTransition(State.Ready, Event.CopyingRequested, State.Copying);

server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java

@@ -25,17 +25,14 @@
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
-import com.cloud.configuration.Resource;
-import com.cloud.event.EventTypes;
-import com.cloud.event.UsageEventUtils;
-import com.cloud.user.ResourceLimitService;
-import org.apache.log4j.Logger;
-import org.springframework.stereotype.Component;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
 import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
 import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.State;
+import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory;
+import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
+import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService;
 import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.framework.config.Configurable;
 import org.apache.cloudstack.managed.context.ManagedContextRunnable;
@@ -48,6 +45,8 @@
 import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO;
 import org.apache.cloudstack.utils.identity.ManagementServerNode;
+import org.apache.log4j.Logger;
+import org.springframework.stereotype.Component;
 
 import com.cloud.agent.Listener;
 import com.cloud.agent.api.AgentControlAnswer;
@@ -56,6 +55,9 @@
 import com.cloud.agent.api.Command;
 import com.cloud.agent.api.StartupCommand;
 import com.cloud.alert.AlertManager;
+import com.cloud.configuration.Resource;
+import com.cloud.event.EventTypes;
+import com.cloud.event.UsageEventUtils;
 import com.cloud.exception.ConnectionException;
 import com.cloud.host.Host;
 import com.cloud.host.Status;
@@ -65,6 +67,7 @@
 import com.cloud.storage.dao.VMTemplateZoneDao;
 import com.cloud.storage.dao.VolumeDao;
 import com.cloud.template.VirtualMachineTemplate;
+import com.cloud.user.ResourceLimitService;
 import com.cloud.utils.component.ManagerBase;
 import com.cloud.utils.concurrency.NamedThreadFactory;
 import com.cloud.utils.db.Transaction;
@@ -102,6 +105,12 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto
     private AlertManager _alertMgr;
     @Inject
     private VMTemplateZoneDao _vmTemplateZoneDao;
+    @Inject
+    private DataStoreManager dataStoreManager;
+    @Inject
+    private TemplateDataFactory templateFactory;
+    @Inject
+    private TemplateService templateService;
 
     private long _nodeId;
     private ScheduledExecutorService _executor = null;
@@ -395,6 +404,20 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
                             VMTemplateVO templateUpdate = _templateDao.createForUpdate();
                             templateUpdate.setSize(answer.getVirtualSize());
                             _templateDao.update(tmpTemplate.getId(), templateUpdate);
+                            // For multi-disk OVA, check and create data disk templates
+                            if (tmpTemplate.getFormat().equals(Storage.ImageFormat.OVA)) {
+                                final DataStore store = dataStoreManager.getDataStore(templateDataStore.getDataStoreId(), templateDataStore.getDataStoreRole());
+                                final TemplateInfo templateInfo = templateFactory.getTemplate(tmpTemplate.getId(), store);
+                                if (!templateService.createOvaDataDiskTemplates(templateInfo)) {
+                                    tmpTemplateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.ABANDONED);
+                                    tmpTemplateDataStore.setState(State.Failed);
+                                    stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.OperationFailed, null, _templateDao);
+                                    msg = "Multi-disk OVA template " + tmpTemplate.getUuid() + " failed to process data disks";
+                                    s_logger.error(msg);
+                                    sendAlert = true;
+                                    break;
+                                }
+                            }
                             stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.OperationSucceeded, null, _templateDao);
                             _resourceLimitMgr.incrementResourceCount(template.getAccountId(), Resource.ResourceType.secondary_storage, answer.getVirtualSize());
                             //publish usage event