Refactor volume migration to remove secondary storage intermediate

nvazquez · nvazquez · commit 9034a8a0b658 · 2020-03-04T12:03:55.000-03:00
diff --git a/core/src/main/java/com/cloud/storage/resource/StorageProcessor.java b/core/src/main/java/com/cloud/storage/resource/StorageProcessor.java
@@ -74,4 +74,6 @@ public interface StorageProcessor {
     public Answer resignature(ResignatureCommand cmd);
 
     public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd);
+
+    Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd);
 }
diff --git a/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java b/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java
@@ -20,6 +20,7 @@
 package com.cloud.storage.resource;
 
 import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
+import org.apache.cloudstack.storage.to.VolumeObjectTO;
 import org.apache.log4j.Logger;
 
 import org.apache.cloudstack.storage.command.AttachCommand;
@@ -95,7 +96,9 @@ protected Answer execute(CopyCommand cmd) {
             //copy volume from image cache to primary
             return processor.copyVolumeFromImageCacheToPrimary(cmd);
         } else if (srcData.getObjectType() == DataObjectType.VOLUME && srcData.getDataStore().getRole() == DataStoreRole.Primary) {
-            if (destData.getObjectType() == DataObjectType.VOLUME) {
+            if (destData.getObjectType() == DataObjectType.VOLUME && srcData instanceof VolumeObjectTO && ((VolumeObjectTO)srcData).isDirectDownload()) {
+                return processor.copyVolumeFromPrimaryToPrimary(cmd);
+            } else if (destData.getObjectType() == DataObjectType.VOLUME) {
                 return processor.copyVolumeFromPrimaryToSecondary(cmd);
             } else if (destData.getObjectType() == DataObjectType.TEMPLATE) {
                 return processor.createTemplateFromVolume(cmd);
diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java
@@ -61,6 +61,7 @@ public class VolumeObjectTO implements DataTO {
     private DiskCacheMode cacheMode;
     private Hypervisor.HypervisorType hypervisorType;
     private MigrationOptions migrationOptions;
+    private boolean directDownload;
 
     public VolumeObjectTO() {
 
@@ -100,6 +101,7 @@ public VolumeObjectTO(VolumeInfo volume) {
         hypervisorType = volume.getHypervisorType();
         setDeviceId(volume.getDeviceId());
         this.migrationOptions = volume.getMigrationOptions();
+        this.directDownload = volume.isDirectDownload();
     }
 
     public String getUuid() {
@@ -307,4 +309,8 @@ public DiskCacheMode getCacheMode() {
     public MigrationOptions getMigrationOptions() {
         return migrationOptions;
     }
+
+    public boolean isDirectDownload() {
+        return directDownload;
+    }
 }
diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java
@@ -79,4 +79,8 @@ public interface VolumeInfo extends DataObject, Volume {
     MigrationOptions getMigrationOptions();
 
     void setMigrationOptions(MigrationOptions migrationOptions);
+
+    boolean isDirectDownload();
+
+    void setDirectDownload(boolean directDownload);
 }
diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java
@@ -325,7 +325,25 @@ protected Answer copyVolumeBetweenPools(DataObject srcData, DataObject destData)
 
         Scope destScope = getZoneScope(destData.getDataStore().getScope());
         DataStore cacheStore = cacheMgr.getCacheStorage(destScope);
+        boolean bypassSecondaryStorage = false;
+        if (srcData instanceof VolumeInfo && ((VolumeInfo)srcData).isDirectDownload()) {
+            bypassSecondaryStorage = true;
+        }
+
         if (cacheStore == null) {
+            if (bypassSecondaryStorage) {
+                CopyCommand cmd = new CopyCommand(srcData.getTO(), destData.getTO(), _copyvolumewait, VirtualMachineManager.ExecuteInSequence.value());
+                EndPoint ep = selector.select(srcData, destData);
+                Answer answer = null;
+                if (ep == null) {
+                    String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";
+                    s_logger.error(errMsg);
+                    answer = new Answer(cmd, false, errMsg);
+                } else {
+                    answer = ep.sendMessage(cmd);
+                }
+                return answer;
+            }
             // need to find a nfs or cifs image store, assuming that can't copy volume
             // directly to s3
             ImageStoreEntity imageStore = (ImageStoreEntity)dataStoreMgr.getImageStoreWithFreeCapacity(destScope.getScopeId());
diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java
@@ -84,6 +84,12 @@ protected boolean moveBetweenPrimaryImage(DataStore srcStore, DataStore destStor
         }
     }
 
+    private boolean moveBetweenPrimaryDirectDownload(DataStore srcStore, DataStore destStore) {
+        DataStoreRole srcRole = srcStore.getRole();
+        DataStoreRole destRole = destStore.getRole();
+        return srcRole == DataStoreRole.Primary && destRole == DataStoreRole.Primary;
+    }
+
     protected boolean moveBetweenCacheAndImage(DataStore srcStore, DataStore destStore) {
         DataStoreRole srcRole = srcStore.getRole();
         DataStoreRole destRole = destStore.getRole();
@@ -182,6 +188,8 @@ public EndPoint select(DataObject srcData, DataObject destData) {
         DataStore destStore = destData.getDataStore();
         if (moveBetweenPrimaryImage(srcStore, destStore)) {
             return findEndPointForImageMove(srcStore, destStore);
+        } else if (moveBetweenPrimaryDirectDownload(srcStore, destStore)) {
+            return findEndPointForImageMove(srcStore, destStore);
         } else if (moveBetweenCacheAndImage(srcStore, destStore)) {
             // pick ssvm based on image cache dc
             DataStore selectedStore = null;
diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java
@@ -23,6 +23,8 @@
 
 import javax.inject.Inject;
 
+import com.cloud.storage.VMTemplateVO;
+import com.cloud.storage.dao.VMTemplateDao;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
 import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
@@ -42,6 +44,8 @@ public class VolumeDataFactoryImpl implements VolumeDataFactory {
     VolumeDataStoreDao volumeStoreDao;
     @Inject
     DataStoreManager storeMgr;
+    @Inject
+    VMTemplateDao templateDao;
 
     @Override
     public VolumeInfo getVolume(long volumeId, DataStore store) {
@@ -90,6 +94,12 @@ public VolumeInfo getVolume(long volumeId) {
             DataStore store = storeMgr.getDataStore(volumeVO.getPoolId(), DataStoreRole.Primary);
             vol = VolumeObject.getVolumeObject(store, volumeVO);
         }
+        if (vol.getTemplateId() != null) {
+            VMTemplateVO template = templateDao.findById(vol.getTemplateId());
+            if (template != null) {
+                vol.setDirectDownload(template.isDirectDownload());
+            }
+        }
         return vol;
     }
 
diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java
@@ -74,6 +74,7 @@ public class VolumeObject implements VolumeInfo {
     DiskOfferingDao diskOfferingDao;
     private Object payload;
     private MigrationOptions migrationOptions;
+    private boolean directDownload;
 
     public VolumeObject() {
         _volStateMachine = Volume.State.getStateMachine();
@@ -327,6 +328,16 @@ public void setMigrationOptions(MigrationOptions migrationOptions) {
         this.migrationOptions = migrationOptions;
     }
 
+    @Override
+    public boolean isDirectDownload() {
+        return directDownload;
+    }
+
+    @Override
+    public void setDirectDownload(boolean directDownload) {
+        this.directDownload = directDownload;
+    }
+
     public void update() {
         volumeDao.update(volumeVO.getId(), volumeVO);
         volumeVO = volumeDao.findById(volumeVO.getId());
diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
@@ -1742,6 +1742,41 @@ public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd)
         return new DirectDownloadAnswer(true, template.getSize(), template.getName());
     }
 
+    @Override
+    public Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd) {
+        final DataTO srcData = cmd.getSrcTO();
+        final DataTO destData = cmd.getDestTO();
+        final VolumeObjectTO srcVol = (VolumeObjectTO)srcData;
+        final VolumeObjectTO destVol = (VolumeObjectTO)destData;
+        final ImageFormat srcFormat = srcVol.getFormat();
+        final ImageFormat destFormat = destVol.getFormat();
+        final DataStoreTO srcStore = srcData.getDataStore();
+        final DataStoreTO destStore = destData.getDataStore();
+        final PrimaryDataStoreTO primaryStore = (PrimaryDataStoreTO)srcStore;
+        final PrimaryDataStoreTO primaryStoreDest = (PrimaryDataStoreTO)destStore;
+        final String srcVolumePath = srcData.getPath();
+        final String destVolumePath = destData.getPath();
+        KVMStoragePool destPool = null;
+
+        try {
+            final String volumeName = UUID.randomUUID().toString();
+
+            final String destVolumeName = volumeName + "." + destFormat.getFileExtension();
+            final KVMPhysicalDisk volume = storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), srcVolumePath);
+            volume.setFormat(PhysicalDiskFormat.valueOf(srcFormat.toString()));
+
+            destPool = storagePoolMgr.getStoragePool(primaryStoreDest.getPoolType(), primaryStoreDest.getUuid());
+            storagePoolMgr.copyPhysicalDisk(volume, destVolumeName, destPool, cmd.getWaitInMillSeconds());
+            final VolumeObjectTO newVol = new VolumeObjectTO();
+            newVol.setPath(destVolumePath + File.separator + destVolumeName);
+            newVol.setFormat(destFormat);
+            return new CopyCmdAnswer(newVol);
+        } catch (final CloudRuntimeException e) {
+            s_logger.debug("Failed to copyVolumeFromPrimaryToPrimary: ", e);
+            return new CopyCmdAnswer(e.toString());
+        }
+    }
+
     /**
      * True if location exists
      */
diff --git a/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3StorageProcessor.java b/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3StorageProcessor.java
@@ -826,6 +826,11 @@ public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd)
         return null;
     }
 
+    @Override
+    public Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd) {
+        return null;
+    }
+
     /**
      * Attach disks
      * @param cmd
diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java
@@ -3554,4 +3554,9 @@ void setFullCloneFlag(boolean value){
     public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd) {
         return null;
     }
+
+    @Override
+    public Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd) {
+        return null;
+    }
 }
diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java
@@ -209,6 +209,11 @@ public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd)
         return null;
     }
 
+    @Override
+    public Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd) {
+        return null;
+    }
+
     @Override
     public AttachAnswer attachIso(final AttachCommand cmd) {
         final DiskTO disk = cmd.getDisk();

Original file line number	Diff line number	Diff line change
`@@ -74,4 +74,6 @@ public interface StorageProcessor {`
`74`	`74`	`public Answer resignature(ResignatureCommand cmd);`
`75`	`75`
`76`	`76`	`public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd);`
	`77`	`+`
	`78`	`+ Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd);`
`77`	`79`	`}`
Original file line number	Diff line number	Diff line change
`@@ -79,4 +79,8 @@ public interface VolumeInfo extends DataObject, Volume {`
`79`	`79`	`MigrationOptions getMigrationOptions();`
`80`	`80`
`81`	`81`	`void setMigrationOptions(MigrationOptions migrationOptions);`
	`82`	`+`
	`83`	`+ boolean isDirectDownload();`
	`84`	`+`
	`85`	`+ void setDirectDownload(boolean directDownload);`
`82`	`86`	`}`