KVM/VMware: warm/change-tracking migration and direct VDDK import into Ceph/RBD and Linstor, plus importVm adoption of existing RBD root volumes

PendingReleaseNotes

@@ -39,3 +39,23 @@ example.ver.1 > example.ver.2:
     which can now be attached to Instances. This is to prevent the Secondary
     Storage to grow to enormous sizes as Linux Distributions keep growing in
     size while a stripped down Linux should fit on a 2.88MB floppy.
+
+4.23.0.0:
+  * VMware-to-KVM import using VDDK can import powered-off VMware VMs directly
+    into Ceph RBD primary storage when forceconverttopool is enabled and the
+    selected KVM conversion host supports qemu RBD access and in-place virt-v2v
+    finalization. Hosts without in-place finalization support can still use the
+    staged VDDK flow, converting to temporary qcow2 storage before copying the
+    finalized disks into RBD.
+
+  * Linstor primary storage can now be the destination for VMware-to-KVM
+    imports in all three modes. Staged imports (virt-v2v to a temporary NFS
+    location) copy the converted disks into pre-created DRBD block devices
+    through the Linstor storage adaptor; the import host must be a LINSTOR
+    satellite connected to the destination pool. Direct VDDK imports with
+    forceconverttopool write each powered-off VMware disk over nbdkit/VDDK
+    straight into the DRBD device and finalize it with virt-v2v in place.
+    VMware CBT warm migrations replicate the initial full copy and all
+    incremental changed-block cycles directly into Linstor volumes and
+    finalize in place at cutover; the conversion host requires VDDK,
+    in-place virt-v2v support and access to the Linstor pool.

api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java

@@ -38,6 +38,7 @@ public class RemoteInstanceTO implements Serializable {
     private String datacenterName;
     private String clusterName;
     private String hostName;
+    private String vmwareMoref;
 
     public RemoteInstanceTO() {
     }
@@ -65,6 +66,11 @@ public RemoteInstanceTO(String instanceName, String instancePath, String vcenter
         this.hostName = hostName;
     }
 
+    public RemoteInstanceTO(String instanceName, String instancePath, String vcenterHost, String vcenterUsername, String vcenterPassword, String datacenterName, String clusterName, String hostName, String vmwareMoref) {
+        this(instanceName, instancePath, vcenterHost, vcenterUsername, vcenterPassword, datacenterName, clusterName, hostName);
+        this.vmwareMoref = vmwareMoref;
+    }
+
     public Hypervisor.HypervisorType getHypervisorType() {
         return this.hypervisorType;
     }
@@ -100,4 +106,12 @@ public String getClusterName() {
     public String getHostName() {
         return hostName;
     }
+
+    public String getVmwareMoref() {
+        return vmwareMoref;
+    }
+
+    public void setVmwareMoref(String vmwareMoref) {
+        this.vmwareMoref = vmwareMoref;
+    }
 }

api/src/main/java/com/cloud/agent/api/to/VmwareCbtChangedBlockRangeTO.java

@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.agent.api.to;
+
+import java.io.Serializable;
+
+public class VmwareCbtChangedBlockRangeTO implements Serializable {
+
+    private String diskId;
+    private long startOffset;
+    private long length;
+
+    public VmwareCbtChangedBlockRangeTO() {
+    }
+
+    public VmwareCbtChangedBlockRangeTO(String diskId, long startOffset, long length) {
+        this.diskId = diskId;
+        this.startOffset = startOffset;
+        this.length = length;
+    }
+
+    public String getDiskId() {
+        return diskId;
+    }
+
+    public long getStartOffset() {
+        return startOffset;
+    }
+
+    public long getLength() {
+        return length;
+    }
+}

api/src/main/java/com/cloud/agent/api/to/VmwareCbtDiskSyncResultTO.java

@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.agent.api.to;
+
+import java.io.Serializable;
+
+public class VmwareCbtDiskSyncResultTO implements Serializable {
+
+    private String diskId;
+    private String targetPath;
+    private String changeId;
+    private String snapshotMor;
+    private long changedBytes;
+    private long durationSeconds;
+    private boolean result;
+    private String details;
+
+    public VmwareCbtDiskSyncResultTO() {
+    }
+
+    public VmwareCbtDiskSyncResultTO(String diskId, String targetPath, String changeId, String snapshotMor,
+                                    long changedBytes, long durationSeconds, boolean result, String details) {
+        this.diskId = diskId;
+        this.targetPath = targetPath;
+        this.changeId = changeId;
+        this.snapshotMor = snapshotMor;
+        this.changedBytes = changedBytes;
+        this.durationSeconds = durationSeconds;
+        this.result = result;
+        this.details = details;
+    }
+
+    public String getDiskId() {
+        return diskId;
+    }
+
+    public String getTargetPath() {
+        return targetPath;
+    }
+
+    public String getChangeId() {
+        return changeId;
+    }
+
+    public String getSnapshotMor() {
+        return snapshotMor;
+    }
+
+    public long getChangedBytes() {
+        return changedBytes;
+    }
+
+    public long getDurationSeconds() {
+        return durationSeconds;
+    }
+
+    public boolean getResult() {
+        return result;
+    }
+
+    public String getDetails() {
+        return details;
+    }
+}

api/src/main/java/com/cloud/agent/api/to/VmwareCbtDiskTO.java

@@ -0,0 +1,85 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.agent.api.to;
+
+import java.io.Serializable;
+
+public class VmwareCbtDiskTO implements Serializable {
+
+    private String diskId;
+    private Integer diskDeviceKey;
+    private String sourceDiskPath;
+    private String datastoreName;
+    private String targetPath;
+    private String targetFormat;
+    private String changeId;
+    private String snapshotMor;
+    private long capacityBytes;
+
+    public VmwareCbtDiskTO() {
+    }
+
+    public VmwareCbtDiskTO(String diskId, Integer diskDeviceKey, String sourceDiskPath, String datastoreName,
+                          String targetPath, String targetFormat, String changeId, String snapshotMor,
+                          long capacityBytes) {
+        this.diskId = diskId;
+        this.diskDeviceKey = diskDeviceKey;
+        this.sourceDiskPath = sourceDiskPath;
+        this.datastoreName = datastoreName;
+        this.targetPath = targetPath;
+        this.targetFormat = targetFormat;
+        this.changeId = changeId;
+        this.snapshotMor = snapshotMor;
+        this.capacityBytes = capacityBytes;
+    }
+
+    public String getDiskId() {
+        return diskId;
+    }
+
+    public Integer getDiskDeviceKey() {
+        return diskDeviceKey;
+    }
+
+    public String getSourceDiskPath() {
+        return sourceDiskPath;
+    }
+
+    public String getDatastoreName() {
+        return datastoreName;
+    }
+
+    public String getTargetPath() {
+        return targetPath;
+    }
+
+    public String getTargetFormat() {
+        return targetFormat;
+    }
+
+    public String getChangeId() {
+        return changeId;
+    }
+
+    public String getSnapshotMor() {
+        return snapshotMor;
+    }
+
+    public long getCapacityBytes() {
+        return capacityBytes;
+    }
+}

api/src/main/java/com/cloud/agent/api/to/VmwareVddkSourceDiskTO.java

@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.agent.api.to;
+
+import java.io.Serializable;
+
+public class VmwareVddkSourceDiskTO implements Serializable {
+
+    private String diskId;
+    private String sourceDiskPath;
+    private long capacityBytes;
+    private Integer position;
+
+    public VmwareVddkSourceDiskTO() {
+    }
+
+    public VmwareVddkSourceDiskTO(String diskId, String sourceDiskPath, long capacityBytes, Integer position) {
+        this.diskId = diskId;
+        this.sourceDiskPath = sourceDiskPath;
+        this.capacityBytes = capacityBytes;
+        this.position = position;
+    }
+
+    public String getDiskId() {
+        return diskId;
+    }
+
+    public String getSourceDiskPath() {
+        return sourceDiskPath;
+    }
+
+    public long getCapacityBytes() {
+        return capacityBytes;
+    }
+
+    public Integer getPosition() {
+        return position;
+    }
+}

api/src/main/java/com/cloud/event/EventTypes.java

@@ -127,6 +127,9 @@ public class EventTypes {
     public static final String EVENT_VM_RESTORE = "VM.RESTORE";
     public static final String EVENT_VM_EXPUNGE = "VM.EXPUNGE";
     public static final String EVENT_VM_IMPORT = "VM.IMPORT";
+    public static final String EVENT_VMWARE_CBT_MIGRATION_START = "VMWARE.CBT.MIGRATION.START";
+    public static final String EVENT_VMWARE_CBT_MIGRATION_SYNC = "VMWARE.CBT.MIGRATION.SYNC";
+    public static final String EVENT_VMWARE_CBT_MIGRATION_CUTOVER = "VMWARE.CBT.MIGRATION.CUTOVER";
     public static final String EVENT_VM_UNMANAGE = "VM.UNMANAGE";
     public static final String EVENT_VM_RECOVER = "VM.RECOVER";
 

api/src/main/java/com/cloud/host/Host.java

@@ -60,8 +60,18 @@ public static String[] toStrings(Host.Type... types) {
     String HOST_VDDK_SUPPORT = "host.vddk.support";
     String HOST_VDDK_LIB_DIR = "vddk.lib.dir";
     String HOST_VDDK_VERSION = "host.vddk.version";
+    String HOST_VDDK_BLOCKCOPY_SUPPORT = "host.vddk.blockcopy.support";
+    String HOST_VDDK_BLOCKCOPY_INPLACE_FINALIZATION_SUPPORT = "host.vddk.blockcopy.inplace.finalization.support";
+    String HOST_VDDK_BLOCKCOPY_RBD_SUPPORT = "host.vddk.blockcopy.rbd.support";
+    String HOST_QEMU_IMG_VERSION = "host.qemu.img.version";
+    String HOST_QEMU_NBD_VERSION = "host.qemu.nbd.version";
+    String HOST_QEMU_IO_VERSION = "host.qemu.io.version";
     String HOST_OVFTOOL_VERSION = "host.ovftool.version";
     String HOST_VIRTV2V_VERSION = "host.virtv2v.version";
+    String HOST_VIRTV2V_INPLACE_VERSION = "host.virtv2v.inplace.version";
+    String HOST_VDDK_RBD_DIRECT_IMPORT_SUPPORT = "host.vddk.rbd.direct.import.support";
+    String HOST_VIRTV2V_INPLACE_SUPPORT = "host.virtv2v.inplace.support";
+    String HOST_QEMU_RBD_SUPPORT = "host.qemu.rbd.support";
     String HOST_SSH_PORT = "host.ssh.port";
     String HOST_CDROM_MAX_COUNT = "host.cdrom.max.count";
     String GUEST_OS_CATEGORY_ID = "guest.os.category.id";

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -652,6 +652,29 @@ public class ApiConstants {
     public static final String USER_SECURITY_GROUP_LIST = "usersecuritygrouplist";
     public static final String USER_SECRET_KEY = "usersecretkey";
     public static final String USE_VDDK = "usevddk";
+    public static final String VMWARE_MIGRATION_MODE = "vmwaremigrationmode";
+    public static final String VMWARE_CBT_MIGRATION_ID = "vmwarecbtmigrationid";
+    public static final String SOURCE_HOST = "sourcehost";
+    public static final String SOURCE_CLUSTER = "sourcecluster";
+    public static final String SOURCE_VM_NAME = "sourcevmname";
+    public static final String SOURCE_DISK_ID = "sourcediskid";
+    public static final String SOURCE_DISK_DEVICE_KEY = "sourcediskdevicekey";
+    public static final String SOURCE_DISK_PATH = "sourcediskpath";
+    public static final String TARGET_PATH = "targetpath";
+    public static final String TARGET_FORMAT = "targetformat";
+    public static final String CHANGE_ID = "changeid";
+    public static final String SNAPSHOT_MOR = "snapshotmor";
+    public static final String CURRENT_STEP = "currentstep";
+    public static final String LAST_ERROR = "lasterror";
+    public static final String COMPLETED_CYCLES = "completedcycles";
+    public static final String QUIET_CYCLES = "quietcycles";
+    public static final String LAST_CHANGED_BYTES = "lastchangedbytes";
+    public static final String LAST_DIRTY_RATE = "lastdirtyrate";
+    public static final String TOTAL_CHANGED_BYTES = "totalchangedbytes";
+    public static final String CYCLE = "cycle";
+    public static final String CYCLE_NUMBER = "cyclenumber";
+    public static final String CHANGED_BYTES = "changedbytes";
+    public static final String DIRTY_RATE = "dirtyrate";
     public static final String USE_VIRTUAL_NETWORK = "usevirtualnetwork";
     public static final String USE_VIRTUAL_ROUTER_IP_RESOLVER = "userouteripresolver";
     public static final String UPDATE_IN_SEQUENCE = "updateinsequence";