qemu: migration: Prepare for non-contiguous vcpu configurations

Introduce a new migration cookie flag that will be used for any
configurations that are not compatible with libvirt that would not
support the specific vcpu hotplug approach. This will make sure that old
libvirt does not fail to reproduce the configuration correctly.

src/qemu/qemu_domain.c

@@ -5964,3 +5964,38 @@ qemuDomainPrepareChannel(virDomainChrDefPtr channel,
 
     return 0;
 }
+
+
+/**
+ * qemuDomainVcpuHotplugIsInOrder:
+ * @def: domain definition
+ *
+ * Returns true if online vcpus were added in order (clustered behind vcpu0
+ * with increasing order).
+ */
+bool
+qemuDomainVcpuHotplugIsInOrder(virDomainDefPtr def)
+{
+    size_t maxvcpus = virDomainDefGetVcpusMax(def);
+    virDomainVcpuDefPtr vcpu;
+    unsigned int prevorder = 0;
+    size_t seenonlinevcpus = 0;
+    size_t i;
+
+    for (i = 0; i < maxvcpus; i++) {
+        vcpu = virDomainDefGetVcpu(def, i);
+
+        if (!vcpu->online)
+            break;
+
+        if (vcpu->order < prevorder)
+            break;
+
+        if (vcpu->order > prevorder)
+            prevorder = vcpu->order;
+
+        seenonlinevcpus++;
+    }
+
+    return seenonlinevcpus == virDomainDefGetVcpus(def);
+}

src/qemu/qemu_domain.h

@@ -722,4 +722,7 @@ int qemuDomainPrepareChannel(virDomainChrDefPtr chr,
                              const char *domainChannelTargetDir)
     ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2);
 
+bool qemuDomainVcpuHotplugIsInOrder(virDomainDefPtr def)
+    ATTRIBUTE_NONNULL(1);
+
 #endif /* __QEMU_DOMAIN_H__ */

src/qemu/qemu_migration.c

@@ -92,6 +92,7 @@ enum qemuMigrationCookieFlags {
     QEMU_MIGRATION_COOKIE_FLAG_NBD,
     QEMU_MIGRATION_COOKIE_FLAG_STATS,
     QEMU_MIGRATION_COOKIE_FLAG_MEMORY_HOTPLUG,
+    QEMU_MIGRATION_COOKIE_FLAG_CPU_HOTPLUG,
 
     QEMU_MIGRATION_COOKIE_FLAG_LAST
 };
@@ -105,7 +106,8 @@ VIR_ENUM_IMPL(qemuMigrationCookieFlag,
               "network",
               "nbd",
               "statistics",
-              "memory-hotplug");
+              "memory-hotplug",
+              "cpu-hotplug");
 
 enum qemuMigrationCookieFeatures {
     QEMU_MIGRATION_COOKIE_GRAPHICS  = (1 << QEMU_MIGRATION_COOKIE_FLAG_GRAPHICS),
@@ -115,6 +117,7 @@ enum qemuMigrationCookieFeatures {
     QEMU_MIGRATION_COOKIE_NBD = (1 << QEMU_MIGRATION_COOKIE_FLAG_NBD),
     QEMU_MIGRATION_COOKIE_STATS = (1 << QEMU_MIGRATION_COOKIE_FLAG_STATS),
     QEMU_MIGRATION_COOKIE_MEMORY_HOTPLUG = (1 << QEMU_MIGRATION_COOKIE_FLAG_MEMORY_HOTPLUG),
+    QEMU_MIGRATION_COOKIE_CPU_HOTPLUG = (1 << QEMU_MIGRATION_COOKIE_FLAG_CPU_HOTPLUG),
 };
 
 typedef struct _qemuMigrationCookieGraphics qemuMigrationCookieGraphics;
@@ -1408,6 +1411,9 @@ qemuMigrationBakeCookie(qemuMigrationCookiePtr mig,
     if (flags & QEMU_MIGRATION_COOKIE_MEMORY_HOTPLUG)
         mig->flagsMandatory |= QEMU_MIGRATION_COOKIE_MEMORY_HOTPLUG;
 
+    if (flags & QEMU_MIGRATION_COOKIE_CPU_HOTPLUG)
+        mig->flagsMandatory |= QEMU_MIGRATION_COOKIE_CPU_HOTPLUG;
+
     if (!(*cookieout = qemuMigrationCookieXMLFormatStr(driver, mig)))
         return -1;
 
@@ -3191,6 +3197,11 @@ qemuMigrationBeginPhase(virQEMUDriverPtr driver,
          vm->newDef && virDomainDefHasMemoryHotplug(vm->newDef)))
         cookieFlags |= QEMU_MIGRATION_COOKIE_MEMORY_HOTPLUG;
 
+    if (!qemuDomainVcpuHotplugIsInOrder(vm->def) ||
+        ((flags & VIR_MIGRATE_PERSIST_DEST) &&
+         vm->newDef && !qemuDomainVcpuHotplugIsInOrder(vm->newDef)))
+        cookieFlags |= QEMU_MIGRATION_COOKIE_CPU_HOTPLUG;
+
     if (!(mig = qemuMigrationEatCookie(driver, vm, NULL, 0, 0)))
         goto cleanup;
 
@@ -3686,7 +3697,8 @@ qemuMigrationPrepareAny(virQEMUDriverPtr driver,
     if (!(mig = qemuMigrationEatCookie(driver, vm, cookiein, cookieinlen,
                                        QEMU_MIGRATION_COOKIE_LOCKSTATE |
                                        QEMU_MIGRATION_COOKIE_NBD |
-                                       QEMU_MIGRATION_COOKIE_MEMORY_HOTPLUG)))
+                                       QEMU_MIGRATION_COOKIE_MEMORY_HOTPLUG |
+                                       QEMU_MIGRATION_COOKIE_CPU_HOTPLUG)))
         goto cleanup;
 
     if (STREQ_NULLABLE(protocol, "rdma") &&