set cpu bandwidth for the vm

The cpu bandwidth is applied at the vcpu group level. We should apply it
at the vm group level too, because the vm may do heavy I/O, and it will affect
the other vm.

We apply cpu bandwidth at the vcpu and the vm group level, so we must ensure
that max(child_quota) <= parent_quota when we modify cpu bandwidth.

src/qemu/qemu_cgroup.c

@@ -435,6 +435,7 @@ int qemuSetupCgroupForVcpu(struct qemud_driver *driver, virDomainObjPtr vm)
     unsigned int i;
     unsigned long long period = vm->def->cputune.period;
     long long quota = vm->def->cputune.quota;
+    long long vm_quota = 0;
 
     if (driver->cgroup == NULL)
         return 0; /* Not supported, so claim success */
@@ -447,26 +448,31 @@ int qemuSetupCgroupForVcpu(struct qemud_driver *driver, virDomainObjPtr vm)
         goto cleanup;
     }
 
+    /* Set cpu bandwidth for the vm */
+    if (period || quota) {
+        if (qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_CPU)) {
+            /* Ensure that we can multiply by vcpus without overflowing. */
+            if (quota > LLONG_MAX / vm->def->vcpus) {
+                virReportSystemError(EINVAL,
+                                     _("%s"),
+                                     "Unable to set cpu bandwidth quota");
+                goto cleanup;
+            }
+
+            if (quota > 0)
+                vm_quota = quota * vm->def->vcpus;
+            else
+                vm_quota = quota;
+            if (qemuSetupCgroupVcpuBW(cgroup, period, vm_quota) < 0)
+                goto cleanup;
+        }
+    }
+
     if (priv->nvcpupids == 0 || priv->vcpupids[0] == vm->pid) {
         /* If we does not know VCPU<->PID mapping or all vcpu runs in the same
          * thread, we cannot control each vcpu.
          */
-        if (period || quota) {
-            if (qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_CPU)) {
-                /* Ensure that we can multiply by vcpus without overflowing. */
-                if (quota > LLONG_MAX / vm->def->vcpus) {
-                    virReportSystemError(EINVAL,
-                                         _("%s"),
-                                         "Unable to set cpu bandwidth quota");
-                    goto cleanup;
-                }
-
-                if (quota > 0)
-                    quota *= vm->def->vcpus;
-                if (qemuSetupCgroupVcpuBW(cgroup, period, quota) < 0)
-                    goto cleanup;
-            }
-        }
+        virCgroupFree(&cgroup);
         return 0;
     }
 

src/qemu/qemu_driver.c

@@ -6064,43 +6064,98 @@ qemuSetVcpusBWLive(virDomainObjPtr vm, virCgroupPtr cgroup,
     qemuDomainObjPrivatePtr priv = vm->privateData;
     virCgroupPtr cgroup_vcpu = NULL;
     int rc;
+    long long vm_quota = 0;
+    long long old_quota = 0;
+    unsigned long long old_period = 0;
 
     if (period == 0 && quota == 0)
         return 0;
 
-    if (priv->nvcpupids == 0 || priv->vcpupids[0] == vm->pid) {
-        /* If we does not know VCPU<->PID mapping or all vcpu runs in the same
-         * thread, we cannot control each vcpu.
-         */
-        /* Ensure that we can multiply by vcpus without overflowing. */
-        if (quota > LLONG_MAX / vm->def->vcpus) {
-            virReportSystemError(EINVAL,
-                                 _("%s"),
-                                 "Unable to set cpu bandwidth quota");
-            goto cleanup;
-        }
+    /* Ensure that we can multiply by vcpus without overflowing. */
+    if (quota > LLONG_MAX / vm->def->vcpus) {
+        virReportSystemError(EINVAL,
+                             _("%s"),
+                             "Unable to set cpu bandwidth quota");
+        goto cleanup;
+    }
+
+    if (quota > 0)
+        vm_quota = quota * vm->def->vcpus;
+    else
+        vm_quota = quota;
 
-        if (quota > 0)
-            quota *= vm->def->vcpus;
-        return qemuSetupCgroupVcpuBW(cgroup, period, quota);
+    rc = virCgroupGetCpuCfsQuota(cgroup, &old_quota);
+    if (rc < 0) {
+        virReportSystemError(-rc, "%s",
+                             _("unable to get cpu bandwidth tunable"));
+        goto cleanup;
     }
 
-    for (i = 0; i < priv->nvcpupids; i++) {
-        rc = virCgroupForVcpu(cgroup, i, &cgroup_vcpu, 0);
-        if (rc < 0) {
-            virReportSystemError(-rc,
-                                 _("Unable to find vcpu cgroup for %s(vcpu:"
-                                   " %d)"),
-                                 vm->def->name, i);
-            goto cleanup;
+    rc = virCgroupGetCpuCfsPeriod(cgroup, &old_period);
+    if (rc < 0) {
+        virReportSystemError(-rc, "%s",
+                             _("unable to get cpu bandwidth period tunable"));
+        goto cleanup;
+    }
+
+    /*
+     * If quota will be changed to a small value, we should modify vcpu's quota
+     * first. Otherwise, we should modify vm's quota first.
+     *
+     * If period will be changed to a small value, we should modify vm's period
+     * first. Otherwise, we should modify vcpu's period first.
+     *
+     * If both quota and period will be changed to a big/small value, we cannot
+     * modify period and quota together.
+     */
+    if ((quota != 0) && (period != 0)) {
+        if (((quota > old_quota) && (period > old_period)) ||
+            ((quota < old_quota) && (period < old_period))) {
+            /* modify period */
+            if (qemuSetVcpusBWLive(vm, cgroup, period, 0) < 0)
+                goto cleanup;
+
+            /* modify quota */
+            if (qemuSetVcpusBWLive(vm, cgroup, 0, quota) < 0)
+                goto cleanup;
+            return 0;
         }
+    }
 
-        if (qemuSetupCgroupVcpuBW(cgroup_vcpu, period, quota) < 0)
+    if (((vm_quota != 0) && (vm_quota > old_quota)) ||
+        ((period != 0) && (period < old_period)))
+        /* Set cpu bandwidth for the vm */
+        if (qemuSetupCgroupVcpuBW(cgroup, period, vm_quota) < 0)
             goto cleanup;
 
-        virCgroupFree(&cgroup_vcpu);
+    /* If we does not know VCPU<->PID mapping or all vcpu runs in the same
+     * thread, we cannot control each vcpu. So we only modify cpu bandwidth
+     * when each vcpu has a separated thread.
+     */
+    if (priv->nvcpupids != 0 && priv->vcpupids[0] != vm->pid) {
+        for (i = 0; i < priv->nvcpupids; i++) {
+            rc = virCgroupForVcpu(cgroup, i, &cgroup_vcpu, 0);
+            if (rc < 0) {
+                virReportSystemError(-rc,
+                                     _("Unable to find vcpu cgroup for %s(vcpu:"
+                                       " %d)"),
+                                     vm->def->name, i);
+                goto cleanup;
+            }
+
+            if (qemuSetupCgroupVcpuBW(cgroup_vcpu, period, quota) < 0)
+                goto cleanup;
+
+            virCgroupFree(&cgroup_vcpu);
+        }
     }
 
+    if (((vm_quota != 0) && (vm_quota <= old_quota)) ||
+        ((period != 0) && (period >= old_period)))
+        /* Set cpu bandwidth for the vm */
+        if (qemuSetupCgroupVcpuBW(cgroup, period, vm_quota) < 0)
+            goto cleanup;
+
     return 0;
 
 cleanup: