qemu_cgroup.c 25.6 KB
Newer Older
1 2 3
/*
 * qemu_cgroup.c: QEMU cgroup management
 *
4
 * Copyright (C) 2006-2012 Red Hat, Inc.
5 6 7 8 9 10 11 12 13 14 15 16 17
 * Copyright (C) 2006 Daniel P. Berrange
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
18
 * License along with this library.  If not, see
O
Osier Yang 已提交
19
 * <http://www.gnu.org/licenses/>.
20 21 22 23 24 25 26
 *
 * Author: Daniel P. Berrange <berrange@redhat.com>
 */

#include <config.h>

#include "qemu_cgroup.h"
27
#include "qemu_domain.h"
28
#include "qemu_process.h"
29
#include "vircgroup.h"
30
#include "virlog.h"
31
#include "viralloc.h"
32
#include "virerror.h"
33
#include "virutil.h"
34
#include "domain_audit.h"
35 36 37 38 39 40 41

#define VIR_FROM_THIS VIR_FROM_QEMU

static const char *const defaultDeviceACL[] = {
    "/dev/null", "/dev/full", "/dev/zero",
    "/dev/random", "/dev/urandom",
    "/dev/ptmx", "/dev/kvm", "/dev/kqemu",
42
    "/dev/rtc", "/dev/hpet",
43 44 45 46 47
    NULL,
};
#define DEVICE_PTY_MAJOR 136
#define DEVICE_SND_MAJOR 116

48
bool qemuCgroupControllerActive(virQEMUDriverPtr driver,
49
                                int controller)
50 51
{
    if (driver->cgroup == NULL)
52 53 54
        return false;
    if (controller < 0 || controller >= VIR_CGROUP_CONTROLLER_LAST)
        return false;
E
Eric Blake 已提交
55 56
    if (!virCgroupMounted(driver->cgroup, controller))
        return false;
57
    if (driver->cgroupControllers & (1 << controller))
58 59
        return true;
    return false;
60 61
}

62
static int
63
qemuSetupDiskPathAllow(virDomainDiskDefPtr disk,
64 65 66
                       const char *path,
                       size_t depth ATTRIBUTE_UNUSED,
                       void *opaque)
67
{
68
    qemuCgroupData *data = opaque;
69 70 71
    int rc;

    VIR_DEBUG("Process path %s for disk", path);
72 73 74
    rc = virCgroupAllowDevicePath(data->cgroup, path,
                                  (disk->readonly ? VIR_CGROUP_DEVICE_READ
                                   : VIR_CGROUP_DEVICE_RW));
75 76
    virDomainAuditCgroupPath(data->vm, data->cgroup, "allow", path,
                             disk->readonly ? "r" : "rw", rc);
77 78
    if (rc < 0) {
        if (rc == -EACCES) { /* Get this for root squash NFS */
79 80 81 82 83 84 85 86 87 88 89 90
            VIR_DEBUG("Ignoring EACCES for %s", path);
        } else {
            virReportSystemError(-rc,
                                 _("Unable to allow access for disk path %s"),
                                 path);
            return -1;
        }
    }
    return 0;
}


91
int qemuSetupDiskCgroup(virDomainObjPtr vm,
92 93 94
                        virCgroupPtr cgroup,
                        virDomainDiskDefPtr disk)
{
95
    qemuCgroupData data = { vm, cgroup };
96 97 98
    return virDomainDiskDefForeachPath(disk,
                                       true,
                                       qemuSetupDiskPathAllow,
99
                                       &data);
100 101 102
}


103 104 105 106 107
static int
qemuTeardownDiskPathDeny(virDomainDiskDefPtr disk ATTRIBUTE_UNUSED,
                         const char *path,
                         size_t depth ATTRIBUTE_UNUSED,
                         void *opaque)
108
{
109
    qemuCgroupData *data = opaque;
110 111 112
    int rc;

    VIR_DEBUG("Process path %s for disk", path);
113 114
    rc = virCgroupDenyDevicePath(data->cgroup, path,
                                 VIR_CGROUP_DEVICE_RWM);
115
    virDomainAuditCgroupPath(data->vm, data->cgroup, "deny", path, "rwm", rc);
116 117
    if (rc < 0) {
        if (rc == -EACCES) { /* Get this for root squash NFS */
118 119 120 121 122 123 124 125 126 127 128 129
            VIR_DEBUG("Ignoring EACCES for %s", path);
        } else {
            virReportSystemError(-rc,
                                 _("Unable to deny access for disk path %s"),
                                 path);
            return -1;
        }
    }
    return 0;
}


130
int qemuTeardownDiskCgroup(virDomainObjPtr vm,
131 132 133
                           virCgroupPtr cgroup,
                           virDomainDiskDefPtr disk)
{
134
    qemuCgroupData data = { vm, cgroup };
135 136 137
    return virDomainDiskDefForeachPath(disk,
                                       true,
                                       qemuTeardownDiskPathDeny,
138
                                       &data);
139 140 141
}


142 143 144 145
static int
qemuSetupChardevCgroup(virDomainDefPtr def,
                       virDomainChrDefPtr dev,
                       void *opaque)
146
{
147
    qemuCgroupData *data = opaque;
148 149
    int rc;

150
    if (dev->source.type != VIR_DOMAIN_CHR_TYPE_DEV)
151 152 153
        return 0;


154
    VIR_DEBUG("Process path '%s' for disk", dev->source.data.file.path);
155 156
    rc = virCgroupAllowDevicePath(data->cgroup, dev->source.data.file.path,
                                  VIR_CGROUP_DEVICE_RW);
157 158
    virDomainAuditCgroupPath(data->vm, data->cgroup, "allow",
                             dev->source.data.file.path, "rw", rc);
159
    if (rc < 0) {
160 161
        virReportSystemError(-rc,
                             _("Unable to allow device %s for %s"),
162
                             dev->source.data.file.path, def->name);
163 164 165 166 167 168 169 170 171 172 173
        return -1;
    }

    return 0;
}


int qemuSetupHostUsbDeviceCgroup(usbDevice *dev ATTRIBUTE_UNUSED,
                                 const char *path,
                                 void *opaque)
{
174
    qemuCgroupData *data = opaque;
175 176 177
    int rc;

    VIR_DEBUG("Process path '%s' for USB device", path);
178 179
    rc = virCgroupAllowDevicePath(data->cgroup, path,
                                  VIR_CGROUP_DEVICE_RW);
180
    virDomainAuditCgroupPath(data->vm, data->cgroup, "allow", path, "rw", rc);
181
    if (rc < 0) {
182 183 184 185 186 187 188 189 190
        virReportSystemError(-rc,
                             _("Unable to allow device %s"),
                             path);
        return -1;
    }

    return 0;
}

191
int qemuSetupCgroup(virQEMUDriverPtr driver,
192
                    virDomainObjPtr vm,
193
                    virBitmapPtr nodemask)
194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214
{
    virCgroupPtr cgroup = NULL;
    int rc;
    unsigned int i;
    const char *const *deviceACL =
        driver->cgroupDeviceACL ?
        (const char *const *)driver->cgroupDeviceACL :
        defaultDeviceACL;

    if (driver->cgroup == NULL)
        return 0; /* Not supported, so claim success */

    rc = virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 1);
    if (rc != 0) {
        virReportSystemError(-rc,
                             _("Unable to create cgroup for %s"),
                             vm->def->name);
        goto cleanup;
    }

    if (qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_DEVICES)) {
215
        qemuCgroupData data = { vm, cgroup };
216
        rc = virCgroupDenyAllDevices(cgroup);
217
        virDomainAuditCgroup(vm, cgroup, "deny", "all", rc == 0);
218 219
        if (rc != 0) {
            if (rc == -EPERM) {
220
                VIR_WARN("Group devices ACL is not accessible, disabling whitelisting");
221 222 223 224 225 226 227 228 229
                goto done;
            }

            virReportSystemError(-rc,
                                 _("Unable to deny all devices for %s"), vm->def->name);
            goto cleanup;
        }

        for (i = 0; i < vm->def->ndisks ; i++) {
230 231
            if (qemuDomainDetermineDiskChain(driver, vm->def->disks[i],
                                             false) < 0 ||
232
                qemuSetupDiskCgroup(vm, cgroup, vm->def->disks[i]) < 0)
233 234 235
                goto cleanup;
        }

236 237
        rc = virCgroupAllowDeviceMajor(cgroup, 'c', DEVICE_PTY_MAJOR,
                                       VIR_CGROUP_DEVICE_RW);
238 239
        virDomainAuditCgroupMajor(vm, cgroup, "allow", DEVICE_PTY_MAJOR,
                                  "pty", "rw", rc == 0);
240 241 242 243 244 245
        if (rc != 0) {
            virReportSystemError(-rc, "%s",
                                 _("unable to allow /dev/pts/ devices"));
            goto cleanup;
        }

246 247 248 249 250
        if (vm->def->nsounds &&
            (!vm->def->ngraphics ||
             ((vm->def->graphics[0]->type == VIR_DOMAIN_GRAPHICS_TYPE_VNC &&
               driver->vncAllowHostAudio) ||
              (vm->def->graphics[0]->type == VIR_DOMAIN_GRAPHICS_TYPE_SDL)))) {
251 252
            rc = virCgroupAllowDeviceMajor(cgroup, 'c', DEVICE_SND_MAJOR,
                                           VIR_CGROUP_DEVICE_RW);
253 254
            virDomainAuditCgroupMajor(vm, cgroup, "allow", DEVICE_SND_MAJOR,
                                      "sound", "rw", rc == 0);
255 256 257 258 259 260 261 262
            if (rc != 0) {
                virReportSystemError(-rc, "%s",
                                     _("unable to allow /dev/snd/ devices"));
                goto cleanup;
            }
        }

        for (i = 0; deviceACL[i] != NULL ; i++) {
263 264
            rc = virCgroupAllowDevicePath(cgroup, deviceACL[i],
                                          VIR_CGROUP_DEVICE_RW);
265
            virDomainAuditCgroupPath(vm, cgroup, "allow", deviceACL[i], "rw", rc);
266 267 268 269 270 271 272 273 274 275 276 277
            if (rc < 0 &&
                rc != -ENOENT) {
                virReportSystemError(-rc,
                                     _("unable to allow device %s"),
                                     deviceACL[i]);
                goto cleanup;
            }
        }

        if (virDomainChrDefForeach(vm->def,
                                   true,
                                   qemuSetupChardevCgroup,
278
                                   &data) < 0)
279 280 281 282 283 284 285 286 287 288
            goto cleanup;

        for (i = 0; i < vm->def->nhostdevs; i++) {
            virDomainHostdevDefPtr hostdev = vm->def->hostdevs[i];
            usbDevice *usb;

            if (hostdev->mode != VIR_DOMAIN_HOSTDEV_MODE_SUBSYS)
                continue;
            if (hostdev->source.subsys.type != VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB)
                continue;
289 290
            if (hostdev->missing)
                continue;
291 292

            if ((usb = usbGetDevice(hostdev->source.subsys.u.usb.bus,
293 294
                                    hostdev->source.subsys.u.usb.device,
                                    NULL)) == NULL)
295 296
                goto cleanup;

297 298
            if (usbDeviceFileIterate(usb, qemuSetupHostUsbDeviceCgroup,
                                     &data) < 0)
299 300 301 302
                goto cleanup;
        }
    }

303 304
    if (vm->def->blkio.weight != 0) {
        if (qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_BLKIO)) {
305
            rc = virCgroupSetBlkioWeight(cgroup, vm->def->blkio.weight);
306
            if (rc != 0) {
307 308 309 310 311
                virReportSystemError(-rc,
                                     _("Unable to set io weight for domain %s"),
                                     vm->def->name);
                goto cleanup;
            }
312
        } else {
313 314
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                           _("Block I/O tuning is not available on this host"));
315 316 317 318 319 320 321 322
            goto cleanup;
        }
    }

    if (vm->def->blkio.ndevices) {
        if (qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_BLKIO)) {
            for (i = 0; i < vm->def->blkio.ndevices; i++) {
                virBlkioDeviceWeightPtr dw = &vm->def->blkio.devices[i];
323 324
                if (!dw->weight)
                    continue;
325 326 327 328 329 330 331 332 333 334 335
                rc = virCgroupSetBlkioDeviceWeight(cgroup, dw->path,
                                                   dw->weight);
                if (rc != 0) {
                    virReportSystemError(-rc,
                                         _("Unable to set io device weight "
                                           "for domain %s"),
                                         vm->def->name);
                    goto cleanup;
                }
            }
        } else {
336
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
337
                           _("Block I/O tuning is not available on this host"));
338
            goto cleanup;
339 340 341
        }
    }

342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370
    if (qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_MEMORY)) {
        unsigned long long hard_limit = vm->def->mem.hard_limit;

        if (!hard_limit) {
            /* If there is no hard_limit set, set a reasonable
             * one to avoid system trashing caused by exploited qemu.
             * As 'reasonable limit' has been chosen:
             *     (1 + k) * (domain memory + total video memory) + F
             * where k = 0.02 and F = 200MB. */
            hard_limit = vm->def->mem.max_balloon;
            for (i = 0; i < vm->def->nvideos; i++)
                hard_limit += vm->def->videos[i]->vram;
            hard_limit = hard_limit * 1.02 + 204800;
        }

        rc = virCgroupSetMemoryHardLimit(cgroup, hard_limit);
        if (rc != 0) {
            virReportSystemError(-rc,
                                 _("Unable to set memory hard limit for domain %s"),
                                 vm->def->name);
            goto cleanup;
        }
        if (vm->def->mem.soft_limit != 0) {
            rc = virCgroupSetMemorySoftLimit(cgroup, vm->def->mem.soft_limit);
            if (rc != 0) {
                virReportSystemError(-rc,
                                     _("Unable to set memory soft limit for domain %s"),
                                     vm->def->name);
                goto cleanup;
371
            }
372
        }
373

374 375 376 377 378 379 380
        if (vm->def->mem.swap_hard_limit != 0) {
            rc = virCgroupSetMemSwapHardLimit(cgroup, vm->def->mem.swap_hard_limit);
            if (rc != 0) {
                virReportSystemError(-rc,
                                     _("Unable to set swap hard limit for domain %s"),
                                     vm->def->name);
                goto cleanup;
381 382
            }
        }
383 384 385 386 387 388 389
    } else if (vm->def->mem.hard_limit != 0 ||
               vm->def->mem.soft_limit != 0 ||
               vm->def->mem.swap_hard_limit != 0) {
        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                       _("Memory cgroup is not available on this host"));
    } else {
        VIR_WARN("Could not autoset a RSS limit for domain %s", vm->def->name);
390 391
    }

392 393
    if (vm->def->cputune.shares != 0) {
        if (qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_CPU)) {
394
            rc = virCgroupSetCpuShares(cgroup, vm->def->cputune.shares);
395
            if (rc != 0) {
396 397 398 399 400
                virReportSystemError(-rc,
                                     _("Unable to set io cpu shares for domain %s"),
                                     vm->def->name);
                goto cleanup;
            }
401
        } else {
402 403
            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                           _("CPU tuning is not available on this host"));
404 405 406
        }
    }

407 408 409
    if ((vm->def->numatune.memory.nodemask ||
         (vm->def->numatune.memory.placement_mode ==
          VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_AUTO)) &&
H
Hu Tao 已提交
410 411
        vm->def->numatune.memory.mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT &&
        qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_CPUSET)) {
412 413 414
        char *mask = NULL;
        if (vm->def->numatune.memory.placement_mode ==
            VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_AUTO)
415
            mask = virBitmapFormat(nodemask);
416
        else
417
            mask = virBitmapFormat(vm->def->numatune.memory.nodemask);
H
Hu Tao 已提交
418
        if (!mask) {
419 420
            virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
                           _("failed to convert memory nodemask"));
H
Hu Tao 已提交
421 422 423 424 425 426 427 428 429 430 431 432
            goto cleanup;
        }

        rc = virCgroupSetCpusetMems(cgroup, mask);
        VIR_FREE(mask);
        if (rc != 0) {
            virReportSystemError(-rc,
                                 _("Unable to set cpuset.mems for domain %s"),
                                 vm->def->name);
            goto cleanup;
        }
    }
433 434 435 436 437 438 439 440 441 442 443 444
done:
    virCgroupFree(&cgroup);
    return 0;

cleanup:
    if (cgroup) {
        virCgroupRemove(cgroup);
        virCgroupFree(&cgroup);
    }
    return -1;
}

445 446 447 448 449 450 451 452 453 454 455 456 457 458
int qemuSetupCgroupVcpuBW(virCgroupPtr cgroup, unsigned long long period,
                          long long quota)
{
    int rc;
    unsigned long long old_period;

    if (period == 0 && quota == 0)
        return 0;

    if (period) {
        /* get old period, and we can rollback if set quota failed */
        rc = virCgroupGetCpuCfsPeriod(cgroup, &old_period);
        if (rc < 0) {
            virReportSystemError(-rc,
W
Wen Congyang 已提交
459
                                 "%s", _("Unable to get cpu bandwidth period"));
460 461 462 463 464 465
            return -1;
        }

        rc = virCgroupSetCpuCfsPeriod(cgroup, period);
        if (rc < 0) {
            virReportSystemError(-rc,
W
Wen Congyang 已提交
466
                                 "%s", _("Unable to set cpu bandwidth period"));
467 468 469 470 471 472 473 474
            return -1;
        }
    }

    if (quota) {
        rc = virCgroupSetCpuCfsQuota(cgroup, quota);
        if (rc < 0) {
            virReportSystemError(-rc,
W
Wen Congyang 已提交
475
                                 "%s", _("Unable to set cpu bandwidth quota"));
476 477 478 479 480 481 482 483 484 485
            goto cleanup;
        }
    }

    return 0;

cleanup:
    if (period) {
        rc = virCgroupSetCpuCfsPeriod(cgroup, old_period);
        if (rc < 0)
486
            virReportSystemError(-rc, "%s",
487
                                 _("Unable to rollback cpu bandwidth period"));
488 489 490 491 492
    }

    return -1;
}

493 494 495 496 497
int qemuSetupCgroupVcpuPin(virCgroupPtr cgroup,
                           virDomainVcpuPinDefPtr *vcpupin,
                           int nvcpupin,
                           int vcpuid)
{
498
    int i;
499 500 501

    for (i = 0; i < nvcpupin; i++) {
        if (vcpuid == vcpupin[i]->vcpuid) {
502
            return qemuSetupCgroupEmulatorPin(cgroup, vcpupin[i]->cpumask);
503 504 505
        }
    }

506 507 508 509
    return -1;
}

int qemuSetupCgroupEmulatorPin(virCgroupPtr cgroup,
510
                               virBitmapPtr cpumask)
511 512 513 514
{
    int rc = 0;
    char *new_cpus = NULL;

515
    new_cpus = virBitmapFormat(cpumask);
516 517 518 519 520 521 522 523 524 525 526 527 528 529 530
    if (!new_cpus) {
        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
                       _("failed to convert cpu mask"));
        rc = -1;
        goto cleanup;
    }

    rc = virCgroupSetCpusetCpus(cgroup, new_cpus);
    if (rc < 0) {
        virReportSystemError(-rc,
                             "%s",
                             _("Unable to set cpuset.cpus"));
        goto cleanup;
    }

531 532 533 534 535
cleanup:
    VIR_FREE(new_cpus);
    return rc;
}

536
int qemuSetupCgroupForVcpu(virQEMUDriverPtr driver, virDomainObjPtr vm)
537 538 539 540
{
    virCgroupPtr cgroup = NULL;
    virCgroupPtr cgroup_vcpu = NULL;
    qemuDomainObjPrivatePtr priv = vm->privateData;
541
    virDomainDefPtr def = vm->def;
542
    int rc;
M
Martin Kletzander 已提交
543
    unsigned int i, j;
544 545 546
    unsigned long long period = vm->def->cputune.period;
    long long quota = vm->def->cputune.quota;

547
    if ((period || quota) &&
548 549 550 551
        (!driver->cgroup ||
         !qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_CPU))) {
        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                       _("cgroup cpu is required for scheduler tuning"));
H
Hu Tao 已提交
552 553 554
        return -1;
    }

555 556 557 558 559 560 561
    /* We are trying to setup cgroups for CPU pinning, which can also be done
     * with virProcessInfoSetAffinity, thus the lack of cgroups is not fatal
     * here.
     */
    if (driver->cgroup == NULL)
        return 0;

562 563 564 565 566 567 568 569 570
    rc = virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0);
    if (rc != 0) {
        virReportSystemError(-rc,
                             _("Unable to find cgroup for %s"),
                             vm->def->name);
        goto cleanup;
    }

    if (priv->nvcpupids == 0 || priv->vcpupids[0] == vm->pid) {
571
        /* If we don't know VCPU<->PID mapping or all vcpu runs in the same
W
Wen Congyang 已提交
572
         * thread, we cannot control each vcpu.
573
         */
574 575 576
        VIR_WARN("Unable to get vcpus' pids.");
        virCgroupFree(&cgroup);
        return 0;
577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598
    }

    for (i = 0; i < priv->nvcpupids; i++) {
        rc = virCgroupForVcpu(cgroup, i, &cgroup_vcpu, 1);
        if (rc < 0) {
            virReportSystemError(-rc,
                                 _("Unable to create vcpu cgroup for %s(vcpu:"
                                   " %d)"),
                                 vm->def->name, i);
            goto cleanup;
        }

        /* move the thread for vcpu to sub dir */
        rc = virCgroupAddTask(cgroup_vcpu, priv->vcpupids[i]);
        if (rc < 0) {
            virReportSystemError(-rc,
                                 _("unable to add vcpu %d task %d to cgroup"),
                                 i, priv->vcpupids[i]);
            goto cleanup;
        }

        if (period || quota) {
H
Hu Tao 已提交
599 600
            if (qemuSetupCgroupVcpuBW(cgroup_vcpu, period, quota) < 0)
                goto cleanup;
601 602
        }

603
        /* Set vcpupin in cgroup if vcpupin xml is provided */
M
Martin Kletzander 已提交
604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619
        if (qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_CPUSET)) {
            /* find the right CPU to pin, otherwise
             * qemuSetupCgroupVcpuPin will fail. */
            for (j = 0; j < def->cputune.nvcpupin; j++) {
                if (def->cputune.vcpupin[j]->vcpuid != i)
                    continue;

                if (qemuSetupCgroupVcpuPin(cgroup_vcpu,
                                           def->cputune.vcpupin,
                                           def->cputune.nvcpupin,
                                           i) < 0)
                    goto cleanup;

                break;
            }
        }
620

621 622 623 624 625 626 627
        virCgroupFree(&cgroup_vcpu);
    }

    virCgroupFree(&cgroup);
    return 0;

cleanup:
628 629 630 631 632
    if (cgroup_vcpu) {
        virCgroupRemove(cgroup_vcpu);
        virCgroupFree(&cgroup_vcpu);
    }

633 634 635 636 637 638 639 640
    if (cgroup) {
        virCgroupRemove(cgroup);
        virCgroupFree(&cgroup);
    }

    return -1;
}

641
int qemuSetupCgroupForEmulator(virQEMUDriverPtr driver,
642 643
                               virDomainObjPtr vm,
                               virBitmapPtr nodemask)
644
{
645
    virBitmapPtr cpumask = NULL;
646
    virBitmapPtr cpumap = NULL;
647 648
    virCgroupPtr cgroup = NULL;
    virCgroupPtr cgroup_emulator = NULL;
649
    virDomainDefPtr def = vm->def;
650 651
    unsigned long long period = vm->def->cputune.emulator_period;
    long long quota = vm->def->cputune.emulator_quota;
652 653
    int rc, i;

654 655 656 657 658 659 660 661
    if ((period || quota) &&
        (!driver->cgroup ||
         !qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_CPU))) {
        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
                       _("cgroup cpu is required for scheduler tuning"));
        return -1;
    }

662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681
    if (driver->cgroup == NULL)
        return 0; /* Not supported, so claim success */

    rc = virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0);
    if (rc != 0) {
        virReportSystemError(-rc,
                             _("Unable to find cgroup for %s"),
                             vm->def->name);
        goto cleanup;
    }

    rc = virCgroupForEmulator(cgroup, &cgroup_emulator, 1);
    if (rc < 0) {
        virReportSystemError(-rc,
                             _("Unable to create emulator cgroup for %s"),
                             vm->def->name);
        goto cleanup;
    }

    for (i = 0; i < VIR_CGROUP_CONTROLLER_LAST; i++) {
682
        if (!qemuCgroupControllerActive(driver, i))
683 684 685 686 687 688 689 690 691 692 693
            continue;
        rc = virCgroupMoveTask(cgroup, cgroup_emulator, i);
        if (rc < 0) {
            virReportSystemError(-rc,
                                 _("Unable to move tasks from domain cgroup to "
                                   "emulator cgroup in controller %d for %s"),
                                 i, vm->def->name);
            goto cleanup;
        }
    }

694 695 696 697 698
    if (def->placement_mode == VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO) {
        if (!(cpumap = qemuPrepareCpumap(driver, nodemask)))
            goto cleanup;
        cpumask = cpumap;
    } else if (def->cputune.emulatorpin) {
699
        cpumask = def->cputune.emulatorpin->cpumask;
700
    } else if (def->cpumask) {
701
        cpumask = def->cpumask;
702
    }
703 704 705 706 707 708 709 710

    if (cpumask) {
        if (qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_CPUSET)) {
            rc = qemuSetupCgroupEmulatorPin(cgroup_emulator, cpumask);
            if (rc < 0)
                goto cleanup;
        }
        cpumask = NULL; /* sanity */
H
Hu Tao 已提交
711
    }
712

713 714
    if (period || quota) {
        if (qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_CPU)) {
H
Hu Tao 已提交
715 716
            if ((rc = qemuSetupCgroupVcpuBW(cgroup_emulator, period,
                                            quota)) < 0)
717 718 719 720
                goto cleanup;
        }
    }

721 722
    virCgroupFree(&cgroup_emulator);
    virCgroupFree(&cgroup);
723
    virBitmapFree(cpumap);
724 725 726
    return 0;

cleanup:
727 728
    virBitmapFree(cpumap);

729 730 731 732 733 734 735 736 737 738 739 740
    if (cgroup_emulator) {
        virCgroupRemove(cgroup_emulator);
        virCgroupFree(&cgroup_emulator);
    }

    if (cgroup) {
        virCgroupRemove(cgroup);
        virCgroupFree(&cgroup);
    }

    return rc;
}
741

742
int qemuRemoveCgroup(virQEMUDriverPtr driver,
743 744 745 746 747 748 749 750 751 752 753 754
                     virDomainObjPtr vm,
                     int quiet)
{
    virCgroupPtr cgroup;
    int rc;

    if (driver->cgroup == NULL)
        return 0; /* Not supported, so claim success */

    rc = virCgroupForDomain(driver->cgroup, vm->def->name, &cgroup, 0);
    if (rc != 0) {
        if (!quiet)
755 756 757
            virReportError(VIR_ERR_INTERNAL_ERROR,
                           _("Unable to find cgroup for %s"),
                           vm->def->name);
758 759 760 761 762 763 764 765
        return rc;
    }

    rc = virCgroupRemove(cgroup);
    virCgroupFree(&cgroup);
    return rc;
}

766
int qemuAddToCgroup(virQEMUDriverPtr driver,
767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797
                    virDomainDefPtr def)
{
    virCgroupPtr cgroup = NULL;
    int ret = -1;
    int rc;

    if (driver->cgroup == NULL)
        return 0; /* Not supported, so claim success */

    rc = virCgroupForDomain(driver->cgroup, def->name, &cgroup, 0);
    if (rc != 0) {
        virReportSystemError(-rc,
                             _("unable to find cgroup for domain %s"),
                             def->name);
        goto cleanup;
    }

    rc = virCgroupAddTask(cgroup, getpid());
    if (rc != 0) {
        virReportSystemError(-rc,
                             _("unable to add domain %s task %d to cgroup"),
                             def->name, getpid());
        goto cleanup;
    }

    ret = 0;

cleanup:
    virCgroupFree(&cgroup);
    return ret;
}