block.c 28.4 KB
Newer Older
L
lirans@il.ibm.com 已提交
1 2 3 4 5 6 7 8 9 10 11
/*
 * QEMU live block migration
 *
 * Copyright IBM, Corp. 2009
 *
 * Authors:
 *  Liran Schour   <lirans@il.ibm.com>
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
12 13
 * Contributions after 2012-01-13 are licensed under the terms of the
 * GNU GPL, version 2 or (at your option) any later version.
L
lirans@il.ibm.com 已提交
14 15
 */

P
Peter Maydell 已提交
16
#include "qemu/osdep.h"
17
#include "qapi/error.h"
L
lirans@il.ibm.com 已提交
18
#include "qemu-common.h"
19 20 21
#include "block/block.h"
#include "qemu/error-report.h"
#include "qemu/main-loop.h"
L
lirans@il.ibm.com 已提交
22
#include "hw/hw.h"
23
#include "qemu/cutils.h"
24 25
#include "qemu/queue.h"
#include "qemu/timer.h"
26 27
#include "block.h"
#include "migration/misc.h"
28
#include "migration/migration.h"
29
#include "sysemu/blockdev.h"
J
Juan Quintela 已提交
30
#include "qemu-file.h"
31
#include "migration/vmstate.h"
32
#include "sysemu/block-backend.h"
L
lirans@il.ibm.com 已提交
33

34 35
#define BLOCK_SIZE                       (1 << 20)
#define BDRV_SECTORS_PER_DIRTY_CHUNK     (BLOCK_SIZE >> BDRV_SECTOR_BITS)
L
lirans@il.ibm.com 已提交
36 37 38

#define BLK_MIG_FLAG_DEVICE_BLOCK       0x01
#define BLK_MIG_FLAG_EOS                0x02
39
#define BLK_MIG_FLAG_PROGRESS           0x04
40
#define BLK_MIG_FLAG_ZERO_BLOCK         0x08
L
lirans@il.ibm.com 已提交
41 42 43

#define MAX_IS_ALLOCATED_SEARCH 65536

44 45
#define MAX_INFLIGHT_IO 512

L
lirans@il.ibm.com 已提交
46 47 48
//#define DEBUG_BLK_MIGRATION

#ifdef DEBUG_BLK_MIGRATION
M
malc 已提交
49
#define DPRINTF(fmt, ...) \
L
lirans@il.ibm.com 已提交
50 51
    do { printf("blk_migration: " fmt, ## __VA_ARGS__); } while (0)
#else
M
malc 已提交
52
#define DPRINTF(fmt, ...) \
L
lirans@il.ibm.com 已提交
53 54 55
    do { } while (0)
#endif

56
typedef struct BlkMigDevState {
57
    /* Written during setup phase.  Can be read without a lock.  */
58 59
    BlockBackend *blk;
    char *blk_name;
60 61
    int shared_base;
    int64_t total_sectors;
62
    QSIMPLEQ_ENTRY(BlkMigDevState) entry;
63
    Error *blocker;
64 65 66 67 68 69

    /* Only used by migration thread.  Does not need a lock.  */
    int bulk_completed;
    int64_t cur_sector;
    int64_t cur_dirty;

70 71 72
    /* Data in the aio_bitmap is protected by block migration lock.
     * Allocation and free happen during setup and cleanup respectively.
     */
73
    unsigned long *aio_bitmap;
74 75

    /* Protected by block migration lock.  */
76
    int64_t completed_sectors;
77 78 79 80

    /* During migration this is protected by iothread lock / AioContext.
     * Allocation and free happen during setup and cleanup respectively.
     */
F
Fam Zheng 已提交
81
    BdrvDirtyBitmap *dirty_bitmap;
82 83
} BlkMigDevState;

L
lirans@il.ibm.com 已提交
84
typedef struct BlkMigBlock {
85
    /* Only used by migration thread.  */
L
lirans@il.ibm.com 已提交
86 87 88
    uint8_t *buf;
    BlkMigDevState *bmds;
    int64_t sector;
89
    int nr_sectors;
L
lirans@il.ibm.com 已提交
90 91
    struct iovec iov;
    QEMUIOVector qiov;
92
    BlockAIOCB *aiocb;
93

P
Paolo Bonzini 已提交
94
    /* Protected by block migration lock.  */
L
lirans@il.ibm.com 已提交
95
    int ret;
96
    QSIMPLEQ_ENTRY(BlkMigBlock) entry;
L
lirans@il.ibm.com 已提交
97 98 99
} BlkMigBlock;

typedef struct BlkMigState {
100
    QSIMPLEQ_HEAD(bmds_list, BlkMigDevState) bmds_list;
101
    int64_t total_sector_sum;
102
    bool zero_blocks;
103

P
Paolo Bonzini 已提交
104
    /* Protected by lock.  */
105
    QSIMPLEQ_HEAD(blk_list, BlkMigBlock) blk_list;
L
lirans@il.ibm.com 已提交
106 107
    int submitted;
    int read_done;
108 109

    /* Only used by migration thread.  Does not need a lock.  */
L
lirans@il.ibm.com 已提交
110
    int transferred;
111
    int prev_progress;
L
Liran Schour 已提交
112
    int bulk_completed;
P
Paolo Bonzini 已提交
113

114
    /* Lock must be taken _inside_ the iothread lock and any AioContexts.  */
P
Paolo Bonzini 已提交
115
    QemuMutex lock;
L
lirans@il.ibm.com 已提交
116 117
} BlkMigState;

118
static BlkMigState block_mig_state;
L
lirans@il.ibm.com 已提交
119

P
Paolo Bonzini 已提交
120 121 122 123 124 125 126 127 128 129
static void blk_mig_lock(void)
{
    qemu_mutex_lock(&block_mig_state.lock);
}

static void blk_mig_unlock(void)
{
    qemu_mutex_unlock(&block_mig_state.lock);
}

130 131 132 133
/* Must run outside of the iothread lock during the bulk phase,
 * or the VM will stall.
 */

134 135 136
static void blk_send(QEMUFile *f, BlkMigBlock * blk)
{
    int len;
137 138 139 140 141 142
    uint64_t flags = BLK_MIG_FLAG_DEVICE_BLOCK;

    if (block_mig_state.zero_blocks &&
        buffer_is_zero(blk->buf, BLOCK_SIZE)) {
        flags |= BLK_MIG_FLAG_ZERO_BLOCK;
    }
143 144 145

    /* sector number and flags */
    qemu_put_be64(f, (blk->sector << BDRV_SECTOR_BITS)
146
                     | flags);
147 148

    /* device name */
149
    len = strlen(blk->bmds->blk_name);
150
    qemu_put_byte(f, len);
151
    qemu_put_buffer(f, (uint8_t *) blk->bmds->blk_name, len);
152

153 154 155 156 157 158 159 160
    /* if a block is zero we need to flush here since the network
     * bandwidth is now a lot higher than the storage device bandwidth.
     * thus if we queue zero blocks we slow down the migration */
    if (flags & BLK_MIG_FLAG_ZERO_BLOCK) {
        qemu_fflush(f);
        return;
    }

161 162 163
    qemu_put_buffer(f, blk->buf, BLOCK_SIZE);
}

164 165 166 167 168 169 170 171 172 173
int blk_mig_active(void)
{
    return !QSIMPLEQ_EMPTY(&block_mig_state.bmds_list);
}

uint64_t blk_mig_bytes_transferred(void)
{
    BlkMigDevState *bmds;
    uint64_t sum = 0;

P
Paolo Bonzini 已提交
174
    blk_mig_lock();
175 176 177
    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
        sum += bmds->completed_sectors;
    }
P
Paolo Bonzini 已提交
178
    blk_mig_unlock();
179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197
    return sum << BDRV_SECTOR_BITS;
}

uint64_t blk_mig_bytes_remaining(void)
{
    return blk_mig_bytes_total() - blk_mig_bytes_transferred();
}

uint64_t blk_mig_bytes_total(void)
{
    BlkMigDevState *bmds;
    uint64_t sum = 0;

    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
        sum += bmds->total_sectors;
    }
    return sum << BDRV_SECTOR_BITS;
}

P
Paolo Bonzini 已提交
198 199 200

/* Called with migration lock held.  */

201 202 203 204
static int bmds_aio_inflight(BlkMigDevState *bmds, int64_t sector)
{
    int64_t chunk = sector / (int64_t)BDRV_SECTORS_PER_DIRTY_CHUNK;

205
    if (sector < blk_nb_sectors(bmds->blk)) {
206 207 208 209 210 211 212
        return !!(bmds->aio_bitmap[chunk / (sizeof(unsigned long) * 8)] &
            (1UL << (chunk % (sizeof(unsigned long) * 8))));
    } else {
        return 0;
    }
}

P
Paolo Bonzini 已提交
213 214
/* Called with migration lock held.  */

215 216 217 218 219 220 221 222 223 224 225 226 227 228
static void bmds_set_aio_inflight(BlkMigDevState *bmds, int64_t sector_num,
                             int nb_sectors, int set)
{
    int64_t start, end;
    unsigned long val, idx, bit;

    start = sector_num / BDRV_SECTORS_PER_DIRTY_CHUNK;
    end = (sector_num + nb_sectors - 1) / BDRV_SECTORS_PER_DIRTY_CHUNK;

    for (; start <= end; start++) {
        idx = start / (sizeof(unsigned long) * 8);
        bit = start % (sizeof(unsigned long) * 8);
        val = bmds->aio_bitmap[idx];
        if (set) {
229
            val |= 1UL << bit;
230
        } else {
231
            val &= ~(1UL << bit);
232 233 234 235 236 237 238
        }
        bmds->aio_bitmap[idx] = val;
    }
}

static void alloc_aio_bitmap(BlkMigDevState *bmds)
{
239
    BlockBackend *bb = bmds->blk;
240 241
    int64_t bitmap_size;

242
    bitmap_size = blk_nb_sectors(bb) + BDRV_SECTORS_PER_DIRTY_CHUNK * 8 - 1;
243 244
    bitmap_size /= BDRV_SECTORS_PER_DIRTY_CHUNK * 8;

245
    bmds->aio_bitmap = g_malloc0(bitmap_size);
246 247
}

P
Paolo Bonzini 已提交
248 249
/* Never hold migration lock when yielding to the main loop!  */

L
lirans@il.ibm.com 已提交
250 251 252
static void blk_mig_read_cb(void *opaque, int ret)
{
    BlkMigBlock *blk = opaque;
253

P
Paolo Bonzini 已提交
254
    blk_mig_lock();
L
lirans@il.ibm.com 已提交
255
    blk->ret = ret;
256

257
    QSIMPLEQ_INSERT_TAIL(&block_mig_state.blk_list, blk, entry);
258
    bmds_set_aio_inflight(blk->bmds, blk->sector, blk->nr_sectors, 0);
259

260 261 262
    block_mig_state.submitted--;
    block_mig_state.read_done++;
    assert(block_mig_state.submitted >= 0);
P
Paolo Bonzini 已提交
263
    blk_mig_unlock();
L
lirans@il.ibm.com 已提交
264 265
}

266 267
/* Called with no lock taken.  */

268
static int mig_save_device_bulk(QEMUFile *f, BlkMigDevState *bmds)
269
{
270 271
    int64_t total_sectors = bmds->total_sectors;
    int64_t cur_sector = bmds->cur_sector;
272
    BlockBackend *bb = bmds->blk;
L
lirans@il.ibm.com 已提交
273
    BlkMigBlock *blk;
274
    int nr_sectors;
275

276
    if (bmds->shared_base) {
277
        qemu_mutex_lock_iothread();
278
        aio_context_acquire(blk_get_aio_context(bb));
279 280
        /* Skip unallocated sectors; intentionally treats failure as
         * an allocated sector */
281
        while (cur_sector < total_sectors &&
282 283
               !bdrv_is_allocated(blk_bs(bb), cur_sector,
                                  MAX_IS_ALLOCATED_SEARCH, &nr_sectors)) {
L
lirans@il.ibm.com 已提交
284 285
            cur_sector += nr_sectors;
        }
286
        aio_context_release(blk_get_aio_context(bb));
287
        qemu_mutex_unlock_iothread();
L
lirans@il.ibm.com 已提交
288
    }
289 290

    if (cur_sector >= total_sectors) {
291
        bmds->cur_sector = bmds->completed_sectors = total_sectors;
L
lirans@il.ibm.com 已提交
292 293
        return 1;
    }
294

295
    bmds->completed_sectors = cur_sector;
296

297 298
    cur_sector &= ~((int64_t)BDRV_SECTORS_PER_DIRTY_CHUNK - 1);

J
Jan Kiszka 已提交
299 300
    /* we are going to transfer a full block even if it is not allocated */
    nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK;
L
lirans@il.ibm.com 已提交
301

J
Jan Kiszka 已提交
302
    if (total_sectors - cur_sector < BDRV_SECTORS_PER_DIRTY_CHUNK) {
303
        nr_sectors = total_sectors - cur_sector;
L
lirans@il.ibm.com 已提交
304
    }
305

306
    blk = g_new(BlkMigBlock, 1);
307
    blk->buf = g_malloc(BLOCK_SIZE);
308 309
    blk->bmds = bmds;
    blk->sector = cur_sector;
310
    blk->nr_sectors = nr_sectors;
311

L
Liran Schour 已提交
312 313 314
    blk->iov.iov_base = blk->buf;
    blk->iov.iov_len = nr_sectors * BDRV_SECTOR_SIZE;
    qemu_iovec_init_external(&blk->qiov, &blk->iov, 1);
315

P
Paolo Bonzini 已提交
316
    blk_mig_lock();
317
    block_mig_state.submitted++;
P
Paolo Bonzini 已提交
318
    blk_mig_unlock();
319

320 321 322 323 324 325 326 327
    /* We do not know if bs is under the main thread (and thus does
     * not acquire the AioContext when doing AIO) or rather under
     * dataplane.  Thus acquire both the iothread mutex and the
     * AioContext.
     *
     * This is ugly and will disappear when we make bdrv_* thread-safe,
     * without the need to acquire the AioContext.
     */
328
    qemu_mutex_lock_iothread();
329 330 331
    aio_context_acquire(blk_get_aio_context(bmds->blk));
    blk->aiocb = blk_aio_preadv(bb, cur_sector * BDRV_SECTOR_SIZE, &blk->qiov,
                                0, blk_mig_read_cb, blk);
332

333
    bdrv_reset_dirty_bitmap(bmds->dirty_bitmap, cur_sector, nr_sectors);
334
    aio_context_release(blk_get_aio_context(bmds->blk));
335
    qemu_mutex_unlock_iothread();
336

337
    bmds->cur_sector = cur_sector + nr_sectors;
338
    return (bmds->cur_sector >= total_sectors);
L
lirans@il.ibm.com 已提交
339 340
}

341 342
/* Called with iothread lock taken.  */

343
static int set_dirty_tracking(void)
L
lirans@il.ibm.com 已提交
344 345
{
    BlkMigDevState *bmds;
346 347 348
    int ret;

    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
349 350 351 352
        aio_context_acquire(blk_get_aio_context(bmds->blk));
        bmds->dirty_bitmap = bdrv_create_dirty_bitmap(blk_bs(bmds->blk),
                                                      BLOCK_SIZE, NULL, NULL);
        aio_context_release(blk_get_aio_context(bmds->blk));
353 354 355 356 357 358
        if (!bmds->dirty_bitmap) {
            ret = -errno;
            goto fail;
        }
    }
    return 0;
359

360
fail:
361
    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
362
        if (bmds->dirty_bitmap) {
363 364 365
            aio_context_acquire(blk_get_aio_context(bmds->blk));
            bdrv_release_dirty_bitmap(blk_bs(bmds->blk), bmds->dirty_bitmap);
            aio_context_release(blk_get_aio_context(bmds->blk));
366
        }
F
Fam Zheng 已提交
367
    }
368
    return ret;
F
Fam Zheng 已提交
369 370
}

371 372
/* Called with iothread lock taken.  */

F
Fam Zheng 已提交
373 374 375 376 377
static void unset_dirty_tracking(void)
{
    BlkMigDevState *bmds;

    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
378 379 380
        aio_context_acquire(blk_get_aio_context(bmds->blk));
        bdrv_release_dirty_bitmap(blk_bs(bmds->blk), bmds->dirty_bitmap);
        aio_context_release(blk_get_aio_context(bmds->blk));
L
lirans@il.ibm.com 已提交
381 382 383
    }
}

384
static int init_blk_migration(QEMUFile *f)
L
lirans@il.ibm.com 已提交
385
{
386
    BlockDriverState *bs;
387
    BlkMigDevState *bmds;
388
    int64_t sectors;
K
Kevin Wolf 已提交
389
    BdrvNextIterator it;
390 391 392 393 394
    int i, num_bs = 0;
    struct {
        BlkMigDevState *bmds;
        BlockDriverState *bs;
    } *bmds_bs;
395 396
    Error *local_err = NULL;
    int ret;
397

398 399 400 401 402 403 404 405
    block_mig_state.submitted = 0;
    block_mig_state.read_done = 0;
    block_mig_state.transferred = 0;
    block_mig_state.total_sector_sum = 0;
    block_mig_state.prev_progress = -1;
    block_mig_state.bulk_completed = 0;
    block_mig_state.zero_blocks = migrate_zero_blocks();

K
Kevin Wolf 已提交
406
    for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
407 408 409 410 411
        num_bs++;
    }
    bmds_bs = g_malloc0(num_bs * sizeof(*bmds_bs));

    for (i = 0, bs = bdrv_first(&it); bs; bs = bdrv_next(&it), i++) {
412 413 414 415
        if (bdrv_is_read_only(bs)) {
            continue;
        }

416
        sectors = bdrv_nb_sectors(bs);
417
        if (sectors <= 0) {
418
            ret = sectors;
419
            goto out;
420 421
        }

422
        bmds = g_new0(BlkMigDevState, 1);
423
        bmds->blk = blk_new(BLK_PERM_CONSISTENT_READ, BLK_PERM_ALL);
424
        bmds->blk_name = g_strdup(bdrv_get_device_name(bs));
425 426 427
        bmds->bulk_completed = 0;
        bmds->total_sectors = sectors;
        bmds->completed_sectors = 0;
428
        bmds->shared_base = migrate_use_block_incremental();
429 430 431 432

        assert(i < num_bs);
        bmds_bs[i].bmds = bmds;
        bmds_bs[i].bs = bs;
433 434 435 436

        block_mig_state.total_sector_sum += sectors;

        if (bmds->shared_base) {
437
            DPRINTF("Start migration for %s with shared base image\n",
438
                    bdrv_get_device_name(bs));
439
        } else {
440
            DPRINTF("Start full migration for %s\n", bdrv_get_device_name(bs));
441 442 443 444
        }

        QSIMPLEQ_INSERT_TAIL(&block_mig_state.bmds_list, bmds, entry);
    }
445 446 447 448 449 450 451 452

    /* Can only insert new BDSes now because doing so while iterating block
     * devices may end up in a deadlock (iterating the new BDSes, too). */
    for (i = 0; i < num_bs; i++) {
        BlkMigDevState *bmds = bmds_bs[i].bmds;
        BlockDriverState *bs = bmds_bs[i].bs;

        if (bmds) {
453 454 455 456 457
            ret = blk_insert_bs(bmds->blk, bs, &local_err);
            if (ret < 0) {
                error_report_err(local_err);
                goto out;
            }
458 459 460 461 462 463 464

            alloc_aio_bitmap(bmds);
            error_setg(&bmds->blocker, "block device is in use by migration");
            bdrv_op_block_all(bs, bmds->blocker);
        }
    }

465
    ret = 0;
466 467
out:
    g_free(bmds_bs);
468
    return ret;
469 470
}

471 472
/* Called with no lock taken.  */

473
static int blk_mig_save_bulked_block(QEMUFile *f)
L
lirans@il.ibm.com 已提交
474
{
475
    int64_t completed_sector_sum = 0;
L
lirans@il.ibm.com 已提交
476
    BlkMigDevState *bmds;
477
    int progress;
478
    int ret = 0;
L
lirans@il.ibm.com 已提交
479

480
    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
481
        if (bmds->bulk_completed == 0) {
482
            if (mig_save_device_bulk(f, bmds) == 1) {
483 484
                /* completed bulk section for this device */
                bmds->bulk_completed = 1;
L
lirans@il.ibm.com 已提交
485
            }
486 487 488 489 490
            completed_sector_sum += bmds->completed_sectors;
            ret = 1;
            break;
        } else {
            completed_sector_sum += bmds->completed_sectors;
L
lirans@il.ibm.com 已提交
491 492
        }
    }
493

494 495 496 497 498 499
    if (block_mig_state.total_sector_sum != 0) {
        progress = completed_sector_sum * 100 /
                   block_mig_state.total_sector_sum;
    } else {
        progress = 100;
    }
500 501 502 503
    if (progress != block_mig_state.prev_progress) {
        block_mig_state.prev_progress = progress;
        qemu_put_be64(f, (progress << BDRV_SECTOR_BITS)
                         | BLK_MIG_FLAG_PROGRESS);
504
        DPRINTF("Completed %d %%\r", progress);
505 506 507
    }

    return ret;
L
lirans@il.ibm.com 已提交
508 509
}

510
static void blk_mig_reset_dirty_cursor(void)
L
lirans@il.ibm.com 已提交
511 512
{
    BlkMigDevState *bmds;
513 514 515 516 517 518

    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
        bmds->cur_dirty = 0;
    }
}

519
/* Called with iothread lock and AioContext taken.  */
520

521 522
static int mig_save_device_dirty(QEMUFile *f, BlkMigDevState *bmds,
                                 int is_async)
523 524
{
    BlkMigBlock *blk;
525
    BlockDriverState *bs = blk_bs(bmds->blk);
526
    int64_t total_sectors = bmds->total_sectors;
L
lirans@il.ibm.com 已提交
527
    int64_t sector;
528
    int nr_sectors;
529
    int ret = -EIO;
530

531
    for (sector = bmds->cur_dirty; sector < bmds->total_sectors;) {
P
Paolo Bonzini 已提交
532
        blk_mig_lock();
533
        if (bmds_aio_inflight(bmds, sector)) {
P
Paolo Bonzini 已提交
534
            blk_mig_unlock();
535
            blk_drain(bmds->blk);
P
Paolo Bonzini 已提交
536 537
        } else {
            blk_mig_unlock();
538
        }
539
        if (bdrv_get_dirty(bs, bmds->dirty_bitmap, sector)) {
540

541 542 543 544 545
            if (total_sectors - sector < BDRV_SECTORS_PER_DIRTY_CHUNK) {
                nr_sectors = total_sectors - sector;
            } else {
                nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK;
            }
546
            blk = g_new(BlkMigBlock, 1);
547
            blk->buf = g_malloc(BLOCK_SIZE);
548 549
            blk->bmds = bmds;
            blk->sector = sector;
550
            blk->nr_sectors = nr_sectors;
551

552
            if (is_async) {
553 554 555 556
                blk->iov.iov_base = blk->buf;
                blk->iov.iov_len = nr_sectors * BDRV_SECTOR_SIZE;
                qemu_iovec_init_external(&blk->qiov, &blk->iov, 1);

557 558 559 560
                blk->aiocb = blk_aio_preadv(bmds->blk,
                                            sector * BDRV_SECTOR_SIZE,
                                            &blk->qiov, 0, blk_mig_read_cb,
                                            blk);
P
Paolo Bonzini 已提交
561 562

                blk_mig_lock();
563
                block_mig_state.submitted++;
564
                bmds_set_aio_inflight(bmds, sector, nr_sectors, 1);
P
Paolo Bonzini 已提交
565
                blk_mig_unlock();
566
            } else {
567 568
                ret = blk_pread(bmds->blk, sector * BDRV_SECTOR_SIZE, blk->buf,
                                nr_sectors * BDRV_SECTOR_SIZE);
569
                if (ret < 0) {
570
                    goto error;
L
lirans@il.ibm.com 已提交
571
                }
572
                blk_send(f, blk);
573

574 575
                g_free(blk->buf);
                g_free(blk);
576
            }
577

578
            bdrv_reset_dirty_bitmap(bmds->dirty_bitmap, sector, nr_sectors);
579 580 581
            sector += nr_sectors;
            bmds->cur_dirty = sector;

582
            break;
L
lirans@il.ibm.com 已提交
583
        }
584 585
        sector += BDRV_SECTORS_PER_DIRTY_CHUNK;
        bmds->cur_dirty = sector;
L
lirans@il.ibm.com 已提交
586
    }
587

588 589
    return (bmds->cur_dirty >= bmds->total_sectors);

590
error:
591
    DPRINTF("Error reading sector %" PRId64 "\n", sector);
592 593
    g_free(blk->buf);
    g_free(blk);
594
    return ret;
595 596
}

597 598 599
/* Called with iothread lock taken.
 *
 * return value:
600 601 602
 * 0: too much data for max_downtime
 * 1: few enough data for max_downtime
*/
603
static int blk_mig_save_dirty_block(QEMUFile *f, int is_async)
604 605
{
    BlkMigDevState *bmds;
606
    int ret = 1;
607 608

    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
609
        aio_context_acquire(blk_get_aio_context(bmds->blk));
610
        ret = mig_save_device_dirty(f, bmds, is_async);
611
        aio_context_release(blk_get_aio_context(bmds->blk));
612
        if (ret <= 0) {
613 614 615 616 617
            break;
        }
    }

    return ret;
L
lirans@il.ibm.com 已提交
618 619
}

620 621
/* Called with no locks taken.  */

622
static int flush_blks(QEMUFile *f)
L
lirans@il.ibm.com 已提交
623
{
624
    BlkMigBlock *blk;
625
    int ret = 0;
626

M
malc 已提交
627
    DPRINTF("%s Enter submitted %d read_done %d transferred %d\n",
628 629
            __FUNCTION__, block_mig_state.submitted, block_mig_state.read_done,
            block_mig_state.transferred);
630

P
Paolo Bonzini 已提交
631
    blk_mig_lock();
632 633 634 635
    while ((blk = QSIMPLEQ_FIRST(&block_mig_state.blk_list)) != NULL) {
        if (qemu_file_rate_limit(f)) {
            break;
        }
636
        if (blk->ret < 0) {
637
            ret = blk->ret;
638 639
            break;
        }
640

641
        QSIMPLEQ_REMOVE_HEAD(&block_mig_state.blk_list, entry);
P
Paolo Bonzini 已提交
642
        blk_mig_unlock();
643
        blk_send(f, blk);
P
Paolo Bonzini 已提交
644
        blk_mig_lock();
645

646 647
        g_free(blk->buf);
        g_free(blk);
648

649 650 651
        block_mig_state.read_done--;
        block_mig_state.transferred++;
        assert(block_mig_state.read_done >= 0);
L
lirans@il.ibm.com 已提交
652
    }
P
Paolo Bonzini 已提交
653
    blk_mig_unlock();
L
lirans@il.ibm.com 已提交
654

M
malc 已提交
655
    DPRINTF("%s Exit submitted %d read_done %d transferred %d\n", __FUNCTION__,
656 657
            block_mig_state.submitted, block_mig_state.read_done,
            block_mig_state.transferred);
658
    return ret;
L
lirans@il.ibm.com 已提交
659 660
}

661 662
/* Called with iothread lock taken.  */

663 664 665 666 667 668
static int64_t get_remaining_dirty(void)
{
    BlkMigDevState *bmds;
    int64_t dirty = 0;

    QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
669
        aio_context_acquire(blk_get_aio_context(bmds->blk));
670
        dirty += bdrv_get_dirty_count(bmds->dirty_bitmap);
671
        aio_context_release(blk_get_aio_context(bmds->blk));
672 673
    }

674
    return dirty << BDRV_SECTOR_BITS;
675 676
}

677 678
/* Called with iothread lock taken.  */

L
Liang Li 已提交
679
static void block_migration_cleanup(void *opaque)
680
{
681 682
    BlkMigDevState *bmds;
    BlkMigBlock *blk;
683
    AioContext *ctx;
684

685 686
    bdrv_drain_all();

F
Fam Zheng 已提交
687
    unset_dirty_tracking();
688

689 690
    while ((bmds = QSIMPLEQ_FIRST(&block_mig_state.bmds_list)) != NULL) {
        QSIMPLEQ_REMOVE_HEAD(&block_mig_state.bmds_list, entry);
691
        bdrv_op_unblock_all(blk_bs(bmds->blk), bmds->blocker);
692
        error_free(bmds->blocker);
693

694 695
        /* Save ctx, because bmds->blk can disappear during blk_unref.  */
        ctx = blk_get_aio_context(bmds->blk);
696
        aio_context_acquire(ctx);
697
        blk_unref(bmds->blk);
698 699
        aio_context_release(ctx);

700
        g_free(bmds->blk_name);
701 702
        g_free(bmds->aio_bitmap);
        g_free(bmds);
703 704
    }

705
    blk_mig_lock();
706 707
    while ((blk = QSIMPLEQ_FIRST(&block_mig_state.blk_list)) != NULL) {
        QSIMPLEQ_REMOVE_HEAD(&block_mig_state.blk_list, entry);
708 709
        g_free(blk->buf);
        g_free(blk);
710
    }
P
Paolo Bonzini 已提交
711
    blk_mig_unlock();
712 713
}

714
static int block_save_setup(QEMUFile *f, void *opaque)
L
lirans@il.ibm.com 已提交
715
{
716 717
    int ret;

718 719
    DPRINTF("Enter save live setup submitted %d transferred %d\n",
            block_mig_state.submitted, block_mig_state.transferred);
720

721
    qemu_mutex_lock_iothread();
722 723 724 725 726
    ret = init_blk_migration(f);
    if (ret < 0) {
        qemu_mutex_unlock_iothread();
        return ret;
    }
727 728

    /* start track dirty blocks */
729 730
    ret = set_dirty_tracking();

731 732
    qemu_mutex_unlock_iothread();

733 734 735 736
    if (ret) {
        return ret;
    }

737
    ret = flush_blks(f);
738 739 740
    blk_mig_reset_dirty_cursor();
    qemu_put_be64(f, BLK_MIG_FLAG_EOS);

741
    return ret;
742 743
}

744
static int block_save_iterate(QEMUFile *f, void *opaque)
745 746
{
    int ret;
747
    int64_t last_ftell = qemu_ftell(f);
G
Gary R Hook 已提交
748
    int64_t delta_ftell;
749

750 751
    DPRINTF("Enter save live iterate submitted %d transferred %d\n",
            block_mig_state.submitted, block_mig_state.transferred);
752

753
    ret = flush_blks(f);
754 755
    if (ret) {
        return ret;
756 757
    }

758 759
    blk_mig_reset_dirty_cursor();

760
    /* control the rate of transfer */
P
Paolo Bonzini 已提交
761
    blk_mig_lock();
762 763
    while ((block_mig_state.submitted +
            block_mig_state.read_done) * BLOCK_SIZE <
764 765 766 767
           qemu_file_get_rate_limit(f) &&
           (block_mig_state.submitted +
            block_mig_state.read_done) <
           MAX_INFLIGHT_IO) {
P
Paolo Bonzini 已提交
768
        blk_mig_unlock();
769 770 771 772 773 774
        if (block_mig_state.bulk_completed == 0) {
            /* first finish the bulk phase */
            if (blk_mig_save_bulked_block(f) == 0) {
                /* finished saving bulk on all devices */
                block_mig_state.bulk_completed = 1;
            }
775
            ret = 0;
776
        } else {
777 778 779 780
            /* Always called with iothread lock taken for
             * simplicity, block_save_complete also calls it.
             */
            qemu_mutex_lock_iothread();
781
            ret = blk_mig_save_dirty_block(f, 1);
782
            qemu_mutex_unlock_iothread();
783 784 785 786
        }
        if (ret < 0) {
            return ret;
        }
P
Paolo Bonzini 已提交
787
        blk_mig_lock();
788 789 790
        if (ret != 0) {
            /* no more dirty blocks */
            break;
791
        }
792
    }
P
Paolo Bonzini 已提交
793
    blk_mig_unlock();
794

795
    ret = flush_blks(f);
796 797
    if (ret) {
        return ret;
798 799
    }

800
    qemu_put_be64(f, BLK_MIG_FLAG_EOS);
G
Gary R Hook 已提交
801 802 803 804 805 806 807 808
    delta_ftell = qemu_ftell(f) - last_ftell;
    if (delta_ftell > 0) {
        return 1;
    } else if (delta_ftell < 0) {
        return -1;
    } else {
        return 0;
    }
809 810
}

811 812
/* Called with iothread lock taken.  */

813 814 815 816 817 818 819
static int block_save_complete(QEMUFile *f, void *opaque)
{
    int ret;

    DPRINTF("Enter save live complete submitted %d transferred %d\n",
            block_mig_state.submitted, block_mig_state.transferred);

820
    ret = flush_blks(f);
821 822 823
    if (ret) {
        return ret;
    }
824

825
    blk_mig_reset_dirty_cursor();
826

827 828
    /* we know for sure that save bulk is completed and
       all async read completed */
P
Paolo Bonzini 已提交
829
    blk_mig_lock();
830
    assert(block_mig_state.submitted == 0);
P
Paolo Bonzini 已提交
831
    blk_mig_unlock();
832

833 834
    do {
        ret = blk_mig_save_dirty_block(f, 0);
835 836 837
        if (ret < 0) {
            return ret;
        }
838
    } while (ret == 0);
839

840 841
    /* report completion */
    qemu_put_be64(f, (100 << BDRV_SECTOR_BITS) | BLK_MIG_FLAG_PROGRESS);
842

843 844
    DPRINTF("Block migration completed\n");

845 846
    qemu_put_be64(f, BLK_MIG_FLAG_EOS);

847
    return 0;
L
lirans@il.ibm.com 已提交
848 849
}

850 851 852
static void block_save_pending(QEMUFile *f, void *opaque, uint64_t max_size,
                               uint64_t *non_postcopiable_pending,
                               uint64_t *postcopiable_pending)
853
{
854
    /* Estimate pending number of bytes to send */
855 856
    uint64_t pending;

857
    qemu_mutex_lock_iothread();
858 859 860
    pending = get_remaining_dirty();
    qemu_mutex_unlock_iothread();

P
Paolo Bonzini 已提交
861
    blk_mig_lock();
862 863 864
    pending += block_mig_state.submitted * BLOCK_SIZE +
               block_mig_state.read_done * BLOCK_SIZE;
    blk_mig_unlock();
865 866

    /* Report at least one block pending during bulk phase */
867 868
    if (pending <= max_size && !block_mig_state.bulk_completed) {
        pending = max_size + BLOCK_SIZE;
869
    }
870

871
    DPRINTF("Enter save live pending  %" PRIu64 "\n", pending);
872 873
    /* We don't do postcopy */
    *non_postcopiable_pending += pending;
874 875
}

L
lirans@il.ibm.com 已提交
876 877
static int block_load(QEMUFile *f, void *opaque, int version_id)
{
878
    static int banner_printed;
L
lirans@il.ibm.com 已提交
879 880 881
    int len, flags;
    char device_name[256];
    int64_t addr;
882
    BlockBackend *blk, *blk_prev = NULL;;
883
    Error *local_err = NULL;
L
lirans@il.ibm.com 已提交
884
    uint8_t *buf;
885 886
    int64_t total_sectors = 0;
    int nr_sectors;
887
    int ret;
888 889
    BlockDriverInfo bdi;
    int cluster_size = BLOCK_SIZE;
890

L
lirans@il.ibm.com 已提交
891 892
    do {
        addr = qemu_get_be64(f);
893

J
Jan Kiszka 已提交
894 895
        flags = addr & ~BDRV_SECTOR_MASK;
        addr >>= BDRV_SECTOR_BITS;
896 897

        if (flags & BLK_MIG_FLAG_DEVICE_BLOCK) {
L
lirans@il.ibm.com 已提交
898 899 900 901
            /* get device name */
            len = qemu_get_byte(f);
            qemu_get_buffer(f, (uint8_t *)device_name, len);
            device_name[len] = '\0';
902

903 904
            blk = blk_by_name(device_name);
            if (!blk) {
905 906 907 908
                fprintf(stderr, "Error unknown block device %s\n",
                        device_name);
                return -EINVAL;
            }
909

910 911 912
            if (blk != blk_prev) {
                blk_prev = blk;
                total_sectors = blk_nb_sectors(blk);
913
                if (total_sectors <= 0) {
914
                    error_report("Error getting length of block device %s",
915 916 917
                                 device_name);
                    return -EINVAL;
                }
918

919
                blk_invalidate_cache(blk, &local_err);
920 921 922 923
                if (local_err) {
                    error_report_err(local_err);
                    return -EINVAL;
                }
924 925 926 927 928 929 930 931 932

                ret = bdrv_get_info(blk_bs(blk), &bdi);
                if (ret == 0 && bdi.cluster_size > 0 &&
                    bdi.cluster_size <= BLOCK_SIZE &&
                    BLOCK_SIZE % bdi.cluster_size == 0) {
                    cluster_size = bdi.cluster_size;
                } else {
                    cluster_size = BLOCK_SIZE;
                }
933 934 935 936 937 938 939 940
            }

            if (total_sectors - addr < BDRV_SECTORS_PER_DIRTY_CHUNK) {
                nr_sectors = total_sectors - addr;
            } else {
                nr_sectors = BDRV_SECTORS_PER_DIRTY_CHUNK;
            }

941
            if (flags & BLK_MIG_FLAG_ZERO_BLOCK) {
942 943 944
                ret = blk_pwrite_zeroes(blk, addr * BDRV_SECTOR_SIZE,
                                        nr_sectors * BDRV_SECTOR_SIZE,
                                        BDRV_REQ_MAY_UNMAP);
945
            } else {
946 947 948 949
                int i;
                int64_t cur_addr;
                uint8_t *cur_buf;

950 951
                buf = g_malloc(BLOCK_SIZE);
                qemu_get_buffer(f, buf, BLOCK_SIZE);
952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969
                for (i = 0; i < BLOCK_SIZE / cluster_size; i++) {
                    cur_addr = addr * BDRV_SECTOR_SIZE + i * cluster_size;
                    cur_buf = buf + i * cluster_size;

                    if ((!block_mig_state.zero_blocks ||
                        cluster_size < BLOCK_SIZE) &&
                        buffer_is_zero(cur_buf, cluster_size)) {
                        ret = blk_pwrite_zeroes(blk, cur_addr,
                                                cluster_size,
                                                BDRV_REQ_MAY_UNMAP);
                    } else {
                        ret = blk_pwrite(blk, cur_addr, cur_buf,
                                         cluster_size, 0);
                    }
                    if (ret < 0) {
                        break;
                    }
                }
970 971
                g_free(buf);
            }
972

973 974 975
            if (ret < 0) {
                return ret;
            }
976 977 978 979 980 981 982 983
        } else if (flags & BLK_MIG_FLAG_PROGRESS) {
            if (!banner_printed) {
                printf("Receiving block device images\n");
                banner_printed = 1;
            }
            printf("Completed %d %%%c", (int)addr,
                   (addr == 100) ? '\n' : '\r');
            fflush(stdout);
984
        } else if (!(flags & BLK_MIG_FLAG_EOS)) {
985
            fprintf(stderr, "Unknown block migration flags: %#x\n", flags);
986 987
            return -EINVAL;
        }
988 989 990
        ret = qemu_file_get_error(f);
        if (ret != 0) {
            return ret;
L
lirans@il.ibm.com 已提交
991
        }
992 993
    } while (!(flags & BLK_MIG_FLAG_EOS));

L
lirans@il.ibm.com 已提交
994 995 996
    return 0;
}

997 998
static bool block_is_active(void *opaque)
{
999
    return migrate_use_block();
1000 1001
}

1002
static SaveVMHandlers savevm_block_handlers = {
1003
    .save_live_setup = block_save_setup,
1004
    .save_live_iterate = block_save_iterate,
1005
    .save_live_complete_precopy = block_save_complete,
1006
    .save_live_pending = block_save_pending,
1007
    .load_state = block_load,
L
Liang Li 已提交
1008
    .cleanup = block_migration_cleanup,
1009
    .is_active = block_is_active,
1010 1011
};

L
lirans@il.ibm.com 已提交
1012
void blk_mig_init(void)
1013
{
1014 1015
    QSIMPLEQ_INIT(&block_mig_state.bmds_list);
    QSIMPLEQ_INIT(&block_mig_state.blk_list);
P
Paolo Bonzini 已提交
1016
    qemu_mutex_init(&block_mig_state.lock);
1017

1018 1019
    register_savevm_live(NULL, "block", 0, 1, &savevm_block_handlers,
                         &block_mig_state);
L
lirans@il.ibm.com 已提交
1020
}