Merge remote-tracking branch 'mreitz/tags/pull-block-2017-10-26' into queue-block

Block patches

# gpg: Signature made Thu Oct 26 15:01:20 2017 CEST
# gpg:                using RSA key F407DB0061D5CF40
# gpg: Good signature from "Max Reitz <mreitz@redhat.com>"
# Primary key fingerprint: 91BE B60A 30DB 3E88 57D1  1829 F407 DB00 61D5 CF40

* mreitz/tags/pull-block-2017-10-26:
  iotests: Add cluster_size=64k to 125
  qcow2: Always execute preallocate() in a coroutine
  qcow2: Fix unaligned preallocated truncation
  qcow2: Emit errp when truncating the image tail
  iotests: Filter actual image size in 184 and 191
  iotests: Pull _filter_actual_image_size from 67/87
  iotests: Add test for dataplane mirroring
  qcow2: Use BDRV_SECTOR_BITS instead of its literal value
Signed-off-by: NKevin Wolf <kwolf@redhat.com>

block/qcow2.c

@@ -1139,7 +1139,7 @@ static int qcow2_do_open(BlockDriverState *bs, QDict *options, int flags,
 
     s->cluster_bits = header.cluster_bits;
     s->cluster_size = 1 << s->cluster_bits;
-    s->cluster_sectors = 1 << (s->cluster_bits - 9);
+    s->cluster_sectors = 1 << (s->cluster_bits - BDRV_SECTOR_BITS);
 
     /* Initialise version 3 header fields */
     if (header.version == 2) {
@@ -1636,7 +1636,7 @@ static int64_t coroutine_fn qcow2_co_get_block_status(BlockDriverState *bs,
 
     bytes = MIN(INT_MAX, nb_sectors * BDRV_SECTOR_SIZE);
     qemu_co_mutex_lock(&s->lock);
-    ret = qcow2_get_cluster_offset(bs, sector_num << 9, &bytes,
+    ret = qcow2_get_cluster_offset(bs, sector_num << BDRV_SECTOR_BITS, &bytes,
                                    &cluster_offset);
     qemu_co_mutex_unlock(&s->lock);
     if (ret < 0) {
@@ -2460,6 +2460,14 @@ static int qcow2_set_up_encryption(BlockDriverState *bs, const char *encryptfmt,
 }
 
 
+typedef struct PreallocCo {
+    BlockDriverState *bs;
+    uint64_t offset;
+    uint64_t new_length;
+
+    int ret;
+} PreallocCo;
+
 /**
  * Preallocates metadata structures for data clusters between @offset (in the
  * guest disk) and @new_length (which is thus generally the new guest disk
@@ -2467,9 +2475,12 @@ static int qcow2_set_up_encryption(BlockDriverState *bs, const char *encryptfmt,
  *
  * Returns: 0 on success, -errno on failure.
  */
-static int preallocate(BlockDriverState *bs,
-                       uint64_t offset, uint64_t new_length)
+static void coroutine_fn preallocate_co(void *opaque)
 {
+    PreallocCo *params = opaque;
+    BlockDriverState *bs = params->bs;
+    uint64_t offset = params->offset;
+    uint64_t new_length = params->new_length;
     BDRVQcow2State *s = bs->opaque;
     uint64_t bytes;
     uint64_t host_offset = 0;
@@ -2477,9 +2488,7 @@ static int preallocate(BlockDriverState *bs,
     int ret;
     QCowL2Meta *meta;
 
-    if (qemu_in_coroutine()) {
-        qemu_co_mutex_lock(&s->lock);
-    }
+    qemu_co_mutex_lock(&s->lock);
 
     assert(offset <= new_length);
     bytes = new_length - offset;
@@ -2533,10 +2542,28 @@ static int preallocate(BlockDriverState *bs,
     ret = 0;
 
 done:
+    qemu_co_mutex_unlock(&s->lock);
+    params->ret = ret;
+}
+
+static int preallocate(BlockDriverState *bs,
+                       uint64_t offset, uint64_t new_length)
+{
+    PreallocCo params = {
+        .bs         = bs,
+        .offset     = offset,
+        .new_length = new_length,
+        .ret        = -EINPROGRESS,
+    };
+
     if (qemu_in_coroutine()) {
-        qemu_co_mutex_unlock(&s->lock);
+        preallocate_co(&params);
+    } else {
+        Coroutine *co = qemu_coroutine_create(preallocate_co, &params);
+        bdrv_coroutine_enter(bs, co);
+        BDRV_POLL_WHILE(bs, params.ret == -EINPROGRESS);
     }
-    return ret;
+    return params.ret;
 }
 
 /* qcow2_refcount_metadata_size:
@@ -3145,12 +3172,13 @@ static int qcow2_truncate(BlockDriverState *bs, int64_t offset,
             return last_cluster;
         }
         if ((last_cluster + 1) * s->cluster_size < old_file_size) {
-            ret = bdrv_truncate(bs->file, (last_cluster + 1) * s->cluster_size,
-                                PREALLOC_MODE_OFF, NULL);
-            if (ret < 0) {
-                warn_report("Failed to truncate the tail of the image: %s",
-                            strerror(-ret));
-                ret = 0;
+            Error *local_err = NULL;
+
+            bdrv_truncate(bs->file, (last_cluster + 1) * s->cluster_size,
+                          PREALLOC_MODE_OFF, &local_err);
+            if (local_err) {
+                warn_reportf_err(local_err,
+                                 "Failed to truncate the tail of the image: ");
             }
         }
     } else {
@@ -3187,6 +3215,7 @@ static int qcow2_truncate(BlockDriverState *bs, int64_t offset,
                              "Failed to inquire current file length");
             return old_file_size;
         }
+        old_file_size = ROUND_UP(old_file_size, s->cluster_size);
 
         nb_new_data_clusters = DIV_ROUND_UP(offset - old_length,
                                             s->cluster_size);

tests/qemu-iotests/067

@@ -56,7 +56,7 @@ _filter_qmp_events()
 function run_qemu()
 {
     do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_qmp | _filter_qemu \
-                          | sed -e 's/\("actual-size":\s*\)[0-9]\+/\1SIZE/g' \
+                          | _filter_actual_image_size \
                           | _filter_generated_node_ids | _filter_qmp_events
 }
 

tests/qemu-iotests/087

@@ -46,7 +46,7 @@ function run_qemu()
 {
     do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_qmp \
                           | _filter_qemu | _filter_imgfmt \
-                          | sed -e 's/\("actual-size":\s*\)[0-9]\+/\1SIZE/g'
+                          | _filter_actual_image_size
 }
 
 size=128M

tests/qemu-iotests/125

@@ -69,13 +69,15 @@ fi
 # in B
 CREATION_SIZE=$((2 * 1024 * 1024 - 48 * 1024))
 
+# 512 is the actual test -- but it's good to test 64k as well, just to be sure.
+for cluster_size in 512 64k; do
 # in kB
 for GROWTH_SIZE in 16 48 80; do
     for create_mode in off metadata falloc full; do
         for growth_mode in off metadata falloc full; do
-            echo "--- growth_size=$GROWTH_SIZE create_mode=$create_mode growth_mode=$growth_mode ---"
+            echo "--- cluster_size=$cluster_size growth_size=$GROWTH_SIZE create_mode=$create_mode growth_mode=$growth_mode ---"
 
-            IMGOPTS="preallocation=$create_mode,cluster_size=512" _make_test_img ${CREATION_SIZE}
+            IMGOPTS="preallocation=$create_mode,cluster_size=$cluster_size" _make_test_img ${CREATION_SIZE}
             $QEMU_IMG resize -f "$IMGFMT" --preallocation=$growth_mode "$TEST_IMG" +${GROWTH_SIZE}K
 
             host_size_0=$(get_image_size_on_host)
@@ -123,6 +125,7 @@ for GROWTH_SIZE in 16 48 80; do
         done
     done
 done
+done
 
 # success, all done
 echo '*** done'

tests/qemu-iotests/127

+#!/bin/bash
+#
+# Test case for mirroring with dataplane
+#
+# Copyright (C) 2017 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq=$(basename $0)
+echo "QA output created by $seq"
+
+here=$PWD
+status=1    # failure is the default!
+
+_cleanup()
+{
+    _cleanup_qemu
+    _cleanup_test_img
+    _rm_test_img "$TEST_IMG.overlay0"
+    _rm_test_img "$TEST_IMG.overlay1"
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and qemu instance handling
+. ./common.rc
+. ./common.filter
+. ./common.qemu
+
+_supported_fmt qcow2
+_supported_proto file
+_supported_os Linux
+
+IMG_SIZE=64K
+
+_make_test_img $IMG_SIZE
+TEST_IMG="$TEST_IMG.overlay0" _make_test_img -b "$TEST_IMG" $IMG_SIZE
+TEST_IMG="$TEST_IMG.overlay1" _make_test_img -b "$TEST_IMG" $IMG_SIZE
+
+# So that we actually have something to mirror and the job does not return
+# immediately (which may be bad because then we cannot know whether the
+# 'return' or the 'BLOCK_JOB_READY' comes first).
+$QEMU_IO -c 'write 0 42' "$TEST_IMG.overlay0" | _filter_qemu_io
+
+# We cannot use virtio-blk here because that does not actually set the attached
+# BB's AioContext in qtest mode
+_launch_qemu \
+    -object iothread,id=iothr \
+    -blockdev node-name=source,driver=$IMGFMT,file.driver=file,file.filename="$TEST_IMG.overlay0" \
+    -device virtio-scsi,id=scsi-bus,iothread=iothr \
+    -device scsi-hd,bus=scsi-bus.0,drive=source
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'qmp_capabilities' }" \
+    'return'
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'drive-mirror',
+       'arguments': {
+           'job-id': 'mirror',
+           'device': 'source',
+           'target': '$TEST_IMG.overlay1',
+           'mode':   'existing',
+           'sync':   'top'
+       } }" \
+    'BLOCK_JOB_READY'
+
+# The backing BDS should be assigned the overlay's AioContext
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'block-job-complete',
+       'arguments': { 'device': 'mirror' } }" \
+    'BLOCK_JOB_COMPLETED'
+
+_send_qemu_cmd $QEMU_HANDLE \
+    "{ 'execute': 'quit' }" \
+    'return'
+
+wait=yes _cleanup_qemu
+
+# success, all done
+echo '*** done'
+rm -f $seq.full
+status=0

tests/qemu-iotests/127.out

+QA output created by 127
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=65536
+Formatting 'TEST_DIR/t.IMGFMT.overlay0', fmt=IMGFMT size=65536 backing_file=TEST_DIR/t.IMGFMT
+Formatting 'TEST_DIR/t.IMGFMT.overlay1', fmt=IMGFMT size=65536 backing_file=TEST_DIR/t.IMGFMT
+wrote 42/42 bytes at offset 0
+42 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+{"return": {}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_READY", "data": {"device": "mirror", "len": 65536, "offset": 65536, "speed": 0, "type": "mirror"}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "BLOCK_JOB_COMPLETED", "data": {"device": "mirror", "len": 65536, "offset": 65536, "speed": 0, "type": "mirror"}}
+{"return": {}}
+{"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN", "data": {"guest": false}}
+*** done

tests/qemu-iotests/184

@@ -51,7 +51,8 @@ function do_run_qemu()
 function run_qemu()
 {
     do_run_qemu "$@" 2>&1 | _filter_testdir | _filter_qemu | _filter_qmp\
-                          | _filter_qemu_io | _filter_generated_node_ids
+                          | _filter_qemu_io | _filter_generated_node_ids \
+                          | _filter_actual_image_size
 }
 
 _make_test_img 64M

tests/qemu-iotests/184.out

@@ -32,7 +32,7 @@ Testing:
                 "filename": "json:{\"throttle-group\": \"group0\", \"driver\": \"throttle\", \"file\": {\"driver\": \"qcow2\", \"file\": {\"driver\": \"file\", \"filename\": \"TEST_DIR/t.qcow2\"}}}",
                 "cluster-size": 65536,
                 "format": "throttle",
-                "actual-size": 200704,
+                "actual-size": SIZE,
                 "dirty-flag": false
             },
             "iops_wr": 0,
@@ -62,7 +62,7 @@ Testing:
                 "filename": "TEST_DIR/t.qcow2",
                 "cluster-size": 65536,
                 "format": "qcow2",
-                "actual-size": 200704,
+                "actual-size": SIZE,
                 "format-specific": {
                     "type": "qcow2",
                     "data": {
@@ -100,7 +100,7 @@ Testing:
                 "virtual-size": 197120,
                 "filename": "TEST_DIR/t.qcow2",
                 "format": "file",
-                "actual-size": 200704,
+                "actual-size": SIZE,
                 "dirty-flag": false
             },
             "iops_wr": 0,

tests/qemu-iotests/191

@@ -92,7 +92,7 @@ echo === Check that both top and top2 point to base now ===
 echo
 
 _send_qemu_cmd $h "{ 'execute': 'query-named-block-nodes' }" "^}" |
-    _filter_generated_node_ids
+    _filter_generated_node_ids | _filter_actual_image_size
 
 _send_qemu_cmd $h "{ 'execute': 'quit' }" "^}"
 wait=1 _cleanup_qemu
@@ -140,7 +140,7 @@ echo === Check that both top and top2 point to base now ===
 echo
 
 _send_qemu_cmd $h "{ 'execute': 'query-named-block-nodes' }" "^}" |
-    _filter_generated_node_ids
+    _filter_generated_node_ids | _filter_actual_image_size
 
 _send_qemu_cmd $h "{ 'execute': 'quit' }" "^}"
 wait=1 _cleanup_qemu

tests/qemu-iotests/191.out

@@ -47,7 +47,7 @@ wrote 65536/65536 bytes at offset 1048576
                     "filename": "TEST_DIR/t.qcow2.base",
                     "cluster-size": 65536,
                     "format": "qcow2",
-                    "actual-size": 397312,
+                    "actual-size": SIZE,
                     "format-specific": {
                         "type": "qcow2",
                         "data": {
@@ -64,7 +64,7 @@ wrote 65536/65536 bytes at offset 1048576
                 "filename": "TEST_DIR/t.qcow2.ovl2",
                 "cluster-size": 65536,
                 "format": "qcow2",
-                "actual-size": 200704,
+                "actual-size": SIZE,
                 "format-specific": {
                     "type": "qcow2",
                     "data": {
@@ -105,7 +105,7 @@ wrote 65536/65536 bytes at offset 1048576
                 "virtual-size": 197120,
                 "filename": "TEST_DIR/t.qcow2.ovl2",
                 "format": "file",
-                "actual-size": 200704,
+                "actual-size": SIZE,
                 "dirty-flag": false
             },
             "iops_wr": 0,
@@ -136,7 +136,7 @@ wrote 65536/65536 bytes at offset 1048576
                     "filename": "TEST_DIR/t.qcow2.base",
                     "cluster-size": 65536,
                     "format": "qcow2",
-                    "actual-size": 397312,
+                    "actual-size": SIZE,
                     "format-specific": {
                         "type": "qcow2",
                         "data": {
@@ -153,7 +153,7 @@ wrote 65536/65536 bytes at offset 1048576
                 "filename": "TEST_DIR/t.qcow2",
                 "cluster-size": 65536,
                 "format": "qcow2",
-                "actual-size": 200704,
+                "actual-size": SIZE,
                 "format-specific": {
                     "type": "qcow2",
                     "data": {
@@ -194,7 +194,7 @@ wrote 65536/65536 bytes at offset 1048576
                 "virtual-size": 197120,
                 "filename": "TEST_DIR/t.qcow2",
                 "format": "file",
-                "actual-size": 200704,
+                "actual-size": SIZE,
                 "dirty-flag": false
             },
             "iops_wr": 0,
@@ -225,7 +225,7 @@ wrote 65536/65536 bytes at offset 1048576
                     "filename": "TEST_DIR/t.qcow2.base",
                     "cluster-size": 65536,
                     "format": "qcow2",
-                    "actual-size": 397312,
+                    "actual-size": SIZE,
                     "format-specific": {
                         "type": "qcow2",
                         "data": {
@@ -242,7 +242,7 @@ wrote 65536/65536 bytes at offset 1048576
                 "filename": "TEST_DIR/t.qcow2.mid",
                 "cluster-size": 65536,
                 "format": "qcow2",
-                "actual-size": 397312,
+                "actual-size": SIZE,
                 "format-specific": {
                     "type": "qcow2",
                     "data": {
@@ -283,7 +283,7 @@ wrote 65536/65536 bytes at offset 1048576
                 "virtual-size": 393216,
                 "filename": "TEST_DIR/t.qcow2.mid",
                 "format": "file",
-                "actual-size": 397312,
+                "actual-size": SIZE,
                 "dirty-flag": false
             },
             "iops_wr": 0,
@@ -313,7 +313,7 @@ wrote 65536/65536 bytes at offset 1048576
                 "filename": "TEST_DIR/t.qcow2.base",
                 "cluster-size": 65536,
                 "format": "qcow2",
-                "actual-size": 397312,
+                "actual-size": SIZE,
                 "format-specific": {
                     "type": "qcow2",
                     "data": {
@@ -351,7 +351,7 @@ wrote 65536/65536 bytes at offset 1048576
                 "virtual-size": 393216,
                 "filename": "TEST_DIR/t.qcow2.base",
                 "format": "file",
-                "actual-size": 397312,
+                "actual-size": SIZE,
                 "dirty-flag": false
             },
             "iops_wr": 0,
@@ -450,7 +450,7 @@ wrote 65536/65536 bytes at offset 1048576
                     "filename": "TEST_DIR/t.qcow2.base",
                     "cluster-size": 65536,
                     "format": "qcow2",
-                    "actual-size": 397312,
+                    "actual-size": SIZE,
                     "format-specific": {
                         "type": "qcow2",
                         "data": {
@@ -467,7 +467,7 @@ wrote 65536/65536 bytes at offset 1048576
                 "filename": "TEST_DIR/t.qcow2.ovl2",
                 "cluster-size": 65536,
                 "format": "qcow2",
-                "actual-size": 200704,
+                "actual-size": SIZE,
                 "format-specific": {
                     "type": "qcow2",
                     "data": {
@@ -508,7 +508,7 @@ wrote 65536/65536 bytes at offset 1048576
                 "virtual-size": 197120,
                 "filename": "TEST_DIR/t.qcow2.ovl2",
                 "format": "file",
-                "actual-size": 200704,
+                "actual-size": SIZE,
                 "dirty-flag": false
             },
             "iops_wr": 0,
@@ -540,7 +540,7 @@ wrote 65536/65536 bytes at offset 1048576
                         "filename": "TEST_DIR/t.qcow2.base",
                         "cluster-size": 65536,
                         "format": "qcow2",
-                        "actual-size": 397312,
+                        "actual-size": SIZE,
                         "format-specific": {
                             "type": "qcow2",
                             "data": {
@@ -557,7 +557,7 @@ wrote 65536/65536 bytes at offset 1048576
                     "filename": "TEST_DIR/t.qcow2.ovl2",
                     "cluster-size": 65536,
                     "format": "qcow2",
-                    "actual-size": 200704,
+                    "actual-size": SIZE,
                     "format-specific": {
                         "type": "qcow2",
                         "data": {
@@ -576,7 +576,7 @@ wrote 65536/65536 bytes at offset 1048576
                 "filename": "TEST_DIR/t.qcow2.ovl3",
                 "cluster-size": 65536,
                 "format": "qcow2",
-                "actual-size": 200704,
+                "actual-size": SIZE,
                 "format-specific": {
                     "type": "qcow2",
                     "data": {
@@ -617,7 +617,7 @@ wrote 65536/65536 bytes at offset 1048576
                 "virtual-size": 197120,
                 "filename": "TEST_DIR/t.qcow2.ovl3",
                 "format": "file",
-                "actual-size": 200704,
+                "actual-size": SIZE,
                 "dirty-flag": false
             },
             "iops_wr": 0,
@@ -647,7 +647,7 @@ wrote 65536/65536 bytes at offset 1048576
                 "filename": "TEST_DIR/t.qcow2.base",
                 "cluster-size": 65536,
                 "format": "qcow2",
-                "actual-size": 397312,
+                "actual-size": SIZE,
                 "format-specific": {
                     "type": "qcow2",
                     "data": {
@@ -685,7 +685,7 @@ wrote 65536/65536 bytes at offset 1048576
                 "virtual-size": 393216,
                 "filename": "TEST_DIR/t.qcow2.base",
                 "format": "file",
-                "actual-size": 397312,
+                "actual-size": SIZE,
                 "dirty-flag": false
             },
             "iops_wr": 0,
@@ -716,7 +716,7 @@ wrote 65536/65536 bytes at offset 1048576
                     "filename": "TEST_DIR/t.qcow2.base",
                     "cluster-size": 65536,
                     "format": "qcow2",
-                    "actual-size": 397312,
+                    "actual-size": SIZE,
                     "format-specific": {
                         "type": "qcow2",
                         "data": {
@@ -733,7 +733,7 @@ wrote 65536/65536 bytes at offset 1048576
                 "filename": "TEST_DIR/t.qcow2",
                 "cluster-size": 65536,
                 "format": "qcow2",
-                "actual-size": 200704,
+                "actual-size": SIZE,
                 "format-specific": {
                     "type": "qcow2",
                     "data": {
@@ -774,7 +774,7 @@ wrote 65536/65536 bytes at offset 1048576
                 "virtual-size": 197120,
                 "filename": "TEST_DIR/t.qcow2",
                 "format": "file",
-                "actual-size": 200704,
+                "actual-size": SIZE,
                 "dirty-flag": false
             },
             "iops_wr": 0,

tests/qemu-iotests/common.filter

@@ -105,6 +105,12 @@ _filter_block_job_len()
     sed -e 's/, "len": [0-9]\+,/, "len": LEN,/g'
 }
 
+# replace actual image size (depends on the host filesystem)
+_filter_actual_image_size()
+{
+    sed -s 's/\("actual-size":\s*\)[0-9]\+/\1SIZE/g'
+}
+
 # replace driver-specific options in the "Formatting..." line
 _filter_img_create()
 {

tests/qemu-iotests/group

@@ -133,6 +133,7 @@
 124 rw auto backing
 125 rw auto
 126 rw auto backing
+127 rw auto backing quick
 128 rw auto quick
 129 rw auto quick
 130 rw auto quick