diff --git a/block/qcow2-cache.c b/block/qcow2-cache.c index 53b8afc3d3c3ecdad24bb5c24afeb2020db87c3f..046f5b8e48b4a0e0cc16d7c365dc0980061ad86c 100644 --- a/block/qcow2-cache.c +++ b/block/qcow2-cache.c @@ -22,16 +22,24 @@ * THE SOFTWARE. */ +/* Needed for CONFIG_MADVISE */ +#include "config-host.h" + +#if defined(CONFIG_MADVISE) || defined(CONFIG_POSIX_MADVISE) +#include +#endif + #include "block/block_int.h" #include "qemu-common.h" +#include "qemu/osdep.h" #include "qcow2.h" #include "trace.h" typedef struct Qcow2CachedTable { int64_t offset; - bool dirty; uint64_t lru_counter; int ref; + bool dirty; } Qcow2CachedTable; struct Qcow2Cache { @@ -41,6 +49,7 @@ struct Qcow2Cache { bool depends_on_flush; void *table_array; uint64_t lru_counter; + uint64_t cache_clean_lru_counter; }; static inline void *qcow2_cache_get_table_addr(BlockDriverState *bs, @@ -60,6 +69,56 @@ static inline int qcow2_cache_get_table_idx(BlockDriverState *bs, return idx; } +static void qcow2_cache_table_release(BlockDriverState *bs, Qcow2Cache *c, + int i, int num_tables) +{ +#if QEMU_MADV_DONTNEED != QEMU_MADV_INVALID + BDRVQcowState *s = bs->opaque; + void *t = qcow2_cache_get_table_addr(bs, c, i); + int align = getpagesize(); + size_t mem_size = (size_t) s->cluster_size * num_tables; + size_t offset = QEMU_ALIGN_UP((uintptr_t) t, align) - (uintptr_t) t; + size_t length = QEMU_ALIGN_DOWN(mem_size - offset, align); + if (length > 0) { + qemu_madvise((uint8_t *) t + offset, length, QEMU_MADV_DONTNEED); + } +#endif +} + +static inline bool can_clean_entry(Qcow2Cache *c, int i) +{ + Qcow2CachedTable *t = &c->entries[i]; + return t->ref == 0 && !t->dirty && t->offset != 0 && + t->lru_counter <= c->cache_clean_lru_counter; +} + +void qcow2_cache_clean_unused(BlockDriverState *bs, Qcow2Cache *c) +{ + int i = 0; + while (i < c->size) { + int to_clean = 0; + + /* Skip the entries that we don't need to clean */ + while (i < c->size && !can_clean_entry(c, i)) { + i++; + } + + /* And count how many we can clean in a row */ + while (i < c->size && can_clean_entry(c, i)) { + c->entries[i].offset = 0; + c->entries[i].lru_counter = 0; + i++; + to_clean++; + } + + if (to_clean > 0) { + qcow2_cache_table_release(bs, c, i - to_clean, to_clean); + } + } + + c->cache_clean_lru_counter = c->lru_counter; +} + Qcow2Cache *qcow2_cache_create(BlockDriverState *bs, int num_tables) { BDRVQcowState *s = bs->opaque; @@ -237,6 +296,8 @@ int qcow2_cache_empty(BlockDriverState *bs, Qcow2Cache *c) c->entries[i].lru_counter = 0; } + qcow2_cache_table_release(bs, c, 0, c->size); + c->lru_counter = 0; return 0; diff --git a/block/qcow2.c b/block/qcow2.c index 76c331b387e147625f46d83787eaf0f84ee6365f..ea34ae2da5e52dfc84c73a1ec4b7f779567bcbb7 100644 --- a/block/qcow2.c +++ b/block/qcow2.c @@ -467,6 +467,11 @@ static QemuOptsList qcow2_runtime_opts = { .type = QEMU_OPT_SIZE, .help = "Maximum refcount block cache size", }, + { + .name = QCOW2_OPT_CACHE_CLEAN_INTERVAL, + .type = QEMU_OPT_NUMBER, + .help = "Clean unused cache entries after this time (in seconds)", + }, { /* end of list */ } }, }; @@ -482,6 +487,49 @@ static const char *overlap_bool_option_names[QCOW2_OL_MAX_BITNR] = { [QCOW2_OL_INACTIVE_L2_BITNR] = QCOW2_OPT_OVERLAP_INACTIVE_L2, }; +static void cache_clean_timer_cb(void *opaque) +{ + BlockDriverState *bs = opaque; + BDRVQcowState *s = bs->opaque; + qcow2_cache_clean_unused(bs, s->l2_table_cache); + qcow2_cache_clean_unused(bs, s->refcount_block_cache); + timer_mod(s->cache_clean_timer, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + + (int64_t) s->cache_clean_interval * 1000); +} + +static void cache_clean_timer_init(BlockDriverState *bs, AioContext *context) +{ + BDRVQcowState *s = bs->opaque; + if (s->cache_clean_interval > 0) { + s->cache_clean_timer = aio_timer_new(context, QEMU_CLOCK_VIRTUAL, + SCALE_MS, cache_clean_timer_cb, + bs); + timer_mod(s->cache_clean_timer, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + + (int64_t) s->cache_clean_interval * 1000); + } +} + +static void cache_clean_timer_del(BlockDriverState *bs) +{ + BDRVQcowState *s = bs->opaque; + if (s->cache_clean_timer) { + timer_del(s->cache_clean_timer); + timer_free(s->cache_clean_timer); + s->cache_clean_timer = NULL; + } +} + +static void qcow2_detach_aio_context(BlockDriverState *bs) +{ + cache_clean_timer_del(bs); +} + +static void qcow2_attach_aio_context(BlockDriverState *bs, + AioContext *new_context) +{ + cache_clean_timer_init(bs, new_context); +} + static void read_cache_sizes(BlockDriverState *bs, QemuOpts *opts, uint64_t *l2_cache_size, uint64_t *refcount_cache_size, Error **errp) @@ -555,6 +603,7 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags, const char *opt_overlap_check, *opt_overlap_check_template; int overlap_check_template = 0; uint64_t l2_cache_size, refcount_cache_size; + uint64_t cache_clean_interval; ret = bdrv_pread(bs->file, 0, &header, sizeof(header)); if (ret < 0) { @@ -848,6 +897,16 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags, goto fail; } + cache_clean_interval = + qemu_opt_get_number(opts, QCOW2_OPT_CACHE_CLEAN_INTERVAL, 0); + if (cache_clean_interval > UINT_MAX) { + error_setg(errp, "Cache clean interval too big"); + ret = -EINVAL; + goto fail; + } + s->cache_clean_interval = cache_clean_interval; + cache_clean_timer_init(bs, bdrv_get_aio_context(bs)); + s->cluster_cache = g_malloc(s->cluster_size); /* one more sector for decompressed data alignment */ s->cluster_data = qemu_try_blockalign(bs->file, QCOW_MAX_CRYPT_CLUSTERS @@ -1013,6 +1072,7 @@ static int qcow2_open(BlockDriverState *bs, QDict *options, int flags, qemu_vfree(s->l1_table); /* else pre-write overlap checks in cache_destroy may crash */ s->l1_table = NULL; + cache_clean_timer_del(bs); if (s->l2_table_cache) { qcow2_cache_destroy(bs, s->l2_table_cache); } @@ -1471,6 +1531,7 @@ static void qcow2_close(BlockDriverState *bs) } } + cache_clean_timer_del(bs); qcow2_cache_destroy(bs, s->l2_table_cache); qcow2_cache_destroy(bs, s->refcount_block_cache); @@ -2977,6 +3038,9 @@ BlockDriver bdrv_qcow2 = { .create_opts = &qcow2_create_opts, .bdrv_check = qcow2_check, .bdrv_amend_options = qcow2_amend_options, + + .bdrv_detach_aio_context = qcow2_detach_aio_context, + .bdrv_attach_aio_context = qcow2_attach_aio_context, }; static void bdrv_qcow2_init(void) diff --git a/block/qcow2.h b/block/qcow2.h index 72e132838a6eab3556b729f1e4899c2959d5688e..71dafd6dc9a290c9f5c5d30efc94c4acf3e4b688 100644 --- a/block/qcow2.h +++ b/block/qcow2.h @@ -96,6 +96,7 @@ #define QCOW2_OPT_CACHE_SIZE "cache-size" #define QCOW2_OPT_L2_CACHE_SIZE "l2-cache-size" #define QCOW2_OPT_REFCOUNT_CACHE_SIZE "refcount-cache-size" +#define QCOW2_OPT_CACHE_CLEAN_INTERVAL "cache-clean-interval" typedef struct QCowHeader { uint32_t magic; @@ -239,6 +240,8 @@ typedef struct BDRVQcowState { Qcow2Cache* l2_table_cache; Qcow2Cache* refcount_block_cache; + QEMUTimer *cache_clean_timer; + unsigned cache_clean_interval; uint8_t *cluster_cache; uint8_t *cluster_data; @@ -581,6 +584,7 @@ int qcow2_cache_set_dependency(BlockDriverState *bs, Qcow2Cache *c, Qcow2Cache *dependency); void qcow2_cache_depends_on_flush(Qcow2Cache *c); +void qcow2_cache_clean_unused(BlockDriverState *bs, Qcow2Cache *c); int qcow2_cache_empty(BlockDriverState *bs, Qcow2Cache *c); int qcow2_cache_get(BlockDriverState *bs, Qcow2Cache *c, uint64_t offset, diff --git a/block/quorum.c b/block/quorum.c index 2f6c45f7606235fd62ac305f4cb0d803584118ac..8fe53b42729bbdec1d9fade5f76e6a0ee969aef4 100644 --- a/block/quorum.c +++ b/block/quorum.c @@ -889,6 +889,12 @@ static int quorum_open(BlockDriverState *bs, QDict *options, int flags, } s->threshold = qemu_opt_get_number(opts, QUORUM_OPT_VOTE_THRESHOLD, 0); + /* and validate it against s->num_children */ + ret = quorum_valid_threshold(s->threshold, s->num_children, &local_err); + if (ret < 0) { + goto exit; + } + ret = parse_read_pattern(qemu_opt_get(opts, QUORUM_OPT_READ_PATTERN)); if (ret < 0) { error_setg(&local_err, "Please set read-pattern as fifo or quorum"); @@ -897,12 +903,6 @@ static int quorum_open(BlockDriverState *bs, QDict *options, int flags, s->read_pattern = ret; if (s->read_pattern == QUORUM_READ_PATTERN_QUORUM) { - /* and validate it against s->num_children */ - ret = quorum_valid_threshold(s->threshold, s->num_children, &local_err); - if (ret < 0) { - goto exit; - } - /* is the driver in blkverify mode */ if (qemu_opt_get_bool(opts, QUORUM_OPT_BLKVERIFY, false) && s->num_children == 2 && s->threshold == 2) { diff --git a/block/raw-posix.c b/block/raw-posix.c index 855febed5222568f8faa3c89a73fb4b5f522cd7f..30df8adf7fb446d49654a7d96c20f04d0c325d4f 100644 --- a/block/raw-posix.c +++ b/block/raw-posix.c @@ -670,11 +670,17 @@ static int raw_reopen_prepare(BDRVReopenState *state, /* If we cannot use fcntl, or fcntl failed, fall back to qemu_open() */ if (raw_s->fd == -1) { - assert(!(raw_s->open_flags & O_CREAT)); - raw_s->fd = qemu_open(state->bs->filename, raw_s->open_flags); - if (raw_s->fd == -1) { - error_setg_errno(errp, errno, "Could not reopen file"); - ret = -1; + const char *normalized_filename = state->bs->filename; + ret = raw_normalize_devicepath(&normalized_filename); + if (ret < 0) { + error_setg_errno(errp, -ret, "Could not normalize device path"); + } else { + assert(!(raw_s->open_flags & O_CREAT)); + raw_s->fd = qemu_open(normalized_filename, raw_s->open_flags); + if (raw_s->fd == -1) { + error_setg_errno(errp, errno, "Could not reopen file"); + ret = -1; + } } } @@ -2314,6 +2320,12 @@ static int hdev_create(const char *filename, QemuOpts *opts, (void)has_prefix; + ret = raw_normalize_devicepath(&filename); + if (ret < 0) { + error_setg_errno(errp, -ret, "Could not normalize device path"); + return ret; + } + /* Read out options */ total_size = ROUND_UP(qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0), BDRV_SECTOR_SIZE); diff --git a/docs/qcow2-cache.txt b/docs/qcow2-cache.txt new file mode 100644 index 0000000000000000000000000000000000000000..5bb06072d36ba50f464ea818ceadf66bdf706f29 --- /dev/null +++ b/docs/qcow2-cache.txt @@ -0,0 +1,164 @@ +qcow2 L2/refcount cache configuration +===================================== +Copyright (C) 2015 Igalia, S.L. +Author: Alberto Garcia + +This work is licensed under the terms of the GNU GPL, version 2 or +later. See the COPYING file in the top-level directory. + +Introduction +------------ +The QEMU qcow2 driver has two caches that can improve the I/O +performance significantly. However, setting the right cache sizes is +not a straightforward operation. + +This document attempts to give an overview of the L2 and refcount +caches, and how to configure them. + +Please refer to the docs/specs/qcow2.txt file for an in-depth +technical description of the qcow2 file format. + + +Clusters +-------- +A qcow2 file is organized in units of constant size called clusters. + +The cluster size is configurable, but it must be a power of two and +its value 512 bytes or higher. QEMU currently defaults to 64 KB +clusters, and it does not support sizes larger than 2MB. + +The 'qemu-img create' command supports specifying the size using the +cluster_size option: + + qemu-img create -f qcow2 -o cluster_size=128K hd.qcow2 4G + + +The L2 tables +------------- +The qcow2 format uses a two-level structure to map the virtual disk as +seen by the guest to the disk image in the host. These structures are +called the L1 and L2 tables. + +There is one single L1 table per disk image. The table is small and is +always kept in memory. + +There can be many L2 tables, depending on how much space has been +allocated in the image. Each table is one cluster in size. In order to +read or write data from the virtual disk, QEMU needs to read its +corresponding L2 table to find out where that data is located. Since +reading the table for each I/O operation can be expensive, QEMU keeps +an L2 cache in memory to speed up disk access. + +The size of the L2 cache can be configured, and setting the right +value can improve the I/O performance significantly. + + +The refcount blocks +------------------- +The qcow2 format also mantains a reference count for each cluster. +Reference counts are used for cluster allocation and internal +snapshots. The data is stored in a two-level structure similar to the +L1/L2 tables described above. + +The second level structures are called refcount blocks, are also one +cluster in size and the number is also variable and dependent on the +amount of allocated space. + +Each block contains a number of refcount entries. Their size (in bits) +is a power of two and must not be higher than 64. It defaults to 16 +bits, but a different value can be set using the refcount_bits option: + + qemu-img create -f qcow2 -o refcount_bits=8 hd.qcow2 4G + +QEMU keeps a refcount cache to speed up I/O much like the +aforementioned L2 cache, and its size can also be configured. + + +Choosing the right cache sizes +------------------------------ +In order to choose the cache sizes we need to know how they relate to +the amount of allocated space. + +The amount of virtual disk that can be mapped by the L2 and refcount +caches (in bytes) is: + + disk_size = l2_cache_size * cluster_size / 8 + disk_size = refcount_cache_size * cluster_size * 8 / refcount_bits + +With the default values for cluster_size (64KB) and refcount_bits +(16), that is + + disk_size = l2_cache_size * 8192 + disk_size = refcount_cache_size * 32768 + +So in order to cover n GB of disk space with the default values we +need: + + l2_cache_size = disk_size_GB * 131072 + refcount_cache_size = disk_size_GB * 32768 + +QEMU has a default L2 cache of 1MB (1048576 bytes) and a refcount +cache of 256KB (262144 bytes), so using the formulas we've just seen +we have + + 1048576 / 131072 = 8 GB of virtual disk covered by that cache + 262144 / 32768 = 8 GB + + +How to configure the cache sizes +-------------------------------- +Cache sizes can be configured using the -drive option in the +command-line, or the 'blockdev-add' QMP command. + +There are three options available, and all of them take bytes: + +"l2-cache-size": maximum size of the L2 table cache +"refcount-cache-size": maximum size of the refcount block cache +"cache-size": maximum size of both caches combined + +There are two things that need to be taken into account: + + - Both caches must have a size that is a multiple of the cluster + size. + + - If you only set one of the options above, QEMU will automatically + adjust the others so that the L2 cache is 4 times bigger than the + refcount cache. + +This means that these options are equivalent: + + -drive file=hd.qcow2,l2-cache-size=2097152 + -drive file=hd.qcow2,refcount-cache-size=524288 + -drive file=hd.qcow2,cache-size=2621440 + +The reason for this 1/4 ratio is to ensure that both caches cover the +same amount of disk space. Note however that this is only valid with +the default value of refcount_bits (16). If you are using a different +value you might want to calculate both cache sizes yourself since QEMU +will always use the same 1/4 ratio. + +It's also worth mentioning that there's no strict need for both caches +to cover the same amount of disk space. The refcount cache is used +much less often than the L2 cache, so it's perfectly reasonable to +keep it small. + + +Reducing the memory usage +------------------------- +It is possible to clean unused cache entries in order to reduce the +memory usage during periods of low I/O activity. + +The parameter "cache-clean-interval" defines an interval (in seconds). +All cache entries that haven't been accessed during that interval are +removed from memory. + +This example removes all unused cache entries every 15 minutes: + + -drive file=hd.qcow2,cache-clean-interval=900 + +If unset, the default value for this parameter is 0 and it disables +this feature. + +Note that this functionality currently relies on the MADV_DONTNEED +argument for madvise() to actually free the memory, so it is not +useful in systems that don't follow that behavior. diff --git a/qapi/block-core.json b/qapi/block-core.json index 7b2efb8678ae77b8640ba10c7934c942ec584081..bb2189ef3a37288a7338b712c2e4a9537c7d9d89 100644 --- a/qapi/block-core.json +++ b/qapi/block-core.json @@ -1592,6 +1592,10 @@ # @refcount-cache-size: #optional the maximum size of the refcount block cache # in bytes (since 2.2) # +# @cache-clean-interval: #optional clean unused entries in the L2 and refcount +# caches. The interval is in seconds. The default value +# is 0 and it disables this feature (since 2.5) +# # Since: 1.7 ## { 'struct': 'BlockdevOptionsQcow2', @@ -1603,7 +1607,8 @@ '*overlap-check': 'Qcow2OverlapChecks', '*cache-size': 'int', '*l2-cache-size': 'int', - '*refcount-cache-size': 'int' } } + '*refcount-cache-size': 'int', + '*cache-clean-interval': 'int' } } ## diff --git a/qemu-img.c b/qemu-img.c index 75f4ee44212ef1ba332a5001d79d76301431f7b0..6ff4e852b176bfe474a2506cf5fcd7b0f3de3567 100644 --- a/qemu-img.c +++ b/qemu-img.c @@ -2931,7 +2931,7 @@ static int img_amend(int argc, char **argv) if (!is_valid_option_list(optarg)) { error_report("Invalid option list: %s", optarg); ret = -1; - goto out; + goto out_no_progress; } if (!options) { options = g_strdup(optarg); @@ -3031,6 +3031,7 @@ static int img_amend(int argc, char **argv) out: qemu_progress_end(); +out_no_progress: blk_unref(blk); qemu_opts_del(opts); qemu_opts_free(create_opts); diff --git a/tests/qemu-iotests/039 b/tests/qemu-iotests/039 index 859705f848ad548dfaa8439a6395874c6bce1dd4..617f3977ccde3fb30e43ad1d0c1dd08ed965d9e0 100755 --- a/tests/qemu-iotests/039 +++ b/tests/qemu-iotests/039 @@ -47,13 +47,6 @@ _supported_os Linux _default_cache_mode "writethrough" _supported_cache_modes "writethrough" -_subshell_exec() -{ - # Executing crashing commands in a subshell prevents information like the - # "Killed" line from being lost - (exec "$@") -} - size=128M echo @@ -74,8 +67,8 @@ echo "== Creating a dirty image file ==" IMGOPTS="compat=1.1,lazy_refcounts=on" _make_test_img $size -_subshell_exec $QEMU_IO -c "write -P 0x5a 0 512" \ - -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 \ +$QEMU_IO -c "write -P 0x5a 0 512" \ + -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 \ | _filter_qemu_io # The dirty bit must be set @@ -109,8 +102,8 @@ echo "== Opening a dirty image read/write should repair it ==" IMGOPTS="compat=1.1,lazy_refcounts=on" _make_test_img $size -_subshell_exec $QEMU_IO -c "write -P 0x5a 0 512" \ - -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 \ +$QEMU_IO -c "write -P 0x5a 0 512" \ + -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 \ | _filter_qemu_io # The dirty bit must be set @@ -127,8 +120,8 @@ echo "== Creating an image file with lazy_refcounts=off ==" IMGOPTS="compat=1.1,lazy_refcounts=off" _make_test_img $size -_subshell_exec $QEMU_IO -c "write -P 0x5a 0 512" \ - -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 \ +$QEMU_IO -c "write -P 0x5a 0 512" \ + -c "sigraise $(kill -l KILL)" "$TEST_IMG" 2>&1 \ | _filter_qemu_io # The dirty bit must not be set since lazy_refcounts=off diff --git a/tests/qemu-iotests/039.out b/tests/qemu-iotests/039.out index d09751f9c816eba57beec123d7886587ab9607ff..b055670b8ce03d6cb09f6a10526f8503ab19a55b 100644 --- a/tests/qemu-iotests/039.out +++ b/tests/qemu-iotests/039.out @@ -11,7 +11,7 @@ No errors were found on the image. Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 wrote 512/512 bytes at offset 0 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -./039: Killed ( exec "$@" ) +./common.config: Killed ( exec "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@" ) incompatible_features 0x1 ERROR cluster 5 refcount=0 reference=1 ERROR OFLAG_COPIED data cluster: l2_entry=8000000000050000 refcount=0 @@ -46,7 +46,7 @@ read 512/512 bytes at offset 0 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 wrote 512/512 bytes at offset 0 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -./039: Killed ( exec "$@" ) +./common.config: Killed ( exec "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@" ) incompatible_features 0x1 ERROR cluster 5 refcount=0 reference=1 Rebuilding refcount structure @@ -60,7 +60,7 @@ incompatible_features 0x0 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 wrote 512/512 bytes at offset 0 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) -./039: Killed ( exec "$@" ) +./common.config: Killed ( exec "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@" ) incompatible_features 0x0 No errors were found on the image. diff --git a/tests/qemu-iotests/041 b/tests/qemu-iotests/041 index 3d46ed705bb9633ede66e67a585d31c40138ac40..38ca5f13c71c21e01cc426241b9d4fff3acc32a6 100755 --- a/tests/qemu-iotests/041 +++ b/tests/qemu-iotests/041 @@ -42,6 +42,8 @@ class TestSingleDrive(iotests.QMPTestCase): iotests.create_image(backing_img, self.image_len) qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % backing_img, test_img) self.vm = iotests.VM().add_drive(test_img) + if iotests.qemu_default_machine == 'pc': + self.vm.add_drive(None, 'media=cdrom', 'ide') self.vm.launch() def tearDown(self): @@ -167,8 +169,11 @@ class TestSingleDrive(iotests.QMPTestCase): 'target image does not match source after mirroring') def test_medium_not_found(self): - result = self.vm.qmp('drive-mirror', device='ide1-cd0', sync='full', - target=target_img) + if iotests.qemu_default_machine != 'pc': + return + + result = self.vm.qmp('drive-mirror', device='drive1', # CD-ROM + sync='full', target=target_img) self.assert_qmp(result, 'error/class', 'GenericError') def test_image_not_found(self): @@ -707,6 +712,9 @@ class TestRepairQuorum(iotests.QMPTestCase): def setUp(self): self.vm = iotests.VM() + if iotests.qemu_default_machine == 'pc': + self.vm.add_drive(None, 'media=cdrom', 'ide') + # Add each individual quorum images for i in self.IMAGES: qemu_img('create', '-f', iotests.imgfmt, i, @@ -831,7 +839,11 @@ class TestRepairQuorum(iotests.QMPTestCase): if not self.has_quorum(): return - result = self.vm.qmp('drive-mirror', device='ide1-cd0', sync='full', + if iotests.qemu_default_machine != 'pc': + return + + result = self.vm.qmp('drive-mirror', device='drive0', # CD-ROM + sync='full', node_name='repair0', replaces='img1', target=quorum_repair_img, format=iotests.imgfmt) diff --git a/tests/qemu-iotests/049.out b/tests/qemu-iotests/049.out index 9f93666c5b539e805a6dd754a44a2673b8ddc887..8884543db99a3ae457903a3af3ae17cffecfa60d 100644 --- a/tests/qemu-iotests/049.out +++ b/tests/qemu-iotests/049.out @@ -95,17 +95,15 @@ qemu-img create -f qcow2 TEST_DIR/t.qcow2 -- -1024 qemu-img: Image size must be less than 8 EiB! qemu-img create -f qcow2 -o size=-1024 TEST_DIR/t.qcow2 -qemu-img: qcow2 doesn't support shrinking images yet -qemu-img: TEST_DIR/t.qcow2: Could not resize image: Operation not supported -Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=-1024 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16 +qemu-img: Parameter 'size' expects a non-negative number below 2^64 +qemu-img: TEST_DIR/t.qcow2: Invalid options for file format 'qcow2' qemu-img create -f qcow2 TEST_DIR/t.qcow2 -- -1k qemu-img: Image size must be less than 8 EiB! qemu-img create -f qcow2 -o size=-1k TEST_DIR/t.qcow2 -qemu-img: qcow2 doesn't support shrinking images yet -qemu-img: TEST_DIR/t.qcow2: Could not resize image: Operation not supported -Formatting 'TEST_DIR/t.qcow2', fmt=qcow2 size=-1024 encryption=off cluster_size=65536 lazy_refcounts=off refcount_bits=16 +qemu-img: Parameter 'size' expects a non-negative number below 2^64 +qemu-img: TEST_DIR/t.qcow2: Invalid options for file format 'qcow2' qemu-img create -f qcow2 TEST_DIR/t.qcow2 -- 1kilobyte qemu-img: Invalid image size specified! You may use k, M, G, T, P or E suffixes for diff --git a/tests/qemu-iotests/055 b/tests/qemu-iotests/055 index 017a609f39d06a1f77cc6bfb6f0dc9f921742906..c8e35787020eef8d14e2a66c13e34db27f37d68f 100755 --- a/tests/qemu-iotests/055 +++ b/tests/qemu-iotests/055 @@ -42,6 +42,8 @@ class TestSingleDrive(iotests.QMPTestCase): qemu_img('create', '-f', iotests.imgfmt, blockdev_target_img, str(TestSingleDrive.image_len)) self.vm = iotests.VM().add_drive(test_img).add_drive(blockdev_target_img) + if iotests.qemu_default_machine == 'pc': + self.vm.add_drive(None, 'media=cdrom', 'ide') self.vm.launch() def tearDown(self): @@ -104,12 +106,18 @@ class TestSingleDrive(iotests.QMPTestCase): self.do_test_pause('blockdev-backup', 'drive1', blockdev_target_img) def test_medium_not_found(self): - result = self.vm.qmp('drive-backup', device='ide1-cd0', + if iotests.qemu_default_machine != 'pc': + return + + result = self.vm.qmp('drive-backup', device='drive2', # CD-ROM target=target_img, sync='full') self.assert_qmp(result, 'error/class', 'GenericError') def test_medium_not_found_blockdev_backup(self): - result = self.vm.qmp('blockdev-backup', device='ide1-cd0', + if iotests.qemu_default_machine != 'pc': + return + + result = self.vm.qmp('blockdev-backup', device='drive2', # CD-ROM target='drive1', sync='full') self.assert_qmp(result, 'error/class', 'GenericError') @@ -249,6 +257,8 @@ class TestSingleTransaction(iotests.QMPTestCase): qemu_img('create', '-f', iotests.imgfmt, blockdev_target_img, str(TestSingleDrive.image_len)) self.vm = iotests.VM().add_drive(test_img).add_drive(blockdev_target_img) + if iotests.qemu_default_machine == 'pc': + self.vm.add_drive(None, 'media=cdrom', 'ide') self.vm.launch() def tearDown(self): @@ -323,9 +333,12 @@ class TestSingleTransaction(iotests.QMPTestCase): self.do_test_pause('blockdev-backup', 'drive1', blockdev_target_img) def do_test_medium_not_found(self, cmd, target): + if iotests.qemu_default_machine != 'pc': + return + result = self.vm.qmp('transaction', actions=[{ 'type': cmd, - 'data': { 'device': 'ide1-cd0', + 'data': { 'device': 'drive2', # CD-ROM 'target': target, 'sync': 'full' }, } diff --git a/tests/qemu-iotests/061 b/tests/qemu-iotests/061 index 8d37f8a65c74800e4edf20bf3e1c1aff2fa57c90..1df887a01dd336a0399997595fca14617a769150 100755 --- a/tests/qemu-iotests/061 +++ b/tests/qemu-iotests/061 @@ -58,7 +58,8 @@ echo echo "=== Testing dirty version downgrade ===" echo IMGOPTS="compat=1.1,lazy_refcounts=on" _make_test_img 64M -$QEMU_IO -c "write -P 0x2a 0 128k" -c flush -c abort "$TEST_IMG" | _filter_qemu_io +$QEMU_IO -c "write -P 0x2a 0 128k" -c flush -c abort "$TEST_IMG" 2>&1 \ + | _filter_qemu_io $PYTHON qcow2.py "$TEST_IMG" dump-header $QEMU_IMG amend -o "compat=0.10" "$TEST_IMG" $PYTHON qcow2.py "$TEST_IMG" dump-header @@ -91,7 +92,8 @@ echo echo "=== Testing dirty lazy_refcounts=off ===" echo IMGOPTS="compat=1.1,lazy_refcounts=on" _make_test_img 64M -$QEMU_IO -c "write -P 0x2a 0 128k" -c flush -c abort "$TEST_IMG" | _filter_qemu_io +$QEMU_IO -c "write -P 0x2a 0 128k" -c flush -c abort "$TEST_IMG" 2>&1 \ + | _filter_qemu_io $PYTHON qcow2.py "$TEST_IMG" dump-header $QEMU_IMG amend -o "lazy_refcounts=off" "$TEST_IMG" $PYTHON qcow2.py "$TEST_IMG" dump-header diff --git a/tests/qemu-iotests/061.out b/tests/qemu-iotests/061.out index 5ec248f79b41dfd0a43fa24941c8a21a4d1e8ebb..45053764337d6a17c1bdab4776ed3f45858a102b 100644 --- a/tests/qemu-iotests/061.out +++ b/tests/qemu-iotests/061.out @@ -57,6 +57,7 @@ No errors were found on the image. Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 wrote 131072/131072 bytes at offset 0 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +./common.config: Aborted (core dumped) ( exec "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@" ) magic 0x514649fb version 3 backing_file_offset 0x0 @@ -214,6 +215,7 @@ No errors were found on the image. Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 wrote 131072/131072 bytes at offset 0 128 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) +./common.config: Aborted (core dumped) ( exec "$QEMU_IO_PROG" $QEMU_IO_OPTIONS "$@" ) magic 0x514649fb version 3 backing_file_offset 0x0 diff --git a/tests/qemu-iotests/067 b/tests/qemu-iotests/067 index 83eefa394eb45860cfa50acff58e99ec1d5eaea7..3e9a053ef3d574c4cfa147242ef825bb098de606 100755 --- a/tests/qemu-iotests/067 +++ b/tests/qemu-iotests/067 @@ -59,7 +59,7 @@ echo echo === -drive/-device and device_del === echo -run_qemu -drive file=$TEST_IMG,format=$IMGFMT,if=none,id=disk -device virtio-blk-pci,drive=disk,id=virtio0 <"${fifo_out}" \ 2>&1 \ <"${fifo_in}" & diff --git a/tests/qemu-iotests/common.rc b/tests/qemu-iotests/common.rc index 22d3514041577739a43b8c852426ed2e40ae87a6..28e4beac15637b93c48b6887fbb1da137c5eed14 100644 --- a/tests/qemu-iotests/common.rc +++ b/tests/qemu-iotests/common.rc @@ -439,7 +439,17 @@ _unsupported_imgopts() # _require_command() { - eval c=\$$1 + if [ "$1" = "QEMU" ]; then + c=$QEMU_PROG + elif [ "$1" = "QEMU_IMG" ]; then + c=$QEMU_IMG_PROG + elif [ "$1" = "QEMU_IO" ]; then + c=$QEMU_IO_PROG + elif [ "$1" = "QEMU_NBD" ]; then + c=$QEMU_NBD_PROG + else + eval c=\$$1 + fi [ -x "$c" ] || _notrun "$1 utility required, skipped this test" } diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py index 8615b107512dd26f05aaa7bc351d6034dde1bfaa..ff5905f37922c747a7c42c8cbe393fee7452a636 100644 --- a/tests/qemu-iotests/iotests.py +++ b/tests/qemu-iotests/iotests.py @@ -31,37 +31,60 @@ import struct __all__ = ['imgfmt', 'imgproto', 'test_dir' 'qemu_img', 'qemu_io', 'VM', 'QMPTestCase', 'notrun', 'main'] -# This will not work if arguments or path contain spaces but is necessary if we +# This will not work if arguments contain spaces but is necessary if we # want to support the override options that ./check supports. -qemu_img_args = os.environ.get('QEMU_IMG', 'qemu-img').strip().split(' ') -qemu_io_args = os.environ.get('QEMU_IO', 'qemu-io').strip().split(' ') -qemu_args = os.environ.get('QEMU', 'qemu').strip().split(' ') +qemu_img_args = [os.environ.get('QEMU_IMG_PROG', 'qemu-img')] +if os.environ.get('QEMU_IMG_OPTIONS'): + qemu_img_args += os.environ['QEMU_IMG_OPTIONS'].strip().split(' ') + +qemu_io_args = [os.environ.get('QEMU_IO_PROG', 'qemu-io')] +if os.environ.get('QEMU_IO_OPTIONS'): + qemu_io_args += os.environ['QEMU_IO_OPTIONS'].strip().split(' ') + +qemu_args = [os.environ.get('QEMU_PROG', 'qemu')] +if os.environ.get('QEMU_OPTIONS'): + qemu_args += os.environ['QEMU_OPTIONS'].strip().split(' ') imgfmt = os.environ.get('IMGFMT', 'raw') imgproto = os.environ.get('IMGPROTO', 'file') test_dir = os.environ.get('TEST_DIR', '/var/tmp') output_dir = os.environ.get('OUTPUT_DIR', '.') cachemode = os.environ.get('CACHEMODE') +qemu_default_machine = os.environ.get('QEMU_DEFAULT_MACHINE') socket_scm_helper = os.environ.get('SOCKET_SCM_HELPER', 'socket_scm_helper') def qemu_img(*args): '''Run qemu-img and return the exit code''' devnull = open('/dev/null', 'r+') - return subprocess.call(qemu_img_args + list(args), stdin=devnull, stdout=devnull) + exitcode = subprocess.call(qemu_img_args + list(args), stdin=devnull, stdout=devnull) + if exitcode < 0: + sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args)))) + return exitcode def qemu_img_verbose(*args): '''Run qemu-img without suppressing its output and return the exit code''' - return subprocess.call(qemu_img_args + list(args)) + exitcode = subprocess.call(qemu_img_args + list(args)) + if exitcode < 0: + sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args)))) + return exitcode def qemu_img_pipe(*args): '''Run qemu-img and return its output''' - return subprocess.Popen(qemu_img_args + list(args), stdout=subprocess.PIPE).communicate()[0] + subp = subprocess.Popen(qemu_img_args + list(args), stdout=subprocess.PIPE) + exitcode = subp.wait() + if exitcode < 0: + sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args)))) + return subp.communicate()[0] def qemu_io(*args): '''Run qemu-io and return the stdout data''' args = qemu_io_args + list(args) - return subprocess.Popen(args, stdout=subprocess.PIPE).communicate()[0] + subp = subprocess.Popen(args, stdout=subprocess.PIPE) + exitcode = subp.wait() + if exitcode < 0: + sys.stderr.write('qemu-io received signal %i: %s\n' % (-exitcode, ' '.join(args))) + return subp.communicate()[0] def compare_images(img1, img2): '''Return True if two image files are identical''' @@ -117,13 +140,16 @@ class VM(object): self._args.append('-monitor') self._args.append(args) - def add_drive(self, path, opts=''): + def add_drive(self, path, opts='', interface='virtio'): '''Add a virtio-blk drive to the VM''' - options = ['if=virtio', + options = ['if=%s' % interface, 'format=%s' % imgfmt, 'cache=%s' % cachemode, - 'file=%s' % path, 'id=drive%d' % self._num_drives] + + if path is not None: + options.append('file=%s' % path) + if opts: options.append(opts) @@ -196,7 +222,9 @@ class VM(object): '''Terminate the VM and clean up''' if not self._popen is None: self._qmp.cmd('quit') - self._popen.wait() + exitcode = self._popen.wait() + if exitcode < 0: + sys.stderr.write('qemu received signal %i: %s\n' % (-exitcode, ' '.join(self._args))) os.remove(self._monitor_path) os.remove(self._qtest_path) os.remove(self._qemu_log_path) diff --git a/util/qemu-option.c b/util/qemu-option.c index efe9d279c423e15c16069b6c091ce5bc8748aa6c..efd6f024e7ebb8c2cc26b7725ff316d326ab3828 100644 --- a/util/qemu-option.c +++ b/util/qemu-option.c @@ -180,6 +180,11 @@ void parse_option_size(const char *name, const char *value, if (value != NULL) { sizef = strtod(value, &postfix); + if (sizef < 0 || sizef > UINT64_MAX) { + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, name, + "a non-negative number below 2^64"); + return; + } switch (*postfix) { case 'T': sizef *= 1024;