Merging pull request 1185

Signed-off-by: NLukáš Doktor <ldoktor@redhat.com>

* https://github.com/autotest/tp-qemu:
  Use the same content for example iSCSI target configuration
  qemu/tests/multi_disk.py: use virttest.error_context library
  Use tabular output from avocado.utils.astring
  Turn the iscsi target into example only
  Improve "dd" regexp to cope with newer versions in cgroup test
  Fix the usage of process in cgroup test
  Make sure images are removed in live_snapshot_chain

qemu/tests/cfg/boot_from_device.cfg

@@ -76,9 +76,10 @@
         - boot_from_iscsi_device:
             start_vm = no
             dev_name = iscsi-dev
-            portal_ip = 10.66.90.100
-            initiator = "iqn.2010-07.com.redhat:kvmautotest"
-            target = "iqn.2001-05.com.equallogic:0-8a0906-db31f7d03-470263b05654c204-kvm-puyiqiao"
+            # Please update these to point to your iscsi server
+            portal_ip = 127.0.0.1
+            initiator = "iqn.2010-07.com.example:my_initiator"
+            target = "iqn.2001-05.com.mytarget:0-0a0000-000000000-0000000000000000-my-target"
             images = "stg"
             image_name_stg = "/dev/sdb"
             image_format_stg = ""

qemu/tests/cfg/drive_mirror.cfg

@@ -70,10 +70,10 @@
             host_setup_flag_target = 2
             # Please replace below iscsi connection params before start your testing
             emulated_image_target = images/iscsi-storage.raw
-            portal_ip_target = "127.0.0.1"
-            initiator_target = "iqn.2010-07.test.org:kvmautotest"
-            target_target = "iqn.2001-05.com.equallogic:0-8a0906-db31f7d03-470263b05654c204-kvm-test"
-            # End
+            # Please update these to point to your iscsi server
+            portal_ip_target = 127.0.0.1
+            initiator_target = "iqn.2010-07.com.example:my_initiator"
+            target_target = "iqn.2001-05.com.mytarget:0-0a0000-000000000-0000000000000000-my-target"
     variants:
         - simple_test:
             variants:

qemu/tests/cfg/drive_mirror_negative_tests.cfg

@@ -72,9 +72,10 @@
             boot_target_image = yes
             # Please replace below iscsi connection params before start your testing
             emulated_image_target = images/iscsi-storage.raw
-            portal_ip_target = "127.0.0.1"
-            initiator_target = "iqn.2010-07.test.org:kvmautotest"
-            target_target = "iqn.2001-05.com.equallogic:0-8a0906-db31f7d03-470263b05654c204-kvm-test"
+            # Please update these to point to your iscsi server
+            portal_ip_target = 127.0.0.1
+            initiator_target = "iqn.2010-07.com.example:my_initiator"
+            target_target = "iqn.2001-05.com.mytarget:0-0a0000-000000000-0000000000000000-my-target"
             start_firewall_cmd = "iptables -A INPUT -p tcp -d ${portal_ip_target} --dport 3260 -j DROP"
             # End
     variants:

qemu/tests/cfg/qemu_img_negative.cfg

@@ -18,10 +18,6 @@
                     lv_size = 10G
                     lv_name_list = "lv_base lv_sn1 lv_sn2"
                     host_setup_flag = 2
-                    #Please update portal_ip, initiator and target params based on your test env
-                    #portal_ip = 127.0.0.1
-                    #initiator = "iqn.2010-07.com.redhat:kvmautotest"
-                    #target = "iqn.2001-05.com.equallogic:0-8a0906-db31f7d03-470263b05654c204-kvm-autotest"
                     image_chain += " sn1"
                     images += " sn1 sn2"
                     image_name_image1 = "lv_base"
@@ -29,3 +25,7 @@
                     image_name_sn2 = "lv_sn2"
                     rebase_list = "sn2 > image1"
                     negtive_test_sn2 = yes
+                    #Please update portal_ip, initiator and target params based on your test env
+                    portal_ip = 127.0.0.1
+                    initiator = "iqn.2010-07.com.example:my_initiator"
+                    target = "iqn.2001-05.com.mytarget:0-0a0000-000000000-0000000000000000-my-target"
\ No newline at end of file

qemu/tests/cgroup.py

@@ -8,7 +8,7 @@ import os
 import re
 import time
 
-from autotest.client import utils
+from avocado.utils import astring
 from avocado.utils import process
 from avocado.core import exceptions
 
@@ -172,7 +172,7 @@ def run(test, params, env):
         :note: params['cgroup_rmmod_scsi_debug'] == "yes" => rmmod scsi_debug
         """
         process.system("echo -%d > /sys/bus/pseudo/drivers/scsi_debug/add_host"
-                       % no_disks)
+                       % no_disks, shell=True)
 
         if params.get('cgroup_rmmod_scsi_debug', "no") == "yes":
             process.system("rmmod scsi_debug")
@@ -182,16 +182,18 @@ def run(test, params, env):
         Adds scsi_debug disk to every VM in params['vms']
         :param prefix: adds prefix to drive name
         """
-        if process.system("lsmod | grep scsi_debug", ignore_status=True):
+        if process.system("lsmod | grep scsi_debug", ignore_status=True,
+                          shell=True):
             process.system("modprobe scsi_debug dev_size_mb=8 add_host=0")
         for name in params['vms'].split(' '):
             disk_name = "scsi-debug-" + name
-            process.system("echo 1 >/sys/bus/pseudo/drivers/scsi_debug/add_host")
+            process.system("echo 1 > /sys/bus/pseudo/drivers/scsi_debug/"
+                           "add_host", shell=True)
             time.sleep(1)   # Wait for device init
             dev = process.system_output("ls /dev/sd* | tail -n 1", shell=True)
             # Enable idling in scsi_debug drive
             process.system("echo 1 > /sys/block/%s/queue/rotational"
-                           % (dev.split('/')[-1]))
+                           % (dev.split('/')[-1]), shell=True)
             vm_disks = params.get('images_%s' % name,
                                   params.get('images', 'image1'))
             params['images_%s' % name] = "%s %s" % (vm_disks, disk_name)
@@ -285,7 +287,7 @@ def run(test, params, env):
                     out[i] = out[i][1:-1]
                 for _ in re.findall(re_dd, out[i])[1:-1]:
                     data += int(_[0])
-                    duration += float(_[1])
+                    duration += float(_[2])
                 out[i] = int(data / duration)
 
             # normalize each output according to cgroup_weights
@@ -308,7 +310,7 @@ def run(test, params, env):
                     err += "%d, " % i
 
             logging.info("blkio_bandwidth_%s: dd statistics\n%s", direction,
-                         utils.matrix_to_string(out, ['status', 'norm_weights',
+                         astring.tabular_output(out, ['status', 'norm_weights',
                                                       'norm_out', 'actual']))
 
             if err:
@@ -357,8 +359,8 @@ def run(test, params, env):
         # ; true is necessarily when there is no dd present at the time
         kill_cmd = "rm -f /tmp/cgroup_lock; killall -9 dd; true"
         stat_cmd = "killall -SIGUSR1 dd; true"
-        re_dd = (r'(\d+) bytes \(\d+\.*\d* \w*\) copied, (\d+\.*\d*) s, '
-                 '\d+\.*\d* \w./s')
+        re_dd = (r'(\d+) bytes \(\d+\.*\d* \w*(, \d+\.*\d* \w*)?\) copied, '
+                 '(\d+\.*\d*) s, \d+\.*\d* \w./s')
         err = ""
         try:
             logging.info("Read test")
@@ -499,7 +501,7 @@ def run(test, params, env):
                     data = 0
                     for _ in re.findall(re_dd, out[i][j]):
                         data += int(_[0])
-                        duration += float(_[1])
+                        duration += float(_[2])
                     output.append(['PASS', j, 'vm%d' % i, speeds[i][j],
                                    int(data / duration)])
                     # Don't measure unlimited speeds
@@ -512,7 +514,7 @@ def run(test, params, env):
 
             # TODO: Unlimited speed fluctuates during test
             logging.info("blkio_throttle_%s: dd statistics\n%s", direction,
-                         utils.matrix_to_string(output, ['result', 'it',
+                         astring.tabular_output(output, ['result', 'it',
                                                          'vm', 'speed', 'actual']))
             if err:
                 err = ("blkio_throttle_%s: limits [%s] were broken"
@@ -602,8 +604,8 @@ def run(test, params, env):
         # ; true is necessarily when there is no dd present at the time
         kill_cmd = "rm -f /tmp/cgroup_lock; killall -9 dd; true"
         stat_cmd = "killall -SIGUSR1 dd; true"
-        re_dd = (r'(\d+) bytes \(\d+\.*\d* \w*\) copied, (\d+\.*\d*) s, '
-                 '\d+\.*\d* \w./s')
+        re_dd = (r'(\d+) bytes \(\d+\.*\d* \w*(, \d+\.*\d* \w*)?\) copied, '
+                 '(\d+\.*\d*) s, \d+\.*\d* \w./s')
         err = ""
         try:
             logging.info("Read test")
@@ -1219,7 +1221,7 @@ def run(test, params, env):
                         matrix[-1].append("%3d ~ %d" % (verify[i][j],
                                                         stats[i][j]))
             logging.info("Results (theoretical ~ actual):\n%s",
-                         utils.matrix_to_string(matrix, header))
+                         astring.tabular_output(matrix, header))
             if err:
                 err = "Scenerios %s FAILED" % err
                 logging.error(err)
@@ -1542,9 +1544,11 @@ def run(test, params, env):
         devices.mk_cgroup()
 
         # Add one scsi_debug disk which will be used in testing
-        if process.system("lsmod | grep scsi_debug", ignore_status=True):
+        if process.system("lsmod | grep scsi_debug", ignore_status=True,
+                          shell=True):
             process.system("modprobe scsi_debug dev_size_mb=8 add_host=0")
-        process.system("echo 1 > /sys/bus/pseudo/drivers/scsi_debug/add_host")
+        process.system("echo 1 > /sys/bus/pseudo/drivers/scsi_debug/add_host",
+                       shell=True)
         time.sleep(0.1)
         disk = process.system_output("ls /dev/sd* | tail -n 1", shell=True)
         dev = "%s:%s" % get_maj_min(disk)
@@ -1664,7 +1668,7 @@ def run(test, params, env):
             for i in range(10):
                 try:
                     out = process.system_output("cat /proc/%s/task/*/stat" %
-                                                pid)
+                                                pid, shell=True)
                 except process.CmdError:
                     out = None
                 else:

qemu/tests/live_snapshot_chain.py

@@ -62,6 +62,25 @@ def run(test, params, env):
                 cmd = cmd[8:]
                 vm.monitor.send_args_cmd(cmd)
 
+    def cleanup_images(snapshot_chain, params):
+        if not params.get("remove_snapshot_images"):
+            return []
+        errs = []
+        for index, image in enumerate(snapshot_chain):
+            try:
+                image_params = params.object_params(image)
+                if index != 0:
+                    image = qemu_storage.QemuImg(
+                        image_params, data_dir.get_data_dir(), image)
+                    if not os.path.exists(image.image_filename):
+                        errs.append("Image %s was not created during test."
+                                    % image.image_filename)
+                    image.remove()
+            except Exception as details:
+                errs.append("Fail to remove image %s: %s"
+                            % (image.image_filename, details))
+        return errs
+
     vm = env.get_vm(params["main_vm"])
     vm.verify_alive()
     timeout = int(params.get("login_timeout", 360))
@@ -79,97 +98,99 @@ def run(test, params, env):
 
     md5_value = {}
     files_in_guest = {}
-    for index, image in enumerate(snapshot_chain):
-        image_params = params.object_params(image)
-        if image_params.get("file_create"):
-            session.cmd(dir_create_cmd % file_dir)
-        if index > 0:
-            snapshot_file = storage.get_image_filename(image_params,
+    try:
+        for index, image in enumerate(snapshot_chain):
+            image_params = params.object_params(image)
+            if image_params.get("file_create"):
+                session.cmd(dir_create_cmd % file_dir)
+            if index > 0:
+                snapshot_file = storage.get_image_filename(image_params,
+                                                           data_dir.get_data_dir())
+                base_image = get_base_image(snapshot_chain, image)
+                base_image_params = params.object_params(base_image)
+                base_file = storage.get_image_filename(base_image_params,
                                                        data_dir.get_data_dir())
-            base_image = get_base_image(snapshot_chain, image)
-            base_image_params = params.object_params(base_image)
-            base_file = storage.get_image_filename(base_image_params,
-                                                   data_dir.get_data_dir())
-            snapshot_format = image_params.get("image_format")
-
-            error.context("Do pre snapshot operates", logging.info)
-            if image_params.get("pre_snapshot_cmd"):
-                do_operate(image_params, "pre_snapshot_cmd")
-
-            error.context("Do live snapshot ", logging.info)
-            vm.live_snapshot(base_file, snapshot_file, snapshot_format)
-
-            error.context("Do post snapshot operates", logging.info)
-            if image_params.get("post_snapshot_cmd"):
-                do_operate(image_params, "post_snapshot_cmd")
-            md5 = ""
+                snapshot_format = image_params.get("image_format")
+
+                error.context("Do pre snapshot operates", logging.info)
+                if image_params.get("pre_snapshot_cmd"):
+                    do_operate(image_params, "pre_snapshot_cmd")
+
+                error.context("Do live snapshot ", logging.info)
+                vm.live_snapshot(base_file, snapshot_file, snapshot_format)
+
+                error.context("Do post snapshot operates", logging.info)
+                if image_params.get("post_snapshot_cmd"):
+                    do_operate(image_params, "post_snapshot_cmd")
+                md5 = ""
+                if image_params.get("file_create"):
+                    session.cmd(file_create_cmd % image)
+                    md5 = session.cmd_output(md5_cmd % image)
+                md5_value[image] = md5_value[base_image].copy()
+                md5_value[image].update({image: md5})
+            elif index == 0:
+                md5 = ""
+                if params.get("file_create"):
+                    session.cmd(file_create_cmd % image)
+                    md5 = session.cmd_output(md5_cmd % image)
+                md5_value[image] = {image: md5}
+            if image_params.get("check_alive_cmd"):
+                session.cmd(image_params.get("check_alive_cmd"))
             if image_params.get("file_create"):
-                session.cmd(file_create_cmd % image)
-                md5 = session.cmd_output(md5_cmd % image)
-            md5_value[image] = md5_value[base_image].copy()
-            md5_value[image].update({image: md5})
-        elif index == 0:
-            md5 = ""
-            if params.get("file_create"):
-                session.cmd(file_create_cmd % image)
-                md5 = session.cmd_output(md5_cmd % image)
-            md5_value[image] = {image: md5}
-        if image_params.get("check_alive_cmd"):
-            session.cmd(image_params.get("check_alive_cmd"))
-        if image_params.get("file_create"):
-            files_check = session.cmd(file_check_cmd % file_dir)
-            files_in_guest[image] = files_check
-    session.close()
-
-    error.context("Reboot guest", logging.info)
-    if image_params.get("need_reboot", "no") == "yes":
-        vm.monitor.cmd("system_reset")
-        vm.verify_alive()
-
-    error.context("Do base files check", logging.info)
-    snapshot_chain_backward = snapshot_chain[:]
-    snapshot_chain_backward.reverse()
-    for index, image in enumerate(snapshot_chain_backward):
-        image_params = params.object_params(image)
-        if image_params.get("check_base_image"):
-            vm.destroy()
-            vm.create(params=image_params)
+                files_check = session.cmd(file_check_cmd % file_dir)
+                files_in_guest[image] = files_check
+        session.close()
+
+        error.context("Reboot guest", logging.info)
+        if image_params.get("need_reboot", "no") == "yes":
+            vm.monitor.cmd("system_reset")
             vm.verify_alive()
 
-            session = vm.wait_for_login(timeout=timeout)
-            if image_params.get("file_create"):
-                for file in md5_value[image]:
-                    md5 = session.cmd_output(md5_cmd % file)
-                    if md5 != md5_value[image][file]:
-                        error_message = "File %s in image %s changed " %\
-                                        (file, image)
-                        error_message += "from '%s' to '%s'(md5)" %\
-                                         (md5_value[image][file], md5)
-                        raise error.TestFail(error_message)
-                files_check = session.cmd(file_check_cmd % file_dir)
-                if files_check != files_in_guest[image]:
-                    error_message = "Files in image %s is not as expect:" %\
-                                    image
-                    error_message += "Before shut down: %s" %\
-                        files_in_guest[image]
-                    error_message += "Now: %s" % files_check
-                    raise error.TestFail(error_message)
-            if image_params.get("image_check"):
-                image = qemu_storage.QemuImg(
-                    image_params, data_dir.get_data_dir(), image)
-                image.check_image(image_params, data_dir.get_data_dir())
-            session.close()
-
-    error.context("Remove snapshot images", logging.info)
-    if vm.is_alive():
-        vm.destroy()
-    if params.get("remove_snapshot_images"):
-        for index, image in enumerate(snapshot_chain):
+        error.context("Do base files check", logging.info)
+        snapshot_chain_backward = snapshot_chain[:]
+        snapshot_chain_backward.reverse()
+
+        for index, image in enumerate(snapshot_chain_backward):
             image_params = params.object_params(image)
-            if index != 0:
-                image = qemu_storage.QemuImg(
-                    image_params, data_dir.get_data_dir(), image)
-                test.assertTrue(os.path.exists(image.image_filename),
-                                "Produced snapshot is not present in "
-                                "filesystem: %s." % image.image_filename)
-                image.remove()
+            if image_params.get("check_base_image"):
+                vm.destroy()
+                vm.create(params=image_params)
+                vm.verify_alive()
+
+                session = vm.wait_for_login(timeout=timeout)
+                if image_params.get("file_create"):
+                    for file in md5_value[image]:
+                        md5 = session.cmd_output(md5_cmd % file)
+                        if md5 != md5_value[image][file]:
+                            error_message = "File %s in image %s changed " %\
+                                            (file, image)
+                            error_message += "from '%s' to '%s'(md5)" %\
+                                             (md5_value[image][file], md5)
+                            raise error.TestFail(error_message)
+                    files_check = session.cmd(file_check_cmd % file_dir)
+                    if files_check != files_in_guest[image]:
+                        error_message = "Files in image %s is not as expect:" %\
+                                        image
+                        error_message += "Before shut down: %s" %\
+                            files_in_guest[image]
+                        error_message += "Now: %s" % files_check
+                        raise error.TestFail(error_message)
+                if image_params.get("image_check"):
+                    image = qemu_storage.QemuImg(
+                        image_params, data_dir.get_data_dir(), image)
+                    image.check_image(image_params, data_dir.get_data_dir())
+                session.close()
+
+        error.context("Remove snapshot images", logging.info)
+        if vm.is_alive():
+            vm.destroy()
+        errs = cleanup_images(snapshot_chain, params)
+        test.assertFalse(errs, "Errors occurred while removing images:\n%s"
+                         % "\n".join(errs))
+    except Exception as details:
+        error.context("Force-cleaning after exception: %s" % details,
+                      logging.error)
+        if vm.is_alive():
+            vm.destroy()
+        cleanup_images(snapshot_chain, params)
+        raise

qemu/tests/multi_disk.py

@@ -8,11 +8,13 @@ import re
 import random
 import string
 
+from avocado.utils import astring
+
 from autotest.client.shared import error
-from autotest.client.shared import utils
 
-from virttest import qemu_qtree
 from virttest import env_process
+from virttest import error_context
+from virttest import qemu_qtree
 from virttest import utils_misc
 
 _RE_RANGE1 = re.compile(r'range\([ ]*([-]?\d+|n).*\)')
@@ -20,7 +22,7 @@ _RE_RANGE2 = re.compile(r',[ ]*([-]?\d+|n)')
 _RE_BLANKS = re.compile(r'^([ ]*)')
 
 
-@error.context_aware
+@error_context.context_aware
 def _range(buf, n=None):
     """
     Converts 'range(..)' string to range. It supports 1-4 args. It supports
@@ -74,7 +76,7 @@ def _range(buf, n=None):
     return out
 
 
-@error.context_aware
+@error_context.context_aware
 def run(test, params, env):
     """
     Test multi disk suport of guest, this case will:
@@ -127,7 +129,7 @@ def run(test, params, env):
                 if o:
                     disk_indexs.append(o[0])
 
-    error.context("Parsing test configuration", logging.info)
+    error_context.context("Parsing test configuration", logging.info)
     stg_image_num = 0
     stg_params = params.get("stg_params", "")
     # Compatibility
@@ -209,17 +211,17 @@ def run(test, params, env):
     if params.get("multi_disk_params_only") == 'yes':
         # Only print the test param_matrix and finish
         logging.info('Newly added disks:\n%s',
-                     utils.matrix_to_string(param_table, param_table_header))
+                     astring.tabular_output(param_table, param_table_header))
         return
 
     # Always recreate VMs and disks
-    error.context("Start the guest with new disks", logging.info)
+    error_context.context("Start the guest with new disks", logging.info)
     for vm_name in params.objects("vms"):
         vm_params = params.object_params(vm_name)
         env_process.process_images(env_process.preprocess_image, test,
                                    vm_params)
 
-    error.context("Start the guest with those disks", logging.info)
+    error_context.context("Start the guest with those disks", logging.info)
     vm = env.get_vm(params["main_vm"])
     vm.create(timeout=max(10, stg_image_num), params=params)
     session = vm.wait_for_login(timeout=int(params.get("login_timeout", 360)))
@@ -238,7 +240,7 @@ def run(test, params, env):
         have_qtree = False
 
     if (params.get("check_guest_proc_scsi") == "yes") and have_qtree:
-        error.context("Verifying qtree vs. test params")
+        error_context.context("Verifying qtree vs. test params")
         err = 0
         qtree = qemu_qtree.QtreeContainer()
         qtree.parse_info_qtree(vm.monitor.info('qtree'))
@@ -263,7 +265,7 @@ def run(test, params, env):
             session.cmd_status_output(cmd)
 
         if params.get("os_type") == "windows":
-            error.context("Create partition on those disks", logging.info)
+            error_context.context("Create partition on those disks", logging.info)
             # Get the disk index
             _get_disk_index(session, stg_image_size, disk_indexs)
             if len(disk_indexs) < stg_image_num:
@@ -278,7 +280,7 @@ def run(test, params, env):
                 utils_misc.format_windows_disk(session, disk_indexs[i], None,
                                                None, fs_type)
 
-        error.context("Get disks dev filenames in guest", logging.info)
+        error_context.context("Get disks dev filenames in guest", logging.info)
         cmd = params["list_volume_command"]
         s, output = session.cmd_status_output(cmd, timeout=cmd_timeout)
         if s != 0:
@@ -315,27 +317,27 @@ def run(test, params, env):
     try:
         for i in range(n_repeat):
             logging.info("iterations: %s", (i + 1))
-            error.context("Format those disks in guest", logging.info)
+            error_context.context("Format those disks in guest", logging.info)
             for disk in disks:
                 disk = disk.strip()
-                error.context("Preparing disk: %s..." % disk)
+                error_context.context("Preparing disk: %s..." % disk)
 
                 # Random select one file system from file_system
                 index = random.randint(0, (len(file_system) - 1))
                 fs = file_system[index].strip()
                 cmd = params["format_command"] % (fs, disk)
-                error.context("formatting test disk")
+                error_context.context("formatting test disk")
                 session.cmd(cmd, timeout=cmd_timeout)
                 cmd = params.get("mount_command")
                 if cmd:
                     cmd = cmd % (disk, disk, disk)
                     session.cmd(cmd)
 
-            error.context("Cope file into / out of those disks", logging.info)
+            error_context.context("Cope file into / out of those disks", logging.info)
             for disk in disks:
                 disk = disk.strip()
 
-                error.context("Performing I/O on disk: %s..." % disk)
+                error_context.context("Performing I/O on disk: %s..." % disk)
                 cmd_list = params["cmd_list"].split()
                 for cmd_l in cmd_list:
                     cmd = params.get(cmd_l)
@@ -356,7 +358,7 @@ def run(test, params, env):
                 disks.sort()
                 for disk in disks:
                     disk = disk.strip()
-                    error.context("Unmounting disk: %s..." % disk)
+                    error_context.context("Unmounting disk: %s..." % disk)
                     cmd = params.get("umount_command") % (disk, disk)
                     session.cmd(cmd)
     finally:
@@ -367,7 +369,7 @@ def run(test, params, env):
                 disks = re.findall(re_str, output)
                 disks.sort()
                 for disk in disks:
-                    error.context("Unmounting disk: %s..." % disk)
+                    error_context.context("Unmounting disk: %s..." % disk)
                     cmd = params["umount_command"] % (disk, disk)
                     session.cmd(cmd)
             except Exception, err: