Merge pull request #2961 from xiaodwan/v2v_code_improvement

v2v: improve codes and add new -o json cases

provider/v2v_vmcheck_helper.py

 import os
 import re
-import logging
 import time
+import json
+import string
+import logging
 from distutils.version import LooseVersion  # pylint: disable=E0611
 
 from avocado.core import exceptions
@@ -765,3 +767,91 @@ class VMChecker(object):
         elif has_genid == 'no':
             if re.search(r'genid', self.vmxml):
                 self.log_err('Unexpected genid in xml')
+
+
+def check_local_output(params):
+    """
+    Check -o local result
+
+    Only do basic checking, '-o libvirt' already does
+    the whole checking process.
+    """
+    logging.info('checking local output')
+
+    os_directory = params.get('os_directory')
+    disk_count = int(params.get('vm_disk_count', 0))
+    vm_name = params.get('main_vm')
+
+    result = True
+    # Checking all disks
+    for i, c in enumerate(string.ascii_lowercase):
+        if i == disk_count:
+            break
+        disk_file_name = "%s-%s" % (vm_name, 'sd%s' % c)
+        disk_file = os.path.join(os_directory, disk_file_name)
+        if not os.path.isfile(disk_file):
+            logging.error('Not found %s' % disk_file)
+            result = False
+
+    # Check json file
+    xml_file = os.path.join(os_directory, '%s.xml' % vm_name)
+    if not os.path.isfile(xml_file):
+        logging.error('Not found %s' % xml_file)
+        result = False
+
+    return result
+
+
+def check_json_output(params):
+    """
+    Check -o json result
+    """
+    logging.info('checking json output')
+
+    os_directory = params.get('os_directory')
+    disk_count = int(params.get('vm_disk_count', 0))
+    vm_name = params.get('main_vm')
+    json_disk_pattern = params.get('json_disk_pattern')
+
+    result = True
+
+    json_disk_dict = {
+        'GuestName': vm_name,
+        'DiskDeviceName': '',
+        'DiskNo': 0}
+
+    if json_disk_pattern:
+        json_disk_pattern = json_disk_pattern.replace('%{', '{')
+        json_disk_pattern = re.sub(
+            r'%{(.*?)}', r'%%{{\g<1>}}', json_disk_pattern)
+
+    # Checking all disks
+    for i, c in enumerate(string.ascii_lowercase):
+        if i == disk_count:
+            break
+
+        json_disk_dict.update({'DiskDeviceName': 'sd%s' % c})
+        json_disk_dict.update({'DiskNo': '%d' % (i + 1)})
+
+        disk_file_name = "%s-%s" % (vm_name, 'sd%s' % c)
+        if json_disk_pattern:
+            disk_file_name = json_disk_pattern.format(**json_disk_dict)
+        disk_file = os.path.join(os_directory, disk_file_name)
+        if not os.path.isfile(disk_file):
+            logging.error('Not found %s' % disk_file)
+            result = False
+
+    # Check json file
+    json_file = os.path.join(os_directory, '%s.json' % vm_name)
+    if not os.path.isfile(json_file):
+        logging.error('Not found %s' % json_file)
+        result = False
+
+    # Check content of the json file
+    with open(json_file) as fp:
+        vm = json.load(fp)
+        if vm['name'] != vm_name and len(vm['disks']) != disk_count:
+            logging.error('Verify content failed in %s' % json_file)
+            result = False
+
+    return result

v2v/tests/cfg/convert_from_file.cfg

@@ -37,9 +37,7 @@
                         - rhv:
                             output_method = "rhev"
                 - local:
-                    only dest_none
-                    output_mode = local
-                    target = ${output_mode}
+                    only dest_local
     variants:
         - input_mode:
             variants:

v2v/tests/cfg/function_test_esx.cfg

@@ -36,6 +36,13 @@
                     src_uri_type = 'esx'
         - it_default:
     variants:
+        - json:
+            only dest_json
+            only uefi.win2019,device_map
+            variants:
+                - default:
+                - pattern:
+                    json_disk_pattern = '%%{GuestName}-%{GuestName}-%{DiskDeviceName}-%{DiskNo}'
         - libvirt:
             only dest_libvirt
             only uefi, GPO_AV, special_name, suse
@@ -233,6 +240,8 @@
             only esx_60
             main_vm = VM_NAME_EMPTY_CDROM_V2V_EXAMPLE
             checkpoint = empty_cdrom
+            skip_vm_check = yes
+            skip_reason = "No vm is created in this case"
         - option_root:
             only esx_67
             main_vm = VM_NAME_MULTIPLE_LINUX_V2V_EXAMPLE

v2v/tests/cfg/specific_kvm.cfg

@@ -32,9 +32,7 @@
         - output_mode:
             variants:
                 - local:
-                    only dest_none
-                    output_mode = 'local'
-                    target = ${output_mode}
+                    only dest_local
                 - rhev:
                     only dest_rhev.NFS
                     variants:

v2v/tests/cfg/v2v_options.cfg

@@ -26,8 +26,7 @@
                     only dest_libvirt
                     target = "libvirt"
                 - local:
-                    only dest_none
-                    output_mode = "local"
+                    only dest_local
                     output_storage = "/tmp"
                 - null:
                     only dest_null

v2v/tests/src/convert_from_file.py

@@ -15,6 +15,8 @@ from virttest import remote
 from virttest.utils_test import libvirt
 
 from provider.v2v_vmcheck_helper import VMChecker
+from provider.v2v_vmcheck_helper import check_json_output
+from provider.v2v_vmcheck_helper import check_local_output
 
 
 def run(test, params, env):
@@ -34,7 +36,7 @@ def run(test, params, env):
     input_file = params.get('input_file')
     output_mode = params.get('output_mode')
     output_format = params.get('output_format')
-    output_storage = params.get('output_storage', 'default')
+    os_pool = output_storage = params.get('output_storage', 'default')
     bridge = params.get('bridge')
     network = params.get('network')
     address_cache = env.get('address_cache')
@@ -122,9 +124,17 @@ def run(test, params, env):
         """
         Check virt-v2v command result
         """
-        libvirt.check_exit_status(result, status_error)
-        output = result.stdout_text + result.stderr_text
-        if not status_error:
+        def vm_check():
+            """
+            Checking the VM
+            """
+            if output_mode == 'json' and not check_json_output(params):
+                test.fail('check json output failed')
+            if output_mode == 'local' and not check_local_output(params):
+                test.fail('check local output failed')
+            if output_mode in ['null', 'json', 'local']:
+                return
+
             # Create vmchecker before virsh.start so that the vm can be undefined
             # if started failed.
             vmchecker = VMChecker(test, params, env)
@@ -148,6 +158,11 @@ def run(test, params, env):
                     check_BSOD()
                 # Merge 2 error lists
                 error_list.extend(vmchecker.errors)
+
+        libvirt.check_exit_status(result, status_error)
+        output = result.stdout_text + result.stderr_text
+        if not status_error:
+            vm_check()
         log_check = utils_v2v.check_log(params, output)
         if log_check:
             log_fail(log_check)
@@ -159,7 +174,7 @@ def run(test, params, env):
     try:
         v2v_params = {
             'main_vm': vm_name, 'target': target, 'v2v_opts': v2v_opts,
-            'storage': output_storage, 'network': network, 'bridge': bridge,
+            'os_storage': output_storage, 'network': network, 'bridge': bridge,
             'input_mode': input_mode, 'input_file': input_file,
             'new_name': 'ova_vm_' + utils_misc.generate_random_string(3),
             'datastore': datastore,
@@ -168,7 +183,8 @@ def run(test, params, env):
             'input_transport': input_transport,
             'vmx_nfs_src': vmx_nfs_src,
             'output_method': output_method,
-            'storage_name': storage_name,
+            'os_storage_name': storage_name,
+            'os_pool': os_pool,
             'rhv_upload_opts': rhv_upload_opts,
             'params': params
         }
@@ -190,7 +206,7 @@ def run(test, params, env):
             else:
                 logging.debug('%s already exists, Skip copying' % dest_dir)
         if output_format:
-            v2v_params.update({'output_format': output_format})
+            v2v_params.update({'of_format': output_format})
         # Create libvirt dir pool
         if output_mode == 'libvirt':
             pvt.pre_pool(pool_name, pool_type, pool_target, '')
@@ -205,7 +221,7 @@ def run(test, params, env):
             v2v_sasl.server_pwd = params.get('remote_pwd')
             v2v_sasl.setup(remote=True)
         if output_mode == 'local':
-            v2v_params['storage'] = data_dir.get_tmp_dir()
+            v2v_params['os_directory'] = data_dir.get_tmp_dir()
 
         if checkpoint == 'ova_relative_path':
             logging.debug('Current dir: %s', os.getcwd())

v2v/tests/src/convert_vm_to_libvirt.py

@@ -116,7 +116,7 @@ def run(test, params, env):
     v2v_params = {'target': target, 'hypervisor': hypervisor,
                   'main_vm': vm_name, 'input_mode': input_mode,
                   'network': network, 'bridge': bridge,
-                  'storage': pool_name, 'hostname': source_ip,
+                  'os_pool': pool_name, 'hostname': source_ip,
                   'password': source_pwd,
                   'input_transport': input_transport, 'vcenter_host': vpx_ip,
                   'vcenter_password': vpx_pwd,

v2v/tests/src/convert_vm_to_ovirt.py

@@ -30,7 +30,7 @@ def run(test, params, env):
     # nfs mount source
     vddk_libdir_src = params.get('vddk_libdir_src')
     vddk_thumbprint = params.get('vddk_thumbprint')
-    storage = params.get('storage')
+    os_pool = storage = params.get('storage')
     storage_name = params.get('storage_name')
     network = params.get('network')
     bridge = params.get('bridge')
@@ -143,11 +143,11 @@ def run(test, params, env):
     v2v_params = {'target': target, 'hypervisor': hypervisor,
                   'main_vm': vm_name, 'input_mode': input_mode,
                   'network': network, 'bridge': bridge,
-                  'storage': storage, 'hostname': source_ip,
+                  'os_storage': storage, 'hostname': source_ip,
                   # For virsh connection
                   'password': source_pwd,
                   'new_name': vm_name + utils_misc.generate_random_string(3),
-                  'output_method': output_method, 'storage_name': storage_name,
+                  'output_method': output_method, 'os_storage_name': storage_name,
                   'input_transport': input_transport, 'vcenter_host': vpx_ip,
                   'vcenter_password': vpx_pwd,
                   'vddk_thumbprint': vddk_thumbprint,
@@ -166,7 +166,7 @@ def run(test, params, env):
     output_format = params.get('output_format')
     # output_format will be set to 'raw' in utils_v2v.v2v_cmd if it's None
     if output_format:
-        v2v_params.update({'output_format': output_format})
+        v2v_params.update({'of_format': output_format})
 
     # Set libguestfs environment variable
     if hypervisor in ('xen', 'kvm'):

v2v/tests/src/function_test_esx.py

@@ -15,6 +15,8 @@ from avocado.utils import download
 from aexpect.exceptions import ShellProcessTerminatedError
 
 from provider.v2v_vmcheck_helper import VMChecker
+from provider.v2v_vmcheck_helper import check_json_output
+from provider.v2v_vmcheck_helper import check_local_output
 
 
 def run(test, params, env):
@@ -58,10 +60,12 @@ def run(test, params, env):
     vddk_thumbprint = params.get('vddk_thumbprint')
     src_uri_type = params.get('src_uri_type')
     esxi_password = params.get('esxi_password')
+    json_disk_pattern = params.get('json_disk_pattern')
     # For construct rhv-upload option in v2v cmd
     output_method = params.get("output_method")
     rhv_upload_opts = params.get("rhv_upload_opts")
     storage_name = params.get('storage_name')
+    os_pool = os_storage = params.get('output_storage', 'default')
     # for get ca.crt file from ovirt engine
     rhv_passwd = params.get("rhv_upload_passwd")
     rhv_passwd_file = params.get("rhv_upload_passwd_file")
@@ -412,12 +416,20 @@ def run(test, params, env):
         """
         Check virt-v2v command result
         """
-        libvirt.check_exit_status(result, status_error)
-        output = result.stdout_text + result.stderr_text
-        if checkpoint == 'empty_cdrom':
+        def vm_check(status_error):
+            """
+            Checking the VM
+            """
             if status_error:
-                log_fail('Virsh dumpxml failed for empty cdrom image')
-        elif not status_error:
+                return
+
+            if output_mode == 'json' and not check_json_output(params):
+                test.fail('check json output failed')
+            if output_mode == 'local' and not check_local_output(params):
+                test.fail('check local output failed')
+            if output_mode in ['null', 'json', 'local']:
+                return
+
             vmchecker = VMChecker(test, params, env)
             params['vmchecker'] = vmchecker
             if output_mode == 'rhev':
@@ -428,25 +440,20 @@ def run(test, params, env):
                 virsh.start(vm_name, debug=True)
             # Check guest following the checkpoint document after convertion
             logging.info('Checking common checkpoints for v2v')
-            if skip_vm_check != 'yes':
-                if checkpoint == 'ogac':
-                    # windows guests will reboot at any time after qemu-ga is
-                    # installed. The process cannot be controled. In order to
-                    # don't break vmchecker.run() process, It's better to put
-                    # check_windows_ogac before vmchecker.run(). Because in
-                    # check_windows_ogac, it waits until rebooting completes.
-                    vmchecker.checker.create_session()
-                    if os_type == 'windows':
-                        check_windows_ogac(vmchecker.checker)
-                    else:
-                        check_linux_ogac(vmchecker.checker)
-                ret = vmchecker.run()
-                if len(ret) == 0:
-                    logging.info("All common checkpoints passed")
-            else:
-                logging.info(
-                    'Skip checking vm after conversion: %s' %
-                    skip_reason)
+            if checkpoint == 'ogac':
+                # windows guests will reboot at any time after qemu-ga is
+                # installed. The process cannot be controled. In order to
+                # don't break vmchecker.run() process, It's better to put
+                # check_windows_ogac before vmchecker.run(). Because in
+                # check_windows_ogac, it waits until rebooting completes.
+                vmchecker.checker.create_session()
+                if os_type == 'windows':
+                    check_windows_ogac(vmchecker.checker)
+                else:
+                    check_linux_ogac(vmchecker.checker)
+            ret = vmchecker.run()
+            if len(ret) == 0:
+                logging.info("All common checkpoints passed")
             # Check specific checkpoints
             if checkpoint == 'cdrom':
                 virsh_session = utils_sasl.VirshSessionSASL(params)
@@ -472,6 +479,16 @@ def run(test, params, env):
                     log_fail("Bridge virbr0 already started during conversion")
             # Merge 2 error lists
             error_list.extend(vmchecker.errors)
+
+        libvirt.check_exit_status(result, status_error)
+        output = result.stdout_text + result.stderr_text
+        if skip_vm_check != 'yes':
+            vm_check(status_error)
+        else:
+            logging.info(
+                'Skip checking vm after conversion: %s' %
+                skip_reason)
+
         log_check = utils_v2v.check_log(params, output)
         if log_check:
             log_fail(log_check)
@@ -489,7 +506,8 @@ def run(test, params, env):
             'vpx_dc': vpx_dc, 'esx_ip': esx_ip,
             'new_name': vm_name + utils_misc.generate_random_string(4),
             'v2v_opts': v2v_opts, 'input_mode': 'libvirt',
-            'storage': params.get('output_storage', 'default'),
+            'os_storage': os_storage,
+            'os_pool': os_pool,
             'network': params.get('network'),
             'bridge': params.get('bridge'),
             'target': params.get('target'),
@@ -504,8 +522,9 @@ def run(test, params, env):
             'esxi_password': esxi_password,
             'esxi_host': esxi_host,
             'output_method': output_method,
-            'storage_name': storage_name,
+            'os_storage_name': storage_name,
             'rhv_upload_opts': rhv_upload_opts,
+            'oo_json_disk_pattern': json_disk_pattern,
             'params': params
         }
 
@@ -523,7 +542,7 @@ def run(test, params, env):
         v2v_params['v2v_opts'] += " -ip %s" % vpx_passwd_file
 
         if params.get('output_format'):
-            v2v_params.update({'output_format': params['output_format']})
+            v2v_params.update({'of_format': params['output_format']})
         # Rename guest with special name while converting to rhev
         if '#' in vm_name and output_mode == 'rhev':
             v2v_params['new_name'] = v2v_params['new_name'].replace('#', '_')

v2v/tests/src/function_test_xen.py

@@ -54,6 +54,7 @@ def run(test, params, env):
     output_method = params.get("output_method")
     rhv_upload_opts = params.get("rhv_upload_opts")
     storage_name = params.get('storage_name')
+    os_pool = os_storage = params.get('output_storage', 'default')
     # for get ca.crt file from ovirt engine
     rhv_passwd = params.get("rhv_upload_passwd")
     rhv_passwd_file = params.get("rhv_upload_passwd_file")
@@ -230,12 +231,13 @@ def run(test, params, env):
             'v2v_opts': v2v_opts, 'input_mode': 'libvirt',
             'new_name': new_vm_name,
             'password': xen_host_passwd,
-            'storage': params.get('output_storage', 'default'),
+            'os_pool': os_pool,
+            'os_storage': os_storage,
+            'os_storage_name': storage_name,
             'network': params.get('network'),
             'bridge': params.get('bridge'),
             'target': params.get('target'),
             'output_method': output_method,
-            'storage_name': storage_name,
             'rhv_upload_opts': rhv_upload_opts,
             'params': params
         }
@@ -257,12 +259,12 @@ def run(test, params, env):
         utils_misc.add_identities_into_ssh_agent()
 
         if params.get('output_format'):
-            v2v_params.update({'output_format': params.get('output_format')})
+            v2v_params.update({'of_format': params.get('output_format')})
 
         # Build rhev related options
         if output_mode == 'rhev':
             # To RHV doesn't support 'qcow2' right now
-            v2v_params['output_format'] = 'raw'
+            v2v_params['of_format'] = 'raw'
             # create different sasl_user name for different job
             params.update({'sasl_user': params.get("sasl_user") +
                            utils_misc.generate_random_string(3)})
@@ -331,7 +333,7 @@ def run(test, params, env):
         if checkpoint == 'pool_uuid':
             virsh.pool_start(pool_name)
             pooluuid = virsh.pool_uuid(pool_name).stdout.strip()
-            v2v_params['storage'] = pooluuid
+            v2v_params['os_pool'] = pooluuid
         if checkpoint.startswith('vnc'):
             vm_xml.VMXML.set_graphics_attr(vm_name, {'type': 'vnc'},
                                            virsh_instance=virsh_instance)

v2v/tests/src/specific_kvm.py

@@ -40,7 +40,7 @@ def run(test, params, env):
     output_mode = params.get('output_mode')
     output_format = params.get('output_format')
     source_user = params.get("username", "root")
-    storage = params.get('output_storage')
+    os_pool = storage = params.get('output_storage')
     bridge = params.get('bridge')
     network = params.get('network')
     ntp_server = params.get('ntp_server')
@@ -665,13 +665,14 @@ def run(test, params, env):
             'input_mode': input_mode,
             'network': network,
             'bridge': bridge,
-            'storage': storage,
+            'os_storage': storage,
+            'os_pool': os_pool,
             'hostname': source_ip,
             'password': source_pwd,
             'v2v_opts': v2v_opts,
             'new_name': vm_name + utils_misc.generate_random_string(3),
             'output_method': output_method,
-            'storage_name': storage_name,
+            'os_storage_name': storage_name,
             'rhv_upload_opts': rhv_upload_opts,
             'input_transport': input_transport,
             'vcenter_host': source_ip,
@@ -687,7 +688,7 @@ def run(test, params, env):
             v2v_params.update({"esx_ip": esx_ip})
         output_format = params.get('output_format')
         if output_format:
-            v2v_params.update({'output_format': output_format})
+            v2v_params.update({'of_format': output_format})
         # Build rhev related options
         if output_mode == 'rhev':
             # Create different sasl_user name for different job
@@ -714,7 +715,7 @@ def run(test, params, env):
                                        ovirt_ca_file_path,
                                        local_ca_file_path)
         if output_mode == 'local':
-            v2v_params['storage'] = data_dir.get_tmp_dir()
+            v2v_params['os_directory'] = data_dir.get_tmp_dir()
         if output_mode == 'libvirt':
             pvt.pre_pool(pool_name, pool_type, pool_target, '')
         # Set libguestfs environment variable