qemu.tests.nic_hotplug: add missed reboot step

1. add steps reboot vm after nic hotplug and hotunplug
2. check vm status after pause & resume
3. add debug info when raise VMAddressVerificationError and VMIPAddressMissingError
Signed-off-by: NXu Tian <xutian@redhat.com>

qemu/tests/nic_hotplug.py

 import logging
 import random
-from autotest.client.shared import error
+from autotest.client.shared import error, utils
 from virttest import utils_test, utils_net, virt_vm
 
 
@@ -31,25 +31,79 @@ def run(test, params, env):
     :param params: Dictionary with the test parameters.
     :param env:    Dictionary with test environment.
     """
+    def renew_ip_address(session, mac, is_linux_guest=True):
+        if not is_linux_guest:
+            utils_net.restart_windows_guest_network_by_key(session,
+                                                           "macaddress",
+                                                           mac)
+            return None
+        ifname = utils_net.get_linux_ifname(session, mac)
+        p_cfg = "/etc/sysconfig/network-scripts/ifcfg-%s" % ifname
+        cfg_con = "DEVICE=%s\nBOOTPROTO=dhcp\nONBOOT=yes" % ifname
+        make_conf = "test -f %s || echo '%s' > %s" % (p_cfg, cfg_con, p_cfg)
+        arp_clean = "arp -n|awk '/^[1-9]/{print \"arp -d \" $1}'|sh"
+        session.cmd_output_safe(make_conf)
+        session.cmd_output_safe("ifconfig %s up" % ifname)
+        session.cmd_output_safe("dhclient -r", timeout=240)
+        session.cmd_output_safe("dhclient %s" % ifname, timeout=240)
+        session.cmd_output_safe(arp_clean)
+        return None
+
+    def get_hotplug_nic_ip(vm, nic, session, is_linux_guest=True):
+        def __get_address():
+            try:
+                return vm.wait_for_get_address(nic["nic_name"], timeout=90)
+            except (virt_vm.VMIPAddressMissingError,
+                    virt_vm.VMAddressVerificationError):
+                renew_ip_address(session, nic["mac"], is_linux_guest)
+            return
+
+        nic_ip = utils.wait_for(__get_address, timeout=360)
+        if nic_ip:
+            return nic_ip
+        cached_ip = vm.address_cache.get(nic["mac"])
+        arps = utils.system_output("arp -aen")
+        logging.debug("Can't get IP address:")
+        logging.debug("\tCached IP: %s" % cached_ip)
+        logging.debug("\tARP table: %s" % arps)
+        return None
+
+    def ping_hotplug_nic(ip, mac, session, is_linux_guest):
+        status, output = utils_test.ping(ip, 10, timeout=30)
+        if status != 0:
+            if not is_linux_guest:
+                return status, output
+            ifname = utils_net.get_linux_ifname(session, mac)
+            add_route_cmd = "route add %s dev %s" % (ip, ifname)
+            del_route_cmd = "route del %s dev %s" % (ip, ifname)
+            logging.warn("Failed to ping %s from host.")
+            logging.info("Add route and try again")
+            session.cmd_output_safe(add_route_cmd)
+            status, output = utils_test.ping(hotnic_ip, 10, timeout=30)
+            logging.info("Del the route.")
+            status, output = session.cmd_output_safe(del_route_cmd)
+        return status, output
+
     login_timeout = int(params.get("login_timeout", 360))
     vm = env.get_vm(params["main_vm"])
     vm.verify_alive()
-    session = vm.wait_for_serial_login(timeout=login_timeout)
     primary_nic = [nic for nic in vm.virtnet]
-
-    run_dhclient = params.get("run_dhclient", "no")
     guest_is_linux = ("linux" == params.get("os_type", ""))
     host_ip_addr = utils_net.get_host_ip_address(params)
     if guest_is_linux:
         # Modprobe the module if specified in config file
         module = params.get("modprobe_module")
         if module:
-            session.cmd_output_safe("modprobe %s" % module)
-
+            s_session = vm.wait_for_serial_login(timeout=login_timeout)
+            s_session.cmd_output_safe("modprobe %s" % module)
+            s_session.close()
     nic_hotplug_count = int(params.get("nic_hotplug_count", 1))
     nic_hotplugged = []
     try:
         for nic_index in xrange(1, nic_hotplug_count + 1):
+            # need to reconnect serial port after
+            # guest reboot for windows guest
+            s_session = vm.wait_for_serial_login(timeout=login_timeout)
             nic_name = "hotplug_nic%s" % nic_index
             nic_params = params.object_params(nic_name)
             nic_model = nic_params["pci_model"]
@@ -59,66 +113,70 @@ def run(test, params, env):
             error.context("Disable other link(s) of guest", logging.info)
             disable_nic_list = primary_nic + nic_hotplugged
             for nic in disable_nic_list:
+                if guest_is_linux:
+                    ifname = utils_net.get_linux_ifname(s_session, nic["mac"])
+                    s_session.cmd_output_safe("ifconfig %s 0.0.0.0" % ifname)
+                else:
+                    s_session.cmd_output_safe("ipconfig /release all")
                 vm.set_link(nic.device_id, up=False)
 
             error.context("Hotplug %sth '%s' nic named '%s'" % (nic_index,
-                                                                nic_model, nic_name))
+                                                                nic_model,
+                                                                nic_name))
             hotplug_nic = vm.hotplug_nic(**nic_params)
-            # Only run dhclient if explicitly set and guest is not windows.
-            # Most modern Linux guests run NetworkManager,
-            #and thus do not need this.
-            s_session = vm.wait_for_serial_login(timeout=login_timeout)
-            if guest_is_linux:
-                if run_dhclient == "yes":
-                    s_session.cmd_output_safe("killall -9 dhclient")
-                    ifname = utils_net.get_linux_ifname(s_session,
-                                                        hotplug_nic['mac'])
-                    s_session.cmd_output_safe("ifconfig %s up" % ifname)
-                    s_session.cmd_output_safe("dhclient %s &" % ifname)
-                arp_clean = "arp -n|awk '/^[1-9]/{print \"arp -d \" $1}'|sh"
-                s_session.cmd_output_safe(arp_clean)
-
             error.context("Check if new interface gets ip address",
                           logging.info)
-            try:
-                hotnic_ip = vm.wait_for_get_address(nic_name)
-            except virt_vm.VMIPAddressMissingError, err:
-                if params.get("additional_operation"):
-                    p_cfg = "/etc/sysconfig/network-scripts/ifcfg-%s" % ifname
-                    cfg_con = "DEVICE=%s\nBOOTPROTO=dhcp\nONBOOT=yes" % ifname
-                    cmd = "echo '%s' > %s" % (cfg_con, p_cfg)
-                    s_session.cmd_output_safe(cmd)
-                    s_session.cmd_output_safe("killall -9 dhclient")
-                    s_session.cmd_output_safe("dhclient %s " % ifname)
-                    hotnic_ip = vm.wait_for_get_address(nic_name)
-                else:
-                    err_msg = "Could not get or verify nic ip address"
-                    err_msg += "error info: '%s' " % err
-                    raise error.TestFail(err_msg)
+            hotnic_ip = get_hotplug_nic_ip(
+                vm,
+                hotplug_nic,
+                s_session,
+                guest_is_linux)
+            if not hotnic_ip:
+                raise error.TestFail("Hotplug nic can get ip address")
             logging.info("Got the ip address of new nic: %s", hotnic_ip)
 
             error.context("Ping guest's new ip from host", logging.info)
-            if params.get("additional_operation"):
-                s_session.cmd_output_safe("route add %s dev %s" %
-                                          (host_ip_addr, ifname))
-            status, output = utils_test.ping(hotnic_ip, 10, timeout=30)
+            status, output = ping_hotplug_nic(host_ip_addr, hotplug_nic["mac"],
+                                              s_session, guest_is_linux)
             if status:
                 err_msg = "New nic failed ping test, error info: '%s'"
                 raise error.TestFail(err_msg % output)
 
-            error.context("Ping guest's new ip after pasue/resume",
-                          logging.info)
+            error.context("Reboot vm after hotplug nic", logging.info)
+            # reboot vm via serial port since some guest can't auto up
+            # hotplug nic and next step will check is hotplug nic works.
+            s_session = vm.reboot(session=s_session, serial=True)
+            vm.verify_alive()
+            hotnic_ip = get_hotplug_nic_ip(
+                vm,
+                hotplug_nic,
+                s_session,
+                guest_is_linux)
+            if not hotnic_ip:
+                raise error.TestFail(
+                    "Hotplug nic can't get ip after reboot vm")
+
+            error.context("Ping guest's new ip from host", logging.info)
+            status, output = ping_hotplug_nic(host_ip_addr, hotplug_nic["mac"],
+                                              s_session, guest_is_linux)
+            if status:
+                err_msg = "New nic failed ping test, error info: '%s'"
+                raise error.TestFail(err_msg % output)
+
+            error.context("Pause vm", logging.info)
             vm.monitor.cmd("stop")
+            vm.verify_status("paused")
+            error.context("Resume vm", logging.info)
             vm.monitor.cmd("cont")
-            status, output = utils_test.ping(hotnic_ip, 10, timeout=30)
+            vm.verify_status("running")
+            error.context("Ping guest's new ip after resume", logging.info)
+            status, output = ping_hotplug_nic(host_ip_addr, hotplug_nic["mac"],
+                                              s_session, guest_is_linux)
             if status:
                 err_msg = "New nic failed ping test after stop/cont, "
                 err_msg += "error info: '%s'" % output
                 raise error.TestFail(err_msg)
 
-            if params.get("additional_operation"):
-                s_session.cmd_output_safe("route del %s dev %s" %
-                                          (host_ip_addr, ifname))
             # random hotunplug nic
             nic_hotplugged.append(hotplug_nic)
             if random.randint(0, 1) and params.get("do_random_unhotplug"):
@@ -127,9 +185,18 @@ def run(test, params, env):
                 unplug_nic_index = random.randint(0, len(nic_hotplugged) - 1)
                 vm.hotunplug_nic(nic_hotplugged[unplug_nic_index].nic_name)
                 nic_hotplugged.pop(unplug_nic_index)
+                s_session = vm.reboot(session=s_session, serial=True)
+                vm.verify_alive()
+            s_session.close()
     finally:
         for nic in nic_hotplugged:
             vm.hotunplug_nic(nic.nic_name)
         error.context("Re-enabling the primary link(s)", logging.info)
         for nic in primary_nic:
             vm.set_link(nic.device_id, up=True)
+        error.context("Reboot vm to verify it alive after hotunplug nic(s)",
+                      logging.info)
+        serial = len(vm.virtnet) > 0 and False or True
+        session = vm.reboot(serial=serial)
+        vm.verify_alive()
+        session.close()