performance: add case "Virio with qemu vhost Back end in DPDK"

Signed-off-by: NWenli Quan <wquan@redhat.com>

qemu/deps/performance/start_testpmd.py

+import logging
+import sys
+import time
+import locale
+import pexpect
+
+from six import string_types
+
+
+nic1_driver = sys.argv[1]
+nic2_driver = sys.argv[2]
+nic1 = sys.argv[3]
+nic2 = sys.argv[4]
+cores = int(sys.argv[5])
+queues = int(sys.argv[6])
+running_time = int(sys.argv[7])
+
+ENCODING = locale.getpreferredencoding()
+
+
+class TestPMD(object):
+
+    def __init__(self):
+
+        self.proc = None
+        testpmd_cmd = "/usr/bin/testpmd "
+        self.testpmd_exec = testpmd_cmd
+
+    def launch(self, nic1_driver, nic2_driver, nic1, nic2, cores, queues):
+
+        cmd = ("-l 1,2,3 -n 4 -d %s -d %s"
+               " -w %s -w %s "
+               " -- "
+               " -i --nb-cores=%d "
+               " --disable-rss --rxd=512 --txd=512 "
+               " --rxq=%d --txq=%d" % (
+                   nic1_driver, nic2_driver, nic1, nic2, cores, queues, queues))
+        cmd_str = self.testpmd_exec + cmd
+        logging.info("[cmd] %s" % cmd_str)
+        try:
+            self.proc = pexpect.spawn(cmd_str)
+            self.proc.expect("testpmd>")
+        except pexpect.ExceptionPexpect as e:
+            logging.error(e)
+            return False
+
+    def start(self):
+        self.command("start")
+
+    def stop(self):
+        self.command("stop")
+
+    def quit(self):
+        self.proc.sendline("quit")
+        logging.info("testpmd> quit")
+        print("testpmd> quit")
+        self.proc.expect("Bye...")
+        logging.info(self.proc.before)
+        line_list = to_text(self.proc.before).split("\n")
+        for subline in line_list:
+            if len(subline.strip()) > 0:
+                print(subline)
+        return to_text(self.proc.before)
+
+    def set_port_stats(self):
+        self.command("show port stats all")
+
+    def set_portlist(self, portlist):
+        self.command("set portlist %s" % portlist)
+
+    def get_config_fwd(self):
+        self.command("show config fwd")
+
+    def set_fwd_mac_retry(self):
+        self.command('set fwd mac retry')
+
+    def set_vlan_0(self):
+        self.command('vlan set strip on 0')
+
+    def set_vlan_1(self):
+        self.command('vlan set strip on 1')
+
+    def command(self, cmd):
+        self.proc.sendline(cmd)
+        self.proc.expect("testpmd>")
+        logging.info("testpmd> %s" % cmd)
+        print("testpmd> %s" % cmd)
+        logging.info(self.proc.before)
+        line_list = to_text(self.proc.before).split("\n")
+        for subline in line_list:
+            if len(subline.strip()) > 0:
+                print(subline)
+
+        return to_text(self.proc.before)
+
+
+def start_testpmd(nic1_driver, nic2_driver, nic1, nic2, cores, queues):
+
+    my_testpmd = TestPMD()
+    my_testpmd.launch(
+        nic1_driver=nic1_driver,
+        nic2_driver=nic2_driver,
+        nic1=nic1,
+        nic2=nic2,
+        cores=cores,
+        queues=queues)
+
+    my_testpmd.set_fwd_mac_retry()
+    my_testpmd.set_vlan_0()
+    my_testpmd.set_vlan_1()
+
+    my_testpmd.start()
+    my_testpmd.set_port_stats()
+
+    # testmpd will quit after running_time
+    start_time = time.time()
+    end_time = start_time + running_time
+    while time.time() < end_time:
+        time.sleep(10)
+        print("time.time=%s" % time.time)
+    my_testpmd.stop()
+    my_testpmd.set_port_stats()
+    my_testpmd.quit()
+
+
+def to_text(data):
+    """
+    Convert anything to text decoded text
+
+    When the data is bytes, it's decoded. When it's not of string types
+    it's re-formatted into text and returned. Otherwise (it's string)
+    it's returned unchanged.
+
+    :param data: data to be transformed into text
+    :type data: either bytes or other data that will be returned
+                unchanged
+    """
+    if isinstance(data, bytes):
+        return data.decode(ENCODING)
+    elif not isinstance(data, string_types):
+        if sys.version_info[0] < 3:
+            return unicode(data)    # pylint: disable=E0602
+        else:
+            return str(data)
+    return data
+
+
+start_testpmd(nic1_driver, nic2_driver, nic1, nic2, cores, queues)

qemu/tests/cfg/virtio_net_dpdk.cfg

+- virtio_net_dpdk:
+    no JeOS
+    no Windows
+    no Host_RHEL.m5, Host_RHEL.m6
+    only bridge
+    virt_test_type = qemu
+    type = virtio_net_dpdk
+    kill_vm = yes
+    machine_type = q35
+    get_status_in_guest = no
+    auto_cpu_model = no
+    cpu_model = "host"
+    category = "rx tx"
+    login_timeout = 90
+    testpmd_running_time = 20
+    mac_changeable = yes
+    kvm_ver_chk_cmd = "rpm -qa qemu-kvm-rhev && rpm -qa qemu-kvm"
+    guest_ver_cmd = "uname -r"
+    guest_dpdk_cmd = "rpm -qa dpdk"
+    record_list = "pkt_size mpps"
+    pkt_size = 60
+    numa_node = 2
+    nics = 'nic1 nic2 nic3'
+    nic_mode = tap
+    netdst = switch
+    nic_model_nic1 = rtl8139
+    nic_model_nic2 = virtio
+    nic_model_nic3 = virtio
+    netdst_nic1 = switch
+    vhost_nic1 =
+    vhost_nic2 = vhost=on
+    nettype_nic2 = macvtap
+    netdst_nic2 = shell:ip link | sed -n '/90:e2:ba:7d:33:3d/{g;1!p;};h'|awk -F': ' '{print $2}'|head -1
+    nic_extra_params_nic2 =
+    pf_filter_re = 82599
+    device_name_nic3 = shell:ip link | sed -n '/90:e2:ba:7d:33:3c/{g;1!p;};h'|awk -F': ' '{print $2}'|head -1
+    vf_filter_re = "Virtual Function"
+    device_driver = vfio-pci
+    pci_assignable_nic3 = vf
+    pci_assignable_nic2 = no
+    pci_assignable_nic1 = no
+    pre_command = echo 8 > /sys/devices/system/node/node1/hugepages/hugepages-1048576kB/nr_hugepages
+    pre_command += ; modprobe vfio; modprobe vfio-pci
+    pre_command += ; echo 2 > /sys/class/net/enp6s0f0/device/sriov_numvfs
+    post_command = modprobe -r ixgbe; modprobe ixgbe
+    #intel-iommu device
+    extra_params = " -device intel-iommu,intremap=on,caching-mode=on,device-iotlb=on"
+    extra_params +=" -object memory-backend-file,id=mem,size=8G,mem-path=/dev/hugepages,share=on -numa node,memdev=mem -mem-prealloc "
+    machine_type_extra_params = "kernel-irqchip=split"
+
+    #set iommu_platform=on for all virtio pci device
+    virtio_dev_iommu_platform = on
+    #set ats=on for all virtio pci device
+    virtio_dev_ats = on
+
+    # Use 'intel_iommu=on, iommu=pt' by default, tester can modify it as needed
+    enable_guest_iommu = yes
+    guest_iommu_option = pt
+    kernel_extra_params_add = "default_hugepagesz=1G"
+
+    #generator host configuration
+    generator = 10.73.8.4
+    username_generator = root
+    password_generator = kvmautotest
+    shell_client_generator = ssh
+    shell_port_generator = 22
+    shell_prompt_generator =  \[root@.{0,50}][\#\$]
+    generator_hugepages_cmd = echo 10 >  /sys/devices/system/node/node0/hugepages/hugepages-1048576kB/nr_hugepages
+    generator_hugepages_cmd += ; echo 10 >  /sys/devices/system/node/node1/hugepages/hugepages-1048576kB/nr_hugepages
+    # VM configuration
+    package_install = tuned-profiles-cpu-partitioning dpdk
+    env_hugepages_cmd = echo 8 >  /sys/devices/system/node/node0/hugepages/hugepages-1048576kB/nr_hugepages
+    env_pkg = tuned-profiles-cpu-partitioning dpdk dpdk-devel dpdk-tools
+    nic_driver = Virtio 82599
+    nic1_dpdk_driver = /usr/lib64/librte_pmd_virtio.so
+    nic2_dpdk_driver = /usr/lib64/librte_pmd_ixgbe.so
+    moongen_pkg = MoonGen.zip
+    moongen_dpdk_nic = 0000:05:00.0 0000:05:00.1
+    testpmd_exec = start_testpmd.py
+    testpmd_queues = 1

qemu/tests/virtio_net_dpdk.py

+import os
+import logging
+import threading
+import six
+import time
+
+from avocado.utils import process
+
+from virttest import error_context
+from virttest import virt_vm
+from virttest import remote
+from virttest import data_dir
+from virttest import utils_misc
+from virttest import utils_test
+
+
+def format_result(result, base="12", fbase="2"):
+    """
+    Format the result to a fixed length string.
+
+    :param result: result need to convert
+    :param base: the length of converted string
+    :param fbase: the decimal digit for float
+    """
+    if isinstance(result, six.string_types):
+        value = "%" + base + "s"
+    elif isinstance(result, int):
+        value = "%" + base + "d"
+    elif isinstance(result, float):
+        value = "%" + base + "." + fbase + "f"
+    return value % result
+
+
+@error_context.context_aware
+def run(test, params, env):
+    """
+    Virtio with qemu vhost backend with dpdk
+
+    1) Boot up VM and reboot VM with 1G hugepages and iommu enabled
+    2) Install dpdk realted packages
+    3) Bind two nics to vfio-pci on VM
+    4) Install and start Moongen on external host
+    5) Start testpmd on VM, collect and analyze the results
+
+    :param test: QEMU test object.
+    :param params: Dictionary with the test parameters.
+    :param env: Dictionary with test environment.
+    """
+
+    def _pin_vm_threads(node):
+        """
+        pin guest vcpu and vhost threads to cpus of a numa node repectively
+
+        :param node: which numa node to pin
+        """
+        if node:
+            if not isinstance(node, utils_misc.NumaNode):
+                node = utils_misc.NumaNode(int(node))
+            utils_test.qemu.pin_vm_threads(vm, node)
+
+    def install_dpdk():
+        """ Install dpdk realted packages"""
+
+        cmd = 'yum install -y %s' % params.get("env_pkg")
+        session.cmd(cmd, timeout=360, ignore_all_errors=True)
+        session.cmd_output('rpm -qa |grep dpdk')
+
+    def env_setup():
+        """
+        Prepare the test environment
+        1) Set 1G hugepages and iommu enabled
+        2) Copy testpmd script to guest
+
+        """
+        error_context.context("Setup env for guest")
+
+        # setup hugepages
+        session.cmd(params.get("env_hugepages_cmd"), ignore_all_errors=True)
+
+        # install dpdk related packages
+        install_dpdk()
+
+        # install python pexpect
+        session.cmd("`command -v pip pip3` install pexpect", ignore_all_errors=True)
+
+        # copy testpmd script to guest
+        testpmd_exec = params.get("testpmd_exec")
+        src = os.path.join(data_dir.get_deps_dir(),
+                           "performance/%s" % testpmd_exec)
+        dst = "/tmp/%s" % testpmd_exec
+        vm.copy_files_to(src, dst, nic_index=0)
+
+        return dst
+
+    def dpdk_devbind():
+        """
+
+        bind two nics to vfio-pci
+        return nic1 and nic2's pci
+        """
+
+        error_context.context("bind two nics to vfio-pci")
+        cmd = "modprobe vfio"
+        cmd += " && modprobe vfio-pci"
+        session.cmd(cmd, timeout=360, ignore_all_errors=True)
+        session.cmd_output("lspci|grep Eth")
+        cmd_nic_pci = "lspci |awk '/%s/ {print $1}'" % params.get("nic_driver")
+        nic_driver = params.get("nic_driver").split()
+        if len(nic_driver) > 1:
+            for i in nic_driver:
+                if i == "Virtio":
+                    nic_pci_1 = "0000:%s" % session.cmd(
+                        "lspci |awk '/%s network/ {print $1}'" % i).strip()
+                    cmd_str = "dpdk-devbind --bind=vfio-pci %s" % nic_pci_1
+                else:
+                    nic_pci_2 = "0000:%s" % session.cmd(
+                        "lspci |awk '/%s/ {print $1}'" % i).strip()
+                    cmd_str = "dpdk-devbind --bind=vfio-pci %s" % nic_pci_2
+                session.cmd_output(cmd_str)
+        session.cmd_output('dpdk-devbind --status')
+        return nic_pci_1, nic_pci_2
+
+    def install_moongen(session, ip, user, port, password):
+        """
+
+        Install moogen on remote moongen host
+
+        """
+
+        # copy MoonGen.zip to remote moongen host
+        moongen_pkg = params.get("moongen_pkg")
+        local_path = os.path.join(
+            data_dir.get_deps_dir(), "performance/%s" % moongen_pkg)
+        remote.scp_to_remote(ip, shell_port, username,
+                             password, local_path, "/home")
+
+        # install moongen
+        cmd_str = "rm -rf /home/MoonGen"
+        cmd_str += " && unzip /home/%s -d /home" % params.get("moongen_pkg")
+        cmd_str += " && cd /home/MoonGen && ./build.sh"
+        if session.cmd_status(cmd_str, timeout=300) != 0:
+            test.error("Fail to install program on monngen host")
+
+        # set hugepages
+        session.cmd(params.get("generator_hugepages_cmd"), ignore_all_errors=True)
+
+        # probe vfio and vfip-pci
+        cmd_probe = "modprobe vfio; modprobe vfio-pci"
+        session.cmd_status(cmd_probe, timeout=300)
+
+        # bind nic
+        moongen_dpdk_nic = params.get("moongen_dpdk_nic").split()
+        for i in list(moongen_dpdk_nic):
+            cmd_bind = "dpdk-devbind --bind=vfio-pci %s" % i
+            if session.cmd_status(cmd_bind) != 0:
+                test.error("Fail to bind nic %s on monngen host" % i)
+
+    def result(recode, dst):
+
+        if os.path.getsize(dst) > 0:
+
+            cmd = "grep -i %s %s | tail -1 | awk  -F ':'  '{print $2}'"\
+                  "| awk '{print $1}'" % (recode, dst)
+            pps_results = process.system_output(cmd, shell=True)
+            power = 10**6
+            mpps_results = float(pps_results) / float(power)
+            pps_results = "%.2f" % mpps_results
+        else:
+            test.error("the content of /tmp/testpmd.log is empty")
+
+        return mpps_results
+
+    vm = env.get_vm(params["main_vm"])
+    vm.verify_alive()
+    login_timeout = int(params.get("login_timeout", 360))
+
+    try:
+        vm.wait_for_serial_login(
+            timeout=login_timeout, restart_network=True).close()
+    except virt_vm.VMIPAddressMissingError:
+        pass
+
+    # print numa information on host and pinning vhost and vcpus to cpus
+    process.system_output("numactl --hardware")
+    process.system_output("numactl --show")
+    _pin_vm_threads(params.get("numa_node"))
+    error_context.context("Prepare env of vm/generator host", logging.info)
+
+    session = vm.wait_for_login(nic_index=0, timeout=login_timeout)
+
+    guest_ip = vm.wait_for_get_address(0, timeout=90)
+    macvtap_mac = vm.get_mac_address(1)
+    vfio_mac = vm.get_mac_address(2)
+
+    # get parameter from dictionary
+    category = params.get("category")
+    pkt_size = params.get("pkt_size")
+    kvm_ver_chk_cmd = params.get("kvm_ver_chk_cmd")
+    guest_ver_cmd = params["guest_ver_cmd"]
+    guest_dpdk_cmd = params["guest_dpdk_cmd"]
+    record_list = params["record_list"]
+
+    # get record_list
+    record_line = ""
+    for record in record_list.split():
+        record_line += "%s|" % format_result(record)
+
+    # setup env and bind nics to vfio-pci in guest
+
+    exec_file = env_setup()
+    nic_pci_1, nic_pci_2 = dpdk_devbind()
+
+    # setup env on moongen host
+    generator_ip = params.get("generator")
+    shell_port = params.get("shell_port_generator")
+    password = params.get("password_generator")
+    username = params.get("username_generator")
+    generator1 = remote.wait_for_login(params.get("shell_client_generator"),
+                                       generator_ip,
+                                       shell_port,
+                                       username,
+                                       password,
+                                       params.get("shell_prompt_generator"))
+    generator2 = remote.wait_for_login(params.get("shell_client_generator"),
+                                       generator_ip,
+                                       shell_port,
+                                       username,
+                                       password,
+                                       params.get("shell_prompt_generator"))
+    install_moongen(generator1, generator_ip, username, shell_port, password)
+
+    # get qemu, guest kernel, kvm version and dpdk version and write them into result
+    result_path = utils_misc.get_path(test.resultsdir, "virtio_net_dpdk.RHS")
+    result_file = open(result_path, "w")
+    kvm_ver = process.system_output(kvm_ver_chk_cmd, shell=True).decode()
+    host_ver = os.uname()[2]
+    guest_ver = session.cmd_output(guest_ver_cmd)
+    dpdk_ver = session.cmd_output(guest_dpdk_cmd)
+    result_file.write("### kvm-userspace-ver : %s\n" % kvm_ver)
+    result_file.write("### kvm_version : %s\n" % host_ver)
+    result_file.write("### guest-kernel-ver :%s\n" % guest_ver)
+    result_file.write("### guest-dpdk-ver :%s\n" % dpdk_ver)
+
+    # get result tested by each scenario
+    for pkt_cate in category.split():
+        result_file.write("Category:%s\n" % pkt_cate)
+        result_file.write("%s\n" % record_line.rstrip("|"))
+        nic1_driver = params.get("nic1_dpdk_driver")
+        nic2_driver = params.get("nic2_dpdk_driver")
+        cores = params.get("vcpu_sockets")
+        queues = params.get("testpmd_queues")
+        running_time = int(params.get("testpmd_running_time"))
+        size = 60
+
+        if pkt_cate == "rx":
+            error_context.context("test guest rx pps performance",
+                                  logging.info)
+            port = 1
+            record = "Rx-pps"
+            mac = vm.get_mac_address(1)
+        if pkt_cate == "tx":
+            error_context.context("test guest tx pps performance",
+                                  logging.info)
+            port = 0
+            record = "Tx-pps"
+            mac = vm.get_mac_address(2)
+
+        status = launch_test(session, generator1, generator2,
+                             mac, port, exec_file,
+                             nic1_driver, nic2_driver,
+                             nic_pci_1, nic_pci_2,
+                             cores, queues, running_time)
+        if status is True:
+            error_context.context("%s test is finished" %
+                                  pkt_cate, logging.info)
+        else:
+            test.fail("test is failed, please check your command and env")
+
+        dst = utils_misc.get_path(test.resultsdir, "testpmd.%s" % pkt_cate)
+        vm.copy_files_from("/tmp/testpmd.log", dst)
+
+        pkt_cate_r = result("%s-pps" % pkt_cate, dst)
+        line = "%s|" % format_result(size)
+        line += "%s" % format_result(pkt_cate_r)
+        result_file.write(("%s\n" % line))
+
+    generator1.close()
+    generator2.close()
+    session.close()
+
+
+@error_context.context_aware
+def launch_test(session, generator1, generator2,
+                mac, port_id, exec_file,
+                nic1_driver, nic2_driver,
+                nic_pci_1, nic_pci_2,
+                cores, queues, running_time):
+    """ Launch MoonGen """
+
+    def start_moongen(generator1, mac, port_id, running_time):
+
+        file = '/home/MoonGen/examples/udp-throughput.lua'
+        cmd = "pkill MoonGen ; rm -rf /tmp/throughput.log ; sleep 3"
+        cmd += r" && \cp %s %s.tmp" % (file, file)
+        tmp_file = "%s.tmp" % file
+        cmd += " && sed -i 's/10:11:12:13:14:15/%s/g' %s" % (mac, tmp_file)
+        cmd += " && cd /home/MoonGen "\
+               " && ./build/MoonGen %s %s > /tmp/throughput.log &" % (
+                       tmp_file, port_id)
+        generator1.cmd_output(cmd)
+
+    def run_moongen_up(generator2):
+
+        cmd = 'grep "1 devices are up" /tmp/throughput.log'
+        if generator2.cmd_status(cmd) == 0:
+            return True
+        else:
+            return False
+
+    def start_testpmd(session, exec_file, nic1_driver, nic2_driver,
+                      nic1_pci_1, nic2_pci_2, cores, queues, running_time):
+        """ Start testpmd on VM """
+
+        cmd = "`command -v python python3` "
+        cmd += " %s %s %s %s %s %s %s %s > /tmp/testpmd.log" % (
+            exec_file, nic1_driver, nic2_driver,
+            nic_pci_1, nic_pci_2, cores, queues, running_time)
+        session.cmd_output(cmd)
+
+    moongen_thread = threading.Thread(
+        target=start_moongen, args=(generator1, mac, port_id, running_time))
+    moongen_thread.start()
+
+    if utils_misc.wait_for(lambda: run_moongen_up(generator2), 30,
+                           text="Wait until devices is up to work"):
+        logging.debug("MoonGen start to work")
+        testpmd_thread = threading.Thread(target=start_testpmd, args=(
+            session, exec_file, nic1_driver, nic2_driver,
+            nic_pci_1, nic_pci_2, cores, queues, running_time))
+        time.sleep(3)
+        testpmd_thread.start()
+        testpmd_thread.join()
+        moongen_thread.join()
+        return True
+    else:
+        return False