Merge branch 'ci_report' into 'master'

add ci report

See merge request !412

tools/bazel_adb_run.py

@@ -96,7 +96,7 @@ def main(unused_args):
     target_socs = None
     if FLAGS.target_socs != "all" and FLAGS.target_socs != "random":
         target_socs = set(FLAGS.target_socs.split(','))
-    target_devices = sh_commands.adb_devices(target_socs=target_socs)
+    target_devices = sh_commands.get_target_socs_serialnos(target_socs)
     if FLAGS.target_socs == "random":
         unlocked_devices = \
             [d for d in target_devices if not sh_commands.is_device_locked(d)]

tools/mace_tools.py

@@ -88,59 +88,69 @@ def get_hexagon_mode(configs):
     return False
 
 
-def gen_opencl_and_tuning_code(target_soc,
-                               target_abi,
+def gen_opencl_and_tuning_code(target_abi,
+                               serialno,
                                model_output_dirs,
                                pull_or_not):
     if pull_or_not:
-        sh_commands.pull_binaries(
-                target_soc, target_abi, model_output_dirs)
+        sh_commands.pull_binaries(target_abi, serialno, model_output_dirs)
 
     codegen_path = "mace/codegen"
 
     # generate opencl binary code
-    sh_commands.gen_opencl_binary_code(target_soc, model_output_dirs)
+    sh_commands.gen_opencl_binary_code(model_output_dirs)
 
-    sh_commands.gen_tuning_param_code(
-            target_soc, model_output_dirs)
+    sh_commands.gen_tuning_param_code(model_output_dirs)
 
 
 def model_benchmark_stdout_processor(stdout,
-                                     target_soc,
                                      abi,
+                                     serialno,
                                      model_name,
-                                     runtime,
-                                     running_round,
-                                     tuning):
-    metrics = {}
-    for line in stdout.split("\n"):
-        if "Aborted" in line:
-            raise Exception("Command failed")
+                                     runtime):
+    metrics = [0] * 5
+    for line in stdout.split('\n'):
         line = line.strip()
         parts = line.split()
         if len(parts) == 6 and parts[0].startswith("time"):
-            metrics["%s.create_net_ms" % model_name] = str(float(parts[1]))
-            metrics["%s.mace_engine_ctor_ms" % model_name] = str(
-                float(parts[2]))
-            metrics["%s.init_ms" % model_name] = str(float(parts[3]))
-            metrics["%s.warmup_ms" % model_name] = str(float(parts[4]))
-            if float(parts[5]) > 0:
-                metrics["%s.avg_latency_ms" % model_name] = str(
-                    float(parts[5]))
-    tags = {
-        "ro.board.platform": target_soc,
-        "abi": abi,
-        "runtime": runtime,
-        "round": running_round,  # TODO(yejianwu) change this to source/binary
-        "tuning": tuning
-    }
-    sh_commands.falcon_push_metrics(
-        metrics, endpoint="mace_model_benchmark", tags=tags)
+            metrics[0] = str(float(parts[1]))
+            metrics[1] = str(float(parts[2]))
+            metrics[2] = str(float(parts[3]))
+            metrics[3] = str(float(parts[4]))
+            metrics[4] = str(float(parts[5]))
+            break
+
+    props = sh_commands.adb_getprop_by_serialno(serialno)
+    device_type = props.get("ro.product.model", "")
+    target_soc = props.get("ro.board.platform", "")
+
+    report_filename = "build/report.csv"
+    if not os.path.exists(report_filename):
+        with open(report_filename, 'w') as f:
+            f.write("model_name,device_type,soc,abi,runtime,create_net,"
+                    "engine_ctor,init,warmup,run_avg\n")
+
+    data_str = "{model_name},{device_type},{soc},{abi},{runtime}," \
+               "{create_net},{engine_ctor},{init},{warmup},{run_avg}\n" \
+        .format(
+            model_name=model_name,
+            device_type=device_type,
+            soc=target_soc,
+            abi=abi,
+            runtime=runtime,
+            create_net=metrics[0],
+            engine_ctor=metrics[1],
+            init=metrics[2],
+            warmup=metrics[3],
+            run_avg=metrics[4]
+        )
+    with open(report_filename, 'a') as f:
+        f.write(data_str)
 
 
 def tuning_run(runtime,
-               target_soc,
                target_abi,
+               serialno,
                vlog_level,
                embed_model_data,
                model_output_dir,
@@ -158,8 +168,8 @@ def tuning_run(runtime,
                limit_opencl_kernel_time=0,
                option_args=""):
     stdout = sh_commands.tuning_run(
-            target_soc,
             target_abi,
+            serialno,
             vlog_level,
             embed_model_data,
             model_output_dir,
@@ -176,21 +186,19 @@ def tuning_run(runtime,
             out_of_range_check,
             phone_data_dir,
             option_args)
-    model_benchmark_stdout_processor(stdout,
-                                     target_soc,
-                                     target_abi,
-                                     model_name,
-                                     runtime,
-                                     running_round,
-                                     tuning)
 
+    if running_round > 0 and FLAGS.collect_report:
+        model_benchmark_stdout_processor(
+            stdout, target_abi, serialno, model_name, runtime)
 
-def build_mace_run_prod(hexagon_mode, runtime, target_soc, target_abi,
-                        vlog_level, embed_model_data, model_output_dir,
-                        input_nodes, output_nodes, input_shapes, output_shapes,
-                        model_name, device_type, running_round, restart_round,
-                        tuning, limit_opencl_kernel_time, phone_data_dir):
-    gen_opencl_and_tuning_code(target_soc, target_abi, [], False)
+
+def build_mace_run_prod(hexagon_mode, runtime, target_abi,
+                        serialno, vlog_level, embed_model_data,
+                        model_output_dir, input_nodes, output_nodes,
+                        input_shapes, output_shapes, model_name, device_type,
+                        running_round, restart_round, tuning,
+                        limit_opencl_kernel_time, phone_data_dir):
+    gen_opencl_and_tuning_code(target_abi, serialno, [], False)
     production_or_not = False
     mace_run_target = "//mace/tools/validation:mace_run"
     sh_commands.bazel_build(
@@ -202,21 +210,20 @@ def build_mace_run_prod(hexagon_mode, runtime, target_soc, target_abi,
     sh_commands.update_mace_run_lib(model_output_dir, target_abi, model_name,
                                     embed_model_data)
 
-    tuning_run(runtime, target_soc, target_abi, vlog_level, embed_model_data,
-               model_output_dir, input_nodes, output_nodes, input_shapes,
-               output_shapes, model_name, device_type, running_round=0,
-               restart_round=1, out_of_range_check=True,
+    tuning_run(runtime, target_abi, serialno, vlog_level,
+               embed_model_data, model_output_dir, input_nodes, output_nodes,
+               input_shapes, output_shapes, model_name, device_type,
+               running_round=0, restart_round=1, out_of_range_check=True,
                phone_data_dir=phone_data_dir, tuning=False)
 
-    tuning_run(runtime, target_soc, target_abi, vlog_level, embed_model_data,
-               model_output_dir, input_nodes, output_nodes, input_shapes,
-               output_shapes, model_name, device_type, running_round=0,
-               restart_round=1, out_of_range_check=False,
+    tuning_run(runtime, target_abi, serialno, vlog_level,
+               embed_model_data, model_output_dir, input_nodes, output_nodes,
+               input_shapes, output_shapes, model_name, device_type,
+               running_round=0, restart_round=1, out_of_range_check=False,
                phone_data_dir=phone_data_dir, tuning=tuning,
                limit_opencl_kernel_time=limit_opencl_kernel_time)
 
-    gen_opencl_and_tuning_code(
-            target_soc, target_abi, [model_output_dir], True)
+    gen_opencl_and_tuning_code(target_abi, serialno, [model_output_dir], True)
     production_or_not = True
     sh_commands.bazel_build(
             mace_run_target,
@@ -230,13 +237,14 @@ def build_mace_run_prod(hexagon_mode, runtime, target_soc, target_abi,
 
 def merge_libs_and_tuning_results(target_soc,
                                   target_abi,
+                                  serialno,
                                   project_name,
                                   output_dir,
                                   model_output_dirs,
                                   hexagon_mode,
                                   embed_model_data):
     gen_opencl_and_tuning_code(
-            target_soc, target_abi, model_output_dirs, False)
+            target_abi, serialno, model_output_dirs, False)
     sh_commands.build_production_code(target_abi)
 
     sh_commands.merge_libs(target_soc,
@@ -322,11 +330,17 @@ def parse_args():
         type="bool",
         default="false",
         help="Enable out of range check for opencl.")
+    parser.add_argument(
+        "--collect_report",
+        type="bool",
+        default="false",
+        help="Collect report.")
     return parser.parse_known_args()
 
 
 def process_models(project_name, configs, embed_model_data, vlog_level,
-                   target_soc, target_abi, phone_data_dir, option_args):
+                   target_soc, target_abi, serialno, phone_data_dir,
+                   option_args):
     hexagon_mode = get_hexagon_mode(configs)
     model_output_dirs = []
     for model_name in configs["models"]:
@@ -344,9 +358,11 @@ def process_models(project_name, configs, embed_model_data, vlog_level,
 
         # Create model build directory
         model_path_digest = md5sum(model_config["model_file_path"])
-        model_output_dir = "%s/%s/%s/%s/%s/%s/%s" % (
+        device_name = sh_commands.adb_get_device_name_by_serialno(serialno)
+        model_output_dir = "%s/%s/%s/%s/%s/%s_%s/%s" % (
             FLAGS.output_dir, project_name, "build",
-            model_name, model_path_digest, target_soc, target_abi)
+            model_name, model_path_digest, device_name.replace(' ', ''),
+            target_soc, target_abi)
         model_output_dirs.append(model_output_dir)
 
         if FLAGS.mode == "build" or FLAGS.mode == "all":
@@ -354,7 +370,7 @@ def process_models(project_name, configs, embed_model_data, vlog_level,
                 sh.rm("-rf", model_output_dir)
             os.makedirs(model_output_dir)
             sh_commands.clear_mace_run_data(
-                    target_abi, target_soc, phone_data_dir)
+                    target_abi, serialno, phone_data_dir)
 
         model_file_path, weight_file_path = get_model_files(
                 model_config["model_file_path"],
@@ -389,8 +405,8 @@ def process_models(project_name, configs, embed_model_data, vlog_level,
                     model_config["obfuscate"])
             build_mace_run_prod(hexagon_mode,
                                 model_config["runtime"],
-                                target_soc,
                                 target_abi,
+                                serialno,
                                 vlog_level,
                                 embed_model_data,
                                 model_output_dir,
@@ -409,8 +425,8 @@ def process_models(project_name, configs, embed_model_data, vlog_level,
         if FLAGS.mode == "run" or FLAGS.mode == "validate" or \
                 FLAGS.mode == "all":
             tuning_run(model_config["runtime"],
-                       target_soc,
                        target_abi,
+                       serialno,
                        vlog_level,
                        embed_model_data,
                        model_output_dir,
@@ -426,8 +442,8 @@ def process_models(project_name, configs, embed_model_data, vlog_level,
                        phone_data_dir)
 
         if FLAGS.mode == "benchmark":
-            sh_commands.benchmark_model(target_soc,
-                                        target_abi,
+            sh_commands.benchmark_model(target_abi,
+                                        serialno,
                                         vlog_level,
                                         embed_model_data,
                                         model_output_dir,
@@ -442,8 +458,8 @@ def process_models(project_name, configs, embed_model_data, vlog_level,
                                         option_args)
 
         if FLAGS.mode == "validate" or FLAGS.mode == "all":
-            sh_commands.validate_model(target_soc,
-                                       target_abi,
+            sh_commands.validate_model(target_abi,
+                                       serialno,
                                        model_file_path,
                                        weight_file_path,
                                        model_config["platform"],
@@ -460,6 +476,7 @@ def process_models(project_name, configs, embed_model_data, vlog_level,
         merge_libs_and_tuning_results(
             target_soc,
             target_abi,
+            serialno,
             project_name,
             FLAGS.output_dir,
             model_output_dirs,
@@ -487,8 +504,8 @@ def process_models(project_name, configs, embed_model_data, vlog_level,
         for model_name in configs["models"]:
             runtime = configs["models"][model_name]["runtime"]
             model_tag_dict[runtime] = model_name
-        sh_commands.build_run_throughput_test(target_soc,
-                                              target_abi,
+        sh_commands.build_run_throughput_test(target_abi,
+                                              serialno,
                                               vlog_level,
                                               FLAGS.run_seconds,
                                               merged_lib_file,
@@ -534,18 +551,26 @@ def main(unused_args):
     phone_data_dir = "/data/local/tmp/mace_run/"
     for target_soc in target_socs:
         for target_abi in configs["target_abis"]:
-            serialno = sh_commands.adb_devices([target_soc]).pop()
-            props = sh_commands.adb_getprop_by_serialno(serialno)
-            print(
-                "============================================================="
-            )
-            print("Trying to lock device", serialno)
-            with sh_commands.device_lock(serialno):
-                print("Run on device: %s, %s, %s" % (
-                      serialno, props["ro.board.platform"],
-                      props["ro.product.model"]))
+            if target_abi != 'host':
+                serialnos = sh_commands.get_target_socs_serialnos([target_soc])
+                for serialno in serialnos:
+                    props = sh_commands.adb_getprop_by_serialno(serialno)
+                    print(
+                        "===================================================="
+                    )
+                    print("Trying to lock device", serialno)
+                    with sh_commands.device_lock(serialno):
+                        print("Run on device: %s, %s, %s" % (
+                            serialno, props["ro.board.platform"],
+                              props["ro.product.model"]))
+                        process_models(project_name, configs, embed_model_data,
+                                       vlog_level, target_soc, target_abi,
+                                       serialno, phone_data_dir, option_args)
+            else:
+                print("====================================================")
+                print("Run on host")
                 process_models(project_name, configs, embed_model_data,
-                               vlog_level, target_soc, target_abi,
+                               vlog_level, target_soc, target_abi, '',
                                phone_data_dir, option_args)
 
     if FLAGS.mode == "build" or FLAGS.mode == "all":

tools/sh_commands.py

@@ -21,6 +21,7 @@ import sh
 import subprocess
 import sys
 import time
+import urllib
 
 
 sys.path.insert(0, "mace/python/tools")
@@ -76,11 +77,10 @@ def formatted_file_name(input_name, input_file_name):
 # clear data
 ################################
 def clear_mace_run_data(abi,
-                        target_soc,
+                        serialno,
                         phone_data_dir,
                         model_codegen_dir="mace/codegen/models"):
     if abi != "host":
-        serialno = adb_devices([target_soc]).pop()
         sh.adb("-s",
                serialno,
                "shell",
@@ -98,24 +98,36 @@ def adb_split_stdout(stdout_str):
     return [l.strip() for l in stdout_str.split('\n') if len(l.strip()) > 0]
 
 
-def adb_devices(target_socs=None):
-    device_ids = []
+def adb_devices():
+    serialnos = []
     p = re.compile(r'(\w+)\s+device')
     for line in adb_split_stdout(sh.adb("devices")):
         m = p.match(line)
         if m:
-            device_ids.append(m.group(1))
-
-    if target_socs is not None:
-        target_socs_set = set(target_socs)
-        target_devices = []
-        for serialno in device_ids:
-            props = adb_getprop_by_serialno(serialno)
-            if props["ro.board.platform"] in target_socs_set:
-                target_devices.append(serialno)
-        return target_devices
-    else:
-        return device_ids
+            serialnos.append(m.group(1))
+
+    return serialnos
+
+
+def get_soc_serialnos_map():
+    serialnos = adb_devices()
+    soc_serialnos_map = {}
+    for serialno in serialnos:
+        props = adb_getprop_by_serialno(serialno)
+        soc_serialnos_map.setdefault(props["ro.board.platform"], [])\
+            .append(serialno)
+
+    return soc_serialnos_map
+
+
+def get_target_socs_serialnos(target_socs=None):
+    soc_serialnos_map = get_soc_serialnos_map()
+    serialnos = []
+    if target_socs is None:
+        target_socs = soc_serialnos_map.keys()
+    for target_soc in target_socs:
+        serialnos.extend(soc_serialnos_map[target_soc])
+    return serialnos
 
 
 def adb_getprop_by_serialno(serialno):
@@ -130,6 +142,11 @@ def adb_getprop_by_serialno(serialno):
     return props
 
 
+def adb_get_device_name_by_serialno(serialno):
+    props = adb_getprop_by_serialno(serialno)
+    return props.get("ro.product.model", "")
+
+
 def adb_supported_abis(serialno):
     props = adb_getprop_by_serialno(serialno)
     abilist_str = props["ro.product.cpu.abilist"]
@@ -303,8 +320,7 @@ def gen_encrypted_opencl_source(codegen_path="mace/codegen"):
                            "mace/codegen/opencl/opencl_encrypt_program.cc")
 
 
-def pull_binaries(target_soc, abi, model_output_dirs):
-    serialno = adb_devices([target_soc]).pop()
+def pull_binaries(abi, serialno, model_output_dirs):
     compiled_opencl_dir = "/data/local/tmp/mace_run/cl_program/"
     mace_run_param_file = "mace_run.config"
 
@@ -323,15 +339,12 @@ def pull_binaries(target_soc, abi, model_output_dirs):
                      cl_bin_dir, serialno)
 
 
-def gen_opencl_binary_code(target_soc,
-                           model_output_dirs,
+def gen_opencl_binary_code(model_output_dirs,
                            codegen_path="mace/codegen"):
     cl_built_kernel_file_name = "mace_cl_compiled_program.bin"
     cl_platform_info_file_name = "mace_cl_platform_info.txt"
     opencl_codegen_file = "%s/opencl/opencl_compiled_program.cc" % codegen_path
 
-    serialno = adb_devices([target_soc]).pop()
-
     cl_bin_dirs = []
     for d in model_output_dirs:
         cl_bin_dirs.append(os.path.join(d, "opencl_bin"))
@@ -342,8 +355,7 @@ def gen_opencl_binary_code(target_soc,
                    cl_platform_info_file_name)
 
 
-def gen_tuning_param_code(target_soc,
-                          model_output_dirs,
+def gen_tuning_param_code(model_output_dirs,
                           codegen_path="mace/codegen"):
     mace_run_param_file = "mace_run.config"
     cl_bin_dirs = []
@@ -444,10 +456,10 @@ def gen_random_input(model_output_dir,
         input_file_list.append(input_files)
     if len(input_file_list) != 0:
         input_name_list = []
-        if isinstance(input_names, list):
-            input_name_list.extend(input_names)
+        if isinstance(input_nodes, list):
+            input_name_list.extend(input_nodes)
         else:
-            input_name_list.append(input_names)
+            input_name_list.append(input_nodes)
         if len(input_file_list) != len(input_name_list):
             raise Exception('If input_files set, the input files should '
                             'match the input names.')
@@ -496,8 +508,8 @@ def update_mace_run_lib(model_output_dir,
           model_output_dir)
 
 
-def tuning_run(target_soc,
-               abi,
+def tuning_run(abi,
+               serialno,
                vlog_level,
                embed_model_data,
                model_output_dir,
@@ -538,7 +550,6 @@ def tuning_run(target_soc,
                 "%s" % option_args])
         p.wait()
     else:
-        serialno = adb_devices([target_soc]).pop()
         sh.adb("-s", serialno, "shell", "mkdir", "-p", phone_data_dir)
         compiled_opencl_dir = "/data/local/tmp/mace_run/cl_program/"
         sh.adb("-s", serialno, "shell", "mkdir", "-p", compiled_opencl_dir)
@@ -590,8 +601,8 @@ def tuning_run(target_soc,
     return "".join(stdout_buff)
 
 
-def validate_model(target_soc,
-                   abi,
+def validate_model(abi,
+                   serialno,
                    model_file_path,
                    weight_file_path,
                    platform,
@@ -605,7 +616,6 @@ def validate_model(target_soc,
                    input_file_name="model_input",
                    output_file_name="model_out"):
     print("* Validate with %s" % platform)
-    serialno = adb_devices([target_soc]).pop()
 
     if platform == "tensorflow":
         if abi != "host":
@@ -821,8 +831,8 @@ def packaging_lib(libmace_output_dir, project_name):
     print("Packaging Done!\n")
 
 
-def benchmark_model(target_soc,
-                    abi,
+def benchmark_model(abi,
+                    serialno,
                     vlog_level,
                     embed_model_data,
                     model_output_dir,
@@ -842,7 +852,7 @@ def benchmark_model(target_soc,
     if os.path.exists(benchmark_binary_file):
         sh.rm("-rf", benchmark_binary_file)
     if not embed_model_data:
-        sh.cp("-f", "codegen/models/%s/%s.data" % (model_tag, model_tag),
+        sh.cp("-f", "mace/codegen/models/%s/%s.data" % (model_tag, model_tag),
               model_output_dir)
 
     benchmark_target = "//mace/benchmark:benchmark_model"
@@ -871,7 +881,6 @@ def benchmark_model(target_soc,
                 "%s" % option_args])
         p.wait()
     else:
-        serialno = adb_devices([target_soc]).pop()
         sh.adb("-s", serialno, "shell", "mkdir", "-p", phone_data_dir)
 
         for input_name in input_nodes:
@@ -911,8 +920,8 @@ def benchmark_model(target_soc,
     return "".join(stdout_buff)
 
 
-def build_run_throughput_test(target_soc,
-                              abi,
+def build_run_throughput_test(abi,
+                              serialno,
                               vlog_level,
                               run_seconds,
                               merged_lib_file,
@@ -929,7 +938,6 @@ def build_run_throughput_test(target_soc,
                               strip="always",
                               input_file_name="model_input"):
     print("* Build and run throughput_test")
-    serialno = adb_devices([target_soc]).pop()
 
     model_tag_build_flag = ""
     if cpu_model_tag: