From 6ef980206cb6fb0afa0cdacba2af907dc2e72cc4 Mon Sep 17 00:00:00 2001 From: wuchenghui Date: Mon, 23 Apr 2018 16:15:21 +0800 Subject: [PATCH] add ci report --- tools/bazel_adb_run.py | 2 +- tools/mace_tools.py | 187 +++++++++++++++++++++++------------------ tools/sh_commands.py | 88 ++++++++++--------- 3 files changed, 155 insertions(+), 122 deletions(-) diff --git a/tools/bazel_adb_run.py b/tools/bazel_adb_run.py index b542f93a..c7e78852 100644 --- a/tools/bazel_adb_run.py +++ b/tools/bazel_adb_run.py @@ -96,7 +96,7 @@ def main(unused_args): target_socs = None if FLAGS.target_socs != "all" and FLAGS.target_socs != "random": target_socs = set(FLAGS.target_socs.split(',')) - target_devices = sh_commands.adb_devices(target_socs=target_socs) + target_devices = sh_commands.get_target_socs_serialnos(target_socs) if FLAGS.target_socs == "random": unlocked_devices = \ [d for d in target_devices if not sh_commands.is_device_locked(d)] diff --git a/tools/mace_tools.py b/tools/mace_tools.py index 27908389..0ccaecf9 100644 --- a/tools/mace_tools.py +++ b/tools/mace_tools.py @@ -88,59 +88,69 @@ def get_hexagon_mode(configs): return False -def gen_opencl_and_tuning_code(target_soc, - target_abi, +def gen_opencl_and_tuning_code(target_abi, + serialno, model_output_dirs, pull_or_not): if pull_or_not: - sh_commands.pull_binaries( - target_soc, target_abi, model_output_dirs) + sh_commands.pull_binaries(target_abi, serialno, model_output_dirs) codegen_path = "mace/codegen" # generate opencl binary code - sh_commands.gen_opencl_binary_code(target_soc, model_output_dirs) + sh_commands.gen_opencl_binary_code(model_output_dirs) - sh_commands.gen_tuning_param_code( - target_soc, model_output_dirs) + sh_commands.gen_tuning_param_code(model_output_dirs) def model_benchmark_stdout_processor(stdout, - target_soc, abi, + serialno, model_name, - runtime, - running_round, - tuning): - metrics = {} - for line in stdout.split("\n"): - if "Aborted" in line: - raise Exception("Command failed") + runtime): + metrics = [0] * 5 + for line in stdout.split('\n'): line = line.strip() parts = line.split() if len(parts) == 6 and parts[0].startswith("time"): - metrics["%s.create_net_ms" % model_name] = str(float(parts[1])) - metrics["%s.mace_engine_ctor_ms" % model_name] = str( - float(parts[2])) - metrics["%s.init_ms" % model_name] = str(float(parts[3])) - metrics["%s.warmup_ms" % model_name] = str(float(parts[4])) - if float(parts[5]) > 0: - metrics["%s.avg_latency_ms" % model_name] = str( - float(parts[5])) - tags = { - "ro.board.platform": target_soc, - "abi": abi, - "runtime": runtime, - "round": running_round, # TODO(yejianwu) change this to source/binary - "tuning": tuning - } - sh_commands.falcon_push_metrics( - metrics, endpoint="mace_model_benchmark", tags=tags) + metrics[0] = str(float(parts[1])) + metrics[1] = str(float(parts[2])) + metrics[2] = str(float(parts[3])) + metrics[3] = str(float(parts[4])) + metrics[4] = str(float(parts[5])) + break + + props = sh_commands.adb_getprop_by_serialno(serialno) + device_type = props.get("ro.product.model", "") + target_soc = props.get("ro.board.platform", "") + + report_filename = "build/report.csv" + if not os.path.exists(report_filename): + with open(report_filename, 'w') as f: + f.write("model_name,device_type,soc,abi,runtime,create_net," + "engine_ctor,init,warmup,run_avg\n") + + data_str = "{model_name},{device_type},{soc},{abi},{runtime}," \ + "{create_net},{engine_ctor},{init},{warmup},{run_avg}\n" \ + .format( + model_name=model_name, + device_type=device_type, + soc=target_soc, + abi=abi, + runtime=runtime, + create_net=metrics[0], + engine_ctor=metrics[1], + init=metrics[2], + warmup=metrics[3], + run_avg=metrics[4] + ) + with open(report_filename, 'a') as f: + f.write(data_str) def tuning_run(runtime, - target_soc, target_abi, + serialno, vlog_level, embed_model_data, model_output_dir, @@ -158,8 +168,8 @@ def tuning_run(runtime, limit_opencl_kernel_time=0, option_args=""): stdout = sh_commands.tuning_run( - target_soc, target_abi, + serialno, vlog_level, embed_model_data, model_output_dir, @@ -176,21 +186,19 @@ def tuning_run(runtime, out_of_range_check, phone_data_dir, option_args) - model_benchmark_stdout_processor(stdout, - target_soc, - target_abi, - model_name, - runtime, - running_round, - tuning) + if running_round > 0 and FLAGS.collect_report: + model_benchmark_stdout_processor( + stdout, target_abi, serialno, model_name, runtime) -def build_mace_run_prod(hexagon_mode, runtime, target_soc, target_abi, - vlog_level, embed_model_data, model_output_dir, - input_nodes, output_nodes, input_shapes, output_shapes, - model_name, device_type, running_round, restart_round, - tuning, limit_opencl_kernel_time, phone_data_dir): - gen_opencl_and_tuning_code(target_soc, target_abi, [], False) + +def build_mace_run_prod(hexagon_mode, runtime, target_abi, + serialno, vlog_level, embed_model_data, + model_output_dir, input_nodes, output_nodes, + input_shapes, output_shapes, model_name, device_type, + running_round, restart_round, tuning, + limit_opencl_kernel_time, phone_data_dir): + gen_opencl_and_tuning_code(target_abi, serialno, [], False) production_or_not = False mace_run_target = "//mace/tools/validation:mace_run" sh_commands.bazel_build( @@ -202,21 +210,20 @@ def build_mace_run_prod(hexagon_mode, runtime, target_soc, target_abi, sh_commands.update_mace_run_lib(model_output_dir, target_abi, model_name, embed_model_data) - tuning_run(runtime, target_soc, target_abi, vlog_level, embed_model_data, - model_output_dir, input_nodes, output_nodes, input_shapes, - output_shapes, model_name, device_type, running_round=0, - restart_round=1, out_of_range_check=True, + tuning_run(runtime, target_abi, serialno, vlog_level, + embed_model_data, model_output_dir, input_nodes, output_nodes, + input_shapes, output_shapes, model_name, device_type, + running_round=0, restart_round=1, out_of_range_check=True, phone_data_dir=phone_data_dir, tuning=False) - tuning_run(runtime, target_soc, target_abi, vlog_level, embed_model_data, - model_output_dir, input_nodes, output_nodes, input_shapes, - output_shapes, model_name, device_type, running_round=0, - restart_round=1, out_of_range_check=False, + tuning_run(runtime, target_abi, serialno, vlog_level, + embed_model_data, model_output_dir, input_nodes, output_nodes, + input_shapes, output_shapes, model_name, device_type, + running_round=0, restart_round=1, out_of_range_check=False, phone_data_dir=phone_data_dir, tuning=tuning, limit_opencl_kernel_time=limit_opencl_kernel_time) - gen_opencl_and_tuning_code( - target_soc, target_abi, [model_output_dir], True) + gen_opencl_and_tuning_code(target_abi, serialno, [model_output_dir], True) production_or_not = True sh_commands.bazel_build( mace_run_target, @@ -230,13 +237,14 @@ def build_mace_run_prod(hexagon_mode, runtime, target_soc, target_abi, def merge_libs_and_tuning_results(target_soc, target_abi, + serialno, project_name, output_dir, model_output_dirs, hexagon_mode, embed_model_data): gen_opencl_and_tuning_code( - target_soc, target_abi, model_output_dirs, False) + target_abi, serialno, model_output_dirs, False) sh_commands.build_production_code(target_abi) sh_commands.merge_libs(target_soc, @@ -322,11 +330,17 @@ def parse_args(): type="bool", default="false", help="Enable out of range check for opencl.") + parser.add_argument( + "--collect_report", + type="bool", + default="false", + help="Collect report.") return parser.parse_known_args() def process_models(project_name, configs, embed_model_data, vlog_level, - target_soc, target_abi, phone_data_dir, option_args): + target_soc, target_abi, serialno, phone_data_dir, + option_args): hexagon_mode = get_hexagon_mode(configs) model_output_dirs = [] for model_name in configs["models"]: @@ -344,9 +358,11 @@ def process_models(project_name, configs, embed_model_data, vlog_level, # Create model build directory model_path_digest = md5sum(model_config["model_file_path"]) - model_output_dir = "%s/%s/%s/%s/%s/%s/%s" % ( + device_name = sh_commands.adb_get_device_name_by_serialno(serialno) + model_output_dir = "%s/%s/%s/%s/%s/%s_%s/%s" % ( FLAGS.output_dir, project_name, "build", - model_name, model_path_digest, target_soc, target_abi) + model_name, model_path_digest, device_name.replace(' ', ''), + target_soc, target_abi) model_output_dirs.append(model_output_dir) if FLAGS.mode == "build" or FLAGS.mode == "all": @@ -354,7 +370,7 @@ def process_models(project_name, configs, embed_model_data, vlog_level, sh.rm("-rf", model_output_dir) os.makedirs(model_output_dir) sh_commands.clear_mace_run_data( - target_abi, target_soc, phone_data_dir) + target_abi, serialno, phone_data_dir) model_file_path, weight_file_path = get_model_files( model_config["model_file_path"], @@ -388,8 +404,8 @@ def process_models(project_name, configs, embed_model_data, vlog_level, model_config["obfuscate"]) build_mace_run_prod(hexagon_mode, model_config["runtime"], - target_soc, target_abi, + serialno, vlog_level, embed_model_data, model_output_dir, @@ -408,8 +424,8 @@ def process_models(project_name, configs, embed_model_data, vlog_level, if FLAGS.mode == "run" or FLAGS.mode == "validate" or \ FLAGS.mode == "all": tuning_run(model_config["runtime"], - target_soc, target_abi, + serialno, vlog_level, embed_model_data, model_output_dir, @@ -425,8 +441,8 @@ def process_models(project_name, configs, embed_model_data, vlog_level, phone_data_dir) if FLAGS.mode == "benchmark": - sh_commands.benchmark_model(target_soc, - target_abi, + sh_commands.benchmark_model(target_abi, + serialno, vlog_level, embed_model_data, model_output_dir, @@ -441,8 +457,8 @@ def process_models(project_name, configs, embed_model_data, vlog_level, option_args) if FLAGS.mode == "validate" or FLAGS.mode == "all": - sh_commands.validate_model(target_soc, - target_abi, + sh_commands.validate_model(target_abi, + serialno, model_file_path, weight_file_path, model_config["platform"], @@ -459,6 +475,7 @@ def process_models(project_name, configs, embed_model_data, vlog_level, merge_libs_and_tuning_results( target_soc, target_abi, + serialno, project_name, FLAGS.output_dir, model_output_dirs, @@ -486,8 +503,8 @@ def process_models(project_name, configs, embed_model_data, vlog_level, for model_name in configs["models"]: runtime = configs["models"][model_name]["runtime"] model_tag_dict[runtime] = model_name - sh_commands.build_run_throughput_test(target_soc, - target_abi, + sh_commands.build_run_throughput_test(target_abi, + serialno, vlog_level, FLAGS.run_seconds, merged_lib_file, @@ -533,18 +550,26 @@ def main(unused_args): phone_data_dir = "/data/local/tmp/mace_run/" for target_soc in target_socs: for target_abi in configs["target_abis"]: - serialno = sh_commands.adb_devices([target_soc]).pop() - props = sh_commands.adb_getprop_by_serialno(serialno) - print( - "=============================================================" - ) - print("Trying to lock device", serialno) - with sh_commands.device_lock(serialno): - print("Run on device: %s, %s, %s" % ( - serialno, props["ro.board.platform"], - props["ro.product.model"])) + if target_abi != 'host': + serialnos = sh_commands.get_target_socs_serialnos([target_soc]) + for serialno in serialnos: + props = sh_commands.adb_getprop_by_serialno(serialno) + print( + "====================================================" + ) + print("Trying to lock device", serialno) + with sh_commands.device_lock(serialno): + print("Run on device: %s, %s, %s" % ( + serialno, props["ro.board.platform"], + props["ro.product.model"])) + process_models(project_name, configs, embed_model_data, + vlog_level, target_soc, target_abi, + serialno, phone_data_dir, option_args) + else: + print("====================================================") + print("Run on host") process_models(project_name, configs, embed_model_data, - vlog_level, target_soc, target_abi, + vlog_level, target_soc, target_abi, '', phone_data_dir, option_args) if FLAGS.mode == "build" or FLAGS.mode == "all": diff --git a/tools/sh_commands.py b/tools/sh_commands.py index 6aa77834..40cc37e2 100644 --- a/tools/sh_commands.py +++ b/tools/sh_commands.py @@ -21,6 +21,7 @@ import sh import subprocess import sys import time +import urllib sys.path.insert(0, "mace/python/tools") @@ -76,11 +77,10 @@ def formatted_file_name(input_name, input_file_name): # clear data ################################ def clear_mace_run_data(abi, - target_soc, + serialno, phone_data_dir, model_codegen_dir="mace/codegen/models"): if abi != "host": - serialno = adb_devices([target_soc]).pop() sh.adb("-s", serialno, "shell", @@ -98,24 +98,36 @@ def adb_split_stdout(stdout_str): return [l.strip() for l in stdout_str.split('\n') if len(l.strip()) > 0] -def adb_devices(target_socs=None): - device_ids = [] +def adb_devices(): + serialnos = [] p = re.compile(r'(\w+)\s+device') for line in adb_split_stdout(sh.adb("devices")): m = p.match(line) if m: - device_ids.append(m.group(1)) - - if target_socs is not None: - target_socs_set = set(target_socs) - target_devices = [] - for serialno in device_ids: - props = adb_getprop_by_serialno(serialno) - if props["ro.board.platform"] in target_socs_set: - target_devices.append(serialno) - return target_devices - else: - return device_ids + serialnos.append(m.group(1)) + + return serialnos + + +def get_soc_serialnos_map(): + serialnos = adb_devices() + soc_serialnos_map = {} + for serialno in serialnos: + props = adb_getprop_by_serialno(serialno) + soc_serialnos_map.setdefault(props["ro.board.platform"], [])\ + .append(serialno) + + return soc_serialnos_map + + +def get_target_socs_serialnos(target_socs=None): + soc_serialnos_map = get_soc_serialnos_map() + serialnos = [] + if target_socs is None: + target_socs = soc_serialnos_map.keys() + for target_soc in target_socs: + serialnos.extend(soc_serialnos_map[target_soc]) + return serialnos def adb_getprop_by_serialno(serialno): @@ -130,6 +142,11 @@ def adb_getprop_by_serialno(serialno): return props +def adb_get_device_name_by_serialno(serialno): + props = adb_getprop_by_serialno(serialno) + return props.get("ro.product.model", "") + + def adb_supported_abis(serialno): props = adb_getprop_by_serialno(serialno) abilist_str = props["ro.product.cpu.abilist"] @@ -303,8 +320,7 @@ def gen_encrypted_opencl_source(codegen_path="mace/codegen"): "mace/codegen/opencl/opencl_encrypt_program.cc") -def pull_binaries(target_soc, abi, model_output_dirs): - serialno = adb_devices([target_soc]).pop() +def pull_binaries(abi, serialno, model_output_dirs): compiled_opencl_dir = "/data/local/tmp/mace_run/cl_program/" mace_run_param_file = "mace_run.config" @@ -323,15 +339,12 @@ def pull_binaries(target_soc, abi, model_output_dirs): cl_bin_dir, serialno) -def gen_opencl_binary_code(target_soc, - model_output_dirs, +def gen_opencl_binary_code(model_output_dirs, codegen_path="mace/codegen"): cl_built_kernel_file_name = "mace_cl_compiled_program.bin" cl_platform_info_file_name = "mace_cl_platform_info.txt" opencl_codegen_file = "%s/opencl/opencl_compiled_program.cc" % codegen_path - serialno = adb_devices([target_soc]).pop() - cl_bin_dirs = [] for d in model_output_dirs: cl_bin_dirs.append(os.path.join(d, "opencl_bin")) @@ -342,8 +355,7 @@ def gen_opencl_binary_code(target_soc, cl_platform_info_file_name) -def gen_tuning_param_code(target_soc, - model_output_dirs, +def gen_tuning_param_code(model_output_dirs, codegen_path="mace/codegen"): mace_run_param_file = "mace_run.config" cl_bin_dirs = [] @@ -438,10 +450,10 @@ def gen_random_input(model_output_dir, input_file_list.append(input_files) if len(input_file_list) != 0: input_name_list = [] - if isinstance(input_names, list): - input_name_list.extend(input_names) + if isinstance(input_nodes, list): + input_name_list.extend(input_nodes) else: - input_name_list.append(input_names) + input_name_list.append(input_nodes) if len(input_file_list) != len(input_name_list): raise Exception('If input_files set, the input files should ' 'match the input names.') @@ -490,8 +502,8 @@ def update_mace_run_lib(model_output_dir, model_output_dir) -def tuning_run(target_soc, - abi, +def tuning_run(abi, + serialno, vlog_level, embed_model_data, model_output_dir, @@ -532,7 +544,6 @@ def tuning_run(target_soc, "%s" % option_args]) p.wait() else: - serialno = adb_devices([target_soc]).pop() sh.adb("-s", serialno, "shell", "mkdir", "-p", phone_data_dir) compiled_opencl_dir = "/data/local/tmp/mace_run/cl_program/" sh.adb("-s", serialno, "shell", "mkdir", "-p", compiled_opencl_dir) @@ -584,8 +595,8 @@ def tuning_run(target_soc, return "".join(stdout_buff) -def validate_model(target_soc, - abi, +def validate_model(abi, + serialno, model_file_path, weight_file_path, platform, @@ -599,7 +610,6 @@ def validate_model(target_soc, input_file_name="model_input", output_file_name="model_out"): print("* Validate with %s" % platform) - serialno = adb_devices([target_soc]).pop() if platform == "tensorflow": if abi != "host": @@ -815,8 +825,8 @@ def packaging_lib(libmace_output_dir, project_name): print("Packaging Done!\n") -def benchmark_model(target_soc, - abi, +def benchmark_model(abi, + serialno, vlog_level, embed_model_data, model_output_dir, @@ -836,7 +846,7 @@ def benchmark_model(target_soc, if os.path.exists(benchmark_binary_file): sh.rm("-rf", benchmark_binary_file) if not embed_model_data: - sh.cp("-f", "codegen/models/%s/%s.data" % (model_tag, model_tag), + sh.cp("-f", "mace/codegen/models/%s/%s.data" % (model_tag, model_tag), model_output_dir) benchmark_target = "//mace/benchmark:benchmark_model" @@ -865,7 +875,6 @@ def benchmark_model(target_soc, "%s" % option_args]) p.wait() else: - serialno = adb_devices([target_soc]).pop() sh.adb("-s", serialno, "shell", "mkdir", "-p", phone_data_dir) for input_name in input_nodes: @@ -905,8 +914,8 @@ def benchmark_model(target_soc, return "".join(stdout_buff) -def build_run_throughput_test(target_soc, - abi, +def build_run_throughput_test(abi, + serialno, vlog_level, run_seconds, merged_lib_file, @@ -923,7 +932,6 @@ def build_run_throughput_test(target_soc, strip="always", input_file_name="model_input"): print("* Build and run throughput_test") - serialno = adb_devices([target_soc]).pop() model_tag_build_flag = "" if cpu_model_tag: -- GitLab