From d84c1a5ad65d509f10a36f941aafe6cdee08326b Mon Sep 17 00:00:00 2001 From: yejianwu Date: Fri, 20 Apr 2018 16:24:59 +0800 Subject: [PATCH] unify phone_data_dir, fix tuning codegen dir, update codegen process --- tools/mace_tools.py | 56 ++++++++++++++++++++------------- tools/sh_commands.py | 75 +++++++++++++++++++------------------------- 2 files changed, 66 insertions(+), 65 deletions(-) diff --git a/tools/mace_tools.py b/tools/mace_tools.py index ddcf29a5..6c18783d 100644 --- a/tools/mace_tools.py +++ b/tools/mace_tools.py @@ -90,20 +90,21 @@ def get_hexagon_mode(configs): def generate_code(target_soc, target_abi, model_output_dirs, pull_or_not): - sh_commands.pull_binaries( - target_soc, target_abi, model_output_dirs, pull_or_not) + if pull_or_not: + sh_commands.pull_binaries( + target_soc, target_abi, model_output_dirs) sh_commands.gen_opencl_binary_code( - target_soc, target_abi, model_output_dirs, pull_or_not) + target_soc, target_abi, model_output_dirs) sh_commands.gen_tuning_param_code( - target_soc, target_abi, model_output_dirs, pull_or_not) + target_soc, target_abi, model_output_dirs) -def ops_benchmark_stdout_processor(stdout, - target_soc, - abi, - runtime, - running_round, - tuning): +def model_benchmark_stdout_processor(stdout, + target_soc, + abi, + runtime, + running_round, + tuning): metrics = {} for line in stdout: if "Aborted" in line: @@ -145,6 +146,7 @@ def tuning_run(runtime, running_round, restart_round, out_of_range_check, + phone_data_dir, tuning=False, limit_opencl_kernel_time=0, option_args=""): @@ -165,8 +167,9 @@ def tuning_run(runtime, limit_opencl_kernel_time, tuning, out_of_range_check, + phone_data_dir, option_args) - ops_benchmark_stdout_processor(stdout, + model_benchmark_stdout_processor(stdout, target_soc, target_abi, runtime, @@ -178,7 +181,7 @@ def build_mace_run_prod(hexagon_mode, runtime, target_soc, target_abi, vlog_level, embed_model_data, model_output_dir, input_nodes, output_nodes, input_shapes, output_shapes, model_name, device_type, running_round, restart_round, - tuning, limit_opencl_kernel_time): + tuning, limit_opencl_kernel_time, phone_data_dir): generate_code(target_soc, target_abi, [], False) production_or_not = False mace_run_target = "//mace/tools/validation:mace_run" @@ -194,12 +197,14 @@ def build_mace_run_prod(hexagon_mode, runtime, target_soc, target_abi, tuning_run(runtime, target_soc, target_abi, vlog_level, embed_model_data, model_output_dir, input_nodes, output_nodes, input_shapes, output_shapes, model_name, device_type, running_round=0, - restart_round=1, out_of_range_check=True, tuning=False) + restart_round=1, out_of_range_check=True, + phone_data_dir=phone_data_dir, tuning=False) tuning_run(runtime, target_soc, target_abi, vlog_level, embed_model_data, model_output_dir, input_nodes, output_nodes, input_shapes, output_shapes, model_name, device_type, running_round=0, - restart_round=1, out_of_range_check=False, tuning=tuning, + restart_round=1, out_of_range_check=False, + phone_data_dir=phone_data_dir, tuning=tuning, limit_opencl_kernel_time=limit_opencl_kernel_time) generate_code(target_soc, target_abi, [model_output_dir], True) @@ -307,7 +312,7 @@ def parse_args(): def process_models(project_name, configs, embed_model_data, vlog_level, - target_soc, target_abi, option_args): + target_soc, target_abi, phone_data_dir, option_args): hexagon_mode = get_hexagon_mode(configs) model_output_dirs = [] for model_name in configs["models"]: @@ -334,7 +339,8 @@ def process_models(project_name, configs, embed_model_data, vlog_level, if os.path.exists(model_output_dir): sh.rm("-rf", model_output_dir) os.makedirs(model_output_dir) - sh_commands.clear_mace_run_data(target_abi, target_soc) + sh_commands.clear_mace_run_data( + target_abi, target_soc, phone_data_dir) model_file_path, weight_file_path = download_model_files( model_config["model_file_path"], @@ -382,8 +388,8 @@ def process_models(project_name, configs, embed_model_data, vlog_level, FLAGS.round, FLAGS.restart_round, FLAGS.tuning, - model_config[ - "limit_opencl_kernel_time"]) + model_config["limit_opencl_kernel_time"], + phone_data_dir) if FLAGS.mode == "run" or FLAGS.mode == "validate" or \ FLAGS.mode == "all": @@ -401,7 +407,8 @@ def process_models(project_name, configs, embed_model_data, vlog_level, device_type, FLAGS.round, FLAGS.restart_round, - FLAGS.out_of_range_check) + FLAGS.out_of_range_check, + phone_data_dir) if FLAGS.mode == "benchmark": sh_commands.benchmark_model(target_soc, @@ -416,6 +423,7 @@ def process_models(project_name, configs, embed_model_data, vlog_level, model_name, device_type, hexagon_mode, + phone_data_dir, option_args) if FLAGS.mode == "validate" or FLAGS.mode == "all": @@ -429,7 +437,8 @@ def process_models(project_name, configs, embed_model_data, vlog_level, model_config["output_nodes"], model_config["input_shapes"], model_config["output_shapes"], - model_output_dir) + model_output_dir, + phone_data_dir) if FLAGS.mode == "build" or FLAGS.mode == "merge" or \ FLAGS.mode == "all": @@ -476,7 +485,8 @@ def process_models(project_name, configs, embed_model_data, vlog_level, model_config["output_shapes"], model_tag_dict.get("cpu", ""), model_tag_dict.get("gpu", ""), - model_tag_dict.get("dsp", "")) + model_tag_dict.get("dsp", ""), + phone_data_dir) def main(unused_args): @@ -506,6 +516,7 @@ def main(unused_args): embed_model_data = configs.get("embed_model_data", 1) vlog_level = configs.get("vlog_level", 0) + phone_data_dir = "/data/local/tmp/mace_run/" for target_soc in target_socs: for target_abi in configs["target_abis"]: serialno = sh_commands.adb_devices([target_soc]).pop() @@ -519,7 +530,8 @@ def main(unused_args): serialno, props["ro.board.platform"], props["ro.product.model"])) process_models(project_name, configs, embed_model_data, - vlog_level, target_soc, target_abi, option_args) + vlog_level, target_soc, target_abi, + phone_data_dir, option_args) if FLAGS.mode == "build" or FLAGS.mode == "all": sh_commands.packaging_lib(FLAGS.output_dir, project_name) diff --git a/tools/sh_commands.py b/tools/sh_commands.py index 55e4578e..3aa613be 100644 --- a/tools/sh_commands.py +++ b/tools/sh_commands.py @@ -63,14 +63,14 @@ def formatted_file_name(input_name, input_file_name): ################################ def clear_mace_run_data(abi, target_soc, - model_codegen_dir="mace/codegen/models", - mace_run_phone_data_dir="/data/local/tmp/mace_run"): + phone_data_dir, + model_codegen_dir="mace/codegen/models"): if abi != "host": serialno = adb_devices([target_soc]).pop() sh.adb("-s", serialno, "shell", - "rm -rf %s" % mace_run_phone_data_dir) + "rm -rf %s" % phone_data_dir) if os.path.exists(model_codegen_dir): sh.rm("-rf", model_codegen_dir) @@ -292,7 +292,7 @@ def gen_encrypted_opencl_source(codegen_path="mace/codegen"): "--output_path=%s/opencl/opencl_encrypt_program.cc" % codegen_path) -def pull_binaries(target_soc, abi, model_output_dirs, pull_or_not): +def pull_binaries(target_soc, abi, model_output_dirs): serialno = adb_devices([target_soc]).pop() compiled_opencl_dir = "/data/local/tmp/mace_run/cl_program/" mace_run_config_file = "mace_run.config" @@ -301,7 +301,7 @@ def pull_binaries(target_soc, abi, model_output_dirs, pull_or_not): for d in model_output_dirs: cl_bin_dirs.append(os.path.join(d, "opencl_bin")) cl_bin_dirs_str = ",".join(cl_bin_dirs) - if cl_bin_dirs and pull_or_not: + if cl_bin_dirs: cl_bin_dir = cl_bin_dirs_str if os.path.exists(cl_bin_dir): sh.rm("-rf", cl_bin_dir) @@ -315,7 +315,6 @@ def pull_binaries(target_soc, abi, model_output_dirs, pull_or_not): def gen_opencl_binary_code(target_soc, abi, model_output_dirs, - pull_or_not, codegen_path="mace/codegen"): cl_built_kernel_file_name = "mace_cl_compiled_program.bin" cl_platform_info_file_name = "mace_cl_platform_info.txt" @@ -347,30 +346,22 @@ def gen_opencl_binary_code(target_soc, def gen_tuning_param_code(target_soc, abi, model_output_dirs, - pull_or_not, codegen_path="mace/codegen"): mace_run_config_file = "mace_run.config" cl_bin_dirs = [] for d in model_output_dirs: cl_bin_dirs.append(os.path.join(d, "opencl_bin")) cl_bin_dirs_str = ",".join(cl_bin_dirs) - if not cl_bin_dirs: - opencl_codegen_dir = "%s/opencl/" % codegen_path - if not os.path.exists(opencl_codegen_dir): - sh.mkdir("-p", opencl_codegen_dir) - sh.python( - "mace/python/tools/binary_codegen.py", - "--binary_file_name=%s" % mace_run_config_file, - "--output_path=%s/opencl/tuning_params.cc" % codegen_path) - else: - tuning_codegen_dir = "%s/tuning/" % codegen_path - if not os.path.exists(tuning_codegen_dir): - sh.mkdir("-p", tuning_codegen_dir) - sh.python( - "mace/python/tools/binary_codegen.py", - "--binary_dirs=%s" % cl_bin_dirs_str, - "--binary_file_name=%s" % mace_run_config_file, - "--output_path=%s/tuning_params.cc" % tuning_codegen_dir) + + tuning_codegen_dir = "%s/tuning/" % codegen_path + if not os.path.exists(tuning_codegen_dir): + sh.mkdir("-p", tuning_codegen_dir) + + sh.python( + "mace/python/tools/binary_codegen.py", + "--binary_dirs=%s" % cl_bin_dirs_str, + "--binary_file_name=%s" % mace_run_config_file, + "--output_path=%s/tuning_params.cc" % tuning_codegen_dir) def gen_mace_version(codegen_path="mace/codegen"): @@ -468,7 +459,7 @@ def gen_random_input(model_output_dir, input_file_list[i].startswith("https://"): urllib.urlretrieve(input_file_list[i], dst_input_file) else: - sh.cp(input_file_list[i], dst_input_file) + sh.cp("-f", input_file_list[i], dst_input_file) def update_mace_run_lib(model_output_dir, @@ -488,19 +479,19 @@ def update_mace_run_lib(model_output_dir, if os.path.exists(model_lib_path): sh.rm("-rf", model_lib_path) - sh.cp(generated_model_lib_dir + "/" + generated_model_lib_name, + sh.cp("-f", generated_model_lib_dir + "/" + generated_model_lib_name, model_lib_path) mace_run_filepath = model_output_dir + "/mace_run" if os.path.exists(mace_run_filepath): sh.rm("-rf", mace_run_filepath) - sh.cp("bazel-bin/mace/tools/validation/mace_run", model_output_dir) + sh.cp("-f", "bazel-bin/mace/tools/validation/mace_run", model_output_dir) if embed_model_data == 0: - sh.cp("mace/codegen/models/%s/%s.data" % (model_tag, model_tag), + sh.cp("-f", "mace/codegen/models/%s/%s.data" % (model_tag, model_tag), model_output_dir) - sh.cp("mace/codegen/models/%s/%s.h" % (model_tag, model_tag), + sh.cp("-f", "mace/codegen/models/%s/%s.h" % (model_tag, model_tag), model_output_dir) @@ -520,6 +511,7 @@ def tuning_run(target_soc, limit_opencl_kernel_time, tuning, out_of_range_check, + phone_data_dir, option_args="", input_file_name="model_input", output_file_name="model_out"): @@ -546,7 +538,6 @@ def tuning_run(target_soc, p.wait() else: serialno = adb_devices([target_soc]).pop() - phone_data_dir = "/data/local/tmp/mace_run/" sh.adb("-s", serialno, "shell", "mkdir", "-p", phone_data_dir) compiled_opencl_dir = "/data/local/tmp/mace_run/cl_program/" sh.adb("-s", serialno, "shell", "mkdir", "-p", compiled_opencl_dir) @@ -609,11 +600,11 @@ def validate_model(target_soc, input_shapes, output_shapes, model_output_dir, + phone_data_dir, input_file_name="model_input", output_file_name="model_out"): print("* Validate with %s" % platform) serialno = adb_devices([target_soc]).pop() - phone_data_dir = "/data/local/tmp/mace_run" stdout_buff = [] process_output = make_output_processor(stdout_buff) @@ -750,10 +741,11 @@ def merge_libs(target_soc, sh.mkdir("-p", model_bin_dir) if not os.path.exists(model_header_dir): sh.mkdir("-p", model_header_dir) + sh.cp("-f", glob.glob("mace/public/*.h"), model_header_dir) if not os.path.exists(model_data_dir): sh.mkdir("-p", model_data_dir) if hexagon_mode: - sh.cp(hexagon_lib_file, model_bin_dir) + sh.cp("-f", hexagon_lib_file, model_bin_dir) mri_stream = "" mri_stream += "create %s/libmace_%s.%s.a\n" % \ @@ -796,12 +788,9 @@ def merge_libs(target_soc, for lib in sh.ls(glob.glob("%s/*.a" % model_output_dir), "-1"): mri_stream += "addlib %s\n" % lib if not embed_model_data: - for data_file in sh.ls(glob.glob("%s/*.data" % - model_output_dir), "-1"): - sh.cp(data_file.strip("\n"), model_data_dir) - for header_file in sh.ls(glob.glob("%s/*.h" % model_output_dir), - "-1"): - sh.cp(header_file.strip("\n"), model_header_dir) + sh.cp("-f", glob.glob("%s/*.data" % model_output_dir), + model_data_dir) + sh.cp("-f", glob.glob("%s/*.h" % model_output_dir), model_header_dir) mri_stream += "save\n" mri_stream += "end\n" @@ -854,6 +843,7 @@ def benchmark_model(target_soc, model_tag, device_type, hexagon_mode, + phone_data_dir, option_args="", input_file_name="model_input", output_file_name="model_out"): @@ -862,7 +852,7 @@ def benchmark_model(target_soc, if os.path.exists(benchmark_binary_file): sh.rm("-rf", benchmark_binary_file) if not embed_model_data: - sh.cp("codegen/models/%s/%s.data" % (model_tag, model_tag), + sh.cp("-f", "codegen/models/%s/%s.data" % (model_tag, model_tag), model_output_dir) benchmark_target = "//mace/benchmark:benchmark_model" @@ -872,7 +862,7 @@ def benchmark_model(target_soc, hexagon_mode=hexagon_mode) target_bin = "/".join(bazel_target_to_bin(benchmark_target)) - sh.cp(target_bin, model_output_dir) + sh.cp("-f", target_bin, model_output_dir) stdout_buff = [] process_output = make_output_processor(stdout_buff) @@ -892,7 +882,6 @@ def benchmark_model(target_soc, p.wait() else: serialno = adb_devices([target_soc]).pop() - phone_data_dir = "/data/local/tmp/mace_run/" sh.adb("-s", serialno, "shell", "mkdir", "-p", phone_data_dir) for input_name in input_nodes: @@ -946,10 +935,10 @@ def build_run_throughput_test(target_soc, cpu_model_tag, gpu_model_tag, dsp_model_tag, + phone_data_dir, strip="always", input_file_name="model_input"): print("* Build and run throughput_test") - phone_data_dir = "/data/local/tmp/mace_run/" serialno = adb_devices([target_soc]).pop() model_tag_build_flag = "" @@ -963,7 +952,7 @@ def build_run_throughput_test(target_soc, model_tag_build_flag += "--copt=-DMACE_DSP_MODEL_TAG=%s " % \ dsp_model_tag - sh.cp(merged_lib_file, "mace/benchmark/libmace_merged.a") + sh.cp("-f", merged_lib_file, "mace/benchmark/libmace_merged.a") stdout_buff = [] process_output = make_output_processor(stdout_buff) p = sh.bazel( -- GitLab