From d61080b2167a30b14510a81f157c6b95e550ae5b Mon Sep 17 00:00:00 2001 From: Liangliang He Date: Mon, 2 Apr 2018 20:51:13 +0800 Subject: [PATCH] Refactor bazel run tools and add metrics logging --- .gitlab-ci.yml | 4 +- mace/core/testing/test_benchmark.cc | 2 + mace/core/testing/test_benchmark_main.cc | 4 +- tools/adb_tools.py | 33 ----- tools/bazel-adb-run.sh | 73 ----------- tools/bazel_adb_run.py | 103 +++++++++++++++ tools/falcon_cli.py | 60 +++++++++ tools/mace_tools.py | 10 +- tools/sh_commands.py | 156 +++++++++++++++++++++++ 9 files changed, 330 insertions(+), 115 deletions(-) delete mode 100644 tools/adb_tools.py delete mode 100755 tools/bazel-adb-run.sh create mode 100644 tools/bazel_adb_run.py create mode 100644 tools/falcon_cli.py create mode 100644 tools/sh_commands.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cbdb6fc7..ab596bca 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,9 +12,9 @@ cpplint: ops_test: stage: ops_test script: - - FAILURE_PATTERN="FAILED\|Aborted" tools/bazel-adb-run.sh //mace/ops:ops_test + - python tools/bazel_adb_run.py --target="//mace/ops:ops_test" --run_target=True --stdout_processor=ops_test_stdout_processor ops_benchmark: stage: ops_benchmark script: - - FAILURE_PATTERN="Aborted" tools/bazel-adb-run.sh //mace/ops:ops_benchmark --pattern=.*CONV.* + - python tools/bazel_adb_run.py --target="//mace/ops:ops_benchmark" --run_target=True --stdout_processor=ops_benchmark_stdout_processor --args="--filter=.*CONV.*" diff --git a/mace/core/testing/test_benchmark.cc b/mace/core/testing/test_benchmark.cc index 4a894a1d..5110bed7 100644 --- a/mace/core/testing/test_benchmark.cc +++ b/mace/core/testing/test_benchmark.cc @@ -47,6 +47,8 @@ void Benchmark::Run(const char *pattern) { width = std::max(width, b->name_.length()); } + // Internal perf regression tools depends on the output formatting, + // please keep in consistent when modifying printf("%-*s %10s %10s %10s %10s\n", width, "Benchmark", "Time(ns)", "Iterations", "Input(MB/s)", "MACC(G/s)"); printf("%s\n", std::string(width + 44, '-').c_str()); diff --git a/mace/core/testing/test_benchmark_main.cc b/mace/core/testing/test_benchmark_main.cc index 70090887..e5bc0e89 100644 --- a/mace/core/testing/test_benchmark_main.cc +++ b/mace/core/testing/test_benchmark_main.cc @@ -9,7 +9,7 @@ #include "mace/public/mace.h" #include "mace/public/mace_runtime.h" -DEFINE_string(pattern, "all", "op benchmark pattern, eg:.*CONV.*"); +DEFINE_string(filter, "all", "op benchmark regex filter, eg:.*CONV.*"); DEFINE_int32(gpu_perf_hint, 3, "0:DEFAULT/1:LOW/2:NORMAL/3:HIGH"); DEFINE_int32(gpu_priority_hint, 3, "0:DEFAULT/1:LOW/2:NORMAL/3:HIGH"); DEFINE_int32(omp_num_threads, 1, "num of openmp threads"); @@ -28,6 +28,6 @@ int main(int argc, char **argv) { static_cast(FLAGS_gpu_perf_hint), static_cast(FLAGS_gpu_priority_hint)); - mace::testing::Benchmark::Run(FLAGS_pattern.c_str()); + mace::testing::Benchmark::Run(FLAGS_filter.c_str()); return 0; } diff --git a/tools/adb_tools.py b/tools/adb_tools.py deleted file mode 100644 index 19a96f7f..00000000 --- a/tools/adb_tools.py +++ /dev/null @@ -1,33 +0,0 @@ -import sh -import re - -def strip_invalid_utf8(str): - return sh.iconv(str, "-c", "-t", "UTF-8") - -def adb_split_stdout(stdout_str): - stdout_str = strip_invalid_utf8(stdout_str) - # Filter out last empty line - return [l.strip() for l in stdout_str.split('\n') if len(l.strip()) > 0] - -def adb_devices(): - outputs = sh.grep(sh.adb("devices"), "^[A-Za-z0-9]\+[[:space:]]\+device$") - raw_lists = sh.cut(outputs, "-f1") - return adb_split_stdout(raw_lists) - -def adb_getprop_by_serialno(serialno): - outputs = sh.adb("-s", serialno, "shell", "getprop") - raw_props = adb_split_stdout(outputs) - props = {} - p = re.compile("\[(.+)\]: \[(.+)\]") - for raw_prop in raw_props: - m = p.match(raw_prop) - if m: - props[m.group(1)] = m.group(2) - return props - -def adb_get_all_socs(): - socs = [] - for d in adb_devices(): - props = adb_getprop_by_serialno(d) - socs.append(props["ro.board.platform"]) - return set(socs) diff --git a/tools/bazel-adb-run.sh b/tools/bazel-adb-run.sh deleted file mode 100755 index 2e3dede0..00000000 --- a/tools/bazel-adb-run.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash - -if [ "$#" -lt 1 ]; then - echo "$0" bazel-target [cmd params] - exit 1 -fi - -MACE_SOURCE_DIR=`/bin/pwd` -CODEGEN_DIR=${MACE_SOURCE_DIR}/mace/codegen -DEVICE_PATH=/data/local/tmp/mace -DEVICE_CL_PATH=$DEVICE_PATH/cl/ -BAZEL_TARGET=$1 -shift - -# change //mace/a/b:c to bazel-bin/mace/a/b/c -BAZEL_BIN_PATH=`echo $BAZEL_TARGET | cut -d: -f1` -BAZEL_BIN_PATH=${BAZEL_BIN_PATH#//} -BAZEL_BIN_PATH=bazel-bin/$BAZEL_BIN_PATH -BIN_NAME=`echo $BAZEL_TARGET | cut -d: -f2` - -ANDROID_ABI=arm64-v8a -ANDROID_ABI=armeabi-v7a -STRIP="--strip always" -VLOG_LEVEL=0 -PROFILING="1" - -echo "Step 1: Generate encrypted opencl source" -python mace/python/tools/encrypt_opencl_codegen.py \ - --cl_kernel_dir=./mace/kernels/opencl/cl/ --output_path=${CODEGEN_DIR}/opencl/opencl_encrypt_program.cc - -echo "Step 2: Generate version source" -mkdir -p ${CODEGEN_DIR}/version -bash mace/tools/git/gen_version_source.sh ${CODEGEN_DIR}/version/version.cc - -echo "Step 3: Build target" -# -D_GLIBCXX_USE_C99_MATH_TR1 is used to solve include error instead -# of linking error which solved by -lm -bazel build -c opt $STRIP --verbose_failures $BAZEL_TARGET \ - --crosstool_top=//external:android/crosstool \ - --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \ - --cpu=$ANDROID_ABI \ - --copt="-std=c++11" \ - --copt="-D_GLIBCXX_USE_C99_MATH_TR1" \ - --copt="-DMACE_DISABLE_NO_TUNING_WARNING" \ - --copt="-Werror=return-type" \ - --copt="-O3" \ - --define neon=true \ - --define openmp=true - -if [ $? -ne 0 ]; then - exit 1 -fi - -echo "Step 4: Run target" -du -hs $BAZEL_BIN_PATH/$BIN_NAME - -for device in `adb devices | grep "^[A-Za-z0-9]\+[[:space:]]\+device$"| cut -f1`; do - echo ====================================================================== - echo "Run on device: ${device}" - adb -s ${device} shell "rm -rf $DEVICE_PATH" - adb -s ${device} shell "mkdir -p $DEVICE_PATH" - adb -s ${device} shell "mkdir -p $DEVICE_PATH/cl" - adb -s ${device} push $BAZEL_BIN_PATH/$BIN_NAME $DEVICE_PATH && \ - TEST_LOG_FILE=`tempfile` - adb -s ${device} shell "MACE_OPENCL_PROFILING=$PROFILING MACE_KERNEL_PATH=$DEVICE_CL_PATH MACE_CPP_MIN_VLOG_LEVEL=$VLOG_LEVEL $DEVICE_PATH/$BIN_NAME $@" | tee $TEST_LOG_FILE - if [ ! -z "$FAILURE_PATTERN" ]; then - grep "$FAILURE_PATTERN" $TEST_LOG_FILE > /dev/null - if [ $? -eq 0 ]; then - exit 1 - fi - fi - rm $TEST_LOG_FILE -done diff --git a/tools/bazel_adb_run.py b/tools/bazel_adb_run.py new file mode 100644 index 00000000..c78f9b9c --- /dev/null +++ b/tools/bazel_adb_run.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python + +# Must run at root dir of libmace project. +# python tools/bazel_adb_run.py \ +# --target_abis=armeabi-v7a \ +# --target_socs=sdm845 +# --target=//mace/ops:ops_test +# --stdout_processor=stdout_processor + + +import argparse +import re +import sys + +import sh_commands + +def stdout_processor(stdout, device_properties, abi): + pass + +def ops_test_stdout_processor(stdout, device_properties, abi): + stdout_lines = stdout.split("\n") + for line in stdout_lines: + if "Aborted" in line or "FAILED" in line: + raise Exception("Command failed") + +def ops_benchmark_stdout_processor(stdout, device_properties, abi): + stdout_lines = stdout.split("\n") + metrics = {} + for line in stdout_lines: + if "Aborted" in line: + raise Exception("Command failed") + line = line.strip() + parts = line.split() + if len(parts) == 5 and parts[0].startswith("BM_"): + metrics["%s.ops_benchmark_time_ms" % parts[0]] = str(float(parts[1])/1000000.0) + metrics["%s.ops_benchmark_input_mb_per_sec" % parts[0]] = parts[3] + metrics["%s.ops_benchmark_gmacc_per_sec" % parts[0]] = parts[4] + sh_commands.falcon_push_metrics(metrics, device_properties, abi) + +def parse_args(): + """Parses command line arguments.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--target_abis", + type=str, + default="armeabi-v7a", + help="Target ABIs, comma seperated list") + parser.add_argument( + "--target_socs", + type=str, + default="all", + help="SoCs to build, comma seperated list (getprop ro.board.platform)") + parser.add_argument( + "--target", + type=str, + default="//...", + help="Bazel target to build") + parser.add_argument( + "--run_target", + type=bool, + default=False, + help="Whether to run the target") + parser.add_argument( + "--args", + type=str, + default="", + help="Command args") + parser.add_argument( + "--stdout_processor", + type=str, + default="stdout_processor", + help="Stdout processing function, default: stdout_processor") + return parser.parse_known_args() + +def main(unused_args): + target_socs = None + if FLAGS.target_socs != "all": + target_socs = set(FLAGS.target_socs.split(',')) + target_devices = sh_commands.adb_devices(target_socs=target_socs) + + target = FLAGS.target + host_bin_path, bin_name = sh_commands.bazel_target_to_bin(target) + target_abis = FLAGS.target_abis.split(',') + + # generate sources + sh_commands.gen_encrypted_opencl_source() + sh_commands.gen_mace_version() + + for target_abi in target_abis: + sh_commands.bazel_build(target, abi=target_abi) + if FLAGS.run_target: + for serialno in target_devices: + device_properties = sh_commands.adb_getprop_by_serialno(serialno) + stdouts = sh_commands.adb_run(serialno, host_bin_path, bin_name, + args=FLAGS.args, + opencl_profiling=1, + vlog_level=0, + device_bin_path="/data/local/tmp/mace") + globals()[FLAGS.stdout_processor](stdouts, device_properties, target_abi) + +if __name__ == "__main__": + FLAGS, unparsed = parse_args() + main(unused_args=[sys.argv[0]] + unparsed) diff --git a/tools/falcon_cli.py b/tools/falcon_cli.py new file mode 100644 index 00000000..7debe05c --- /dev/null +++ b/tools/falcon_cli.py @@ -0,0 +1,60 @@ +#-*- coding:utf8 -*- + +import json, socket, itertools + +class FalconCli(object): + + def __init__(self, addr, debug=True, buf_size=1000): + self.socket_ = socket.create_connection(addr) + self.stream = self.socket_.makefile() + self.id_counter = itertools.count() + self.debug = debug + self.buf_size = buf_size + + def __del__(self): + self.socket_.close() + self.stream.close() + + @classmethod + def connect(cls, server="transfer.falcon.miliao.srv", port=8433, debug=True, buf_size=1000): + try: + return FalconCli((server, port), debug, buf_size) + except socket.error, exc: + print "error: connect to %s:%s error: %s" %(server, port, exc) + + def call(self, name, *params): + request = dict(id=next(self.id_counter), + params=list(params), + method=name) + payload = json.dumps(request).encode() + if self.debug: + print "--> req:", payload + self.socket_.sendall(payload) + + response = self.stream.readline() + if not response: + raise Exception('empty response') + + if self.debug: + print "<-- resp:", response + + response = json.loads(response.decode("utf8")) + if response.get('error') is not None: + raise Exception(response.get('error')) + + return response.get('result') + + def update(self, lines): + s = 0 + resp = [] + + while True: + buf = lines[s:s+self.buf_size] + s = s + self.buf_size + if len(buf) == 0: + break + r = self.call("Transfer.Update", buf) + resp.append(r) + + return resp + diff --git a/tools/mace_tools.py b/tools/mace_tools.py index f6892fa1..f9dd9181 100644 --- a/tools/mace_tools.py +++ b/tools/mace_tools.py @@ -16,7 +16,7 @@ import urllib import yaml import re -import adb_tools +import sh_commands from ConfigParser import ConfigParser @@ -237,7 +237,7 @@ def parse_args(): default="all", help="[build|run|validate|merge|all|throughput_test].") parser.add_argument( - "--socs", + "--target_socs", type=str, default="all", help="SoCs to build, comma seperated list (getprop ro.board.platform)") @@ -271,14 +271,14 @@ def main(unused_args): generate_opencl_and_version_code() option_args = ' '.join([arg for arg in unused_args if arg.startswith('--')]) - available_socs = adb_tools.adb_get_all_socs() + available_socs = sh_commands.adb_get_all_socs() target_socs = available_socs if hasattr(configs, "target_socs"): target_socs = set(configs["target_socs"]) target_socs = target_socs & available_socs - if FLAGS.socs != "all": - socs = set(FLAGS.socs.split(',')) + if FLAGS.target_socs != "all": + socs = set(FLAGS.target_socs.split(',')) target_socs = target_socs & socs missing_socs = socs.difference(target_socs) if len(missing_socs) > 0: diff --git a/tools/sh_commands.py b/tools/sh_commands.py new file mode 100644 index 00000000..fe5bc1ca --- /dev/null +++ b/tools/sh_commands.py @@ -0,0 +1,156 @@ +import sh +import re +import time +import falcon_cli + +################################ +# common +################################ +def strip_invalid_utf8(str): + return sh.iconv(str, "-c", "-t", "UTF-8") + +def make_output_processor(buff): + def process_output(line): + print(line.strip()) + buff.append(line) + return process_output + +################################ +# adb commands +################################ +def adb_split_stdout(stdout_str): + stdout_str = strip_invalid_utf8(stdout_str) + # Filter out last empty line + return [l.strip() for l in stdout_str.split('\n') if len(l.strip()) > 0] + +def adb_devices(target_socs=None): + outputs = sh.grep(sh.adb("devices"), "^[A-Za-z0-9]\+[[:space:]]\+device$") + raw_lists = sh.cut(outputs, "-f1") + device_ids = adb_split_stdout(raw_lists) + if target_socs != None: + target_socs_set = set(target_socs) + target_devices = [] + for serialno in device_ids: + props = adb_getprop_by_serialno(serialno) + if props["ro.board.platform"] in target_socs_set: + target_devices.append(serialno) + return target_devices + else: + return device_ids + +def adb_getprop_by_serialno(serialno): + outputs = sh.adb("-s", serialno, "shell", "getprop") + raw_props = adb_split_stdout(outputs) + props = {} + p = re.compile("\[(.+)\]: \[(.+)\]") + for raw_prop in raw_props: + m = p.match(raw_prop) + if m: + props[m.group(1)] = m.group(2) + return props + +def adb_get_all_socs(): + socs = [] + for d in adb_devices(): + props = adb_getprop_by_serialno(d) + socs.append(props["ro.board.platform"]) + return set(socs) + +def adb_run(serialno, host_bin_path, bin_name, + args="", + opencl_profiling=1, + vlog_level=0, + device_bin_path="/data/local/tmp/mace"): + host_bin_full_path = "%s/%s" % (host_bin_path, bin_name) + device_bin_full_path = "%s/%s" % (device_bin_path, bin_name) + device_cl_path = "%s/cl" % device_bin_path + props = adb_getprop_by_serialno(serialno) + print("Run on device: %s, %s, %s" % (serialno, props["ro.board.platform"], props["ro.product.model"])) + sh.adb("-s", serialno, "shell", "rm -rf %s" % device_bin_path) + sh.adb("-s", serialno, "shell", "mkdir -p %s" % device_bin_path) + sh.adb("-s", serialno, "shell", "mkdir -p %s" % device_cl_path) + print("Push %s to device" % device_bin_full_path) + sh.adb("-s", serialno, "push", host_bin_full_path, device_bin_path) + print("Run %s on device" % device_bin_full_path) + stdout_buff=[] + process_output = make_output_processor(stdout_buff) + p = sh.adb("-s", serialno, "shell", + "MACE_OPENCL_PROFILING=%d MACE_KERNEL_PATH=%s MACE_CPP_MIN_VLOG_LEVEL=%d %s %s" % (opencl_profiling, device_cl_path, vlog_level, device_bin_full_path, args), + _out=process_output, _bg=True) + p.wait() + return "".join(stdout_buff) + + +################################ +# bazel commands +################################ +def bazel_build(target, strip="always", abi="armeabi-v7a"): + stdout_buff=[] + process_output = make_output_processor(stdout_buff) + p= sh.bazel("build", + "-c", "opt", + "--strip", strip, + "--verbose_failures", + target, + "--crosstool_top=//external:android/crosstool", + "--host_crosstool_top=@bazel_tools//tools/cpp:toolchain", + "--cpu=%s" % abi, + "--copt=-std=c++11", + "--copt=-D_GLIBCXX_USE_C99_MATH_TR1", + "--copt=-DMACE_DISABLE_NO_TUNING_WARNING", + "--copt=-Werror=return-type", + "--copt=-O3", + "--define", "neon=true", + "--define", "openmp=true", + _out=process_output, _bg=True) + p.wait() + return "".join(stdout_buff) + +def bazel_target_to_bin(target): + # change //mace/a/b:c to bazel-bin/mace/a/b/c + prefix, bin_name = target.split(':') + prefix = prefix.replace('//', '/') + if prefix.startswith('/'): + prefix = prefix[1:] + host_bin_path = "bazel-bin/%s" % prefix + return host_bin_path, bin_name + +################################ +# mace commands +################################ +# TODO this should be refactored +def gen_encrypted_opencl_source(codegen_path="mace/codegen"): + sh.python("mace/python/tools/encrypt_opencl_codegen.py", + "--cl_kernel_dir=./mace/kernels/opencl/cl/", + "--output_path=%s/opencl/opencl_encrypt_program.cc" % codegen_path) + +def gen_mace_version(codegen_path="mace/codegen"): + sh.mkdir("-p", "%s/version" % codegen_path) + sh.bash("mace/tools/git/gen_version_source.sh", + "%s/version/version.cc" % codegen_path) + +################################ +# falcon +################################ +def falcon_tags(platform, model, abi): + return "ro.board.platform=%s,ro.product.model=%s,abi=%s" % (platform, model, abi) + +def falcon_push_metrics(metrics, device_properties, abi): + cli = falcon_cli.FalconCli.connect(server="transfer.falcon.miliao.srv", + port=8433, + debug=False) + platform = device_properties["ro.board.platform"].replace(" ", "-") + model = device_properties["ro.product.model"].replace(" ", "-") + tags = falcon_tags(platform, model, abi) + ts = int(time.time()) + falcon_metrics = [{ + "endpoint": "mace_dev", + "metric": key, + "tags": tags, + "timestamp": ts, + "value": value, + "step": 3600, + "counterType": "GAUGE" + } for key, value in metrics.iteritems()] + cli.update(falcon_metrics) + -- GitLab