diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cbdb6fc7414c3316f7a1eac9ecba82f39efbcb31..b66f8bc33ebad368a3849b9047a671c085c021ff 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,9 +12,11 @@ cpplint: ops_test: stage: ops_test script: - - FAILURE_PATTERN="FAILED\|Aborted" tools/bazel-adb-run.sh //mace/ops:ops_test + - if [ -z "$TARGET_SOCS" ]; then TARGET_SOCS=random; fi + - python tools/bazel_adb_run.py --target="//mace/ops:ops_test" --run_target=True --stdout_processor=ops_test_stdout_processor --target_socs=$TARGET_SOCS ops_benchmark: stage: ops_benchmark script: - - FAILURE_PATTERN="Aborted" tools/bazel-adb-run.sh //mace/ops:ops_benchmark --pattern=.*CONV.* + - if [ -z "$TARGET_SOCS" ]; then TARGET_SOCS=random; fi + - python tools/bazel_adb_run.py --target="//mace/ops:ops_benchmark" --run_target=True --stdout_processor=ops_benchmark_stdout_processor --target_socs=$TARGET_SOCS diff --git a/mace/core/testing/test_benchmark.cc b/mace/core/testing/test_benchmark.cc index 4a894a1d4ff39d421bc05ff1630759f4bf1bb4d7..0d1b68fe664d7bab859c7f6474b49ea730a37b61 100644 --- a/mace/core/testing/test_benchmark.cc +++ b/mace/core/testing/test_benchmark.cc @@ -33,6 +33,11 @@ void Benchmark::Run() { Run("all"); } void Benchmark::Run(const char *pattern) { if (!all_benchmarks) return; + std::sort(all_benchmarks->begin(), all_benchmarks->end(), + [](const Benchmark *lhs, const Benchmark *rhs) { + return lhs->name_ < rhs->name_; + }); + if (std::string(pattern) == "all") { pattern = ".*"; } @@ -47,6 +52,8 @@ void Benchmark::Run(const char *pattern) { width = std::max(width, b->name_.length()); } + // Internal perf regression tools depends on the output formatting, + // please keep in consistent when modifying printf("%-*s %10s %10s %10s %10s\n", width, "Benchmark", "Time(ns)", "Iterations", "Input(MB/s)", "MACC(G/s)"); printf("%s\n", std::string(width + 44, '-').c_str()); diff --git a/mace/core/testing/test_benchmark_main.cc b/mace/core/testing/test_benchmark_main.cc index 70090887a87c8da264049ba7ae14f9c6bd7f25f9..e5bc0e896f79ecd0e6a3a9e3de46461b57ad0094 100644 --- a/mace/core/testing/test_benchmark_main.cc +++ b/mace/core/testing/test_benchmark_main.cc @@ -9,7 +9,7 @@ #include "mace/public/mace.h" #include "mace/public/mace_runtime.h" -DEFINE_string(pattern, "all", "op benchmark pattern, eg:.*CONV.*"); +DEFINE_string(filter, "all", "op benchmark regex filter, eg:.*CONV.*"); DEFINE_int32(gpu_perf_hint, 3, "0:DEFAULT/1:LOW/2:NORMAL/3:HIGH"); DEFINE_int32(gpu_priority_hint, 3, "0:DEFAULT/1:LOW/2:NORMAL/3:HIGH"); DEFINE_int32(omp_num_threads, 1, "num of openmp threads"); @@ -28,6 +28,6 @@ int main(int argc, char **argv) { static_cast(FLAGS_gpu_perf_hint), static_cast(FLAGS_gpu_priority_hint)); - mace::testing::Benchmark::Run(FLAGS_pattern.c_str()); + mace::testing::Benchmark::Run(FLAGS_filter.c_str()); return 0; } diff --git a/tools/adb_tools.py b/tools/adb_tools.py deleted file mode 100644 index 19a96f7ffae1feedd33faaf7a1dd12b3f0fe3251..0000000000000000000000000000000000000000 --- a/tools/adb_tools.py +++ /dev/null @@ -1,33 +0,0 @@ -import sh -import re - -def strip_invalid_utf8(str): - return sh.iconv(str, "-c", "-t", "UTF-8") - -def adb_split_stdout(stdout_str): - stdout_str = strip_invalid_utf8(stdout_str) - # Filter out last empty line - return [l.strip() for l in stdout_str.split('\n') if len(l.strip()) > 0] - -def adb_devices(): - outputs = sh.grep(sh.adb("devices"), "^[A-Za-z0-9]\+[[:space:]]\+device$") - raw_lists = sh.cut(outputs, "-f1") - return adb_split_stdout(raw_lists) - -def adb_getprop_by_serialno(serialno): - outputs = sh.adb("-s", serialno, "shell", "getprop") - raw_props = adb_split_stdout(outputs) - props = {} - p = re.compile("\[(.+)\]: \[(.+)\]") - for raw_prop in raw_props: - m = p.match(raw_prop) - if m: - props[m.group(1)] = m.group(2) - return props - -def adb_get_all_socs(): - socs = [] - for d in adb_devices(): - props = adb_getprop_by_serialno(d) - socs.append(props["ro.board.platform"]) - return set(socs) diff --git a/tools/bazel-adb-run.sh b/tools/bazel-adb-run.sh deleted file mode 100755 index 2e3dede092a6ddb0852ae2e75730310fd91ddeec..0000000000000000000000000000000000000000 --- a/tools/bazel-adb-run.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash - -if [ "$#" -lt 1 ]; then - echo "$0" bazel-target [cmd params] - exit 1 -fi - -MACE_SOURCE_DIR=`/bin/pwd` -CODEGEN_DIR=${MACE_SOURCE_DIR}/mace/codegen -DEVICE_PATH=/data/local/tmp/mace -DEVICE_CL_PATH=$DEVICE_PATH/cl/ -BAZEL_TARGET=$1 -shift - -# change //mace/a/b:c to bazel-bin/mace/a/b/c -BAZEL_BIN_PATH=`echo $BAZEL_TARGET | cut -d: -f1` -BAZEL_BIN_PATH=${BAZEL_BIN_PATH#//} -BAZEL_BIN_PATH=bazel-bin/$BAZEL_BIN_PATH -BIN_NAME=`echo $BAZEL_TARGET | cut -d: -f2` - -ANDROID_ABI=arm64-v8a -ANDROID_ABI=armeabi-v7a -STRIP="--strip always" -VLOG_LEVEL=0 -PROFILING="1" - -echo "Step 1: Generate encrypted opencl source" -python mace/python/tools/encrypt_opencl_codegen.py \ - --cl_kernel_dir=./mace/kernels/opencl/cl/ --output_path=${CODEGEN_DIR}/opencl/opencl_encrypt_program.cc - -echo "Step 2: Generate version source" -mkdir -p ${CODEGEN_DIR}/version -bash mace/tools/git/gen_version_source.sh ${CODEGEN_DIR}/version/version.cc - -echo "Step 3: Build target" -# -D_GLIBCXX_USE_C99_MATH_TR1 is used to solve include error instead -# of linking error which solved by -lm -bazel build -c opt $STRIP --verbose_failures $BAZEL_TARGET \ - --crosstool_top=//external:android/crosstool \ - --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \ - --cpu=$ANDROID_ABI \ - --copt="-std=c++11" \ - --copt="-D_GLIBCXX_USE_C99_MATH_TR1" \ - --copt="-DMACE_DISABLE_NO_TUNING_WARNING" \ - --copt="-Werror=return-type" \ - --copt="-O3" \ - --define neon=true \ - --define openmp=true - -if [ $? -ne 0 ]; then - exit 1 -fi - -echo "Step 4: Run target" -du -hs $BAZEL_BIN_PATH/$BIN_NAME - -for device in `adb devices | grep "^[A-Za-z0-9]\+[[:space:]]\+device$"| cut -f1`; do - echo ====================================================================== - echo "Run on device: ${device}" - adb -s ${device} shell "rm -rf $DEVICE_PATH" - adb -s ${device} shell "mkdir -p $DEVICE_PATH" - adb -s ${device} shell "mkdir -p $DEVICE_PATH/cl" - adb -s ${device} push $BAZEL_BIN_PATH/$BIN_NAME $DEVICE_PATH && \ - TEST_LOG_FILE=`tempfile` - adb -s ${device} shell "MACE_OPENCL_PROFILING=$PROFILING MACE_KERNEL_PATH=$DEVICE_CL_PATH MACE_CPP_MIN_VLOG_LEVEL=$VLOG_LEVEL $DEVICE_PATH/$BIN_NAME $@" | tee $TEST_LOG_FILE - if [ ! -z "$FAILURE_PATTERN" ]; then - grep "$FAILURE_PATTERN" $TEST_LOG_FILE > /dev/null - if [ $? -eq 0 ]; then - exit 1 - fi - fi - rm $TEST_LOG_FILE -done diff --git a/tools/bazel_adb_run.py b/tools/bazel_adb_run.py new file mode 100644 index 0000000000000000000000000000000000000000..675a20dd79ac9a8349ad6514204b78168724e778 --- /dev/null +++ b/tools/bazel_adb_run.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python + +# Must run at root dir of libmace project. +# python tools/bazel_adb_run.py \ +# --target_abis=armeabi-v7a \ +# --target_socs=sdm845 +# --target=//mace/ops:ops_test +# --stdout_processor=stdout_processor + + +import argparse +import random +import re +import sys + +import sh_commands + +def stdout_processor(stdout, device_properties, abi): + pass + +def ops_test_stdout_processor(stdout, device_properties, abi): + stdout_lines = stdout.split("\n") + for line in stdout_lines: + if "Aborted" in line or "FAILED" in line: + raise Exception("Command failed") + +def ops_benchmark_stdout_processor(stdout, device_properties, abi): + stdout_lines = stdout.split("\n") + metrics = {} + for line in stdout_lines: + if "Aborted" in line: + raise Exception("Command failed") + line = line.strip() + parts = line.split() + if len(parts) == 5 and parts[0].startswith("BM_"): + metrics["%s.time_ms" % parts[0]] = str(float(parts[1])/1000000.0) + metrics["%s.input_mb_per_sec" % parts[0]] = parts[3] + metrics["%s.gmacc_per_sec" % parts[0]] = parts[4] + sh_commands.falcon_push_metrics(metrics, device_properties, abi, + endpoint="mace_ops_benchmark") + +def parse_args(): + """Parses command line arguments.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--target_abis", + type=str, + default="armeabi-v7a", + help="Target ABIs, comma seperated list") + parser.add_argument( + "--target_socs", + type=str, + default="all", + help="SoCs(ro.board.platform) to build, comma seperated list or all/random") + parser.add_argument( + "--target", + type=str, + default="//...", + help="Bazel target to build") + parser.add_argument( + "--run_target", + type=bool, + default=False, + help="Whether to run the target") + parser.add_argument( + "--args", + type=str, + default="", + help="Command args") + parser.add_argument( + "--stdout_processor", + type=str, + default="stdout_processor", + help="Stdout processing function, default: stdout_processor") + return parser.parse_known_args() + +def main(unused_args): + target_socs = None + if FLAGS.target_socs != "all" and FLAGS.target_socs != "random": + target_socs = set(FLAGS.target_socs.split(',')) + target_devices = sh_commands.adb_devices(target_socs=target_socs) + if FLAGS.target_socs == "random": + target_devices = [random.choice(target_devices)] + + target = FLAGS.target + host_bin_path, bin_name = sh_commands.bazel_target_to_bin(target) + target_abis = FLAGS.target_abis.split(',') + + # generate sources + sh_commands.gen_encrypted_opencl_source() + sh_commands.gen_mace_version() + + for target_abi in target_abis: + sh_commands.bazel_build(target, abi=target_abi) + if FLAGS.run_target: + for serialno in target_devices: + device_properties = sh_commands.adb_getprop_by_serialno(serialno) + stdouts = sh_commands.adb_run(serialno, host_bin_path, bin_name, + args=FLAGS.args, + opencl_profiling=1, + vlog_level=0, + device_bin_path="/data/local/tmp/mace") + globals()[FLAGS.stdout_processor](stdouts, device_properties, target_abi) + +if __name__ == "__main__": + FLAGS, unparsed = parse_args() + main(unused_args=[sys.argv[0]] + unparsed) diff --git a/tools/falcon_cli.py b/tools/falcon_cli.py new file mode 100644 index 0000000000000000000000000000000000000000..7debe05c95d0a1d4772b50844c59705a7ac56479 --- /dev/null +++ b/tools/falcon_cli.py @@ -0,0 +1,60 @@ +#-*- coding:utf8 -*- + +import json, socket, itertools + +class FalconCli(object): + + def __init__(self, addr, debug=True, buf_size=1000): + self.socket_ = socket.create_connection(addr) + self.stream = self.socket_.makefile() + self.id_counter = itertools.count() + self.debug = debug + self.buf_size = buf_size + + def __del__(self): + self.socket_.close() + self.stream.close() + + @classmethod + def connect(cls, server="transfer.falcon.miliao.srv", port=8433, debug=True, buf_size=1000): + try: + return FalconCli((server, port), debug, buf_size) + except socket.error, exc: + print "error: connect to %s:%s error: %s" %(server, port, exc) + + def call(self, name, *params): + request = dict(id=next(self.id_counter), + params=list(params), + method=name) + payload = json.dumps(request).encode() + if self.debug: + print "--> req:", payload + self.socket_.sendall(payload) + + response = self.stream.readline() + if not response: + raise Exception('empty response') + + if self.debug: + print "<-- resp:", response + + response = json.loads(response.decode("utf8")) + if response.get('error') is not None: + raise Exception(response.get('error')) + + return response.get('result') + + def update(self, lines): + s = 0 + resp = [] + + while True: + buf = lines[s:s+self.buf_size] + s = s + self.buf_size + if len(buf) == 0: + break + r = self.call("Transfer.Update", buf) + resp.append(r) + + return resp + diff --git a/tools/mace_tools.py b/tools/mace_tools.py index f6892fa1e0727a9aa16a84865f77d35f1772b116..f9dd9181b2e55717598b9d526ccf4505750ee0e0 100644 --- a/tools/mace_tools.py +++ b/tools/mace_tools.py @@ -16,7 +16,7 @@ import urllib import yaml import re -import adb_tools +import sh_commands from ConfigParser import ConfigParser @@ -237,7 +237,7 @@ def parse_args(): default="all", help="[build|run|validate|merge|all|throughput_test].") parser.add_argument( - "--socs", + "--target_socs", type=str, default="all", help="SoCs to build, comma seperated list (getprop ro.board.platform)") @@ -271,14 +271,14 @@ def main(unused_args): generate_opencl_and_version_code() option_args = ' '.join([arg for arg in unused_args if arg.startswith('--')]) - available_socs = adb_tools.adb_get_all_socs() + available_socs = sh_commands.adb_get_all_socs() target_socs = available_socs if hasattr(configs, "target_socs"): target_socs = set(configs["target_socs"]) target_socs = target_socs & available_socs - if FLAGS.socs != "all": - socs = set(FLAGS.socs.split(',')) + if FLAGS.target_socs != "all": + socs = set(FLAGS.target_socs.split(',')) target_socs = target_socs & socs missing_socs = socs.difference(target_socs) if len(missing_socs) > 0: diff --git a/tools/sh_commands.py b/tools/sh_commands.py new file mode 100644 index 0000000000000000000000000000000000000000..d7e55e39d552f5c152568648069d40692eceb958 --- /dev/null +++ b/tools/sh_commands.py @@ -0,0 +1,159 @@ +import sh +import re +import time +import falcon_cli + +################################ +# common +################################ +def strip_invalid_utf8(str): + return sh.iconv(str, "-c", "-t", "UTF-8") + +def make_output_processor(buff): + def process_output(line): + print(line.strip()) + buff.append(line) + return process_output + +################################ +# adb commands +################################ +def adb_split_stdout(stdout_str): + stdout_str = strip_invalid_utf8(stdout_str) + # Filter out last empty line + return [l.strip() for l in stdout_str.split('\n') if len(l.strip()) > 0] + +def adb_devices(target_socs=None): + outputs = sh.grep(sh.adb("devices"), "^[A-Za-z0-9]\+[[:space:]]\+device$") + raw_lists = sh.cut(outputs, "-f1") + device_ids = adb_split_stdout(raw_lists) + if target_socs != None: + target_socs_set = set(target_socs) + target_devices = [] + for serialno in device_ids: + props = adb_getprop_by_serialno(serialno) + if props["ro.board.platform"] in target_socs_set: + target_devices.append(serialno) + return target_devices + else: + return device_ids + +def adb_getprop_by_serialno(serialno): + outputs = sh.adb("-s", serialno, "shell", "getprop") + raw_props = adb_split_stdout(outputs) + props = {} + p = re.compile("\[(.+)\]: \[(.+)\]") + for raw_prop in raw_props: + m = p.match(raw_prop) + if m: + props[m.group(1)] = m.group(2) + return props + +def adb_get_all_socs(): + socs = [] + for d in adb_devices(): + props = adb_getprop_by_serialno(d) + socs.append(props["ro.board.platform"]) + return set(socs) + +def adb_run(serialno, host_bin_path, bin_name, + args="", + opencl_profiling=1, + vlog_level=0, + device_bin_path="/data/local/tmp/mace"): + host_bin_full_path = "%s/%s" % (host_bin_path, bin_name) + device_bin_full_path = "%s/%s" % (device_bin_path, bin_name) + device_cl_path = "%s/cl" % device_bin_path + props = adb_getprop_by_serialno(serialno) + print("=====================================================================") + print("Run on device: %s, %s, %s" % (serialno, props["ro.board.platform"], + props["ro.product.model"])) + sh.adb("-s", serialno, "shell", "rm -rf %s" % device_bin_path) + sh.adb("-s", serialno, "shell", "mkdir -p %s" % device_bin_path) + sh.adb("-s", serialno, "shell", "mkdir -p %s" % device_cl_path) + print("Push %s to %s" % (host_bin_full_path, device_bin_full_path)) + sh.adb("-s", serialno, "push", host_bin_full_path, device_bin_path) + print("Run %s" % device_bin_full_path) + stdout_buff=[] + process_output = make_output_processor(stdout_buff) + p = sh.adb("-s", serialno, "shell", + "MACE_OPENCL_PROFILING=%d MACE_KERNEL_PATH=%s MACE_CPP_MIN_VLOG_LEVEL=%d %s %s" % + (opencl_profiling, device_cl_path, vlog_level, device_bin_full_path, args), + _out=process_output, _bg=True, _err_to_out=True) + p.wait() + return "".join(stdout_buff) + + +################################ +# bazel commands +################################ +def bazel_build(target, strip="always", abi="armeabi-v7a"): + stdout_buff=[] + process_output = make_output_processor(stdout_buff) + p= sh.bazel("build", + "-c", "opt", + "--strip", strip, + "--verbose_failures", + target, + "--crosstool_top=//external:android/crosstool", + "--host_crosstool_top=@bazel_tools//tools/cpp:toolchain", + "--cpu=%s" % abi, + "--copt=-std=c++11", + "--copt=-D_GLIBCXX_USE_C99_MATH_TR1", + "--copt=-DMACE_DISABLE_NO_TUNING_WARNING", + "--copt=-Werror=return-type", + "--copt=-O3", + "--define", "neon=true", + "--define", "openmp=true", + _out=process_output, _bg=True, _err_to_out=True) + p.wait() + return "".join(stdout_buff) + +def bazel_target_to_bin(target): + # change //mace/a/b:c to bazel-bin/mace/a/b/c + prefix, bin_name = target.split(':') + prefix = prefix.replace('//', '/') + if prefix.startswith('/'): + prefix = prefix[1:] + host_bin_path = "bazel-bin/%s" % prefix + return host_bin_path, bin_name + +################################ +# mace commands +################################ +# TODO this should be refactored +def gen_encrypted_opencl_source(codegen_path="mace/codegen"): + sh.python("mace/python/tools/encrypt_opencl_codegen.py", + "--cl_kernel_dir=./mace/kernels/opencl/cl/", + "--output_path=%s/opencl/opencl_encrypt_program.cc" % codegen_path) + +def gen_mace_version(codegen_path="mace/codegen"): + sh.mkdir("-p", "%s/version" % codegen_path) + sh.bash("mace/tools/git/gen_version_source.sh", + "%s/version/version.cc" % codegen_path) + +################################ +# falcon +################################ +def falcon_tags(platform, model, abi): + return "ro.board.platform=%s,ro.product.model=%s,abi=%s" % (platform, model, abi) + +def falcon_push_metrics(metrics, device_properties, abi, endpoint="mace_dev"): + cli = falcon_cli.FalconCli.connect(server="transfer.falcon.miliao.srv", + port=8433, + debug=False) + platform = device_properties["ro.board.platform"].replace(" ", "-") + model = device_properties["ro.product.model"].replace(" ", "-") + tags = falcon_tags(platform, model, abi) + ts = int(time.time()) + falcon_metrics = [{ + "endpoint": endpoint, + "metric": key, + "tags": tags, + "timestamp": ts, + "value": value, + "step": 86400, + "counterType": "GAUGE" + } for key, value in metrics.iteritems()] + cli.update(falcon_metrics) +