Refactor bazel run tools and add metrics logging

.gitlab-ci.yml

@@ -12,9 +12,9 @@ cpplint:
 ops_test:
   stage: ops_test
   script:
-    - FAILURE_PATTERN="FAILED\|Aborted" tools/bazel-adb-run.sh //mace/ops:ops_test
+    - python tools/bazel_adb_run.py --target="//mace/ops:ops_test" --run_target=True --stdout_processor=ops_test_stdout_processor
 
 ops_benchmark:
   stage: ops_benchmark
   script:
-    - FAILURE_PATTERN="Aborted" tools/bazel-adb-run.sh //mace/ops:ops_benchmark --pattern=.*CONV.*
+    - python tools/bazel_adb_run.py --target="//mace/ops:ops_benchmark" --run_target=True --stdout_processor=ops_benchmark_stdout_processor --args="--filter=.*CONV.*"

mace/core/testing/test_benchmark.cc

@@ -47,6 +47,8 @@ void Benchmark::Run(const char *pattern) {
     width = std::max<int>(width, b->name_.length());
   }
 
+  // Internal perf regression tools depends on the output formatting,
+  // please keep in consistent when modifying
   printf("%-*s %10s %10s %10s %10s\n", width, "Benchmark", "Time(ns)",
          "Iterations", "Input(MB/s)", "MACC(G/s)");
   printf("%s\n", std::string(width + 44, '-').c_str());

mace/core/testing/test_benchmark_main.cc

@@ -9,7 +9,7 @@
 #include "mace/public/mace.h"
 #include "mace/public/mace_runtime.h"
 
-DEFINE_string(pattern, "all", "op benchmark pattern, eg:.*CONV.*");
+DEFINE_string(filter, "all", "op benchmark regex filter, eg:.*CONV.*");
 DEFINE_int32(gpu_perf_hint, 3, "0:DEFAULT/1:LOW/2:NORMAL/3:HIGH");
 DEFINE_int32(gpu_priority_hint, 3, "0:DEFAULT/1:LOW/2:NORMAL/3:HIGH");
 DEFINE_int32(omp_num_threads, 1, "num of openmp threads");
@@ -28,6 +28,6 @@ int main(int argc, char **argv) {
       static_cast<mace::GPUPerfHint>(FLAGS_gpu_perf_hint),
       static_cast<mace::GPUPriorityHint>(FLAGS_gpu_priority_hint));
 
-  mace::testing::Benchmark::Run(FLAGS_pattern.c_str());
+  mace::testing::Benchmark::Run(FLAGS_filter.c_str());
   return 0;
 }

tools/adb_tools.py

-import sh
-import re
-
-def strip_invalid_utf8(str):
-  return sh.iconv(str, "-c", "-t", "UTF-8")
-
-def adb_split_stdout(stdout_str):
-  stdout_str = strip_invalid_utf8(stdout_str)
-  # Filter out last empty line
-  return [l.strip() for l in stdout_str.split('\n') if len(l.strip()) > 0]
-
-def adb_devices():
-  outputs = sh.grep(sh.adb("devices"), "^[A-Za-z0-9]\+[[:space:]]\+device$")
-  raw_lists = sh.cut(outputs, "-f1")
-  return adb_split_stdout(raw_lists)
-
-def adb_getprop_by_serialno(serialno):
-  outputs = sh.adb("-s", serialno, "shell", "getprop")
-  raw_props = adb_split_stdout(outputs)
-  props = {}
-  p = re.compile("\[(.+)\]: \[(.+)\]")
-  for raw_prop in raw_props:
-    m = p.match(raw_prop)
-    if m:
-      props[m.group(1)] = m.group(2)
-  return props
-
-def adb_get_all_socs():
-  socs = []
-  for d in adb_devices():
-    props = adb_getprop_by_serialno(d)
-    socs.append(props["ro.board.platform"])
-  return set(socs)

tools/bazel-adb-run.sh

-#!/bin/bash
-
-if [ "$#" -lt 1 ]; then
-  echo "$0" bazel-target [cmd params]
-  exit 1
-fi
-
-MACE_SOURCE_DIR=`/bin/pwd`
-CODEGEN_DIR=${MACE_SOURCE_DIR}/mace/codegen
-DEVICE_PATH=/data/local/tmp/mace
-DEVICE_CL_PATH=$DEVICE_PATH/cl/
-BAZEL_TARGET=$1
-shift
-
-# change //mace/a/b:c to bazel-bin/mace/a/b/c
-BAZEL_BIN_PATH=`echo $BAZEL_TARGET | cut -d: -f1`
-BAZEL_BIN_PATH=${BAZEL_BIN_PATH#//}
-BAZEL_BIN_PATH=bazel-bin/$BAZEL_BIN_PATH
-BIN_NAME=`echo $BAZEL_TARGET | cut -d: -f2`
-
-ANDROID_ABI=arm64-v8a
-ANDROID_ABI=armeabi-v7a
-STRIP="--strip always"
-VLOG_LEVEL=0
-PROFILING="1"
-
-echo "Step 1: Generate encrypted opencl source"
-python mace/python/tools/encrypt_opencl_codegen.py \
-    --cl_kernel_dir=./mace/kernels/opencl/cl/ --output_path=${CODEGEN_DIR}/opencl/opencl_encrypt_program.cc
-
-echo "Step 2: Generate version source"
-mkdir -p ${CODEGEN_DIR}/version
-bash mace/tools/git/gen_version_source.sh ${CODEGEN_DIR}/version/version.cc
-
-echo "Step 3: Build target"
-# -D_GLIBCXX_USE_C99_MATH_TR1 is used to solve include error instead
-# of linking error which solved by -lm
-bazel build -c opt $STRIP --verbose_failures $BAZEL_TARGET \
-   --crosstool_top=//external:android/crosstool \
-   --host_crosstool_top=@bazel_tools//tools/cpp:toolchain  \
-   --cpu=$ANDROID_ABI \
-   --copt="-std=c++11" \
-   --copt="-D_GLIBCXX_USE_C99_MATH_TR1" \
-   --copt="-DMACE_DISABLE_NO_TUNING_WARNING" \
-   --copt="-Werror=return-type" \
-   --copt="-O3" \
-   --define neon=true \
-   --define openmp=true
-
-if [ $? -ne 0 ]; then
-  exit 1
-fi
-
-echo "Step 4: Run target"
-du -hs $BAZEL_BIN_PATH/$BIN_NAME
-
-for device in `adb devices | grep "^[A-Za-z0-9]\+[[:space:]]\+device$"| cut -f1`; do
-  echo ======================================================================
-  echo "Run on device: ${device}"
-  adb -s ${device} shell "rm -rf $DEVICE_PATH"
-  adb -s ${device} shell "mkdir -p $DEVICE_PATH"
-  adb -s ${device} shell "mkdir -p $DEVICE_PATH/cl"
-  adb -s ${device} push $BAZEL_BIN_PATH/$BIN_NAME $DEVICE_PATH && \
-  TEST_LOG_FILE=`tempfile`
-  adb -s ${device} shell "MACE_OPENCL_PROFILING=$PROFILING MACE_KERNEL_PATH=$DEVICE_CL_PATH MACE_CPP_MIN_VLOG_LEVEL=$VLOG_LEVEL $DEVICE_PATH/$BIN_NAME $@" | tee $TEST_LOG_FILE 
-  if [ ! -z "$FAILURE_PATTERN" ]; then
-    grep "$FAILURE_PATTERN" $TEST_LOG_FILE > /dev/null
-    if [ $? -eq 0 ]; then
-      exit 1
-    fi
-  fi
-  rm $TEST_LOG_FILE
-done

tools/bazel_adb_run.py

+#!/usr/bin/env python
+
+# Must run at root dir of libmace project.
+# python tools/bazel_adb_run.py \
+#     --target_abis=armeabi-v7a \
+#     --target_socs=sdm845
+#     --target=//mace/ops:ops_test
+#     --stdout_processor=stdout_processor
+
+
+import argparse
+import re
+import sys
+
+import sh_commands
+
+def stdout_processor(stdout, device_properties, abi):
+  pass
+
+def ops_test_stdout_processor(stdout, device_properties, abi):
+  stdout_lines = stdout.split("\n")
+  for line in stdout_lines:
+    if "Aborted" in line or "FAILED" in line:
+      raise Exception("Command failed")
+
+def ops_benchmark_stdout_processor(stdout, device_properties, abi):
+  stdout_lines = stdout.split("\n")
+  metrics = {}
+  for line in stdout_lines:
+    if "Aborted" in line:
+      raise Exception("Command failed")
+    line = line.strip()
+    parts = line.split()
+    if len(parts) == 5 and parts[0].startswith("BM_"):
+      metrics["%s.ops_benchmark_time_ms" % parts[0]] = str(float(parts[1])/1000000.0)
+      metrics["%s.ops_benchmark_input_mb_per_sec" % parts[0]] = parts[3]
+      metrics["%s.ops_benchmark_gmacc_per_sec" % parts[0]] = parts[4]
+  sh_commands.falcon_push_metrics(metrics, device_properties, abi)
+
+def parse_args():
+  """Parses command line arguments."""
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
+      "--target_abis",
+      type=str,
+      default="armeabi-v7a",
+      help="Target ABIs, comma seperated list")
+  parser.add_argument(
+      "--target_socs",
+      type=str,
+      default="all",
+      help="SoCs to build, comma seperated list (getprop ro.board.platform)")
+  parser.add_argument(
+      "--target",
+      type=str,
+      default="//...",
+      help="Bazel target to build")
+  parser.add_argument(
+      "--run_target",
+      type=bool,
+      default=False,
+      help="Whether to run the target")
+  parser.add_argument(
+      "--args",
+      type=str,
+      default="",
+      help="Command args")
+  parser.add_argument(
+      "--stdout_processor",
+      type=str,
+      default="stdout_processor",
+      help="Stdout processing function, default: stdout_processor")
+  return parser.parse_known_args()
+
+def main(unused_args):
+  target_socs = None
+  if FLAGS.target_socs != "all":
+    target_socs = set(FLAGS.target_socs.split(','))
+  target_devices = sh_commands.adb_devices(target_socs=target_socs)
+
+  target = FLAGS.target
+  host_bin_path, bin_name = sh_commands.bazel_target_to_bin(target)
+  target_abis = FLAGS.target_abis.split(',')
+
+  # generate sources
+  sh_commands.gen_encrypted_opencl_source()
+  sh_commands.gen_mace_version()
+
+  for target_abi in target_abis:
+    sh_commands.bazel_build(target, abi=target_abi)
+    if FLAGS.run_target:
+      for serialno in target_devices:
+        device_properties = sh_commands.adb_getprop_by_serialno(serialno)
+        stdouts = sh_commands.adb_run(serialno, host_bin_path, bin_name,
+                                      args=FLAGS.args,
+                                      opencl_profiling=1,
+                                      vlog_level=0,
+                                      device_bin_path="/data/local/tmp/mace")
+        globals()[FLAGS.stdout_processor](stdouts, device_properties, target_abi)
+
+if __name__ == "__main__":
+  FLAGS, unparsed = parse_args()
+  main(unused_args=[sys.argv[0]] + unparsed)

tools/falcon_cli.py

+#-*- coding:utf8 -*-
+
+import json, socket, itertools
+
+class FalconCli(object):
+
+    def __init__(self, addr, debug=True, buf_size=1000):
+        self.socket_ = socket.create_connection(addr)
+        self.stream = self.socket_.makefile()
+        self.id_counter = itertools.count()
+        self.debug = debug
+        self.buf_size = buf_size
+
+    def __del__(self):
+        self.socket_.close()
+        self.stream.close()
+
+    @classmethod
+    def connect(cls, server="transfer.falcon.miliao.srv", port=8433, debug=True, buf_size=1000):
+        try:
+            return FalconCli((server, port), debug, buf_size)
+        except socket.error, exc:
+            print "error: connect to %s:%s error: %s" %(server, port, exc)
+
+    def call(self, name, *params):
+        request = dict(id=next(self.id_counter),
+                    params=list(params),
+                    method=name)
+        payload = json.dumps(request).encode()
+        if self.debug:
+            print "--> req:", payload
+        self.socket_.sendall(payload)
+
+        response = self.stream.readline()
+        if not response:
+            raise Exception('empty response')
+
+        if self.debug:
+            print "<-- resp:", response
+
+        response = json.loads(response.decode("utf8"))
+        if response.get('error') is not None:
+            raise Exception(response.get('error'))
+
+        return response.get('result')
+
+    def update(self, lines):
+        s = 0
+        resp = []
+
+        while True:
+            buf = lines[s:s+self.buf_size]
+            s = s + self.buf_size
+            if len(buf) == 0:
+                break
+            r = self.call("Transfer.Update", buf)
+            resp.append(r)
+
+        return resp
+

tools/mace_tools.py

@@ -16,7 +16,7 @@ import urllib
 import yaml
 import re
 
-import adb_tools
+import sh_commands
 
 from ConfigParser import ConfigParser
 
@@ -237,7 +237,7 @@ def parse_args():
       default="all",
       help="[build|run|validate|merge|all|throughput_test].")
   parser.add_argument(
-      "--socs",
+      "--target_socs",
       type=str,
       default="all",
       help="SoCs to build, comma seperated list (getprop ro.board.platform)")
@@ -271,14 +271,14 @@ def main(unused_args):
   generate_opencl_and_version_code()
   option_args = ' '.join([arg for arg in unused_args if arg.startswith('--')])
 
-  available_socs = adb_tools.adb_get_all_socs()
+  available_socs = sh_commands.adb_get_all_socs()
   target_socs = available_socs
   if hasattr(configs, "target_socs"):
     target_socs = set(configs["target_socs"])
     target_socs = target_socs & available_socs
 
-  if FLAGS.socs != "all":
-    socs = set(FLAGS.socs.split(','))
+  if FLAGS.target_socs != "all":
+    socs = set(FLAGS.target_socs.split(','))
     target_socs = target_socs & socs
     missing_socs = socs.difference(target_socs)
     if len(missing_socs) > 0:

tools/sh_commands.py

+import sh
+import re
+import time
+import falcon_cli
+
+################################
+# common
+################################
+def strip_invalid_utf8(str):
+  return sh.iconv(str, "-c", "-t", "UTF-8")
+
+def make_output_processor(buff):
+  def process_output(line):
+    print(line.strip())
+    buff.append(line)
+  return process_output
+
+################################
+# adb commands
+################################
+def adb_split_stdout(stdout_str):
+  stdout_str = strip_invalid_utf8(stdout_str)
+  # Filter out last empty line
+  return [l.strip() for l in stdout_str.split('\n') if len(l.strip()) > 0]
+
+def adb_devices(target_socs=None):
+  outputs = sh.grep(sh.adb("devices"), "^[A-Za-z0-9]\+[[:space:]]\+device$")
+  raw_lists = sh.cut(outputs, "-f1")
+  device_ids = adb_split_stdout(raw_lists)
+  if target_socs != None:
+    target_socs_set = set(target_socs)
+    target_devices = []
+    for serialno in device_ids:
+      props = adb_getprop_by_serialno(serialno)
+      if props["ro.board.platform"] in target_socs_set:
+        target_devices.append(serialno)
+    return target_devices
+  else:
+    return device_ids
+
+def adb_getprop_by_serialno(serialno):
+  outputs = sh.adb("-s", serialno, "shell", "getprop")
+  raw_props = adb_split_stdout(outputs)
+  props = {}
+  p = re.compile("\[(.+)\]: \[(.+)\]")
+  for raw_prop in raw_props:
+    m = p.match(raw_prop)
+    if m:
+      props[m.group(1)] = m.group(2)
+  return props
+
+def adb_get_all_socs():
+  socs = []
+  for d in adb_devices():
+    props = adb_getprop_by_serialno(d)
+    socs.append(props["ro.board.platform"])
+  return set(socs)
+
+def adb_run(serialno, host_bin_path, bin_name,
+            args="",
+            opencl_profiling=1,
+            vlog_level=0,
+            device_bin_path="/data/local/tmp/mace"):
+  host_bin_full_path = "%s/%s" % (host_bin_path, bin_name)
+  device_bin_full_path = "%s/%s" % (device_bin_path, bin_name)
+  device_cl_path = "%s/cl" % device_bin_path
+  props = adb_getprop_by_serialno(serialno)
+  print("Run on device: %s, %s, %s" % (serialno, props["ro.board.platform"], props["ro.product.model"]))
+  sh.adb("-s", serialno, "shell", "rm -rf %s" % device_bin_path)
+  sh.adb("-s", serialno, "shell", "mkdir -p %s" % device_bin_path)
+  sh.adb("-s", serialno, "shell", "mkdir -p %s" % device_cl_path)
+  print("Push %s to device" % device_bin_full_path)
+  sh.adb("-s", serialno, "push", host_bin_full_path, device_bin_path)
+  print("Run %s on device" % device_bin_full_path)
+  stdout_buff=[]
+  process_output = make_output_processor(stdout_buff)
+  p = sh.adb("-s", serialno, "shell",
+             "MACE_OPENCL_PROFILING=%d MACE_KERNEL_PATH=%s MACE_CPP_MIN_VLOG_LEVEL=%d %s %s" % (opencl_profiling, device_cl_path, vlog_level, device_bin_full_path, args),
+             _out=process_output, _bg=True)
+  p.wait()
+  return "".join(stdout_buff)
+
+
+################################
+# bazel commands
+################################
+def bazel_build(target, strip="always", abi="armeabi-v7a"):
+  stdout_buff=[]
+  process_output = make_output_processor(stdout_buff)
+  p= sh.bazel("build",
+              "-c", "opt",
+              "--strip", strip,
+              "--verbose_failures",
+              target,
+              "--crosstool_top=//external:android/crosstool",
+              "--host_crosstool_top=@bazel_tools//tools/cpp:toolchain",
+              "--cpu=%s" % abi,
+              "--copt=-std=c++11",
+              "--copt=-D_GLIBCXX_USE_C99_MATH_TR1",
+              "--copt=-DMACE_DISABLE_NO_TUNING_WARNING",
+              "--copt=-Werror=return-type",
+              "--copt=-O3",
+              "--define", "neon=true",
+              "--define", "openmp=true",
+             _out=process_output, _bg=True)
+  p.wait()
+  return "".join(stdout_buff)
+
+def bazel_target_to_bin(target):
+  # change //mace/a/b:c to bazel-bin/mace/a/b/c
+  prefix, bin_name = target.split(':')
+  prefix = prefix.replace('//', '/')
+  if prefix.startswith('/'):
+    prefix = prefix[1:]
+  host_bin_path = "bazel-bin/%s" % prefix
+  return host_bin_path, bin_name
+
+################################
+# mace commands
+################################
+# TODO this should be refactored
+def gen_encrypted_opencl_source(codegen_path="mace/codegen"):
+  sh.python("mace/python/tools/encrypt_opencl_codegen.py",
+            "--cl_kernel_dir=./mace/kernels/opencl/cl/",
+            "--output_path=%s/opencl/opencl_encrypt_program.cc" % codegen_path)
+
+def gen_mace_version(codegen_path="mace/codegen"):
+  sh.mkdir("-p", "%s/version" % codegen_path)
+  sh.bash("mace/tools/git/gen_version_source.sh",
+          "%s/version/version.cc" % codegen_path)
+
+################################
+# falcon
+################################
+def falcon_tags(platform, model, abi):
+  return "ro.board.platform=%s,ro.product.model=%s,abi=%s" % (platform, model, abi)
+
+def falcon_push_metrics(metrics, device_properties, abi):
+  cli = falcon_cli.FalconCli.connect(server="transfer.falcon.miliao.srv",
+                                     port=8433,
+                                     debug=False)
+  platform = device_properties["ro.board.platform"].replace(" ", "-")
+  model = device_properties["ro.product.model"].replace(" ", "-")
+  tags = falcon_tags(platform, model, abi)
+  ts = int(time.time())
+  falcon_metrics = [{
+      "endpoint": "mace_dev",
+      "metric": key,
+      "tags": tags,
+      "timestamp": ts,
+      "value": value,
+      "step": 3600,
+      "counterType": "GAUGE"
+      } for key, value in metrics.iteritems()]
+  cli.update(falcon_metrics)
+