From 2dfaaedd8ddf83a04ee2d9c1699a0d04a60209f7 Mon Sep 17 00:00:00 2001
From: wuchenghui <wuchenghui@xiaomi.com>
Date: Thu, 8 Mar 2018 17:09:12 +0800
Subject: [PATCH] CPU+GPU+DSP full throughput benchmark

---
 build_run_throughput_test.sh | 73 ++++++++++++++++++++++++++++++++++++
 mace_tools.py                | 21 ++++++++++-
 2 files changed, 92 insertions(+), 2 deletions(-)
 create mode 100644 build_run_throughput_test.sh

diff --git a/build_run_throughput_test.sh b/build_run_throughput_test.sh
new file mode 100644
index 00000000..e87e2ff8
--- /dev/null
+++ b/build_run_throughput_test.sh
@@ -0,0 +1,73 @@
+#!/bin/bash
+
+Usage() {
+  echo "Usage: bash tools/build_run_throughput_test.sh run_seconds merged_lib_file model_input_dir"
+}
+
+if [ $# -lt 1 ]; then
+  Usage
+  exit 1
+fi
+
+RUN_SECONDS=$1
+MERGED_LIB_FILE=$2
+MODEL_INPUT_DIR=$3
+
+CURRENT_DIR=`dirname $0`
+source ${CURRENT_DIR}/env.sh
+
+
+if [ "$CPU_MODEL_TAG" != '' ]; then
+  CPU_MODEL_TAG_BUILD_FLAGS="--copt=-DMACE_CPU_MODEL_TAG=${CPU_MODEL_TAG}"
+fi
+
+if [ "$GPU_MODEL_TAG" != '' ]; then
+  GPU_MODEL_TAG_BUILD_FLAGS="--copt=-DMACE_GPU_MODEL_TAG=${GPU_MODEL_TAG}"
+fi
+
+if [ "$DSP_MODEL_TAG" != '' ]; then
+  DSP_MODEL_TAG_BUILD_FLAGS="--copt=-DMACE_DSP_MODEL_TAG=${DSP_MODEL_TAG}"
+fi
+
+cp $MERGED_LIB_FILE benchmark/libmace_merged.a
+
+bazel build --verbose_failures -c opt --strip always benchmark:model_throughput_test \
+    --crosstool_top=//external:android/crosstool \
+    --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \
+    --cpu=${TARGET_ABI} \
+    --copt="-std=c++11" \
+    --copt="-D_GLIBCXX_USE_C99_MATH_TR1" \
+    --copt="-Werror=return-type" \
+    ${CPU_MODEL_TAG_BUILD_FLAGS} \
+    ${GPU_MODEL_TAG_BUILD_FLAGS} \
+    ${DSP_MODEL_TAG_BUILD_FLAGS} \
+    --define openmp=true \
+    --copt="-O3" || exit 1
+
+rm benchmark/libmace_merged.a
+
+adb shell "mkdir -p ${PHONE_DATA_DIR}" || exit 1
+
+adb push ${MODEL_INPUT_DIR}/${INPUT_FILE_NAME} ${PHONE_DATA_DIR} || exit 1
+adb push bazel-bin/benchmark/model_throughput_test ${PHONE_DATA_DIR} || exit 1
+if [ "$EMBED_MODEL_DATA" = 0 ]; then
+  adb push codegen/models/${CPU_MODEL_TAG}/${CPU_MODEL_TAG}.data ${PHONE_DATA_DIR} || exit 1
+  adb push codegen/models/${GPU_MODEL_TAG}/${GPU_MODEL_TAG}.data ${PHONE_DATA_DIR} || exit 1
+  adb push codegen/models/${DSP_MODEL_TAG}/${DSP_MODEL_TAG}.data ${PHONE_DATA_DIR} || exit 1
+fi
+adb push lib/hexagon/libhexagon_controller.so ${PHONE_DATA_DIR} || exit 1
+
+adb </dev/null shell \
+LD_LIBRARY_PATH=${PHONE_DATA_DIR} \
+MACE_CPP_MIN_VLOG_LEVEL=$VLOG_LEVEL \
+MACE_RUN_PARAMETER_PATH=${PHONE_DATA_DIR}/mace_run.config \
+MACE_KERNEL_PATH=$KERNEL_DIR \
+MACE_LIMIT_OPENCL_KERNEL_TIME=${LIMIT_OPENCL_KERNEL_TIME} \
+${PHONE_DATA_DIR}/model_throughput_test \
+--input_shape="${INPUT_SHAPE}" \
+--output_shape="${OUTPUT_SHAPE}" \
+--input_file=${PHONE_DATA_DIR}/${INPUT_FILE_NAME} \
+--cpu_model_data_file=${PHONE_DATA_DIR}/${CPU_MODEL_TAG}.data \
+--gpu_model_data_file=${PHONE_DATA_DIR}/${GPU_MODEL_TAG}.data \
+--dsp_model_data_file=${PHONE_DATA_DIR}/${DSP_MODEL_TAG}.data \
+--run_seconds=$RUN_SECONDS || exit 1
\ No newline at end of file
diff --git a/mace_tools.py b/mace_tools.py
index 690c30f2..0d420cd0 100644
--- a/mace_tools.py
+++ b/mace_tools.py
@@ -125,6 +125,12 @@ def build_mace_run_prod(model_output_dir, tuning, libmace_name):
   build_mace_run(production_or_not, model_output_dir, hexagon_mode)
 
 
+def build_run_throughput_test(run_seconds, merged_lib_file, model_input_dir):
+  command = "bash tools/build_run_throughput_test.sh {} {} {}".format(
+    run_seconds, merged_lib_file, model_input_dir)
+  run_command(command)
+
+
 def validate_model(model_output_dir):
   generate_data_or_not = False
   command = "bash tools/validate_tools.sh {} {}".format(
@@ -167,10 +173,12 @@ def parse_args():
       "--output_dir", type=str, default="build", help="The output dir.")
   parser.add_argument(
       "--round", type=int, default=1, help="The model running round.")
+  parser.add_argument("--run_seconds", type=int, default=10,
+                      help="The model throughput test running seconds.")
   parser.add_argument(
       "--tuning", type="bool", default="true", help="Tune opencl params.")
-  parser.add_argument(
-      "--mode", type=str, default="all", help="[build|run|validate|merge|all].")
+  parser.add_argument("--mode", type=str, default="all",
+                      help="[build|run|validate|merge|all|throughput_test].")
   return parser.parse_known_args()
 
 
@@ -256,6 +264,15 @@ def main(unused_args):
       merge_libs_and_tuning_results(FLAGS.output_dir + "/" + target_abi,
                                     model_output_dirs)
 
+  if FLAGS.mode == "throughput_test":
+    merged_lib_file = FLAGS.output_dir + "/%s/libmace/lib/libmace_%s.a" % \
+        (configs["target_abis"][0], os.environ["PROJECT_NAME"])
+    generate_random_input(FLAGS.output_dir)
+    for model_name in configs["models"]:
+      runtime = configs["models"][model_name]["runtime"]
+      os.environ["%s_MODEL_TAG" % runtime.upper()] = model_name
+    build_run_throughput_test(FLAGS.run_seconds, merged_lib_file, FLAGS.output_dir)
+
 
 if __name__ == "__main__":
   FLAGS, unparsed = parse_args()
-- 
GitLab