refactor validate_model.sh with python

build_mace_run.sh

+#!/bin/bash
+
+Usage() {
+  echo "Usage: bash tools/build_mace_run.sh production_mode model_output_dir"
+}
+
+if [ $# -lt 2 ]; then
+  Usage
+  exit 1
+fi
+
+CURRENT_DIR=`dirname $0`
+source ${CURRENT_DIR}/env.sh
+
+PRODUCTION_MODE=$1
+MODEL_OUTPUT_DIR=$2
+
+if [ "$PRODUCTION_MODE" = 1 ]; then
+  PRODUCTION_MODE_BUILD_FLAGS="--define production=true"
+fi
+
+if [ x"$RUNTIME" = x"local" ]; then
+  bazel build --verbose_failures -c opt --strip always codegen:generated_models \
+    --copt="-std=c++11" \
+    --copt="-D_GLIBCXX_USE_C99_MATH_TR1" \
+    --copt="-Werror=return-type" \
+    --copt="-DMACE_MODEL_TAG=${MODEL_TAG}" \
+    --define openmp=true \
+    $PRODUCTION_MODE_BUILD_FLAGS || exit 1
+
+  bazel build --verbose_failures -c opt --strip always examples:mace_run \
+    --copt="-std=c++11" \
+    --copt="-D_GLIBCXX_USE_C99_MATH_TR1" \
+    --copt="-Werror=return-type" \
+    --copt="-DMACE_MODEL_TAG=${MODEL_TAG}" \
+    --define openmp=true \
+    $PRODUCTION_MODE_BUILD_FLAGS || exit 1
+else
+  if [ x"$RUNTIME" = x"dsp" ]; then
+    HEXAGON_MODE_BUILD_FLAG="--define hexagon=true"
+  fi
+
+  bazel build --verbose_failures -c opt --strip always examples:mace_run \
+    --crosstool_top=//external:android/crosstool \
+    --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \
+    --cpu=${ANDROID_ABI} \
+    --copt="-std=c++11" \
+    --copt="-D_GLIBCXX_USE_C99_MATH_TR1" \
+    --copt="-Werror=return-type" \
+    --copt="-DMACE_MODEL_TAG=${MODEL_TAG}" \
+    $PRODUCTION_MODE_BUILD_FLAGS \
+    $HEXAGON_MODE_BUILD_FLAG || exit 1
+fi
+
+if [ "$PRODUCTION_MODE" = 1 ]; then
+  cp $GENERATED_MODEL_LIB_PATH $MODEL_OUTPUT_DIR/libmace_${MODEL_TAG}.a
+fi

build_production_code.sh

+#!/bin/bash
+
+Usage() {
+  echo "Usage: bash tools/build_production_code.sh"
+}
+
+CURRENT_DIR=`dirname $0`
+source ${CURRENT_DIR}/env.sh
+
+
+build_target()
+{
+  BAZEL_TARGET=$1
+  bazel build --verbose_failures -c opt --strip always $BAZEL_TARGET \
+    --crosstool_top=//external:android/crosstool \
+    --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \
+     --cpu=$ANDROID_ABI \
+    --copt="-std=c++11" \
+    --copt="-D_GLIBCXX_USE_C99_MATH_TR1" \
+    --copt="-Werror=return-type" \
+    --copt="-DMACE_OBFUSCATE_LITERALS" || exit 1
+}
+
+build_target //codegen:generated_opencl_prod
+build_target //codegen:generated_tuning_params

clear_env.sh

+CURRENT_DIR=`dirname $0`
+source ${CURRENT_DIR}/env.sh
+
+adb shell rm -rf $PHONE_DATA_DIR
+rm -rf codegen/models codegen/opencl codegen/tuning

download_and_link_lib.sh

+#!/bin/bash
+
+Usage() {
+  echo "Usage: bash tools/download_and_link_lib.sh libmace_v7_dsp"
+}
+
+if [ $# -lt 1 ]; then
+  Usage
+  exit 1
+fi
+
+CURRENT_DIR=`dirname $0`
+source ${CURRENT_DIR}/env.sh
+
+LIB_FOLDER_NAME=$1
+
+if [ ! -d "${LIBMACE_SOURCE_DIR}/lib/${LIB_FOLDER_NAME}" ]; then
+  wget -P ${LIBMACE_SOURCE_DIR}/lib http://cnbj1-inner-fds.api.xiaomi.net/libmace/libs/${LIBMACE_TAG}/${LIB_FOLDER_NAME}.tar.gz && \
+    tar xvzf ${LIBMACE_SOURCE_DIR}/lib/${LIB_FOLDER_NAME}.tar.gz -C ${LIBMACE_SOURCE_DIR}/lib/ || exit 1
+  echo "${LIB_FOLDER_NAME} download successfully!"
+else
+  echo "${LIB_FOLDER_NAME} already exists!"
+fi
+
+echo "Create link 'mace' of downloaded or existed ${LIB_FOLDER_NAME}"
+if [ -L ${LIBMACE_SOURCE_DIR}/lib/mace ]; then
+  unlink ${LIBMACE_SOURCE_DIR}/lib/mace
+fi
+ln -s ${LIBMACE_SOURCE_DIR}/lib/${LIB_FOLDER_NAME} ${LIBMACE_SOURCE_DIR}/lib/mace && \
+  rm -rf ${LIBMACE_SOURCE_DIR}/lib/${LIB_FOLDER_NAME}.tar.gz || exit 1

env.sh

+LIBMACE_TAG=`git describe --abbrev=0 --tags`
+
+VLOG_LEVEL=0
+LIBMACE_SOURCE_DIR=`/bin/pwd`
+INPUT_FILE_NAME="model_input"
+OUTPUT_FILE_NAME="model.out"
+OUTPUT_LIST_FILE="model.list"
+TF_MODEL_NAME="model.pb"
+TF_MODEL_FILE_PATH=$TF_MODEL_FILE_DIR/$TF_MODEL_NAME
+PHONE_DATA_DIR="/data/local/tmp/mace_run"
+KERNEL_DIR="${PHONE_DATA_DIR}/cl/"
+CODEGEN_DIR=${LIBMACE_SOURCE_DIR}/codegen
+MODEL_CODEGEN_DIR=${CODEGEN_DIR}/models/${MODEL_TAG}
+CL_CODEGEN_DIR=${CODEGEN_DIR}/opencl
+TUNING_CODEGEN_DIR=${CODEGEN_DIR}/tuning
+VERSION_SOURCE_PATH=${CODEGEN_DIR}/version
+GENERATED_MODEL_LIB_NAME="libgenerated_models.a"
+GENERATED_MODEL_LIB_PATH="bazel-bin/codegen/${GENERATED_MODEL_LIB_NAME}"
+
+MACE_RUNTIME=cpu
+if [ x"$RUNTIME" = x"dsp" ]; then
+  DATA_TYPE="DT_UINT8"
+  DEVICE_TYPE="HEXAGON"
+  LIB_FOLDER_NAME="${LIB_FOLDER_NAME}_dsp"
+  MACE_RUNTIME=$RUNTIME
+elif [ x"$RUNTIME" = x"gpu" ]; then
+  DATA_TYPE="DT_HALF"
+  DEVICE_TYPE="OPENCL"
+  MACE_RUNTIME=$RUNTIME
+elif [ x"$RUNTIME" = x"cpu" ]; then
+  DATA_TYPE="DT_FLOAT"
+  DEVICE_TYPE="CPU"
+elif [ x"$RUNTIME" = x"local" ];then
+  DATA_TYPE="DT_FLOAT"
+  DEVICE_TYPE="CPU"
+fi

generate_model_code.sh

+#!/bin/bash
+
+CURRENT_DIR=`dirname $0`
+source ${CURRENT_DIR}/env.sh
+
+bazel build //lib/python/tools:tf_converter || exit 1
+rm -rf ${MODEL_CODEGEN_DIR}
+mkdir -p ${MODEL_CODEGEN_DIR}
+if [ ${DSP_MODE} ]; then
+    DSP_MODE_FLAG="--dsp_mode=${DSP_MODE}"
+fi
+bazel-bin/lib/python/tools/tf_converter --input=${TF_MODEL_FILE_PATH} \
+                                        --output=${MODEL_CODEGEN_DIR}/model.cc \
+                                        --input_node=${TF_INPUT_NODE} \
+                                        --output_node=${TF_OUTPUT_NODE} \
+                                        --data_type=${DATA_TYPE} \
+                                        --runtime=${MACE_RUNTIME} \
+                                        --output_type=source \
+                                        --template=${LIBMACE_SOURCE_DIR}/lib/python/tools/model.template \
+                                        --model_tag=${MODEL_TAG} \
+                                        ${DSP_MODE_FLAG} \
+                                        --obfuscate=True || exit 1

generate_production_code.sh

+#!/bin/bash
+
+Usage() {
+  echo "Usage: bash tools/generate_production_code.sh cl_bin_dirs pull_or_not"
+}
+
+if [ $# -lt 2 ]; then
+  Usage
+  exit 1
+fi
+
+CURRENT_DIR=`dirname $0`
+source ${CURRENT_DIR}/env.sh
+
+CL_BIN_DIRS=$1
+PULL_OR_NOT=$2
+
+if [ "$PULL_OR_NOT" = 1 ]; then
+  CL_BIN_DIR=${CL_BIN_DIRS}
+  rm -rf ${CL_BIN_DIR}
+  mkdir -p ${CL_BIN_DIR}
+  if [ x"$RUNTIME" != x"local" ]; then
+    adb pull ${KERNEL_DIR}/. ${CL_BIN_DIR}
+    adb pull ${PHONE_DATA_DIR}/mace_run.config ${CL_BIN_DIR}
+  fi
+fi
+
+rm -rf ${CL_CODEGEN_DIR}
+mkdir -p ${CL_CODEGEN_DIR}
+rm -rf ${TUNING_CODEGEN_DIR}
+mkdir -p ${TUNING_CODEGEN_DIR}
+
+set -x
+
+python lib/python/tools/opencl_codegen.py \
+  --cl_binary_dirs=${CL_BIN_DIRS} \
+  --output_path=${CL_CODEGEN_DIR}/opencl_compiled_program.cc
+
+python lib/python/tools/binary_codegen.py \
+  --binary_dirs=${CL_BIN_DIRS} \
+  --binary_file_name=mace_run.config \
+  --output_path=${TUNING_CODEGEN_DIR}/tuning_params.cc

mace_tools.py

+#!/usr/bin/env python
+
+# python tools/mace_tools.py \
+#     --global_config=models/config \
+#     --round=100 \
+#     --mode=all
+
+import argparse
+import os
+import shutil
+import subprocess
+import sys
+
+from ConfigParser import ConfigParser
+
+tf_model_file_dir_key = "TF_MODEL_FILE_DIR"
+
+
+def run_command(command):
+  print("Run command: {}".format(command))
+  result = subprocess.Popen(
+      command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+  out, err = result.communicate()
+
+  if out:
+    print("Stdout msg:\n{}".format(out))
+  if err:
+    print("Stderr msg:\n{}".format(err))
+
+  if result.returncode != 0:
+    raise Exception("Exit not 0 from bash with code: {}, command: {}".format(
+        result.returncode, command))
+
+
+def get_libs(configs):
+  libmace_name = "libmace"
+  for config in configs:
+    if config["ANDROID_ABI"] == "armeabi-v7a":
+      libmace_name += "_v7"
+      break
+    elif config["ANDROID_ABI"] == "arm64-v8a":
+      libmace_name += "_v8"
+      break
+
+  for config in configs:
+    if config["RUNTIME"] == "dsp":
+      libmace_name += "_dsp"
+      break
+    if config["RUNTIME"] == "local":
+      libmace_name += "_local"
+      break
+
+  command = "bash tools/download_and_link_lib.sh " + libmace_name
+  run_command(command)
+
+
+def clear_env():
+  command = "bash tools/clear_env.sh"
+  run_command(command)
+
+
+def generate_random_input(model_output_dir):
+  generate_data_or_not = True
+  command = "bash tools/validate_tools.sh {} {}".format(
+      model_output_dir, int(generate_data_or_not))
+  run_command(command)
+
+
+def generate_model_code():
+  command = "bash tools/generate_model_code.sh"
+  run_command(command)
+
+
+def build_mace_run(production_mode, model_output_dir):
+  command = "bash tools/build_mace_run.sh {} {}".format(
+      int(production_mode), model_output_dir)
+  run_command(command)
+
+
+def tuning_run(model_output_dir, running_round, tuning, production_mode):
+  command = "bash tools/tuning_run.sh {} {} {} {}".format(
+      model_output_dir, running_round, int(tuning), int(production_mode))
+  run_command(command)
+
+
+def run_model(model_output_dir, running_round):
+  tuning_run(model_output_dir, running_round, False, False)
+
+
+def generate_production_code(model_output_dirs, pull_or_not):
+  cl_bin_dirs = []
+  for d in model_output_dirs:
+    cl_bin_dirs.append(os.path.join(d, "opencl_bin"))
+  cl_bin_dirs_str = ",".join(cl_bin_dirs)
+  command = "bash tools/generate_production_code.sh {} {}".format(
+      cl_bin_dirs_str, int(pull_or_not))
+  run_command(command)
+
+
+def build_mace_run_prod(model_output_dir, tuning):
+  production_or_not = False
+  build_mace_run(production_or_not, model_output_dir)
+  tuning_run(
+      model_output_dir,
+      running_round=0,
+      tuning=tuning,
+      production_mode=production_or_not)
+
+  production_or_not = True
+  pull_or_not = True
+  generate_production_code([model_output_dir], pull_or_not)
+  build_mace_run(production_or_not, model_output_dir)
+
+
+def validate_model(model_output_dir):
+  generate_data_or_not = False
+  command = "bash tools/validate_tools.sh {} {}".format(
+      model_output_dir, int(generate_data_or_not))
+  run_command(command)
+
+
+def build_production_code():
+  command = "bash tools/build_production_code.sh"
+  run_command(command)
+
+
+def merge_libs_and_tuning_results(output_dir, model_output_dirs):
+  pull_or_not = False
+  generate_production_code(model_output_dirs, pull_or_not)
+  production_or_not = True
+  build_production_code()
+
+  model_output_dirs_str = ",".join(model_output_dirs)
+  command = "bash tools/merge_libs.sh {} {}".format(output_dir,
+                                                    model_output_dirs_str)
+  run_command(command)
+
+
+def parse_sub_model_configs(model_dirs, global_configs):
+  model_configs = []
+  for model_dir in model_dirs:
+    model_config = {}
+
+    model_config_path = os.path.join(model_dir, "config")
+    if os.path.exists(model_config_path):
+      cf = ConfigParser()
+      # Preserve character case
+      cf.optionxform = str
+      cf.read(model_config_path)
+      if "configs" in cf.sections():
+        config_list = cf.items("configs")
+        for config_map in config_list:
+          model_config[config_map[0]] = config_map[1]
+      else:
+        print("No config msg found in {}".format(model_config_path))
+
+    model_config[tf_model_file_dir_key] = model_dir
+
+    for config_map in global_configs:
+      model_config[config_map[0]] = config_map[1]
+
+    model_configs.append(model_config)
+
+  return model_configs
+
+
+def parse_model_configs():
+  config_parser = ConfigParser()
+  # Preserve character case
+  config_parser.optionxform = str
+
+  global_config_dir = os.path.dirname(FLAGS.global_config)
+
+  try:
+    config_parser.read(FLAGS.global_config)
+    config_sections = config_parser.sections()
+
+    model_dirs = []
+    model_output_map = {}
+    if ("models" in config_sections) and (config_parser.items("models")):
+      model_dirs_relative_to_config_str = config_parser.get(
+          "models", "DIRECTORIES")
+      model_dirs_relative_to_config_str = model_dirs_relative_to_config_str.rstrip(
+          ",")
+
+      # Remove repetition element
+      model_dirs_relative_to_config_str_list = list(
+          set(model_dirs_relative_to_config_str.split(",")))
+
+      for d in model_dirs_relative_to_config_str_list:
+        model_dir = os.path.join(global_config_dir, d)
+        model_dirs.append(model_dir)
+
+        # Create output dirs
+        model_output_dir = os.path.join(FLAGS.output_dir, d)
+
+        model_output_map[model_dir] = model_output_dir
+    else:
+      model_dirs = [global_config_dir]
+
+      # Create output dirs
+      model_output_dir = os.path.join(FLAGS.output_dir, global_config_dir)
+      model_output_map[global_config_dir] = model_output_dir
+  except Exception as e:
+    print("Error in read model path msg. Exception: {}".format(e))
+    return
+
+  global_configs = []
+  if "configs" in config_sections:
+    global_configs = config_parser.items("configs")
+
+  return parse_sub_model_configs(model_dirs, global_configs), model_output_map
+
+
+def parse_args():
+  """Parses command line arguments."""
+  parser = argparse.ArgumentParser()
+  parser.register("type", "bool", lambda v: v.lower() == "true")
+  parser.add_argument(
+      "--global_config",
+      type=str,
+      default="./tool/config",
+      help="The global config file of models.")
+  parser.add_argument(
+      "--output_dir", type=str, default="./build/", help="The output dir.")
+  parser.add_argument(
+      "--round", type=int, default=1, help="The model running round.")
+  parser.add_argument(
+      "--tuning", type="bool", default="true", help="Tune opencl params.")
+  parser.add_argument(
+      "--mode", type=str, default="all", help="[build|run|merge|all].")
+  return parser.parse_known_args()
+
+
+def main(unused_args):
+  configs, model_output_map = parse_model_configs()
+
+  if FLAGS.mode == "build" or FLAGS.mode == "all":
+    # Remove previous output dirs
+    if not os.path.exists(FLAGS.output_dir):
+      os.makedirs(FLAGS.output_dir)
+    elif os.path.exists(os.path.join(FLAGS.output_dir, "libmace")):
+      shutil.rmtree(os.path.join(FLAGS.output_dir, "libmace"))
+
+  get_libs(configs)
+
+  model_output_dirs = []
+  for config in configs:
+    # Transfer params by environment
+    for key in config:
+      os.environ[key] = config[key]
+    model_output_dir = model_output_map[config[tf_model_file_dir_key]]
+    model_output_dirs.append(model_output_dir)
+
+    if FLAGS.mode == "build" or FLAGS.mode == "all":
+      if os.path.exists(model_output_dir):
+        shutil.rmtree(model_output_dir)
+      os.makedirs(model_output_dir)
+      clear_env()
+
+    if FLAGS.mode == "build" or FLAGS.mode == "run" or FLAGS.mode == "all":
+      generate_random_input(model_output_dir)
+
+    if FLAGS.mode == "build" or FLAGS.mode == "all":
+      generate_model_code()
+      build_mace_run_prod(model_output_dir, FLAGS.tuning)
+
+    if FLAGS.mode == "run" or FLAGS.mode == "all":
+      run_model(model_output_dir, FLAGS.round)
+
+    if FLAGS.mode == "all":
+      validate_model(model_output_dir)
+
+  if FLAGS.mode == "build" or FLAGS.mode == "merge" or FLAGS.mode == "all":
+    merge_libs_and_tuning_results(FLAGS.output_dir, model_output_dirs)
+
+
+if __name__ == '__main__':
+  FLAGS, unparsed = parse_args()
+  main(unused_args=[sys.argv[0]] + unparsed)

model.config

+; [models]
+; DIRECTORIES = mobile_net/64/
+
+[configs]
+; If exists 'models' section, this section will be global variable to sub models
 TF_INPUT_NODE=input
 TF_OUTPUT_NODE=output
 TF_MODEL_FILE_PATH=path/to/tf_model_opt.pb
@@ -6,6 +11,6 @@ INPUT_SHAPE=1,64,64,3
 OUTPUT_SHAPE=1,64,64,2
 RUNTIME=gpu
 TUNING_OR_NOT=1
-ANDROID_ABI=armeabi-v7a # Or 'arm64-v8a'
+ANDROID_ABI=armeabi-v7a ; Or 'arm64-v8a'
 LIMIT_OPENCL_KERNEL_TIME=0
-DSP_MODE=0 # used only for dsp
+DSP_MODE=0 ; used only for dsp

tuning_run.sh

+#!/bin/bash
+
+Usage() {
+  echo "Usage: bash tools/run_and_tuning.sh model_output_dir round tuning production_mode"
+}
+
+if [ $# -lt 4 ]; then
+  Usage
+  exit 1
+fi
+
+CURRENT_DIR=`dirname $0`
+source ${CURRENT_DIR}/env.sh
+
+MODEL_OUTPUT_DIR=$1
+ROUND=$2
+TUNING_OR_NOT=$3
+PRODUCTION_MODE=$4
+
+if [ x"$RUNTIME" = x"local" ]; then
+  MACE_CPP_MIN_VLOG_LEVEL=$VLOG_LEVEL \
+  bazel-bin/examples/mace_run \
+      --input_shape="${INPUT_SHAPE}"\
+      --output_shape="${OUTPUT_SHAPE}"\
+      --input_file=${MODEL_OUTPUT_DIR}/${INPUT_FILE_NAME} \
+      --output_file=${MODEL_OUTPUT_DIR}/${OUTPUT_FILE_NAME} \
+      --device=${DEVICE_TYPE}   \
+      --round=1 || exit 1
+else
+  if [[ "${TUNING_OR_NOT}" != "0" && "$PRODUCTION_MODE" != 1 ]];then
+    tuning_flag=1
+  else
+    tuning_flag=0
+  fi
+  
+  adb shell "mkdir -p ${PHONE_DATA_DIR}" || exit 1
+  if [ "$PRODUCTION_MODE" = 0 ]; then
+    adb shell "mkdir -p ${KERNEL_DIR}" || exit 1
+  fi
+  adb push ${MODEL_OUTPUT_DIR}/${INPUT_FILE_NAME} ${PHONE_DATA_DIR} || exit 1
+  adb push bazel-bin/examples/mace_run ${PHONE_DATA_DIR} || exit 1
+  adb push lib/hexagon/libhexagon_controller.so ${PHONE_DATA_DIR} || exit 1
+  
+  adb </dev/null shell \
+    LD_LIBRARY_PATH=${PHONE_DATA_DIR} \
+    MACE_TUNING=${tuning_flag} \
+    MACE_CPP_MIN_VLOG_LEVEL=$VLOG_LEVEL \
+    MACE_RUN_PARAMETER_PATH=${PHONE_DATA_DIR}/mace_run.config \
+    MACE_KERNEL_PATH=$KERNEL_DIR \
+    MACE_LIMIT_OPENCL_KERNEL_TIME=${LIMIT_OPENCL_KERNEL_TIME} \
+    ${PHONE_DATA_DIR}/mace_run \
+    --input_shape="${INPUT_SHAPE}"\
+    --output_shape="${OUTPUT_SHAPE}"\
+    --input_file=${PHONE_DATA_DIR}/${INPUT_FILE_NAME} \
+    --output_file=${PHONE_DATA_DIR}/${OUTPUT_FILE_NAME} \
+    --device=${DEVICE_TYPE}   \
+    --round=$ROUND || exit 1
+fi

validate_model.sh

-#!/bin/bash
-# Must run at root dir of mace project.
-set +x
-Usage() {
-  echo 'Usage: bash tools/validate_model.sh tools/model.config'
-}
-
-if [ $# -lt 1 ];then
-  Usage
-  exit -1
-fi
-
-source $1
-
-LIB_FOLDER_NAME="libmace"
-if [ x"${ANDROID_ABI}" = x"armeabi-v7a" ]; then
-  LIB_FOLDER_NAME="${LIB_FOLDER_NAME}_v7"
-elif [ x"${ANDROID_ABI}" = x"arm64-v8a" ]; then
-  LIB_FOLDER_NAME="${LIB_FOLDER_NAME}_v8"
-else
-  echo "Unsopported android abi"
-  exit -1
-fi
-
-MACE_RUNTIME=cpu
-
-if [ x"$RUNTIME" = x"dsp" ]; then
-  DATA_TYPE="DT_UINT8"
-  DEVICE_TYPE="HEXAGON"
-  LIB_FOLDER_NAME="${LIB_FOLDER_NAME}_dsp"
-  MACE_RUNTIME=$RUNTIME
-elif [ x"$RUNTIME" = x"gpu" ]; then
-  DATA_TYPE="DT_HALF"
-  DEVICE_TYPE="OPENCL"
-  MACE_RUNTIME=$RUNTIME
-elif [ x"$RUNTIME" = x"cpu" ]; then
-  DATA_TYPE="DT_FLOAT"
-  DEVICE_TYPE="CPU"
-elif [ x"$RUNTIME" = x"local" ];then
-  DATA_TYPE="DT_FLOAT"
-  DEVICE_TYPE="CPU"
-else
-  Usage
-  exit -1
-fi
-
-LIBMACE_TAG=`git describe --abbrev=0 --tags` || exit -1
-
-VLOG_LEVEL=0
-MODEL_DIR=$(dirname ${TF_MODEL_FILE_PATH})
-LIBMACE_SOURCE_DIR=`/bin/pwd`
-LIBMACE_BUILD_DIR="${LIBMACE_SOURCE_DIR}/build"
-INPUT_FILE_NAME="model_input"
-OUTPUT_FILE_NAME="model.out"
-OUTPUT_LIST_FILE="model.list"
-PHONE_DATA_DIR="/data/local/tmp/mace_run"
-KERNEL_DIR="${PHONE_DATA_DIR}/cl/"
-CODEGEN_DIR=${LIBMACE_SOURCE_DIR}/codegen
-MODEL_CODEGEN_DIR=${CODEGEN_DIR}/models/${MODEL_TAG}
-CL_CODEGEN_DIR=${CODEGEN_DIR}/opencl
-CL_BIN_DIR=${CODEGEN_DIR}/opencl_bin
-TUNING_CODEGEN_DIR=${CODEGEN_DIR}/tuning
-VERSION_SOURCE_PATH=${CODEGEN_DIR}/version
-
-build_and_run()
-{
-  PRODUCTION_MODE=$1
-  if [ "$PRODUCTION_MODE" = true ]; then
-    PRODUCTION_MODE_BUILD_FLAGS="--define production=true"
-  fi
-
-  if [[ "${TUNING_OR_NOT}" != "0" && "$PRODUCTION_MODE" != true ]];then
-    tuning_flag=1
-    round=0 # only warm up
-  elif [[ "$PRODUCTION_MODE" != true ]];then
-    tuning_flag=0
-    round=1
-  else
-    tuning_flag=0
-    round=2
-  fi
-
-  bazel build --verbose_failures -c opt --strip always examples:mace_run \
-    --crosstool_top=//external:android/crosstool \
-    --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \
-    --cpu=${ANDROID_ABI} \
-    --copt="-std=c++11" \
-    --copt="-D_GLIBCXX_USE_C99_MATH_TR1" \
-    --copt="-Werror=return-type" \
-    --copt="-DMACE_MODEL_TAG=${MODEL_TAG}" \
-    $PRODUCTION_MODE_BUILD_FLAGS \
-    --define hexagon=true || exit -1
-
-  adb shell "mkdir -p ${PHONE_DATA_DIR}" || exit -1
-  if [ "$PRODUCTION_MODE" = false ]; then
-    adb shell "mkdir -p ${KERNEL_DIR}" || exit -1
-  fi
-  adb push ${MODEL_DIR}/${INPUT_FILE_NAME} ${PHONE_DATA_DIR} || exit -1
-  adb push bazel-bin/examples/mace_run ${PHONE_DATA_DIR} || exit -1
-  adb push lib/hexagon/libhexagon_controller.so ${PHONE_DATA_DIR} || exit 0
-
-  adb </dev/null shell \
-    LD_LIBRARY_PATH=${PHONE_DATA_DIR} \
-    MACE_TUNING=${tuning_flag} \
-    MACE_CPP_MIN_VLOG_LEVEL=$VLOG_LEVEL \
-    MACE_RUN_PARAMETER_PATH=${PHONE_DATA_DIR}/mace_run.config \
-    MACE_KERNEL_PATH=$KERNEL_DIR \
-    MACE_LIMIT_OPENCL_KERNEL_TIME=${LIMIT_OPENCL_KERNEL_TIME} \
-    ${PHONE_DATA_DIR}/mace_run \
-    --input_shape="${INPUT_SHAPE}"\
-    --output_shape="${OUTPUT_SHAPE}"\
-    --input_file=${PHONE_DATA_DIR}/${INPUT_FILE_NAME} \
-    --output_file=${PHONE_DATA_DIR}/${OUTPUT_FILE_NAME} \
-    --device=${DEVICE_TYPE}   \
-    --round=$round || exit -1
-}
-
-local_build_and_run()
-{
-  bazel build --verbose_failures -c opt --strip always codegen:generated_models \
-    --copt="-std=c++11" \
-    --copt="-D_GLIBCXX_USE_C99_MATH_TR1" \
-    --copt="-Werror=return-type" \
-    --copt="-DMACE_MODEL_TAG=${MODEL_TAG}" \
-    --define openmp=true \
-    --define production=true || exit -1
-
-  bazel build --verbose_failures -c opt --strip always examples:mace_run \
-    --copt="-std=c++11" \
-    --copt="-D_GLIBCXX_USE_C99_MATH_TR1" \
-    --copt="-Werror=return-type" \
-    --copt="-DMACE_MODEL_TAG=${MODEL_TAG}" \
-    --define openmp=true \
-    --define production=true || exit -1
-
-  MACE_CPP_MIN_VLOG_LEVEL=$VLOG_LEVEL \
-  bazel-bin/examples/mace_run \
-      --input_shape="${INPUT_SHAPE}"\
-      --output_shape="${OUTPUT_SHAPE}"\
-      --input_file=${MODEL_DIR}/${INPUT_FILE_NAME} \
-      --output_file=${MODEL_DIR}/${OUTPUT_FILE_NAME} \
-      --device=${DEVICE_TYPE}   \
-      --round=1 || exit -1
-}
-
-download_and_link_lib()
-{
-  if [ ! -d "${LIBMACE_SOURCE_DIR}/lib/${LIB_FOLDER_NAME}" ]; then
-    wget -P ${LIBMACE_SOURCE_DIR}/lib http://cnbj1-inner-fds.api.xiaomi.net/libmace/libs/${LIBMACE_TAG}/${LIB_FOLDER_NAME}.tar.gz && \
-      tar xvzf ${LIBMACE_SOURCE_DIR}/lib/${LIB_FOLDER_NAME}.tar.gz -C ${LIBMACE_SOURCE_DIR}/lib/ || exit -1
-    echo "${LIB_FOLDER_NAME} download successfully!"
-  else
-    echo "${LIB_FOLDER_NAME} already exists!"
-  fi
-
-  echo "Create link 'mace' of downloaded or existed ${LIB_FOLDER_NAME}"
-  if [ -L ${LIBMACE_SOURCE_DIR}/lib/mace ]; then
-    unlink ${LIBMACE_SOURCE_DIR}/lib/mace
-  fi
-  ln -s ${LIBMACE_SOURCE_DIR}/lib/${LIB_FOLDER_NAME} ${LIBMACE_SOURCE_DIR}/lib/mace && \
-    rm -rf ${LIBMACE_SOURCE_DIR}/lib/${LIB_FOLDER_NAME}.tar.gz || exit -1
-}
-
-echo "Step 1: Generate input data"
-rm -rf ${MODEL_DIR}/${INPUT_FILE_NAME}
-python tools/validate.py --generate_data true \
- --input_file=${MODEL_DIR}/${INPUT_FILE_NAME} \
- --input_shape="${INPUT_SHAPE}" || exit -1
-
-echo "Step 2: Convert tf model to mace model and optimize memory"
-bazel build //lib/python/tools:tf_converter || exit -1
-rm -rf ${MODEL_CODEGEN_DIR}
-mkdir -p ${MODEL_CODEGEN_DIR}
-if [ ${DSP_MODE} ]; then
-    DSP_MODE_FLAG="--dsp_mode=${DSP_MODE}"
-fi
-bazel-bin/lib/python/tools/tf_converter --input=${TF_MODEL_FILE_PATH} \
-                                        --output=${MODEL_CODEGEN_DIR}/model.cc \
-                                        --input_node=${TF_INPUT_NODE} \
-                                        --output_node=${TF_OUTPUT_NODE} \
-                                        --data_type=${DATA_TYPE} \
-                                        --runtime=${MACE_RUNTIME} \
-                                        --output_type=source \
-                                        --template=${LIBMACE_SOURCE_DIR}/lib/python/tools/model.template \
-                                        --model_tag=${MODEL_TAG} \
-                                        ${DSP_MODE_FLAG} \
-                                        --input_shape="${INPUT_SHAPE}" \
-                                        --obfuscate=True || exit -1
-
-echo "Step 3: Download mace static library"
-download_and_link_lib
-
-if [ x"$RUNTIME" = x"local" ]; then
-
-  echo "Step 4: remove the mace run result."
-  rm -rf ${MODEL_DIR}/${OUTPUT_FILE_NAME}
-
-  echo "Step 8: Run model on the local pc using binary"
-  local_build_and_run
-
-else
-  echo "Step 4: Run model on the phone with files"
-  build_and_run false
-
-  echo "Step 5: Generate OpenCL binary program and config code"
-  rm -rf ${CL_BIN_DIR}
-  rm -rf ${CL_CODEGEN_DIR}
-  mkdir -p ${CL_BIN_DIR}
-  mkdir -p ${CL_CODEGEN_DIR}
-  adb pull ${KERNEL_DIR}/. ${CL_BIN_DIR}
-  python lib/python/tools/opencl_codegen.py \
-    --cl_binary_dir=${CL_BIN_DIR} --output_path=${CL_CODEGEN_DIR}/opencl_compiled_program.cc
-
-  echo "Step 6: Generate tuning source file"
-  adb pull ${PHONE_DATA_DIR}/mace_run.config ${CL_BIN_DIR}
-  rm -rf ${TUNING_CODEGEN_DIR}
-  mkdir -p ${TUNING_CODEGEN_DIR}
-  python lib/python/tools/binary_codegen.py \
-    --binary_file=${CL_BIN_DIR}/mace_run.config --output_path=${TUNING_CODEGEN_DIR}/tuning_params.cc
-
-  echo "Step 7: Run model on the phone using binary"
-  build_and_run true
-
-  echo "Step 8: Pull the mace run result."
-  rm -rf ${MODEL_DIR}/${OUTPUT_FILE_NAME}
-  adb </dev/null pull ${PHONE_DATA_DIR}/${OUTPUT_FILE_NAME} ${MODEL_DIR}
-
-fi
-
-echo "Step 9: Validate the result"
-python tools/validate.py --model_file ${TF_MODEL_FILE_PATH} \
-    --input_file ${MODEL_DIR}/${INPUT_FILE_NAME} \
-    --mace_out_file ${MODEL_DIR}/${OUTPUT_FILE_NAME} \
-    --mace_runtime ${MACE_RUNTIME} \
-    --input_node ${TF_INPUT_NODE} \
-    --output_node ${TF_OUTPUT_NODE} \
-    --input_shape ${INPUT_SHAPE} \
-    --output_shape ${OUTPUT_SHAPE}
-
-echo "Step 10: Generate project static lib"
-rm -rf ${LIBMACE_BUILD_DIR}
-mkdir -p ${LIBMACE_BUILD_DIR}/lib
-cp -rf ${LIBMACE_SOURCE_DIR}/include ${LIBMACE_BUILD_DIR}
-
-if [ x"$RUNTIME" = x"local" ]; then
-  $ANDROID_NDK_HOME/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/bin/aarch64-linux-android-ar \
-    -M < tools/libmace_local.mri || exit -1
-else
-  cp ${LIBMACE_SOURCE_DIR}/lib/hexagon/libhexagon_controller.so ${LIBMACE_BUILD_DIR}/lib
-  $ANDROID_NDK_HOME/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/bin/aarch64-linux-android-ar \
-    -M < tools/libmace.mri || exit -1
-fi
-
-echo "Done"

validate_tools.sh

+#!/bin/bash
+
+Usage() {
+  echo "Usage: bash tools/validate_tools.sh model_output_dir generate_data_or_not"
+}
+
+if [ $# -lt 2 ]; then
+  Usage
+  exit 1
+fi
+
+CURRENT_DIR=`dirname $0`
+source ${CURRENT_DIR}/env.sh
+
+MODEL_OUTPUT_DIR=$1
+GENERATE_DATA_OR_NOT=$2
+
+if [ "$GENERATE_DATA_OR_NOT" = 1 ]; then
+  rm -rf ${MODEL_OUTPUT_DIR}/${INPUT_FILE_NAME}
+  python tools/validate.py --generate_data true \
+     --input_file=${MODEL_OUTPUT_DIR}/${INPUT_FILE_NAME} \
+      --input_shape="${INPUT_SHAPE}" || exit 1
+else
+  rm -rf ${MODEL_OUTPUT_DIR}/${OUTPUT_FILE_NAME}
+  adb </dev/null pull ${PHONE_DATA_DIR}/${OUTPUT_FILE_NAME} ${MODEL_OUTPUT_DIR}
+  python tools/validate.py --model_file ${TF_MODEL_FILE_PATH} \
+      --input_file ${MODEL_OUTPUT_DIR}/${INPUT_FILE_NAME} \
+      --mace_out_file ${MODEL_OUTPUT_DIR}/${OUTPUT_FILE_NAME} \
+      --mace_runtime ${MACE_RUNTIME} \
+      --input_node ${TF_INPUT_NODE} \
+      --output_node ${TF_OUTPUT_NODE} \
+      --input_shape ${INPUT_SHAPE} \
+      --output_shape ${OUTPUT_SHAPE}
+fi