Add opencl binary program support

mace/BUILD

@@ -31,3 +31,11 @@ config_setting(
     },
     visibility = ["//visibility:public"],
 )
+
+config_setting(
+    name = "embed_binary_program",
+    define_values = {
+        "embed_binary_program": "true",
+    },
+    visibility = ["//visibility:public"],
+)

mace/codegen/BUILD

+# Description:
+# Generated model and runtime code.
+#
+package(
+    default_visibility = ["//visibility:public"],
+)
+
+load("//mace:mace.bzl", "if_embed_binary_program")
+
+cc_library(
+    name = "generated_models_lib",
+    srcs = glob(["models/*/*.cc"]),
+    copts = ["-std=c++11", "-D_GLIBCXX_USE_C99_MATH_TR1"],
+    linkstatic = 1,
+    deps = [
+        "//mace/core",
+        "//mace/ops",
+    ] + if_embed_binary_program(['//mace/codegen:generated_opencl_lib']),
+)
+
+cc_library(
+    name = "generated_opencl_lib",
+    srcs = glob(["opencl/*.cc"]),
+    copts = ["-std=c++11", "-D_GLIBCXX_USE_C99_MATH_TR1"],
+    linkstatic = 1,
+)

mace/core/BUILD

@@ -7,7 +7,7 @@ package(
 
 licenses(["notice"])  # Apache 2.0
 
-load("//mace:mace.bzl", "if_android", "if_profiling_enabled")
+load("//mace:mace.bzl", "if_android", "if_profiling_enabled", "if_embed_binary_program")
 
 cc_library(
     name = "opencl_runtime",
@@ -19,7 +19,8 @@ cc_library(
         "runtime/opencl/*.h",
     ]),
     copts = ["-std=c++11", "-D_GLIBCXX_USE_C99_MATH_TR1"] +
-            if_profiling_enabled(["-DMACE_OPENCL_PROFILING"]),
+            if_profiling_enabled(["-DMACE_OPENCL_PROFILING"]) +
+            if_embed_binary_program(["-DMACE_EMBED_BINARY_PROGRAM"]),
     linkopts = ["-ldl"],
     deps = [
         ":core",

mace/core/runtime/opencl/opencl_runtime.cc

@@ -180,6 +180,17 @@ void OpenCLRuntime::BuildProgram(const std::string &program_file_name,
                                  cl::Program *program) {
   MACE_CHECK_NOTNULL(program);
 
+#ifdef MACE_EMBED_BINARY_PROGRAM
+  extern const std::map<std::string, std::vector<unsigned char>> kCompiledProgramMap;
+  VLOG(1) << "Create program with merged binary map";
+  auto it_binary = kCompiledProgramMap.find(binary_file_name_prefix);
+  if (it_binary == kCompiledProgramMap.end()) {
+    LOG(FATAL) << "Cannot found the binary key '" << binary_file_name_prefix
+      << "' in kCompiledProgramMap";
+  }
+  std::vector<unsigned char> binary = it_binary->second;
+  *program = cl::Program(this->context(), {device()}, {binary});
+#else
   std::string source_filename = kernel_path_ + program_file_name;
   std::string binary_filename = kernel_path_ + binary_file_name_prefix + ".bin";
 
@@ -187,10 +198,10 @@ void OpenCLRuntime::BuildProgram(const std::string &program_file_name,
   bool is_binary_filename_exist = std::ifstream(binary_filename).is_open();
   if (is_binary_filename_exist) {
     VLOG(1) << "Create program with binary: " << binary_filename;
-    std::vector<unsigned char> binaries;
-    MACE_CHECK(ReadFile(binary_filename, true, &binaries));
+    std::vector<unsigned char> binary;
+    MACE_CHECK(ReadFile(binary_filename, true, &binary));
 
-    *program = cl::Program(this->context(), {device()}, {binaries});
+    *program = cl::Program(this->context(), {device()}, {binary});
 
   } else if (std::ifstream(source_filename).is_open()) {
     VLOG(1) << "Create program with source: " << source_filename;
@@ -206,6 +217,7 @@ void OpenCLRuntime::BuildProgram(const std::string &program_file_name,
     LOG(FATAL) << "Failed to open kernel file " << binary_filename << " or "
                << source_filename;
   }
+#endif
 
   // Build program
   std::string build_options_str =
@@ -223,6 +235,7 @@ void OpenCLRuntime::BuildProgram(const std::string &program_file_name,
     LOG(FATAL) << "Build program failed: " << ret;
   }
 
+#ifndef MACE_EMBED_BINARY_PROGRAM
   // Write binary if necessary
   if (!is_binary_filename_exist) {
     size_t device_list_size = 1;
@@ -250,6 +263,7 @@ void OpenCLRuntime::BuildProgram(const std::string &program_file_name,
 
     MACE_CHECK(WriteFile(binary_filename, true, content));
   }
+#endif
 }
 
 cl::Kernel OpenCLRuntime::BuildKernel(

mace/examples/BUILD

 # Examples
-load("//mace:mace.bzl", "if_android")
+load("//mace:mace.bzl", "if_android", "if_embed_binary_program")
 
 cc_binary(
     name = "helloworld",
@@ -30,13 +30,12 @@ cc_test(
 
 cc_binary(
     name = "mace_run",
-    srcs = glob(["models/*/*.cc"] + ["mace_run.cc"]),
+    srcs = ["mace_run.cc"],
     copts = ["-std=c++11", "-D_GLIBCXX_USE_C99_MATH_TR1"],
     linkopts = ["-fopenmp"],
     linkstatic = 1,
     deps = [
-        "//mace/core",
-        "//mace/ops",
+        "//mace/codegen:generated_models_lib",
         "//mace/utils:command_line_flags",
     ],
 )

mace/mace.bzl

@@ -28,4 +28,10 @@ def if_profiling_enabled(a):
   return select({
       "//mace:profiling_enabled": a,
       "//conditions:default": [],
-})
+  })
+
+def if_embed_binary_program(a):
+  return select({
+      "//mace:embed_binary_program": a,
+      "//conditions:default": [],
+  })

mace/python/tools/opencl_codegen.py

+import argparse
+import os
+import subprocess
+import sys
+
+import numpy as np
+
+import jinja2
+
+# python mace/python/tools/opencl_codegen.py \
+#     --cl_binary_dir=${CL_BIN_DIR} --output_path=${CL_HEADER_PATH}
+
+FLAGS = None
+
+
+def generate_cpp_source():
+  maps = {"binary_maps": []}
+  for file_name in os.listdir(FLAGS.cl_binary_dir):
+    file_path = os.path.join(FLAGS.cl_binary_dir, file_name)
+    if file_path[-4:] == ".bin":
+      # read binary
+      f = open(file_path, "rb")
+      binary_array = np.fromfile(f, dtype=np.uint8)
+      f.close()
+
+      binary_dict = {"name": file_name[:-4], "content": []}
+      for ele in binary_array:
+        binary_dict["content"].append(hex(ele))
+      maps["binary_maps"].append(binary_dict)
+
+  env = jinja2.Environment(loader=jinja2.FileSystemLoader(sys.path[0]))
+  return env.get_template('opencl_compiled_program.cc.tmpl').render(maps)
+
+
+def main(unused_args):
+  if not os.path.exists(FLAGS.cl_binary_dir):
+    print("Input cl_binary_dir " + FLAGS.cl_binary_dir + " doesn't exist!")
+
+  cpp_cl_binary_source = generate_cpp_source()
+  if os.path.isfile(FLAGS.output_path):
+    os.remove(FLAGS.output_path)
+  w_file = open(FLAGS.output_path, "w")
+  w_file.write(cpp_cl_binary_source)
+  w_file.close()
+
+
+def parse_args():
+  """Parses command line arguments."""
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
+      "--cl_binary_dir",
+      type=str,
+      default="./cl_bin/",
+      help="The cl binaries directory.")
+  parser.add_argument(
+      "--output_path",
+      type=str,
+      default="./mace/examples/codegen/opencl/opencl_compiled_program.cc",
+      help="The path of generated C++ header file which contains cl binaries.")
+  return parser.parse_known_args()
+
+
+if __name__ == '__main__':
+  FLAGS, unparsed = parse_args()
+  main(unused_args=[sys.argv[0]] + unparsed)

mace/python/tools/opencl_compiled_program.cc.tmpl

+//
+// Copyright (c) 2017 XiaoMi All rights reserved.
+//
+
+// This is a generated file, DO NOT EDIT
+
+#include <map>
+#include <string>
+#include <vector>
+
+namespace mace {
+
+{% for map in binary_maps %}
+//  {{map.name}}
+{% endfor %}
+
+extern const std::map<std::string, std::vector<unsigned char>> kCompiledProgramMap =
+{
+  {% for map in binary_maps %}
+  {
+    "{{map.name}}",
+    {
+      {%- for ele in map.content -%}
+      {{ele}},
+      {%- endfor -%}
+    }
+  },  // {{map.name}}
+{% endfor %}
+};
+
+}  // namespace

mace/python/tools/source_converter_lib.py

@@ -106,7 +106,7 @@ def convert_to_source(net_def, template, confuse, model_tag, output):
       tag = model_tag,
       mode = 0,
     )
-    with gfile.GFile(output_dir + str(counter) + '.cc', "wb") as f:
+    with gfile.GFile(output_dir + 'tensor' + str(counter) + '.cc', "wb") as f:
       f.write(source)
     counter += 1
 

tools/validate_gcn.sh

@@ -21,20 +21,62 @@ PHONE_DATA_DIR="/data/local/tmp/${MACE_MODEL_NAME}"
 KERNEL_DIR="${PHONE_DATA_DIR}/cl/"
 IMAGE_SIZE=$2
 MODEL_TAG=GCN${IMAGE_SIZE}
+CODEGEN_DIR=${MACE_SOURCE_DIR}/mace/codegen
+MODEL_CODEGEN_DIR=${CODEGEN_DIR}/models/gcn-$IMAGE_SIZE
+CL_CODEGEN_DIR=${CODEGEN_DIR}/opencl
+CL_BIN_DIR=${CODEGEN_DIR}/opencl_bin
+
+build_and_run()
+{
+  EMBED_OPENCL_BINARY=$1
+  if [ "$EMBED_OPENCL_BINARY" = true ]; then
+    EMBED_OPENCL_BINARY_BUILD_FLAGS="--define embed_binary_program=true"
+  fi
+
+  bazel build -c opt --strip always mace/examples:mace_run \
+    --crosstool_top=//external:android/crosstool \
+    --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \
+    --cpu=arm64-v8a \
+    $EMBED_OPENCL_BINARY_BUILD_FLAGS \
+    --copt=-DMACE_MODEL_FUNCTION=Create${MODEL_TAG}
+
+  adb shell "rm -rf ${PHONE_DATA_DIR}"
+  adb shell "mkdir -p ${PHONE_DATA_DIR}"
+  if [ "$EMBED_OPENCL_BINARY" = false ]; then
+    adb shell "mkdir -p ${KERNEL_DIR}"
+    adb push mace/kernels/opencl/cl/ ${KERNEL_DIR}
+  fi
+  adb push ${MODEL_DIR}/${INPUT_FILE_NAME} ${PHONE_DATA_DIR}
+  adb push bazel-bin/mace/examples/mace_run ${PHONE_DATA_DIR}
+
+  num_threads=${1:-4}
+
+  adb </dev/null shell MACE_CPP_MIN_VLOG_LEVEL=0 \
+    MACE_RUN_PARAMETER_PATH=${PHONE_DATA_DIR}/mace_run.config \
+    MACE_KERNEL_PATH=$KERNEL_DIR \
+    OMP_NUM_THREADS=$num_threads \
+    ${PHONE_DATA_DIR}/mace_run \
+    --model=${PHONE_DATA_DIR}/${MACE_MODEL_NAME} \
+    --input=mace_input_node \
+    --output=mace_output_node \
+    --input_shape="1,${IMAGE_SIZE},${IMAGE_SIZE},3"\
+    --input_file=${PHONE_DATA_DIR}/${INPUT_FILE_NAME} \
+    --output_file=${PHONE_DATA_DIR}/${OUTPUT_FILE_NAME} \
+    --device=OPENCL   \
+    --round=1
+}
 
-# Step 1: Generate input data
 echo "Step 1: Generate input data"
 python tools/validate.py --generate_data true --random_seed 1 \
  --input_file=${MODEL_DIR}/${INPUT_FILE_NAME} \
  --input_shape="${IMAGE_SIZE},${IMAGE_SIZE},3"
 
-# Step 2: convert tf model to mace model
-echo "Step 2: convert tf model to mace model and optimize memory"
+echo "Step 2: Convert tf model to mace model and optimize memory"
 bazel build //mace/python/tools:tf_converter
-rm -rf ${MACE_SOURCE_DIR}/mace/examples/models/gcn
-mkdir -p ${MACE_SOURCE_DIR}/mace/examples/models/gcn
+rm -rf ${MODEL_CODEGEN_DIR}
+mkdir -p ${MODEL_CODEGEN_DIR}
 bazel-bin/mace/python/tools/tf_converter --input=${TF_MODEL_FILE_PATH} \
-                                         --output=${MACE_SOURCE_DIR}/mace/examples/models/gcn/mace_gcn.cc \
+                                         --output=${MODEL_CODEGEN_DIR}/mace_gcn${IMAGE_SIZE}.cc \
                                          --input_node=input \
                                          --output_node=GCN/br_result_2/fcn_br \
                                          --data_type=DT_HALF \
@@ -44,44 +86,25 @@ bazel-bin/mace/python/tools/tf_converter --input=${TF_MODEL_FILE_PATH} \
                                          --model_tag=${MODEL_TAG} \
                                          --confuse=True
 
-# Step 3: Run model on the phone
 echo "Step 3: Run model on the phone"
-echo Create${MODEL_TAG}
-bazel build -c opt --strip always mace/examples:mace_run \
-    --crosstool_top=//external:android/crosstool \
-    --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \
-    --cpu=arm64-v8a \
-    --copt=-DMACE_MODEL_FUNCTION=Create${MODEL_TAG}
-
-adb shell "mkdir -p ${PHONE_DATA_DIR}"
-adb shell "mkdir -p ${KERNEL_DIR}"
-adb push mace/kernels/opencl/cl/ ${KERNEL_DIR}
-adb push ${MODEL_DIR}/${INPUT_FILE_NAME} ${PHONE_DATA_DIR}
-adb push bazel-bin/mace/examples/mace_run ${PHONE_DATA_DIR}
+build_and_run false
 
-num_threads=${1:-4}
+echo "Step 4: Generate OpenCL binary program and config code"
+rm -rf ${CL_BIN_DIR}/*
+adb pull ${KERNEL_DIR} ${CL_BIN_DIR}
+rm -rf ${CL_CODEGEN_DIR}
+mkdir -p ${CL_CODEGEN_DIR}
+python mace/python/tools/opencl_codegen.py \
+  --cl_binary_dir=${CL_BIN_DIR} --output_path=${CL_CODEGEN_DIR}/opencl_compiled_program.cc
 
-adb </dev/null shell MACE_CPP_MIN_VLOG_LEVEL=0 \
-        MACE_RUN_PARAMETER_PATH=${PHONE_DATA_DIR}/mace_run.config \
-        MACE_KERNEL_PATH=$KERNEL_DIR \
-        OMP_NUM_THREADS=$num_threads \
-        ${PHONE_DATA_DIR}/mace_run \
-          --model=${PHONE_DATA_DIR}/${MACE_MODEL_NAME} \
-          --input=mace_input_node \
-          --output=mace_output_node \
-          --input_shape="1,${IMAGE_SIZE},${IMAGE_SIZE},3"\
-          --input_file=${PHONE_DATA_DIR}/${INPUT_FILE_NAME} \
-          --output_file=${PHONE_DATA_DIR}/${OUTPUT_FILE_NAME} \
-          --device=OPENCL   \
-          --round=1
+echo "Step 5: Run model on the phone"
+build_and_run true
 
-# Step 4: pull the mace run result.
-echo "Step 4: pull the mace run result."
+echo "Step 6: Pull the mace run result."
 rm -rf ${MODEL_DIR}/${OUTPUT_FILE_NAME}
 adb </dev/null pull ${PHONE_DATA_DIR}/${OUTPUT_FILE_NAME} ${MODEL_DIR}
 
-# Step 5: validate the result
-echo "Step 5: validate the result"
+echo "Step 7: Validate the result"
 python tools/validate.py --model_file ${TF_MODEL_FILE_PATH} \
     --input_file ${MODEL_DIR}/${INPUT_FILE_NAME} \
     --mace_out_file ${MODEL_DIR}/${OUTPUT_FILE_NAME} \