Merge branch 'support_for_libmace' into 'master'

encrypt cl source, extract embed code for tuning and get opencl program

See merge request !191

mace/codegen/BUILD

@@ -5,7 +5,7 @@ package(
     default_visibility = ["//visibility:public"],
 )
 
-load("//mace:mace.bzl", "if_embed_binary_program")
+load("//mace:mace.bzl", "if_embed_binary_program", "if_use_source_program")
 
 cc_library(
     name = "generated_models_lib",
@@ -15,16 +15,25 @@ cc_library(
     deps = [
         "//mace/core",
         "//mace/ops",
-        '//mace/codegen:version_lib',
-    ] + if_embed_binary_program([
-        '//mace/codegen:generated_opencl_lib',
-        '//mace/codegen:generated_tuning_lib',
-    ]),
+    ] + if_embed_binary_program(["//mace/core:opencl_binary_linkage"]) +
+    if_use_source_program(["//mace/core:opencl_source_linkage"]) + [
+        "//mace/codegen:version_lib",
+        "//mace/codegen:generated_opencl_source_lib",
+        "//mace/codegen:generated_opencl_compiled_lib",
+        "//mace/codegen:generated_tuning_lib",
+    ],
 )
 
 cc_library(
-    name = "generated_opencl_lib",
-    srcs = glob(["opencl/*.cc"]),
+    name = "generated_opencl_source_lib",
+    srcs = glob(["opencl/opencl_encrypt_program.cc"]),
+    copts = ["-std=c++11", "-D_GLIBCXX_USE_C99_MATH_TR1", "-Werror=return-type"],
+    linkstatic = 1,
+)
+
+cc_library(
+    name = "generated_opencl_compiled_lib",
+    srcs = glob(["opencl/opencl_compiled_program.cc"]),
     copts = ["-std=c++11", "-D_GLIBCXX_USE_C99_MATH_TR1", "-Werror=return-type"],
     linkstatic = 1,
 )

mace/core/BUILD

@@ -8,8 +8,7 @@ package(
 licenses(["notice"])  # Apache 2.0
 
 load("//mace:mace.bzl", "if_android", "if_android_armv7", "if_android_arm64",
-
- "if_profiling_enabled", "if_embed_binary_program")
+ "if_profiling_enabled")
 
 cc_library(
     name = "core",
@@ -19,6 +18,8 @@ cc_library(
         "runtime/hexagon/*.cc",
     ],
     exclude = [
+        "runtime/opencl/opencl_binary_linkage.cc",
+        "runtime/opencl/opencl_source_linkage.cc",
         "*_test.cc",
         "runtime/hexagon/hexagon_controller_dummy.cc",
     ]) + if_android_armv7([
@@ -34,8 +35,7 @@ cc_library(
         "runtime/hexagon/*.h",
     ]),
     copts = ["-std=c++11", "-D_GLIBCXX_USE_C99_MATH_TR1", "-Werror=return-type"] +
-            if_profiling_enabled(["-DMACE_OPENCL_PROFILING"]) +
-            if_embed_binary_program(["-DMACE_EMBED_BINARY_PROGRAM"]),
+            if_profiling_enabled(["-DMACE_OPENCL_PROFILING"]),
     linkopts = if_android(["-pie", "-ldl"]),
     deps = [
         "//mace/utils:utils_hdrs",
@@ -62,3 +62,23 @@ cc_library(
         "//mace/utils:utils_hdrs",
     ],
 )
+
+cc_library(
+    name = "opencl_source_linkage",
+    srcs = ["runtime/opencl/opencl_source_linkage.cc"],
+    copts = ["-std=c++11", "-D_GLIBCXX_USE_C99_MATH_TR1", "-Werror=return-type"],
+    linkstatic = 1,
+    deps = [
+        ":core",
+    ],
+)
+
+cc_library(
+    name = "opencl_binary_linkage",
+    srcs = ["runtime/opencl/opencl_binary_linkage.cc"],
+    copts = ["-std=c++11", "-D_GLIBCXX_USE_C99_MATH_TR1", "-Werror=return-type"],
+    linkstatic = 1,
+    deps = [
+        ":core",
+    ],
+)

mace/core/runtime/opencl/opencl_binary_linkage.cc

+#include <map>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "mace/core/runtime/opencl/cl2_header.h"
+
+namespace mace {
+
+bool GetSourceOrBinaryProgram(const std::string &program_name,
+                              const std::string &binary_file_name_prefix,
+                              cl::Context &context,
+                              cl::Device &device,
+                              cl::Program *program,
+                              bool *is_binary) {
+  extern const std::map<std::string, std::vector<unsigned char>> kCompiledProgramMap;
+  *is_binary = true;
+  auto it_binary = kCompiledProgramMap.find(binary_file_name_prefix);
+  if (it_binary == kCompiledProgramMap.end()) {
+    return false;
+  }
+  *program = cl::Program(context, {device}, {it_binary->second});
+  return true;
+}
+
+bool GetTuningParams(const char *path,
+                     std::unordered_map<std::string, std::vector<unsigned int>> *param_table) {
+  extern const std::map<std::string, std::vector<unsigned int>> kTuningParamsData;
+  for (auto it = kTuningParamsData.begin(); it != kTuningParamsData.end(); ++it) {
+    param_table->emplace(it->first, std::vector<unsigned int>(it->second.begin(), it->second.end()));
+  }
+  return true;
+}
+
+}  // namespace mace

mace/core/runtime/opencl/opencl_runtime.cc

@@ -16,47 +16,6 @@
 namespace mace {
 namespace {
 
-bool ReadFile(const std::string &filename,
-              bool binary,
-              std::vector<unsigned char> *content_ptr) {
-  MACE_CHECK_NOTNULL(content_ptr);
-
-  std::ios_base::openmode mode = std::ios::in;
-  if (binary) {
-    mode |= std::ios::binary;
-  }
-
-  std::ifstream ifs(filename, mode);
-
-  // Stop eating new lines and whitespace
-  ifs.unsetf(std::ios::skipws);
-
-  if (!ifs.is_open()) {
-    LOG(ERROR) << "Failed to open file " << filename;
-    return false;
-  }
-
-  ifs.seekg(0, std::ios::end);
-  const size_t filesize = ifs.tellg();
-  if (filesize > 10 * 1024 * 1024) {
-    LOG(ERROR) << "Filesize overflow 10MB";
-    return false;
-  }
-  content_ptr->reserve(filesize);
-  ifs.seekg(0, std::ios::beg);
-  content_ptr->insert(content_ptr->begin(), std::istreambuf_iterator<char>(ifs),
-                      std::istreambuf_iterator<char>());
-
-  if (ifs.fail()) {
-    LOG(ERROR) << "Failed to read from file " << filename;
-    return false;
-  }
-
-  ifs.close();
-
-  return true;
-}
-
 bool WriteFile(const std::string &filename,
                bool binary,
                const std::vector<unsigned char> &content) {
@@ -174,45 +133,33 @@ std::string OpenCLRuntime::GenerateCLBinaryFilenamePrefix(
   return filename_prefix;
 }
 
-void OpenCLRuntime::BuildProgram(const std::string &program_file_name,
+extern bool GetSourceOrBinaryProgram(const std::string &program_name,
+                                     const std::string &binary_file_name_prefix,
+                                     cl::Context &context,
+                                     cl::Device &device,
+                                     cl::Program *program,
+                                     bool *is_opencl_binary);
+
+void OpenCLRuntime::BuildProgram(const std::string &program_name,
                                  const std::string &binary_file_name_prefix,
                                  const std::string &build_options,
                                  cl::Program *program) {
   MACE_CHECK_NOTNULL(program);
 
-#ifdef MACE_EMBED_BINARY_PROGRAM
-  extern const std::map<std::string, std::vector<unsigned char>> kCompiledProgramMap;
-  VLOG(1) << "Create program with merged binary map";
-  auto it_binary = kCompiledProgramMap.find(binary_file_name_prefix);
-  if (it_binary == kCompiledProgramMap.end()) {
-    LOG(FATAL) << "Cannot found the binary key '" << binary_file_name_prefix
-      << "' in kCompiledProgramMap";
-  }
-  std::vector<unsigned char> binary = it_binary->second;
-  *program = cl::Program(this->context(), {device()}, {binary});
-#else
-  std::string source_filename = kernel_path_ + program_file_name;
-
-  // Create program
-  if (std::ifstream(source_filename).is_open()) {
-    VLOG(1) << "Create program with source: " << source_filename;
-    std::vector<unsigned char> kernel_source;
-    MACE_CHECK(ReadFile(source_filename, false, &kernel_source));
-
-    cl::Program::Sources sources;
-    sources.push_back(std::string(kernel_source.begin(), kernel_source.end()));
-
-    *program = cl::Program(this->context(), sources);
-
-  } else {
-    LOG(FATAL) << "Failed to open kernel file " << source_filename;
-  }
-#endif
+  bool is_opencl_binary = false;
+  std::vector<unsigned char> program_vec;
+  const bool found = GetSourceOrBinaryProgram(program_name,
+                                              binary_file_name_prefix,
+                                              context(),
+                                              device(),
+                                              program,
+                                              &is_opencl_binary);
+  MACE_CHECK(found, "Program not found source: ", program_name, ", or binary: ",
+             binary_file_name_prefix);
 
   // Build program
   std::string build_options_str =
-      build_options + " -Werror -cl-mad-enable -cl-fast-relaxed-math -I" +
-      kernel_path_;
+      build_options + " -Werror -cl-mad-enable -cl-fast-relaxed-math";
   // TODO(heliangliang) -cl-unsafe-math-optimizations -cl-fast-relaxed-math
   cl_int ret = program->build({device()}, build_options_str.c_str());
   if (ret != CL_SUCCESS) {
@@ -225,41 +172,40 @@ void OpenCLRuntime::BuildProgram(const std::string &program_file_name,
     LOG(FATAL) << "Build program failed: " << ret;
   }
 
-#ifndef MACE_EMBED_BINARY_PROGRAM
-  // Write binary if necessary
-  std::string binary_filename = kernel_path_ + binary_file_name_prefix + ".bin";
-  size_t device_list_size = 1;
-  std::unique_ptr<size_t[]> program_binary_sizes(
-      new size_t[device_list_size]);
-  cl_int err = clGetProgramInfo((*program)(), CL_PROGRAM_BINARY_SIZES,
-                                sizeof(size_t) * device_list_size,
-                                program_binary_sizes.get(), nullptr);
-  MACE_CHECK(err == CL_SUCCESS) << "Error code: " << err;
-  std::unique_ptr<std::unique_ptr<unsigned char[]>[]> program_binaries(
-      new std::unique_ptr<unsigned char[]>[device_list_size]);
-  for (cl_uint i = 0; i < device_list_size; ++i) {
-    program_binaries[i] = std::unique_ptr<unsigned char[]>(
-        new unsigned char[program_binary_sizes[i]]);
-  }
+  if (!is_opencl_binary) {
+    // Write binary if necessary
+    std::string binary_filename = kernel_path_ + binary_file_name_prefix + ".bin";
+    size_t device_list_size = 1;
+    std::unique_ptr<size_t[]> program_binary_sizes(
+        new size_t[device_list_size]);
+    cl_int err = clGetProgramInfo((*program)(), CL_PROGRAM_BINARY_SIZES,
+                                  sizeof(size_t) * device_list_size,
+                                  program_binary_sizes.get(), nullptr);
+    MACE_CHECK(err == CL_SUCCESS) << "Error code: " << err;
+    std::unique_ptr<std::unique_ptr<unsigned char[]>[]> program_binaries(
+        new std::unique_ptr<unsigned char[]>[device_list_size]);
+    for (cl_uint i = 0; i < device_list_size; ++i) {
+      program_binaries[i] = std::unique_ptr<unsigned char[]>(
+          new unsigned char[program_binary_sizes[i]]);
+    }
 
-  err = clGetProgramInfo((*program)(), CL_PROGRAM_BINARIES,
-                         sizeof(unsigned char *) * device_list_size,
-                         program_binaries.get(), nullptr);
-  MACE_CHECK(err == CL_SUCCESS) << "Error code: " << err;
-  std::vector<unsigned char> content(
-      reinterpret_cast<unsigned char const *>(program_binaries[0].get()),
-      reinterpret_cast<unsigned char const *>(program_binaries[0].get()) +
-          program_binary_sizes[0]);
+    err = clGetProgramInfo((*program)(), CL_PROGRAM_BINARIES,
+                           sizeof(unsigned char *) * device_list_size,
+                           program_binaries.get(), nullptr);
+    MACE_CHECK(err == CL_SUCCESS) << "Error code: " << err;
+    std::vector<unsigned char> content(
+        reinterpret_cast<unsigned char const *>(program_binaries[0].get()),
+        reinterpret_cast<unsigned char const *>(program_binaries[0].get()) +
+            program_binary_sizes[0]);
 
-  MACE_CHECK(WriteFile(binary_filename, true, content));
-#endif
+    MACE_CHECK(WriteFile(binary_filename, true, content));
+  }
 }
 
 cl::Kernel OpenCLRuntime::BuildKernel(
     const std::string &program_name,
     const std::string &kernel_name,
     const std::set<std::string> &build_options) {
-  std::string program_file_name = program_name + ".cl";
   std::string build_options_str;
   for (auto &option : build_options) {
     build_options_str += " " + option;
@@ -274,7 +220,7 @@ cl::Kernel OpenCLRuntime::BuildKernel(
   } else {
     std::string binary_file_name_prefix =
         GenerateCLBinaryFilenamePrefix(built_program_key);
-    this->BuildProgram(program_file_name, binary_file_name_prefix,
+    this->BuildProgram(program_name, binary_file_name_prefix,
                        build_options_str, &program);
     built_program_map_.emplace(built_program_key, program);
   }

mace/core/runtime/opencl/opencl_source_linkage.cc

+#include <fstream>
+#include <map>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "mace/core/runtime/opencl/cl2_header.h"
+
+
+namespace mace {
+
+namespace {
+inline void DecryptOpenCLSource(const std::vector<unsigned char> &src,
+                                std::vector<unsigned char> *dst) {
+  dst->reserve(src.size());
+
+  // Keep consistent with encrypt in python tool
+  const std::string decrypt_lookup_table = "Xiaomi-AI-Platform-Mace";
+  size_t lookup_table_size = decrypt_lookup_table.size();
+  for (int i = 0; i < src.size(); i++) {
+    dst->push_back(src[i] ^ decrypt_lookup_table[i % lookup_table_size]);
+  }
+}
+}  // namespace
+
+bool GetSourceOrBinaryProgram(const std::string &program_name,
+                              const std::string &binary_file_name_prefix,
+                              cl::Context &context,
+                              cl::Device &device,
+                              cl::Program *program,
+                              bool *is_binary) {
+  extern const std::map<std::string, std::vector<unsigned char>> kEncryptedProgramMap;
+  *is_binary = false;
+  auto it_source = kEncryptedProgramMap.find(program_name);
+  if (it_source == kEncryptedProgramMap.end()) {
+    return false;
+  }
+  cl::Program::Sources sources;
+  std::vector<unsigned char> decrypt_source;
+  DecryptOpenCLSource(it_source->second, &decrypt_source);
+  sources.push_back(std::string(decrypt_source.begin(), decrypt_source.end()));
+  *program = cl::Program(context, sources);
+
+  return true;
+}
+
+bool GetTuningParams(const char *path,
+                     std::unordered_map<std::string, std::vector<unsigned int>> *param_table) {
+  if (path != nullptr) {
+    std::ifstream ifs(path, std::ios::binary | std::ios::in);
+    if (ifs.is_open()) {
+      int32_t key_size = 0;
+      int32_t params_size = 0;
+      int32_t params_count = 0;
+      int64_t num_pramas = 0;
+      ifs.read(reinterpret_cast<char *>(&num_pramas), sizeof(num_pramas));
+      while (num_pramas--) {
+        ifs.read(reinterpret_cast<char *>(&key_size), sizeof(key_size));
+        std::string key(key_size, ' ');
+        ifs.read(&key[0], key_size);
+
+        ifs.read(reinterpret_cast<char *>(&params_size), sizeof(params_size));
+        params_count = params_size / sizeof(unsigned int);
+        std::vector<unsigned int> params(params_count);
+        for (int i = 0; i < params_count; ++i) {
+          ifs.read(reinterpret_cast<char *>(&params[i]), sizeof(unsigned int));
+        }
+        param_table->emplace(key, params);
+      }
+      ifs.close();
+    } else {
+      return false;
+    }
+  }
+  return true;
+}
+
+}  // namespace mace

mace/examples/BUILD

 # Examples
-load("//mace:mace.bzl", "if_android", "if_enable_neon", "if_embed_binary_program")
+load("//mace:mace.bzl", "if_android", "if_enable_neon")
 
 cc_binary(
     name = "helloworld",

mace/kernels/BUILD

@@ -7,7 +7,7 @@ package(
 
 licenses(["notice"])  # Apache 2.0
 
-load("//mace:mace.bzl", "if_android", "if_enable_neon", "if_embed_binary_program")
+load("//mace:mace.bzl", "if_android", "if_enable_neon")
 
 cc_library(
     name = "kernels",
@@ -22,7 +22,6 @@ cc_library(
         "-D_GLIBCXX_USE_C99_MATH_TR1",
         "-Werror=return-type",
     ] +
-    if_embed_binary_program(["-DMACE_EMBED_BINARY_PROGRAM"]) +
     if_enable_neon(["-fopenmp"]),
     linkopts = if_android(["-lm"]),
     deps = [

mace/mace.bzl

@@ -36,6 +36,12 @@ def if_embed_binary_program(a):
       "//conditions:default": [],
   })
 
+def if_use_source_program(a):
+  return select({
+      "//mace:embed_binary_program": [],
+      "//conditions:default": a,
+  })
+
 def if_enable_neon(a):
   return select({
       "//mace:enable_neon": a,

mace/python/tools/binary_codegen.py

@@ -38,14 +38,13 @@ def generate_cpp_source():
     idx += params_size
 
   env = jinja2.Environment(loader=jinja2.FileSystemLoader(sys.path[0]))
-  return env.get_template('binary.cc.tmpl').render(
+  return env.get_template('str2vec_maps.cc.tmpl').render(
     maps = data_map,
-    data_type = 'int',
+    data_type = 'unsigned int',
     variable_name = FLAGS.variable_name
   )
 
 def main(unused_args):
-
   cpp_binary_source = generate_cpp_source()
   if os.path.isfile(FLAGS.output_path):
     os.remove(FLAGS.output_path)

mace/python/tools/encrypt_opencl_codegen.py

+import argparse
+import os
+import sys
+
+import jinja2
+
+# python encrypt_opencl_codegen.py --cl_kernel_dir=./mace/kernels/opencl/cl/    \
+#     --output_path=./mace/codegen/opencl_encrypt/opencl_encrypted_program.cc
+
+FLAGS = None
+
+encrypt_lookup_table = "Xiaomi-AI-Platform-Mace"
+
+def encrypt_code(code_str):
+  encrypted_arr = []
+  for i in range(len(code_str)):
+    encrypted_char = hex(ord(code_str[i]) ^ ord(encrypt_lookup_table[i % len(encrypt_lookup_table)]))
+    encrypted_arr.append(encrypted_char)
+  return encrypted_arr
+
+
+def main(unused_args):
+  if not os.path.exists(FLAGS.cl_kernel_dir):
+    print("Input cl_kernel_dir " + FLAGS.cl_kernel_dir + " doesn't exist!")
+
+  header_code = ""
+  for file_name in os.listdir(FLAGS.cl_kernel_dir):
+    file_path = os.path.join(FLAGS.cl_kernel_dir, file_name)
+    if file_path[-2:] == ".h":
+      f = open(file_path, "r")
+      header_code += f.read()
+
+  encrypted_code_maps = {}
+  for file_name in os.listdir(FLAGS.cl_kernel_dir):
+    file_path = os.path.join(FLAGS.cl_kernel_dir, file_name)
+    if file_path[-3:] == ".cl":
+      f = open(file_path, "r")
+      code_str = ""
+      for line in f.readlines():
+        if "#include <common.h>" in line:
+          code_str += header_code
+        else:
+          code_str += line
+      encrypted_code_arr = encrypt_code(code_str)
+      encrypted_code_maps[file_name[:-3]] = encrypted_code_arr
+
+  env = jinja2.Environment(loader=jinja2.FileSystemLoader(sys.path[0]))
+  cpp_cl_encrypted_kernel = env.get_template('str2vec_maps.cc.tmpl').render(
+      maps=encrypted_code_maps,
+      data_type='unsigned char',
+      variable_name='kEncryptedProgramMap')
+
+  if os.path.isfile(FLAGS.output_path):
+    os.remove(FLAGS.output_path)
+  w_file = open(FLAGS.output_path, "w")
+  w_file.write(cpp_cl_encrypted_kernel)
+  w_file.close()
+
+  print("Generate encrypted opencl source done!")
+
+
+def parse_args():
+  """Parses command line arguments."""
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
+      "--cl_kernel_dir",
+      type=str,
+      default="./mace/kernels/opencl/cl/",
+      help="The cl kernels directory.")
+  parser.add_argument(
+      "--output_path",
+      type=str,
+      default="./mace/examples/codegen/opencl/opencl_encrypted_program.cc",
+      help="The path of encrypted opencl kernels.")
+  return parser.parse_known_args()
+
+
+if __name__ == '__main__':
+  FLAGS, unparsed = parse_args()
+  main(unused_args=[sys.argv[0]] + unparsed)

mace/python/tools/opencl_codegen.py

 import argparse
 import os
-import subprocess
 import sys
 
 import numpy as np
@@ -28,7 +27,7 @@ def generate_cpp_source():
         maps[file_name[:-4]].append(hex(ele))
 
   env = jinja2.Environment(loader=jinja2.FileSystemLoader(sys.path[0]))
-  return env.get_template('binary.cc.tmpl').render(
+  return env.get_template('str2vec_maps.cc.tmpl').render(
     maps = maps,
     data_type = 'unsigned char',
     variable_name = 'kCompiledProgramMap'

mace/utils/tuner.h

@@ -18,6 +18,9 @@
 
 namespace mace {
 
+extern bool GetTuningParams(const char *path,
+    std::unordered_map<std::string, std::vector<unsigned int>> *param_table);
+
 template <typename param_type>
 class Tuner {
  public:
@@ -99,40 +102,10 @@ class Tuner {
   }
 
   inline void ReadRunParamters() {
-#ifdef MACE_EMBED_BINARY_PROGRAM
-    extern const std::map<std::string, std::vector<int>> kTuningParamsData;
-    VLOG(1) << "Read tuning parameters from source";
-    for (auto it = kTuningParamsData.begin(); it != kTuningParamsData.end(); ++it) {
-      param_table_.emplace(it->first, std::vector<param_type>(it->second.begin(), it->second.end()));
-    }
-#else
-    if (path_ != nullptr) {
-      std::ifstream ifs(path_, std::ios::binary | std::ios::in);
-      if (ifs.is_open()) {
-        int32_t key_size = 0;
-        int32_t params_size = 0;
-        int32_t params_count = 0;
-        int64_t num_pramas = 0;
-        ifs.read(reinterpret_cast<char *>(&num_pramas), sizeof(num_pramas));
-        while (num_pramas--) {
-          ifs.read(reinterpret_cast<char *>(&key_size), sizeof(key_size));
-          std::string key(key_size, ' ');
-          ifs.read(&key[0], key_size);
-
-          ifs.read(reinterpret_cast<char *>(&params_size), sizeof(params_size));
-          params_count = params_size / sizeof(param_type);
-          std::vector<param_type> params(params_count);
-          for (int i = 0; i < params_count; ++i) {
-            ifs.read(reinterpret_cast<char *>(&params[i]), sizeof(param_type));
-          }
-          param_table_.emplace(key, params);
-        }
-        ifs.close();
-      } else {
-        LOG(WARNING) << "Read run parameter file failed.";
-      }
+    bool success = GetTuningParams(path_, &param_table_);
+    if (!success) {
+      LOG(WARNING) << "Get run parameter failed.";
     }
-#endif
   }
 
   template <typename RetType>

tools/bazel-adb-run.sh

@@ -5,7 +5,8 @@ if [ "$#" -lt 1 ]; then
   exit 1
 fi
 
-CL_PATH=mace/kernels/opencl/cl/
+MACE_SOURCE_DIR=`/bin/pwd`
+CL_CODEGEN_DIR=${MACE_SOURCE_DIR}/mace/codegen/opencl
 DEVICE_PATH=/data/local/tmp/mace
 DEVICE_CL_PATH=$DEVICE_PATH/cl/
 BAZEL_TARGET=$1
@@ -26,6 +27,11 @@ PROFILINE="--define profiling=true"
 BRANCH=$(git symbolic-ref --short HEAD)
 COMMIT_ID=$(git rev-parse --short HEAD)
 
+echo "Step 1: Generate encrypted opencl source"
+python mace/python/tools/encrypt_opencl_codegen.py \
+    --cl_kernel_dir=./mace/kernels/opencl/cl/ --output_path=${CL_CODEGEN_DIR}/opencl_encrypt_program.cc
+
+echo "Step 2: Build target"
 bazel build -c opt $STRIP --verbose_failures $BAZEL_TARGET \
    --crosstool_top=//external:android/crosstool \
    --host_crosstool_top=@bazel_tools//tools/cpp:toolchain  \
@@ -36,6 +42,7 @@ if [ $? -ne 0 ]; then
   exit 1
 fi
 
+echo "Step 3: Run target"
 du -hs $BAZEL_BIN_PATH/$BIN_NAME
 
 for device in `adb devices | grep "^[A-Za-z0-9]\+[[:space:]]\+device$"| cut -f1`; do
@@ -43,7 +50,6 @@ for device in `adb devices | grep "^[A-Za-z0-9]\+[[:space:]]\+device$"| cut -f1`
   echo "Run on device: ${device}"
   adb -s ${device} shell "rm -rf $DEVICE_PATH"
   adb -s ${device} shell "mkdir -p $DEVICE_PATH"
-  adb -s ${device} push $CL_PATH $DEVICE_CL_PATH && \
   adb -s ${device} push $BAZEL_BIN_PATH/$BIN_NAME $DEVICE_PATH && \
   adb -s ${device} shell "MACE_KERNEL_PATH=$DEVICE_CL_PATH MACE_CPP_MIN_VLOG_LEVEL=$VLOG_LEVEL $DEVICE_PATH/$BIN_NAME $@"
 done

tools/validate_gcn.sh

@@ -49,7 +49,6 @@ build_and_run()
   adb shell "mkdir -p ${PHONE_DATA_DIR}"
   if [ "$EMBED_OPENCL_BINARY" = false ]; then
     adb shell "mkdir -p ${KERNEL_DIR}"
-    adb push mace/kernels/opencl/cl/. ${KERNEL_DIR}
   fi
   adb push ${MODEL_DIR}/${INPUT_FILE_NAME} ${PHONE_DATA_DIR}
   adb push bazel-bin/mace/examples/mace_run ${PHONE_DATA_DIR}
@@ -101,31 +100,36 @@ rm -rf ${VERSION_SOURCE_PATH}
 mkdir -p ${VERSION_SOURCE_PATH}
 bash mace/tools/git/gen_version_source.sh ${VERSION_SOURCE_PATH}/version.cc
 
-echo "Step 4: Run model on the phone with files"
+echo "Step 4: Generate encrypted opencl source"
+rm -rf ${CL_CODEGEN_DIR}
+mkdir -p ${CL_CODEGEN_DIR}
+python mace/python/tools/encrypt_opencl_codegen.py \
+  --cl_kernel_dir=./mace/kernels/opencl/cl/ --output_path=${CL_CODEGEN_DIR}/opencl_encrypt_program.cc
+
+echo "Step 5: Run model on the phone with files"
 build_and_run false
 
-echo "Step 5: Generate OpenCL binary program and config code"
+echo "Step 6: Generate OpenCL binary program and config code"
 rm -rf ${CL_BIN_DIR}
 adb pull ${KERNEL_DIR} ${CL_BIN_DIR}
-rm -rf ${CL_CODEGEN_DIR}
-mkdir -p ${CL_CODEGEN_DIR}
 python mace/python/tools/opencl_codegen.py \
   --cl_binary_dir=${CL_BIN_DIR} --output_path=${CL_CODEGEN_DIR}/opencl_compiled_program.cc
 
-echo "Step 6: Generate tuning source file"
+echo "Step 7: Generate tuning source file"
 adb pull ${PHONE_DATA_DIR}/mace_run.config ${CL_BIN_DIR}
+rm -rf ${TUNING_CODEGEN_DIR}
 mkdir -p ${TUNING_CODEGEN_DIR}
 python mace/python/tools/binary_codegen.py \
   --binary_file=${CL_BIN_DIR}/mace_run.config --output_path=${TUNING_CODEGEN_DIR}/tuning_params.cc
 
-echo "Step 7: Run model on the phone using binary"
+echo "Step 8: Run model on the phone using binary"
 build_and_run true
 
-echo "Step 8: Pull the mace run result."
+echo "Step 9: Pull the mace run result."
 rm -rf ${MODEL_DIR}/${OUTPUT_FILE_NAME}
 adb </dev/null pull ${PHONE_DATA_DIR}/${OUTPUT_FILE_NAME} ${MODEL_DIR}
 
-echo "Step 9: Validate the result"
+echo "Step 10: Validate the result"
 python tools/validate.py --model_file ${TF_MODEL_FILE_PATH} \
     --input_file ${MODEL_DIR}/${INPUT_FILE_NAME} \
     --mace_out_file ${MODEL_DIR}/${OUTPUT_FILE_NAME} \