enable fast test compilation && push opencl kernels by run.py (#1852)

* enable fast test compilation && push opencl kernels by run.py

* merge cl kernels into so

* restore pre-commit

mobile/.clang-format

+---
+Language:        Cpp
+BasedOnStyle: Google 
+Standard:  Cpp11 
+...

mobile/.clang-tidy

+Checks: >
+  *
+  -android-*
+  -bugprone-bool-pointer-implicit-conversion
+  -cert-env33-c
+  -cert-dcl50-cpp
+  -cert-dcl59-cpp
+  -cppcoreguidelines-*
+  -fuchsia-*
+  -google-*
+  google-default-arguments
+  google-explicit-constructor
+  google-runtime-member-string-references
+  google-runtime-operator
+  -hicpp-braces-around-statements
+  -hicpp-named-parameter
+  -hicpp-no-array-decay
+  -hicpp-no-assembler
+  -hicpp-no-malloc
+  -hicpp-function-size
+  -hicpp-special-member-functions
+  -hicpp-vararg
+  -llvm-*
+  -objc-*
+  -readability-else-after-return
+  -readability-implicit-bool-conversion
+  -readability-named-parameter
+  -readability-simplify-boolean-expr
+  -readability-braces-around-statements
+  -readability-identifier-naming
+  -readability-function-size
+  -readability-redundant-member-init
+  -misc-bool-pointer-implicit-conversion
+  -misc-definitions-in-headers
+  -misc-unused-alias-decls
+  -misc-unused-parameters
+  -misc-unused-using-decls
+  -modernize-use-using
+  -modernize-use-default-member-init
+  -clang-diagnostic-*
+  -clang-analyzer-*
+WarningsAsErrors: '*'
+HeaderFilterRegex: ''
+AnalyzeTemporaryDtors: false
+FormatStyle:     none
+User:            allonli
+CheckOptions:    
+  - key:             google-readability-braces-around-statements.ShortStatementLines
+    value:           '1'
+  - key:             google-readability-function-size.StatementThreshold
+    value:           '800'
+  - key:             google-readability-namespace-comments.ShortNamespaceLines
+    value:           '10'
+  - key:             google-readability-namespace-comments.SpacesBeforeComments
+    value:           '2'
+  - key:             modernize-loop-convert.MaxCopySize
+    value:           '16'
+  - key:             modernize-loop-convert.MinConfidence
+    value:           reasonable
+  - key:             modernize-loop-convert.NamingStyle
+    value:           CamelCase
+  - key:             modernize-pass-by-value.IncludeStyle
+    value:           llvm
+  - key:             modernize-replace-auto-ptr.IncludeStyle
+    value:           llvm
+  - key:             modernize-use-nullptr.NullMacros
+    value:           'NULL'

mobile/.gitignore

+opencl_kernels.cpp
+# Prerequisites
+*.d
+
+# Compiled Object files
+*.slo
+*.lo
+*.o
+*.obj
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
+# Fortran module files
+*.mod
+*.smod
+
+# Compiled Static libraries
+*.lai
+*.la
+*.lib
+*.a
+
+# Executables
+*.exe
+*.out
+*.app
+
+.DS_Store
+
+build/
+
+.idea/
+
+CMakeCache.txt
+
+CMakeFiles/
+
+Makefile
+
+cmake_install.cmake
+
+
+*.cbp
+
+paddle-mobile.cbp
+
+.idea
+
+compile_commands.json
+
+cmake-build-debug/
+cmake-build-release/
+
+test/models/
+
+test/images/
+
+# Emacs intermediate files
+*~
+
+# CMake building directory
+build
+
+# clion building directories
+cmake-build-debug
+cmake-build-release
+
+# ios
+tools/libomp.a
+
+# ios demo
+demo/ios/PaddleMobileDemo/PaddleMobileDemo/googlenet_combine/
+demo/ios/PaddleMobileDemo/PaddleMobileDemo/*.jpg
+demo/ios/PaddleMobileDemo/PaddleMobileDemo/PaddleMobile/*.a
+*.xcuserstate
+/tools/quantification/quantify
+
+# metal
+Podfile.lock
+metal/Pods/
+SwiftProtobuf.framework
+paddle-mobile.xcworkspace
+metal/models/
+metal/images/
+*.a
+metal/paddle-mobile/paddle-mobile/CPU/libpaddle-mobile.a
+*.xcuserdatad/
+*/xcuserdata/
+/venv/
+
+metal/paddle-mobile-demo/paddle-mobile-demo/images
+metal/paddle-mobile-demo/paddle-mobile-demo/models
+metal/paddle-mobile-demo/paddle-mobile-demo/Resources
+metal/paddle-mobile-demo/paddle-mobile-demo/Resources/images
+metal/paddle-mobile-demo/paddle-mobile-demo/Resources/models
+metal/MobileNetDemo/MobileNetDemo/Resources

mobile/.pre-commit-config.yaml

+repos:
+-   repo: https://github.com/Lucas-C/pre-commit-hooks.git
+    sha: v1.0.1
+    hooks:
+    -   id: remove-crlf
+        files: ^(mobile/src/).*\.(md|py|mm|swift|java|c|cc|cxx|cpp|cu|h|hpp|hxx)$
+        exclude: ^(lite/)
+    -   id: remove-tabs
+        files: ^(mobile/test/|mobile/src/).*\.(md|py|mm|swift|java|c|cc|cxx|cpp|cu|h|hpp|hxx)$
+        exclude: ^(lite/)
+
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    sha: 5bf6c09bfa1297d3692cadd621ef95f1284e33c0
+    hooks:
+    -   id: check-added-large-files
+        exclude: ^(lite/)
+    -   id: check-merge-conflict
+        exclude: ^(lite/)
+    -   id: check-symlinks
+        exclude: ^(lite/)
+    -   id: detect-private-key
+        files: (?!.*tar.gz)^.*$ 
+        exclude: ^(lite/)
+    -   id: end-of-file-fixer
+        files: ^(mobile/test/|mobile/src/).*\.(md|py|mm|swift|java|c|cc|cxx|cpp|h|hpp|hxx)$
+        exclude: ^(lite/)
+    -   id: trailing-whitespace
+        files: ^(mobile/test/|mobile/src/).*\.(md|py|mm|swift|java|c|cc|cxx|cpp|h|hpp|hxx)$
+        exclude: ^(lite/)
+
+-   repo: local
+    hooks:
+    -   id: copyright
+        name: copyright
+        entry: python ./mobile/tools/pre-commit.hooks/copyright.hook
+        language: system
+        files: ^(mobile/test/|mobile/src/).*\.(c|cc|cxx|cpp|h|hpp|hxx|py)$
+        exclude: (?!.*third_party)^.*$ | (?!.*book)^.*$ | ^(lite/)
+
+-   repo: local
+    hooks:
+    -   id: clang-format
+        name: clang-format
+        description: Format files with ClangFormat.
+        entry: bash ./mobile/tools/pre-commit.hooks/clang-format.hook -i
+        language: system
+        files: ^(mobile/test/|mobile/src/).*\.(c|cc|cxx|cpp|h|hpp|hxx)$
+        exclude: ^(lite/)
+
+-   repo: local
+    hooks:
+    -   id: cpplint
+        name: cpplint
+        description: Check C++ code style using cpplint.
+        entry: bash ./mobile/tools/pre-commit.hooks/cpplint.hook
+        language: system
+        files: ^(mobile/test/|mobile/src/).*\.(c|cc|cxx|cpp|h|hpp|hxx)$
+        exclude: (?!.*third_party)^.*$ | (?!.*book)^.*$i | *\.pb\.cpp | ^(lite/)
+
+
+#
+#-   repo: local
+#    hooks:
+#    -   id: clang-tidy
+#        name: clang-tidy
+#        description: Check C++ code style using clang-tidy.
+#        entry: bash ./tools/pre-commit.hooks/.clang-tidy.hook -i
+#        language: system
+#        files: (src).*\.(c|cc|cxx|cpp|h|hpp|hxx)$

mobile/.travis.yml

+language: cpp
+cache: ccache
+sudo: required
+dist: trusty
+
+os:
+  - linux
+
+addons:
+  apt:
+    packages:
+      - git
+      - python
+      - python-pip
+      - python2.7-dev
+      - libc6-i386
+      - curl
+
+compiler:
+  - clang
+        
+before_install:
+  - sudo pip install -U virtualenv pre-commit pip
+  # Download and install recent cmake
+
+script:
+  - | 
+    function timeout() { perl -e 'alarm shift; exec @ARGV' "$@"; }
+  - |
+    timeout 600 .travis/pre-commit-job.sh # 10min timeout
+    RESULT=$?; if [ $RESULT -eq 0 ] || [ $RESULT -eq 142 ]; then true; else exit 1; fi;
+
+notifications:
+  email:
+    on_success: change
+    on_failure: always

mobile/.travis/pre-commit-job.sh

+#!/bin/bash
+function abort(){
+    echo "Your change doesn't follow Paddle-Moible's code style" 1>&2
+    echo "Please use pre-commit to auto-format your code." 1>&2
+    exit 1
+}
+
+trap 'abort' 0
+set -e
+cd `dirname $0`
+cd ..
+export PATH=/usr/bin:$PATH
+pre-commit install
+
+if ! pre-commit run -a ; then
+  ls -lh
+  git diff  --exit-code
+  exit 1
+fi
+
+trap : 0

mobile/src/framework/cl/cl_engine.h

@@ -96,6 +96,21 @@ class CLEngine {
     return std::move(program_ptr);
   }
 
+  std::unique_ptr<_cl_program, CLProgramDeleter> CreateProgramWithSource(
+      cl_context context, const char *source) {
+    size_t sourceSize[] = {strlen(source)};
+    cl_program p =
+        clCreateProgramWithSource(context, 1, &source, sourceSize, &status_);
+
+    DLOG << " cl kernel from source";
+    DLOG << " source size: " << sourceSize[0];
+    CL_CHECK_ERRORS(status_);
+
+    std::unique_ptr<_cl_program, CLProgramDeleter> program_ptr(p);
+
+    return std::move(program_ptr);
+  }
+
   std::unique_ptr<_cl_event, CLEventDeleter> CreateEvent(cl_context context) {
     cl_event event = clCreateUserEvent(context, &status_);
     std::unique_ptr<_cl_event, CLEventDeleter> event_ptr(event);

mobile/src/framework/cl/cl_scope.h

@@ -14,9 +14,11 @@ limitations under the License. */
 
 #pragma once
 
+#include <map>
 #include <memory>
 #include <string>
 #include <unordered_map>
+#include <vector>
 
 #include "CL/cl.h"
 #include "framework/cl/cl_deleter.h"
@@ -24,6 +26,10 @@ limitations under the License. */
 #include "framework/cl/cl_tool.h"
 
 namespace paddle_mobile {
+
+extern const std::map<std::string, std::vector<unsigned char>> opencl_kernels;
+extern const std::vector<std::string> need_conv_header_kernels;
+
 namespace framework {
 
 class CLScope {
@@ -62,15 +68,35 @@ class CLScope {
       return it->second.get();
     }
 
-    auto program = CLEngine::Instance()->CreateProgramWith(
-        context_,
-        CLEngine::Instance()->GetCLPath() + "/cl_kernel/" + file_name);
-
-    DLOG << " --- begin build program -> " << program_key << " --- ";
-    CLEngine::Instance()->BuildProgram(program.get(), options);
-    DLOG << " --- end build program -> " << program_key << " --- ";
-
-    programs_[program_key] = std::move(program);
+    if (opencl_kernels.find(file_name) != opencl_kernels.end()) {
+      auto it = opencl_kernels.find(file_name);
+      std::string source(it->second.begin(), it->second.end());
+      if (std::find(need_conv_header_kernels.begin(),
+                    need_conv_header_kernels.end(),
+                    file_name) != need_conv_header_kernels.end()) {
+        auto it = opencl_kernels.find("conv_kernel.inc.cl");
+        std::string header(it->second.begin(), it->second.end());
+        source = header + source;
+      }
+      auto program = CLEngine::Instance()->CreateProgramWithSource(
+          context_, source.c_str());
+
+      DLOG << " --- begin build program -> " << program_key << " --- ";
+      CLEngine::Instance()->BuildProgram(program.get(), options);
+      DLOG << " --- end build program -> " << program_key << " --- ";
+
+      programs_[program_key] = std::move(program);
+    } else {
+      auto program = CLEngine::Instance()->CreateProgramWith(
+          context_,
+          CLEngine::Instance()->GetCLPath() + "/cl_kernel/" + file_name);
+
+      DLOG << " --- begin build program -> " << program_key << " --- ";
+      CLEngine::Instance()->BuildProgram(program.get(), options);
+      DLOG << " --- end build program -> " << program_key << " --- ";
+
+      programs_[program_key] = std::move(program);
+    }
 
     return programs_[program_key].get();
   }

mobile/src/io/paddle_mobile_wrap.h

@@ -16,9 +16,9 @@ limitations under the License. */
 
 #include <cstdint>
 #include <map>
+#include <memory>
 #include <string>
 #include <unordered_map>
-#include <memory>
 #include <utility>
 #include <vector>
 

mobile/src/operators/kernel/arm/conditional_block_kernel.cpp

@@ -14,10 +14,10 @@ limitations under the License. */
 
 #ifdef CONDITIONAL_BLOCK_OP
 
-#include <algorithm>
 #include "operators/kernel/conditional_block_kernel.h"
 #include <framework/program/block_desc.h>
 #include <framework/program/op_desc.h>
+#include <algorithm>
 #include "framework/data_type.h"
 
 namespace paddle_mobile {

mobile/src/operators/kernel/cl/gen_code.py

+#  Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import re
+import os
+import sys
+
+source = """
+#pragma
+#ifdef PADDLE_MOBILE_CL
+#include <map>
+#include <string>
+#include <vector>
+namespace paddle_mobile {
+    extern const std::map<std::string, std::vector<unsigned char>> opencl_kernels = {
+%s
+    };
+    extern const std::vector<std::string> need_conv_header_kernels = {
+        %s
+    };
+}
+#endif
+"""
+
+def string_to_hex(str):
+    hex_list = []
+    for i in range(len(code_str)):
+        hex_ = hex(ord(code_str[i]))
+        hex_list.append(hex_)
+    return hex_list
+
+infile = open("cl_kernel/cl_common.h", "r")
+common_content = infile.read()
+infile.close()
+common_content = re.sub(r"/\*[^*]*\*/", "", common_content, flags=re.DOTALL)
+lines = common_content.split("\n")
+new_lines = []
+for i in range(len(lines)):
+    line = lines[i]
+    line = line.strip()
+    if line == "":
+        continue
+    if line.startswith("//"):
+        continue
+    line = re.sub(r"//.*$", "", line)
+    new_lines.append(line)
+common_content = "\n".join(new_lines)
+
+need_conv_header_kernels = []
+
+cores = ""
+filenames = os.listdir("cl_kernel")
+file_count = len(filenames)
+for i in range(file_count):
+    filename = filenames[i]
+    infile = open("cl_kernel/" + filename, "r")
+    new_lines = []
+    content = infile.read()
+    content = re.sub(r"/\*[^*]*\*/", "", content, flags=re.DOTALL)
+    infile.close()
+    lines = content.split("\n")
+    for i in range(len(lines)):
+        line = lines[i]
+        line = line.strip()
+        if line == "":
+            continue
+        if line.startswith("//"):
+            continue
+        line = re.sub(r"//.*$", "", line)
+        if "cl_common.h" in line:
+            line = common_content
+        elif "conv_kernel.inc.cl" in line:
+            need_conv_header_kernels.append("\"%s\"" % filename)
+            continue
+        new_lines.append(line)
+    content = "\n".join(new_lines)
+    if content == "":
+        content = " "
+    hexes = []
+    for char in content:
+        hexes.append(hex(ord(char)))
+    core = "        {\"%s\", {" % filename
+    for item in hexes:
+        core += str(item) + ", "
+    core = core[: -2]
+    core += "}}"
+    if i != file_count - 1:
+        core += ",\n"
+    cores += core
+
+source = source % (cores, ",".join(need_conv_header_kernels))
+print(source)

mobile/tools/build.sh

@@ -2,6 +2,15 @@
 NETS=""
 declare -a supportedNets=("googlenet" "mobilenet" "yolo" "squeezenet" "resnet" "mobilenetssd" "nlp" "mobilenetfssd" "genet" "super" "op")
 
+# merge cl to so
+merge_cl_to_so=1
+rm ../src/operators/kernel/cl/opencl_kernels.cpp
+if [ $merge_cl_to_so == 1 ]; then
+    cd ../src/operators/kernel/cl
+    python gen_code.py > opencl_kernels.cpp
+    cd -
+fi
+
 build_for_mac() {
     if [ ! `which brew` ]; then
         echo "building failed! homebrew not found, please install homebrew."

mobile/tools/pre-commit.hooks/clang-format.hook

 #!/bin/bash
-set -e
+# set -e
 
 readonly VERSION="5.0"
 

mobile/tools/python/fluidtools/run.py

@@ -535,6 +535,7 @@ def main():
     push(checked_model_path)
     push(feed_path + "/" + last_feed_file_name, "input.txt")
     push(mobile_src_root + "/build/release/arm-v7a/build/libpaddle-mobile.so")
+    push(mobile_src_root + "/build/release/arm-v7a/build/cl_kernel")
     push(mobile_src_root + "/test/build/test-net")
     last_feed_var_shape = get_feed_var_shape(last_feed_var_name)
     args = str(len(last_feed_var_shape))