Merge remote-tracking branch 'upstream/develop' into develop

README.md

@@ -26,8 +26,15 @@ Paddle-Moible是PaddlePaddle组织下的项目，是一个致力于嵌入式平
 
 - **ARM CPU**
 
-
-![](http://mms-graph.bj.bcebos.com/paddle-mobile%2F2018_07_29.png)
+|mobilenet arm v7|1线程|2线程|4线程|
+|------------|----|-----|-----|
+|麒麟960(ms)|110.586|72.474|49.833|
+|||||
+|mobilenetssd arm v7|1线程|2线程|4线程|
+|麒麟960(ms)|224.464|142.544|96.068|
+|||||
+|googlenet(v1) arm v7|1线程|2线程|4线程|
+|麒麟960(ms)|348.018|242.689|169.998|
 
     arm cpu是paddle-mobile的主要支持方向，cpu的通用性一直是其优势。嵌入式深度学习，需要大量的cpu汇编实现。我们正在紧锣密鼓的编码，为的是能充分硬件的每一点加速能力。
     arm cpu的优化工作还在进行中，现在使用了常规的cpu优化。在arm a73上paddle-mobile arm-v7现在单核运行一次mobilenet1.0是110+ms，显然这不是我们的最终目标，我们正在用大量的汇编改写，后续性能仍会有巨大提升空间, 目前只支持armv7, 未来我们也会支持armv8。

src/io/api.cc

@@ -12,6 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
 
+#include "cstring"
 #include "io/paddle_inference_api.h"
 
 namespace paddle_mobile {

src/operators/kernel/fpga/elementwise_add_relu_kernel.cpp

@@ -25,9 +25,9 @@ bool ElementwiseAddReluKernel<FPGA, float>::Init(
   const Tensor *input_x = param->InputX();
   const Tensor *input_y = param->InputY();
   Tensor *out = param->Out();
-  auto input_x_ptr = input_x->data<float>();
-  auto input_y_ptr = input_y->data<float>();
-  auto out_ptr = out->mutable_data<float>();
+  auto input_x_ptr = input_x->data<half>();
+  auto input_y_ptr = input_y->data<half>();
+  auto out_ptr = out->mutable_data<half>();
 
   fpga::EWAddArgs ewaddArgs;
   ewaddArgs.relu_enabled = relu_enabled;

src/operators/kernel/fpga/fc_relu_kernel.cpp

@@ -22,13 +22,13 @@ template <>
 bool FusionFcReluKernel<FPGA, float>::Init(FusionFcReluParam *param) {
   bool relu_enabled = true;
   const Tensor *input_x = param->InputX();
-  auto input_x_ptr = input_x->data<float>();
+  auto input_x_ptr = input_x->data<half>();
   const Tensor *input_y = param->InputY();
   auto input_y_ptr = input_y->data<float>();
   const Tensor *input_z = param->InputZ();
   auto input_z_ptr = input_z->data<float>();
   Tensor *out = param->Out();
-  auto out_ptr = out->mutable_data<float>();
+  auto out_ptr = out->mutable_data<half>();
 
   PADDLE_MOBILE_ENFORCE(input_x->dims()[1] == input_y->dims()[0],
                         "Image channel should be equal to weight number");

src/operators/kernel/fpga/fusion_fc_kernel.cpp

@@ -22,13 +22,13 @@ template <>
 bool FusionFcKernel<FPGA, float>::Init(FusionFcParam *param) {
   bool relu_enabled = false;
   const Tensor *input_x = param->InputX();
-  auto input_x_ptr = input_x->data<float>();
+  auto input_x_ptr = input_x->data<half>();
   const Tensor *input_y = param->InputY();
   auto input_y_ptr = input_y->data<float>();
   const Tensor *input_z = param->InputZ();
   auto input_z_ptr = input_z->data<float>();
   Tensor *out = param->Out();
-  auto out_ptr = out->mutable_data<float>();
+  auto out_ptr = out->mutable_data<half>();
 
   PADDLE_MOBILE_ENFORCE(input_x->dims()[1] == input_y->dims()[0],
                         "Image channel should be equal to weight number");

src/operators/kernel/fpga/pool_kernel.cpp

@@ -22,9 +22,9 @@ namespace operators {
 template <>
 bool PoolKernel<FPGA, float>::Init(PoolParam *param) {
   const Tensor *input = param->Input();
-  auto input_ptr = input->data<float>();
+  auto input_ptr = input->data<half>();
   Tensor *output = param->Output();
-  auto output_ptr = output->mutable_data<float>();
+  auto output_ptr = output->mutable_data<half>();
   vector<int> ksize = param->Ksize();
   vector<int> strides = param->Strides();
   vector<int> paddings = param->Paddings();

src/operators/math/depthwise_conv_3x3.cpp

@@ -529,6 +529,252 @@ void DepthwiseConvAddBNRelu3x3s1p1(const Tensor *input, const Tensor *filter,
   const float *newscale_data = new_scale->data<float>();
   const float *newbias_data = new_bias->data<float>();
 
+  const int batch_size = static_cast<int>(input->dims()[0]);
+  const int input_channel = static_cast<int>(input->dims()[1]);
+
+  const int input_height = static_cast<int>(input->dims()[2]);
+  const int input_width = static_cast<int>(input->dims()[3]);
+  const int output_height = static_cast<int>(output->dims()[2]);
+  const int output_width = static_cast<int>(output->dims()[3]);
+
+  const int hxw = input_height * input_width;
+
+  const int l = input_height;
+  float32x4_t vnewbias = vdupq_n_f32(0.0);
+  float32x4_t vnewscale = vdupq_n_f32(1.0);
+  float32x4_t vzero = vdupq_n_f32(0);
+
+  for (int b = 0; b < batch_size; b++) {
+    filter_data = filter->data<float>();
+    for (int c = 0; c < input_channel; c++) {
+      vnewbias = vdupq_n_f32(newbias_data[c]);
+      vnewscale = vdupq_n_f32(newscale_data[c]);
+
+      float w00 = filter_data[0];
+      float w01 = filter_data[1];
+      float w02 = filter_data[2];
+      float w10 = filter_data[3];
+      float w11 = filter_data[4];
+      float w12 = filter_data[5];
+      float w20 = filter_data[6];
+      float w21 = filter_data[7];
+      float w22 = filter_data[8];
+
+      output_data[0] = w11 * input_data[0] + w12 * input_data[1] +
+                       w21 * input_data[l] + w22 * input_data[l + 1];
+      output_data[l - 1] = w10 * input_data[l - 2] + w11 * input_data[l - 1] +
+                           w20 * input_data[2 * l - 2] +
+                           w21 * input_data[2 * l - 1];
+      output_data[(l - 1) * l] =
+          w01 * input_data[(l - 2) * l] + w02 * input_data[(l - 2) * l + 1] +
+          w11 * input_data[(l - 1) * l] + w12 * input_data[(l - 1) * l + 1];
+      output_data[l * l - 1] = w00 * input_data[(l - 2) * (l + 1)] +
+                               w01 * input_data[(l - 2) * (l + 1) + 1] +
+                               w10 * input_data[l * l - 2] +
+                               w11 * input_data[l * l - 1];
+      output_data[0] = output_data[0] * newscale_data[c] + newbias_data[c];
+      output_data[l - 1] =
+          output_data[l - 1] * newscale_data[c] + newbias_data[c];
+      output_data[(l - 1) * l] =
+          output_data[(l - 1) * l] * newscale_data[c] + newbias_data[c];
+      output_data[l * l - 1] =
+          output_data[l * l - 1] * newscale_data[c] + newbias_data[c];
+
+      if (if_relu) {
+        output_data[0] = output_data[0] < 0 ? 0 : output_data[0];
+        output_data[l - 1] = output_data[l - 1] < 0 ? 0 : output_data[l - 1];
+        output_data[(l - 1) * l] =
+            output_data[(l - 1) * l] < 0 ? 0 : output_data[(l - 1) * l];
+        output_data[l * l - 1] =
+            output_data[l * l - 1] < 0 ? 0 : output_data[l * l - 1];
+      }
+      for (int i = 1; i < l - 1; ++i) {
+        output_data[i * l] =
+            w01 * input_data[i * l - l] + w02 * input_data[i * l - l + 1] +
+            w11 * input_data[i * l] + w12 * input_data[i * l + 1] +
+            w21 * input_data[i * l + l] + w22 * input_data[i * l + l + 1];
+
+        output_data[i * l + l - 1] = w00 * input_data[i * l + l - 1 - l - 1] +
+                                     w01 * input_data[i * l + l - 1 - l] +
+                                     w10 * input_data[i * l + l - 1 - 1] +
+                                     w11 * input_data[i * l + l - 1] +
+                                     w20 * input_data[i * l + l - 1 + l - 1] +
+                                     w21 * input_data[i * l + l - 1 + l];
+        output_data[i * l] =
+            output_data[i * l] * newscale_data[c] + newbias_data[c];
+        output_data[i * l + l - 1] =
+            output_data[i * l + l - 1] * newscale_data[c] + newbias_data[c];
+
+        if (if_relu) {
+          output_data[i * l] = output_data[i * l] < 0 ? 0 : output_data[i * l];
+          output_data[i * l + l - 1] =
+              output_data[i * l + l - 1] < 0 ? 0 : output_data[i * l + l - 1];
+        }
+      }
+
+      int m;
+
+      for (m = 1; m < output_width - 4; m += 4) {
+        float *output_ptr = output_data + m;
+        float32x4_t in0, in1, in2, in3, tmp0, tmp1, tmp2, tmp3, out0;
+        in0 = vld1q_f32(input_data + m - 1);
+        in1 = vld1q_f32(input_data + m + 3);
+        in2 = vld1q_f32(input_data + input_width + m - 1);
+        in3 = vld1q_f32(input_data + input_width + m + 3);
+        tmp0 = vextq_f32(in0, in1, 1);
+        tmp1 = vextq_f32(in0, in1, 2);
+        tmp2 = vextq_f32(in2, in3, 1);
+        tmp3 = vextq_f32(in2, in3, 2);
+        out0 = vmulq_n_f32(in0, w10);
+        out0 = vmlaq_n_f32(out0, tmp0, w11);
+        out0 = vmlaq_n_f32(out0, tmp1, w12);
+        out0 = vmlaq_n_f32(out0, in2, w20);
+        out0 = vmlaq_n_f32(out0, tmp2, w21);
+        out0 = vmlaq_n_f32(out0, tmp3, w22);
+        out0 = vmlaq_f32(vnewbias, vnewscale, out0);
+        if (if_relu) {
+          out0 = vmaxq_f32(out0, vzero);
+        }
+        vst1q_f32(output_ptr, out0);
+      }
+      for (m = 1; (m + 3) < output_width - 1; m = m + 4) {
+      }
+      for (int j = m; j < output_width - 1; j++) {
+        output_data[j] = input_data[j - 1] * w10 + input_data[j] * w11 +
+                         input_data[j + 1] * w12 +
+                         input_data[input_width + j - 1] * w20 +
+                         input_data[input_width + j] * w21 +
+                         input_data[input_width + j + 1] * w22;
+        output_data[j] = output_data[j] * newscale_data[c] + newbias_data[c];
+
+        if (if_relu) {
+          output_data[j] = output_data[j] < 0 ? 0 : output_data[j];
+        }
+      }
+
+      for (m = 1; (m + 3) < output_width - 1; m = m + 4) {
+        float *output_ptr =
+            output_data + (output_height - 1) * output_width + m;
+
+        float32x4_t in0, in1, in2, in3, tmp0, tmp1, tmp2, tmp3, out0;
+        in0 = vld1q_f32(input_data + (output_height - 2) * input_width + m - 1);
+        in1 = vld1q_f32(input_data + (output_height - 2) * input_width + m + 3);
+        in2 = vld1q_f32(input_data + (output_height - 1) * input_width + m - 1);
+        in3 = vld1q_f32(input_data + (output_height - 1) * input_width + m + 3);
+        tmp0 = vextq_f32(in0, in1, 1);
+        tmp1 = vextq_f32(in0, in1, 2);
+        tmp2 = vextq_f32(in2, in3, 1);
+        tmp3 = vextq_f32(in2, in3, 2);
+        out0 = vmulq_n_f32(in0, w00);
+        out0 = vmlaq_n_f32(out0, tmp0, w01);
+        out0 = vmlaq_n_f32(out0, tmp1, w02);
+        out0 = vmlaq_n_f32(out0, in2, w10);
+        out0 = vmlaq_n_f32(out0, tmp2, w11);
+        out0 = vmlaq_n_f32(out0, tmp3, w12);
+        out0 = vmlaq_f32(vnewbias, vnewscale, out0);
+        if (if_relu) {
+          out0 = vmaxq_f32(out0, vzero);
+        }
+        vst1q_f32(output_ptr, out0);
+      }
+      for (m = 1; (m + 3) < output_width - 1; m = m + 4) {
+      }
+      for (int j = m; j < output_width - 1; j++) {
+        output_data[(output_height - 1) * input_width + j] =
+            input_data[(output_height - 2) * input_width + j - 1] * w00 +
+            input_data[(output_height - 2) * input_width + j] * w01 +
+            input_data[(output_height - 2) * input_width + j + 1] * w02 +
+            input_data[(output_height - 1) * input_width + j - 1] * w10 +
+            input_data[(output_height - 1) * input_width + j] * w11 +
+            input_data[(output_height - 1) * input_width + j + 1] * w12;
+        output_data[(output_height - 1) * output_width + j] =
+            output_data[(output_height - 1) * output_width + j] *
+                newscale_data[c] +
+            newbias_data[c];
+
+        if (if_relu) {
+          output_data[(output_height - 1) * output_width + j] =
+              output_data[(output_height - 1) * output_width + j] < 0
+                  ? 0
+                  : output_data[(output_height - 1) * output_width + j];
+        }
+      }
+      #pragma omp parallel for
+      for (int i = 1; i < output_height - 1; i++) {
+        for (int m = 1; (m + 3) < output_width - 1; m = m + 4) {
+          float *output_ptr = output_data + i * output_width + m;
+          float32x4_t in0, in1, in2, in3, in4, in5, tmp0, tmp1, tmp2, tmp3,
+              tmp4, tmp5, out0;
+          in0 = vld1q_f32(input_data + (i - 1) * input_width + m - 1);
+          in1 = vld1q_f32(input_data + (i - 1) * input_width + m + 3);
+          in2 = vld1q_f32(input_data + i * input_width + m - 1);
+          in3 = vld1q_f32(input_data + i * input_width + m + 3);
+          in4 = vld1q_f32(input_data + (i + 1) * input_width + m - 1);
+          in5 = vld1q_f32(input_data + (i + 1) * input_width + m + 3);
+
+          tmp0 = vextq_f32(in0, in1, 1);
+          tmp1 = vextq_f32(in0, in1, 2);
+          tmp2 = vextq_f32(in2, in3, 1);
+          tmp3 = vextq_f32(in2, in3, 2);
+          tmp4 = vextq_f32(in4, in5, 1);
+          tmp5 = vextq_f32(in4, in5, 2);
+
+          out0 = vmulq_n_f32(in0, w00);
+          out0 = vmlaq_n_f32(out0, tmp0, w01);
+          out0 = vmlaq_n_f32(out0, tmp1, w02);
+          out0 = vmlaq_n_f32(out0, in2, w10);
+          out0 = vmlaq_n_f32(out0, tmp2, w11);
+          out0 = vmlaq_n_f32(out0, tmp3, w12);
+          out0 = vmlaq_n_f32(out0, in4, w20);
+          out0 = vmlaq_n_f32(out0, tmp4, w21);
+          out0 = vmlaq_n_f32(out0, tmp5, w22);
+
+          out0 = vmlaq_f32(vnewbias, vnewscale, out0);
+          if (if_relu) {
+            out0 = vmaxq_f32(out0, vzero);
+          }
+          vst1q_f32(output_ptr, out0);
+        }
+        int m;
+        for (m = 1; (m + 3) < output_width - 1; m = m + 4) {
+        }
+
+        for (int j = m; j < output_width - 1; j++) {
+          output_data[i * output_width + j] =
+              input_data[(i - 1) * input_width + j - 1] * w00 +
+              input_data[(i - 1) * input_width + j] * w01 +
+              input_data[(i - 1) * input_width + j + 1] * w02 +
+              input_data[(i)*input_width + j - 1] * w10 +
+              input_data[(i)*input_width + j] * w11 +
+              input_data[(i)*input_width + j + 1] * w12 +
+              input_data[(i + 1) * input_width + j - 1] * w20 +
+              input_data[(i + 1) * input_width + j] * w21 +
+              input_data[(i + 1) * input_width + j + 1] * w22;
+          output_data[i * output_width + j] =
+              newscale_data[c] * output_data[i * output_width + j] +
+              newbias_data[c];
+          if (if_relu) {
+            output_data[i * output_width + j] =
+                output_data[i * output_width + j] < 0
+                    ? 0
+                    : output_data[i * output_width + j];
+          }
+        }
+      }
+
+      input_data = input_data + hxw;
+      output_data = output_data + hxw;
+      filter_data = filter_data + 9;
+    }
+  }
+
+    /*
+    const float *input_data = input->data<float>();
+    const float *filter_data = filter->data<float>();
+    float *output_data = output->data<float>();
+    const float *newscale_data = new_scale->data<float>();
+    const float *newbias_data = new_bias->data<float>();
+
     const int h = static_cast<int>(input->dims()[2]);
     const int w = static_cast<int>(input->dims()[3]);
     const int l = h;
@@ -605,8 +851,8 @@ void DepthwiseConvAddBNRelu3x3s1p1(const Tensor *input, const Tensor *filter,
               output_data[i * l + l - 1] * newscale_data[j] + newbias_data[j];
 
           if (if_relu) {
-          output_data[i * l] = output_data[i * l] < 0 ? 0 : output_data[i * l];
-          output_data[i * l + l - 1] =
+            output_data[i * l] = output_data[i * l] < 0 ? 0 : output_data[i *
+            l]; output_data[i * l + l - 1] =
                 output_data[i * l + l - 1] < 0 ? 0 : output_data[i * l + l - 1];
           }
         }
@@ -738,6 +984,7 @@ void DepthwiseConvAddBNRelu3x3s1p1(const Tensor *input, const Tensor *filter,
         }
         // mid
 
+
         for (int i = 0; i < l - 2; ++i) {
           auto output_ptr = output_data + (i + 1) * l + 1;
           input_tmp = input_data + i * l;
@@ -820,6 +1067,7 @@ void DepthwiseConvAddBNRelu3x3s1p1(const Tensor *input, const Tensor *filter,
         filter_data_tmp += 9;
       }
     }
+  */
 #endif
 }
 

src/operators/math/gemm.h

@@ -50,6 +50,10 @@ void PackMatrixA_6r(int m, int k, int m_tail, const float *A, int lda,
                     float *buffer);
 void PackMatrixA_8r(int m, int k, int m_tail, const float *A, int lda,
                     float *buffer);
+void PackMatrixA_omp_6r(int m, int k, int m_tail, const float *A, int lda,
+                        float *buffer);
+void PackMatrixA_omp_8r(int m, int k, int m_tail, const float *A, int lda,
+                        float *buffer);
 
 // 将 B 矩阵分块复制到连续内存(RowMajor)
 void PackMatrixB_8c(int k, int n, int n_tail, const float *B, int ldb,
@@ -58,6 +62,12 @@ void PackMatrixB_12c(int k, int n, int n_tail, const float *B, int ldb,
                      float *buffer);
 void PackMatrixB_16c(int k, int n, int n_tail, const float *B, int ldb,
                      float *buffer);
+void PackMatrixB_omp_8c(int k, int n, int n_tail, const float *B, int ldb,
+                        float *buffer);
+void PackMatrixB_omp_12c(int k, int n, int n_tail, const float *B, int ldb,
+                         float *buffer);
+void PackMatrixB_omp_16c(int k, int n, int n_tail, const float *B, int ldb,
+                         float *buffer);
 
 // 分块矩阵乘法
 void InnerKernel(int mc, int nc, float alpha, const float *a, const float *b,
@@ -136,6 +146,16 @@ void SgemmWithBn(int m, int n, int k, float alpha, const float *A, int lda,
                  const float *B, int ldb, float beta, float *C, int ldc,
                  bool relu, float *new_scale, float *new_bias);
 
+// 32位 float 矩阵乘法（openmp 多线程版本）
+void Sgemm_omp(int m, int n, int k, float alpha, const float *A, int lda,
+               const float *B, int ldb, float beta, float *C, int ldc,
+               bool relu, float *bias);
+
+// 32位 float 矩阵乘法, 并对结果进行 batchnrom（openmp 多线程版本）
+void SgemmWithBn_omp(int m, int n, int k, float alpha, const float *A, int lda,
+                     const float *B, int ldb, float beta, float *C, int ldc,
+                     bool relu, float *new_scale, float *new_bias);
+
 }  // namespace math
 }  // namespace operators
 }  // namespace paddle_mobile

src/operators/math/math_function.cpp

@@ -42,8 +42,13 @@ void matmul<float>(const framework::Tensor &matrix_a, bool trans_a,
   int N = dim_out[1];
   int K = (!trans_a) ? dim_a[1] : dim_a[0];
 
+#ifdef _OPENMP
+  Sgemm_omp(M, N, K, alpha, matrix_a.data<float>(), K, matrix_b.data<float>(),
+            N, beta, matrix_out->data<float>(), N, relu, bias);
+#else
   Sgemm(M, N, K, alpha, matrix_a.data<float>(), K, matrix_b.data<float>(), N,
         beta, matrix_out->data<float>(), N, relu, bias);
+#endif
 }
 
 template <>
@@ -70,10 +75,17 @@ void matmulWithBn<float>(const framework::Tensor &matrix_a, bool trans_a,
   int N = dim_out[1];
   int K = (!trans_a) ? dim_a[1] : dim_a[0];
 
+#ifdef _OPENMP
+  SgemmWithBn_omp(M, N, K, alpha, matrix_a.data<float>(), K,
+                  matrix_b.data<float>(), N, beta, matrix_out->data<float>(), N,
+                  relu, new_scale->data<float>() + group,
+                  new_bias->data<float>() + group);
+#else
   SgemmWithBn(M, N, K, alpha, matrix_a.data<float>(), K, matrix_b.data<float>(),
               N, beta, matrix_out->data<float>(), N, relu,
               new_scale->data<float>() + group,
               new_bias->data<float>() + group);
+#endif
 }
 
 }  // namespace math