diff --git a/CMakeLists.txt b/CMakeLists.txt index a00d179a0d4972080c8fd392160f8ec451692e4d..097986546601ddf2f7f25e14c10ef4dc104c9e3a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,7 +9,6 @@ option(LOG_PROFILE "log profile" ON) option(CPU "armv7 with neon" ON) option(MALI_GPU "mali gpu" OFF) option(FPGA "fpga" OFF) -set(DEBUGING ON) if (ARM_LINUX) include("${CMAKE_CURRENT_LIST_DIR}/tools/arm-platform.cmake") @@ -134,6 +133,9 @@ else () endif () if(DEBUGING) - add_subdirectory(test) + if(IS_IOS) + else() + add_subdirectory(test) + endif() endif() diff --git a/demo/android/PaddleMobile_Android/app/src/main/java/com/baidu/paddle/MainActivity.java b/demo/android/PaddleMobile_Android/app/src/main/java/com/baidu/paddle/MainActivity.java index febe816681d3845a61c5a8b40630e82ac9b4ea95..6a6665dd334d1c7a47fea04ef708b84498f0e357 100755 --- a/demo/android/PaddleMobile_Android/app/src/main/java/com/baidu/paddle/MainActivity.java +++ b/demo/android/PaddleMobile_Android/app/src/main/java/com/baidu/paddle/MainActivity.java @@ -121,7 +121,14 @@ public class MainActivity extends Activity { String assetPath = "pml_demo"; String sdcardPath = Environment.getExternalStorageDirectory() + File.separator + assetPath + File.separator + type; - PML.load(sdcardPath); + //PML.load(sdcardPath); + String modelPath = Environment.getExternalStorageDirectory() + + File.separator + assetPath + + File.separator + "googlenet_combine" + File.separator + "model"; + String paramPath = Environment.getExternalStorageDirectory() + + File.separator + assetPath + + File.separator + "googlenet_combine" + File.separator + "params"; + PML.loadCombined(modelPath, paramPath); } }); diff --git a/demo/android/PaddleMobile_Android/app/src/main/java/com/baidu/paddle/PML.java b/demo/android/PaddleMobile_Android/app/src/main/java/com/baidu/paddle/PML.java index 7649d4c081223bace01b806d1eb7dca57129ed7c..e67f04e47a77b28bfd8ce98866b1539797c217cd 100644 --- a/demo/android/PaddleMobile_Android/app/src/main/java/com/baidu/paddle/PML.java +++ b/demo/android/PaddleMobile_Android/app/src/main/java/com/baidu/paddle/PML.java @@ -8,6 +8,14 @@ public class PML { */ public static native boolean load(String modelPath); + /** + * Load + * @param modelPath + * @param paramPath + * @return + */ + public static native boolean loadCombined(String modelPath,String paramPath); + /** * object detection diff --git a/demo/ios/PaddleMobileDemo/PaddleMobileDemo.xcodeproj/project.pbxproj b/demo/ios/PaddleMobileDemo/PaddleMobileDemo.xcodeproj/project.pbxproj index 8500c89c9af5ab2d56e08b576dc007a424262d15..469c228e82841a8ceb79842bd9f021566b9fbf6f 100644 --- a/demo/ios/PaddleMobileDemo/PaddleMobileDemo.xcodeproj/project.pbxproj +++ b/demo/ios/PaddleMobileDemo/PaddleMobileDemo.xcodeproj/project.pbxproj @@ -15,9 +15,9 @@ FC12E94120EB6B2900807EF4 /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = FC12E94020EB6B2900807EF4 /* main.m */; }; FC12E94A20EB6B6800807EF4 /* libpaddle-mobile.a in Frameworks */ = {isa = PBXBuildFile; fileRef = FC12E94820EB6B6800807EF4 /* libpaddle-mobile.a */; }; FC12E94D20EB6BBB00807EF4 /* libstdc++.tbd in Frameworks */ = {isa = PBXBuildFile; fileRef = FC12E94C20EB6BBB00807EF4 /* libstdc++.tbd */; }; - FC12E95120EB6BED00807EF4 /* params in Resources */ = {isa = PBXBuildFile; fileRef = FC12E94F20EB6BED00807EF4 /* params */; }; - FC12E95220EB6BED00807EF4 /* model in Resources */ = {isa = PBXBuildFile; fileRef = FC12E95020EB6BED00807EF4 /* model */; }; FC12E95420EB6C0D00807EF4 /* apple.jpg in Resources */ = {isa = PBXBuildFile; fileRef = FC12E95320EB6C0D00807EF4 /* apple.jpg */; }; + FC51640120EF758D00636C28 /* params in Resources */ = {isa = PBXBuildFile; fileRef = FC5163FF20EF758D00636C28 /* params */; }; + FC51640220EF758D00636C28 /* model in Resources */ = {isa = PBXBuildFile; fileRef = FC51640020EF758D00636C28 /* model */; }; /* End PBXBuildFile section */ /* Begin PBXFileReference section */ @@ -34,9 +34,9 @@ FC12E94820EB6B6800807EF4 /* libpaddle-mobile.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; path = "libpaddle-mobile.a"; sourceTree = ""; }; FC12E94920EB6B6800807EF4 /* PaddleMobile.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = PaddleMobile.h; sourceTree = ""; }; FC12E94C20EB6BBB00807EF4 /* libstdc++.tbd */ = {isa = PBXFileReference; lastKnownFileType = "sourcecode.text-based-dylib-definition"; name = "libstdc++.tbd"; path = "usr/lib/libstdc++.tbd"; sourceTree = SDKROOT; }; - FC12E94F20EB6BED00807EF4 /* params */ = {isa = PBXFileReference; lastKnownFileType = file; path = params; sourceTree = ""; }; - FC12E95020EB6BED00807EF4 /* model */ = {isa = PBXFileReference; lastKnownFileType = file; path = model; sourceTree = ""; }; FC12E95320EB6C0D00807EF4 /* apple.jpg */ = {isa = PBXFileReference; lastKnownFileType = image.jpeg; path = apple.jpg; sourceTree = ""; }; + FC5163FF20EF758D00636C28 /* params */ = {isa = PBXFileReference; lastKnownFileType = file; path = params; sourceTree = ""; }; + FC51640020EF758D00636C28 /* model */ = {isa = PBXFileReference; lastKnownFileType = file; path = model; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -72,9 +72,9 @@ FC12E93020EB6B2800807EF4 /* PaddleMobileDemo */ = { isa = PBXGroup; children = ( - FC12E95320EB6C0D00807EF4 /* apple.jpg */, - FC12E94E20EB6BED00807EF4 /* googlenet_combine */, FC12E94720EB6B6800807EF4 /* PaddleMobile */, + FC5163FE20EF758D00636C28 /* googlenet_combine */, + FC12E95320EB6C0D00807EF4 /* apple.jpg */, FC12E93120EB6B2800807EF4 /* AppDelegate.h */, FC12E93220EB6B2800807EF4 /* AppDelegate.m */, FC12E93420EB6B2800807EF4 /* ViewController.h */, @@ -105,11 +105,11 @@ name = Frameworks; sourceTree = ""; }; - FC12E94E20EB6BED00807EF4 /* googlenet_combine */ = { + FC5163FE20EF758D00636C28 /* googlenet_combine */ = { isa = PBXGroup; children = ( - FC12E94F20EB6BED00807EF4 /* params */, - FC12E95020EB6BED00807EF4 /* model */, + FC5163FF20EF758D00636C28 /* params */, + FC51640020EF758D00636C28 /* model */, ); path = googlenet_combine; sourceTree = ""; @@ -171,10 +171,10 @@ isa = PBXResourcesBuildPhase; buildActionMask = 2147483647; files = ( + FC51640220EF758D00636C28 /* model in Resources */, + FC51640120EF758D00636C28 /* params in Resources */, FC12E93E20EB6B2900807EF4 /* LaunchScreen.storyboard in Resources */, - FC12E95220EB6BED00807EF4 /* model in Resources */, FC12E93B20EB6B2900807EF4 /* Assets.xcassets in Resources */, - FC12E95120EB6BED00807EF4 /* params in Resources */, FC12E95420EB6C0D00807EF4 /* apple.jpg in Resources */, FC12E93920EB6B2800807EF4 /* Main.storyboard in Resources */, ); diff --git a/demo/ios/PaddleMobileDemo/PaddleMobileDemo.xcodeproj/project.xcworkspace/xcuserdata/liuruilong.xcuserdatad/UserInterfaceState.xcuserstate b/demo/ios/PaddleMobileDemo/PaddleMobileDemo.xcodeproj/project.xcworkspace/xcuserdata/liuruilong.xcuserdatad/UserInterfaceState.xcuserstate index 395136a63bb50378df8c37256880d4bbf9fd2f83..a5b221642f9973f6abd0b18cc017a58d9a61e434 100644 Binary files a/demo/ios/PaddleMobileDemo/PaddleMobileDemo.xcodeproj/project.xcworkspace/xcuserdata/liuruilong.xcuserdatad/UserInterfaceState.xcuserstate and b/demo/ios/PaddleMobileDemo/PaddleMobileDemo.xcodeproj/project.xcworkspace/xcuserdata/liuruilong.xcuserdatad/UserInterfaceState.xcuserstate differ diff --git a/demo/ios/PaddleMobileDemo/PaddleMobileDemo/PaddleMobile/PaddleMobile.h b/demo/ios/PaddleMobileDemo/PaddleMobileDemo/PaddleMobile/PaddleMobile.h index ec58371de032c265b2c32a5bac61ca6cf682ff28..5854c5c3a4d4c899feb88822b2f7993860d1ed76 100644 --- a/demo/ios/PaddleMobileDemo/PaddleMobileDemo/PaddleMobile/PaddleMobile.h +++ b/demo/ios/PaddleMobileDemo/PaddleMobileDemo/PaddleMobile/PaddleMobile.h @@ -1,26 +1,52 @@ /* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. - + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ +#pragma once + #import #import @interface PaddleMobile : NSObject +/* + 创建对象 +*/ - (instancetype)init; + +/* + load 模型, 开辟内存 +*/ - (BOOL)load:(NSString *)modelPath andWeightsPath:(NSString *)weighsPath; + +/* + 加载散开形式的模型, 需传入模型的目录 +*/ +- (BOOL)load:(NSString *)modelAndWeightPath; + +/* + 进行预测, means 和 scale 为训练模型时的预处理参数, 如训练时没有做这些预处理则直接使用 predict +*/ - (NSArray *)predict:(CGImageRef)image dim:(NSArray *)dim means:(NSArray *)means scale:(float)scale; + +/* + 进行预测 +*/ - (NSArray *)predict:(CGImageRef)image dim:(NSArray *)dim; + +/* + 清理内存 +*/ - (void)clear; @end diff --git a/doc/development_doc.md b/doc/development_doc.md index 90f07a65ffe00bd20a8277dce97286d92875670b..18a1974576d8e488fe7f4e70cad18ae088b794a9 100644 --- a/doc/development_doc.md +++ b/doc/development_doc.md @@ -4,9 +4,11 @@ ## 编译 -### 一. 使用 build.sh 编译 - ```sh + +# 在 paddle-mobile 目录下: +cd tools + sh build.sh ios # 如果只想编译某个特定模型的 op, 则需执行以下命令 @@ -17,9 +19,7 @@ cd ../build/release/ios/build ``` -### 二. 集成 - -#### 如使用 oc 接口 +## 集成 ``` 将上一步生成的: @@ -28,7 +28,11 @@ libpaddle-mobile.a /src/ios_io/ 下的 PaddleMobile.h ``` -拖入工程, 接口如下: +拖入工程 + +#### oc 接口 + +接口如下: ``` /* diff --git a/src/ios_io/PaddleMobile.h b/src/ios_io/PaddleMobile.h index 863c0d004440dc6098eb7dc1ed490fde20f237c9..5854c5c3a4d4c899feb88822b2f7993860d1ed76 100644 --- a/src/ios_io/PaddleMobile.h +++ b/src/ios_io/PaddleMobile.h @@ -19,10 +19,34 @@ @interface PaddleMobile : NSObject +/* + 创建对象 +*/ - (instancetype)init; + +/* + load 模型, 开辟内存 +*/ - (BOOL)load:(NSString *)modelPath andWeightsPath:(NSString *)weighsPath; + +/* + 加载散开形式的模型, 需传入模型的目录 +*/ +- (BOOL)load:(NSString *)modelAndWeightPath; + +/* + 进行预测, means 和 scale 为训练模型时的预处理参数, 如训练时没有做这些预处理则直接使用 predict +*/ - (NSArray *)predict:(CGImageRef)image dim:(NSArray *)dim means:(NSArray *)means scale:(float)scale; + +/* + 进行预测 +*/ - (NSArray *)predict:(CGImageRef)image dim:(NSArray *)dim; + +/* + 清理内存 +*/ - (void)clear; @end diff --git a/src/ios_io/PaddleMobile.mm b/src/ios_io/PaddleMobile.mm index f5ec2afb2a996ec4932d99ea93362e06ddf28a14..e3ed909394a1057302fb0f747b582b944c89cc65 100644 --- a/src/ios_io/PaddleMobile.mm +++ b/src/ios_io/PaddleMobile.mm @@ -62,6 +62,15 @@ static std::mutex shared_mutex; } } +- (BOOL)load:(NSString *)modelAndWeightPath{ + std::string model_path_str = std::string([modelAndWeightPath UTF8String]); + if (loaded_ = pam_->Load(model_path_str)) { + return YES; + } else { + return NO; + } +} + -(void)preprocess:(const UInt8 *)input output:(float *)output imageWidth:(int)imageWidth imageHeight:(int)imageHeight imageChannels:(int)imageChannels means:(NSArray *)means scale:(float)scale dim:(std::vector)dim{ if (means == nil) { means = @[@0, @0, @0]; diff --git a/src/jni/paddle_mobile_jni.cpp b/src/jni/paddle_mobile_jni.cpp index 01d4e52a4b1308a7ff97bc672d1a15d329dbf318..b14f095c1d82f167c1e3f15897b907e730a4a5a8 100644 --- a/src/jni/paddle_mobile_jni.cpp +++ b/src/jni/paddle_mobile_jni.cpp @@ -60,6 +60,15 @@ JNIEXPORT jboolean JNICALL Java_com_baidu_paddle_PML_load(JNIEnv *env, optimize); } +JNIEXPORT jboolean JNICALL Java_com_baidu_paddle_PML_loadCombined( + JNIEnv *env, jclass thiz, jstring modelPath, jstring paramPath) { + ANDROIDLOGI("load invoked"); + bool optimize = true; + return getPaddleMobileInstance()->Load(jstring2cppstring(env, modelPath), + jstring2cppstring(env, paramPath), + optimize); +} + JNIEXPORT jfloatArray JNICALL Java_com_baidu_paddle_PML_predict(JNIEnv *env, jclass thiz, jfloatArray buf) { jfloatArray result = NULL; diff --git a/src/jni/paddle_mobile_jni.h b/src/jni/paddle_mobile_jni.h index 86caa9a273ab11124f6ea67efe27dc3529cea69f..ab88816dcb7ec6ba88f12cb270812c4af0923b32 100644 --- a/src/jni/paddle_mobile_jni.h +++ b/src/jni/paddle_mobile_jni.h @@ -22,11 +22,16 @@ extern "C" { namespace paddle_mobile { namespace jni { /** - * load model & params of the net for android + * load separated model for android */ JNIEXPORT jboolean JNICALL Java_com_baidu_paddle_PML_load(JNIEnv *env, jclass thiz, jstring modelPath); +/** + * load combined model for android + */ +JNIEXPORT jboolean JNICALL Java_com_baidu_paddle_PML_loadCombined( + JNIEnv *env, jclass thiz, jstring modelPath, jstring paramPath); /** * object detection for anroid diff --git a/src/operators/fusion_conv_add.cpp b/src/operators/fusion_conv_add.cpp index 656d30c4e1921914d3fe80d930f4219d73f025ea..b1dba23be0d8ea010b38844b1897381fbf578617 100644 --- a/src/operators/fusion_conv_add.cpp +++ b/src/operators/fusion_conv_add.cpp @@ -45,28 +45,6 @@ void FusionConvAddOp::InferShape() const { this->param_.Output()->Resize(ddim); } -#ifdef PADDLE_MOBILE_CPU - -#ifndef CONV_ADD_REGISTER -framework::FusionOpRegistrar convadd_registrar(new FusionConvAddMatcher()); -#define CONV_ADD_REGISTER -#endif - -#endif - -#ifdef PADDLE_MOBILE_MALI_GPU - -#ifndef CONV_ADD_REGISTER -static framework::FusionOpRegistrar convadd_registrar( - new FusionConvAddMatcher()); -#define CONV_ADD_REGISTER -#endif - -#endif - -#ifdef PADDLE_MOBILE_FPGA -#endif - template class FusionConvAddOp; } // namespace operators } // namespace paddle_mobile diff --git a/src/operators/fusion_conv_add.h b/src/operators/fusion_conv_add.h index bc623efc8c9288a6007a8141220a2d40b29d7bdb..ae030ba5767e4039cfa3effe0a7ded4886f261cf 100644 --- a/src/operators/fusion_conv_add.h +++ b/src/operators/fusion_conv_add.h @@ -69,7 +69,7 @@ class FusionConvAddOp : public framework::OperatorWithKernel< #ifdef PADDLE_MOBILE_CPU #ifndef CONV_ADD_REGISTER -extern framework::FusionOpRegistrar convadd_registrar( +static framework::FusionOpRegistrar convadd_registrar( new FusionConvAddMatcher()); #define CONV_ADD_REGISTER #endif @@ -82,6 +82,7 @@ extern framework::FusionOpRegistrar convadd_registrar( static framework::FusionOpRegistrar convadd_registrar( new FusionConvAddMatcher()); #define CONV_ADD_REGISTER + #endif #endif diff --git a/src/operators/fusion_fc_op.cpp b/src/operators/fusion_fc_op.cpp index 0ca3c26c47e91771ab7019261f78815ad8463408..57a8b1b53f2f98b3218ee8fc40c6c9774ec5a5c7 100644 --- a/src/operators/fusion_fc_op.cpp +++ b/src/operators/fusion_fc_op.cpp @@ -50,27 +50,6 @@ void FusionFcOp::InferShape() const { this->param_.Out()->Resize(ddim); } -#ifdef PADDLE_MOBILE_CPU - -#ifndef CONV_CPU_REGISTER -#define CONV_CPU_REGISTER -framework::FusionOpRegistrar fc_registrar(new FusionFcMatcher()); -#endif - -#endif - -#ifdef PADDLE_MOBILE_MALI_GPU - -#ifndef CONV_CPU_REGISTER -#define CONV_CPU_REGISTER -framework::FusionOpRegistrar fc_registrar(new FusionFcMatcher()); -#endif - -#endif - -#ifdef PADDLE_MOBILE_FPGA -#endif - template class FusionFcOp; } // namespace operators } // namespace paddle_mobile diff --git a/src/operators/fusion_fc_op.h b/src/operators/fusion_fc_op.h index b545bb8380aedcc31200080beaec4c56f137ad13..c6f13c8e347f525ecee915759a5f4df492a73a92 100644 --- a/src/operators/fusion_fc_op.h +++ b/src/operators/fusion_fc_op.h @@ -69,7 +69,7 @@ class FusionFcOp #ifndef CONV_CPU_REGISTER #define CONV_CPU_REGISTER -extern framework::FusionOpRegistrar fc_registrar(new FusionFcMatcher()); +static framework::FusionOpRegistrar fc_registrar(new FusionFcMatcher()); #endif #endif @@ -78,7 +78,7 @@ extern framework::FusionOpRegistrar fc_registrar(new FusionFcMatcher()); #ifndef CONV_CPU_REGISTER #define CONV_CPU_REGISTER -extern framework::FusionOpRegistrar fc_registrar(new FusionFcMatcher()); +static framework::FusionOpRegistrar fc_registrar(new FusionFcMatcher()); #endif #endif diff --git a/src/operators/kernel/arm/prelu_kernel.cpp b/src/operators/kernel/arm/prelu_kernel.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a83783a078f4ec680fbab238a2839226546f894c --- /dev/null +++ b/src/operators/kernel/arm/prelu_kernel.cpp @@ -0,0 +1,113 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#ifdef PRELU_OP + +#include "operators/kernel/prelu_kernel.h" +#include + +namespace paddle_mobile { +namespace operators { + +template +struct PReluFunctor { + explicit PReluFunctor(float slope) { this->slope_ = slope; } + inline T operator()(T in) const { return in > 0 ? in : in * slope_; } + + float slope_ = 0.0f; +}; + +/* + * @b 特化到具体平台的实现, param 从 op 层传入 + * */ +template <> +void PReluKernel::Compute(const PReluParam ¶m) const { + const auto *input_x = param.InputX(); + auto *input_x_ptr = input_x->data(); + auto *out = param.Out(); + auto *out_ptr = out->mutable_data(); + + if (param.Slopes().size() == 1) { + PReluFunctor func_(param.Slopes()[0]); + math::Transform trans; + trans(input_x_ptr, input_x_ptr + input_x->numel(), out_ptr, func_); + } else if (param.Slopes().size() > 1) { + const int dim_size = input_x->dims().size(); + switch (dim_size) { + case 0: + break; + case 1: { + const int input_width = input_x->dims()[0]; + math::Transform trans; + + #pragma omp parallel for + for (int w = 0; w < input_width; ++w) { + out_ptr[w] = input_x_ptr[w] * param.Slopes()[w]; + } + } break; + case 2: { + const int input_height = input_x->dims()[0]; + const int input_width = input_x->dims()[1]; + + math::Transform trans; + #pragma omp parallel for + for (int h = 0; h < input_height; ++h) { + PReluFunctor func_(param.Slopes()[h]); + const float *ptr = input_x_ptr + h * input_width; + float *optr = out_ptr + +h * input_width; + trans(ptr, ptr + input_width, optr, func_); + } + } break; + case 3: { + const int chan_size = input_x->dims()[0]; + const int input_height = input_x->dims()[1]; + const int input_width = input_x->dims()[2]; + + math::Transform trans; + #pragma omp parallel for + for (int c = 0; c < chan_size; ++c) { + PReluFunctor func_(param.Slopes()[c]); + int size = input_height * input_width; + const float *ptr = input_x_ptr + c * size; + float *optr = out_ptr + c * size; + trans(ptr, ptr + size, optr, func_); + } + } break; + case 4: + default: { + const int batch_size = input_x->dims()[0]; + const int chan_size = input_x->dims()[1]; + const int input_height = input_x->dims()[2]; + const int input_width = input_x->dims()[3]; + math::Transform trans; + + #pragma omp parallel for + for (int b = 0; b < batch_size; ++b) { + for (int c = 0; c < chan_size; ++c) { + PReluFunctor func_(param.Slopes()[c]); + int size = input_height * input_width; + const float *ptr = input_x_ptr + b * c * size; + float *optr = out_ptr + +b * c * size; + trans(ptr, ptr + size, optr, func_); + } + } + } // case 3,default + break; + } + } +} +} // namespace operators +} // namespace paddle_mobile + +#endif diff --git a/src/operators/kernel/arm/resize_kernel.cpp b/src/operators/kernel/arm/resize_kernel.cpp new file mode 100644 index 0000000000000000000000000000000000000000..bef24f1c56f656d25261c9c837e5455244d1ad87 --- /dev/null +++ b/src/operators/kernel/arm/resize_kernel.cpp @@ -0,0 +1,124 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#ifdef RESIZE_OP + +#include "operators/kernel/resize_kernel.h" +#include + +namespace paddle_mobile { +namespace operators { +void BiLinearResizeTensor(const float* src, const int src_height, + const int src_width, float* dst, const int dst_height, + const int dst_width) { + const float scale_w = src_width / (float)dst_width; + const float scale_h = src_height / (float)dst_height; + float* dst_data = dst; + const float* src_data = src; + + for (int dst_h = 0; dst_h < dst_height; ++dst_h) { + float fh = dst_h * scale_h; + + int src_h = std::floor(fh); + + fh -= src_h; + const float w_h0 = std::abs((float)1.0 - fh); + const float w_h1 = std::abs(fh); + + const int dst_offset_1 = dst_h * dst_width; + const int src_offset_1 = src_h * src_width; + + float* dst_data_ptr = dst_data + dst_offset_1; + + for (int dst_w = 0; dst_w < dst_width; ++dst_w) { + float fw = dst_w * scale_w; + int src_w = std::floor(fw); + fw -= src_w; + const float w_w0 = std::abs((float)1.0 - fw); + const float w_w1 = std::abs(fw); + + float dst_value = 0; + + const int src_idx = src_offset_1 + src_w; + dst_value += (w_h0 * w_w0 * src_data[src_idx]); + int flag = 0; + if (src_w + 1 < src_width) { + dst_value += (w_h0 * w_w1 * src_data[src_idx + 1]); + ++flag; + } + if (src_h + 1 < src_height) { + dst_value += (w_h1 * w_w0 * src_data[src_idx + src_width]); + ++flag; + } + + if (flag > 1) { + dst_value += (w_h1 * w_w1 * src_data[src_idx + src_width + 1]); + // ++flag; + } + *(dst_data_ptr++) = dst_value; + } + } +} + +void ResizeTensor(const Tensor* src, const int src_n, const int src_c, + Tensor* dst, const int dst_n, const int dst_c) { + framework::DDim in_dims = src->dims(); + const int src_chans = in_dims[1]; + const int src_height = in_dims[2]; + const int src_width = in_dims[3]; + const int src_offset = (src_n * src_chans + src_c) * src_height * src_width; + + framework::DDim out_dims = dst->dims(); + const int dst_chans = out_dims[1]; + const int dst_height = out_dims[2]; + const int dst_width = out_dims[3]; + const int dst_offset = (dst_n * dst_chans + dst_c) * dst_height * dst_width; + + const auto* src_ptr = src->data(); + auto* dst_ptr = dst->data(); + const auto* src_data = &(src_ptr[src_offset]); + auto* dst_data = &(dst_ptr[dst_offset]); + BiLinearResizeTensor(src_data, src_height, src_width, dst_data, dst_height, + dst_width); +} + +void ResizeTensor(const Tensor* src, Tensor* dst) { + framework::DDim in_dims = src->dims(); + framework::DDim out_dims = dst->dims(); + PADDLE_MOBILE_ENFORCE(in_dims[0] == out_dims[0], + "src tensor batch num not equal to dst tensor"); + PADDLE_MOBILE_ENFORCE(in_dims[1] == out_dims[1], + "src tensor channel num not equal to dst tensor"); + for (int n = 0, batch_num = in_dims[0]; n < batch_num; ++n) { + for (int c = 0, chan_num = in_dims[1]; c < chan_num; ++c) { + ResizeTensor(src, n, c, dst, n, c); + } + } +} + +template <> +void ResizeKernel::Compute(const ResizeParam& param) const { + const auto* input_x = param.InputX(); + const auto& input_x_dims = input_x->dims(); + auto* out = param.Out(); + framework::DDim out_dims = CalOutputShape(param); + + out->Resize(out_dims); + ResizeTensor(input_x, out); +} + +} // namespace operators +} // namespace paddle_mobile + +#endif diff --git a/src/operators/kernel/arm/scale_kernel.cpp b/src/operators/kernel/arm/scale_kernel.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c20e70896145958aa91de4f00de7ad2eeba1bb5c --- /dev/null +++ b/src/operators/kernel/arm/scale_kernel.cpp @@ -0,0 +1,146 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#ifdef SCALE_OP + +#include "operators/kernel/scale_kernel.h" + +namespace paddle_mobile { +namespace operators { + +/* + * @b 特化到具体平台的实现, param 从 op 层传入 + * */ +template <> +void ScaleKernel::Compute(const ScaleParam ¶m) const { + const auto *input_x = param.InputX(); + auto *input_x_ptr = input_x->data(); + auto *out = param.Out(); + auto *out_ptr = out->mutable_data(); + + const vector scales = param.Scales(); + bool has_bias = param.HasBias(); + + const int dim_size = input_x->dims().size(); + switch (dim_size) { + case 1: { + const int input_width = input_x->dims()[0]; + if (has_bias) { + const vector biases = param.Biases(); + #pragma omp parallel for + for (int w = 0; w < input_width; w++) { + out_ptr[w] = input_x_ptr[w] * scales[w] + biases[w]; + } + } else { + #pragma omp parallel for + for (int w = 0; w < input_width; w++) { + out_ptr[w] = input_x_ptr[w] * scales[w]; + } + } + } break; + case 2: { + const int input_height = input_x->dims()[0]; + const int input_width = input_x->dims()[1]; + + if (has_bias) { + const vector biases = param.Biases(); + #pragma omp parallel for + for (int h = 0; h < input_height; ++h) { + const float *iptr = input_x_ptr + h * input_width; + float *optr = out_ptr + h * input_width; + for (int w = 0; w < input_width; ++w) { + optr[w] = iptr[w] * scales[w] + biases[w]; + } + } + } else { + #pragma omp parallel for + for (int h = 0; h < input_height; ++h) { + const float *iptr = input_x_ptr + h * input_width; + float *optr = out_ptr + h * input_width; + for (int w = 0; w < input_width; ++w) { + optr[w] = iptr[w] * scales[w]; + } + } + } + } break; + case 3: { + const int chan_size = input_x->dims()[0]; + const int input_height = input_x->dims()[1]; + const int input_width = input_x->dims()[2]; + int size = input_width * input_height; + + if (has_bias) { + const vector biases = param.Biases(); + + #pragma omp parallel for + for (int c = 0; c < chan_size; ++c) { + const float *iptr = input_x_ptr + c * size; + float *optr = out_ptr + c * size; + for (int i = 0; i < size; ++i) { + optr[i] = iptr[i] * scales[c] + biases[c]; + } + } + } else { + #pragma omp parallel for + for (int c = 0; c < chan_size; ++c) { + const float *iptr = input_x_ptr + c * size; + float *optr = out_ptr + c * size; + for (int i = 0; i < size; ++i) { + optr[i] = iptr[i] * scales[c]; + } + } + } + } break; + + case 4: { + const int batch_size = input_x->dims()[0]; + const int chan_size = input_x->dims()[0]; + const int input_height = input_x->dims()[1]; + const int input_width = input_x->dims()[2]; + int size = input_width * input_height; + + if (has_bias) { + const vector biases = param.Biases(); + + #pragma omp parallel for + for (int b = 0; b < batch_size; ++b) { + for (int c = 0; c < chan_size; ++c) { + const float *iptr = input_x_ptr + b * c * size; + float *optr = out_ptr + b * c * size; + for (int i = 0; i < size; ++i) { + optr[i] = iptr[i] * scales[c] + biases[c]; + } + } + } + } else { + #pragma omp parallel for + for (int b = 0; b < batch_size; ++b) { + for (int c = 0; c < chan_size; ++c) { + const float *iptr = input_x_ptr + b * c * size; + float *optr = out_ptr + b * c * size; + for (int i = 0; i < size; ++i) { + optr[i] = iptr[i] * scales[c]; + } + } + } + } + } break; + default: + break; + } +} +} // namespace operators +} // namespace paddle_mobile + +#endif diff --git a/src/operators/kernel/arm/slice_kernel.cpp b/src/operators/kernel/arm/slice_kernel.cpp new file mode 100644 index 0000000000000000000000000000000000000000..62efec9d2fb01568a108df8f3516085d81865bf7 --- /dev/null +++ b/src/operators/kernel/arm/slice_kernel.cpp @@ -0,0 +1,22 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#ifdef SLICE_OP + +#include "operators/kernel/slice_kernel.h" + +namespace paddle_mobile { +namespace operators {} +} // namespace paddle_mobile +#endif diff --git a/src/operators/kernel/prelu_kernel.h b/src/operators/kernel/prelu_kernel.h new file mode 100644 index 0000000000000000000000000000000000000000..9f5dcb23ee9bf44ffa8bbdd98879d533d07c39f9 --- /dev/null +++ b/src/operators/kernel/prelu_kernel.h @@ -0,0 +1,29 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "framework/operator.h" +#include "operators/op_param.h" + +#pragma once; + +namespace paddle_mobile { +namespace operators { + +template +class PReluKernel : public framework::OpKernelBase { + public: + void Compute(const PReluParam& param) const; +}; +} // namespace operators +} // namespace paddle_mobile diff --git a/src/operators/kernel/resize_kernel.h b/src/operators/kernel/resize_kernel.h new file mode 100644 index 0000000000000000000000000000000000000000..f1b9e64fc00863e83caa1c44606cdb3ec9f44817 --- /dev/null +++ b/src/operators/kernel/resize_kernel.h @@ -0,0 +1,79 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#ifdef RESIZE_OP + +#pragma once + +#include +#include "framework/operator.h" + +#include "operators/op_param.h" + +namespace paddle_mobile { +namespace operators { + +inline framework::DDim CalOutputShape(const ResizeParam ¶m) { + const auto *input_x = param.InputX(); + const auto &input_x_dims = input_x->dims(); + auto *out = param.Out(); + framework::DDim out_dims = out->dims(); + const auto *input_shape = param.InputShape(); + + if (input_shape) { + auto *shape_data = input_shape->data(); + framework::Tensor cpu_shape_tensor; + auto shape = + std::vector(shape_data, shape_data + input_shape->numel()); + const int in_batch_size = input_x->dims()[0]; + const int in_chan_size = input_x->dims()[1]; + const int in_height = input_x->dims()[2]; + const int in_width = input_x->dims()[3]; + + int out_height = 0; + int out_width = 0; + bool is_pyramid_test = param.IsPyramidTest(); + if (is_pyramid_test == false) { + out_height = param.Height(); + out_width = param.Width(); + PADDLE_MOBILE_ENFORCE(out_height > 0, "output height is required"); + PADDLE_MOBILE_ENFORCE(out_width > 0, "output width is required"); + + } else { + float out_height_scale = param.OutHeightScale(); + float out_width_scale = param.OutWidthScale(); + PADDLE_MOBILE_ENFORCE(out_height_scale > 0, + "output height scale is required"); + PADDLE_MOBILE_ENFORCE(out_width_scale > 0, + "output width scale is required"); + + out_height = int(out_height_scale * in_height); + out_width = int(out_width_scale * in_width); + } + + out_dims = framework::make_ddim( + {in_batch_size, in_chan_size, in_height, in_width}); + } + return out_dims; +} + +template +class ResizeKernel : public framework::OpKernelBase { + public: + void Compute(const ResizeParam ¶m) const; +}; +} // namespace operators +} // namespace paddle_mobile + +#endif diff --git a/src/operators/kernel/scale_kernel.h b/src/operators/kernel/scale_kernel.h new file mode 100644 index 0000000000000000000000000000000000000000..cc76a4b59b0be24dd3a3fb82c0e3d9fb1a4dbf24 --- /dev/null +++ b/src/operators/kernel/scale_kernel.h @@ -0,0 +1,29 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "framework/operator.h" +#include "operators/op_param.h" + +#pragma once; + +namespace paddle_mobile { +namespace operators { + +template +class ScaleKernel : public framework::OpKernelBase { + public: + void Compute(const ScaleParam& param) const; +}; +} // namespace operators +} // namespace paddle_mobile diff --git a/src/operators/kernel/slice_kernel.h b/src/operators/kernel/slice_kernel.h new file mode 100644 index 0000000000000000000000000000000000000000..e308364602f401b1c6c6f8e2e35385aefa055360 --- /dev/null +++ b/src/operators/kernel/slice_kernel.h @@ -0,0 +1,29 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "framework/operator.h" +#include "operators/op_param.h" + +#pragma once; + +namespace paddle_mobile { +namespace operators { + +template +class SliceKernel : public framework::OpKernelBase { + public: + void Compute(const SliceParam& param) const {} +}; +} // namespace operators +} // namespace paddle_mobile diff --git a/src/operators/math/gemm.cpp b/src/operators/math/gemm.cpp index e9974df967b293317c3014803bec27d2da73fca3..9582c18cbcfb6e502c42ab4195b553bd3b20093b 100644 --- a/src/operators/math/gemm.cpp +++ b/src/operators/math/gemm.cpp @@ -22,9 +22,14 @@ limitations under the License. */ namespace paddle_mobile { namespace operators { namespace math { -alignas(64) float packedA[MC * KC]; -alignas(64) float packedB[KC * NC]; -alignas(64) float ab[MR * NR]; +int MC = 0; +int KC = 0; +int NC = 0; + +float *packedA; +float *packedB; +float *packedC; +float *zero; // 将A矩阵分块复制到连续内存(ColMajor) void PackMatrixA(int m, int k, int m_tail, const float *A, int lda, float *buffer) { @@ -55,28 +60,39 @@ void PackMatrixA(int m, int k, int m_tail, const float *A, int lda, // 将A矩阵分块复制到连续内存(RowMajor) void PackMatrixA_(int m, int k, int m_tail, const float *A, int lda, float *buffer) { - int i, j; - const float *Ai, *Ai1, *Ai2, *Ai3; - for (i = 0; i < m - m_tail; i += MR) { - Ai = &A(i, 0); - Ai1 = &A(i + 1, 0); - Ai2 = &A(i + 2, 0); - Ai3 = &A(i + 3, 0); + const float *a0, *a1, *a2, *a3; + for (int i = 0; i < m - m_tail; i += MR) { + a0 = A + i * lda; + a1 = A + (i + 1) * lda; + a2 = A + (i + 2) * lda; + a3 = A + (i + 3) * lda; for (int j = 0; j < k; ++j) { - *buffer++ = *Ai++; - *buffer++ = *Ai1++; - *buffer++ = *Ai2++; - *buffer++ = *Ai3++; + *buffer++ = *a0++; + *buffer++ = *a1++; + *buffer++ = *a2++; + *buffer++ = *a3++; } } + int i = m - m_tail; + a0 = &A(i, 0); + a1 = a0 + lda; + a2 = a0 + 2 * lda; + a3 = a0 + 3 * lda; if (m_tail != 0) { - for (j = 0; j < k; ++j) { - for (i = m - m_tail; i < m; ++i) { - *buffer++ = A(i, j); - } - for (i = m; i < m + (MR - m_tail); ++i) { - *buffer++ = 0; - } + if (m_tail <= 3) { + a3 = zero; + } + if (m_tail <= 2) { + a2 = zero; + } + if (m_tail <= 1) { + a1 = zero; + } + for (int j = 0; j < k; ++j) { + *buffer++ = *a0++; + *buffer++ = *a1++; + *buffer++ = *a2++; + *buffer++ = *a3++; } } } @@ -113,35 +129,24 @@ void PackMatrixB(int k, int n, int n_tail, const float *B, int ldb, // 将B矩阵分块复制到连续内存(RowMajor) void PackMatrixB_(int k, int n, int n_tail, const float *B, int ldb, float *buffer) { - int i, j; - const float *Bij; - for (j = 0; j < n - n_tail; j += NR) { -#ifdef ARMV7 - - for (i = 0; i < k; ++i) { - Bij = &B(i, j); + const float *b0; + for (int j = 0; j < n - n_tail; j += NR) { + for (int i = 0; i < k; ++i) { + b0 = &B(i, j); asm volatile( - "vld1.32 {q0}, [%[Bij]] \n\t" - "vst1.32 {q0}, [%[buffer]]! \n\t" + "pld [%[b0]] \n\t" + "vld1.32 {q0, q1}, [%[b0]] \n\t" + "vst1.32 {q0, q1}, [%[buffer]]! \n\t" : [buffer] "+r"(buffer) - : [Bij] "r"(Bij) - : "memory", "q0"); - } -#else - for (i = 0; i < k; ++i) { - Bij = &B(i, j); - *buffer++ = *Bij; - *buffer++ = *(Bij + 1); - *buffer++ = *(Bij + 2); - *buffer++ = *(Bij + 3); + : [b0] "r"(b0) + : "memory", "q0", "q0"); } -#endif } if (n_tail != 0) { - for (i = 0; i < k; ++i) { - Bij = &B(i, n - n_tail); + for (int i = 0; i < k; ++i) { + b0 = &B(i, n - n_tail); for (int j = n - n_tail; j < n; ++j) { - *buffer++ = *Bij++; + *buffer++ = *b0++; } for (int j = n; j < n + (NR - n_tail); ++j) { *buffer++ = 0; @@ -151,118 +156,53 @@ void PackMatrixB_(int k, int n, int n_tail, const float *B, int ldb, } // 分块矩阵乘法 -void InnerKernel(int m, int n, int k, float alpha, const float *A, int lda, - const float *B, int ldb, float beta, float *C, int ldc, - int first_time) { - int m_block = (m + MR - 1) / MR * MR; - int n_block = (n + NR - 1) / NR * NR; - - int m_tail = m % MR; - int n_tail = n % NR; +void InnerKernel(int mc, int nc, float alpha, const float *a, const float *b, + float beta, float *c, float *C, int ldc, bool relu) { + for (int j = 0; j < nc; j += NR) { + for (int i = 0; i < mc; i += MR) { + // AddDot4x4(KC, a + i * KC, b + j * KC, c + i * NC + j, NC); + AddDot4x8(KC, a + i * KC, b + j * KC, c + i * NC + j, NC); + } + } - if (first_time) { - PackMatrixB_(k, n, n_tail, B, ldb, packedB); + if (alpha != 1) { + WriteWithAlphaBeta(mc, nc, c, C, ldc); + return; } - PackMatrixA_(m, k, m_tail, A, lda, packedA); - - int i, j, mc, nc; - - // B 取 4 列, 打包预热 - for (j = 0; j < n_block; j += NR) { - nc = (n - j) < NR ? n_tail : NR; - // A 取 4 行，打包预热 - for (i = 0; i < m_block; i += MR) { - mc = (m - i) < MR ? m_tail : MR; - AddDot4x4(k, alpha, &packedA[i * k], 4, &packedB[j * k], k, beta, - &C(i, j), ldc, mc, nc); - } + if (beta == 0) { + WriteBasic(mc, nc, c, C, ldc); + return; + } + if (beta == 1 && !relu) { + WriteWithAdd(mc, nc, c, C, ldc); + return; + } + if (beta == 1 && relu) { + WriteWithAddRelu(mc, nc, c, C, ldc); + return; } } // 分块矩阵乘法 -void InnerKernel_relu(int m, int n, int k, float alpha, const float *A, int lda, - const float *B, int ldb, float beta, float *C, int ldc, - int first_time, bool relu = false) { - int m_block = (m + MR - 1) / MR * MR; - int n_block = (n + NR - 1) / NR * NR; - - int m_tail = m % MR; - int n_tail = n % NR; - - if (first_time) { - PackMatrixB_(k, n, n_tail, B, ldb, packedB); - } - PackMatrixA_(m, k, m_tail, A, lda, packedA); - - int i, j, mc, nc; - - // B 取 4 列, 打包预热 - for (j = 0; j < n_block; j += NR) { - nc = (n - j) < NR ? n_tail : NR; - // A 取 4 行，打包预热 - for (i = 0; i < m_block; i += MR) { - mc = (m - i) < MR ? m_tail : MR; - AddDot4x4_relu(k, alpha, &packedA[i * k], 4, &packedB[j * k], k, beta, - &C(i, j), ldc, mc, nc, relu); +void InnerKernelWithBn(int mc, int nc, float alpha, const float *a, + const float *b, float beta, float *c, float *C, int ldc, + bool relu, float *new_scale, float *new_bias) { + for (int j = 0; j < nc; j += NR) { + for (int i = 0; i < mc; i += MR) { + // AddDot4x4(KC, a + i * KC, b + j * KC, c + i * NC + j, NC); + AddDot4x8(KC, a + i * KC, b + j * KC, c + i * NC + j, NC); } } -} - -// 计算一个更小的 4 * 4 的 C 矩阵分块 -#if defined(IOS) -void AddDot4x4(int k, float alpha, const float *a, int lda, const float *b, - int ldb, float beta, float *C, int ldc, int mc, int nc) { - // init C - float32x4_t cv0 = vdupq_n_f32(0.0); - float32x4_t cv1 = vdupq_n_f32(0.0); - float32x4_t cv2 = vdupq_n_f32(0.0); - float32x4_t cv3 = vdupq_n_f32(0.0); - - float32x4_t av; - float32x4_t bv; - - float32x2_t av01; - float32x2_t av23; - - for (int p = 0; p < k; p += 1) { - av = vld1q_f32(a); - bv = vld1q_f32(b); - - av01 = vget_low_f32(av); - cv0 = vmlaq_lane_f32(cv0, bv, av01, 0); - cv1 = vmlaq_lane_f32(cv1, bv, av01, 1); - av23 = vget_high_f32(av); - cv2 = vmlaq_lane_f32(cv2, bv, av23, 0); - cv3 = vmlaq_lane_f32(cv3, bv, av23, 1); - a += MR; - b += NR; - } - float32x4x4_t cv = {cv0, cv1, cv2, cv3}; - int i, j; - for (i = 0; i < mc; ++i) { - for (j = 0; j < nc; ++j) { - if (beta == 0.0) { - C(i, j) = 0.0; - } else if (beta != 1.0) { - C(i, j) *= beta; - } - if (j == 0) { - C(i, j) += alpha * vgetq_lane_f32(cv.val[i], 0); - } else if (j == 1) { - C(i, j) += alpha * vgetq_lane_f32(cv.val[i], 1); - } else if (j == 2) { - C(i, j) += alpha * vgetq_lane_f32(cv.val[i], 2); - } else if (j == 3) { - C(i, j) += alpha * vgetq_lane_f32(cv.val[i], 3); - } - } + if (relu) { + WriteWithBnRelu(mc, nc, c, C, ldc, new_scale, new_bias); + } else { + WriteWithBn(mc, nc, c, C, ldc, new_scale, new_bias); } } -void AddDot4x4_relu(int k, float alpha, const float *a, int lda, const float *b, - int ldb, float beta, float *C, int ldc, int mc, int nc, - bool relu = false) { +#if defined(IOS) +void AddDot4x4(int k, const float *a, const float *b, float *C, int ldc) { // init C float32x4_t cv0 = vdupq_n_f32(0.0); float32x4_t cv1 = vdupq_n_f32(0.0); @@ -307,183 +247,22 @@ void AddDot4x4_relu(int k, float alpha, const float *a, int lda, const float *b, } else if (j == 3) { C(i, j) += alpha * vgetq_lane_f32(cv.val[i], 3); } - if (C(i, j) < 0) { - C(i, j) = 0; - } } } } +} // namespace math #elif defined(ARMV7) -void AddDot4x4(int k, float alpha, const float *a, int lda, const float *b, - int ldb, float beta, float *C, int ldc, int mc, int nc) { - int kc1 = k / 4, kc2 = k % 4; - int bytes_ldc = 4 * ldc; - int flag_alpha = (alpha == 1.0) ? 1 : 2; - int flag_beta; - if (beta == 0.0) { - flag_beta = 0; - } else if (beta == 1.0) { - flag_beta = 1; - } else { - flag_beta = 2; - } - asm volatile( - "pld [%[a]] \n\t" - "pld [%[b]] \n\t" - "vmov.f32 q10, #0.0 \n\t" - "vmov.f32 q11, #0.0 \n\t" - "vmov.f32 q12, #0.0 \n\t" - "vmov.f32 q13, #0.0 \n\t" - - "subs %[kc1], %[kc1], #1 \n\t" - "blt end_kc1_%= \n\t" - "loop_kc1_%=: \n\t" - "pld [%[a], #64] \n\t" - "pld [%[b], #64] \n\t" - "vld1.32 {q0, q1}, [%[a]]! \n\t" - "vld1.32 {q2, q3}, [%[b]]! \n\t" - "vmla.f32 q10, q2, d0[0] \n\t" - "vmla.f32 q11, q2, d0[1] \n\t" - "vmla.f32 q12, q2, d1[0] \n\t" - "vmla.f32 q13, q2, d1[1] \n\t" - "vmla.f32 q10, q3, d2[0] \n\t" - "vmla.f32 q11, q3, d2[1] \n\t" - "vmla.f32 q12, q3, d3[0] \n\t" - "vmla.f32 q13, q3, d3[1] \n\t" - "vld1.32 {q0, q1}, [%[a]]! \n\t" - "vld1.32 {q2, q3}, [%[b]]! \n\t" - "vmla.f32 q10, q2, d0[0] \n\t" - "vmla.f32 q11, q2, d0[1] \n\t" - "vmla.f32 q12, q2, d1[0] \n\t" - "vmla.f32 q13, q2, d1[1] \n\t" - "vmla.f32 q10, q3, d2[0] \n\t" - "vmla.f32 q11, q3, d2[1] \n\t" - "vmla.f32 q12, q3, d3[0] \n\t" - "vmla.f32 q13, q3, d3[1] \n\t" - "subs %[kc1], %[kc1], #1 \n\t" - "bge loop_kc1_%= \n\t" - "end_kc1_%=: \n\t" - - "subs %[kc2], %[kc2], #1 \n\t" - "blt end_kc2_%= \n\t" - "loop_kc2_%=: \n\t" - "vld1.32 {q0}, [%[a]]! \n\t" - "vld1.32 {q1}, [%[b]]! \n\t" - "vmla.f32 q10, q1, d0[0] \n\t" - "vmla.f32 q11, q1, d0[1] \n\t" - "vmla.f32 q12, q1, d1[0] \n\t" - "vmla.f32 q13, q1, d1[1] \n\t" - "subs %[kc2], %[kc2], #1 \n\t" - "bge loop_kc2_%= \n\t" - "end_kc2_%=: \n\t" - - "cmp %[mc], #4 \n\t" - "bne temp_%= \n\t" - "cmp %[nc], #4 \n\t" - "bne temp_%= \n\t" - - "vmov.f32 d8[0], %[alpha] \n\t" - "vmov.f32 d8[1], %[beta] \n\t" - - "cmp %[flag_alpha], #1 \n\t" - "bne alpha_%= \n\t" - - "alpha_%=: \n\t" - "vmul.f32 q10, q10, d8[0] \n\t" - "vmul.f32 q11, q11, d8[0] \n\t" - "vmul.f32 q12, q12, d8[0] \n\t" - "vmul.f32 q13, q13, d8[0] \n\t" - - "beta_%=: \n\t" - "cmp %[flag_beta], #0 \n\t" - "beq memory_%= \n\t" - - "mov r4, %[C] \n\t" - "mov r6, %[bytes_ldc]\n\t" - "vld1.32 {q0}, [r4], r6 \n\t" - "vld1.32 {q1}, [r4], r6 \n\t" - "vld1.32 {q2}, [r4], r6 \n\t" - "vld1.32 {q3}, [r4] \n\t" - "cmp %[flag_beta], #1 \n\t" - "beq beta_eq1_%= \n\t" - "bne beta_ne1_%= \n\t" - - "beta_eq1_%=: \n\t" - "vadd.f32 q10, q10, q0 \n\t" - "vadd.f32 q11, q11, q1 \n\t" - "vadd.f32 q12, q12, q2 \n\t" - "vadd.f32 q13, q13, q3 \n\t" - "b memory_%= \n\t" - - "beta_ne1_%=: \n\t" - "vmla.f32 q10, q0, d8[1] \n\t" - "vmla.f32 q11, q1, d8[1] \n\t" - "vmla.f32 q12, q2, d8[1] \n\t" - "vmla.f32 q13, q3, d8[1] \n\t" - - "memory_%=: \n\t" - "mov r5, %[C] \n\t" - "mov r6, %[bytes_ldc]\n\t" - "vst1.32 {q10}, [r5], r6 \n\t" - "vst1.32 {q11}, [r5], r6 \n\t" - "vst1.32 {q12}, [r5], r6 \n\t" - "vst1.32 {q13}, [r5] \n\t" - "b end_%= \n\t" - - "temp_%=: \n\t" - "vst1.32 {q10, q11}, [%[ab]]!\n\t" - "vst1.32 {q12, q13}, [%[ab]] \n\t" - "end_%=: \n\t" - : - : [a] "r"(a), [b] "r"(b), [C] "r"(C), [ab] "r"(ab), [kc1] "r"(kc1), - [kc2] "r"(kc2), [mc] "r"(mc), [nc] "r"(nc), [alpha] "r"(alpha), - [beta] "r"(beta), [bytes_ldc] "r"(bytes_ldc), - [flag_alpha] "r"(flag_alpha), [flag_beta] "r"(flag_beta) - : "memory", "q0", "q1", "q2", "q3", "q4", "q10", "q11", "q12", "q13"); - - if (mc != MR || nc != NR) { - int i, j; - for (i = 0; i < mc; ++i) { - for (j = 0; j < nc; ++j) { - if (beta == 0.0) { - if (alpha != 1.0) { - C(i, j) = alpha * ab[i * MR + j]; - } else { - C(i, j) = ab[i * MR + j]; - } - } else { - if (beta != 1.0) { - C(i, j) *= beta; - } - if (alpha != 1.0) { - C(i, j) += alpha * ab[i * MR + j]; - } else { - C(i, j) += ab[i * MR + j]; - } - } - } - } - } -} - -void AddDot4x4_relu(int k, float alpha, const float *a, int lda, const float *b, - int ldb, float beta, float *C, int ldc, int mc, int nc, - bool relu = false) { - int kc1 = k / 4, kc2 = k % 4; - int bytes_ldc = 4 * ldc; - int flag_alpha = (alpha == 1.0) ? 1 : 2; - int flag_beta; - if (beta == 0.0) { - flag_beta = 0; - } else if (beta == 1.0) { - flag_beta = 1; - } else { - flag_beta = 2; - } +void AddDot4x4(int k, const float *a, const float *b, float *c, int ldc) { + const float *a_ptr, *b_ptr; + a_ptr = a; + b_ptr = b; + int kc1 = k / 4; + int kc2 = k % 4; + int step = 4 * ldc; asm volatile( - "pld [%[a]] \n\t" - "pld [%[b]] \n\t" + "pld [%[a_ptr]] \n\t" + "pld [%[b_ptr]] \n\t" "vmov.f32 q10, #0.0 \n\t" "vmov.f32 q11, #0.0 \n\t" "vmov.f32 q12, #0.0 \n\t" @@ -492,20 +271,10 @@ void AddDot4x4_relu(int k, float alpha, const float *a, int lda, const float *b, "subs %[kc1], %[kc1], #1 \n\t" "blt end_kc1_%= \n\t" "loop_kc1_%=: \n\t" - "pld [%[a], #64] \n\t" - "pld [%[b], #64] \n\t" - "vld1.32 {q0, q1}, [%[a]]! \n\t" - "vld1.32 {q2, q3}, [%[b]]! \n\t" - "vmla.f32 q10, q2, d0[0] \n\t" - "vmla.f32 q11, q2, d0[1] \n\t" - "vmla.f32 q12, q2, d1[0] \n\t" - "vmla.f32 q13, q2, d1[1] \n\t" - "vmla.f32 q10, q3, d2[0] \n\t" - "vmla.f32 q11, q3, d2[1] \n\t" - "vmla.f32 q12, q3, d3[0] \n\t" - "vmla.f32 q13, q3, d3[1] \n\t" - "vld1.32 {q0, q1}, [%[a]]! \n\t" - "vld1.32 {q2, q3}, [%[b]]! \n\t" + "pld [%[a_ptr], #64] \n\t" + "pld [%[b_ptr], #64] \n\t" + "vld1.32 {q0, q1}, [%[a_ptr]]! \n\t" + "vld1.32 {q2, q3}, [%[b_ptr]]! \n\t" "vmla.f32 q10, q2, d0[0] \n\t" "vmla.f32 q11, q2, d0[1] \n\t" "vmla.f32 q12, q2, d1[0] \n\t" @@ -514,6 +283,16 @@ void AddDot4x4_relu(int k, float alpha, const float *a, int lda, const float *b, "vmla.f32 q11, q3, d2[1] \n\t" "vmla.f32 q12, q3, d3[0] \n\t" "vmla.f32 q13, q3, d3[1] \n\t" + "vld1.32 {q4, q5}, [%[a_ptr]]! \n\t" + "vld1.32 {q6, q7}, [%[b_ptr]]! \n\t" + "vmla.f32 q10, q6, d8[0] \n\t" + "vmla.f32 q11, q6, d8[1] \n\t" + "vmla.f32 q12, q6, d9[0] \n\t" + "vmla.f32 q13, q6, d9[1] \n\t" + "vmla.f32 q10, q7, d10[0] \n\t" + "vmla.f32 q11, q7, d10[1] \n\t" + "vmla.f32 q12, q7, d11[0] \n\t" + "vmla.f32 q13, q7, d11[1] \n\t" "subs %[kc1], %[kc1], #1 \n\t" "bge loop_kc1_%= \n\t" "end_kc1_%=: \n\t" @@ -521,8 +300,8 @@ void AddDot4x4_relu(int k, float alpha, const float *a, int lda, const float *b, "subs %[kc2], %[kc2], #1 \n\t" "blt end_kc2_%= \n\t" "loop_kc2_%=: \n\t" - "vld1.32 {q0}, [%[a]]! \n\t" - "vld1.32 {q1}, [%[b]]! \n\t" + "vld1.32 {q0}, [%[a_ptr]]! \n\t" + "vld1.32 {q1}, [%[b_ptr]]! \n\t" "vmla.f32 q10, q1, d0[0] \n\t" "vmla.f32 q11, q1, d0[1] \n\t" "vmla.f32 q12, q1, d1[0] \n\t" @@ -531,290 +310,168 @@ void AddDot4x4_relu(int k, float alpha, const float *a, int lda, const float *b, "bge loop_kc2_%= \n\t" "end_kc2_%=: \n\t" - "cmp %[mc], #4 \n\t" - "bne temp_%= \n\t" - "cmp %[nc], #4 \n\t" - "bne temp_%= \n\t" - - "vmov.f32 d8[0], %[alpha] \n\t" - "vmov.f32 d8[1], %[beta] \n\t" - - "cmp %[flag_alpha], #1 \n\t" - "bne alpha_%= \n\t" - - "alpha_%=: \n\t" - "vmul.f32 q10, q10, d8[0] \n\t" - "vmul.f32 q11, q11, d8[0] \n\t" - "vmul.f32 q12, q12, d8[0] \n\t" - "vmul.f32 q13, q13, d8[0] \n\t" - - "beta_%=: \n\t" - "cmp %[flag_beta], #0 \n\t" - "beq memory_%= \n\t" - - "mov r4, %[C] \n\t" - "mov r6, %[bytes_ldc]\n\t" - "vld1.32 {q0}, [r4], r6 \n\t" - "vld1.32 {q1}, [r4], r6 \n\t" - "vld1.32 {q2}, [r4], r6 \n\t" - "vld1.32 {q3}, [r4] \n\t" - "cmp %[flag_beta], #1 \n\t" - "beq beta_eq1_%= \n\t" - "bne beta_ne1_%= \n\t" - - "beta_eq1_%=: \n\t" - "vadd.f32 q10, q10, q0 \n\t" - "vadd.f32 q11, q11, q1 \n\t" - "vadd.f32 q12, q12, q2 \n\t" - "vadd.f32 q13, q13, q3 \n\t" - "b memory_%= \n\t" - - "beta_ne1_%=: \n\t" - "vmla.f32 q10, q0, d8[1] \n\t" - "vmla.f32 q11, q1, d8[1] \n\t" - "vmla.f32 q12, q2, d8[1] \n\t" - "vmla.f32 q13, q3, d8[1] \n\t" - - "memory_%=: \n\t" - "vmax.f32 q10, q10, q14 \n\t" - "vmax.f32 q11, q11, q14 \n\t" - "vmax.f32 q12, q12, q14 \n\t" - "vmax.f32 q13, q13, q14 \n\t" - "mov r5, %[C] \n\t" - "mov r6, %[bytes_ldc]\n\t" + "mov r5, %[c] \n\t" + "mov r6, %[step] \n\t" "vst1.32 {q10}, [r5], r6 \n\t" "vst1.32 {q11}, [r5], r6 \n\t" "vst1.32 {q12}, [r5], r6 \n\t" "vst1.32 {q13}, [r5] \n\t" - "b end_%= \n\t" - - "temp_%=: \n\t" - "vst1.32 {q10, q11}, [%[ab]]!\n\t" - "vst1.32 {q12, q13}, [%[ab]] \n\t" - "end_%=: \n\t" : - : [a] "r"(a), [b] "r"(b), [C] "r"(C), [ab] "r"(ab), [kc1] "r"(kc1), - [kc2] "r"(kc2), [mc] "r"(mc), [nc] "r"(nc), [alpha] "r"(alpha), - [beta] "r"(beta), [bytes_ldc] "r"(bytes_ldc), - [flag_alpha] "r"(flag_alpha), [flag_beta] "r"(flag_beta) - : "memory", "q0", "q1", "q2", "q3", "q4", "q10", "q11", "q12", "q13", - "q14"); - - if (mc != MR || nc != NR) { - int i, j; - for (i = 0; i < mc; ++i) { - for (j = 0; j < nc; ++j) { - if (beta == 0.0) { - if (alpha != 1.0) { - C(i, j) = alpha * ab[i * MR + j]; - } else { - C(i, j) = ab[i * MR + j]; - } - } else { - if (beta != 1.0) { - C(i, j) *= beta; - } - if (alpha != 1.0) { - C(i, j) += alpha * ab[i * MR + j]; - } else { - C(i, j) += ab[i * MR + j]; - } - } - if (relu) { - if (C(i, j) < 0) { - C(i, j) = 0; - } - } - } - } - } + : [a_ptr] "r"(a_ptr), [b_ptr] "r"(b_ptr), [c] "r"(c), [kc1] "r"(kc1), + [kc2] "r"(kc2), [step] "r"(step) + : "memory", "r5", "r6", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", + "q10", "q11", "q12", "q13"); } #else -void AddDot4x4(int k, float alpha, const float *a, int lda, const float *b, - int ldb, float beta, float *C, int ldc, int mc, int nc) { - float c[16] = {0}; - float reg_a0, reg_a1, reg_a2, reg_a3, reg_b0, reg_b1, reg_b2, reg_b3; - +void AddDot4x4(int k, const float *a, const float *b, float *c, int ldc) { + float *c0, *c1, *c2, *c3; + c0 = c; + c1 = c + ldc; + c2 = c + 2 * ldc; + c3 = c + 3 * ldc; for (int p = 0; p < k; p += 1) { - reg_b0 = *b++; - reg_b1 = *b++; - reg_b2 = *b++; - reg_b3 = *b++; - - reg_a0 = *a++; - reg_a1 = *a++; - reg_a2 = *a++; - reg_a3 = *a++; - // first row - c[0] += reg_a0 * reg_b0; - c[1] += reg_a0 * reg_b1; - c[2] += reg_a0 * reg_b2; - c[3] += reg_a0 * reg_b3; + c0[0] += a[0] * b[0]; + c0[1] += a[0] * b[1]; + c0[2] += a[0] * b[2]; + c0[3] += a[0] * b[3]; // second row - c[4] += reg_a1 * reg_b0; - c[5] += reg_a1 * reg_b1; - c[6] += reg_a1 * reg_b2; - c[7] += reg_a1 * reg_b3; + c1[0] += a[1] * b[0]; + c1[1] += a[1] * b[1]; + c1[2] += a[1] * b[2]; + c1[3] += a[1] * b[3]; // third row - c[8] += reg_a2 * reg_b0; - c[9] += reg_a2 * reg_b1; - c[10] += reg_a2 * reg_b2; - c[11] += reg_a2 * reg_b3; + c2[0] += a[2] * b[0]; + c2[1] += a[2] * b[1]; + c2[2] += a[2] * b[2]; + c2[3] += a[2] * b[3]; // fourth row - c[12] += reg_a3 * reg_b0; - c[13] += reg_a3 * reg_b1; - c[14] += reg_a3 * reg_b2; - c[15] += reg_a3 * reg_b3; - } - int i, j; - for (i = 0; i < mc; ++i) { - for (j = 0; j < nc; ++j) { - if (beta == 0.0) { - C(i, j) = 0.0; - } else if (beta != 1.0) { - C(i, j) *= beta; - } - if (alpha != 1.0) { - C(i, j) += alpha * c[i * MR + j]; - } else { - C(i, j) += c[i * MR + j]; - } - } - } -} - -void AddDot4x4_relu(int k, float alpha, const float *a, int lda, const float *b, - int ldb, float beta, float *C, int ldc, int mc, int nc, - bool relu) { - float c[16] = {0}; - float reg_a0, reg_a1, reg_a2, reg_a3, reg_b0, reg_b1, reg_b2, reg_b3; - - for (int p = 0; p < k; p += 1) { - reg_b0 = *b++; - reg_b1 = *b++; - reg_b2 = *b++; - reg_b3 = *b++; - - reg_a0 = *a++; - reg_a1 = *a++; - reg_a2 = *a++; - reg_a3 = *a++; - - // first row - c[0] += reg_a0 * reg_b0; - c[1] += reg_a0 * reg_b1; - c[2] += reg_a0 * reg_b2; - c[3] += reg_a0 * reg_b3; - - // second row - c[4] += reg_a1 * reg_b0; - c[5] += reg_a1 * reg_b1; - c[6] += reg_a1 * reg_b2; - c[7] += reg_a1 * reg_b3; - - // third row - c[8] += reg_a2 * reg_b0; - c[9] += reg_a2 * reg_b1; - c[10] += reg_a2 * reg_b2; - c[11] += reg_a2 * reg_b3; + c3[0] += a[3] * b[0]; + c3[1] += a[3] * b[1]; + c3[2] += a[3] * b[2]; + c3[3] += a[3] * b[3]; - // fourth row - c[12] += reg_a3 * reg_b0; - c[13] += reg_a3 * reg_b1; - c[14] += reg_a3 * reg_b2; - c[15] += reg_a3 * reg_b3; - } - int i, j; - for (i = 0; i < mc; ++i) { - for (j = 0; j < nc; ++j) { - if (beta == 0.0) { - C(i, j) = 0.0; - } else if (beta != 1.0) { - C(i, j) *= beta; - } - if (alpha != 1.0) { - C(i, j) += alpha * c[i * MR + j]; - } else { - C(i, j) += c[i * MR + j]; - } - if (relu) { - if (C(i, j) < 0) { - C(i, j) = 0; - } - } - } + a += 4; + b += 4; } } #endif // 32位 float 矩阵乘法 -void sgemm(int m, int n, int k, float alpha, const float *A, int lda, - const float *B, int ldb, float beta, float *C, int ldc) { - int i, j, p, mc, nc, kc; - float beta_; - -#ifdef ARMV7 - if (m == 1) { - VectorKernel(1, n, k, alpha, A, lda, B, ldb, beta, C, ldc); - return; +void Sgemm(int m, int n, int k, float alpha, const float *A, int lda, + const float *B, int ldb, float beta, float *C, int ldc, bool relu) { + // L1 data cache is 32 kib (Per Contex-A57, Contex-A72, Contex-A73) + // L2 cache is 0.5~4 Mib (Contex-A72 cluster) + int L1 = 30 * 1024; + int L2 = 1 * 1024 * 1024; + + KC = k; + MC = L2 / (2 * KC * sizeof(float)); + NC = MC; + + // make sure MC is multiple of 4, and NC is multiple of 8 + int mblock_num = (m + MC - 1) / MC; + MC = (m + mblock_num - 1) / mblock_num; + MC = (MC + 4 - 1) / 4 * 4; + // DLOG << "mblock_num = " << mblock_num << ", MC = " << MC << "\n"; + + int nblock_num = (n + NC - 1) / NC; + NC = (n + nblock_num - 1) / nblock_num; + NC = (NC + 8 - 1) / 8 * 8; + // DLOG << "nblock_num = " << nblock_num << ", NC = " << NC << "\n"; + + packedA = static_cast( + paddle_mobile::memory::Alloc(sizeof(float) * MC * KC)); + packedB = static_cast( + paddle_mobile::memory::Alloc(sizeof(float) * KC * NC)); + packedC = static_cast( + paddle_mobile::memory::Alloc(sizeof(float) * MC * NC)); + zero = static_cast(paddle_mobile::memory::Alloc(sizeof(float) * KC)); + + for (int l = 0; l < KC; ++l) { + zero[l] = 0; } -#endif - for (j = 0; j < n; j += NC) { + int mc, nc; + for (int j = 0; j < n; j += NC) { nc = s_min(n - j, NC); - for (p = 0; p < k; p += KC) { - kc = s_min(k - p, KC); - for (i = 0; i < m; i += MC) { - mc = s_min(m - i, MC); - if (p != 0) { - beta_ = 1.0; - } else { - beta_ = beta; - } - InnerKernel(mc, nc, kc, alpha, &A(i, p), lda, &B(p, j), ldb, beta_, - &C(i, j), ldc, i == 0); - } + PackMatrixB_(KC, nc, nc % NR, &B(0, j), ldb, packedB); + for (int i = 0; i < m; i += MC) { + mc = s_min(m - i, MC); + PackMatrixA_(mc, KC, mc % MR, &A(i, 0), lda, packedA); + InnerKernel(mc, nc, alpha, packedA, packedB, beta, packedC, &C(i, j), ldc, + relu); } } + + paddle_mobile::memory::Free(packedA); + paddle_mobile::memory::Free(packedB); + paddle_mobile::memory::Free(packedC); + paddle_mobile::memory::Free(zero); } -void sgemm_relu(int m, int n, int k, float alpha, const float *A, int lda, - const float *B, int ldb, float beta, float *C, int ldc) { - int i, j, p, mc, nc, kc; - float beta_; - for (j = 0; j < n; j += NC) { - nc = s_min(n - j, NC); - for (p = 0; p < k; p += KC) { - kc = s_min(k - p, KC); - for (i = 0; i < m; i += MC) { - mc = s_min(m - i, MC); - if (p != 0) { - beta_ = 1.0; - } else { - beta_ = beta; - } +void SgemmWithBn(int m, int n, int k, float alpha, const float *A, int lda, + const float *B, int ldb, float beta, float *C, int ldc, + bool relu, float *new_scale, float *new_bias) { + // L1 data cache is 32 kib (Per Contex-A57, Contex-A72, Contex-A73) + // L2 cache is 0.5~4 Mib (Contex-A72 cluster) + int L1 = 30 * 1024; + int L2 = 1 * 1024 * 1024; + + KC = k; + MC = L2 / (2 * KC * sizeof(float)); + NC = MC; + + // make sure MC is multiple of 4, and NC is multiple of 8 + int mblock_num = (m + MC - 1) / MC; + MC = (m + mblock_num - 1) / mblock_num; + MC = (MC + 4 - 1) / 4 * 4; + // DLOG << "mblock_num = " << mblock_num << ", MC = " << MC << "\n"; + + int nblock_num = (n + NC - 1) / NC; + NC = (n + nblock_num - 1) / nblock_num; + NC = (NC + 8 - 1) / 8 * 8; + // DLOG << "nblock_num = " << nblock_num << ", NC = " << NC << "\n"; + + packedA = static_cast( + paddle_mobile::memory::Alloc(sizeof(float) * MC * KC)); + packedB = static_cast( + paddle_mobile::memory::Alloc(sizeof(float) * KC * NC)); + packedC = static_cast( + paddle_mobile::memory::Alloc(sizeof(float) * MC * NC)); + zero = static_cast(paddle_mobile::memory::Alloc(sizeof(float) * KC)); + + for (int l = 0; l < KC; ++l) { + zero[l] = 0; + } - if (p + KC >= k) { - InnerKernel_relu(mc, nc, kc, alpha, &A(i, p), lda, &B(p, j), ldb, - beta_, &C(i, j), ldc, i == 0, true); - } else { - InnerKernel(mc, nc, kc, alpha, &A(i, p), lda, &B(p, j), ldb, beta_, - &C(i, j), ldc, i == 0); - } - } + int mc, nc; + for (int j = 0; j < n; j += NC) { + nc = s_min(n - j, NC); + PackMatrixB_(KC, nc, nc % NR, &B(0, j), ldb, packedB); + for (int i = 0; i < m; i += MC) { + mc = s_min(m - i, MC); + PackMatrixA_(mc, KC, mc % MR, &A(i, 0), lda, packedA); + InnerKernelWithBn(mc, nc, alpha, packedA, packedB, beta, packedC, + &C(i, j), ldc, relu, new_scale + ldc * i + j, + new_bias + ldc * i + j); } } + + paddle_mobile::memory::Free(packedA); + paddle_mobile::memory::Free(packedB); + paddle_mobile::memory::Free(packedC); + paddle_mobile::memory::Free(zero); } -#ifdef ARMV7 void VectorKernel(int m, int n, int k, float alpha, const float *A, int lda, - const float *B, int ldb, float beta, float *C, int ldc) { + const float *B, int ldb, float beta, float *C, int ldc, + bool relu) { float *bufferC = static_cast(memory::Alloc(sizeof(float) * n)); const float *a0, *b0, *b1, *b2, *b3; @@ -1016,18 +673,995 @@ void VectorKernel(int m, int n, int k, float alpha, const float *A, int lda, } } - c0 = bufferC; - C0 = C; - for (int i = 0; i < n; i++) { - if (beta == 1.0) { - *C0++ += *c0++; - } else { - *C0++ = *c0++; - } + if (alpha != 1) { + VecWriteWithAlphaBeta(n, bufferC, C, ldc); + return; + } + if (beta == 0) { + VecWriteBasic(n, bufferC, C, ldc); + return; + } + if (beta == 1 && !relu) { + VecWriteWithAdd(n, bufferC, C, ldc); + return; + } + if (beta == 1 && relu) { + VecWriteWithAddRelu(n, bufferC, C, ldc); + return; } } -#endif -} // namespace math -} // namespace operators +void VectorKernelWithBn(int m, int n, int k, float alpha, const float *A, + int lda, const float *B, int ldb, float beta, float *C, + int ldc, bool relu, float *new_scale, float *new_bias) { + float *bufferC = static_cast(memory::Alloc(sizeof(float) * n)); + + const float *a0, *b0, *b1, *b2, *b3; + float *c0, *C0; + + int volatile kc1 = k / 4; + int volatile kc2 = k % 4; + int volatile nc1 = n / 16; + int _nc1 = n % 16; + int volatile nc2 = _nc1 / 4; + int volatile nc3 = _nc1 % 4; + for (int i = 0; i < kc1; i++) { + a0 = A + i * 4; + b0 = B + i * 4 * ldb; + b1 = b0 + ldb; + b2 = b1 + ldb; + b3 = b2 + ldb; + c0 = bufferC; + asm volatile( + "pld [%[a0], #16] \n\t" + "vld1.32 {q0}, [%[a0]] \n\t" + + "subs %[nc1], %[nc1], #1 \n\t" + "blt end_nc1_%= \n\t" + "loop_nc1_%=: \n\t" + + "cmp %[i], #0 \n\t" + "beq i_eq0_%= \n\t" + "bne i_ne0_%= \n\t" + + "i_eq0_%=: \n\t" + "vmov.f32 q10, #0.0 \n\t" + "vmov.f32 q11, #0.0 \n\t" + "vmov.f32 q12, #0.0 \n\t" + "vmov.f32 q13, #0.0 \n\t" + "b gemm_nc1_%= \n\t" + + "i_ne0_%=: \n\t" + "pld [%[c0], #64] \n\t" + "vld1.32 {q10, q11}, [%[c0]]! \n\t" + "vld1.32 {q12, q13}, [%[c0]] \n\t" + "sub %[c0], %[c0], #32 \n\t" + + "gemm_nc1_%=: \n\t" + "pld [%[b0], #64] \n\t" + "vld1.32 {q2, q3}, [%[b0]]! \n\t" + "vld1.32 {q4, q5}, [%[b0]]! \n\t" + "vmla.f32 q10, q2, d0[0] \n\t" + "vmla.f32 q11, q3, d0[0] \n\t" + "vmla.f32 q12, q4, d0[0] \n\t" + "vmla.f32 q13, q5, d0[0] \n\t" + + "pld [%[b1], #64] \n\t" + "vld1.32 {q2, q3}, [%[b1]]! \n\t" + "vld1.32 {q4, q5}, [%[b1]]! \n\t" + "vmla.f32 q10, q2, d0[1] \n\t" + "vmla.f32 q11, q3, d0[1] \n\t" + "vmla.f32 q12, q4, d0[1] \n\t" + "vmla.f32 q13, q5, d0[1] \n\t" + + "pld [%[b2], #64] \n\t" + "vld1.32 {q2, q3}, [%[b2]]! \n\t" + "vld1.32 {q4, q5}, [%[b2]]! \n\t" + "vmla.f32 q10, q2, d1[0] \n\t" + "vmla.f32 q11, q3, d1[0] \n\t" + "vmla.f32 q12, q4, d1[0] \n\t" + "vmla.f32 q13, q5, d1[0] \n\t" + + "pld [%[b3], #64] \n\t" + "vld1.32 {q2, q3}, [%[b3]]! \n\t" + "vld1.32 {q4, q5}, [%[b3]]! \n\t" + "vmla.f32 q10, q2, d1[1] \n\t" + "vmla.f32 q11, q3, d1[1] \n\t" + "vmla.f32 q12, q4, d1[1] \n\t" + "vmla.f32 q13, q5, d1[1] \n\t" + + "vst1.32 {q10, q11}, [%[c0]]! \n\t" + "vst1.32 {q12, q13}, [%[c0]]! \n\t" + + "subs %[nc1], %[nc1], #1 \n\t" + "bge loop_nc1_%= \n\t" + "end_nc1_%=: \n\t" + + "subs %[nc2], %[nc2], #1 \n\t" + "blt end_nc2_%= \n\t" + "loop_nc2_%=: \n\t" + + "cmp %[i], #0 \n\t" + "beq ii_eq0_%= \n\t" + "bne ii_ne0_%= \n\t" + + "ii_eq0_%=: \n\t" + "vmov.f32 q10, #0.0 \n\t" + "b gemm_nc2_%= \n\t" + + "ii_ne0_%=: \n\t" + "pld [%[c0], #16] \n\t" + "vld1.32 {q10}, [%[c0]] \n\t" + + "gemm_nc2_%=: \n\t" + "pld [%[b0], #16] \n\t" + "vld1.32 {q2}, [%[b0]]! \n\t" + "vmla.f32 q10, q2, d0[0] \n\t" + + "pld [%[b1], #16] \n\t" + "vld1.32 {q3}, [%[b1]]! \n\t" + "vmla.f32 q10, q3, d0[1] \n\t" + + "pld [%[b2], #16] \n\t" + "vld1.32 {q4}, [%[b2]]! \n\t" + "vmla.f32 q10, q4, d1[0] \n\t" + + "pld [%[b3], #16] \n\t" + "vld1.32 {q5}, [%[b3]]! \n\t" + "vmla.f32 q10, q5, d1[1] \n\t" + + "vst1.32 {q10}, [%[c0]]! \n\t" + + "subs %[nc2], %[nc2], #1 \n\t" + "bge loop_nc2_%= \n\t" + "end_nc2_%=: \n\t" + + : [b0] "+r"(b0), [b1] "+r"(b1), [b2] "+r"(b2), [b3] "+r"(b3), + [c0] "+r"(c0) + : [a0] "r"(a0), [i] "r"(i), [nc1] "r"(nc1), [nc2] "r"(nc2) + : "memory", "q0", "q2", "q3", "q4", "q5", "q10", "q11", "q12", "q13"); + + for (int j = 0; j < nc3; j++) { + if (i == 0) { + *c0 = (*a0) * (*b0++); + } else { + *c0 += (*a0) * (*b0++); + } + *c0 += (*(a0 + 1)) * (*b1++); + *c0 += (*(a0 + 2)) * (*b2++); + *c0 += (*(a0 + 3)) * (*b3++); + c0++; + } + } + + for (int i = 0; i < kc2; ++i) { + a0 = A + 4 * kc1 + i; + b0 = B + (4 * kc1 + i) * ldb; + c0 = bufferC; + asm volatile( + "pld [%[a0], #16] \n\t" + "vld1.32 {d0}, [%[a0]] \n\t" + + "subs %[nc1], %[nc1], #1 \n\t" + "blt end_nc1_%= \n\t" + "loop_nc1_%=: \n\t" + + "pld [%[c0], #64] \n\t" + "vld1.32 {q10, q11}, [%[c0]]! \n\t" + "vld1.32 {q12, q13}, [%[c0]] \n\t" + "sub %[c0], %[c0], #32 \n\t" + + "gemm_nc1_%=: \n\t" + "pld [%[b0], #64] \n\t" + "vld1.32 {q2, q3}, [%[b0]]! \n\t" + "vld1.32 {q4, q5}, [%[b0]]! \n\t" + "vmla.f32 q10, q2, d0[0] \n\t" + "vmla.f32 q11, q3, d0[0] \n\t" + "vmla.f32 q12, q4, d0[0] \n\t" + "vmla.f32 q13, q5, d0[0] \n\t" + + "vst1.32 {q10, q11}, [%[c0]]! \n\t" + "vst1.32 {q12, q13}, [%[c0]]! \n\t" + + "subs %[nc1], %[nc1], #1 \n\t" + "bge loop_nc1_%= \n\t" + "end_nc1_%=: \n\t" + + "subs %[nc2], %[nc2], #1 \n\t" + "blt end_nc2_%= \n\t" + "loop_nc2_%=: \n\t" + + "pld [%[c0], #16] \n\t" + "vld1.32 {q10}, [%[c0]] \n\t" + + "gemm_nc2_%=: \n\t" + "vld1.32 {q2}, [%[b0]]! \n\t" + "vmla.f32 q10, q2, d0[0] \n\t" + + "vst1.32 {q10}, [%[c0]]! \n\t" + + "subs %[nc2], %[nc2], #1 \n\t" + "bge loop_nc2_%= \n\t" + "end_nc2_%=: \n\t" + + : [b0] "+r"(b0), [b1] "+r"(b1), [b2] "+r"(b2), [b3] "+r"(b3), + [c0] "+r"(c0) + : [a0] "r"(a0), [nc1] "r"(nc1), [nc2] "r"(nc2) + : "memory", "q0", "q2", "q3", "q4", "q5", "q10", "q11", "q12", "q13"); + + for (int j = 0; j < nc3; j++) { + *c0 += (*a0) * (*b0++); + c0++; + } + } + + if (relu) { + VecWriteWithBnRelu(n, bufferC, C, ldc, new_scale, new_bias); + } else { + VecWriteWithBn(n, bufferC, C, ldc, new_scale, new_bias); + } +} + +void AddDot4x8(int k, const float *a, const float *b, float *c, int ldc) { + const float *a_ptr, *b_ptr; + a_ptr = a; + b_ptr = b; + int kc1 = k / 4; + int kc2 = k % 4; + int step = 4 * ldc; + asm volatile( + "pld [%[a_ptr]] \n\t" + "pld [%[b_ptr]] \n\t" + + "vmov.f32 q8, #0.0 \n\t" + "vmov.f32 q9, #0.0 \n\t" + "vmov.f32 q10, #0.0 \n\t" + "vmov.f32 q11, #0.0 \n\t" + "vmov.f32 q12, #0.0 \n\t" + "vmov.f32 q13, #0.0 \n\t" + "vmov.f32 q14, #0.0 \n\t" + "vmov.f32 q15, #0.0 \n\t" + + "subs %[kc1], %[kc1], #1 \n\t" + "blt end_kc1_%= \n\t" + "loop_kc1_%=: \n\t" + + "pld [%[a_ptr], #64] \n\t" + "pld [%[b_ptr], #64] \n\t" + + "vld1.32 {q0, q1}, [%[a_ptr]]! \n\t" + "vld1.32 {q2, q3}, [%[b_ptr]]! \n\t" + "vld1.32 {q4, q5}, [%[b_ptr]]! \n\t" + + "vmla.f32 q8, q2, d0[0] \n\t" + "vmla.f32 q9, q3, d0[0] \n\t" + "vmla.f32 q10, q2, d0[1] \n\t" + "vmla.f32 q11, q3, d0[1] \n\t" + "vmla.f32 q12, q2, d1[0] \n\t" + "vmla.f32 q13, q3, d1[0] \n\t" + "vmla.f32 q14, q2, d1[1] \n\t" + "vmla.f32 q15, q3, d1[1] \n\t" + + "vmla.f32 q8, q4, d2[0] \n\t" + "vmla.f32 q9, q5, d2[0] \n\t" + "vmla.f32 q10, q4, d2[1] \n\t" + "vmla.f32 q11, q5, d2[1] \n\t" + "vmla.f32 q12, q4, d3[0] \n\t" + "vmla.f32 q13, q5, d3[0] \n\t" + "vmla.f32 q14, q4, d3[1] \n\t" + "vmla.f32 q15, q5, d3[1] \n\t" + + "pld [%[b_ptr], #64] \n\t" + + "vld1.32 {q0, q1}, [%[a_ptr]]! \n\t" + "vld1.32 {q2, q3}, [%[b_ptr]]! \n\t" + "vld1.32 {q4, q5}, [%[b_ptr]]! \n\t" + + "vmla.f32 q8, q2, d0[0] \n\t" + "vmla.f32 q9, q3, d0[0] \n\t" + "vmla.f32 q10, q2, d0[1] \n\t" + "vmla.f32 q11, q3, d0[1] \n\t" + "vmla.f32 q12, q2, d1[0] \n\t" + "vmla.f32 q13, q3, d1[0] \n\t" + "vmla.f32 q14, q2, d1[1] \n\t" + "vmla.f32 q15, q3, d1[1] \n\t" + + "vmla.f32 q8, q4, d2[0] \n\t" + "vmla.f32 q9, q5, d2[0] \n\t" + "vmla.f32 q10, q4, d2[1] \n\t" + "vmla.f32 q11, q5, d2[1] \n\t" + "vmla.f32 q12, q4, d3[0] \n\t" + "vmla.f32 q13, q5, d3[0] \n\t" + "vmla.f32 q14, q4, d3[1] \n\t" + "vmla.f32 q15, q5, d3[1] \n\t" + + "subs %[kc1], %[kc1], #1 \n\t" + "bge loop_kc1_%= \n\t" + "end_kc1_%=: \n\t" + + "subs %[kc2], %[kc2], #1 \n\t" + "blt end_kc2_%= \n\t" + "loop_kc2_%=: \n\t" + "vld1.32 {q0}, [%[a_ptr]]! \n\t" + "vld1.32 {q2, q3}, [%[b_ptr]]! \n\t" + "vmla.f32 q8, q2, d0[0] \n\t" + "vmla.f32 q9, q3, d0[0] \n\t" + "vmla.f32 q10, q2, d0[1] \n\t" + "vmla.f32 q11, q3, d0[1] \n\t" + "vmla.f32 q12, q2, d1[0] \n\t" + "vmla.f32 q13, q3, d1[0] \n\t" + "vmla.f32 q14, q2, d1[1] \n\t" + "vmla.f32 q15, q3, d1[1] \n\t" + "subs %[kc2], %[kc2], #1 \n\t" + "bge loop_kc2_%= \n\t" + "end_kc2_%=: \n\t" + + "mov r5, %[c] \n\t" + "mov r6, %[step] \n\t" + "vst1.32 {q8, q9}, [r5], r6 \n\t" + "vst1.32 {q10, q11}, [r5], r6 \n\t" + "vst1.32 {q12, q13}, [r5], r6 \n\t" + "vst1.32 {q14, q15}, [r5] \n\t" + : + : [a_ptr] "r"(a_ptr), [b_ptr] "r"(b_ptr), [c] "r"(c), [kc1] "r"(kc1), + [kc2] "r"(kc2), [step] "r"(step) + : "memory", "r5", "r6", "q0", "q1", "q2", "q3", "q4", "q5", "q8", "q9", + "q10", "q11", "q12", "q13", "q14", "q15"); +} + +// C = A * B +void WriteBasic(int mc, int nc, float *c, float *C, int ldc) { + int nc1 = nc / 16; + int _nc1 = nc % 16; + int step = 4 * ldc; + int step1 = 4 * (NC - 16 * nc1); + int volatile m = mc; + + float *volatile c_ptr, *volatile C_ptr; + float *C0, *c0; + c_ptr = c; + C_ptr = C; + if (nc1 > 0) { + asm volatile( + "subs %[mc], %[mc], #1 \n\t" + "blt end_mc_%= \n\t" + "loop_mc_%=: \n\t" + + "mov r6, %[C_ptr] \n\t" + "mov r5, %[nc1] \n\t" + "subs r5, r5, #1 \n\t" + "blt end_nc1_%= \n\t" + "loop_nc1_%=: \n\t" + + "vld1.32 {q0, q1}, [%[c_ptr]]! \n\t" + "vst1.32 {q0, q1}, [r6]! \n\t" + + "vld1.32 {q2, q3}, [%[c_ptr]]! \n\t" + "vst1.32 {q2, q3}, [r6]! \n\t" + + "subs r5, r5, #1 \n\t" + "bge loop_nc1_%= \n\t" + "end_nc1_%=: \n\t" + + "add %[C_ptr], %[C_ptr], %[step] \n\t" + "add %[c_ptr], %[c_ptr], %[step1] \n\t" + "subs %[mc], %[mc], #1 \n\t" + "bge loop_mc_%= \n\t" + "end_mc_%=: \n\t" + + : + : [C_ptr] "r"(C_ptr), [c_ptr] "r"(c_ptr), [mc] "r"(m), [nc1] "r"(nc1), + [step] "r"(step), [step1] "r"(step1) + : "memory", "r5", "r6", "q0", "q1", "q2", "q3"); + } + + if (_nc1 != 0) { + for (int i = 0; i < mc; i++) { + C0 = C_ptr + nc1 * 16 + i * ldc; + c0 = c_ptr + nc1 * 16 + i * NC; + for (int j = 0; j < _nc1; j++) { + *C0++ = *c0++; + } + } + } +} + +// C = alpha * A * B + beta * C +void WriteWithAlphaBeta(int mc, int nc, float *c, float *C, int ldc) {} + +// C = A * B + C +void WriteWithAdd(int mc, int nc, float *c, float *C, int ldc) { + int nc1 = nc / 16; + int _nc1 = nc % 16; + int step = 4 * ldc; + int step1 = 4 * (NC - 16 * nc1); + int volatile m = mc; + + float *volatile c_ptr, *volatile C_ptr; + float *C0, *c0; + c_ptr = c; + C_ptr = C; + if (nc1 > 0) { + asm volatile( + "subs %[mc], %[mc], #1 \n\t" + "blt end_mc_%= \n\t" + "loop_mc_%=: \n\t" + + "mov r6, %[C_ptr] \n\t" + "mov r5, %[nc1] \n\t" + "subs r5, r5, #1 \n\t" + "blt end_nc1_%= \n\t" + "loop_nc1_%=: \n\t" + + "vld1.32 {q0, q1}, [r6] \n\t" + "vld1.32 {q2, q3}, [%[c_ptr]]! \n\t" + "vadd.f32 q10, q0, q2 \n\t" + "vadd.f32 q11, q1, q3 \n\t" + "vst1.32 {q10, q11}, [r6]! \n\t" + + "vld1.32 {q4, q5}, [r6] \n\t" + "vld1.32 {q6, q7}, [%[c_ptr]]! \n\t" + "vadd.f32 q12, q4, q6 \n\t" + "vadd.f32 q13, q5, q7 \n\t" + "vst1.32 {q12, q13}, [r6]! \n\t" + + "subs r5, r5, #1 \n\t" + "bge loop_nc1_%= \n\t" + "end_nc1_%=: \n\t" + + "add %[C_ptr], %[C_ptr], %[step] \n\t" + "add %[c_ptr], %[c_ptr], %[step1] \n\t" + "subs %[mc], %[mc], #1 \n\t" + "bge loop_mc_%= \n\t" + "end_mc_%=: \n\t" + + : + : [C_ptr] "r"(C_ptr), [c_ptr] "r"(c_ptr), [mc] "r"(m), [nc1] "r"(nc1), + [step] "r"(step), [step1] "r"(step1) + : "memory", "r5", "r6", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", + "q10", "q11", "q12", "q13"); + } + + if (_nc1 != 0) { + for (int i = 0; i < mc; i++) { + C0 = C_ptr + nc1 * 16 + i * ldc; + c0 = c_ptr + nc1 * 16 + i * NC; + for (int j = 0; j < _nc1; j++) { + *C0++ += *c0++; + } + } + } +} + +// C = A * B + C, relu(C) +void WriteWithAddRelu(int mc, int nc, float *c, float *C, int ldc) { + int nc1 = nc / 16; + int _nc1 = nc % 16; + int step = 4 * ldc; + int step1 = 4 * (NC - 16 * nc1); + int volatile m = mc; + + float *volatile c_ptr, *volatile C_ptr; + float *C0, *c0; + c_ptr = c; + C_ptr = C; + if (nc1 > 0) { + asm volatile( + "vmov.f32 q14, #0.0 \n\t" + "subs %[mc], %[mc], #1 \n\t" + "blt end_mc_%= \n\t" + "loop_mc_%=: \n\t" + + "mov r6, %[C_ptr] \n\t" + "mov r5, %[nc1] \n\t" + "subs r5, r5, #1 \n\t" + "blt end_nc1_%= \n\t" + "loop_nc1_%=: \n\t" + + "vld1.32 {q0, q1}, [r6] \n\t" + "vld1.32 {q2, q3}, [%[c_ptr]]! \n\t" + "vadd.f32 q10, q0, q2 \n\t" + "vadd.f32 q11, q1, q3 \n\t" + "vmax.f32 q10, q10, q14 \n\t" + "vmax.f32 q11, q11, q14 \n\t" + "vst1.32 {q10, q11}, [r6]! \n\t" + + "vld1.32 {q4, q5}, [r6] \n\t" + "vld1.32 {q6, q7}, [%[c_ptr]]! \n\t" + "vadd.f32 q12, q4, q6 \n\t" + "vadd.f32 q13, q5, q7 \n\t" + "vmax.f32 q12, q12, q14 \n\t" + "vmax.f32 q13, q13, q14 \n\t" + "vst1.32 {q12, q13}, [r6]! \n\t" + + "subs r5, r5, #1 \n\t" + "bge loop_nc1_%= \n\t" + "end_nc1_%=: \n\t" + + "add %[C_ptr], %[C_ptr], %[step] \n\t" + "add %[c_ptr], %[c_ptr], %[step1] \n\t" + "subs %[mc], %[mc], #1 \n\t" + "bge loop_mc_%= \n\t" + "end_mc_%=: \n\t" + + : + : [C_ptr] "r"(C_ptr), [c_ptr] "r"(c_ptr), [mc] "r"(m), [nc1] "r"(nc1), + [step] "r"(step), [step1] "r"(step1) + : "memory", "r5", "r6", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", + "q10", "q11", "q12", "q13"); + } + + if (_nc1 != 0) { + for (int i = 0; i < mc; i++) { + C0 = C_ptr + nc1 * 16 + i * ldc; + c0 = c_ptr + nc1 * 16 + i * NC; + for (int j = 0; j < _nc1; j++) { + *C0 += *c0; + if (*C0 < 0) { + *C0 = 0; + } + C0++; + c0++; + } + } + } +} + +// C = A * B, batchnorm(C) +void WriteWithBn(int mc, int nc, float *c, float *C, int ldc, float *scale, + float *bias) { + int nc1 = nc / 16; + int _nc1 = nc % 16; + int nc2 = _nc1 / 4; + int nc3 = 16 - 4 * (_nc1 % 4); + int step = 4 * (ldc - nc); + int step1 = 4 * (NC - nc); + + asm volatile( + "subs %[mc], %[mc], #1 \n\t" + "blt end_mc_%= \n\t" + "loop_mc_%=: \n\t" + + "mov r5, %[nc1] \n\t" + "mov r6, %[nc2] \n\t" + + "subs r5, r5, #1 \n\t" + "blt end_nc1_%= \n\t" + "loop_nc1_%=: \n\t" + + "vld1.32 {q0, q1}, [%[c]]! \n\t" + "vld1.32 {q2, q3}, [%[scale]]! \n\t" + "vld1.32 {q10, q11}, [%[bias]]! \n\t" + "vmla.f32 q10, q0, q2 \n\t" + "vmla.f32 q11, q1, q3 \n\t" + "vst1.32 {q10, q11}, [%[C]]! \n\t" + + "vld1.32 {q4, q5}, [%[c]]! \n\t" + "vld1.32 {q6, q7}, [%[scale]]! \n\t" + "vld1.32 {q12, q13}, [%[bias]]! \n\t" + "vmla.f32 q12, q4, q6 \n\t" + "vmla.f32 q13, q5, q7 \n\t" + "vst1.32 {q12, q13}, [%[C]]! \n\t" + + "subs r5, r5, #1 \n\t" + "bge loop_nc1_%= \n\t" + "end_nc1_%=: \n\t" + + "subs r6, r6, #1 \n\t" + "blt end_nc2_%= \n\t" + "loop_nc2_%=: \n\t" + + "vld1.32 {q0}, [%[c]]! \n\t" + "vld1.32 {q1}, [%[scale]]! \n\t" + "vld1.32 {q10}, [%[bias]]! \n\t" + "vmla.f32 q10, q0, q1 \n\t" + "vst1.32 {q10}, [%[C]]! \n\t" + + "subs r6, r6, #1 \n\t" + "bge loop_nc2_%= \n\t" + "end_nc2_%=: \n\t" + + "cmp %[nc3], #16 \n\t" + "beq end_nc3_%= \n\t" + + "sub %[c], %[c], %[nc3] \n\t" + "sub %[scale], %[scale], %[nc3] \n\t" + "sub %[bias], %[bias], %[nc3] \n\t" + "sub %[C], %[C], %[nc3] \n\t" + + "vld1.32 {q0}, [%[c]]! \n\t" + "vld1.32 {q1}, [%[scale]]! \n\t" + "vld1.32 {q10}, [%[bias]]! \n\t" + "vmla.f32 q10, q0, q1 \n\t" + "vst1.32 {q10}, [%[C]]! \n\t" + "end_nc3_%=: \n\t" + + "add %[c], %[c], %[step1] \n\t" + "add %[scale], %[scale], %[step] \n\t" + "add %[bias], %[bias], %[step] \n\t" + "add %[C], %[C], %[step] \n\t" + + "subs %[mc], %[mc], #1 \n\t" + "bge loop_mc_%= \n\t" + "end_mc_%=: \n\t" + + : + : [C] "r"(C), [c] "r"(c), [mc] "r"(mc), [nc1] "r"(nc1), [nc2] "r"(nc2), + [nc3] "r"(nc3), [step] "r"(step), [step1] "r"(step1), + [scale] "r"(scale), [bias] "r"(bias) + : "memory", "cc", "r5", "r6", "r7", "r8", "q0", "q1", "q2", "q3", "q4", + "q5", "q6", "q7", "q10", "q11", "q12", "q13"); +} + +// C = A * B, batchnorm(C), relu(C) +void WriteWithBnRelu(int mc, int nc, float *c, float *C, int ldc, float *scale, + float *bias) { + int nc1 = nc / 16; + int _nc1 = nc % 16; + int nc2 = _nc1 / 4; + int nc3 = 16 - 4 * (_nc1 % 4); + int step = 4 * (ldc - nc); + int step1 = 4 * (NC - nc); + + asm volatile( + "vmov.f32 q14, #0.0 \n\t" + "subs %[mc], %[mc], #1 \n\t" + "blt end_mc_%= \n\t" + "loop_mc_%=: \n\t" + + "mov r5, %[nc1] \n\t" + "mov r6, %[nc2] \n\t" + + "subs r5, r5, #1 \n\t" + "blt end_nc1_%= \n\t" + "loop_nc1_%=: \n\t" + + "vld1.32 {q0, q1}, [%[c]]! \n\t" + "vld1.32 {q2, q3}, [%[scale]]! \n\t" + "vld1.32 {q10, q11}, [%[bias]]! \n\t" + "vmla.f32 q10, q0, q2 \n\t" + "vmla.f32 q11, q1, q3 \n\t" + "vmax.f32 q10, q10, q14 \n\t" + "vmax.f32 q11, q11, q14 \n\t" + "vst1.32 {q10, q11}, [%[C]]! \n\t" + + "vld1.32 {q4, q5}, [%[c]]! \n\t" + "vld1.32 {q6, q7}, [%[scale]]! \n\t" + "vld1.32 {q12, q13}, [%[bias]]! \n\t" + "vmla.f32 q12, q4, q6 \n\t" + "vmla.f32 q13, q5, q7 \n\t" + "vmax.f32 q12, q12, q14 \n\t" + "vmax.f32 q13, q13, q14 \n\t" + "vst1.32 {q12, q13}, [%[C]]! \n\t" + + "subs r5, r5, #1 \n\t" + "bge loop_nc1_%= \n\t" + "end_nc1_%=: \n\t" + + "subs r6, r6, #1 \n\t" + "blt end_nc2_%= \n\t" + "loop_nc2_%=: \n\t" + + "vld1.32 {q0}, [%[c]]! \n\t" + "vld1.32 {q1}, [%[scale]]! \n\t" + "vld1.32 {q10}, [%[bias]]! \n\t" + "vmla.f32 q10, q0, q1 \n\t" + "vmax.f32 q10, q10, q14 \n\t" + "vst1.32 {q10}, [%[C]]! \n\t" + + "subs r6, r6, #1 \n\t" + "bge loop_nc2_%= \n\t" + "end_nc2_%=: \n\t" + + "cmp %[nc3], #16 \n\t" + "beq end_nc3_%= \n\t" + + "sub %[c], %[c], %[nc3] \n\t" + "sub %[scale], %[scale], %[nc3] \n\t" + "sub %[bias], %[bias], %[nc3] \n\t" + "sub %[C], %[C], %[nc3] \n\t" + + "vld1.32 {q0}, [%[c]]! \n\t" + "vld1.32 {q1}, [%[scale]]! \n\t" + "vld1.32 {q10}, [%[bias]]! \n\t" + "vmla.f32 q10, q0, q1 \n\t" + "vmax.f32 q10, q10, q14 \n\t" + "vst1.32 {q10}, [%[C]]! \n\t" + "end_nc3_%=: \n\t" + + "add %[c], %[c], %[step1] \n\t" + "add %[scale], %[scale], %[step] \n\t" + "add %[bias], %[bias], %[step] \n\t" + "add %[C], %[C], %[step] \n\t" + + "subs %[mc], %[mc], #1 \n\t" + "bge loop_mc_%= \n\t" + "end_mc_%=: \n\t" + + : + : [C] "r"(C), [c] "r"(c), [mc] "r"(mc), [nc1] "r"(nc1), [nc2] "r"(nc2), + [nc3] "r"(nc3), [step] "r"(step), [step1] "r"(step1), + [scale] "r"(scale), [bias] "r"(bias) + : "memory", "r5", "r6", "r7", "r8", "q0", "q1", "q2", "q3", "q4", "q5", + "q6", "q7", "q10", "q11", "q12", "q13", "q14"); +} + +// C = A * B +void VecWriteBasic(int n, float *c, float *C, int ldc) { + int nc1 = n / 16; + int _nc1 = n % 16; + int nc2 = _nc1 / 4; + int nc3 = 16 - 4 * (_nc1 % 4); + + asm volatile( + "subs %[nc1], %[nc1], #1 \n\t" + "blt end_nc1_%= \n\t" + "loop_nc1_%=: \n\t" + + "vld1.32 {q0, q1}, [%[c]]! \n\t" + "vst1.32 {q0, q1}, [%[C]]! \n\t" + + "vld1.32 {q2, q3}, [%[c]]! \n\t" + "vst1.32 {q2, q3}, [%[C]]! \n\t" + + "subs %[nc1], %[nc1], #1 \n\t" + "bge loop_nc1_%= \n\t" + "end_nc1_%=: \n\t" + + "subs %[nc2], %[nc2], #1 \n\t" + "blt end_nc2_%= \n\t" + "loop_nc2_%=: \n\t" + + "vld1.32 {q4}, [%[c]]! \n\t" + "vst1.32 {q4}, [%[C]]! \n\t" + + "subs %[nc2], %[nc2], #1 \n\t" + "bge loop_nc2_%= \n\t" + "end_nc2_%=: \n\t" + + "cmp %[nc3], #16 \n\t" + "beq end_nc3_%= \n\t" + "sub %[c], %[c], %[nc3] \n\t" + "sub %[C], %[C], %[nc3] \n\t" + "vld1.32 {q5}, [%[c]]! \n\t" + "vst1.32 {q5}, [%[C]]! \n\t" + "end_nc3_%=: \n\t" + + : + : [C] "r"(C), [c] "r"(c), [nc1] "r"(nc1), [nc2] "r"(nc2), [nc3] "r"(nc3) + : "memory", "q0", "q1", "q2", "q3", "q4", "q5"); +} + +// C = alpha * A * B + beta * C +void VecWriteWithAlphaBeta(int n, float *c, float *C, int ldc) {} + +// C = A * B + C +void VecWriteWithAdd(int n, float *c, float *C, int ldc) { + int nc1 = n / 16; + int _nc1 = n % 16; + + asm volatile( + "subs %[nc1], %[nc1], #1 \n\t" + "blt end_nc1_%= \n\t" + "loop_nc1_%=: \n\t" + + "vld1.32 {q0, q1}, [%[c]]! \n\t" + "vld1.32 {q2, q3}, [%[C]] \n\t" + "vadd.f32 q10, q0, q2 \n\t" + "vadd.f32 q11, q1, q3 \n\t" + "vst1.32 {q10, q11}, [%[C]]! \n\t" + + "vld1.32 {q4, q5}, [%[c]]! \n\t" + "vld1.32 {q6, q7}, [%[C]] \n\t" + "vadd.f32 q12, q4, q6 \n\t" + "vadd.f32 q13, q5, q7 \n\t" + "vst1.32 {q12, q13}, [%[C]]! \n\t" + + "subs %[nc1], %[nc1], #1 \n\t" + "bge loop_nc1_%= \n\t" + "end_nc1_%=: \n\t" + + : [C] "+r"(C), [c] "+r"(c) + : [nc1] "r"(nc1) + : "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q10", "q11", + "q12", "q13"); + + if (_nc1 != 0) { + for (int j = 0; j < _nc1; j++) { + *C++ += *c++; + } + } +} + +// C = A * B + C, relu(C) +void VecWriteWithAddRelu(int n, float *c, float *C, int ldc) { + int nc1 = n / 16; + int _nc1 = n % 16; + + asm volatile( + "vmov.f32 q14, #0.0 \n\t" + "subs %[nc1], %[nc1], #1 \n\t" + "blt end_nc1_%= \n\t" + "loop_nc1_%=: \n\t" + + "vld1.32 {q0, q1}, [%[c]]! \n\t" + "vld1.32 {q2, q3}, [%[C]] \n\t" + "vadd.f32 q10, q0, q2 \n\t" + "vadd.f32 q11, q1, q3 \n\t" + "vmax.f32 q10, q10, q14 \n\t" + "vmax.f32 q11, q11, q14 \n\t" + "vst1.32 {q10, q11}, [%[C]]! \n\t" + + "vld1.32 {q4, q5}, [%[c]]! \n\t" + "vld1.32 {q6, q7}, [%[C]] \n\t" + "vadd.f32 q12, q4, q6 \n\t" + "vadd.f32 q13, q5, q7 \n\t" + "vmax.f32 q12, q12, q14 \n\t" + "vmax.f32 q13, q13, q14 \n\t" + "vst1.32 {q12, q13}, [%[C]]! \n\t" + + "subs %[nc1], %[nc1], #1 \n\t" + "bge loop_nc1_%= \n\t" + "end_nc1_%=: \n\t" + + : [C] "+r"(C), [c] "+r"(c) + : [nc1] "r"(nc1) + : "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q10", "q11", + "q12", "q13"); + + if (_nc1 != 0) { + for (int j = 0; j < _nc1; j++) { + *C += *c; + if (*C < 0) { + *C = 0; + } + C++; + c++; + } + } +} + +// C = A * B, batchnorm(C) +void VecWriteWithBn(int n, float *c, float *C, int ldc, float *scale, + float *bias) { + int nc1 = n / 16; + int _nc1 = n % 16; + int nc2 = _nc1 / 4; + int nc3 = 16 - 4 * (_nc1 % 4); + + asm volatile( + "subs %[nc1], %[nc1], #1 \n\t" + "blt end_nc1_%= \n\t" + "loop_nc1_%=: \n\t" + + "vld1.32 {q0, q1}, [%[c]]! \n\t" + "vld1.32 {q2, q3}, [%[scale]]! \n\t" + "vld1.32 {q10, q11}, [%[bias]]! \n\t" + "vmla.f32 q10, q0, q2 \n\t" + "vmla.f32 q11, q1, q3 \n\t" + "vst1.32 {q10, q11}, [%[C]]! \n\t" + + "vld1.32 {q4, q5}, [%[c]]! \n\t" + "vld1.32 {q6, q7}, [%[scale]]! \n\t" + "vld1.32 {q12, q13}, [%[bias]]! \n\t" + "vmla.f32 q12, q4, q6 \n\t" + "vmla.f32 q13, q5, q7 \n\t" + "vst1.32 {q12, q13}, [%[C]]! \n\t" + + "subs %[nc1], %[nc1], #1 \n\t" + "bge loop_nc1_%= \n\t" + "end_nc1_%=: \n\t" + + "subs %[nc2], %[nc2], #1 \n\t" + "blt end_nc2_%= \n\t" + "loop_nc2_%=: \n\t" + + "vld1.32 {q0}, [%[c]]! \n\t" + "vld1.32 {q1}, [%[scale]]! \n\t" + "vld1.32 {q10}, [%[bias]]! \n\t" + "vmla.f32 q10, q0, q1 \n\t" + "vst1.32 {q10}, [%[C]]! \n\t" + + "subs %[nc2], %[nc2], #1 \n\t" + "bge loop_nc2_%= \n\t" + "end_nc2_%=: \n\t" + + "cmp %[nc3], #16 \n\t" + "beq end_nc3_%= \n\t" + + "sub %[c], %[c], %[nc3] \n\t" + "sub %[scale], %[scale], %[nc3] \n\t" + "sub %[bias], %[bias], %[nc3] \n\t" + "sub %[C], %[C], %[nc3] \n\t" + + "vld1.32 {q0}, [%[c]]! \n\t" + "vld1.32 {q1}, [%[scale]]! \n\t" + "vld1.32 {q10}, [%[bias]]! \n\t" + "vmla.f32 q10, q0, q1 \n\t" + "vst1.32 {q10}, [%[C]]! \n\t" + "end_nc3_%=: \n\t" + + : + : [C] "r"(C), [c] "r"(c), [nc1] "r"(nc1), [nc2] "r"(nc2), [nc3] "r"(nc3), + [scale] "r"(scale), [bias] "r"(bias) + : "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q10", "q11", + "q12", "q13"); +} + +// C = A * B, batchnorm(C), relu(C) +void VecWriteWithBnRelu(int n, float *c, float *C, int ldc, float *scale, + float *bias) { + int nc1 = n / 16; + int _nc1 = n % 16; + int nc2 = _nc1 / 4; + int nc3 = 16 - 4 * (_nc1 % 4); + + asm volatile( + "vmov.f32 q14, #0.0 \n\t" + "subs %[nc1], %[nc1], #1 \n\t" + "blt end_nc1_%= \n\t" + "loop_nc1_%=: \n\t" + + "vld1.32 {q0, q1}, [%[c]]! \n\t" + "vld1.32 {q2, q3}, [%[scale]]! \n\t" + "vld1.32 {q10, q11}, [%[bias]]! \n\t" + "vmla.f32 q10, q0, q2 \n\t" + "vmla.f32 q11, q1, q3 \n\t" + "vmax.f32 q10, q10, q14 \n\t" + "vmax.f32 q11, q11, q14 \n\t" + "vst1.32 {q10, q11}, [%[C]]! \n\t" + + "vld1.32 {q4, q5}, [%[c]]! \n\t" + "vld1.32 {q6, q7}, [%[scale]]! \n\t" + "vld1.32 {q12, q13}, [%[bias]]! \n\t" + "vmla.f32 q12, q4, q6 \n\t" + "vmla.f32 q13, q5, q7 \n\t" + "vmax.f32 q12, q12, q14 \n\t" + "vmax.f32 q13, q13, q14 \n\t" + "vst1.32 {q12, q13}, [%[C]]! \n\t" + + "subs %[nc1], %[nc1], #1 \n\t" + "bge loop_nc1_%= \n\t" + "end_nc1_%=: \n\t" + + "subs %[nc2], %[nc2], #1 \n\t" + "blt end_nc2_%= \n\t" + "loop_nc2_%=: \n\t" + + "vld1.32 {q0}, [%[c]]! \n\t" + "vld1.32 {q1}, [%[scale]]! \n\t" + "vld1.32 {q10}, [%[bias]]! \n\t" + "vmla.f32 q10, q0, q1 \n\t" + "vmax.f32 q10, q10, q14 \n\t" + "vst1.32 {q10}, [%[C]]! \n\t" + + "subs %[nc2], %[nc2], #1 \n\t" + "bge loop_nc2_%= \n\t" + "end_nc2_%=: \n\t" + + "cmp %[nc3], #16 \n\t" + "beq end_nc3_%= \n\t" + + "sub %[c], %[c], %[nc3] \n\t" + "sub %[scale], %[scale], %[nc3] \n\t" + "sub %[bias], %[bias], %[nc3] \n\t" + "sub %[C], %[C], %[nc3] \n\t" + + "vld1.32 {q0}, [%[c]]! \n\t" + "vld1.32 {q1}, [%[scale]]! \n\t" + "vld1.32 {q10}, [%[bias]]! \n\t" + "vmla.f32 q10, q0, q1 \n\t" + "vmax.f32 q10, q10, q14 \n\t" + "vst1.32 {q10}, [%[C]]! \n\t" + "end_nc3_%=: \n\t" + + : + : [C] "r"(C), [c] "r"(c), [nc1] "r"(nc1), [nc2] "r"(nc2), [nc3] "r"(nc3), + [scale] "r"(scale), [bias] "r"(bias) + : "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q10", "q11", + "q12", "q13", "q14"); +} + +} // namespace operators +} // namespace paddle_mobile } // namespace paddle_mobile diff --git a/src/operators/math/gemm.h b/src/operators/math/gemm.h index 6d7ae6d2bcdbd7e24cb3c2389dd3cdf09a807892..b4bce43c7a29fba09ade7512cbc660f0ac2888ab 100644 --- a/src/operators/math/gemm.h +++ b/src/operators/math/gemm.h @@ -19,12 +19,8 @@ limitations under the License. */ #define B(i, j) B[(i)*ldb + (j)] #define C(i, j) C[(i)*ldc + (j)] -// 分块计算的块大小，mc 与 kc 分别对应分块计算时的 m 与 k -#define MC 128 -#define KC 128 -#define NC 1024 #define MR 4 -#define NR 4 +#define NR 8 #define s_min(i, j) ((i) < (j) ? (i) : (j)) @@ -49,28 +45,66 @@ void PackMatrixB_(int k, int n, int n_tail, const float *B, int ldb, float *buffer); // 分块矩阵乘法 -void InnerKernel(int m, int n, int k, float alpha, const float *A, int lda, - const float *B, int ldb, float beta, float *C, int ldc, - int first_time); +void InnerKernel(int mc, int nc, float alpha, const float *a, const float *b, + float beta, float *c, float *C, int ldc, bool relu); + +void InnerKernelWithBn(int mc, int nc, float alpha, const float *a, + const float *b, float beta, float *c, float *C, int ldc, + bool relu, float *new_scale, float *new_bias); // 向量矩阵乘法 (M = 1) void VectorKernel(int m, int n, int k, float alpha, const float *A, int lda, - const float *B, int ldb, float beta, float *C, int ldc); - -// 计算一个更小的 4 * 4 的 C 矩阵分块 -void AddDot4x4(int k, float alpha, const float *A, int lda, const float *B, - int ldb, float beta, float *C, int ldc, int mc, int nc); - -void AddDot4x4_relu(int k, float alpha, const float *a, int lda, const float *b, - int ldb, float beta, float *C, int ldc, int mc, int nc, - bool relu); + const float *B, int ldb, float beta, float *C, int ldc, + bool relu); + +void VectorKernelWithBn(int m, int n, int k, float alpha, const float *A, + int lda, const float *B, int ldb, float beta, float *C, + int ldc, bool relu, float *new_scale, float *new_bias); + +// 计算一个更小的 C 矩阵分块 +void AddDot4x4(int k, const float *a, const float *b, float *c, int ldc); +void AddDot4x8(int k, const float *a, const float *b, float *c, int ldc); + +// 分块矩阵乘法结果回写 +// C = A * B +void WriteBasic(int mc, int nc, float *c, float *C, int ldc); +// C = alpha * A * B + beta * C +void WriteWithAlphaBeta(int mc, int nc, float *c, float *C, int ldc); +// C = A * B + C +void WriteWithAdd(int mc, int nc, float *c, float *C, int ldc); +// C = A * B + C, relu(C) +void WriteWithAddRelu(int mc, int nc, float *c, float *C, int ldc); +// C = A * B, batchnorm(C) +void WriteWithBn(int mc, int nc, float *c, float *C, int ldc, float *new_scale, + float *new_bias); +// C = A * B, batchnorm(C), relu(C) +void WriteWithBnRelu(int mc, int nc, float *c, float *C, int ldc, + float *new_scale, float *new_bias); + +// 向量矩阵乘法结果回写 +// C = A * B +void VecWriteBasic(int n, float *c, float *C, int ldc); +// C = alpha * A * B + beta * C +void VecWriteWithAlphaBeta(int n, float *c, float *C, int ldc); +// C = A * B + C +void VecWriteWithAdd(int n, float *c, float *C, int ldc); +// C = A * B + C, relu(C) +void VecWriteWithAddRelu(int n, float *c, float *C, int ldc); +// C = A * B, batchnorm(C) +void VecWriteWithBn(int n, float *c, float *C, int ldc, float *new_scale, + float *new_bias); +// C = A * B, batchnorm(C), relu(C) +void VecWriteWithBnRelu(int n, float *c, float *C, int ldc, float *new_scale, + float *new_bias); // 32位 float 矩阵乘法 -void sgemm(int m, int n, int k, float alpha, const float *A, int lda, - const float *B, int ldb, float beta, float *C, int ldc); +void Sgemm(int m, int n, int k, float alpha, const float *A, int lda, + const float *B, int ldb, float beta, float *C, int ldc, bool relu); -void sgemm_relu(int m, int n, int k, float alpha, const float *A, int lda, - const float *B, int ldb, float beta, float *C, int ldc); +// 32位 float 矩阵乘法, 并对结果进行 batchnrom +void SgemmWithBn(int m, int n, int k, float alpha, const float *A, int lda, + const float *B, int ldb, float beta, float *C, int ldc, + bool relu, float *new_scale, float *new_bias); // 64位 double 矩阵乘法 void dgemm(int m, int n, int k, float alpha, const double *A, int lda, diff --git a/src/operators/math/math_function.cpp b/src/operators/math/math_function.cpp index fd4106038c7446e659736c6b3c61b5aa05127e72..ca5367788ed87da070dd19900e8d546e51caf337 100644 --- a/src/operators/math/math_function.cpp +++ b/src/operators/math/math_function.cpp @@ -39,22 +39,18 @@ void matmul(const framework::Tensor &matrix_a, bool trans_a, int M = dim_out[0]; int N = dim_out[1]; - int K = (trans_a == false) ? dim_a[1] : dim_a[0]; + int K = (!trans_a) ? dim_a[1] : dim_a[0]; - if (relu) { - sgemm_relu(M, N, K, alpha, matrix_a.data(), K, - matrix_b.data(), N, beta, matrix_out->data(), N); - } else { - sgemm(M, N, K, alpha, matrix_a.data(), K, matrix_b.data(), N, - beta, matrix_out->data(), N); - } + Sgemm(M, N, K, alpha, matrix_a.data(), K, matrix_b.data(), N, + beta, matrix_out->data(), N, relu); } template <> -void matmul(const framework::Tensor &matrix_a, bool trans_a, - const framework::Tensor &matrix_b, bool trans_b, - double alpha, framework::Tensor *matrix_out, double beta, - bool relu) { +void matmulWithBn(const framework::Tensor &matrix_a, bool trans_a, + const framework::Tensor &matrix_b, bool trans_b, + float alpha, framework::Tensor *matrix_out, float beta, + bool relu, framework::Tensor *new_scale, + framework::Tensor *new_bias) { auto dim_a = matrix_a.dims(); auto dim_b = matrix_b.dims(); auto dim_out = matrix_out->dims(); @@ -71,7 +67,11 @@ void matmul(const framework::Tensor &matrix_a, bool trans_a, int M = dim_out[0]; int N = dim_out[1]; - int K = (trans_a == false) ? dim_a[1] : dim_a[0]; + int K = (!trans_a) ? dim_a[1] : dim_a[0]; + + SgemmWithBn(M, N, K, alpha, matrix_a.data(), K, matrix_b.data(), + N, beta, matrix_out->data(), N, relu, + new_scale->data(), new_bias->data()); } } // namespace math diff --git a/src/operators/math/math_function.h b/src/operators/math/math_function.h index 0b953ec6a3b2a03a94a91884b9daf3ed88523a22..0ca7815fc2bcff2be0345b581d3dfb26cf55794c 100644 --- a/src/operators/math/math_function.h +++ b/src/operators/math/math_function.h @@ -26,6 +26,12 @@ template void matmul(const framework::Tensor &matrix_a, bool trans_a, const framework::Tensor &matrix_b, bool trans_b, T alpha, framework::Tensor *matrix_out, T beta, bool relu = false); + +template +void matmulWithBn(const framework::Tensor &matrix_a, bool trans_a, + const framework::Tensor &matrix_b, bool trans_b, T alpha, + framework::Tensor *matrix_out, T beta, bool relu, + framework::Tensor *new_scale, framework::Tensor *new_bias); } // namespace math } // namespace operators } // namespace paddle_mobile diff --git a/src/operators/op_param.h b/src/operators/op_param.h index 892b08e6da0ce92df95e81dd9896df3ee8899fb9..4ecc1622f91d5ff63d6abe9434ba0222b10d34e6 100644 --- a/src/operators/op_param.h +++ b/src/operators/op_param.h @@ -715,6 +715,123 @@ class ReshapeParam : public OpParam { }; #endif +#ifdef SCALE_OP +class ScaleParam : public OpParam { + public: + ScaleParam(const VariableNameMap &inputs, const VariableNameMap &outputs, + const AttributeMap &attrs, const Scope &scope) { + input_x_ = InputXFrom(inputs, scope); + input_bias_ = InputBiasFrom(inputs, scope); + out_ = OutFrom(outputs, scope); + inplace_ = GetAttr("inplace", attrs); + has_bias_ = GetAttr("has_bias", attrs); + scales_ = GetAttr>("scales", attrs); + biases_ = GetAttr>("biases", attrs); + } + + const Tensor *InputX() const { return input_x_; } + + const Tensor *InputBias() const { return input_bias_; } + + Tensor *Out() const { return out_; } + + const bool &Inplace() const { return inplace_; } + + const bool &HasBias() const { return has_bias_; } + + const vector &Scales() const { return scales_; } + + const vector &Biases() const { return biases_; } + + private: + Tensor *input_x_; + Tensor *input_bias_; + Tensor *out_; + bool inplace_; + bool has_bias_; + vector scales_; + vector biases_; +}; +#endif + +#ifdef SLICE_OP +class SliceParam : public OpParam { + public: + SliceParam(const VariableNameMap &inputs, const VariableNameMap &outputs, + const AttributeMap &attrs, const Scope &scope) { + input_x_ = InputXFrom(inputs, scope); + input_shape_ = InputShapeFrom(inputs, scope); + out_ = OutFrom(outputs, scope); + axis_ = GetAttr("axis", attrs); + slice_points_ = GetAttr>("slice_points", attrs); + inplace_ = GetAttr("inplace", attrs); + } + + const Tensor *InputX() const { return input_x_; } + + const Tensor *InputShape() const { return input_shape_; } + + Tensor *Out() const { return out_; } + + const int &Axis() const { return axis_; } + + const vector &SlicePoints() const { return slice_points_; } + + const bool &Inplace() const { return inplace_; } + + private: + Tensor *input_x_; + Tensor *input_shape_; + Tensor *out_; + int axis_; + vector slice_points_; + bool inplace_; +}; +#endif + +#ifdef RESIZE_OP +class ResizeParam : public OpParam { + public: + ResizeParam(const VariableNameMap &inputs, const VariableNameMap &outputs, + const AttributeMap &attrs, const Scope &scope) { + input_x_ = InputXFrom(inputs, scope); + input_shape_ = InputShapeFrom(inputs, scope); + out_ = OutFrom(outputs, scope); + is_pyramid_test_ = GetAttr("is_pyramid_test", attrs); + height_ = GetAttr("height", attrs); + width_ = GetAttr("width", attrs); + out_height_scale_ = GetAttr("out_height_scale", attrs); + out_width_scale_ = GetAttr("out_width_scale", attrs); + } + + const Tensor *InputX() const { return input_x_; } + + const Tensor *InputShape() const { return input_shape_; } + + Tensor *Out() const { return out_; } + + const bool &IsPyramidTest() const { return is_pyramid_test_; } + + const int &Height() const { return height_; } + + const int &Width() const { return width_; } + + const float &OutHeightScale() const { return out_height_scale_; } + + const float &OutWidthScale() const { return out_width_scale_; } + + private: + Tensor *input_x_; + Tensor *input_shape_; + Tensor *out_; + bool is_pyramid_test_; + int height_; + int width_; + float out_height_scale_; + float out_width_scale_; +}; +#endif + #ifdef RELU_OP /* * @b op 层实例化好这个 param 传递给 kernel 层使用 @@ -737,6 +854,27 @@ class ReluParam : public OpParam { }; #endif +#ifdef PRELU_OP +class PReluParam : public OpParam { + public: + PReluParam(const VariableNameMap &inputs, const VariableNameMap &outputs, + const AttributeMap &attrs, const Scope &scope) { + input_x_ = InputXFrom(inputs, scope); + out_ = OutFrom(outputs, scope); + slopes_ = GetAttr>("slopes", attrs); + } + + const Tensor *InputX() const { return input_x_; } + Tensor *Out() const { return out_; } + const vector &Slopes() const { return slopes_; } + + private: + Tensor *input_x_; + Tensor *out_; + vector slopes_; +}; +#endif + #ifdef FUSION_FC_OP class FusionFcParam : public OpParam { public: diff --git a/src/operators/prelu_op.cpp b/src/operators/prelu_op.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e78f6b0374336a3d891a1f3e73f63c706b321ccc --- /dev/null +++ b/src/operators/prelu_op.cpp @@ -0,0 +1,47 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#ifdef PRELU_OP + +#include "operators/prelu_op.h" +namespace paddle_mobile { +namespace operators { + +template +void PReluOp::InferShape() const { + auto input_dims = this->param_.InputX()->dims(); + this->param_.Out()->Resize(input_dims); +} +template class PReluOp; +} // namespace operators +} // namespace paddle_mobile + +/* + * @b 每一个 op 都需要注册一下的, + * USE_OP的参数 和 REGISTER_OPERATOR的第一个参数 + * 都是需要和model中类型对应起来的 + * */ +namespace ops = paddle_mobile::operators; +#ifdef PADDLE_MOBILE_CPU +USE_OP_CPU(prelu); +REGISTER_OPERATOR_CPU(prelu, ops::PReluOp); +#endif +#ifdef PADDLE_MOBILE_MALI_GPU +USE_OP_MALI_GPU(prelu); +REGISTER_OPERATOR_MALI_GPU(prelu, ops::PReluOp); +#endif +#ifdef PADDLE_MOBILE_FPGA +#endif + +#endif diff --git a/src/operators/prelu_op.h b/src/operators/prelu_op.h new file mode 100644 index 0000000000000000000000000000000000000000..330d1d4a567b10bddf5879b4897a945350a34ac5 --- /dev/null +++ b/src/operators/prelu_op.h @@ -0,0 +1,53 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#ifdef PRELU_OP + +#pragma once + +#include + +#include "framework/operator.h" +#include "operators/kernel/prelu_kernel.h" +#include "operators/op_param.h" + +namespace paddle_mobile { +namespace operators { + +using paddle_mobile::framework::Tensor; + +template +class PReluOp + : public framework::OperatorWithKernel< + DeviceType, PReluParam, operators::PReluKernel> { + public: + PReluOp(const std::string &type, const VariableNameMap &inputs, + const VariableNameMap &outputs, const framework::AttributeMap &attrs, + std::shared_ptr scope) + : framework::OperatorWithKernel>( + type, inputs, outputs, attrs, scope) {} + + using framework::OperatorWithKernel< + DeviceType, PReluParam, + operators::PReluKernel>::OperatorWithKernel; + void InferShape() const override; + + protected: +}; + +} // namespace operators +} // namespace paddle_mobile + +#endif diff --git a/src/operators/resize_op.cpp b/src/operators/resize_op.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f378ff53f513ccf7cfb986f606378895b5af4b9f --- /dev/null +++ b/src/operators/resize_op.cpp @@ -0,0 +1,43 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#ifdef RESIZE_OP + +#include "operators/resize_op.h" +#include +namespace paddle_mobile { +namespace operators { + +template +void ResizeOp::InferShape() const { + auto out_dims = CalOutputShape(this->param_); + this->param_.Out()->Resize(out_dims); +} +template class ResizeOp; +} // namespace operators +} // namespace paddle_mobile + +namespace ops = paddle_mobile::operators; +#ifdef PADDLE_MOBILE_CPU +USE_OP_CPU(resize); +REGISTER_OPERATOR_CPU(resize, ops::ResizeOp); +#endif +#ifdef PADDLE_MOBILE_MALI_GPU +USE_OP_MALI_GPU(resize); +REGISTER_OPERATOR_MALI_GPU(resize, ops::ResizeOp); +#endif +#ifdef PADDLE_MOBILE_FPGA +#endif + +#endif diff --git a/src/operators/resize_op.h b/src/operators/resize_op.h new file mode 100644 index 0000000000000000000000000000000000000000..9e7fd6b8029aebfdf4b7c53439936189b0c8eb8e --- /dev/null +++ b/src/operators/resize_op.h @@ -0,0 +1,52 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#ifdef RESIZE_OP + +#pragma once + +#include + +#include "framework/operator.h" +#include "operators/kernel/resize_kernel.h" +#include "operators/op_param.h" + +namespace paddle_mobile { +namespace operators { + +using paddle_mobile::framework::Tensor; + +template +class ResizeOp + : public framework::OperatorWithKernel< + DeviceType, ResizeParam, operators::ResizeKernel> { + public: + ResizeOp(const std::string &type, const VariableNameMap &inputs, + const VariableNameMap &outputs, const framework::AttributeMap attrs, + std::shared_ptr scope) + : framework::OperatorWithKernel>( + type, inputs, outputs, attrs, scope) {} + + using framework::OperatorWithKernel< + DeviceType, ResizeParam, + operators::ResizeKernel>::OperatorWithKernel; + void InferShape() const override; + + protected: +}; +} // namespace operators +} // namespace paddle_mobile + +#endif diff --git a/src/operators/scale_op.cpp b/src/operators/scale_op.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c1931ed4fdc4c058c979fdceba11ea25f7d752f4 --- /dev/null +++ b/src/operators/scale_op.cpp @@ -0,0 +1,43 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#ifdef SCALE_OP + +#include "operators/scale_op.h" +#include +namespace paddle_mobile { +namespace operators { + +template +void ScaleOp::InferShape() const { + auto input_dims = this->param_.InputX()->dims(); + this->param_.Out()->Resize(input_dims); +} +template class ScaleOp; +} // namespace operators +} // namespace paddle_mobile + +namespace ops = paddle_mobile::operators; +#ifdef PADDLE_MOBILE_CPU +USE_OP_CPU(scale); +REGISTER_OPERATOR_CPU(scale, ops::ScaleOp); +#endif +#ifdef PADDLE_MOBILE_MALI_GPU +USE_OP_MALI_GPU(scale); +REGISTER_OPERATOR_MALI_GPU(scale, ops::ScaleOp); +#endif +#ifdef PADDLE_MOBILE_FPGA +#endif + +#endif diff --git a/src/operators/scale_op.h b/src/operators/scale_op.h new file mode 100644 index 0000000000000000000000000000000000000000..e6a900d646a53cb8aa2a0ab624a31781f420db06 --- /dev/null +++ b/src/operators/scale_op.h @@ -0,0 +1,53 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#ifdef SCALE_OP + +#pragma once + +#include + +#include "framework/operator.h" +#include "operators/kernel/scale_kernel.h" +#include "operators/op_param.h" + +namespace paddle_mobile { +namespace operators { + +using paddle_mobile::framework::Tensor; + +template +class ScaleOp + : public framework::OperatorWithKernel< + DeviceType, ScaleParam, operators::ScaleKernel> { + public: + ScaleOp(const std::string &type, const VariableNameMap &inputs, + const VariableNameMap &outputs, const framework::AttributeMap &attrs, + std::shared_ptr scope) + : framework::OperatorWithKernel>( + type, inputs, outputs, attrs, scope) {} + + using framework::OperatorWithKernel< + DeviceType, ScaleParam, + operators::ScaleKernel>::OperatorWithKernel; + void InferShape() const override; + + protected: +}; + +} // namespace operators +} // namespace paddle_mobile + +#endif diff --git a/src/operators/slice_op.cpp b/src/operators/slice_op.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6d70895fcc5edf75f73368813212f7d9177c760b --- /dev/null +++ b/src/operators/slice_op.cpp @@ -0,0 +1,42 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#ifdef SLICE_OP + +#include "operators/slice_op.h" +#include +namespace paddle_mobile { +namespace operators { + +template +void SliceOp::InferShape() const { + /// todo: add InputShape() detection. +} +template class SliceOp; +} // namespace operators +} // namespace paddle_mobile + +namespace ops = paddle_mobile::operators; +#ifdef PADDLE_MOBILE_CPU +USE_OP_CPU(slice); +REGISTER_OPERATOR_CPU(slice, ops::SliceOp); +#endif +#ifdef PADDLE_MOBILE_MALI_GPU +USE_OP_MALI_GPU(slice); +REGISTER_OPERATOR_MALI_GPU(slice, ops::SliceOp); +#endif +#ifdef PADDLE_MOBILE_FPGA +#endif + +#endif diff --git a/src/operators/slice_op.h b/src/operators/slice_op.h new file mode 100644 index 0000000000000000000000000000000000000000..4c9d074b24c1aaf1bf28c862f3731ca130f3c462 --- /dev/null +++ b/src/operators/slice_op.h @@ -0,0 +1,53 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#ifdef SLICE_OP + +#pragma once + +#include + +#include "framework/operator.h" +#include "operators/kernel/slice_kernel.h" +#include "operators/op_param.h" + +namespace paddle_mobile { +namespace operators { + +using paddle_mobile::framework::Tensor; + +template +class SliceOp + : public framework::OperatorWithKernel< + DeviceType, SliceParam, operators::SliceKernel> { + public: + SliceOp(const std::string &type, const VariableNameMap &inputs, + const VariableNameMap &outputs, const framework::AttributeMap &attrs, + std::shared_ptr scope) + : framework::OperatorWithKernel>( + type, inputs, outputs, attrs, scope) {} + + using framework::OperatorWithKernel< + DeviceType, SliceParam, + operators::SliceKernel>::OperatorWithKernel; + void InferShape() const override; + + protected: +}; + +} // namespace operators +} // namespace paddle_mobile + +#endif diff --git a/test/operators/test_prelu_op.cpp b/test/operators/test_prelu_op.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e93d8732d18496721b24cfba1df296250169f8b2 --- /dev/null +++ b/test/operators/test_prelu_op.cpp @@ -0,0 +1,58 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "../executor_for_test.h" +#include "../test_include.h" +#include "operators/prelu_op.h" + +int main() { + paddle_mobile::Loader loader; + auto program = loader.Load(g_resnet); + PADDLE_MOBILE_ENFORCE(program.originProgram != nullptr, + "program file read fail"); + + Executor4Test> + executor(program, "prelu"); + + // 1. input_tensors; + vector input_tensors; + + Tensor input1; + auto input1_data = CreateInput(&input1, {1, 2, 3, 4}, -1, 1); + input_tensors.push_back(input1); + + // 2. input_names + vector input_names({ + "batch_norm_0.tmp_2", + }); + + // 3. output_names + vector output_names({"batch_norm_0.tmp_3"}); + + // 4. out_dims; + vector out_ddims; + auto out_ddim = paddle_mobile::framework::make_ddim({1, 2, 3, 4}); + out_ddims.push_back(out_ddim); + + auto output = executor.Predict(input_tensors, input_names, + output_names, out_ddims); + + auto output0_data = output[0]->data(); + + for (int j = 0; j < output[0]->numel(); ++j) { + DLOG << " value of output: " << output0_data[j]; + } + return 0; +} diff --git a/test/operators/test_resize_op.cpp b/test/operators/test_resize_op.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f4dcaa6885d92a727e8c97d5106c3b6913a4ab33 --- /dev/null +++ b/test/operators/test_resize_op.cpp @@ -0,0 +1,47 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "../test_include.h" +#include "operators/resize_op.h" + +int main() { + paddle_mobile::Loader loader; + auto program = loader.Load(std::string(g_mobilenet_ssd)); + if (program.originProgram == nullptr) { + DLOG << "program read file"; + } + Executor4Test> + executor(program, "resize"); + paddle_mobile::framework::Tensor input; + SetupTensor(&input, {2, 3, 3, 2}, static_cast(0), + static_cast(1)); + auto input_ptr = input.data(); + auto out_ddim = paddle_mobile::framework::make_ddim({2, 9, 2}); + auto output = + executor.Predict(input, "transpose_0.tmp_0", "reshape_0.tmp_0", out_ddim); + auto *output_ptr = output->data(); + + DLOG << "input : "; + for (int j = 0; j < input.numel(); ++j) { + DLOG << " index " << j << " : " << input_ptr[j]; + } + + DLOG << "output : "; + for (int j = 0; j < output->numel(); ++j) { + DLOG << " index " << j << " : " << output_ptr[j]; + } + + return 0; +} diff --git a/test/operators/test_scale_op.cpp b/test/operators/test_scale_op.cpp new file mode 100644 index 0000000000000000000000000000000000000000..574779d71e5ebc5f06fe5cd8fb33422726f39464 --- /dev/null +++ b/test/operators/test_scale_op.cpp @@ -0,0 +1,18 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "../test_include.h" +#include "operators/scale_op.h" + +int main() {} diff --git a/test/operators/test_slice_op.cpp b/test/operators/test_slice_op.cpp new file mode 100644 index 0000000000000000000000000000000000000000..9306bc53c6ae23b10c27a71071c11c9ddf1c0d25 --- /dev/null +++ b/test/operators/test_slice_op.cpp @@ -0,0 +1,18 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "../test_include.h" +#include "operators/slice_op.h" + +int main() {} diff --git a/tools/build.sh b/tools/build.sh index 0b891e56a44e06a53f01b792304d33a49c760f68..e1a47e64e6ea87dd9d70969214412fc0c2fa2d67 100755 --- a/tools/build.sh +++ b/tools/build.sh @@ -119,6 +119,7 @@ build_for_ios() { fi cd "${BUILD_DIR}" make -j 8 + cp ../../../src/ios_io/PaddleMobile.h ./build/PaddleMobile.h cd ./build # 生成符号表 ranlib *.a @@ -160,4 +161,4 @@ else build_error fi fi -fi +fi \ No newline at end of file diff --git a/tools/op.cmake b/tools/op.cmake index b413cb346261f03b955fa4b32aff371f6a4a32e0..71defeffcc919848e165ea836f4bfed2fcc7e0ff 100644 --- a/tools/op.cmake +++ b/tools/op.cmake @@ -64,6 +64,10 @@ else () set(TRANSPOSE_OP ON) set(FUSION_CONVADD_RELU_OP ON) set(FUSION_CONVADDBNRELU_OP ON) + set(PRELU_OP ON) + set(RESIZE_OP ON) + set(SCALE_OP ON) + set(SLICE_OP ON) set(DROPOUT_OP ON) set(IM2SEQUENCE_OP ON) # option(BATCHNORM_OP "" ON) @@ -151,6 +155,18 @@ endif() if (FUSION_CONVADDBNRELU_OP) add_definitions(-DFUSION_CONVADDBNRELU_OP) endif() +if (PRELU_OP) + add_definitions(-DPRELU_OP) +endif() +if (RESIZE_OP) + add_definitions(-DRESIZE_OP) +endif() +if (SCALE_OP) + add_definitions(-DSCALE_OP) +endif() +if (SLICE_OP) + add_definitions(-DSLICE_OP) +endif() if (DROPOUT_OP) add_definitions(-DDROPOUT_OP) endif()