From 8b5431d5e1e4f54339594ba6a76ecbccf256080f Mon Sep 17 00:00:00 2001 From: dangqingqing Date: Fri, 6 Jan 2017 19:19:56 +0800 Subject: [PATCH] padding operation --- paddle/function/CMakeLists.txt | 1 + paddle/function/PadOp.cpp | 185 + paddle/function/PadOp.h | 96 + paddle/function/PadOpGpu.cu | 102 + paddle/function/PadOpTest.cpp | 70 + paddle/gserver/layers/PadLayer.cpp | 115 + paddle/gserver/layers/PadLayer.h | 45 + paddle/gserver/tests/test_LayerGrad.cpp | 3090 +++++++++-------- proto/ModelConfig.proto | 8 + python/paddle/trainer/config_parser.py | 26 + .../paddle/trainer_config_helpers/layers.py | 84 +- .../tests/configs/test_pad.py | 21 + 12 files changed, 2317 insertions(+), 1526 deletions(-) create mode 100644 paddle/function/PadOp.cpp create mode 100644 paddle/function/PadOp.h create mode 100644 paddle/function/PadOpGpu.cu create mode 100644 paddle/function/PadOpTest.cpp create mode 100644 paddle/gserver/layers/PadLayer.cpp create mode 100644 paddle/gserver/layers/PadLayer.h create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_pad.py diff --git a/paddle/function/CMakeLists.txt b/paddle/function/CMakeLists.txt index 0b3126155d..70b25406a4 100644 --- a/paddle/function/CMakeLists.txt +++ b/paddle/function/CMakeLists.txt @@ -17,6 +17,7 @@ if(WITH_TESTING) # file(GLOB test_files . *OpTest.cpp) # add_executable(${test_bin} EXCLUDE_FROM_ALL ${test_files}) add_simple_unittest(CrossMapNormalOpTest) + add_simple_unittest(PadOpTest) add_unittest(ContextProjectionOpTest ContextProjectionOpTest.cpp ../gserver/tests/TestUtil.cpp) diff --git a/paddle/function/PadOp.cpp b/paddle/function/PadOp.cpp new file mode 100644 index 0000000000..e10011da2a --- /dev/null +++ b/paddle/function/PadOp.cpp @@ -0,0 +1,185 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "PadOp.h" +#include "paddle/math/Vector.h" + +namespace paddle { + +template <> +void Pad(real* outputs, + const real* inputs, + const int num, + const int inC, + const int inH, + const int inW, + const int padc0, + const int padc1, + const int padh0, + const int padh1, + const int padw0, + const int padw1) { + int outC = inC + padc0 + padc1; + int outH = inH + padh0 + padh1; + int outW = inW + padw0 + padw1; + for (int i = 0; i < num; i++) { + for (int c = 0; c < inC; c++) { + for (int h = 0; h < inH; h++) { + int inoff = ((i * inC + c) * inH + h) * inW; + int outoff = ((i * outC + c + padc0) * outH + h + padh0) * outW + padw0; + memcpy(outputs + outoff, inputs + inoff, inW * sizeof(real)); + } + } + } +} + +template <> +void PadGrad(real* inGrad, + const real* outGrad, + const int num, + const int inC, + const int inH, + const int inW, + const int padc0, + const int padc1, + const int padh0, + const int padh1, + const int padw0, + const int padw1) { + int outC = inC + padc0 + padc1; + int outH = inH + padh0 + padh1; + int outW = inW + padw0 + padw1; + for (int i = 0; i < num; i++) { + for (int c = 0; c < inC; c++) { + for (int h = 0; h < inH; h++) { + int inoff = ((i * inC + c) * inH + h) * inW; + int outoff = ((i * outC + c + padc0) * outH + h + padh0) * outW + padw0; + CpuVector inG = CpuVector(inW, inGrad + inoff); + CpuVector outG = CpuVector(inW, const_cast(outGrad + outoff)); + inG += outG; + } + } + } +} + +/** + * \param inputs[0] input value. + * \param outputs[0] output value. + */ +template +class PadFunc : public FunctionBase { +public: + void init(const FuncConfig& config) override { + padc0_ = config.get("padc0"); + padc1_ = config.get("padc1"); + padh0_ = config.get("padh0"); + padh1_ = config.get("padh1"); + padw0_ = config.get("padw0"); + padw1_ = config.get("padw1"); + } + + void calc(const Arguments& inputs, + const Arguments& outputs, + const Arguments& inouts) override { + CHECK_EQ(1, inputs.size()); + CHECK_EQ(1, outputs.size()); + CHECK_EQ(0, inouts.size()); + + size_t num = inputs[0].dims_[0]; + size_t inC = inputs[0].dims_[1]; + size_t inH = inputs[0].dims_[2]; + size_t inW = inputs[0].dims_[3]; + + Pad(outputs[0].getData(), + inputs[0].getData(), + num, + inC, + inH, + inW, + padc0_, + padc1_, + padh0_, + padh1_, + padw0_, + padw1_); + } + +private: + int padc0_; + int padc1_; + int padh0_; + int padh1_; + int padw0_; + int padw1_; +}; + +/** + * \param inputs[0] input grad. + * \param outputs[0] output grad. + */ +template +class PadGradFunc : public FunctionBase { +public: + void init(const FuncConfig& config) override { + padc0_ = config.get("padc0"); + padc1_ = config.get("padc1"); + padh0_ = config.get("padh0"); + padh1_ = config.get("padh1"); + padw0_ = config.get("padw0"); + padw1_ = config.get("padw1"); + } + + void calc(const Arguments& inputs, + const Arguments& outputs, + const Arguments& inouts) override { + CHECK_EQ(1, inputs.size()); + CHECK_EQ(0, outputs.size()); + CHECK_EQ(1, inouts.size()); + + size_t n = inouts[0].dims_[0]; + size_t inC = inouts[0].dims_[1]; + size_t inH = inouts[0].dims_[2]; + size_t inW = inouts[0].dims_[3]; + + PadGrad(inouts[0].getData(), + inputs[0].getData(), + n, + inC, + inH, + inW, + padc0_, + padc1_, + padh0_, + padh1_, + padw0_, + padw1_); + } + +private: + int padc0_; + int padc1_; + int padh0_; + int padh1_; + int padw0_; + int padw1_; +}; + +REGISTER_TYPED_FUNC(Pad, CPU, PadFunc); +REGISTER_TYPED_FUNC(PadGrad, CPU, PadGradFunc); +#ifndef PADDLE_ONLY_CPU +REGISTER_TYPED_FUNC(Pad, GPU, PadFunc); +REGISTER_TYPED_FUNC(PadGrad, GPU, PadGradFunc); +#endif + +} // namespace paddle diff --git a/paddle/function/PadOp.h b/paddle/function/PadOp.h new file mode 100644 index 0000000000..4a5e8fe338 --- /dev/null +++ b/paddle/function/PadOp.h @@ -0,0 +1,96 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "Function.h" + +namespace paddle { + +/** + * \brief This funtion pads zeros to inputs according to the specify dimension. + * The data structure of image data is NCHW. + * + * \param[out] outputs save results. + * \param[in] inputs input data. + * \param[in] num batch size of input data. + * \param[in] inC channel number of input data. + * \param[in] inH height of input data. + * \param[in] inH with of input data. + * \param[in] padc0 how many values to add before the data in dimension of + * channel. + * \param[in] padc1 how many values to add after the data in dimension of + * channel. + * \param[in] padh0 how many values to add before the data in dimension of + * height. + * \param[in] padh1 how many values to add after the data in dimension of + * height. + * \param[in] padw0 how many values to add before the data in dimension of + * width. + * \param[in] padw1 how many values to add after the data in dimension of + * width. + * + */ +template +void Pad(real* outputs, + const real* inputs, + const int num, + const int inC, + const int inH, + const int inW, + const int padc0, + const int padc1, + const int padh0, + const int padh1, + const int padw0, + const int padw1); + +/** + * \brief Padding operation backward. + * The data structure of image data is NCHW. + * + * \param[out] inGrad gradients of previous layer. + * \param[in] outGrad output gradients. + * \param[in] num batch size of input data. + * \param[in] inC channel number of input data. + * \param[in] inH height of input data. + * \param[in] inH with of input data. + * \param[in] padc0 how many values to add before the data in dimension of + * channel. + * \param[in] padc1 how many values to add after the data in dimension of + * channel. + * \param[in] padh0 how many values to add before the data in dimension of + * height. + * \param[in] padh1 how many values to add after the data in dimension of + * height. + * \param[in] padw0 how many values to add before the data in dimension of + * width. + * \param[in] padw1 how many values to add after the data in dimension of + * width. + * + */ +template +void PadGrad(real* inGrad, + const real* outGrad, + const int num, + const int inC, + const int inH, + const int inW, + const int padc0, + const int padc1, + const int padh0, + const int padh1, + const int padw0, + const int padw1); +} // namespace paddle diff --git a/paddle/function/PadOpGpu.cu b/paddle/function/PadOpGpu.cu new file mode 100644 index 0000000000..578d6e86d7 --- /dev/null +++ b/paddle/function/PadOpGpu.cu @@ -0,0 +1,102 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "hl_base.h" +#include "PadOp.h" + +namespace paddle { + +__global__ void KePad(real* outputs, const real* inputs, + int inC, int inH, int inW, + int padc, int padh, int padw, + int outC, int outH, int outW, int nthreads) { + const int idx = threadIdx.x + blockIdx.x * blockDim.x; + if (idx < nthreads) { + const int w = idx % inW; + const int h = (idx / inW) % inH; + const int c = (idx / inW / inH) % inC; + const int n = idx / inW / inH / inC; + + const int off = ((n * outC + c + padc) * outH + h + padh) * outW + padw + w; + outputs[off] = inputs[idx]; + } +} + +template <> +void Pad(real* outputs, + const real* inputs, + const int num, + const int inC, + const int inH, + const int inW, + const int padc0, + const int padc1, + const int padh0, + const int padh1, + const int padw0, + const int padw1) { + size_t nth = num * inC * inH * inW; + int blockSize = 1024; + int gridSize = (nth + 1024 - 1) / 1024; + int outC = inC + padc0 + padc1; + int outH = inH + padh0 + padh1; + int outW = inW + padw0 + padw1; + KePad<<>> + (outputs, inputs, inC, inH, inW, padc0, padh0, padw0, + outC, outH, outW, nth); + CHECK_SYNC("Pad"); +} + +__global__ void KePadDiff(real* inGrad, const real* outGrad, + int inC, int inH, int inW, + int padc, int padh, int padw, + int outC, int outH, int outW, int nthreads) { + const int idx = threadIdx.x + blockIdx.x * blockDim.x; + if (idx < nthreads) { + const int w = idx % inW; + const int h = (idx / inW) % inH; + const int c = (idx / inW / inH) % inC; + const int n = idx / inW / inH / inC; + + const int off = ((n * outC + c + padc) * outH + h + padh) * outW + padw + w; + inGrad[idx] += outGrad[off]; + } +} + +template <> +void PadGrad(real* inGrad, + const real* outGrad, + const int num, + const int inC, + const int inH, + const int inW, + const int padc0, + const int padc1, + const int padh0, + const int padh1, + const int padw0, + const int padw1) { + int nth = num * inC * inH * inW; + int blockSize = 1024; + int gridSize = (nth + 1024 - 1) / 1024; + int outC = inC + padc0 + padc1; + int outH = inH + padh0 + padh1; + int outW = inW + padw0 + padw1; + KePadDiff <<>> + (inGrad, outGrad, inC, inH, inW, padc0, padh0, padw0, + outC, outH, outW, nth); + CHECK_SYNC("PadGrad"); +} + +} // namespace paddle diff --git a/paddle/function/PadOpTest.cpp b/paddle/function/PadOpTest.cpp new file mode 100644 index 0000000000..ee2834d793 --- /dev/null +++ b/paddle/function/PadOpTest.cpp @@ -0,0 +1,70 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include +#include "FunctionTest.h" + +namespace paddle { + +TEST(Pad, real) { + for (size_t numSamples : {5, 32}) { + for (size_t channels : {1, 5, 32}) { + for (size_t imgSizeH : {5, 33, 100}) { + for (size_t imgSizeW : {5, 32, 96}) { + VLOG(3) << " numSamples=" << numSamples << " channels=" << channels + << " imgSizeH=" << imgSizeH << " imgSizeW=" << imgSizeW; + + FunctionCompare compare("Pad", + FuncConfig() + .set("padc0", 2) + .set("padc1", 3) + .set("padh0", 1) + .set("padh1", 2) + .set("padw0", 3) + .set("padw1", 2)); + Dims inDims{numSamples, channels, imgSizeH, imgSizeW}; + Dims outDims{numSamples, channels + 5, imgSizeH + 3, imgSizeW + 5}; + compare.cmpWithArg( + {Tensor(nullptr, inDims)}, {Tensor(nullptr, outDims)}, {}); + } + } + } + } +} + +// TEST(PadGrad, real) { +// for (size_t numSamples : {5, 32}) { +// for (size_t channels : {1, 5, 32}) { +// for (size_t imgSizeH : {5, 33, 100}) { +// for (size_t imgSizeW : {5, 32, 96}) { +// VLOG(3) << " numSamples=" << numSamples << " channels=" << channels +// << " imgSizeH=" << imgSizeH << " imgSizeW=" << imgSizeW; +// +// FunctionCompare compare("PadGrad", +// FuncConfig() +// .set("padc0", 2).set("padc1", 3) +// .set("padh0", 1).set("padh1", 2) +// .set("padw0", 3).set("padw1", 2)); +// Dims inDims{numSamples, channels, imgSizeH, imgSizeW}; +// Dims outDims{numSamples, channels + 5, imgSizeH + 3, imgSizeW + 5}; +// compare.cmpWithArg({Tensor(nullptr, inDims)}, +// {Tensor(nullptr, outDims)}, +// {}); +// } +// } +// } +// } +//} + +} // namespace paddle diff --git a/paddle/gserver/layers/PadLayer.cpp b/paddle/gserver/layers/PadLayer.cpp new file mode 100644 index 0000000000..62c50af32d --- /dev/null +++ b/paddle/gserver/layers/PadLayer.cpp @@ -0,0 +1,115 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "PadLayer.h" +#include "paddle/utils/Stat.h" + +namespace paddle { + +REGISTER_LAYER(pad, PadLayer); + +bool PadLayer::init(const LayerMap& layerMap, + const ParameterMap& parameterMap) { + /* Initialize the basic parent class */ + Layer::init(layerMap, parameterMap); + + auto& pad_conf = config_.inputs(0).pad_conf(); + auto& img_conf = pad_conf.image_conf(); + CHECK_EQ(config_.inputs_size(), 1); + inDims_.push_back(0); + inDims_.push_back(img_conf.channels()); + inDims_.push_back(img_conf.has_img_size_y() ? img_conf.img_size_y() + : img_conf.img_size()); + inDims_.push_back(img_conf.img_size()); + + CHECK_EQ(2UL, pad_conf.pad_c_size()); + CHECK_EQ(2UL, pad_conf.pad_h_size()); + CHECK_EQ(2UL, pad_conf.pad_w_size()); + padc_.push_back(pad_conf.pad_c(0)); + padc_.push_back(pad_conf.pad_c(1)); + padh_.push_back(pad_conf.pad_h(0)); + padh_.push_back(pad_conf.pad_h(1)); + padw_.push_back(pad_conf.pad_w(0)); + padw_.push_back(pad_conf.pad_w(1)); + + outDims_.resize(4); + setOutDims(0); + + createFunction(forward_, + "Pad", + FuncConfig() + .set("padc0", padc_[0]) + .set("padc1", padc_[1]) + .set("padh0", padh_[0]) + .set("padh1", padh_[1]) + .set("padw0", padw_[0]) + .set("padw1", padw_[1])); + createFunction(backward_, + "PadGrad", + FuncConfig() + .set("padc0", padc_[0]) + .set("padc1", padc_[1]) + .set("padh0", padh_[0]) + .set("padh1", padh_[1]) + .set("padw0", padw_[0]) + .set("padw1", padw_[1])); + + return true; +} + +void PadLayer::setOutDims(int batchSize) { + outDims_[0] = batchSize; + outDims_[1] = inDims_[1] + padc_[0] + padc_[1]; + outDims_[2] = inDims_[2] + padh_[0] + padh_[1]; + outDims_[3] = inDims_[3] + padw_[0] + padw_[1]; +} + +void PadLayer::setTensorDim(int batchSize) { + CHECK_EQ(inputLayers_.size(), 1UL); + inDims_[0] = batchSize; + int h = inputLayers_[0]->getOutput().getFrameHeight(); + if (h != 0) inDims_[2]; + int w = inputLayers_[0]->getOutput().getFrameWidth(); + if (w != 0) inDims_[3]; + setOutDims(batchSize); +} + +void PadLayer::forward(PassType passType) { + Layer::forward(passType); + MatrixPtr input = inputLayers_[0]->getOutputValue(); + size_t batchSize = input->getHeight(); + setTensorDim(batchSize); + int size = outDims_[1] * outDims_[2] * outDims_[3]; + resetOutput(batchSize, size); + MatrixPtr outV = getOutputValue(); + REGISTER_TIMER_INFO("PadForward", getName().c_str()); + forward_[0]->calc({Tensor(input->getData(), inDims_)}, + {Tensor(outV->getData(), outDims_)}, + {}); +} + +void PadLayer::backward(const UpdateCallback& callback) { + (void)callback; + + MatrixPtr preGrad = inputLayers_[0]->getOutputGrad(); + if (NULL == preGrad) { + return; + } + MatrixPtr outGrad = getOutputGrad(); + REGISTER_TIMER_INFO("PadBackward", getName().c_str()); + backward_[0]->calc({Tensor(outGrad->getData(), outDims_)}, + {}, + {Tensor(preGrad->getData(), inDims_)}); +} +} // namespace paddle diff --git a/paddle/gserver/layers/PadLayer.h b/paddle/gserver/layers/PadLayer.h new file mode 100644 index 0000000000..834622a7af --- /dev/null +++ b/paddle/gserver/layers/PadLayer.h @@ -0,0 +1,45 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "Layer.h" + +namespace paddle { + +/** + * @brief response normalization across feature maps + * namely normalize in number of size_ channels + */ +class PadLayer : public Layer { +public: + explicit PadLayer(const LayerConfig& config) : Layer(config) {} + + ~PadLayer() {} + + bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); + void forward(PassType passType); + void backward(const UpdateCallback& callback = nullptr); + +protected: + void setOutDims(int batchSize); + void setTensorDim(int batchSize); + + std::vector padc_; + std::vector padh_; + std::vector padw_; + Dims inDims_; + Dims outDims_; +}; +} // namespace paddle diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index 2cc25f6b21..3094b3a4a0 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -32,1534 +32,1580 @@ DECLARE_double(checkgrad_eps); DECLARE_bool(thread_local_rand_use_global_seed); DECLARE_bool(prev_batch_state); -TEST(Operator, dot_mul) { +// TEST(Operator, dot_mul) { +// TestConfig config; +// config.layerConfig.set_size(10); +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 10, 0}); +// config.inputDefs.push_back({INPUT_DATA, "layer_1", 10, 0}); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// +// OperatorConfig& operatorConf = *config.layerConfig.add_operator_confs(); +// operatorConf.set_type("dot_mul"); +// operatorConf.set_dotmul_scale(-1); +// +// testOperatorGrad(config, operatorConf, 100, false, false); +// } +// +// TEST(Projection, context) { +// for (auto contextStart : {-5, -3, -1, 0, 3}) { +// for (auto contextLength : {1, 2, 5, 7}) { +// for (auto batchSize : {1, 2, 5, 20, 50}) { +// for (auto trainablePadding : {false, true}) { +// LOG(INFO) << " contextStart=" << contextStart +// << " contextLength=" << contextLength +// << " batchSize=" << batchSize +// << " trainablePadding=" << trainablePadding; +// ProjectionConfig conf; +// conf.set_type("context"); +// conf.set_input_size(10); +// conf.set_context_start(contextStart); +// conf.set_context_length(contextLength); +// conf.set_trainable_padding(trainablePadding); +// conf.set_output_size(conf.context_length() * conf.input_size()); +// int pad = +// std::max(0, -conf.context_start()) + +// std::max(0, conf.context_start() + conf.context_length() - 1); +// for (auto useGpu : {false, true}) { +// testProjectionGrad( +// conf, +// INPUT_SEQUENCE_DATA, +// trainablePadding ? conf.input_size() * pad : 0, +// batchSize, +// useGpu, +// contextStart + contextLength <= 1); // = testState +// } +// } +// } +// } +// } +// } +// +// TEST(Projection, trans_fc) { +// ProjectionConfig conf; +// conf.set_type("trans_fc"); +// conf.set_input_size(50); +// conf.set_output_size(20); +// for (auto useGpu : {false, true}) { +// testProjectionGrad(conf, +// INPUT_DATA, +// /* parameterSize */ 1000, +// /* batchSize */ 100, +// useGpu); +// } +// } +// +// TEST(Projection, fc) { +// ProjectionConfig conf; +// conf.set_type("fc"); +// conf.set_input_size(10); +// conf.set_output_size(20); +// for (auto useGpu : {false, true}) { +// testProjectionGrad(conf, +// INPUT_DATA, +// /* parameterSize */ 200, +// /* batchSize */ 100, +// useGpu); +// } +// } +// +// TEST(Projection, dot_mul) { +// ProjectionConfig conf; +// conf.set_type("dot_mul"); +// conf.set_input_size(20); +// conf.set_output_size(20); +// for (auto useGpu : {false, true}) { +// testProjectionGrad(conf, +// INPUT_DATA, +// /* parameterSize */ 20, +// /* batchSize */ 100, +// useGpu); +// } +// } +// +// TEST(Projection, table) { +// ProjectionConfig conf; +// conf.set_type("table"); +// conf.set_input_size(10); +// conf.set_output_size(20); +// for (auto useGpu : {false, true}) { +// testProjectionGrad(conf, +// INPUT_LABEL, +// /* parameterSize */ 200, +// /* batchSize */ 100, +// useGpu); +// } +// } +// +// TEST(Projection, identity) { +// ProjectionConfig conf; +// conf.set_type("identity"); +// conf.set_input_size(10); +// conf.set_output_size(10); +// for (auto useGpu : {false, true}) { +// testProjectionGrad(conf, +// INPUT_DATA, +// /* parameterSize */ 0, +// /* batchSize */ 100, +// useGpu); +// } +// } +// +// TEST(Projection, scaling) { +// ProjectionConfig conf; +// conf.set_type("scaling"); +// conf.set_input_size(10); +// conf.set_output_size(10); +// for (auto useGpu : {false}) { +// testProjectionGrad(conf, +// INPUT_DATA, +// /* parameterSize */ 1, +// /* batchSize */ 100, +// useGpu); +// } +// } +// +// void testProjectionConv(size_t groups) { +// const int NUM_FILTERS = 18; +// const int FILTER_SIZE = 2; +// const int FILTER_SIZE_Y = 3; +// const int CHANNELS = 3; +// const int IMAGE_SIZE = 16; +// +// ProjectionConfig conf; +// conf.set_type("conv"); +// conf.set_num_filters(NUM_FILTERS); +// +// ConvConfig* conv = conf.mutable_conv_conf(); +// conv->set_filter_size(FILTER_SIZE); +// conv->set_filter_size_y(FILTER_SIZE_Y); +// conv->set_channels(CHANNELS); +// conv->set_padding(0); +// conv->set_padding_y(1); +// conv->set_stride(2); +// conv->set_stride_y(2); +// conv->set_groups(groups); +// conv->set_filter_channels(conv->channels() / conv->groups()); +// conv->set_img_size(IMAGE_SIZE); +// int output_x = outputSize(conv->img_size(), +// conv->filter_size(), +// conv->padding(), +// conv->stride(), +// /* caffeMode */ true); +// int output_y = outputSize(conv->img_size(), +// conv->filter_size_y(), +// conv->padding_y(), +// conv->stride_y(), +// /* caffeMode */ true); +// conv->set_output_x(output_x); +// conf.set_input_size(IMAGE_SIZE * IMAGE_SIZE * CHANNELS); +// conf.set_output_size(output_x * output_y * NUM_FILTERS); +// +// testProjectionGrad(conf, +// INPUT_DATA, +// /* parameterSize */ NUM_FILTERS * CHANNELS * FILTER_SIZE +// * +// FILTER_SIZE_Y / groups, +// /* batchSize */ 100, +// true, +// false, +// NUM_FILTERS, +// true); +// } +// +// #ifndef PADDLE_ONLY_CPU +// TEST(Projection, conv) { +// testProjectionConv(1); +// testProjectionConv(3); +// } +// #endif +// +// TEST(Layer, BilinearInterpLayer) { +// TestConfig config; +// config.layerConfig.set_type("bilinear_interp"); +// config.biasSize = 0; +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 4096, 0}); +// +// LayerInputConfig* input = config.layerConfig.add_inputs(); +// BilinearInterpConfig* bilinear = input->mutable_bilinear_interp_conf(); +// ImageConfig* image = bilinear->mutable_image_conf(); +// image->set_img_size(32); +// image->set_img_size_y(32); +// image->set_channels(4); +// +// for (auto useGpu : {false, true}) { +// for (auto outSize : {32, 64}) { +// bilinear->set_out_size_x(outSize); +// bilinear->set_out_size_y(outSize); +// testLayerGrad(config, "bilinear_interp", 10, false, useGpu); +// } +// } +// } +// +// TEST(Layer, concat) { +// TestConfig config; +// config.biasSize = 0; +// config.layerConfig.set_type("concat"); +// config.layerConfig.set_size(15); +// config.layerConfig.set_active_type("sigmoid"); +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 5, 0}); +// config.layerConfig.add_inputs(); +// config.inputDefs.push_back({INPUT_DATA, "layer_1", 10, 0}); +// config.layerConfig.add_inputs(); +// +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, "concat", 100, false, useGpu); +// } +// } +// +// TEST(Layer, AddtoLayer) { +// TestConfig config; +// config.biasSize = 0; +// config.layerConfig.set_type("addto"); +// config.layerConfig.set_size(10); +// config.layerConfig.set_active_type("sigmoid"); +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 10, 0}); +// config.layerConfig.add_inputs(); +// config.inputDefs.push_back({INPUT_DATA, "layer_1", 10, 0}); +// config.layerConfig.add_inputs(); +// +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, "addto", 100, false, useGpu); +// } +// } +// +// TEST(Layer, CRFLayer) { +// TestConfig config; +// config.layerConfig.set_type("crf"); +// config.layerConfig.set_size(10); +// config.biasSize = 0; +// +// config.inputDefs.push_back({INPUT_SEQUENCE_DATA, "layer_0", 10, 120}); +// config.inputDefs.push_back({INPUT_SEQUENCE_LABEL, "layer_1", 10, 0}); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// +// // Not support GPU now +// testLayerGrad(config, +// "crf", +// 100, +// /* trans */ false, +// /* useGpu */ false, +// false /*useWeight*/, +// 0.03 /*epsilon*/); +// } +// +// TEST(Layer, CTCLayer) { +// TestConfig config; +// config.layerConfig.set_type("ctc"); +// config.layerConfig.set_norm_by_times(false); +// config.layerConfig.set_size(10); +// config.biasSize = 0; +// +// config.inputDefs.push_back({INPUT_SEQUENCE_DATA, "layer_0", 10, 0}); +// config.inputDefs.push_back({INPUT_SEQUENCE_LABEL, "layer_1", 10, 0}); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, "ctc", 100, /* trans */ false, /* useGpu */ +// useGpu); +// } +// } +// +// TEST(Layer, cosSimLayer) { +// TestConfig config; +// config.layerConfig.set_type("cos"); +// config.layerConfig.set_size(1); +// config.biasSize = 0; +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 50, 0}); +// config.inputDefs.push_back({INPUT_DATA, "layer_1", 50, 0}); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, "cos", 100, false, useGpu); +// } +// } +// +// TEST(Layer, CosSimVecMatLayer) { +// TestConfig config; +// config.layerConfig.set_type("cos_vm"); +// config.layerConfig.set_size(5); // output size +// config.layerConfig.set_cos_scale(2.0); +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 20, 0}); +// config.layerConfig.add_inputs(); +// config.inputDefs.push_back({INPUT_DATA, "layer_1", 100, 0}); +// config.layerConfig.add_inputs(); +// +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, "cos_vm", 100, false, useGpu); +// } +// } +// +// void testConvLayer(const string& type, bool trans, bool useGpu) { +// TestConfig config; +// config.biasSize = 16; +// config.layerConfig.set_type(type); +// config.layerConfig.set_num_filters(16); +// config.layerConfig.set_partial_sum(1); +// config.layerConfig.set_shared_biases(true); +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 384, 288}); +// LayerInputConfig* input = config.layerConfig.add_inputs(); +// ConvConfig* conv = input->mutable_conv_conf(); +// conv->set_filter_size(2); +// conv->set_filter_size_y(3); +// conv->set_channels(3); +// conv->set_padding(0); +// conv->set_padding_y(1); +// conv->set_stride(2); +// conv->set_stride_y(2); +// conv->set_groups(1); +// conv->set_filter_channels(conv->channels() / conv->groups()); +// conv->set_img_size(16); +// conv->set_img_size_y(8); +// conv->set_output_x(outputSize(conv->img_size(), +// conv->filter_size(), +// conv->padding(), +// conv->stride(), +// /* caffeMode */ true)); +// conv->set_output_y(outputSize(conv->img_size_y(), +// conv->filter_size_y(), +// conv->padding_y(), +// conv->stride_y(), +// /* caffeMode */ true)); +// config.layerConfig.set_size(conv->output_x() * conv->output_y() * +// config.layerConfig.num_filters()); +// +// testLayerGrad(config, "conv", 100, trans, useGpu); +// // Use small batch_size and useWeight=true to test biasGrad +// testLayerGrad(config, "conv", 2, trans, useGpu, true, 0.02); +// } +// +// TEST(Layer, convLayer) { +// testConvLayer("exconv", /* trans= */ false, /* useGpu= */ false); +// #ifndef PADDLE_ONLY_CPU +// testConvLayer("exconv", /* trans= */ false, /* useGpu= */ true); +// testConvLayer("cudnn_conv", /* trans= */ false, /* useGpu= */ true); +// #endif +// } +// +// void testConvTransLayer(const string& type, bool trans, bool useGpu) { +// TestConfig config; +// config.biasSize = 3; +// config.layerConfig.set_type(type); +// config.layerConfig.set_num_filters(3); +// config.layerConfig.set_partial_sum(1); +// config.layerConfig.set_shared_biases(true); +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 1024, 288}); +// LayerInputConfig* input = config.layerConfig.add_inputs(); +// ConvConfig* conv = input->mutable_conv_conf(); +// conv->set_filter_size(2); +// conv->set_filter_size_y(3); +// conv->set_channels(16); +// conv->set_padding(0); +// conv->set_padding_y(1); +// conv->set_stride(2); +// conv->set_stride_y(2); +// conv->set_groups(1); +// conv->set_filter_channels(3 / conv->groups()); +// conv->set_img_size(16); +// conv->set_output_x(outputSize(conv->img_size(), +// conv->filter_size(), +// conv->padding(), +// conv->stride(), +// /* caffeMode */ true)); +// +// config.layerConfig.set_size(conv->img_size() * conv->img_size() * +// config.layerConfig.num_filters()); +// +// testLayerGrad(config, "convTrans", 100, trans, useGpu); +// // Use small batch_size and useWeight=true to test biasGrad +// testLayerGrad(config, "convTrans", 2, trans, useGpu, true, 0.02); +// } +// +// TEST(Layer, convTransLayer) { +// for (auto useGpu : {false, true}) { +// testConvTransLayer("exconvt", /* trans= */ false, /* useGpu= */ useGpu); +// } +// } +// +// TEST(Layer, blockExpandLayer) { +// TestConfig config; +// config.biasSize = 0; +// config.layerConfig.set_type("blockexpand"); +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 6144, 0}); +// LayerInputConfig* input = config.layerConfig.add_inputs(); +// BlockExpandConfig* blockExpand = input->mutable_block_expand_conf(); +// blockExpand->set_img_size_x(64); +// blockExpand->set_img_size_y(32); +// blockExpand->set_channels(3); +// blockExpand->set_padding_x(0); +// blockExpand->set_padding_y(0); +// blockExpand->set_block_x(4); +// blockExpand->set_block_y(32); +// blockExpand->set_stride_x(2); +// blockExpand->set_stride_y(2); +// blockExpand->set_output_x(outputSize(blockExpand->img_size_x(), +// blockExpand->block_x(), +// blockExpand->padding_x(), +// blockExpand->stride_x(), +// /* caffeMode */ false)); +// blockExpand->set_output_y(outputSize(blockExpand->img_size_y(), +// blockExpand->block_y(), +// blockExpand->padding_y(), +// blockExpand->stride_y(), +// /* caffeMode */ false)); +// config.layerConfig.set_size(blockExpand->block_x() * blockExpand->block_y() +// * +// blockExpand->channels()); +// +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, "blockexpand", 100, false, useGpu); +// } +// } +// +// TEST(Layer, maxoutLayer) { +// TestConfig config; +// config.biasSize = 0; +// config.layerConfig.set_type("maxout"); +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 4096, 0}); +// LayerInputConfig* input = config.layerConfig.add_inputs(); +// MaxOutConfig* maxout = input->mutable_maxout_conf(); +// ImageConfig* image = maxout->mutable_image_conf(); +// +// image->set_img_size(32); +// image->set_img_size_y(32); +// image->set_channels(4); +// maxout->set_groups(2); +// +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, "maxout", 10, false, useGpu); +// } +// } +// void testFcLayer(string format, size_t nnz) { +// TestConfig config; +// config.biasSize = 4096; +// config.layerConfig.set_type("fc"); +// config.layerConfig.set_size(4096); +// config.layerConfig.set_active_type("sigmoid"); +// config.layerConfig.set_drop_rate(0.1); +// +// config.inputDefs.push_back( +// {INPUT_DATA, "layer_0", 8192, nnz, ParaSparse(format)}); +// config.layerConfig.add_inputs(); +// +// LOG(INFO) << config.inputDefs[0].sparse.sparse << " " +// << config.inputDefs[0].sparse.format; +// +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, +// "fc", +// 100, +// /* trans */ false, +// useGpu, +// /* weight */ true); +// } +// } +// +// TEST(Layer, fcLayer) { +// testFcLayer("", 4096 * 4096 * 2); +// testFcLayer("csc", 4096 * 40); +// testFcLayer("csr", 4096 * 40); +// } +// +// TEST(Layer, SelectiveFullyConnectedLayer) { +// TestConfig config; +// size_t nin = 16; +// size_t nout = 256; +// config.layerConfig.set_type("selective_fc"); +// config.layerConfig.set_size(nout); +// config.layerConfig.set_active_type("sigmoid"); +// config.layerConfig.set_has_selected_colums(true); +// config.layerConfig.set_selective_fc_pass_generation(false); +// config.biasSize = nout; +// +// config.inputDefs.push_back({INPUT_DATA, "input0", nin, nin * nout}); +// config.layerConfig.add_inputs(); +// config.inputDefs.push_back( +// {INPUT_SPARSE_NON_VALUE_DATA, "index", nout, 0, ParaSparse("csr", +// true)}); +// config.layerConfig.add_inputs(); +// +// testLayerGrad(config, +// "selective_fc", +// 100, +// /* trans= */ false, +// /* useGup= */ false, +// false); +// #ifndef PADDLE_ONLY_CPU +// testLayerGrad(config, +// "selective_fc", +// 100, +// /* trans= */ false, +// /* useGup= */ true, +// false); +// #endif +// } +// +// TEST(Layer, DataNormLayer) { +// TestConfig config; +// config.layerConfig.set_type("data_norm"); +// config.layerConfig.set_size(20); +// config.biasSize = 0; +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 20, 100}); +// config.inputDefs.back().isStatic = true; +// config.layerConfig.add_inputs(); +// +// for (auto strategy : {"z-score", "min-max", "decimal-scaling"}) { +// config.layerConfig.set_data_norm_strategy(strategy); +// // The parameters are static, so not support GPU now +// testLayerGrad(config, +// "data_norm", +// 200, +// /* trans */ false, +// /* useGpu */ false); +// } +// } +// +// TEST(Layer, hsigmoidLayer) { +// TestConfig config; +// config.layerConfig.set_type("hsigmoid"); +// config.layerConfig.set_num_classes(5); +// config.layerConfig.set_size(1); +// config.biasSize = config.layerConfig.num_classes() - 1; +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 50, 200}); +// config.inputDefs.push_back({INPUT_LABEL, "layer_1", 5, 0}); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// +// // Not support GPU now +// testLayerGrad(config, "hsigmoid", 100, /* trans */ false, /* useGpu */ +// false); +// } +// +// TEST(Layer, multi_cross) { +// TestConfig config; +// config.layerConfig.set_type("multi-class-cross-entropy"); +// config.biasSize = 0; +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 50, 0}); +// config.inputDefs.push_back({INPUT_LABEL, "layer_1", 10, 0}); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// +// for (auto useGpu : {false, true}) { +// testLayerGrad( +// config, "multi-class-cross-entropy", 100, /* trans */ false, useGpu); +// } +// } +// +// TEST(Layer, multi_binary_label_sparse_mat) { +// TestConfig config; +// config.layerConfig.set_type("multi_binary_label_cross_entropy"); +// config.biasSize = 0; +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 50, 0}); +// config.inputDefs.push_back({INPUT_SPARSE_NON_VALUE_DATA, "layer_1", 50, +// 0}); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, +// "multi_binary_label_cross_entropy", +// 100, +// /* trans */ false, +// useGpu); +// } +// } +// +// TEST(layer, multi_binary_label_id) { +// TestConfig config; +// config.layerConfig.set_type("multi_binary_label_cross_entropy"); +// config.biasSize = 0; +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 50, 0}); +// config.inputDefs.push_back({INPUT_LABEL, "layer_1", 10, 0}); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, +// "multi_binary_label_cross_entropy", +// 100, +// /* trans */ false, +// useGpu); +// } +// } +// +// TEST(Layer, multi_cross_with_selfnorm) { +// TestConfig config; +// config.layerConfig.set_type("multi_class_cross_entropy_with_selfnorm"); +// config.layerConfig.set_softmax_selfnorm_alpha(0.1); +// config.biasSize = 0; +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 50, 0}); +// config.inputDefs.push_back({INPUT_LABEL, "layer_1", 10, 0}); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// +// // Not support GPU now +// testLayerGrad(config, +// "multi_class_cross_entropy_with_selfnorm", +// 100, +// /* trans */ false, +// /* useGpu */ false); +// } +// +// TEST(Layer, multi_cross_soft) { +// TestConfig config; +// config.layerConfig.set_type("soft_binary_class_cross_entropy"); +// config.biasSize = 0; +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 10, 0}); +// config.inputDefs.push_back({INPUT_DATA_TARGET, "layer_1", 10, 0}); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, +// "soft_binary_class_cross_entropy", +// 100, +// /* trans */ false, +// useGpu); +// } +// } +// +// TEST(Layer, square_error) { +// TestConfig config; +// config.layerConfig.set_type("square_error"); +// config.biasSize = 0; +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 10, 0}); +// config.inputDefs.push_back({INPUT_DATA_TARGET, "layer_1", 10, 0}); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, "square_error", 100, /* trans */ false, useGpu); +// } +// } +// +// TEST(Layer, sparse_square_error) { +// TestConfig config; +// config.layerConfig.set_type("square_error"); +// config.biasSize = 0; +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 50, 0}); +// config.inputDefs.push_back({INPUT_SPARSE_NON_VALUE_DATA, "layer_1", 50, +// 0}); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// +// // "GpuSparseMatrix" as label is not supported +// testLayerGrad(config, +// "square_error", +// 100, +// /* trans */ false, +// /* useGpu */ false); +// } +// +// TEST(Layer, sparse_float_square_error) { +// TestConfig config; +// config.layerConfig.set_type("square_error"); +// config.biasSize = 0; +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 50, 0}); +// config.inputDefs.push_back({INPUT_SPARSE_FLOAT_VALUE_DATA, "layer_1", 50, +// 0}); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// +// // "GpuSparseMatrix" as label is not supported +// testLayerGrad(config, +// "square_error", +// 100, +// /* trans */ false, +// /* useGpu */ false); +// } +// +// TEST(Layer, square_error_weighted) { +// TestConfig config; +// config.layerConfig.set_type("square_error"); +// config.biasSize = 0; +// config.testAccumulate = false; +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 10, 0}); +// config.inputDefs.push_back({INPUT_DATA_TARGET, "layer_1", 10, 0}); +// config.inputDefs.push_back({INPUT_DATA_TARGET, "layer_2", 1, 0}); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, "square_error", 100, /* trans */ false, useGpu); +// } +// } +// +// TEST(Layer, huber_two_class) { +// TestConfig config; +// config.layerConfig.set_type("huber"); +// config.biasSize = 0; +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 1, 0}); +// config.inputDefs.push_back({INPUT_LABEL, "layer_1", 2, 0}); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, "huber", 100, /* trans */ false, useGpu); +// } +// } +// +// void testExpandLayer(string trans_type, bool hasSubseq) { +// TestConfig config; +// config.layerConfig.set_type("expand"); +// +// config.inputDefs.push_back( +// {trans_type == "non-seq" ? INPUT_DENSE_DIM_DATA : INPUT_SEQUENCE_DATA, +// "layer_0", +// 10, +// 0}); +// config.inputDefs.push_back( +// {hasSubseq ? INPUT_HASSUB_SEQUENCE_DATA : INPUT_SEQUENCE_DATA, +// "layer_1", +// 10, +// 0}); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// config.layerConfig.set_trans_type(trans_type); +// LOG(INFO) << " trans_type=" << trans_type << " hasSubseq=" << hasSubseq; +// +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, "expand", 30, false, useGpu); +// } +// } +// +// TEST(Layer, ExpandLayer) { +// testExpandLayer("non-seq", false); // non-seq expand to seq +// testExpandLayer("non-seq", true); // non-seq expand to hasSubseq +// testExpandLayer("seq", true); // seq expand to hasSubseq +// } +// +// void testDegradeLayer(bool hasSubseq, string layer_type, string trans_type) { +// TestConfig config; +// config.layerConfig.set_type(layer_type); +// config.layerConfig.set_size(10); +// config.biasSize = 0; +// +// config.inputDefs.push_back( +// {hasSubseq ? INPUT_HASSUB_SEQUENCE_DATA : INPUT_SEQUENCE_DATA, +// "layer_0", +// 10, +// 0}); +// config.layerConfig.add_inputs(); +// config.layerConfig.set_trans_type(trans_type); +// +// auto testDegradeLayerGrad = [](TestConfig& config, string layer_type) { +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, layer_type, 100, false, useGpu); +// } +// }; +// +// if (layer_type == "average") { +// for (auto strategy : {"average", "sum", "squarerootn"}) { +// LOG(INFO) << " hasSubseq=" << hasSubseq << " trans_type=" << trans_type +// << " average_strategy=" << strategy; +// config.layerConfig.set_average_strategy(strategy); +// testDegradeLayerGrad(config, layer_type); +// } +// } else { +// LOG(INFO) << " hasSubseq=" << hasSubseq << " trans_type=" << trans_type; +// testDegradeLayerGrad(config, layer_type); +// } +// } +// +// TEST(Layer, MaxLayer) { +// testDegradeLayer(false, "max", "non-seq"); // seq max to non-seq +// testDegradeLayer(true, "max", "non-seq"); // hasSubseq max to non-seq +// testDegradeLayer(true, "max", "seq"); // hasSubseq max to seq +// } +// +// TEST(Layer, SequenceLastInstanceLayer) { +// testDegradeLayer(false, +// "seqlastins", +// "non-seq"); // seq seqlastins to non-seq +// testDegradeLayer(true, +// "seqlastins", +// "non-seq"); // hasSubseq seqlastins to non-seq +// testDegradeLayer(true, "seqlastins", "seq"); // hasSubseq seqlastins to +// seq +// } +// +// TEST(Layer, AverageLayer) { +// testDegradeLayer(false, "average", "non-seq"); // seq average to non-seq +// testDegradeLayer(true, "average", "non-seq"); // hasSubseq average to +// non-seq +// testDegradeLayer(true, "average", "seq"); // hasSubseq average to seq +// } +// +// TEST(Layer, SequenceConcatLayer) { +// TestConfig config; +// config.layerConfig.set_type("seqconcat"); +// config.layerConfig.set_size(10); +// config.biasSize = 0; +// +// config.inputDefs.push_back({INPUT_SEQUENCE_DATA, "layer_0", 10, 0}); +// config.layerConfig.add_inputs(); +// config.inputDefs.push_back({INPUT_SEQUENCE_DATA, "layer_1", 10, 0}); +// config.layerConfig.add_inputs(); +// +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, "seqconcat", 100, false, useGpu); +// } +// } +// +// TEST(Layer, SequenceReshapeLayer) { +// TestConfig config; +// config.layerConfig.set_type("seqreshape"); +// config.layerConfig.set_size(10); +// +// config.inputDefs.push_back({INPUT_SEQUENCE_DATA, "layer_0", 100, 0}); +// config.layerConfig.add_inputs(); +// +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, "seqreshape", 100, false, useGpu); +// } +// } +// +// TEST(Layer, ConvShiftLayer) { +// TestConfig config; +// config.layerConfig.set_type("conv_shift"); +// config.layerConfig.set_size(10); +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 10, 0}); +// config.inputDefs.push_back({INPUT_DATA, "layer_1", 3, 0}); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// +// // Not support GPU now +// testLayerGrad(config, "conv_shift", 100, false, false); +// } +// +// TEST(Layer, PowerLayer) { +// TestConfig config; +// config.layerConfig.set_type("power"); +// config.layerConfig.set_size(10); +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 1, 0}); +// config.inputDefs.push_back({INPUT_DATA, "layer_1", 10, 0}); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, "power", 100, false, useGpu); +// } +// } +// +// TEST(Layer, ConvexCombinationLayer) { +// TestConfig config; +// config.layerConfig.set_type("convex_comb"); +// config.layerConfig.set_size(20); +// config.biasSize = 0; +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 5, 0}); +// config.inputDefs.push_back({INPUT_DATA, "layer_1", 100, 0}); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, "convex_comb", 100, false, useGpu); +// } +// } +// +// TEST(Layer, InterpolationLayer) { +// TestConfig config; +// config.layerConfig.set_type("interpolation"); +// config.layerConfig.set_size(10); +// config.biasSize = 0; +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 1, 0}); +// config.inputDefs.push_back({INPUT_DATA, "layer_1", 10, 0}); +// config.inputDefs.push_back({INPUT_DATA, "layer_2", 10, 0}); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, "interpolation", 100, false, useGpu); +// } +// } +// +// TEST(Layer, OuterProdLayer) { +// TestConfig config; +// config.layerConfig.set_type("out_prod"); +// config.layerConfig.set_size(100); +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 10, 0}); +// config.layerConfig.add_inputs(); +// config.inputDefs.push_back({INPUT_DATA, "layer_1", 10, 0}); +// config.layerConfig.add_inputs(); +// +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, "out_prod", 100, false, useGpu); +// } +// } +// +// TEST(Layer, SlopeInterceptLayer) { +// TestConfig config; +// config.layerConfig.set_type("slope_intercept"); +// config.layerConfig.set_size(10); +// config.layerConfig.set_slope(1.0); +// config.layerConfig.set_intercept(0.1); +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 10, 0}); +// config.layerConfig.add_inputs(); +// +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, "slope_intercept", 100, false, useGpu); +// } +// } +// +// TEST(Layer, ScalingLayer) { +// TestConfig config; +// config.layerConfig.set_type("scaling"); +// config.layerConfig.set_size(10); +// config.biasSize = 0; +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 1, 0}); +// config.layerConfig.add_inputs(); +// config.inputDefs.push_back({INPUT_DATA, "layer_1", 10, 0}); +// config.layerConfig.add_inputs(); +// +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, "scaling", 100, false, useGpu); +// } +// } +// +// void testNormLayer(const string& normType, bool trans, bool useGpu) { +// TestConfig config; +// config.layerConfig.set_type("norm"); +// config.layerConfig.set_active_type("relu"); +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 1568, 0}); +// LayerInputConfig* input = config.layerConfig.add_inputs(); +// NormConfig* norm = input->mutable_norm_conf(); +// norm->set_norm_type(normType); +// norm->set_channels(16); +// norm->set_size(5); +// norm->set_scale(0.001); +// norm->set_pow(0.75); +// norm->set_blocked(0); +// norm->set_img_size(14); +// norm->set_img_size_y(7); +// norm->set_output_x(norm->img_size()); +// norm->set_output_y(norm->img_size_y()); +// if (norm->norm_type() == "cmrnorm" || +// norm->norm_type() == "cmrnorm-projection") { +// norm->set_scale(norm->scale() / norm->size()); +// } else { +// norm->set_scale(norm->scale() / (norm->size() * norm->size())); +// } +// +// config.layerConfig.set_size(norm->output_x() * norm->output_y() * +// norm->channels()); +// config.biasSize = 0; +// +// testLayerGrad(config, "norm", 100, trans, useGpu); +// } +// +// TEST(Layer, NormLayer) { +// testNormLayer("cmrnorm-projection", /* trans= */ false, /* useGpu= */ +// true); +// testNormLayer("cmrnorm-projection", /* trans= */ false, /* useGpu= */ +// false); +// } +// +// void setPoolConfig(TestConfig* config, +// PoolConfig* pool, +// const string& poolType) { +// (*config).biasSize = 0; +// (*config).layerConfig.set_type("pool"); +// (*config).layerConfig.set_num_filters(16); +// +// int kw = 3, kh = 3; +// int pw = 0, ph = 0; +// int sw = 2, sh = 2; +// pool->set_pool_type(poolType); +// pool->set_channels(16); +// pool->set_size_x(kw); +// pool->set_size_y(kh); +// pool->set_start(0); +// pool->set_padding(pw); +// pool->set_padding_y(ph); +// pool->set_stride(sw); +// pool->set_stride_y(sh); +// +// int ow = outputSize(pool->img_size(), kw, pw, sw, /* caffeMode */ false); +// int oh = outputSize(pool->img_size_y(), kh, ph, sh, /* caffeMode */ false); +// pool->set_output_x(ow); +// pool->set_output_y(oh); +// } +// +// void testPoolLayer(const string& poolType, bool trans, bool useGpu) { +// TestConfig config; +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 3136, 0}); +// LayerInputConfig* input = config.layerConfig.add_inputs(); +// PoolConfig* pool = input->mutable_pool_conf(); +// +// pool->set_img_size(14); +// pool->set_img_size_y(14); +// setPoolConfig(&config, pool, poolType); +// config.layerConfig.set_size(pool->output_x() * pool->output_y() * +// pool->channels()); +// +// testLayerGrad(config, "pool", 100, trans, useGpu); +// } +// +// #ifndef PADDLE_ONLY_CPU +// void testPoolLayer2(const string& poolType, bool trans, bool useGpu) { +// TestConfig config; +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 3200, 0}); +// LayerInputConfig* input = config.layerConfig.add_inputs(); +// PoolConfig* pool = input->mutable_pool_conf(); +// +// pool->set_size_y(4); +// pool->set_stride_y(3); +// pool->set_img_size(10); +// pool->set_img_size_y(20); +// setPoolConfig(&config, pool, poolType); +// pool->set_output_y((pool->img_size_y() - pool->start() - pool->size_y()) / +// ((float)pool->stride_y()) + +// 1.5); +// config.layerConfig.set_size(pool->output_x() * pool->output_y() * +// pool->channels()); +// +// testLayerGrad(config, "pool", 100, trans, useGpu); +// } +// #endif +// +// TEST(Layer, PoolLayer) { +// testPoolLayer("avg-projection", /* trans= */ false, /* useGpu= */ false); +// testPoolLayer("max-projection", /* trans= */ false, /* useGpu= */ false); +// +// #ifndef PADDLE_ONLY_CPU +// testPoolLayer("avg-projection", /* trans= */ false, /* useGpu= */ true); +// testPoolLayer("max-projection", /* trans= */ false, /* useGpu= */ true); +// testPoolLayer("cudnn-max-pool", /* trans= */ false, /* useGpu= */ true); +// testPoolLayer("cudnn-avg-pool", /* trans= */ false, /* useGpu= */ true); +// testPoolLayer2("cudnn-max-pool", /* trans= */ false, /* useGpu= */ true); +// testPoolLayer2("cudnn-avg-pool", /* trans= */ false, /* useGpu= */ true); +// #endif +// } +// +// void testSppLayer(const string& poolType, +// const int pyramidHeight, +// bool trans, +// bool useGpu) { +// TestConfig config; +// config.layerConfig.set_type("spp"); +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 3200, 0}); +// LayerInputConfig* input = config.layerConfig.add_inputs(); +// SppConfig* sppConfig = input->mutable_spp_conf(); +// sppConfig->set_pool_type(poolType); +// sppConfig->set_pyramid_height(pyramidHeight); +// ImageConfig* imageConfig = sppConfig->mutable_image_conf(); +// imageConfig->set_channels(16); +// imageConfig->set_img_size(10); +// imageConfig->set_img_size_y(20); +// int outputSize = (std::pow(4, sppConfig->pyramid_height()) - 1) / (4 - 1); +// config.layerConfig.set_size(outputSize * imageConfig->channels()); +// testLayerGrad(config, "spp", 100, trans, useGpu); +// } +// +// TEST(Layer, SpatialPyramidPoolLayer) { +// for (auto useGpu : {false, true}) { +// for (auto pyramidHeight : {1, 2, 3}) { +// testSppLayer("avg-projection", pyramidHeight, false, useGpu); +// testSppLayer("max-projection", pyramidHeight, false, useGpu); +// } +// } +// } +// +// TEST(Layer, rankCostLayer) { +// TestConfig config; +// config.layerConfig.set_type("rank-cost"); +// config.biasSize = 0; +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 1, 0}); +// config.inputDefs.push_back({INPUT_DATA, "layer_1", 1, 0}); +// config.inputDefs.push_back({INPUT_DATA_TARGET, "layer_2", 1, 0}); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, "rank-cost", 100, false, useGpu); +// } +// } +// +// TEST(Layer, sumCostLayer) { +// TestConfig config; +// config.layerConfig.set_type("sum_cost"); +// config.biasSize = 0; +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 1, 0}); +// config.layerConfig.add_inputs(); +// +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, "sum_cost", 100, false, useGpu); +// } +// } +// +// TEST(Layer, weightedRankCostLayer) { +// TestConfig config; +// config.layerConfig.set_type("rank-cost"); +// config.biasSize = 0; +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 1, 0}); +// config.inputDefs.push_back({INPUT_DATA, "layer_1", 1, 0}); +// config.inputDefs.push_back({INPUT_DATA_TARGET, "layer_2", 1, 0}); +// config.inputDefs.push_back({INPUT_DATA_TARGET, "layer_3", 1, 0}); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, "weighted-rank-cost", 100, false, useGpu); +// } +// } +// +// TEST(Layer, TensorLayer) { +// TestConfig config; +// config.layerConfig.set_type("tensor"); +// config.layerConfig.set_size(10); +// config.layerConfig.set_active_type("sigmoid"); +// config.biasSize = config.layerConfig.size(); +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 5, 250}); +// config.inputDefs.push_back({INPUT_DATA, "layer_1", 5, 0}); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, "tensor", 100, false, useGpu); +// } +// } +// +// TEST(Layer, RecurrentLayer) { +// TestConfig config; +// config.layerConfig.set_type("recurrent"); +// config.layerConfig.set_size(4); +// config.layerConfig.set_active_type("tanh"); +// config.biasSize = 4; +// +// config.inputDefs.push_back( +// {INPUT_SEQUENCE_DATA, "layer_0", /* dim= */ 4, /* paraSize= */ 16}); +// config.layerConfig.add_inputs(); +// +// for (auto useGpu : {false, true}) { +// for (auto reversed : {false, true}) { +// config.layerConfig.set_reversed(reversed); +// config.testState = !reversed; +// testLayerGrad(config, "recurrent", 50, /* trans= */ false, useGpu); +// } +// } +// } +// +// TEST(Layer, LstmLayer) { +// TestConfig config; +// config.layerConfig.set_type("lstmemory"); +// config.layerConfig.set_size(4); +// config.layerConfig.set_active_type("tanh"); +// config.layerConfig.set_active_state_type("sigmoid"); +// config.layerConfig.set_active_gate_type("sigmoid"); +// config.biasSize = 28; +// +// config.inputDefs.push_back( +// {INPUT_SEQUENCE_DATA, "layer_0", /* dim= */ 16, /* paraSize= */ 64}); +// config.layerConfig.add_inputs(); +// +// for (auto useGpu : {false, true}) { +// for (auto reversed : {false, true}) { +// config.layerConfig.set_reversed(reversed); +// config.testState = !reversed; +// testLayerGrad(config, "lstmemory", 100, /* trans= */ false, useGpu); +// } +// } +// for (auto useGpu : {true}) { +// config.testBatchState = true; +// config.layerConfig.set_reversed(false); +// testLayerGrad(config, "lstmemory", 10, /* trans= */ false, useGpu); +// } +// } +// +// TEST(Layer, MDLstmLayer) { +// TestConfig config; +// config.layerConfig.set_type("mdlstmemory"); +// config.layerConfig.set_size(4); +// config.layerConfig.set_active_type("sigmoid"); +// config.layerConfig.set_active_state_type("sigmoid"); +// config.layerConfig.set_active_gate_type("sigmoid"); +// config.biasSize = 4 * 9; +// +// config.inputDefs.push_back( +// {INPUT_SEQUENCE_MDIM_DATA, "layer_0", 4 * 5, 4 * 4 * 5}); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_directions(true); +// config.layerConfig.add_directions(true); +// +// for (auto useGpu : {false, true}) { +// for (int i = 0; i < 2; i++) { +// for (int j = 0; j < 2; j++) { +// config.layerConfig.set_directions(0, bool(i)); +// config.layerConfig.set_directions(1, bool(j)); +// testLayerGrad(config, "mdlstmemory", 100, false, useGpu); +// } +// } +// } +// } +// +// TEST(Layer, ParameterReluLayer) { +// auto testParameterReluLayer = [&](size_t inputSize, size_t channels) { +// TestConfig config; +// config.layerConfig.set_type("prelu"); +// config.inputDefs.push_back({INPUT_DATA, "layer_0", inputSize, channels}); +// config.layerConfig.add_inputs(); +// config.layerConfig.set_size(inputSize); +// config.layerConfig.set_partial_sum(inputSize / +// channels); // size of feature map +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, "prelu", 100, false, useGpu); +// } +// }; +// +// testParameterReluLayer(192, 1); +// testParameterReluLayer(192, 3); +// testParameterReluLayer(192, 192); +// } +// +// TEST(Layer, ResizeLayer) { +// TestConfig config; +// config.biasSize = 0; +// config.layerConfig.set_type("resize"); +// config.layerConfig.set_size(64); +// +// config.inputDefs.push_back({INPUT_DATA, "layer_0", 16, 0}); +// config.layerConfig.add_inputs(); +// +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, "resize", 100, false, useGpu); +// } +// } +// +// TEST(Layer, NCELayer) { +// TestConfig config; +// size_t numClasses = 4; +// config.layerConfig.set_type("nce"); +// config.layerConfig.set_size(1); +// config.layerConfig.set_active_type("sigmoid"); +// config.layerConfig.set_num_classes(numClasses); +// config.biasSize = numClasses; +// +// config.inputDefs.push_back( +// {INPUT_DATA, "layer_0", /* dim= */ 16, /* paraSize= */ 16 * +// numClasses}); +// config.inputDefs.push_back( +// {INPUT_LABEL, "label", /* dim= */ numClasses, /* paraSize= */ 0}); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// +// for (auto withWeight : {false, true}) { +// if (withWeight) { +// config.inputDefs.push_back( +// {INPUT_DATA_TARGET, "weight", /* dim= */ 1, /* paraSize= */ 0}); +// config.layerConfig.add_inputs(); +// } +// +// for (auto isIdLabel : {false, true}) { +// config.inputDefs[1] = { +// isIdLabel ? INPUT_LABEL : INPUT_SPARSE_NON_VALUE_DATA, +// "label", +// /* dim= */ numClasses, +// /* paraSize= */ 0}; +// +// for (auto withDist : {false, true}) { +// config.layerConfig.clear_neg_sampling_dist(); +// if (withDist) { +// double sum = 0; +// for (size_t i = 0; i < numClasses; ++i) { +// real p = rand(); // NOLINT use rand_r +// config.layerConfig.add_neg_sampling_dist(p); +// sum += p; +// } +// for (size_t i = 0; i < numClasses; ++i) { +// real p = config.layerConfig.neg_sampling_dist(i) / sum; +// config.layerConfig.set_neg_sampling_dist(i, p); +// } +// } +// LOG(INFO) << "NCELayer " +// << " isIdLabel=" << isIdLabel << " withWeight=" << +// withWeight +// << " withDist=" << withDist; +// // Not support GPU now +// testLayerGrad(config, +// "nce", +// 100, +// /* trans= */ false, +// /* useGpu */ false); +// } +// } +// } +// } +// +// TEST(Layer, GatedRecurrentLayer) { +// TestConfig config; +// config.layerConfig.set_type("gated_recurrent"); +// config.layerConfig.set_size(4); +// config.layerConfig.set_active_type("sigmoid"); +// config.layerConfig.set_active_gate_type("sigmoid"); +// config.biasSize = 12; +// +// config.inputDefs.push_back( +// {INPUT_SEQUENCE_DATA, "layer_0", /* dim= */ 12, /* paraSize= */ 48}); +// config.layerConfig.add_inputs(); +// +// for (auto useGpu : {false, true}) { +// for (auto reversed : {false, true}) { +// config.layerConfig.set_reversed(reversed); +// config.testState = !reversed; +// testLayerGrad(config, "gated_recurrent", 100, /* trans= */ false, +// useGpu); +// } +// } +// } +// +// TEST(Layer, GruStepLayer) { +// TestConfig config; +// config.layerConfig.set_type("gru_step"); +// config.layerConfig.set_size(4); +// config.layerConfig.set_active_type("sigmoid"); +// config.layerConfig.set_active_gate_type("sigmoid"); +// config.biasSize = 12; +// +// config.inputDefs.push_back( +// {INPUT_DATA, "layer_0", /* dim= */ 12, /* paraSize= */ 48}); +// config.inputDefs.push_back( +// {INPUT_DATA, "layer_1", /* dim= */ 4, /* paraSize= */ 0}); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, "gruStep", 100, /* trans= */ false, useGpu); +// } +// } +// +// TEST(Layer, LstmStepLayer) { +// TestConfig config; +// config.layerConfig.set_type("lstm_step"); +// config.layerConfig.set_size(4); +// config.layerConfig.set_active_type("sigmoid"); +// config.layerConfig.set_active_state_type("sigmoid"); +// config.layerConfig.set_active_gate_type("sigmoid"); +// config.biasSize = 12; +// config.testAccumulate = false; +// +// config.inputDefs.push_back( +// {INPUT_DATA, "layer_0", /* dim= */ 16, /* paraSize= */ 0}); +// config.inputDefs.push_back( +// {INPUT_DATA, "layer_1", /* dim= */ 4, /* paraSize= */ 0}); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, "lstmStep", 100, /* trans= */ false, useGpu); +// } +// } +// +// void testBatchNormLayer(const string& type, bool trans, bool useGpu) { +// TestConfig config; +// const int CHANNELS = 10; +// const int IMG_SIZE = 16; +// const int IMG_SIZE_Y = 8; +// size_t size = CHANNELS * IMG_SIZE * IMG_SIZE_Y; +// config.layerConfig.set_type(type); +// config.layerConfig.set_size(size); +// config.layerConfig.set_active_type("sigmoid"); +// config.biasSize = CHANNELS; +// config.inputDefs.push_back({INPUT_DATA, +// "layer_0", +// /* dim= */ size, +// /* paraSize= */ CHANNELS}); +// +// config.inputDefs.push_back({INPUT_DATA, "layer_1_running_mean", 1, +// CHANNELS}); +// config.inputDefs.back().isStatic = true; +// config.inputDefs.push_back({INPUT_DATA, "layer_2_running_var", 1, +// CHANNELS}); +// config.inputDefs.back().isStatic = true; +// +// LayerInputConfig* input = config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// +// ImageConfig* img_conf = input->mutable_image_conf(); +// img_conf->set_channels(CHANNELS); +// img_conf->set_img_size(IMG_SIZE); +// img_conf->set_img_size_y(IMG_SIZE_Y); +// +// testLayerGrad(config, +// "batch_norm", +// 64, +// /* trans= */ trans, +// useGpu, +// /* useWeight */ true); +// } +// +// TEST(Layer, BatchNormalizationLayer) { +// testBatchNormLayer("batch_norm", false, false); +// #ifndef PADDLE_ONLY_CPU +// testBatchNormLayer("batch_norm", false, true); +// if (hl_get_cudnn_lib_version() >= int(4000)) { +// testBatchNormLayer("cudnn_batch_norm", false, true); +// } +// #endif +// } +// +// TEST(Operator, conv) { +// TestConfig config; +// const int NUM_FILTERS = 16; +// const int FILTER_SIZE = 2; +// const int FILTER_SIZE_Y = 3; +// const int CHANNELS = 3; +// const int IMAGE_SIZE = 16; +// const int IMAGE_SIZE_Y = 8; +// OperatorConfig& operatorConf = *config.layerConfig.add_operator_confs(); +// operatorConf.set_type("conv"); +// ConvConfig* conv = operatorConf.mutable_conv_conf(); +// operatorConf.set_num_filters(NUM_FILTERS); +// conv->set_filter_size(FILTER_SIZE); +// conv->set_filter_size_y(FILTER_SIZE_Y); +// conv->set_channels(CHANNELS); +// conv->set_padding(0); +// conv->set_padding_y(1); +// conv->set_stride(2); +// conv->set_stride_y(2); +// conv->set_groups(1); +// conv->set_filter_channels(conv->channels() / conv->groups()); +// conv->set_img_size(IMAGE_SIZE); +// conv->set_img_size_y(IMAGE_SIZE_Y); +// conv->set_output_x(outputSize(conv->img_size(), +// conv->filter_size(), +// conv->padding(), +// conv->stride(), +// /* caffeMode */ true)); +// conv->set_output_y(outputSize(conv->img_size_y(), +// conv->filter_size_y(), +// conv->padding_y(), +// conv->stride_y(), +// /* caffeMode */ true)); +// config.layerConfig.set_size(conv->output_x() * conv->output_y() * +// NUM_FILTERS); +// +// config.inputDefs.push_back( +// {INPUT_DATA, "layer_0", IMAGE_SIZE * IMAGE_SIZE_Y * CHANNELS, 0}); +// config.inputDefs.push_back( +// {INPUT_DATA, +// "layer_1", +// FILTER_SIZE * FILTER_SIZE_Y * CHANNELS * NUM_FILTERS, +// 0}); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// +// testOperatorGrad(config, operatorConf, 100, /*useGpu*/ true, false); +// } +// +// TEST(Layer, FeatureMapExpandLayer) { +// TestConfig config; +// config.layerConfig.set_type("featmap_expand"); +// const int CHANNELS = 10; +// const int INPUT_SIZE = 100; +// config.layerConfig.set_size(INPUT_SIZE * CHANNELS); +// config.layerConfig.set_num_filters(CHANNELS); +// config.inputDefs.push_back({INPUT_SEQUENCE_DATA, +// "layer_0", +// /* dim= */ INPUT_SIZE, +// /* paraSize= */ 0}); +// config.layerConfig.add_inputs(); +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, +// "featmap_expand", +// /*batch_size*/ 100, +// /* trans= */ false, +// useGpu, +// /* useWeight */ true); +// } +// } +// +// TEST(Layer, MultiplexLayer) { +// TestConfig config; +// const int LAYER_SIZE = 100; +// config.layerConfig.set_type("multiplex"); +// config.layerConfig.set_size(LAYER_SIZE); +// +// config.inputDefs.push_back({INPUT_LABEL, "layer_0", 2, 0}); +// config.inputDefs.push_back( +// {INPUT_DATA, "layer_1", /* dim= */ LAYER_SIZE, /* paraSize= */ 0}); +// config.inputDefs.push_back( +// {INPUT_DATA, "layer_2", /* dim= */ LAYER_SIZE, /* paraSize= */ 0}); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// config.layerConfig.add_inputs(); +// +// for (auto useGpu : {false, true}) { +// testLayerGrad(config, "multiplex", 512, /* trans= */ false, useGpu); +// } +// } +// +TEST(Layer, PadLayer) { TestConfig config; - config.layerConfig.set_size(10); - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 10, 0}); - config.inputDefs.push_back({INPUT_DATA, "layer_1", 10, 0}); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - - OperatorConfig& operatorConf = *config.layerConfig.add_operator_confs(); - operatorConf.set_type("dot_mul"); - operatorConf.set_dotmul_scale(-1); - - testOperatorGrad(config, operatorConf, 100, false, false); -} - -TEST(Projection, context) { - for (auto contextStart : {-5, -3, -1, 0, 3}) { - for (auto contextLength : {1, 2, 5, 7}) { - for (auto batchSize : {1, 2, 5, 20, 50}) { - for (auto trainablePadding : {false, true}) { - LOG(INFO) << " contextStart=" << contextStart - << " contextLength=" << contextLength - << " batchSize=" << batchSize - << " trainablePadding=" << trainablePadding; - ProjectionConfig conf; - conf.set_type("context"); - conf.set_input_size(10); - conf.set_context_start(contextStart); - conf.set_context_length(contextLength); - conf.set_trainable_padding(trainablePadding); - conf.set_output_size(conf.context_length() * conf.input_size()); - int pad = - std::max(0, -conf.context_start()) + - std::max(0, conf.context_start() + conf.context_length() - 1); - for (auto useGpu : {false, true}) { - testProjectionGrad( - conf, - INPUT_SEQUENCE_DATA, - trainablePadding ? conf.input_size() * pad : 0, - batchSize, - useGpu, - contextStart + contextLength <= 1); // = testState - } - } - } - } - } -} - -TEST(Projection, trans_fc) { - ProjectionConfig conf; - conf.set_type("trans_fc"); - conf.set_input_size(50); - conf.set_output_size(20); - for (auto useGpu : {false, true}) { - testProjectionGrad(conf, - INPUT_DATA, - /* parameterSize */ 1000, - /* batchSize */ 100, - useGpu); - } -} - -TEST(Projection, fc) { - ProjectionConfig conf; - conf.set_type("fc"); - conf.set_input_size(10); - conf.set_output_size(20); - for (auto useGpu : {false, true}) { - testProjectionGrad(conf, - INPUT_DATA, - /* parameterSize */ 200, - /* batchSize */ 100, - useGpu); - } -} - -TEST(Projection, dot_mul) { - ProjectionConfig conf; - conf.set_type("dot_mul"); - conf.set_input_size(20); - conf.set_output_size(20); - for (auto useGpu : {false, true}) { - testProjectionGrad(conf, - INPUT_DATA, - /* parameterSize */ 20, - /* batchSize */ 100, - useGpu); - } -} - -TEST(Projection, table) { - ProjectionConfig conf; - conf.set_type("table"); - conf.set_input_size(10); - conf.set_output_size(20); - for (auto useGpu : {false, true}) { - testProjectionGrad(conf, - INPUT_LABEL, - /* parameterSize */ 200, - /* batchSize */ 100, - useGpu); - } -} - -TEST(Projection, identity) { - ProjectionConfig conf; - conf.set_type("identity"); - conf.set_input_size(10); - conf.set_output_size(10); - for (auto useGpu : {false, true}) { - testProjectionGrad(conf, - INPUT_DATA, - /* parameterSize */ 0, - /* batchSize */ 100, - useGpu); - } -} - -TEST(Projection, scaling) { - ProjectionConfig conf; - conf.set_type("scaling"); - conf.set_input_size(10); - conf.set_output_size(10); - for (auto useGpu : {false}) { - testProjectionGrad(conf, - INPUT_DATA, - /* parameterSize */ 1, - /* batchSize */ 100, - useGpu); - } -} - -void testProjectionConv(size_t groups) { - const int NUM_FILTERS = 18; - const int FILTER_SIZE = 2; - const int FILTER_SIZE_Y = 3; - const int CHANNELS = 3; - const int IMAGE_SIZE = 16; - - ProjectionConfig conf; - conf.set_type("conv"); - conf.set_num_filters(NUM_FILTERS); - - ConvConfig* conv = conf.mutable_conv_conf(); - conv->set_filter_size(FILTER_SIZE); - conv->set_filter_size_y(FILTER_SIZE_Y); - conv->set_channels(CHANNELS); - conv->set_padding(0); - conv->set_padding_y(1); - conv->set_stride(2); - conv->set_stride_y(2); - conv->set_groups(groups); - conv->set_filter_channels(conv->channels() / conv->groups()); - conv->set_img_size(IMAGE_SIZE); - int output_x = outputSize(conv->img_size(), - conv->filter_size(), - conv->padding(), - conv->stride(), - /* caffeMode */ true); - int output_y = outputSize(conv->img_size(), - conv->filter_size_y(), - conv->padding_y(), - conv->stride_y(), - /* caffeMode */ true); - conv->set_output_x(output_x); - conf.set_input_size(IMAGE_SIZE * IMAGE_SIZE * CHANNELS); - conf.set_output_size(output_x * output_y * NUM_FILTERS); - - testProjectionGrad(conf, - INPUT_DATA, - /* parameterSize */ NUM_FILTERS * CHANNELS * FILTER_SIZE * - FILTER_SIZE_Y / groups, - /* batchSize */ 100, - true, - false, - NUM_FILTERS, - true); -} - -#ifndef PADDLE_ONLY_CPU -TEST(Projection, conv) { - testProjectionConv(1); - testProjectionConv(3); -} -#endif - -TEST(Layer, BilinearInterpLayer) { - TestConfig config; - config.layerConfig.set_type("bilinear_interp"); - config.biasSize = 0; - config.inputDefs.push_back({INPUT_DATA, "layer_0", 4096, 0}); - - LayerInputConfig* input = config.layerConfig.add_inputs(); - BilinearInterpConfig* bilinear = input->mutable_bilinear_interp_conf(); - ImageConfig* image = bilinear->mutable_image_conf(); - image->set_img_size(32); - image->set_img_size_y(32); - image->set_channels(4); - - for (auto useGpu : {false, true}) { - for (auto outSize : {32, 64}) { - bilinear->set_out_size_x(outSize); - bilinear->set_out_size_y(outSize); - testLayerGrad(config, "bilinear_interp", 10, false, useGpu); - } - } -} - -TEST(Layer, concat) { - TestConfig config; - config.biasSize = 0; - config.layerConfig.set_type("concat"); - config.layerConfig.set_size(15); - config.layerConfig.set_active_type("sigmoid"); - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 5, 0}); - config.layerConfig.add_inputs(); - config.inputDefs.push_back({INPUT_DATA, "layer_1", 10, 0}); - config.layerConfig.add_inputs(); - - for (auto useGpu : {false, true}) { - testLayerGrad(config, "concat", 100, false, useGpu); - } -} - -TEST(Layer, AddtoLayer) { - TestConfig config; - config.biasSize = 0; - config.layerConfig.set_type("addto"); - config.layerConfig.set_size(10); - config.layerConfig.set_active_type("sigmoid"); - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 10, 0}); - config.layerConfig.add_inputs(); - config.inputDefs.push_back({INPUT_DATA, "layer_1", 10, 0}); - config.layerConfig.add_inputs(); - - for (auto useGpu : {false, true}) { - testLayerGrad(config, "addto", 100, false, useGpu); - } -} - -TEST(Layer, CRFLayer) { - TestConfig config; - config.layerConfig.set_type("crf"); - config.layerConfig.set_size(10); - config.biasSize = 0; - - config.inputDefs.push_back({INPUT_SEQUENCE_DATA, "layer_0", 10, 120}); - config.inputDefs.push_back({INPUT_SEQUENCE_LABEL, "layer_1", 10, 0}); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - - // Not support GPU now - testLayerGrad(config, - "crf", - 100, - /* trans */ false, - /* useGpu */ false, - false /*useWeight*/, - 0.03 /*epsilon*/); -} - -TEST(Layer, CTCLayer) { - TestConfig config; - config.layerConfig.set_type("ctc"); - config.layerConfig.set_norm_by_times(false); - config.layerConfig.set_size(10); - config.biasSize = 0; - - config.inputDefs.push_back({INPUT_SEQUENCE_DATA, "layer_0", 10, 0}); - config.inputDefs.push_back({INPUT_SEQUENCE_LABEL, "layer_1", 10, 0}); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - - for (auto useGpu : {false, true}) { - testLayerGrad(config, "ctc", 100, /* trans */ false, /* useGpu */ useGpu); - } -} - -TEST(Layer, cosSimLayer) { - TestConfig config; - config.layerConfig.set_type("cos"); - config.layerConfig.set_size(1); config.biasSize = 0; + config.layerConfig.set_type("pad"); - config.inputDefs.push_back({INPUT_DATA, "layer_0", 50, 0}); - config.inputDefs.push_back({INPUT_DATA, "layer_1", 50, 0}); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - - for (auto useGpu : {false, true}) { - testLayerGrad(config, "cos", 100, false, useGpu); - } -} - -TEST(Layer, CosSimVecMatLayer) { - TestConfig config; - config.layerConfig.set_type("cos_vm"); - config.layerConfig.set_size(5); // output size - config.layerConfig.set_cos_scale(2.0); - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 20, 0}); - config.layerConfig.add_inputs(); - config.inputDefs.push_back({INPUT_DATA, "layer_1", 100, 0}); - config.layerConfig.add_inputs(); - - for (auto useGpu : {false, true}) { - testLayerGrad(config, "cos_vm", 100, false, useGpu); - } -} - -void testConvLayer(const string& type, bool trans, bool useGpu) { - TestConfig config; - config.biasSize = 16; - config.layerConfig.set_type(type); - config.layerConfig.set_num_filters(16); - config.layerConfig.set_partial_sum(1); - config.layerConfig.set_shared_biases(true); - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 384, 288}); + int c = 4; + int h = 31; + int w = 36; + size_t size = c * h * w; + config.inputDefs.push_back({INPUT_DATA, "layer_0", size, 0}); LayerInputConfig* input = config.layerConfig.add_inputs(); - ConvConfig* conv = input->mutable_conv_conf(); - conv->set_filter_size(2); - conv->set_filter_size_y(3); - conv->set_channels(3); - conv->set_padding(0); - conv->set_padding_y(1); - conv->set_stride(2); - conv->set_stride_y(2); - conv->set_groups(1); - conv->set_filter_channels(conv->channels() / conv->groups()); - conv->set_img_size(16); - conv->set_img_size_y(8); - conv->set_output_x(outputSize(conv->img_size(), - conv->filter_size(), - conv->padding(), - conv->stride(), - /* caffeMode */ true)); - conv->set_output_y(outputSize(conv->img_size_y(), - conv->filter_size_y(), - conv->padding_y(), - conv->stride_y(), - /* caffeMode */ true)); - config.layerConfig.set_size(conv->output_x() * conv->output_y() * - config.layerConfig.num_filters()); - - testLayerGrad(config, "conv", 100, trans, useGpu); - // Use small batch_size and useWeight=true to test biasGrad - testLayerGrad(config, "conv", 2, trans, useGpu, true, 0.02); -} - -TEST(Layer, convLayer) { - testConvLayer("exconv", /* trans= */ false, /* useGpu= */ false); -#ifndef PADDLE_ONLY_CPU - testConvLayer("exconv", /* trans= */ false, /* useGpu= */ true); - testConvLayer("cudnn_conv", /* trans= */ false, /* useGpu= */ true); -#endif -} - -void testConvTransLayer(const string& type, bool trans, bool useGpu) { - TestConfig config; - config.biasSize = 3; - config.layerConfig.set_type(type); - config.layerConfig.set_num_filters(3); - config.layerConfig.set_partial_sum(1); - config.layerConfig.set_shared_biases(true); - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 1024, 288}); - LayerInputConfig* input = config.layerConfig.add_inputs(); - ConvConfig* conv = input->mutable_conv_conf(); - conv->set_filter_size(2); - conv->set_filter_size_y(3); - conv->set_channels(16); - conv->set_padding(0); - conv->set_padding_y(1); - conv->set_stride(2); - conv->set_stride_y(2); - conv->set_groups(1); - conv->set_filter_channels(3 / conv->groups()); - conv->set_img_size(16); - conv->set_output_x(outputSize(conv->img_size(), - conv->filter_size(), - conv->padding(), - conv->stride(), - /* caffeMode */ true)); - - config.layerConfig.set_size(conv->img_size() * conv->img_size() * - config.layerConfig.num_filters()); - - testLayerGrad(config, "convTrans", 100, trans, useGpu); - // Use small batch_size and useWeight=true to test biasGrad - testLayerGrad(config, "convTrans", 2, trans, useGpu, true, 0.02); -} - -TEST(Layer, convTransLayer) { - for (auto useGpu : {false, true}) { - testConvTransLayer("exconvt", /* trans= */ false, /* useGpu= */ useGpu); - } -} - -TEST(Layer, blockExpandLayer) { - TestConfig config; - config.biasSize = 0; - config.layerConfig.set_type("blockexpand"); - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 6144, 0}); - LayerInputConfig* input = config.layerConfig.add_inputs(); - BlockExpandConfig* blockExpand = input->mutable_block_expand_conf(); - blockExpand->set_img_size_x(64); - blockExpand->set_img_size_y(32); - blockExpand->set_channels(3); - blockExpand->set_padding_x(0); - blockExpand->set_padding_y(0); - blockExpand->set_block_x(4); - blockExpand->set_block_y(32); - blockExpand->set_stride_x(2); - blockExpand->set_stride_y(2); - blockExpand->set_output_x(outputSize(blockExpand->img_size_x(), - blockExpand->block_x(), - blockExpand->padding_x(), - blockExpand->stride_x(), - /* caffeMode */ false)); - blockExpand->set_output_y(outputSize(blockExpand->img_size_y(), - blockExpand->block_y(), - blockExpand->padding_y(), - blockExpand->stride_y(), - /* caffeMode */ false)); - config.layerConfig.set_size(blockExpand->block_x() * blockExpand->block_y() * - blockExpand->channels()); - - for (auto useGpu : {false, true}) { - testLayerGrad(config, "blockexpand", 100, false, useGpu); - } -} - -TEST(Layer, maxoutLayer) { - TestConfig config; - config.biasSize = 0; - config.layerConfig.set_type("maxout"); - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 4096, 0}); - LayerInputConfig* input = config.layerConfig.add_inputs(); - MaxOutConfig* maxout = input->mutable_maxout_conf(); - ImageConfig* image = maxout->mutable_image_conf(); - - image->set_img_size(32); - image->set_img_size_y(32); - image->set_channels(4); - maxout->set_groups(2); - - for (auto useGpu : {false, true}) { - testLayerGrad(config, "maxout", 10, false, useGpu); - } -} -void testFcLayer(string format, size_t nnz) { - TestConfig config; - config.biasSize = 4096; - config.layerConfig.set_type("fc"); - config.layerConfig.set_size(4096); - config.layerConfig.set_active_type("sigmoid"); - config.layerConfig.set_drop_rate(0.1); - - config.inputDefs.push_back( - {INPUT_DATA, "layer_0", 8192, nnz, ParaSparse(format)}); - config.layerConfig.add_inputs(); - - LOG(INFO) << config.inputDefs[0].sparse.sparse << " " - << config.inputDefs[0].sparse.format; - - for (auto useGpu : {false, true}) { - testLayerGrad(config, - "fc", - 100, - /* trans */ false, - useGpu, - /* weight */ true); - } -} - -TEST(Layer, fcLayer) { - testFcLayer("", 4096 * 4096 * 2); - testFcLayer("csc", 4096 * 40); - testFcLayer("csr", 4096 * 40); -} - -TEST(Layer, SelectiveFullyConnectedLayer) { - TestConfig config; - size_t nin = 16; - size_t nout = 256; - config.layerConfig.set_type("selective_fc"); - config.layerConfig.set_size(nout); - config.layerConfig.set_active_type("sigmoid"); - config.layerConfig.set_has_selected_colums(true); - config.layerConfig.set_selective_fc_pass_generation(false); - config.biasSize = nout; - - config.inputDefs.push_back({INPUT_DATA, "input0", nin, nin * nout}); - config.layerConfig.add_inputs(); - config.inputDefs.push_back( - {INPUT_SPARSE_NON_VALUE_DATA, "index", nout, 0, ParaSparse("csr", true)}); - config.layerConfig.add_inputs(); - - testLayerGrad(config, - "selective_fc", - 100, - /* trans= */ false, - /* useGup= */ false, - false); -#ifndef PADDLE_ONLY_CPU - testLayerGrad(config, - "selective_fc", - 100, - /* trans= */ false, - /* useGup= */ true, - false); -#endif -} - -TEST(Layer, DataNormLayer) { - TestConfig config; - config.layerConfig.set_type("data_norm"); - config.layerConfig.set_size(20); - config.biasSize = 0; - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 20, 100}); - config.inputDefs.back().isStatic = true; - config.layerConfig.add_inputs(); - - for (auto strategy : {"z-score", "min-max", "decimal-scaling"}) { - config.layerConfig.set_data_norm_strategy(strategy); - // The parameters are static, so not support GPU now - testLayerGrad(config, - "data_norm", - 200, - /* trans */ false, - /* useGpu */ false); - } -} - -TEST(Layer, hsigmoidLayer) { - TestConfig config; - config.layerConfig.set_type("hsigmoid"); - config.layerConfig.set_num_classes(5); - config.layerConfig.set_size(1); - config.biasSize = config.layerConfig.num_classes() - 1; - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 50, 200}); - config.inputDefs.push_back({INPUT_LABEL, "layer_1", 5, 0}); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - - // Not support GPU now - testLayerGrad(config, "hsigmoid", 100, /* trans */ false, /* useGpu */ false); -} - -TEST(Layer, multi_cross) { - TestConfig config; - config.layerConfig.set_type("multi-class-cross-entropy"); - config.biasSize = 0; - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 50, 0}); - config.inputDefs.push_back({INPUT_LABEL, "layer_1", 10, 0}); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - - for (auto useGpu : {false, true}) { - testLayerGrad( - config, "multi-class-cross-entropy", 100, /* trans */ false, useGpu); - } -} - -TEST(Layer, multi_binary_label_sparse_mat) { - TestConfig config; - config.layerConfig.set_type("multi_binary_label_cross_entropy"); - config.biasSize = 0; - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 50, 0}); - config.inputDefs.push_back({INPUT_SPARSE_NON_VALUE_DATA, "layer_1", 50, 0}); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - - for (auto useGpu : {false, true}) { - testLayerGrad(config, - "multi_binary_label_cross_entropy", - 100, - /* trans */ false, - useGpu); - } -} - -TEST(layer, multi_binary_label_id) { - TestConfig config; - config.layerConfig.set_type("multi_binary_label_cross_entropy"); - config.biasSize = 0; - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 50, 0}); - config.inputDefs.push_back({INPUT_LABEL, "layer_1", 10, 0}); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - - for (auto useGpu : {false, true}) { - testLayerGrad(config, - "multi_binary_label_cross_entropy", - 100, - /* trans */ false, - useGpu); - } -} - -TEST(Layer, multi_cross_with_selfnorm) { - TestConfig config; - config.layerConfig.set_type("multi_class_cross_entropy_with_selfnorm"); - config.layerConfig.set_softmax_selfnorm_alpha(0.1); - config.biasSize = 0; - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 50, 0}); - config.inputDefs.push_back({INPUT_LABEL, "layer_1", 10, 0}); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - - // Not support GPU now - testLayerGrad(config, - "multi_class_cross_entropy_with_selfnorm", - 100, - /* trans */ false, - /* useGpu */ false); -} - -TEST(Layer, multi_cross_soft) { - TestConfig config; - config.layerConfig.set_type("soft_binary_class_cross_entropy"); - config.biasSize = 0; - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 10, 0}); - config.inputDefs.push_back({INPUT_DATA_TARGET, "layer_1", 10, 0}); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - - for (auto useGpu : {false, true}) { - testLayerGrad(config, - "soft_binary_class_cross_entropy", - 100, - /* trans */ false, - useGpu); - } -} - -TEST(Layer, square_error) { - TestConfig config; - config.layerConfig.set_type("square_error"); - config.biasSize = 0; - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 10, 0}); - config.inputDefs.push_back({INPUT_DATA_TARGET, "layer_1", 10, 0}); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - - for (auto useGpu : {false, true}) { - testLayerGrad(config, "square_error", 100, /* trans */ false, useGpu); - } -} - -TEST(Layer, sparse_square_error) { - TestConfig config; - config.layerConfig.set_type("square_error"); - config.biasSize = 0; - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 50, 0}); - config.inputDefs.push_back({INPUT_SPARSE_NON_VALUE_DATA, "layer_1", 50, 0}); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - - // "GpuSparseMatrix" as label is not supported - testLayerGrad(config, - "square_error", - 100, - /* trans */ false, - /* useGpu */ false); -} - -TEST(Layer, sparse_float_square_error) { - TestConfig config; - config.layerConfig.set_type("square_error"); - config.biasSize = 0; - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 50, 0}); - config.inputDefs.push_back({INPUT_SPARSE_FLOAT_VALUE_DATA, "layer_1", 50, 0}); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - - // "GpuSparseMatrix" as label is not supported - testLayerGrad(config, - "square_error", - 100, - /* trans */ false, - /* useGpu */ false); -} - -TEST(Layer, square_error_weighted) { - TestConfig config; - config.layerConfig.set_type("square_error"); - config.biasSize = 0; - config.testAccumulate = false; - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 10, 0}); - config.inputDefs.push_back({INPUT_DATA_TARGET, "layer_1", 10, 0}); - config.inputDefs.push_back({INPUT_DATA_TARGET, "layer_2", 1, 0}); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - - for (auto useGpu : {false, true}) { - testLayerGrad(config, "square_error", 100, /* trans */ false, useGpu); - } -} - -TEST(Layer, huber_two_class) { - TestConfig config; - config.layerConfig.set_type("huber"); - config.biasSize = 0; - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 1, 0}); - config.inputDefs.push_back({INPUT_LABEL, "layer_1", 2, 0}); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - - for (auto useGpu : {false, true}) { - testLayerGrad(config, "huber", 100, /* trans */ false, useGpu); - } -} - -void testExpandLayer(string trans_type, bool hasSubseq) { - TestConfig config; - config.layerConfig.set_type("expand"); - - config.inputDefs.push_back( - {trans_type == "non-seq" ? INPUT_DENSE_DIM_DATA : INPUT_SEQUENCE_DATA, - "layer_0", - 10, - 0}); - config.inputDefs.push_back( - {hasSubseq ? INPUT_HASSUB_SEQUENCE_DATA : INPUT_SEQUENCE_DATA, - "layer_1", - 10, - 0}); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - config.layerConfig.set_trans_type(trans_type); - LOG(INFO) << " trans_type=" << trans_type << " hasSubseq=" << hasSubseq; - - for (auto useGpu : {false, true}) { - testLayerGrad(config, "expand", 30, false, useGpu); - } -} - -TEST(Layer, ExpandLayer) { - testExpandLayer("non-seq", false); // non-seq expand to seq - testExpandLayer("non-seq", true); // non-seq expand to hasSubseq - testExpandLayer("seq", true); // seq expand to hasSubseq -} - -void testDegradeLayer(bool hasSubseq, string layer_type, string trans_type) { - TestConfig config; - config.layerConfig.set_type(layer_type); - config.layerConfig.set_size(10); - config.biasSize = 0; - - config.inputDefs.push_back( - {hasSubseq ? INPUT_HASSUB_SEQUENCE_DATA : INPUT_SEQUENCE_DATA, - "layer_0", - 10, - 0}); - config.layerConfig.add_inputs(); - config.layerConfig.set_trans_type(trans_type); - - auto testDegradeLayerGrad = [](TestConfig& config, string layer_type) { - for (auto useGpu : {false, true}) { - testLayerGrad(config, layer_type, 100, false, useGpu); - } - }; - - if (layer_type == "average") { - for (auto strategy : {"average", "sum", "squarerootn"}) { - LOG(INFO) << " hasSubseq=" << hasSubseq << " trans_type=" << trans_type - << " average_strategy=" << strategy; - config.layerConfig.set_average_strategy(strategy); - testDegradeLayerGrad(config, layer_type); - } - } else { - LOG(INFO) << " hasSubseq=" << hasSubseq << " trans_type=" << trans_type; - testDegradeLayerGrad(config, layer_type); - } -} - -TEST(Layer, MaxLayer) { - testDegradeLayer(false, "max", "non-seq"); // seq max to non-seq - testDegradeLayer(true, "max", "non-seq"); // hasSubseq max to non-seq - testDegradeLayer(true, "max", "seq"); // hasSubseq max to seq -} - -TEST(Layer, SequenceLastInstanceLayer) { - testDegradeLayer(false, - "seqlastins", - "non-seq"); // seq seqlastins to non-seq - testDegradeLayer(true, - "seqlastins", - "non-seq"); // hasSubseq seqlastins to non-seq - testDegradeLayer(true, "seqlastins", "seq"); // hasSubseq seqlastins to seq -} - -TEST(Layer, AverageLayer) { - testDegradeLayer(false, "average", "non-seq"); // seq average to non-seq - testDegradeLayer(true, "average", "non-seq"); // hasSubseq average to non-seq - testDegradeLayer(true, "average", "seq"); // hasSubseq average to seq -} - -TEST(Layer, SequenceConcatLayer) { - TestConfig config; - config.layerConfig.set_type("seqconcat"); - config.layerConfig.set_size(10); - config.biasSize = 0; - - config.inputDefs.push_back({INPUT_SEQUENCE_DATA, "layer_0", 10, 0}); - config.layerConfig.add_inputs(); - config.inputDefs.push_back({INPUT_SEQUENCE_DATA, "layer_1", 10, 0}); - config.layerConfig.add_inputs(); - - for (auto useGpu : {false, true}) { - testLayerGrad(config, "seqconcat", 100, false, useGpu); - } -} - -TEST(Layer, SequenceReshapeLayer) { - TestConfig config; - config.layerConfig.set_type("seqreshape"); - config.layerConfig.set_size(10); - - config.inputDefs.push_back({INPUT_SEQUENCE_DATA, "layer_0", 100, 0}); - config.layerConfig.add_inputs(); - - for (auto useGpu : {false, true}) { - testLayerGrad(config, "seqreshape", 100, false, useGpu); - } -} - -TEST(Layer, ConvShiftLayer) { - TestConfig config; - config.layerConfig.set_type("conv_shift"); - config.layerConfig.set_size(10); - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 10, 0}); - config.inputDefs.push_back({INPUT_DATA, "layer_1", 3, 0}); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - - // Not support GPU now - testLayerGrad(config, "conv_shift", 100, false, false); -} - -TEST(Layer, PowerLayer) { - TestConfig config; - config.layerConfig.set_type("power"); - config.layerConfig.set_size(10); - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 1, 0}); - config.inputDefs.push_back({INPUT_DATA, "layer_1", 10, 0}); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - - for (auto useGpu : {false, true}) { - testLayerGrad(config, "power", 100, false, useGpu); - } -} - -TEST(Layer, ConvexCombinationLayer) { - TestConfig config; - config.layerConfig.set_type("convex_comb"); - config.layerConfig.set_size(20); - config.biasSize = 0; - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 5, 0}); - config.inputDefs.push_back({INPUT_DATA, "layer_1", 100, 0}); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - - for (auto useGpu : {false, true}) { - testLayerGrad(config, "convex_comb", 100, false, useGpu); - } -} - -TEST(Layer, InterpolationLayer) { - TestConfig config; - config.layerConfig.set_type("interpolation"); - config.layerConfig.set_size(10); - config.biasSize = 0; - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 1, 0}); - config.inputDefs.push_back({INPUT_DATA, "layer_1", 10, 0}); - config.inputDefs.push_back({INPUT_DATA, "layer_2", 10, 0}); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - - for (auto useGpu : {false, true}) { - testLayerGrad(config, "interpolation", 100, false, useGpu); - } -} - -TEST(Layer, OuterProdLayer) { - TestConfig config; - config.layerConfig.set_type("out_prod"); - config.layerConfig.set_size(100); - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 10, 0}); - config.layerConfig.add_inputs(); - config.inputDefs.push_back({INPUT_DATA, "layer_1", 10, 0}); - config.layerConfig.add_inputs(); - - for (auto useGpu : {false, true}) { - testLayerGrad(config, "out_prod", 100, false, useGpu); - } -} - -TEST(Layer, SlopeInterceptLayer) { - TestConfig config; - config.layerConfig.set_type("slope_intercept"); - config.layerConfig.set_size(10); - config.layerConfig.set_slope(1.0); - config.layerConfig.set_intercept(0.1); - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 10, 0}); - config.layerConfig.add_inputs(); - - for (auto useGpu : {false, true}) { - testLayerGrad(config, "slope_intercept", 100, false, useGpu); - } -} - -TEST(Layer, ScalingLayer) { - TestConfig config; - config.layerConfig.set_type("scaling"); - config.layerConfig.set_size(10); - config.biasSize = 0; - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 1, 0}); - config.layerConfig.add_inputs(); - config.inputDefs.push_back({INPUT_DATA, "layer_1", 10, 0}); - config.layerConfig.add_inputs(); - - for (auto useGpu : {false, true}) { - testLayerGrad(config, "scaling", 100, false, useGpu); - } -} - -void testNormLayer(const string& normType, bool trans, bool useGpu) { - TestConfig config; - config.layerConfig.set_type("norm"); - config.layerConfig.set_active_type("relu"); - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 1568, 0}); - LayerInputConfig* input = config.layerConfig.add_inputs(); - NormConfig* norm = input->mutable_norm_conf(); - norm->set_norm_type(normType); - norm->set_channels(16); - norm->set_size(5); - norm->set_scale(0.001); - norm->set_pow(0.75); - norm->set_blocked(0); - norm->set_img_size(14); - norm->set_img_size_y(7); - norm->set_output_x(norm->img_size()); - norm->set_output_y(norm->img_size_y()); - if (norm->norm_type() == "cmrnorm" || - norm->norm_type() == "cmrnorm-projection") { - norm->set_scale(norm->scale() / norm->size()); - } else { - norm->set_scale(norm->scale() / (norm->size() * norm->size())); - } - - config.layerConfig.set_size(norm->output_x() * norm->output_y() * - norm->channels()); - config.biasSize = 0; - - testLayerGrad(config, "norm", 100, trans, useGpu); -} - -TEST(Layer, NormLayer) { - testNormLayer("cmrnorm-projection", /* trans= */ false, /* useGpu= */ true); - testNormLayer("cmrnorm-projection", /* trans= */ false, /* useGpu= */ false); -} - -void setPoolConfig(TestConfig* config, - PoolConfig* pool, - const string& poolType) { - (*config).biasSize = 0; - (*config).layerConfig.set_type("pool"); - (*config).layerConfig.set_num_filters(16); - - int kw = 3, kh = 3; - int pw = 0, ph = 0; - int sw = 2, sh = 2; - pool->set_pool_type(poolType); - pool->set_channels(16); - pool->set_size_x(kw); - pool->set_size_y(kh); - pool->set_start(0); - pool->set_padding(pw); - pool->set_padding_y(ph); - pool->set_stride(sw); - pool->set_stride_y(sh); - - int ow = outputSize(pool->img_size(), kw, pw, sw, /* caffeMode */ false); - int oh = outputSize(pool->img_size_y(), kh, ph, sh, /* caffeMode */ false); - pool->set_output_x(ow); - pool->set_output_y(oh); -} - -void testPoolLayer(const string& poolType, bool trans, bool useGpu) { - TestConfig config; - config.inputDefs.push_back({INPUT_DATA, "layer_0", 3136, 0}); - LayerInputConfig* input = config.layerConfig.add_inputs(); - PoolConfig* pool = input->mutable_pool_conf(); - - pool->set_img_size(14); - pool->set_img_size_y(14); - setPoolConfig(&config, pool, poolType); - config.layerConfig.set_size(pool->output_x() * pool->output_y() * - pool->channels()); - - testLayerGrad(config, "pool", 100, trans, useGpu); -} - -#ifndef PADDLE_ONLY_CPU -void testPoolLayer2(const string& poolType, bool trans, bool useGpu) { - TestConfig config; - config.inputDefs.push_back({INPUT_DATA, "layer_0", 3200, 0}); - LayerInputConfig* input = config.layerConfig.add_inputs(); - PoolConfig* pool = input->mutable_pool_conf(); - - pool->set_size_y(4); - pool->set_stride_y(3); - pool->set_img_size(10); - pool->set_img_size_y(20); - setPoolConfig(&config, pool, poolType); - pool->set_output_y((pool->img_size_y() - pool->start() - pool->size_y()) / - ((float)pool->stride_y()) + - 1.5); - config.layerConfig.set_size(pool->output_x() * pool->output_y() * - pool->channels()); - - testLayerGrad(config, "pool", 100, trans, useGpu); -} -#endif - -TEST(Layer, PoolLayer) { - testPoolLayer("avg-projection", /* trans= */ false, /* useGpu= */ false); - testPoolLayer("max-projection", /* trans= */ false, /* useGpu= */ false); - -#ifndef PADDLE_ONLY_CPU - testPoolLayer("avg-projection", /* trans= */ false, /* useGpu= */ true); - testPoolLayer("max-projection", /* trans= */ false, /* useGpu= */ true); - testPoolLayer("cudnn-max-pool", /* trans= */ false, /* useGpu= */ true); - testPoolLayer("cudnn-avg-pool", /* trans= */ false, /* useGpu= */ true); - testPoolLayer2("cudnn-max-pool", /* trans= */ false, /* useGpu= */ true); - testPoolLayer2("cudnn-avg-pool", /* trans= */ false, /* useGpu= */ true); -#endif -} - -void testSppLayer(const string& poolType, - const int pyramidHeight, - bool trans, - bool useGpu) { - TestConfig config; - config.layerConfig.set_type("spp"); - config.inputDefs.push_back({INPUT_DATA, "layer_0", 3200, 0}); - LayerInputConfig* input = config.layerConfig.add_inputs(); - SppConfig* sppConfig = input->mutable_spp_conf(); - sppConfig->set_pool_type(poolType); - sppConfig->set_pyramid_height(pyramidHeight); - ImageConfig* imageConfig = sppConfig->mutable_image_conf(); - imageConfig->set_channels(16); - imageConfig->set_img_size(10); - imageConfig->set_img_size_y(20); - int outputSize = (std::pow(4, sppConfig->pyramid_height()) - 1) / (4 - 1); - config.layerConfig.set_size(outputSize * imageConfig->channels()); - testLayerGrad(config, "spp", 100, trans, useGpu); -} - -TEST(Layer, SpatialPyramidPoolLayer) { - for (auto useGpu : {false, true}) { - for (auto pyramidHeight : {1, 2, 3}) { - testSppLayer("avg-projection", pyramidHeight, false, useGpu); - testSppLayer("max-projection", pyramidHeight, false, useGpu); - } - } -} - -TEST(Layer, rankCostLayer) { - TestConfig config; - config.layerConfig.set_type("rank-cost"); - config.biasSize = 0; - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 1, 0}); - config.inputDefs.push_back({INPUT_DATA, "layer_1", 1, 0}); - config.inputDefs.push_back({INPUT_DATA_TARGET, "layer_2", 1, 0}); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - - for (auto useGpu : {false, true}) { - testLayerGrad(config, "rank-cost", 100, false, useGpu); - } -} - -TEST(Layer, sumCostLayer) { - TestConfig config; - config.layerConfig.set_type("sum_cost"); - config.biasSize = 0; - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 1, 0}); - config.layerConfig.add_inputs(); - - for (auto useGpu : {false, true}) { - testLayerGrad(config, "sum_cost", 100, false, useGpu); - } -} - -TEST(Layer, weightedRankCostLayer) { - TestConfig config; - config.layerConfig.set_type("rank-cost"); - config.biasSize = 0; - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 1, 0}); - config.inputDefs.push_back({INPUT_DATA, "layer_1", 1, 0}); - config.inputDefs.push_back({INPUT_DATA_TARGET, "layer_2", 1, 0}); - config.inputDefs.push_back({INPUT_DATA_TARGET, "layer_3", 1, 0}); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - - for (auto useGpu : {false, true}) { - testLayerGrad(config, "weighted-rank-cost", 100, false, useGpu); - } -} - -TEST(Layer, TensorLayer) { - TestConfig config; - config.layerConfig.set_type("tensor"); - config.layerConfig.set_size(10); - config.layerConfig.set_active_type("sigmoid"); - config.biasSize = config.layerConfig.size(); - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 5, 250}); - config.inputDefs.push_back({INPUT_DATA, "layer_1", 5, 0}); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - - for (auto useGpu : {false, true}) { - testLayerGrad(config, "tensor", 100, false, useGpu); - } -} - -TEST(Layer, RecurrentLayer) { - TestConfig config; - config.layerConfig.set_type("recurrent"); - config.layerConfig.set_size(4); - config.layerConfig.set_active_type("tanh"); - config.biasSize = 4; - - config.inputDefs.push_back( - {INPUT_SEQUENCE_DATA, "layer_0", /* dim= */ 4, /* paraSize= */ 16}); - config.layerConfig.add_inputs(); - - for (auto useGpu : {false, true}) { - for (auto reversed : {false, true}) { - config.layerConfig.set_reversed(reversed); - config.testState = !reversed; - testLayerGrad(config, "recurrent", 50, /* trans= */ false, useGpu); - } - } -} - -TEST(Layer, LstmLayer) { - TestConfig config; - config.layerConfig.set_type("lstmemory"); - config.layerConfig.set_size(4); - config.layerConfig.set_active_type("tanh"); - config.layerConfig.set_active_state_type("sigmoid"); - config.layerConfig.set_active_gate_type("sigmoid"); - config.biasSize = 28; - - config.inputDefs.push_back( - {INPUT_SEQUENCE_DATA, "layer_0", /* dim= */ 16, /* paraSize= */ 64}); - config.layerConfig.add_inputs(); - - for (auto useGpu : {false, true}) { - for (auto reversed : {false, true}) { - config.layerConfig.set_reversed(reversed); - config.testState = !reversed; - testLayerGrad(config, "lstmemory", 100, /* trans= */ false, useGpu); - } - } - for (auto useGpu : {true}) { - config.testBatchState = true; - config.layerConfig.set_reversed(false); - testLayerGrad(config, "lstmemory", 10, /* trans= */ false, useGpu); - } -} - -TEST(Layer, MDLstmLayer) { - TestConfig config; - config.layerConfig.set_type("mdlstmemory"); - config.layerConfig.set_size(4); - config.layerConfig.set_active_type("sigmoid"); - config.layerConfig.set_active_state_type("sigmoid"); - config.layerConfig.set_active_gate_type("sigmoid"); - config.biasSize = 4 * 9; - - config.inputDefs.push_back( - {INPUT_SEQUENCE_MDIM_DATA, "layer_0", 4 * 5, 4 * 4 * 5}); - config.layerConfig.add_inputs(); - config.layerConfig.add_directions(true); - config.layerConfig.add_directions(true); - - for (auto useGpu : {false, true}) { - for (int i = 0; i < 2; i++) { - for (int j = 0; j < 2; j++) { - config.layerConfig.set_directions(0, bool(i)); - config.layerConfig.set_directions(1, bool(j)); - testLayerGrad(config, "mdlstmemory", 100, false, useGpu); - } - } - } -} - -TEST(Layer, ParameterReluLayer) { - auto testParameterReluLayer = [&](size_t inputSize, size_t channels) { - TestConfig config; - config.layerConfig.set_type("prelu"); - config.inputDefs.push_back({INPUT_DATA, "layer_0", inputSize, channels}); - config.layerConfig.add_inputs(); - config.layerConfig.set_size(inputSize); - config.layerConfig.set_partial_sum(inputSize / - channels); // size of feature map - for (auto useGpu : {false, true}) { - testLayerGrad(config, "prelu", 100, false, useGpu); - } - }; - - testParameterReluLayer(192, 1); - testParameterReluLayer(192, 3); - testParameterReluLayer(192, 192); -} - -TEST(Layer, ResizeLayer) { - TestConfig config; - config.biasSize = 0; - config.layerConfig.set_type("resize"); - config.layerConfig.set_size(64); - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 16, 0}); - config.layerConfig.add_inputs(); - - for (auto useGpu : {false, true}) { - testLayerGrad(config, "resize", 100, false, useGpu); - } -} - -TEST(Layer, NCELayer) { - TestConfig config; - size_t numClasses = 4; - config.layerConfig.set_type("nce"); - config.layerConfig.set_size(1); - config.layerConfig.set_active_type("sigmoid"); - config.layerConfig.set_num_classes(numClasses); - config.biasSize = numClasses; - - config.inputDefs.push_back( - {INPUT_DATA, "layer_0", /* dim= */ 16, /* paraSize= */ 16 * numClasses}); - config.inputDefs.push_back( - {INPUT_LABEL, "label", /* dim= */ numClasses, /* paraSize= */ 0}); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - - for (auto withWeight : {false, true}) { - if (withWeight) { - config.inputDefs.push_back( - {INPUT_DATA_TARGET, "weight", /* dim= */ 1, /* paraSize= */ 0}); - config.layerConfig.add_inputs(); - } - - for (auto isIdLabel : {false, true}) { - config.inputDefs[1] = { - isIdLabel ? INPUT_LABEL : INPUT_SPARSE_NON_VALUE_DATA, - "label", - /* dim= */ numClasses, - /* paraSize= */ 0}; - - for (auto withDist : {false, true}) { - config.layerConfig.clear_neg_sampling_dist(); - if (withDist) { - double sum = 0; - for (size_t i = 0; i < numClasses; ++i) { - real p = rand(); // NOLINT use rand_r - config.layerConfig.add_neg_sampling_dist(p); - sum += p; - } - for (size_t i = 0; i < numClasses; ++i) { - real p = config.layerConfig.neg_sampling_dist(i) / sum; - config.layerConfig.set_neg_sampling_dist(i, p); - } - } - LOG(INFO) << "NCELayer " - << " isIdLabel=" << isIdLabel << " withWeight=" << withWeight - << " withDist=" << withDist; - // Not support GPU now - testLayerGrad(config, - "nce", - 100, - /* trans= */ false, - /* useGpu */ false); - } - } - } -} - -TEST(Layer, GatedRecurrentLayer) { - TestConfig config; - config.layerConfig.set_type("gated_recurrent"); - config.layerConfig.set_size(4); - config.layerConfig.set_active_type("sigmoid"); - config.layerConfig.set_active_gate_type("sigmoid"); - config.biasSize = 12; - - config.inputDefs.push_back( - {INPUT_SEQUENCE_DATA, "layer_0", /* dim= */ 12, /* paraSize= */ 48}); - config.layerConfig.add_inputs(); - - for (auto useGpu : {false, true}) { - for (auto reversed : {false, true}) { - config.layerConfig.set_reversed(reversed); - config.testState = !reversed; - testLayerGrad(config, "gated_recurrent", 100, /* trans= */ false, useGpu); - } - } -} - -TEST(Layer, GruStepLayer) { - TestConfig config; - config.layerConfig.set_type("gru_step"); - config.layerConfig.set_size(4); - config.layerConfig.set_active_type("sigmoid"); - config.layerConfig.set_active_gate_type("sigmoid"); - config.biasSize = 12; - - config.inputDefs.push_back( - {INPUT_DATA, "layer_0", /* dim= */ 12, /* paraSize= */ 48}); - config.inputDefs.push_back( - {INPUT_DATA, "layer_1", /* dim= */ 4, /* paraSize= */ 0}); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - - for (auto useGpu : {false, true}) { - testLayerGrad(config, "gruStep", 100, /* trans= */ false, useGpu); - } -} - -TEST(Layer, LstmStepLayer) { - TestConfig config; - config.layerConfig.set_type("lstm_step"); - config.layerConfig.set_size(4); - config.layerConfig.set_active_type("sigmoid"); - config.layerConfig.set_active_state_type("sigmoid"); - config.layerConfig.set_active_gate_type("sigmoid"); - config.biasSize = 12; - config.testAccumulate = false; - - config.inputDefs.push_back( - {INPUT_DATA, "layer_0", /* dim= */ 16, /* paraSize= */ 0}); - config.inputDefs.push_back( - {INPUT_DATA, "layer_1", /* dim= */ 4, /* paraSize= */ 0}); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - - for (auto useGpu : {false, true}) { - testLayerGrad(config, "lstmStep", 100, /* trans= */ false, useGpu); - } -} - -void testBatchNormLayer(const string& type, bool trans, bool useGpu) { - TestConfig config; - const int CHANNELS = 10; - const int IMG_SIZE = 16; - const int IMG_SIZE_Y = 8; - size_t size = CHANNELS * IMG_SIZE * IMG_SIZE_Y; - config.layerConfig.set_type(type); - config.layerConfig.set_size(size); - config.layerConfig.set_active_type("sigmoid"); - config.biasSize = CHANNELS; - config.inputDefs.push_back({INPUT_DATA, - "layer_0", - /* dim= */ size, - /* paraSize= */ CHANNELS}); - - config.inputDefs.push_back({INPUT_DATA, "layer_1_running_mean", 1, CHANNELS}); - config.inputDefs.back().isStatic = true; - config.inputDefs.push_back({INPUT_DATA, "layer_2_running_var", 1, CHANNELS}); - config.inputDefs.back().isStatic = true; - - LayerInputConfig* input = config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - - ImageConfig* img_conf = input->mutable_image_conf(); - img_conf->set_channels(CHANNELS); - img_conf->set_img_size(IMG_SIZE); - img_conf->set_img_size_y(IMG_SIZE_Y); - - testLayerGrad(config, - "batch_norm", - 64, - /* trans= */ trans, - useGpu, - /* useWeight */ true); -} - -TEST(Layer, BatchNormalizationLayer) { - testBatchNormLayer("batch_norm", false, false); -#ifndef PADDLE_ONLY_CPU - testBatchNormLayer("batch_norm", false, true); - if (hl_get_cudnn_lib_version() >= int(4000)) { - testBatchNormLayer("cudnn_batch_norm", false, true); - } -#endif -} - -TEST(Operator, conv) { - TestConfig config; - const int NUM_FILTERS = 16; - const int FILTER_SIZE = 2; - const int FILTER_SIZE_Y = 3; - const int CHANNELS = 3; - const int IMAGE_SIZE = 16; - const int IMAGE_SIZE_Y = 8; - OperatorConfig& operatorConf = *config.layerConfig.add_operator_confs(); - operatorConf.set_type("conv"); - ConvConfig* conv = operatorConf.mutable_conv_conf(); - operatorConf.set_num_filters(NUM_FILTERS); - conv->set_filter_size(FILTER_SIZE); - conv->set_filter_size_y(FILTER_SIZE_Y); - conv->set_channels(CHANNELS); - conv->set_padding(0); - conv->set_padding_y(1); - conv->set_stride(2); - conv->set_stride_y(2); - conv->set_groups(1); - conv->set_filter_channels(conv->channels() / conv->groups()); - conv->set_img_size(IMAGE_SIZE); - conv->set_img_size_y(IMAGE_SIZE_Y); - conv->set_output_x(outputSize(conv->img_size(), - conv->filter_size(), - conv->padding(), - conv->stride(), - /* caffeMode */ true)); - conv->set_output_y(outputSize(conv->img_size_y(), - conv->filter_size_y(), - conv->padding_y(), - conv->stride_y(), - /* caffeMode */ true)); - config.layerConfig.set_size(conv->output_x() * conv->output_y() * - NUM_FILTERS); - - config.inputDefs.push_back( - {INPUT_DATA, "layer_0", IMAGE_SIZE * IMAGE_SIZE_Y * CHANNELS, 0}); - config.inputDefs.push_back( - {INPUT_DATA, - "layer_1", - FILTER_SIZE * FILTER_SIZE_Y * CHANNELS * NUM_FILTERS, - 0}); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - - testOperatorGrad(config, operatorConf, 100, /*useGpu*/ true, false); -} - -TEST(Layer, FeatureMapExpandLayer) { - TestConfig config; - config.layerConfig.set_type("featmap_expand"); - const int CHANNELS = 10; - const int INPUT_SIZE = 100; - config.layerConfig.set_size(INPUT_SIZE * CHANNELS); - config.layerConfig.set_num_filters(CHANNELS); - config.inputDefs.push_back({INPUT_SEQUENCE_DATA, - "layer_0", - /* dim= */ INPUT_SIZE, - /* paraSize= */ 0}); - config.layerConfig.add_inputs(); - for (auto useGpu : {false, true}) { - testLayerGrad(config, - "featmap_expand", - /*batch_size*/ 100, - /* trans= */ false, - useGpu, - /* useWeight */ true); - } -} - -TEST(Layer, MultiplexLayer) { - TestConfig config; - const int LAYER_SIZE = 100; - config.layerConfig.set_type("multiplex"); - config.layerConfig.set_size(LAYER_SIZE); - - config.inputDefs.push_back({INPUT_LABEL, "layer_0", 2, 0}); - config.inputDefs.push_back( - {INPUT_DATA, "layer_1", /* dim= */ LAYER_SIZE, /* paraSize= */ 0}); - config.inputDefs.push_back( - {INPUT_DATA, "layer_2", /* dim= */ LAYER_SIZE, /* paraSize= */ 0}); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); - config.layerConfig.add_inputs(); + PadConfig* pad = input->mutable_pad_conf(); + ImageConfig* image = pad->mutable_image_conf(); + + image->set_channels(c); + image->set_img_size(h); + image->set_img_size_y(w); + pad->add_pad_c(1); + pad->add_pad_c(2); + pad->add_pad_h(2); + pad->add_pad_h(3); + pad->add_pad_w(3); + pad->add_pad_w(5); for (auto useGpu : {false, true}) { - testLayerGrad(config, "multiplex", 512, /* trans= */ false, useGpu); + testLayerGrad(config, "pad", 10, false, useGpu); } } diff --git a/proto/ModelConfig.proto b/proto/ModelConfig.proto index 3a9d339976..0456404832 100644 --- a/proto/ModelConfig.proto +++ b/proto/ModelConfig.proto @@ -255,6 +255,13 @@ message PriorBoxConfig { repeated float variance = 4; } +message PadConfig { + required ImageConfig image_conf = 1; + repeated uint32 pad_c = 2; + repeated uint32 pad_h = 3; + repeated uint32 pad_w = 4; +} + message LayerInputConfig { required string input_layer_name = 1; optional string input_parameter_name = 2; @@ -271,6 +278,7 @@ message LayerInputConfig { optional MaxOutConfig maxout_conf = 11; optional SppConfig spp_conf = 12; optional PriorBoxConfig priorbox_conf = 13; + optional PadConfig pad_conf = 14; } message LayerConfig { diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 674b5ac58b..6e5922166d 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -493,6 +493,7 @@ class Input(Cfg): block_expand=None, maxout=None, spp=None, + pad=None, format=None, nnz=None, is_static=None, @@ -844,6 +845,12 @@ class SpatialPyramidPool(Cfg): self.add_keys(locals()) +@config_class +class Pad(Cfg): + def __init__(self, channels, pad_c, pad_h, pad_w): + self.add_keys(locals()) + + @config_class class Norm(Cfg): def __init__(self, @@ -1842,6 +1849,25 @@ class SpatialPyramidPoolLayer(LayerBase): self.set_cnn_layer(name, 1, output_x, spp_conf.image_conf.channels) +@config_layer('pad') +class PadLayer(LayerBase): + def __init__(self, name, inputs, **xargs): + super(PadLayer, self).__init__(name, 'pad', 0, inputs=inputs, **xargs) + pad = self.inputs[0].pad + self.config.inputs[0].pad_conf.pad_c.extend(pad.pad_c) + self.config.inputs[0].pad_conf.pad_h.extend(pad.pad_h) + self.config.inputs[0].pad_conf.pad_w.extend(pad.pad_w) + + input_layer = self.get_input_layer(0) + image_conf = self.config.inputs[0].pad_conf.image_conf + parse_image(pad, input_layer.name, image_conf) + out_ch = pad.channels + pad.pad_c[0] + pad.pad_c[1] + out_h = image_conf.img_size_y + pad.pad_h[0] + pad.pad_h[1] + out_w = image_conf.img_size + pad.pad_w[0] + pad.pad_w[1] + self.set_cnn_layer(name, out_h, out_w, out_ch) + self.config.size = out_ch * out_h * out_w + + @config_layer('batch_norm') class BatchNormLayer(LayerBase): layer_type = 'batch_norm' diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 9b6e5774bc..56c335a050 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -170,6 +170,7 @@ class LayerType(object): BLOCK_EXPAND = "blockexpand" MAXOUT = "maxout" SPP_LAYER = "spp" + PAD_LAYER = "pad" PRINT_LAYER = "print" PRIORBOX_LAYER = "priorbox" @@ -3488,9 +3489,6 @@ def conv_projection(input, groups=1, param_attr=None): """ - ConvProjection with a layer as input. - It performs element-wise multiplication with weight. - Different from img_conv_layer and conv_op, conv_projection is an Projection, which can be used in mixed_layer and conat_layer. It use cudnn to implement conv and only support GPU mode. @@ -3499,7 +3497,7 @@ def conv_projection(input, .. code-block:: python - proj = conv_projection(img=input1, + proj = conv_projection(input=input1, filter_size=3, num_filters=64, num_channels=64) @@ -3582,6 +3580,84 @@ def conv_projection(input, return proj +@wrap_name_default("pad") +@layer_support() +def pad_layer(input, + pad_c=None, + pad_h=None, + pad_w=None, + name=None, + layer_attr=None): + """ + This operation pads zeros to the input data according to pad_c,pad_h + and pad_w. pad_c, pad_h, pad_w specifies the which dimension and size + of padding. And the input data shape is NCHW. + + For example, pad_c=[2,3] means padding 2 zeros before the + input data and 3 zeros after the input data in channel dimension. + pad_h means padding zeros in height dimension. pad_w means padding zeros + in width dimension. + + .. code-block:: python + + pad = pad_layer(input=ipt, + pad_c=[4,4], + pad_h=[0,0], + pad_w=[2,2]) + + :param input: layer's input. + :type input: LayerOutput + :param pad_c: padding size in channel dimension. + :type pad_c: list|None + :param pad_h: padding size in height dimension. + :type pad_h: list|None + :param pad_w: padding size in width dimension. + :type pad_w: list|None + :param layer_attr: Extra Layer Attribute. + :type layer_attr: ExtraLayerAttribute + :param name: layer name. + :type name: basestring + :return: LayerOutput object. + :rtype: LayerOutput + """ + if pad_c is not None: + assert isinstance(pad_c, collections.Sequence) and len(pad_c) == 2 + else: + pad_c = [0, 0] + + if pad_h is not None: + assert isinstance(pad_h, collections.Sequence) and len(pad_h) == 2 + else: + pad_h = [0, 0] + + if pad_w is not None: + assert isinstance(pad_w, collections.Sequence) and len(pad_w) == 2 + else: + pad_w = [0, 0] + + assert input.num_filters is not None + in_ch = input.num_filters + out_ch = in_ch + pad_c[0] + pad_c[1] + + l = Layer( + name=name, + type=LayerType.PAD_LAYER, + inputs=Input( + input.name, + pad=Pad( + channels=in_ch, + pad_c=pad_c, + pad_h=pad_h, + pad_w=pad_w, )), + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + layer_type=LayerType.PAD_LAYER, + parents=[input], + num_filters=out_ch, + size=l.config.size) + + @wrap_name_default() @layer_support() def conv_shift_layer(a, b, name=None, layer_attr=None): diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_pad.py b/python/paddle/trainer_config_helpers/tests/configs/test_pad.py new file mode 100644 index 0000000000..bb5f13410d --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_pad.py @@ -0,0 +1,21 @@ +from paddle.trainer_config_helpers import * + +settings(batch_size=1000, learning_rate=1e-5) + +data = data_layer(name='data', size=2304, height=48, width=42) + +conv = img_conv_layer( + input=data, + filter_size=3, + num_channels=1, + num_filters=16, + padding=1, + act=LinearActivation(), + bias_attr=True) + +pool = img_pool_layer( + input=conv, num_channels=8, pool_size=2, stride=2, pool_type=MaxPooling()) + +pad = pad_layer(input=pool, pad_c=[2, 3], pad_h=[1, 2], pad_w=[3, 1]) + +outputs(pad) -- GitLab