conv_mkldnn_op.cc 52.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License. */

#include "paddle/fluid/operators/conv_op.h"
#include "paddle/fluid/platform/mkldnn_helper.h"
X
xiaolil1 已提交
17
#include "paddle/fluid/framework/data_layout_transform.h"
X
xiaolil1 已提交
18 19
#include <unordered_map>
#include <map>
20 21 22 23

namespace paddle {
namespace operators {

24 25 26 27 28 29 30 31
using framework::DataLayout;
using mkldnn::memory;
using mkldnn::primitive;
using mkldnn::reorder;
using mkldnn::stream;
using platform::to_void_cast;
using platform::GetMKLDNNFormat;

32 33 34 35 36 37 38 39 40 41
class ConvMKLDNNHandler : public platform::MKLDNNHandler {
 public:
  ConvMKLDNNHandler(
      std::shared_ptr<mkldnn::convolution_forward::primitive_desc> conv_pd,
      const platform::MKLDNNDeviceContext& dev_ctx, mkldnn::engine engine,
      const std::string& base_key)
      : platform::MKLDNNHandler(dev_ctx, engine, base_key) {
    conv_pd_ = conv_pd;
  }

42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58
  ConvMKLDNNHandler(
      std::shared_ptr<mkldnn::convolution_forward::primitive_desc> conv_pd,
      std::shared_ptr<mkldnn::convolution_backward_data::primitive_desc>
          conv_bwd_data_pd,
      std::shared_ptr<mkldnn::convolution_backward_weights::primitive_desc>
          conv_bwd_weights_pd,
      const platform::MKLDNNDeviceContext& dev_ctx, mkldnn::engine engine,
      const std::string& base_key)
      : platform::MKLDNNHandler(dev_ctx, engine, base_key),
        conv_pd_(conv_pd),
        conv_bwd_weights_pd_(conv_bwd_weights_pd),
        conv_bwd_data_pd_(conv_bwd_data_pd) {
    // If we are in Grad operatgor then update a key with BWD suffix to
    // distinguish from FWD memory primitives
    key_ += "-BWD";
  }

59
  size_t GetDstMemorySize() const {
60 61
    return conv_pd_->dst_primitive_desc().get_size();
  }
Z
Zhang, Guoming 已提交
62 63 64 65 66
  
  mkldnn::memory::format GetDstFormat() const {
    return static_cast<mkldnn::memory::format>(
        conv_pd_->dst_primitive_desc().desc().data.format);
  }
67

68
  size_t GetDiffWeightsMemorySize() const {
69 70 71
    return conv_bwd_weights_pd_->diff_weights_primitive_desc().get_size();
  }

72
  size_t GetDiffSourceMemorySize() const {
73 74 75
    return conv_bwd_data_pd_->diff_src_primitive_desc().get_size();
  }

76 77
  std::shared_ptr<mkldnn::memory> AcquireSrcMemoryFromWeightsPrimitive(
      const std::shared_ptr<mkldnn::memory> user_memory_p,
G
gongweibao 已提交
78
      std::vector<mkldnn::primitive>& pipeline) {  // NOLINT
79 80 81 82 83 84 85 86
    auto src_pd = conv_bwd_weights_pd_->src_primitive_desc();
    auto user_pd = user_memory_p->get_primitive_desc();
    return this->AcquireMemory(src_pd, user_pd, user_memory_p,
                               "@weights-src_mem_p", pipeline);
  }

  std::shared_ptr<mkldnn::memory> AcquireDiffDstMemoryFromWeightsPrimitive(
      const std::shared_ptr<mkldnn::memory> user_memory_p,
G
gongweibao 已提交
87
      std::vector<mkldnn::primitive>& pipeline) {  // NOLINT
88 89 90 91 92 93 94 95 96 97 98 99 100 101 102
    auto diff_dst_pd = conv_bwd_weights_pd_->diff_dst_primitive_desc();
    auto user_pd = user_memory_p->get_primitive_desc();
    return this->AcquireMemory(diff_dst_pd, user_pd, user_memory_p,
                               "@weights-diff_dst_mem_p", pipeline);
  }

  std::shared_ptr<mkldnn::memory> AcquireDiffWeightsMemoryFromWeightsPrimitive(
      void* ptr) {
    return this->AcquireMemoryFromPrimitive(
        conv_bwd_weights_pd_->diff_weights_primitive_desc(), ptr,
        "@diff_weights_mem_p");
  }

  std::shared_ptr<mkldnn::memory> AcquireDiffDstMemoryFromDataPrimitive(
      const std::shared_ptr<mkldnn::memory> user_memory_p,
G
gongweibao 已提交
103
      std::vector<mkldnn::primitive>& pipeline) {  // NOLINT
104 105 106 107 108 109 110 111
    auto diff_dst_pd = conv_bwd_data_pd_->diff_dst_primitive_desc();
    auto user_pd = user_memory_p->get_primitive_desc();
    return this->AcquireMemory(diff_dst_pd, user_pd, user_memory_p,
                               "@data-diff_dst_mem_p", pipeline);
  }

  std::shared_ptr<mkldnn::memory> AcquireWeightsMemoryFromDataPrimitive(
      const std::shared_ptr<mkldnn::memory> user_weights_memory_p,
G
gongweibao 已提交
112
      std::vector<mkldnn::primitive>& pipeline) {  // NOLINT
113 114 115 116 117 118
    auto weights_pd = conv_bwd_data_pd_->weights_primitive_desc();
    auto user_pd = user_weights_memory_p->get_primitive_desc();
    return this->AcquireMemory(weights_pd, user_pd, user_weights_memory_p,
                               "@data-weights_mem_p", pipeline);
  }

Z
Zhang, Guoming 已提交
119 120 121 122 123 124 125 126 127 128 129 130 131 132 133

  std::shared_ptr<mkldnn::memory> AcquireResidualDataMemory(
      const mkldnn::memory::desc& md, void* ptr) {
    return this->AcquireMemory(md, ptr, "@user_residual_data_mem_p");
  }

  std::shared_ptr<mkldnn::memory> AcquireDstMemoryFromResidualDataMemory(
      const std::shared_ptr<mkldnn::memory>& user_residual_memory_p,
      void* dst_ptr,
      std::vector<mkldnn::primitive>& pipeline) {  // NOLINT
    return this->AcquireMemory(user_residual_memory_p,
                               this->AcquireDstMemoryFromPrimitive(dst_ptr),
                               "@residual_data_mem_p", pipeline);
  }
  
134 135 136 137 138 139
  std::shared_ptr<mkldnn::memory> AcquireDiffSrcMemoryFromDataPrimitive(
      void* ptr) {
    return this->AcquireMemoryFromPrimitive(
        conv_bwd_data_pd_->diff_src_primitive_desc(), ptr, "@diff_src_mem_p");
  }

140 141 142 143 144 145 146
  std::shared_ptr<mkldnn::memory> AcquireDstMemoryFromPrimitive(void* ptr) {
    return this->AcquireMemoryFromPrimitive(conv_pd_->dst_primitive_desc(), ptr,
                                            "@dst_mem_p");
  }

  std::shared_ptr<mkldnn::memory> AcquireSrcMemoryFromPrimitive(
      const std::shared_ptr<mkldnn::memory> user_memory_p,
147
      std::vector<mkldnn::primitive>& pipeline) {  // NOLINT
148
    auto src_pd = conv_pd_->src_primitive_desc();
149
    auto user_pd = user_memory_p->get_primitive_desc();
150 151 152 153 154 155
    return this->AcquireMemory(src_pd, user_pd, user_memory_p, "@src_mem_p",
                               pipeline);
  }

  std::shared_ptr<mkldnn::memory> AcquireWeightsMemoryFromPrimitive(
      const std::shared_ptr<mkldnn::memory> user_weights_memory_p,
K
Krzysztof Binias 已提交
156
      std::vector<mkldnn::primitive>& pipeline,  // NOLINT
X
xiaolil1 已提交
157 158 159 160
      bool is_persistent = false,
      bool is_INT8 = false,
      std::vector<float> scale_data = {1.0f},
      int mask = 0) { 
161 162 163 164
    auto user_weights_pd = user_weights_memory_p->get_primitive_desc();
    auto weights_pd = conv_pd_->weights_primitive_desc();
    return this->AcquireMemory(weights_pd, user_weights_pd,
                               user_weights_memory_p, "@weights_mem_p",
X
xiaolil1 已提交
165 166
                               pipeline, is_persistent,
                               is_INT8, scale_data, mask);
167 168
  }

169 170
  std::shared_ptr<mkldnn::memory> AcquireBiasMemoryFromPrimitive(
      const std::shared_ptr<mkldnn::memory> user_bias_memory_p,
X
xiaolil1 已提交
171
      std::vector<mkldnn::primitive>& pipeline,
X
xiaolil1 已提交
172
      bool is_persistent = false,
X
xiaolil1 已提交
173 174 175
      bool is_INT8 = false,
      std::vector<float> scale_data = {1.0f},
      int mask = 0) {  // NOLINT
176 177 178
    auto user_bias_pd = user_bias_memory_p->get_primitive_desc();
    auto bias_pd = conv_pd_->bias_primitive_desc();
    return this->AcquireMemory(bias_pd, user_bias_pd, user_bias_memory_p,
X
xiaolil1 已提交
179 180
                               "@bias_mem_p", pipeline, is_persistent,
                               is_INT8, scale_data, mask);
181 182
  }

183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203
  std::shared_ptr<mkldnn::convolution_forward> AcquireConvolution(
      std::shared_ptr<mkldnn::memory> src_memory_p,
      std::shared_ptr<mkldnn::memory> weights_memory_p,
      std::shared_ptr<mkldnn::memory> dst_memory_p) {
    auto prim_key = key_ + "@conv_p";
    auto conv_p = std::static_pointer_cast<mkldnn::convolution_forward>(
        dev_ctx_.GetBlob(prim_key));
    PADDLE_ENFORCE((conv_p != nullptr) || (is_reusing_ == false),
                   "Fail to find convolution primitive in device context");
    if (conv_p == nullptr) {
      conv_p = std::make_shared<mkldnn::convolution_forward>(
          *conv_pd_, *(src_memory_p), *(weights_memory_p.get()),
          *(dst_memory_p.get()));

      dev_ctx_.SetBlob(prim_key, conv_p);
    } else {
      is_reusing_ = true;
    }
    return conv_p;
  }

204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225
  std::shared_ptr<mkldnn::convolution_forward> AcquireConvolution(
      std::shared_ptr<mkldnn::memory> src_memory_p,
      std::shared_ptr<mkldnn::memory> weights_memory_p,
      std::shared_ptr<mkldnn::memory> bias_memory_p,
      std::shared_ptr<mkldnn::memory> dst_memory_p) {
    auto prim_key = key_ + "@conv_p";
    auto conv_p = std::static_pointer_cast<mkldnn::convolution_forward>(
        dev_ctx_.GetBlob(prim_key));
    PADDLE_ENFORCE((conv_p != nullptr) || (is_reusing_ == false),
                   "Fail to find convolution primitive in device context");
    if (conv_p == nullptr) {
      conv_p = std::make_shared<mkldnn::convolution_forward>(
          *conv_pd_, *(src_memory_p), *(weights_memory_p.get()),
          *(bias_memory_p.get()), *(dst_memory_p.get()));

      dev_ctx_.SetBlob(prim_key, conv_p);
    } else {
      is_reusing_ = true;
    }
    return conv_p;
  }

226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273
  std::shared_ptr<mkldnn::convolution_backward_weights>
  AcquireConvolutionBackwardWeights(
      std::shared_ptr<mkldnn::memory> src_memory_p,
      std::shared_ptr<mkldnn::memory> diff_dst_memory_p,
      std::shared_ptr<mkldnn::memory> diff_weights_memory_p) {
    auto prim_key = key_ + "@conv_bwd_weights_p";
    auto conv_bwd_weights_p =
        std::static_pointer_cast<mkldnn::convolution_backward_weights>(
            dev_ctx_.GetBlob(prim_key));
    PADDLE_ENFORCE(
        (conv_bwd_weights_p != nullptr) || (is_reusing_ == false),
        "Fail to find convolution bwd weights primitive in device context");
    if (conv_bwd_weights_p == nullptr) {
      // create backward conv primitive for weights
      conv_bwd_weights_p =
          std::make_shared<mkldnn::convolution_backward_weights>(
              *conv_bwd_weights_pd_, *src_memory_p, *diff_dst_memory_p,
              *diff_weights_memory_p);
      dev_ctx_.SetBlob(prim_key, conv_bwd_weights_p);
    } else {
      is_reusing_ = true;
    }
    return conv_bwd_weights_p;
  }

  std::shared_ptr<mkldnn::convolution_backward_data>
  AcquireConvolutionBackwardData(
      std::shared_ptr<mkldnn::memory> diff_dst_memory_p,
      std::shared_ptr<mkldnn::memory> weights_memory_p,
      std::shared_ptr<mkldnn::memory> diff_src_memory_p) {
    auto prim_key = key_ + "@conv_bwd_data_p";
    auto conv_bwd_data_p =
        std::static_pointer_cast<mkldnn::convolution_backward_data>(
            dev_ctx_.GetBlob(prim_key));
    PADDLE_ENFORCE(
        (conv_bwd_data_p != nullptr) || (is_reusing_ == false),
        "Fail to find convolution bwd data primitive in device context");
    if (conv_bwd_data_p == nullptr) {
      conv_bwd_data_p = std::make_shared<mkldnn::convolution_backward_data>(
          *conv_bwd_data_pd_, *diff_dst_memory_p, *weights_memory_p,
          *diff_src_memory_p);
      dev_ctx_.SetBlob(prim_key, conv_bwd_data_p);
    } else {
      is_reusing_ = true;
    }
    return conv_bwd_data_p;
  }

274 275
  // Generate keys for storing/retriving primitives for this operator
  // TODO(jczaja): Make hashing function more optimial
G
gongweibao 已提交
276 277 278 279 280 281
  static std::string GetHash(memory::dims& input_dims,     // NOLINT
                             memory::dims& weights_dims,   // NOLINT
                             std::vector<int>& strides,    // NOLINT
                             std::vector<int>& paddings,   // NOLINT
                             std::vector<int>& dilations,  // NOLINT
                             int groups, const std::string& suffix) {
282 283 284 285 286 287 288
    return dims2str(input_dims) + dims2str(weights_dims) + dims2str(strides) +
           dims2str(paddings) + dims2str(dilations) + std::to_string(groups) +
           suffix;
  }

 private:
  std::shared_ptr<mkldnn::convolution_forward::primitive_desc> conv_pd_;
289 290 291 292
  std::shared_ptr<mkldnn::convolution_backward_weights::primitive_desc>
      conv_bwd_weights_pd_;
  std::shared_ptr<mkldnn::convolution_backward_data::primitive_desc>
      conv_bwd_data_pd_;
293 294
};

295
template <typename T>
296
class ConvMKLDNNOpKernel : public paddle::framework::OpKernel<T> {
297 298 299 300
 public:
  void Compute(const paddle::framework::ExecutionContext& ctx) const override {
    PADDLE_ENFORCE(paddle::platform::is_cpu_place(ctx.GetPlace()),
                   "It must use CPUPlace.");
K
Krzysztof Binias 已提交
301 302
    const bool is_test = ctx.Attr<bool>("is_test");

303 304
    auto& dev_ctx =
        ctx.template device_context<paddle::platform::MKLDNNDeviceContext>();
305 306 307 308
    const auto& mkldnn_engine = dev_ctx.GetEngine();

    auto* input = ctx.Input<Tensor>("Input");
    auto* filter = ctx.Input<Tensor>("Filter");
309
    auto* bias = ctx.HasInput("Bias") ? ctx.Input<Tensor>("Bias") : nullptr;
310 311
    auto* output = ctx.Output<Tensor>("Output");

312 313 314 315
    auto* scale_in = ctx.HasInput("Scale_in") ? ctx.Input<Tensor>("Scale_in") : nullptr;
    auto* scale_in_eltwise = ctx.HasInput("Scale_in_eltwise")? ctx.Input<Tensor>("Scale_in_eltwise") : nullptr;
    auto* scale_weights = ctx.HasInput("Scale_weights")? ctx.Input<Tensor>("Scale_weights") : nullptr;
    auto* scale_out = ctx.HasInput("Scale_out")? ctx.Input<Tensor>("Scale_out") : nullptr;
X
xiaolil1 已提交
316 317

    bool is_INT8 = ctx.HasInput("Scale_in")? true : false;
X
xiaolil1 已提交
318
    bool is_multi_channel = (is_INT8 && scale_weights->memory_size() > 1) ? true : false;
319

320 321 322 323 324 325
    PADDLE_ENFORCE(input->layout() == DataLayout::kMKLDNN &&
                       input->format() != memory::format::format_undef,
                   "Wrong layout/format set for Input tensor");
    PADDLE_ENFORCE(filter->layout() == DataLayout::kMKLDNN &&
                       filter->format() != memory::format::format_undef,
                   "Wrong layout/format set for Filter tensor");
326 327 328 329 330 331 332 333 334 335 336
    PADDLE_ENFORCE(input->dims().size() == 4,
                   "Input must be with 4 dimensions, i.e. NCHW");
    PADDLE_ENFORCE(filter->dims().size() == 4,
                   "Filter must be with 4 dimensions, i.e. OIHW");
    if (bias) {
      PADDLE_ENFORCE(bias->layout() == DataLayout::kMKLDNN &&
                         bias->format() != memory::format::format_undef,
                     "Wrong layout/format set for Bias tensor");
      PADDLE_ENFORCE(bias->dims().size() == 1,
                     "Bias must only have 1 dimension, i.e. X");
    }
337 338 339 340

    std::vector<int> strides = ctx.Attr<std::vector<int>>("strides");
    std::vector<int> paddings = ctx.Attr<std::vector<int>>("paddings");
    std::vector<int> dilations = ctx.Attr<std::vector<int>>("dilations");
M
Michal Gallus 已提交
341
    bool fuse_relu = ctx.Attr<bool>("fuse_relu");
342
    bool force_fp32_output = ctx.Attr<bool>("force_fp32_output");
Z
Zhang, Guoming 已提交
343
    bool fuse_residual_conn = ctx.Attr<bool>("fuse_residual_connection");
344 345
    int groups = ctx.Attr<int>("groups");

Z
Zhang, Guoming 已提交
346
    // TODO(tpatejko): add support for dilation
347 348 349 350 351
    PADDLE_ENFORCE(
        dilations.size() == 2 && dilations[0] == 1 && dilations[1] == 1,
        "dilation in convolution is not implemented yet");

    const T* input_data = input->data<T>();
X
xiaolil1 已提交
352
    const float* filter_data = filter->data<float>();
353 354 355 356

    std::vector<int> src_tz = paddle::framework::vectorize2int(input->dims());
    std::vector<int> weights_tz =
        paddle::framework::vectorize2int(filter->dims());
357 358 359 360 361 362 363 364 365 366 367 368 369
    int g = std::max(groups, 1);
    if (g > 1) {
      int o = weights_tz[0];
      int i = weights_tz[1];
      int h = weights_tz[2];
      int w = weights_tz[3];
      weights_tz.resize(5);
      weights_tz[0] = g;
      weights_tz[1] = o / g;
      weights_tz[2] = i;
      weights_tz[3] = h;
      weights_tz[4] = w;
    }
370 371
    std::vector<int> dst_tz = paddle::framework::vectorize2int(output->dims());

X
xiaolil1 已提交
372 373 374 375 376
    // Get unique name for storing MKLDNN primitives
    const std::string key = ConvMKLDNNHandler::GetHash(
        src_tz, weights_tz, strides, paddings, dilations, groups,
        ctx.op().Output("Output"));
    const std::string key_conv_pd = key + "@conv_pd";
X
xiaolil1 已提交
377
    static std::unordered_map<std::string, std::vector<std::vector<float>>> scale_map;
X
xiaolil1 已提交
378 379
    //scale_map.insert({key_conv_pd,{1.0f}});
    //scale_map[key_conv_pd]={0.1f};
380
    bool scale_reuse = true;
X
xiaolil1 已提交
381 382 383 384 385 386
    //auto scale_in_key = key + "@scale_in";
    //auto scale_weights_key = key + "@scale_weights";
    //auto scale_out_key = key + "@scale_out";
    //auto output_shift_scale_key = key + "@output_shift_scale";
    //auto sum_scale_key = key + "@sum_scale";
    //auto scale_in_eltwise_key = key + "@scale_in_eltwise";
X
xiaolil1 已提交
387 388 389
    std::vector<float> scale_in_data;
    std::vector<float> scale_out_data;
    std::vector<float> scale_weights_data;
X
xiaolil1 已提交
390
    std::vector<float> scale_in_eltwise_data = {1.0f};
X
xiaolil1 已提交
391
    std::vector<float> output_shift_scale;
X
xiaolil1 已提交
392
    std::vector<float> sum_scale = {1.0f};
X
xiaolil1 已提交
393 394 395
    std::vector<float> scale_bias_data = {1.0f};
    std::vector<std::vector<float>> none_scale = {{0.0f}};
    std::vector<std::vector<float>> scale_datas(7,{1.0f});
396

X
xiaolil1 已提交
397 398 399
//scale_in_data 0, scale_in_eltwise_data 1, scale_weights_data 2, scale_bias_data 3, scale_out_data 4, output_shift_scale 5, sum_scale 6

    if (is_INT8 && GetScaleMap(scale_map, key) == none_scale){
400
        scale_reuse = false;
X
xiaolil1 已提交
401 402
    } else{
        scale_datas = GetScaleMap(scale_map, key);
X
xiaolil1 已提交
403 404
    }
//std::cout<<"scale_reuse = "<<scale_reuse<<std::endl;
405
    if(is_INT8){
406
        if(!scale_reuse){
X
xiaolil1 已提交
407 408 409 410
//std::cout<<"load scale!!!!!!!!"<<std::endl;
            int count = is_multi_channel? (g>1? weights_tz[1]*weights_tz[0] : weights_tz[0]) : 1; 
            scale_in_data = {*(scale_in->data<float>())};
            scale_weights_data.resize(count);
X
xiaolil1 已提交
411
            #pragma omp parallel for if (count > 1)
X
xiaolil1 已提交
412 413 414 415 416
            for(int i=0; i<count; i++){
                scale_weights_data[i] =*(scale_weights->data<float>() + i);
            }
            scale_out_data = {*(scale_out->data<float>())};
            output_shift_scale.resize(count);
X
xiaolil1 已提交
417
            #pragma omp parallel for if (count > 1)
X
xiaolil1 已提交
418 419 420 421 422 423 424 425 426
            for(int i=0; i<count; i++){
                if(scale_weights_data[i] == 0.0)
                    output_shift_scale[i] = scale_out_data[0];
                else 
                    output_shift_scale[i] = scale_out_data[0] / (scale_in_data[0] * scale_weights_data[i]);
            }
            if(fuse_residual_conn){
                scale_in_eltwise_data = {*(scale_in_eltwise->data<float>())};
                sum_scale[0] = scale_out_data[0] / scale_in_eltwise_data[0];
X
xiaolil1 已提交
427
                //SetScaleMap(scale_map, scale_in_eltwise_key, scale_in_eltwise_data);
X
xiaolil1 已提交
428 429 430
            }

            //scale reuse
X
xiaolil1 已提交
431 432 433 434 435 436 437 438 439 440 441
            scale_datas[0] = scale_in_data;
            scale_datas[1] = scale_in_eltwise_data;
            scale_datas[2] = scale_weights_data;
            scale_datas[4] = scale_out_data;
            scale_datas[5] = output_shift_scale;
            scale_datas[6] = sum_scale;
            //SetScaleMap(scale_map, key, scale_datas);
            //SetScaleMap(scale_map, scale_weights_key, scale_weights_data);
            //SetScaleMap(scale_map, scale_out_key, scale_out_data);
            //SetScaleMap(scale_map, output_shift_scale_key, output_shift_scale);
            //SetScaleMap(scale_map, sum_scale_key, sum_scale);
X
xiaolil1 已提交
442
        } else{
X
xiaolil1 已提交
443 444 445
            scale_in_data = scale_datas[0];
            scale_out_data = scale_datas[3];
            scale_weights_data = scale_datas[2];
X
xiaolil1 已提交
446
            if(fuse_residual_conn){
X
xiaolil1 已提交
447
                scale_in_eltwise_data = scale_datas[1];
X
xiaolil1 已提交
448
            }
X
xiaolil1 已提交
449 450
            output_shift_scale = scale_datas[5];
            sum_scale = scale_datas[6]; 
X
xiaolil1 已提交
451
            //printf("pause!!!");
X
xiaolil1 已提交
452
        }
X
xiaolil1 已提交
453

454 455
    }

X
xiaolil1 已提交
456
    static std::unordered_map<std::string, std::vector<std::shared_ptr<mkldnn::memory::desc>>> md_map;
457
    bool md_reuse = true;
X
xiaolil1 已提交
458 459 460 461
    std::vector<std::shared_ptr<mkldnn::memory::desc>> mds(8, nullptr);
    std::vector<std::shared_ptr<mkldnn::memory::desc>> none_mds = {};
    //auto user_src_md_key = key + "@user_src_md";
    if (GetMdMap(md_map, key) == none_mds){
462
        md_reuse = false;   //we suppose all mds are reused if the first md is in the map.
X
xiaolil1 已提交
463 464
    } else{
        mds = GetMdMap(md_map, key);
465
    }
X
xiaolil1 已提交
466
    //auto user_weights_md_key = key + "@user_weights_md";
467 468
    std::shared_ptr<mkldnn::memory::desc> user_src_md;
    std::shared_ptr<mkldnn::memory::desc> user_weights_md;
X
xiaolil1 已提交
469
    std::vector<primitive> pipeline;
470 471 472 473 474 475 476 477
//std::cout<<"md_reuse = "<<md_reuse<<std::endl;
    if(!md_reuse){
//std::cout<<"create md.......... "<<std::endl;
        user_src_md.reset(new mkldnn::memory::desc(platform::MKLDNNMemDesc(
                {src_tz}, paddle::framework::ToMKLDNNDataType(input->type()), input->format())));
        user_weights_md.reset(new mkldnn::memory::desc(platform::MKLDNNMemDesc(
                {weights_tz}, platform::MKLDNNGetDataType<float>(),
                (g == 1) ? mkldnn::memory::format::oihw : mkldnn::memory::format::goihw)));
X
xiaolil1 已提交
478 479 480 481 482

        mds[0] = user_src_md;
        mds[1] = user_weights_md;        
        //SetMdMap(md_map, user_src_md_key, user_src_md);
        //SetMdMap(md_map, user_weights_md_key, user_weights_md);
483
    } else{
X
xiaolil1 已提交
484 485 486 487
        user_src_md = mds[0];
        user_weights_md = mds[1];
        //user_src_md = GetMdMap(md_map, user_src_md_key);
        //user_weights_md = GetMdMap(md_map, user_weights_md_key);
488
    }
489 490 491 492 493

    /* create memory descriptor for convolution without specified format
     * ('any') which lets a primitive (convolution in this case) choose
     * the memory format preferred for best performance
     */
494
    std::string data_format = ctx.Attr<std::string>("data_format");
X
xiaolil1 已提交
495
    auto chosen_memory_format = 
496
        platform::data_format_to_memory_format(data_format);
497

X
xiaolil1 已提交
498 499
    std::shared_ptr<mkldnn::convolution_forward::primitive_desc> conv_pd;
    auto bias_tz = paddle::framework::vectorize2int(bias->dims());
500

X
xiaolil1 已提交
501 502 503 504
    //auto src_md_key = key + "@src_md";
    //auto weights_md_key = key + "@weights_md_key";
    //auto dst_md_key = key + "@dst_md_key";
    //auto bias_md_key = key + "@bias_md_key";
505 506 507 508
    std::shared_ptr<mkldnn::memory::desc> src_md;
    std::shared_ptr<mkldnn::memory::desc> weights_md;
    std::shared_ptr<mkldnn::memory::desc> dst_md;

X
xiaolil1 已提交
509
    if(is_INT8){
510 511 512 513 514 515 516 517 518 519 520 521 522
        if(!md_reuse){
            src_md.reset(new mkldnn::memory::desc(platform::MKLDNNMemDesc(
                src_tz, memory::data_type::u8, chosen_memory_format)));
            weights_md.reset(new mkldnn::memory::desc(platform::MKLDNNMemDesc(
                weights_tz, memory::data_type::s8,
                (g == 1) ? chosen_memory_format : mkldnn::memory::format::goihw)));
            auto dst_dt = fuse_relu? paddle::framework::ToMKLDNNDataType(std::type_index(typeid(unsigned char))) : paddle::framework::ToMKLDNNDataType(std::type_index(typeid(signed char)));
            if(fuse_residual_conn){
                auto residual = ctx.Input<Tensor>("ResidualData");
                auto residual_dt = paddle::framework::ToMKLDNNDataType(residual->type());
                if(dst_dt != residual_dt)
                    dst_dt = residual_dt;
            }
523
            if(force_fp32_output)
H
Haihao Shen 已提交
524
                dst_dt = paddle::framework::ToMKLDNNDataType(std::type_index(typeid(float)));
525
            dst_md.reset(new mkldnn::memory::desc(platform::MKLDNNMemDesc(dst_tz, dst_dt, chosen_memory_format)));
X
xiaolil1 已提交
526 527 528 529 530 531
            mds[2] = src_md;
            mds[3] = weights_md;
            mds[4] = dst_md;
            //SetMdMap(md_map, src_md_key, src_md);
            //SetMdMap(md_map, weights_md_key, weights_md);
            //SetMdMap(md_map, dst_md_key, dst_md);
532
        } else{
X
xiaolil1 已提交
533 534 535 536 537 538
            src_md = mds[2];
            weights_md = mds[3];
            dst_md = mds[4];
            //src_md = GetMdMap(md_map, src_md_key);
            //weights_md = GetMdMap(md_map, weights_md_key);
            //dst_md = GetMdMap(md_map, dst_md_key);
539
        }
540

X
xiaolil1 已提交
541 542
        // create a conv primitive descriptor and save it for usage in backward
        if (bias) {
543 544 545 546
            std::shared_ptr<mkldnn::memory::desc> bias_md;
            if(!md_reuse){
                bias_md.reset(new mkldnn::memory::desc(platform::MKLDNNMemDesc(
                    bias_tz, memory::data_type::s32, memory::format::x)));
X
xiaolil1 已提交
547 548
                mds[5] = bias_md;
                //SetMdMap(md_map, bias_md_key, bias_md);
549
            } else{
X
xiaolil1 已提交
550 551
                bias_md = mds[5];
                //bias_md = GetMdMap(md_map, bias_md_key);
552 553 554
            }
             
            conv_pd = ConvFwdPrimitiveDesc(*src_md, *weights_md, *bias_md, *dst_md,
X
xiaolil1 已提交
555
                                           strides, paddings, mkldnn_engine,
X
xiaolil1 已提交
556
                                           fuse_relu, fuse_residual_conn,
X
xiaolil1 已提交
557
                                           output_shift_scale, sum_scale[0], is_test);
X
xiaolil1 已提交
558
        } else {
X
xiaolil1 已提交
559
            conv_pd =
560
                ConvFwdPrimitiveDesc(*src_md, *weights_md, *dst_md, strides, paddings,
X
xiaolil1 已提交
561
                                     mkldnn_engine, fuse_relu, fuse_residual_conn,
X
xiaolil1 已提交
562
                                     output_shift_scale, sum_scale[0], is_test);
X
xiaolil1 已提交
563 564
        }
    } else{
565 566 567 568 569 570 571 572
        if(!md_reuse){
            src_md.reset(new mkldnn::memory::desc(platform::MKLDNNMemDesc(
                src_tz, platform::MKLDNNGetDataType<float>(), chosen_memory_format)));
            weights_md.reset(new mkldnn::memory::desc(platform::MKLDNNMemDesc(
                weights_tz, platform::MKLDNNGetDataType<float>(),
                (g == 1) ? chosen_memory_format : mkldnn::memory::format::goihw)));
            dst_md.reset(new mkldnn::memory::desc(platform::MKLDNNMemDesc(
                dst_tz, platform::MKLDNNGetDataType<float>(), chosen_memory_format)));
X
xiaolil1 已提交
573 574 575 576 577 578
            mds[2] = src_md;
            mds[3] = weights_md;
            mds[4] = dst_md;
            //SetMdMap(md_map, src_md_key, src_md);
            //SetMdMap(md_map, weights_md_key, weights_md);
            //SetMdMap(md_map, dst_md_key, dst_md);
579
        } else{
X
xiaolil1 已提交
580 581 582 583 584 585
            src_md = mds[2];
            weights_md = mds[3];
            dst_md = mds[4];
            //src_md = GetMdMap(md_map, src_md_key);
            //weights_md = GetMdMap(md_map, weights_md_key);
            //dst_md = GetMdMap(md_map, dst_md_key);
586
        }
X
xiaolil1 已提交
587 588
        // create a conv primitive descriptor and save it for usage in backward
        if (bias) {
589 590 591 592
            std::shared_ptr<mkldnn::memory::desc> bias_md;
            if(!md_reuse){
                bias_md.reset(new mkldnn::memory::desc(platform::MKLDNNMemDesc(
                    bias_tz, platform::MKLDNNGetDataType<float>(), memory::format::x)));
X
xiaolil1 已提交
593 594
                mds[5] = bias_md;
                //SetMdMap(md_map, bias_md_key, bias_md);
595
            } else{
X
xiaolil1 已提交
596 597
                bias_md = mds[5];
                //bias_md = GetMdMap(md_map, bias_md_key);
598 599 600 601
            }
            conv_pd = ConvFwdPrimitiveDesc(*src_md, *weights_md, *bias_md, *dst_md,
                                           strides, paddings, mkldnn_engine,
                                           fuse_relu, fuse_residual_conn, is_test);
X
xiaolil1 已提交
602
        } else {
603 604
            conv_pd =
                ConvFwdPrimitiveDesc(*src_md, *weights_md, *dst_md, strides, paddings,
X
xiaolil1 已提交
605
                                         mkldnn_engine, fuse_relu, fuse_residual_conn, is_test);
X
xiaolil1 已提交
606
        }
607
    }
608 609
    // Save conv_pd/src_memory/weights_memory for backward pass
    dev_ctx.SetBlob(key_conv_pd, conv_pd);
610

611
    ConvMKLDNNHandler handler(conv_pd, dev_ctx, mkldnn_engine, key);
612

613 614
    // create mkldnn memory from input tensors (data/weights)
    auto user_src_memory_p =
615
        handler.AcquireSrcMemory(*user_src_md, to_void_cast<T>(input_data));
616
    auto user_weights_memory_p = handler.AcquireWeightsMemory(
617
        *user_weights_md, to_void_cast<float>(filter_data));
Z
Zhang, Guoming 已提交
618

619 620
    // create reorder primitive if the input format is not the preferred one
    auto src_memory_p =
621
        handler.AcquireSrcMemoryFromPrimitive(user_src_memory_p, pipeline);
Z
Zhang, Guoming 已提交
622
        
X
xiaolil1 已提交
623
    std::shared_ptr<mkldnn::memory> weights_memory_p;
X
xiaolil1 已提交
624
    if(is_INT8){
625
        int mask_reorder = is_multi_channel? ((g!= 1) ? (1<<1)+(1<<0) : 1<<0) : 0;
X
xiaolil1 已提交
626
        weights_memory_p = handler.AcquireWeightsMemoryFromPrimitive(
X
xiaolil1 已提交
627
            user_weights_memory_p, pipeline, is_test, is_INT8, scale_weights_data, mask_reorder);
X
xiaolil1 已提交
628 629 630 631 632 633
    } else{
        weights_memory_p = handler.AcquireWeightsMemoryFromPrimitive(
            user_weights_memory_p, pipeline, is_test);
    }

    std::shared_ptr<mkldnn::memory> dst_memory_p;
634
    bool need_s8_to_u8 = false;
X
xiaolil1 已提交
635
    //auto user_residual_md_key = key + "@user_residual_md";
636 637 638 639 640 641 642
    if(fuse_residual_conn) {
        auto residual_param = ctx.Input<Tensor>("ResidualData");
        PADDLE_ENFORCE_EQ(output->dims(), residual_param->dims(),
              "Output and elementwise parameter need to have the "
              "same dimension sizes");
        auto residual_dt = paddle::framework::ToMKLDNNDataType(residual_param->type());
        if(residual_param->format() != handler.GetDstFormat()) {
643 644 645 646 647 648 649 650
            std::shared_ptr<mkldnn::memory::desc> user_residual_md;
            if(!md_reuse){
                auto residual_data_tz =
                    paddle::framework::vectorize2int(residual_param->dims());
                auto residual_data_type =
                    paddle::framework::ToMKLDNNDataType(residual_param->type());
                user_residual_md.reset(new mkldnn::memory::desc(platform::MKLDNNMemDesc(
                    residual_data_tz, residual_data_type, residual_param->format())));
X
xiaolil1 已提交
651 652
                mds[6] = user_residual_md;
                //SetMdMap(md_map, user_residual_md_key, user_residual_md);
653
            } else{
X
xiaolil1 已提交
654 655
                user_residual_md = mds[6];
                //user_residual_md = GetMdMap(md_map, user_residual_md_key);
656
            }
657 658
            if(is_INT8){
                if(residual_dt == mkldnn::memory::data_type::u8){
659 660 661 662 663 664 665 666 667 668
                    auto residual_param_data = residual_param->data<uint8_t>();
                    auto user_residual_memory_p = handler.AcquireResidualDataMemory(
                        *user_residual_md, to_void_cast<uint8_t>(residual_param_data));
                    PADDLE_ENFORCE(
                          residual_param_data != nullptr,
                          "Provide data if you want MKLDNN conv+elementwise_add fusion");
                        uint8_t* output_data = output->mutable_data<uint8_t>(ctx.GetPlace());
                        dst_memory_p =
                            handler.AcquireDstMemoryFromResidualDataMemory(
                                user_residual_memory_p, to_void_cast<uint8_t>(output_data), pipeline);
669
                } else{
670 671 672 673 674 675 676 677 678 679
                    auto residual_param_data = residual_param->data<int8_t>();
                    auto user_residual_memory_p = handler.AcquireResidualDataMemory(
                        *user_residual_md, to_void_cast<int8_t>(residual_param_data));
                    PADDLE_ENFORCE(
                          residual_param_data != nullptr,
                          "Provide data if you want MKLDNN conv+elementwise_add fusion");
                        int8_t* output_data = output->mutable_data<int8_t>(ctx.GetPlace());
                        dst_memory_p =
                            handler.AcquireDstMemoryFromResidualDataMemory(
                                user_residual_memory_p, to_void_cast<int8_t>(output_data), pipeline);
680 681 682 683 684 685
                    if(fuse_relu)
                      need_s8_to_u8 = true;
                }
            } else{
                auto residual_param_data = residual_param->data<T>();
                auto user_residual_memory_p = handler.AcquireResidualDataMemory(
686
                    *user_residual_md, to_void_cast<T>(residual_param_data));
687 688 689 690 691 692 693 694
                PADDLE_ENFORCE(
                      residual_param_data != nullptr,
                      "Provide data if you want MKLDNN conv+elementwise_add fusion");
                 auto output_data =
                     output->mutable_data<T>(ctx.GetPlace(), handler.GetDstMemorySize());
                 dst_memory_p = handler.AcquireDstMemoryFromResidualDataMemory(
                      user_residual_memory_p, to_void_cast<T>(output_data), pipeline);
            }
X
xiaolil1 已提交
695
        } else {
696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717
             output->ShareDataWith(*residual_param);
             if(is_INT8){
                 if(residual_dt == mkldnn::memory::data_type::u8){

                     uint8_t* output_data = output->mutable_data<uint8_t>(ctx.GetPlace());
                     dst_memory_p =
                         handler.AcquireDstMemoryFromPrimitive(to_void_cast<uint8_t>(output_data));
                 } else{
                     int8_t* output_data = output->mutable_data<int8_t>(ctx.GetPlace());
                     dst_memory_p =
                         handler.AcquireDstMemoryFromPrimitive(to_void_cast<int8_t>(output_data));
                     if(fuse_relu)
                       need_s8_to_u8 = true;
                 }
             } else{
                  auto output_data = output->mutable_data<T>(ctx.GetPlace());
                  dst_memory_p =
                      handler.AcquireDstMemoryFromPrimitive(to_void_cast<T>(output_data));               
             }
        }
    } else {
        if(is_INT8){
X
xiaolil1 已提交
718 719 720 721 722 723 724 725 726
          if(fuse_relu){
              uint8_t* output_data = output->mutable_data<uint8_t>(ctx.GetPlace(), handler.GetDstMemorySize());
              dst_memory_p =
                  handler.AcquireDstMemoryFromPrimitive(to_void_cast<uint8_t>(output_data));
          } else{
              int8_t* output_data = output->mutable_data<int8_t>(ctx.GetPlace(), handler.GetDstMemorySize());
              dst_memory_p =
                  handler.AcquireDstMemoryFromPrimitive(to_void_cast<int8_t>(output_data));
          }
727 728 729 730 731
        } else{
        auto output_data =
            output->mutable_data<T>(ctx.GetPlace(), handler.GetDstMemorySize());
        dst_memory_p =
            handler.AcquireDstMemoryFromPrimitive(to_void_cast<T>(output_data));
X
xiaolil1 已提交
732
        }
X
xiaolil1 已提交
733
    }
734 735

    // create convolution op primitive
736
    std::shared_ptr<mkldnn::convolution_forward> conv_p;
X
xiaolil1 已提交
737
    //auto scale_bias_key = key + "@scale_bias";
X
xiaolil1 已提交
738
    //auto user_bias_md_key = key + "@user_bias_md";
739
    if (bias) {
X
xiaolil1 已提交
740
      const float* bias_data = bias->data<float>();
741 742 743 744
      std::shared_ptr<mkldnn::memory::desc> user_bias_md;
      if(!md_reuse){
          user_bias_md.reset(new mkldnn::memory::desc(platform::MKLDNNMemDesc(
              {bias_tz}, platform::MKLDNNGetDataType<float>(), memory::format::x)));
X
xiaolil1 已提交
745 746
          mds[7] = user_bias_md;
          //SetMdMap(md_map, user_bias_md_key, user_bias_md);
747
      } else{
X
xiaolil1 已提交
748 749
          user_bias_md = mds[7];
          //user_bias_md = GetMdMap(md_map, user_bias_md_key);
750
      }
751
      auto user_bias_memory_p =
752
          handler.AcquireBiasMemory(*user_bias_md, to_void_cast<float>(bias_data));
X
xiaolil1 已提交
753
      std::shared_ptr<mkldnn::memory>  bias_memory_p;
X
xiaolil1 已提交
754
      if(is_INT8){
755
          int mask_reorder = is_multi_channel? 1<<0 : 1;
756
          if(!scale_reuse){
X
xiaolil1 已提交
757 758
              int count = is_multi_channel? (g>1? weights_tz[1]*weights_tz[0] : weights_tz[0]) : 1;
              scale_bias_data.resize(count);
X
xiaolil1 已提交
759
              #pragma omp parallel for if (count > 1)
X
xiaolil1 已提交
760 761 762
              for(int i=0; i<count; i++){
                  scale_bias_data[i] = scale_in_data[0] * scale_weights_data[i];
              }
X
xiaolil1 已提交
763 764
              scale_datas[3] = scale_bias_data;
              //SetScaleMap(scale_map, scale_bias_key, scale_bias_data);
X
xiaolil1 已提交
765
          } else{
X
xiaolil1 已提交
766
              scale_bias_data = scale_datas[3];
X
xiaolil1 已提交
767
          }
X
xiaolil1 已提交
768
          bias_memory_p =
X
xiaolil1 已提交
769
              handler.AcquireBiasMemoryFromPrimitive(user_bias_memory_p, pipeline, is_test, is_INT8, scale_bias_data, mask_reorder);
X
xiaolil1 已提交
770 771 772 773
      } else{
          bias_memory_p =
              handler.AcquireBiasMemoryFromPrimitive(user_bias_memory_p, pipeline);
      } 
774 775 776 777 778 779
      conv_p = handler.AcquireConvolution(src_memory_p, weights_memory_p,
                                          bias_memory_p, dst_memory_p);
    } else {
      conv_p = handler.AcquireConvolution(src_memory_p, weights_memory_p,
                                          dst_memory_p);
    }
780

X
xiaolil1 已提交
781
    SetScaleMap(scale_map, key, scale_datas);
X
xiaolil1 已提交
782
    SetMdMap(md_map, key, mds);
X
xiaolil1 已提交
783

784
    // push primitive to stream and wait until it's executed
785
    pipeline.push_back(*conv_p);
786 787
    stream(stream::kind::eager).submit(pipeline).wait();

788
    if(need_s8_to_u8){
789 790 791
        output->mutable_data<uint8_t>(ctx.GetPlace());
    }

792
    output->set_layout(DataLayout::kMKLDNN);
793
    output->set_format(GetMKLDNNFormat(*dst_memory_p));
794
  }
795

796
 private:
X
xiaolil1 已提交
797

X
xiaolil1 已提交
798 799
    void SetScaleMap(std::unordered_map<std::string, std::vector<std::vector<float>>> &scale_map,
                       const std::string& name, std::vector<std::vector<float>> scale_datas) const {
X
xiaolil1 已提交
800 801
      auto it = scale_map.find(name);
      if (it == scale_map.end()) {
X
xiaolil1 已提交
802
        scale_map[name] = scale_datas;  // create new blob
X
xiaolil1 已提交
803
      } else {
X
xiaolil1 已提交
804
        (*it).second = scale_datas;  // set data to existing blob
X
xiaolil1 已提交
805 806 807 808
      }
      return;
    }

X
xiaolil1 已提交
809
    std::vector<std::vector<float>> GetScaleMap(std::unordered_map<std::string, std::vector<std::vector<float>>> scale_map,
X
xiaolil1 已提交
810 811 812 813 814
         const std::string& name) const {
      auto it = scale_map.find(name);
      if (it != scale_map.end()) {
        return (*it).second;
      }
X
xiaolil1 已提交
815
      return {{0.0f}};
816 817
    }

X
xiaolil1 已提交
818 819
    void SetMdMap(std::unordered_map<std::string, std::vector<std::shared_ptr<mkldnn::memory::desc>>> &md_map,
                       const std::string& name, std::vector<std::shared_ptr<mkldnn::memory::desc>> mds) const {
820 821
      auto it = md_map.find(name);
      if (it == md_map.end()) {
X
xiaolil1 已提交
822
        md_map[name] = mds;  // create new blob
823
      } else {
X
xiaolil1 已提交
824
        (*it).second = mds;  // set data to existing blob
825 826 827 828
      }
      return;
    }

X
xiaolil1 已提交
829
    std::vector<std::shared_ptr<mkldnn::memory::desc>> GetMdMap(std::unordered_map<std::string, std::vector<std::shared_ptr<mkldnn::memory::desc>>> md_map,
830 831 832 833 834
         const std::string& name) const {
      auto it = md_map.find(name);
      if (it != md_map.end()) {
        return (*it).second;
      }
X
xiaolil1 已提交
835
      return {};
X
xiaolil1 已提交
836 837
    }

Z
Zhang, Guoming 已提交
838
    mkldnn::primitive_attr CreatePostOps(bool fuse_relu, bool fuse_residual_conn,
X
xiaolil1 已提交
839
                          const std::vector<float> output_shift_scale, float sum_scale) const {
840 841
      mkldnn::primitive_attr conv_attr;
      mkldnn::post_ops post_operations;
842
    // Fusion with Elementwise layer relies on adding a sum post-operation with
Z
Zhang, Guoming 已提交
843 844 845 846
    // the scale parameter. It is assumed that when fuse_residual_connection is
    // true, the output tensor contains the data coming from residual
    // connection. The result of this post_op is:
    // Output = scale * Output + Conv_Out.
X
xiaolil1 已提交
847
      int mask = output_shift_scale.size() > 1 ? 1<<1 : 0;
848
      conv_attr.set_output_scales(mask, output_shift_scale);
Z
Zhang, Guoming 已提交
849
      if (fuse_residual_conn) {
850 851 852 853 854
        post_operations.append_sum(sum_scale);
      }
      if (fuse_relu) {
        constexpr float scale = 1.0f;
        constexpr float negative_slope = 0.0f;
855
        constexpr float placeholder = 1.0f; //beta
856 857 858 859 860
        post_operations.append_eltwise(scale, mkldnn::algorithm::eltwise_relu,
                                       negative_slope, placeholder);
      }
      conv_attr.set_post_ops(post_operations);
      return conv_attr;
861
    }
862

X
xiaolil1 已提交
863
      mkldnn::primitive_attr CreatePostOps(bool fuse_relu, bool fuse_residual_conn) const {
864 865 866 867

      mkldnn::primitive_attr conv_attr;
      mkldnn::post_ops post_operations;
      // Fusion with Elementwise layer relies on adding a sum post-operation with
X
xiaolil1 已提交
868
      // the scale parameter. It is assumed that when fuse_residual_conn is true, the
869 870
      // Output tensor contains the data coming from residual connection. The
      // result of this post_op is: Output = scale * Output + Conv_Out.
X
xiaolil1 已提交
871
      conv_attr.set_output_scales(0, {1.0f});
X
xiaolil1 已提交
872
      if (fuse_residual_conn) {
873 874 875 876 877 878 879 880 881 882 883 884 885
        post_operations.append_sum(1.0f);
      }
      // Fusion with ReLU layer is executed through the PostOps feature. Create a
      // PostOps object and configure it to execute an eltwise relu operation.
      if (fuse_relu) {
        constexpr float scale = 1.0f;
        constexpr float negative_slope = 0.0f;
        constexpr float placeholder = 0.0f;
        post_operations.append_eltwise(scale, mkldnn::algorithm::eltwise_relu,
                                       negative_slope, placeholder);
      }
      conv_attr.set_post_ops(post_operations);
      return conv_attr;
886
    }
M
Michal Gallus 已提交
887

Z
Zhang, Guoming 已提交
888
    std::unique_ptr<mkldnn::convolution_forward::primitive_desc>
889 890 891 892
    ConvFwdPrimitiveDesc(const memory::desc& src, const memory::desc& weights,
                         const memory::desc& dst, const std::vector<int>& strides,
                         const std::vector<int>& paddings,
                         const mkldnn::engine& engine, const bool fuse_relu,
Z
Zhang, Guoming 已提交
893
                         const bool fuse_residual_conn,
X
xiaolil1 已提交
894
                         const std::vector<float> output_shift_scale, const float sum_scale, bool is_test) const {
895 896 897
      memory::dims stride_dims = {strides[0], strides[1]};
      memory::dims padding_dims = {paddings[0], paddings[1]};

X
xiaolil1 已提交
898 899
      auto propagation = is_test ? mkldnn::prop_kind::forward_scoring : mkldnn::prop_kind::forward_training;

900
      auto conv_desc = mkldnn::convolution_forward::desc(
X
xiaolil1 已提交
901
          propagation, mkldnn::convolution_direct, src, weights,
902 903 904 905
          dst, stride_dims, padding_dims, padding_dims,
          mkldnn::padding_kind::zero);

      mkldnn::primitive_attr conv_attr =
Z
Zhang, Guoming 已提交
906
          CreatePostOps(fuse_relu, fuse_residual_conn, output_shift_scale, sum_scale);
907 908 909 910 911 912

      auto p_conv_pd = new mkldnn::convolution_forward::primitive_desc(
          conv_desc, conv_attr, engine);

      return std::unique_ptr<mkldnn::convolution_forward::primitive_desc>(
          p_conv_pd);
913
    }
M
Michal Gallus 已提交
914

915
  std::unique_ptr<mkldnn::convolution_forward::primitive_desc>
916 917 918 919
    ConvFwdPrimitiveDesc(const memory::desc& src, const memory::desc& weights,
                         const memory::desc& dst, const std::vector<int>& strides,
                         const std::vector<int>& paddings,
                         const mkldnn::engine& engine, const bool fuse_relu,
X
xiaolil1 已提交
920
                         const bool fuse_residual_conn, bool is_test) const{
921 922
      memory::dims stride_dims = {strides[0], strides[1]};
      memory::dims padding_dims = {paddings[0], paddings[1]};
X
xiaolil1 已提交
923 924 925
 
      auto propagation = is_test ? mkldnn::prop_kind::forward_scoring : mkldnn::prop_kind::forward_training;
 
926
      auto conv_desc = mkldnn::convolution_forward::desc(
X
xiaolil1 已提交
927
          propagation, mkldnn::convolution_direct, src, weights,
928 929 930
          dst, stride_dims, padding_dims, padding_dims,
          mkldnn::padding_kind::zero);
  
Z
Zhang, Guoming 已提交
931
      mkldnn::primitive_attr conv_attr = CreatePostOps(fuse_relu, fuse_residual_conn);
932 933 934 935 936 937 938
  
      auto p_conv_pd = new mkldnn::convolution_forward::primitive_desc(
          conv_desc, conv_attr, engine);
  
      return std::unique_ptr<mkldnn::convolution_forward::primitive_desc>(
          p_conv_pd);
    }
939 940

  std::unique_ptr<mkldnn::convolution_forward::primitive_desc>
941 942 943 944 945
    ConvFwdPrimitiveDesc(const memory::desc& src, const memory::desc& weights,
                         const memory::desc& bias, const memory::desc& dst,
                         const std::vector<int>& strides,
                         const std::vector<int>& paddings,
                         const mkldnn::engine& engine, const bool fuse_relu,
Z
Zhang, Guoming 已提交
946
                         const bool fuse_residual_conn,
X
xiaolil1 已提交
947
                         const std::vector<float> output_shift_scale, const float sum_scale, bool is_test) const {
948 949 950
      memory::dims stride_dims = {strides[0], strides[1]};
      memory::dims padding_dims = {paddings[0], paddings[1]};

X
xiaolil1 已提交
951 952
      auto propagation = is_test ? mkldnn::prop_kind::forward_scoring : mkldnn::prop_kind::forward_training;

953
      auto conv_desc = mkldnn::convolution_forward::desc(
X
xiaolil1 已提交
954
          propagation, mkldnn::convolution_direct, src, weights,
955 956 957 958
          bias, dst, stride_dims, padding_dims, padding_dims,
          mkldnn::padding_kind::zero);

      mkldnn::primitive_attr conv_attr = 
Z
Zhang, Guoming 已提交
959
          CreatePostOps(fuse_relu, fuse_residual_conn, output_shift_scale, sum_scale);
960 961 962 963 964 965 966 967 968 969 970 971 972 973

      auto p_conv_pd = new mkldnn::convolution_forward::primitive_desc(
          conv_desc, conv_attr, engine);

      return std::unique_ptr<mkldnn::convolution_forward::primitive_desc>(
          p_conv_pd);
    }

  std::unique_ptr<mkldnn::convolution_forward::primitive_desc>
    ConvFwdPrimitiveDesc(const memory::desc& src, const memory::desc& weights,
                         const memory::desc& bias, const memory::desc& dst,
                         const std::vector<int>& strides,
                         const std::vector<int>& paddings,
                         const mkldnn::engine& engine, const bool fuse_relu,
X
xiaolil1 已提交
974
                         const bool fuse_residual_conn, bool is_test) const{
975 976 977
      memory::dims stride_dims = {strides[0], strides[1]};
      memory::dims padding_dims = {paddings[0], paddings[1]};

X
xiaolil1 已提交
978 979
      auto propagation = is_test ? mkldnn::prop_kind::forward_scoring : mkldnn::prop_kind::forward_training;

980
      auto conv_desc = mkldnn::convolution_forward::desc(
X
xiaolil1 已提交
981
          propagation, mkldnn::convolution_direct, src, weights,
982 983 984
          bias, dst, stride_dims, padding_dims, padding_dims,
          mkldnn::padding_kind::zero);

Z
Zhang, Guoming 已提交
985
      mkldnn::primitive_attr conv_attr = CreatePostOps(fuse_relu, fuse_residual_conn);
986 987 988 989 990 991 992 993

      auto p_conv_pd = new mkldnn::convolution_forward::primitive_desc(
          conv_desc, conv_attr, engine);

      return std::unique_ptr<mkldnn::convolution_forward::primitive_desc>(
          p_conv_pd);
    }

994 995 996
};

template <typename T>
997
class ConvMKLDNNGradOpKernel : public paddle::framework::OpKernel<T> {
998 999 1000 1001 1002
 public:
  void Compute(const paddle::framework::ExecutionContext& ctx) const override {
    PADDLE_ENFORCE(paddle::platform::is_cpu_place(ctx.GetPlace()),
                   "It must use CPUPlace.");

1003 1004
    auto& dev_ctx =
        ctx.template device_context<platform::MKLDNNDeviceContext>();
1005 1006 1007 1008 1009 1010 1011 1012 1013 1014
    const auto& mkldnn_engine = dev_ctx.GetEngine();

    const Tensor* input = ctx.Input<Tensor>("Input");
    const Tensor* filter = ctx.Input<Tensor>("Filter");
    const Tensor* output = ctx.Input<Tensor>("Output");
    const Tensor* output_grad =
        ctx.Input<Tensor>(framework::GradVarName("Output"));
    Tensor* input_grad = ctx.Output<Tensor>(framework::GradVarName("Input"));
    Tensor* filter_grad = ctx.Output<Tensor>(framework::GradVarName("Filter"));

1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027
    PADDLE_ENFORCE(input->layout() == DataLayout::kMKLDNN &&
                       input->format() != memory::format::format_undef,
                   "Wrong layout/format set for Input tensor");
    PADDLE_ENFORCE(filter->layout() == DataLayout::kMKLDNN &&
                       filter->format() != memory::format::format_undef,
                   "Wrong layout/format set for Filter tensor");
    PADDLE_ENFORCE(output->layout() == DataLayout::kMKLDNN &&
                       output->format() != memory::format::format_undef,
                   "Wrong layout/format set for Output tensor");
    PADDLE_ENFORCE(output_grad->layout() == DataLayout::kMKLDNN &&
                       output_grad->format() != memory::format::format_undef,
                   "Wrong layout/format set for output_grad tensor");

1028 1029 1030 1031
    if (!input_grad && !filter_grad) return;

    std::vector<int> strides = ctx.Attr<std::vector<int>>("strides");
    std::vector<int> paddings = ctx.Attr<std::vector<int>>("paddings");
1032 1033
    std::vector<int> dilations = ctx.Attr<std::vector<int>>("dilations");
    int groups = ctx.Attr<int>("groups");
1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045

    const T* input_data = input->data<T>();
    const T* filter_data = filter->data<T>();
    const T* output_grad_data = output_grad->data<T>();
    T* input_grad_data = nullptr;
    T* filter_grad_data = nullptr;

    std::vector<int> src_tz = paddle::framework::vectorize2int(input->dims());
    std::vector<int> weights_tz =
        paddle::framework::vectorize2int(filter->dims());
    std::vector<int> dst_tz = paddle::framework::vectorize2int(output->dims());

1046
    // Get an unique name from "argument" name of "Output" variable
J
Jacek Czaja 已提交
1047
    // as well as attributes of primitive to be created
1048 1049 1050 1051 1052 1053
    // This name will be used as key when saving info into device context
    const std::string key =
        ConvMKLDNNHandler::GetHash(src_tz, weights_tz, strides, paddings,
                                   dilations, groups, ctx.op().Input("Output"));

    const std::string key_conv_pd = key + "@conv_pd";
1054
    std::vector<primitive> pipeline;
1055

1056 1057 1058 1059 1060 1061 1062
    // Create user memory descriptors
    auto user_src_md = platform::MKLDNNMemDesc(
        {src_tz}, platform::MKLDNNGetDataType<T>(), input->format());
    auto user_weights_md = platform::MKLDNNMemDesc(
        {weights_tz}, platform::MKLDNNGetDataType<T>(), filter->format());
    auto user_diff_dst_md = platform::MKLDNNMemDesc(
        {dst_tz}, platform::MKLDNNGetDataType<T>(), output_grad->format());
1063 1064 1065 1066 1067

    /* create memory descriptor for conv backward without specified format
     * ('any') which lets a primitive (conv backward in this case) choose
     * the memory format preferred for best performance
     */
1068 1069 1070 1071
    std::string data_format = ctx.Attr<std::string>("data_format");
    auto chosen_memory_format =
        platform::data_format_to_memory_format(data_format);

1072
    auto src_md = platform::MKLDNNMemDesc(
1073
        src_tz, platform::MKLDNNGetDataType<T>(), chosen_memory_format);
1074
    auto diff_src_md = platform::MKLDNNMemDesc(
1075
        src_tz, platform::MKLDNNGetDataType<T>(), chosen_memory_format);
1076
    auto weights_md = platform::MKLDNNMemDesc(
1077
        weights_tz, platform::MKLDNNGetDataType<T>(), chosen_memory_format);
1078
    auto diff_weights_md = platform::MKLDNNMemDesc(
1079
        weights_tz, platform::MKLDNNGetDataType<T>(), chosen_memory_format);
1080
    auto diff_dst_md = platform::MKLDNNMemDesc(
1081
        dst_tz, platform::MKLDNNGetDataType<T>(), chosen_memory_format);
1082

1083
    // Retrieve conv_pd from device context
1084 1085 1086
    auto conv_pd =
        std::static_pointer_cast<mkldnn::convolution_forward::primitive_desc>(
            dev_ctx.GetBlob(key_conv_pd));
1087 1088 1089
    PADDLE_ENFORCE(conv_pd != nullptr,
                   "Fail to find conv_pd in device context");

1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115
    // create backward convolution weights primitive descriptor
    auto conv_bwd_weights_desc = mkldnn::convolution_backward_weights::desc(
        mkldnn::convolution_direct, src_md, diff_weights_md, diff_dst_md,
        strides, paddings, paddings, mkldnn::padding_kind::zero);
    auto conv_bwd_weights_pd =
        std::make_shared<mkldnn::convolution_backward_weights::primitive_desc>(
            conv_bwd_weights_desc, mkldnn_engine, *conv_pd);

    // create backward convolution data primitive descriptor
    auto conv_bwd_data_desc = mkldnn::convolution_backward_data::desc(
        mkldnn::convolution_direct, diff_src_md, weights_md, diff_dst_md,
        strides, paddings, paddings, mkldnn::padding_kind::zero);
    auto conv_bwd_data_pd =
        std::make_shared<mkldnn::convolution_backward_data::primitive_desc>(
            conv_bwd_data_desc, mkldnn_engine, *conv_pd);

    ConvMKLDNNHandler handler(conv_pd, conv_bwd_data_pd, conv_bwd_weights_pd,
                              dev_ctx, mkldnn_engine, key);

    // create mkldnn memory from input tensors (data/weights)
    auto user_src_memory_p =
        handler.AcquireSrcMemory(user_src_md, to_void_cast<T>(input_data));
    auto user_weights_memory_p = handler.AcquireWeightsMemory(
        user_weights_md, to_void_cast<T>(filter_data));
    auto user_diff_dst_memory_p = handler.AcquireDiffDstMemory(
        user_diff_dst_md, to_void_cast<T>(output_grad_data));
1116 1117
    // create backward conv primitive for weights
    if (filter_grad) {
1118 1119
      auto src_memory_p = handler.AcquireSrcMemoryFromWeightsPrimitive(
          user_src_memory_p, pipeline);
1120

1121 1122 1123 1124
      auto diff_dst_memory_4filter_p =
          handler.AcquireDiffDstMemoryFromWeightsPrimitive(
              user_diff_dst_memory_p, pipeline);

1125
      const size_t size = handler.GetDiffWeightsMemorySize();
1126 1127
      filter_grad_data = filter_grad->mutable_data<T>(ctx.GetPlace(), size);

1128 1129 1130 1131 1132 1133 1134 1135 1136
      auto diff_weights_memory_p =
          handler.AcquireDiffWeightsMemoryFromWeightsPrimitive(
              reinterpret_cast<void*>(filter_grad_data));

      auto conv_bwd_weights_p = handler.AcquireConvolutionBackwardWeights(
          src_memory_p, diff_dst_memory_4filter_p, diff_weights_memory_p);

      // push primitive to stream and wait until it's executed
      pipeline.push_back(*conv_bwd_weights_p);
1137 1138

      filter_grad->set_layout(DataLayout::kMKLDNN);
1139
      filter_grad->set_format(GetMKLDNNFormat(*diff_weights_memory_p));
1140 1141 1142
    }

    if (input_grad) {
1143 1144 1145 1146 1147 1148 1149
      auto weights_memory_p = handler.AcquireWeightsMemoryFromDataPrimitive(
          user_weights_memory_p, pipeline);

      auto diff_dst_memory_4data_p =
          handler.AcquireDiffDstMemoryFromDataPrimitive(user_diff_dst_memory_p,
                                                        pipeline);

1150
      const size_t size = handler.GetDiffSourceMemorySize();
1151 1152
      input_grad_data = input_grad->mutable_data<T>(ctx.GetPlace(), size);

1153 1154 1155 1156 1157 1158 1159
      auto diff_src_memory_p = handler.AcquireDiffSrcMemoryFromDataPrimitive(
          reinterpret_cast<void*>(input_grad_data));

      auto conv_bwd_data_p = handler.AcquireConvolutionBackwardData(
          diff_dst_memory_4data_p, weights_memory_p, diff_src_memory_p);

      pipeline.push_back(*conv_bwd_data_p);
1160 1161

      input_grad->set_layout(DataLayout::kMKLDNN);
1162
      input_grad->set_format(GetMKLDNNFormat(*diff_src_memory_p));
1163
    }
1164
    stream(stream::kind::eager).submit(pipeline).wait();
1165 1166 1167 1168 1169 1170 1171 1172 1173
  }  // Compute()
};

}  // namespace operators
}  // namespace paddle

namespace ops = paddle::operators;

REGISTER_OP_KERNEL(conv2d, MKLDNN, ::paddle::platform::CPUPlace,
X
xiaolil1 已提交
1174 1175
                   ops::ConvMKLDNNOpKernel<float>,
                   ops::ConvMKLDNNOpKernel<uint8_t>);
1176 1177

REGISTER_OP_KERNEL(conv2d_grad, MKLDNN, ::paddle::platform::CPUPlace,
1178
                   ops::ConvMKLDNNGradOpKernel<float>);