data_layout_transform.cc 7.5 KB
Newer Older
1
//   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
D
dzhwinter 已提交
2 3 4 5 6 7 8 9 10 11 12 13
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
14

Y
Yi Wang 已提交
15
#include "paddle/fluid/framework/data_layout_transform.h"
16

Y
Yi Wang 已提交
17
#include "paddle/fluid/operators/math/math_function.h"
M
mozga-intel 已提交
18
#ifdef PADDLE_WITH_MKLDNN
19
#include "paddle/fluid/platform/mkldnn_reuse.h"
M
mozga-intel 已提交
20
#endif
21 22 23 24

namespace paddle {
namespace framework {

25
std::vector<int> GetAxis(const DataLayout& from, const DataLayout& to) {
26 27 28 29
  PADDLE_ENFORCE_NE(
      from, to,
      platform::errors::InvalidArgument(
          "Layout transform should transform between different layout."));
30 31 32 33 34
  if (from == DataLayout::kNCHW && to == DataLayout::kNHWC) {
    return {0, 2, 3, 1};
  } else if (from == DataLayout::kNHWC && to == DataLayout::kNCHW) {
    return {0, 3, 1, 2};
  } else {
35 36
    PADDLE_THROW(
        platform::errors::InvalidArgument("Unsupported layout transform."));
37 38 39
  }
}

40 41 42
template <typename T>
void CastDataLayout::apply() {
  auto place = ctx_->GetPlace();
43

44 45 46 47 48 49 50
  if (platform::is_cpu_place(place)) {
    operators::math::Transpose<platform::CPUDeviceContext, T, 4> trans4;
    auto* context = static_cast<const platform::CPUDeviceContext*>(ctx_);
    trans4(*context, in_, out_, axis_);
  } else {
    PADDLE_THROW(platform::errors::PreconditionNotMet(
        "Unsupported data layout cast from CPU to GPU."));
51
  }
52
}
53

54 55 56
void TransDataLayout(const OpKernelType& kernel_type_for_var,
                     const OpKernelType& expected_kernel_type, const Tensor& in,
                     Tensor* out) {
57
  PADDLE_ENFORCE(
58 59
      platform::places_are_same_class(kernel_type_for_var.place_,
                                      expected_kernel_type.place_),
60 61
      platform::errors::PreconditionNotMet(
          "TransDataLayout only support DataLayout transform on same place."));
62

63 64 65 66 67
  PADDLE_ENFORCE_EQ(
      arity(in.dims()), 4,
      platform::errors::InvalidArgument(
          "Input dimension arity only can be 4, the input dimension is %s.",
          in.dims()));
68 69

  auto& pool = platform::DeviceContextPool::Instance();
70

71
  auto src_dim = in.dims();
72 73
  std::vector<int64_t> dst_dim;

74 75
  auto axis = GetAxis(kernel_type_for_var.data_layout_,
                      expected_kernel_type.data_layout_);
76 77 78 79 80
  dst_dim.resize(axis.size());
  for (size_t i = 0; i < axis.size(); i++) {
    dst_dim[i] = src_dim[axis[i]];
  }

81 82
  out->Resize(make_ddim(dst_dim));
  out->mutable_data(expected_kernel_type.place_, in.type());
83

84
  framework::VisitDataType(
Y
Yu Yang 已提交
85
      in.type(),
86
      CastDataLayout(pool.Get(expected_kernel_type.place_), axis, in, out));
87

88
  out->set_layout(expected_kernel_type.data_layout_);
89 90
}

M
mozga-intel 已提交
91
#ifdef PADDLE_WITH_MKLDNN
92 93 94
using dnnl::memory;
using dnnl::primitive;
using dnnl::reorder;
M
mozga-intel 已提交
95

96
void* GetDataFromTensor(const Tensor& tensor, dnnl::memory::data_type type) {
M
mozga-intel 已提交
97
  switch (type) {
98
    case dnnl::memory::data_type::f32:
M
mozga-intel 已提交
99
      return platform::to_void_cast(tensor.data<float>());
100
    case dnnl::memory::data_type::s8:
Y
Yu Yang 已提交
101
      return platform::to_void_cast(tensor.data<int8_t>());
102
    case dnnl::memory::data_type::u8:
M
mozga-intel 已提交
103
      return platform::to_void_cast(tensor.data<unsigned char>());
104
    case dnnl::memory::data_type::s32:
M
mozga-intel 已提交
105
      return platform::to_void_cast(tensor.data<int32_t>());
106
    case dnnl::memory::data_type::bf16:
107
      return platform::to_void_cast(tensor.data<paddle::platform::bfloat16>());
M
mozga-intel 已提交
108
    default:
109 110
      PADDLE_THROW(
          platform::errors::InvalidArgument("Wrong mkldnn type provided."));
M
mozga-intel 已提交
111 112 113 114 115 116 117 118
  }
}

void TransDataLayoutFromMKLDNN(const OpKernelType& kernel_type_for_var,
                               const OpKernelType& expected_kernel_type,
                               const Tensor& in, Tensor* out) {
  auto in_layout = kernel_type_for_var.data_layout_;
  auto out_layout = expected_kernel_type.data_layout_;
119
  auto place = expected_kernel_type.place_;
M
mozga-intel 已提交
120 121 122

  PADDLE_ENFORCE(
      in_layout == DataLayout::kMKLDNN && out_layout != DataLayout::kMKLDNN,
123 124 125
      platform::errors::InvalidArgument(
          "TransDataLayoutFromMKLDNN only supports transform from MKLDNN to "
          "non-MKLDNN"));
M
mozga-intel 已提交
126

127 128 129 130
  innerTransDataLayoutFromMKLDNN(
      in_layout,
      paddle::platform::MKLDNNDeviceContext::tls().get_cur_paddle_data_layout(),
      in, out, place);
131 132 133 134
}

void innerTransDataLayoutFromMKLDNN(DataLayout in_layout, DataLayout out_layout,
                                    const Tensor& in, Tensor* out,
135
                                    platform::Place place, bool always_copy) {
A
Adam 已提交
136
  PADDLE_ENFORCE_NE(in.format(), MKLDNNMemoryFormat::undef,
137 138 139
                    platform::errors::InvalidArgument(
                        "Input tensor format is invalid. Input tensor should "
                        "have specified memory format."));
140
  PADDLE_ENFORCE_NE(in.format(), MKLDNNMemoryFormat::any,
141 142 143
                    platform::errors::InvalidArgument(
                        "Input tensor format is invalid. Input tensor should "
                        "have specified memory format."));
M
mozga-intel 已提交
144 145 146 147 148

  // Set default as NCHW in case not specified
  out_layout =
      out_layout == DataLayout::kAnyLayout ? DataLayout::kNCHW : out_layout;

149
  auto& pool = platform::DeviceContextPool::Instance();
150
  auto* dev_ctx = dynamic_cast<platform::MKLDNNDeviceContext*>(pool.Get(place));
151 152
  auto& cpu_engine = dev_ctx->GetEngine();

A
Adam 已提交
153
  auto in_tz = paddle::framework::vectorize<int64_t>(in.dims());
154
  auto out_tz = in_tz;
M
mozga-intel 已提交
155 156

  memory::data_type in_type = ToMKLDNNDataType(in.type());
A
Adam 已提交
157
  PADDLE_ENFORCE_NE(in_type, memory::data_type::undef,
158 159 160
                    platform::errors::InvalidArgument(
                        "Input tensor type (%s) is not supported.",
                        DataTypeToString(in.type())));
M
mozga-intel 已提交
161

162 163 164 165
  auto in_format = platform::MKLDNNFormatForSize(in_tz.size(), in.format());
  auto out_format =
      platform::MKLDNNFormatForSize(in_tz.size(), ToMKLDNNFormat(out_layout));

M
mozga-intel 已提交
166 167 168
  // output tensor has the same dims as input. Reorder don't change dims
  out->Resize(in.dims());

169
  if ((in_format != out_format) || always_copy) {
170
    void* in_data = GetDataFromTensor(in, in_type);
M
mozga-intel 已提交
171

172 173
    platform::ReorderMKLDNNHandler handler(in_tz, in.type(), in_type,
                                           cpu_engine);
M
mozga-intel 已提交
174

175 176
    auto reorder_src_memory_p = handler.AcquireSrcMemory(in_format, in_data);
    auto reorder_dst_memory_p =
177
        handler.AcquireDstMemory(out, out_format, place);
178 179 180
    auto reorder_p =
        handler.AcquireReorder(reorder_dst_memory_p, reorder_src_memory_p);

181
    auto& astream = platform::MKLDNNDeviceContext::tls().get_stream();
182 183
    platform::RecordEvent record_reorder("ext_reorder",
                                         platform::EventRole::kUniqueOp);
A
Adam 已提交
184 185
    reorder_p->execute(astream, *reorder_src_memory_p, *reorder_dst_memory_p);
    astream.wait();
186 187 188
  } else {
    out->ShareDataWith(in);
  }
189 190
  // For exepected NHWC data format we need to reshape the Output tensor
  // As MKL-DNN description was in NCHW and paddle is expecting NHWC
191 192
  platform::MatchShapeToLayout(out, in_layout, out_layout);

J
Jacek Czaja 已提交
193
  out->set_layout(DataLayout::kNCHW);
194
  // reset format since the out tensor will be feed to non-MKLDNN OPkernel
A
Adam 已提交
195
  out->set_format(MKLDNNMemoryFormat::undef);
M
mozga-intel 已提交
196
}
197
#endif
M
mozga-intel 已提交
198

199 200
}  // namespace framework
}  // namespace paddle