tensor_util.h 17.1 KB
Newer Older
1
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
D
dzhwinter 已提交
2

L
Luo Tao 已提交
3 4 5
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
D
dzhwinter 已提交
6

L
Luo Tao 已提交
7
    http://www.apache.org/licenses/LICENSE-2.0
D
dzhwinter 已提交
8

L
Luo Tao 已提交
9 10 11 12 13
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
D
dzhwinter 已提交
14 15

#pragma once
S
Steffy-zxf 已提交
16 17 18 19 20
#include <algorithm>
#include <codecvt>
#include <locale>
#include <string>
#include <unordered_map>
21
#include <vector>
W
wanghuancoder 已提交
22

Y
Yi Wang 已提交
23
#include "paddle/fluid/framework/data_type.h"
6
633WHU 已提交
24
#include "paddle/fluid/framework/dlpack_tensor.h"
Y
Yi Wang 已提交
25
#include "paddle/fluid/framework/eigen.h"
S
Steffy-zxf 已提交
26
#include "paddle/fluid/framework/string_array.h"
Y
Yi Wang 已提交
27
#include "paddle/fluid/framework/tensor.h"
28 29 30 31
#include "paddle/fluid/memory/allocation/allocator_facade.h"
#ifdef PADDLE_WITH_ASCEND_CL
#include "paddle/fluid/memory/allocation/npu_pinned_allocator.h"
#endif
Y
Yi Wang 已提交
32
#include "paddle/fluid/platform/device_context.h"
F
fwenguang 已提交
33 34 35
#ifdef PADDLE_WITH_MLU
#include "paddle/fluid/platform/device/mlu/device_context.h"
#endif
D
dzhwinter 已提交
36

37 38
#include "paddle/pten/core/dense_tensor.h"

D
dzhwinter 已提交
39 40 41
namespace paddle {
namespace framework {

42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61
class PrintOptions {
 public:
  static PrintOptions& Instance() {
    static PrintOptions instance;
    return instance;
  }
  ~PrintOptions() {}
  PrintOptions(const PrintOptions& o) = delete;
  const PrintOptions& operator=(const PrintOptions& o) = delete;

  int precision = 8;
  int threshold = 1000;
  int edgeitems = 3;
  int linewidth = 75;
  bool sci_mode = false;

 private:
  PrintOptions() {}
};

S
Steffy-zxf 已提交
62 63 64 65 66 67 68 69
void TensorToStream(std::ostream& os, const Tensor& tensor,
                    const platform::DeviceContext& dev_ctx);
void TensorFromStream(std::istream& is, Tensor* tensor,
                      const platform::DeviceContext& dev_ctx);
void TensorFromStream(std::istream& is, Tensor* tensor,
                      const platform::DeviceContext& dev_ctx,
                      const size_t& seek, const std::vector<int64_t>& shape);

C
chengduo 已提交
70 71 72 73 74 75
// NOTE(zcd): Because TensorCopy is an async operation, when the src_place
// and dst_place are two different GPU, to ensure that the operation can
// be carried out correctly, there is a src_ctx wait operation in TensorCopy.
// If ctx_place and src_place are the same, src_ctx.Wait() is added
// after memory::Copy; if ctx_place and dst_place are the same,
// src_ctx.Wait() is added before memory::Copy.
Y
Yi Wang 已提交
76
void TensorCopy(const Tensor& src, const platform::Place& dst_place,
F
fengjiayi 已提交
77
                const platform::DeviceContext& ctx, Tensor* dst);
C
chengduo 已提交
78 79 80 81 82 83 84 85

// NOTE(zcd): If the src.place() and dst_place are two different GPU,
// the copy operation is carried out on the dst_place's stream. This is
// very important, because TensorCopy is an async operator, and in most
// case, once this copy operator returns, dst is to be used in dst_place's
// stream, if this copy operation is carried out on the src_place's stream,
// when dst is used in dst_place's stream the copy operation may be
// not completed.
Y
Yi Wang 已提交
86 87
void TensorCopy(const Tensor& src, const platform::Place& dst_place,
                Tensor* dst);
C
chengduo 已提交
88

F
fengjiayi 已提交
89 90
void TensorCopySync(const Tensor& src, const platform::Place& dst_place,
                    Tensor* dst);
D
dzhwinter 已提交
91

Y
Yi Wang 已提交
92 93 94 95 96
template <typename T>
void TensorFromVector(const std::vector<T>& src,
                      const platform::DeviceContext& ctx, Tensor* dst);
template <typename T>
void TensorFromVector(const std::vector<T>& src, Tensor* dst);
D
dzhwinter 已提交
97

Y
Yi Wang 已提交
98 99 100 101 102
template <typename T>
void TensorToVector(const Tensor& src, const platform::DeviceContext& ctx,
                    std::vector<T>* dst);
template <typename T>
void TesnorToVector(const Tensor& src, std::vector<T>* dst);
D
dzhwinter 已提交
103

104
// copy the result bool to cpu
Y
Yi Wang 已提交
105 106
bool TensorContainsNAN(const framework::Tensor& tensor);
bool TensorContainsInf(const framework::Tensor& tensor);
107 108 109 110 111 112
bool TensorIsfinite(const framework::Tensor& tensor);

// store the result bool in gpu tensor, async operation. Faster than above ones.
void TensorContainsNAN(const framework::Tensor& tensor, framework::Tensor* out);
void TensorContainsInf(const framework::Tensor& tensor, framework::Tensor* out);
void TensorIsfinite(const framework::Tensor& tensor, framework::Tensor* out);
D
dzhwinter 已提交
113

Y
Yi Wang 已提交
114 115 116 117
void TensorToStream(std::ostream& os, const Tensor& tensor,
                    const platform::DeviceContext& dev_ctx);
void TensorFromStream(std::istream& is, Tensor* tensor,
                      const platform::DeviceContext& dev_ctx);
T
tangwei12 已提交
118 119 120
void TensorFromStream(std::istream& is, Tensor* tensor,
                      const platform::DeviceContext& dev_ctx,
                      const size_t& seek, const std::vector<int64_t>& shape);
D
dzhwinter 已提交
121

J
Jack Zhou 已提交
122 123 124 125 126 127 128
// store the bool result tensor in out tensor
void TensorContainsNANV2(const framework::Tensor& tensor,
                         framework::Tensor* out);
void TensorContainsInfV2(const framework::Tensor& tensor,
                         framework::Tensor* out);
void TensorIsfiniteV2(const framework::Tensor& tensor, framework::Tensor* out);

6
633WHU 已提交
129 130 131
// convert dlpack's DLTensor to tensor
void TensorFromDLPack(const ::DLTensor& dl_tensor, framework::Tensor* dst);

Y
Yi Wang 已提交
132 133 134
//
// The implementation of template functions.
//
D
dzhwinter 已提交
135

136 137 138 139 140 141 142 143 144 145 146
template <typename T>
void TensorFromArray(const T* src, const size_t& array_size,
                     const platform::DeviceContext& ctx, Tensor* dst) {
  auto dst_place = ctx.GetPlace();
  auto src_ptr = static_cast<const void*>(src);
  platform::CPUPlace src_place;
  dst->Resize({static_cast<int64_t>(array_size)});
  auto dst_ptr = static_cast<void*>(dst->mutable_data<T>(dst_place));
  auto size = array_size * sizeof(T);

  if (platform::is_cpu_place(dst_place)) {
147
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size);
148
  }
149
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
150 151
  else if (platform::is_gpu_place(dst_place)) {  // NOLINT
    memory::Copy(
152
        dst_place, dst_ptr, src_place, src_ptr, size,
153 154 155
        reinterpret_cast<const platform::CUDADeviceContext&>(ctx).stream());
  }
#endif
156 157 158 159 160 161 162 163 164 165 166 167
#ifdef PADDLE_WITH_ASCEND_CL
  else if (platform::is_npu_place(dst_place)) {  // NOLINT
    //  1. vector -> npu pinned tensor
    platform::NPUPinnedPlace npu_pinned_place;
    Tensor npu_pinned_tensor;
    npu_pinned_tensor.Resize(dst->dims());
    auto npu_pinned_ptr =
        npu_pinned_tensor.mutable_data(npu_pinned_place, dst->type());
    memory::Copy(npu_pinned_place, npu_pinned_ptr, src_place, src_ptr, size);

    //  2. async copy npu pinned tensor -> npu tensor
    memory::Copy(
168
        dst_place, dst_ptr, npu_pinned_place, npu_pinned_ptr, size,
169 170 171 172 173 174 175 176
        reinterpret_cast<const platform::NPUDeviceContext&>(ctx).stream());

    //  3. record event
    auto npu_pinned_allocator =
        static_cast<paddle::memory::allocation::NPUPinnedAllocator*>(
            paddle::memory::allocation::AllocatorFacade::Instance()
                .GetAllocator(npu_pinned_place)
                .get());
177
    pten::Allocation* allocation = npu_pinned_tensor.Holder().get();
178 179 180 181 182
    npu_pinned_allocator->RecordEvent(
        allocation,
        reinterpret_cast<const platform::NPUDeviceContext&>(ctx).stream());
  }
#endif
183
}
184

D
dzhwinter 已提交
185
template <typename T>
Y
Yi Wang 已提交
186 187
void TensorFromVector(const std::vector<T>& src,
                      const platform::DeviceContext& ctx, Tensor* dst) {
D
dzhwinter 已提交
188 189 190 191 192 193 194 195
  auto dst_place = ctx.GetPlace();
  auto src_ptr = static_cast<const void*>(src.data());
  platform::CPUPlace src_place;
  dst->Resize({static_cast<int64_t>(src.size())});
  auto dst_ptr = static_cast<void*>(dst->mutable_data<T>(dst_place));
  auto size = src.size() * sizeof(T);

  if (platform::is_cpu_place(dst_place)) {
196
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size);
D
dzhwinter 已提交
197
  }
198
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
D
dzhwinter 已提交
199 200
  else if (platform::is_gpu_place(dst_place)) {  // NOLINT
    memory::Copy(
201
        dst_place, dst_ptr, src_place, src_ptr, size,
D
dzhwinter 已提交
202 203 204
        reinterpret_cast<const platform::CUDADeviceContext&>(ctx).stream());
  }
#endif
205
#ifdef PADDLE_WITH_ASCEND_CL
206 207 208 209 210 211
  // NOTE(zhiqiu): Becareful that aclrtMemcpyAsync is different from
  // cudaMemcpyAsync.
  // cudaMemcpyAsync is actually "sync" between cpu <-> gpu.
  // aclrtMemcpyAsync is really "async" between cpu <-> npu.
  // Since vector is on cpu, I think this function should be a "sync" operation,
  // so pass nullptr as stream to  memory::Copy().
212
  else if (platform::is_npu_place(dst_place)) {  // NOLINT
213 214 215 216 217 218 219 220 221
    //  1. vector -> npu pinned tensor
    Tensor npu_pinned_tensor(dst->type());
    platform::NPUPinnedPlace npu_pinned_place;
    auto npu_pinned_ptr =
        npu_pinned_tensor.mutable_data<T>(dst->dims(), npu_pinned_place);
    memory::Copy(npu_pinned_place, npu_pinned_ptr, src_place, src_ptr, size);

    //  2. async copy npu pinned tensor -> npu tensor
    memory::Copy(
222
        dst_place, dst_ptr, npu_pinned_place, npu_pinned_ptr, size,
223 224 225 226 227 228 229 230
        reinterpret_cast<const platform::NPUDeviceContext&>(ctx).stream());

    //  3. record event
    auto npu_pinned_allocator =
        static_cast<paddle::memory::allocation::NPUPinnedAllocator*>(
            paddle::memory::allocation::AllocatorFacade::Instance()
                .GetAllocator(npu_pinned_place)
                .get());
231
    pten::Allocation* allocation = npu_pinned_tensor.Holder().get();
232 233 234
    npu_pinned_allocator->RecordEvent(
        allocation,
        reinterpret_cast<const platform::NPUDeviceContext&>(ctx).stream());
235 236
  }
#endif
F
fwenguang 已提交
237
#ifdef PADDLE_WITH_MLU
F
fwenguang 已提交
238
  else if (platform::is_mlu_place(dst_place)) {  // NOLINT
F
fwenguang 已提交
239
    memory::Copy(
240
        dst_place, dst_ptr, src_place, src_ptr, size,
F
fwenguang 已提交
241 242 243
        reinterpret_cast<const platform::MLUDeviceContext&>(ctx).stream());
  }
#endif
D
dzhwinter 已提交
244 245
}

246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266
// The fully specialized function should be inline to avoid
// multi-definition.
template <>
inline void TensorFromVector(const std::vector<bool>& src,
                             const platform::DeviceContext& ctx, Tensor* dst) {
  // vector<bool> has no data() member, use array instead.
  // See details:
  // https://stackoverflow.com/questions/46115669/why-does-stdvectorbool-have-no-data/46115714
  bool* array = new bool[src.size()];
  for (unsigned int i = 0; i < src.size(); i++) {
    array[i] = static_cast<bool>(src[i]);
  }

  auto dst_place = ctx.GetPlace();
  auto src_ptr = static_cast<const void*>(array);
  platform::CPUPlace src_place;
  dst->Resize({static_cast<int64_t>(src.size())});
  auto dst_ptr = static_cast<void*>(dst->mutable_data<bool>(dst_place));
  auto size = src.size() * sizeof(bool);

  if (platform::is_cpu_place(dst_place)) {
267
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size);
268 269 270 271
  }
#ifdef PADDLE_WITH_CUDA
  else if (platform::is_gpu_place(dst_place)) {  // NOLINT
    memory::Copy(
272
        dst_place, dst_ptr, src_place, src_ptr, size,
273 274 275 276 277
        reinterpret_cast<const platform::CUDADeviceContext&>(ctx).stream());
  }
#endif
#ifdef PADDLE_WITH_ASCEND_CL
  else if (platform::is_npu_place(dst_place)) {  // NOLINT
278 279 280 281 282 283 284 285 286 287
    //  1. vector -> npu pinned tensor
    platform::NPUPinnedPlace npu_pinned_place;
    Tensor npu_pinned_tensor;
    npu_pinned_tensor.Resize(dst->dims());
    auto npu_pinned_ptr =
        npu_pinned_tensor.mutable_data(npu_pinned_place, dst->type());
    memory::Copy(npu_pinned_place, npu_pinned_ptr, src_place, src_ptr, size);

    //  2. async copy npu pinned tensor -> npu tensor
    memory::Copy(
288
        dst_place, dst_ptr, npu_pinned_place, npu_pinned_ptr, size,
289 290 291 292 293 294 295 296
        reinterpret_cast<const platform::NPUDeviceContext&>(ctx).stream());

    //  3. record event
    auto npu_pinned_allocator =
        static_cast<paddle::memory::allocation::NPUPinnedAllocator*>(
            paddle::memory::allocation::AllocatorFacade::Instance()
                .GetAllocator(npu_pinned_place)
                .get());
297
    pten::Allocation* allocation = npu_pinned_tensor.Holder().get();
298 299 300
    npu_pinned_allocator->RecordEvent(
        allocation,
        reinterpret_cast<const platform::NPUDeviceContext&>(ctx).stream());
301 302 303 304 305
  }
#endif
  delete[] array;
}

D
dzhwinter 已提交
306
template <typename T>
Y
Yi Wang 已提交
307
void TensorFromVector(const std::vector<T>& src, Tensor* dst) {
D
dzhwinter 已提交
308 309 310 311 312 313 314 315 316 317
  platform::CPUPlace dst_place = platform::CPUPlace();
  auto src_ptr = static_cast<const void*>(src.data());
  platform::CPUPlace src_place;
  dst->Resize({static_cast<int64_t>(src.size())});
  auto dst_ptr = static_cast<void*>(dst->mutable_data<T>(dst_place));
  auto size = src.size() * sizeof(T);

  memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size);
}

318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334
template <>
inline void TensorFromVector(const std::vector<bool>& src, Tensor* dst) {
  bool* array = new bool[src.size()];
  for (unsigned int i = 0; i < src.size(); i++) {
    array[i] = static_cast<bool>(src[i]);
  }
  platform::CPUPlace dst_place = platform::CPUPlace();
  auto src_ptr = static_cast<const void*>(array);
  platform::CPUPlace src_place;
  dst->Resize({static_cast<int64_t>(src.size())});
  auto dst_ptr = static_cast<void*>(dst->mutable_data<bool>(dst_place));
  auto size = src.size() * sizeof(bool);

  memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size);
  delete[] array;
}

D
dzhwinter 已提交
335
template <typename T>
Y
Yi Wang 已提交
336 337
void TensorToVector(const Tensor& src, const platform::DeviceContext& ctx,
                    std::vector<T>* dst) {
D
dzhwinter 已提交
338 339 340 341 342 343 344 345
  auto src_ptr = static_cast<const void*>(src.data<T>());
  auto size = src.numel() * sizeof(T);

  platform::CPUPlace dst_place;
  dst->resize(src.numel());
  auto dst_ptr = static_cast<void*>(dst->data());

  if (platform::is_cpu_place(src.place())) {
346
    memory::Copy(dst_place, dst_ptr, src.place(), src_ptr, size);
D
dzhwinter 已提交
347
  }
348
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
D
dzhwinter 已提交
349 350
  else if (platform::is_gpu_place(src.place())) {  // NOLINT
    memory::Copy(
351
        dst_place, dst_ptr, src.place(), src_ptr, size,
D
dzhwinter 已提交
352 353 354
        reinterpret_cast<const platform::CUDADeviceContext&>(ctx).stream());
  }
#endif
355 356
#if defined(PADDLE_WITH_XPU)
  else if (platform::is_xpu_place(src.place())) {  // NOLINT
357
    memory::Copy(dst_place, dst_ptr, src.place(), src_ptr, size);
358 359
  }
#endif
360 361
#ifdef PADDLE_WITH_ASCEND_CL
  else if (platform::is_npu_place(src.place())) {  // NOLINT
362
    memory::Copy(dst_place, dst_ptr, src.place(), src_ptr, size, nullptr);
363 364
  }
#endif
F
fwenguang 已提交
365 366 367
#ifdef PADDLE_WITH_MLU
  else if (platform::is_mlu_place(src.place())) {  // NOLINT
    memory::Copy(
368
        dst_place, dst_ptr, src.place(), src_ptr, size,
F
fwenguang 已提交
369 370 371
        reinterpret_cast<const platform::MLUDeviceContext&>(ctx).stream());
  }
#endif
D
dzhwinter 已提交
372 373
}

374 375 376 377 378 379 380 381 382 383 384 385 386 387
template <>
inline void TensorToVector(const Tensor& src,
                           const platform::DeviceContext& ctx,
                           std::vector<bool>* dst) {
  auto src_ptr = static_cast<const void*>(src.data<bool>());
  auto size = src.numel() * sizeof(bool);

  bool* array = new bool[src.numel()];

  platform::CPUPlace dst_place;
  dst->resize(src.numel());
  auto dst_ptr = static_cast<void*>(array);

  if (platform::is_cpu_place(src.place())) {
388
    memory::Copy(dst_place, dst_ptr, src.place(), src_ptr, size);
389
  }
390
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
391 392
  else if (platform::is_gpu_place(src.place())) {  // NOLINT
    memory::Copy(
393
        dst_place, dst_ptr, src.place(), src_ptr, size,
394 395 396
        reinterpret_cast<const platform::CUDADeviceContext&>(ctx).stream());
  }
#endif
397 398
#if defined(PADDLE_WITH_XPU)
  else if (platform::is_xpu_place(src.place())) {  // NOLINT
399
    memory::Copy(dst_place, dst_ptr, src.place(), src_ptr, size);
400 401
  }
#endif
402 403
#ifdef PADDLE_WITH_ASCEND_CL
  else if (platform::is_npu_place(src.place())) {  // NOLINT
404
    memory::Copy(dst_place, dst_ptr, src.place(), src_ptr, size, nullptr);
405
  }
F
fwenguang 已提交
406 407 408 409
#endif
#ifdef PADDLE_WITH_MLU
  else if (platform::is_mlu_place(src.place())) {  // NOLINT
    memory::Copy(
410
        dst_place, dst_ptr, src.place(), src_ptr, size,
F
fwenguang 已提交
411 412
        reinterpret_cast<const platform::MLUDeviceContext&>(ctx).stream());
  }
413 414 415 416 417 418 419
#endif
  for (unsigned int i = 0; i < src.numel(); i++) {
    (*dst)[i] = static_cast<bool>(array[i]);
  }
  delete[] array;
}

D
dzhwinter 已提交
420
template <typename T>
Y
Yi Wang 已提交
421
void TensorToVector(const Tensor& src, std::vector<T>* dst) {
D
dzhwinter 已提交
422 423 424 425 426 427 428
  auto src_ptr = static_cast<const void*>(src.data<T>());
  auto size = src.numel() * sizeof(T);

  platform::CPUPlace dst_place;
  dst->resize(src.numel());
  auto dst_ptr = static_cast<void*>(dst->data());

429 430 431 432 433
  PADDLE_ENFORCE_EQ(
      platform::is_cpu_place(src.place()), true,
      platform::errors::InvalidArgument(
          "The input tensor should be CPU device, but actually it is in %s.",
          src.place()));
D
dzhwinter 已提交
434

435
  memory::Copy(dst_place, dst_ptr, src.place(), src_ptr, size);
D
dzhwinter 已提交
436
}
437

438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454
template <>
inline void TensorToVector(const Tensor& src, std::vector<bool>* dst) {
  auto src_ptr = static_cast<const void*>(src.data<bool>());
  auto size = src.numel() * sizeof(bool);

  bool* array = new bool[src.numel()];

  platform::CPUPlace dst_place;
  dst->resize(src.numel());
  auto dst_ptr = static_cast<void*>(array);

  PADDLE_ENFORCE_EQ(
      platform::is_cpu_place(src.place()), true,
      platform::errors::InvalidArgument(
          "The input tensor should be CPU device, but actually it is in %s.",
          src.place()));

455
  memory::Copy(dst_place, dst_ptr, src.place(), src_ptr, size);
456 457 458 459 460 461 462

  for (unsigned int i = 0; i < src.numel(); i++) {
    (*dst)[i] = static_cast<bool>(array[i]);
  }
  delete[] array;
}

463 464
std::ostream& operator<<(std::ostream& os, const LoD& lod);

D
dzhwinter 已提交
465 466
}  // namespace framework
}  // namespace paddle
467 468 469 470

namespace pten {
std::ostream& operator<<(std::ostream& os, const DenseTensor& t);
}