tensor_util.cc 57.1 KB
Newer Older
Y
Yang Yu 已提交
1 2
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.

3 4 5
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
Y
Yang Yu 已提交
6

7 8 9 10 11 12 13
    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
Y
Yang Yu 已提交
14

C
chengduoZH 已提交
15 16
#include <algorithm>
#include <limits>
17
#include <memory>
18
#include <string>
19
#include <utility>
C
chengduoZH 已提交
20
#include <vector>
21

22
#include "paddle/fluid/framework/convert_utils.h"
Y
yuyang18 已提交
23
#include "paddle/fluid/framework/data_type.h"
24
#include "paddle/fluid/framework/tensor_util.h"
25
#include "paddle/fluid/platform/complex.h"
26
#include "paddle/fluid/platform/profiler.h"
27 28 29

#include "paddle/pten/core/dense_tensor.h"

30
#ifdef PADDLE_WITH_MKLDNN
31
#include "dnnl_debug.h"  // NOLINT
32
#endif
Y
Yang Yu 已提交
33 34 35

namespace paddle {
namespace framework {
Y
Yi Wang 已提交
36

37 38 39
template <typename TENSOR>
void TensorCopyImpl(const TENSOR& src, const platform::Place& dst_place,
                    const platform::DeviceContext& ctx, TENSOR* dst) {
40 41
  if (&src == dst) {
    auto src_copy = src;
42
    TensorCopyImpl(src_copy, dst_place, ctx, dst);
43 44 45
    return;
  }

M
minqiyang 已提交
46 47
  VLOG(3) << "TensorCopy " << src.dims() << " from " << src.place() << " to "
          << dst_place;
Y
Yi Wang 已提交
48 49 50 51
  src.check_memory_size();
  dst->Resize(src.dims());
  dst->set_layout(src.layout());
  auto src_place = src.place();
52
  auto src_ptr = src.data();
53 54 55 56 57 58
#ifdef PADDLE_WITH_MKLDNN
  dst->set_format(src.format());
  // oneDNN tensors due to padding may be of bigger size
  // than numel()*size(type())
  auto dst_ptr =
      src.layout() == DataLayout::kMKLDNN
59 60
          ? dst->mutable_data(dst_place, src.dtype(), src.memory_size())
          : dst->mutable_data(dst_place, src.dtype());
61
#else
62
  auto dst_ptr = dst->mutable_data(dst_place, src.dtype());
63
#endif
64 65 66 67 68
  if (src_ptr == dst_ptr && src_place == dst_place) {
    VLOG(3) << "Skip copy the same data async from " << src_place << " to "
            << dst_place;
    return;
  }
69
  VLOG(4) << "src:" << src_ptr << ", dst:" << dst_ptr;
70

71 72 73
#ifdef PADDLE_WITH_MKLDNN
  auto size = src.layout() == DataLayout::kMKLDNN
                  ? src.memory_size()
74
                  : src.numel() * framework::DataTypeSize(src.dtype());
75
#else
76
  auto size = src.numel() * framework::DataTypeSize(src.dtype());
77
#endif
Y
Yi Wang 已提交
78 79

  if (platform::is_cpu_place(src_place) && platform::is_cpu_place(dst_place)) {
80
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size);
Y
Yi Wang 已提交
81
  }
82 83 84
#ifdef PADDLE_WITH_IPU
  else if (platform::is_ipu_place(src_place) &&  // NOLINT
           platform::is_cpu_place(dst_place)) {
85
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size);
86 87
  } else if (platform::is_cpu_place(src_place) &&
             platform::is_ipu_place(dst_place)) {
88
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size);
89 90
  } else if (platform::is_ipu_place(src_place) &&
             platform::is_ipu_place(dst_place)) {
91
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size);
92 93
  }
#endif
94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116
#ifdef PADDLE_WITH_CUSTOM_DEVICE
  else if (platform::is_custom_place(src_place) &&  // NOLINT
           platform::is_cpu_place(dst_place)) {
    auto stream =
        reinterpret_cast<const platform::CustomDeviceContext&>(ctx).stream();
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size, stream);
  } else if (platform::is_cpu_place(src_place) &&  // NOLINT
             platform::is_custom_place(dst_place)) {
    auto stream =
        reinterpret_cast<const platform::CustomDeviceContext&>(ctx).stream();
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size, stream);
  } else if (platform::is_custom_place(src_place) &&  // NOLINT
             platform::is_custom_place(dst_place)) {
    if (src_ptr == dst_ptr) {
      VLOG(3) << "Skip copy the same data async from " << src_place << " to "
              << dst_place;
      return;
    }
    auto stream =
        reinterpret_cast<const platform::CustomDeviceContext&>(ctx).stream();
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size, stream);
  }
#endif
117 118 119
#ifdef PADDLE_WITH_XPU
  else if (platform::is_xpu_place(src_place) &&  // NOLINT
           platform::is_cpu_place(dst_place)) {
120
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size);
121 122
  } else if (platform::is_cpu_place(src_place) &&
             platform::is_xpu_place(dst_place)) {
123
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size);
124 125 126 127 128 129 130
  } else if (platform::is_xpu_place(src_place) &&
             platform::is_xpu_place(dst_place)) {
    if (src_ptr == dst_ptr) {
      VLOG(3) << "Skip copy the same data async from " << src_place << " to "
              << dst_place;
      return;
    }
131
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size);
132 133 134 135 136
  } else {
    PADDLE_THROW(platform::errors::Unimplemented(
        "Copy from %s to %s is not supported.", src_place, dst_place));
  }
#endif
137 138 139 140 141 142
#ifdef PADDLE_WITH_ASCEND_CL
  // TODO(zhiqiu): handle different condition like CUDA code below
  else if (platform::is_npu_place(src_place) &&  // NOLINT
           platform::is_cpu_place(dst_place)) {
    auto stream =
        reinterpret_cast<const platform::NPUDeviceContext&>(ctx).stream();
143
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size, stream);
144 145 146
  }
  else if (platform::is_cpu_place(src_place) &&  // NOLINT
           platform::is_npu_place(dst_place)) {
147 148 149 150 151
    //  1. cpu tensor -> npu pinned tensor
    platform::NPUPinnedPlace npu_pinned_place;
    Tensor npu_pinned_tensor;
    npu_pinned_tensor.Resize(src.dims());
    auto npu_pinned_ptr =
152
        npu_pinned_tensor.mutable_data(npu_pinned_place, src.dtype());
153
    memory::Copy(npu_pinned_place, npu_pinned_ptr, src_place, src_ptr, size);
154 155 156

    //  2. async copy npu pinned tensor -> npu tensor
    memory::Copy(
157
        dst_place, dst_ptr, npu_pinned_place, npu_pinned_ptr, size,
158 159 160 161 162 163 164 165
        reinterpret_cast<const platform::NPUDeviceContext&>(ctx).stream());

    //  3. record event
    auto npu_pinned_allocator =
        static_cast<paddle::memory::allocation::NPUPinnedAllocator*>(
            paddle::memory::allocation::AllocatorFacade::Instance()
                .GetAllocator(npu_pinned_place)
                .get());
166
    pten::Allocation* allocation = npu_pinned_tensor.Holder().get();
167 168 169
    npu_pinned_allocator->RecordEvent(
        allocation,
        reinterpret_cast<const platform::NPUDeviceContext&>(ctx).stream());
170 171 172 173 174 175 176 177 178 179
  }
  else if (platform::is_npu_place(src_place) &&  // NOLINT
           platform::is_npu_place(dst_place)) {
    if (src_ptr == dst_ptr) {
      VLOG(3) << "Skip copy the same data async from " << src_place << " to "
              << dst_place;
      return;
    }
    auto stream =
        reinterpret_cast<const platform::NPUDeviceContext&>(ctx).stream();
180
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size, stream);
181
  }
182 183
  else if (platform::is_npu_pinned_place(src_place) &&  // NOLINT
           platform::is_npu_place(dst_place)) {         /* npu_pinned->npu */
184 185
    auto src_npu_pinned_place = src_place;
    auto dst_npu_place = dst_place;
186 187 188 189 190 191
    auto ctx_place = ctx.GetPlace();
    PADDLE_ENFORCE_EQ(platform::is_npu_place(ctx_place), true,
                      platform::errors::PreconditionNotMet(
                          "Device context place mismatch. When copying Tensor "
                          "data from NPU Pinned memory to NPU memory, current "
                          "device context place should be NPU."));
192
    auto ctx_npu_place = ctx_place;
193 194 195 196 197 198 199 200 201 202 203 204 205
    PADDLE_ENFORCE_EQ(dst_npu_place, ctx_npu_place,
                      platform::errors::PreconditionNotMet(
                          "The target NPU device and current device context do "
                          "not match. The target NPU device number is %d, but "
                          "device context NPU number is %d.",
                          dst_npu_place.device, ctx_npu_place.device));
    auto stream =
        reinterpret_cast<const platform::NPUDeviceContext&>(ctx).stream();
    memory::Copy(dst_npu_place, dst_ptr, src_npu_pinned_place, src_ptr, size,
                 stream);
  }
  else if (platform::is_npu_place(src_place) &&        // NOLINT
           platform::is_npu_pinned_place(dst_place)) { /* npu->npu_pinned */
206 207
    auto src_npu_place = src_place;
    auto dst_npu_pinned_place = dst_place;
208 209 210 211 212 213
    auto ctx_place = ctx.GetPlace();
    PADDLE_ENFORCE_EQ(platform::is_npu_place(ctx_place), true,
                      platform::errors::PreconditionNotMet(
                          "Device context place mismatch. When copying Tensor "
                          "data from NPU memory to NPU Pinned memory, current "
                          "device context place should be NPU."));
214
    auto ctx_npu_place = ctx_place;
215 216 217 218 219 220 221 222 223 224 225
    PADDLE_ENFORCE_EQ(src_place, ctx_npu_place,
                      platform::errors::PreconditionNotMet(
                          "The source NPU device and current device context do "
                          "not match. The source NPU device number is %d, but "
                          "device context NPU number is %d.",
                          src_npu_place.device, ctx_npu_place.device));
    auto stream =
        reinterpret_cast<const platform::NPUDeviceContext&>(ctx).stream();
    memory::Copy(dst_npu_pinned_place, dst_ptr, src_npu_place, src_ptr, size,
                 stream);
  }
226 227 228 229 230
  else {  // NOLINT
    PADDLE_THROW(platform::errors::Unimplemented(
        "Copy from %s to %s is not supported.", src_place, dst_place));
  }
#endif
231
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
232 233
  else if (platform::is_cuda_pinned_place(src_place) &&  // NOLINT
           platform::is_cuda_pinned_place(dst_place)) {
234
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size);
235
  }
236
  else if (platform::is_cuda_pinned_place(src_place) &&  // NOLINT
Y
Yi Wang 已提交
237
           platform::is_cpu_place(dst_place)) {
238
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size);
239 240 241
  }
  else if (platform::is_cpu_place(src_place) &&  // NOLINT
           platform::is_cuda_pinned_place(dst_place)) {
242
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size);
243 244 245
  }
  else if (platform::is_gpu_place(src_place) &&  // NOLINT
           platform::is_cpu_place(dst_place)) {
246 247
    auto src_gpu_place = src_place;
    auto dst_cpu_place = dst_place;
Y
Yi Wang 已提交
248
    auto ctx_place = ctx.GetPlace();
249 250 251 252 253
    PADDLE_ENFORCE_EQ(
        platform::is_gpu_place(ctx_place), true,
        platform::errors::PreconditionNotMet(
            "Context place error, excepted GPUPlace, but actually %s.",
            ctx_place));
254
    auto ctx_gpu_place = ctx_place;
255 256 257 258 259
    PADDLE_ENFORCE_EQ(src_gpu_place, ctx_gpu_place,
                      platform::errors::Unavailable(
                          "Source place and context place do not match, source "
                          "place is %s, context place is %s.",
                          src_gpu_place, ctx_gpu_place));
260
    auto stream =
F
fengjiayi 已提交
261
        reinterpret_cast<const platform::CUDADeviceContext&>(ctx).stream();
262
    memory::Copy(dst_cpu_place, dst_ptr, src_gpu_place, src_ptr, size, stream);
263 264 265
  }
  else if (platform::is_cpu_place(src_place) &&  // NOLINT
           platform::is_gpu_place(dst_place)) {
266 267
    auto src_cpu_place = src_place;
    auto dst_gpu_place = dst_place;
Y
Yi Wang 已提交
268
    auto ctx_place = ctx.GetPlace();
269 270 271 272 273
    PADDLE_ENFORCE_EQ(
        platform::is_gpu_place(ctx_place), true,
        platform::errors::PreconditionNotMet(
            "Context place error, excepted GPUPlace, but actually %s.",
            ctx_place));
274
    auto ctx_gpu_place = ctx_place;
275 276 277 278 279
    PADDLE_ENFORCE_EQ(dst_gpu_place, ctx_gpu_place,
                      platform::errors::Unavailable(
                          "Destination place and context place do not match, "
                          "destination place is %s, context place is %s.",
                          dst_gpu_place, ctx_gpu_place));
280
    auto stream =
F
fengjiayi 已提交
281
        reinterpret_cast<const platform::CUDADeviceContext&>(ctx).stream();
282
    memory::Copy(dst_gpu_place, dst_ptr, src_cpu_place, src_ptr, size, stream);
283 284 285
  }
  else if (platform::is_gpu_place(src_place) &&  // NOLINT
           platform::is_cuda_pinned_place(dst_place)) {
286 287
    auto src_gpu_place = src_place;
    auto dst_cuda_pinned_place = dst_place;
288 289 290 291 292 293
    auto ctx_place = ctx.GetPlace();
    PADDLE_ENFORCE_EQ(platform::is_gpu_place(ctx_place), true,
                      platform::errors::PreconditionNotMet(
                          "Device context place mismatch. When copying Tensor "
                          "data from GPU memory to CUDA Pinned memory, current "
                          "device context place should be GPU."));
294
    auto ctx_gpu_place = ctx_place;
295 296 297 298 299 300 301 302 303 304
    PADDLE_ENFORCE_EQ(src_gpu_place, ctx_gpu_place,
                      platform::errors::PreconditionNotMet(
                          "The source GPU device and current device context do "
                          "not match. The source GPU device number is %d, but "
                          "device context GPU number is %d.",
                          src_gpu_place.device, ctx_gpu_place.device));
    auto stream =
        reinterpret_cast<const platform::CUDADeviceContext&>(ctx).stream();
    memory::Copy(dst_cuda_pinned_place, dst_ptr, src_gpu_place, src_ptr, size,
                 stream);
305 306 307
  }
  else if (platform::is_cuda_pinned_place(src_place) &&  // NOLINT
           platform::is_gpu_place(dst_place)) {
308 309
    auto src_cuda_pinned_place = src_place;
    auto dst_gpu_place = dst_place;
310 311 312 313 314 315
    auto ctx_place = ctx.GetPlace();
    PADDLE_ENFORCE_EQ(platform::is_gpu_place(ctx_place), true,
                      platform::errors::PreconditionNotMet(
                          "Device context place mismatch. When copying Tensor "
                          "data from CUDA Pinned memory to GPU memory, current "
                          "device context place should be GPU."));
316
    auto ctx_gpu_place = ctx_place;
317 318 319 320 321 322 323 324 325 326
    PADDLE_ENFORCE_EQ(dst_gpu_place, ctx_gpu_place,
                      platform::errors::PreconditionNotMet(
                          "The target GPU device and current device context do "
                          "not match. The target GPU device number is %d, but "
                          "device context GPU number is %d.",
                          dst_gpu_place.device, ctx_gpu_place.device));
    auto stream =
        reinterpret_cast<const platform::CUDADeviceContext&>(ctx).stream();
    memory::Copy(dst_gpu_place, dst_ptr, src_cuda_pinned_place, src_ptr, size,
                 stream);
327 328 329
  }
  else if (platform::is_gpu_place(src_place) &&  // NOLINT
           platform::is_gpu_place(dst_place)) {
330 331
    auto src_gpu_place = src_place;
    auto dst_gpu_place = dst_place;
Y
Yi Wang 已提交
332
    auto ctx_place = ctx.GetPlace();
333 334 335 336 337
    PADDLE_ENFORCE_EQ(
        platform::is_gpu_place(ctx_place), true,
        platform::errors::PreconditionNotMet(
            "Context place error, excepted GPUPlace, but actually %s.",
            ctx_place));
338
    auto stream =
F
fengjiayi 已提交
339
        reinterpret_cast<const platform::CUDADeviceContext&>(ctx).stream();
C
chengduo 已提交
340 341 342 343 344 345 346
    if (platform::is_same_place(src_place, dst_place)) {
      memory::Copy(dst_gpu_place, dst_ptr, src_gpu_place, src_ptr, size,
                   stream);
    } else {
      if (platform::is_same_place(ctx_place, src_place)) {
        memory::Copy(dst_gpu_place, dst_ptr, src_gpu_place, src_ptr, size,
                     stream);
C
chengduo 已提交
347
        platform::DeviceContextPool::Instance().Get(src.place())->Wait();
C
chengduo 已提交
348
      } else if (platform::is_same_place(ctx_place, dst_place)) {
C
chengduo 已提交
349
        platform::DeviceContextPool::Instance().Get(src.place())->Wait();
C
chengduo 已提交
350 351 352
        memory::Copy(dst_gpu_place, dst_ptr, src_gpu_place, src_ptr, size,
                     stream);
      } else {
353 354
        PADDLE_THROW(platform::errors::Unavailable(
            "Context place dose not match the source and destination place."));
C
chengduo 已提交
355 356
      }
    }
357 358
  }
  else {  // NOLINT
359 360
    PADDLE_THROW(platform::errors::Unimplemented(
        "Copying from %s to %s is not supported.", src_place, dst_place));
Y
Yi Wang 已提交
361 362
  }
#endif
363 364 365
#ifdef PADDLE_WITH_MLU
  else if (platform::is_mlu_place(src_place) &&  // NOLINT
           platform::is_cpu_place(dst_place)) {
366 367
    auto src_mlu_place = src_place;
    auto dst_cpu_place = dst_place;
368 369 370 371 372 373
    auto stream =
        reinterpret_cast<const platform::MLUDeviceContext&>(ctx).stream();
    memory::Copy(dst_cpu_place, dst_ptr, src_mlu_place, src_ptr, size, stream);
  }
  else if (platform::is_cpu_place(src_place) &&  // NOLINT
           platform::is_mlu_place(dst_place)) {
374 375
    auto src_cpu_place = src_place;
    auto dst_mlu_place = dst_place;
376 377 378 379 380 381
    auto stream =
        reinterpret_cast<const platform::MLUDeviceContext&>(ctx).stream();
    memory::Copy(dst_mlu_place, dst_ptr, src_cpu_place, src_ptr, size, stream);
  }
  else if (platform::is_mlu_place(src_place) &&  // NOLINT
           platform::is_mlu_place(dst_place)) {
382 383
    auto src_mlu_place = src_place;
    auto dst_mlu_place = dst_place;
384 385 386 387 388 389 390 391 392
    auto stream =
        reinterpret_cast<const platform::MLUDeviceContext&>(ctx).stream();
    memory::Copy(dst_mlu_place, dst_ptr, src_mlu_place, src_ptr, size, stream);
  }
  else {  // NOLINT
    PADDLE_THROW(platform::errors::Unimplemented(
        "Copying from %s to %s is not supported.", src_place, dst_place));
  }
#endif
Y
Yi Wang 已提交
393 394
}

395 396 397
template <typename TENSOR>
void TensorCopyImpl(const TENSOR& src, const platform::Place& dst_place,
                    TENSOR* dst) {
Y
Yi Wang 已提交
398 399
  platform::DeviceContextPool& pool = platform::DeviceContextPool::Instance();
  const platform::DeviceContext* dev_ctx;
400
  if (platform::is_gpu_place(dst_place) || platform::is_npu_place(dst_place) ||
401 402
      platform::is_mlu_place(dst_place) ||
      platform::is_custom_place(dst_place)) {
Y
Yi Wang 已提交
403
    dev_ctx = pool.Get(dst_place);
C
chengduo 已提交
404 405
  } else {
    dev_ctx = pool.Get(src.place());
Y
Yi Wang 已提交
406
  }
407 408 409 410 411 412 413 414 415 416 417
  TensorCopyImpl(src, dst_place, *dev_ctx, dst);
}

void TensorCopy(const Tensor& src, const platform::Place& dst_place,
                Tensor* dst) {
  TensorCopyImpl<Tensor>(src, dst_place, dst);
}
void TensorCopy(const Tensor& src, const platform::Place& dst_place,
                const platform::DeviceContext& ctx, Tensor* dst) {
  TensorCopyImpl<Tensor>(src, dst_place, ctx, dst);
}
Y
Yi Wang 已提交
418

F
fengjiayi 已提交
419 420
void TensorCopySync(const Tensor& src, const platform::Place& dst_place,
                    Tensor* dst) {
421 422 423 424 425 426
  if (&src == dst) {
    auto src_copy = src;
    TensorCopySync(src_copy, dst_place, dst);
    return;
  }

M
minqiyang 已提交
427 428
  VLOG(3) << "TensorCopySync " << src.dims() << " from " << src.place()
          << " to " << dst_place;
F
fengjiayi 已提交
429 430 431
  src.check_memory_size();
  dst->Resize(src.dims());
  dst->set_layout(src.layout());
J
Jacek Czaja 已提交
432 433 434
#ifdef PADDLE_WITH_MKLDNN
  dst->set_format(src.format());
#endif
F
fengjiayi 已提交
435
  auto src_place = src.place();
436
  auto src_ptr = src.data();
437
  auto dst_ptr = dst->mutable_data(dst_place, src.dtype());
438
  VLOG(4) << "src:" << src_ptr << ", dst:" << dst_ptr;
439 440 441 442 443 444 445

  if (src_ptr == dst_ptr && src_place == dst_place) {
    VLOG(3) << "Skip copy the same data from " << src_place << " to "
            << dst_place;
    return;
  }

446
  auto size = src.numel() * framework::DataTypeSize(src.dtype());
F
fengjiayi 已提交
447
  if (platform::is_cpu_place(src_place) && platform::is_cpu_place(dst_place)) {
448
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size);
F
fengjiayi 已提交
449
  }
450 451 452
#ifdef PADDLE_WITH_IPU
  else if (platform::is_ipu_place(src_place) &&  // NOLINT
           platform::is_cpu_place(dst_place)) {
453
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size);
454 455
  } else if (platform::is_cpu_place(src_place) &&  // NOLINT
             platform::is_ipu_place(dst_place)) {
456
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size);
457 458 459 460 461
  } else {  // NOLINT
    PADDLE_THROW(platform::errors::Unimplemented(
        "Copy from %s to %s is not supported.", src_place, dst_place));
  }
#endif
462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481
#ifdef PADDLE_WITH_CUSTOM_DEVICE
  else if (platform::is_custom_place(src_place) &&  // NOLINT
           platform::is_cpu_place(dst_place)) {     /* custom_device -> cpu*/
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size, nullptr);
  }
  else if (platform::is_cpu_place(src_place) &&    // NOLINT
           platform::is_custom_place(dst_place)) { /* cpu -> custom_device*/
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size, nullptr);
  }
  else if (platform::is_custom_place(src_place) &&  // NOLINT
           platform::is_custom_place(
               dst_place)) { /* custom_device -> custom_device*/
    if (src_ptr == dst_ptr) {
      VLOG(3) << "Skip copy the same data sync from " << src_place << " to "
              << dst_place;
      return;
    }
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size, nullptr);
  }
#endif
482 483 484
#ifdef PADDLE_WITH_XPU
  else if (platform::is_xpu_place(src_place) &&  // NOLINT
           platform::is_cpu_place(dst_place)) {
485
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size);
486 487 488
  }
  else if (platform::is_cpu_place(src_place) &&  // NOLINT
           platform::is_xpu_place(dst_place)) {
489
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size);
490 491 492
  }
  else if (platform::is_xpu_place(src_place) &&  // NOLINT
           platform::is_xpu_place(dst_place)) {
493 494 495 496 497
    if (src_ptr == dst_ptr) {
      VLOG(3) << "Skip copy the same data async from " << src_place << " to "
              << dst_place;
      return;
    }
498 499 500
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size);
    platform::XPUPlace xpu_dst_place = dst_place;
    platform::XPUPlace xpu_src_place = src_place;
501 502 503 504
    if (xpu_dst_place.device == xpu_src_place.device) {
      auto xpu_ctx = platform::DeviceContextPool::Instance().Get(xpu_dst_place);
      xpu_ctx->Wait();
    }
505 506
  }
  else {  // NOLINT
507 508 509 510
    PADDLE_THROW(platform::errors::Unimplemented(
        "Copy from %s to %s is not supported.", src_place, dst_place));
  }
#endif
511 512 513
#ifdef PADDLE_WITH_ASCEND_CL
  else if (platform::is_npu_place(src_place) &&  // NOLINT
           platform::is_cpu_place(dst_place)) {  /* npu -> cpu*/
514
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size, nullptr);
515 516 517
  }
  else if (platform::is_cpu_place(src_place) &&  // NOLINT
           platform::is_npu_place(dst_place)) {  /* cpu -> npu*/
518
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size, nullptr);
519 520 521 522 523 524 525 526
  }
  else if (platform::is_npu_place(src_place) &&  // NOLINT
           platform::is_npu_place(dst_place)) {  /* npu -> npu*/
    if (src_ptr == dst_ptr) {
      VLOG(3) << "Skip copy the same data sync from " << src_place << " to "
              << dst_place;
      return;
    }
527
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size, nullptr);
528 529 530 531 532 533
  }
  else {  // NOLINT
    PADDLE_THROW(platform::errors::Unimplemented(
        "Copy from %s to %s is not supported.", src_place, dst_place));
  }
#endif
534
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
535 536
  else if (platform::is_cuda_pinned_place(src_place) &&  // NOLINT
           platform::is_cuda_pinned_place(dst_place)) {
537
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size);
538
  }
539
  else if (platform::is_cuda_pinned_place(src_place) &&  // NOLINT
F
fengjiayi 已提交
540
           platform::is_cpu_place(dst_place)) {
541
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size);
542 543 544
  }
  else if (platform::is_cpu_place(src_place) &&  // NOLINT
           platform::is_cuda_pinned_place(dst_place)) {
545
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size);
546 547 548
  }
  else if (platform::is_gpu_place(src_place) &&  // NOLINT
           platform::is_cuda_pinned_place(dst_place)) {
549
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size, nullptr);
550 551 552
  }
  else if (platform::is_gpu_place(src_place) &&  // NOLINT
           platform::is_cpu_place(dst_place)) {
553 554
    auto src_gpu_place = src_place;
    auto dst_cpu_place = dst_place;
F
fengjiayi 已提交
555
    memory::Copy(dst_cpu_place, dst_ptr, src_gpu_place, src_ptr, size, nullptr);
556 557 558
  }
  else if (platform::is_cpu_place(src_place) &&  // NOLINT
           platform::is_gpu_place(dst_place)) {
559 560
    auto src_cpu_place = src_place;
    auto dst_gpu_place = dst_place;
F
fengjiayi 已提交
561
    memory::Copy(dst_gpu_place, dst_ptr, src_cpu_place, src_ptr, size, nullptr);
562 563 564
  }
  else if (platform::is_gpu_place(src_place) &&  // NOLINT
           platform::is_gpu_place(dst_place)) {
565 566
    auto src_gpu_place = src_place;
    auto dst_gpu_place = dst_place;
F
fengjiayi 已提交
567
    memory::Copy(dst_gpu_place, dst_ptr, src_gpu_place, src_ptr, size, nullptr);
568 569 570
  }
  else if (platform::is_cuda_pinned_place(src_place) &&  // NOLINT
           platform::is_gpu_place(dst_place)) {
571 572
    auto src_pinned_place = src_place;
    auto dst_gpu_place = dst_place;
573 574
    memory::Copy(dst_gpu_place, dst_ptr, src_pinned_place, src_ptr, size,
                 nullptr);
575 576
  }
  else {  // NOLINT
577 578
    PADDLE_THROW(platform::errors::Unimplemented(
        "Copy from %s to %s is not supported.", src_place, dst_place));
F
fengjiayi 已提交
579 580
  }
#endif
581 582 583
#ifdef PADDLE_WITH_MLU
  else if (platform::is_mlu_place(src_place) &&  // NOLINT
           platform::is_cpu_place(dst_place)) {
584
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size, nullptr);
585 586 587
  }
  else if (platform::is_cpu_place(src_place) &&  // NOLINT
           platform::is_mlu_place(dst_place)) {
588
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size, nullptr);
589 590 591 592 593 594 595 596
  }
  else if (platform::is_mlu_place(src_place) &&  // NOLINT
           platform::is_mlu_place(dst_place)) {
    if (src_ptr == dst_ptr) {
      VLOG(3) << "Skip copy the same data async from " << src_place << " to "
              << dst_place;
      return;
    }
597
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size, nullptr);
598 599 600 601 602 603
  }
  else {  // NOLINT
    PADDLE_THROW(platform::errors::Unimplemented(
        "Copy from %s to %s is not supported.", src_place, dst_place));
  }
#endif
F
fengjiayi 已提交
604 605
}

Y
Yang Yu 已提交
606 607 608 609 610 611 612 613 614 615 616 617
template <typename Predicate, typename DevCtx>
struct AnyDTypeVisitor {
  Predicate predicate_;
  const Tensor& tensor_;
  const DevCtx& ctx_;
  Tensor* out_;

  AnyDTypeVisitor(Predicate predicate, const Tensor& tensor, const DevCtx& ctx,
                  Tensor* out)
      : predicate_(predicate), tensor_(tensor), ctx_(ctx), out_(out) {}

  template <typename T>
D
dzhwinter 已提交
618
  void apply() const {
Y
Yang Yu 已提交
619 620
    auto t = EigenVector<T>::Flatten(tensor_);
    auto o = EigenScalar<bool>::From(*out_);
Y
Yang Yu 已提交
621
    // return any of predicate_(t) is true.
Y
Yang Yu 已提交
622 623 624 625 626 627 628
    o.device(*ctx_.eigen_device()) = predicate_(t).any();
  }
};

template <typename Predicate, typename DevCtx>
inline void AnyImpl(Predicate predicate, const framework::Tensor& tensor,
                    const DevCtx& ctx, framework::Tensor* out) {
629 630 631
  VisitDataType(
      framework::TransToProtoVarType(tensor.dtype()),
      AnyDTypeVisitor<Predicate, DevCtx>(predicate, tensor, ctx, out));
Y
Yang Yu 已提交
632 633 634
}

template <typename Predicate>
635 636
class AnyVisitor : public boost::static_visitor<bool> {
 private:
Y
Yang Yu 已提交
637 638 639
  const framework::Tensor& tensor_;
  Predicate predicate_;

640 641 642 643 644 645 646 647 648 649 650 651 652
  bool GetResultHelper(const framework::Tensor& out,
                       const platform::Place& place) const {
    platform::CPUPlace cpu;
    framework::Tensor tmp;
    tmp.Resize({1});
    tmp.mutable_data<bool>(cpu);
    auto ctx = platform::DeviceContextPool::Instance().Get(place);
    ctx->Wait();
    TensorCopy(out, cpu, *ctx, &tmp);
    ctx->Wait();
    return GetResult(tmp, cpu);
  }

653
 public:
Y
Yang Yu 已提交
654 655 656 657 658 659 660 661 662 663 664 665 666
  AnyVisitor(const framework::Tensor& tensor, Predicate predicate)
      : tensor_(tensor), predicate_(std::move(predicate)) {}

  template <typename Place>
  bool operator()(const Place& place) const {
    framework::Tensor out;
    out.Resize({1});
    out.mutable_data<bool>(place);
    auto* ctx = platform::DeviceContextPool::Instance().GetByPlace(place);
    AnyImpl(predicate_, tensor_, *ctx, &out);
    return this->GetResult(out, place);
  }

667 668 669 670 671
  bool GetResult(const framework::Tensor& out,
                 const platform::XPUPlace& xpu) const {
    return GetResultHelper(out, xpu);
  }

672 673 674 675 676 677 678
  bool GetResult(const framework::Tensor& out,
                 const platform::MLUPlace& mlu) const {
    PADDLE_THROW(
        platform::errors::Unimplemented("Not supported on place (%s) ", mlu));
    return true;
  }

Y
Yang Yu 已提交
679 680
  bool GetResult(const framework::Tensor& out,
                 const platform::CUDAPlace& gpu) const {
681
    return GetResultHelper(out, gpu);
Y
Yang Yu 已提交
682 683
  }

684 685 686 687 688 689
  bool GetResult(const framework::Tensor& out,
                 const platform::NPUPlace& npu) const {
    PADDLE_THROW(
        platform::errors::Unimplemented("Not supported on place (%s) ", npu));
    // return GetResultHelper(out, npu);
  }
690 691 692 693 694
  bool GetResult(const framework::Tensor& out,
                 const platform::IPUPlace& ipu) const {
    PADDLE_THROW(
        platform::errors::Unimplemented("Not supported on place (%s) ", ipu));
  }
695

696 697 698 699 700
  bool GetResult(const framework::Tensor& out,
                 const platform::NPUPinnedPlace& cpu) const {
    return *out.data<bool>();
  }

Y
Yang Yu 已提交
701 702 703 704
  bool GetResult(const framework::Tensor& out,
                 const platform::CPUPlace& cpu) const {
    return *out.data<bool>();
  }
C
chengduoZH 已提交
705 706 707 708 709

  bool GetResult(const framework::Tensor& out,
                 const platform::CUDAPinnedPlace& cpu) const {
    return *out.data<bool>();
  }
710 711 712 713 714 715 716

  bool GetResult(const framework::Tensor& out,
                 const platform::CustomPlace& custom_dev) const {
    PADDLE_THROW(platform::errors::Unimplemented("Not supported on place (%s) ",
                                                 custom_dev));
    return false;
  }
Y
Yang Yu 已提交
717 718
};

719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739
template <typename Predicate>
class AnyOutVisitor : public boost::static_visitor<> {
 private:
  const framework::Tensor& tensor_;
  mutable framework::Tensor* out_;
  Predicate predicate_;

 public:
  AnyOutVisitor(const framework::Tensor& tensor, Predicate predicate,
                framework::Tensor* out)
      : tensor_(tensor), out_(out), predicate_(std::move(predicate)) {}

  template <typename Place>
  void operator()(const Place& place) const {
    auto* ctx = platform::DeviceContextPool::Instance().GetByPlace(place);
    out_->Resize({1});
    out_->mutable_data<bool>(place);
    AnyImpl(predicate_, tensor_, *ctx, out_);
  }
};

Y
Yang Yu 已提交
740 741 742 743 744 745 746
template <typename Predicate>
inline bool Any(const framework::Tensor& tensor, Predicate predicate) {
  AnyVisitor<Predicate> visitor(tensor, predicate);
  auto place = tensor.place();
  return platform::VisitPlace(place, visitor);
}

747 748 749 750 751 752 753 754
template <typename Predicate>
inline void Any(const framework::Tensor& tensor, Predicate predicate,
                framework::Tensor* out) {
  AnyOutVisitor<Predicate> visitor(tensor, predicate, out);
  auto place = tensor.place();
  platform::VisitPlace(place, visitor);
}

J
Jack Zhou 已提交
755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776
template <typename Predicate, typename DevCtx>
struct AllDTypeVisitor {
  Predicate predicate_;
  const Tensor& tensor_;
  const DevCtx& ctx_;
  Tensor* out_;

  AllDTypeVisitor(Predicate predicate, const Tensor& tensor, const DevCtx& ctx,
                  Tensor* out)
      : predicate_(predicate), tensor_(tensor), ctx_(ctx), out_(out) {}

  template <typename T>
  void apply() const {
    auto t = EigenVector<T>::Flatten(tensor_);
    auto o = EigenVector<bool>::Flatten(*out_);
    o.device(*ctx_.eigen_device()) = predicate_(t);
  }
};

template <typename Predicate, typename DevCtx>
inline void AllImpl(Predicate predicate, const framework::Tensor& tensor,
                    const DevCtx& ctx, framework::Tensor* out) {
777 778 779
  VisitDataType(
      framework::TransToProtoVarType(tensor.dtype()),
      AllDTypeVisitor<Predicate, DevCtx>(predicate, tensor, ctx, out));
J
Jack Zhou 已提交
780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810
}

template <typename Predicate>
class AllOutVisitor : public boost::static_visitor<> {
 private:
  const framework::Tensor& tensor_;
  mutable framework::Tensor* out_;
  Predicate predicate_;

 public:
  AllOutVisitor(const framework::Tensor& tensor, Predicate predicate,
                framework::Tensor* out)
      : tensor_(tensor), out_(out), predicate_(predicate) {}

  template <typename Place>
  void operator()(const Place& place) const {
    auto* ctx = platform::DeviceContextPool::Instance().GetByPlace(place);
    out_->Resize(tensor_.dims());
    out_->mutable_data<bool>(place);
    AllImpl(predicate_, tensor_, *ctx, out_);
  }
};

template <typename Predicate>
inline void All(const framework::Tensor& tensor, Predicate predicate,
                framework::Tensor* out) {
  AllOutVisitor<Predicate> visitor(tensor, predicate, out);
  auto place = tensor.place();
  platform::VisitPlace(place, visitor);
}

Y
Yi Wang 已提交
811
struct ContainsNANPredicate {
Y
Yang Yu 已提交
812 813 814
  template <typename T>
  auto operator()(const T& eigen_vec) const
      -> decltype(std::declval<T>().isnan()) {
Y
Yang Yu 已提交
815
    // Cast eigen_vector to vector of bool. true if is inf.
Y
Yang Yu 已提交
816 817 818 819
    return eigen_vec.isnan();
  }
};

Y
Yi Wang 已提交
820 821
bool TensorContainsNAN(const framework::Tensor& tensor) {
  ContainsNANPredicate predicate;
Y
Yang Yu 已提交
822 823 824
  return Any(tensor, predicate);
}

825 826 827 828 829 830
void TensorContainsNAN(const framework::Tensor& tensor,
                       framework::Tensor* out) {
  ContainsNANPredicate predicate;
  Any(tensor, predicate, out);
}

J
Jack Zhou 已提交
831 832 833 834 835 836
void TensorContainsNANV2(const framework::Tensor& tensor,
                         framework::Tensor* out) {
  ContainsNANPredicate predicate;
  All(tensor, predicate, out);
}

Y
Yi Wang 已提交
837
struct ContainsInfPredicate {
Y
Yang Yu 已提交
838 839 840
  template <typename T>
  auto operator()(const T& eigen_vec) const
      -> decltype(std::declval<T>().isinf()) {
Y
Yang Yu 已提交
841
    // Cast eigen_vector to vector of bool. true if is inf.
Y
Yang Yu 已提交
842 843 844 845
    return eigen_vec.isinf();
  }
};

Y
Yi Wang 已提交
846 847
bool TensorContainsInf(const framework::Tensor& tensor) {
  ContainsInfPredicate predicate;
Y
Yang Yu 已提交
848 849 850
  return Any(tensor, predicate);
}

851 852 853 854 855 856
void TensorContainsInf(const framework::Tensor& tensor,
                       framework::Tensor* out) {
  ContainsInfPredicate predicate;
  Any(tensor, predicate, out);
}

J
Jack Zhou 已提交
857 858 859 860 861 862
void TensorContainsInfV2(const framework::Tensor& tensor,
                         framework::Tensor* out) {
  ContainsInfPredicate predicate;
  All(tensor, predicate, out);
}

863 864 865 866 867 868 869 870 871 872
// NOTE(dzhwinter):
// Isfinite need a AllVisitor to loop through all the elements.
// We choose two cuda call instead of one allvisitor. The AllVisitor
// should be implemented if the performance hurts.
bool TensorIsfinite(const framework::Tensor& tensor) {
  ContainsInfPredicate pred_inf;
  ContainsNANPredicate pred_nan;
  return !Any(tensor, pred_inf) && !Any(tensor, pred_nan);
}

873
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
874
template <typename T>
J
Jack Zhou 已提交
875 876
static inline void __global__ BothFalse(const T* cmp, T* out, int element_num) {
  CUDA_KERNEL_LOOP(i, element_num) { out[i] = (!cmp[i]) && (!out[i]); }
877 878 879 880 881 882 883 884 885 886 887 888 889 890
}
#endif

struct BothFalseVisitor : public boost::static_visitor<> {
  const framework::Tensor& in_;
  mutable framework::Tensor* out_;
  BothFalseVisitor(const framework::Tensor& in, framework::Tensor* out)
      : in_(in), out_(out) {}

  template <typename Place>
  void operator()(const Place& place) const {
    VisitorImpl(place);
  }

891 892 893
  void VisitorImpl(const platform::XPUPlace& xpu) const {
    PADDLE_THROW(platform::errors::Unimplemented("XPUPlace is not supported"));
  }
894 895 896
  void VisitorImpl(const platform::IPUPlace& ipu) const {
    PADDLE_THROW(platform::errors::Unimplemented("IPUPlace is not supported"));
  }
897

898
  void VisitorImpl(const platform::CUDAPlace& gpu) const {
899
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
900
    auto* ctx = platform::DeviceContextPool::Instance().GetByPlace(gpu);
J
Jack Zhou 已提交
901 902 903 904 905 906 907 908 909 910
    constexpr int MAX_BLOCK_DIM = 512;
    const int MAX_GRID_DIM = ctx->GetMaxPhysicalThreadCount() / MAX_BLOCK_DIM;
    int element_num = in_.numel();
    int block_size = (element_num >= MAX_BLOCK_DIM)
                         ? MAX_BLOCK_DIM
                         : (1 << static_cast<int>(std::log2(element_num)));
    int grid_size = element_num / block_size;
    grid_size = (grid_size >= MAX_GRID_DIM) ? MAX_GRID_DIM : grid_size;
    BothFalse<bool><<<grid_size, block_size, 0, ctx->stream()>>>(
        in_.data<bool>(), out_->mutable_data<bool>(gpu), element_num);
911 912 913
#endif
  }

914 915 916 917
  void VisitorImpl(const platform::NPUPlace& npu) const {
    // TODO(zhiqiu)
  }

918 919 920 921
  void VisitorImpl(const platform::MLUPlace& mlu) const {
    PADDLE_THROW(platform::errors::Unimplemented("MLUPlace is not supported"));
  }

922
  void VisitorImpl(const platform::CPUPlace& cpu) const {
J
Jack Zhou 已提交
923 924 925 926 927 928 929 930
    int num = in_.numel();
    const bool* in_ptr = in_.data<bool>();
    bool* out_ptr = out_->data<bool>();
    for (int i = 0; i < num; ++i) {
      bool lhs = !in_ptr[i];
      bool rhs = !out_ptr[i];
      out_ptr[i] = lhs && rhs;
    }
931 932 933 934
  }

  void VisitorImpl(
      const platform::CUDAPinnedPlace& cpu /* equals to cpu*/) const {
J
Jack Zhou 已提交
935 936 937 938 939 940 941 942
    int num = in_.numel();
    const bool* in_ptr = in_.data<bool>();
    bool* out_ptr = out_->data<bool>();
    for (int i = 0; i < num; ++i) {
      bool lhs = !in_ptr[i];
      bool rhs = !out_ptr[i];
      out_ptr[i] = lhs && rhs;
    }
943
  }
944 945 946 947 948 949 950 951 952 953 954 955

  void VisitorImpl(
      const platform::NPUPinnedPlace& cpu /* equals to cpu*/) const {
    int num = in_.numel();
    const bool* in_ptr = in_.data<bool>();
    bool* out_ptr = out_->data<bool>();
    for (int i = 0; i < num; ++i) {
      bool lhs = !in_ptr[i];
      bool rhs = !out_ptr[i];
      out_ptr[i] = lhs && rhs;
    }
  }
956 957 958 959 960

  void VisitorImpl(const platform::CustomPlace& custom_dev) const {
    PADDLE_THROW(
        platform::errors::Unimplemented("CustomPlace is not supported"));
  }
961 962 963 964 965 966 967 968 969 970 971
};

void TensorIsfinite(const framework::Tensor& tensor, framework::Tensor* out) {
  framework::Tensor tmp;
  TensorContainsInf(tensor, &tmp);
  TensorContainsNAN(tensor, out);
  BothFalseVisitor visitor(tmp, out);
  auto place = tensor.place();
  platform::VisitPlace(place, visitor);
}

J
Jack Zhou 已提交
972 973 974 975 976 977 978 979 980
void TensorIsfiniteV2(const framework::Tensor& tensor, framework::Tensor* out) {
  framework::Tensor tmp;
  TensorContainsInfV2(tensor, &tmp);
  TensorContainsNANV2(tensor, out);
  BothFalseVisitor visitor(tmp, out);
  auto place = tensor.place();
  platform::VisitPlace(place, visitor);
}

Y
Yi Wang 已提交
981 982 983 984 985 986 987 988 989 990
void TensorToStream(std::ostream& os, const Tensor& tensor,
                    const platform::DeviceContext& dev_ctx) {
  {  // the 1st field, uint32_t version
    constexpr uint32_t version = 0;
    os.write(reinterpret_cast<const char*>(&version), sizeof(version));
  }
  {  // the 2nd field, tensor description
     // int32_t  size
     // void*    protobuf message
    proto::VarType::TensorDesc desc;
991
    desc.set_data_type(framework::TransToProtoVarType(tensor.dtype()));
992
    auto dims = pten::vectorize(tensor.dims());
Y
Yi Wang 已提交
993 994 995 996 997 998 999 1000 1001
    auto* pb_dims = desc.mutable_dims();
    pb_dims->Resize(static_cast<int>(dims.size()), 0);
    std::copy(dims.begin(), dims.end(), pb_dims->begin());
    int32_t size = desc.ByteSize();
    os.write(reinterpret_cast<const char*>(&size), sizeof(size));
    auto out = desc.SerializeAsString();
    os.write(out.data(), size);
  }
  {  // the 3rd field, tensor data
1002
    uint64_t size = tensor.numel() * framework::DataTypeSize(tensor.dtype());
Y
yuyang18 已提交
1003

1004
    auto* data_ptr = tensor.data();
1005
    PADDLE_ENFORCE_LT(size, (std::numeric_limits<std::streamsize>::max)(),
1006 1007
                      platform::errors::ResourceExhausted(
                          "tensor size %d overflow when writing tensor", size));
Y
Yi Wang 已提交
1008
    if (platform::is_gpu_place(tensor.place())) {
1009
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
Y
Yi Wang 已提交
1010 1011 1012 1013 1014 1015 1016 1017
      constexpr size_t kBufSize = 1024 * 1024 * 64;  // 64MB
      std::unique_ptr<char[]> buf(new char[kBufSize]);
      auto& gpu_dev_ctx =
          static_cast<const platform::CUDADeviceContext&>(dev_ctx);
      platform::CPUPlace cpu;
      uintptr_t data = reinterpret_cast<uintptr_t>(data_ptr);
      while (size != 0) {
        size_t size_to_write = std::min(kBufSize, static_cast<size_t>(size));
1018
        memory::Copy(cpu, buf.get(), tensor.place(),
Y
Yi Wang 已提交
1019 1020 1021 1022 1023 1024 1025 1026
                     reinterpret_cast<const void*>(data), size_to_write,
                     gpu_dev_ctx.stream());
        gpu_dev_ctx.Wait();
        os.write(buf.get(), size_to_write);
        data += size_to_write;
        size -= size_to_write;
      }
#else
1027 1028
      PADDLE_THROW(platform::errors::Unimplemented(
          "CUDAPlace is not supported when not compiled with CUDA"));
1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039
#endif
    } else if (platform::is_xpu_place(tensor.place())) {
#ifdef PADDLE_WITH_XPU
      constexpr size_t kBufSize = 1024 * 1024 * 64;  // 64MB
      std::unique_ptr<char[]> buf(new char[kBufSize]);
      auto& xpu_dev_ctx =
          static_cast<const platform::XPUDeviceContext&>(dev_ctx);
      platform::CPUPlace cpu;
      uintptr_t data = reinterpret_cast<uintptr_t>(data_ptr);
      while (size != 0) {
        size_t size_to_write = std::min(kBufSize, static_cast<size_t>(size));
1040
        memory::Copy(cpu, buf.get(), tensor.place(),
1041 1042 1043 1044 1045 1046 1047 1048 1049
                     reinterpret_cast<const void*>(data), size_to_write);
        xpu_dev_ctx.Wait();
        os.write(buf.get(), size_to_write);
        data += size_to_write;
        size -= size_to_write;
      }
#else
      PADDLE_THROW(platform::errors::Unimplemented(
          "XPUPlace is not supported when not compiled with XPU"));
1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060
#endif
    } else if (platform::is_mlu_place(tensor.place())) {
#ifdef PADDLE_WITH_MLU
      constexpr size_t kBufSize = 1024 * 1024 * 64;  // 64MB
      std::unique_ptr<char[]> buf(new char[kBufSize]);
      auto& mlu_dev_ctx =
          static_cast<const platform::MLUDeviceContext&>(dev_ctx);
      platform::CPUPlace cpu;
      uintptr_t data = reinterpret_cast<uintptr_t>(data_ptr);
      while (size != 0) {
        size_t size_to_write = std::min(kBufSize, static_cast<size_t>(size));
1061
        memory::Copy(cpu, buf.get(), tensor.place(),
1062 1063 1064 1065 1066 1067 1068 1069 1070 1071
                     reinterpret_cast<const void*>(data), size_to_write,
                     mlu_dev_ctx.stream());
        mlu_dev_ctx.Wait();
        os.write(buf.get(), size_to_write);
        data += size_to_write;
        size -= size_to_write;
      }
#else
      PADDLE_THROW(platform::errors::Unimplemented(
          "MLUPlace is not supported when not compiled with MLU"));
1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082
#endif
    } else if (platform::is_npu_place(tensor.place())) {
#ifdef PADDLE_WITH_ASCEND_CL
      constexpr size_t kBufSize = 1024 * 1024 * 64;  // 64MB
      std::unique_ptr<char[]> buf(new char[kBufSize]);
      auto& npu_dev_ctx =
          static_cast<const platform::NPUDeviceContext&>(dev_ctx);
      platform::CPUPlace cpu;
      uintptr_t data = reinterpret_cast<uintptr_t>(data_ptr);
      while (size != 0) {
        size_t size_to_write = std::min(kBufSize, static_cast<size_t>(size));
1083
        memory::Copy(cpu, buf.get(), tensor.place(),
1084 1085 1086 1087 1088 1089 1090 1091 1092 1093
                     reinterpret_cast<const void*>(data), size_to_write,
                     npu_dev_ctx.stream());
        npu_dev_ctx.Wait();
        os.write(buf.get(), size_to_write);
        data += size_to_write;
        size -= size_to_write;
      }
#else
      PADDLE_THROW(platform::errors::Unimplemented(
          "NPUPlace is not supported when not compiled with NPU"));
1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116
#endif
    } else if (platform::is_custom_place(tensor.place())) {
#ifdef PADDLE_WITH_CUSTOM_DEVICE
      constexpr size_t kBufSize = 1024 * 1024 * 64;  // 64MB
      std::unique_ptr<char[]> buf(new char[kBufSize]);
      auto& custom_device_context =
          static_cast<const platform::CustomDeviceContext&>(dev_ctx);
      platform::CPUPlace cpu;
      uintptr_t data = reinterpret_cast<uintptr_t>(data_ptr);
      while (size != 0) {
        size_t size_to_write = std::min(kBufSize, static_cast<size_t>(size));
        memory::Copy(cpu, buf.get(), tensor.place(),
                     reinterpret_cast<const void*>(data), size_to_write,
                     custom_device_context.stream());
        custom_device_context.Wait();
        os.write(buf.get(), size_to_write);
        data += size_to_write;
        size -= size_to_write;
      }
#else
      PADDLE_THROW(platform::errors::Unimplemented(
          "CustomPlace is not supported when not compiled with "
          "CustomDevice"));
Y
Yi Wang 已提交
1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130
#endif
    } else {
      os.write(static_cast<const char*>(data_ptr),
               static_cast<std::streamsize>(size));
    }
  }
}

struct DeserializedDataFunctor {
  DeserializedDataFunctor(void** buf, Tensor* tensor,
                          const platform::Place& place)
      : buf_(buf), tensor_(tensor), place_(place) {}

  template <typename T>
D
dzhwinter 已提交
1131
  void apply() {
Y
Yi Wang 已提交
1132 1133 1134 1135 1136 1137 1138 1139
    *buf_ = tensor_->mutable_data<T>(place_);
  }

  void** buf_;
  Tensor* tensor_;
  platform::Place place_;
};

1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163
void TensorFromStream(std::istream& is, Tensor* tensor,
                      const platform::DeviceContext& dev_ctx,
                      const size_t& seek, const std::vector<int64_t>& shape) {
  uint32_t version;
  is.read(reinterpret_cast<char*>(&version), sizeof(version));

  PADDLE_ENFORCE_EQ(
      version, 0U,
      platform::errors::InvalidArgument(
          "tensor version %u is not supported, Only version 0 is supported",
          version));

  proto::VarType::TensorDesc desc;
  {  // int32_t size
    // proto buffer
    int32_t size;
    is.read(reinterpret_cast<char*>(&size), sizeof(size));
    std::unique_ptr<char[]> buf(new char[size]);
    is.read(reinterpret_cast<char*>(buf.get()), size);
    PADDLE_ENFORCE_EQ(
        desc.ParseFromArray(buf.get(), size), true,
        platform::errors::InvalidArgument("Cannot parse tensor desc"));
  }
  {  // read tensor
1164
    tensor->Resize(pten::make_ddim(shape));
1165 1166 1167 1168
    size_t seekg = seek * framework::SizeOfType(desc.data_type());
    is.seekg(seekg, is.cur);

    void* buf;
1169
    platform::CPUDeviceContext ctx;
1170
    size_t size = tensor->numel() * framework::SizeOfType(desc.data_type());
1171
    if (platform::is_gpu_place(dev_ctx.GetPlace()) ||
1172
        platform::is_xpu_place(dev_ctx.GetPlace()) ||
1173
        platform::is_mlu_place(dev_ctx.GetPlace()) ||
1174 1175
        platform::is_npu_place(dev_ctx.GetPlace()) ||
        platform::is_custom_place(dev_ctx.GetPlace())) {
1176
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) || \
1177
    defined(PADDLE_WITH_XPU) || defined(PADDLE_WITH_MLU) ||  \
1178
    defined(PADDLE_WITH_ASCEND_CL) || defined(PADDLE_WITH_CUSTOM_DEVICE)
1179
      Tensor cpu_tensor;
1180
      cpu_tensor.Resize(pten::make_ddim(shape));
1181 1182 1183 1184 1185 1186
      framework::VisitDataType(
          desc.data_type(),
          DeserializedDataFunctor(&buf, &cpu_tensor, ctx.GetPlace()));
      is.read(static_cast<char*>(buf), size);
      auto dst_place = dev_ctx.GetPlace();
      framework::TensorCopy(cpu_tensor, dst_place, dev_ctx, tensor);
1187 1188
      if (platform::is_npu_place(dev_ctx.GetPlace()) ||
          platform::is_custom_place(dev_ctx.GetPlace())) {
1189 1190
        dev_ctx.Wait();
      }
1191
#else
1192 1193 1194
      if (platform::is_gpu_place(dev_ctx.GetPlace())) {
        PADDLE_THROW(platform::errors::Unimplemented(
            "CUDAPlace is not supported when not compiled with CUDA"));
1195
      } else if (platform::is_xpu_place(dev_ctx.GetPlace())) {
1196 1197
        PADDLE_THROW(platform::errors::Unimplemented(
            "XPUPlace is not supported when not compiled with XPU"));
1198 1199 1200
      } else if (platform::is_mlu_place(dev_ctx.GetPlace())) {
        PADDLE_THROW(platform::errors::Unimplemented(
            "MLUPlace is not supported when not compiled with MLU"));
1201 1202 1203
      } else {
        PADDLE_THROW(platform::errors::Unimplemented(
            "NPUPlace is not supported when not compiled with NPU"));
1204
      }
1205 1206 1207 1208 1209 1210 1211 1212 1213 1214
#endif
    } else {
      framework::VisitDataType(
          desc.data_type(),
          DeserializedDataFunctor(&buf, tensor, ctx.GetPlace()));
      is.read(static_cast<char*>(buf), size);
    }
  }
}

Y
Yi Wang 已提交
1215 1216 1217 1218
void TensorFromStream(std::istream& is, Tensor* tensor,
                      const platform::DeviceContext& dev_ctx) {
  uint32_t version;
  is.read(reinterpret_cast<char*>(&version), sizeof(version));
1219 1220 1221 1222 1223
  PADDLE_ENFORCE_EQ(
      version, 0U,
      platform::errors::InvalidArgument(
          "tensor version %u is not supported, Only version 0 is supported",
          version));
Y
Yi Wang 已提交
1224 1225 1226 1227 1228 1229 1230
  proto::VarType::TensorDesc desc;
  {  // int32_t size
     // proto buffer
    int32_t size;
    is.read(reinterpret_cast<char*>(&size), sizeof(size));
    std::unique_ptr<char[]> buf(new char[size]);
    is.read(reinterpret_cast<char*>(buf.get()), size);
1231 1232 1233
    PADDLE_ENFORCE_EQ(
        desc.ParseFromArray(buf.get(), size), true,
        platform::errors::InvalidArgument("Cannot parse tensor desc"));
Y
Yi Wang 已提交
1234 1235 1236 1237 1238
  }
  {  // read tensor
    std::vector<int64_t> dims;
    dims.reserve(static_cast<size_t>(desc.dims().size()));
    std::copy(desc.dims().begin(), desc.dims().end(), std::back_inserter(dims));
1239
    tensor->Resize(pten::make_ddim(dims));
Y
Yi Wang 已提交
1240
    void* buf;
1241
    platform::CPUDeviceContext ctx;
1242
    size_t size = tensor->numel() * framework::SizeOfType(desc.data_type());
1243
    if (platform::is_gpu_place(dev_ctx.GetPlace()) ||
1244
        platform::is_xpu_place(dev_ctx.GetPlace()) ||
1245
        platform::is_mlu_place(dev_ctx.GetPlace()) ||
1246 1247
        platform::is_npu_place(dev_ctx.GetPlace()) ||
        platform::is_custom_place(dev_ctx.GetPlace())) {
1248
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) || \
1249
    defined(PADDLE_WITH_XPU) || defined(PADDLE_WITH_MLU) ||  \
1250
    defined(PADDLE_WITH_ASCEND_CL) || defined(PADDLE_WITH_CUSTOM_DEVICE)
Y
Yi Wang 已提交
1251
      Tensor cpu_tensor;
1252
      cpu_tensor.Resize(pten::make_ddim(dims));
Y
Yi Wang 已提交
1253 1254 1255
      framework::VisitDataType(
          desc.data_type(),
          DeserializedDataFunctor(&buf, &cpu_tensor, ctx.GetPlace()));
Y
yuyang18 已提交
1256
      is.read(static_cast<char*>(buf), size);
Y
Yi Wang 已提交
1257 1258
      auto dst_place = dev_ctx.GetPlace();
      framework::TensorCopy(cpu_tensor, dst_place, dev_ctx, tensor);
1259 1260
      if (platform::is_npu_place(dev_ctx.GetPlace()) ||
          platform::is_custom_place(dev_ctx.GetPlace())) {
1261 1262
        dev_ctx.Wait();
      }
Y
Yi Wang 已提交
1263
#else
1264 1265 1266
      if (platform::is_gpu_place(dev_ctx.GetPlace())) {
        PADDLE_THROW(platform::errors::Unimplemented(
            "CUDAPlace is not supported when not compiled with CUDA"));
1267
      } else if (platform::is_xpu_place(dev_ctx.GetPlace())) {
1268 1269
        PADDLE_THROW(platform::errors::Unimplemented(
            "XPUPlace is not supported when not compiled with XPU"));
1270 1271 1272
      } else if (platform::is_mlu_place(dev_ctx.GetPlace())) {
        PADDLE_THROW(platform::errors::Unimplemented(
            "MLUPlace is not supported when not compiled with MLU"));
1273
      } else if (platform::is_npu_place(dev_ctx.GetPlace())) {
1274 1275
        PADDLE_THROW(platform::errors::Unimplemented(
            "NPUPlace is not supported when not compiled with NPU"));
1276 1277 1278
      } else {
        PADDLE_THROW(platform::errors::Unimplemented(
            "CutomPlace is not supported when not compiled with CustomDevice"));
1279
      }
Y
Yi Wang 已提交
1280 1281 1282 1283 1284
#endif
    } else {
      framework::VisitDataType(
          desc.data_type(),
          DeserializedDataFunctor(&buf, tensor, ctx.GetPlace()));
Y
yuyang18 已提交
1285
      is.read(static_cast<char*>(buf), size);
Y
Yi Wang 已提交
1286 1287 1288 1289
    }
  }
}

6
633WHU 已提交
1290 1291 1292 1293
// get tensor data point by DLDataType
void* GetDstPtrByDLDataType(DLDataType type, framework::Tensor* dst,
                            const platform::Place& dst_place) {
  // vector types not currently supported
1294 1295 1296
  PADDLE_ENFORCE_LE(type.lanes, 1,
                    platform::errors::Unimplemented(
                        "Vector type is not supported currently."));
6
633WHU 已提交
1297 1298 1299 1300 1301 1302 1303

  switch (type.bits) {
    case 8:
      if (type.code == kDLInt)
        return static_cast<void*>(dst->mutable_data<int8_t>(dst_place));
      if (type.code == kDLUInt)
        return static_cast<void*>(dst->mutable_data<uint8_t>(dst_place));
1304 1305 1306
      PADDLE_THROW(platform::errors::Unimplemented(
          "DLDataType code <%d> is illegal when DLDataType.bits is <%d>.",
          type.code, type.bits));
6
633WHU 已提交
1307 1308 1309 1310 1311 1312
    case 16:
      if (type.code == kDLInt)
        return static_cast<void*>(dst->mutable_data<int16_t>(dst_place));
      if (type.code == kDLFloat)
        return static_cast<void*>(
            dst->mutable_data<paddle::platform::float16>(dst_place));
S
Siming Dai 已提交
1313 1314 1315
      if (type.code == kDLBfloat)
        return static_cast<void*>(
            dst->mutable_data<paddle::platform::bfloat16>(dst_place));
1316 1317 1318
      PADDLE_THROW(platform::errors::Unimplemented(
          "DLDataType code <%d> is illegal when DLDataType.bits is <%d>.",
          type.code, type.bits));
6
633WHU 已提交
1319 1320 1321 1322 1323
    case 32:
      if (type.code == kDLInt)
        return static_cast<void*>(dst->mutable_data<int32_t>(dst_place));
      if (type.code == kDLFloat)
        return static_cast<void*>(dst->mutable_data<float>(dst_place));
1324 1325 1326
      PADDLE_THROW(platform::errors::Unimplemented(
          "DLDataType code <%d> is illegal when DLDataType.bits is <%d>.",
          type.code, type.bits));
6
633WHU 已提交
1327 1328 1329 1330 1331
    case 64:
      if (type.code == kDLInt)
        return static_cast<void*>(dst->mutable_data<int64_t>(dst_place));
      if (type.code == kDLFloat)
        return static_cast<void*>(dst->mutable_data<double>(dst_place));
S
Siming Dai 已提交
1332 1333 1334 1335 1336 1337 1338 1339 1340 1341
      if (type.code == kDLComplex)
        return static_cast<void*>(
            dst->mutable_data<paddle::platform::complex<float>>(dst_place));
      PADDLE_THROW(platform::errors::Unimplemented(
          "DLDataType code <%d> is illegal when DLDataType.bits is <%d>.",
          type.code, type.bits));
    case 128:
      if (type.code == kDLComplex)
        return static_cast<void*>(
            dst->mutable_data<paddle::platform::complex<double>>(dst_place));
1342 1343 1344
      PADDLE_THROW(platform::errors::Unimplemented(
          "DLDataType code <%d> is illegal when DLDataType.bits is <%d>.",
          type.code, type.bits));
6
633WHU 已提交
1345
    default:
1346 1347
      PADDLE_THROW(platform::errors::Unimplemented(
          "Unsupported DLDataType.bits %d.", type.bits));
6
633WHU 已提交
1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358
  }
}

void TensorFromDLPack(const ::DLTensor& dl_tensor, framework::Tensor* dst) {
  platform::CPUPlace dst_place = platform::CPUPlace();
  platform::CPUPlace src_place = platform::CPUPlace();

  std::vector<int64_t> vec;
  std::copy(dl_tensor.shape, dl_tensor.shape + dl_tensor.ndim,
            std::back_inserter(vec));

1359
  framework::DDim vddim = pten::make_ddim(vec);
6
633WHU 已提交
1360 1361 1362 1363 1364 1365

  dst->Resize(vddim);
  ::DLDataType type = dl_tensor.dtype;
  void* dst_ptr = GetDstPtrByDLDataType(type, dst, dst_place);

  auto src_ptr = static_cast<const void*>(dl_tensor.data);
1366
  auto size = pten::product(vddim) * type.bits / 8;
6
633WHU 已提交
1367

S
Siming Dai 已提交
1368
  if (dl_tensor.device.device_type == kDLCPU) {
6
633WHU 已提交
1369 1370
    memory::Copy(dst_place, dst_ptr, src_place, src_ptr, size);
  }
1371
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
S
Siming Dai 已提交
1372
  if (dl_tensor.device.device_type == kDLGPU) {
6
633WHU 已提交
1373
    platform::CUDAPlace dst_place =
S
Siming Dai 已提交
1374
        platform::CUDAPlace(dl_tensor.device.device_id);
6
633WHU 已提交
1375
    platform::CUDAPlace src_place =
S
Siming Dai 已提交
1376
        platform::CUDAPlace(dl_tensor.device.device_id);
6
633WHU 已提交
1377 1378 1379 1380 1381 1382 1383
    dst_ptr = GetDstPtrByDLDataType(type, dst, dst_place);
    auto* ctx = platform::DeviceContextPool::Instance().GetByPlace(dst_place);
    memory::Copy(
        dst_place, dst_ptr, src_place, src_ptr, size,
        reinterpret_cast<const platform::CUDADeviceContext&>(*ctx).stream());
  }
#endif
1384 1385 1386
#ifdef PADDLE_WITH_XPU
  PADDLE_THROW(platform::errors::Unimplemented("XPUPlace is not supported"));
#endif
6
633WHU 已提交
1387 1388
}

1389 1390 1391 1392 1393 1394
template <typename T>
std::string format_tensor(const framework::Tensor& tensor) {
  // TODO(zhiqiu): use the print option to format tensor.
  return "NOT IMPLEMENTED";
}

1395 1396 1397 1398 1399
template <typename T>
std::ostream& print_tensor(std::ostream& os, const framework::Tensor& tensor) {
  auto inspect = tensor.data<T>();
  auto element_num = tensor.numel();

1400
  os << "  - data: [";
1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414
  // Note: int8_t && uint8_t is typedf of char, ostream unable to print properly
  if (typeid(int8_t) == typeid(T) || typeid(uint8_t) == typeid(T)) {
    if (element_num > 0) {
      os << signed(inspect[0]);
      for (int j = 1; j < element_num; ++j) {
        os << " " << signed(inspect[j]);
      }
    }
  } else {
    if (element_num > 0) {
      os << inspect[0];
      for (int j = 1; j < element_num; ++j) {
        os << " " << inspect[j];
      }
1415 1416 1417 1418 1419 1420
    }
  }
  os << "]";
  return os;
}

1421
template <>
1422
std::ostream& print_tensor<paddle::platform::complex<float>>(
1423
    std::ostream& os, const framework::Tensor& tensor) {
1424
  auto inspect = tensor.data<paddle::platform::complex<float>>();
1425 1426 1427 1428
  auto element_num = tensor.numel();

  os << "  - data: [";
  if (element_num > 0) {
1429
    os << signed(inspect[0].real) << "+" << signed(inspect[0].imag) << "j";
1430
    for (int j = 1; j < element_num; ++j) {
1431 1432
      os << " " << signed(inspect[j].real) << "+" << signed(inspect[j].imag)
         << "j";
1433 1434 1435 1436 1437 1438 1439
    }
  }
  os << "]";
  return os;
}

template <>
1440
std::ostream& print_tensor<paddle::platform::complex<double>>(
1441
    std::ostream& os, const framework::Tensor& tensor) {
1442
  auto inspect = tensor.data<paddle::platform::complex<double>>();
1443 1444 1445 1446
  auto element_num = tensor.numel();

  os << "  - data: [";
  if (element_num > 0) {
1447
    os << signed(inspect[0].real) << "+" << signed(inspect[0].imag) << "j";
1448
    for (int j = 1; j < element_num; ++j) {
1449 1450
      os << " " << signed(inspect[j].real) << "+" << signed(inspect[j].imag)
         << "j";
1451 1452 1453 1454 1455 1456
    }
  }
  os << "]";
  return os;
}

1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476
std::ostream& operator<<(std::ostream& os, const LoD& lod) {
  os << "{";
  for (auto& v : lod) {
    os << "{";
    bool is_first = true;
    for (auto& i : v) {
      if (is_first) {
        os << i;
        is_first = false;
      } else {
        os << ", " << i;
      }
    }
    os << "}";
  }
  os << "}";

  return os;
}

1477 1478 1479 1480 1481 1482
}  // namespace framework
}  // namespace paddle

namespace pten {

std::ostream& operator<<(std::ostream& os, const pten::DenseTensor& t) {
1483 1484 1485 1486
  if (t.lod().size() > 0) {
    os << "  - lod: " << t.lod() << "\n";
  }

1487 1488
  os << "  - place: " << t.place() << "\n";
  os << "  - shape: [" << t.dims() << "]\n";
1489 1490
  os << "  - layout: " << paddle::framework::DataLayoutToString(t.layout())
     << "\n";
1491

1492 1493 1494 1495 1496
#ifdef PADDLE_WITH_MKLDNN
  os << "  - format: "
     << dnnl_fmt_tag2str(static_cast<dnnl_format_tag_t>(t.format())) << "\n";
#endif

1497
  DenseTensor tensor;
1498
  tensor.Resize(t.dims());
1499
  if (paddle::platform::is_cpu_place(t.place())) {
1500 1501
    tensor.ShareDataWith(t);
  } else {
1502 1503 1504 1505
    paddle::platform::CPUPlace place;
    paddle::framework::TensorCopy(t, place, &tensor);
    paddle::platform::DeviceContextPool& pool =
        paddle::platform::DeviceContextPool::Instance();
1506 1507 1508 1509
    auto& dev_ctx = *pool.Get(t.place());
    dev_ctx.Wait();
  }

1510 1511 1512 1513 1514 1515 1516 1517
#define PrintTensorCallback(cpp_type, proto_type)                 \
  do {                                                            \
    if (paddle::framework::TransToProtoVarType(tensor.dtype()) == \
        proto_type) {                                             \
      os << "  - dtype: " << proto_type << "\n";                  \
      paddle::framework::print_tensor<cpp_type>(os, tensor);      \
      return os;                                                  \
    }                                                             \
1518 1519 1520 1521 1522 1523
  } while (0)

  _ForEachDataType_(PrintTensorCallback);
  VLOG(1) << "PrintVar: unrecognized data type:" << t.type();
  return os;
}
1524
}  // namespace pten
新手
引导
客服 返回
顶部