diff --git a/paddle/fluid/framework/variable.h b/paddle/fluid/framework/variable.h index 182ddafe3ea3da08ddbe6937155d0ff27dc61d4f..52bf3a12a043f63f6d370f528ac55368aed63527 100644 --- a/paddle/fluid/framework/variable.h +++ b/paddle/fluid/framework/variable.h @@ -72,7 +72,7 @@ class Variable { private: // This method hides type T, so it doesn't appear as a template parameter of // Variable. - pten::TensorInplaceVersion* InplaceVersionCounter(); + pten::DenseTensor::InplaceVersion* InplaceVersionCounter(); public: void SetInplaceVersionToZero(); @@ -114,8 +114,8 @@ class Variable { std::shared_ptr holder_; }; -inline pten::TensorInplaceVersion* Variable::InplaceVersionCounter() { - pten::TensorInplaceVersion* version_counter_ptr(nullptr); +inline pten::DenseTensor::InplaceVersion* Variable::InplaceVersionCounter() { + pten::DenseTensor::InplaceVersion* version_counter_ptr(nullptr); if (IsType()) { version_counter_ptr = &GetMutable()->InplaceVersionCounter(); diff --git a/paddle/pten/core/CMakeLists.txt b/paddle/pten/core/CMakeLists.txt index e89d2cd3b3c38789d63ff6a744e94e3caab68453..757eed25974e34fc55c7b0ef39df0a1e11a13f3c 100644 --- a/paddle/pten/core/CMakeLists.txt +++ b/paddle/pten/core/CMakeLists.txt @@ -19,7 +19,7 @@ cc_library(kernel_context SRCS kernel_context.cc DEPS pten_enforce pten_context) cc_library(tensor_base SRCS tensor_base.cc allocator.cc storage.cc DEPS pten_enforce) cc_library(tensor_meta SRCS tensor_meta.cc DEPS pten_enforce mixed_vector) cc_library(lod_utils SRCS lod_utils.cc DEPS pten_enforce mixed_vector) -cc_library(dense_tensor SRCS dense_tensor.cc DEPS convert_utils tensor_meta tensor_base) +cc_library(dense_tensor SRCS dense_tensor.cc dense_tensor_impl.cc DEPS convert_utils tensor_meta tensor_base) cc_library(pten_device_context SRCS device_context.cc DEPS tensor_base ) cc_library(meta_tensor SRCS meta_tensor.cc DEPS tensor_base tensor_meta dense_tensor) diff --git a/paddle/pten/core/dense_tensor.cc b/paddle/pten/core/dense_tensor.cc index 45eacf342fab4a2cfdfbf3bd64434c2d3f44484f..15f9f0bda3c25e2b8a4125d1025d8b0a673f2dc5 100644 --- a/paddle/pten/core/dense_tensor.cc +++ b/paddle/pten/core/dense_tensor.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -22,14 +22,6 @@ limitations under the License. */ #include "paddle/pten/api/lib/utils/storage.h" #include "paddle/pten/core/convert_utils.h" -namespace paddle { -namespace framework { -extern void TensorCopy(const pten::DenseTensor& src, - const paddle::platform::Place& dst_place, - pten::DenseTensor* dst); -} -} - namespace pten { DenseTensor::DenseTensor(Allocator* a, const DenseTensorMeta& meta) @@ -180,375 +172,4 @@ DATA_MEMBER_FUNC_INSTANTIATION(::paddle::experimental::complex128); #undef DATA_MEMBER_FUNC_INSTANTIATION -/* --------------------------- */ -/* From framework::Tensor */ -/* --------------------------- */ -DenseTensor::DenseTensor() { - inplace_version_counter_ = std::make_shared(0); - meta_.dtype = paddle::experimental::DataType::FLOAT32; - meta_.offset = 0; -} - -DenseTensor::DenseTensor(paddle::framework::proto::VarType::Type dtype) { - inplace_version_counter_ = std::make_shared(0); - meta_.dtype = TransToPtenDataType(dtype); - meta_.offset = 0; -} - -size_t DenseTensor::memory_size() const { - return holder_ == nullptr ? 0UL : holder_->size() - meta_.offset; -} - -void DenseTensor::check_memory_size() const { - PADDLE_ENFORCE_NOT_NULL(holder_, - paddle::platform::errors::PreconditionNotMet( - "Tensor holds no memory. " - "Call Tensor::mutable_data firstly.")); - PADDLE_ENFORCE_LE( - numel() * SizeOf(dtype()), - memory_size(), - paddle::platform::errors::PreconditionNotMet( - "Tensor's dimension is out of bound." - "Tensor's dimension must be equal or less than the size of its " - "memory." - "But received Tensor's dimension is d%, memory's size is %d.", - numel() * SizeOf(dtype()), - memory_size())); -} - -const paddle::platform::Place& DenseTensor::place() const { - PADDLE_ENFORCE_NOT_NULL( - holder_, - paddle::platform::errors::PreconditionNotMet( - "Tensor not initialized yet when DenseTensor::place() is called.")); - return holder_->place(); -} - -paddle::framework::proto::VarType::Type DenseTensor::type() const { - return TransToProtoVarType(meta_.dtype); -} - -paddle::framework::proto::VarType::Type DenseTensor::saved_type() const { - return TransToProtoVarType(meta_.dtype); -} - -void DenseTensor::set_layout(const paddle::framework::DataLayout layout) { - meta_.layout = layout; -} - -void DenseTensor::ResetHolder(const std::shared_ptr& holder) { - PADDLE_ENFORCE_EQ( - meta_.offset, - 0, - paddle::platform::errors::Fatal( - "Only the offset is supported to zero when the holder is reset.")); - - if (holder_) { - // TODO(zyfncg): The change of static_cast<> in check will recover back - // when SetAllocationForOutputTenosr is deleted. - // Now the numel() may return -1, and will cast to a very large number when - // compare with a data with unsigned long type, this will make checking - // failed, so it's a temporary solution to deal with this problem. - PADDLE_ENFORCE_LE( - numel() * static_cast(SizeOf(dtype())), - static_cast(holder->size()), - paddle::platform::errors::InvalidArgument( - "The size of Holder is not enough to store the Tensor.")); - } - holder_ = holder; -} - -void DenseTensor::ResetHolderWithType( - const std::shared_ptr& holder, - paddle::framework::proto::VarType::Type type) { - set_type(type); - ResetHolder(holder); -} - -void DenseTensor::set_type(paddle::framework::proto::VarType::Type type) { - meta_.dtype = TransToPtenDataType(type); -} - -void* DenseTensor::mutable_data(const paddle::platform::Place& place, - paddle::framework::proto::VarType::Type type, - size_t requested_size) { - set_type(type); - PADDLE_ENFORCE_GE( - numel(), - 0, - paddle::platform::errors::PreconditionNotMet( - "The Tensor's element number must be equal or greater than zero. " - "The Tensor's shape is [", - dims(), - "] now")); - size_t size = numel() * SizeOf(dtype()); - if (requested_size && (requested_size > size)) { - size = requested_size; - } - - /* some versions of boost::variant don't have operator!= */ - if (holder_ == nullptr || !(holder_->place() == place) || - holder_->size() < size + meta_.offset) { - holder_.reset(); - holder_ = paddle::memory::AllocShared(place, size); - meta_.offset = 0; - } - return reinterpret_cast(reinterpret_cast(holder_->ptr()) + - meta_.offset); -} - -void* DenseTensor::mutable_data(const paddle::platform::Place& place, - size_t requested_size) { - return mutable_data(place, type(), requested_size); -} - -void* DenseTensor::mutable_data(const paddle::platform::Place& place, - paddle::framework::proto::VarType::Type type, - const paddle::platform::Stream& stream) { - set_type(type); - PADDLE_ENFORCE_GE( - numel(), - 0, - paddle::platform::errors::PreconditionNotMet( - "The Tensor's element number must be equal or greater than zero. " - "The Tensor's shape is [", - dims(), - "] now")); - size_t size = numel() * SizeOf(dtype()); - - /* some versions of boost::variant don't have operator!= */ - if (holder_ == nullptr || !(holder_->place() == place) || - holder_->size() < size + meta_.offset || - !(paddle::platform::is_gpu_place(place) && - paddle::memory::InSameStream(holder_, stream))) { - holder_.reset(); - holder_ = paddle::memory::AllocShared(place, size, stream); - meta_.offset = 0; - } - return reinterpret_cast(reinterpret_cast(holder_->ptr()) + - meta_.offset); -} - -/* @jim19930609: The following "mutable_data" only supports specific dtypes - defined in OpProto. This part need another clean up once the data type across - Fluid - and Pten get unified. - */ -template -inline T* DenseTensor::mutable_data(const DDim& dims, - const paddle::platform::Place& place, - size_t requested_size) { - static_assert(std::is_pod::value, "T must be POD"); - meta_.dims = dims; - return mutable_data(place, requested_size); -} - -template -inline T* DenseTensor::mutable_data(const paddle::platform::Place& place, - size_t requested_size) { - static_assert(std::is_pod::value, "T must be POD"); - return reinterpret_cast(mutable_data( - place, paddle::framework::DataTypeTrait::DataType(), requested_size)); -} - -void DenseTensor::ShareBufferWith(const DenseTensor& tensor) { - holder_ = tensor.holder_; - meta_.offset = tensor.meta().offset; - meta_.dtype = tensor.dtype(); -} - -#define LEGACY_DATA_MEMBER_FUNC_INSTANTIATION(dtype) \ - template dtype* DenseTensor::mutable_data( \ - const DDim& dims, \ - const paddle::platform::Place& place, \ - size_t requested_size); \ - template dtype* DenseTensor::mutable_data( \ - const paddle::platform::Place& place, size_t requested_size); - -LEGACY_DATA_MEMBER_FUNC_INSTANTIATION(bool) -LEGACY_DATA_MEMBER_FUNC_INSTANTIATION(int8_t) -LEGACY_DATA_MEMBER_FUNC_INSTANTIATION(uint8_t) -LEGACY_DATA_MEMBER_FUNC_INSTANTIATION(int16_t) -LEGACY_DATA_MEMBER_FUNC_INSTANTIATION(int32_t) -LEGACY_DATA_MEMBER_FUNC_INSTANTIATION(int64_t) -LEGACY_DATA_MEMBER_FUNC_INSTANTIATION(float) -LEGACY_DATA_MEMBER_FUNC_INSTANTIATION(double) -LEGACY_DATA_MEMBER_FUNC_INSTANTIATION(::paddle::platform::bfloat16) -LEGACY_DATA_MEMBER_FUNC_INSTANTIATION(::paddle::platform::float16) -LEGACY_DATA_MEMBER_FUNC_INSTANTIATION(::paddle::experimental::complex64) -LEGACY_DATA_MEMBER_FUNC_INSTANTIATION(::paddle::experimental::complex128) - -#undef LEGACY_DATA_MEMBER_FUNC_INSTANTIATION - -/* ------------------------------ */ -/* From framework::LoDTensor */ -/* ------------------------------ */ - -DenseTensor::DenseTensor(intrusive_ptr storage, - const DenseTensorMeta& meta) - : meta_(meta), holder_(storage->move_data_shared()) {} - -DenseTensor::DenseTensor(intrusive_ptr storage, DenseTensorMeta&& meta) - : meta_(std::move(meta)), holder_(storage->move_data_shared()) {} - -DenseTensor::DenseTensor(const LoD& lod) : DenseTensor() { meta_.lod = lod; } - -void DenseTensor::set_lod(const LoD& lod) { meta_.lod = lod; } - -LoD* DenseTensor::mutable_lod() { return &meta_.lod; } - -std::pair DenseTensor::lod_element(size_t level, - size_t elem) const { - PADDLE_ENFORCE_LT( - level, - NumLevels(), - paddle::platform::errors::InvalidArgument( - "The input level of LoD is invalid, it should be less than LoD " - "size. The input level is %zu, the LoD size is %zu.", - level, - NumLevels())); - - PADDLE_ENFORCE_LT(elem, - NumElements(level), - paddle::platform::errors::InvalidArgument( - "The input element of LoD is invalid, it should be " - "less than the number of elements in its level." - "The input element is %zu, the number of elements in " - "its level is %zu.", - elem, - NumElements(level))); - - return std::make_pair((meta_.lod)[level][elem], (meta_.lod)[level][elem + 1]); -} - -size_t DenseTensor::NumLevels() const { return meta_.lod.size(); } - -size_t DenseTensor::NumElements(size_t level) const { - PADDLE_ENFORCE_LT( - level, - NumLevels(), - paddle::platform::errors::InvalidArgument( - "The input level of LoD is invalid, it should be less than LoD " - "size. The input level is %zu, the LoD size is %zu.", - level, - NumLevels())); - - // the last offset is the end of last element - return (meta_.lod)[level].size() - 1; -} - -DenseTensor& DenseTensor::Resize(const DDim& dims) { - meta_.dims = dims; - return *this; -} - -DenseTensor DenseTensor::Slice(int64_t begin_idx, int64_t end_idx) const { - check_memory_size(); - PADDLE_ENFORCE_GE(begin_idx, - 0, - paddle::platform::errors::OutOfRange( - "The start row index must be greater than 0." - "But received the start index is d%.", - begin_idx)); - PADDLE_ENFORCE_LE(end_idx, - meta_.dims[0], - paddle::platform::errors::OutOfRange( - "The end row index is out of bound.")); - PADDLE_ENFORCE_LT( - begin_idx, - end_idx, - paddle::platform::errors::InvalidArgument( - "The start row index must be less than the end row index." - "But received the start index = %d, the end index = %d.", - begin_idx, - end_idx)); - - if (meta_.dims[0] == 1) { - return *this; - } else { - size_t base = numel() / meta_.dims[0]; - DenseTensor dst; - dst.holder_ = holder_; - dst.set_layout(meta_.layout); - dst.meta_.dtype = meta_.dtype; - DDim dst_dims = meta_.dims; - dst_dims[0] = end_idx - begin_idx; - dst.Resize(dst_dims); - dst.meta_.offset = meta_.offset + begin_idx * base * SizeOf(dtype()); - return dst; - } -} - -std::vector DenseTensor::Split(int64_t split_size, - int64_t axis) const { - check_memory_size(); - - PADDLE_ENFORCE_GE(meta_.dims.size(), - 0, - paddle::platform::errors::OutOfRange( - "split expects at least a 1-dimensional tensor")); - - PADDLE_ENFORCE_GE( - split_size, - 0, - paddle::platform::errors::OutOfRange( - "split expects split_size be non-negative, but got split_size is %d", - split_size)); - - int64_t numel_size = meta_.dims[axis]; - - int64_t num_splits = 1; - if (split_size != 0) { - num_splits = - std::max((numel_size + split_size - 1) / split_size, 1); - } - - std::vector splits(num_splits); - int64_t last_split_size = split_size - (split_size * num_splits - numel_size); - - for (int64_t i = 0; i < num_splits; ++i) { - int64_t length = i < num_splits - 1 ? split_size : last_split_size; - splits[i] = Slice(i * split_size, i * split_size + length); - } - return splits; -} - -std::vector DenseTensor::Chunk(int64_t chunks, - int64_t axis) const { - check_memory_size(); - PADDLE_ENFORCE_GE(meta_.dims.size(), - 0, - paddle::platform::errors::OutOfRange( - "split expects at least a 1-dimensional tensor")); - PADDLE_ENFORCE_GE( - chunks, - 0, - paddle::platform::errors::OutOfRange( - "chunks expects to be greater than 0, but got chunks is %d", chunks)); - - int64_t numel_size = meta_.dims[axis]; - int64_t split_size = (numel_size + chunks - 1) / chunks; - return Split(split_size, axis); -} - -DenseTensor& DenseTensor::ShareDataWith(const DenseTensor& src) { - src.check_memory_size(); - // Preserve LoD - auto lod = meta_.lod; - *this = src; - meta_.lod = lod; - return *this; -} - -DenseTensor& DenseTensor::ShareInplaceVersionCounterWith( - const DenseTensor& src) { - PADDLE_ENFORCE_NOT_NULL( - inplace_version_counter_, - paddle::platform::errors::PreconditionNotMet( - "Tensor does not hold inplace_version_counter_.")); - - inplace_version_counter_ = src.inplace_version_counter_; - return *this; -} - } // namespace pten diff --git a/paddle/pten/core/dense_tensor.h b/paddle/pten/core/dense_tensor.h index 47c88aefa53a79b0079a2ed1d2a3b8cd7cdb42bd..2823441f97da2a784d6fb175429a0496e50d6aaa 100644 --- a/paddle/pten/core/dense_tensor.h +++ b/paddle/pten/core/dense_tensor.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -33,25 +33,6 @@ namespace pten { class CompatibleDenseTensorUtils; -/* --------------------------- */ -/* From framework::Tensor */ -/* --------------------------- */ -/* Temporarily put TensorInplaceVersion inside DenseTensor. - Will move to AutogradMeta as soon as we switch to Eager Dygraph. - */ -class TensorInplaceVersion { - public: - explicit TensorInplaceVersion(uint32_t inplace_version = 0) - : inplace_version_(inplace_version) {} - bool IsUnique() const { return inplace_version_ == 0; } - void Bump() { ++inplace_version_; } - uint32_t CurrentVersion() const { return inplace_version_; } - void SetInplaceVersionToZero() { inplace_version_ = 0; } - - private: - uint32_t inplace_version_; -}; - /// \brief The Dense tensor store values in a contiguous sequential block /// of memory where all values are represented. Tensors or multi-dimensional /// arrays are used in math operators. @@ -90,6 +71,8 @@ class DenseTensor : public TensorBase, DenseTensor& operator=(DenseTensor&& other); + DenseTensor(); + /// \brief Destroy the tensor object and release exclusive resources. virtual ~DenseTensor() = default; @@ -179,181 +162,6 @@ class DenseTensor : public TensorBase, DenseTensorMeta meta_; std::shared_ptr holder_; - /* --------------------------- */ - /* From framework::Tensor */ - /* --------------------------- */ - /* The following members & interfaces were copied from framework::Tensor, - so as to facilitate the unification of different Tensors - - Will be adjusted/removed/moved in the near future - */ - public: - /* @jim19930609: The way default constructor handles allocator might change, - according to - the final design of Allocation - Allocator. - */ - DenseTensor(); - - /* @jim19930609: Remove dependency on protobuf after Tensor Unification. - */ - explicit DenseTensor(paddle::framework::proto::VarType::Type dtype); - - /// \brief Use existing storage space to create dense tensor. This interface - /// can be used to deliberately create an uninitialized dense tensor. - /// \param storage The existing storage. - /// \param meta The meta data of dense tensor. - DenseTensor(intrusive_ptr storage, const DenseTensorMeta& meta); - - /// \brief Use existing storage space to create dense tensor. This interface - /// can be used to deliberately create an uninitialized dense tensor. - /// \param storage The existing storage. - /// \param meta The meta data of dense tensor. - DenseTensor(intrusive_ptr storage, DenseTensorMeta&& meta); - - inline bool IsInitialized() const { return holder_ != nullptr; } - - template - T* data(); - - void* data(); - - template - T* mutable_data(const paddle::platform::Place& place, - size_t requested_size = 0); - - template - T* mutable_data(const DDim& dims, - const paddle::platform::Place& place, - size_t requested_size = 0); - - void* mutable_data(const paddle::platform::Place& place, - paddle::framework::proto::VarType::Type type, - size_t requested_size = 0); - - void* mutable_data(const paddle::platform::Place& place, - size_t requested_size = 0); - - void* mutable_data(const paddle::platform::Place& place, - paddle::framework::proto::VarType::Type type, - const paddle::platform::Stream& stream); - - /* @jim19930609: Remove dependency on protobuf after Tensor Unification. - */ - paddle::framework::proto::VarType::Type type() const; - - /* @jim19930609: Remove dependency on protobuf after Tensor Unification. - */ - paddle::framework::proto::VarType::Type saved_type() const; - - // memory size returns the holding memory size in byte. - size_t memory_size() const; - - void check_memory_size() const; - - void set_layout(const paddle::framework::DataLayout layout); - - void clear() { - holder_.reset(); - meta_.offset = 0; - } - - void ShareBufferWith(const DenseTensor& tensor); - - void ShareDataTypeWith(const DenseTensor& tensor) { - meta_.dtype = tensor.meta().dtype; - } - - bool IsSharedBufferWith(const DenseTensor& src) const { - return holder_ && holder_ == src.Holder(); - } - - const std::shared_ptr& Holder() const { return holder_; } - - void set_offset(size_t offset) { meta_.offset = offset; } - size_t offset() const { return meta_.offset; } - - std::shared_ptr MoveMemoryHolder() { - return std::move(holder_); - } - - void ResetHolder(const std::shared_ptr& holder); - - void ResetHolderWithType(const std::shared_ptr& holder, - paddle::framework::proto::VarType::Type type); - - void set_type(paddle::framework::proto::VarType::Type type); - - TensorInplaceVersion& InplaceVersionCounter() { - return *inplace_version_counter_; - } - - /*! The internal of two tensors share the same memory block. */ - DenseTensor& ShareDataWith(const DenseTensor& src); - - /*! The internal of two tensors share the same inplace version counter. */ - DenseTensor& ShareInplaceVersionCounterWith(const DenseTensor& src); - - DenseTensor Slice(int64_t begin_idx, int64_t end_idx) const; - - std::vector Split(int64_t split_size, int64_t axis) const; - - std::vector Chunk(int64_t chunks, int64_t axis) const; - - protected: - std::shared_ptr inplace_version_counter_; - -/* @jim19930609: This is a hack - In general, it is badly designed to fuse MKLDNN-specific objects into a - generic Tensor. - We temporarily leave them here to unblock Tensor Unification progress. - In the final state, we should come up with a MKLDNN_Tensor and move the - following codes there. - */ -#ifdef PADDLE_WITH_MKLDNN - - public: - inline dnnl::memory::format_tag format() const { return format_; } - - inline void set_format(const dnnl::memory::format_tag format) { - format_ = format; - } - - protected: - /** - * @brief the detail format of memory block which have layout as kMKLDNN - * - * @note MKLDNN lib support various memory format like nchw, nhwc, nChw8C, - * nChw16c, etc. For a MKLDNN memory block, layout will be set as - * DataLayout::kMKLDNN meanwhile detail memory format will be kept in - * this field. - */ - - dnnl::memory::format_tag format_ = dnnl::memory::format_tag::undef; -#endif - - /* ------------------------------ */ - /* From framework::LoDTensor */ - /* ------------------------------ */ - /* The following members & interfaces were copied from framework::Tensor, - so as to facilitate the unification of different Tensors - - Will be adjusted/removed/moved in the near future - */ - public: - explicit DenseTensor(const LoD& lod); - - void set_lod(const LoD& lod); - - LoD* mutable_lod(); - - /* - * Get the start offset and end offset of an element from LoD. - */ - std::pair lod_element(size_t level, size_t elem) const; - - size_t NumLevels() const; - - size_t NumElements(size_t level = 0) const; +#include "paddle/pten/core/dense_tensor.inl" }; - } // namespace pten diff --git a/paddle/pten/core/dense_tensor.inl b/paddle/pten/core/dense_tensor.inl new file mode 100644 index 0000000000000000000000000000000000000000..754baeb73c90c2b494bf774588219c877a2fb8e9 --- /dev/null +++ b/paddle/pten/core/dense_tensor.inl @@ -0,0 +1,197 @@ +/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +/* --------------------------- */ +/* From framework::Tensor */ +/* --------------------------- */ +/* The following members & interfaces were copied from framework::Tensor, + so as to facilitate the unification of different Tensors + + Will be adjusted/removed/moved in the near future +*/ +public: +/* Temporarily put InplaceVersion inside DenseTensor. +Will move to AutogradMeta as soon as we switch to Eager Dygraph. +*/ +class InplaceVersion { +public: + bool IsUnique() const { return inplace_version_ == 0; } + void Bump() { ++inplace_version_; } + uint32_t CurrentVersion() const { return inplace_version_; } + void SetInplaceVersionToZero() { inplace_version_ = 0; } + +private: + uint32_t inplace_version_{0}; +}; + +/* @jim19930609: Remove dependency on protobuf after Tensor Unification. +*/ +explicit DenseTensor(paddle::framework::proto::VarType::Type dtype); + +/// \brief Use existing storage space to create dense tensor. This interface +/// can be used to deliberately create an uninitialized dense tensor. +/// \param storage The existing storage. +/// \param meta The meta data of dense tensor. +DenseTensor(intrusive_ptr storage, const DenseTensorMeta& meta); + +/// \brief Use existing storage space to create dense tensor. This interface +/// can be used to deliberately create an uninitialized dense tensor. +/// \param storage The existing storage. +/// \param meta The meta data of dense tensor. +DenseTensor(intrusive_ptr storage, DenseTensorMeta&& meta); + +inline bool IsInitialized() const { return holder_ != nullptr; } + +template +T* data(); + +void* data(); + +template +T* mutable_data(const paddle::platform::Place& place, + size_t requested_size = 0); + +template +T* mutable_data(const DDim& dims, + const paddle::platform::Place& place, + size_t requested_size = 0); + +void* mutable_data(const paddle::platform::Place& place, + paddle::framework::proto::VarType::Type type, + size_t requested_size = 0); + +void* mutable_data(const paddle::platform::Place& place, + size_t requested_size = 0); + +void* mutable_data(const paddle::platform::Place& place, + paddle::framework::proto::VarType::Type type, + const paddle::platform::Stream& stream); + +/* @jim19930609: Remove dependency on protobuf after Tensor Unification. +*/ +paddle::framework::proto::VarType::Type type() const; + +/* @jim19930609: Remove dependency on protobuf after Tensor Unification. +*/ +paddle::framework::proto::VarType::Type saved_type() const; + +// memory size returns the holding memory size in byte. +size_t memory_size() const; + +void check_memory_size() const; + +void set_layout(const paddle::framework::DataLayout layout); + +void clear() { + holder_.reset(); + meta_.offset = 0; +} + +void ShareBufferWith(const DenseTensor& tensor); + +void ShareDataTypeWith(const DenseTensor& tensor) { + meta_.dtype = tensor.meta().dtype; +} + +bool IsSharedBufferWith(const DenseTensor& src) const { + return holder_ && holder_ == src.Holder(); +} + +const std::shared_ptr& Holder() const { return holder_; } + +void set_offset(size_t offset) { meta_.offset = offset; } +size_t offset() const { return meta_.offset; } + +std::shared_ptr MoveMemoryHolder() { + return std::move(holder_); +} + +void ResetHolder(const std::shared_ptr& holder); + +void ResetHolderWithType(const std::shared_ptr& holder, + paddle::framework::proto::VarType::Type type); + +void set_type(paddle::framework::proto::VarType::Type type); + +InplaceVersion& InplaceVersionCounter() { + return *inplace_version_counter_; +} + +/*! The internal of two tensors share the same memory block. */ +DenseTensor& ShareDataWith(const DenseTensor& src); + +/*! The internal of two tensors share the same inplace version counter. */ +DenseTensor& ShareInplaceVersionCounterWith(const DenseTensor& src); + +DenseTensor Slice(int64_t begin_idx, int64_t end_idx) const; + +std::vector Split(int64_t split_size, int64_t axis) const; + +std::vector Chunk(int64_t chunks, int64_t axis) const; + +protected: +std::shared_ptr inplace_version_counter_{std::make_shared()}; + +/* @jim19930609: This is a hack +In general, it is badly designed to fuse MKLDNN-specific objects into a +generic Tensor. +We temporarily leave them here to unblock Tensor Unification progress. +In the final state, we should come up with a MKLDNN_Tensor and move the +following codes there. +*/ +#ifdef PADDLE_WITH_MKLDNN + +public: +inline dnnl::memory::format_tag format() const { return format_; } + +inline void set_format(const dnnl::memory::format_tag format) { + format_ = format; +} + +protected: +/** + * @brief the detail format of memory block which have layout as kMKLDNN + * + * @note MKLDNN lib support various memory format like nchw, nhwc, nChw8C, + * nChw16c, etc. For a MKLDNN memory block, layout will be set as + * DataLayout::kMKLDNN meanwhile detail memory format will be kept in + * this field. + */ + +dnnl::memory::format_tag format_ = dnnl::memory::format_tag::undef; +#endif + +/* ------------------------------ */ +/* From framework::LoDTensor */ +/* ------------------------------ */ +/* The following members & interfaces were copied from framework::Tensor, + so as to facilitate the unification of different Tensors + + Will be adjusted/removed/moved in the near future +*/ +public: +explicit DenseTensor(const LoD& lod); + +void set_lod(const LoD& lod); + +LoD* mutable_lod(); + +/* +* Get the start offset and end offset of an element from LoD. +*/ +std::pair lod_element(size_t level, size_t elem) const; + +size_t NumLevels() const; + +size_t NumElements(size_t level = 0) const; diff --git a/paddle/pten/core/dense_tensor_impl.cc b/paddle/pten/core/dense_tensor_impl.cc new file mode 100644 index 0000000000000000000000000000000000000000..f825d3619b92bcfd8d66ea47d9f176630ccbb525 --- /dev/null +++ b/paddle/pten/core/dense_tensor_impl.cc @@ -0,0 +1,394 @@ +/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/pten/core/dense_tensor.h" + +// See Note [ Why still include the fluid headers? ] +#include "paddle/pten/common/bfloat16.h" +#include "paddle/pten/common/complex.h" +#include "paddle/pten/common/float16.h" + +#include "paddle/pten/api/lib/utils/storage.h" +#include "paddle/pten/core/convert_utils.h" + +namespace pten { +/* --------------------------- */ +/* From framework::Tensor */ +/* --------------------------- */ +DenseTensor::DenseTensor() { + meta_.dtype = paddle::experimental::DataType::FLOAT32; + meta_.offset = 0; +} + +DenseTensor::DenseTensor(paddle::framework::proto::VarType::Type dtype) { + meta_.dtype = TransToPtenDataType(dtype); + meta_.offset = 0; +} + +size_t DenseTensor::memory_size() const { + return holder_ == nullptr ? 0UL : holder_->size() - meta_.offset; +} + +void DenseTensor::check_memory_size() const { + PADDLE_ENFORCE_NOT_NULL(holder_, + paddle::platform::errors::PreconditionNotMet( + "Tensor holds no memory. " + "Call Tensor::mutable_data firstly.")); + PADDLE_ENFORCE_LE( + numel() * SizeOf(dtype()), + memory_size(), + paddle::platform::errors::PreconditionNotMet( + "Tensor's dimension is out of bound." + "Tensor's dimension must be equal or less than the size of its " + "memory." + "But received Tensor's dimension is d%, memory's size is %d.", + numel() * SizeOf(dtype()), + memory_size())); +} + +const paddle::platform::Place& DenseTensor::place() const { + PADDLE_ENFORCE_NOT_NULL( + holder_, + paddle::platform::errors::PreconditionNotMet( + "Tensor not initialized yet when DenseTensor::place() is called.")); + return holder_->place(); +} + +paddle::framework::proto::VarType::Type DenseTensor::type() const { + return TransToProtoVarType(meta_.dtype); +} + +paddle::framework::proto::VarType::Type DenseTensor::saved_type() const { + return TransToProtoVarType(meta_.dtype); +} + +void DenseTensor::set_layout(const paddle::framework::DataLayout layout) { + meta_.layout = layout; +} + +void DenseTensor::ResetHolder(const std::shared_ptr& holder) { + PADDLE_ENFORCE_EQ( + meta_.offset, + 0, + paddle::platform::errors::Fatal( + "Only the offset is supported to zero when the holder is reset.")); + + if (holder_) { + // TODO(zyfncg): The change of static_cast<> in check will recover back + // when SetAllocationForOutputTenosr is deleted. + // Now the numel() may return -1, and will cast to a very large number when + // compare with a data with unsigned long type, this will make checking + // failed, so it's a temporary solution to deal with this problem. + PADDLE_ENFORCE_LE( + numel() * static_cast(SizeOf(dtype())), + static_cast(holder->size()), + paddle::platform::errors::InvalidArgument( + "The size of Holder is not enough to store the Tensor.")); + } + holder_ = holder; +} + +void DenseTensor::ResetHolderWithType( + const std::shared_ptr& holder, + paddle::framework::proto::VarType::Type type) { + set_type(type); + ResetHolder(holder); +} + +void DenseTensor::set_type(paddle::framework::proto::VarType::Type type) { + meta_.dtype = TransToPtenDataType(type); +} + +void* DenseTensor::mutable_data(const paddle::platform::Place& place, + paddle::framework::proto::VarType::Type type, + size_t requested_size) { + set_type(type); + PADDLE_ENFORCE_GE( + numel(), + 0, + paddle::platform::errors::PreconditionNotMet( + "The Tensor's element number must be equal or greater than zero. " + "The Tensor's shape is [", + dims(), + "] now")); + size_t size = numel() * SizeOf(dtype()); + if (requested_size && (requested_size > size)) { + size = requested_size; + } + + /* some versions of boost::variant don't have operator!= */ + if (holder_ == nullptr || !(holder_->place() == place) || + holder_->size() < size + meta_.offset) { + holder_.reset(); + holder_ = paddle::memory::AllocShared(place, size); + meta_.offset = 0; + } + return reinterpret_cast(reinterpret_cast(holder_->ptr()) + + meta_.offset); +} + +void* DenseTensor::mutable_data(const paddle::platform::Place& place, + size_t requested_size) { + return mutable_data(place, type(), requested_size); +} + +void* DenseTensor::mutable_data(const paddle::platform::Place& place, + paddle::framework::proto::VarType::Type type, + const paddle::platform::Stream& stream) { + set_type(type); + PADDLE_ENFORCE_GE( + numel(), + 0, + paddle::platform::errors::PreconditionNotMet( + "The Tensor's element number must be equal or greater than zero. " + "The Tensor's shape is [", + dims(), + "] now")); + size_t size = numel() * SizeOf(dtype()); + + /* some versions of boost::variant don't have operator!= */ + if (holder_ == nullptr || !(holder_->place() == place) || + holder_->size() < size + meta_.offset || + !(paddle::platform::is_gpu_place(place) && + paddle::memory::InSameStream(holder_, stream))) { + holder_.reset(); + holder_ = paddle::memory::AllocShared(place, size, stream); + meta_.offset = 0; + } + return reinterpret_cast(reinterpret_cast(holder_->ptr()) + + meta_.offset); +} + +/* @jim19930609: The following "mutable_data" only supports specific dtypes + defined in OpProto. This part need another clean up once the data type across + Fluid + and Pten get unified. + */ +template +inline T* DenseTensor::mutable_data(const DDim& dims, + const paddle::platform::Place& place, + size_t requested_size) { + static_assert(std::is_pod::value, "T must be POD"); + meta_.dims = dims; + return mutable_data(place, requested_size); +} + +template +inline T* DenseTensor::mutable_data(const paddle::platform::Place& place, + size_t requested_size) { + static_assert(std::is_pod::value, "T must be POD"); + return reinterpret_cast(mutable_data( + place, paddle::framework::DataTypeTrait::DataType(), requested_size)); +} + +void DenseTensor::ShareBufferWith(const DenseTensor& tensor) { + holder_ = tensor.holder_; + meta_.offset = tensor.meta().offset; + meta_.dtype = tensor.dtype(); +} + +#define LEGACY_DATA_MEMBER_FUNC_INSTANTIATION(dtype) \ + template dtype* DenseTensor::mutable_data( \ + const DDim& dims, \ + const paddle::platform::Place& place, \ + size_t requested_size); \ + template dtype* DenseTensor::mutable_data( \ + const paddle::platform::Place& place, size_t requested_size); + +LEGACY_DATA_MEMBER_FUNC_INSTANTIATION(bool) +LEGACY_DATA_MEMBER_FUNC_INSTANTIATION(int8_t) +LEGACY_DATA_MEMBER_FUNC_INSTANTIATION(uint8_t) +LEGACY_DATA_MEMBER_FUNC_INSTANTIATION(int16_t) +LEGACY_DATA_MEMBER_FUNC_INSTANTIATION(int32_t) +LEGACY_DATA_MEMBER_FUNC_INSTANTIATION(int64_t) +LEGACY_DATA_MEMBER_FUNC_INSTANTIATION(float) +LEGACY_DATA_MEMBER_FUNC_INSTANTIATION(double) +LEGACY_DATA_MEMBER_FUNC_INSTANTIATION(::paddle::platform::bfloat16) +LEGACY_DATA_MEMBER_FUNC_INSTANTIATION(::paddle::platform::float16) +LEGACY_DATA_MEMBER_FUNC_INSTANTIATION(::paddle::experimental::complex64) +LEGACY_DATA_MEMBER_FUNC_INSTANTIATION(::paddle::experimental::complex128) + +#undef LEGACY_DATA_MEMBER_FUNC_INSTANTIATION + +/* ------------------------------ */ +/* From framework::LoDTensor */ +/* ------------------------------ */ + +DenseTensor::DenseTensor(intrusive_ptr storage, + const DenseTensorMeta& meta) + : meta_(meta), holder_(storage->move_data_shared()) {} + +DenseTensor::DenseTensor(intrusive_ptr storage, DenseTensorMeta&& meta) + : meta_(std::move(meta)), holder_(storage->move_data_shared()) {} + +DenseTensor::DenseTensor(const LoD& lod) : DenseTensor() { meta_.lod = lod; } + +void DenseTensor::set_lod(const LoD& lod) { meta_.lod = lod; } + +LoD* DenseTensor::mutable_lod() { return &meta_.lod; } + +std::pair DenseTensor::lod_element(size_t level, + size_t elem) const { + PADDLE_ENFORCE_LT( + level, + NumLevels(), + paddle::platform::errors::InvalidArgument( + "The input level of LoD is invalid, it should be less than LoD " + "size. The input level is %zu, the LoD size is %zu.", + level, + NumLevels())); + + PADDLE_ENFORCE_LT(elem, + NumElements(level), + paddle::platform::errors::InvalidArgument( + "The input element of LoD is invalid, it should be " + "less than the number of elements in its level." + "The input element is %zu, the number of elements in " + "its level is %zu.", + elem, + NumElements(level))); + + return std::make_pair((meta_.lod)[level][elem], (meta_.lod)[level][elem + 1]); +} + +size_t DenseTensor::NumLevels() const { return meta_.lod.size(); } + +size_t DenseTensor::NumElements(size_t level) const { + PADDLE_ENFORCE_LT( + level, + NumLevels(), + paddle::platform::errors::InvalidArgument( + "The input level of LoD is invalid, it should be less than LoD " + "size. The input level is %zu, the LoD size is %zu.", + level, + NumLevels())); + + // the last offset is the end of last element + return (meta_.lod)[level].size() - 1; +} + +DenseTensor& DenseTensor::Resize(const DDim& dims) { + meta_.dims = dims; + return *this; +} + +DenseTensor DenseTensor::Slice(int64_t begin_idx, int64_t end_idx) const { + check_memory_size(); + PADDLE_ENFORCE_GE(begin_idx, + 0, + paddle::platform::errors::OutOfRange( + "The start row index must be greater than 0." + "But received the start index is d%.", + begin_idx)); + PADDLE_ENFORCE_LE(end_idx, + meta_.dims[0], + paddle::platform::errors::OutOfRange( + "The end row index is out of bound.")); + PADDLE_ENFORCE_LT( + begin_idx, + end_idx, + paddle::platform::errors::InvalidArgument( + "The start row index must be less than the end row index." + "But received the start index = %d, the end index = %d.", + begin_idx, + end_idx)); + + if (meta_.dims[0] == 1) { + return *this; + } else { + size_t base = numel() / meta_.dims[0]; + DenseTensor dst; + dst.holder_ = holder_; + dst.set_layout(meta_.layout); + dst.meta_.dtype = meta_.dtype; + DDim dst_dims = meta_.dims; + dst_dims[0] = end_idx - begin_idx; + dst.Resize(dst_dims); + dst.meta_.offset = meta_.offset + begin_idx * base * SizeOf(dtype()); + return dst; + } +} + +std::vector DenseTensor::Split(int64_t split_size, + int64_t axis) const { + check_memory_size(); + + PADDLE_ENFORCE_GE(meta_.dims.size(), + 0, + paddle::platform::errors::OutOfRange( + "split expects at least a 1-dimensional tensor")); + + PADDLE_ENFORCE_GE( + split_size, + 0, + paddle::platform::errors::OutOfRange( + "split expects split_size be non-negative, but got split_size is %d", + split_size)); + + int64_t numel_size = meta_.dims[axis]; + + int64_t num_splits = 1; + if (split_size != 0) { + num_splits = + std::max((numel_size + split_size - 1) / split_size, 1); + } + + std::vector splits(num_splits); + int64_t last_split_size = split_size - (split_size * num_splits - numel_size); + + for (int64_t i = 0; i < num_splits; ++i) { + int64_t length = i < num_splits - 1 ? split_size : last_split_size; + splits[i] = Slice(i * split_size, i * split_size + length); + } + return splits; +} + +std::vector DenseTensor::Chunk(int64_t chunks, + int64_t axis) const { + check_memory_size(); + PADDLE_ENFORCE_GE(meta_.dims.size(), + 0, + paddle::platform::errors::OutOfRange( + "split expects at least a 1-dimensional tensor")); + PADDLE_ENFORCE_GE( + chunks, + 0, + paddle::platform::errors::OutOfRange( + "chunks expects to be greater than 0, but got chunks is %d", chunks)); + + int64_t numel_size = meta_.dims[axis]; + int64_t split_size = (numel_size + chunks - 1) / chunks; + return Split(split_size, axis); +} + +DenseTensor& DenseTensor::ShareDataWith(const DenseTensor& src) { + src.check_memory_size(); + // Preserve LoD + auto lod = meta_.lod; + *this = src; + meta_.lod = lod; + return *this; +} + +DenseTensor& DenseTensor::ShareInplaceVersionCounterWith( + const DenseTensor& src) { + PADDLE_ENFORCE_NOT_NULL( + inplace_version_counter_, + paddle::platform::errors::PreconditionNotMet( + "Tensor does not hold inplace_version_counter_.")); + + inplace_version_counter_ = src.inplace_version_counter_; + return *this; +} +} // namespace pten diff --git a/paddle/pten/core/tensor_status.h b/paddle/pten/core/tensor_status.h deleted file mode 100644 index e426a27eabb882adf447d610c957173a46903c49..0000000000000000000000000000000000000000 --- a/paddle/pten/core/tensor_status.h +++ /dev/null @@ -1,62 +0,0 @@ -/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#pragma once - -#include "paddle/pten/common/backend.h" -#include "paddle/pten/common/data_type.h" -#include "paddle/pten/common/layout.h" -namespace pten { -class TensorInplaceVersion { - public: - explicit TensorInplaceVersion(uint32_t inplace_version = 0) - : inplace_version_(inplace_version) {} - bool IsUnique() const { return inplace_version_ == 0; } - void Bump() { ++inplace_version_; } - uint32_t CurrentVersion() const { return inplace_version_; } - - private: - uint32_t inplace_version_; -}; - -/** - * The Status data member of DenseTensor. - * - * Here the `static` represents information describing the status of Tensor, - * such as version counter, or other bool status members. - * - * Note: TensorStatus is a struct, the members are named like - * ordinary nonmember variables, such as `type` instead of `type_`. - * And we direct access its members, in addition to constructor, destructor - * and functions for setting data members, can not provide other functions. - * - * Note: polish impl later - */ -struct TensorStatus { - TensorStatus() = default; - TensorStatus(const TensorStatus&) = default; - TensorStatus(TensorStatus&&) = default; - - TensorStatus& operator=(const TensorStatus&) = delete; - TensorStatus& operator=(TensorStatus&&) = delete; - - TensorInplaceVersion inplace_version_counter{0}; - - /** - * For Scalar Tensor design - */ - bool is_scalar{false}; -}; - -} // namespace pten