diff --git a/cmake/cuda.cmake b/cmake/cuda.cmake index b7a93cd9ee2160090c0142d62d96da72e4c58717..7a94bda0f5f73e48081f68d7b2730e3df1e46232 100644 --- a/cmake/cuda.cmake +++ b/cmake/cuda.cmake @@ -16,6 +16,7 @@ else() set(paddle_known_gpu_archs8 "30 35 50 52 60 61") set(paddle_known_gpu_archs9 "30 35 50 52 60 61 70") set(paddle_known_gpu_archs10 "30 35 50 52 60 61 70 75") + set(paddle_known_gpu_archs11 "52 60 61 70 75 80") endif() ###################################################################################### @@ -188,6 +189,10 @@ elseif (${CMAKE_CUDA_COMPILER_VERSION} LESS 11.0) # CUDA 10.x set(paddle_known_gpu_archs ${paddle_known_gpu_archs10}) set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -D_MWAITXINTRIN_H_INCLUDED") set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -D__STRICT_ANSI__") +elseif (${CMAKE_CUDA_COMPILER_VERSION} LESS 12.0) # CUDA 11.x + set(paddle_known_gpu_archs ${paddle_known_gpu_archs11}) + set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -D_MWAITXINTRIN_H_INCLUDED") + set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -D__STRICT_ANSI__") endif() add_definitions("-DPADDLE_CUDA_BINVER=\"${CUDA_VERSION_MAJOR}${CUDA_VERSION_MINOR}\"") diff --git a/cmake/external/dgc.cmake b/cmake/external/dgc.cmake index 8472a0743b91e69d823ed62f94b55045a31aaabc..bc8611f3862cd14c0de493564ea82a1c9ce66667 100644 --- a/cmake/external/dgc.cmake +++ b/cmake/external/dgc.cmake @@ -19,7 +19,7 @@ SET(DGC_SOURCES_DIR "${THIRD_PARTY_PATH}/dgc/src/extern_dgc") SET(DGC_INSTALL_DIR "${THIRD_PARTY_PATH}/install/dgc") SET(DGC_INCLUDE_DIR "${DGC_INSTALL_DIR}/include" CACHE PATH "dgc include directory." FORCE) SET(DGC_LIBRARIES "${DGC_INSTALL_DIR}/lib/libdgc.a" CACHE FILEPATH "dgc library." FORCE) -SET(DGC_URL "http://fleet.bj.bcebos.com/collective_ef2216a.tgz") +SET(DGC_URL "https://fleet.bj.bcebos.com/dgc/collective_f66ef73.tgz") INCLUDE_DIRECTORIES(${DGC_INCLUDE_DIR}) cache_third_party(extern_dgc @@ -30,7 +30,7 @@ ExternalProject_Add( extern_dgc ${EXTERNAL_PROJECT_LOG_ARGS} "${DGC_DOWNLOAD_CMD}" - URL_MD5 "2f67549fd5f1262383d83289abc4f88f" + URL_MD5 "94e6fa1bc97169d0e1aad44570fe3251" PREFIX "${DGC_PREFIX_DIR}" SOURCE_DIR "${DGC_SOURCES_DIR}" CONFIGURE_COMMAND "" diff --git a/cmake/external/lite.cmake b/cmake/external/lite.cmake index 8a655b2954dea5d6b864616ed2f4d19b167c4be8..3da550519bae2a12139873a2a84680debbaa8f4c 100644 --- a/cmake/external/lite.cmake +++ b/cmake/external/lite.cmake @@ -34,7 +34,7 @@ if (NOT LITE_SOURCE_DIR OR NOT LITE_BINARY_DIR) set(LITE_INSTALL_DIR ${THIRD_PARTY_PATH}/install/lite) if(NOT LITE_GIT_TAG) - set(LITE_GIT_TAG dfdfa6440c83bf0b415f9f5a9ff84842ce0bb0fa) + set(LITE_GIT_TAG 6d2b2a4028a58715b01887b04eb9bff8432eb184) endif() if(NOT CUDA_ARCH_NAME) diff --git a/cmake/external/mkldnn.cmake b/cmake/external/mkldnn.cmake index ae870b766fc3349ea53628e14c68ab9a5826213f..c0adda0da31ae1e7425ddfb352971444c09d5615 100644 --- a/cmake/external/mkldnn.cmake +++ b/cmake/external/mkldnn.cmake @@ -19,8 +19,8 @@ SET(MKLDNN_PREFIX_DIR ${THIRD_PARTY_PATH}/mkldnn) SET(MKLDNN_SOURCE_DIR ${THIRD_PARTY_PATH}/mkldnn/src/extern_mkldnn) SET(MKLDNN_INSTALL_DIR ${THIRD_PARTY_PATH}/install/mkldnn) SET(MKLDNN_INC_DIR "${MKLDNN_INSTALL_DIR}/include" CACHE PATH "mkldnn include directory." FORCE) -SET(MKLDNN_REPOSITORY https://github.com/intel/mkl-dnn.git) -SET(MKLDNN_TAG 1ea812f4f5aa1bd989372a23ab50d0f0f81ee677) +SET(MKLDNN_REPOSITORY https://github.com/oneapi-src/oneDNN.git) +SET(MKLDNN_TAG 64a48f9565aa72f6359917b3406328075a409939) # Introduce variables: # * CMAKE_INSTALL_LIBDIR diff --git a/cmake/external/warpctc.cmake b/cmake/external/warpctc.cmake index 0f6b1c182d5590354c8a970eea339a3e23846f39..ac6cf624e82c0a346fea42fa29fe9bab6ace8d47 100644 --- a/cmake/external/warpctc.cmake +++ b/cmake/external/warpctc.cmake @@ -18,7 +18,7 @@ SET(WARPCTC_PREFIX_DIR ${THIRD_PARTY_PATH}/warpctc) SET(WARPCTC_SOURCE_DIR ${THIRD_PARTY_PATH}/warpctc/src/extern_warpctc) SET(WARPCTC_INSTALL_DIR ${THIRD_PARTY_PATH}/install/warpctc) set(WARPCTC_REPOSITORY https://github.com/baidu-research/warp-ctc.git) -set(WARPCTC_TAG bc29dcfff07ced1c7a19a4ecee48e5ad583cef8e) +set(WARPCTC_TAG fc7f226b93758216a03b1be9d24593a12819b984) SET(WARPCTC_INCLUDE_DIR "${WARPCTC_INSTALL_DIR}/include" CACHE PATH "Warp-ctc Directory" FORCE) diff --git a/cmake/flags.cmake b/cmake/flags.cmake index 9d07a0979d9392c9b2ab78562f8e0ceb8fc5d722..415e07c75425345f5f1ad29a8544e02a5bfb12e4 100644 --- a/cmake/flags.cmake +++ b/cmake/flags.cmake @@ -28,7 +28,15 @@ function(CheckCompilerCXX11Flag) endfunction() CheckCompilerCXX11Flag() -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") +if (WITH_GPU) + if (${CMAKE_CUDA_COMPILER_VERSION} GREATER_EQUAL 11.0) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14") + else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + endif() +else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") +endif() # safe_set_flag # # Set a compile flag only if compiler is support diff --git a/cmake/third_party.cmake b/cmake/third_party.cmake index c9442e8f843ac152cac02908799a8d24f5951e58..9edfcb967abc26a25a94d368298c1c475295019f 100644 --- a/cmake/third_party.cmake +++ b/cmake/third_party.cmake @@ -243,9 +243,10 @@ IF(WITH_TESTING OR (WITH_DISTRIBUTE AND NOT WITH_GRPC)) ENDIF() if(WITH_GPU) - include(external/cub) # download cub - list(APPEND third_party_deps extern_cub) - + if (${CMAKE_CUDA_COMPILER_VERSION} LESS 11.0) + include(external/cub) # download cub + list(APPEND third_party_deps extern_cub) + endif() set(CUDAERROR_URL "http://paddlepaddledeps.bj.bcebos.com/cudaErrorMessage.tar.gz" CACHE STRING "" FORCE) file_download_and_uncompress(${CUDAERROR_URL} "cudaerror") # download file cudaErrorMessage endif(WITH_GPU) diff --git a/paddle/fluid/framework/c/c_api.cc b/paddle/fluid/framework/c/c_api.cc index ab987fb56686594f505e63b6664c2176e5a4ad89..0dd2768ccb9ffa1dc7b85dca500095f8c10479c3 100644 --- a/paddle/fluid/framework/c/c_api.cc +++ b/paddle/fluid/framework/c/c_api.cc @@ -49,7 +49,8 @@ std::vector PD_GetGradOpDescStrs( for (size_t i = 0; i < op_num; ++i) { PADDLE_ENFORCE_EQ( grad_op_descs[i]->Proto()->SerializePartialToString(&ret[i]), true, - "Cannot serialize message."); + paddle::platform::errors::Unavailable( + "Cannot serialize operator desc message.")); } } return ret; diff --git a/paddle/fluid/framework/distributed_strategy.proto b/paddle/fluid/framework/distributed_strategy.proto old mode 100755 new mode 100644 index 551d1342edeb335d1cad4782f85ae9f94f8739bd..edd1700ae7284c77883af6abd2cd7d511097685f --- a/paddle/fluid/framework/distributed_strategy.proto +++ b/paddle/fluid/framework/distributed_strategy.proto @@ -36,7 +36,10 @@ message AMPConfig { repeated string custom_black_varnames = 9; } -message LocalSGDConfig { optional int32 k_steps = 1 [ default = 4 ]; } +message LocalSGDConfig { + optional int32 k_steps = 1 [ default = 1 ]; + optional int32 begin_step = 2 [ default = 1 ]; +} message GradientMergeConfig { optional int32 k_steps = 1 [ default = 1 ]; @@ -52,6 +55,8 @@ message DGCConfig { message LarsConfig { optional float lars_coeff = 1 [ default = 0.001 ]; optional float lars_weight_decay = 2 [ default = 0.0005 ]; + optional float epsilon = 3 [ default = 0.0 ]; + repeated string exclude_from_weight_decay = 4; } message LambConfig { diff --git a/paddle/fluid/framework/fleet/nccl_wrapper.cc b/paddle/fluid/framework/fleet/nccl_wrapper.cc index d5a25605cf81147b520bf541e38f4f75e53ae756..33a91388fd8cc97d181df46ab826d384860d38f5 100644 --- a/paddle/fluid/framework/fleet/nccl_wrapper.cc +++ b/paddle/fluid/framework/fleet/nccl_wrapper.cc @@ -25,7 +25,7 @@ bool NCCLWrapper::is_initialized_ = false; void NCCLWrapper::InitNCCL() { #if defined(PADDLE_WITH_NCCL) - PADDLE_ENFORCE(platform::dynload::ncclCommInitRank( + PADDLE_ENFORCE_CUDA_SUCCESS(platform::dynload::ncclCommInitRank( &(nccl_info_.comm_), nccl_info_.global_ranks_, nccl_info_.nccl_id_, nccl_info_.my_global_rank_)); #endif @@ -41,7 +41,8 @@ void NCCLWrapper::SetNCCLId(const NCCLInfo& nccl_info) { NCCLInfo NCCLWrapper::GetNCCLId() { #if defined(PADDLE_WITH_NCCL) - PADDLE_ENFORCE(platform::dynload::ncclGetUniqueId(&(nccl_info_.nccl_id_))); + PADDLE_ENFORCE_CUDA_SUCCESS( + platform::dynload::ncclGetUniqueId(&(nccl_info_.nccl_id_))); #endif return nccl_info_; } @@ -52,8 +53,8 @@ void NCCLWrapper::SetRankInfo(const int local_rank, const int global_rank, nccl_info_.local_rank_ = local_rank; nccl_info_.my_global_rank_ = global_rank; nccl_info_.global_ranks_ = ranks; - PADDLE_ENFORCE(cudaSetDevice(local_rank)); - PADDLE_ENFORCE(cudaStreamCreate(&(nccl_info_.stream_))); + PADDLE_ENFORCE_CUDA_SUCCESS(cudaSetDevice(local_rank)); + PADDLE_ENFORCE_CUDA_SUCCESS(cudaStreamCreate(&(nccl_info_.stream_))); #endif return; } @@ -65,7 +66,7 @@ void NCCLWrapper::SyncVar(const int root_rank, const Scope& scope, auto var = scope.FindVar(name); LoDTensor* tensor = var->GetMutable(); int32_t total_size = tensor->numel(); - PADDLE_ENFORCE(platform::dynload::ncclBcast( + PADDLE_ENFORCE_CUDA_SUCCESS(platform::dynload::ncclBcast( reinterpret_cast(tensor->data()), total_size, ncclFloat, root_rank, nccl_info_.comm_, nccl_info_.stream_)); cudaStreamSynchronize(nccl_info_.stream_); diff --git a/paddle/fluid/framework/ir/multihead_matmul_fuse_pass.cc b/paddle/fluid/framework/ir/multihead_matmul_fuse_pass.cc index 40e01c75bb99157aedccd0692d7410b99393c009..198107ea082dc86d9e65a926bf9befe2fc4abfa4 100644 --- a/paddle/fluid/framework/ir/multihead_matmul_fuse_pass.cc +++ b/paddle/fluid/framework/ir/multihead_matmul_fuse_pass.cc @@ -615,6 +615,16 @@ static int BuildFusionV2(Graph* graph, const std::string& name_scope, GET_IR_NODE_FROM_SUBGRAPH(transpose2_qkv_out, transpose2_qkv_out, multihead_pattern); + // If weights or biases in qkv's fc are shared by multiple multihead_matmul + // patterns, we do not support this kind of fusion, this pass will not take + // effect. + bool is_fc_params_shared = + mul0_w->outputs.size() > 1 || mul1_w->outputs.size() > 1 || + mul2_w->outputs.size() > 1 || eltadd0_b->outputs.size() > 1 || + eltadd1_b->outputs.size() > 1 || eltadd2_b->outputs.size() > 1; + if (is_fc_params_shared) { + return; + } fuse_creater(input0, mul0, mul1, mul2, mul0_out, mul1_out, mul2_out, mul0_w, mul1_w, mul2_w, eltadd0_b, eltadd1_b, eltadd2_b, eltadd_qk_b, reshape2_0, reshape2_qkv_out, scale, scale_out); diff --git a/paddle/fluid/framework/tensor.cc b/paddle/fluid/framework/tensor.cc index 544c014eaf98a99b1737809f2cbad39b46fdb276..0b22bab26789a3e2ebd20428adc236faa8b38dee 100644 --- a/paddle/fluid/framework/tensor.cc +++ b/paddle/fluid/framework/tensor.cc @@ -19,13 +19,17 @@ namespace paddle { namespace framework { extern size_t SizeOfType(proto::VarType::Type type); void Tensor::check_memory_size() const { - PADDLE_ENFORCE_NOT_NULL( - holder_, "Tensor holds no memory. Call Tensor::mutable_data first."); + PADDLE_ENFORCE_NOT_NULL(holder_, platform::errors::PreconditionNotMet( + "Tensor holds no memory. " + "Call Tensor::mutable_data firstly.")); PADDLE_ENFORCE_LE( numel() * SizeOfType(type()), memory_size(), - "Tensor's dims_ is out of bound. Call Tensor::mutable_data " - "first to re-allocate memory.\n" - "or maybe the required data-type mismatches the data already stored."); + platform::errors::PreconditionNotMet( + "Tensor's dimension is out of bound." + "Tensor's dimension must be equal or less than the size of its " + "memory." + "But received Tensor's dimension is d%, memory's size is %d.", + numel() * SizeOfType(type()), memory_size())); } Tensor::Tensor(const proto::VarType::Type& dtype) : type_(dtype), offset_(0) {} @@ -37,15 +41,21 @@ size_t Tensor::memory_size() const { void* Tensor::mutable_data(const platform::Place& place, proto::VarType::Type type, size_t requested_size) { type_ = type; - PADDLE_ENFORCE_GE(numel(), 0, - "When calling this method, the Tensor's numel must be " - "equal or larger than zero. " - "Please check Tensor::dims, or Tensor::Resize has been " - "called first. The Tensor's shape is [", - dims(), "] now"); + PADDLE_ENFORCE_GE( + numel(), 0, + platform::errors::PreconditionNotMet( + "The Tensor's element number must be equal or greater than zero. " + "The Tensor's shape is [", + dims(), "] now")); size_t size = numel() * SizeOfType(type); if (requested_size) { - PADDLE_ENFORCE_GE(requested_size, size); + PADDLE_ENFORCE_GE( + requested_size, size, + platform::errors::InvalidArgument( + "The requested memory size is less than the memory size of Tensor. " + "But received requested memory size is d%, " + "memory size of Tensor is %d.", + requested_size, size)); size = requested_size; } /* some versions of boost::variant don't have operator!= */ @@ -62,8 +72,8 @@ void* Tensor::mutable_data(const platform::Place& place, void* Tensor::mutable_data(const platform::Place& place, size_t requested_size) { - PADDLE_ENFORCE_NOT_NULL( - this->holder_, "Cannot invoke mutable data if current hold nothing."); + PADDLE_ENFORCE_NOT_NULL(this->holder_, platform::errors::PreconditionNotMet( + "The tensor is not initialized.")); return mutable_data(place, type_, requested_size); } @@ -75,12 +85,20 @@ Tensor& Tensor::ShareDataWith(const Tensor& src) { Tensor Tensor::Slice(int64_t begin_idx, int64_t end_idx) const { check_memory_size(); - PADDLE_ENFORCE_GE(begin_idx, 0, - "The start row index must be greater than 0."); - PADDLE_ENFORCE_LE(end_idx, dims_[0], "The end row index is out of bound."); + PADDLE_ENFORCE_GE( + begin_idx, 0, + platform::errors::OutOfRange("The start row index must be greater than 0." + "But received the start index is d%.", + begin_idx)); + PADDLE_ENFORCE_LE( + end_idx, dims_[0], + platform::errors::OutOfRange("The end row index is out of bound.")); PADDLE_ENFORCE_LT( begin_idx, end_idx, - "The start row index must be lesser than the end row index."); + platform::errors::InvalidArgument( + "The start row index must be less than the end row index." + "But received the start index = %d, the end index = %d.", + begin_idx, end_idx)); if (dims_[0] == 1) { return *this; diff --git a/paddle/fluid/framework/tensor.h b/paddle/fluid/framework/tensor.h index d9fddc4c77d99efcb119ae37591b26bff0e51c0a..f2ccff2c133a238d02e25c65faf41dd519fdb506 100644 --- a/paddle/fluid/framework/tensor.h +++ b/paddle/fluid/framework/tensor.h @@ -131,13 +131,17 @@ class Tensor { const platform::Place& place() const { PADDLE_ENFORCE_NOT_NULL( - holder_, "Tensor not initialized yet when Tensor::place() is called."); + holder_, + platform::errors::PreconditionNotMet( + "Tensor not initialized yet when Tensor::place() is called.")); return holder_->place(); } proto::VarType::Type type() const { PADDLE_ENFORCE_NOT_NULL( - holder_, "Tensor not initialized yet when Tensor::type() is called."); + holder_, + platform::errors::PreconditionNotMet( + "Tensor not initialized yet when Tensor::type() is called.")); return type_; } diff --git a/paddle/fluid/framework/tensor_impl.h b/paddle/fluid/framework/tensor_impl.h index f5171b0a8d1efc33521c2d731e88f0f96bdf41c7..986551b935e8811a6b257c2b4a613a493b3b644b 100644 --- a/paddle/fluid/framework/tensor_impl.h +++ b/paddle/fluid/framework/tensor_impl.h @@ -43,9 +43,13 @@ inline T* Tensor::data() { check_memory_size(); bool valid = std::is_same::value || type_ == DataTypeTrait::DataType(); - PADDLE_ENFORCE( - valid, "Tensor holds the wrong type, it holds %s, but desires to be %s", - DataTypeToString(type_), DataTypeToString(DataTypeTrait::DataType())); + PADDLE_ENFORCE_EQ( + valid, true, + platform::errors::InvalidArgument( + "Tensor holds the wrong type, it holds %s, but desires to be %s", + DataTypeToString(type_), + DataTypeToString(DataTypeTrait::DataType()))); + return reinterpret_cast(reinterpret_cast(holder_->ptr()) + offset_); } @@ -69,9 +73,12 @@ inline T* Tensor::mutable_data(const platform::Place& place, inline Tensor ReshapeToMatrix(const Tensor& src, int num_col_dims) { int rank = src.dims().size(); PADDLE_ENFORCE_GE( - rank, 2, - "'ReshapeToMatrix()' is only used for flatten high rank " - "tensors to matrixs. Can not be used in reshaping vectors."); + rank, 2, platform::errors::InvalidArgument( + "'ReshapeToMatrix()' is only used for flatten high rank " + "tensors to matrixs. The dimensions of Tensor must be " + "greater or equal than 2. " + "But received dimensions of Tensor is %d", + rank)); if (rank == 2) { return src; } diff --git a/paddle/fluid/framework/tensor_test.cc b/paddle/fluid/framework/tensor_test.cc index 84f98d339a29b7d23de6e8be4b069b216be31ab2..cc972dd93d032c19015c86debebc27f7c8c0d155 100644 --- a/paddle/fluid/framework/tensor_test.cc +++ b/paddle/fluid/framework/tensor_test.cc @@ -41,7 +41,7 @@ TEST(Tensor, DataAssert) { std::string ex_msg = err.what(); EXPECT_TRUE(ex_msg.find("holder_ should not be null") != std::string::npos); EXPECT_TRUE(ex_msg.find("Tensor holds no memory. Call " - "Tensor::mutable_data first.") != + "Tensor::mutable_data firstly.") != std::string::npos); } ASSERT_TRUE(caught); @@ -157,7 +157,7 @@ TEST(Tensor, ShareDataWith) { EXPECT_TRUE(ex_msg.find("holder_ should not be null") != std::string::npos); EXPECT_TRUE(ex_msg.find("Tensor holds no memory. Call " - "Tensor::mutable_data first.") != + "Tensor::mutable_data firstly.") != std::string::npos); } ASSERT_TRUE(caught); diff --git a/paddle/fluid/framework/threadpool.cc b/paddle/fluid/framework/threadpool.cc index 7f7f426d0e28224932fc96a3fefa0df1279e6475..4682bfc264b68997abd0a87233c5ed39e7e50a63 100644 --- a/paddle/fluid/framework/threadpool.cc +++ b/paddle/fluid/framework/threadpool.cc @@ -42,7 +42,8 @@ void ThreadPool::Init() { num_threads = FLAGS_dist_threadpool_size; VLOG(1) << "set dist_threadpool_size to " << num_threads; } - PADDLE_ENFORCE_GT(num_threads, 0); + PADDLE_ENFORCE_GT(num_threads, 0, platform::errors::InvalidArgument( + "The number of threads is 0.")); threadpool_.reset(new ThreadPool(num_threads)); } } @@ -83,7 +84,8 @@ void ThreadPool::TaskLoop() { } if (tasks_.empty()) { - PADDLE_THROW("This thread has no task to Run"); + PADDLE_THROW(platform::errors::Unavailable( + "Current thread has no task to Run.")); } // pop a task from the task queue diff --git a/paddle/fluid/framework/threadpool.h b/paddle/fluid/framework/threadpool.h index 654d81116b280bb6a52af3f83aeec284387f3b63..09528b6fc35bf49ac3110440a62aba3200341e15 100644 --- a/paddle/fluid/framework/threadpool.h +++ b/paddle/fluid/framework/threadpool.h @@ -91,7 +91,8 @@ class ThreadPool { { std::unique_lock lock(mutex_); if (!running_) { - PADDLE_THROW("enqueue on stopped ThreadPool"); + PADDLE_THROW(platform::errors::Unavailable( + "Task is enqueued into stopped ThreadPool.")); } tasks_.push(std::move(task)); } diff --git a/paddle/fluid/framework/var_desc.cc b/paddle/fluid/framework/var_desc.cc index f3ea1f624ee836a483c37c2addb4d9766e87c107..2ee0b17b64b6df7a2f66b208f5b5879683db6656 100644 --- a/paddle/fluid/framework/var_desc.cc +++ b/paddle/fluid/framework/var_desc.cc @@ -43,8 +43,9 @@ void VarDesc::SetTensorDescNum(size_t num) { } break; default: PADDLE_THROW( - "Setting 'sub_tensor_number' is not supported by the type of var %s.", - this->Name()); + platform::errors::Unavailable("Setting 'sub_tensor_number' is not " + "supported by the %s type variable.", + this->Name())); } } @@ -55,8 +56,9 @@ size_t VarDesc::GetTensorDescNum() const { break; default: PADDLE_THROW( - "Getting 'sub_tensor_number' is not supported by the type of var %s.", - this->Name()); + platform::errors::Unavailable("Getting 'sub_tensor_number' is not " + "supported by the %s type variable.", + this->Name())); } } @@ -133,9 +135,9 @@ void VarDesc::SetLoDLevel(int32_t lod_level) { desc_.mutable_type()->mutable_tensor_array()->set_lod_level(lod_level); break; default: - PADDLE_THROW( - "Setting 'lod_level' is not supported by the type of var %s.", - this->Name()); + PADDLE_THROW(platform::errors::Unavailable( + "Setting 'lod_level' is not supported by the %s type variable.", + this->Name())); } } @@ -157,9 +159,9 @@ void VarDesc::SetLoDLevels(const std::vector &multiple_lod_level) { } } break; default: - PADDLE_THROW( - "Setting 'lod_levels' is not supported by the type of var %s.", - this->Name()); + PADDLE_THROW(platform::errors::Unavailable( + "Setting 'lod_levels' is not supported by the %s type variable", + this->Name())); } } @@ -170,9 +172,9 @@ int32_t VarDesc::GetLoDLevel() const { case proto::VarType::LOD_TENSOR_ARRAY: return desc_.type().tensor_array().lod_level(); default: - PADDLE_THROW( - "Getting 'lod_level' is not supported by the type of var %s.", - this->Name()); + PADDLE_THROW(platform::errors::Unavailable( + "Getting 'lod_level' is not supported by the %s type variable.", + this->Name())); } } @@ -187,15 +189,19 @@ std::vector VarDesc::GetLoDLevels() const { return res; break; default: - PADDLE_THROW( - "Getting 'lod_levels' is not supported by the type of var %s.", - this->Name()); + PADDLE_THROW(platform::errors::Unavailable( + "Getting 'lod_levels' is not supported by the %s type variable.", + this->Name())); } } const proto::VarType::TensorDesc &VarDesc::tensor_desc() const { - PADDLE_ENFORCE(desc_.has_type(), "The var's type hasn't been set."); - PADDLE_ENFORCE(desc_.type().has_type(), "The var type hasn't been set."); + PADDLE_ENFORCE_EQ( + desc_.has_type(), true, + platform::errors::NotFound("The variable's type was not be set.")); + PADDLE_ENFORCE_EQ( + desc_.type().has_type(), true, + platform::errors::NotFound("The variable's type was not be set.")); switch (desc_.type().type()) { case proto::VarType::SELECTED_ROWS: return desc_.type().selected_rows(); @@ -204,14 +210,16 @@ const proto::VarType::TensorDesc &VarDesc::tensor_desc() const { case proto::VarType::LOD_TENSOR_ARRAY: return desc_.type().tensor_array().tensor(); default: - PADDLE_THROW( - "Getting 'tensor_desc' is not supported by the type of var %s.", - this->Name()); + PADDLE_THROW(platform::errors::Unavailable( + "Getting 'tensor_desc' is not supported by the %s type variable.", + this->Name())); } } std::vector VarDesc::tensor_descs() const { - PADDLE_ENFORCE(desc_.has_type(), "The var type hasn't been set."); + PADDLE_ENFORCE_EQ( + desc_.has_type(), true, + platform::errors::NotFound("The variable's type was not be set.")); std::vector res; res.reserve(GetTensorDescNum()); switch (desc_.type().type()) { @@ -221,16 +229,19 @@ std::vector VarDesc::tensor_descs() const { } return res; default: - PADDLE_THROW( - "Getting 'tensor_descs' is not supported by the type of var " - "%s.", - this->Name()); + PADDLE_THROW(platform::errors::Unavailable( + "Getting 'tensor_descs' is not supported by the %s type variable.", + this->Name())); } } proto::VarType::TensorDesc *VarDesc::mutable_tensor_desc() { - PADDLE_ENFORCE(desc_.has_type(), "The var type hasn't been set."); - PADDLE_ENFORCE(desc_.type().has_type(), "The var type hasn't been set."); + PADDLE_ENFORCE_EQ( + desc_.has_type(), true, + platform::errors::NotFound("The variable's type was not be set.")); + PADDLE_ENFORCE_EQ( + desc_.type().has_type(), true, + platform::errors::NotFound("The variable's type was not be set.")); switch (desc_.type().type()) { case proto::VarType::SELECTED_ROWS: return desc_.mutable_type()->mutable_selected_rows(); @@ -240,15 +251,19 @@ proto::VarType::TensorDesc *VarDesc::mutable_tensor_desc() { return desc_.mutable_type()->mutable_tensor_array()->mutable_tensor(); default: PADDLE_THROW( - "Getting 'mutable_tensor_desc' is not supported by the type of var " - "%s.", - this->Name()); + platform::errors::Unavailable("Getting 'mutable_tensor_desc' is not " + "supported by the %s type variable.", + this->Name())); } } std::vector VarDesc::mutable_tensor_descs() { - PADDLE_ENFORCE(desc_.has_type(), "The var type hasn't been set."); - PADDLE_ENFORCE(desc_.type().has_type(), "The var type hasn't been set."); + PADDLE_ENFORCE_EQ( + desc_.has_type(), true, + platform::errors::NotFound("The variable's type was not be set.")); + PADDLE_ENFORCE_EQ( + desc_.type().has_type(), true, + platform::errors::NotFound("The variable's type was not be set.")); std::vector res; res.reserve(GetTensorDescNum()); switch (desc_.type().type()) { @@ -259,10 +274,9 @@ std::vector VarDesc::mutable_tensor_descs() { } return res; default: - PADDLE_THROW( - "Getting 'tensor_descs' is not supported by the type of var " - "%s.", - this->Name()); + PADDLE_THROW(platform::errors::Unavailable( + "Getting 'tensor_descs' is not supported by the %s type variable.", + this->Name())); } } diff --git a/paddle/fluid/framework/var_type.h b/paddle/fluid/framework/var_type.h index 43e9ed553bea84aaaaa18a46fe81f06a18b124af..8affeda67b3d07d67ceed2b657b285210e1bd076 100644 --- a/paddle/fluid/framework/var_type.h +++ b/paddle/fluid/framework/var_type.h @@ -40,7 +40,8 @@ inline proto::VarType::Type ToVarType(int type) { case proto::VarType::READER: return static_cast(type); default: - PADDLE_THROW("ToVarType:Unsupported type %d", type); + PADDLE_THROW(platform::errors::Unavailable( + "ToVarType method Unsupported type %d.", type)); } } @@ -66,7 +67,8 @@ inline void VisitVarType(const framework::Variable& var, Visitor visitor) { visitor(var.Get()); return; default: - PADDLE_THROW("Not supported visit type, %s", ToTypeName(var.Type())); + PADDLE_THROW(platform::errors::Unavailable("Not supported visit type %s.", + ToTypeName(var.Type()))); } } diff --git a/paddle/fluid/framework/var_type_traits.cc b/paddle/fluid/framework/var_type_traits.cc index 5c90b07149ec5575f9907e41cc65a826421cf3ec..1e5e8d657556059bae8129e7c0b6ea6b57cbf63f 100644 --- a/paddle/fluid/framework/var_type_traits.cc +++ b/paddle/fluid/framework/var_type_traits.cc @@ -46,12 +46,14 @@ struct VarIdToTypeIndexMapInitializerImpl { static_assert(!std::is_same::value, "Type cannot be void"); constexpr int kId = VarTypeTrait::kId; auto type = std::type_index(typeid(Type)); - PADDLE_ENFORCE(id_to_type->count(kId) == 0, - "Registered duplicate type id %d for type %s", kId, - type.name()); - PADDLE_ENFORCE(type_to_id->count(type) == 0, - "Registered duplicate type_index %s for id %d", type.name(), - kId); + PADDLE_ENFORCE_EQ( + id_to_type->count(kId), 0, + platform::errors::AlreadyExists( + "Registered duplicate type id %d for type %s.", kId, type.name())); + PADDLE_ENFORCE_EQ( + type_to_id->count(type), 0, + platform::errors::AlreadyExists( + "Registered duplicate type index %s for id %d.", type.name(), kId)); id_to_type->emplace(kId, type); type_to_id->emplace(type, kId); VarIdToTypeIndexMapInitializerImplsecond; } static int ToTypeId(const std::type_index &type) { auto it = Instance().type_to_id_map_.find(type); - PADDLE_ENFORCE(it != Instance().type_to_id_map_.end(), - "VarType %s is not registered.", type.name()); + PADDLE_ENFORCE_NE(it, Instance().type_to_id_map_.end(), + platform::errors::NotFound( + "Variable Type %s is not registered.", type.name())); return it->second; } diff --git a/paddle/fluid/framework/variable_helper.cc b/paddle/fluid/framework/variable_helper.cc index 67e17410a29aff435921f46eeb2691a025d5a9eb..ec42aa30e5abb3dc3d03633cae31d95999d82731 100644 --- a/paddle/fluid/framework/variable_helper.cc +++ b/paddle/fluid/framework/variable_helper.cc @@ -50,11 +50,11 @@ void InitializeVariable(Variable *var, proto::VarType::Type var_type) { } else if (var_type == proto::VarType::RAW) { // GetMutable will be called in operator } else { - PADDLE_THROW( + PADDLE_THROW(platform::errors::Unavailable( "Variable type %d is not in " "[LOD_TENSOR, SELECTED_ROWS, FEED_MINIBATCH, FETCH_LIST, " - "LOD_RANK_TABLE, PLACE_LIST, READER, RAW]", - var_type); + "LOD_RANK_TABLE, PLACE_LIST, READER, RAW].", + var_type)); } } @@ -76,7 +76,8 @@ void CopyVariable(const Variable &src_var, Variable *dst_var) { auto *dst_t = tmp_grad_slr->mutable_value(); framework::TensorCopy(src_t, cpu_place, dst_t); } else { - PADDLE_THROW("unknown var type to copy"); + PADDLE_THROW( + platform::errors::Unavailable("Unknown variable type to copy.")); } } diff --git a/paddle/fluid/inference/analysis/argument.h b/paddle/fluid/inference/analysis/argument.h index 27bae7a71ea192ac08e4e87cb7bcdb8b84e29dc8..8d28b8ace26ae51b8fb6b3dcb240c08b1686b143 100644 --- a/paddle/fluid/inference/analysis/argument.h +++ b/paddle/fluid/inference/analysis/argument.h @@ -218,6 +218,10 @@ struct Argument { DECL_ARGUMENT_FIELD(fusion_statis, FusionStatis, fusion_statis_t); + // Only used in paddle-lite subgraph. + DECL_ARGUMENT_FIELD(cpu_math_library_num_threads, CpuMathLibraryNumThreads, + int); + private: std::unordered_set valid_fields_; }; diff --git a/paddle/fluid/inference/analysis/ir_pass_manager.cc b/paddle/fluid/inference/analysis/ir_pass_manager.cc index cd8d86d72938417112e17e86e5cc6dd12254a8d1..d52d71f148c36fa456aaa703c0df2dbccd901205 100644 --- a/paddle/fluid/inference/analysis/ir_pass_manager.cc +++ b/paddle/fluid/inference/analysis/ir_pass_manager.cc @@ -150,6 +150,8 @@ void IRPassManager::CreatePasses(Argument *argument, pass->Set("use_xpu", new bool(argument->use_xpu())); pass->Set("xpu_l3_workspace_size", new int(argument->xpu_l3_workspace_size())); + pass->Set("cpu_math_library_num_threads", + new int(argument->cpu_math_library_num_threads())); } disable_logs_ = argument->disable_logs(); if (pass_name == "fc_fuse_pass") { diff --git a/paddle/fluid/inference/analysis/ir_passes/lite_subgraph_pass.cc b/paddle/fluid/inference/analysis/ir_passes/lite_subgraph_pass.cc index 6b16a481ddedbad0956d1358de95842ea9a3a101..e78d5ef017b7f8451556d388bf3b8c0a55276a59 100644 --- a/paddle/fluid/inference/analysis/ir_passes/lite_subgraph_pass.cc +++ b/paddle/fluid/inference/analysis/ir_passes/lite_subgraph_pass.cc @@ -244,6 +244,7 @@ void LiteSubgraphPass::SetUpEngine( bool enable_int8 = Get("enable_int8"); bool use_xpu = Get("use_xpu"); int xpu_l3_workspace_size = Get("xpu_l3_workspace_size"); + int cpu_math_library_num_threads = Get("cpu_math_library_num_threads"); lite_api::TargetType target_type; if (use_gpu) { @@ -263,11 +264,12 @@ void LiteSubgraphPass::SetUpEngine( // Notice: The ordering here determines the device where the // input tensor of the Lite engine is located, and then affects // whether tensor sharing is feasible. - paddle::lite::Place({target_type, precision_type}), - paddle::lite::Place({target_type, PRECISION(kInt64)}), - paddle::lite::Place({target_type, PRECISION(kFloat)}), - paddle::lite::Place({TARGET(kHost), PRECISION(kFloat)}), + paddle::lite_api::Place({target_type, precision_type}), + paddle::lite_api::Place({target_type, PRECISION(kInt64)}), + paddle::lite_api::Place({target_type, PRECISION(kFloat)}), + paddle::lite_api::Place({TARGET(kHost), PRECISION(kFloat)}), }; + config.cpu_math_library_num_threads = cpu_math_library_num_threads; config.xpu_l3_workspace_size = xpu_l3_workspace_size; if (dump_model) { lite::StrToBinaryFile("./model.bin", config.model); diff --git a/paddle/fluid/inference/api/CMakeLists.txt b/paddle/fluid/inference/api/CMakeLists.txt index fb0ad31a3e612201de32813a65970c73b73b611b..c0d3b14e0e43e10332d18ddd217a8a50245ab5ed 100755 --- a/paddle/fluid/inference/api/CMakeLists.txt +++ b/paddle/fluid/inference/api/CMakeLists.txt @@ -53,12 +53,10 @@ if(WITH_TESTING) inference_base_test(test_api_impl SRCS api_impl_tester.cc DEPS paddle_fluid_shared ARGS --word2vec_dirname=${WORD2VEC_MODEL_DIR} --book_dirname=${PYTHON_TESTS_DIR}/book) set_tests_properties(test_api_impl PROPERTIES DEPENDS test_image_classification) - set_tests_properties(test_api_impl PROPERTIES LABELS "RUN_TYPE=DIST") elseif(WIN32) inference_base_test(test_api_impl SRCS api_impl_tester.cc DEPS ${inference_deps} ARGS --word2vec_dirname=${WORD2VEC_MODEL_DIR} --book_dirname=${PYTHON_TESTS_DIR}/book) set_tests_properties(test_api_impl PROPERTIES DEPENDS test_image_classification) - set_tests_properties(test_api_impl PROPERTIES LABELS "RUN_TYPE=DIST") endif() endif() diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc index 500aa8341d6a61056f6f80f82c6f28bb569eb772..64dfdda54aceefef1d89ccb2e3a917ad47c53966 100644 --- a/paddle/fluid/inference/api/analysis_predictor.cc +++ b/paddle/fluid/inference/api/analysis_predictor.cc @@ -461,6 +461,8 @@ void AnalysisPredictor::PrepareArgument() { } if (config_.lite_engine_enabled()) { + argument_.SetCpuMathLibraryNumThreads( + config_.cpu_math_library_num_threads()); argument_.SetLitePrecisionMode(config_.lite_precision_mode_); argument_.SetLitePassesFilter(config_.lite_passes_filter_); argument_.SetLiteOpsFilter(config_.lite_ops_filter_); diff --git a/paddle/fluid/inference/api/details/zero_copy_tensor.cc b/paddle/fluid/inference/api/details/zero_copy_tensor.cc index a5a0405ac88ad8e94a65d728557ab9298eae56dc..46755eeda660ae8f4c54d318f6450fbf1d48b1f7 100644 --- a/paddle/fluid/inference/api/details/zero_copy_tensor.cc +++ b/paddle/fluid/inference/api/details/zero_copy_tensor.cc @@ -21,15 +21,21 @@ namespace paddle { void ZeroCopyTensor::Reshape(const std::vector &shape) { - PADDLE_ENFORCE(!name_.empty(), - "Need to SetName first, so that the corresponding tensor can " - "be retrieved."); - PADDLE_ENFORCE(input_or_output_, - "Can't reshape the output tensor, it is readonly"); - PADDLE_ENFORCE(scope_); + PADDLE_ENFORCE_EQ( + name_.empty(), false, + platform::errors::PreconditionNotMet( + "Need to SetName first, so that the corresponding tensor can " + "be retrieved.")); + PADDLE_ENFORCE_EQ(input_or_output_, true, + platform::errors::PermissionDenied( + "Can't reshape the output tensor, it is readonly")); + PADDLE_ENFORCE_NOT_NULL(scope_, platform::errors::PreconditionNotMet( + "The scope should not be nullptr.")); auto *scope = static_cast(scope_); auto *var = scope->FindVar(name_); - PADDLE_ENFORCE(var, "No tensor called [%s] in the runtime scope", name_); + PADDLE_ENFORCE_NOT_NULL( + var, platform::errors::PreconditionNotMet( + "No tensor called [%s] in the runtime scope", name_)); auto *tensor = var->GetMutable(); tensor->Resize(framework::make_ddim(shape)); } @@ -45,8 +51,10 @@ T *ZeroCopyTensor::mutable_data(PaddlePlace place) { EAGER_GET_TENSOR; PADDLE_ENFORCE_GT( tensor->numel(), 0, - "You should call ZeroCopyTensor::Reshape(const std::vector &shape)" - "function before retrieving mutable_data from input tensor."); + platform::errors::PreconditionNotMet( + "You should call ZeroCopyTensor::Reshape(const std::vector " + "&shape)" + "function before retrieving mutable_data from input tensor.")); switch (static_cast(place)) { case static_cast(PaddlePlace::kCPU): { return tensor->mutable_data(platform::CPUPlace()); @@ -55,7 +63,8 @@ T *ZeroCopyTensor::mutable_data(PaddlePlace place) { return tensor->mutable_data(platform::CUDAPlace(device_)); } default: - PADDLE_THROW("Unsupported place: %d", static_cast(place)); + PADDLE_THROW(platform::errors::Unavailable("Unsupported place: %d", + static_cast(place))); break; } return nullptr; @@ -96,10 +105,11 @@ PaddleDType ZeroCopyTensor::type() const { template void ZeroCopyTensor::copy_from_cpu(const T *data) { EAGER_GET_TENSOR; - PADDLE_ENFORCE_GE( - tensor->numel(), 0, - "You should call ZeroCopyTensor::Reshape(const std::vector &shape)" - "function before copying data from cpu."); + PADDLE_ENFORCE_GE(tensor->numel(), 0, + platform::errors::PreconditionNotMet( + "You should call ZeroCopyTensor::Reshape(const " + "std::vector &shape)" + "function before copying data from cpu.")); size_t ele_size = tensor->numel() * sizeof(T); if (place_ == PaddlePlace::kCPU) { @@ -116,7 +126,8 @@ void ZeroCopyTensor::copy_from_cpu(const T *data) { memory::Copy(gpu_place, static_cast(t_data), platform::CPUPlace(), data, ele_size, dev_ctx->stream()); #else - PADDLE_THROW("Not compiled with CUDA, should not reach here."); + PADDLE_THROW(platform::errors::Unavailable( + "Not compiled with CUDA, should not reach here.")); #endif } } @@ -141,7 +152,8 @@ void ZeroCopyTensor::copy_to_cpu(T *data) { cudaStreamSynchronize(dev_ctx->stream()); #else - PADDLE_THROW("Not compile with CUDA, should not reach here."); + PADDLE_THROW(platform::errors::Unavailable( + "Not compile with CUDA, should not reach here.")); #endif } } @@ -176,20 +188,27 @@ template PD_INFER_DECL uint8_t *ZeroCopyTensor::mutable_data( PaddlePlace place); void *ZeroCopyTensor::FindTensor() const { - PADDLE_ENFORCE(!name_.empty(), - "Need to SetName first, so that the corresponding tensor can " - "be retrieved."); - PADDLE_ENFORCE(scope_); + PADDLE_ENFORCE_EQ( + name_.empty(), false, + platform::errors::PreconditionNotMet( + "Need to SetName first, so that the corresponding tensor can " + "be retrieved.")); + PADDLE_ENFORCE_NOT_NULL(scope_, platform::errors::PreconditionNotMet( + "The scope should not be nullptr.")); auto *scope = static_cast(scope_); auto *var = scope->FindVar(name_); - PADDLE_ENFORCE(var, "No tensor called [%s] in the runtime scope", name_); + PADDLE_ENFORCE_NOT_NULL( + var, platform::errors::PreconditionNotMet( + "No tensor called [%s] in the runtime scope", name_)); auto *tensor = var->GetMutable(); return tensor; } std::vector ZeroCopyTensor::shape() const { EAGER_GET_TENSOR; - PADDLE_ENFORCE(tensor_, "not found tensor called %s in the scope", name_); + PADDLE_ENFORCE_NOT_NULL( + tensor_, platform::errors::PreconditionNotMet( + "Not found tensor called %s in the scope", name_)); return framework::vectorize(tensor->dims()); } diff --git a/paddle/fluid/inference/api/paddle_inference_api.h b/paddle/fluid/inference/api/paddle_inference_api.h index da5d7411693c92eaa2066c7f76d56970f8939bc7..5dc4430fde4715fe11c19ce8adc7397f77391fc3 100644 --- a/paddle/fluid/inference/api/paddle_inference_api.h +++ b/paddle/fluid/inference/api/paddle_inference_api.h @@ -31,12 +31,30 @@ limitations under the License. */ #include "paddle_analysis_config.h" // NOLINT #include "paddle_api.h" // NOLINT +/// +/// \file paddle_inference_api.h +/// +/// \brief Paddle Inference API +/// +/// \author paddle-infer@baidu.com +/// \date 2020-09-01 +/// \since 2.0.0-beta +/// + namespace paddle_infer { using DataType = paddle::PaddleDType; using PlaceType = paddle::PaddlePlace; using PrecisionType = paddle::AnalysisConfig::Precision; using Config = paddle::AnalysisConfig; +/// +/// \class Tensor +/// +/// \brief Represents an n-dimensional array of values. +/// The Tensor is used to store the input or output of the network. +/// It is obtained through Predictor::GetinputHandle() +/// and Predictor::GetOutputHandle() interface. +/// class PD_INFER_DECL Tensor { public: // Can only be created by predictor->GetInputHandle(cosnt std::string& name) @@ -44,60 +62,186 @@ class PD_INFER_DECL Tensor { Tensor() = delete; explicit Tensor(std::unique_ptr&& tensor) : tensor_(std::move(tensor)) {} + + /// + /// \brief Reset the shape of the tensor. + /// Generally it's only used for the input tensor. + /// Reshape must be called before calling mutable_data() or CopyFromCpu() + /// \param shape The shape to set. + /// void Reshape(const std::vector& shape); + /// + /// \brief Copy the host memory to tensor data. + /// It's usually used to set the input tensor data. + /// \param data The pointer of the data, from which the tensor will copy. + /// template void CopyFromCpu(const T* data); - // should add the place + /// + /// \brief Get the memory pointer in CPU or GPU with specific data type. + /// Please Reshape the tensor first before call this. + /// It's usually used to get input data pointer. + /// \param place The place of the tensor. + /// \return The tensor data buffer pointer. + /// template T* mutable_data(PlaceType place); + /// + /// \brief Copy the tensor data to the host memory. + /// It's usually used to get the output tensor data. + /// \param[out] data The tensor will copy the data to the address. + /// template void CopyToCpu(T* data); + /// + /// \brief Get the memory pointer directly. + /// It's usually used to get the output data pointer. + /// \param[out] place To get the device type of the tensor. + /// \param[out] size To get the data size of the tensor. + /// \return The tensor data buffer pointer. + /// template T* data(PlaceType* place, int* size) const; + /// + /// \brief Set lod info of the tensor. + /// More about LOD can be seen here: + /// https://www.paddlepaddle.org.cn/documentation/docs/zh/beginners_guide/basic_concept/lod_tensor.html#lodtensor + /// \param x the lod info. + /// void SetLoD(const std::vector>& x); + + /// \brief Return the lod info of the tensor. std::vector> lod() const; + /// \brief Return the data type of the tensor. + /// It's usually used to get the output tensor data type. + /// \return The data type of the tensor. DataType type() const; + /// \brief Return the shape of the Tensor. std::vector shape() const; + + /// \brief Return the name of the tensor. const std::string& name() const; private: std::unique_ptr tensor_; }; +/// +/// \class Predictor +/// +/// \brief Predictor is the interface for model prediction. +/// +/// The predictor has the following typical uses: +/// +/// Get predictor +/// \code{cpp} +/// auto predictor = CreatePredictor(config); +/// \endcode +/// +/// Get input or output names +/// \code{cpp} +/// auto input_names = predictor->GetInputNames(); +/// auto output_names = predictor->GetOutputNames(); +/// \endcode +/// +/// Get input or output handle +/// \code{cpp} +/// auto input_t = predictor->GetInputHandle(input_names[0]); +/// auto output_t = predictor->GetOutputHandle(output_names[0]); +/// \endcode +/// +/// Run predictor +/// \code{cpp} +/// predictor->Run(); +/// \endcode +/// class PD_INFER_DECL Predictor { public: - Predictor() = default; + Predictor() = delete; ~Predictor() {} // Use for clone explicit Predictor(std::unique_ptr&& pred) : predictor_(std::move(pred)) {} + /// + /// \brief Construct a new Predictor object + /// + /// \param[in] Config config + /// explicit Predictor(const Config& config); + /// + /// \brief Get the input names + /// + /// \return input names + /// std::vector GetInputNames(); + + /// + /// \brief Get the Input Tensor object + /// + /// \param[in] name input name + /// \return input tensor + /// std::unique_ptr GetInputHandle(const std::string& name); + /// + /// \brief Run the prediction engine + /// + /// \return Whether the function executed successfully + /// bool Run(); + /// + /// \brief Get the output names + /// + /// \return output names + /// std::vector GetOutputNames(); + + /// + /// \brief Get the Output Tensor object + /// + /// \param[in] name otuput name + /// \return output tensor + /// std::unique_ptr GetOutputHandle(const std::string& name); + /// + /// \brief Clone to get the new predictor. thread safe. + /// + /// \return get a new predictor + /// std::unique_ptr Clone(); + + /// \brief Clear the intermediate tensors of the predictor void ClearIntermediateTensor(); private: std::unique_ptr predictor_; }; +/// +/// \brief A factory to help create predictors. +/// +/// Usage: +/// +/// \code{.cpp} +/// Config config; +/// ... // change the configs. +/// auto predictor = CreatePredictor(config); +/// \endcode +/// PD_INFER_DECL std::shared_ptr CreatePredictor( const Config& config); // NOLINT + PD_INFER_DECL int GetNumBytesOfDataType(DataType dtype); PD_INFER_DECL std::string GetVersion(); @@ -128,13 +272,24 @@ T* Tensor::data(PlaceType* place, int* size) const { namespace paddle_infer { namespace services { +/// +/// \class PredictorPool +/// +/// \brief PredictorPool is a simple encapsulation of Predictor, suitable for +/// use in multi-threaded situations. According to the thread id, the +/// corresponding Predictor is taken out from PredictorPool to complete the +/// prediction. +/// class PD_INFER_DECL PredictorPool { public: PredictorPool() = delete; PredictorPool(const PredictorPool&) = delete; PredictorPool& operator=(const PredictorPool&) = delete; + /// \brief Construct the predictor pool with \param size predictor instances. explicit PredictorPool(const Config& config, size_t size = 1); + + /// \brief Get \param id-th predictor. Predictor* Retrive(size_t idx); private: diff --git a/paddle/fluid/inference/capi/c_api.cc b/paddle/fluid/inference/capi/c_api.cc index 821dff2f036c1892570a8ade5b40363251c7f531..07493c742c4fa906e7c4817e328e7d4f81afbffa 100644 --- a/paddle/fluid/inference/capi/c_api.cc +++ b/paddle/fluid/inference/capi/c_api.cc @@ -16,6 +16,7 @@ #include #include "paddle/fluid/inference/capi/c_api_internal.h" #include "paddle/fluid/inference/capi/paddle_c_api.h" +#include "paddle/fluid/platform/enforce.h" using paddle::ConvertToACPrecision; using paddle::ConvertToPaddleDType; @@ -34,27 +35,37 @@ void PD_DeletePaddleBuf(PD_PaddleBuf* buf) { } void PD_PaddleBufResize(PD_PaddleBuf* buf, size_t length) { - PADDLE_ENFORCE_NOT_NULL(buf); + PADDLE_ENFORCE_NOT_NULL(buf, + paddle::platform::errors::InvalidArgument( + "The pointer of Buffer shouldn't be nullptr")); buf->buf.Resize(length); } void PD_PaddleBufReset(PD_PaddleBuf* buf, void* data, size_t length) { - PADDLE_ENFORCE_NOT_NULL(buf); + PADDLE_ENFORCE_NOT_NULL(buf, + paddle::platform::errors::InvalidArgument( + "The pointer of Buffer shouldn't be nullptr")); buf->buf.Reset(data, length); } bool PD_PaddleBufEmpty(PD_PaddleBuf* buf) { - PADDLE_ENFORCE_NOT_NULL(buf); + PADDLE_ENFORCE_NOT_NULL(buf, + paddle::platform::errors::InvalidArgument( + "The pointer of Buffer shouldn't be nullptr")); return buf->buf.empty(); } void* PD_PaddleBufData(PD_PaddleBuf* buf) { - PADDLE_ENFORCE_NOT_NULL(buf); + PADDLE_ENFORCE_NOT_NULL(buf, + paddle::platform::errors::InvalidArgument( + "The pointer of Buffer shouldn't be nullptr")); return buf->buf.data(); } size_t PD_PaddleBufLength(PD_PaddleBuf* buf) { - PADDLE_ENFORCE_NOT_NULL(buf); + PADDLE_ENFORCE_NOT_NULL(buf, + paddle::platform::errors::InvalidArgument( + "The pointer of Buffer shouldn't be nullptr")); return buf->buf.length(); } diff --git a/paddle/fluid/inference/capi/c_api_internal.h b/paddle/fluid/inference/capi/c_api_internal.h index 2dd827229779d34384df2b3ba5f398c77db8369a..7e69b7210768e5af9e8f4150883a608a1517a13c 100644 --- a/paddle/fluid/inference/capi/c_api_internal.h +++ b/paddle/fluid/inference/capi/c_api_internal.h @@ -18,7 +18,6 @@ #include "paddle/fluid/inference/api/paddle_analysis_config.h" #include "paddle/fluid/inference/api/paddle_api.h" #include "paddle/fluid/inference/capi/paddle_c_api.h" -#include "paddle/fluid/platform/enforce.h" using PD_PaddleDType = paddle::PaddleDType; using PD_ACPrecision = paddle::AnalysisConfig::Precision; diff --git a/paddle/fluid/inference/capi/pd_config.cc b/paddle/fluid/inference/capi/pd_config.cc index b99abc06b27ecb9686b4c6e883aaaf8b3e592415..af8d4a69ecf24862ca5f282655b72ef37307c1c8 100644 --- a/paddle/fluid/inference/capi/pd_config.cc +++ b/paddle/fluid/inference/capi/pd_config.cc @@ -20,6 +20,7 @@ #include #include "paddle/fluid/inference/capi/c_api_internal.h" #include "paddle/fluid/inference/capi/paddle_c_api.h" +#include "paddle/fluid/platform/enforce.h" using paddle::ConvertToACPrecision; using paddle::ConvertToPaddleDType; @@ -40,7 +41,10 @@ void PD_DeleteAnalysisConfig(PD_AnalysisConfig* config) { void PD_SetModel(PD_AnalysisConfig* config, const char* model_dir, const char* params_path) { LOG(INFO) << model_dir; - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); LOG(INFO) << std::string(model_dir); if (!params_path) { config->config.SetModel(std::string(model_dir)); @@ -50,104 +54,164 @@ void PD_SetModel(PD_AnalysisConfig* config, const char* model_dir, } void PD_SetProgFile(PD_AnalysisConfig* config, const char* x) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); config->config.SetProgFile(std::string(x)); } void PD_SetParamsFile(PD_AnalysisConfig* config, const char* x) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); config->config.SetParamsFile(std::string(x)); } void PD_SetOptimCacheDir(PD_AnalysisConfig* config, const char* opt_cache_dir) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); config->config.SetOptimCacheDir(std::string(opt_cache_dir)); } const char* PD_ModelDir(const PD_AnalysisConfig* config) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); return config->config.model_dir().c_str(); } const char* PD_ProgFile(const PD_AnalysisConfig* config) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); return config->config.prog_file().c_str(); } const char* PD_ParamsFile(const PD_AnalysisConfig* config) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); return config->config.params_file().c_str(); } void PD_EnableUseGpu(PD_AnalysisConfig* config, int memory_pool_init_size_mb, int device_id) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); config->config.EnableUseGpu(static_cast(memory_pool_init_size_mb), device_id); } void PD_DisableGpu(PD_AnalysisConfig* config) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); config->config.DisableGpu(); } bool PD_UseGpu(const PD_AnalysisConfig* config) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); return config->config.use_gpu(); } int PD_GpuDeviceId(const PD_AnalysisConfig* config) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); return config->config.gpu_device_id(); } int PD_MemoryPoolInitSizeMb(const PD_AnalysisConfig* config) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); return config->config.memory_pool_init_size_mb(); } float PD_FractionOfGpuMemoryForPool(const PD_AnalysisConfig* config) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); return config->config.fraction_of_gpu_memory_for_pool(); } void PD_EnableCUDNN(PD_AnalysisConfig* config) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); config->config.EnableCUDNN(); } bool PD_CudnnEnabled(const PD_AnalysisConfig* config) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); return config->config.cudnn_enabled(); } void PD_SwitchIrOptim(PD_AnalysisConfig* config, bool x) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); config->config.SwitchIrOptim(x); } bool PD_IrOptim(const PD_AnalysisConfig* config) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); return config->config.ir_optim(); } void PD_SwitchUseFeedFetchOps(PD_AnalysisConfig* config, bool x) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); config->config.SwitchUseFeedFetchOps(x); } bool PD_UseFeedFetchOpsEnabled(const PD_AnalysisConfig* config) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); return config->config.use_feed_fetch_ops_enabled(); } void PD_SwitchSpecifyInputNames(PD_AnalysisConfig* config, bool x) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); config->config.SwitchSpecifyInputNames(x); } bool PD_SpecifyInputName(const PD_AnalysisConfig* config) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); return config->config.specify_input_name(); } @@ -155,110 +219,168 @@ void PD_EnableTensorRtEngine(PD_AnalysisConfig* config, int workspace_size, int max_batch_size, int min_subgraph_size, Precision precision, bool use_static, bool use_calib_mode) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); config->config.EnableTensorRtEngine( workspace_size, max_batch_size, min_subgraph_size, paddle::ConvertToACPrecision(precision), use_static, use_calib_mode); } bool PD_TensorrtEngineEnabled(const PD_AnalysisConfig* config) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); return config->config.tensorrt_engine_enabled(); } void PD_SwitchIrDebug(PD_AnalysisConfig* config, bool x) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); config->config.SwitchIrDebug(x); } void PD_EnableMKLDNN(PD_AnalysisConfig* config) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); config->config.EnableMKLDNN(); } void PD_SetMkldnnCacheCapacity(PD_AnalysisConfig* config, int capacity) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); config->config.SetMkldnnCacheCapacity(capacity); } bool PD_MkldnnEnabled(const PD_AnalysisConfig* config) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); return config->config.mkldnn_enabled(); } void PD_SetCpuMathLibraryNumThreads(PD_AnalysisConfig* config, int cpu_math_library_num_threads) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); config->config.SetCpuMathLibraryNumThreads(cpu_math_library_num_threads); } int PD_CpuMathLibraryNumThreads(const PD_AnalysisConfig* config) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); return config->config.cpu_math_library_num_threads(); } void PD_EnableMkldnnQuantizer(PD_AnalysisConfig* config) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); config->config.EnableMkldnnQuantizer(); } bool PD_MkldnnQuantizerEnabled(const PD_AnalysisConfig* config) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); return config->config.mkldnn_quantizer_enabled(); } void PD_EnableMkldnnBfloat16(PD_AnalysisConfig* config) { - PADDLE_ENFORCE_NOT_NULL(config, paddle::platform::errors::NotFound( - "PD_AnalysisConfig should not be null")); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); config->config.EnableMkldnnBfloat16(); } bool PD_MkldnnBfloat16Enabled(const PD_AnalysisConfig* config) { - PADDLE_ENFORCE_NOT_NULL(config, paddle::platform::errors::NotFound( - "PD_AnalysisConfig should not be null")); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); return config->config.mkldnn_bfloat16_enabled(); } void PD_SetModelBuffer(PD_AnalysisConfig* config, const char* prog_buffer, size_t prog_buffer_size, const char* params_buffer, size_t params_buffer_size) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); config->config.SetModelBuffer(prog_buffer, prog_buffer_size, params_buffer, params_buffer_size); } bool PD_ModelFromMemory(const PD_AnalysisConfig* config) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); return config->config.model_from_memory(); } void PD_EnableMemoryOptim(PD_AnalysisConfig* config) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); config->config.EnableMemoryOptim(); } bool PD_MemoryOptimEnabled(const PD_AnalysisConfig* config) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); return config->config.enable_memory_optim(); } void PD_EnableProfile(PD_AnalysisConfig* config) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); config->config.EnableProfile(); } bool PD_ProfileEnabled(const PD_AnalysisConfig* config) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); return config->config.profile_enabled(); } void PD_SetInValid(PD_AnalysisConfig* config) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); config->config.SetInValid(); } bool PD_IsValid(const PD_AnalysisConfig* config) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); return config->config.is_valid(); } diff --git a/paddle/fluid/inference/capi/pd_predictor.cc b/paddle/fluid/inference/capi/pd_predictor.cc index 8aa1e2a7b7f9b99a1636ca2e7396089ab2ae7e15..0509a6190211c25b6461c1d683daa6b33110b4e0 100644 --- a/paddle/fluid/inference/capi/pd_predictor.cc +++ b/paddle/fluid/inference/capi/pd_predictor.cc @@ -22,6 +22,7 @@ #include "paddle/fluid/inference/api/paddle_api.h" #include "paddle/fluid/inference/capi/c_api_internal.h" #include "paddle/fluid/inference/capi/paddle_c_api.h" +#include "paddle/fluid/platform/enforce.h" using paddle::ConvertToACPrecision; using paddle::ConvertToPaddleDType; @@ -81,7 +82,10 @@ extern "C" { bool PD_PredictorRun(const PD_AnalysisConfig* config, PD_Tensor* inputs, int in_size, PD_Tensor** output_data, int* out_size, int batch_size) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); VLOG(3) << "Predoctor: PD_PredictorRun. "; static std::map> predictors; @@ -111,7 +115,10 @@ bool PD_PredictorRun(const PD_AnalysisConfig* config, PD_Tensor* inputs, bool PD_PredictorZeroCopyRun(const PD_AnalysisConfig* config, PD_ZeroCopyData* inputs, int in_size, PD_ZeroCopyData** output, int* out_size) { - PADDLE_ENFORCE_NOT_NULL(config); + PADDLE_ENFORCE_NOT_NULL( + config, + paddle::platform::errors::InvalidArgument( + "The pointer of analysis configuration shouldn't be nullptr")); static std::map> predictors; if (!predictors.count(config->config.model_dir())) { @@ -144,7 +151,8 @@ bool PD_PredictorZeroCopyRun(const PD_AnalysisConfig* config, input_t->copy_from_cpu(static_cast(inputs[i].data)); break; default: - CHECK(false) << "Unsupport data type."; + PADDLE_THROW(paddle::platform::errors::InvalidArgument( + "Unsupported data type.")); break; } } @@ -227,7 +235,8 @@ void PD_SetZeroCopyInput(PD_Predictor* predictor, input->copy_from_cpu(static_cast(tensor->data.data)); break; default: - CHECK(false) << "Unsupport data type."; + PADDLE_THROW( + paddle::platform::errors::InvalidArgument("Unsupported data type.")); break; } @@ -294,7 +303,8 @@ void PD_GetZeroCopyOutput(PD_Predictor* predictor, PD_ZeroCopyTensor* tensor) { output->copy_to_cpu(reinterpret_cast(tensor->data.data)); break; default: - CHECK(false) << "Unsupport data type."; + PADDLE_THROW( + paddle::platform::errors::InvalidArgument("Unsupported data type.")); break; } } diff --git a/paddle/fluid/inference/capi/pd_tensor.cc b/paddle/fluid/inference/capi/pd_tensor.cc index b4811f1d6ff192659fa12b33008fe5ac07e6a6c5..9b1eedd7c5a8106a6f6b7be3f682913e2431a3e5 100644 --- a/paddle/fluid/inference/capi/pd_tensor.cc +++ b/paddle/fluid/inference/capi/pd_tensor.cc @@ -19,6 +19,7 @@ #include #include "paddle/fluid/inference/capi/c_api_internal.h" #include "paddle/fluid/inference/capi/paddle_c_api.h" +#include "paddle/fluid/platform/enforce.h" using paddle::ConvertToACPrecision; using paddle::ConvertToPaddleDType; @@ -37,44 +38,60 @@ void PD_DeletePaddleTensor(PD_Tensor* tensor) { } void PD_SetPaddleTensorName(PD_Tensor* tensor, char* name) { - PADDLE_ENFORCE_NOT_NULL(tensor); + PADDLE_ENFORCE_NOT_NULL(tensor, + paddle::platform::errors::InvalidArgument( + "The pointer of tensor shouldn't be nullptr")); tensor->tensor.name = std::string(name); } void PD_SetPaddleTensorDType(PD_Tensor* tensor, PD_DataType dtype) { - PADDLE_ENFORCE_NOT_NULL(tensor); + PADDLE_ENFORCE_NOT_NULL(tensor, + paddle::platform::errors::InvalidArgument( + "The pointer of tensor shouldn't be nullptr")); tensor->tensor.dtype = paddle::ConvertToPaddleDType(dtype); } void PD_SetPaddleTensorData(PD_Tensor* tensor, PD_PaddleBuf* buf) { - PADDLE_ENFORCE_NOT_NULL(tensor); + PADDLE_ENFORCE_NOT_NULL(tensor, + paddle::platform::errors::InvalidArgument( + "The pointer of tensor shouldn't be nullptr")); tensor->tensor.data = buf->buf; } void PD_SetPaddleTensorShape(PD_Tensor* tensor, int* shape, int size) { - PADDLE_ENFORCE_NOT_NULL(tensor); + PADDLE_ENFORCE_NOT_NULL(tensor, + paddle::platform::errors::InvalidArgument( + "The pointer of tensor shouldn't be nullptr")); tensor->tensor.shape.assign(shape, shape + size); } const char* PD_GetPaddleTensorName(const PD_Tensor* tensor) { - PADDLE_ENFORCE_NOT_NULL(tensor); + PADDLE_ENFORCE_NOT_NULL(tensor, + paddle::platform::errors::InvalidArgument( + "The pointer of tensor shouldn't be nullptr")); return tensor->tensor.name.c_str(); } PD_DataType PD_GetPaddleTensorDType(const PD_Tensor* tensor) { - PADDLE_ENFORCE_NOT_NULL(tensor); + PADDLE_ENFORCE_NOT_NULL(tensor, + paddle::platform::errors::InvalidArgument( + "The pointer of tensor shouldn't be nullptr")); return ConvertToPDDataType(tensor->tensor.dtype); } PD_PaddleBuf* PD_GetPaddleTensorData(const PD_Tensor* tensor) { - PADDLE_ENFORCE_NOT_NULL(tensor); + PADDLE_ENFORCE_NOT_NULL(tensor, + paddle::platform::errors::InvalidArgument( + "The pointer of tensor shouldn't be nullptr")); PD_PaddleBuf* ret = PD_NewPaddleBuf(); ret->buf = tensor->tensor.data; return ret; } const int* PD_GetPaddleTensorShape(const PD_Tensor* tensor, int* size) { - PADDLE_ENFORCE_NOT_NULL(tensor); + PADDLE_ENFORCE_NOT_NULL(tensor, + paddle::platform::errors::InvalidArgument( + "The pointer of tensor shouldn't be nullptr")); const std::vector& shape = tensor->tensor.shape; *size = shape.size(); return shape.data(); diff --git a/paddle/fluid/inference/lite/engine.cc b/paddle/fluid/inference/lite/engine.cc index 8e88c94493952ff257ef69bf73f8edebb6ba2eee..5f24ef00bce59e5886d8448cf3f8356e9aeba481 100644 --- a/paddle/fluid/inference/lite/engine.cc +++ b/paddle/fluid/inference/lite/engine.cc @@ -20,8 +20,12 @@ #define LITE_WITH_XPU 1 #endif +#ifndef PADDLE_WITH_ARM +#define LITE_WITH_X86 1 +#endif + #include "paddle/fluid/inference/lite/engine.h" -#include "lite/api/paddle_use_passes.h" +#include namespace paddle { namespace inference { @@ -36,32 +40,40 @@ bool EngineManager::Has(const std::string& name) const { return engines_.at(name).get() != nullptr; } -paddle::lite::Predictor* EngineManager::Get(const std::string& name) const { +paddle::lite_api::PaddlePredictor* EngineManager::Get( + const std::string& name) const { return engines_.at(name).get(); } -paddle::lite::Predictor* EngineManager::Create(const std::string& name, - const EngineConfig& cfg) { - if (cfg.valid_places.front().target == TARGET(kCUDA)) { -#ifdef PADDLE_WITH_CUDA - paddle::lite::Env::Init(); +paddle::lite_api::PaddlePredictor* EngineManager::Create( + const std::string& name, const EngineConfig& cfg) { + // config info for predictor. + paddle::lite_api::CxxConfig lite_cxx_config; + lite_cxx_config.set_model_buffer(cfg.model.c_str(), cfg.model.size(), + cfg.param.c_str(), cfg.param.size()); + lite_cxx_config.set_valid_places(cfg.valid_places); +#ifdef PADDLE_WITH_ARM + set_threads.set_threads(cfg.cpu_math_library_num_threads); +#else + lite_cxx_config.set_x86_math_library_num_threads( + cfg.cpu_math_library_num_threads); #endif - } else if (cfg.valid_places.front().target == TARGET(kXPU)) { + #ifdef PADDLE_WITH_XPU - paddle::lite::TargetWrapper::workspace_l3_size_per_thread = - cfg.xpu_l3_workspace_size; + lite_cxx_config.set_xpu_workspace_l3_size_per_thread( + cfg.xpu_l3_workspace_size); #endif - } - auto* p = new paddle::lite::Predictor(); - p->Build("", cfg.model, cfg.param, cfg.valid_places, cfg.neglected_passes, - cfg.model_type, cfg.model_from_memory); - engines_[name].reset(p); - return p; + + // create predictor + std::shared_ptr p = + paddle::lite_api::CreatePaddlePredictor(lite_cxx_config); + engines_[name] = std::move(p); + return engines_[name].get(); } void EngineManager::DeleteAll() { for (auto& item : engines_) { - item.second.reset(nullptr); + item.second.reset(); } } diff --git a/paddle/fluid/inference/lite/engine.h b/paddle/fluid/inference/lite/engine.h index 345eb682e9fe81d4ec67a31082c1d347a694fd96..5ba487cc24d7d58cd87853a58fc12f1a82c3610d 100644 --- a/paddle/fluid/inference/lite/engine.h +++ b/paddle/fluid/inference/lite/engine.h @@ -23,12 +23,9 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wall" #include "lite/api/cxx_api.h" +#include "lite/api/paddle_api.h" #include "lite/api/paddle_place.h" -#include "lite/core/context.h" -#include "lite/core/device_info.h" -#include "lite/core/memory.h" -#include "lite/core/op_registry.h" -#include "lite/core/tensor.h" +#include "lite/api/paddle_use_passes.h" #pragma GCC diagnostic pop namespace paddle { @@ -38,25 +35,33 @@ namespace lite { struct EngineConfig { std::string model; std::string param; - paddle::lite::Place prefer_place; - std::vector valid_places; + std::vector valid_places; std::vector neglected_passes; lite_api::LiteModelType model_type{lite_api::LiteModelType::kProtobuf}; bool model_from_memory{true}; + + // for xpu size_t xpu_l3_workspace_size; + + // for x86 or arm + int cpu_math_library_num_threads{1}; + + // for cuda + bool use_multi_stream{false}; }; class EngineManager { public: bool Empty() const; bool Has(const std::string& name) const; - paddle::lite::Predictor* Get(const std::string& name) const; - paddle::lite::Predictor* Create(const std::string& name, - const EngineConfig& cfg); + paddle::lite_api::PaddlePredictor* Get(const std::string& name) const; + paddle::lite_api::PaddlePredictor* Create(const std::string& name, + const EngineConfig& cfg); void DeleteAll(); private: - std::unordered_map> + std::unordered_map> engines_; }; diff --git a/paddle/fluid/inference/lite/tensor_utils.cc b/paddle/fluid/inference/lite/tensor_utils.cc index d79a041ccf8a1611247b65b034c03940eabfcccd..33661594b926f284052c85c6a816a17dfff1ce20 100644 --- a/paddle/fluid/inference/lite/tensor_utils.cc +++ b/paddle/fluid/inference/lite/tensor_utils.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "paddle/fluid/inference/lite/tensor_utils.h" +#include #include #include #include "paddle/fluid/framework/data_type.h" @@ -144,16 +145,55 @@ void MemoryCopyAsync(const platform::Place& dst_place, void* dst_data, } } -void InitDstTensor(paddle::lite::Tensor* dst, const framework::LoDTensor& src) { +void* GetLiteTensorDataPtr(paddle::lite_api::Tensor* src, + PrecisionType precision_type, + TargetType target_type) { + void* res{nullptr}; + switch (precision_type) { + case PrecisionType::kFloat: + res = static_cast(src->mutable_data(target_type)); + break; + case PrecisionType::kInt8: + res = static_cast(src->mutable_data(target_type)); + break; + case PrecisionType::kInt32: + res = static_cast(src->mutable_data(target_type)); + break; + case PrecisionType::kInt64: + res = static_cast(src->mutable_data(target_type)); + break; + default: + PADDLE_THROW(platform::errors::Unimplemented( + "Unsupported precision type. Now only supports FP32, INT8, INT32 and " + "INT64.")); + break; + } + return res; +} + +int64_t GetLiteTensorNumel(const paddle::lite_api::Tensor& tensor) { + auto shape = tensor.shape(); + int64_t numel = std::accumulate(shape.begin(), shape.end(), 1, + std::multiplies()); + return numel; +} + +void InitDstTensor(paddle::lite_api::Tensor* dst, + const framework::LoDTensor& src) { // Currently, Lite needs to explicitly specify the target type of // the input tensor. constexpr int empty_size = 0; - dst->mutable_data(GetLiteTargetType(src.place()), empty_size); - dst->set_precision(GetLitePrecisionType(src.type())); - SetLoD(dst->mutable_lod(), src.lod()); + dst->Resize({empty_size}); + GetLiteTensorDataPtr(dst, GetLitePrecisionType(src.type()), + GetLiteTargetType(src.place())); + dst->SetPrecision(GetLitePrecisionType(src.type())); + paddle::lite::LoD lite_lod; + SetLoD(&lite_lod, src.lod()); + dst->SetLoD(lite_lod); } -void InitDstTensor(framework::LoDTensor* dst, const paddle::lite::Tensor& src) { +void InitDstTensor(framework::LoDTensor* dst, + const paddle::lite_api::Tensor& src) { constexpr framework::proto::VarType::Type dtype = framework::proto::VarType_Type_FP32; dst->mutable_data(inference::lite::utils::GetNativePlace(src.target()), @@ -162,7 +202,8 @@ void InitDstTensor(framework::LoDTensor* dst, const paddle::lite::Tensor& src) { } template <> -void TensorCopyAsync(paddle::lite::Tensor* dst, const framework::LoDTensor& src, +void TensorCopyAsync(paddle::lite_api::Tensor* dst, + const framework::LoDTensor& src, const platform::DeviceContext& ctx) { InitDstTensor(dst, src); const platform::Place& src_place = src.place(); @@ -171,52 +212,56 @@ void TensorCopyAsync(paddle::lite::Tensor* dst, const framework::LoDTensor& src, static_cast(src.numel()) * framework::SizeOfType(src.type()); dst->Resize(framework::vectorize(src.dims())); const void* src_data = src.data(); - void* dst_data = dst->mutable_data(bytes); + void* dst_data{nullptr}; + dst_data = GetLiteTensorDataPtr(dst, GetLitePrecisionType(src.type()), + GetLiteTargetType(src.place())); VLOG(3) << "[CopyAsync fluid -> lite] Bytes = " << bytes << ", src = " << &src << ", dst = " << dst << ", src_type = " << src.type(); MemoryCopyAsync(dst_place, dst_data, src_place, src_data, bytes, ctx); - VLOG(3) << "[Lite memory size] Bytes = " << dst->memory_size(); + VLOG(3) << "[Lite memory size] Bytes = " << bytes; } template <> -void TensorCopyAsync(framework::LoDTensor* dst, const paddle::lite::Tensor& src, +void TensorCopyAsync(framework::LoDTensor* dst, + const paddle::lite_api::Tensor& src, const platform::DeviceContext& ctx) { - dst->Resize(paddle::framework::make_ddim(src.dims().Vectorize())); + dst->Resize(paddle::framework::make_ddim(src.shape())); InitDstTensor(dst, src); const platform::Place& src_place = GetNativePlace(src.target()); const platform::Place& dst_place = dst->place(); - const size_t bytes = - static_cast(src.numel()) * framework::SizeOfType(dst->type()); - const void* src_data = src.raw_data(); + int64_t src_numel = GetLiteTensorNumel(src); + const size_t bytes = src_numel * framework::SizeOfType(dst->type()); + const void* src_data = src.data(); // When Lite is ready, the source type needs to be modified here. void* dst_data = dst->mutable_data(dst_place, dst->type()); VLOG(3) << "[CopyAsync lite -> fluid] Bytes = " << bytes << ", src = " << &src << ", dst = " << dst << ", src_type = " << dst->type(); MemoryCopyAsync(dst_place, dst_data, src_place, src_data, bytes, ctx); - VLOG(3) << "[Lite memory size] Bytes = " << src.memory_size(); + VLOG(3) << "[Lite memory size] Bytes = " << bytes; } template <> -void TensorDataShare(paddle::lite::Tensor* dst, framework::LoDTensor* src) { - const size_t bytes = - static_cast(src->numel()) * framework::SizeOfType(src->type()); - auto buf = std::make_shared(paddle::lite::Buffer( - src->data(), GetLiteTargetType(src->place()), src->memory_size())); +void TensorDataShare(paddle::lite_api::Tensor* dst, framework::LoDTensor* src) { dst->Resize(framework::vectorize(src->dims())); - dst->set_precision(GetLitePrecisionType(src->type())); - SetLoD(dst->mutable_lod(), src->lod()); - dst->ResetBuffer(buf, bytes); + dst->ShareExternalMemory(src->data(), src->memory_size(), + GetLiteTargetType(src->place())); + dst->SetPrecision(GetLitePrecisionType(src->type())); + paddle::lite::LoD lite_lod; + SetLoD(&lite_lod, src->lod()); + dst->SetLoD(lite_lod); } template <> -void TensorDataShare(framework::LoDTensor* dst, paddle::lite::Tensor* src) { +void TensorDataShare(framework::LoDTensor* dst, paddle::lite_api::Tensor* src) { constexpr framework::proto::VarType::Type dtype = framework::proto::VarType_Type_FP32; - void* src_raw_data = src->raw_data(); + void* src_raw_data = + GetLiteTensorDataPtr(src, GetLitePrecisionType(dtype), src->target()); + size_t memory_size = GetLiteTensorNumel(*src) * sizeof(float); std::shared_ptr holder( - new memory::allocation::Allocation(src_raw_data, src->memory_size(), + new memory::allocation::Allocation(src_raw_data, memory_size, GetNativePlace(src->target()))); - dst->Resize(paddle::framework::make_ddim(src->dims().Vectorize())); + dst->Resize(paddle::framework::make_ddim(src->shape())); SetLoD(dst->mutable_lod(), src->lod()); dst->ResetHolderWithType(holder, dtype); } diff --git a/paddle/fluid/inference/lite/test_engine.cc b/paddle/fluid/inference/lite/test_engine.cc index d29bcb76be78f151dc606d9f335e9df9ed19b16b..e505af19d5389c074c5777d0235dfa055d1395a7 100644 --- a/paddle/fluid/inference/lite/test_engine.cc +++ b/paddle/fluid/inference/lite/test_engine.cc @@ -102,10 +102,10 @@ TEST(EngineManager, engine) { config.model_from_memory = true; config.valid_places = { #ifdef PADDLE_WITH_CUDA - paddle::lite::Place({TARGET(kCUDA), PRECISION(kFloat)}), + paddle::lite_api::Place({TARGET(kCUDA), PRECISION(kFloat)}), #endif - paddle::lite::Place({TARGET(kX86), PRECISION(kFloat)}), - paddle::lite::Place({TARGET(kHost), PRECISION(kAny)}), + paddle::lite_api::Place({TARGET(kX86), PRECISION(kFloat)}), + paddle::lite_api::Place({TARGET(kHost), PRECISION(kAny)}), }; LOG(INFO) << "Create EngineManager"; @@ -118,7 +118,7 @@ TEST(EngineManager, engine) { ASSERT_EQ(inference::Singleton::Global().Has( unique_key), true); - paddle::lite::Predictor* engine_0 = + paddle::lite_api::PaddlePredictor* engine_0 = inference::Singleton::Global().Get( unique_key); CHECK_NOTNULL(engine_0); diff --git a/paddle/fluid/inference/lite/test_tensor_utils.cc b/paddle/fluid/inference/lite/test_tensor_utils.cc index eef7bfb68fe06537d09f3f3e7e5c35283d4739ef..a792fb77d6ad483601402506685e2f91066571da 100644 --- a/paddle/fluid/inference/lite/test_tensor_utils.cc +++ b/paddle/fluid/inference/lite/test_tensor_utils.cc @@ -73,6 +73,33 @@ TEST(LiteEngineOp, GetNativeLayoutType) { EXPECT_ANY_THROW(GetNativeLayoutType(DataLayoutType::kNHWC)); } +template +void test_lite_tensor_data_ptr(PrecisionType precision_type) { + void* GetLiteTensorDataPtr(paddle::lite_api::Tensor * src, + PrecisionType precision_type, + TargetType target_type); + const int count = 4; + paddle::lite::Tensor lite_tensor; + lite_tensor.Resize({count}); + auto* lite_tensor_data = lite_tensor.mutable_data(); + for (size_t i = 0; i < count; ++i) { + lite_tensor_data[i] = i; + } + paddle::lite_api::Tensor lite_api_tensor(&lite_tensor); + T* data = static_cast(GetLiteTensorDataPtr( + &lite_api_tensor, precision_type, TargetType::kHost)); + for (size_t i = 0; i < count; ++i) { + CHECK_EQ(data[i], static_cast(i)) << "the i-th num is not correct."; + } +} + +TEST(LiteEngineOp, GetLiteTensorDataPtr) { + test_lite_tensor_data_ptr(PrecisionType::kInt64); + test_lite_tensor_data_ptr(PrecisionType::kInt32); + test_lite_tensor_data_ptr(PrecisionType::kInt8); + EXPECT_ANY_THROW(test_lite_tensor_data_ptr(PrecisionType::kUnk)); +} + void test_tensor_copy(const platform::DeviceContext& ctx) { // Create LoDTensor. std::vector vector({1, 2, 3, 4}); @@ -83,10 +110,11 @@ void test_tensor_copy(const platform::DeviceContext& ctx) { lod_tensor.set_lod(lod); // Create lite::Tensor and copy. paddle::lite::Tensor lite_tensor; - TensorCopyAsync(&lite_tensor, lod_tensor, ctx); + paddle::lite_api::Tensor lite_api_tensor(&lite_tensor); + TensorCopyAsync(&lite_api_tensor, lod_tensor, ctx); // Copy to LoDTensor. framework::LoDTensor lod_tensor_n; - TensorCopyAsync(&lod_tensor_n, lite_tensor, ctx); + TensorCopyAsync(&lod_tensor_n, lite_api_tensor, ctx); #ifdef PADDLE_WITH_CUDA if (platform::is_gpu_place(ctx.GetPlace())) { platform::GpuStreamSync( @@ -108,10 +136,11 @@ void test_tensor_share(const platform::DeviceContext& ctx) { lod_tensor.set_lod(lod); // Create lite::Tensor and share. paddle::lite::Tensor lite_tensor; - TensorDataShare(&lite_tensor, &lod_tensor); + paddle::lite_api::Tensor lite_api_tensor(&lite_tensor); + TensorDataShare(&lite_api_tensor, &lod_tensor); // Copy to LoDTensor. framework::LoDTensor lod_tensor_n; - TensorCopyAsync(&lod_tensor_n, lite_tensor, ctx); + TensorCopyAsync(&lod_tensor_n, lite_api_tensor, ctx); std::vector result; TensorToVector(lod_tensor_n, ctx, &result); ASSERT_EQ(result, vector); diff --git a/paddle/fluid/inference/tensorrt/engine.cc b/paddle/fluid/inference/tensorrt/engine.cc index 22be877493272cd393538fd4f04184e77d38e2db..754979f77acd7a3b4818cdf16ef9c525bf1d82ea 100644 --- a/paddle/fluid/inference/tensorrt/engine.cc +++ b/paddle/fluid/inference/tensorrt/engine.cc @@ -63,11 +63,13 @@ void TensorRTEngine::Execute(int batch_size, std::vector *buffers, void TensorRTEngine::FreezeNetwork() { freshDeviceId(); VLOG(3) << "TRT to freeze network"; - PADDLE_ENFORCE(infer_builder_ != nullptr, - "Call InitNetwork first to initialize network."); - PADDLE_ENFORCE_EQ(network() != nullptr, true, - platform::errors::InvalidArgument( - "Call InitNetwork first to initialize network.")); + PADDLE_ENFORCE_NOT_NULL(infer_builder_, + platform::errors::InvalidArgument( + "Inference builder of TRT is null. Please make " + "sure you call InitNetwork first.")); + PADDLE_ENFORCE_NOT_NULL(network(), + platform::errors::InvalidArgument( + "Call InitNetwork first to initialize network.")); // build engine. infer_builder_->setMaxBatchSize(max_batch_); infer_builder_->setMaxWorkspaceSize(max_workspace_); @@ -210,7 +212,10 @@ void TensorRTEngine::FreezeNetwork() { } else { infer_engine_.reset(infer_builder_->buildCudaEngine(*network())); } - PADDLE_ENFORCE(infer_engine_ != nullptr, "build cuda engine failed!"); + PADDLE_ENFORCE_NOT_NULL( + infer_engine_, platform::errors::Fatal( + "Build TensorRT cuda engine failed! Please recheck " + "you configurations related to paddle-TensorRT.")); } nvinfer1::ITensor *TensorRTEngine::DeclareInput(const std::string &name, @@ -220,8 +225,16 @@ nvinfer1::ITensor *TensorRTEngine::DeclareInput(const std::string &name, platform::errors::InvalidArgument( "The TRT network should be initialized first.")); auto *input = network()->addInput(name.c_str(), dtype, dims); - PADDLE_ENFORCE(input, "infer network add input %s failed", name); - PADDLE_ENFORCE(input->isNetworkInput()); + PADDLE_ENFORCE_NOT_NULL( + input, platform::errors::InvalidArgument("Adding input %s failed in " + "TensorRT inference network. " + "Please recheck your input.", + name)); + PADDLE_ENFORCE_EQ(input->isNetworkInput(), true, + platform::errors::InvalidArgument( + "Input %s is not the input of TRT inference network. " + "Please recheck your input.", + name)); TensorRTEngine::SetITensor(name, input); return input; } @@ -230,31 +243,53 @@ void TensorRTEngine::DeclareOutput(const nvinfer1::ILayer *layer, int offset, const std::string &name) { auto *output = layer->getOutput(offset); SetITensor(name, output); - PADDLE_ENFORCE(output != nullptr); + PADDLE_ENFORCE_NOT_NULL( + output, platform::errors::InvalidArgument( + "The output %s of TRT engine should not be null.", name)); output->setName(name.c_str()); - PADDLE_ENFORCE(!output->isNetworkInput()); + PADDLE_ENFORCE_EQ(output->isNetworkInput(), false, + platform::errors::InvalidArgument( + "The output %s of TRT engine should not be the input " + "of the network at the same time.", + name)); network()->markOutput(*output); - PADDLE_ENFORCE(output->isNetworkOutput()); + PADDLE_ENFORCE_EQ( + output->isNetworkOutput(), true, + platform::errors::InvalidArgument( + "The output %s of TRT engine should be the output of the network.", + name)); } void TensorRTEngine::DeclareOutput(const std::string &name) { auto *output = TensorRTEngine::GetITensor(name); - PADDLE_ENFORCE(output != nullptr); + PADDLE_ENFORCE_NOT_NULL( + output, platform::errors::InvalidArgument( + "The output %s of TRT engine should not be null.", name)); output->setName(name.c_str()); - PADDLE_ENFORCE(!output->isNetworkInput()); + PADDLE_ENFORCE_EQ(output->isNetworkInput(), false, + platform::errors::InvalidArgument( + "The output %s of TRT engine should not be the input " + "of the network at the same time.", + name)); network()->markOutput(*output); } void TensorRTEngine::SetITensor(const std::string &name, nvinfer1::ITensor *tensor) { - PADDLE_ENFORCE(tensor != nullptr); - PADDLE_ENFORCE_EQ(0, itensor_map_.count(name), "duplicate ITensor name %s", - name); + PADDLE_ENFORCE_NOT_NULL( + tensor, platform::errors::InvalidArgument( + "Tensor named %s of TRT engine should not be null.", name)); + PADDLE_ENFORCE_EQ( + 0, itensor_map_.count(name), + platform::errors::InvalidArgument( + "Tensor named %s of TRT engine should not be duplicated", name)); itensor_map_[name] = tensor; } nvinfer1::ITensor *TensorRTEngine::GetITensor(const std::string &name) { - PADDLE_ENFORCE(itensor_map_.count(name), "no ITensor %s", name); + PADDLE_ENFORCE_EQ(itensor_map_.count(name), true, + platform::errors::NotFound( + "Tensor named %s is not found in TRT engine", name)); return itensor_map_[name]; } @@ -271,11 +306,11 @@ float *TensorRTEngine::GetWeightCPUData(const std::string &name, std::string splitter = "__"; std::string name_with_suffix = name + splitter + name_suffix; platform::CPUPlace cpu_place; - PADDLE_ENFORCE_EQ( - weight_map.count(name_with_suffix), 0, - "During TRT Op converter: We set weight %s with the same name " - "twice into the weight_map", - name_with_suffix); + PADDLE_ENFORCE_EQ(weight_map.count(name_with_suffix), 0, + platform::errors::AlreadyExists( + "The weight named %s is set into the weight map " + "twice in TRT OP converter.", + name_with_suffix)); weight_map[name_with_suffix].reset(new framework::Tensor()); weight_map[name_with_suffix]->Resize(weight_tensor->dims()); TensorCopySync(*weight_tensor, cpu_place, weight_map[name_with_suffix].get()); @@ -297,7 +332,10 @@ nvinfer1::IPluginLayer *TensorRTEngine::AddPlugin( void TensorRTEngine::freshDeviceId() { int count; cudaGetDeviceCount(&count); - PADDLE_ENFORCE_LT(device_id_, count); + PADDLE_ENFORCE_LT(device_id_, count, + platform::errors::OutOfRange( + "Device id %d exceeds the current device count: %d.", + device_id_, count)); cudaSetDevice(device_id_); } diff --git a/paddle/fluid/inference/tensorrt/engine.h b/paddle/fluid/inference/tensorrt/engine.h index 1a3413657ce6fac41603d691dcdb61ddb1d6320a..a85ed483c1d12c3f2eecc5ed4bcb99937397a765 100644 --- a/paddle/fluid/inference/tensorrt/engine.h +++ b/paddle/fluid/inference/tensorrt/engine.h @@ -196,8 +196,10 @@ class TensorRTEngine { } nvinfer1::IHostMemory* Serialize() { - PADDLE_ENFORCE(infer_engine_ != nullptr, - "You should build engine first and then serialize"); + PADDLE_ENFORCE_NOT_NULL( + infer_engine_, + platform::errors::InvalidArgument( + "The TensorRT engine must be built first before serialization")); ihost_memory_.reset(infer_engine_->serialize()); return ihost_memory_.get(); } @@ -222,8 +224,14 @@ class TensorRTEngine { engine_serialized_data.c_str(), engine_serialized_data.size(), &inference::Singleton::Global())); } - PADDLE_ENFORCE(infer_engine_ != nullptr, - "build cuda engine failed when deserialize engine info.!"); + PADDLE_ENFORCE_NOT_NULL( + infer_engine_, + platform::errors::Fatal( + "Building TRT cuda engine failed when deserializing engine info. " + "Please check:\n1. Your TRT serialization is generated and loaded " + "on the same GPU architecture;\n2. The Paddle Inference version of " + "generating serialization file and doing inference are " + "consistent.")); } void SetRuntimeBatch(size_t batch_size); diff --git a/paddle/fluid/inference/tensorrt/plugin/elementwise_op_plugin.cu b/paddle/fluid/inference/tensorrt/plugin/elementwise_op_plugin.cu index 0ec803fe64afadd970777e3b0d0ab5d37fcc4d22..457d9dd87375477926480bce0a84e8f89c409698 100644 --- a/paddle/fluid/inference/tensorrt/plugin/elementwise_op_plugin.cu +++ b/paddle/fluid/inference/tensorrt/plugin/elementwise_op_plugin.cu @@ -56,14 +56,27 @@ __global__ void elementwise_kernel(const size_t total, const T *x_data, nvinfer1::Dims ElementWisePlugin::getOutputDimensions( int index, const nvinfer1::Dims *input_dims, int num_inputs) { - PADDLE_ENFORCE_EQ(index, 0); - PADDLE_ENFORCE_EQ(num_inputs, 2); - PADDLE_ENFORCE_NOT_NULL(input_dims); + PADDLE_ENFORCE_EQ(index, 0, platform::errors::InvalidArgument( + "There is only one output in TRT elementwise " + "op plugin, but got output index: %d.", + index)); + PADDLE_ENFORCE_EQ(num_inputs, 2, platform::errors::InvalidArgument( + "There are 2 inputs in TRT elementwise " + "op plugin, but got input number: %d.", + num_inputs)); + PADDLE_ENFORCE_NOT_NULL( + input_dims, + platform::errors::InvalidArgument( + "The input dims of TRT elementwise op plugin should not be null.")); return input_dims[0]; } int ElementWisePlugin::initialize() { - PADDLE_ENFORCE_GT(dims_y_.nbDims, 0); + PADDLE_ENFORCE_GT(dims_y_.nbDims, 0, + platform::errors::InvalidArgument( + "The dimension of input Y of TRT elementwise op plugin " + "should be greater than 0, but got %d.", + dims_y_.nbDims)); axis_ = (axis_ == -1) ? dims_x_.nbDims - dims_y_.nbDims : axis_; int trimed_nb_dims = dims_y_.nbDims; @@ -74,8 +87,18 @@ int ElementWisePlugin::initialize() { } dims_y_.nbDims = trimed_nb_dims; - PADDLE_ENFORCE_GE(dims_x_.nbDims, dims_y_.nbDims + axis_); - PADDLE_ENFORCE_LT(axis_, dims_x_.nbDims); + PADDLE_ENFORCE_GE(dims_x_.nbDims, dims_y_.nbDims + axis_, + platform::errors::InvalidArgument( + "We expect [number of x dims] >= [number of y dims + " + "axis] in TRT elementwise op plugin, but got [number " + "of x dims] = %d, [number of y dims + axis] = %d.", + dims_x_.nbDims, dims_y_.nbDims + axis_)); + PADDLE_ENFORCE_LT( + axis_, dims_x_.nbDims, + platform::errors::InvalidArgument("We expect [axis] < [number of x dims] " + "in TRT elementwise op plugin, but got " + "[axis] = %d, [number of x dims] = %d.", + axis_, dims_x_.nbDims)); prev_size_ = 1; midd_size_ = 1; @@ -86,7 +109,9 @@ int ElementWisePlugin::initialize() { for (int i = 0; i < dims_y_.nbDims; ++i) { PADDLE_ENFORCE_EQ(dims_x_.d[i + axis_], dims_y_.d[i], - "Broadcast dimension mismatch."); + platform::errors::InvalidArgument( + "Broadcast dimension mismatch. The dims of input Y " + "should be a subsequence of X.")); midd_size_ *= dims_y_.d[i]; } @@ -221,7 +246,10 @@ int ElementwisePluginDynamic::enqueue( elementwise_kernel<<>>( num, x, y, out, prev_size, midd_size, post_size, details::Mul()); } else { - PADDLE_THROW("Not implemented."); + PADDLE_THROW(platform::errors::Unimplemented( + "Paddle-TRT only support elementwise operation: {add, mul} currently, " + "but got %s.", + type_)); } return cudaGetLastError() != cudaSuccess; diff --git a/paddle/fluid/inference/tensorrt/test_engine.cc b/paddle/fluid/inference/tensorrt/test_engine.cc index a03dd45db0f80487cb4c2e6b68f94944e8558ae4..72962c733ecf6a7bc6871fd3a5c65d6156b084d4 100644 --- a/paddle/fluid/inference/tensorrt/test_engine.cc +++ b/paddle/fluid/inference/tensorrt/test_engine.cc @@ -74,7 +74,9 @@ TEST_F(TensorRTEngineTest, add_layer) { nvinfer1::DimsCHW{1, 1, 1}); auto *fc_layer = TRT_ENGINE_ADD_LAYER(engine_, FullyConnected, *x, size, weight.get(), bias.get()); - PADDLE_ENFORCE(fc_layer != nullptr); + PADDLE_ENFORCE_NOT_NULL(fc_layer, + platform::errors::InvalidArgument( + "TRT fully connected layer building failed.")); engine_->DeclareOutput(fc_layer, 0, "y"); LOG(INFO) << "freeze network"; @@ -116,7 +118,9 @@ TEST_F(TensorRTEngineTest, add_layer_multi_dim) { nvinfer1::DimsCHW{1, 2, 1}); auto *fc_layer = TRT_ENGINE_ADD_LAYER(engine_, FullyConnected, *x, 2, weight.get(), bias.get()); - PADDLE_ENFORCE(fc_layer != nullptr); + PADDLE_ENFORCE_NOT_NULL(fc_layer, + platform::errors::InvalidArgument( + "TRT fully connected layer building failed.")); engine_->DeclareOutput(fc_layer, 0, "y"); engine_->FreezeNetwork(); @@ -160,7 +164,9 @@ TEST_F(TensorRTEngineTest, test_conv2d) { auto *conv_layer = TRT_ENGINE_ADD_LAYER(engine_, Convolution, *x, 1, nvinfer1::DimsHW{3, 3}, weight.get(), bias.get()); - PADDLE_ENFORCE(conv_layer != nullptr); + PADDLE_ENFORCE_NOT_NULL(conv_layer, + platform::errors::InvalidArgument( + "TRT convolution layer building failed.")); conv_layer->setStride(nvinfer1::DimsHW{1, 1}); conv_layer->setPadding(nvinfer1::DimsHW{1, 1}); @@ -199,7 +205,9 @@ TEST_F(TensorRTEngineTest, test_pool2d) { auto *pool_layer = TRT_ENGINE_ADD_LAYER(engine_, Pooling, *x, pool_t, nvinfer1::DimsHW{2, 2}); - PADDLE_ENFORCE(pool_layer != nullptr); + PADDLE_ENFORCE_NOT_NULL( + pool_layer, + platform::errors::InvalidArgument("TRT pooling layer building failed.")); pool_layer->setStride(nvinfer1::DimsHW{1, 1}); pool_layer->setPadding(nvinfer1::DimsHW{0, 0}); diff --git a/paddle/fluid/inference/tensorrt/trt_int8_calibrator.cc b/paddle/fluid/inference/tensorrt/trt_int8_calibrator.cc index 34b7072b2ee688c2ac01229ff5d3a234af3680b5..743f7740e5faaa1991172ef2a8d1cd38ad47fab5 100644 --- a/paddle/fluid/inference/tensorrt/trt_int8_calibrator.cc +++ b/paddle/fluid/inference/tensorrt/trt_int8_calibrator.cc @@ -83,9 +83,8 @@ bool TRTInt8Calibrator::setBatch( engine_name_, it.first)); } const auto& d = dataptr->second; - PADDLE_ENFORCE( - cudaMemcpy(d.first, it.second, d.second, cudaMemcpyDeviceToDevice), - "Fail to cudaMemcpy %s for %s", engine_name_, it.first); + PADDLE_ENFORCE_CUDA_SUCCESS( + cudaMemcpy(d.first, it.second, d.second, cudaMemcpyDeviceToDevice)); } data_is_set_ = true; diff --git a/paddle/fluid/inference/tests/api/CMakeLists.txt b/paddle/fluid/inference/tests/api/CMakeLists.txt index 6dd13d32e6e25f1657f351ff3a54562435b098f3..b3ec4b5714eb17032039eb234e148cdbd38c7877 100644 --- a/paddle/fluid/inference/tests/api/CMakeLists.txt +++ b/paddle/fluid/inference/tests/api/CMakeLists.txt @@ -342,9 +342,9 @@ if(WITH_MKLDNN) ### Lexcial analysis GRU model set(GRU_PATH "${INFERENCE_DEMO_INSTALL_DIR}/gru") download_GRU_data("${GRU_PATH}" "GRU_eval_data.tar.gz") - download_GRU_data("${GRU_PATH}" "GRU_eval_model.tar.gz") + download_GRU_data("${GRU_PATH}" "GRU_eval_model_v2.tar.gz") set(GRU_DATA_PATH "${GRU_PATH}/GRU_eval_data.bin") - set(GRU_MODEL_PATH "${GRU_PATH}/GRU_eval_model") + set(GRU_MODEL_PATH "${GRU_PATH}/GRU_eval_model_v2") set(LEXICAL_TEST_APP "test_analyzer_lexical_analysis") set(LEXICAL_TEST_APP_SRC "analyzer_lexical_analysis_gru_tester.cc") @@ -363,9 +363,12 @@ if(WITH_MKLDNN) inference_analysis_api_test_build(${QUANT_IMG_CLASS_TEST_APP} ${QUANT_IMG_CLASS_TEST_APP_SRC}) # MobileNetV1 FP32 vs. Quant INT8 + # The FP32 model should already be downloaded for slim Quant unit tests on Linux set(QUANT2_MobileNetV1_MODEL_DIR "${QUANT_DATA_DIR}/MobileNetV1_quant2") set(QUANT2_INT8_MobileNetV1_MODEL_DIR "${QUANT_DATA_DIR}/MobileNetV1_quant2_int8") - download_quant_data(${QUANT2_MobileNetV1_MODEL_DIR} "MobileNet_qat_perf.tar.gz") + if(NOT LINUX) + download_quant_data(${QUANT2_MobileNetV1_MODEL_DIR} "MobileNet_qat_perf.tar.gz") + endif(NOT LINUX) download_quant_data(${QUANT2_INT8_MobileNetV1_MODEL_DIR} "MobileNet_qat_perf_int8.tar.gz") inference_analysis_api_quant_test_run(test_analyzer_quant_performance_benchmark ${QUANT_IMG_CLASS_TEST_APP} ${QUANT2_MobileNetV1_MODEL_DIR}/MobileNet_qat_perf/float ${QUANT2_INT8_MobileNetV1_MODEL_DIR}/MobileNet_qat_perf_int8 ${IMAGENET_DATA_PATH}) diff --git a/paddle/fluid/inference/tests/api/lite_resnet50_test.cc b/paddle/fluid/inference/tests/api/lite_resnet50_test.cc index 31701c59ec33dfced5745f7f16d8f00ffce462ef..9ae073e9e5b142254b32396e0355f59ae1826909 100644 --- a/paddle/fluid/inference/tests/api/lite_resnet50_test.cc +++ b/paddle/fluid/inference/tests/api/lite_resnet50_test.cc @@ -27,7 +27,7 @@ TEST(AnalysisPredictor, use_gpu) { AnalysisConfig config; config.EnableUseGpu(100, 0); config.SetModel(model_dir + "/model", model_dir + "/params"); - config.EnableLiteEngine(paddle::AnalysisConfig::Precision::kFloat32); + config.EnableLiteEngine(paddle::AnalysisConfig::Precision::kFloat32, true); std::vector inputs; auto predictor = CreatePaddlePredictor(config); diff --git a/paddle/fluid/operators/CMakeLists.txt b/paddle/fluid/operators/CMakeLists.txt index 6e8ff52ed4a8846f5f6060e10cfd9bec22308e9e..f0a04d850dff01e0776e96bbe518cde2ce8bb88b 100644 --- a/paddle/fluid/operators/CMakeLists.txt +++ b/paddle/fluid/operators/CMakeLists.txt @@ -45,7 +45,9 @@ endif() SET(OP_HEADER_DEPS xxhash executor) if (WITH_GPU) - SET(OP_HEADER_DEPS ${OP_HEADER_DEPS} cub) + if (${CMAKE_CUDA_COMPILER_VERSION} LESS 11.0) + SET(OP_HEADER_DEPS ${OP_HEADER_DEPS} cub) + endif() endif() SET(OP_PREFETCH_DEPS "") diff --git a/paddle/fluid/operators/amp/amp_check_finite_and_scale_op.cc b/paddle/fluid/operators/amp/amp_check_finite_and_scale_op.cc deleted file mode 100644 index 7f0ca1493f712f7f4809a56bf6a23f8757f94c2d..0000000000000000000000000000000000000000 --- a/paddle/fluid/operators/amp/amp_check_finite_and_scale_op.cc +++ /dev/null @@ -1,104 +0,0 @@ -/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#include "paddle/fluid/operators/amp/amp_check_finite_and_scale_op.h" - -#include -#include - -namespace paddle { -namespace operators { - -class AmpCheckFiniteAndScaleOp : public framework::OperatorWithKernel { - public: - AmpCheckFiniteAndScaleOp(const std::string &type, - const framework::VariableNameMap &inputs, - const framework::VariableNameMap &outputs, - const framework::AttributeMap &attrs) - : OperatorWithKernel(type, inputs, outputs, attrs) {} - - void InferShape(framework::InferShapeContext *ctx) const override { - OP_INOUT_CHECK(ctx->HasInputs("X"), "Input", "X", - "amp_check_finite_and_unscale"); - OP_INOUT_CHECK(ctx->HasOutputs("Out"), "Output", "Out", - "amp_check_finite_and_unscale"); - PADDLE_ENFORCE_EQ( - ctx->Inputs("X").size(), ctx->Outputs("Out").size(), - platform::errors::InvalidArgument( - "The input(X) and output(Out) should have same size in " - "Operator(amp_check_finite_and_unscale), size of input(X) is %d " - "and size of output(Out) is %d.", - ctx->Inputs("X").size(), ctx->Outputs("Out").size())); - auto x_dims = ctx->GetInputsDim("X"); - ctx->SetOutputsDim("Out", x_dims); - ctx->SetOutputDim("FoundInfinite", {1}); - } - - protected: - framework::OpKernelType GetExpectedKernelType( - const framework::ExecutionContext &ctx) const override { - return framework::OpKernelType( - OperatorWithKernel::IndicateVarDataType(ctx, "X"), ctx.GetPlace()); - } -}; - -class AmpCheckFiniteAndScaleOpMaker : public framework::OpProtoAndCheckerMaker { - public: - void Make() override { - AddInput( - "X", - "(Tensors) The input tensors of amp_check_finite_and_scale operator.") - .AsDuplicable(); - AddInput("Scale", - "(Tensor) 1-dim tensor, the scale of amp_check_finite_and_scale " - "operator."); - AddOutput("Out", - "(Tensors) The scaled output tensor of " - "amp_check_finite_and_unscale operator.") - .AsDuplicable(); - AddOutput("FoundInfinite", - "(Tensor) 1-dim tensor, contains a bool scalar, which indicates " - "if there there is infinite or nan item in input X."); - AddComment(R"DOC( -amp_check_finite_and_scale operator. -Check if input X contains all finite data, if yes, scale it by input Scale. - -$$Out = X * scale$$ - -If any tensor in X contains Inf or Nan, the Out will generate a indicator. -FoundInfinite will be 1 (True), and Out will not be scaled. In this case, the data of -Out should not be used, and its data may not be deterministic. -Otherwise, FoundInfinite will be 0 (False). - -)DOC"); - } -}; - -} // namespace operators -} // namespace paddle - -namespace ops = paddle::operators; - -REGISTER_OPERATOR( - amp_check_finite_and_scale, ops::AmpCheckFiniteAndScaleOp, - ops::AmpCheckFiniteAndScaleOpMaker, - paddle::framework::EmptyGradOpMaker, - paddle::framework::EmptyGradOpMaker); - -REGISTER_OP_CPU_KERNEL( - amp_check_finite_and_scale, - ops::AmpCheckFiniteAndScaleKernel, - ops::AmpCheckFiniteAndScaleKernel); diff --git a/paddle/fluid/operators/amp/amp_check_finite_and_scale_op.h b/paddle/fluid/operators/amp/amp_check_finite_and_scale_op.h deleted file mode 100644 index 6c2c4eb8a615c4c04a98601c25b5de43b4262e6b..0000000000000000000000000000000000000000 --- a/paddle/fluid/operators/amp/amp_check_finite_and_scale_op.h +++ /dev/null @@ -1,66 +0,0 @@ -/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#pragma once - -#include -#include -#include "paddle/fluid/framework/tensor_util.h" -#include "paddle/fluid/operators/elementwise/elementwise_op_function.h" -#include "paddle/fluid/operators/isfinite_op.h" - -namespace paddle { -namespace operators { - -template -class AmpCheckFiniteAndScaleKernel : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext& ctx) const { - auto& dev_ctx = ctx.template device_context(); - const auto xs = ctx.MultiInput("X"); - const auto* scale = ctx.Input("Scale"); - auto outs = ctx.MultiOutput("Out"); - auto* found_inf = ctx.Output("FoundInfinite"); - - const T* scale_data = scale->data(); - bool* found_inf_data = found_inf->mutable_data(dev_ctx.GetPlace()); - - *found_inf_data = false; - framework::Tensor is_finite = - ctx.AllocateTmpTensor({1}, dev_ctx); - bool* is_finite_data = is_finite.template data(); - - auto& dev = *ctx.template device_context().eigen_device(); - for (size_t i = 0; i < xs.size(); ++i) { - const auto* x = xs[i]; - auto* out = outs[i]; - out->mutable_data(dev_ctx.GetPlace()); - if (!(*found_inf_data)) { - framework::TensorIsfinite(*x, &is_finite); - if (*is_finite_data) { - auto eigen_out = framework::EigenVector::Flatten(*out); - auto eigen_in = framework::EigenVector::Flatten(*x); - eigen_out.device(dev) = (*scale_data) * eigen_in; - } else { - *found_inf_data = true; - break; - } - } - } - return; - } -}; - -} // namespace operators -} // namespace paddle diff --git a/paddle/fluid/operators/amp/check_finite_and_unscale_op.cc b/paddle/fluid/operators/amp/check_finite_and_unscale_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..51c659d5db1c33d5e2db261b998a0673f5e766cb --- /dev/null +++ b/paddle/fluid/operators/amp/check_finite_and_unscale_op.cc @@ -0,0 +1,141 @@ +/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/operators/amp/check_finite_and_unscale_op.h" +#include "paddle/fluid/framework/tensor_util.h" + +namespace paddle { +namespace operators { + +class CheckFiniteAndUnscaleOp : public framework::OperatorWithKernel { + public: + CheckFiniteAndUnscaleOp(const std::string& type, + const framework::VariableNameMap& inputs, + const framework::VariableNameMap& outputs, + const framework::AttributeMap& attrs) + : OperatorWithKernel(type, inputs, outputs, attrs) {} + + void InferShape(framework::InferShapeContext* ctx) const override { + OP_INOUT_CHECK(ctx->HasInputs("X"), "Input", "X", + "check_finite_and_unscale"); + OP_INOUT_CHECK(ctx->HasOutputs("Out"), "Output", "Out", + "check_finite_and_unscale"); + PADDLE_ENFORCE_EQ( + ctx->Inputs("X").size(), ctx->Outputs("Out").size(), + platform::errors::InvalidArgument( + "The input(X) and output(Out) should have same size in " + "Operator(check_finite_and_unscale), size of input(X) is %d " + "and size of output(Out) is %d.", + ctx->Inputs("X").size(), ctx->Outputs("Out").size())); + auto x_dims = ctx->GetInputsDim("X"); + ctx->SetOutputsDim("Out", x_dims); + ctx->SetOutputDim("FoundInfinite", {1}); + } + + protected: + framework::OpKernelType GetExpectedKernelType( + const framework::ExecutionContext& ctx) const override { + return framework::OpKernelType( + OperatorWithKernel::IndicateVarDataType(ctx, "X"), ctx.GetPlace()); + } +}; + +class CheckFiniteAndUnscaleOpMaker : public framework::OpProtoAndCheckerMaker { + public: + void Make() override { + AddInput( + "X", + "(Tensors) The input tensors of check_finite_and_unscale operator.") + .AsDuplicable(); + AddInput("Scale", + "(Tensor) 1-dim tensor, the scale of check_finite_and_unscale " + "operator."); + AddOutput("Out", + "(Tensors) The scaled output tensor of " + "check_finite_and_unscale operator.") + .AsDuplicable(); + AddOutput("FoundInfinite", + "(Tensor) 1-dim tensor, contains a bool scalar, which indicates " + "if there there is infinite or nan item in input X."); + AddComment(R"DOC( +check_finite_and_unscale operator. +Check if input X contains all finite data, if yes, scale it by input Scale. + +$$Out = X / scale$$ + +If any tensor in X contains Inf or Nan, the Out will generate a indicator. +FoundInfinite will be 1 (True), and Out will not be scaled. In this case, the data of +Out should not be used, and its data may not be deterministic. +Otherwise, FoundInfinite will be 0 (False). + +)DOC"); + } +}; + +template +class CheckFiniteAndUnscaleCpuKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const { + auto& dev_ctx = ctx.template device_context(); + const auto xs = ctx.MultiInput("X"); + const auto* scale = ctx.Input("Scale"); + auto outs = ctx.MultiOutput("Out"); + auto* found_inf = ctx.Output("FoundInfinite"); + + const T* scale_data = scale->data(); + bool* found_inf_data = found_inf->mutable_data(dev_ctx.GetPlace()); + + *found_inf_data = false; + framework::Tensor is_finite = + ctx.AllocateTmpTensor({1}, dev_ctx); + bool* is_finite_data = is_finite.template data(); + + auto& dev = *ctx.template device_context() + .eigen_device(); + + T inverse_scale = Inverse(*scale_data); + for (size_t i = 0; i < xs.size(); ++i) { + const auto* x = xs[i]; + auto* out = outs[i]; + out->mutable_data(dev_ctx.GetPlace()); + if (!(*found_inf_data)) { + framework::TensorIsfinite(*x, &is_finite); + *found_inf_data = !(*is_finite_data); + } + auto eigen_out = framework::EigenVector::Flatten(*out); + auto eigen_in = framework::EigenVector::Flatten(*x); + if (!(*found_inf_data)) { + eigen_out.device(dev) = eigen_in * inverse_scale; + } else { + eigen_out.device(dev) = eigen_in * static_cast(0); + } + } + return; + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; + +REGISTER_OPERATOR( + check_finite_and_unscale, ops::CheckFiniteAndUnscaleOp, + ops::CheckFiniteAndUnscaleOpMaker, + paddle::framework::EmptyGradOpMaker, + paddle::framework::EmptyGradOpMaker); + +REGISTER_OP_CPU_KERNEL(check_finite_and_unscale, + ops::CheckFiniteAndUnscaleCpuKernel, + ops::CheckFiniteAndUnscaleCpuKernel); diff --git a/paddle/fluid/operators/amp/amp_check_finite_and_scale_op.cu b/paddle/fluid/operators/amp/check_finite_and_unscale_op.cu similarity index 63% rename from paddle/fluid/operators/amp/amp_check_finite_and_scale_op.cu rename to paddle/fluid/operators/amp/check_finite_and_unscale_op.cu index ee00d7c5f4499867c2c706ddcf314c1bfae0a866..cf9df34a2467f8461c4c284b4848c54b76edf452 100644 --- a/paddle/fluid/operators/amp/amp_check_finite_and_scale_op.cu +++ b/paddle/fluid/operators/amp/check_finite_and_unscale_op.cu @@ -14,28 +14,31 @@ limitations under the License. */ #include -#include "paddle/fluid/operators/amp/amp_check_finite_and_scale_op.h" -#include "paddle/fluid/platform/float16.h" +#include "paddle/fluid/operators/amp/check_finite_and_unscale_op.h" namespace paddle { namespace operators { template -__global__ void AmpCheckFiniteAndScale(const T* in, const T* scale, int num, - bool* found_inf, T* out) { +__global__ void GpuInverse(const T* s, T* o) { + *o = Inverse(*s); +} + +template +__global__ void CheckFiniteAndUnscale(const T* in, const T* scale, int num, + bool* found_inf, T* out) { const int idx = threadIdx.x + blockIdx.x * blockDim.x; if (idx < num) { if (!isfinite(in[idx])) { - *found_inf = 1; + *found_inf = true; } - out[idx] = *found_inf ? in[idx] : in[idx] * scale[0]; + out[idx] = *found_inf ? in[idx] : in[idx] * (*scale); } } template -class AmpCheckFiniteAndScaleKernel - : public framework::OpKernel { +class CheckFiniteAndUnscaleGpuKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const { auto& dev_ctx = ctx.template device_context(); @@ -48,6 +51,12 @@ class AmpCheckFiniteAndScaleKernel bool* found_inf_data = found_inf->mutable_data(dev_ctx.GetPlace()); cudaMemset(found_inf_data, false, found_inf->numel() * sizeof(bool)); + framework::Tensor inverse_scale = + ctx.AllocateTmpTensor({1}, dev_ctx); + T* inverse_scale_v = inverse_scale.template data(); + + GpuInverse<<<1, 1, 0, dev_ctx.stream()>>>(scale_data, inverse_scale_v); + for (size_t i = 0; i < xs.size(); ++i) { const auto* x = xs[i]; auto* out = outs[i]; @@ -55,11 +64,11 @@ class AmpCheckFiniteAndScaleKernel T* out_data = out->mutable_data(dev_ctx.GetPlace()); int num = x->numel(); - int block = 512; + int block = 1024; int grid = (num + block - 1) / block; VLOG(3) << "launch kernel"; - AmpCheckFiniteAndScale<<>>( - x_data, scale_data, num, found_inf_data, out_data); + CheckFiniteAndUnscale<<>>( + x_data, inverse_scale_v, num, found_inf_data, out_data); VLOG(3) << "finish kernel"; } } @@ -68,9 +77,6 @@ class AmpCheckFiniteAndScaleKernel } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP_CUDA_KERNEL( - amp_check_finite_and_scale, - ops::AmpCheckFiniteAndScaleKernel, - ops::AmpCheckFiniteAndScaleKernel); +REGISTER_OP_CUDA_KERNEL(check_finite_and_unscale, + ops::CheckFiniteAndUnscaleGpuKernel, + ops::CheckFiniteAndUnscaleGpuKernel); diff --git a/paddle/fluid/operators/amp/check_finite_and_unscale_op.h b/paddle/fluid/operators/amp/check_finite_and_unscale_op.h new file mode 100644 index 0000000000000000000000000000000000000000..4fb8744d0eee3c58f2948c5a466e08c2700b4332 --- /dev/null +++ b/paddle/fluid/operators/amp/check_finite_and_unscale_op.h @@ -0,0 +1,31 @@ +/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include +#include +#include "paddle/fluid/operators/isfinite_op.h" +#include "paddle/fluid/platform/hostdevice.h" + +namespace paddle { +namespace operators { + +template +inline HOSTDEVICE T Inverse(T s) { + return 1.0 / s; +} + +} // namespace operators +} // namespace paddle diff --git a/paddle/fluid/operators/amp/update_loss_scaling_op.cc b/paddle/fluid/operators/amp/update_loss_scaling_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..fca3c531b40550952273f03f41bbc62cbff170fc --- /dev/null +++ b/paddle/fluid/operators/amp/update_loss_scaling_op.cc @@ -0,0 +1,170 @@ +/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/operators/amp/update_loss_scaling_op.h" +#include +#include +#include +#include "paddle/fluid/framework/op_registry.h" + +namespace paddle { +namespace operators { + +class UpdateLossScalingOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext* ctx) const override { + OP_INOUT_CHECK(ctx->HasInputs("X"), "Input", "X", "update_loss_scaling"); + OP_INOUT_CHECK(ctx->HasInput("FoundInfinite"), "Input", "FoundInfinite", + "update_loss_scaling"); + OP_INOUT_CHECK(ctx->HasInput("PrevLossScaling"), "Input", "PrevLossScaling", + "update_loss_scaling"); + OP_INOUT_CHECK(ctx->HasInput("InGoodSteps"), "Input", "InGoodSteps", + "update_loss_scaling"); + OP_INOUT_CHECK(ctx->HasInput("InBadSteps"), "Input", "InBadSteps", + "update_loss_scaling"); + OP_INOUT_CHECK(ctx->HasOutputs("Out"), "Output", "Out", + "update_loss_scaling"); + OP_INOUT_CHECK(ctx->HasOutput("LossScaling"), "Output", "LossScaling", + "update_loss_scaling"); + OP_INOUT_CHECK(ctx->HasOutput("OutGoodSteps"), "Output", "OutGoodSteps", + "update_loss_scaling"); + OP_INOUT_CHECK(ctx->HasOutput("OutBadSteps"), "Output", "OutBadSteps", + "update_loss_scaling"); + auto x_dims = ctx->GetInputsDim("X"); + ctx->SetOutputsDim("Out", x_dims); + ctx->SetOutputDim("LossScaling", {1}); + ctx->SetOutputDim("OutGoodSteps", {1}); + ctx->SetOutputDim("OutBadSteps", {1}); + } + + protected: + framework::OpKernelType GetExpectedKernelType( + const framework::ExecutionContext& ctx) const override { + return framework::OpKernelType( + OperatorWithKernel::IndicateVarDataType(ctx, "PrevLossScaling"), + ctx.device_context()); + } +}; + +class UpdateLossScalingOpMaker : public framework::OpProtoAndCheckerMaker { + public: + void Make() override { + AddInput("X", + "(Tensors) The input tensors of update_loss_scaling operator.") + .AsDuplicable(); + AddInput("FoundInfinite", + "(Tensor) 1-dim tensor, contains a bool scalar, which indicates " + "whether there is any infinite gradient."); + AddInput("PrevLossScaling", + "(Tensor) 1-dim tensor, previous loss scaling."); + AddInput("InGoodSteps", + "(Tensor) 1-dim tensor, accumulates good steps in which all " + "gradients are finite."); + AddInput("InBadSteps", + "(Tensor) 1-dim tensor, accumulates bad steps in which some " + "gradients are infinite."); + AddOutput("Out", + "(Tensors) The output tensor of update_loss_scaling operator.") + .AsDuplicable(); + AddOutput("LossScaling", "(Tensor) 1-dim tensor, updated loss scaling."); + AddOutput("OutGoodSteps", "(Tensor) 1-dim tensor, pdated good steps."); + AddOutput("OutBadSteps", "(Tensor) 1-dim tensor, updated bad steps."); + AddAttr("incr_every_n_steps", + "A value represents increasing loss scaling every n " + "consecutive steps with finite gradients."); + AddAttr("decr_every_n_nan_or_inf", + "A value represents decreasing loss scaling every n " + "accumulated steps with nan or inf gradients."); + AddAttr("incr_ratio", + "The multiplier to use when increasing the loss scaling.") + .AddCustomChecker([](float incr_ratio) { + PADDLE_ENFORCE_EQ(incr_ratio > 1.0f, true, + platform::errors::InvalidArgument( + "'incr_ratio' should be greater than 1, but " + "the received is %f", + incr_ratio)); + }); + AddAttr( + "decr_ratio", + "The less-than-one-multiplier to use when decreasing loss scaling.") + .AddCustomChecker([](float decr_ratio) { + PADDLE_ENFORCE_EQ(decr_ratio > 0.0f && decr_ratio < 1.0f, true, + platform::errors::InvalidArgument( + "'incr_ratio' should be between 0 and 1, but " + "the received is %f", + decr_ratio)); + }); + AddComment(R"DOC( +Update loss scaling according to overall gradients. If all gradients is +finite after incr_every_n_steps, loss scaling will increase by incr_ratio. +Otherwise, loss scaling will decrease by decr_ratio after +decr_every_n_nan_or_inf steps and each step some gradients are infinite. + +)DOC"); + } +}; + +template +class UpdateLossScalingFunctor { + public: + void operator()(const platform::CPUDeviceContext& ctx, + const bool* found_inf_data, const T* pre_loss_scaling_data, + const int* good_in_data, const int* bad_in_data, + const int incr_every_n_steps, + const int decr_every_n_nan_or_inf, const float incr_ratio, + const float decr_ratio, T* updated_loss_scaling_data, + int* good_out_data, int* bad_out_data) const { + Update(found_inf_data, pre_loss_scaling_data, good_in_data, bad_in_data, + incr_every_n_steps, decr_every_n_nan_or_inf, incr_ratio, + decr_ratio, updated_loss_scaling_data, good_out_data, + bad_out_data); + } +}; + +template +class LazyZeroInputs { + public: + void operator()(const platform::CPUDeviceContext& dev_ctx, + const bool* found_inf_data, + const std::vector& xs, + const std::vector& outs) const { + if (*found_inf_data) { + VLOG(1) << "-- UpdateLossScaling: Infinite values are found in grads. --"; + for (size_t i = 0; i < xs.size(); ++i) { + auto* out = outs[i]; + T* out_data = out->mutable_data(dev_ctx.GetPlace()); + int num = out->numel(); + std::memset(out_data, 0, num * sizeof(T)); + } + } + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +using CPU = paddle::platform::CPUDeviceContext; + +REGISTER_OPERATOR( + update_loss_scaling, ops::UpdateLossScalingOp, + ops::UpdateLossScalingOpMaker, + paddle::framework::EmptyGradOpMaker, + paddle::framework::EmptyGradOpMaker); + +REGISTER_OP_CPU_KERNEL(update_loss_scaling, + ops::UpdateLossScalingKernel, + ops::UpdateLossScalingKernel); diff --git a/paddle/fluid/operators/amp/update_loss_scaling_op.cu b/paddle/fluid/operators/amp/update_loss_scaling_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..2bc60423d247447adf18eb3ef050ca9b395a2e2f --- /dev/null +++ b/paddle/fluid/operators/amp/update_loss_scaling_op.cu @@ -0,0 +1,84 @@ +/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include +#include "paddle/fluid/framework/op_registry.h" +#include "paddle/fluid/operators/amp/update_loss_scaling_op.h" +#include "paddle/fluid/platform/enforce.h" + +namespace paddle { +namespace operators { + +template +__global__ void GpuUpdateLossScaling( + const bool* found_inf_data, const T* pre_loss_scaling_data, + const int* good_in_data, const int* bad_in_data, + const int incr_every_n_steps, const int decr_every_n_nan_or_inf, + const float incr_ratio, const float decr_ratio, + T* updated_loss_scaling_data, int* good_out_data, int* bad_out_data) { + Update(found_inf_data, pre_loss_scaling_data, good_in_data, bad_in_data, + incr_every_n_steps, decr_every_n_nan_or_inf, incr_ratio, decr_ratio, + updated_loss_scaling_data, good_out_data, bad_out_data); +} + +template +class UpdateLossScalingFunctor { + public: + void operator()(const platform::CUDADeviceContext& dev_ctx, + const bool* found_inf_data, const T* pre_loss_scaling_data, + const int* good_in_data, const int* bad_in_data, + const int incr_every_n_steps, + const int decr_every_n_nan_or_inf, const float incr_ratio, + const float decr_ratio, T* updated_loss_scaling_data, + int* good_out_data, int* bad_out_data) const { + GpuUpdateLossScaling<<<1, 1, 0, dev_ctx.stream()>>>( + found_inf_data, pre_loss_scaling_data, good_in_data, bad_in_data, + incr_every_n_steps, decr_every_n_nan_or_inf, incr_ratio, decr_ratio, + updated_loss_scaling_data, good_out_data, bad_out_data); + } +}; + +template +class LazyZeroInputs { + public: + void operator()(const platform::CUDADeviceContext& dev_ctx, + const bool* found_inf_data, + const std::vector& xs, + const std::vector& outs) const { + const auto gpu_place = + BOOST_GET_CONST(platform::CUDAPlace, dev_ctx.GetPlace()); + bool has_inf{false}; + memory::Copy(platform::CPUPlace(), &has_inf, gpu_place, found_inf_data, + sizeof(bool), dev_ctx.stream()); + if (has_inf) { + VLOG(1) << "-- UpdateLossScaling: Infinite values are found in grads. --"; + for (size_t i = 0; i < xs.size(); ++i) { + auto* out = outs[i]; + T* out_data = out->mutable_data(dev_ctx.GetPlace()); + int num = out->numel(); + cudaMemset(out_data, 0, num * sizeof(T)); + } + } + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +using GPU = paddle::platform::CUDADeviceContext; + +REGISTER_OP_CUDA_KERNEL(update_loss_scaling, + ops::UpdateLossScalingKernel, + ops::UpdateLossScalingKernel); diff --git a/paddle/fluid/operators/amp/update_loss_scaling_op.h b/paddle/fluid/operators/amp/update_loss_scaling_op.h new file mode 100644 index 0000000000000000000000000000000000000000..ca23b72eff0e85ab94c4d1f11e986f69b4e2d776 --- /dev/null +++ b/paddle/fluid/operators/amp/update_loss_scaling_op.h @@ -0,0 +1,123 @@ +// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include "paddle/fluid/framework/operator.h" +#include "paddle/fluid/platform/device_context.h" +#include "paddle/fluid/platform/enforce.h" +#include "paddle/fluid/platform/errors.h" +#include "paddle/fluid/platform/hostdevice.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; + +template +HOSTDEVICE void Update(const bool* found_inf_data, + const T* pre_loss_scaling_data, const int* good_in_data, + const int* bad_in_data, const int incr_every_n_steps, + const int decr_every_n_nan_or_inf, + const float incr_ratio, const float decr_ratio, + T* updated_loss_scaling_data, int* good_out_data, + int* bad_out_data) { + if (*found_inf_data) { + *good_out_data = 0; + *bad_out_data = *bad_in_data + 1; + if (*bad_out_data == decr_every_n_nan_or_inf) { + T new_loss_scaling = *pre_loss_scaling_data * decr_ratio; + *updated_loss_scaling_data = new_loss_scaling < static_cast(1) + ? static_cast(1) + : new_loss_scaling; + *bad_out_data = 0; + } + } else { + *bad_out_data = 0; + *good_out_data = *good_in_data + 1; + if (*good_out_data == incr_every_n_steps) { + T new_loss_scaling = *pre_loss_scaling_data * incr_ratio; + *updated_loss_scaling_data = std::isfinite(new_loss_scaling) + ? new_loss_scaling + : *pre_loss_scaling_data; + *good_out_data = 0; + } + } +} + +template +class UpdateLossScalingFunctor { + public: + void operator()(const DeviceContext& dev_ctx, const bool* found_inf_data, + const T* pre_loss_scaling_data, const int* good_in_data, + const int* bad_in_data, const int incr_every_n_steps, + const int decr_every_n_nan_or_inf, const float incr_ratio, + const float decr_ratio, T* updated_loss_scaling_data, + int* good_out_data, int* bad_out_data) const; +}; + +template +class LazyZeroInputs { + public: + void operator()(const DeviceContext& dev_ctx, const bool* found_inf_data, + const std::vector& xs, + const std::vector& outs) const; +}; + +template +class UpdateLossScalingKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + const auto xs = ctx.MultiInput("X"); + const auto* found_inf = ctx.Input("FoundInfinite"); + const auto* pre_loss_scaling = ctx.Input("PrevLossScaling"); + const auto* good_in = ctx.Input("InGoodSteps"); + const auto* bad_in = ctx.Input("InBadSteps"); + auto outs = ctx.MultiOutput("Out"); + auto* updated_loss_scaling = ctx.Output("LossScaling"); + auto* good_out = ctx.Output("OutGoodSteps"); + auto* bad_out = ctx.Output("OutBadSteps"); + + PADDLE_ENFORCE_EQ(found_inf->numel(), 1, + platform::errors::InvalidArgument( + "FoundInfinite must has only one element.")); + + const bool* found_inf_data = found_inf->data(); + const T* pre_loss_scaling_data = pre_loss_scaling->data(); + const int* good_in_data = good_in->data(); + const int* bad_in_data = bad_in->data(); + + auto& dev_ctx = ctx.template device_context(); + T* updated_loss_scaling_data = + updated_loss_scaling->mutable_data(dev_ctx.GetPlace()); + int* good_out_data = good_out->mutable_data(dev_ctx.GetPlace()); + int* bad_out_data = bad_out->mutable_data(dev_ctx.GetPlace()); + + const int incr_every_n_steps = ctx.Attr("incr_every_n_steps"); + const int decr_every_n_nan_or_inf = + ctx.Attr("decr_every_n_nan_or_inf"); + const float incr_ratio = ctx.Attr("incr_ratio"); + const float decr_ratio = ctx.Attr("decr_ratio"); + UpdateLossScalingFunctor{}( + dev_ctx, found_inf_data, pre_loss_scaling_data, good_in_data, + bad_in_data, incr_every_n_steps, decr_every_n_nan_or_inf, incr_ratio, + decr_ratio, updated_loss_scaling_data, good_out_data, bad_out_data); + LazyZeroInputs{}(dev_ctx, found_inf_data, xs, outs); + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/fluid/operators/controlflow/compare_op.cc b/paddle/fluid/operators/controlflow/compare_op.cc index 60f29ba39a8ee64f9fe5d95e685cac1fb52dfd21..4940649c2a32649a068c364081071ac840b4e25a 100644 --- a/paddle/fluid/operators/controlflow/compare_op.cc +++ b/paddle/fluid/operators/controlflow/compare_op.cc @@ -111,8 +111,16 @@ class CompareOp : public framework::OperatorWithKernel { framework::OpKernelType kt = OperatorWithKernel::GetExpectedKernelType(ctx); // CompareOp kernel's device type is decided by input tensor place bool force_cpu = ctx.Attr("force_cpu"); - kt.place_ = force_cpu ? platform::CPUPlace() - : ctx.Input("X")->place(); + if (force_cpu) { + kt.place_ = platform::CPUPlace(); + } else { + if (ctx.Input("X")->place().type() != + typeid(platform::CUDAPinnedPlace)) { + kt.place_ = ctx.Input("X")->place(); + } else { + kt.place_ = ctx.GetPlace(); + } + } return kt; } }; diff --git a/paddle/fluid/operators/correlation_op.cc b/paddle/fluid/operators/correlation_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..a2e6ff214bfa30af2e25b7feac41d22f02ab75a7 --- /dev/null +++ b/paddle/fluid/operators/correlation_op.cc @@ -0,0 +1,181 @@ +/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include +#include +#include +#include +#include "paddle/fluid/framework/op_registry.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; + +inline std::vector CorrelationOutputSize(int batch, int input_height, + int input_width, int stride1, + int stride2, int kernel_size, + int pad_size, + int max_displacement) { + std::vector output_shape({batch}); + int kernel_radius = (kernel_size - 1) / 2; + int border_radius = kernel_radius + max_displacement; + int padded_input_height = input_height + 2 * pad_size; + int padded_input_width = input_width + 2 * pad_size; + int output_channel = ((max_displacement / stride2) * 2 + 1) * + ((max_displacement / stride2) * 2 + 1); + output_shape.push_back(output_channel); + int output_height = + std::ceil(static_cast(padded_input_height - 2 * border_radius) / + static_cast(stride1)); + int output_width = + std::ceil(static_cast(padded_input_width - 2 * border_radius) / + static_cast(stride1)); + output_shape.push_back(output_height); + output_shape.push_back(output_width); + return output_shape; +} + +class CorrelationOpMaker : public framework::OpProtoAndCheckerMaker { + public: + void Make() override { + AddInput("Input1", "Input is a 4-D Tensor with shape [N, C, H, W]"); + AddInput("Input2", "Input is a 4-D Tensor with shape [N, C, H, W]"); + AddOutput("Output", + "(Tensor) The output tensor of correlation operator. " + "It has same data fromat and data type as the Input."); + AddAttr("pad_size", "pad size for input1 and input2"); + AddAttr("kernel_size", "kernel size of input1 and input2"); + AddAttr("max_displacement", "max displacement of input1 and input2"); + AddAttr("stride1", "Input1 stride"); + AddAttr("stride2", "Input2 stride"); + AddAttr("corr_type_multiply", "correlation coefficient").SetDefault(1); + AddComment( + R"DOC(Correlation of two feature map. Only support NCHW data format.)DOC"); + } +}; + +class CorrelationOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext* ctx) const override { + OP_INOUT_CHECK(ctx->HasInput("Input1"), "Input", "X", "CorrelationOp"); + OP_INOUT_CHECK(ctx->HasInput("Input2"), "Input", "Y", "CorrelationOp"); + int stride1 = ctx->Attrs().Get("stride1"); + int stride2 = ctx->Attrs().Get("stride2"); + int max_displacement = ctx->Attrs().Get("max_displacement"); + int pad_size = ctx->Attrs().Get("pad_size"); + int kernel_size = ctx->Attrs().Get("kernel_size"); + + auto in_dims = ctx->GetInputDim("Input1"); + auto in2_dims = ctx->GetInputDim("Input2"); + + PADDLE_ENFORCE_EQ(in_dims.size() == 4, true, + platform::errors::InvalidArgument( + "Input(X) of CorrelationOp must be 4 dims." + "But received dims is %d.", + in_dims.size())); + + PADDLE_ENFORCE_EQ(in2_dims.size() == 4, true, + platform::errors::InvalidArgument( + "Input(Y) of CorrelationOp must be 4 dims." + "But received dims is %d.", + in2_dims.size())); + std::vector output_shape = + CorrelationOutputSize(in_dims[0], in_dims[2], in_dims[3], stride1, + stride2, kernel_size, pad_size, max_displacement); + ctx->SetOutputDim("Output", framework::make_ddim(output_shape)); + } + + protected: + framework::OpKernelType GetExpectedKernelType( + const framework::ExecutionContext& ctx) const override { + auto input_data_type = + OperatorWithKernel::IndicateVarDataType(ctx, "Input1"); + PADDLE_ENFORCE_EQ(input_data_type, ctx.Input("Input2")->type(), + platform::errors::InvalidArgument( + "X and Y shoule have the same datatype")); + return framework::OpKernelType(input_data_type, ctx.GetPlace()); + } + + framework::OpKernelType GetKernelTypeForVar( + const std::string& var_name, const Tensor& tensor, + const framework::OpKernelType& expected_kernel_type) const override { + return framework::OpKernelType(expected_kernel_type.data_type_, + tensor.place(), tensor.layout()); + } +}; + +template +class CorrelationOpGradMaker : public framework::SingleGradOpMaker { + public: + using framework::SingleGradOpMaker::SingleGradOpMaker; + + protected: + void Apply(GradOpPtr op) const override { + op->SetType("correlation_grad"); + op->SetInput("Input1", this->Input("Input1")); + op->SetInput("Input2", this->Input("Input2")); + op->SetInput(framework::GradVarName("Output"), this->OutputGrad("Output")); + op->SetOutput(framework::GradVarName("Input1"), this->InputGrad("Input1")); + op->SetOutput(framework::GradVarName("Input2"), this->InputGrad("Input2")); + op->SetAttrMap(this->Attrs()); + } +}; + +class CorrelationOpGrad : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext* ctx) const override { + OP_INOUT_CHECK(ctx->HasInput("Input1"), "Input", "X", "CorrelationOp"); + OP_INOUT_CHECK(ctx->HasInput("Input2"), "Input", "Y", "CorrelationOp"); + OP_INOUT_CHECK(ctx->HasInput(framework::GradVarName("Output")), "Input", + "Output@GRAD", "CorrelationGradOp"); + + auto in1_dims = ctx->GetInputDim("Input1"); + auto in2_dims = ctx->GetInputDim("Input2"); + ctx->SetOutputDim(framework::GradVarName("Input1"), in1_dims); + ctx->SetOutputDim(framework::GradVarName("Input2"), in2_dims); + } + + protected: + framework::OpKernelType GetExpectedKernelType( + const framework::ExecutionContext& ctx) const override { + return framework::OpKernelType( + OperatorWithKernel::IndicateVarDataType(ctx, "Input1"), ctx.GetPlace()); + } +}; + +template +class CorrelationKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + PADDLE_ENFORCE_EQ( + platform::is_gpu_place(ctx.GetPlace()), true, + platform::errors::Unimplemented("Correlation only supports GPU now.")); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OPERATOR(correlation, ops::CorrelationOp, ops::CorrelationOpMaker, + ops::CorrelationOpGradMaker, + ops::CorrelationOpGradMaker); +REGISTER_OPERATOR(correlation_grad, ops::CorrelationOpGrad); +REGISTER_OP_CPU_KERNEL(correlation, ops::CorrelationKernel, + ops::CorrelationKernel); diff --git a/paddle/fluid/operators/correlation_op.cu b/paddle/fluid/operators/correlation_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..0d177f653ec3d5cacbd4d938e6e5da6689b1bc74 --- /dev/null +++ b/paddle/fluid/operators/correlation_op.cu @@ -0,0 +1,483 @@ +/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include +#include +#include "paddle/fluid/framework/op_registry.h" + +namespace paddle { +namespace operators { + +#define THREADS_PER_BLOCK 32 +#define FULL_MASK 0xffffffff + +using framework::Tensor; +using DataLayout = framework::DataLayout; + +template +__forceinline__ __device__ T warpReduceSum(T val) { + for (int offset = 16; offset > 0; offset /= 2) { + val += __shfl_down_sync(FULL_MASK, val, offset); + } + return val; +} + +template +__forceinline__ __device__ T blockReduceSum(T val) { + static __shared__ T shared[32]; + int lane = threadIdx.x % warpSize; + int wid = threadIdx.x / warpSize; + + val = warpReduceSum(val); + if (lane == 0) shared[wid] = val; + + __syncthreads(); + val = (threadIdx.x < blockDim.x / warpSize) ? shared[lane] : 0; + + if (wid == 0) val = warpReduceSum(val); + + return val; +} + +template +__global__ void set_zero(T *x, int num) { + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < num; + i += blockDim.x * gridDim.x) + x[i] = static_cast(0); +} + +template +__global__ void channel_first(const T *input, T *rinput, const int channel, + const int height, const int width, + const int pad_size) { + int n = blockIdx.x; + int h = blockIdx.y; + int w = blockIdx.z; + + int ch_off = threadIdx.x; + T value; + int dimchw = channel * height * width; + int dimhw = height * width; + + int p_dimw = (width + 2 * pad_size); + int p_dimh = (height + 2 * pad_size); + int p_dimchw = channel * p_dimw * p_dimh; + int p_dimcw = channel * p_dimw; + + for (int c = ch_off; c < channel; c += THREADS_PER_BLOCK) { + value = input[n * dimchw + c * dimhw + h * width + w]; + rinput[n * p_dimchw + (h + pad_size) * p_dimcw + (w + pad_size) * channel + + c] = value; + } +} + +template +__global__ void correlation_forward( + T *output, const int output_channel, const int output_height, + const int output_width, const T *rinput1, const int input_channel, + const int input_height, const int input_width, const T *rinput2, + const int pad_size, const int kernel_size, const int max_displacement, + const int stride1, const int stride2) { + int p_input_width = input_width + 2 * pad_size; + int p_input_height = input_height + 2 * pad_size; + + int kernel_rad = (kernel_size - 1) / 2; + int displacement_rad = max_displacement / stride2; + + int displacement_size = 2 * displacement_rad + 1; + + int n = blockIdx.x; + int h1 = blockIdx.y * stride1 + max_displacement; + int w1 = blockIdx.z * stride1 + max_displacement; + int c = threadIdx.x; + + int p_dimchw = p_input_height * p_input_width * input_channel; + int p_dimcw = p_input_width * input_channel; + int p_dimc = input_channel; + + int t_dimchw = output_channel * output_height * output_width; + int t_dimhw = output_height * output_width; + int t_dimw = output_width; + + int nelems = kernel_size * kernel_size * p_dimc; + + for (int tj = -displacement_rad; tj <= displacement_rad; ++tj) { + for (int ti = -displacement_rad; ti <= displacement_rad; ++ti) { + int w2 = w1 + ti * stride2; + int h2 = h1 + tj * stride2; + + T acc0 = 0; + for (int j = -kernel_rad; j <= kernel_rad; ++j) { + for (int i = -kernel_rad; i <= kernel_rad; ++i) { + for (int ch = c; ch < p_dimc; ch += blockDim.x) { + int index1 = + n * p_dimchw + (h1 + j) * p_dimcw + (w1 + i) * p_dimc + ch; + int index2 = + n * p_dimchw + (h2 + j) * p_dimcw + (w2 + i) * p_dimc + ch; + acc0 += static_cast(rinput1[index1] * rinput2[index2]); + } + } + } + if (blockDim.x == warpSize) { + __syncwarp(); + acc0 = warpReduceSum(acc0); + } else { + __syncthreads(); + acc0 = blockReduceSum(acc0); + } + + if (threadIdx.x == 0) { + int tc = (tj + displacement_rad) * displacement_size + + (ti + displacement_rad); + const int t_index = + n * t_dimchw + tc * t_dimhw + blockIdx.y * t_dimw + blockIdx.z; + output[t_index] = static_cast(acc0 / nelems); + } + } + } +} + +// class CorrelationKernel +template +class CorrelationCUDAKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext &ctx) const override { + PADDLE_ENFORCE_EQ(platform::is_gpu_place(ctx.GetPlace()), true, + platform::errors::InvalidArgument( + "Correlation only supports GPU now.")); + + auto *input1 = ctx.Input("Input1"); + auto *input2 = ctx.Input("Input2"); + int pad_size = ctx.Attr("pad_size"); + int kernel_size = ctx.Attr("kernel_size"); + int stride1 = ctx.Attr("stride1"); + int stride2 = ctx.Attr("stride2"); + int max_displacement = ctx.Attr("max_displacement"); + int corr_type_multiply = ctx.Attr("corr_type_multiply"); + + auto *output = ctx.Output("Output"); + output->mutable_data(ctx.GetPlace()); + auto &dev_ctx = ctx.template device_context(); + + // base on input1, NCHW + auto in_dims = input1->dims(); + int N = in_dims[0]; + int C = in_dims[1]; + int H = in_dims[2]; + int W = in_dims[3]; + + int padded_input_height = H + 2 * pad_size; + int padded_input_width = W + 2 * pad_size; + + Tensor rinput1 = ctx.AllocateTmpTensor( + {N, padded_input_height, padded_input_width, C}, dev_ctx); + rinput1.mutable_data(ctx.GetPlace()); + + Tensor rinput2 = ctx.AllocateTmpTensor( + {N, padded_input_height, padded_input_width, C}, dev_ctx); + rinput2.mutable_data(ctx.GetPlace()); + + set_zero<<<(rinput1.numel() + 512 - 1) / 512, 512, 0, dev_ctx.stream()>>>( + rinput1.data(), rinput1.numel()); + set_zero<<<(rinput2.numel() + 512 - 1) / 512, 512, 0, dev_ctx.stream()>>>( + rinput2.data(), rinput2.numel()); + set_zero<<<(output->numel() + 512 - 1) / 512, 512, 0, dev_ctx.stream()>>>( + output->data(), output->numel()); + + auto out_dims = output->dims(); + int OC = out_dims[1]; + int OH = out_dims[2]; + int OW = out_dims[3]; + + dim3 blocks_grid(N, H, W); + dim3 threads_block(THREADS_PER_BLOCK); + + channel_first<<>>( + input1->data(), rinput1.data(), C, H, W, pad_size); + channel_first<<>>( + input2->data(), rinput2.data(), C, H, W, pad_size); + + dim3 threadsPerBlock(THREADS_PER_BLOCK); + dim3 totalBlocksCorr(N, OH, OW); + + correlation_forward< + T><<>>( + output->data(), OC, OH, OW, rinput1.data(), C, H, W, + rinput2.data(), pad_size, kernel_size, max_displacement, stride1, + stride2); + } +}; + +template +__global__ void correlation_backward_input1( + int item, T *grad_input1, const int input_channel, const int input_height, + const int input_width, const T *grad_output, const int output_channel, + const int output_height, const int output_width, const T *rinput2, + const int pad_size, const int kernel_size, const int max_displacement, + const int stride1, const int stride2) { + int n = item; + int h = blockIdx.x * stride1 + pad_size; + int w = blockIdx.y * stride1 + pad_size; + int c = blockIdx.z; + int tch_off = threadIdx.x; + + int kernel_rad = (kernel_size - 1) / 2; + int displacement_rad = max_displacement / stride2; + int displacement_size = 2 * displacement_rad + 1; + + int xmin = (w - kernel_rad - max_displacement) / stride1; + int ymin = (h - kernel_rad - max_displacement) / stride1; + + int xmax = (w + kernel_rad - max_displacement) / stride1; + int ymax = (h + kernel_rad - max_displacement) / stride1; + + if (xmax < 0 || ymax < 0 || xmin >= output_width || ymin >= output_height) { + return; + } + + if (xmin > xmax || ymin > ymax) { + return; + } + + xmin = max(0, xmin); + xmax = min(output_width - 1, xmax); + + ymin = max(0, ymin); + ymax = min(output_height - 1, ymax); + + int p_input_width = input_width + 2 * pad_size; + int p_input_height = input_height + 2 * pad_size; + int p_dimchw = input_channel * p_input_height * p_input_width; + int p_dimcw = input_channel * p_input_width; + int p_dimc = input_channel; + + int t_dimchw = output_channel * output_height * output_width; + int t_dimhw = output_height * output_width; + int t_dimw = output_width; + + int o_dimchw = input_channel * input_height * input_width; + int o_dimhw = input_height * input_width; + int o_dimw = input_width; + + int nelems = kernel_size * kernel_size * input_channel; + + __shared__ T prod_sum[THREADS_PER_BLOCK]; + prod_sum[tch_off] = 0; + + for (int tc = tch_off; tc < output_channel; tc += THREADS_PER_BLOCK) { + int i2 = (tc % displacement_size - displacement_rad) * stride2; + int j2 = (tc / displacement_size - displacement_rad) * stride2; + + int index2 = n * p_dimchw + (h + j2) * p_dimcw + (w + i2) * p_dimc + c; + + T val2 = rinput2[index2]; + for (int j = ymin; j <= ymax; ++j) { + for (int i = xmin; i <= xmax; ++i) { + int t_index = n * t_dimchw + tc * t_dimhw + j * t_dimw + i; + prod_sum[tch_off] += grad_output[t_index] * val2; + } + } + } + + __syncthreads(); + + if (tch_off == 0) { + T reduce_sum = 0; + for (int index = 0; index < THREADS_PER_BLOCK; index++) { + reduce_sum += prod_sum[index]; + } + const int index1 = + n * o_dimchw + c * o_dimhw + (h - pad_size) * o_dimw + (w - pad_size); + grad_input1[index1] = static_cast(reduce_sum / nelems); + } +} + +template +__global__ void correlation_backward_input2( + int item, T *grad_input2, const int input_channel, const int input_height, + const int input_width, const T *grad_output, const int output_channel, + const int output_height, const int output_width, const T *rinput1, + const int pad_size, const int kernel_size, const int max_displacement, + const int stride1, const int stride2) { + int n = item; + int h = blockIdx.x * stride1 + pad_size; + int w = blockIdx.y * stride1 + pad_size; + int c = blockIdx.z; + + int tch_off = threadIdx.x; + + int kernel_rad = (kernel_size - 1) / 2; + int displacement_rad = max_displacement / stride2; + int displacement_size = 2 * displacement_rad + 1; + + int p_input_width = input_width + 2 * pad_size; + int p_input_height = input_height + 2 * pad_size; + int p_dimchw = input_channel * p_input_height * p_input_width; + int p_dimcw = input_channel * p_input_width; + int p_dimc = input_channel; + + int t_dimchw = output_channel * output_height * output_width; + int t_dimhw = output_height * output_width; + int t_dimw = output_width; + + int o_dimchw = input_channel * input_height * input_width; + int o_dimhw = input_height * input_width; + int o_dimw = input_width; + + int nelems = kernel_size * kernel_size * input_channel; + + __shared__ T prod_sum[THREADS_PER_BLOCK]; + prod_sum[tch_off] = 0; + + for (int tc = tch_off; tc < output_channel; tc += THREADS_PER_BLOCK) { + int i2 = (tc % displacement_size - displacement_rad) * stride2; + int j2 = (tc / displacement_size - displacement_rad) * stride2; + + int xmin = (w - kernel_rad - max_displacement - i2) / stride1; + int ymin = (h - kernel_rad - max_displacement - j2) / stride1; + + int xmax = (w + kernel_rad - max_displacement - i2) / stride1; + int ymax = (h + kernel_rad - max_displacement - j2) / stride1; + + if (xmax < 0 || ymax < 0 || xmin >= output_width || ymin >= output_height) { + continue; + } + + if (xmin > xmax || ymin > ymax) { + continue; + } + + xmin = max(0, xmin); + xmax = min(output_width - 1, xmax); + + ymin = max(0, ymin); + ymax = min(output_height - 1, ymax); + + int index1 = n * p_dimchw + (h - j2) * p_dimcw + (w - i2) * p_dimc + c; + T val1 = rinput1[index1]; + for (int j = ymin; j <= ymax; ++j) { + for (int i = xmin; i <= xmax; ++i) { + int t_index = n * t_dimchw + tc * t_dimhw + j * t_dimw + i; + prod_sum[tch_off] += grad_output[t_index] * val1; + } + } + } + + __syncthreads(); + + if (tch_off == 0) { + T reduce_sum = 0; + for (int index = 0; index < THREADS_PER_BLOCK; index++) { + reduce_sum += prod_sum[index]; + } + const int index2 = + n * o_dimchw + c * o_dimhw + (h - pad_size) * o_dimw + (w - pad_size); + grad_input2[index2] = static_cast(reduce_sum / nelems); + } +} + +template +class CorrelationCUDAGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext &ctx) const override { + PADDLE_ENFORCE_EQ(platform::is_gpu_place(ctx.GetPlace()), true, + platform::errors::InvalidArgument( + "Correlation only supports GPU now.")); + const auto *input1 = ctx.Input("Input1"); + const auto *input2 = ctx.Input("Input2"); + const auto *grad_output = + ctx.Input(framework::GradVarName("Output")); + const int pad_size = ctx.Attr("pad_size"); + const int kernel_size = ctx.Attr("kernel_size"); + const int stride1 = ctx.Attr("stride1"); + const int stride2 = ctx.Attr("stride2"); + const int max_displacement = ctx.Attr("max_displacement"); + const int corr_type_multiply = ctx.Attr("corr_type_multiply"); + + auto *grad_input1 = ctx.Output(framework::GradVarName("Input1")); + grad_input1->mutable_data(ctx.GetPlace()); + auto *grad_input2 = ctx.Output(framework::GradVarName("Input2")); + grad_input2->mutable_data(ctx.GetPlace()); + auto &dev_ctx = ctx.template device_context(); + + auto in_dims = input1->dims(); + int N = in_dims[0]; + int C = in_dims[1]; + int H = in_dims[2]; + int W = in_dims[3]; + + int padded_input_height = H + 2 * pad_size; + int padded_input_width = W + 2 * pad_size; + + Tensor rinput1 = ctx.AllocateTmpTensor( + {N, padded_input_height, padded_input_width, C}, dev_ctx); + rinput1.mutable_data(ctx.GetPlace()); + + Tensor rinput2 = ctx.AllocateTmpTensor( + {N, padded_input_height, padded_input_width, C}, dev_ctx); + rinput2.mutable_data(ctx.GetPlace()); + + set_zero<<<(rinput1.numel() + 512 - 1) / 512, 512, 0, dev_ctx.stream()>>>( + rinput1.data(), rinput1.numel()); + set_zero<<<(rinput2.numel() + 512 - 1) / 512, 512, 0, dev_ctx.stream()>>>( + rinput2.data(), rinput2.numel()); + set_zero<<<(grad_input1->numel() + 512 - 1) / 512, 512, 0, + dev_ctx.stream()>>>(grad_input1->data(), + grad_input1->numel()); + set_zero<<<(grad_input2->numel() + 512 - 1) / 512, 512, 0, + dev_ctx.stream()>>>(grad_input2->data(), + grad_input2->numel()); + + auto grad_out_dims = grad_output->dims(); + int GOC = grad_out_dims[1]; + int GOH = grad_out_dims[2]; + int GOW = grad_out_dims[3]; + + dim3 blocks_grid(N, H, W); + dim3 threads_block(THREADS_PER_BLOCK); + + channel_first<<>>( + input1->data(), rinput1.data(), C, H, W, pad_size); + channel_first<<>>( + input2->data(), rinput2.data(), C, H, W, pad_size); + + dim3 threadsPerBlock(THREADS_PER_BLOCK); + dim3 totalBlocksCorr(H, W, C); + + for (int n = 0; n < N; n++) { + correlation_backward_input1< + T><<>>( + n, grad_input1->data(), C, H, W, grad_output->data(), GOC, GOH, + GOW, rinput2.data(), pad_size, kernel_size, max_displacement, + stride1, stride2); + } + + for (int n = 0; n < N; n++) { + correlation_backward_input2< + T><<>>( + n, grad_input2->data(), C, H, W, grad_output->data(), GOC, GOH, + GOW, rinput1.data(), pad_size, kernel_size, max_displacement, + stride1, stride2); + } + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP_CUDA_KERNEL(correlation, ops::CorrelationCUDAKernel, + ops::CorrelationCUDAKernel); +REGISTER_OP_CUDA_KERNEL(correlation_grad, ops::CorrelationCUDAGradKernel, + ops::CorrelationCUDAGradKernel); diff --git a/paddle/fluid/operators/detection/CMakeLists.txt b/paddle/fluid/operators/detection/CMakeLists.txt index 537063640e4ef6e49f7b991482f0f3122ecef02f..c2b7c27ab4adb5282ad7aa5f7a16c15f81ba5f5e 100644 --- a/paddle/fluid/operators/detection/CMakeLists.txt +++ b/paddle/fluid/operators/detection/CMakeLists.txt @@ -41,9 +41,13 @@ detection_library(sigmoid_focal_loss_op SRCS sigmoid_focal_loss_op.cc sigmoid_fo detection_library(retinanet_detection_output_op SRCS retinanet_detection_output_op.cc) if(WITH_GPU) - detection_library(generate_proposals_op SRCS generate_proposals_op.cc generate_proposals_op.cu DEPS memory cub) - detection_library(distribute_fpn_proposals_op SRCS distribute_fpn_proposals_op.cc distribute_fpn_proposals_op.cu DEPS memory cub) - detection_library(collect_fpn_proposals_op SRCS collect_fpn_proposals_op.cc collect_fpn_proposals_op.cu DEPS memory cub) + set(TMPDEPS memory) + if (${CMAKE_CUDA_COMPILER_VERSION} LESS 11.0) + set(TMPDEPS memory cub) + endif() + detection_library(generate_proposals_op SRCS generate_proposals_op.cc generate_proposals_op.cu DEPS ${TMPDEPS}) + detection_library(distribute_fpn_proposals_op SRCS distribute_fpn_proposals_op.cc distribute_fpn_proposals_op.cu DEPS ${TMPDEPS}) + detection_library(collect_fpn_proposals_op SRCS collect_fpn_proposals_op.cc collect_fpn_proposals_op.cu DEPS ${TMPDEPS}) else() detection_library(generate_proposals_op SRCS generate_proposals_op.cc) detection_library(distribute_fpn_proposals_op SRCS distribute_fpn_proposals_op.cc) diff --git a/paddle/fluid/operators/detection/collect_fpn_proposals_op.cc b/paddle/fluid/operators/detection/collect_fpn_proposals_op.cc index b3e3332fe3425301a3dede3a3d810697ad4debf3..44f602237da2e2c8fa26e39326f977d10235155d 100644 --- a/paddle/fluid/operators/detection/collect_fpn_proposals_op.cc +++ b/paddle/fluid/operators/detection/collect_fpn_proposals_op.cc @@ -10,6 +10,7 @@ See the License for the specific language governing permissions and limitations under the License.*/ #include "paddle/fluid/operators/detection/collect_fpn_proposals_op.h" +#include "paddle/fluid/framework/op_version_registry.h" namespace paddle { namespace operators { @@ -54,11 +55,14 @@ class CollectFpnProposalsOp : public framework::OperatorWithKernel { score_dim[1])); } context->SetOutputDim("FpnRois", {post_nms_topN, 4}); + if (context->HasOutput("RoisNum")) { + context->SetOutputDim("RoisNum", {-1}); + } if (!context->IsRuntime()) { // Runtime LoD infershape will be computed // in Kernel. context->ShareLoD("MultiLevelRois", "FpnRois"); } - if (context->IsRuntime()) { + if (context->IsRuntime() && !context->HasInputs("MultiLevelRoIsNum")) { std::vector roi_inputs = context->GetInputVarPtrs("MultiLevelRois"); std::vector score_inputs = @@ -99,7 +103,16 @@ class CollectFpnProposalsOpMaker : public framework::OpProtoAndCheckerMaker { "(LoDTensor) Multiple score LoDTensors from each level in shape" " (N, 1), N is the number of RoIs.") .AsDuplicable(); + AddInput( + "MultiLevelRoIsNum", + "(List of Tensor) The RoIs' number of each image on multiple levels." + "The number on each level has the shape of (N), N is the number of " + "images.") + .AsDuplicable() + .AsDispensable(); AddOutput("FpnRois", "(LoDTensor) All selected RoIs with highest scores"); + AddOutput("RoisNum", "(Tensor), Number of RoIs in each images.") + .AsDispensable(); AddAttr("post_nms_topN", "Select post_nms_topN RoIs from" " all images and all fpn layers"); @@ -123,3 +136,14 @@ REGISTER_OPERATOR( REGISTER_OP_CPU_KERNEL(collect_fpn_proposals, ops::CollectFpnProposalsOpKernel, ops::CollectFpnProposalsOpKernel); +REGISTER_OP_VERSION(collect_fpn_proposals) + .AddCheckpoint( + R"ROC( + Upgrade collect_fpn_proposals add a new input + [MultiLevelRoIsNum] and add a new output [RoisNum].)ROC", + paddle::framework::compatible::OpVersionDesc() + .NewInput("MultiLevelRoIsNum", + "The RoIs' number of each image on multiple levels." + "The number on each level has the shape of (N), " + "N is the number of images.") + .NewOutput("RoisNum", "The number of RoIs in each image.")); diff --git a/paddle/fluid/operators/detection/collect_fpn_proposals_op.cu b/paddle/fluid/operators/detection/collect_fpn_proposals_op.cu index 35222a85cd388f6fef3c61c440be7b36598d9e01..86207052bb2bef4f7bea34c2614fe7686f579de8 100644 --- a/paddle/fluid/operators/detection/collect_fpn_proposals_op.cu +++ b/paddle/fluid/operators/detection/collect_fpn_proposals_op.cu @@ -80,14 +80,27 @@ class GPUCollectFpnProposalsOpKernel : public framework::OpKernel { int lod_size; auto place = BOOST_GET_CONST(platform::CUDAPlace, dev_ctx.GetPlace()); + auto multi_rois_num = ctx.MultiInput("MultiLevelRoIsNum"); for (size_t i = 0; i < roi_ins.size(); ++i) { auto roi_in = roi_ins[i]; auto score_in = score_ins[i]; - auto roi_lod = roi_in->lod().back(); - lod_size = roi_lod.size() - 1; - for (size_t n = 0; n < lod_size; ++n) { - for (size_t j = roi_lod[n]; j < roi_lod[n + 1]; ++j) { - roi_batch_id_data[index++] = n; + if (multi_rois_num.size() > 0) { + framework::Tensor temp; + TensorCopySync(*multi_rois_num[i], platform::CPUPlace(), &temp); + const int* length_in = temp.data(); + lod_size = multi_rois_num[i]->numel(); + for (size_t n = 0; n < lod_size; ++n) { + for (size_t j = 0; j < length_in[n]; ++j) { + roi_batch_id_data[index++] = n; + } + } + } else { + auto length_in = roi_in->lod().back(); + lod_size = length_in.size() - 1; + for (size_t n = 0; n < lod_size; ++n) { + for (size_t j = length_in[n]; j < length_in[n + 1]; ++j) { + roi_batch_id_data[index++] = n; + } } } @@ -190,6 +203,13 @@ class GPUCollectFpnProposalsOpKernel : public framework::OpKernel { offset.emplace_back(offset.back() + length_lod_cpu[i]); } + if (ctx.HasOutput("RoisNum")) { + auto* rois_num = ctx.Output("RoisNum"); + int* rois_num_data = rois_num->mutable_data({lod_size}, place); + memory::Copy(place, rois_num_data, place, length_lod_data, + lod_size * sizeof(int), dev_ctx.stream()); + } + framework::LoD lod; lod.emplace_back(offset); fpn_rois->set_lod(lod); diff --git a/paddle/fluid/operators/detection/collect_fpn_proposals_op.h b/paddle/fluid/operators/detection/collect_fpn_proposals_op.h index badd88f0689ba9defcb3f26eb57fef89308aa877..950b8b78933bff6bf1692df61142258dfbc87a8c 100644 --- a/paddle/fluid/operators/detection/collect_fpn_proposals_op.h +++ b/paddle/fluid/operators/detection/collect_fpn_proposals_op.h @@ -17,6 +17,7 @@ limitations under the License.*/ #include #include #include +#include #include #include #include "paddle/fluid/framework/op_registry.h" @@ -65,6 +66,8 @@ class CollectFpnProposalsOpKernel : public framework::OpKernel { auto multi_layer_scores = context.MultiInput("MultiLevelScores"); + auto multi_rois_num = context.MultiInput("MultiLevelRoIsNum"); + int num_size = multi_rois_num.size(); auto* fpn_rois = context.Output("FpnRois"); @@ -88,11 +91,21 @@ class CollectFpnProposalsOpKernel : public framework::OpKernel { const int num_fpn_level = multi_layer_rois.size(); std::vector integral_of_all_rois(num_fpn_level + 1, 0); for (int i = 0; i < num_fpn_level; ++i) { - auto cur_rois_lod = multi_layer_rois[i]->lod().back(); - integral_of_all_rois[i + 1] = - integral_of_all_rois[i] + cur_rois_lod[cur_rois_lod.size() - 1]; + int all_rois = 0; + if (num_size == 0) { + auto cur_rois_lod = multi_layer_rois[i]->lod().back(); + all_rois = cur_rois_lod[cur_rois_lod.size() - 1]; + } else { + const int* cur_rois_num = multi_rois_num[i]->data(); + all_rois = std::accumulate( + cur_rois_num, cur_rois_num + multi_rois_num[i]->numel(), 0); + } + integral_of_all_rois[i + 1] = integral_of_all_rois[i] + all_rois; } + const int batch_size = (num_size == 0) + ? multi_layer_rois[0]->lod().back().size() - 1 + : multi_rois_num[0]->numel(); // concatenate all fpn rois scores into a list // create a vector to store all scores std::vector> scores_of_all_rois( @@ -100,11 +113,20 @@ class CollectFpnProposalsOpKernel : public framework::OpKernel { for (int i = 0; i < num_fpn_level; ++i) { const T* cur_level_scores = multi_layer_scores[i]->data(); int cur_level_num = integral_of_all_rois[i + 1] - integral_of_all_rois[i]; - auto cur_scores_lod = multi_layer_scores[i]->lod().back(); int cur_batch_id = 0; + int pre_num = 0; for (int j = 0; j < cur_level_num; ++j) { - if (static_cast(j) >= cur_scores_lod[cur_batch_id + 1]) { - cur_batch_id++; + if (num_size == 0) { + auto cur_scores_lod = multi_layer_scores[i]->lod().back(); + if (static_cast(j) >= cur_scores_lod[cur_batch_id + 1]) { + cur_batch_id++; + } + } else { + const int* rois_num_data = multi_rois_num[i]->data(); + if (j >= pre_num + rois_num_data[cur_batch_id]) { + pre_num += rois_num_data[cur_batch_id]; + cur_batch_id++; + } } int cur_index = j + integral_of_all_rois[i]; scores_of_all_rois[cur_index].score = cur_level_scores[j]; @@ -134,6 +156,9 @@ class CollectFpnProposalsOpKernel : public framework::OpKernel { T* fpn_rois_data = fpn_rois->data(); std::vector lod0(1, 0); int cur_batch_id = 0; + std::vector num_per_batch; + int pre_idx = 0; + int cur_num = 0; for (int i = 0; i < post_nms_topN; ++i) { int cur_fpn_level = scores_of_all_rois[i].level; int cur_level_index = scores_of_all_rois[i].index; @@ -144,6 +169,18 @@ class CollectFpnProposalsOpKernel : public framework::OpKernel { if (scores_of_all_rois[i].batch_id != cur_batch_id) { cur_batch_id = scores_of_all_rois[i].batch_id; lod0.emplace_back(i); + cur_num = i - pre_idx; + pre_idx = i; + num_per_batch.emplace_back(cur_num); + } + } + num_per_batch.emplace_back(post_nms_topN - pre_idx); + if (context.HasOutput("RoisNum")) { + auto* rois_num = context.Output("RoisNum"); + int* rois_num_data = + rois_num->mutable_data({batch_size}, context.GetPlace()); + for (int i = 0; i < batch_size; i++) { + rois_num_data[i] = num_per_batch[i]; } } lod0.emplace_back(post_nms_topN); diff --git a/paddle/fluid/operators/detection/distribute_fpn_proposals_op.cc b/paddle/fluid/operators/detection/distribute_fpn_proposals_op.cc index 160d43a917b3c74ff905e070714415d35c5c877c..614b37e703e721337057e04c5611386ff87a1e9e 100644 --- a/paddle/fluid/operators/detection/distribute_fpn_proposals_op.cc +++ b/paddle/fluid/operators/detection/distribute_fpn_proposals_op.cc @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/fluid/operators/detection/distribute_fpn_proposals_op.h" +#include "paddle/fluid/framework/op_version_registry.h" namespace paddle { namespace operators { @@ -48,6 +49,14 @@ class DistributeFpnProposalsOp : public framework::OperatorWithKernel { } ctx->SetOutputsDim("MultiFpnRois", outs_dims); ctx->SetOutputDim("RestoreIndex", {-1, 1}); + + if (ctx->HasOutputs("MultiLevelRoIsNum")) { + std::vector outs_num_dims; + for (size_t i = 0; i < num_out_rois; ++i) { + outs_num_dims.push_back({-1}); + } + ctx->SetOutputsDim("MultiLevelRoIsNum", outs_num_dims); + } if (!ctx->IsRuntime()) { for (size_t i = 0; i < num_out_rois; ++i) { ctx->SetLoDLevel("MultiFpnRois", ctx->GetLoDLevel("FpnRois"), i); @@ -66,12 +75,22 @@ class DistributeFpnProposalsOp : public framework::OperatorWithKernel { class DistributeFpnProposalsOpMaker : public framework::OpProtoAndCheckerMaker { public: void Make() override { - AddInput("FpnRois", "(LoDTensor) The rois at all levels in shape (-1, 4)"); + AddInput("FpnRois", "(LoDTensor) The RoIs at all levels in shape (-1, 4)"); + AddInput("RoisNum", + "(Tensor) The number of RoIs in shape (B)," + "B is the number of images") + .AsDispensable(); AddOutput("MultiFpnRois", "(LoDTensor) Output with distribute operator") .AsDuplicable(); AddOutput("RestoreIndex", "(Tensor) An array of positive number which is " "used to restore the order of FpnRois"); + AddOutput("MultiLevelRoIsNum", + "(List of Tensor) The RoIs' number of each image on multiple " + "levels. The number on each level has the shape of (B)," + "B is the number of images.") + .AsDuplicable() + .AsDispensable(); AddAttr("min_level", "The lowest level of FPN layer where the" " proposals come from"); @@ -105,3 +124,14 @@ REGISTER_OPERATOR( REGISTER_OP_CPU_KERNEL(distribute_fpn_proposals, ops::DistributeFpnProposalsOpKernel, ops::DistributeFpnProposalsOpKernel); +REGISTER_OP_VERSION(distribute_fpn_proposals) + .AddCheckpoint( + R"ROC( + Upgrade distribute_fpn_proposals add a new input + [RoisNum] and add a new output [MultiLevelRoIsNum].)ROC", + paddle::framework::compatible::OpVersionDesc() + .NewInput("RoIsNum", "The number of RoIs in each image.") + .NewOutput("MultiLevelRoisNum", + "The RoIs' number of each image on multiple " + "levels. The number on each level has the shape of (B)," + "B is the number of images.")); diff --git a/paddle/fluid/operators/detection/distribute_fpn_proposals_op.cu b/paddle/fluid/operators/detection/distribute_fpn_proposals_op.cu index 1e3cd9f36c595f978f5b5e5f5c5cf5cad6dc9059..27c06a0f8fb207b5dc85c7875ea91428b16e606c 100644 --- a/paddle/fluid/operators/detection/distribute_fpn_proposals_op.cu +++ b/paddle/fluid/operators/detection/distribute_fpn_proposals_op.cu @@ -76,12 +76,20 @@ class GPUDistributeFpnProposalsOpKernel : public framework::OpKernel { int num_level = max_level - min_level + 1; // check that the fpn_rois is not empty - PADDLE_ENFORCE_EQ( - fpn_rois->lod().size(), 1UL, - platform::errors::InvalidArgument("DistributeFpnProposalsOp needs LoD" - "with one level")); + if (!ctx.HasInput("RoisNum")) { + PADDLE_ENFORCE_EQ( + fpn_rois->lod().size(), 1UL, + platform::errors::InvalidArgument("DistributeFpnProposalsOp needs LoD" + "with one level")); + } - auto fpn_rois_lod = fpn_rois->lod().back(); + std::vector fpn_rois_lod; + if (ctx.HasInput("RoisNum")) { + auto* rois_num = ctx.Input("RoisNum"); + fpn_rois_lod = GetLodFromRoisNum(rois_num); + } else { + fpn_rois_lod = fpn_rois->lod().back(); + } int lod_size = fpn_rois_lod.size() - 1; int roi_num = fpn_rois_lod[lod_size]; @@ -154,6 +162,8 @@ class GPUDistributeFpnProposalsOpKernel : public framework::OpKernel { restore_idx_data, roi_num); int start = 0; + auto multi_rois_num = ctx.MultiOutput("MultiLevelRoIsNum"); + for (int i = 0; i < num_level; ++i) { Tensor sub_lod = sub_lod_list.Slice(i, i + 1); int* sub_lod_data = sub_lod.data(); @@ -180,6 +190,11 @@ class GPUDistributeFpnProposalsOpKernel : public framework::OpKernel { multi_fpn_rois[i]->mutable_data({sub_rois_num, kBoxDim}, dev_ctx.GetPlace()); } + if (multi_rois_num.size() > 0) { + Tensor* rois_num_t = multi_rois_num[i]; + TensorCopySync(sub_lod, dev_ctx.GetPlace(), rois_num_t); + rois_num_t->Resize({lod_size}); + } framework::LoD lod; lod.emplace_back(offset); multi_fpn_rois[i]->set_lod(lod); diff --git a/paddle/fluid/operators/detection/distribute_fpn_proposals_op.h b/paddle/fluid/operators/detection/distribute_fpn_proposals_op.h index 0c84b385ccbc1dd26453bd957661c0310b7137e3..79498f01536d2fb2616921a2ef1ffa04f13fae64 100644 --- a/paddle/fluid/operators/detection/distribute_fpn_proposals_op.h +++ b/paddle/fluid/operators/detection/distribute_fpn_proposals_op.h @@ -28,6 +28,21 @@ namespace operators { const int kBoxDim = 4; +inline std::vector GetLodFromRoisNum(const Tensor* rois_num) { + std::vector rois_lod; + auto* rois_num_data = rois_num->data(); + Tensor cpu_tensor; + if (platform::is_gpu_place(rois_num->place())) { + TensorCopySync(*rois_num, platform::CPUPlace(), &cpu_tensor); + rois_num_data = cpu_tensor.data(); + } + rois_lod.push_back(static_cast(0)); + for (int i = 0; i < rois_num->numel(); ++i) { + rois_lod.push_back(rois_lod.back() + static_cast(rois_num_data[i])); + } + return rois_lod; +} + template static inline T BBoxArea(const T* box, bool normalized) { if (box[2] < box[0] || box[3] < box[1]) { @@ -65,13 +80,22 @@ class DistributeFpnProposalsOpKernel : public framework::OpKernel { const int num_level = max_level - min_level + 1; // check that the fpn_rois is not empty - PADDLE_ENFORCE_EQ( - fpn_rois->lod().size(), 1UL, - platform::errors::InvalidArgument("DistributeFpnProposalsOp needs LoD " - "with one level.")); + if (!context.HasInput("RoisNum")) { + PADDLE_ENFORCE_EQ(fpn_rois->lod().size(), 1UL, + platform::errors::InvalidArgument( + "DistributeFpnProposalsOp needs LoD " + "with one level.")); + } - auto fpn_rois_lod = fpn_rois->lod().back(); - int fpn_rois_num = fpn_rois_lod[fpn_rois_lod.size() - 1]; + std::vector fpn_rois_lod; + int fpn_rois_num; + if (context.HasInput("RoisNum")) { + auto* rois_num = context.Input("RoisNum"); + fpn_rois_lod = GetLodFromRoisNum(rois_num); + } else { + fpn_rois_lod = fpn_rois->lod().back(); + } + fpn_rois_num = fpn_rois_lod[fpn_rois_lod.size() - 1]; std::vector target_level; // std::vector target_level(fpn_rois_num, -1); // record the number of rois in each level @@ -136,6 +160,18 @@ class DistributeFpnProposalsOpKernel : public framework::OpKernel { for (int i = 0; i < fpn_rois_num; ++i) { restore_index_data[restore_index_inter[i]] = i; } + auto multi_rois_num = context.MultiOutput("MultiLevelRoIsNum"); + if (multi_rois_num.size() > 0) { + int batch_size = fpn_rois_lod.size() - 1; + for (int i = 0; i < num_level; ++i) { + int* rois_num_data = multi_rois_num[i]->mutable_data( + {batch_size}, context.GetPlace()); + for (int j = 0; j < batch_size; ++j) { + rois_num_data[j] = static_cast(multi_fpn_rois_lod0[i][j + 1] - + multi_fpn_rois_lod0[i][j]); + } + } + } // merge lod information into LoDTensor for (int i = 0; i < num_level; ++i) { framework::LoD lod; diff --git a/paddle/fluid/operators/detection/generate_proposals_op.cc b/paddle/fluid/operators/detection/generate_proposals_op.cc index 981a368e8564fbcd3d688bc67d2def8664bcfe8d..06e560f86d4e0a74f7ae04b155829618ce634697 100644 --- a/paddle/fluid/operators/detection/generate_proposals_op.cc +++ b/paddle/fluid/operators/detection/generate_proposals_op.cc @@ -17,6 +17,7 @@ limitations under the License. */ #include #include #include "paddle/fluid/framework/op_registry.h" +#include "paddle/fluid/framework/op_version_registry.h" #include "paddle/fluid/operators/gather.h" #include "paddle/fluid/operators/math/math_function.h" @@ -61,6 +62,10 @@ class GenerateProposalsOp : public framework::OperatorWithKernel { ctx->SetOutputDim("RpnRois", {-1, 4}); ctx->SetOutputDim("RpnRoiProbs", {-1, 1}); + if (!ctx->IsRuntime()) { + ctx->SetLoDLevel("RpnRois", std::max(ctx->GetLoDLevel("Scores"), 1)); + ctx->SetLoDLevel("RpnRoiProbs", std::max(ctx->GetLoDLevel("Scores"), 1)); + } } protected: @@ -347,7 +352,7 @@ class GenerateProposalsKernel : public framework::OpKernel { lod0.push_back(0); anchors.Resize({anchors.numel() / 4, 4}); variances.Resize({variances.numel() / 4, 4}); - std::vector tmp_lod; + std::vector tmp_num; int64_t num_proposals = 0; for (int64_t i = 0; i < num; ++i) { @@ -369,16 +374,16 @@ class GenerateProposalsKernel : public framework::OpKernel { AppendProposals(rpn_roi_probs, num_proposals, scores); num_proposals += proposals.dims()[0]; lod0.push_back(num_proposals); - tmp_lod.push_back(num_proposals); + tmp_num.push_back(proposals.dims()[0]); } - if (context.HasOutput("RpnRoisLod")) { - auto *rpn_rois_lod = context.Output("RpnRoisLod"); - rpn_rois_lod->mutable_data({num}, context.GetPlace()); - int64_t *lod_data = rpn_rois_lod->data(); + if (context.HasOutput("RpnRoisNum")) { + auto *rpn_rois_num = context.Output("RpnRoisNum"); + rpn_rois_num->mutable_data({num}, context.GetPlace()); + int *num_data = rpn_rois_num->data(); for (int i = 0; i < num; i++) { - lod_data[i] = tmp_lod[i]; + num_data[i] = tmp_num[i]; } - rpn_rois_lod->Resize({num}); + rpn_rois_num->Resize({num}); } rpn_rois->set_lod(lod); rpn_roi_probs->set_lod(lod); @@ -433,6 +438,16 @@ class GenerateProposalsKernel : public framework::OpKernel { Tensor keep; FilterBoxes(ctx, &proposals, min_size, im_info_slice, &keep); + // Handle the case when there is no keep index left + if (keep.numel() == 0) { + math::SetConstant set_zero; + bbox_sel.mutable_data({1, 4}, ctx.GetPlace()); + set_zero(ctx, &bbox_sel, static_cast(0)); + Tensor scores_filter; + scores_filter.mutable_data({1, 1}, ctx.GetPlace()); + set_zero(ctx, &scores_filter, static_cast(0)); + return std::make_pair(bbox_sel, scores_filter); + } Tensor scores_filter; bbox_sel.mutable_data({keep.numel(), 4}, ctx.GetPlace()); @@ -481,7 +496,8 @@ class GenerateProposalsOpMaker : public framework::OpProtoAndCheckerMaker { "(LoDTensor), Output proposals with shape (rois_num, 4)."); AddOutput("RpnRoiProbs", "(LoDTensor) Scores of proposals with shape (rois_num, 1)."); - AddOutput("RpnRoisLod", "(Tensor), rpn rois's lod info").AsDispensable(); + AddOutput("RpnRoisNum", "(Tensor), The number of Rpn RoIs in each image") + .AsDispensable(); AddAttr("pre_nms_topN", "Number of top scoring RPN proposals to keep before " "applying NMS."); @@ -515,3 +531,11 @@ REGISTER_OPERATOR( paddle::framework::EmptyGradOpMaker); REGISTER_OP_CPU_KERNEL(generate_proposals, ops::GenerateProposalsKernel, ops::GenerateProposalsKernel); +REGISTER_OP_VERSION(generate_proposals) + .AddCheckpoint( + R"ROC( + Upgrade generate_proposals add a new output [RpnRoisNum])ROC", + paddle::framework::compatible::OpVersionDesc().NewOutput( + "RpnRoisNum", + "The number of Rpn RoIs in each image. RpnRoisNum is " + "dispensable.")); diff --git a/paddle/fluid/operators/detection/generate_proposals_op.cu b/paddle/fluid/operators/detection/generate_proposals_op.cu index fa7670f6d680a95da1c1abd5befe1651ccb7265f..485136d8e2f7ab66f6b1c58deb09036ea5d4e1ec 100644 --- a/paddle/fluid/operators/detection/generate_proposals_op.cu +++ b/paddle/fluid/operators/detection/generate_proposals_op.cu @@ -330,6 +330,15 @@ static std::pair ProposalForOneImage( keep_index.Resize({keep_num}); Tensor scores_filter, proposals_filter; + // Handle the case when there is no keep index left + if (keep_num == 0) { + math::SetConstant set_zero; + proposals_filter.mutable_data({1, 4}, ctx.GetPlace()); + scores_filter.mutable_data({1, 1}, ctx.GetPlace()); + set_zero(ctx, &proposals_filter, static_cast(0)); + set_zero(ctx, &scores_filter, static_cast(0)); + return std::make_pair(proposals_filter, scores_filter); + } proposals_filter.mutable_data({keep_num, 4}, ctx.GetPlace()); scores_filter.mutable_data({keep_num, 1}, ctx.GetPlace()); GPUGather(ctx, proposals, keep_index, &proposals_filter); @@ -421,7 +430,7 @@ class CUDAGenerateProposalsKernel : public framework::OpKernel { int64_t num_proposals = 0; std::vector offset(1, 0); - std::vector tmp_lod; + std::vector tmp_num; for (int64_t i = 0; i < num; ++i) { Tensor im_info_slice = im_info->Slice(i, i + 1); @@ -448,15 +457,15 @@ class CUDAGenerateProposalsKernel : public framework::OpKernel { dev_ctx.Wait(); num_proposals += proposals.dims()[0]; offset.emplace_back(num_proposals); - tmp_lod.push_back(num_proposals); + tmp_num.push_back(proposals.dims()[0]); } - if (context.HasOutput("RpnRoisLod")) { - auto *rpn_rois_lod = context.Output("RpnRoisLod"); - rpn_rois_lod->mutable_data({num}, context.GetPlace()); - int64_t *lod_data = rpn_rois_lod->data(); - memory::Copy(place, lod_data, cpu_place, &tmp_lod[0], - sizeof(int64_t) * num, dev_ctx.stream()); - rpn_rois_lod->Resize({num}); + if (context.HasOutput("RpnRoisNum")) { + auto *rpn_rois_num = context.Output("RpnRoisNum"); + rpn_rois_num->mutable_data({num}, context.GetPlace()); + int *num_data = rpn_rois_num->data(); + memory::Copy(place, num_data, cpu_place, &tmp_num[0], sizeof(int) * num, + dev_ctx.stream()); + rpn_rois_num->Resize({num}); } framework::LoD lod; lod.emplace_back(offset); diff --git a/paddle/fluid/operators/dist_op.h b/paddle/fluid/operators/dist_op.h index ca03400cfd1ef9a27ba8e725381515d5e4ebc0ba..a2279e40623b4ba2f0421e73a6148b89eb970e71 100644 --- a/paddle/fluid/operators/dist_op.h +++ b/paddle/fluid/operators/dist_op.h @@ -176,14 +176,26 @@ static void DistGradFunction(const framework::ExecutionContext& context) { } else if (p == INFINITY || p == -INFINITY) { // p=inf or -inf, Lp-norm = |z_i|, the j-th element of dz tends to 0 if // j!=i, or equals to sign(z_i) * dout if j=i. - grad_t.device(place) = - (x_minux_y_abs == out_t.broadcast(out_bcast_dims)).template cast() * - sign * out_grad_t.broadcast(out_bcast_dims); + if (platform::is_cpu_place(context.GetPlace())) { + grad_t.device(place) = (x_minux_y_abs == out_t.broadcast(out_bcast_dims)) + .template cast() * + sign.eval() * out_grad_t.broadcast(out_bcast_dims); + } else { + grad_t.device(place) = (x_minux_y_abs == out_t.broadcast(out_bcast_dims)) + .template cast() * + sign * out_grad_t.broadcast(out_bcast_dims); + } } else { // dz = pow(abs(x-y)/out, p-1) * sign(x-y) * dout - grad_t.device(place) = - (x_minux_y_abs / out_t.broadcast(out_bcast_dims)).pow(p - 1) * sign * - out_grad_t.broadcast(out_bcast_dims); + if (platform::is_cpu_place(context.GetPlace())) { + grad_t.device(place) = + (x_minux_y_abs / out_t.broadcast(out_bcast_dims)).pow(p - 1) * + sign.eval() * out_grad_t.broadcast(out_bcast_dims); + } else { + grad_t.device(place) = + (x_minux_y_abs / out_t.broadcast(out_bcast_dims)).pow(p - 1) * sign * + out_grad_t.broadcast(out_bcast_dims); + } } Eigen::DSizes x_reshape_dims; diff --git a/paddle/fluid/operators/elementwise/elementwise_floordiv_op.cc b/paddle/fluid/operators/elementwise/elementwise_floordiv_op.cc index 457d9e79d7da171ef526d5cab0e59b021cb64f98..5a398fa50febe2efffd588ce8f3612f1f9cec0b6 100644 --- a/paddle/fluid/operators/elementwise/elementwise_floordiv_op.cc +++ b/paddle/fluid/operators/elementwise/elementwise_floordiv_op.cc @@ -49,8 +49,6 @@ REGISTER_OP_WITHOUT_GRADIENT(elementwise_floordiv, ops::ElementwiseOp, REGISTER_OP_CPU_KERNEL( elementwise_floordiv, - ops::ElementwiseFloorDivKernel, - ops::ElementwiseFloorDivKernel, ops::ElementwiseFloorDivKernel, ops::ElementwiseFloorDivKernel); diff --git a/paddle/fluid/operators/elementwise/elementwise_floordiv_op.cu b/paddle/fluid/operators/elementwise/elementwise_floordiv_op.cu index f63d6f037632c1a6a05726b933b2258adc113ee3..60846d1e8fee1c7f68ac101f18355750c2c15a4d 100644 --- a/paddle/fluid/operators/elementwise/elementwise_floordiv_op.cu +++ b/paddle/fluid/operators/elementwise/elementwise_floordiv_op.cu @@ -19,7 +19,5 @@ namespace plat = paddle::platform; REGISTER_OP_CUDA_KERNEL( elementwise_floordiv, - ops::ElementwiseFloorDivKernel, - ops::ElementwiseFloorDivKernel, ops::ElementwiseFloorDivKernel, ops::ElementwiseFloorDivKernel); diff --git a/paddle/fluid/operators/elementwise/elementwise_floordiv_op.h b/paddle/fluid/operators/elementwise/elementwise_floordiv_op.h index 8afe2133c0488bbe04ec4803aac5dce6573f634d..5dc93740949e6e7c25be564927c8fcffde1a18d6 100644 --- a/paddle/fluid/operators/elementwise/elementwise_floordiv_op.h +++ b/paddle/fluid/operators/elementwise/elementwise_floordiv_op.h @@ -14,7 +14,6 @@ limitations under the License. */ #pragma once -#include #include "paddle/fluid/framework/eigen.h" #include "paddle/fluid/operators/elementwise/elementwise_op.h" #include "paddle/fluid/operators/elementwise/elementwise_op_function.h" @@ -62,15 +61,8 @@ void elementwise_floor_div(const framework::ExecutionContext &ctx, const framework::Tensor *x, const framework::Tensor *y, framework::Tensor *z) { int axis = ctx.Attr("axis"); - auto x_dims = x->dims(); - auto y_dims = y->dims(); - if (x_dims.size() >= y_dims.size()) { - ElementwiseComputeEx, DeviceContext, T>( - ctx, x, y, axis, FloorDivFunctor(), z); - } else { - ElementwiseComputeEx, DeviceContext, T>( - ctx, x, y, axis, InverseFloorDivFunctor(), z); - } + ElementwiseComputeEx, DeviceContext, T>( + ctx, x, y, axis, FloorDivFunctor(), z); } template diff --git a/paddle/fluid/operators/elementwise/elementwise_mul_op.h b/paddle/fluid/operators/elementwise/elementwise_mul_op.h index 718321b441b2025afea9d913855b26a82cda8075..e4d3ea6d7291eff8911d8419cda96f2d2738b9a1 100644 --- a/paddle/fluid/operators/elementwise/elementwise_mul_op.h +++ b/paddle/fluid/operators/elementwise/elementwise_mul_op.h @@ -33,22 +33,7 @@ class ElementwiseMulOp : public ElementwiseOp { auto input_data_type = OperatorWithKernel::IndicateVarDataType(ctx, "X"); #ifdef PADDLE_WITH_MKLDNN - using mkldnn::memory; - auto CanMKLDNNElementwiseMulBeUsed = [&]() { - auto x_dims = ctx.Input("X")->dims(); - auto y_dims = ctx.Input("Y")->dims(); - int rankdiff = x_dims.size() - y_dims.size(); - // TODO(jczaja): Remove this when oneDNN performance for scalar - // broadcasting - // is improved (Ernie large situation) - if (rankdiff != 0 && y_dims.size() == 1 && y_dims[0] == 1) { - return false; - } - - return true; - }; - - if (platform::CanMKLDNNBeUsed(ctx) && CanMKLDNNElementwiseMulBeUsed()) { + if (platform::CanMKLDNNBeUsed(ctx)) { return framework::OpKernelType(input_data_type, ctx.GetPlace(), framework::DataLayout::kMKLDNN, framework::LibraryType::kMKLDNN); diff --git a/paddle/fluid/operators/empty_op.cc b/paddle/fluid/operators/empty_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..f539e2e6f6d2d6faa084d1e62ec894b4b65e96bf --- /dev/null +++ b/paddle/fluid/operators/empty_op.cc @@ -0,0 +1,132 @@ +/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/operators/empty_op.h" +#include "paddle/fluid/framework/op_registry.h" + +namespace paddle { +namespace operators { + +class EmptyOpMaker : public framework::OpProtoAndCheckerMaker { + public: + void Make() override { + AddInput("ShapeTensor", + "(Tensor), optional). The shape of the output." + "It has a higher priority than Attr(shape).") + .AsDispensable(); + AddInput("ShapeTensorList", + "(vector>, optional). The shape of the output. " + "It has a higher priority than Attr(shape)." + "The shape of the element in vector must be [1].") + .AsDuplicable() + .AsDispensable(); + AddAttr>("shape", + "(vector) The shape of the output") + .SetDefault({}); + AddAttr("dtype", "The data type of output tensor, Default is float") + .SetDefault(framework::proto::VarType::FP32); + AddOutput("Out", "(Tensor) The output tensor."); + AddComment(R"DOC(empty operator +Returns a tensor filled with uninitialized data. The shape of the tensor is +defined by the variable argument shape. + + +The type of the tensor is specify by `dtype`. +)DOC"); + } +}; + +class EmptyOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext* context) const override { + OP_INOUT_CHECK(context->HasOutput("Out"), "Output", "Out", "empty"); + + if (context->HasInput("ShapeTensor")) { + auto dims = context->GetInputDim("ShapeTensor"); + int num_ele = 1; + for (int i = 0; i < dims.size(); ++i) { + num_ele *= dims[i]; + } + + context->SetOutputDim("Out", framework::make_ddim({num_ele})); + } else if (context->HasInputs("ShapeTensorList")) { + std::vector out_dims; + auto dims_list = context->GetInputsDim("ShapeTensorList"); + for (size_t i = 0; i < dims_list.size(); ++i) { + auto& dims = dims_list[i]; + PADDLE_ENFORCE_EQ( + dims, framework::make_ddim({1}), + "ShapeError: The shape of Tensor in list must be [1]. " + "But received the shape " + "is [%s]", + dims); + + out_dims.push_back(dims[0]); + } + + context->SetOutputDim("Out", framework::make_ddim(out_dims)); + } else { + auto& shape = context->Attrs().Get>("shape"); + context->SetOutputDim("Out", framework::make_ddim(shape)); + } + } + + protected: + framework::OpKernelType GetKernelTypeForVar( + const std::string& var_name, const framework::Tensor& tensor, + const framework::OpKernelType& expected_kernel_type) const override { + if (var_name == "ShapeTensor" || var_name == "ShapeTensorList") { + return expected_kernel_type; + } else { + return framework::OpKernelType(expected_kernel_type.data_type_, + tensor.place(), tensor.layout()); + } + } + + framework::OpKernelType GetExpectedKernelType( + const framework::ExecutionContext& context) const override { + return framework::OpKernelType( + framework::proto::VarType::Type(context.Attr("dtype")), + context.GetPlace()); + } +}; + +class EmptyOpVarTypeInference : public framework::VarTypeInference { + public: + void operator()(framework::InferVarTypeContext* context) const override { + auto data_type = static_cast( + BOOST_GET_CONST(int, context->GetAttr("dtype"))); + context->SetOutputDataType("Out", data_type); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +namespace plat = paddle::platform; + +REGISTER_OPERATOR( + empty, ops::EmptyOp, ops::EmptyOpMaker, ops::EmptyOpVarTypeInference, + paddle::framework::EmptyGradOpMaker, + paddle::framework::EmptyGradOpMaker); + +REGISTER_OP_CPU_KERNEL(empty, ops::EmptyKernel, + ops::EmptyKernel, + ops::EmptyKernel, + ops::EmptyKernel, + ops::EmptyKernel, + ops::EmptyKernel); diff --git a/paddle/fluid/operators/empty_op.cu.cc b/paddle/fluid/operators/empty_op.cu.cc new file mode 100644 index 0000000000000000000000000000000000000000..22799e507aeff7940274f729b174f50bfd9132a5 --- /dev/null +++ b/paddle/fluid/operators/empty_op.cu.cc @@ -0,0 +1,26 @@ +/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/operators/empty_op.h" + +namespace ops = paddle::operators; +namespace plat = paddle::platform; + +REGISTER_OP_CUDA_KERNEL( + empty, ops::EmptyKernel, + ops::EmptyKernel, + ops::EmptyKernel, + ops::EmptyKernel, + ops::EmptyKernel, + ops::EmptyKernel); diff --git a/paddle/fluid/operators/empty_op.h b/paddle/fluid/operators/empty_op.h new file mode 100644 index 0000000000000000000000000000000000000000..9c91377683870917db28f6f6a5f3f3b1b4a1962f --- /dev/null +++ b/paddle/fluid/operators/empty_op.h @@ -0,0 +1,45 @@ +// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +#include "paddle/fluid/framework/op_registry.h" +#include "paddle/fluid/operators/utils.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; + +template +class EmptyKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext &context) const override { + auto dtype = static_cast( + context.Attr("dtype")); + + Tensor *out_tensor = context.Output("Out"); + + auto shape = GetShape(context); + out_tensor->Resize(shape); + + out_tensor->mutable_data(context.GetPlace(), dtype); + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/fluid/operators/expand_op.cc b/paddle/fluid/operators/expand_op.cc index 3c898ac29f0cab572d199eaafe951751682d4834..83e205367a7af62c52825297d92571c306be2c42 100644 --- a/paddle/fluid/operators/expand_op.cc +++ b/paddle/fluid/operators/expand_op.cc @@ -228,6 +228,26 @@ class ExpandGradOpMaker : public framework::SingleGradOpMaker { } }; +template +class ExpandDoubleGradOpMaker : public framework::SingleGradOpMaker { + public: + using framework::SingleGradOpMaker::SingleGradOpMaker; + + protected: + void Apply(GradOpPtr op) const override { + op->SetInput("X", this->OutputGrad(framework::GradVarName("X"))); + op->SetOutput("Out", this->InputGrad(framework::GradVarName("Out"))); + if (this->HasInput("expand_times_tensor")) { + op->SetInput("expand_times_tensor", this->Input("expand_times_tensor")); + } + if (this->HasInput("ExpandTimes")) { + op->SetInput("ExpandTimes", this->Input("ExpandTimes")); + } + op->SetAttrMap(this->Attrs()); + op->SetType("expand"); + } +}; + DECLARE_NO_NEED_BUFFER_VARS_INFERER(ExpandGradNoNeedBufVarsInferer, "X"); } // namespace operators @@ -238,6 +258,8 @@ REGISTER_OPERATOR(expand, ops::ExpandOp, ops::ExpandOpMaker, ops::ExpandGradOpMaker, ops::ExpandGradOpMaker); REGISTER_OPERATOR(expand_grad, ops::ExpandGradOp, + ops::ExpandDoubleGradOpMaker, + ops::ExpandDoubleGradOpMaker, ops::ExpandGradNoNeedBufVarsInferer); REGISTER_OP_CPU_KERNEL( expand, ops::ExpandKernel, diff --git a/paddle/fluid/operators/expand_v2_op.cc b/paddle/fluid/operators/expand_v2_op.cc index 359d512c341529579a56dbe840e5eef0aa3062a5..a1ee47b7f93910a481c6e0793c306e2b190c774d 100644 --- a/paddle/fluid/operators/expand_v2_op.cc +++ b/paddle/fluid/operators/expand_v2_op.cc @@ -230,6 +230,26 @@ class ExpandV2GradOpMaker : public framework::SingleGradOpMaker { } }; +template +class ExpandV2DoubleGradOpMaker : public framework::SingleGradOpMaker { + public: + using framework::SingleGradOpMaker::SingleGradOpMaker; + + protected: + void Apply(GradOpPtr op) const override { + op->SetType("expand_v2"); + op->SetInput("X", this->OutputGrad(framework::GradVarName("X"))); + op->SetOutput("Out", this->InputGrad(framework::GradVarName("Out"))); + if (this->HasInput("expand_shapes_tensor")) { + op->SetInput("expand_shapes_tensor", this->Input("expand_shapes_tensor")); + } + if (this->HasInput("Shape")) { + op->SetInput("Shape", this->Input("Shape")); + } + op->SetAttrMap(this->Attrs()); + } +}; + DECLARE_NO_NEED_BUFFER_VARS_INFERER(ExpandV2GradNoNeedBufVarsInferer, "X"); } // namespace operators @@ -240,6 +260,8 @@ REGISTER_OPERATOR(expand_v2, ops::ExpandV2Op, ops::ExpandV2OpMaker, ops::ExpandV2GradOpMaker, ops::ExpandV2GradOpMaker); REGISTER_OPERATOR(expand_v2_grad, ops::ExpandV2GradOp, + ops::ExpandV2DoubleGradOpMaker, + ops::ExpandV2DoubleGradOpMaker, ops::ExpandV2GradNoNeedBufVarsInferer); REGISTER_OP_CPU_KERNEL( expand_v2, ops::ExpandV2Kernel, diff --git a/paddle/fluid/operators/fill_constant_op.h b/paddle/fluid/operators/fill_constant_op.h index 74939da08b38dc147c156011759757a605db9444..6fea8fe98bf0e19bbbb023c91f4f9900f5ec1859 100644 --- a/paddle/fluid/operators/fill_constant_op.h +++ b/paddle/fluid/operators/fill_constant_op.h @@ -27,27 +27,6 @@ namespace operators { using Tensor = framework::Tensor; -inline framework::DDim GetShape(const framework::ExecutionContext &ctx, - std::string op_type) { - // 1. shape is a Tensor - if (ctx.HasInput("ShapeTensor")) { - auto *shape_tensor = ctx.Input("ShapeTensor"); - auto vec_shape = GetDataFromTensor(shape_tensor); - return framework::make_ddim(vec_shape); - } - - // 2. shape is a list/tuple containing Tensor - auto shape_tensor_list = ctx.MultiInput("ShapeTensorList"); - if (shape_tensor_list.size() > 0) { - auto vec_shape = GetDataFromTensorList(shape_tensor_list); - return framework::make_ddim(vec_shape); - } - - // 3. shape is a list/tuple without containing Tensor - auto vec_shape = ctx.Attr>("shape"); - return framework::make_ddim(vec_shape); -} - template class FillConstantKernel : public framework::OpKernel { public: @@ -93,8 +72,7 @@ class FillConstantKernel : public framework::OpKernel { } value = tensor_data[0]; } - const std::string op_type = "fill_constant"; - auto shape = GetShape(ctx, op_type); + auto shape = GetShape(ctx); if (out_var->IsType()) { tensor = out_var->GetMutable(); diff --git a/paddle/fluid/operators/fused/fused_embedding_fc_lstm_op.cc b/paddle/fluid/operators/fused/fused_embedding_fc_lstm_op.cc index c698cb1405fd6f049e01b23613e175ba39c4976e..79fa268f3884b2710fe08eb2907dbd989479d7e6 100644 --- a/paddle/fluid/operators/fused/fused_embedding_fc_lstm_op.cc +++ b/paddle/fluid/operators/fused/fused_embedding_fc_lstm_op.cc @@ -367,8 +367,13 @@ class FusedEmbeddingFCLSTMKernel : public framework::OpKernel { auto blas = math::GetBlas(ctx); for (int64_t i = 0; i < ids_numel; ++i) { - PADDLE_ENFORCE_LT(ids_data[i], row_number); - PADDLE_ENFORCE_GE(ids_data[i], 0, "ids %d", i); + PADDLE_ENFORCE_LT( + ids_data[i], row_number, + platform::errors::OutOfRange( + "Value of Ids %d should less than dict size %d.", i, row_number)); + PADDLE_ENFORCE_GE(ids_data[i], 0, + platform::errors::OutOfRange( + "Value of Ids %d should greater than ZERO.", i)); memcpy(xx_data + i * row_width, embeddings_data + ids_data[i] * row_width, row_width * sizeof(T)); } @@ -473,8 +478,13 @@ class FusedEmbeddingFCLSTMKernel : public framework::OpKernel { auto blas = math::GetBlas(dev_ctx); for (int64_t i = 0; i < ids_numel; ++i) { - PADDLE_ENFORCE_LT(ids_data[i], row_number); - PADDLE_ENFORCE_GE(ids_data[i], 0, "ids %d", i); + PADDLE_ENFORCE_LT( + ids_data[i], row_number, + platform::errors::OutOfRange( + "Value of Ids %d should less than dict size %d.", i, row_number)); + PADDLE_ENFORCE_GE(ids_data[i], 0, + platform::errors::OutOfRange( + "Value of Ids %d should greater than ZERO.", i)); memcpy(xx_data + i * row_width, embeddings_data + ids_data[i] * row_width, row_width * sizeof(T)); } diff --git a/paddle/fluid/operators/fused/fusion_gru_op.cc b/paddle/fluid/operators/fused/fusion_gru_op.cc index d0920098f606e49d4d1a3e4cb6d8a2b6c44ca267..4013906609603e31b798e333d55ecccba197506a 100644 --- a/paddle/fluid/operators/fused/fusion_gru_op.cc +++ b/paddle/fluid/operators/fused/fusion_gru_op.cc @@ -30,16 +30,18 @@ void FusionGRUOp::InferShape(framework::InferShapeContext* ctx) const { OP_INOUT_CHECK(ctx->HasInput("X"), "Input", "X", "fusion_gru"); OP_INOUT_CHECK(ctx->HasInput("WeightX"), "Input", "WeightX", "fusion_gru"); OP_INOUT_CHECK(ctx->HasInput("WeightH"), "Input", "WeightH", "fusion_gru"); - OP_INOUT_CHECK(ctx->HasOutput("XX"), "Output", "XX", "fusion_gru"); OP_INOUT_CHECK(ctx->HasOutput("Hidden"), "Output", "Hidden", "fusion_gru"); - auto x_dims = ctx->GetInputDim("X"); - PADDLE_ENFORCE_EQ(x_dims.size(), 2, - platform::errors::InvalidArgument( - "Input(X)'s rank must be 2, but received input dim " - "size is:%d, input dim is:[%s]", - x_dims.size(), x_dims)); + auto x_mat_dims = (x_dims.size() == 3 && x_dims[1] == 1) + ? framework::flatten_to_2d(x_dims, 1) + : x_dims; + PADDLE_ENFORCE_EQ( + x_mat_dims.size(), 2, + platform::errors::InvalidArgument("The size of input X dims should be 2, " + "or 3 with second dimension equal to " + "1, but now Input X dim is:[%s] ", + x_dims)); auto wx_dims = ctx->GetInputDim("WeightX"); PADDLE_ENFORCE_EQ(wx_dims.size(), 2, @@ -47,12 +49,14 @@ void FusionGRUOp::InferShape(framework::InferShapeContext* ctx) const { "The rank of Input(WeightX) should be 2, but received " "WeightX dim size is:%d, WeightX dim is:[%s] ", wx_dims.size(), wx_dims)); - PADDLE_ENFORCE_EQ(wx_dims[0], x_dims[1], - platform::errors::InvalidArgument( - "The first dimension of Input(WeightX) " - "should equal to second dimension of input x, but " - "received WeightX dimension is:%d, x dimension is:%d", - wx_dims[0], x_dims[1])); + PADDLE_ENFORCE_EQ( + wx_dims[0], x_mat_dims[1], + platform::errors::InvalidArgument( + "The first dimension of flattened WeightX" + "should equal to last dimension of flattened input X, but " + "received fattened WeightX dimension is:%d, flattened X dimension " + "is:%d", + wx_dims[0], x_mat_dims[1])); int frame_size = wx_dims[1] / 3; auto wh_dims = ctx->GetInputDim("WeightH"); @@ -102,24 +106,24 @@ void FusionGRUOp::InferShape(framework::InferShapeContext* ctx) const { "received bias dim is:[%s], frame size is:%d", b_dims, frame_size)); } - framework::DDim out_dims({x_dims[0], frame_size}); + framework::DDim out_dims({x_mat_dims[0], frame_size}); ctx->SetOutputDim("Hidden", out_dims); ctx->ShareLoD("X", "Hidden"); int xx_width; if (ctx->Attrs().Get("use_seq")) { xx_width = wx_dims[1]; } else { - xx_width = x_dims[1] > wx_dims[1] ? wx_dims[1] : x_dims[1]; + xx_width = x_mat_dims[1] > wx_dims[1] ? wx_dims[1] : x_mat_dims[1]; OP_INOUT_CHECK(ctx->HasOutput("ReorderedH0"), "Output", "ReorderedH0", "fusion_gru"); OP_INOUT_CHECK(ctx->HasOutput("BatchedInput"), "Output", "BatchedInput", "fusion_gru"); OP_INOUT_CHECK(ctx->HasOutput("BatchedOut"), "Output", "BatchedOut", "fusion_gru"); - ctx->SetOutputDim("BatchedInput", {x_dims[0], wx_dims[1]}); + ctx->SetOutputDim("BatchedInput", {x_mat_dims[0], wx_dims[1]}); ctx->SetOutputDim("BatchedOut", out_dims); } - ctx->SetOutputDim("XX", {x_dims[0], xx_width}); + ctx->SetOutputDim("XX", {x_mat_dims[0], xx_width}); ctx->ShareLoD("X", "XX"); } @@ -202,6 +206,27 @@ void FusionGRUOpMaker::Make() { AddAttr("use_mkldnn", "(bool, default false) Only used in mkldnn kernel") .SetDefault(false); + AddAttr( + "mkldnn_data_type", + "(string, default \"float32\"). Data type of mkldnn kernel") + .SetDefault("float32") + .InEnum({"float32", "int8", "bfloat16"}); + AddAttr("Scale_data", + "Scale to be used for int8 input/output data." + "Only used with MKL-DNN INT8.") + .SetDefault(1.0f); + AddAttr("Shift_data", + "Shift to be used for int8 input/output data." + "Only used with MKL-DNN INT8.") + .SetDefault(0.0f); + AddAttr>("Scale_weights", + "Scale_weights to be used for int8 weights data." + "Only used with MKL-DNN INT8.") + .SetDefault({1.0f}); + AddAttr("force_fp32_output", + "(bool, default false) Force INT8 kernel output FP32, only " + "used in MKL-DNN INT8") + .SetDefault(false); AddComment(R"DOC( The Fusion complete GRU Operator. This operator fuse the fully-connected operator into GRU, @@ -220,14 +245,17 @@ class FusionGRUKernel : public framework::OpKernel { } } -#define INIT_BASE_DEFINES \ - auto* x = ctx.Input("X"); \ - auto* wh = ctx.Input("WeightH"); \ - auto* xx = ctx.Output("XX"); \ - auto x_lod = x->lod(); \ - auto x_dims = x->dims(); /* T x M*/ \ - auto wh_dims = wh->dims(); /* D x 3D*/ \ - const int total_T = x_dims[0]; \ +#define INIT_BASE_DEFINES \ + auto* x = ctx.Input("X"); \ + auto* wh = ctx.Input("WeightH"); \ + auto* xx = ctx.Output("XX"); \ + auto x_lod = x->lod(); \ + auto x_dims = x->dims(); /* T x M*/ \ + auto x_mat_dims = (x_dims.size() == 3 && x_dims[1] == 1) \ + ? framework::flatten_to_2d(x_dims, 1) \ + : x_dims; \ + auto wh_dims = wh->dims(); /* D x 3D*/ \ + const int total_T = x_mat_dims[0]; \ const int D3 = wh_dims[1] #define INIT_OTHER_DEFINES \ @@ -236,7 +264,7 @@ class FusionGRUKernel : public framework::OpKernel { auto* bias = ctx.Input("Bias"); \ auto* hidden_out = ctx.Output("Hidden"); \ bool is_reverse = ctx.Attr("is_reverse"); \ - const int M = x_dims[1]; \ + const int M = x_mat_dims[1]; \ const int D = wh_dims[0]; \ const int D2 = D * 2; \ const jit::gru_attr_t attr( \ diff --git a/paddle/fluid/operators/fused/mkldnn/fusion_gru_mkldnn_op.cc b/paddle/fluid/operators/fused/mkldnn/fusion_gru_mkldnn_op.cc index 3940aae53b8ef70c15311305ce13f8929400d405..5fad1b116de6437e62e311318832ad77e24a40cc 100644 --- a/paddle/fluid/operators/fused/mkldnn/fusion_gru_mkldnn_op.cc +++ b/paddle/fluid/operators/fused/mkldnn/fusion_gru_mkldnn_op.cc @@ -21,11 +21,12 @@ namespace operators { using paddle::framework::LoDTensor; using paddle::framework::Tensor; using paddle::platform::CPUDeviceContext; +using paddle::platform::CreateKey; using paddle::platform::MKLDNNGetDataType; using paddle::platform::MKLDNNMemDesc; using platform::to_void_cast; -template +template class GRUMKLDNNHandler : public platform::MKLDNNHandlerT { public: GRUMKLDNNHandler(const paddle::framework::ExecutionContext& ctx, @@ -38,7 +39,7 @@ class GRUMKLDNNHandler : public platform::MKLDNNHandlerT { const std::string& unique_name) : platform::MKLDNNHandlerT( dev_ctx, dev_ctx.GetEngine(), cpu_place, - platform::CreateKey(unique_name, Ti)), + CreateKey(unique_name, MKLDNNGetDataType(), Ti)), N(N), Ti(Ti), IC(IC), @@ -47,9 +48,29 @@ class GRUMKLDNNHandler : public platform::MKLDNNHandlerT { // do not depend on Ti size but primitive and input/output memory do if (platform::MKLDNNDeviceContext::tls().get_cur_mkldnn_session_id() != platform::MKLDNNDeviceContextThreadLocals::kMKLDNNSessionID_Default) { - memory_key_ = unique_name; + memory_key_ = CreateKey(unique_name, MKLDNNGetDataType()); } else { - memory_key_ = unique_name + "-t:" + platform::ThreadIDasStr(); + memory_key_ = CreateKey(unique_name, MKLDNNGetDataType(), "-t:", + platform::ThreadIDasStr()); + } + + // Is it int8 kernel + const bool is_INT8 = std::is_same::value; + + if (is_INT8) { + // Int8 attributes + const float scale_data = ctx.Attr("Scale_data"); + const float shift_data = ctx.Attr("Shift_data"); + const auto scale_weights = ctx.Attr>("Scale_weights"); + + const int weights_scale_mask = + 0 + + (1 << 3) // bit, indicating the unique scales for `g` dim in `ldigo` + + + (1 << 4); // bit, indicating the unique scales for `o` dim in `ldigo` + + attr_.set_rnn_data_qparams(scale_data, shift_data); + attr_.set_rnn_weights_qparams(weights_scale_mask, scale_weights); } if (!this->isCached()) { @@ -63,6 +84,10 @@ class GRUMKLDNNHandler : public platform::MKLDNNHandlerT { platform::errors::Unimplemented( "oneDNN fusion_gru supports only tanh as an activation.")); + // Weights for int8 kernel are of a type s8 + const auto weights_dt = + is_INT8 ? dnnl::memory::data_type::s8 : dnnl::memory::data_type::f32; + // oneDNN RNN dimensions const int64_t D = 1; // Directions const int64_t L = 1; // Layers (PP supports only 1 stacked layer) @@ -71,19 +96,16 @@ class GRUMKLDNNHandler : public platform::MKLDNNHandlerT { // Create memory descriptors auto input_md = MKLDNNMemDesc({Ti, N, IC}, MKLDNNGetDataType(), MKLDNNMemoryFormat::any); - auto weight_x_md = MKLDNNMemDesc( - {L, D, IC, G, OC}, MKLDNNGetDataType(), MKLDNNMemoryFormat::any); - auto weight_h_md = MKLDNNMemDesc( - {L, D, OC, G, OC}, MKLDNNGetDataType(), MKLDNNMemoryFormat::any); + auto weight_x_md = + MKLDNNMemDesc({L, D, IC, G, OC}, weights_dt, MKLDNNMemoryFormat::any); + auto weight_h_md = + MKLDNNMemDesc({L, D, OC, G, OC}, weights_dt, MKLDNNMemoryFormat::any); auto bias_md = MKLDNNMemDesc({L, D, G, OC}, MKLDNNGetDataType(), MKLDNNMemoryFormat::ldgo); - auto hidden_md = MKLDNNMemDesc({Ti, N, OC}, MKLDNNGetDataType(), + auto hidden_md = MKLDNNMemDesc({Ti, N, OC}, MKLDNNGetDataType(), MKLDNNMemoryFormat::any); - auto h0_md = dnnl::memory::desc(); - if (h0) { - h0_md = MKLDNNMemDesc({L, D, N, OC}, MKLDNNGetDataType(), - MKLDNNMemoryFormat::ldnc); - } + auto h0_md = MKLDNNMemDesc({L, D, N, OC}, MKLDNNGetDataType(), + MKLDNNMemoryFormat::ldnc); // Create GRU oneDNN primitive const auto direction = @@ -91,7 +113,7 @@ class GRUMKLDNNHandler : public platform::MKLDNNHandlerT { : dnnl::rnn_direction::unidirectional_left2right; this->AcquireForwardPrimitiveDescriptor( - dnnl::prop_kind::forward_inference, direction, input_md, h0_md, + attr_, dnnl::prop_kind::forward_inference, direction, input_md, h0_md, weight_x_md, weight_h_md, bias_md, hidden_md, dnnl::memory::desc()); } } @@ -101,29 +123,31 @@ class GRUMKLDNNHandler : public platform::MKLDNNHandlerT { dnnl::memory::format_tag::ntc); } - void reorderRNNdata(const T* input_data, T* output_data, + void reorderRNNdata(void* input_data, void* output_data, std::vector lod, const bool is_reverse, platform::RNNReorderType reorder_type) { switch (reorder_type) { // Reorder input memory [WORDS, C] + LoD -> [N, T, C] case platform::RNNReorderType::PP_NTC: { - auto* input_data_iter = input_data; + auto* input_data_iter = reinterpret_cast(input_data); + auto* output_data_iter = reinterpret_cast(output_data); for (int n = 0; n < N; ++n) { const auto num_elements = (lod[n + 1] - lod[n]) * IC; const auto offset = is_reverse ? (Ti * IC - num_elements) : 0; - memcpy(output_data + n * Ti * IC + offset, input_data_iter, + memcpy(output_data_iter + n * Ti * IC + offset, input_data_iter, sizeof(T) * num_elements); input_data_iter += num_elements; } } break; // Reorder input memory [WORDS, C] + LoD -> [T, N, C] case platform::RNNReorderType::PP_TNC: { - auto* input_data_iter = input_data; + auto* input_data_iter = reinterpret_cast(input_data); + auto* output_data_iter = reinterpret_cast(output_data); for (int n = 0; n < N; ++n) { const auto num_elements = (lod[n + 1] - lod[n]); const auto offset = is_reverse ? (Ti - num_elements) : 0; for (size_t t = 0; t < num_elements; ++t) { - memcpy(output_data + (t + offset) * N * IC + n * IC, + memcpy(output_data_iter + (t + offset) * N * IC + n * IC, input_data_iter, sizeof(T) * IC); input_data_iter += IC; } @@ -131,24 +155,27 @@ class GRUMKLDNNHandler : public platform::MKLDNNHandlerT { } break; // Reorder output values to PP format [N, T, C] -> [WORDS, C] case platform::RNNReorderType::NTC_PP: { - auto* output_data_iter = output_data; + auto* input_data_iter = reinterpret_cast(input_data); + auto* output_data_iter = reinterpret_cast(output_data); for (int n = 0; n < N; ++n) { const auto num_elements = (lod[n + 1] - lod[n]) * OC; const auto offset = is_reverse ? (Ti * OC - num_elements) : 0; - memcpy(output_data_iter, input_data + n * Ti * OC + offset, - sizeof(T) * num_elements); + memcpy(output_data_iter, input_data_iter + n * Ti * OC + offset, + sizeof(T_out) * num_elements); output_data_iter += num_elements; } } break; // Reorder output values to PP format [T, N, C] -> [WORDS, C] case platform::RNNReorderType::TNC_PP: { - auto* output_data_iter = output_data; + auto* input_data_iter = reinterpret_cast(input_data); + auto* output_data_iter = reinterpret_cast(output_data); for (int n = 0; n < N; ++n) { const auto num_elements = lod[n + 1] - lod[n]; const auto offset = is_reverse ? (Ti - num_elements) : 0; for (size_t t = 0; t < num_elements; ++t) { memcpy(output_data_iter, - input_data + (t + offset) * N * OC + n * OC, sizeof(T) * OC); + input_data_iter + (t + offset) * N * OC + n * OC, + sizeof(T_out) * OC); output_data_iter += OC; } } @@ -169,9 +196,9 @@ class GRUMKLDNNHandler : public platform::MKLDNNHandlerT { } const auto& input_lod = input->lod()[0]; - auto* x_data = input->data(); + auto* x_data = to_void_cast(input->data()); - auto* x_onednn_data = reinterpret_cast(memory_p->get_data_handle()); + auto* x_onednn_data = memory_p->get_data_handle(); memset(x_onednn_data, 0, sizeof(T) * N * Ti * IC); if (platform::GetMKLDNNFormat(this->fwd_pd_->src_desc()) == @@ -198,19 +225,35 @@ class GRUMKLDNNHandler : public platform::MKLDNNHandlerT { return memory_p; } + // TODO(grygielski) H0 is for now persistable std::shared_ptr AcquireH0Memory(const Tensor* h0) { const std::string h0_key = memory_key_ + "@h0"; auto memory_p = std::static_pointer_cast(this->dev_ctx_.GetBlob(h0_key)); - auto* h0_data = to_void_cast(h0->data()); - if (!memory_p) { - memory_p = std::make_shared( - this->fwd_pd_->weights_layer_desc(), this->engine_, h0_data); + auto user_h0_memory = dnnl::memory(); + if (h0) { + user_h0_memory = + dnnl::memory({{1, 1, N, OC}, + MKLDNNGetDataType(), + MKLDNNMemoryFormat::ldnc}, + this->engine_, to_void_cast(h0->data())); + } else { + user_h0_memory = dnnl::memory({{1, 1, N, OC}, + MKLDNNGetDataType(), + MKLDNNMemoryFormat::ldnc}, + this->engine_); + memset(user_h0_memory.get_data_handle(), 0, sizeof(float) * N * OC); + } + memory_p = std::make_shared(this->fwd_pd_->src_iter_desc(), + this->engine_); + + dnnl::stream astream(this->engine_); + dnnl::reorder(user_h0_memory, *memory_p, attr_) + .execute(astream, user_h0_memory, *memory_p); + this->dev_ctx_.SetBlob(h0_key, memory_p); - } else { - memory_p->set_data_handle(h0_data); } return memory_p; } @@ -245,7 +288,7 @@ class GRUMKLDNNHandler : public platform::MKLDNNHandlerT { this->fwd_pd_->weights_layer_desc(), this->engine_); dnnl::stream astream(this->engine_); - dnnl::reorder(user_memory, *memory_p) + dnnl::reorder(user_memory, *memory_p, attr_) .execute(astream, user_memory, *memory_p); this->dev_ctx_.SetBlob(wx_key, memory_p); @@ -298,7 +341,7 @@ class GRUMKLDNNHandler : public platform::MKLDNNHandlerT { this->fwd_pd_->weights_iter_desc(), this->engine_); dnnl::stream astream(this->engine_); - dnnl::reorder(user_memory, *memory_p) + dnnl::reorder(user_memory, *memory_p, attr_) .execute(astream, user_memory, *memory_p); this->dev_ctx_.SetBlob(wh_key, memory_p); @@ -347,12 +390,26 @@ class GRUMKLDNNHandler : public platform::MKLDNNHandlerT { // Memory size of weights, bias and h0 does not depend // on Ti size, thus we need another key to cache them std::string memory_key_; + dnnl::primitive_attr attr_; }; template class FusionGRUMKLDNNKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { + const bool is_INT8 = std::is_same::value; + const bool force_fp32_output = ctx.Attr("force_fp32_output"); + + // TODO(grygielski) Add option for bfloat + if (!is_INT8 || force_fp32_output) { + RunKernel(ctx); + } else { + RunKernel(ctx); + } + } + + template + void RunKernel(const framework::ExecutionContext& ctx) const { auto& dev_ctx = ctx.template device_context(); const auto& mkldnn_engine = dev_ctx.GetEngine(); @@ -364,13 +421,16 @@ class FusionGRUMKLDNNKernel : public framework::OpKernel { const auto* weight_h = ctx.Input("WeightH"); const auto* bias = ctx.Input("Bias"); auto* hidden = ctx.Output("Hidden"); - + auto x_dims = input->dims(); + auto x_mat_dims = (x_dims.size() == 3 && x_dims[1] == 1) + ? framework::flatten_to_2d(x_dims, 1) + : x_dims; // Get attributes const bool is_reverse = ctx.Attr("is_reverse"); const bool origin_mode = ctx.Attr("origin_mode"); // Get tensor dimensions - const auto x_dims = framework::vectorize(input->dims()); + const auto x_mat_dims_vec = framework::vectorize(x_mat_dims); const auto weight_h_dims = framework::vectorize(weight_h->dims()); const auto& input_lod = input->lod()[0]; @@ -384,15 +444,17 @@ class FusionGRUMKLDNNKernel : public framework::OpKernel { } return res; }(); - const int64_t IC = x_dims[1]; // Input channels - const int64_t OC = weight_h_dims[0]; // Output channels + const int64_t IC = x_mat_dims_vec[1]; // Input channels + const int64_t OC = weight_h_dims[0]; // Output channels - GRUMKLDNNHandler handler(ctx, dev_ctx, mkldnn_engine, ctx.GetPlace(), - input, weight_h, h0, is_reverse, N, Ti, IC, OC, - ctx.InputName("X") + ctx.InputName("WeightH")); + GRUMKLDNNHandler handler( + ctx, dev_ctx, mkldnn_engine, ctx.GetPlace(), input, weight_h, h0, + is_reverse, N, Ti, IC, OC, + ctx.InputName("X") + ctx.InputName("WeightH")); auto input_memory_p = handler.AcquireInputMemoryWithReorder(input, is_reverse); + auto h0_memory_p = handler.AcquireH0Memory(h0); auto weight_x_memory_p = handler.AcquireWeightXMemory(weight_x, origin_mode); auto weight_h_memory_p = @@ -402,25 +464,21 @@ class FusionGRUMKLDNNKernel : public framework::OpKernel { std::unordered_map gru_args = { {DNNL_ARG_SRC_LAYER, *input_memory_p}, + {DNNL_ARG_SRC_ITER, *h0_memory_p}, {DNNL_ARG_WEIGHTS_LAYER, *weight_x_memory_p}, {DNNL_ARG_WEIGHTS_ITER, *weight_h_memory_p}, {DNNL_ARG_BIAS, *bias_memory_p}, {DNNL_ARG_DST_LAYER, *hidden_onednn_memory_p}}; - if (h0) { - auto h0_memory_p = handler.AcquireH0Memory(h0); - gru_args.insert({DNNL_ARG_SRC_ITER, *h0_memory_p}); - } - auto gru_forward_p = handler.AcquireForwardPrimitive(); dnnl::stream astream(mkldnn_engine); gru_forward_p->execute(astream, gru_args); astream.wait(); - auto* hidden_onednn_data = - reinterpret_cast(hidden_onednn_memory_p->get_data_handle()); - auto* hidden_data = hidden->mutable_data(ctx.GetPlace()); + auto* hidden_onednn_data = hidden_onednn_memory_p->get_data_handle(); + auto* hidden_data = + to_void_cast(hidden->mutable_data(ctx.GetPlace())); if (handler.is_NTC()) { handler.reorderRNNdata(hidden_onednn_data, hidden_data, input_lod, is_reverse, platform::RNNReorderType::NTC_PP); @@ -436,4 +494,5 @@ class FusionGRUMKLDNNKernel : public framework::OpKernel { namespace ops = paddle::operators; REGISTER_OP_KERNEL(fusion_gru, MKLDNN, paddle::platform::CPUPlace, - ops::FusionGRUMKLDNNKernel); + ops::FusionGRUMKLDNNKernel, + ops::FusionGRUMKLDNNKernel); diff --git a/paddle/fluid/operators/gaussian_random_op.cc b/paddle/fluid/operators/gaussian_random_op.cc index 4f128463375b91803a7a4d02a27dd78157961aac..17a71c67b8a084c114497eb97568e9b536161711 100644 --- a/paddle/fluid/operators/gaussian_random_op.cc +++ b/paddle/fluid/operators/gaussian_random_op.cc @@ -34,8 +34,7 @@ class CPUGaussianRandomKernel : public framework::OpKernel { auto* tensor = context.Output("Out"); std::normal_distribution dist(mean, std); - const std::string op_type = "gaussian_random"; - auto shape = GetShape(context, op_type); + auto shape = GetShape(context); tensor->Resize(shape); int64_t size = tensor->numel(); T* data = tensor->mutable_data(context.GetPlace()); diff --git a/paddle/fluid/operators/gaussian_random_op.cu b/paddle/fluid/operators/gaussian_random_op.cu index 69c8b60040651179784cd6b77c31c66e892231be..7a0c93eb1b2eaa7afaae7f0a604a0da5ac0fd75d 100644 --- a/paddle/fluid/operators/gaussian_random_op.cu +++ b/paddle/fluid/operators/gaussian_random_op.cu @@ -58,8 +58,7 @@ class GPUGaussianRandomKernel : public framework::OpKernel { T mean = static_cast(context.Attr("mean")); T std = static_cast(context.Attr("std")); thrust::counting_iterator index_sequence_begin(0); - const std::string op_type = "gaussian_random"; - auto shape = GetShape(context, op_type); + auto shape = GetShape(context); tensor->Resize(shape); T* data = tensor->mutable_data(context.GetPlace()); diff --git a/paddle/fluid/operators/grid_sampler_op.cc b/paddle/fluid/operators/grid_sampler_op.cc index deb71b807128e5c0b173b517e60832894ced41e5..f5224239eb2ded9a156aadc9185eca89f4e3396f 100644 --- a/paddle/fluid/operators/grid_sampler_op.cc +++ b/paddle/fluid/operators/grid_sampler_op.cc @@ -115,7 +115,7 @@ class GridSampleOpMaker : public framework::OpProtoAndCheckerMaker { AddAttr( "padding_mode", "(bool, default true) The padding method used when source" - "index is out of input images. It can be 'zeros', 'reflect' and " + "index is out of input images. It can be 'zeros', 'reflection' and " "'border'.") .SetDefault("zeros"); @@ -174,6 +174,10 @@ class GridSampleOpGrad : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; void InferShape(framework::InferShapeContext* ctx) const override { + OP_INOUT_CHECK(ctx->HasOutput(framework::GradVarName("X")), "Output", + framework::GradVarName("X"), "grid_sampler"); + OP_INOUT_CHECK(ctx->HasOutput(framework::GradVarName("Grid")), "Output", + framework::GradVarName("Grid"), "grid_sampler"); auto input_dims = ctx->GetInputDim("X"); auto grid_dims = ctx->GetInputDim("Grid"); if (ctx->HasOutput(framework::GradVarName("X"))) { diff --git a/paddle/fluid/operators/grid_sampler_op.cu b/paddle/fluid/operators/grid_sampler_op.cu index 999f990448ca6370dadacbdaee5bf3bcadcaca0e..4e61d0c2ea7f91e4199c3e9daa3e93ac45bc0eb8 100644 --- a/paddle/fluid/operators/grid_sampler_op.cu +++ b/paddle/fluid/operators/grid_sampler_op.cu @@ -268,7 +268,7 @@ class GridSampleOpCUDAKernel : public framework::OpKernel { Mode mode; if (padding_mode_s == "border") { padding_mode = PaddingMode::border; - } else if (padding_mode_s == "reflect") { + } else if (padding_mode_s == "reflection") { padding_mode = PaddingMode::reflect; } else { padding_mode = PaddingMode::zeros; @@ -432,7 +432,7 @@ class GridSampleGradOpCUDAKernel : public framework::OpKernel { Mode mode; if (padding_mode_s == "border") { padding_mode = PaddingMode::border; - } else if (padding_mode_s == "reflect") { + } else if (padding_mode_s == "reflection") { padding_mode = PaddingMode::reflect; } else { padding_mode = PaddingMode::zeros; diff --git a/paddle/fluid/operators/grid_sampler_op.h b/paddle/fluid/operators/grid_sampler_op.h index eda800e78faf5da2bb379b8101e4823c5bc2d2f8..b8faef759ae90e14d1e83b66130bfe957b51907b 100644 --- a/paddle/fluid/operators/grid_sampler_op.h +++ b/paddle/fluid/operators/grid_sampler_op.h @@ -76,7 +76,7 @@ static inline void clip(const platform::CPUDeviceContext& ctx, if (padding_mode == "border") { grid_slice_t.device(place) = grid_slice_t.cwiseMax(static_cast(0)) .cwiseMin(static_cast(max_val)); - } else if (padding_mode == "reflect") { + } else if (padding_mode == "reflection") { if (align_corners) { auto double_range = static_cast(max_val * 2); auto grid_abs = grid_slice_t.abs(); @@ -117,7 +117,7 @@ static inline void clipWithMask(const platform::CPUDeviceContext& ctx, auto in_bound = (res == grid_slice_t); grid_scale_t.device(place) = grid_scale_t * in_bound.template cast(); grid_slice_t.device(place) = res; - } else if (padding_mode == "reflect") { + } else if (padding_mode == "reflection") { if (align_corners) { auto double_range = static_cast(max_val * 2); auto is_neg = (grid_slice_t < static_cast(0)); diff --git a/paddle/fluid/operators/interpolate_v2_op.cc b/paddle/fluid/operators/interpolate_v2_op.cc index 12733a0d9f1689a020f77d23cc31b0d19b412746..1f7dde9b931dafa4b8e0bee211e64461b1c21dc5 100644 --- a/paddle/fluid/operators/interpolate_v2_op.cc +++ b/paddle/fluid/operators/interpolate_v2_op.cc @@ -67,7 +67,7 @@ static void Interpolate1DInferShapeCheck(framework::InferShapeContext* ctx) { scale_tensor[0], 1, platform::errors::InvalidArgument( "Scale's shape must be 1, but got shape = %d .", scale_tensor[0])); - // out_w = -1; + out_w = -1; } else { auto scale = ctx->Attrs().Get>("scale"); if (scale.size() > 0) { @@ -159,8 +159,8 @@ static void Interpolate2DInferShapeCheck(framework::InferShapeContext* ctx) { platform::errors::InvalidArgument( "Scale's shape must be 2 or 1, but got shape = %d .", scale_tensor[0])); - // out_h = -1; - // out_w = -1; + out_h = -1; + out_w = -1; } else { auto scale = ctx->Attrs().Get>("scale"); if (scale.size() > 0) { @@ -264,9 +264,9 @@ static void Interpolate3DInferShapeCheck(framework::InferShapeContext* ctx) { platform::errors::InvalidArgument( "Scale's shape must be 3 or 1, but got shape = %d .", scale_tensor[0])); - // out_d = -1; - // out_h = -1; - // out_w = -1; + out_d = -1; + out_h = -1; + out_w = -1; } else { auto scale = ctx->Attrs().Get>("scale"); if (scale.size() > 0) { @@ -633,6 +633,9 @@ DECLARE_NO_NEED_BUFFER_VARS_INFERER(InterpolateV2GradNoNeedBufferVarsInferer, } // namespace operators } // namespace paddle +// interp_v2 support scale_factor whose input type is list, this operation is +// not +// compatible with interp_op, so a new one is added in paddle2.0 namespace ops = paddle::operators; REGISTER_OPERATOR(bilinear_interp_v2, ops::InterpolateV2Op, ops::InterpolateV2OpMaker, diff --git a/paddle/fluid/operators/interpolate_v2_op.cu b/paddle/fluid/operators/interpolate_v2_op.cu index 6cb8104638dea458743374014e7bef35df2dbfcc..816539c3b5fdb805d16fb8224b7c960f797613cb 100644 --- a/paddle/fluid/operators/interpolate_v2_op.cu +++ b/paddle/fluid/operators/interpolate_v2_op.cu @@ -836,12 +836,12 @@ static void Interpolate1DCUDAFwd(const framework::ExecutionContext& ctx, int out_w = ctx.Attr("out_w"); auto list_new_shape_tensor = ctx.MultiInput("SizeTensor"); + float scale_w = -1; if (list_new_shape_tensor.size() > 0) { // have size tensor auto new_size = get_new_shape(list_new_shape_tensor); out_w = new_size[0]; } else { - float scale_w = -1; auto scale_tensor = ctx.Input("Scale"); auto scale = ctx.Attr>("scale"); if (scale_tensor != nullptr) { @@ -887,8 +887,11 @@ static void Interpolate1DCUDAFwd(const framework::ExecutionContext& ctx, float ratio_w = 0.f; if (out_w > 1) { + float new_scale_w = 0.f; + new_scale_w = (scale_w > 0) ? static_cast(1. / scale_w) + : static_cast(in_w) / out_w; ratio_w = (align_corners) ? static_cast(in_w - 1.0) / (out_w - 1.0) - : static_cast(in_w) / out_w; + : static_cast(new_scale_w); } int in_cw = c * in_w; @@ -924,14 +927,14 @@ static void Interpolate2DCUDAFwd(const framework::ExecutionContext& ctx, int out_w = ctx.Attr("out_w"); auto list_new_shape_tensor = ctx.MultiInput("SizeTensor"); + float scale_w = -1; + float scale_h = -1; if (list_new_shape_tensor.size() > 0) { // have size tensor auto new_size = get_new_shape(list_new_shape_tensor); out_h = new_size[0]; out_w = new_size[1]; } else { - float scale_h = -1; - float scale_w = -1; auto scale_tensor = ctx.Input("Scale"); auto scale = ctx.Attr>("scale"); if (scale_tensor != nullptr) { @@ -993,12 +996,18 @@ static void Interpolate2DCUDAFwd(const framework::ExecutionContext& ctx, float ratio_h = 0.f; float ratio_w = 0.f; if (out_h > 1) { + float new_scale_h = 0.f; + new_scale_h = (scale_h > 0) ? static_cast(1. / scale_h) + : static_cast(in_h) / out_h; ratio_h = (align_corners) ? static_cast(in_h - 1) / (out_h - 1) - : static_cast(in_h) / out_h; + : static_cast(new_scale_h); } if (out_w > 1) { + float new_scale_w = 0.f; + new_scale_w = (scale_w > 0) ? static_cast(1. / scale_w) + : static_cast(in_w) / out_w; ratio_w = (align_corners) ? static_cast(in_w - 1) / (out_w - 1) - : static_cast(in_w) / out_w; + : static_cast(new_scale_w); } int in_hw = in_h * in_w; @@ -1048,6 +1057,9 @@ static void Interpolate3DCUDAFwd(const framework::ExecutionContext& ctx, int out_w = ctx.Attr("out_w"); auto list_new_shape_tensor = ctx.MultiInput("SizeTensor"); + float scale_w = -1; + float scale_d = -1; + float scale_h = -1; if (list_new_shape_tensor.size() > 0) { // have size tensor auto new_size = get_new_shape(list_new_shape_tensor); @@ -1055,9 +1067,6 @@ static void Interpolate3DCUDAFwd(const framework::ExecutionContext& ctx, out_h = new_size[1]; out_w = new_size[2]; } else { - float scale_d = -1; - float scale_h = -1; - float scale_w = -1; auto scale_tensor = ctx.Input("Scale"); auto scale = ctx.Attr>("scale"); if (scale_tensor != nullptr) { @@ -1129,16 +1138,25 @@ static void Interpolate3DCUDAFwd(const framework::ExecutionContext& ctx, float ratio_h = 0.f; float ratio_w = 0.f; if (out_d > 1) { + float new_scale_d = 0.f; + new_scale_d = (scale_d > 0) ? static_cast(1. / scale_d) + : static_cast(in_d) / out_d; ratio_d = (align_corners) ? static_cast(in_d - 1) / (out_d - 1) - : static_cast(in_d) / out_d; + : static_cast(new_scale_d); } if (out_h > 1) { + float new_scale_h = 0.f; + new_scale_h = (scale_h > 0) ? static_cast(1. / scale_h) + : static_cast(in_h) / out_h; ratio_h = (align_corners) ? static_cast(in_h - 1) / (out_h - 1) - : static_cast(in_h) / out_h; + : static_cast(new_scale_h); } if (out_w > 1) { + float new_scale_w = 0.f; + new_scale_w = (scale_w > 0) ? static_cast(1. / scale_w) + : static_cast(in_w) / out_w; ratio_w = (align_corners) ? static_cast(in_w - 1) / (out_w - 1) - : static_cast(in_w) / out_w; + : static_cast(new_scale_w); } int in_dhw = in_d * in_h * in_w; @@ -1230,8 +1248,11 @@ static void Interpolate1DCUDABwd(const framework::ExecutionContext& ctx, float ratio_w = 0.f; if (out_w > 1) { + float new_scale_w = 0.f; + new_scale_w = (scale_w > 0) ? static_cast(1. / scale_w) + : static_cast(in_w) / out_w; ratio_w = (align_corners) ? static_cast(in_w - 1) / (out_w - 1) - : static_cast(in_w) / out_w; + : static_cast(new_scale_w); } int in_cw = c * in_w; int out_cw = c * out_w; @@ -1333,12 +1354,18 @@ static void Interpolate2DCUDABwd(const framework::ExecutionContext& ctx, float ratio_h = 0.f; float ratio_w = 0.f; if (out_h > 1) { + float new_scale_h = 0.f; + new_scale_h = (scale_h > 0) ? static_cast(1. / scale_h) + : static_cast(in_h) / out_h; ratio_h = (align_corners) ? static_cast(in_h - 1) / (out_h - 1) - : static_cast(in_h) / out_h; + : static_cast(new_scale_h); } if (out_w > 1) { + float new_scale_w = 0.f; + new_scale_w = (scale_w > 0) ? static_cast(1. / scale_w) + : static_cast(in_w) / out_w; ratio_w = (align_corners) ? static_cast(in_w - 1) / (out_w - 1) - : static_cast(in_w) / out_w; + : static_cast(new_scale_w); } int in_hw = in_h * in_w; @@ -1464,16 +1491,25 @@ static void Interpolate3DCUDABwd(const framework::ExecutionContext& ctx, float ratio_h = 0.f; float ratio_w = 0.f; if (out_d > 1) { + float new_scale_d = 0.f; + new_scale_d = (scale_d > 0) ? static_cast(1. / scale_d) + : static_cast(in_d) / out_d; ratio_d = (align_corners) ? static_cast(in_d - 1) / (out_d - 1) - : static_cast(in_d) / out_d; + : static_cast(new_scale_d); } if (out_h > 1) { + float new_scale_h = 0.f; + new_scale_h = (scale_h > 0) ? static_cast(1. / scale_h) + : static_cast(in_h) / out_h; ratio_h = (align_corners) ? static_cast(in_h - 1) / (out_h - 1) - : static_cast(in_h) / out_h; + : static_cast(new_scale_h); } if (out_w > 1) { + float new_scale_w = 0.f; + new_scale_w = (scale_w > 0) ? static_cast(1. / scale_w) + : static_cast(in_w) / out_w; ratio_w = (align_corners) ? static_cast(in_w - 1) / (out_w - 1) - : static_cast(in_w) / out_w; + : static_cast(new_scale_w); } int in_dhw = in_d * in_h * in_w; diff --git a/paddle/fluid/operators/interpolate_v2_op.h b/paddle/fluid/operators/interpolate_v2_op.h index 111766934b8300c0a7b46ae9a065b8c42460e577..4e4fd9ff63ba47b41363a81d6cc527486671d695 100644 --- a/paddle/fluid/operators/interpolate_v2_op.h +++ b/paddle/fluid/operators/interpolate_v2_op.h @@ -783,12 +783,13 @@ static void Interpolate1DCPUFwd(const framework::ExecutionContext& ctx, int out_w = ctx.Attr("out_w"); auto list_new_size_tensor = ctx.MultiInput("SizeTensor"); + float scale_w = -1.; if (list_new_size_tensor.size() > 0) { // have size tensor auto new_size = get_new_shape(list_new_size_tensor); out_w = new_size[0]; } else { - float scale_w = -1; + // float scale_w = -1; auto scale_tensor = ctx.Input("Scale"); auto scale = ctx.Attr>("scale"); if (scale_tensor != nullptr) { @@ -833,8 +834,11 @@ static void Interpolate1DCPUFwd(const framework::ExecutionContext& ctx, float ratio_w = 0.f; if (out_w > 1) { + float new_scale_w = 0.f; + new_scale_w = (scale_w > 0) ? static_cast(1. / scale_w) + : static_cast(in_w) / out_w; ratio_w = (align_corners) ? static_cast(in_w - 1) / (out_w - 1) - : static_cast(in_w) / out_w; + : static_cast(new_scale_w); } if ("linear" == interp_method) { LinearInterpolation(input, output, ratio_w, in_w, n, c, out_w, @@ -856,6 +860,8 @@ static void Interpolate2DCPUFwd(const framework::ExecutionContext& ctx, int out_h = ctx.Attr("out_h"); int out_w = ctx.Attr("out_w"); + float scale_h = -1; + float scale_w = -1; auto list_new_size_tensor = ctx.MultiInput("SizeTensor"); if (list_new_size_tensor.size() > 0) { @@ -864,8 +870,6 @@ static void Interpolate2DCPUFwd(const framework::ExecutionContext& ctx, out_h = new_size[0]; out_w = new_size[1]; } else { - float scale_h = -1; - float scale_w = -1; auto scale_tensor = ctx.Input("Scale"); auto scale = ctx.Attr>("scale"); if (scale_tensor != nullptr) { @@ -925,12 +929,18 @@ static void Interpolate2DCPUFwd(const framework::ExecutionContext& ctx, float ratio_h = 0.f; float ratio_w = 0.f; if (out_h > 1) { + float new_scale_h = 0.f; + new_scale_h = (scale_h > 0) ? static_cast(1. / scale_h) + : static_cast(in_h) / out_h; ratio_h = (align_corners) ? static_cast(in_h - 1) / (out_h - 1) - : static_cast(in_h) / out_h; + : static_cast(new_scale_h); } if (out_w > 1) { + float new_scale_w = 0.f; + new_scale_w = (scale_w > 0) ? static_cast(1. / scale_w) + : static_cast(in_w) / out_w; ratio_w = (align_corners) ? static_cast(in_w - 1) / (out_w - 1) - : static_cast(in_w) / out_w; + : static_cast(new_scale_w); } if ("bilinear" == interp_method) { @@ -962,6 +972,10 @@ static void Interpolate3DCPUFwd(const framework::ExecutionContext& ctx, int out_h = ctx.Attr("out_h"); int out_w = ctx.Attr("out_w"); + float scale_d = -1; + float scale_h = -1; + float scale_w = -1; + auto list_new_size_tensor = ctx.MultiInput("SizeTensor"); if (list_new_size_tensor.size() > 0) { // have size tensor @@ -970,9 +984,6 @@ static void Interpolate3DCPUFwd(const framework::ExecutionContext& ctx, out_h = new_size[1]; out_w = new_size[2]; } else { - float scale_d = -1; - float scale_h = -1; - float scale_w = -1; auto scale_tensor = ctx.Input("Scale"); auto scale = ctx.Attr>("scale"); if (scale_tensor != nullptr) { @@ -1043,16 +1054,25 @@ static void Interpolate3DCPUFwd(const framework::ExecutionContext& ctx, float ratio_h = 0.f; float ratio_w = 0.f; if (out_d > 1) { + float new_scale_d = 0.f; + new_scale_d = (scale_d > 0) ? static_cast(1. / scale_d) + : static_cast(in_d) / out_d; ratio_d = (align_corners) ? static_cast(in_d - 1) / (out_d - 1) - : static_cast(in_d) / out_d; + : static_cast(new_scale_d); } if (out_h > 1) { + float new_scale_h = 0.f; + new_scale_h = (scale_h > 0) ? static_cast(1. / scale_h) + : static_cast(in_h) / out_h; ratio_h = (align_corners) ? static_cast(in_h - 1) / (out_h - 1) - : static_cast(in_h) / out_h; + : static_cast(new_scale_h); } if (out_w > 1) { + float new_scale_w = 0.f; + new_scale_w = (scale_w > 0) ? static_cast(1. / scale_w) + : static_cast(in_w) / out_w; ratio_w = (align_corners) ? static_cast(in_w - 1) / (out_w - 1) - : static_cast(in_w) / out_w; + : static_cast(new_scale_w); } if ("trilinear" == interp_method) { @@ -1127,8 +1147,11 @@ static void Interpolate1DCPUBwd(const framework::ExecutionContext& ctx, float ratio_w = 0.f; if (out_w > 1) { + float new_scale_w = 0.f; + new_scale_w = (scale_w > 0) ? static_cast(1. / scale_w) + : static_cast(in_w) / out_w; ratio_w = (align_corners) ? static_cast(in_w - 1) / (out_w - 1) - : static_cast(in_w) / out_w; + : static_cast(new_scale_w); } if ("linear" == interp_method) { LinearInterpolationGrad(output_grad, input_grad, ratio_w, in_w, n, c, @@ -1216,12 +1239,18 @@ static void Interpolate2DCPUBwd(const framework::ExecutionContext& ctx, float ratio_h = 0.f; float ratio_w = 0.f; if (out_h > 1) { + float new_scale_h = 0.f; + new_scale_h = (scale_h > 0) ? static_cast(1. / scale_h) + : static_cast(in_h) / out_h; ratio_h = (align_corners) ? static_cast(in_h - 1) / (out_h - 1) - : static_cast(in_h) / out_h; + : static_cast(new_scale_h); } if (out_w > 1) { + float new_scale_w = 0.f; + new_scale_w = (scale_w > 0) ? static_cast(1. / scale_w) + : static_cast(in_w) / out_w; ratio_w = (align_corners) ? static_cast(in_w - 1) / (out_w - 1) - : static_cast(in_w) / out_w; + : static_cast(new_scale_w); } if ("bilinear" == interp_method) { @@ -1327,16 +1356,25 @@ static void Interpolate3DCPUBwd(const framework::ExecutionContext& ctx, float ratio_h = 0.f; float ratio_w = 0.f; if (out_d > 1) { + float new_scale_d = 0.f; + new_scale_d = (scale_d > 0) ? static_cast(1. / scale_d) + : static_cast(in_d) / out_d; ratio_d = (align_corners) ? static_cast(in_d - 1) / (out_d - 1) - : static_cast(in_d) / out_d; + : static_cast(new_scale_d); } if (out_h > 1) { + float new_scale_h = 0.f; + new_scale_h = (scale_h > 0) ? static_cast(1. / scale_h) + : static_cast(in_h) / out_h; ratio_h = (align_corners) ? static_cast(in_h - 1) / (out_h - 1) - : static_cast(in_h) / out_h; + : static_cast(new_scale_h); } if (out_w > 1) { + float new_scale_w = 0.f; + new_scale_w = (scale_w > 0) ? static_cast(1. / scale_w) + : static_cast(in_w) / out_w; ratio_w = (align_corners) ? static_cast(in_w - 1) / (out_w - 1) - : static_cast(in_w) / out_w; + : static_cast(new_scale_w); } if ("trilinear" == interp_method) { diff --git a/paddle/fluid/operators/kldiv_loss_op.h b/paddle/fluid/operators/kldiv_loss_op.h index 369fdb4872b4184d706a5264b58f70f63051fca1..857ecda303c2607b1b6fb9a5d2ec132b335d6c29 100644 --- a/paddle/fluid/operators/kldiv_loss_op.h +++ b/paddle/fluid/operators/kldiv_loss_op.h @@ -72,7 +72,11 @@ class KLDivLossKernel : public framework::OpKernel { loss_t.device(place) = output; } else if ("batchmean" == reduction) { auto output_sum = output.sum(); - loss_t.device(place) = output_sum / output_sum.constant(n); + if (n > 0) { + loss_t.device(place) = output_sum / output_sum.constant(n); + } else { + loss_t.device(place) = output_sum; + } } else if ("mean" == reduction) { loss_t.device(place) = output.mean(); } else if ("sum" == reduction) { diff --git a/paddle/fluid/operators/lite/lite_engine_op.h b/paddle/fluid/operators/lite/lite_engine_op.h index a920bf7c3f505b839f8f1fd252c9f8505393f3a9..f6d65704388e6ec90c9209475e5f4b19061085fa 100644 --- a/paddle/fluid/operators/lite/lite_engine_op.h +++ b/paddle/fluid/operators/lite/lite_engine_op.h @@ -39,7 +39,7 @@ class LiteEngineOp : public framework::OperatorBase { private: std::vector in_names_; std::vector out_names_; - paddle::lite::Predictor *engine_; + paddle::lite_api::PaddlePredictor *engine_; framework::proto::VarType::Type precision_; bool use_gpu_; bool zero_copy_; @@ -78,10 +78,10 @@ class LiteEngineOp : public framework::OperatorBase { framework::LoDTensor src_t = inference::analysis::GetFromScope(scope, in_names_[i]); - paddle::lite::Tensor *dst_t = engine_->GetInput(i); + paddle::lite_api::Tensor dst_t = *(engine_->GetInput(i)); VLOG(3) << "== fluid -> lite (" << in_names_[i] << " -> " << engine_->GetInputNames()[i] << ")"; - inference::lite::utils::TensorCopy(dst_t, &src_t, *ctx, zero_copy_); + inference::lite::utils::TensorCopy(&dst_t, &src_t, *ctx, zero_copy_); } #ifdef PADDLE_WITH_CUDA if (platform::is_gpu_place(dev_place)) { @@ -93,7 +93,7 @@ class LiteEngineOp : public framework::OperatorBase { engine_->Run(); VLOG(3) << "lite engine run done"; for (size_t i = 0; i < out_names_.size(); i++) { - paddle::lite::Tensor src_t = *(engine_->GetOutput(i)); + paddle::lite_api::Tensor src_t = *(engine_->GetOutput(i)); framework::LoDTensor *dst_t = &inference::analysis::GetFromScope( scope, out_names_[i]); diff --git a/paddle/fluid/operators/lite/lite_engine_op_test.cc b/paddle/fluid/operators/lite/lite_engine_op_test.cc index fb5c0dcb3514de815b97944d0fdbf3bd7853b628..76c963ac652687cb0f65a0497b5c994f82d0d7aa 100644 --- a/paddle/fluid/operators/lite/lite_engine_op_test.cc +++ b/paddle/fluid/operators/lite/lite_engine_op_test.cc @@ -84,10 +84,10 @@ TEST(LiteEngineOp, engine_op) { inference::lite::EngineConfig config; config.valid_places = { #ifdef PADDLE_WITH_CUDA - paddle::lite::Place({TARGET(kCUDA), PRECISION(kFloat)}), + paddle::lite_api::Place({TARGET(kCUDA), PRECISION(kFloat)}), #endif - paddle::lite::Place({TARGET(kHost), PRECISION(kAny)}), - paddle::lite::Place({TARGET(kX86), PRECISION(kFloat)}), + paddle::lite_api::Place({TARGET(kX86), PRECISION(kFloat)}), + paddle::lite_api::Place({TARGET(kHost), PRECISION(kAny)}), }; serialize_params(&(config.param), &scope, repetitive_params); config.model = program.Proto()->SerializeAsString(); diff --git a/paddle/fluid/operators/math/CMakeLists.txt b/paddle/fluid/operators/math/CMakeLists.txt index 3a19c7edff3569d503480fd060a6432dc59d2108..10d335b828b516fe08871f314ba4667c06f04714 100644 --- a/paddle/fluid/operators/math/CMakeLists.txt +++ b/paddle/fluid/operators/math/CMakeLists.txt @@ -9,7 +9,11 @@ function(math_library TARGET) set(hip_srcs) set(math_common_deps device_context framework_proto enforce) if (WITH_GPU) - list(APPEND math_common_deps cub) + if (${CMAKE_CUDA_COMPILER_VERSION} LESS 11.0) + list(APPEND math_common_deps cub) + else() + list(APPEND math_common_deps) + endif() endif() set(multiValueArgs DEPS) cmake_parse_arguments(math_library "${options}" "${oneValueArgs}" diff --git a/paddle/fluid/operators/math/concat_test.cc b/paddle/fluid/operators/math/concat_test.cc index 411dbca25bb48c99dfd16779f54e46a3e80d0d4e..270a9d3f80a80d5ea2c8b97d4a69125355ddef61 100644 --- a/paddle/fluid/operators/math/concat_test.cc +++ b/paddle/fluid/operators/math/concat_test.cc @@ -79,8 +79,16 @@ void ConcatCase1(DeviceContext* context) { concat_functor(*context, input, 0, &out); // check the dim of input_a, input_b - PADDLE_ENFORCE_EQ(input_a.dims(), dim_a); - PADDLE_ENFORCE_EQ(input_b.dims(), dim_b); + PADDLE_ENFORCE_EQ(input_a.dims(), dim_a, + paddle::platform::errors::InvalidArgument( + "The dims of Input tensor should be the same as the " + "declared dims. Tensor dims: [%s], declared dims: [%s]", + input_a.dims(), dim_a)); + PADDLE_ENFORCE_EQ(input_b.dims(), dim_b, + paddle::platform::errors::InvalidArgument( + "The dims of Input tensor should be the same as the " + "declared dims. Tensor dims: [%s], declared dims: [%s]", + input_b.dims(), dim_b)); int* out_ptr = nullptr; if (paddle::platform::is_gpu_place(Place())) { @@ -95,10 +103,14 @@ void ConcatCase1(DeviceContext* context) { int idx_a = 0, idx_b = 0; for (int j = 0; j < 5 * 3 * 4; ++j) { if (j >= cols) { - PADDLE_ENFORCE_EQ(out_ptr[j], b_ptr[idx_b]); + PADDLE_ENFORCE_EQ(out_ptr[j], b_ptr[idx_b], + paddle::platform::errors::InvalidArgument( + "Concat test failed, the result should be equal.")); ++idx_b; } else { - PADDLE_ENFORCE_EQ(out_ptr[j], a_ptr[idx_a]); + PADDLE_ENFORCE_EQ(out_ptr[j], a_ptr[idx_a], + paddle::platform::errors::InvalidArgument( + "Concat test failed, the result should be equal.")); ++idx_a; } } @@ -166,8 +178,16 @@ void ConcatCase2(DeviceContext* context) { concat_functor(*context, input, 1, &out); // check the dim of input_a, input_b - PADDLE_ENFORCE_EQ(input_a.dims(), dim_a); - PADDLE_ENFORCE_EQ(input_b.dims(), dim_b); + PADDLE_ENFORCE_EQ(input_a.dims(), dim_a, + paddle::platform::errors::InvalidArgument( + "The dims of Input tensor should be the same as the " + "declared dims. Tensor dims: [%s], declared dims: [%s]", + input_a.dims(), dim_a)); + PADDLE_ENFORCE_EQ(input_b.dims(), dim_b, + paddle::platform::errors::InvalidArgument( + "The dims of Input tensor should be the same as the " + "declared dims. Tensor dims: [%s], declared dims: [%s]", + input_b.dims(), dim_b)); int* out_ptr = nullptr; if (paddle::platform::is_gpu_place(Place())) { @@ -183,10 +203,16 @@ void ConcatCase2(DeviceContext* context) { for (int i = 0; i < 2; ++i) { for (int j = 0; j < 28; ++j) { if (j >= cols) { - PADDLE_ENFORCE_EQ(out_ptr[i * 28 + j], b_ptr[idx_b]); + PADDLE_ENFORCE_EQ( + out_ptr[i * 28 + j], b_ptr[idx_b], + paddle::platform::errors::InvalidArgument( + "Concat test failed, the result should be equal.")); ++idx_b; } else { - PADDLE_ENFORCE_EQ(out_ptr[i * 28 + j], a_ptr[idx_a]); + PADDLE_ENFORCE_EQ( + out_ptr[i * 28 + j], a_ptr[idx_a], + paddle::platform::errors::InvalidArgument( + "Concat test failed, the result should be equal.")); ++idx_a; } } @@ -255,8 +281,16 @@ void ConcatCase3(DeviceContext* context) { concat_functor(*context, input, 2, &out); // check the dim of input_a, input_b - PADDLE_ENFORCE_EQ(input_a.dims(), dim_a); - PADDLE_ENFORCE_EQ(input_b.dims(), dim_b); + PADDLE_ENFORCE_EQ(input_a.dims(), dim_a, + paddle::platform::errors::InvalidArgument( + "The dims of Input tensor should be the same as the " + "declared dims. Tensor dims: [%s], declared dims: [%s]", + input_a.dims(), dim_a)); + PADDLE_ENFORCE_EQ(input_b.dims(), dim_b, + paddle::platform::errors::InvalidArgument( + "The dims of Input tensor should be the same as the " + "declared dims. Tensor dims: [%s], declared dims: [%s]", + input_b.dims(), dim_b)); int* out_ptr = nullptr; if (paddle::platform::is_gpu_place(Place())) { @@ -273,10 +307,16 @@ void ConcatCase3(DeviceContext* context) { for (int i = 0; i < 6; ++i) { for (int j = 0; j < 9; ++j) { if (j >= cols) { - PADDLE_ENFORCE_EQ(out_ptr[i * 9 + j], b_ptr[idx_b]); + PADDLE_ENFORCE_EQ( + out_ptr[i * 9 + j], b_ptr[idx_b], + paddle::platform::errors::InvalidArgument( + "Concat test failed, the result should be equal.")); ++idx_b; } else { - PADDLE_ENFORCE_EQ(out_ptr[i * 9 + j], a_ptr[idx_a]); + PADDLE_ENFORCE_EQ( + out_ptr[i * 9 + j], a_ptr[idx_a], + paddle::platform::errors::InvalidArgument( + "Concat test failed, the result should be equal.")); ++idx_a; } } @@ -347,8 +387,16 @@ void ConcatCase4(DeviceContext* context) { context->Wait(); // check the dim of input_a, input_b - PADDLE_ENFORCE_EQ(input_a.dims(), dim_a); - PADDLE_ENFORCE_EQ(input_b.dims(), dim_b); + PADDLE_ENFORCE_EQ(input_a.dims(), dim_a, + paddle::platform::errors::InvalidArgument( + "The dims of Input tensor should be the same as the " + "declared dims. Tensor dims: [%s], declared dims: [%s]", + input_a.dims(), dim_a)); + PADDLE_ENFORCE_EQ(input_b.dims(), dim_b, + paddle::platform::errors::InvalidArgument( + "The dims of Input tensor should be the same as the " + "declared dims. Tensor dims: [%s], declared dims: [%s]", + input_b.dims(), dim_b)); int* out_ptr = nullptr; if (paddle::platform::is_gpu_place(Place())) { @@ -365,10 +413,16 @@ void ConcatCase4(DeviceContext* context) { for (int i = 0; i < 2; ++i) { for (int j = 0; j < 24; ++j) { if (j >= cols) { - PADDLE_ENFORCE_EQ(out_ptr[i * 24 + j], b_ptr[idx_b]); + PADDLE_ENFORCE_EQ( + out_ptr[i * 24 + j], b_ptr[idx_b], + paddle::platform::errors::InvalidArgument( + "Concat test failed, the result should be equal.")); ++idx_b; } else { - PADDLE_ENFORCE_EQ(out_ptr[i * 24 + j], a_ptr[idx_a]); + PADDLE_ENFORCE_EQ( + out_ptr[i * 24 + j], a_ptr[idx_a], + paddle::platform::errors::InvalidArgument( + "Concat test failed, the result should be equal.")); ++idx_a; } } diff --git a/paddle/fluid/operators/math/context_project.h b/paddle/fluid/operators/math/context_project.h index e9019c6d2fe6890ee92cb5a3b047666e3c2a7e04..051c6019d74f7d2820dc0ba668da3cafe8864346 100644 --- a/paddle/fluid/operators/math/context_project.h +++ b/paddle/fluid/operators/math/context_project.h @@ -134,7 +134,10 @@ class ContextProjectFunctor { } } if (padding_trainable) { - PADDLE_ENFORCE_NOT_NULL(padding_data); + PADDLE_ENFORCE_NOT_NULL( + padding_data, + platform::errors::InvalidArgument( + "The input tensor 'padding_data' should not be NULL.")); for (int i = 0; i < static_cast(lod_level_0.size()) - 1; ++i) { if (lod_level_0[i] == lod_level_0[i + 1]) continue; diff --git a/paddle/fluid/operators/math/cpu_vec.h b/paddle/fluid/operators/math/cpu_vec.h index 8940a41424b01c975f1264ca309cc09fc3c7ae85..925f3b6161ae8506107f917196e77ecb2d9c5593 100644 --- a/paddle/fluid/operators/math/cpu_vec.h +++ b/paddle/fluid/operators/math/cpu_vec.h @@ -621,7 +621,10 @@ class VecActivations { } else if (type == "identity" || type == "") { return vec_identity; } - PADDLE_THROW("Not support type: %s", type); + PADDLE_THROW(platform::errors::InvalidArgument( + "Expected type should be one of sigmod, relu, tanh, identity. But got " + "not support type: %s.", + type)); } }; diff --git a/paddle/fluid/operators/math/cross_entropy.cu b/paddle/fluid/operators/math/cross_entropy.cu index c7fac60dd3e663088813f795352e4d751059de39..84fa0d6af990e22083ec1a0e3993893cefad1ab5 100644 --- a/paddle/fluid/operators/math/cross_entropy.cu +++ b/paddle/fluid/operators/math/cross_entropy.cu @@ -27,8 +27,8 @@ __global__ void CrossEntropyKernel(T* Y, const T* X, const int64_t* label, const int ignore_index) { CUDA_KERNEL_LOOP(i, N) { PADDLE_ENFORCE(label[i] >= 0 && label[i] < D || label[i] == ignore_index, - "label[%d] expected >= 0 and < %ld, or == %ld, but got " - "%ld. Please check input value.", + "The value of label[%d] expected >= 0 and < %ld, or == %ld, " + "but got %ld. Please check input value.", i, D, ignore_index, label[i]); Y[i] = ignore_index == label[i] ? static_cast(0) diff --git a/paddle/fluid/operators/math/im2col.cc b/paddle/fluid/operators/math/im2col.cc index 094a7237826610af574061263e5b0df5eafdf239..6fb393d791cc2a077dbcd0a912bcf31b5d59ad65 100644 --- a/paddle/fluid/operators/math/im2col.cc +++ b/paddle/fluid/operators/math/im2col.cc @@ -34,9 +34,16 @@ class Im2ColFunctor& stride, const std::vector& padding, framework::Tensor* col, const DataLayout data_layout) { - PADDLE_ENFORCE_EQ(im.dims().size(), 3, "The dimension of im should be 3."); + PADDLE_ENFORCE_EQ(im.dims().size(), 3, + platform::errors::InvalidArgument( + "The dimension of tensor 'im' should be 3. But got " + "the dims of tensor 'im' is [%s].", + im.dims())); PADDLE_ENFORCE_EQ(col->dims().size(), 5, - "The dimension of col should be 5."); + platform::errors::InvalidArgument( + "The dimension of tensor 'col' should be 5. But got " + "the dims of tensor 'col' is [%s].", + col->dims())); if (stride[0] == 1 && stride[1] == 1 && dilation[0] == 1 && dilation[1] == 1) { @@ -70,9 +77,16 @@ class Col2ImFunctor& stride, const std::vector& padding, framework::Tensor* im, const DataLayout data_layout) { - PADDLE_ENFORCE_EQ(im->dims().size(), 3, "The dimension of im should be 3."); + PADDLE_ENFORCE_EQ(im->dims().size(), 3, + platform::errors::InvalidArgument( + "The dimension of tensor 'im' should be 3. But got " + "the dims of tensor 'im' is [%s].", + im->dims())); PADDLE_ENFORCE_EQ(col.dims().size(), 5, - "The dimension of col should be 5."); + platform::errors::InvalidArgument( + "The dimension of tensor 'col' should be 5. But got " + "the dims of tensor 'col' is [%s].", + col.dims())); int im_channels = (data_layout != DataLayout::kNHWC ? im->dims()[0] : im->dims()[2]); int im_height = @@ -88,16 +102,16 @@ class Col2ImFunctor& stride, const std::vector& padding, framework::Tensor* col, const DataLayout data_layout) { - PADDLE_ENFORCE_EQ(im.dims().size(), 3, "The dimension of im should be 3."); + PADDLE_ENFORCE_EQ(im.dims().size(), 3, + platform::errors::InvalidArgument( + "The dimension of tensor 'im' should be 3. But got " + "the dims of tensor 'im' is [%s].", + im.dims())); PADDLE_ENFORCE_EQ(col->dims().size(), 5, - "The dimension of col should be 5."); + platform::errors::InvalidArgument( + "The dimension of tensor 'col' should be 5. But got " + "the dims of tensor 'col' is [%s].", + col->dims())); int im_channels = im.dims()[0]; int im_height = im.dims()[1]; int im_width = im.dims()[2]; @@ -218,9 +239,16 @@ class Col2ImFunctor& stride, const std::vector& padding, framework::Tensor* im, const DataLayout data_layout) { - PADDLE_ENFORCE_EQ(im->dims().size(), 3, "The dimension of im should be 3."); + PADDLE_ENFORCE_EQ(im->dims().size(), 3, + platform::errors::InvalidArgument( + "The dimension of tensor 'im' should be 3. But got " + "the dims of tensor 'im' is [%s].", + im->dims())); PADDLE_ENFORCE_EQ(col.dims().size(), 5, - "The dimension of col should be 5."); + platform::errors::InvalidArgument( + "The dimension of tensor 'col' should be 5. But got " + "the dims of tensor 'col' is [%s].", + col.dims())); int im_channels = im->dims()[0]; int im_height = im->dims()[1]; int im_width = im->dims()[2]; @@ -231,14 +259,14 @@ class Col2ImFunctordata(); const T* col_data = col.data(); diff --git a/paddle/fluid/operators/math/im2col.cu b/paddle/fluid/operators/math/im2col.cu index 97719300daed9c02a716f31d853e3a381312961c..f2a2148ba6954f50cf59ae30f4f4be6aa070739f 100644 --- a/paddle/fluid/operators/math/im2col.cu +++ b/paddle/fluid/operators/math/im2col.cu @@ -81,9 +81,16 @@ class Im2ColFunctor& stride, const std::vector& padding, framework::Tensor* col, const DataLayout data_layout) { - PADDLE_ENFORCE_EQ(im.dims().size(), 3, "The dimension of im should be 3."); + PADDLE_ENFORCE_EQ(im.dims().size(), 3, + platform::errors::InvalidArgument( + "The dimension of tensor 'im' should be 3. But got " + "the dims of tensor 'im' is [%s].", + im.dims())); PADDLE_ENFORCE_EQ(col->dims().size(), 5, - "The dimension of col should be 5."); + platform::errors::InvalidArgument( + "The dimension of tensor 'col' should be 5. But got " + "the dims of tensor 'col' is [%s].", + col->dims())); int im_channels = (data_layout != DataLayout::kNHWC ? im.dims()[0] : im.dims()[2]); @@ -182,9 +189,16 @@ class Col2ImFunctor& stride, const std::vector& padding, framework::Tensor* im, const DataLayout data_layout) { - PADDLE_ENFORCE_EQ(im->dims().size(), 3, "The dimension of im should be 3."); + PADDLE_ENFORCE_EQ(im->dims().size(), 3, + platform::errors::InvalidArgument( + "The dimension of tensor 'im' should be 3. But got " + "the dims of tensor 'im' is [%s].", + im->dims())); PADDLE_ENFORCE_EQ(col.dims().size(), 5, - "The dimension of col should be 5."); + platform::errors::InvalidArgument( + "The dimension of tensor 'col' should be 5. But got " + "the dims of tensor 'col' is [%s].", + col.dims())); int im_channels = (data_layout != DataLayout::kNHWC ? im->dims()[0] : im->dims()[2]); @@ -201,16 +215,16 @@ class Col2ImFunctor& stride, const std::vector& padding, framework::Tensor* col, const DataLayout data_layout) { - PADDLE_ENFORCE_EQ(im.dims().size(), 3, "The dimension of im should be 3."); + PADDLE_ENFORCE_EQ(im.dims().size(), 3, + platform::errors::InvalidArgument( + "The dimension of tensor 'im' should be 3. But got " + "the dims of tensor 'im' is [%s].", + im.dims())); PADDLE_ENFORCE_EQ(col->dims().size(), 5, - "The dimension of col should be 5."); + platform::errors::InvalidArgument( + "The dimension of tensor 'col' should be 5. But got " + "the dims of tensor 'col' is [%s].", + col->dims())); int im_channels = im.dims()[0]; int im_height = im.dims()[1]; @@ -370,9 +391,16 @@ class Col2ImFunctor& stride, const std::vector& padding, framework::Tensor* im, const DataLayout data_layout) { - PADDLE_ENFORCE_EQ(im->dims().size(), 3, "The dimension of im should be 3."); + PADDLE_ENFORCE_EQ(im->dims().size(), 3, + platform::errors::InvalidArgument( + "The dimension of tensor 'im' should be 3. But got " + "the dims of tensor 'im' is [%s].", + im->dims())); PADDLE_ENFORCE_EQ(col.dims().size(), 5, - "The dimension of col should be 5."); + platform::errors::InvalidArgument( + "The dimension of tensor 'col' should be 5. But got " + "the dims of tensor 'col' is [%s].", + col.dims())); int im_channels = im->dims()[0]; int im_height = im->dims()[1]; @@ -386,16 +414,16 @@ class Col2ImFunctor { const framework::Tensor& input, const framework::Tensor& vector, framework::Tensor* output) { auto in_dims = input.dims(); + auto out_dims = output->dims(); auto size = input.numel() / in_dims[0]; - PADDLE_ENFORCE_EQ(vector.numel(), size); - PADDLE_ENFORCE_EQ(output->dims(), in_dims); + PADDLE_ENFORCE_EQ( + vector.numel(), size, + platform::errors::InvalidArgument( + "The input vector size" + " should be equal to the size of each row of input tensor." + " Expected vector size=%d, but received %d", + size, vector.numel())); + const char* in_dims_cstr = in_dims.to_str().c_str(); + const char* out_dims_cstr = out_dims.to_str().c_str(); + PADDLE_ENFORCE_EQ(out_dims, in_dims, + platform::errors::InvalidArgument( + "The output tensor shape should be same as the input" + " tensor shape. Expected output tensor shape: %s," + " but received %s", + in_dims_cstr, out_dims_cstr)); auto in = framework::EigenMatrix::From(input); auto vec = framework::EigenVector::Flatten(vector); diff --git a/paddle/fluid/operators/math/math_function.cu b/paddle/fluid/operators/math/math_function.cu index fba143d017deb4b4814ad8b10e614357a7ebee23..1c519d226ebfe5ff19876f17b79fd36aa12c4130 100644 --- a/paddle/fluid/operators/math/math_function.cu +++ b/paddle/fluid/operators/math/math_function.cu @@ -88,9 +88,24 @@ struct RowwiseAdd { const framework::Tensor& input, const framework::Tensor& vector, framework::Tensor* output) { auto in_dims = input.dims(); + auto out_dims = output->dims(); auto size = input.numel() / in_dims[0]; - PADDLE_ENFORCE_EQ(vector.numel(), size); - PADDLE_ENFORCE_EQ(output->dims(), in_dims); + PADDLE_ENFORCE_EQ( + vector.numel(), size, + platform::errors::InvalidArgument( + "The input vector size" + " should be equal to the size of each row of input tensor." + " Expected vector size=%d, but received %d", + size, vector.numel())); + const char* in_dims_cstr = in_dims.to_str().c_str(); + const char* out_dims_cstr = out_dims.to_str().c_str(); + PADDLE_ENFORCE_EQ( + out_dims, in_dims, + platform::errors::InvalidArgument( + "The output tensor shape should be same as the input tensor" + " shape. Expected output tensor shape: %s," + " but received %s", + in_dims_cstr, out_dims_cstr)); int blocks = 512; int grids = (input.numel() + blocks - 1) / blocks; RowwiseAddKernel<<>>( @@ -113,7 +128,12 @@ void ColwiseSum::operator()( framework::Tensor* vector) { auto in_dims = input.dims(); auto size = input.numel() / in_dims[0]; - PADDLE_ENFORCE_EQ(vector->numel(), size); + PADDLE_ENFORCE_EQ(vector->numel(), size, + platform::errors::InvalidArgument( + "The size of input vector" + " should be equal to the size of input tensor column" + " dimension. Expected vector size=%d, but received %d", + size, vector->numel())); framework::Tensor one; one.mutable_data({in_dims[0]}, context.GetPlace()); SetConstant set; @@ -134,7 +154,12 @@ void RowwiseSum::operator()( framework::Tensor* vector) { auto in_dims = input.dims(); auto size = input.numel() / in_dims[0]; - PADDLE_ENFORCE_EQ(vector->numel(), in_dims[0]); + PADDLE_ENFORCE_EQ(vector->numel(), in_dims[0], + platform::errors::InvalidArgument( + "The size of input vector" + " should be equal to the size of input tensor row" + " dimension. Expected vector size=%d, but received %d", + in_dims[0], vector->numel())); framework::Tensor one; one.mutable_data({size}, context.GetPlace()); SetConstant set; diff --git a/paddle/fluid/operators/math/math_function_impl.h b/paddle/fluid/operators/math/math_function_impl.h index 693d5620460e1fe6f6d82bd0749b0780b64841f5..869a3054598da9cd2223ca0e705c0f910ba043ec 100644 --- a/paddle/fluid/operators/math/math_function_impl.h +++ b/paddle/fluid/operators/math/math_function_impl.h @@ -59,7 +59,12 @@ void ColwiseSum::operator()(const DeviceContext& context, framework::Tensor* out) { auto in_dims = input.dims(); auto size = input.numel() / in_dims[0]; - PADDLE_ENFORCE_EQ(out->numel(), size); + PADDLE_ENFORCE_EQ(out->numel(), size, + platform::errors::InvalidArgument( + "The size of output tensor " + "should be equal to the size of input tensor column" + " dimension. Expected output size=%d, but received %d", + size, out->numel())); auto in = framework::EigenMatrix::From(input); auto vec = framework::EigenVector::Flatten(*out); @@ -78,7 +83,13 @@ class ColwiseSum { auto& in_dims = input.dims(); auto height = in_dims[0]; auto size = in_dims[1]; - PADDLE_ENFORCE_EQ(out->numel(), size); + PADDLE_ENFORCE_EQ( + out->numel(), size, + platform::errors::InvalidArgument( + "The size of output tensor " + "should be equal to the size of input tensor column" + " dimension. Expected output size=%d, but received %d", + size, out->numel())); T* out_buf = out->mutable_data(out->place()); const T* in_buf = input.data(); @@ -100,8 +111,16 @@ void RowwiseMean::operator()(const DeviceContext& context, const framework::Tensor& input, framework::Tensor* out) { auto in_dims = input.dims(); - PADDLE_ENFORCE_EQ(in_dims.size(), 2U); - PADDLE_ENFORCE_EQ(out->numel(), in_dims[0]); + PADDLE_ENFORCE_EQ(in_dims.size(), 2U, platform::errors::InvalidArgument( + "The rank of input tensor " + "should be 2, but received %d", + in_dims.size())); + PADDLE_ENFORCE_EQ(out->numel(), in_dims[0], + platform::errors::InvalidArgument( + "The size of output tensor " + "should be equal to the size of input tensor row" + " dimension. Expected output size=%d, but received %d", + in_dims[0], out->numel())); auto in = framework::EigenMatrix::From(input); auto vec = framework::EigenVector::Flatten(*out); @@ -118,10 +137,19 @@ class RowwiseMean { void operator()(const platform::CPUDeviceContext& context, const framework::Tensor& input, framework::Tensor* out) { auto& in_dims = input.dims(); - PADDLE_ENFORCE_EQ(in_dims.size(), 2U); + PADDLE_ENFORCE_EQ(in_dims.size(), 2U, platform::errors::InvalidArgument( + "The rank of input tensor " + "should be 2, but received %d", + in_dims.size())); auto height = in_dims[0]; auto size = in_dims[1]; - PADDLE_ENFORCE_EQ(out->numel(), height); + PADDLE_ENFORCE_EQ( + out->numel(), height, + platform::errors::InvalidArgument( + "The size of output tensor " + "should be equal to the size of input tensor row" + " dimension. Expected output size=%d, but received %d", + height, out->numel())); auto inv_size = 1.0 / size; T* out_buf = out->mutable_data(out->place()); const T* in_buf = input.data(); @@ -141,8 +169,16 @@ void RowwiseSum::operator()(const DeviceContext& context, const framework::Tensor& input, framework::Tensor* out) { auto in_dims = input.dims(); - PADDLE_ENFORCE_EQ(in_dims.size(), 2U); - PADDLE_ENFORCE_EQ(out->numel(), in_dims[0]); + PADDLE_ENFORCE_EQ(in_dims.size(), 2U, platform::errors::InvalidArgument( + "The rank of input tensor " + "should be 2, but received %d", + in_dims.size())); + PADDLE_ENFORCE_EQ(out->numel(), in_dims[0], + platform::errors::InvalidArgument( + "The size of output tensor " + "should be equal to the size of input tensor row" + " dimension. Expected output size=%d, but received %d", + in_dims[0], out->numel())); auto in = framework::EigenMatrix::From(input); auto vec = framework::EigenVector::Flatten(*out); @@ -159,10 +195,19 @@ class RowwiseSum { void operator()(const platform::CPUDeviceContext& context, const framework::Tensor& input, framework::Tensor* out) { auto& in_dims = input.dims(); - PADDLE_ENFORCE_EQ(in_dims.size(), 2U); + PADDLE_ENFORCE_EQ(in_dims.size(), 2U, platform::errors::InvalidArgument( + "The rank of input tensor " + "should be 2, but received %d", + in_dims.size())); auto height = in_dims[0]; auto size = in_dims[1]; - PADDLE_ENFORCE_EQ(out->numel(), height); + PADDLE_ENFORCE_EQ( + out->numel(), height, + platform::errors::InvalidArgument( + "The size of output tensor " + "should be equal to the size of input tensor row" + " dimension. Expected output size=%d, but received %d", + height, out->numel())); T* out_buf = out->mutable_data(out->place()); const T* in_buf = input.data(); diff --git a/paddle/fluid/operators/math/math_function_test.cc b/paddle/fluid/operators/math/math_function_test.cc index 2343e0ee965303c9fdb2ad3faf9ddf6e5bb7782f..587823e535ac67f926fd469d2f43df536c8c88b6 100644 --- a/paddle/fluid/operators/math/math_function_test.cc +++ b/paddle/fluid/operators/math/math_function_test.cc @@ -224,7 +224,11 @@ TEST(math_funciton, set_constant) { auto* ctx = new paddle::platform::CPUDeviceContext(); paddle::operators::math::set_constant(*ctx, &t, 10); for (int64_t i = 0; i < t.numel(); ++i) { - PADDLE_ENFORCE_EQ(10, t.data()[i]); + PADDLE_ENFORCE_EQ(10, t.data()[i], + paddle::platform::errors::InvalidArgument( + "Each value of input" + "tensor should be 10, but received %d.", + t.data()[i])); } delete ctx; } diff --git a/paddle/fluid/operators/math/math_function_test.cu b/paddle/fluid/operators/math/math_function_test.cu index bcbb4a8274f149240b9f0990f38d9f38bdd0e5b1..44b1ee45a4fe9b6f2ea7ba5e09c7cbc60c1aff28 100644 --- a/paddle/fluid/operators/math/math_function_test.cu +++ b/paddle/fluid/operators/math/math_function_test.cu @@ -18,7 +18,12 @@ void fill_fp16_data(paddle::platform::float16* in_ptr, size_t size, const std::vector& data) { - PADDLE_ENFORCE_EQ(size, data.size()); + PADDLE_ENFORCE_EQ( + size, data.size(), + paddle::platform::errors::InvalidArgument( + "The size of argument data should" + " be equal to the argument size. Expected %d, but received %d.", + size, data.size())); for (size_t i = 0; i < data.size(); ++i) { in_ptr[i] = paddle::platform::float16(data[i]); } diff --git a/paddle/fluid/operators/math/padding.h b/paddle/fluid/operators/math/padding.h index 63f793433de07ea2e43ad03ea3ccae1a259f7ae2..379b21c3c18888989663221052e6e99df80e7e9d 100644 --- a/paddle/fluid/operators/math/padding.h +++ b/paddle/fluid/operators/math/padding.h @@ -85,8 +85,9 @@ void PaddingFunctor(int rank, const framework::ExecutionContext& context, PadFunction(context, pads, src, pad_value, out); break; default: - PADDLE_THROW( - "PadOp only support tensors with no more than 6 dimensions."); + PADDLE_THROW(platform::errors::Unimplemented( + "PadOp only support tensors with no more" + " than 6 dimensions currently.")); } } @@ -114,8 +115,9 @@ void PaddingGradFunctor(int rank, const framework::ExecutionContext& context, PadGradFunction(context, pads, src, out); break; default: - PADDLE_THROW( - "PadOp only support tensors with no more than 6 dimensions."); + PADDLE_THROW(platform::errors::Unimplemented( + "PadOp only support tensors with no more" + " than 6 dimensions currently.")); } } diff --git a/paddle/fluid/operators/math/sampler.h b/paddle/fluid/operators/math/sampler.h index 480576ef9dc8c21811a1a867d553ccc6d97fa22a..de9113f2bb616b489747d8d960154f55bb988847 100644 --- a/paddle/fluid/operators/math/sampler.h +++ b/paddle/fluid/operators/math/sampler.h @@ -19,6 +19,8 @@ limitations under the License. */ #include #include +#include "paddle/fluid/platform/enforce.h" + namespace paddle { namespace operators { namespace math { @@ -31,7 +33,10 @@ namespace math { class Sampler { public: explicit Sampler(int64_t range, unsigned int seed = 0UL) : range_(range) { - // PADDLE_ENFORCE_GT(range, 0, "Range should be greater than 0."); + PADDLE_ENFORCE_GT(range, 0, platform::errors::InvalidArgument( + "Range should be" + " greater than 0, but recevied %d.", + range)); if (seed == 0) { std::random_device r; seed_ = r(); diff --git a/paddle/fluid/operators/math/selected_rows_functor.cc b/paddle/fluid/operators/math/selected_rows_functor.cc index 3bb9efc5315fcacf0b50682b65c89ac3ad0d2d4e..c2595beb0cb4dc37104a91ac8a2647c7d787c5c5 100644 --- a/paddle/fluid/operators/math/selected_rows_functor.cc +++ b/paddle/fluid/operators/math/selected_rows_functor.cc @@ -29,7 +29,12 @@ struct SelectedRowsAdd { const framework::SelectedRows& input2, framework::SelectedRows* output) { auto in1_height = input1.height(); - PADDLE_ENFORCE_EQ(in1_height, input2.height()); + PADDLE_ENFORCE_EQ( + in1_height, input2.height(), + platform::errors::InvalidArgument("The two inputs height must be equal." + "But recieved first input height = " + "[%d], second input height = [%d]", + in1_height, input2.height())); output->set_height(in1_height); auto& in1_rows = input1.rows(); @@ -47,15 +52,31 @@ struct SelectedRowsAdd { auto& in2_value = input2.value(); auto in1_row_numel = in1_value.numel() / in1_rows.size(); - PADDLE_ENFORCE_EQ(in1_row_numel, in2_value.numel() / in2_rows.size()); - PADDLE_ENFORCE_EQ(in1_row_numel, out_value->numel() / out_rows.size()); + PADDLE_ENFORCE_EQ( + in1_row_numel, in2_value.numel() / in2_rows.size(), + platform::errors::InvalidArgument( + "The two inputs width must be equal." + "But recieved first input width = [%d], second input width = [%d]", + in1_row_numel, in2_value.numel() / in2_rows.size())); + PADDLE_ENFORCE_EQ( + in1_row_numel, out_value->numel() / out_rows.size(), + platform::errors::InvalidArgument( + "The input and oupput width must be equal." + "But recieved input width = [%d], output width = [%d]", + in1_row_numel, out_value->numel() / out_rows.size())); auto in1_place = input1.place(); - PADDLE_ENFORCE(platform::is_cpu_place(in1_place)); + PADDLE_ENFORCE_EQ(platform::is_cpu_place(in1_place), true, + platform::errors::InvalidArgument( + "The running enviroment is not on the CPU place.")); auto in2_place = input2.place(); - PADDLE_ENFORCE(platform::is_cpu_place(in2_place)); + PADDLE_ENFORCE_EQ(platform::is_cpu_place(in2_place), true, + platform::errors::InvalidArgument( + "The running enviroment is not on the CPU place.")); auto out_place = context.GetPlace(); - PADDLE_ENFORCE(platform::is_cpu_place(out_place)); + PADDLE_ENFORCE_EQ(platform::is_cpu_place(out_place), true, + platform::errors::InvalidArgument( + "The running enviroment is not on the CPU place.")); auto* out_data = out_value->data(); auto* in1_data = in1_value.data(); @@ -82,15 +103,35 @@ struct SelectedRowsAddTensor { auto in1_height = input1.height(); auto in2_dims = input2.dims(); auto out_dims = output->dims(); - PADDLE_ENFORCE_EQ(in1_height, in2_dims[0]); - PADDLE_ENFORCE_EQ(in1_height, out_dims[0]); + PADDLE_ENFORCE_EQ( + in1_height, in2_dims[0], + platform::errors::InvalidArgument("The two inputs height must be equal." + "But recieved first input height = " + "[%d], second input height = [%d]", + in1_height, in2_dims[0])); + PADDLE_ENFORCE_EQ( + in1_height, out_dims[0], + platform::errors::InvalidArgument( + "The input and output height must be equal." + "But recieved input height = [%d], output height = [%d]", + in1_height, out_dims[0])); auto& in1_value = input1.value(); auto& in1_rows = input1.rows(); int64_t in1_row_numel = in1_value.numel() / in1_rows.size(); - PADDLE_ENFORCE_EQ(in1_row_numel, input2.numel() / in1_height); - PADDLE_ENFORCE_EQ(in1_row_numel, output->numel() / in1_height); + PADDLE_ENFORCE_EQ( + in1_row_numel, input2.numel() / in1_height, + platform::errors::InvalidArgument( + "The two inputs width must be equal." + "But recieved first input width = [%d], second input width = [%d]", + in1_row_numel, input2.numel() / in1_height)); + PADDLE_ENFORCE_EQ( + in1_row_numel, output->numel() / in1_height, + platform::errors::InvalidArgument( + "The input and output width must be equal." + "But recieved input width = [%d], output width = [%d]", + in1_row_numel, output->numel() / in1_height)); SetConstant functor; functor(context, output, 0.0); @@ -121,7 +162,12 @@ struct SelectedRowsAddTo { const int64_t input2_offset, framework::SelectedRows* input2) { auto in1_height = input1.height(); - PADDLE_ENFORCE_EQ(in1_height, input2->height()); + PADDLE_ENFORCE_EQ( + in1_height, input2->height(), + platform::errors::InvalidArgument("The two inputs height must be equal." + "But recieved first input height = " + "[%d], second input height = [%d]", + in1_height, input2->height())); auto& in1_rows = input1.rows(); auto& in2_rows = *(input2->mutable_rows()); @@ -133,9 +179,13 @@ struct SelectedRowsAddTo { in2_rows.Extend(in1_rows.begin(), in1_rows.end()); auto in1_place = input1.place(); - PADDLE_ENFORCE(platform::is_cpu_place(in1_place)); + PADDLE_ENFORCE_EQ(platform::is_cpu_place(in1_place), true, + platform::errors::InvalidArgument( + "The running enviroment is not on the CPU place.")); auto in2_place = input2->place(); - PADDLE_ENFORCE(platform::is_cpu_place(in2_place)); + PADDLE_ENFORCE_EQ(platform::is_cpu_place(in2_place), true, + platform::errors::InvalidArgument( + "The running enviroment is not on the CPU place.")); auto* in1_data = in1_value.data(); auto* in2_data = in2_value->data(); @@ -163,7 +213,12 @@ struct SelectedRowsSumTo { auto& in_rows = (*iter)->rows(); size += in_rows.end() - in_rows.begin(); auto in1_height = (*iter)->height(); - PADDLE_ENFORCE_EQ(in1_height, input2->height()); + PADDLE_ENFORCE_EQ(in1_height, input2->height(), + platform::errors::InvalidArgument( + "The two inputs height must be equal." + "But recieved first input height = [%d], second " + "input height = [%d]", + in1_height, input2->height())); } // concat rows std::vector in2_rows; @@ -201,13 +256,23 @@ struct SelectedRowsAddToTensor { } auto in1_height = input1.height(); auto in2_dims = input2->dims(); - PADDLE_ENFORCE_EQ(in1_height, in2_dims[0]); + PADDLE_ENFORCE_EQ( + in1_height, in2_dims[0], + platform::errors::InvalidArgument("The two inputs height must be equal." + "But recieved first input height = " + "[%d], second input height = [%d]", + in1_height, in2_dims[0])); auto& in1_value = input1.value(); auto& in1_rows = input1.rows(); int64_t in1_row_numel = in1_value.numel() / in1_rows.size(); - PADDLE_ENFORCE_EQ(in1_row_numel, input2->numel() / in1_height); + PADDLE_ENFORCE_EQ( + in1_row_numel, input2->numel() / in1_height, + platform::errors::InvalidArgument( + "The two inputs width must be equal." + "But recieved first input width = [%d], second input width = [%d]", + in1_row_numel, input2->numel() / in1_height)); auto* in1_data = in1_value.data(); auto* input2_data = input2->data(); @@ -302,10 +367,12 @@ struct MergeAdd { continue; } PADDLE_ENFORCE_EQ(input_width, input->value().dims()[1], - "all input should have same " - "dimension except for the first one"); + platform::errors::InvalidArgument( + "All inputs should have same " + "dimension except for the first one.")); PADDLE_ENFORCE_EQ(input_height, input->height(), - "all input should have same height"); + platform::errors::InvalidArgument( + "All inputs should have same height.")); row_num += input->rows().size(); merged_row_set.insert(input->rows().begin(), input->rows().end()); } @@ -421,10 +488,12 @@ struct MergeAverage { continue; } PADDLE_ENFORCE_EQ(input_width, input->value().dims()[1], - "all input should have same " - "dimension except for the first one"); + platform::errors::InvalidArgument( + "All inputs should have same " + "dimension except for the first one.")); PADDLE_ENFORCE_EQ(input_height, input->height(), - "all input should have same height"); + platform::errors::InvalidArgument( + "All input should have same height.")); row_num += input->rows().size(); merged_row_set.insert(input->rows().begin(), input->rows().end()); } @@ -492,13 +561,23 @@ struct UpdateToTensor { framework::Tensor* input2) { auto in1_height = input1.height(); auto in2_dims = input2->dims(); - PADDLE_ENFORCE_EQ(in1_height, in2_dims[0]); + PADDLE_ENFORCE_EQ( + in1_height, in2_dims[0], + platform::errors::InvalidArgument("The two inputs height must be equal." + "But recieved first input height = " + "[%d], second input height = [%d]", + in1_height, in2_dims[0])); auto& in1_value = input1.value(); auto& in1_rows = input1.rows(); int64_t in1_row_numel = in1_value.numel() / in1_rows.size(); - PADDLE_ENFORCE_EQ(in1_row_numel, input2->numel() / in1_height); + PADDLE_ENFORCE_EQ( + in1_row_numel, input2->numel() / in1_height, + platform::errors::InvalidArgument( + "The two inputs width must be equal." + "But recieved first input width = [%d], second input width = [%d]", + in1_row_numel, input2->numel() / in1_height)); auto* in1_data = in1_value.data(); auto* input2_data = input2->data(); diff --git a/paddle/fluid/operators/math/selected_rows_functor.cu b/paddle/fluid/operators/math/selected_rows_functor.cu index 9cce52c6d4587baf01ba22eebc9c57da04c26590..35bd02ad35b71eb7deb3299490fa545ef8b23dc6 100644 --- a/paddle/fluid/operators/math/selected_rows_functor.cu +++ b/paddle/fluid/operators/math/selected_rows_functor.cu @@ -30,7 +30,12 @@ struct SelectedRowsAdd { const framework::SelectedRows& input2, framework::SelectedRows* output) { auto in1_height = input1.height(); - PADDLE_ENFORCE_EQ(in1_height, input2.height()); + PADDLE_ENFORCE_EQ( + in1_height, input2.height(), + platform::errors::InvalidArgument("The two inputs height must be equal." + "But recieved first input height = " + "[%d], second input height = [%d]", + in1_height, input2.height())); output->set_height(in1_height); framework::Vector in1_rows(input1.rows()); @@ -48,18 +53,34 @@ struct SelectedRowsAdd { auto& in2_value = input2.value(); auto in1_row_numel = in1_value.numel() / in1_rows.size(); - PADDLE_ENFORCE_EQ(in1_row_numel, in2_value.numel() / in2_rows.size()); - PADDLE_ENFORCE_EQ(in1_row_numel, out_value->numel() / out_rows.size()); + PADDLE_ENFORCE_EQ( + in1_row_numel, in2_value.numel() / in2_rows.size(), + platform::errors::InvalidArgument( + "The two inputs width must be equal." + "But recieved first input width = [%d], second input width = [%d]", + in1_row_numel, in2_value.numel() / in2_rows.size())); + PADDLE_ENFORCE_EQ( + in1_row_numel, out_value->numel() / out_rows.size(), + platform::errors::InvalidArgument( + "The input and oupput width must be equal." + "But recieved input width = [%d], output width = [%d]", + in1_row_numel, out_value->numel() / out_rows.size())); auto* out_data = out_value->data(); auto* in1_data = in1_value.data(); auto in1_place = input1.place(); - PADDLE_ENFORCE_EQ(platform::is_gpu_place(in1_place), true); + PADDLE_ENFORCE_EQ(platform::is_gpu_place(in1_place), true, + platform::errors::InvalidArgument( + "The running enviroment is not on the GPU place.")); auto in2_place = input2.place(); - PADDLE_ENFORCE_EQ(platform::is_gpu_place(in2_place), true); + PADDLE_ENFORCE_EQ(platform::is_gpu_place(in2_place), true, + platform::errors::InvalidArgument( + "The running enviroment is not on the GPU place.")); auto out_place = context.GetPlace(); - PADDLE_ENFORCE_EQ(platform::is_gpu_place(out_place), true); + PADDLE_ENFORCE_EQ(platform::is_gpu_place(out_place), true, + platform::errors::InvalidArgument( + "The running enviroment is not on the GPU place.")); memory::Copy(BOOST_GET_CONST(platform::CUDAPlace, out_place), out_data, BOOST_GET_CONST(platform::CUDAPlace, in1_place), in1_data, @@ -104,15 +125,35 @@ struct SelectedRowsAddTensor { auto in1_height = input1.height(); auto in2_dims = input2.dims(); auto out_dims = output->dims(); - PADDLE_ENFORCE_EQ(in1_height, in2_dims[0]); - PADDLE_ENFORCE_EQ(in1_height, out_dims[0]); + PADDLE_ENFORCE_EQ( + in1_height, in2_dims[0], + platform::errors::InvalidArgument( + "The two inputs height must be equal." + "But recieved first input height = [%d], first input height = [%d]", + in1_height, in2_dims[0])); + PADDLE_ENFORCE_EQ( + in1_height, out_dims[0], + platform::errors::InvalidArgument( + "The input and output height must be equal." + "But recieved input height = [%d], output height = [%d]", + in1_height, out_dims[0])); auto& in1_value = input1.value(); auto& in1_rows = input1.rows(); int64_t in1_row_numel = in1_value.numel() / in1_rows.size(); - PADDLE_ENFORCE_EQ(in1_row_numel, input2.numel() / in1_height); - PADDLE_ENFORCE_EQ(in1_row_numel, output->numel() / in1_height); + PADDLE_ENFORCE_EQ( + in1_row_numel, input2.numel() / in1_height, + platform::errors::InvalidArgument( + "The two inputs width must be equal." + "But recieved first input width = [%d], second input width = [%d]", + in1_row_numel, input2.numel() / in1_height)); + PADDLE_ENFORCE_EQ( + in1_row_numel, output->numel() / in1_height, + platform::errors::InvalidArgument( + "The input and output width must be equal." + "But recieved input width = [%d], output width = [%d]", + in1_row_numel, output->numel() / in1_height)); auto* in1_data = in1_value.data(); auto* in2_data = input2.data(); @@ -148,7 +189,12 @@ struct SelectedRowsAddTo { const int64_t input2_offset, framework::SelectedRows* input2) { auto in1_height = input1.height(); - PADDLE_ENFORCE_EQ(in1_height, input2->height()); + PADDLE_ENFORCE_EQ( + in1_height, input2->height(), + platform::errors::InvalidArgument("The two inputs height must be equal." + "But recieved first input height = " + "[%d], second input height = [%d]", + in1_height, input2->height())); auto& in1_rows = input1.rows(); auto& in2_rows = *(input2->mutable_rows()); @@ -162,9 +208,13 @@ struct SelectedRowsAddTo { } auto in1_place = input1.place(); - PADDLE_ENFORCE_EQ(platform::is_gpu_place(in1_place), true); + PADDLE_ENFORCE_EQ(platform::is_gpu_place(in1_place), true, + platform::errors::InvalidArgument( + "The running enviroment is not on the GPU place.")); auto in2_place = input2->place(); - PADDLE_ENFORCE_EQ(platform::is_gpu_place(in2_place), true); + PADDLE_ENFORCE_EQ(platform::is_gpu_place(in1_place), true, + platform::errors::InvalidArgument( + "The running enviroment is not on the GPU place.")); auto* in1_data = in1_value.data(); auto* in2_data = in2_value->data(); @@ -209,13 +259,23 @@ struct SelectedRowsAddToTensor { framework::Tensor* input2) { auto in1_height = input1.height(); auto in2_dims = input2->dims(); - PADDLE_ENFORCE_EQ(in1_height, in2_dims[0]); + PADDLE_ENFORCE_EQ( + in1_height, in2_dims[0], + platform::errors::InvalidArgument("The two inputs height must be equal." + "But recieved first input height = " + "[%d], second input height = [%d]", + in1_height, in2_dims[0])); auto& in1_value = input1.value(); auto& in1_rows = input1.rows(); int64_t in1_row_numel = in1_value.numel() / in1_rows.size(); - PADDLE_ENFORCE_EQ(in1_row_numel, input2->numel() / in1_height); + PADDLE_ENFORCE_EQ( + in1_row_numel, input2->numel() / in1_height, + platform::errors::InvalidArgument( + "The two inputs width must be equal." + "But recieved first input width = [%d], second input width = [%d]", + in1_row_numel, input2->numel() / in1_height)); auto* in1_data = in1_value.data(); auto* in2_data = input2->data(); @@ -340,10 +400,12 @@ struct MergeAdd { continue; } PADDLE_ENFORCE_EQ(input_width, input->value().dims()[1], - "all input should have same " - "dimension except for the first one"); + platform::errors::InvalidArgument( + "All input should have same " + "dimension except for the first one.")); PADDLE_ENFORCE_EQ(input_height, input->height(), - "all input should have same height"); + platform::errors::InvalidArgument( + "All input should have same height.")); merged_row_set.insert(input->rows().begin(), input->rows().end()); } std::vector merge_rows_cpu(merged_row_set.begin(), @@ -448,13 +510,23 @@ struct UpdateToTensor { auto in1_height = merged_in1.height(); auto in2_dims = input2->dims(); - PADDLE_ENFORCE_EQ(in1_height, in2_dims[0]); + PADDLE_ENFORCE_EQ( + in1_height, in2_dims[0], + platform::errors::InvalidArgument("The two inputs height must be equal." + "But recieved first input height = " + "[%d], second input height = [%d]", + in1_height, in2_dims[0])); auto& in1_value = merged_in1.value(); auto& in1_rows = merged_in1.rows(); int64_t in1_row_numel = in1_value.numel() / in1_rows.size(); - PADDLE_ENFORCE_EQ(in1_row_numel, input2->numel() / in1_height); + PADDLE_ENFORCE_EQ( + in1_row_numel, input2->numel() / in1_height, + platform::errors::InvalidArgument( + "The two inputs width must be equal." + "But recieved first input width = [%d], second input width = [%d]", + in1_row_numel, input2->numel() / in1_height)); auto* in1_data = in1_value.template data(); auto* in2_data = input2->data(); diff --git a/paddle/fluid/operators/math/selected_rows_functor_test.cu.cc b/paddle/fluid/operators/math/selected_rows_functor_test.cu.cc index 74892316e6decdeab3a08396fa2f4bdeb8eb7b73..81ad620466ee3d9fcd9d3e057cfd0dd9053089f0 100644 --- a/paddle/fluid/operators/math/selected_rows_functor_test.cu.cc +++ b/paddle/fluid/operators/math/selected_rows_functor_test.cu.cc @@ -38,7 +38,9 @@ TEST(selected_rows_functor, gpu_add) { {static_cast(rows1.size()), row_numel}), gpu_place); functor(ctx, in1_value, 1.0); - PADDLE_ENFORCE(cudaDeviceSynchronize()); + PADDLE_ENFORCE_EQ(cudaDeviceSynchronize(), 0, + paddle::platform::errors::PreconditionNotMet( + "The all synchronization on the cuda is error!")); std::vector rows2{0, 5, 7, 9}; std::unique_ptr selected_rows2{ diff --git a/paddle/fluid/operators/math/sequence2batch.cc b/paddle/fluid/operators/math/sequence2batch.cc index e4ffeedb5a0061dd60ca3a30aa9928ef8b05887c..300a3692012ab9631d7049d2042e91fb99ad3c21 100644 --- a/paddle/fluid/operators/math/sequence2batch.cc +++ b/paddle/fluid/operators/math/sequence2batch.cc @@ -29,11 +29,24 @@ class CopyMatrixRowsFunctor { auto src_dims = src.dims(); auto dst_dims = dst->dims(); PADDLE_ENFORCE_EQ(src_dims.size(), 2UL, - "The src must be matrix with rank 2."); + platform::errors::InvalidArgument( + "The source tensor must be a matrix with rank 2, but " + "got the source tensor rank is %lu. " + "Please check the rank of the source tensor", + src_dims.size())); PADDLE_ENFORCE_EQ(dst_dims.size(), 2UL, - "The dst must be matrix with rank 2."); - PADDLE_ENFORCE_EQ(src_dims[1], dst_dims[1], - "The width of src and dst must be same."); + platform::errors::InvalidArgument( + "The destination tensor must be a matrix with rank, " + "but got the destination tensor rank is %lu. " + "Please check the rank of the destination tensor", + dst_dims.size())); + PADDLE_ENFORCE_EQ( + src_dims[1], dst_dims[1], + platform::errors::InvalidArgument( + "The width of the source tensor and the destination tensor must be " + "same. But got %lu != %lu.Please check the rank of the source " + "tensor", + src_dims.size(), dst_dims.size())); auto height = dst_dims[0]; auto width = dst_dims[1]; auto* src_data = src.data(); diff --git a/paddle/fluid/operators/math/sequence2batch.cu b/paddle/fluid/operators/math/sequence2batch.cu index 9ab13659c1cc5b59d28395bcebcfb43fac5b4544..cd1ca572689bc701da801384e5ed08fe6dc10749 100644 --- a/paddle/fluid/operators/math/sequence2batch.cu +++ b/paddle/fluid/operators/math/sequence2batch.cu @@ -46,11 +46,24 @@ class CopyMatrixRowsFunctor { auto src_dims = src.dims(); auto dst_dims = dst->dims(); PADDLE_ENFORCE_EQ(src_dims.size(), 2, - "The src must be matrix with rank 2."); + platform::errors::InvalidArgument( + "The source tensor must be a matrix with rank 2, but " + "got the source tensor rank is %lu. " + "Please check the rank of the source tensor", + src_dims.size())); PADDLE_ENFORCE_EQ(dst_dims.size(), 2, - "The dst must be matrix with rank 2."); - PADDLE_ENFORCE_EQ(src_dims[1], dst_dims[1], - "The width of src and dst must be same."); + platform::errors::InvalidArgument( + "The destination tensor must be a matrix with rank, " + "but got the destination tensor rank is %lu. " + "Please check the rank of the destination tensor", + dst_dims.size())); + PADDLE_ENFORCE_EQ( + src_dims[1], dst_dims[1], + platform::errors::InvalidArgument( + "The width of the source tensor and the destination tensor must be " + "same. But got %lu != %lu.Please check the rank of the source " + "tensor", + src_dims.size(), dst_dims.size())); auto height = dst_dims[0]; auto width = dst_dims[1]; auto* src_data = src.data(); diff --git a/paddle/fluid/operators/math/sequence2batch.h b/paddle/fluid/operators/math/sequence2batch.h index 9d9f7ef00b8a12088225fd3620cb30b43ef9dce9..6aa513e4d10eef49c02417e98b31cddd57088d7c 100644 --- a/paddle/fluid/operators/math/sequence2batch.h +++ b/paddle/fluid/operators/math/sequence2batch.h @@ -64,19 +64,30 @@ class LoDTensor2BatchFunctor { bool is_reverse = false) const { if (!is_cal_batch_lod) { auto lods = batch->lod(); - PADDLE_ENFORCE_GT(lods.size(), 2UL, - "The LoD of LoDTensor should inlcude at least 2-level " - "sequence information."); + PADDLE_ENFORCE_GT( + lods.size(), 2UL, + platform::errors::InvalidArgument( + "The LoD of LoDTensor should inlcude at least 2-level " + "sequence information, but got the LoD level is %lu. Please " + "check the input value.", + lods.size())); PADDLE_ENFORCE_EQ( lods[1].size(), static_cast(lod_tensor.dims()[0]), - "The LoD information should be consistent with the dims."); + platform::errors::InvalidArgument( + "The LoD information should be consistent with the dims, but got " + "%lu != %lu. Please check the input value.", + lods[1].size(), static_cast(lod_tensor.dims()[0]))); CopyMatrixRowsFunctor to_batch; to_batch(context, lod_tensor, lods[1], batch, true); return; } auto lods = lod_tensor.lod(); - PADDLE_ENFORCE_EQ(lods.size(), 1UL, "Only support one level sequence now."); + PADDLE_ENFORCE_EQ(lods.size(), 1UL, + platform::errors::InvalidArgument( + "Only support one level sequence now, but got the " + "LoD level is %lu. Please check the input value.", + lods.size())); const auto& lod = lods[0]; @@ -161,12 +172,19 @@ class Batch2LoDTensorFunctor { const framework::LoDTensor& batch, framework::LoDTensor* lod_tensor) const { auto in_lod = batch.lod(); - PADDLE_ENFORCE_GT(in_lod.size(), 2UL, - "The LoD of LoDTensor should inlcude at least 2-level " - "sequence information."); + PADDLE_ENFORCE_GT( + in_lod.size(), 2UL, + platform::errors::InvalidArgument( + "The LoD of LoDTensor should inlcude at least 2-level " + "sequence information, but got the LoD level is %lu. Please check " + "the input value.", + in_lod.size())); PADDLE_ENFORCE_EQ( in_lod[1].size(), static_cast(lod_tensor->dims()[0]), - "The LoD information should be consistent with the dims."); + platform::errors::InvalidArgument( + "The LoD information should be consistent with the dims, but got " + "%lu != %lu. Please check the input value.", + in_lod[1].size(), static_cast(lod_tensor->dims()[0]))); CopyMatrixRowsFunctor to_seq; to_seq(context, batch, in_lod[1], lod_tensor, false); } diff --git a/paddle/fluid/operators/math/sequence_padding.cc b/paddle/fluid/operators/math/sequence_padding.cc index 4630689dec160da145e607f662a802444ac98b55..076df0176429c7bbd350698af0137fbcca18f806 100644 --- a/paddle/fluid/operators/math/sequence_padding.cc +++ b/paddle/fluid/operators/math/sequence_padding.cc @@ -35,7 +35,11 @@ void CopyValidData(framework::Tensor* dst_tensor, int valid_seq_len = seq_offsets[seq_idx + 1] - seq_offsets[seq_idx]; PADDLE_ENFORCE_GE( pad_seq_len, valid_seq_len, - "The padded sequence length can not be less than its original length."); + platform::errors::InvalidArgument( + "The padded sequence length can not " + "be less than its original length. Expected %ld >= %ld, but got " + "%ld < %ld. Please check input value.", + pad_seq_len, valid_seq_len, pad_seq_len, valid_seq_len)); int seq_data_offset = seq_offsets[seq_idx] * step_width; int pad_data_offset = layout == kBatchLengthWidth ? seq_idx * pad_seq_len * step_width @@ -95,9 +99,14 @@ class PaddingLoDTensorFunctor { CheckDims(seq_tensor_dims, pad_tensor_dims, seq_offsets, pad_seq_len, step_width, layout); - PADDLE_ENFORCE(pad_value.numel() == 1 || pad_value.numel() == step_width, - "The numel of 'pad_value' can only be 1 or be equal to the " - "'step_width'."); + + PADDLE_ENFORCE_EQ( + pad_value.numel() == 1 || pad_value.numel() == step_width, true, + platform::errors::InvalidArgument( + "The numel of 'pad_value' can only be 1 or be equal to the " + "'step_width', but got %ld != 1 and %ld. Please check the input " + "value.", + pad_value.numel(), step_width)); // fill padding value T* pad_data = pad_tensor->data(); diff --git a/paddle/fluid/operators/math/sequence_padding.cu b/paddle/fluid/operators/math/sequence_padding.cu index 1b433067900af71bb8a6833cef019d41f9c76858..19c3af03411b8ce95d274532707c7ee3e93f1d55 100644 --- a/paddle/fluid/operators/math/sequence_padding.cu +++ b/paddle/fluid/operators/math/sequence_padding.cu @@ -66,17 +66,25 @@ class PaddingLoDTensorFunctor { if (pad_seq_len == -1) { pad_seq_len = max_seq_len; } - PADDLE_ENFORCE_GE(pad_seq_len, max_seq_len, - "The pad_seq_len must be equal to or greater than the " - "original max sequence length."); + PADDLE_ENFORCE_GE( + pad_seq_len, max_seq_len, + platform::errors::InvalidArgument( + "The pad_seq_len must be equal to or greater than the " + "original max sequence length. Expected %ld >= %ld, but got %ld < " + "%ld. Please check the input value.", + pad_seq_len, max_seq_len, pad_seq_len, max_seq_len)); int step_width = seq_tensor.numel() / seq_tensor_dims[0]; int seq_num = seq_offsets.size() - 1; CheckDims(seq_tensor_dims, pad_tensor_dims, seq_offsets, pad_seq_len, step_width, layout); - PADDLE_ENFORCE(pad_value.numel() == 1 || pad_value.numel() == step_width, - "The numel of 'pad_value' can only be 1 or be equal to the " - "'step_width'."); + PADDLE_ENFORCE_EQ( + pad_value.numel() == 1 || pad_value.numel() == step_width, true, + platform::errors::InvalidArgument( + "The numel of 'pad_value' can only be 1 or be equal to " + "the 'step_width', but got %ld != 1 and %ld. Please check the " + "input value.", + pad_value.numel(), step_width)); const int kBlockSize = 512; diff --git a/paddle/fluid/operators/math/sequence_padding.h b/paddle/fluid/operators/math/sequence_padding.h index 5580ee5374658c3b7b8e31962cd50f1d72113ba0..956a4ff6a2d45cb619183f9beba1b7e35b7f229c 100644 --- a/paddle/fluid/operators/math/sequence_padding.h +++ b/paddle/fluid/operators/math/sequence_padding.h @@ -52,14 +52,25 @@ inline static void CheckDims(const framework::DDim& seq_tensor_dims, const framework::Vector& seq_offset, int64_t padded_seq_len, int64_t step_width, const PadLayout& layout) { - PADDLE_ENFORCE_EQ(static_cast(seq_tensor_dims[0]), seq_offset.back(), - "Value of 1st dimension of the sequence tensor should be " - "equal to sum of lengths of all sequences."); + PADDLE_ENFORCE_EQ( + static_cast(seq_tensor_dims[0]), seq_offset.back(), + platform::errors::InvalidArgument( + "Value of 1st dimension of the sequence tensor should be " + "equal to sum of lengths of all sequences. Expected %ld == %ld, but " + "got %ld != %ld. Please check the input value.", + static_cast(seq_tensor_dims[0]), seq_offset.back(), + static_cast(seq_tensor_dims[0]), seq_offset.back())); - PADDLE_ENFORCE(seq_tensor_dims.size() + 1 == pad_tensor_dims.size() || - seq_tensor_dims.size() == pad_tensor_dims.size(), - "pad_tensor's rank should be 1 greater than seq_tensor's " - "rank, or be equal with it."); + PADDLE_ENFORCE_EQ( + seq_tensor_dims.size() + 1 == pad_tensor_dims.size() || + seq_tensor_dims.size() == pad_tensor_dims.size(), + true, platform::errors::InvalidArgument( + "pad_tensor's rank should be 1 greater than seq_tensor's " + "rank, or be equal with it. The pad_tensor's rank is %ld, " + "expected the seq_tensor's rank is %ld or %ld, but got %ld. " + "Please check the input value.", + pad_tensor_dims.size(), pad_tensor_dims.size(), + pad_tensor_dims.size() - 1, seq_tensor_dims.size())); } /* diff --git a/paddle/fluid/operators/math/sequence_pooling.cc b/paddle/fluid/operators/math/sequence_pooling.cc index cc3fbd587668b17b7edde50b157adca83e81eddc..2eee4d0a6c14e8b6134b71294745c71302450347 100644 --- a/paddle/fluid/operators/math/sequence_pooling.cc +++ b/paddle/fluid/operators/math/sequence_pooling.cc @@ -42,15 +42,29 @@ class MaxSeqPoolFunctor { auto out_dims = output->dims(); auto idx_dims = index->dims(); PADDLE_ENFORCE_GT(in_dims.size(), 1, - "The rank of input shall be greater than 1."); + platform::errors::InvalidArgument( + "The rank of input shall be greater than 1, but got " + "the rank is %ld. Please check the input value", + in_dims.size())); PADDLE_ENFORCE_GT(out_dims.size(), 1, - "The rank of output shall be greater than 1."); + platform::errors::InvalidArgument( + "The rank of output shall be greater than 1, but got " + "the rank is %ld. Please check the input value", + out_dims.size())); for (int64_t i = 1; i < in_dims.size(); ++i) { - PADDLE_ENFORCE_EQ(in_dims[i], out_dims[i], - "The dimension of input and output shall be same."); + PADDLE_ENFORCE_EQ( + in_dims[i], out_dims[i], + platform::errors::InvalidArgument( + "The dimension of input and output shall be same. Expected %ld " + "== %ld, but got %ld != %ld. Please check the input value.", + in_dims[i], out_dims[i], in_dims[i], out_dims[i])); } - PADDLE_ENFORCE_EQ(idx_dims, out_dims, - "The dimension of index and output shall be same."); + PADDLE_ENFORCE_EQ( + idx_dims, out_dims, + platform::errors::InvalidArgument( + "The dimension of index and output shall be same. Expected %ld == " + "%ld, but got %ld != %ld. Please check the input value.", + idx_dims, out_dims, idx_dims, out_dims)); auto lod_level = input.lod().size(); auto starts = input.lod()[lod_level - 1]; @@ -94,12 +108,22 @@ class MaxSeqPoolFunctor { auto in_dims = input.dims(); auto out_dims = output->dims(); PADDLE_ENFORCE_GT(in_dims.size(), 1, - "The rank of input shall be greater than 1."); + platform::errors::InvalidArgument( + "The rank of input shall be greater than 1, but got " + "%ld <= 1. Please check the input value.", + in_dims.size())); PADDLE_ENFORCE_GT(out_dims.size(), 1, - "The rank of output shall be greater than 1."); + platform::errors::InvalidArgument( + "The rank of output shall be greater than 1, but got " + "%ld <= 1. Please check the input value.", + out_dims.size())); for (int64_t i = 1; i < in_dims.size(); ++i) { - PADDLE_ENFORCE_EQ(in_dims[i], out_dims[i], - "The dimension of input and output shall be same."); + PADDLE_ENFORCE_EQ( + in_dims[i], out_dims[i], + platform::errors::InvalidArgument( + "The dimension of input and output shall be same. Expected %ld " + "== %ld, but got %ld != %ld. Please check the input value.", + in_dims[i], out_dims[i], in_dims[i], out_dims[i])); } auto lod_level = input.lod().size(); @@ -139,16 +163,29 @@ class MaxSeqPoolGradFunctor { auto ig_dims = in_grad->dims(); auto idx_dims = index.dims(); PADDLE_ENFORCE_GT(og_dims.size(), 1, - "The rank of output@Grad shall be greater than 1."); + platform::errors::InvalidArgument( + "The rank of output@Grad shall be greater than 1, " + "but got %ld <= 1. Please check the input value.", + og_dims.size())); PADDLE_ENFORCE_GT(ig_dims.size(), 1, - "The rank of input@Grad shall be greater than 1."); + platform::errors::InvalidArgument( + "The rank of input@Grad shall be greater than 1, but " + "got %ld <= 1. Please check the input value.", + ig_dims.size())); for (int64_t i = 1; i < og_dims.size(); ++i) { - PADDLE_ENFORCE_EQ( - og_dims[i], ig_dims[i], - "The dimension of input@Grad and output@Grad shall be same."); + PADDLE_ENFORCE_EQ(og_dims[i], ig_dims[i], + platform::errors::InvalidArgument( + "The dimension of input@Grad and output@Grad shall " + "be same. Expected %ld == %ld, but got %ld != %ld. " + "Please check the input value.", + og_dims[i], ig_dims[i], og_dims[i], ig_dims[i])); } - PADDLE_ENFORCE_EQ(idx_dims, og_dims, - "The dimension of index and output@Grad shall be same."); + PADDLE_ENFORCE_EQ( + idx_dims, og_dims, + platform::errors::InvalidArgument( + "The dimension of index and output@Grad shall be same. Expected " + "%ld == %ld, but got %ld != %ld. Please check the input value.", + idx_dims, og_dims, idx_dims, og_dims)); const T* og_data = out_grad.data(); const int* max_index = index.data(); @@ -244,9 +281,12 @@ class SumSeqPoolGradFunctor { auto lod = in_grad->lod()[lod_level - 1]; int64_t out_w = out_grad.numel() / out_grad.dims()[0]; int64_t in_w = in_grad->numel() / in_grad->dims()[0]; - PADDLE_ENFORCE_EQ( - in_w, out_w, - "The feature size of input@Grad and output@Grad shall be same."); + PADDLE_ENFORCE_EQ(in_w, out_w, + platform::errors::InvalidArgument( + "The feature size of input@Grad and output@Grad " + "shall be same. Expected %ld == %ld, but got %ld != " + "%ld. Please check the input value.", + in_w, out_w, in_w, out_w)); const T* out_g_data = out_grad.data(); T* in_g_data = in_grad->mutable_data(context.GetPlace()); auto blas = math::GetBlas(context); @@ -298,7 +338,8 @@ class SequencePoolFunctor { auto place = context.GetPlace(); PADDLE_ENFORCE_EQ( platform::is_cpu_place(place), true, - "Sequence_pool should run on CPU Device when pooltype is SUM"); + platform::errors::InvalidArgument( + "Sequence_pool should run on CPU Device when pooltype is SUM")); const T* src = input.data(); T* dst = output->mutable_data(place); jit::seq_pool_attr_t attr( @@ -342,7 +383,10 @@ class SequencePoolFunctor { out_e.device(place) = in_e.sum(Eigen::array({{0}})) / std::sqrt(static_cast(h)); } else { - PADDLE_THROW("unsupported pooling pooltype"); + PADDLE_THROW(platform::errors::InvalidArgument( + "unsupported pooling pooltype: %s. Only support \"AVERAGE\" and " + "\"SQRT\"", + pooltype)); } } } @@ -400,7 +444,10 @@ class SequencePoolGradFunctor { } else if (pooltype == "FIRST") { in_g_e.chip(0, 0).device(place) = out_g_e_v; } else { - PADDLE_THROW("unsupported pooling pooltype"); + PADDLE_THROW(platform::errors::InvalidArgument( + "unsupported pooling pooltype: %s. Only support \"AVERAGE\", " + "\"SQRT\", \"LAST\" and \"FIRST\"", + pooltype)); } } } diff --git a/paddle/fluid/operators/math/sequence_pooling.cu b/paddle/fluid/operators/math/sequence_pooling.cu index 422b06c70eb2107659666edf58223ae8e4666b1d..cba8dd935ef1b3625f5f68578e411aec81eaa4f4 100644 --- a/paddle/fluid/operators/math/sequence_pooling.cu +++ b/paddle/fluid/operators/math/sequence_pooling.cu @@ -205,7 +205,10 @@ class SequencePoolFunctor { lod.CUDAData(context.GetPlace()), lod.size(), item_dim, output->mutable_data(context.GetPlace()), nullptr); } else { - PADDLE_THROW("unsupported pooling pooltype"); + PADDLE_THROW(platform::errors::InvalidArgument( + "unsupported pooling pooltype: %s. Only support \"MAX\", " + "\"AVERAGE\", \"SUM\", \"SQRT\", \"LAST\" and \"FIRST\"", + pooltype)); } } }; @@ -370,7 +373,10 @@ class SequencePoolGradFunctor { in_grad->mutable_data(context.GetPlace()), nullptr); } else { - PADDLE_THROW("unsupported pooling pooltype"); + PADDLE_THROW(platform::errors::InvalidArgument( + "unsupported pooling pooltype: %s. Only support \"MAX\", " + "\"AVERAGE\", \"SUM\", \"SQRT\", \"LAST\" and \"FIRST\"", + pooltype)); } } }; diff --git a/paddle/fluid/operators/math/sequence_pooling_test.cc b/paddle/fluid/operators/math/sequence_pooling_test.cc index efab1a375b56bea3caec2c8169dc390298a37cbe..4b5f484e52c6acb2d7fb2cea6265c8dd7826571b 100644 --- a/paddle/fluid/operators/math/sequence_pooling_test.cc +++ b/paddle/fluid/operators/math/sequence_pooling_test.cc @@ -50,9 +50,21 @@ void TestSequencePoolingSum(const DeviceContext &context, in_grad.mutable_data(in_dims, place); // check tensor contruction result - PADDLE_ENFORCE_EQ(in_grad.dims().size(), out_grad.dims().size()); + PADDLE_ENFORCE_EQ( + in_grad.dims().size(), out_grad.dims().size(), + paddle::platform::errors::InvalidArgument( + "The dimension of input and output shall be same. Expected %ld == " + "%ld, but got %ld != %ld. Please check the input value.", + in_grad.dims().size(), out_grad.dims().size(), in_grad.dims().size(), + out_grad.dims().size())); for (int64_t i = 1; i < out_grad.dims().size(); ++i) { - PADDLE_ENFORCE_EQ(in_grad.dims()[i], out_grad.dims()[i]); + PADDLE_ENFORCE_EQ( + in_grad.dims()[i], out_grad.dims()[i], + paddle::platform::errors::InvalidArgument( + "The dimension of input and output shall be same. Expected %ld == " + "%ld, but got %ld != %ld. Please check the input value.", + in_grad.dims()[i], out_grad.dims()[i], in_grad.dims()[i], + out_grad.dims()[i])); } // call functor diff --git a/paddle/fluid/operators/math/tree2col.cc b/paddle/fluid/operators/math/tree2col.cc index cafcf6319326cc9a496de8ee6aa1033e1320f4b0..0344226ea66e2a64fce0574cb45a8c2ce918359c 100644 --- a/paddle/fluid/operators/math/tree2col.cc +++ b/paddle/fluid/operators/math/tree2col.cc @@ -55,7 +55,11 @@ void Tree2ColUtil::construct_tree(const paddle::Tensor &EdgeSet, std::vector> *tr, size_t *node_count) { auto edge_set_dims = EdgeSet.dims(); - PADDLE_ENFORCE_EQ(edge_set_dims[1], 2); + PADDLE_ENFORCE_EQ(edge_set_dims[1], 2, + platform::errors::InvalidArgument( + "The second dimension of the EdgeSet shall be 2, but " + "got %ld != 2. Please check the input value.", + edge_set_dims[1])); int64_t edge_count = EdgeSet.numel(); const int *edge_data = EdgeSet.data(); diff --git a/paddle/fluid/operators/math/unpooling.cc b/paddle/fluid/operators/math/unpooling.cc index 13f0845bb8579615381c06072eb1e32507ffd3cf..9ad2ec5005203d02032aaa6f7e48b27f52631059 100644 --- a/paddle/fluid/operators/math/unpooling.cc +++ b/paddle/fluid/operators/math/unpooling.cc @@ -37,7 +37,13 @@ class Unpool2dMaxFunctor { for (int c = 0; c < output_channels; ++c) { for (int i = 0; i < input_feasize; ++i) { int index = indices_data[i]; - PADDLE_ENFORCE(index < output_feasize, "err index in unpooling!"); + PADDLE_ENFORCE_LT( + index, output_feasize, + platform::errors::InvalidArgument( + "index should less than output tensor height * output tensor " + "width. Expected %ld < %ld, but got " + "%ld >= %ld. Please check input value.", + index, output_feasize, index, output_feasize)); output_data[index] = input_data[i]; } input_data += input_feasize; @@ -72,7 +78,13 @@ class Unpool2dMaxGradFunctor { for (int c = 0; c < output_channels; ++c) { for (int i = 0; i < input_feasize; ++i) { int index = indices_data[i]; - PADDLE_ENFORCE(index < output_feasize, "err index in unpooling!"); + PADDLE_ENFORCE_LT( + index, output_feasize, + platform::errors::InvalidArgument( + "index should less than output tensor height * output tensor " + "width. Expected %ld < %ld, but got " + "%ld >= %ld. Please check input value.", + index, output_feasize, index, output_feasize)); input_grad_data[i] = output_grad_data[index]; } input_grad_data += input_feasize; diff --git a/paddle/fluid/operators/math/vol2col.cc b/paddle/fluid/operators/math/vol2col.cc index 01f50727b442579fa62059560d0c75d329d6e288..c05da0062f2bab66746feb9d8ebedeca0c0f9688 100644 --- a/paddle/fluid/operators/math/vol2col.cc +++ b/paddle/fluid/operators/math/vol2col.cc @@ -34,10 +34,16 @@ class Vol2ColFunctor { const std::vector& strides, const std::vector& paddings, framework::Tensor* col, const DataLayout data_layout) const { - PADDLE_ENFORCE_EQ(vol.dims().size(), 4, - "The dimension of vol should be 4."); - PADDLE_ENFORCE_EQ(col->dims().size(), 7, - "The dimension of col should be 7."); + PADDLE_ENFORCE_EQ( + vol.dims().size(), 4, + platform::errors::InvalidArgument("The dimension of" + " vol should be 4, but received %d.", + vol.dims().size())); + PADDLE_ENFORCE_EQ( + col->dims().size(), 7, + platform::errors::InvalidArgument("The dimension of" + "col should be 7, but received %d.", + col->dims().size())); int input_channels = (data_layout != DataLayout::kNHWC ? vol.dims()[0] : vol.dims()[3]); @@ -65,27 +71,33 @@ class Vol2ColFunctor { int pad_w_left = paddings_size_is_6 ? paddings[4] : paddings[2]; int pad_w_right = paddings_size_is_6 ? paddings[5] : paddings[2]; - PADDLE_ENFORCE_EQ((input_depth + pad_d_forth + pad_d_back - - ((dilations[0] * (filter_depth - 1) + 1))) / - strides[0] + - 1, - output_depth, - "input_depth and output_depth are " - "mismatching."); - PADDLE_ENFORCE_EQ((input_height + pad_h_up + pad_h_down - - ((dilations[1] * (filter_height - 1) + 1))) / - strides[1] + - 1, - output_height, - "input_height and output_height are " - "mismatching."); - PADDLE_ENFORCE_EQ((input_width + pad_w_left + pad_w_right - - ((dilations[2] * (filter_width - 1) + 1))) / - strides[2] + - 1, - output_width, - "input_width and output_width are " - "mismatching."); + auto input_depth_tmp = (input_depth + pad_d_forth + pad_d_back - + ((dilations[0] * (filter_depth - 1) + 1))) / + strides[0] + + 1; + PADDLE_ENFORCE_EQ( + input_depth_tmp, output_depth, + platform::errors::InvalidArgument( + "input_depth(%d) and output_depth(%d) are mismatching.", + input_depth_tmp, output_depth)); + auto input_height_tmp = (input_height + pad_h_up + pad_h_down - + ((dilations[1] * (filter_height - 1) + 1))) / + strides[1] + + 1; + PADDLE_ENFORCE_EQ( + input_height_tmp, output_height, + platform::errors::InvalidArgument( + "input_height(%d) and output_height(%d) are mismatching.", + input_height_tmp, output_height)); + auto input_width_tmp = (input_width + pad_w_left + pad_w_right - + ((dilations[2] * (filter_width - 1) + 1))) / + strides[2] + + 1; + PADDLE_ENFORCE_EQ( + input_width_tmp, output_width, + platform::errors::InvalidArgument( + "input_width(%d) and output_width(%d) are mismatching.", + input_width_tmp, output_width)); const T* vol_data = vol.data(); T* col_data = col->data(); @@ -140,10 +152,16 @@ class Col2VolFunctor { const std::vector& strides, const std::vector& paddings, framework::Tensor* vol, const DataLayout data_layout) const { - PADDLE_ENFORCE_EQ(vol->dims().size(), 4, - "The dimension of vol should be 4."); - PADDLE_ENFORCE_EQ(col.dims().size(), 7, - "The dimension of col should be 7."); + PADDLE_ENFORCE_EQ( + vol->dims().size(), 4, + platform::errors::InvalidArgument("The dimension of vol" + " should be 4, but received %d.", + vol->dims().size())); + PADDLE_ENFORCE_EQ( + col.dims().size(), 7, + platform::errors::InvalidArgument("The dimension of col" + " should be 7, but received %d.", + col.dims().size())); int input_channels = (data_layout != DataLayout::kNHWC ? vol->dims()[0] : vol->dims()[3]); @@ -170,27 +188,33 @@ class Col2VolFunctor { int pad_w_left = paddings_size_is_6 ? paddings[4] : paddings[2]; int pad_w_right = paddings_size_is_6 ? paddings[5] : paddings[2]; - PADDLE_ENFORCE_EQ((input_depth + pad_d_forth + pad_d_back - - ((dilations[0] * (filter_depth - 1) + 1))) / - strides[0] + - 1, - output_depth, - "input_depth and output_depth are " - "mismatching."); - PADDLE_ENFORCE_EQ((input_height + pad_h_up + pad_h_down - - ((dilations[1] * (filter_height - 1) + 1))) / - strides[1] + - 1, - output_height, - "input_height and output_height are " - "mismatching."); - PADDLE_ENFORCE_EQ((input_width + pad_w_left + pad_w_right - - ((dilations[2] * (filter_width - 1) + 1))) / - strides[2] + - 1, - output_width, - "input_width and output_width are " - "mismatching."); + auto input_depth_tmp = (input_depth + pad_d_forth + pad_d_back - + ((dilations[0] * (filter_depth - 1) + 1))) / + strides[0] + + 1; + PADDLE_ENFORCE_EQ(input_depth_tmp, output_depth, + platform::errors::InvalidArgument( + "input_depth(%d)" + " and output_depth(%d) are mismatching.", + input_depth_tmp, output_depth)); + auto input_height_tmp = (input_height + pad_h_up + pad_h_down - + ((dilations[1] * (filter_height - 1) + 1))) / + strides[1] + + 1; + PADDLE_ENFORCE_EQ(input_height_tmp, output_height, + platform::errors::InvalidArgument( + "input_height(%d)" + " and output_height(%d) are mismatching.", + input_height_tmp, output_height)); + auto input_width_tmp = (input_width + pad_w_left + pad_w_right - + ((dilations[2] * (filter_width - 1) + 1))) / + strides[2] + + 1; + PADDLE_ENFORCE_EQ(input_width_tmp, output_width, + platform::errors::InvalidArgument( + "input_width(%d)" + " and output_width(%d) are mismatching.", + input_width_tmp, output_width)); T* vol_data = vol->data(); const T* col_data = col.data(); diff --git a/paddle/fluid/operators/math/vol2col.cu b/paddle/fluid/operators/math/vol2col.cu index 9de9051f512348f2567bfc35ae775b1852ed25fc..fe5a600909893b8313d470923ef4d43eae155e76 100644 --- a/paddle/fluid/operators/math/vol2col.cu +++ b/paddle/fluid/operators/math/vol2col.cu @@ -90,10 +90,16 @@ class Vol2ColFunctor { const std::vector& strides, const std::vector& paddings, framework::Tensor* col, const DataLayout data_layout) const { - PADDLE_ENFORCE_EQ(vol.dims().size(), 4, - "The dimension of vol should be 4."); - PADDLE_ENFORCE_EQ(col->dims().size(), 7, - "The dimension of col should be 7."); + PADDLE_ENFORCE_EQ( + vol.dims().size(), 4, + platform::errors::InvalidArgument("The dimension of" + " vol should be 4, but received %d.", + vol.dims().size())); + PADDLE_ENFORCE_EQ( + col->dims().size(), 7, + platform::errors::InvalidArgument("The dimension of" + "col should be 7, but received %d.", + col->dims().size())); int input_channels = (data_layout != DataLayout::kNHWC ? vol.dims()[0] : vol.dims()[3]); @@ -117,27 +123,33 @@ class Vol2ColFunctor { int pad_h_down = paddings_size_is_6 ? paddings[3] : paddings[1]; int pad_w_left = paddings_size_is_6 ? paddings[4] : paddings[2]; int pad_w_right = paddings_size_is_6 ? paddings[5] : paddings[2]; - PADDLE_ENFORCE_EQ((input_depth + pad_d_forth + pad_d_back - - ((dilations[0] * (filter_depth - 1) + 1))) / - strides[0] + - 1, - output_depth, - "input_depth and output_depth are " - "mismatching."); - PADDLE_ENFORCE_EQ((input_height + pad_h_up + pad_h_down - - ((dilations[1] * (filter_height - 1) + 1))) / - strides[1] + - 1, - output_height, - "input_height and output_height are " - "mismatching."); - PADDLE_ENFORCE_EQ((input_width + pad_w_left + pad_w_right - - ((dilations[2] * (filter_width - 1) + 1))) / - strides[2] + - 1, - output_width, - "input_width and output_width are " - "mismatching."); + auto input_depth_tmp = (input_depth + pad_d_forth + pad_d_back - + ((dilations[0] * (filter_depth - 1) + 1))) / + strides[0] + + 1; + PADDLE_ENFORCE_EQ( + input_depth_tmp, output_depth, + platform::errors::InvalidArgument( + "input_depth(%d) and output_depth(%d) are mismatching.", + input_depth_tmp, output_depth)); + auto input_height_tmp = (input_height + pad_h_up + pad_h_down - + ((dilations[1] * (filter_height - 1) + 1))) / + strides[1] + + 1; + PADDLE_ENFORCE_EQ( + input_height_tmp, output_height, + platform::errors::InvalidArgument( + "input_height(%d) and output_height(%d) are mismatching.", + input_height_tmp, output_height)); + auto input_width_tmp = (input_width + pad_w_left + pad_w_right - + ((dilations[2] * (filter_width - 1) + 1))) / + strides[2] + + 1; + PADDLE_ENFORCE_EQ( + input_width_tmp, output_width, + platform::errors::InvalidArgument( + "input_width(%d) and output_width(%d) are mismatching.", + input_width_tmp, output_width)); int num_outputs = input_channels * output_depth * output_height * output_width; @@ -241,10 +253,16 @@ class Col2VolFunctor { const std::vector& strides, const std::vector& paddings, framework::Tensor* vol, const DataLayout data_layout) const { - PADDLE_ENFORCE_EQ(vol->dims().size(), 4, - "The dimension of vol should be 4."); - PADDLE_ENFORCE_EQ(col.dims().size(), 7, - "The dimension of col should be 7."); + PADDLE_ENFORCE_EQ( + vol->dims().size(), 4, + platform::errors::InvalidArgument("The dimension of vol" + " should be 4, but received %d.", + vol->dims().size())); + PADDLE_ENFORCE_EQ( + col.dims().size(), 7, + platform::errors::InvalidArgument("The dimension of col" + " should be 7, but received %d.", + col.dims().size())); int input_channels = (data_layout != DataLayout::kNHWC ? vol->dims()[0] : vol->dims()[3]); @@ -269,27 +287,33 @@ class Col2VolFunctor { int pad_w_left = paddings_size_is_6 ? paddings[4] : paddings[2]; int pad_w_right = paddings_size_is_6 ? paddings[5] : paddings[2]; - PADDLE_ENFORCE_EQ((input_depth + pad_d_forth + pad_d_back - - ((dilations[0] * (filter_depth - 1) + 1))) / - strides[0] + - 1, - output_depth, - "input_depth and output_depth are " - "mismatching."); - PADDLE_ENFORCE_EQ((input_height + pad_h_up + pad_h_down - - ((dilations[1] * (filter_height - 1) + 1))) / - strides[1] + - 1, - output_height, - "input_height and output_height are " - "mismatching."); - PADDLE_ENFORCE_EQ((input_width + pad_w_left + pad_w_right - - ((dilations[2] * (filter_width - 1) + 1))) / - strides[2] + - 1, - output_width, - "input_width and output_width are " - "mismatching."); + auto input_depth_tmp = (input_depth + pad_d_forth + pad_d_back - + ((dilations[0] * (filter_depth - 1) + 1))) / + strides[0] + + 1; + PADDLE_ENFORCE_EQ(input_depth_tmp, output_depth, + platform::errors::InvalidArgument( + "input_depth(%d)" + " and output_depth(%d) are mismatching.", + input_depth_tmp, output_depth)); + auto input_height_tmp = (input_height + pad_h_up + pad_h_down - + ((dilations[1] * (filter_height - 1) + 1))) / + strides[1] + + 1; + PADDLE_ENFORCE_EQ(input_height_tmp, output_height, + platform::errors::InvalidArgument( + "input_height(%d)" + " and output_height(%d) are mismatching.", + input_height_tmp, output_height)); + auto input_width_tmp = (input_width + pad_w_left + pad_w_right - + ((dilations[2] * (filter_width - 1) + 1))) / + strides[2] + + 1; + PADDLE_ENFORCE_EQ(input_width_tmp, output_width, + platform::errors::InvalidArgument( + "input_width(%d)" + " and output_width(%d) are mismatching.", + input_width_tmp, output_width)); int num_kernels = input_channels * input_depth * input_height * input_width; diff --git a/paddle/fluid/operators/mkldnn/gaussian_random_mkldnn_op.cc b/paddle/fluid/operators/mkldnn/gaussian_random_mkldnn_op.cc index 98200caca8cf66960632b88966f23e99fcd4c299..51fa5ad021a2b284cd75f297d83326b2102c1e41 100644 --- a/paddle/fluid/operators/mkldnn/gaussian_random_mkldnn_op.cc +++ b/paddle/fluid/operators/mkldnn/gaussian_random_mkldnn_op.cc @@ -30,8 +30,7 @@ class GaussianMKLDNNKernel : public paddle::framework::OpKernel { float std = context.Attr("std"); auto* tensor = context.Output("Out"); - const std::string op_type = "gaussian_random"; - auto shape = GetShape(context, op_type); + auto shape = GetShape(context); tensor->Resize(shape); T* data = tensor->mutable_data(context.GetPlace()); int64_t size = tensor->numel(); diff --git a/paddle/fluid/operators/optimizers/lars_momentum_op.cc b/paddle/fluid/operators/optimizers/lars_momentum_op.cc old mode 100644 new mode 100755 index 5f0500d2faa77f7c2e901c0d30ab2c42036d2a86..479f9643749d63c673158ad055409a0925f3d576 --- a/paddle/fluid/operators/optimizers/lars_momentum_op.cc +++ b/paddle/fluid/operators/optimizers/lars_momentum_op.cc @@ -48,6 +48,9 @@ class LarsMomentumOpMaker : public framework::OpProtoAndCheckerMaker { AddAttr("lars_weight_decay", "(float, default 0.0005) LARS weight decay") .SetDefault(0.0005); + AddAttr("epsilon", + "(float, default 0.0) epsilon to avoid Division by Zero.") + .SetDefault(0.0); AddComment(R"DOC( Lars Momentum Optimizer. diff --git a/paddle/fluid/operators/optimizers/lars_momentum_op.cu b/paddle/fluid/operators/optimizers/lars_momentum_op.cu index 1dace4ed6ab3e17b348035e34f6d9ea6d31edae9..eb0111ae4de2f066359e26406f6c7ec3eb54d5fc 100644 --- a/paddle/fluid/operators/optimizers/lars_momentum_op.cu +++ b/paddle/fluid/operators/optimizers/lars_momentum_op.cu @@ -23,14 +23,16 @@ __global__ void MomentumLarsKernel(const T* p, const T* g, const T* v, const T* learning_rate, const T mu, const int64_t num, const T lars_coeff, const T lars_weight_decay, const T* p_norm, - const T* g_norm, T* p_out, T* v_out) { + const T* g_norm, T* p_out, T* v_out, + const T epsilon) { T lr = learning_rate[0]; T local_lr = learning_rate[0]; CUDA_KERNEL_LOOP(i, num) { - if (p_norm[0] > 0 && g_norm[0] > 0) { + if (lars_weight_decay > 0 && p_norm[0] > 0 && g_norm[0] > 0) { local_lr = lr * lars_coeff * p_norm[0] / - (g_norm[0] + lars_weight_decay * p_norm[0]); + (g_norm[0] + lars_weight_decay * p_norm[0] + epsilon); } + T v_new = v[i] * mu + local_lr * (g[i] + lars_weight_decay * p[i]); v_out[i] = v_new; p_out[i] = p[i] - v_new; @@ -54,6 +56,7 @@ class LarsMomentumOpCUDAKernel : public framework::OpKernel { T mu = static_cast(ctx.Attr("mu")); T lars_coeff = ctx.Attr("lars_coeff"); T lars_weight_decay = ctx.Attr("lars_weight_decay"); + T epsilon = ctx.Attr("epsilon"); auto* p = param->data(); auto* v = velocity->data(); @@ -79,7 +82,7 @@ class LarsMomentumOpCUDAKernel : public framework::OpKernel { eg_norm.device(*place) = eigen_g.square().sum().sqrt(); MomentumLarsKernel<<>>( p, g, v, lr, mu, param->numel(), lars_coeff, lars_weight_decay, - p_norm_data, g_norm_data, p_out, v_out); + p_norm_data, g_norm_data, p_out, v_out, epsilon); } }; diff --git a/paddle/fluid/operators/optimizers/lars_momentum_op.h b/paddle/fluid/operators/optimizers/lars_momentum_op.h old mode 100644 new mode 100755 index e0064c201825b1f074eb53c591dc3abdd7bc1e1b..b579b5143ddbe6221738f9864f13fb7bea4ac509 --- a/paddle/fluid/operators/optimizers/lars_momentum_op.h +++ b/paddle/fluid/operators/optimizers/lars_momentum_op.h @@ -39,6 +39,7 @@ class LarsMomentumOpKernel : public framework::OpKernel { T mu = static_cast(ctx.Attr("mu")); T lars_coeff = ctx.Attr("lars_coeff"); T lars_weight_decay = ctx.Attr("lars_weight_decay"); + T epsilon = ctx.Attr("epsilon"); auto p_out = framework::EigenVector::Flatten(*param_out); auto v_out = framework::EigenVector::Flatten(*velocity_out); @@ -59,9 +60,9 @@ class LarsMomentumOpKernel : public framework::OpKernel { ep_norm = p.square().sum().sqrt(); eg_norm = g.square().sum().sqrt(); T local_lr = lr[0]; - if (ep_norm(0) > 0 && eg_norm(0) > 0) { + if (lars_weight_decay > 0 && ep_norm(0) > 0 && eg_norm(0) > 0) { local_lr = lr[0] * lars_coeff * ep_norm(0) / - (eg_norm(0) + lars_weight_decay * ep_norm(0)); + (eg_norm(0) + lars_weight_decay * ep_norm(0) + epsilon); } v_out = v * mu + local_lr * (g + lars_weight_decay * p); p_out = p - v_out; diff --git a/paddle/fluid/operators/p_norm_op.cc b/paddle/fluid/operators/p_norm_op.cc index 59035d5a8ca5d4214f1370e1b14b2be9b234fa6a..cd7a8c6d24eaaca096b630b74dd7cd1bb9d35d09 100644 --- a/paddle/fluid/operators/p_norm_op.cc +++ b/paddle/fluid/operators/p_norm_op.cc @@ -105,6 +105,12 @@ class PnormOp : public framework::OperatorWithKernel { bool asvector = ctx->Attrs().Get("asvector"); if (asvector) { reduce_dims.emplace_back(1); + if (keepdim) { + for (int i = 1; i < x_dim.size(); ++i) { + reduce_dims.emplace_back(1); + } + x_dim = framework::make_ddim(reduce_dims); + } } else { if (axis < 0) axis = x_dim.size() + axis; for (int i = 0; i < x_dim.size(); ++i) { diff --git a/paddle/fluid/operators/reduce_ops/CMakeLists.txt b/paddle/fluid/operators/reduce_ops/CMakeLists.txt index 3da481a142aa2282aade661de7679cf4edf597a0..a68666b100cb52c722c4fefc849e94947130010f 100644 --- a/paddle/fluid/operators/reduce_ops/CMakeLists.txt +++ b/paddle/fluid/operators/reduce_ops/CMakeLists.txt @@ -1,6 +1,10 @@ include(operators) if(WITH_GPU) - register_operators(DEPS cub) + if(${CMAKE_CUDA_COMPILER_VERSION} LESS 11.0) + register_operators(DEPS cub) + else() + register_operators() + endif() else() register_operators() endif() @@ -24,5 +28,9 @@ if(WITH_GPU) endif() if(WITH_GPU) - nv_test(check_reduce_rank_test SRCS check_reduce_rank_test.cu DEPS tensor cub) + if (${CMAKE_CUDA_COMPILER_VERSION} LESS 11.0) + nv_test(check_reduce_rank_test SRCS check_reduce_rank_test.cu DEPS tensor cub) + else() + nv_test(check_reduce_rank_test SRCS check_reduce_rank_test.cu DEPS tensor) + endif() endif() diff --git a/paddle/fluid/operators/reduce_ops/logsumexp_op.cc b/paddle/fluid/operators/reduce_ops/logsumexp_op.cc index 322a1637f5deec909db13f1bd0433446cd7606ae..7cd164bfd3a3d77288b59c40f147ae9cdd8215e0 100644 --- a/paddle/fluid/operators/reduce_ops/logsumexp_op.cc +++ b/paddle/fluid/operators/reduce_ops/logsumexp_op.cc @@ -13,18 +13,138 @@ // limitations under the License. #include "paddle/fluid/operators/reduce_ops/logsumexp_op.h" -#include +#include #include -#include #include namespace paddle { namespace operators { -class LogsumexpOpMaker : public ops::ReduceOpMaker { - protected: - virtual std::string GetName() const { return "logsumexp"; } - virtual std::string GetOpType() const { return "Reduce logsumexp"; } +class LogsumexpOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext* ctx) const override { + OP_INOUT_CHECK(ctx->HasInput("X"), "Input", "X", "logsumexp"); + OP_INOUT_CHECK(ctx->HasOutput("Out"), "Output", "Out", "logsumexp"); + auto x_dims = ctx->GetInputDim("X"); + auto x_rank = x_dims.size(); + PADDLE_ENFORCE_LE(x_rank, 4, + platform::errors::InvalidArgument( + "The input tensor X's dimensions of logsumexp " + "should be less equal than 4. But received X's " + "dimensions = %d, X's shape = [%s].", + x_rank, x_dims)); + auto axis = ctx->Attrs().Get>("axis"); + PADDLE_ENFORCE_GT( + axis.size(), 0, + platform::errors::InvalidArgument( + "The size of axis of logsumexp " + "should be greater than 0. But received the size of axis " + "of logsumexp is %d.", + axis.size())); + + for (size_t i = 0; i < axis.size(); i++) { + PADDLE_ENFORCE_LT( + axis[i], x_rank, + platform::errors::InvalidArgument( + "axis[%d] should be in the " + "range [-dimension(X), dimension(X)] " + "where dimesion(X) is %d. But received axis[i] = %d.", + i, x_rank, axis[i])); + PADDLE_ENFORCE_GE( + axis[i], -x_rank, + platform::errors::InvalidArgument( + "axis[%d] should be in the " + "range [-dimension(X), dimension(X)] " + "where dimesion(X) is %d. But received axis[i] = %d.", + i, x_rank, axis[i])); + if (axis[i] < 0) { + axis[i] += x_rank; + } + } + + bool keepdim = ctx->Attrs().Get("keepdim"); + bool reduce_all = ctx->Attrs().Get("reduce_all"); + auto dims_vector = vectorize(x_dims); + if (reduce_all) { + if (keepdim) + ctx->SetOutputDim( + "Out", framework::make_ddim(std::vector(x_rank, 1))); + else + ctx->SetOutputDim("Out", {1}); + } else { + auto dims_vector = vectorize(x_dims); + if (keepdim) { + for (size_t i = 0; i < axis.size(); ++i) { + dims_vector[axis[i]] = 1; + } + } else { + const int kDelFlag = -1; + for (size_t i = 0; i < axis.size(); ++i) { + dims_vector[axis[i]] = kDelFlag; + } + dims_vector.erase( + std::remove(dims_vector.begin(), dims_vector.end(), kDelFlag), + dims_vector.end()); + } + if (!keepdim && dims_vector.size() == 0) { + dims_vector.push_back(1); + } + auto out_dims = framework::make_ddim(dims_vector); + ctx->SetOutputDim("Out", out_dims); + if (axis.size() > 0 && axis[0] != 0) { + // Only pass LoD when not reducing on the first dim. + ctx->ShareLoD("X", /*->*/ "Out"); + } + } + } +}; + +class LogsumexpOpMaker : public framework::OpProtoAndCheckerMaker { + public: + void Make() override { + AddInput("X", + "(Tensor) The input tensor. Tensors with rank at most 4 are " + "supported."); + AddOutput("Out", "(Tensor) The result tensor."); + AddAttr>( + "axis", + "(list, default {0}) The dimensions to reduce. " + "Must be in the range [-rank(input), rank(input)). " + "If `axis[i] < 0`, the axis[i] to reduce is `rank + axis[i]`. " + "Note that reducing on the first dim will make the LoD info lost.") + .SetDefault({0}); + AddAttr("keepdim", + "(bool, default false) " + "If true, retain the reduced dimension with length 1.") + .SetDefault(false); + AddAttr("reduce_all", + "(bool, default false) " + "If true, output a scalar reduced along all dimensions.") + .SetDefault(false); + AddComment(string::Sprintf(R"DOC( +logsumexp Operator. + +This operator computes the logsumexp of input tensor along the given axis. +The result tensor has 1 fewer dimension than the input unless keep_dim is true. +If reduce_all is true, just reduce along all dimensions and output a scalar. + +)DOC")); + } +}; + +class LogsumexpGrapOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext* ctx) const override { + OP_INOUT_CHECK(ctx->HasInput("X"), "Input", "X", "logsumexp"); + OP_INOUT_CHECK(ctx->HasInput("Out"), "Input", "Out", "logsumexp"); + OP_INOUT_CHECK(ctx->HasInput(framework::GradVarName("Out")), "Input", + "Out@GRAD", "logsumexp"); + ctx->SetOutputDim(framework::GradVarName("X"), ctx->GetInputDim("X")); + } }; template @@ -32,7 +152,6 @@ class LogsumexpGradOpMaker : public framework::SingleGradOpMaker { public: using framework::SingleGradOpMaker::SingleGradOpMaker; - protected: void Apply(GradOpPtr op) const override { op->SetType("logsumexp_grad"); op->SetInput("X", this->Input("X")); @@ -46,18 +165,17 @@ class LogsumexpGradOpMaker : public framework::SingleGradOpMaker { } // namespace operators } // namespace paddle -REGISTER_OPERATOR(logsumexp, ops::ReduceOp, ops::LogsumexpOpMaker, +namespace ops = paddle::operators; + +REGISTER_OPERATOR(logsumexp, ops::LogsumexpOp, ops::LogsumexpOpMaker, ops::LogsumexpGradOpMaker, ops::LogsumexpGradOpMaker); -REGISTER_OPERATOR(logsumexp_grad, ops::ReduceGradOp); +REGISTER_OPERATOR(logsumexp_grad, ops::LogsumexpGrapOp); -REGISTER_OP_CPU_KERNEL(logsumexp, - ops::ReduceKernel, - ops::ReduceKernel); REGISTER_OP_CPU_KERNEL( - logsumexp_grad, ops::ReduceGradKernel, - ops::ReduceGradKernel); + logsumexp, ops::LogsumexpKernel, + ops::LogsumexpKernel); +REGISTER_OP_CPU_KERNEL( + logsumexp_grad, + ops::LogsumexpGradKernel, + ops::LogsumexpGradKernel); diff --git a/paddle/fluid/operators/reduce_ops/logsumexp_op.cu b/paddle/fluid/operators/reduce_ops/logsumexp_op.cu index c9ad1075c0c3c1c6f405144dbfde2e81b85124aa..86a31595ebaabcbc07fab64779c33566d5b020eb 100644 --- a/paddle/fluid/operators/reduce_ops/logsumexp_op.cu +++ b/paddle/fluid/operators/reduce_ops/logsumexp_op.cu @@ -14,8 +14,8 @@ #include "paddle/fluid/operators/reduce_ops/logsumexp_op.h" -REGISTER_OP_CUDA_KERNEL(logsumexp, - ops::ReduceKernel, - ops::ReduceKernel); +namespace ops = paddle::operators; + +REGISTER_OP_CUDA_KERNEL( + logsumexp, ops::LogsumexpKernel, + ops::LogsumexpKernel); diff --git a/paddle/fluid/operators/reduce_ops/logsumexp_op.h b/paddle/fluid/operators/reduce_ops/logsumexp_op.h index 1d0e00262a37ff7160abd7a865e63377f8b30461..a478690976bd396db921b465d171a422451e0742 100644 --- a/paddle/fluid/operators/reduce_ops/logsumexp_op.h +++ b/paddle/fluid/operators/reduce_ops/logsumexp_op.h @@ -14,11 +14,20 @@ #pragma once -#include "paddle/fluid/operators/reduce_ops/reduce_op.h" +#include +#include +#include "paddle/fluid/operators/reduce_ops/reduce_op_function.h" namespace paddle { namespace operators { +#define HANDLE_DIM(NDIM, RDIM) \ + if (ndim == NDIM && rdim == RDIM) { \ + ReduceFunctor( \ + context.template device_context(), *input, output, \ + axis, keepdim); \ + } + struct LogsumexpFunctor { template void operator()(const DeviceContext& place, X* x, Y* y, const Dim& dim) { @@ -54,5 +63,106 @@ struct LogsumexpGradFunctor { } }; +template +class LogsumexpKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* input = context.Input("X"); + auto* output = context.Output("Out"); + output->mutable_data(context.GetPlace()); + + auto axis = context.Attr>("axis"); + auto keepdim = context.Attr("keepdim"); + auto reduce_all = context.Attr("reduce_all"); + + const auto& input_dim_size = input->dims().size(); + // The dims has full dim, set the reduce_all is True + reduce_all |= (static_cast(axis.size()) == input_dim_size); + + if (reduce_all) { + // Flatten and reduce 1-D tensor + auto x = EigenVector::Flatten(*input); + auto out = EigenScalar::From(*output); + auto& place = + *context.template device_context().eigen_device(); + auto reduce_dim = Eigen::array({{0}}); + LogsumexpFunctor()(place, &x, &out, reduce_dim); + } else { + int ndim = input_dim_size; + int rdim = axis.size(); + // comments for accelerating compiling temporarily. + // HANDLE_DIM(6, 5); + // HANDLE_DIM(6, 4); + // HANDLE_DIM(6, 3); + // HANDLE_DIM(6, 2); + // HANDLE_DIM(6, 1); + // HANDLE_DIM(5, 4); + // HANDLE_DIM(5, 3); + // HANDLE_DIM(5, 2); + // HANDLE_DIM(5, 1); + HANDLE_DIM(4, 3); + HANDLE_DIM(4, 2); + HANDLE_DIM(4, 1); + HANDLE_DIM(3, 2); + HANDLE_DIM(3, 1); + HANDLE_DIM(2, 1); + } + } +}; + +template +class LogsumexpGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* input = context.Input("X"); + auto* output = context.Input("Out"); + auto* output_grad = context.Input(framework::GradVarName("Out")); + auto* input_grad = context.Output(framework::GradVarName("X")); + input_grad->mutable_data(context.GetPlace()); + + auto axis = context.Attr>("axis"); + auto reduce_all = context.Attr("reduce_all"); + const auto input_dim_size = context.Input("X")->dims().size(); + reduce_all |= (static_cast(axis.size()) == input_dim_size); + + if (reduce_all) { + auto x = EigenVector::Flatten(*input); + auto y = EigenVector::Flatten(*output); + auto dy = EigenVector::Flatten(*output_grad); + auto dx = EigenVector::Flatten(*input_grad); + auto& place = + *context.template device_context().eigen_device(); + auto broadcast_dim = + Eigen::array({{static_cast(input->numel())}}); + LogsumexpGradFunctor()(place, &x, &y, &dx, &dy, broadcast_dim, + broadcast_dim[0]); + } else { + int rank = input->dims().size(); + switch (rank) { + case 1: + ReduceGradFunctor( + context.template device_context(), *input, *output, + *output_grad, input_grad, axis); + break; + case 2: + ReduceGradFunctor( + context.template device_context(), *input, *output, + *output_grad, input_grad, axis); + break; + case 3: + ReduceGradFunctor( + context.template device_context(), *input, *output, + *output_grad, input_grad, axis); + break; + case 4: + ReduceGradFunctor( + context.template device_context(), *input, *output, + *output_grad, input_grad, axis); + break; + } + } + } +}; + } // namespace operators } // namespace paddle diff --git a/paddle/fluid/operators/reduce_ops/logsumexp_op.part.cu b/paddle/fluid/operators/reduce_ops/logsumexp_op.part.cu index d6ad4863092a50233b806c944db0b8c161ed9dd0..81124e4f070a54444f4305dc903280548ac10b60 100644 --- a/paddle/fluid/operators/reduce_ops/logsumexp_op.part.cu +++ b/paddle/fluid/operators/reduce_ops/logsumexp_op.part.cu @@ -15,8 +15,9 @@ // .part used to speed up nvcc compile #include "paddle/fluid/operators/reduce_ops/logsumexp_op.h" +namespace ops = paddle::operators; + REGISTER_OP_CUDA_KERNEL( - logsumexp_grad, ops::ReduceGradKernel, - ops::ReduceGradKernel); + logsumexp_grad, + ops::LogsumexpGradKernel, + ops::LogsumexpGradKernel); diff --git a/paddle/fluid/operators/roi_align_op.cc b/paddle/fluid/operators/roi_align_op.cc index 911dfea50e2e2cf8ec8f230bfc1e0bf4836463b6..0eeb7e0bb24f512aa6859e92de9f490e491543aa 100644 --- a/paddle/fluid/operators/roi_align_op.cc +++ b/paddle/fluid/operators/roi_align_op.cc @@ -11,6 +11,7 @@ limitations under the License. */ #include "paddle/fluid/operators/roi_align_op.h" #include +#include "paddle/fluid/framework/op_version_registry.h" namespace paddle { namespace operators { @@ -35,13 +36,13 @@ class ROIAlignOp : public framework::OperatorWithKernel { auto input_dims = ctx->GetInputDim("X"); auto rois_dims = ctx->GetInputDim("ROIs"); - if (ctx->HasInput("RoisLod")) { - auto rois_lod_dims = ctx->GetInputDim("RoisLod"); + if (ctx->HasInput("RoisNum")) { + auto rois_num_dims = ctx->GetInputDim("RoisNum"); PADDLE_ENFORCE_EQ( - rois_lod_dims.size(), 1, - platform::errors::InvalidArgument("The RoisLod dimension should be 1" - ", but got dimension = %d", - rois_lod_dims.size())); + rois_num_dims.size(), 1, + platform::errors::InvalidArgument("The size of RoisNum should be 1" + ", but received size = %d", + rois_num_dims.size())); } PADDLE_ENFORCE_EQ( input_dims.size(), 4, @@ -145,9 +146,9 @@ class ROIAlignOpMaker : public framework::OpProtoAndCheckerMaker { "given as [[x1, y1, x2, y2], ...]. " "(x1, y1) is the top left coordinates, and " "(x2, y2) is the bottom right coordinates."); - AddInput("RoisLod", + AddInput("RoisNum", "(Tensor), " - "The lod info of rois.") + "The number of RoIs in each image.") .AsDispensable(); AddOutput("Out", "(Tensor), " @@ -203,7 +204,7 @@ class ROIAlignGradMaker : public framework::SingleGradOpMaker { op->SetType("roi_align_grad"); op->SetInput("X", this->Input("X")); op->SetInput("ROIs", this->Input("ROIs")); - op->SetInput("RoisLod", this->Input("RoisLod")); + op->SetInput("RoisNum", this->Input("RoisNum")); op->SetInput(framework::GradVarName("Out"), this->OutputGrad("Out")); op->SetOutput(framework::GradVarName("X"), this->InputGrad("X")); op->SetAttrMap(this->Attrs()); @@ -231,3 +232,10 @@ REGISTER_OP_CPU_KERNEL( ops::CPUROIAlignGradOpKernel, ops::CPUROIAlignGradOpKernel, ops::CPUROIAlignGradOpKernel); +REGISTER_OP_VERSION(roi_align) + .AddCheckpoint( + R"ROC( + Upgrade roi_align add a new input [RoisNum])ROC", + paddle::framework::compatible::OpVersionDesc().NewInput( + "RoisNum", + "The number of RoIs in each image. RoisNum is dispensable.")); diff --git a/paddle/fluid/operators/roi_align_op.cu b/paddle/fluid/operators/roi_align_op.cu index f7ec13e5bccd63d2f6552ed52f8d709a57320ddd..3a4ce55f4fb77160e7fc645539c1868fe2864b19 100644 --- a/paddle/fluid/operators/roi_align_op.cu +++ b/paddle/fluid/operators/roi_align_op.cu @@ -257,24 +257,26 @@ class GPUROIAlignOpKernel : public framework::OpKernel { int* roi_batch_id_data = roi_batch_id_list.mutable_data(cplace); auto& dev_ctx = ctx.cuda_device_context(); auto gplace = BOOST_GET_CONST(platform::CUDAPlace, ctx.GetPlace()); - if (ctx.HasInput("RoisLod")) { - auto* rois_lod = ctx.Input("RoisLod"); - int rois_batch_size = rois_lod->numel(); + if (ctx.HasInput("RoisNum")) { + auto* rois_num_t = ctx.Input("RoisNum"); + int rois_batch_size = rois_num_t->numel(); PADDLE_ENFORCE_EQ( - rois_batch_size - 1, batch_size, + rois_batch_size, batch_size, platform::errors::InvalidArgument( "The rois_batch_size and imgs " "batch_size must be the same. But received rois_batch_size = %d, " "batch_size = %d", rois_batch_size, batch_size)); - std::vector rois_lod_(rois_batch_size); - memory::Copy(cplace, rois_lod_.data(), gplace, rois_lod->data(), - sizeof(int64_t) * rois_batch_size, 0); - for (int n = 0; n < rois_batch_size - 1; ++n) { - for (size_t i = rois_lod_[n]; i < rois_lod_[n + 1]; ++i) { + std::vector rois_num_list(rois_batch_size); + memory::Copy(cplace, rois_num_list.data(), gplace, + rois_num_t->data(), sizeof(int) * rois_batch_size, 0); + int start = 0; + for (int n = 0; n < rois_batch_size; ++n) { + for (int i = start; i < start + rois_num_list[n]; ++i) { roi_batch_id_data[i] = n; } + start += rois_num_list[n]; } } else { auto lod = rois->lod(); @@ -348,16 +350,18 @@ class GPUROIAlignGradOpKernel : public framework::OpKernel { auto& dev_ctx = ctx.cuda_device_context(); auto gplace = BOOST_GET_CONST(platform::CUDAPlace, ctx.GetPlace()); - if (ctx.HasInput("RoisLod")) { - auto* rois_lod = ctx.Input("RoisLod"); - int rois_batch_size = rois_lod->numel(); - std::vector rois_lod_(rois_batch_size); - memory::Copy(cplace, rois_lod_.data(), gplace, rois_lod->data(), - sizeof(int64_t) * rois_batch_size, 0); - for (int n = 0; n < rois_batch_size - 1; ++n) { - for (size_t i = rois_lod_[n]; i < rois_lod_[n + 1]; ++i) { + if (ctx.HasInput("RoisNum")) { + auto* rois_num_t = ctx.Input("RoisNum"); + int rois_batch_size = rois_num_t->numel(); + std::vector rois_num_list(rois_batch_size); + memory::Copy(cplace, rois_num_list.data(), gplace, + rois_num_t->data(), sizeof(int) * rois_batch_size, 0); + int start = 0; + for (int n = 0; n < rois_batch_size; ++n) { + for (size_t i = start; i < start + rois_num_list[n]; ++i) { roi_batch_id_data[i] = n; } + start += rois_num_list[n]; } } else { auto rois_lod = rois->lod().back(); diff --git a/paddle/fluid/operators/roi_align_op.h b/paddle/fluid/operators/roi_align_op.h index 366f865411461c91b4ec88203390b15fdba4414c..066125a92fbd9d1d49f0ba023366865620674e1f 100644 --- a/paddle/fluid/operators/roi_align_op.h +++ b/paddle/fluid/operators/roi_align_op.h @@ -165,21 +165,23 @@ class CPUROIAlignOpKernel : public framework::OpKernel { int* roi_batch_id_data = roi_batch_id_list.mutable_data(ctx.GetPlace()); int rois_batch_size; - if (ctx.HasInput("RoisLod")) { - auto* rois_lod_t = ctx.Input("RoisLod"); - rois_batch_size = rois_lod_t->numel(); + if (ctx.HasInput("RoisNum")) { + auto* rois_num_t = ctx.Input("RoisNum"); + rois_batch_size = rois_num_t->numel(); PADDLE_ENFORCE_EQ( - rois_batch_size - 1, batch_size, + rois_batch_size, batch_size, platform::errors::InvalidArgument( "The batch size of rois and the batch size of images " " must be the same. But received the batch size of rois is %d, " "and the batch size of images is %d", rois_batch_size, batch_size)); - auto* rois_lod = rois_lod_t->data(); - for (int n = 0; n < rois_batch_size - 1; ++n) { - for (int i = rois_lod[n]; i < rois_lod[n + 1]; ++i) { + auto* rois_num_data = rois_num_t->data(); + int start = 0; + for (int n = 0; n < rois_batch_size; ++n) { + for (int i = start; i < start + rois_num_data[n]; ++i) { roi_batch_id_data[i] = n; } + start += rois_num_data[n]; } } else { auto lod = rois->lod(); @@ -303,14 +305,16 @@ class CPUROIAlignGradOpKernel : public framework::OpKernel { roi_batch_id_list.mutable_data(ctx.GetPlace()); int rois_batch_size; - if (ctx.HasInput("RoisLod")) { - auto* rois_lod_t = ctx.Input("RoisLod"); - rois_batch_size = rois_lod_t->numel(); - auto* rois_lod = rois_lod_t->data(); - for (int n = 0; n < rois_batch_size - 1; ++n) { - for (int i = rois_lod[n]; i < rois_lod[n + 1]; ++i) { + if (ctx.HasInput("RoisNum")) { + auto* rois_num_t = ctx.Input("RoisNum"); + rois_batch_size = rois_num_t->numel(); + auto* rois_num_data = rois_num_t->data(); + int start = 0; + for (int n = 0; n < rois_batch_size; ++n) { + for (int i = start; i < start + rois_num_data[n]; ++i) { roi_batch_id_data[i] = n; } + start += rois_num_data[n]; } } else { auto rois_lod = rois->lod().back(); diff --git a/paddle/fluid/operators/roi_pool_op.cc b/paddle/fluid/operators/roi_pool_op.cc index 8a34cb35f6bf8dde97d29c02749d8a52fdf5f090..be3187b7513144f583458f3d7902a102e531a981 100644 --- a/paddle/fluid/operators/roi_pool_op.cc +++ b/paddle/fluid/operators/roi_pool_op.cc @@ -14,6 +14,7 @@ limitations under the License. */ #include "paddle/fluid/operators/roi_pool_op.h" #include +#include "paddle/fluid/framework/op_version_registry.h" namespace paddle { namespace operators { @@ -34,12 +35,13 @@ class ROIPoolOp : public framework::OperatorWithKernel { auto input_dims = ctx->GetInputDim("X"); auto rois_dims = ctx->GetInputDim("ROIs"); - if (ctx->HasInput("RoisLod")) { - auto rois_lod_dims = ctx->GetInputDim("RoisLod"); - PADDLE_ENFORCE_EQ(rois_lod_dims.size(), 1, + if (ctx->HasInput("RoisNum")) { + auto rois_num_dims = ctx->GetInputDim("RoisNum"); + PADDLE_ENFORCE_EQ(rois_num_dims.size(), 1, platform::errors::InvalidArgument( - "The lod information tensor of ROIs should " - "be one-dimensional")); + "The second dimension of RoisNum should " + "be 1, but received dimension is %d", + rois_num_dims.size())); } PADDLE_ENFORCE_EQ(input_dims.size(), 4, platform::errors::InvalidArgument( @@ -140,7 +142,8 @@ class ROIPoolOpMaker : public framework::OpProtoAndCheckerMaker { "Where batch_id is the id of the data, " "(x1, y1) is the top left coordinates, and " "(x2, y2) is the bottom right coordinates."); - AddInput("RoisLod", "(Tensor), The lod info of rois.").AsDispensable(); + AddInput("RoisNum", "(Tensor), The number of RoIs in each image.") + .AsDispensable(); AddOutput("Out", "(Tensor), " "The output of ROIPoolOp is a 4-D tensor with shape " @@ -197,7 +200,7 @@ class ROIPoolGradMaker : public framework::SingleGradOpMaker { op->SetType("roi_pool_grad"); op->SetInput("X", this->Input("X")); op->SetInput("ROIs", this->Input("ROIs")); - op->SetInput("RoisLod", this->Input("RoisLod")); + op->SetInput("RoisNum", this->Input("RoisNum")); op->SetInput("Argmax", this->Output("Argmax")); op->SetInput(framework::GradVarName("Out"), this->OutputGrad("Out")); op->SetOutput(framework::GradVarName("X"), this->InputGrad("X")); @@ -223,3 +226,10 @@ REGISTER_OP_CPU_KERNEL( ops::CPUROIPoolGradOpKernel, ops::CPUROIPoolGradOpKernel, ops::CPUROIPoolGradOpKernel); +REGISTER_OP_VERSION(roi_pool) + .AddCheckpoint( + R"ROC( + Upgrade roi_pool add a new input [RoisNum])ROC", + paddle::framework::compatible::OpVersionDesc().NewInput( + "RoisNum", + "The number of RoIs in each image. RoisNum is dispensable.")); diff --git a/paddle/fluid/operators/roi_pool_op.cu b/paddle/fluid/operators/roi_pool_op.cu index 1e8a8e3037d84f980d963d359ce791b1ddba47d3..98d9ef6b6e11440d38abbedbfd93f6d3544d77bc 100644 --- a/paddle/fluid/operators/roi_pool_op.cu +++ b/paddle/fluid/operators/roi_pool_op.cu @@ -157,19 +157,21 @@ class GPUROIPoolOpKernel : public framework::OpKernel { int* roi_batch_id_data = roi_batch_id_list.mutable_data(cplace); auto& dev_ctx = ctx.cuda_device_context(); auto gplace = BOOST_GET_CONST(platform::CUDAPlace, ctx.GetPlace()); - if (ctx.HasInput("RoisLod")) { - auto* rois_lod = ctx.Input("RoisLod"); - int rois_batch_size = rois_lod->numel(); + if (ctx.HasInput("RoisNum")) { + auto* rois_num_t = ctx.Input("RoisNum"); + int rois_batch_size = rois_num_t->numel(); PADDLE_ENFORCE_EQ( - rois_batch_size - 1, batch_size, + rois_batch_size, batch_size, "The rois_batch_size and imgs batch_size must be the same."); - std::vector rois_lod_(rois_batch_size); - memory::Copy(cplace, rois_lod_.data(), gplace, rois_lod->data(), - sizeof(int64_t) * rois_batch_size, 0); - for (int n = 0; n < rois_batch_size - 1; ++n) { - for (size_t i = rois_lod_[n]; i < rois_lod_[n + 1]; ++i) { + std::vector rois_num_list(rois_batch_size); + memory::Copy(cplace, rois_num_list.data(), gplace, + rois_num_t->data(), sizeof(int) * rois_batch_size, 0); + int start = 0; + for (int n = 0; n < rois_batch_size; ++n) { + for (int i = start; i < start + rois_num_list[n]; ++i) { roi_batch_id_data[i] = n; } + start += rois_num_list[n]; } } else { auto rois_lod = rois->lod().back(); @@ -206,7 +208,7 @@ class GPUROIPoolGradOpKernel : public framework::OpKernel { void Compute(const framework::ExecutionContext& ctx) const override { auto* in = ctx.Input("X"); auto* rois = ctx.Input("ROIs"); - auto* rois_lod = ctx.Input("RoisLod"); + auto* rois_lod = ctx.Input("RoisNum"); auto* argmax = ctx.Input("Argmax"); auto* out_grad = ctx.Input(framework::GradVarName("Out")); @@ -229,17 +231,18 @@ class GPUROIPoolGradOpKernel : public framework::OpKernel { auto& dev_ctx = ctx.cuda_device_context(); auto gplace = BOOST_GET_CONST(platform::CUDAPlace, ctx.GetPlace()); - if (ctx.HasInput("RoisLod")) { - auto* rois_lod = ctx.Input("RoisLod"); - int rois_batch_size = rois_lod->numel(); - std::vector rois_lod_(rois_batch_size); - memory::Copy(cplace, rois_lod_.data(), gplace, - rois_lod->data(), - sizeof(int64_t) * rois_batch_size, 0); - for (int n = 0; n < rois_batch_size - 1; ++n) { - for (size_t i = rois_lod_[n]; i < rois_lod_[n + 1]; ++i) { + if (ctx.HasInput("RoisNum")) { + auto* rois_num_t = ctx.Input("RoisNum"); + int rois_batch_size = rois_num_t->numel(); + std::vector rois_num_list(rois_batch_size); + memory::Copy(cplace, rois_num_list.data(), gplace, + rois_num_t->data(), sizeof(int) * rois_batch_size, 0); + int start = 0; + for (int n = 0; n < rois_batch_size; ++n) { + for (int i = start; i < start + rois_num_list[n]; ++i) { roi_batch_id_data[i] = n; } + start += rois_num_list[n]; } } else { auto rois_lod = rois->lod().back(); diff --git a/paddle/fluid/operators/roi_pool_op.h b/paddle/fluid/operators/roi_pool_op.h index 145b170dedf0613328223526b0a40a3c064f3028..40de6d0cf6abbcc4a1505cb6eb121ca70813c780 100644 --- a/paddle/fluid/operators/roi_pool_op.h +++ b/paddle/fluid/operators/roi_pool_op.h @@ -58,18 +58,20 @@ class CPUROIPoolOpKernel : public framework::OpKernel { roi_batch_id_list.mutable_data(ctx.GetPlace()); int rois_batch_size; - if (ctx.HasInput("RoisLod")) { - auto* rois_lod_t = ctx.Input("RoisLod"); - rois_batch_size = rois_lod_t->numel(); + if (ctx.HasInput("RoisNum")) { + auto* rois_num_t = ctx.Input("RoisNum"); + rois_batch_size = rois_num_t->numel(); PADDLE_ENFORCE_EQ( - rois_batch_size - 1, batch_size, + rois_batch_size, batch_size, platform::errors::InvalidArgument("The rois_batch_size and imgs " "batch_size must be the same.")); - auto* rois_lod = rois_lod_t->data(); - for (int n = 0; n < rois_batch_size - 1; ++n) { - for (int i = rois_lod[n]; i < rois_lod[n + 1]; ++i) { + auto* rois_num_data = rois_num_t->data(); + int start = 0; + for (int n = 0; n < rois_batch_size; ++n) { + for (int i = start; i < start + rois_num_data[n]; ++i) { roi_batch_id_data[i] = n; } + start += rois_num_data[n]; } } else { auto rois_lod = rois->lod().back(); @@ -185,14 +187,16 @@ class CPUROIPoolGradOpKernel : public framework::OpKernel { roi_batch_id_list.mutable_data(ctx.GetPlace()); int rois_batch_size; - if (ctx.HasInput("RoisLod")) { - auto* rois_lod_t = ctx.Input("RoisLod"); - rois_batch_size = rois_lod_t->numel(); - auto* rois_lod = rois_lod_t->data(); - for (int n = 0; n < rois_batch_size - 1; ++n) { - for (int i = rois_lod[n]; i < rois_lod[n + 1]; ++i) { + if (ctx.HasInput("RoisNum")) { + auto* rois_num_t = ctx.Input("RoisNum"); + rois_batch_size = rois_num_t->numel(); + auto* rois_num_data = rois_num_t->data(); + int start = 0; + for (int n = 0; n < rois_batch_size; ++n) { + for (int i = start; i < start + rois_num_data[n]; ++i) { roi_batch_id_data[i] = n; } + start += rois_num_data[n]; } } else { auto rois_lod = rois->lod().back(); diff --git a/paddle/fluid/operators/run_program_op.cc b/paddle/fluid/operators/run_program_op.cc index 04559a93c866c72f2d0b309a5005557134355666..2d599716443901053aa3d5dc8e93759320175b24 100644 --- a/paddle/fluid/operators/run_program_op.cc +++ b/paddle/fluid/operators/run_program_op.cc @@ -27,9 +27,6 @@ class RunProgramOp : public framework::OperatorWithKernel { PADDLE_ENFORCE_EQ(ctx->HasInputs("X"), true, platform::errors::NotFound( "Input(X) of RunProgramOp should not be null.")); - PADDLE_ENFORCE_EQ(ctx->HasInputs("Params"), true, - platform::errors::NotFound( - "Input(Params) of RunProgramOp should not be null.")); PADDLE_ENFORCE_EQ(ctx->HasOutputs("Out"), true, platform::errors::NotFound( "Output(Out) of RunProgramOp should not be null.")); @@ -73,7 +70,8 @@ class RunProgramOpMaker : public framework::OpProtoAndCheckerMaker { "(vector)" "The input parameter of RunProgram operator, also the parameters " "of the loaded program.") - .AsDuplicable(); + .AsDuplicable() + .AsDispensable(); AddOutput("Out", "(vector)" "The output tensors of RunProgram operator, also the fetch " @@ -121,10 +119,6 @@ class RunProgramGradOp : public framework::OperatorWithKernel { PADDLE_ENFORCE_EQ(ctx->HasInputs("X"), true, platform::errors::NotFound( "Input(X) of RunProgramGradOp should not be null.")); - PADDLE_ENFORCE_EQ( - ctx->HasInputs("Params"), true, - platform::errors::NotFound( - "Input(Params) of RunProgramGradOp should not be null.")); PADDLE_ENFORCE_EQ( ctx->HasInputs(framework::GradVarName("Out")), true, platform::errors::NotFound( diff --git a/paddle/fluid/operators/run_program_op.h b/paddle/fluid/operators/run_program_op.h index 1c493fc6be093a2af8f58c8e78d1be43de34306f..5afe25cf687fc96d1eaac33b2d0516c96c394a46 100644 --- a/paddle/fluid/operators/run_program_op.h +++ b/paddle/fluid/operators/run_program_op.h @@ -209,9 +209,14 @@ class RunProgramOpKernel : public framework::OpKernel { auto output_vars = ctx.MultiOutputVar("Out"); auto input_var_names = ctx.InputNames("X"); - auto param_names = ctx.InputNames("Params"); auto output_var_names = ctx.OutputNames("Out"); + // current program may not hold parameters + std::vector param_names; + if (!param_vars.empty()) { + param_names = ctx.InputNames("Params"); + } + auto *block = ctx.Attr("global_block"); auto *program = block->Program(); auto start_op_index = ctx.Attr("start_op_index"); diff --git a/paddle/fluid/operators/sequence_ops/sequence_expand_op.h b/paddle/fluid/operators/sequence_ops/sequence_expand_op.h index 013170199a6bbe0246406b9c35a326bd063875a9..1186ed891e8c080c023aae5076cf1cb086fbc231 100644 --- a/paddle/fluid/operators/sequence_ops/sequence_expand_op.h +++ b/paddle/fluid/operators/sequence_ops/sequence_expand_op.h @@ -92,9 +92,11 @@ class SequenceExpandKernel : public framework::OpKernel { auto& x_lod = x->lod(); auto& y_lod = y->lod(); - PADDLE_ENFORCE_EQ(y_lod.empty(), false, - "Input(Y) Tensor of SequenceExpandOp does not contain " - "LoD information."); + PADDLE_ENFORCE_EQ( + y_lod.empty(), false, + platform::errors::InvalidArgument( + "Input(Y) Tensor of SequenceExpandOp does not contain " + "LoD information.")); if (ref_level == -1) ref_level = y_lod.size() - 1; diff --git a/paddle/fluid/operators/size_op.cc b/paddle/fluid/operators/size_op.cc index b45fa7c791ff22be422ce12a8348a071c60ddd0f..70733d643673ad8acde9a45f273a52a9723fb0d3 100644 --- a/paddle/fluid/operators/size_op.cc +++ b/paddle/fluid/operators/size_op.cc @@ -53,7 +53,7 @@ REGISTER_OPERATOR( size, ops::SizeOp, ops::SizeOpMaker, paddle::framework::EmptyGradOpMaker, paddle::framework::EmptyGradOpMaker); -REGISTER_OP_CPU_KERNEL(size, ops::SizeKernel, ops::SizeKernel, +REGISTER_OP_CPU_KERNEL(size, ops::SizeKernel, ops::SizeKernel, ops::SizeKernel, ops::SizeKernel, ops::SizeKernel, ops::SizeKernel); diff --git a/paddle/fluid/operators/size_op.cu b/paddle/fluid/operators/size_op.cu index 3ea3032693236d5618ff6f0c858cbd85e34633ab..de56ecd95270577689f699462b9273b43f34595e 100644 --- a/paddle/fluid/operators/size_op.cu +++ b/paddle/fluid/operators/size_op.cu @@ -16,7 +16,7 @@ limitations under the License. */ REGISTER_OP_CUDA_KERNEL( size, paddle::operators::SizeKernel, - paddle::operators::SizeKernel, + paddle::operators::SizeKernel, paddle::operators::SizeKernel, paddle::operators::SizeKernel, paddle::operators::SizeKernel, paddle::operators::SizeKernel); diff --git a/paddle/fluid/operators/size_op.h b/paddle/fluid/operators/size_op.h index fb44070897156ef88062231322e28a2db1f244a7..e8c53d6e683305bfc1ff7c052a2dc54ecf465936 100644 --- a/paddle/fluid/operators/size_op.h +++ b/paddle/fluid/operators/size_op.h @@ -26,8 +26,18 @@ class SizeKernel : public framework::OpKernel { void Compute(const framework::ExecutionContext& ctx) const override { auto* in_t = ctx.Input("Input"); auto* out_t = ctx.Output("Out"); - auto out_data = out_t->mutable_data(platform::CPUPlace()); - out_data[0] = in_t->numel(); + auto place = ctx.GetPlace(); + auto out_data = out_t->mutable_data(place); + auto cpu_place = platform::CPUPlace(); + if (place == cpu_place) { + out_data[0] = in_t->numel(); + } else { + Tensor cpu_tensor; + auto cpu_data = + cpu_tensor.mutable_data(out_t->dims(), cpu_place); + cpu_data[0] = in_t->numel(); + TensorCopy(cpu_tensor, place, out_t); + } } }; } // namespace operators diff --git a/paddle/fluid/operators/tensorrt/tensorrt_engine_op.h b/paddle/fluid/operators/tensorrt/tensorrt_engine_op.h index cc6ee7b19ea99fe61ef00beaf475fc35c8a0a809..9cfe47da5db7ba15c9b24a8d551606f805ad9b15 100644 --- a/paddle/fluid/operators/tensorrt/tensorrt_engine_op.h +++ b/paddle/fluid/operators/tensorrt/tensorrt_engine_op.h @@ -208,8 +208,11 @@ class TensorRTEngineOp : public framework::OperatorBase { auto stream = reinterpret_cast(dev_ctx).stream(); - PADDLE_ENFORCE_EQ(input_names_.empty(), false, - "should pass at least one input"); + PADDLE_ENFORCE_EQ( + input_names_.empty(), false, + platform::errors::PreconditionNotMet( + "TensorRT engine needs at least one input, but no input is found. " + "Please check if you set the input correctly.")); std::vector output_maps = Attr>("output_name_mapping"); @@ -295,12 +298,19 @@ class TensorRTEngineOp : public framework::OperatorBase { #endif } auto *fluid_v = scope.FindVar(y); - PADDLE_ENFORCE_NOT_NULL(fluid_v, "no output variable called %s", y); + PADDLE_ENFORCE_NOT_NULL( + fluid_v, + platform::errors::NotFound( + "Output variable %s is not found in TensorRT subgraph.", y)); auto *fluid_t = fluid_v->GetMutable(); fluid_t->Resize(framework::make_ddim(ddim)); - PADDLE_ENFORCE(bind_index < num_bindings, - "The bind index should be less than num_bindings"); + PADDLE_ENFORCE_LT(bind_index, num_bindings, + platform::errors::InvalidArgument( + "The binding index in TRT engine should be less " + "than the number of bindings, but got binding " + "index = %d, number of bindings = %d.", + bind_index, num_bindings)); buffers[bind_index] = static_cast(fluid_t->mutable_data( BOOST_GET_CONST(platform::CUDAPlace, dev_place))); diff --git a/paddle/fluid/operators/tile_op.cc b/paddle/fluid/operators/tile_op.cc index da4ca87296d92fc1052f462ae6ee8a3acb05eb49..bc1cb3b4aa1c1bdd0a9be39a4e113301d65ce5b5 100644 --- a/paddle/fluid/operators/tile_op.cc +++ b/paddle/fluid/operators/tile_op.cc @@ -241,6 +241,26 @@ class TileGradOpMaker : public framework::SingleGradOpMaker { } }; +template +class TileDoubleGradOpMaker : public framework::SingleGradOpMaker { + public: + using framework::SingleGradOpMaker::SingleGradOpMaker; + + protected: + void Apply(GradOpPtr op) const override { + op->SetType("tile"); + op->SetInput("X", this->OutputGrad(framework::GradVarName("X"))); + op->SetOutput("Out", this->InputGrad(framework::GradVarName("Out"))); + if (this->HasInput("repeat_times_tensor")) { + op->SetInput("repeat_times_tensor", this->Input("repeat_times_tensor")); + } + if (this->HasInput("RepeatTimes")) { + op->SetInput("RepeatTimes", this->Input("RepeatTimes")); + } + op->SetAttrMap(this->Attrs()); + } +}; + DECLARE_NO_NEED_BUFFER_VARS_INFERER(TileGradNoNeedBufVarsInferer, "X"); } // namespace operators @@ -251,6 +271,8 @@ REGISTER_OPERATOR(tile, ops::TileOp, ops::TileOpMaker, ops::TileGradOpMaker, ops::TileGradOpMaker); REGISTER_OPERATOR(tile_grad, ops::TileGradOp, + ops::TileDoubleGradOpMaker, + ops::TileDoubleGradOpMaker, ops::TileGradNoNeedBufVarsInferer); REGISTER_OP_CPU_KERNEL( tile, ops::TileKernel, diff --git a/paddle/fluid/operators/utils.h b/paddle/fluid/operators/utils.h index e53981a53653a4830a39ceae47f4024bb757b039..aec995304a77118ecbf788ca3984c7e9da531f18 100644 --- a/paddle/fluid/operators/utils.h +++ b/paddle/fluid/operators/utils.h @@ -81,5 +81,26 @@ inline std::vector GetDataFromTensorList( } return vec_new_data; } + +inline framework::DDim GetShape(const framework::ExecutionContext& ctx) { + // 1. shape is a Tensor + if (ctx.HasInput("ShapeTensor")) { + auto* shape_tensor = ctx.Input("ShapeTensor"); + auto vec_shape = GetDataFromTensor(shape_tensor); + return framework::make_ddim(vec_shape); + } + + // 2. shape is a list/tuple containing Tensor + auto shape_tensor_list = ctx.MultiInput("ShapeTensorList"); + if (shape_tensor_list.size() > 0) { + auto vec_shape = GetDataFromTensorList(shape_tensor_list); + return framework::make_ddim(vec_shape); + } + + // 3. shape is a list/tuple without containing Tensor + auto vec_shape = ctx.Attr>("shape"); + return framework::make_ddim(vec_shape); +} + } // namespace operators } // namespace paddle diff --git a/paddle/fluid/pybind/inference_api.cc b/paddle/fluid/pybind/inference_api.cc index 040dd313f1c538b5792538f9da04635ff805b9a8..be4d90597e1e1c647ac6750ee7cebdc2ede8a551 100644 --- a/paddle/fluid/pybind/inference_api.cc +++ b/paddle/fluid/pybind/inference_api.cc @@ -60,6 +60,9 @@ void BindAnalysisConfig(py::module *m); void BindAnalysisPredictor(py::module *m); void BindZeroCopyTensor(py::module *m); void BindPaddlePassBuilder(py::module *m); +void BindPaddleInferPredictor(py::module *m); +void BindPaddleInferTensor(py::module *m); +void BindPredictorPool(py::module *m); #ifdef PADDLE_WITH_MKLDNN void BindMkldnnQuantizerConfig(py::module *m); @@ -139,6 +142,15 @@ void ZeroCopyTensorCreate(ZeroCopyTensor &tensor, // NOLINT tensor.copy_from_cpu(static_cast(data.data())); } +template +void PaddleInferTensorCreate(paddle_infer::Tensor &tensor, // NOLINT + py::array_t data) { + std::vector shape; + std::copy_n(data.shape(), data.ndim(), std::back_inserter(shape)); + tensor.Reshape(std::move(shape)); + tensor.CopyFromCpu(static_cast(data.data())); +} + size_t PaddleGetDTypeSize(PaddleDType dt) { size_t size{0}; switch (dt) { @@ -183,6 +195,30 @@ py::array ZeroCopyTensorToNumpy(ZeroCopyTensor &tensor) { // NOLINT return array; } +py::array PaddleInferTensorToNumpy(paddle_infer::Tensor &tensor) { // NOLINT + py::dtype dt = PaddleDTypeToNumpyDType(tensor.type()); + auto tensor_shape = tensor.shape(); + py::array::ShapeContainer shape(tensor_shape.begin(), tensor_shape.end()); + py::array array(dt, std::move(shape)); + + switch (tensor.type()) { + case PaddleDType::INT32: + tensor.CopyToCpu(static_cast(array.mutable_data())); + break; + case PaddleDType::INT64: + tensor.CopyToCpu(static_cast(array.mutable_data())); + break; + case PaddleDType::FLOAT32: + tensor.CopyToCpu(static_cast(array.mutable_data())); + break; + default: + PADDLE_THROW(platform::errors::Unimplemented( + "Unsupported data type. Now only supports INT32, INT64 and " + "FLOAT32.")); + } + return array; +} + py::bytes SerializePDTensorToBytes(PaddleTensor &tensor) { // NOLINT std::stringstream ss; paddle::inference::SerializePDTensorToStream(&ss, tensor); @@ -200,8 +236,11 @@ void BindInferenceApi(py::module *m) { BindNativePredictor(m); BindAnalysisConfig(m); BindAnalysisPredictor(m); + BindPaddleInferPredictor(m); BindZeroCopyTensor(m); + BindPaddleInferTensor(m); BindPaddlePassBuilder(m); + BindPredictorPool(m); #ifdef PADDLE_WITH_MKLDNN BindMkldnnQuantizerConfig(m); #endif @@ -209,8 +248,17 @@ void BindInferenceApi(py::module *m) { &paddle::CreatePaddlePredictor, py::arg("config")); m->def("create_paddle_predictor", &paddle::CreatePaddlePredictor, py::arg("config")); + m->def("create_predictor", [](const paddle_infer::Config &config) + -> std::unique_ptr { + auto pred = + std::unique_ptr( + new paddle_infer::Predictor(config)); + return std::move(pred); + }); m->def("paddle_dtype_size", &paddle::PaddleDtypeSize); m->def("paddle_tensor_to_bytes", &SerializePDTensorToBytes); + m->def("get_version", &paddle_infer::GetVersion); + m->def("get_num_bytes_of_data_type", &paddle_infer::GetNumBytesOfDataType); } namespace { @@ -525,6 +573,19 @@ void BindAnalysisPredictor(py::module *m) { py::arg("dir")); } +void BindPaddleInferPredictor(py::module *m) { + py::class_(*m, "PaddleInferPredictor") + .def(py::init()) + .def("get_input_names", &paddle_infer::Predictor::GetInputNames) + .def("get_output_names", &paddle_infer::Predictor::GetOutputNames) + .def("get_input_handle", &paddle_infer::Predictor::GetInputHandle) + .def("get_output_handle", &paddle_infer::Predictor::GetOutputHandle) + .def("run", &paddle_infer::Predictor::Run) + .def("clone", &paddle_infer::Predictor::Clone) + .def("clear_intermediate_tensor", + &paddle_infer::Predictor::ClearIntermediateTensor); +} + void BindZeroCopyTensor(py::module *m) { py::class_(*m, "ZeroCopyTensor") .def("reshape", &ZeroCopyTensor::Reshape) @@ -538,6 +599,26 @@ void BindZeroCopyTensor(py::module *m) { .def("type", &ZeroCopyTensor::type); } +void BindPaddleInferTensor(py::module *m) { + py::class_(*m, "PaddleInferTensor") + .def("reshape", &paddle_infer::Tensor::Reshape) + .def("copy_from_cpu", &PaddleInferTensorCreate) + .def("copy_from_cpu", &PaddleInferTensorCreate) + .def("copy_from_cpu", &PaddleInferTensorCreate) + .def("copy_to_cpu", &PaddleInferTensorToNumpy) + .def("shape", &paddle_infer::Tensor::shape) + .def("set_lod", &paddle_infer::Tensor::SetLoD) + .def("lod", &paddle_infer::Tensor::lod) + .def("type", &paddle_infer::Tensor::type); +} + +void BindPredictorPool(py::module *m) { + py::class_(*m, "PredictorPool") + .def(py::init()) + .def("retrive", &paddle_infer::services::PredictorPool::Retrive, + py::return_value_policy::reference); +} + void BindPaddlePassBuilder(py::module *m) { py::class_(*m, "PaddlePassBuilder") .def(py::init &>()) diff --git a/paddle/fluid/pybind/op_function_generator.cc b/paddle/fluid/pybind/op_function_generator.cc index 256faf04ea6de5835f22113537caac49ca1dbab4..f751136640caad6acd3230bc22cd0e3f0fafe9fb 100644 --- a/paddle/fluid/pybind/op_function_generator.cc +++ b/paddle/fluid/pybind/op_function_generator.cc @@ -43,6 +43,11 @@ std::map> op_ins_map = { {"nll_loss", {"X", "Label", "Weight"}}, {"bilinear_tensor_product", {"X", "Y", "Weight", "Bias"}}, {"gather", {"X", "Index", "Axis"}}, + {"roi_pool", {"X", "ROIs", "RoisNum"}}, + {"roi_align", {"X", "ROIs", "RoisNum"}}, + {"collect_fpn_proposals", + {"MultiLevelRois", "MultiLevelScores", "MultiLevelRoIsNum"}}, + {"distribute_fpn_proposals", {"FpnRois", "RoisNum"}}, }; // NOTE(zhiqiu): Like op_ins_map. @@ -63,6 +68,10 @@ std::map> op_outs_map = { {"Y", "MeanOut", "VarianceOut", "SavedMean", "SavedVariance", "ReserveSpace"}}, {"unique", {"Out", "Index", "Indices", "Counts"}}, + {"generate_proposals", {"RpnRois", "RpnRoiProbs", "RpnRoisNum"}}, + {"collect_fpn_proposals", {"FpnRois", "RoisNum"}}, + {"distribute_fpn_proposals", + {"MultiFpnRois", "RestoreIndex", "MultiLevelRoIsNum"}}, }; // NOTE(zhiqiu): Commonly, the outputs in auto-generated OP function are @@ -102,7 +111,9 @@ std::map> op_passing_outs_map = { {"fake_quantize_dequantize_moving_average_abs_max", {"Out", "OutScale", "OutAccum", "OutState"}}, {"fake_quantize_dequantize_abs_max", {"Out", "OutScale"}}, - {"amp_check_finite_and_scale", {"Out", "FoundInfinite"}}, + {"check_finite_and_unscale", {"Out", "FoundInfinite"}}, + {"update_loss_scaling", + {"Out", "LossScaling", "OutGoodSteps", "OutBadSteps"}}, }; // clang-format off diff --git a/paddle/fluid/train/CMakeLists.txt b/paddle/fluid/train/CMakeLists.txt index 7eab677fac1683fdc95c9e338b1099d78b5cabc3..235d92ac4f9e88947cea04425b0916b8a0290979 100644 --- a/paddle/fluid/train/CMakeLists.txt +++ b/paddle/fluid/train/CMakeLists.txt @@ -27,8 +27,6 @@ function(train_test TARGET_NAME) endif() set_tests_properties(test_train_${TARGET_NAME}${arg} PROPERTIES DEPENDS test_${TARGET_NAME}) - set_tests_properties(test_train_${TARGET_NAME}${arg} - PROPERTIES LABELS "RUN_TYPE=DIST") if(NOT WIN32 AND NOT APPLE) set_tests_properties(test_train_${TARGET_NAME}${arg} PROPERTIES TIMEOUT 150) diff --git a/paddle/scripts/paddle_build.bat b/paddle/scripts/paddle_build.bat index 27edf1f677aec452995d81ebcf8e8533b5343ce9..15610abef0f2d07eeb02e37bb0d4cbf394c94d90 100644 --- a/paddle/scripts/paddle_build.bat +++ b/paddle/scripts/paddle_build.bat @@ -111,6 +111,8 @@ goto:success :CASE_wincheck_openblas set WITH_MKL=OFF set WITH_GPU=ON +rem Temporarily turn off WITH_INFERENCE_API_TEST on GPU due to compile hang +set WITH_INFERENCE_API_TEST=OFF call :cmake || goto cmake_error call :build || goto build_error call :test_whl_pacakage || goto test_whl_pacakage_error @@ -152,6 +154,7 @@ echo Step 2. Buile Paddle ... echo ======================================== call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" amd64 +for /F %%# in ('wmic cpu get NumberOfLogicalProcessors^|findstr [0-9]') do set /a PARALLEL_PROJECT_COUNT=%%#*8/10 set build_times=1 :build_tp echo Build third_party the %build_times% time: @@ -170,7 +173,7 @@ echo Build third_party successfully! set build_times=1 :build_paddle echo Build Paddle the %build_times% time: -msbuild /m /p:Configuration=Release /verbosity:minimal paddle.sln +msbuild /m:%PARALLEL_PROJECT_COUNT% /p:Configuration=Release /verbosity:minimal paddle.sln if %ERRORLEVEL% NEQ 0 ( set /a build_times=%build_times%+1 if %build_times% GTR 2 ( @@ -242,7 +245,7 @@ dir %THIRD_PARTY_PATH:/=\%\install\mkldnn\bin dir %THIRD_PARTY_PATH:/=\%\install\warpctc\bin set PATH=%THIRD_PARTY_PATH:/=\%\install\openblas\lib;%THIRD_PARTY_PATH:/=\%\install\openblas\bin;%THIRD_PARTY_PATH:/=\%\install\zlib\bin;%THIRD_PARTY_PATH:/=\%\install\mklml\lib;%THIRD_PARTY_PATH:/=\%\install\mkldnn\bin;%THIRD_PARTY_PATH:/=\%\install\warpctc\bin;%PATH% -ctest.exe --output-on-failure -C Release -j 8 +ctest.exe --output-on-failure -C Release -j 8 --repeat until-pass:4 after-timeout:4 goto:eof :unit_test_error diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index 9c1c95f37ed0785bfd770e7cbc02002daba8447b..ec07565c5af6c7ba79c15d9a335313775719c682 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -296,13 +296,13 @@ function check_style() { commit_files=on for file_name in `git diff --numstat upstream/$BRANCH |awk '{print $NF}'`;do if ! pre-commit run --files $file_name ; then - git diff commit_files=off fi done if [ $commit_files == 'off' ];then echo "code format error" + git diff 2>&1 exit 4 fi trap : 0 @@ -528,9 +528,50 @@ EOF elif [ "$1" == "cp37-cp37m" ]; then pip3.7 install --user ${INSTALL_PREFIX:-/paddle/build}/opt/paddle/share/wheels/*.whl fi + tmpfile_rand=`date +%s%N` + tmpfile=$tmp_dir/$tmpfile_rand set +e ut_startTime_s=`date +%s` - ctest --output-on-failure -j $2;mactest_error=$? + ctest --output-on-failure -j $2 | tee $tmpfile + failed_test_lists='' + collect_failed_tests + set +x + mactest_error=0 + retry_unittests_record='' + retry_time=3 + exec_times=0 + exec_time_array=('first' 'second' 'third') + if [ -n "$failed_test_lists" ];then + mactest_error=1 + while ( [ $exec_times -lt $retry_time ] && [ -n "${failed_test_lists}" ] ) + do + retry_unittests_record="$retry_unittests_record$failed_test_lists" + failed_test_lists_ult=`echo "${failed_test_lists}"` + read retry_unittests <<< $(echo "$failed_test_lists" | grep -oEi "\-.+\(" | sed 's/(//' | sed 's/- //' ) + echo "=========================================" + echo "This is the ${exec_time_array[$exec_times]} time to re-run" + echo "=========================================" + echo "The following unittest will be re-run:" + echo "${retry_unittests}" + echo "=========================================" + + retry_unittests_regular='' + for line in ${retry_unittests[@]} ; + do + if [[ "$retry_unittests_regular" == "" ]];then + retry_unittests_regular="^$line$" + else + retry_unittests_regular="$retry_unittests_regular|^$line$" + fi + done + rm -f $tmp_dir/* + failed_test_lists='' + ctest -R "($retry_unittests_regular)" --output-on-failure -j $2 | tee $tmpfile + collect_failed_tests + exec_times=$[$exec_times+1] + done + fi + #mactest_error=$? ut_endTime_s=`date +%s` echo "Mac testCase Time: $[ $ut_endTime_s - $ut_startTime_s ]s" paddle version @@ -538,7 +579,21 @@ EOF export http_proxy=$my_proxy export https_proxy=$my_proxy if [ "$mactest_error" != 0 ];then - exit 8; + if [[ "$failed_test_lists" == "" ]]; then + echo "========================================" + echo "There are failed tests, which have been successful after re-run:" + echo "========================================" + echo "The following tests have been re-ran:" + echo "${retry_unittests_record}" + else + failed_test_lists_ult=`echo "${failed_test_lists}"` + echo "========================================" + echo "Summary Failed Tests... " + echo "========================================" + echo "The following tests FAILED: " + echo "${failed_test_lists_ult}" + exit 8; + fi fi fi } @@ -562,6 +617,7 @@ function fetch_upstream_develop_if_not_exist() { function generate_upstream_develop_api_spec() { fetch_upstream_develop_if_not_exist cur_branch=`git branch | grep \* | cut -d ' ' -f2` + git checkout . git checkout -b develop_base_pr upstream/$BRANCH cmake_gen $1 build $2 @@ -1391,7 +1447,7 @@ function example() { cd ${PADDLE_ROOT}/tools python sampcd_processor.py cpu;example_error=$? if [ "$example_error" != "0" ];then - echo "Code instance execution failed" + echo "Code instance execution failed" >&2 exit 5 fi } @@ -1400,15 +1456,25 @@ function summary_check_problems() { set +x local check_style_code=$1 local example_code=$2 + local check_style_info=$3 + local example_info=$4 if [ $check_style_code -ne 0 -o $example_code -ne 0 ];then echo "========================================" echo "summary problems:" + if [ $check_style_code -ne 0 -a $example_code -ne 0 ];then + echo "There are 2 errors: Code format error and Example code error." + else + [ $check_style_code -ne 0 ] && echo "There is 1 error: Code format error." + [ $example_code -ne 0 ] && echo "There is 1 error: Example code error." + fi echo "========================================" if [ $check_style_code -ne 0 ];then - echo "- Check code style failed! Please check the log and fix problems." + echo "*****Code format error***** Please fix it according to the diff information:" + echo "$check_style_info" | grep "code format error" -A $(echo "$check_style_info" | wc -l) fi if [ $example_code -ne 0 ];then - echo "- Check example code failed! Please check the log and fix problems." + echo "*****Example code error***** Please fix the error listed in the information:" + echo "$example_info" | grep "API check -- Example Code" -A $(echo "$example_info" | wc -l) fi [ $check_style_code -ne 0 ] && exit $check_style_code [ $example_code -ne 0 ] && exit $example_code @@ -1430,15 +1496,16 @@ function main() { ;; build_and_check) set +e - $(check_style >&2) + check_style_info=$(check_style) check_style_code=$? generate_upstream_develop_api_spec ${PYTHON_ABI:-""} ${parallel_number} cmake_gen_and_build ${PYTHON_ABI:-""} ${parallel_number} check_sequence_op_unittest generate_api_spec ${PYTHON_ABI:-""} "PR" - $(example >&2) + set +e + example_info=$(example) example_code=$? - summary_check_problems $check_style_code $example_code + summary_check_problems $check_style_code $example_code "$check_style_info" "$example_info" assert_api_spec_approvals ;; build) diff --git a/python/paddle/__init__.py b/python/paddle/__init__.py index d5793eb424ab794e3e8af8ef2312aac927c272e5..ed0b415d0bfd86b5160d339a286cfddac37cf4df 100755 --- a/python/paddle/__init__.py +++ b/python/paddle/__init__.py @@ -75,6 +75,7 @@ from .tensor.creation import full_like #DEFINE_ALIAS from .tensor.creation import triu #DEFINE_ALIAS from .tensor.creation import tril #DEFINE_ALIAS from .tensor.creation import meshgrid #DEFINE_ALIAS +from .tensor.creation import empty #DEFINE_ALIAS from .tensor.linalg import matmul #DEFINE_ALIAS from .tensor.linalg import dot #DEFINE_ALIAS # from .tensor.linalg import einsum #DEFINE_ALIAS diff --git a/python/paddle/dataset/uci_housing.py b/python/paddle/dataset/uci_housing.py index 5bc9c1444d2b34f057cd92782eb50e5fc23916eb..f7930d34f93e21bf3f832da828fb0036742b5091 100644 --- a/python/paddle/dataset/uci_housing.py +++ b/python/paddle/dataset/uci_housing.py @@ -74,7 +74,8 @@ def load_data(filename, feature_num=14, ratio=0.8): data = data.reshape(data.shape[0] // feature_num, feature_num) maximums, minimums, avgs = data.max(axis=0), data.min(axis=0), data.sum( axis=0) / data.shape[0] - feature_range(maximums[:-1], minimums[:-1]) + # if you want to print the distribution of input data, you could use function of feature_range + #feature_range(maximums[:-1], minimums[:-1]) for i in six.moves.range(feature_num - 1): data[:, i] = (data[:, i] - avgs[i]) / (maximums[i] - minimums[i]) offset = int(data.shape[0] * ratio) diff --git a/python/paddle/distributed/fleet/base/distributed_strategy.py b/python/paddle/distributed/fleet/base/distributed_strategy.py index 9c1793fd5b56eb728ae7d16840cf4fb09cf975c8..1b86056c00443be4170757cee3cc60bbafd0f40b 100755 --- a/python/paddle/distributed/fleet/base/distributed_strategy.py +++ b/python/paddle/distributed/fleet/base/distributed_strategy.py @@ -15,10 +15,26 @@ import paddle from paddle.distributed.fleet.proto import distributed_strategy_pb2 from paddle.fluid.framework import Variable, set_flags, core +from paddle.fluid.wrapped_decorator import wrap_decorator import google.protobuf.text_format +import google.protobuf __all__ = ["DistributedStrategy"] +non_auto_func_called = True + + +def __non_auto_func_called__(func): + def __impl__(*args, **kwargs): + global non_auto_func_called + non_auto_func_called = False + return func(*args, **kwargs) + + return __impl__ + + +is_strict_auto = wrap_decorator(__non_auto_func_called__) + def get_msg_dict(msg): res_dict = {} @@ -164,6 +180,7 @@ class DistributedStrategy(object): return execution_strategy @execution_strategy.setter + @is_strict_auto def execution_strategy(self, strategy): fields = self.strategy.execution_strategy.DESCRIPTOR.fields for f in fields: @@ -203,6 +220,7 @@ class DistributedStrategy(object): return build_strategy @build_strategy.setter + @is_strict_auto def build_strategy(self, strategy): fields = self.strategy.build_strategy.DESCRIPTOR.fields for f in fields: @@ -237,6 +255,7 @@ class DistributedStrategy(object): return self.strategy.a_sync @a_sync.setter + @is_strict_auto def a_sync(self, flag): if isinstance(flag, bool): self.strategy.a_sync = flag @@ -287,6 +306,7 @@ class DistributedStrategy(object): return get_msg_dict(self.strategy.a_sync_configs) @a_sync_configs.setter + @is_strict_auto def a_sync_configs(self, configs): check_configs_key(self.strategy.a_sync_configs, configs, "a_sync_configs") @@ -309,6 +329,7 @@ class DistributedStrategy(object): return self.strategy.amp @amp.setter + @is_strict_auto def amp(self, flag): if isinstance(flag, bool): self.strategy.amp = flag @@ -351,6 +372,7 @@ class DistributedStrategy(object): return get_msg_dict(self.strategy.amp_configs) @amp_configs.setter + @is_strict_auto def amp_configs(self, configs): check_configs_key(self.strategy.amp_configs, configs, "amp_configs") assign_configs_value(self.strategy.amp_configs, configs) @@ -388,6 +410,7 @@ class DistributedStrategy(object): return self.strategy.sync_nccl_allreduce @sync_nccl_allreduce.setter + @is_strict_auto def sync_nccl_allreduce(self, flag): if isinstance(flag, bool): self.strategy.sync_nccl_allreduce = flag @@ -411,6 +434,7 @@ class DistributedStrategy(object): return self.strategy.use_hierarchical_allreduce @use_hierarchical_allreduce.setter + @is_strict_auto def use_hierarchical_allreduce(self, flag): if isinstance(flag, bool): self.strategy.use_hierarchical_allreduce = flag @@ -435,6 +459,7 @@ class DistributedStrategy(object): return self.strategy.hierarchical_allreduce_inter_nranks @hierarchical_allreduce_inter_nranks.setter + @is_strict_auto def hierarchical_allreduce_inter_nranks(self, value): if isinstance(value, int): self.strategy.hierarchical_allreduce_inter_nranks = value @@ -461,6 +486,7 @@ class DistributedStrategy(object): return self.strategy.sync_batch_norm @sync_batch_norm.setter + @is_strict_auto def sync_batch_norm(self, flag): if isinstance(flag, bool): self.strategy.sync_batch_norm = flag @@ -483,6 +509,7 @@ class DistributedStrategy(object): return self.strategy.fuse_all_reduce_ops @fuse_all_reduce_ops.setter + @is_strict_auto def fuse_all_reduce_ops(self, flag): if isinstance(flag, bool): self.strategy.fuse_all_reduce_ops = flag @@ -506,6 +533,7 @@ class DistributedStrategy(object): return self.strategy.fuse_grad_size_in_MB @fuse_grad_size_in_MB.setter + @is_strict_auto def fuse_grad_size_in_MB(self, value): if isinstance(value, int): self.strategy.fuse_grad_size_in_MB = value @@ -517,6 +545,7 @@ class DistributedStrategy(object): return self.strategy.fuse_grad_size_in_TFLOPS @_fuse_grad_size_in_TFLOPS.setter + @is_strict_auto def _fuse_grad_size_in_TFLOPS(self, value): if isinstance(value, float): self.strategy.fuse_grad_size_in_TFLOPS = value @@ -543,6 +572,7 @@ class DistributedStrategy(object): return self.strategy.nccl_comm_num @nccl_comm_num.setter + @is_strict_auto def nccl_comm_num(self, value): if isinstance(value, int): self.strategy.nccl_comm_num = value @@ -550,6 +580,7 @@ class DistributedStrategy(object): print("WARNING: nccl_comm_num should have value of int type") @recompute.setter + @is_strict_auto def recompute(self, flag): if isinstance(flag, bool): self.strategy.recompute = flag @@ -574,6 +605,7 @@ class DistributedStrategy(object): return get_msg_dict(self.strategy.recompute_configs) @recompute_configs.setter + @is_strict_auto def recompute_configs(self, configs): check_configs_key(self.strategy.recompute_configs, configs, "checkpoint_configs") @@ -598,6 +630,7 @@ class DistributedStrategy(object): return self.strategy.pipeline @pipeline.setter + @is_strict_auto def pipeline(self, flag): if isinstance(flag, bool): self.strategy.pipeline = flag @@ -634,6 +667,7 @@ class DistributedStrategy(object): return get_msg_dict(self.strategy.pipeline_configs) @pipeline_configs.setter + @is_strict_auto def pipeline_configs(self, configs): check_configs_key(self.strategy.pipeline_configs, configs, "pipeline_configs") @@ -658,6 +692,7 @@ class DistributedStrategy(object): return self.strategy.localsgd @localsgd.setter + @is_strict_auto def localsgd(self, flag): if isinstance(flag, bool): self.strategy.localsgd = flag @@ -672,11 +707,7 @@ class DistributedStrategy(object): **Notes**: k_steps(int) The local steps for training before parameter synchronization. Default 1. - - If strategy.auto is set True, the local steps will be calculated automatically during training. - The algorithm is referenced in this paper: - `Adaptive Communication Strategies to Achieve the Best Error-Runtime Trade-off in Local-Update SGD `_. - In this case, k_steps indicates the first local steps which is suggested setting to 1. + begin_step(int) The step of begining training by localsgd. Default 1. Examples: .. code-block:: python @@ -684,12 +715,14 @@ class DistributedStrategy(object): import paddle.distributed.fleet as fleet strategy = fleet.DistributedStrategy() strategy.localsgd = True - strategy.localsgd_configs = {"k_steps": 4} + strategy.localsgd_configs = {"k_steps": 4, + "begin_step": 30} """ return get_msg_dict(self.strategy.localsgd_configs) @localsgd_configs.setter + @is_strict_auto def localsgd_configs(self, configs): check_configs_key(self.strategy.localsgd_configs, configs, "localsgd_configs") @@ -714,6 +747,7 @@ class DistributedStrategy(object): return self.strategy.dgc @dgc.setter + @is_strict_auto def dgc(self, flag): if isinstance(flag, bool): self.strategy.dgc = flag @@ -749,6 +783,7 @@ class DistributedStrategy(object): return get_msg_dict(self.strategy.dgc_configs) @dgc_configs.setter + @is_strict_auto def dgc_configs(self, configs): check_configs_key(self.strategy.dgc_configs, configs, "dgc_configs") assign_configs_value(self.strategy.dgc_configs, configs) @@ -776,6 +811,7 @@ class DistributedStrategy(object): return self.strategy.gradient_merge @gradient_merge.setter + @is_strict_auto def gradient_merge(self, flag): if isinstance(flag, bool): self.strategy.gradient_merge = flag @@ -803,6 +839,7 @@ class DistributedStrategy(object): return get_msg_dict(self.strategy.gradient_merge_configs) @gradient_merge_configs.setter + @is_strict_auto def gradient_merge_configs(self, configs): check_configs_key(self.strategy.gradient_merge_configs, configs, "gradient_configs") @@ -827,6 +864,7 @@ class DistributedStrategy(object): return self.strategy.lars @lars.setter + @is_strict_auto def lars(self, flag): if isinstance(flag, bool): self.strategy.lars = flag @@ -862,6 +900,7 @@ class DistributedStrategy(object): return get_msg_dict(self.strategy.lars_configs) @lars_configs.setter + @is_strict_auto def lars_configs(self, configs): check_configs_key(self.strategy.lars_configs, configs, "lars_configs") assign_configs_value(self.strategy.lars_configs, configs) @@ -887,6 +926,7 @@ class DistributedStrategy(object): return self.strategy.lamb @lamb.setter + @is_strict_auto def lamb(self, flag): if isinstance(flag, bool): self.strategy.lamb = flag @@ -917,15 +957,21 @@ class DistributedStrategy(object): return get_msg_dict(self.strategy.lamb_configs) @lamb_configs.setter + @is_strict_auto def lamb_configs(self, configs): check_configs_key(self.strategy.lamb_configs, configs, "lamb_configs") assign_configs_value(self.strategy.lamb_configs, configs) @property def elastic(self): + """ + Indicating whether we want to do current distributed training on clusters with elastic resources. + Currently, this is configuration is not valid. + """ return self.strategy.elastic @elastic.setter + @is_strict_auto def elastic(self, flag): if isinstance(flag, bool): self.strategy.elastic = flag @@ -934,6 +980,25 @@ class DistributedStrategy(object): @property def auto(self): + """ + Indicating whether we are using auto-parallel configuration + This feature is currently an experimental feature. Currently, + auto-parallelism can be used only when a user does not set any other + strategy configs except auto. For details, please reference the following + code example + Default Value: False + + Examples: + .. code-block:: python + + import paddle + import paddle.distributed.fleet as fleet + strategy = fleet.DistributedStrategy() + strategy.auto = True + + optimizer = paddle.optimizer.SGD(learning_rate=0.01) + optimizer = fleet.distributed_optimizer(optimizer, strategy) + """ return self.strategy.auto @auto.setter @@ -945,9 +1010,27 @@ class DistributedStrategy(object): @property def cudnn_exhaustive_search(self): + """ + Indicating whether to use exhaustive search method to choose convolution algorithms. + Exhaustive search attempts all cuDNN algorithms to choose the fastest algorithm. + This method is time-consuming, the choosed algorithm will be cached for the given layer specifications. + Once the layer specifications (like batch size, feature map size) are changed, it will search again. + Default Value: True + + Examples: + .. code-block:: python + + import paddle.distributed.fleet as fleet + strategy = fleet.DistributedStrategy() + strategy.cudnn_exhaustive_search = False + + optimizer = paddle.optimizer.SGD(learning_rate=0.01) + optimizer = fleet.distributed_optimizer(optimizer, strategy) + """ return self.strategy.cudnn_exhaustive_search @cudnn_exhaustive_search.setter + @is_strict_auto def cudnn_exhaustive_search(self, flag): if isinstance(flag, bool): self.strategy.cudnn_exhaustive_search = flag @@ -958,9 +1041,28 @@ class DistributedStrategy(object): @property def conv_workspace_size_limit(self): + """ + The workspace limit size in MB unit for choosing cuDNN convolution algorithms. + The inner funciton of cuDNN obtain the fastest suited algorithm that fits within this memory limit. + Usually, large workspace size may lead to choose faster algorithms, + but significant increasing memory workspace. Users need to trade-off between memory and speed. + Default Value: 4000 + + Examples: + .. code-block:: python + + import paddle.distributed.fleet as fleet + strategy = fleet.DistributedStrategy() + strategy.conv_workspace_size_limit = 1024 + + optimizer = paddle.optimizer.SGD(learning_rate=0.01) + optimizer = fleet.distributed_optimizer(optimizer, strategy) + + """ return self.strategy.conv_workspace_size_limit @conv_workspace_size_limit.setter + @is_strict_auto def conv_workspace_size_limit(self, value): if isinstance(value, int): self.strategy.conv_workspace_size_limit = value @@ -971,9 +1073,26 @@ class DistributedStrategy(object): @property def cudnn_batchnorm_spatial_persistent(self): + """ + Indicates whether to use the mode CUDNN_BATCHNORM_SPATIAL_PERSISTENT function in batchnorm. + This is only useful in cudnn. + Default Value: True + + Examples: + .. code-block:: python + + import paddle.distributed.fleet as fleet + strategy = fleet.DistributedStrategy() + strategy.cudnn_batchnorm_spatial_persistent = True + + optimizer = paddle.optimizer.SGD(learning_rate=0.01) + optimizer = fleet.distributed_optimizer(optimizer, strategy) + + """ return self.strategy.cudnn_batchnorm_spatial_persistent @cudnn_batchnorm_spatial_persistent.setter + @is_strict_auto def cudnn_batchnorm_spatial_persistent(self, flag): if isinstance(flag, bool): self.strategy.cudnn_batchnorm_spatial_persistent = flag @@ -1005,8 +1124,98 @@ class DistributedStrategy(object): if core.globals().is_public(key): core.globals()[key] = values[i] + def _is_strict_auto(self): + global non_auto_func_called + if self.strategy.auto and non_auto_func_called: + return True + return False + def __repr__(self): + spacing = 2 + max_k = 38 + max_v = 38 + + length = max_k + max_v + spacing + + h1_format = " " + "|{{:^{}s}}|\n".format(length) + h2_format = " " + "|{{:>{}s}}{}{{:^{}s}}|\n".format(max_k, " " * + spacing, max_v) + + border = " +" + "".join(["="] * length) + "+" + line = " +" + "".join(["-"] * length) + "+" + + draws = border + "\n" + draws += h1_format.format("") + draws += h1_format.format("DistributedStrategy Overview") + draws += h1_format.format("") + fields = self.strategy.DESCRIPTOR.fields + str_res = "" + + env_draws = line + "\n" for f in fields: - print("{}: {}".format(f.name, f.default_value)) - return str(self.strategy) + if "build_strategy" in f.name or "execution_strategy" in f.name: + continue + if "_configs" in f.name: + continue + else: + if isinstance(getattr(self.strategy, f.name), bool): + if hasattr(self.strategy, f.name + "_configs"): + if getattr(self.strategy, f.name): + draws += border + "\n" + draws += h1_format.format( + "{} = True, please check {}_configs".format( + f.name, f.name)) + draws += line + "\n" + my_configs = getattr(self.strategy, + f.name + "_configs") + config_fields = my_configs.DESCRIPTOR.fields + for ff in config_fields: + if isinstance( + getattr(my_configs, ff.name), + google.protobuf.pyext._message. + RepeatedScalarContainer): + values = getattr(my_configs, ff.name) + for i, v in enumerate(values): + if i == 0: + draws += h2_format.format(ff.name, + str(v)) + else: + draws += h2_format.format("", + str(v)) + else: + draws += h2_format.format( + ff.name, + str(getattr(my_configs, ff.name))) + else: + env_draws += h2_format.format( + f.name, str(getattr(self.strategy, f.name))) + else: + env_draws += h2_format.format( + f.name, str(getattr(self.strategy, f.name))) + + result_res = draws + border + "\n" + h1_format.format( + "Environment Flags, Communication Flags") + result_res += env_draws + + build_strategy_str = border + "\n" + build_strategy_str += h1_format.format("Build Strategy") + build_strategy_str += line + "\n" + + fields = self.strategy.build_strategy.DESCRIPTOR.fields + for f in fields: + build_strategy_str += h2_format.format( + f.name, str(getattr(self.strategy.build_strategy, f.name))) + build_strategy_str += border + "\n" + + execution_strategy_str = h1_format.format("Execution Strategy") + execution_strategy_str += line + "\n" + + fields = self.strategy.execution_strategy.DESCRIPTOR.fields + for f in fields: + execution_strategy_str += h2_format.format( + f.name, str(getattr(self.strategy.execution_strategy, f.name))) + execution_strategy_str += border + "\n" + + result_res += build_strategy_str + execution_strategy_str + return result_res diff --git a/python/paddle/distributed/fleet/base/fleet_base.py b/python/paddle/distributed/fleet/base/fleet_base.py index 8c748060e630079af362759b1e4c1c0b09d58063..0dfcd5f3255efa945bbd4ac94b00433960eeaa22 100644 --- a/python/paddle/distributed/fleet/base/fleet_base.py +++ b/python/paddle/distributed/fleet/base/fleet_base.py @@ -13,6 +13,7 @@ # limitations under the License. from __future__ import print_function +import copy import warnings import paddle from paddle.fluid.framework import dygraph_only @@ -230,7 +231,7 @@ class Fleet(object): Returns: int: worker numbers - + Examples: .. code-block:: python @@ -736,7 +737,7 @@ class Fleet(object): """ Set the value of the learning rate manually in the optimizer. Only work in dygraph mode - + Args: value (float|Tensor): the value of learning rate @@ -876,7 +877,7 @@ class Fleet(object): """ Execute the optimizer once. Only work in dygraph mode - + Returns: None Examples: @@ -1008,6 +1009,18 @@ class Fleet(object): MetaOptimizerFactory()._get_valid_meta_optimizers( self.user_defined_optimizer) + context["user_defined_strategy"] = copy.copy(self.user_defined_strategy) + + # trigger the auto-parallel in very strict condition + # strategy = DistributedStrategy() + # strategy.auto = True + # optimizer = paddle.optimizer.SGD(learning_rate=0.1) + # optimizer = fleet.distributed_optimizer(optimizer, strategy) + if self.user_defined_strategy._is_strict_auto(): + # turn on all the strategy for each optimizer + for opt in distributed_optimizer_list: + opt._enable_strategy(self.user_defined_strategy, context) + valid_optimizer_list = [] valid_graph_optimizer_list = [] can_not_apply_optimizer_list = [] diff --git a/python/paddle/distributed/fleet/launch_utils.py b/python/paddle/distributed/fleet/launch_utils.py index 3da5aed8201ace6ccf9eed1ff322a7c6304de4a6..0e995200dde035842d89d9c503566b7b70ee67b7 100644 --- a/python/paddle/distributed/fleet/launch_utils.py +++ b/python/paddle/distributed/fleet/launch_utils.py @@ -347,12 +347,13 @@ def pretty_print_envs(envs, header=None): for k, v in envs.items(): max_k = max(max_k, len(k)) - h_format = "{{:^{}s}}{}{{:<{}s}}\n".format(max_k, " " * spacing, max_v) - l_format = "{{:<{}s}}{{}}{{:<{}s}}\n".format(max_k, max_v) + h_format = " " + "|{{:>{}s}}{}{{:^{}s}}|\n".format(max_k, " " * spacing, + max_v) + l_format = " " + "|{{:>{}s}}{{}}{{:^{}s}}|\n".format(max_k, max_v) length = max_k + max_v + spacing - border = "".join(["="] * length) - line = "".join(["-"] * length) + border = " +" + "".join(["="] * length) + "+" + line = " +" + "".join(["-"] * length) + "+" draws = "" draws += border + "\n" diff --git a/python/paddle/distributed/fleet/meta_optimizers/amp_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/amp_optimizer.py index b1952276e44cd1466bc443440505462924115ab7..31a9913701c3e08f5268d578d09c15f5bf8a86f8 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/amp_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/amp_optimizer.py @@ -34,6 +34,9 @@ class AMPOptimizer(MetaOptimizerBase): loss, role_maker, user_defined_optimizer, user_defined_strategy) def _can_apply(self): + if not self.role_maker._is_collective: + return False + if self.user_defined_strategy.amp: return True return False @@ -42,6 +45,17 @@ class AMPOptimizer(MetaOptimizerBase): dist_strategy.amp = False dist_strategy.amp_configs = {} + def _enable_strategy(self, dist_strategy, context): + dist_strategy.amp = True + dist_strategy.amp_configs = { + "init_loss_scaling": 32768.0, + "incr_every_n_steps": 1000, + "decr_every_n_nan_or_inf": 2, + "incr_ratio": 2.0, + "decr_ratio": 8.0, + "use_dynamic_loss_scaling": True + } + def minimize_impl(self, loss, startup_program=None, diff --git a/python/paddle/distributed/fleet/meta_optimizers/dgc_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/dgc_optimizer.py index f1c6defc5c982c7d56980642898aaa333c199bbe..3f6ed1ed2f23d4595b3aadff6f259f9e27f129b2 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/dgc_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/dgc_optimizer.py @@ -53,6 +53,9 @@ class DGCOptimizer(MetaOptimizerBase): name=opt._name) def _can_apply(self): + if not self.role_maker._is_collective: + return False + if self.user_defined_strategy.dgc: if not isinstance(self.inner_opt, Momentum): logging.warn("dgc only works on Momentum optimizer") @@ -69,6 +72,10 @@ class DGCOptimizer(MetaOptimizerBase): dist_strategy.dgc = False dist_strategy.dgc_configs = {} + def _enable_strategy(self, dist_strategy, context): + dist_strategy.dgc = True + dist_strategy.dgc_configs = {"rampup_begin_step": 0, "rampup_step": 1} + def backward(self, loss, startup_program=None, @@ -85,5 +92,5 @@ class DGCOptimizer(MetaOptimizerBase): no_grad_set=None): optimize_ops, params_grads = \ self.dgc_opt.minimize(loss, startup_program, - parameter_list, no_grad_set) + parameter_list, no_grad_set) return optimize_ops, params_grads diff --git a/python/paddle/distributed/fleet/meta_optimizers/gradient_merge_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/gradient_merge_optimizer.py index 7db79ad7b5b7081172209faa2396d9f2a31bbdb3..f1b3680976541806d96ca815be64b03bcd499469 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/gradient_merge_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/gradient_merge_optimizer.py @@ -37,14 +37,21 @@ class GradientMergeOptimizer(MetaOptimizerBase): self.user_defined_strategy.gradient_merge_configs["avg"]) def _can_apply(self): + if not self.role_maker._is_collective: + return False + can_apply = (self.user_defined_strategy.gradient_merge == True) and \ - self.user_defined_strategy.gradient_merge_configs["k_steps"] > 1 + self.user_defined_strategy.gradient_merge_configs["k_steps"] > 1 return can_apply def _disable_strategy(self, dist_strategy): dist_strategy.gradient_merge = False dist_strategy.gradient_merge_configs = {} + def _enable_strategy(self, dist_strategy, context): + # we currently do not support auto-enable gradient merge + return + def minimize_impl(self, loss, startup_program=None, diff --git a/python/paddle/distributed/fleet/meta_optimizers/graph_execution_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/graph_execution_optimizer.py index ace31687338f918ef260b3134b0bd429795542d0..6c1cc3d7a9769a5c61997ab761a5458b7e8df4a3 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/graph_execution_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/graph_execution_optimizer.py @@ -48,7 +48,7 @@ class GraphExecutionOptimizer(MetaOptimizerBase): callbacks=None): pass - # should fix the variable + # should fix the variable def _setup_nccl_op(self, startup_program, main_program, build_strategy): trainer_endpoints = self.role_maker.get_trainer_endpoints() trainers = trainer_endpoints @@ -94,31 +94,31 @@ class GraphExecutionOptimizer(MetaOptimizerBase): dist_strategy = self.user_defined_strategy local_build_strategy = paddle.fluid.BuildStrategy() local_build_strategy.enable_sequential_execution = \ - dist_strategy.build_strategy.enable_sequential_execution + dist_strategy.build_strategy.enable_sequential_execution local_build_strategy.fuse_elewise_add_act_ops = \ - dist_strategy.build_strategy.fuse_elewise_add_act_ops + dist_strategy.build_strategy.fuse_elewise_add_act_ops local_build_strategy.fuse_bn_act_ops = \ - dist_strategy.build_strategy.fuse_bn_act_ops + dist_strategy.build_strategy.fuse_bn_act_ops local_build_strategy.enable_auto_fusion = \ - dist_strategy.build_strategy.enable_auto_fusion + dist_strategy.build_strategy.enable_auto_fusion local_build_strategy.fuse_relu_depthwise_conv = \ - dist_strategy.build_strategy.fuse_relu_depthwise_conv + dist_strategy.build_strategy.fuse_relu_depthwise_conv local_build_strategy.fuse_broadcast_ops = \ - dist_strategy.build_strategy.fuse_broadcast_ops + dist_strategy.build_strategy.fuse_broadcast_ops local_build_strategy.fuse_all_optimizer_ops = \ - dist_strategy.build_strategy.fuse_all_optimizer_ops + dist_strategy.build_strategy.fuse_all_optimizer_ops local_build_strategy.enable_inplace = \ - dist_strategy.build_strategy.enable_inplace + dist_strategy.build_strategy.enable_inplace local_build_strategy.use_hierarchical_allreduce = \ - dist_strategy.use_hierarchical_allreduce + dist_strategy.use_hierarchical_allreduce local_build_strategy.hierarchical_allreduce_inter_nranks = \ - dist_strategy.hierarchical_allreduce_inter_nranks + dist_strategy.hierarchical_allreduce_inter_nranks local_build_strategy.sync_batch_norm = \ - dist_strategy.sync_batch_norm + dist_strategy.sync_batch_norm local_build_strategy.fuse_all_reduce_ops = \ - dist_strategy.fuse_all_reduce_ops + dist_strategy.fuse_all_reduce_ops local_build_strategy.nccl_comm_num = \ - dist_strategy.nccl_comm_num + dist_strategy.nccl_comm_num if self.user_defined_strategy.recompute == True: logging.warn( @@ -148,9 +148,6 @@ class GraphExecutionOptimizer(MetaOptimizerBase): sync_allreduce = dist_strategy.sync_nccl_allreduce if sync_allreduce: - paddle.fluid.framework.set_flags({ - "FLAGS_sync_nccl_allreduce": True - }) exe_strategy.num_threads = local_build_strategy.nccl_comm_num + 1 if local_build_strategy.use_hierarchical_allreduce: exe_strategy.num_threads = 2 * local_build_strategy.nccl_comm_num + 1 @@ -191,7 +188,11 @@ class GraphExecutionOptimizer(MetaOptimizerBase): def _disable_strategy(self, dist_strategy): # TODO(guru4elephant): should close all PE related flags here - pass + return + + def _enable_strategy(self, dist_strategy, context): + # by default, graph execution strategy is enabled + return def minimize(self, loss, diff --git a/python/paddle/distributed/fleet/meta_optimizers/lamb_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/lamb_optimizer.py index 9fa29c4078e9f579a740ef8c0591979e7fbb962d..df9887759e16fddb0579abdcdf3ef5f9024825e7 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/lamb_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/lamb_optimizer.py @@ -62,6 +62,9 @@ class LambOptimizer(MetaOptimizerBase): name=opt._name) def _can_apply(self): + if not self.role_maker._is_collective: + return False + if self.user_defined_strategy.lamb: if not isinstance(self.inner_opt, AdamOptimizer): logging.warn( @@ -75,6 +78,13 @@ class LambOptimizer(MetaOptimizerBase): dist_strategy.lamb = False dist_strategy.lamb_configs = {} + def _enable_strategy(self, dist_strategy, context): + dist_strategy.lamb = True + dist_strategy.lamb_configs = { + "lamb_weight_decay": 0.01, + "exclude_from_weight_decay": [] + } + def backward(self, loss, startup_program=None, @@ -84,6 +94,10 @@ class LambOptimizer(MetaOptimizerBase): return self.lamb_opt.backward(loss, startup_program, parameter_list, no_grad_set, callbacks) + # the following function will be used by AMP if both LARS and AMP are turn on together. + def apply_gradients(self, params_grads): + return self.lamb_opt.apply_gradients(params_grads=params_grads) + def minimize_impl(self, loss, startup_program=None, @@ -91,5 +105,5 @@ class LambOptimizer(MetaOptimizerBase): no_grad_set=None): optimize_ops, params_grads = \ self.lamb_opt.minimize(loss, startup_program, - parameter_list, no_grad_set) + parameter_list, no_grad_set) return optimize_ops, params_grads diff --git a/python/paddle/distributed/fleet/meta_optimizers/lars_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/lars_optimizer.py index a7b856ff5b0dcb1ab30de82a12c91a2e1c14fe76..609d8b85e714c1c7247898f8d506f9dadab9f499 100755 --- a/python/paddle/distributed/fleet/meta_optimizers/lars_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/lars_optimizer.py @@ -44,13 +44,19 @@ class LarsOptimizer(MetaOptimizerBase): parameter_list=opt._parameter_list, regularization=opt.regularization, grad_clip=opt._grad_clip, - name=opt._name) + name=opt._name, + exclude_from_weight_decay=configs['exclude_from_weight_decay'], + epsilon=configs['epsilon']) def _can_apply(self): + if not self.role_maker._is_collective: + return False + if self.user_defined_strategy.lars: if not isinstance(self.inner_opt, Momentum): logging.warn( - "lars need the inner optimizer to be Momentum optimizer.") + "lars need the inner optimizer to be Momentum optimizer but got {}.". + format(self.inner_opt.type)) return False return True return False @@ -59,6 +65,13 @@ class LarsOptimizer(MetaOptimizerBase): dist_strategy.lars = False dist_strategy.lars_configs = {} + def _enable_strategy(self, dist_strategy, context): + dist_strategy.lars = True + dist_strategy.lars_configs = { + "lars_coeff": 0.01, + "lars_weight_decay": 0.0005, + } + def backward(self, loss, startup_program=None, @@ -68,6 +81,10 @@ class LarsOptimizer(MetaOptimizerBase): return self.lars_opt.backward(loss, startup_program, parameter_list, no_grad_set, callbacks) + # the following function will be used by AMP if both LARS and AMP are turn on together. + def apply_gradients(self, params_grads): + return self.lars_opt.apply_gradients(params_grads=params_grads) + def minimize_impl(self, loss, startup_program=None, @@ -75,5 +92,5 @@ class LarsOptimizer(MetaOptimizerBase): no_grad_set=None): optimize_ops, params_grads = \ self.lars_opt.minimize(loss, startup_program, - parameter_list, no_grad_set) + parameter_list, no_grad_set) return optimize_ops, params_grads diff --git a/python/paddle/distributed/fleet/meta_optimizers/localsgd_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/localsgd_optimizer.py index e22127c13999bfde7aa753ad1a66536913ab04f9..6fa34d8d28a907d936500907db3e4c65ab4f4da8 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/localsgd_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/localsgd_optimizer.py @@ -14,8 +14,8 @@ from __future__ import print_function +import paddle from paddle.fluid import program_guard, layers, default_main_program -from paddle.fluid.optimizer import Momentum, SGD from .meta_optimizer_base import MetaOptimizerBase from .common import OpRole, OP_ROLE_KEY, CollectiveHelper, is_update_op @@ -29,19 +29,28 @@ class LocalSGDOptimizer(MetaOptimizerBase): self.snapshot_key = '@SNAPSHOT' def _can_apply(self): + if not self.role_maker._is_collective: + return False + if not self.user_defined_strategy.localsgd: return False if self.role_maker.worker_num() <= 1: return False - return isinstance(self.inner_opt, Momentum) \ - or isinstance(self.inner_opt, SGD) + return isinstance(self.inner_opt, paddle.optimizer.momentum.Momentum) \ + or isinstance(self.inner_opt, paddle.fluid.optimizer.Momentum) \ + or isinstance(self.inner_opt, paddle.optimizer.sgd.SGD) \ + or isinstance(self.inner_opt, paddle.fluid.optimizer.SGD) def _disable_strategy(self, dist_strategy): dist_strategy.localsgd = False dist_strategy.localsgd_configs = {} + def _enable_strategy(self, dist_strategy, context): + dist_strategy.localsgd = True + dist_strategy.localsgd_configs = {"k_steps": 1, "begin_step": 1} + def snapshot_name(self, param_name): return param_name + self.snapshot_key @@ -77,8 +86,9 @@ class LocalSGDOptimizer(MetaOptimizerBase): minimized = self.inner_opt.minimize( loss, startup_program=startup_program) - init_k_steps = self.user_defined_strategy.localsgd_configs['k_steps'] - auto_steps = self.user_defined_strategy.auto + k_steps_value = self.user_defined_strategy.localsgd_configs['k_steps'] + begin_step_value = self.user_defined_strategy.localsgd_configs[ + 'begin_step'] if startup_program is None: startup_program = default_startup_program() @@ -92,45 +102,28 @@ class LocalSGDOptimizer(MetaOptimizerBase): p2s = self.create_snapshot_vars(main_block.program) with program_guard(main_block.program, startup_program): - step = layers.autoincreased_step_counter(begin=0) + step = layers.autoincreased_step_counter(begin=1) k_steps = layers.create_global_var( name="k_steps", shape=[1], - value=init_k_steps, + value=k_steps_value, + dtype='int64', + persistable=True) + + begin_step = layers.create_global_var( + name="begin_step", + shape=[1], + value=begin_step_value, dtype='int64', persistable=True) + last_step = layers.create_global_var( name="last_step", shape=[1], - value=int(0), + value=begin_step_value, dtype='int64', persistable=True) - if auto_steps: - avg_loss = layers.collective._c_allreduce( - loss) / self.role_maker.worker_num() - - lr_0 = layers.create_global_var( - name="lr_0", - shape=[1], - value=float(0), - dtype='float32', - persistable=True) - loss_0 = layers.create_global_var( - name="loss_0", - shape=[1], - value=float(0), - dtype='float32', - persistable=True) - - global_lr = self.inner_opt._global_learning_rate() - - def initialize(): - layers.assign(loss, loss_0) - layers.assign(global_lr, lr_0) - - layers.cond(step == 0, initialize) - def communicate(): sub_block = default_main_program().current_block() ring_id = -1 @@ -186,20 +179,10 @@ class LocalSGDOptimizer(MetaOptimizerBase): inputs={'X': [param]}, outputs={'Out': [snapshot]}, attrs={OP_ROLE_KEY: OpRole.Optimize}) - - if auto_steps: - next_local_steps = layers.cast( - layers.ceil( - layers.sqrt(lr_0 * loss / (global_lr * loss_0) * - float(init_k_steps))), - dtype='int64') - max_local_steps = layers.fill_constant( - shape=[1], dtype='int64', value=16) - next_local_steps = layers.elementwise_min(next_local_steps, - max_local_steps) - layers.assign(next_local_steps, k_steps) layers.assign(step, last_step) - layers.cond(step - last_step == k_steps, communicate) + def begin_localsgd(): + layers.cond(step - last_step == k_steps, communicate) + layers.cond(step > begin_step, begin_localsgd, communicate) return minimized diff --git a/python/paddle/distributed/fleet/meta_optimizers/meta_optimizer_base.py b/python/paddle/distributed/fleet/meta_optimizers/meta_optimizer_base.py index 073148e11a0a2b08253b89d36d7a014b830518f8..a12ca50442b1c3499d62216d1fecc709f3351382 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/meta_optimizer_base.py +++ b/python/paddle/distributed/fleet/meta_optimizers/meta_optimizer_base.py @@ -48,6 +48,10 @@ class MetaOptimizerBase(Optimizer): raise NotImplementedError("you should implement disable strategy in {}". format(type(self).__name__)) + def _enable_strategy(self, dist_strategy, context=None): + raise NotImplementedError("you should implement enable strategy in {}". + format(type(self).__name__)) + def apply_gradients(self, params_grads): return self.inner_opt.apply_gradients(params_grads=params_grads) diff --git a/python/paddle/distributed/fleet/meta_optimizers/parameter_server_graph_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/parameter_server_graph_optimizer.py index 878ed7422d733d3e2828e0395ec63ed16b4c489a..7dc532c86ea681d8479710732ec33e96c58c35d5 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/parameter_server_graph_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/parameter_server_graph_optimizer.py @@ -24,6 +24,9 @@ class ParameterServerGraphOptimizer(ParameterServerOptimizer): self.meta_optimizers_white_list = [] def _can_apply(self): + if self.role_maker._is_collective: + return False + k_steps = self.user_defined_strategy.a_sync_configs["k_steps"] if k_steps < 0: return False @@ -37,7 +40,11 @@ class ParameterServerGraphOptimizer(ParameterServerOptimizer): return True def _disable_strategy(self, dist_strategy): - dist_strategy.a_sync_configs = {} + return + + def _enable_strategy(self, dist_strategy, context): + # only open up the async mode for auto-parallel + return def _is_graph_out(self): return True diff --git a/python/paddle/distributed/fleet/meta_optimizers/parameter_server_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/parameter_server_optimizer.py index ecb198bedf9041aa3ffc929a72cce3c209f03b61..51d4d343165b9057c803a22aa428081109d7d35f 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/parameter_server_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/parameter_server_optimizer.py @@ -13,6 +13,10 @@ from paddle import fluid from .meta_optimizer_base import MetaOptimizerBase +from paddle.fluid import core +import subprocess +import re +import platform class ParameterServerOptimizer(MetaOptimizerBase): @@ -28,6 +32,7 @@ class ParameterServerOptimizer(MetaOptimizerBase): def _can_apply(self): if self.role_maker._is_collective: return False + k_steps = self.user_defined_strategy.a_sync_configs["k_steps"] return True if k_steps >= 0 else False @@ -127,6 +132,95 @@ class ParameterServerOptimizer(MetaOptimizerBase): return _main, _startup + def _can_apply_geo(self, dist_strategy, program): + def get_sys_free_mem(): + plat = platform.system() + if platform.system() == "Darwin": + vm = subprocess.Popen( + ['vm_stat'], stdout=subprocess.PIPE).communicate()[0] + # Process vm_stat + vmLines = vm.split('\n') + sep = re.compile(':[\s]+') + vmStats = {} + for row in range(1, len(vmLines) - 2): + rowText = vmLines[row].strip() + rowElements = sep.split(rowText) + vmStats[(rowElements[0] + )] = int(rowElements[1].strip('\.')) * 4096 + return vmStats["Pages free"] + elif platform.system() == "Linux": + mems = {} + with open('/proc/meminfo', 'rb') as f: + for line in f: + fields = line.split() + mems[fields[0]] = int(fields[1]) * 1024 + free = mems[b'MemFree:'] + return free + else: + raise ValueError( + "%s platform is unsupported is parameter server optimizer" % + (platform.system())) + + if not isinstance(self.inner_opt, fluid.optimizer.SGDOptimizer): + return False + + free = get_sys_free_mem() + + from paddle.fluid.incubate.fleet.parameter_server.ir import vars_metatools + + processed_var_names = set(["@EMPTY@"]) + param_memory_size = 0 + for varname in program.global_block().vars: + var = program.global_block().vars[varname] + if not var.persistable or var.desc.type( + ) != core.VarDesc.VarType.LOD_TENSOR: + continue + param = vars_metatools.create_var_struct(var) + param_memory_size += param.m_size + processed_var_names.add(varname) + + upper_mem_use = param_memory_size * 5.0 + + program_tmp_vars = dict() + eval_batch_size = 1024 + for op in program.global_block().ops: + for var_name in op.output_arg_names: + if var_name in processed_var_names: + continue + processed_var_names.add(var_name) + var = program.global_block().vars[var_name] + + if var.desc.type() != core.VarDesc.VarType.LOD_TENSOR: + continue + + data_count = 1 + neg_dim_count = 0 + for x in var.shape: + if x < 0: + if neg_dim_count >= 1: + raise ValueError( + "Var %s has more than one negative dim." % + (var_name)) + neg_dim_count += 1 + data_count *= (-x) + else: + data_count *= x + program_tmp_vars[var_name] = ( + data_count, neg_dim_count, + vars_metatools.dtype_to_size[var.dtype]) + + for varname in program_tmp_vars: + data_count, neg_dim_count, type_size = program_tmp_vars[varname] + if neg_dim_count == 1: + data_count *= eval_batch_size + var_memory = data_count * type_size + upper_mem_use += var_memory + + if upper_mem_use < free: + return True + else: + return False + def minimize_impl(self, loss, startup_program=None, @@ -143,6 +237,7 @@ class ParameterServerOptimizer(MetaOptimizerBase): compiled_config = public.CompileTimeStrategy(_origin_main_program, _origin_startup_program, strategy, self.role_maker) + compiled_config.strategy = strategy if self.role_maker.is_worker() or self.role_maker._is_heter_worker(): main_program, startup_program = self._build_trainer_programs( @@ -157,4 +252,24 @@ class ParameterServerOptimizer(MetaOptimizerBase): return None, None def _disable_strategy(self, dist_strategy): - self.user_defined_strategy.a_sync_configs = {} + dist_strategy.a_sync = False + a_sync_configs = dist_strategy.a_sync_configs + a_sync_configs["k_steps"] = -1 + dist_strategy.a_sync_configs = a_sync_configs + + def _enable_strategy(self, dist_strategy, context): + a_sync_configs = dist_strategy.a_sync_configs + if a_sync_configs["k_steps"] >= 0: + return + + dist_strategy.a_sync = True + a_sync_configs = dist_strategy.a_sync_configs + + is_geo = self._can_apply_geo(dist_strategy, + context["origin_main_program"]) + + if is_geo: + a_sync_configs["k_steps"] = 800 + else: + a_sync_configs["k_steps"] = 0 + dist_strategy.a_sync_configs = a_sync_configs diff --git a/python/paddle/distributed/fleet/meta_optimizers/pipeline_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/pipeline_optimizer.py index d5a45e2b4e1aeda2e1c66c0a5a36236622f093ec..87fa70779111ea485319f50b58901c605fffa23c 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/pipeline_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/pipeline_optimizer.py @@ -103,6 +103,9 @@ class PipelineOptimizer(MetaOptimizerBase): self.wrapped_opt = PO(self.inner_opt, num_microbatches=num_microbatches) def _can_apply(self): + if not self.role_maker._is_collective: + return False + if self.user_defined_strategy.pipeline == True: return True return False @@ -111,6 +114,10 @@ class PipelineOptimizer(MetaOptimizerBase): dist_strategy.pipeline = False dist_strategy.pipeline_configs = {} + def _enable_strategy(self, dist_strategy, context): + # we do not support enable pipeline automatically right now + return + def minimize_impl(self, loss, startup_program=None, @@ -176,7 +183,7 @@ class PipelineOptimizer(MetaOptimizerBase): grad = None for idx, op in reversed(list(enumerate(block.ops))): if is_backward_op(op) and \ - OP_ROLE_VAR_KEY in op.attr_names: + OP_ROLE_VAR_KEY in op.attr_names: op_role_var = op.all_attrs()[OP_ROLE_VAR_KEY] if len(op_role_var) == 0: continue diff --git a/python/paddle/distributed/fleet/meta_optimizers/recompute_optimizer.py b/python/paddle/distributed/fleet/meta_optimizers/recompute_optimizer.py index 3eb3ca6127cfe0d0a7a458c6c44e09ce22e7b24a..8f9595486922a37cff02d1ac96c1c4c2bbf4b0d5 100644 --- a/python/paddle/distributed/fleet/meta_optimizers/recompute_optimizer.py +++ b/python/paddle/distributed/fleet/meta_optimizers/recompute_optimizer.py @@ -38,6 +38,9 @@ class RecomputeOptimizer(MetaOptimizerBase): list(user_defined_strategy.recompute_configs["checkpoints"])) def _can_apply(self): + if self.role_maker._is_collective: + return False + if self.user_defined_strategy.recompute == True: if len(self.user_defined_strategy.recompute_configs[ "checkpoints"]) == 0: @@ -49,6 +52,10 @@ class RecomputeOptimizer(MetaOptimizerBase): dist_strategy.recompute = False dist_strategy.recompute_configs = {} + def _enable_strategy(self, dist_strategy, context): + # we do not support automatically recompute checkpoints currently + return + def backward(self, loss, startup_program=None, diff --git a/python/paddle/distribution.py b/python/paddle/distribution.py index 918ebce07825139fabe8ddd4c1e266dd04eb7f6d..35204affb3fd168b8bd137d78c3413a08885e2bb 100644 --- a/python/paddle/distribution.py +++ b/python/paddle/distribution.py @@ -138,7 +138,7 @@ class Distribution(object): convert value's dtype to be consistent with param's dtype. Args: - param (int|float|list|numpy.ndarray|Tensor): low and high in Uniform class, loc and scale in Normal class. + param (Tensor): low and high in Uniform class, loc and scale in Normal class. value (Tensor): The input tensor. Returns: @@ -152,6 +152,7 @@ class Distribution(object): ) return core.ops.cast(value, 'in_dtype', value.dtype, 'out_dtype', param.dtype) + return value check_variable_and_dtype(value, 'value', ['float32', 'float64'], 'log_prob') diff --git a/python/paddle/fluid/backward.py b/python/paddle/fluid/backward.py index 898c7d295641863740288e3f4e1da39266bce183..d51cacd1a5cad53ef77b325e5380100c537e057e 100644 --- a/python/paddle/fluid/backward.py +++ b/python/paddle/fluid/backward.py @@ -1756,6 +1756,12 @@ def calc_gradient(targets, inputs, target_gradients=None, no_grad_set=None): op_path_dict = dict() op_path = _find_op_path_(block, targets, inputs, block_no_grad_set, op_path_dict) + + # find no grad var by op_path + no_grad_vars = _find_no_grad_vars(block, op_path, targets, + block_no_grad_set) + block_no_grad_set.update(no_grad_vars) + no_grad_dict[0].update(list(map(_append_grad_suffix_, block_no_grad_set))) grad_to_var = dict() grad_info_map = dict() diff --git a/python/paddle/fluid/contrib/layers/nn.py b/python/paddle/fluid/contrib/layers/nn.py index 0e187d4174cd5cca65f79e4ab84b4cc32ecefd21..7b564b3f837c001673bdd272ba60edf31cde21fb 100644 --- a/python/paddle/fluid/contrib/layers/nn.py +++ b/python/paddle/fluid/contrib/layers/nn.py @@ -37,7 +37,7 @@ import warnings import inspect import numpy as np - +import paddle from paddle.fluid.layer_helper import LayerHelper from paddle.fluid.layers import utils from ... import unique_name @@ -56,7 +56,8 @@ __all__ = [ 'match_matrix_tensor', 'tree_conv', 'fused_embedding_seq_pool', 'multiclass_nms2', 'search_pyramid_hash', 'shuffle_batch', 'partial_concat', 'sparse_embedding', 'partial_sum', 'tdm_child', 'rank_attention', - 'tdm_sampler', 'batch_fc', '_pull_box_extended_sparse', 'bilateral_slice' + 'tdm_sampler', 'batch_fc', '_pull_box_extended_sparse', 'bilateral_slice', + 'correlation' ] @@ -1546,3 +1547,81 @@ def bilateral_slice(x, guide, grid, has_offset, name=None): attrs={'has_offset': has_offset}, outputs={'Out': out}) return out + + +def correlation(x, + y, + pad_size, + kernel_size, + max_displacement, + stride1, + stride2, + corr_type_multiply=1): + """ + + This operation compute correlation of two tensor. + For more information of correlation, please refer to PWC-Net: + CNNs for Optical Flow Using Pyramid, Warping, and Cost Volume + _ + + Args: + x(Tensor): The input x is 4-D Tensor with shape [N, C, H, W]. The data type is float32 and float64. + y(Tensor): The input y is 4-D Tensor with shape [N, C, H, W]. The data type is float32 and float64. + pad_size(int): Pad size. The data type is int. + max_displacement(int): Max displacement. The data type is int. + stride1(int): stride size of x. The data type is int. + stride2(int): stride size of y. The data type is int. + corr_type_multiply(int, optional): The type of multiply. The data type is int. Default: 1. + + Returns: + Tensor: The data type is same as input tensor. + + Examples: + + .. code-block:: python + + import paddle.fluid as fluid + + x1 = fluid.layers.data(name='x1', + shape=x_shape, + dtype=x_type, + append_batch_size=False) + x2 = fluid.layers.data(name='x2', + shape=x_shape, + dtype=x_type, + append_batch_size=False) + + + out = fluid.contrib.correlation( + x1, + x2, + pad_size=4, + kernel_size=1, + max_displacement=4, + stride1=1, + stride2=1) + + """ + + helper = LayerHelper("correlation", **locals()) + output = helper.create_variable_for_type_inference(dtype=x.dtype) + if paddle.fluid.in_dygraph_mode(): + attrs = ("pad_size", pad_size, "kernel_size", kernel_size, + "max_displacement", max_displacement, "stride1", stride1, + "stride2", stride2, "corr_type_multiply", corr_type_multiply) + output = getattr(core.ops, "correlation")(x, y, *attrs) + else: + helper.append_op( + type="correlation", + inputs={"Input1": x, + "Input2": y}, + attrs={ + "pad_size": pad_size, + "kernel_size": kernel_size, + "max_displacement": max_displacement, + "stride1": stride1, + "stride2": stride2, + "corr_type_multiply": corr_type_multiply + }, + outputs={"Output": output}) + return output diff --git a/python/paddle/fluid/contrib/mixed_precision/amp_nn.py b/python/paddle/fluid/contrib/mixed_precision/amp_nn.py new file mode 100644 index 0000000000000000000000000000000000000000..d4dc968ca0de44b01741bf1f1fbaac7a9a65287e --- /dev/null +++ b/python/paddle/fluid/contrib/mixed_precision/amp_nn.py @@ -0,0 +1,124 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.fluid.data_feeder import check_variable_and_dtype, check_type +from paddle.fluid.layer_helper import LayerHelper +from paddle.fluid.framework import Variable + +__all__ = ['check_finite_and_unscale', 'update_loss_scaling'] + + +def check_finite_and_unscale(x, scale, name=None): + """ + Check if input X contains all finite data, if yes, scale it by input Scale. + + $$Out = X / scale$$ + + If any tensor in X contains Inf or Nan, the Out will generate a indicator. + FoundInfinite will be 1 (True), and Out will not be scaled. In this case, the data of + Out should not be used, and its data may not be deterministic. + Otherwise, FoundInfinite will be 0 (False). + Args: + x(list|tuple): The input tensors of check_finite_and_unscale operator. + scale: The scale of check_finite_and_unscale operator. + """ + check_type(x, 'x', (tuple, list), 'check_finite_and_unscale') + for e in x: + check_variable_and_dtype(e, "x", ['float32', 'float64'], + 'check_finite_and_unscale') + + helper = LayerHelper("check_finite_and_unscale", **locals()) + found_inf = helper.create_variable_for_type_inference(dtype='bool') + + inputs = {'X': x, 'Scale': scale} + outputs = {'Out': x, 'FoundInfinite': found_inf} + helper.append_op( + type='check_finite_and_unscale', inputs=inputs, outputs=outputs) + + return x, found_inf + + +def update_loss_scaling(x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + incr_every_n_steps, + decr_every_n_nan_or_inf, + incr_ratio, + decr_ratio, + name=None): + """ + Update loss scaling according to overall gradients. If all gradients is + finite after incr_every_n_steps, loss scaling will increase by incr_ratio. + Otherwise, loss scaling will decrease by decr_ratio after + decr_every_n_nan_or_inf steps and each step some gradients are infinite. + + Args: + x(list|tuple): The input tensors of update_loss_scaling operator. + found_inf (Variable): A boolean variable indicates whether + there is any infinite gradient. + prev_loss_scaling (Variable): Previous loss scaling. + num_good_steps (Variable): A variable accumulates good steps in which + all gradients are finite. + num_bad_steps (Variable): A variable accumulates bad steps in which + some gradients are infinite. + incr_every_n_steps (int): A variable represents increasing loss + scaling every n consecutive steps with + finite gradients. + decr_every_n_nan_or_inf (int): A variable represents decreasing + loss scaling every n accumulated + steps with nan or inf gradients. + incr_ratio(float): The multiplier to use when increasing the loss + scaling. + decr_ratio(float): The less-than-one-multiplier to use when decreasing + loss scaling. + """ + + check_variable_and_dtype(prev_loss_scaling, "prev_loss_scaling", + ['float32', 'float64'], "update_loss_scaling") + check_type(x, 'x', (tuple, list), 'update_loss_scaling') + for e in x: + check_variable_and_dtype(e, "x", ['float32', 'float64'], + 'update_loss_scaling') + assert prev_loss_scaling.dtype == e.dtype, "The dtype of prev_loss_scaling should be equal to the dtype of x." + + helper = LayerHelper("update_loss_scaling", **locals()) + + inputs = { + 'X': x, + 'FoundInfinite': found_inf, + 'PrevLossScaling': prev_loss_scaling, + 'InGoodSteps': num_good_steps, + 'InBadSteps': num_bad_steps + } + + outputs = { + 'Out': x, + 'LossScaling': prev_loss_scaling, + 'OutGoodSteps': num_good_steps, + 'OutBadSteps': num_bad_steps + } + + attrs = { + 'incr_every_n_steps': incr_every_n_steps, + 'decr_every_n_nan_or_inf': decr_every_n_nan_or_inf, + 'incr_ratio': incr_ratio, + 'decr_ratio': decr_ratio, + } + + helper.append_op( + type='update_loss_scaling', inputs=inputs, outputs=outputs, attrs=attrs) + + return x diff --git a/python/paddle/fluid/contrib/mixed_precision/decorator.py b/python/paddle/fluid/contrib/mixed_precision/decorator.py index bfbd2700ae10bac4ad37462b5d7844b90dd05bbe..c9112ac849ce0506b7afd941b2213710e06bd1c6 100644 --- a/python/paddle/fluid/contrib/mixed_precision/decorator.py +++ b/python/paddle/fluid/contrib/mixed_precision/decorator.py @@ -17,9 +17,11 @@ from ... import default_startup_program from ... import layers from ... import unique_name from . import fp16_utils -from .fp16_utils import update_loss_scaling, rewrite_program +from .fp16_utils import rewrite_program from .fp16_utils import update_role_var_grad from .fp16_lists import AutoMixedPrecisionLists +from .amp_nn import check_finite_and_unscale +from .amp_nn import update_loss_scaling __all__ = ["decorate"] @@ -67,10 +69,8 @@ class OptimizerWithMixedPrecision(object): persistable=True) self._use_dynamic_loss_scaling = use_dynamic_loss_scaling if self._use_dynamic_loss_scaling: - self._incr_every_n_steps = layers.fill_constant( - shape=[1], dtype='int32', value=incr_every_n_steps) - self._decr_every_n_nan_or_inf = layers.fill_constant( - shape=[1], dtype='int32', value=decr_every_n_nan_or_inf) + self._incr_every_n_steps = incr_every_n_steps + self._decr_every_n_nan_or_inf = decr_every_n_nan_or_inf self._incr_ratio = incr_ratio self._decr_ratio = decr_ratio self._num_good_steps = layers.create_global_var( @@ -139,49 +139,46 @@ class OptimizerWithMixedPrecision(object): # Change the op_role_var attr for some ops, so that gradients # transferred across GPUs can be FP16. update_role_var_grad(self._train_program, self._params_grads) - scaled_params_grads = [] - for p, g in self._params_grads: - with self._train_program._optimized_guard([p, g]): - scaled_g = g / self._loss_scaling - scaled_params_grads.append([p, scaled_g]) - return scaled_params_grads + return self._params_grads - def apply_gradients(self, scaled_params_grads): + def apply_gradients(self, params_grads): """ Check scaled gradients to determine whether to update loss scaling and update parameters by their scaled gradients, Args: - scaled_params_grads (list): A list of params and scaled grads. + params_grads (list): A list of params and scaled grads. Returns: A list of optimize operators. """ - if self._use_dynamic_loss_scaling: + grads = [g for _, g in params_grads] + with self._train_program._optimized_guard(grads): + grads, found_inf = check_finite_and_unscale( + grads, self._loss_scaling, name="find_infinite_scale") - grads = [layers.reduce_sum(g) for [_, g] in scaled_params_grads] - all_grads = layers.concat(grads) - all_grads_sum = layers.reduce_sum(all_grads) - is_overall_finite = layers.isfinite(all_grads_sum) - - update_loss_scaling(is_overall_finite, self._loss_scaling, - self._num_good_steps, self._num_bad_steps, - self._incr_every_n_steps, - self._decr_every_n_nan_or_inf, self._incr_ratio, - self._decr_ratio) - - # apply_gradient append all ops in global block, thus we shouldn't - # apply gradient in the switch branch. - with layers.Switch() as switch: - with switch.case(is_overall_finite): - pass - with switch.default(): - for _, g in scaled_params_grads: - layers.assign(layers.zeros_like(g), g) - - optimize_ops = self._optimizer.apply_gradients(scaled_params_grads) + if self._use_dynamic_loss_scaling: + with self._train_program._optimized_guard(grads): + grads = update_loss_scaling( + grads, + found_inf, + self._loss_scaling, + self._num_good_steps, + self._num_bad_steps, + self._incr_every_n_steps, + self._decr_every_n_nan_or_inf, + self._incr_ratio, + self._decr_ratio, + name="update_loss_scaling") + + params_unscaled_grads = [] + for pg, new_g in zip(params_grads, grads): + params_unscaled_grads.append((pg[0], new_g)) + # apply_gradient append all ops in global block, thus we shouldn't + # apply gradient in the switch branch. + optimize_ops = self._optimizer.apply_gradients(params_unscaled_grads) return optimize_ops diff --git a/python/paddle/fluid/contrib/mixed_precision/fp16_utils.py b/python/paddle/fluid/contrib/mixed_precision/fp16_utils.py index 328dafe6219adb3c6355de0bafc430c52725024f..0b142ff33de55f36410eb9c23cb75210fc9d6321 100644 --- a/python/paddle/fluid/contrib/mixed_precision/fp16_utils.py +++ b/python/paddle/fluid/contrib/mixed_precision/fp16_utils.py @@ -328,77 +328,3 @@ def update_role_var_grad(main_prog, params_grads): raise ValueError("The op {0} is not in program".format(op)) block.desc._remove_op(op_idx, op_idx + 1) block._sync_with_cpp() - - -def update_loss_scaling(is_overall_finite, prev_loss_scaling, num_good_steps, - num_bad_steps, incr_every_n_steps, - decr_every_n_nan_or_inf, incr_ratio, decr_ratio): - """ - Update loss scaling according to overall gradients. If all gradients is - finite after incr_every_n_steps, loss scaling will increase by incr_ratio. - Otherwise, loss scaling will decrease by decr_ratio after - decr_every_n_nan_or_inf steps and each step some gradients are infinite. - - Args: - is_overall_finite (Variable): A boolean variable indicates whether - all gradients are finite. - prev_loss_scaling (Variable): Previous loss scaling. - num_good_steps (Variable): A variable accumulates good steps in which - all gradients are finite. - num_bad_steps (Variable): A variable accumulates bad steps in which - some gradients are infinite. - incr_every_n_steps (Variable): A variable represents increasing loss - scaling every n consecutive steps with - finite gradients. - decr_every_n_nan_or_inf (Variable): A variable represents decreasing - loss scaling every n accumulated - steps with nan or inf gradients. - incr_ratio(float): The multiplier to use when increasing the loss - scaling. - decr_ratio(float): The less-than-one-multiplier to use when decreasing - loss scaling. - """ - zero_steps = layers.fill_constant(shape=[1], dtype='int32', value=0) - with layers.Switch() as switch: - with switch.case(is_overall_finite): - should_incr_loss_scaling = layers.less_than(incr_every_n_steps, - num_good_steps + 1) - with layers.Switch() as switch1: - with switch1.case(should_incr_loss_scaling): - new_loss_scaling = prev_loss_scaling * incr_ratio - loss_scaling_is_finite = layers.isfinite(new_loss_scaling) - with layers.Switch() as switch2: - with switch2.case(loss_scaling_is_finite): - layers.assign(new_loss_scaling, prev_loss_scaling) - with switch2.default(): - pass - layers.assign(zero_steps, num_good_steps) - layers.assign(zero_steps, num_bad_steps) - - with switch1.default(): - layers.increment(num_good_steps) - layers.assign(zero_steps, num_bad_steps) - - with switch.default(): - should_decr_loss_scaling = layers.less_than(decr_every_n_nan_or_inf, - num_bad_steps + 1) - with layers.Switch() as switch3: - with switch3.case(should_decr_loss_scaling): - new_loss_scaling = prev_loss_scaling * decr_ratio - static_loss_scaling = \ - layers.fill_constant(shape=[1], - dtype='float32', - value=1.0) - less_than_one = layers.less_than(new_loss_scaling, - static_loss_scaling) - with layers.Switch() as switch4: - with switch4.case(less_than_one): - layers.assign(static_loss_scaling, - prev_loss_scaling) - with switch4.default(): - layers.assign(new_loss_scaling, prev_loss_scaling) - layers.assign(zero_steps, num_good_steps) - layers.assign(zero_steps, num_bad_steps) - with switch3.default(): - layers.assign(zero_steps, num_good_steps) - layers.increment(num_bad_steps) diff --git a/python/paddle/fluid/contrib/slim/quantization/imperative/qat.py b/python/paddle/fluid/contrib/slim/quantization/imperative/qat.py index 5662284483bf529034e42178c8a431f6286e31b8..8d399c929018f08eb3d02e50981566705536bbf5 100644 --- a/python/paddle/fluid/contrib/slim/quantization/imperative/qat.py +++ b/python/paddle/fluid/contrib/slim/quantization/imperative/qat.py @@ -192,7 +192,6 @@ class ImperativeQuantAware(object): assert len(input_dtype) == len( feed), "The length of input_shape should be equal to feed's." - prog_trans = dygraph.ProgramTranslator() with dygraph.guard(): model.eval() input_vars = [] diff --git a/python/paddle/fluid/contrib/slim/quantization/imperative/quant_nn.py b/python/paddle/fluid/contrib/slim/quantization/imperative/quant_nn.py index 59dd9867abb95dea74e1cdc362b671e7d4120d70..e22c980b0a7c6030c5d6a2fbc4fd58d2ec66958a 100644 --- a/python/paddle/fluid/contrib/slim/quantization/imperative/quant_nn.py +++ b/python/paddle/fluid/contrib/slim/quantization/imperative/quant_nn.py @@ -209,15 +209,24 @@ class FakeQuantAbsMax(layers.Layer): return quant_out -def _get_fake_quant_type(quant_type, name, moving_rate, quant_bits, dtype, - quant_on_weight): +def _get_fake_quant_type(quant_type, **kwargs): + call_args = { + "name": kwargs.get("name", None), + "quant_bits": kwargs.get("quant_bits", 8), + "dtype": kwargs.get("dtype", "float32") + } + + if quant_type == 'abs_max': + call_args["quant_on_weight"] = kwargs.get("quant_on_weight", False) + elif quant_type == 'moving_average_abs_max': + call_args["moving_rate"] = kwargs.get("moving_rate", 0.9) + fake_quant_map = { - 'abs_max': - lambda: FakeQuantAbsMax(name, quant_bits, dtype, quant_on_weight), - 'moving_average_abs_max': - lambda: FakeQuantMovingAverage(name, moving_rate, quant_bits, dtype) + 'abs_max': FakeQuantAbsMax, + 'moving_average_abs_max': FakeQuantMovingAverage } - return fake_quant_map[quant_type]() + + return fake_quant_map[quant_type](**call_args) class QuantizedConv2D(layers.Layer): @@ -247,11 +256,18 @@ class QuantizedConv2D(layers.Layer): self.bias = getattr(layer, 'bias') # For FakeQuant self._fake_quant_weight = _get_fake_quant_type( - weight_quantize_type, self.weight.name, moving_rate, weight_bits, - self._dtype, True) + weight_quantize_type, + name=self.weight.name, + moving_rate=moving_rate, + quant_bits=weight_bits, + dtype=self._dtype, + quant_on_weight=True) self._fake_quant_input = _get_fake_quant_type( activation_quantize_type, - layer.full_name(), moving_rate, activation_bits, self._dtype, False) + name=layer.full_name(), + moving_rate=moving_rate, + quant_bits=activation_bits, + dtype=self._dtype) def forward(self, input): quant_input = self._fake_quant_input(input) @@ -326,11 +342,18 @@ class QuantizedLinear(layers.Layer): self.bias = getattr(layer, 'bias') # For FakeQuant self._fake_quant_weight = _get_fake_quant_type( - weight_quantize_type, self.weight.name, moving_rate, weight_bits, - self._dtype, True) + weight_quantize_type, + name=self.weight.name, + moving_rate=moving_rate, + quant_bits=weight_bits, + dtype=self._dtype, + quant_on_weight=True) self._fake_quant_input = _get_fake_quant_type( activation_quantize_type, - layer.full_name(), moving_rate, activation_bits, self._dtype, False) + name=layer.full_name(), + moving_rate=moving_rate, + quant_bits=activation_bits, + dtype=self._dtype) def forward(self, input): quant_input = self._fake_quant_input(input) diff --git a/python/paddle/fluid/contrib/slim/quantization/quant2_int8_mkldnn_pass.py b/python/paddle/fluid/contrib/slim/quantization/quant2_int8_mkldnn_pass.py index 75e1ea43d15e432d2f6cbec271acd67624de1e01..dadc756c43ecc782a72c1c7d6626e00bc182f2c6 100644 --- a/python/paddle/fluid/contrib/slim/quantization/quant2_int8_mkldnn_pass.py +++ b/python/paddle/fluid/contrib/slim/quantization/quant2_int8_mkldnn_pass.py @@ -299,11 +299,14 @@ class Quant2Int8MkldnnPass(object): # Convert int8 range weights to fp32 range weights scales = self._weight_scales[output_var_name] weight = self._load_param(self._scope, weight_var_name) - assert scales.size == 1 or scales.size == len( - weight - ), "The size of weight scales vector ({}) does not match the number of output channels ({}) in the weights tensor {}.".format( - scales.size, len(weight), weight_var_name) - w_fp32 = np.divide(np.multiply(weight, self._s8_max).T, scales.T).T + if scales.size == 1 or scales.size == weight.shape[0]: + w_fp32 = np.divide(np.multiply(weight, self._s8_max).T, scales.T).T + elif len(weight.shape) > 1 and scales.size == weight.shape[1]: + w_fp32 = np.divide(np.multiply(weight, self._s8_max), scales) + else: + raise ValueError( + "The size of weight scales vector ({}) does not match the dimensions ({}) of the weights tensor {}." + .format(scales.size, weight.shape, weight_var_name)) w_fp32 = w_fp32.reshape(weight.shape).astype(np.float32) self._restore_var(weight_var_name, w_fp32) diff --git a/python/paddle/fluid/contrib/slim/tests/test_quant2_int8_mkldnn_pass.py b/python/paddle/fluid/contrib/slim/tests/test_quant2_int8_mkldnn_pass.py index fcbb1b66ad1fd73a152b9128fa75a152baecd223..7b51973131496172d61b7ad968417eb41fa11c08 100644 --- a/python/paddle/fluid/contrib/slim/tests/test_quant2_int8_mkldnn_pass.py +++ b/python/paddle/fluid/contrib/slim/tests/test_quant2_int8_mkldnn_pass.py @@ -43,7 +43,7 @@ class TestQuant2Int8MkldnnPass(unittest.TestCase): self.conv_output = np.ndarray(self.conv_output_size).astype(self.dtype) self.conv_output2 = np.ndarray(self.conv_output2_size).astype( self.dtype) - self.quantized_ops = 'conv2d' + self.quantized_ops = 'conv2d,mul' self.variables = { "input": self.input, "filter": self.filter, @@ -51,6 +51,22 @@ class TestQuant2Int8MkldnnPass(unittest.TestCase): "conv_output": self.conv_output, "conv_output2": self.conv_output2, } + self.mul_input_size = [1, 3] + self.mul_weights_size = [3, 5] + self.mul_output_size = [1, 5] + self.mul_input = np.random.random(self.mul_input_size).astype( + self.dtype) + self.mul_weights = np.ones(self.mul_weights_size, self.dtype) + self.mul_weights_bad = np.ones([1, 1], self.dtype) + self.mul_output = np.ndarray(self.mul_output_size).astype(self.dtype) + self.mul_output_scale = np.linspace(1, 5, num=5).astype(self.dtype) + + self.variables_mul = { + "mul_input": self.mul_input, + "mul_weights": self.mul_weights, + "mul_output": self.mul_output, + "mul_weights_bad": self.mul_weights_bad + } def prepare_program(self, program): block = program.global_block() @@ -92,6 +108,23 @@ class TestQuant2Int8MkldnnPass(unittest.TestCase): 'fuse_brelu': True }) + def prepare_program_mul(self, program): + block = program.global_block() + for name in self.variables_mul: + block.create_var( + name=name, + dtype="float32", + shape=self.variables_mul[name].shape) + + mul_op1 = block.append_op( + type="mul", + inputs={ + "X": block.var('mul_input'), + "Y": block.var('mul_weights') + }, + outputs={"Out": block.var('mul_output')}, + attrs={'use_mkldnn': self.use_mkldnn}) + def remove_fuse_activation_attribute(self, graph): for op in graph.all_op_nodes(): op.op().remove_attr("fuse_activation") @@ -103,11 +136,13 @@ class TestQuant2Int8MkldnnPass(unittest.TestCase): def check_graph_after_pass(self, graph): for op in graph.all_op_nodes(): - self.assertTrue(op.op().has_attr("fuse_activation")) - if op.op().has_attr("fuse_relu") and op.op().attr("fuse_relu"): - self.assertTrue(op.op().attr("fuse_activation") == "relu") - if op.op().has_attr("fuse_brelu") and op.op().attr("fuse_brelu"): - self.assertTrue(op.op().attr("fuse_activation") == "relu6") + if op.op().type() == "conv2d": + self.assertTrue(op.op().has_attr("fuse_activation")) + if op.op().has_attr("fuse_relu") and op.op().attr("fuse_relu"): + self.assertTrue(op.op().attr("fuse_activation") == "relu") + if op.op().has_attr("fuse_brelu") and op.op().attr( + "fuse_brelu"): + self.assertTrue(op.op().attr("fuse_activation") == "relu6") def test_quant_update_activation(self): program = fluid.Program() @@ -125,6 +160,39 @@ class TestQuant2Int8MkldnnPass(unittest.TestCase): graph = quant2_int8_mkldnn_pass._update_activations(graph) self.check_graph_after_pass(graph) + def test_dequantize_op_weights(self): + program = fluid.Program() + with fluid.program_guard(program): + self.prepare_program_mul(program) + graph = IrGraph(core.Graph(program.desc), for_test=True) + + for op in graph.all_op_nodes(): + if op.op().type() == "mul": + op_node = op + break + + qpass = Quant2Int8MkldnnPass( + self.quantized_ops, + _scope=self.scope, + _place=self.place, + _core=core, + _debug=False) + qpass._weight_scales["mul_output"] = self.mul_output_scale + param = self.scope.var("mul_weights").get_tensor() + param.set(self.variables_mul["mul_weights"], self.place) + qpass._dequantize_op_weights(graph, op_node, "Y", "Out") + + assert np.allclose( + self.scope.find_var("mul_weights").get_tensor(), + [[127, 63.5, 42.3333, 31.75, 25.4], + [127, 63.5, 42.3333, 31.75, 25.4], + [127, 63.5, 42.3333, 31.75, 25.4]]) + + param = self.scope.var("mul_weights").get_tensor() + param.set(self.variables_mul["mul_weights_bad"], self.place) + with self.assertRaises(ValueError): + qpass._dequantize_op_weights(graph, op_node, "Y", "Out") + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/contrib/tests/test_correlation.py b/python/paddle/fluid/contrib/tests/test_correlation.py new file mode 100644 index 0000000000000000000000000000000000000000..7fcef4dbcd1efd3655b6339ed5ec880d8cd33fc0 --- /dev/null +++ b/python/paddle/fluid/contrib/tests/test_correlation.py @@ -0,0 +1,163 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import numpy as np +import paddle.fluid as fluid +from paddle.fluid.dygraph.base import to_variable + + +def corr(x_1, + x_2, + pad_size=4, + kernel_size=1, + max_displacement=4, + stride1=1, + stride2=1, + corr_multiply=1): + K = kernel_size + + rinput1 = np.pad(x_1, ((0, 0), (0, 0), (pad_size, pad_size), + (pad_size, pad_size)), + mode='constant') + rinput2 = np.pad(x_2, ((0, 0), (0, 0), (pad_size, pad_size), + (pad_size, pad_size)), + mode='constant') + rinput1 = np.transpose(rinput1, (0, 2, 3, 1)) + rinput2 = np.transpose(rinput2, (0, 2, 3, 1)) + B = int(rinput1.shape[0]) + H = int(x_1.shape[2]) + W = int(x_2.shape[3]) + d = max_displacement + D = 2 * d + 1 + output = np.zeros((B, D * D, H, W), dtype=np.float32) + + for b in range(B): + for i in range(H): + for j in range(W): + for k in range(-d, d + 1): + for l in range(-d, d + 1): + x1_index = i + pad_size + y1_index = j + pad_size + x2_index = x1_index + k + y2_index = y1_index + l + output[b, l + d + D * (k + d), i, j] = np.mean( + rinput1[b, x1_index:x1_index + K, y1_index:y1_index + + K] * rinput2[b, x2_index:x2_index + K, + y2_index:y2_index + K]) + + return output + + +class TestCorrelationOp(unittest.TestCase): + def test_check_output(self): + if not fluid.core.is_compiled_with_cuda(): + return + np.random.seed(13) + np.set_printoptions(threshold=np.inf) + x_shape = (2, 10, 3, 3) + x_type = 'float32' + x1 = fluid.layers.data( + name='x1', + shape=x_shape, + dtype=x_type, + append_batch_size=False, + stop_gradient=False) + x2 = fluid.layers.data( + name='x2', + shape=x_shape, + dtype=x_type, + append_batch_size=False, + stop_gradient=False) + + x1_np = np.random.randn(2, 3, 4, 5).astype(x_type) + x2_np = np.random.randn(2, 3, 4, 5).astype(x_type) + out_np = corr( + x1_np, + x2_np, + pad_size=4, + kernel_size=1, + max_displacement=4, + stride1=1, + stride2=1) + + out = fluid.contrib.correlation( + x1, + x2, + pad_size=4, + kernel_size=1, + max_displacement=4, + stride1=1, + stride2=1) + + loss = fluid.layers.reduce_mean(out) + optimizer = fluid.optimizer.Momentum(0.0001, 0.9) + optimizer.minimize(loss) + + place = fluid.CUDAPlace(0) + exe = fluid.Executor(place) + res = exe.run(feed={'x1': x1_np, + 'x2': x2_np}, + fetch_list=[out.name, loss.name]) + + self.assertTrue(np.allclose(res[0], out_np)) + + +class Net(fluid.dygraph.Layer): + def __init__(self, name_scope): + super(Net, self).__init__(name_scope) + + def forward(self, x1, x2): + y = fluid.contrib.correlation( + x1, + x2, + pad_size=4, + kernel_size=1, + max_displacement=4, + stride1=1, + stride2=1) + return y + + +class TestCorrelationOpDyGraph(unittest.TestCase): + def test_check_output(self): + if not fluid.core.is_compiled_with_cuda(): + return + np.random.seed(13) + np.set_printoptions(threshold=np.inf) + x_shape = (2, 10, 3, 3) + x_type = 'float32' + place = fluid.CUDAPlace(0) + with fluid.dygraph.guard(place): + x1_np = np.random.randn(2, 3, 4, 5).astype(x_type) + x2_np = np.random.randn(2, 3, 4, 5).astype(x_type) + out_np = corr( + x1_np, + x2_np, + pad_size=4, + kernel_size=1, + max_displacement=4, + stride1=1, + stride2=1) + + x1 = to_variable(x1_np) + x2 = to_variable(x2_np) + corr_pd = Net('corr_pd') + y = corr_pd(x1, x2) + out = y.numpy() + self.assertTrue(np.allclose(out, out_np)) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/fluid/dataloader/dataloader_iter.py b/python/paddle/fluid/dataloader/dataloader_iter.py index 6a996493e4df1e1facc6ccd205a8ae5105f92c5b..1ef0d494e0725084b0ddfddcafe93d49da0525d7 100644 --- a/python/paddle/fluid/dataloader/dataloader_iter.py +++ b/python/paddle/fluid/dataloader/dataloader_iter.py @@ -347,6 +347,92 @@ class _DataLoaderIterSingleProcess(_DataLoaderIterBase): return self.__next__() +# NOTE(chenweihang): _worker_loop must be top level method to be pickled +def _worker_loop(dataset, dataset_kind, indices_queue, out_queue, done_event, + collate_fn, init_fn, worker_id, num_workers, + use_shared_memory): + try: + # NOTE: [ mmap files clear ] When the child process exits unexpectedly, + # some shared memory objects may have been applied for but have not yet + # been put into the inter-process Queue. This part of the object needs + # to be cleaned up when the process ends. + CleanupFuncRegistrar.register(_cleanup_mmap) + + # set signal handler + core._set_process_signal_handler() + + global _worker_info + _worker_info = WorkerInfo( + id=worker_id, num_workers=num_workers, dataset=dataset) + + init_exception = None + try: + if init_fn is not None: + init_fn(worker_id) + fetcher = _DatasetKind.create_fetcher(dataset_kind, dataset, + collate_fn, True) + except: + init_exception = Exception("init_fn failed in worker {}: " \ + "{}".format(worker_id, sys.exc_info())) + + iterator_drained = False + parent_watch_dog = ParentWatchDog() + + while parent_watch_dog.is_alive(): + try: + data = indices_queue.get(MP_INDICES_CHECK_INTERVAL) + except queue.Empty: + continue + + # None as poison piil, so worker event should be set + if data is None: + assert done_event.is_set() or iterator_drained, \ + "get None when worker done_event set" + break + # If worker done event is set but get still get data in + # indices_queue, remaining data should be get and skipped. + if done_event.is_set() or iterator_drained: + continue + + idx, indices = data + try: + if init_exception is not None: + batch = init_exception + init_exception = None + else: + batch = fetcher.fetch(indices) + except Exception as e: + if isinstance( + e, StopIteration) and dataset_kind == _DatasetKind.ITER: + out_queue.put(_IterableDatasetStopIteration(worker_id)) + iterator_drained = True + else: + out_queue.put((idx, e)) + else: + if use_shared_memory: + # FIXME(dkp): _convert_to_tensor_list only support np.array + # list now, should support paddle.Tensor list + if isinstance(batch[0][0], paddle.Tensor): + np_batch = [] + for sample in batch: + np_batch.append([s.numpy() for s in sample]) + batch = np_batch + + tensor_list = core._convert_to_tensor_list(batch) + out_queue.put((idx, tensor_list)) + core._remove_tensor_list_mmap_fds(tensor_list) + else: + out_queue.put((idx, batch)) + except KeyboardInterrupt: + # NOTE: Main process will raise KeyboardInterrupt anyways, ignore it in child process + pass + except: + six.reraise(*sys.exc_info()) + finally: + if use_shared_memory: + _cleanup_mmap() + + class _DataLoaderIterMultiProcess(_DataLoaderIterBase): def __init__(self, loader): super(_DataLoaderIterMultiProcess, self).__init__(loader) @@ -404,11 +490,11 @@ class _DataLoaderIterMultiProcess(_DataLoaderIterBase): indices_queue = multiprocessing.Queue() self._indices_queues.append(indices_queue) worker = multiprocessing.Process( - target=self._worker_loop, + target=_worker_loop, args=(self._dataset, self._dataset_kind, indices_queue, self._data_queue, self._workers_done_event, self._collate_fn, self._worker_init_fn, i, - self._num_workers)) + self._num_workers, self._use_shared_memory)) worker.daemon = True worker.start() self._workers.append(worker) @@ -483,90 +569,6 @@ class _DataLoaderIterMultiProcess(_DataLoaderIterBase): self._blocking_queue.kill() logging.error("DataLoader reader thread raised an exception!") - def _worker_loop(self, dataset, dataset_kind, indices_queue, out_queue, - done_event, collate_fn, init_fn, worker_id, num_workers): - try: - # NOTE: [ mmap files clear ] When the child process exits unexpectedly, - # some shared memory objects may have been applied for but have not yet - # been put into the inter-process Queue. This part of the object needs - # to be cleaned up when the process ends. - CleanupFuncRegistrar.register(_cleanup_mmap) - - # set signal handler - core._set_process_signal_handler() - - global _worker_info - _worker_info = WorkerInfo( - id=worker_id, num_workers=num_workers, dataset=dataset) - - init_exception = None - try: - if init_fn is not None: - init_fn(worker_id) - fetcher = _DatasetKind.create_fetcher(dataset_kind, dataset, - collate_fn, True) - except: - init_exception = Exception("init_fn failed in worker {}: " \ - "{}".format(worker_id, sys.exc_info())) - - iterator_drained = False - parent_watch_dog = ParentWatchDog() - - while parent_watch_dog.is_alive(): - try: - data = indices_queue.get(MP_INDICES_CHECK_INTERVAL) - except queue.Empty: - continue - - # None as poison piil, so worker event should be set - if data is None: - assert done_event.is_set() or iterator_drained, \ - "get None when worker done_event set" - break - # If worker done event is set but get still get data in - # indices_queue, remaining data should be get and skipped. - if done_event.is_set() or iterator_drained: - continue - - idx, indices = data - try: - if init_exception is not None: - batch = init_exception - init_exception = None - else: - batch = fetcher.fetch(indices) - except Exception as e: - if isinstance( - e, - StopIteration) and dataset_kind == _DatasetKind.ITER: - out_queue.put(_IterableDatasetStopIteration(worker_id)) - iterator_drained = True - else: - out_queue.put((idx, e)) - else: - if self._use_shared_memory: - # FIXME(dkp): _convert_to_tensor_list only support np.array - # list now, should support paddle.Tensor list - if isinstance(batch[0][0], paddle.Tensor): - np_batch = [] - for sample in batch: - np_batch.append([s.numpy() for s in sample]) - batch = np_batch - - tensor_list = core._convert_to_tensor_list(batch) - out_queue.put((idx, tensor_list)) - core._remove_tensor_list_mmap_fds(tensor_list) - else: - out_queue.put((idx, batch)) - except KeyboardInterrupt: - # NOTE: Main process will raise KeyboardInterrupt anyways, ignore it in child process - pass - except: - six.reraise(*sys.exc_info()) - finally: - if self._use_shared_memory: - _cleanup_mmap() - def _thread_loop(self): while not self._thread_done_event.is_set(): batch = self._get_data() diff --git a/python/paddle/fluid/dygraph/amp/loss_scaler.py b/python/paddle/fluid/dygraph/amp/loss_scaler.py index 8f3ca9ec007ef5c1ab8769dde741a5d2b3697600..ff57f30dcd2ec73d55ff06e751767deea0a2eead 100644 --- a/python/paddle/fluid/dygraph/amp/loss_scaler.py +++ b/python/paddle/fluid/dygraph/amp/loss_scaler.py @@ -210,13 +210,12 @@ class AmpScaler(object): def _unscale(self, optimizer): if not self._enable: return - inv_scale = 1.0 / self._scale param_grads = [ param._grad_ivar() for param in optimizer._parameter_list if param._grad_ivar() is not None ] - core.ops.amp_check_finite_and_scale(param_grads, inv_scale, param_grads, - self._found_inf) + core.ops.check_finite_and_unscale(param_grads, self._scale, param_grads, + self._found_inf) def _update(self): """ diff --git a/python/paddle/fluid/dygraph/base.py b/python/paddle/fluid/dygraph/base.py index c548bdfeba19510b26c0f80d356fa6a6b7bbaed7..2f95c2b9007a53483fda86dda8d77e9baff0d8d2 100644 --- a/python/paddle/fluid/dygraph/base.py +++ b/python/paddle/fluid/dygraph/base.py @@ -25,6 +25,7 @@ from .tracer import Tracer import logging import objgraph from ..data_feeder import convert_dtype +import warnings __all__ = [ 'no_grad', 'no_grad_', 'grad', 'guard', 'enable_dygraph', 'disable_dygraph', @@ -609,10 +610,10 @@ def to_variable(value, name=None, zero_copy=None, dtype=None): uint8, uint16, complex64, complex128}. name(str, optional): The default value is None. Normally there is no need for user to set this property. For more information, please - refer to :ref:`api_guide_Name` . + refer to :ref:`api_guide_Name` . zero_copy(bool, optional): Whether to share memory with the input numpy array. This parameter only works with CPUPlace and will be set to - True when it is None. Default: None. + True when it is None. Default: None. (Note: zero_copy is discarded temporally for some reason.) dtype(str, optional): The desired data type of returned ``Variable`` . Can be 'bool' , 'float16' , 'float32' , 'float64' , 'int8' , 'int16' , 'int32' , 'int64' , 'uint8' . Default: None. @@ -665,8 +666,17 @@ def to_variable(value, name=None, zero_copy=None, dtype=None): else: if isinstance(framework._current_expected_place(), framework.core.CPUPlace): - if zero_copy is None: - zero_copy = True + #TODO(zhiqiu): we found two problems when enable zero_copy on CPUPlace. + # (1): eigen requires 16-bytes alignments, but the data of numpy array may not statisfy. + # Details: https://eigen.tuxfamily.org/dox/group__TopicUnalignedArrayAssert.html + # (2): when used in flask framework, it may result in hang. + # Details: https://github.com/PaddlePaddle/Paddle/issues/26635 + # So, we temporally diable the zero_copy strategy. + if zero_copy == True: + warnings.warn( + "Currently, zero_copy is not supported, and it will be discarded." + ) + zero_copy = False else: assert not zero_copy, "zero_copy mode can only be used with CPUPlace" diff --git a/python/paddle/fluid/dygraph/checkpoint.py b/python/paddle/fluid/dygraph/checkpoint.py index f85b184f68111bbc0930b36e2ba6e05c2dbd006a..9876fc620b870f47b10e9f99e4de34f5cb81fde1 100644 --- a/python/paddle/fluid/dygraph/checkpoint.py +++ b/python/paddle/fluid/dygraph/checkpoint.py @@ -24,8 +24,8 @@ from . import learning_rate_scheduler import warnings from .. import core from .base import guard -from paddle.fluid.dygraph.jit import SaveLoadConfig -from paddle.fluid.dygraph.io import _construct_program_holders, _construct_params_and_buffers +from paddle.fluid.dygraph.jit import SaveLoadConfig, deprecate_save_load_configs +from paddle.fluid.dygraph.io import _construct_program_holders, _construct_params_and_buffers, EXTRA_VAR_INFO_FILENAME __all__ = [ 'save_dygraph', @@ -42,9 +42,9 @@ def deprecate_keep_name_table(func): warnings.warn( "The argument `keep_name_table` has deprecated, please use `SaveLoadConfig.keep_name_table`.", DeprecationWarning) - configs = SaveLoadConfig() - configs.keep_name_table = keep_name_table - return configs + config = SaveLoadConfig() + config.keep_name_table = keep_name_table + return config # deal with arg `keep_name_table` if len(args) > 1 and isinstance(args[1], bool): @@ -52,7 +52,7 @@ def deprecate_keep_name_table(func): args[1] = __warn_and_build_configs__(args[1]) # deal with kwargs elif 'keep_name_table' in kwargs: - kwargs['configs'] = __warn_and_build_configs__(kwargs[ + kwargs['config'] = __warn_and_build_configs__(kwargs[ 'keep_name_table']) kwargs.pop('keep_name_table') else: @@ -135,8 +135,9 @@ def save_dygraph(state_dict, model_path): # TODO(qingqing01): remove dygraph_only to support loading static model. # maybe need to unify the loading interface after 2.0 API is ready. # @dygraph_only +@deprecate_save_load_configs @deprecate_keep_name_table -def load_dygraph(model_path, configs=None): +def load_dygraph(model_path, config=None): ''' :api_attr: imperative @@ -151,7 +152,7 @@ def load_dygraph(model_path, configs=None): Args: model_path(str) : The file prefix store the state_dict. (The path should Not contain suffix '.pdparams') - configs (SaveLoadConfig, optional): :ref:`api_imperative_jit_saveLoadConfig` + config (SaveLoadConfig, optional): :ref:`api_imperative_jit_saveLoadConfig` object that specifies additional configuration options, these options are for compatibility with ``jit.save/io.save_inference_model`` formats. Default None. @@ -195,6 +196,7 @@ def load_dygraph(model_path, configs=None): opti_file_path = model_prefix + ".pdopt" # deal with argument `configs` + configs = config if configs is None: configs = SaveLoadConfig() @@ -231,6 +233,19 @@ def load_dygraph(model_path, configs=None): para_dict = dict() for var_name in persistable_var_dict: para_dict[var_name] = persistable_var_dict[var_name].numpy() + + # if __variables.info__ exists, we can recover structured_name + var_info_path = os.path.join(model_prefix, EXTRA_VAR_INFO_FILENAME) + if os.path.exists(var_info_path): + with open(var_info_path, 'rb') as f: + extra_var_info = pickle.load(f) + structured_para_dict = dict() + for var_name in para_dict: + structured_name = extra_var_info[var_name].get( + 'structured_name', None) + assert structured_name is not None, "Cannot find saved variable (%s)'s structured name in saved model." % var_name + structured_para_dict[structured_name] = para_dict[var_name] + para_dict = structured_para_dict else: # Load state dict by `save_dygraph` save format para_dict = {} diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/error.py b/python/paddle/fluid/dygraph/dygraph_to_static/error.py index 5aba7ca0fdc0cfda5d79f5a66d78785df49c0baf..be21ab6d5394ed5f89c23988a9405b57e05b56fb 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/error.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/error.py @@ -13,6 +13,7 @@ # limitations under the License. import os +import six import sys import traceback @@ -20,6 +21,14 @@ from paddle.fluid.dygraph.dygraph_to_static.origin_info import Location, OriginI ERROR_DATA = "Error data about original source code information and traceback." +# A flag to set whether to open the dygraph2static error reporting module +SIMPLIFY_ERROR_ENV_NAME = "TRANSLATOR_SIMPLIFY_NEW_ERROR" +DEFAULT_SIMPLIFY_NEW_ERROR = 1 + +# A flag to set whether to display the simplified error stack +DISABLE_ERROR_ENV_NAME = "TRANSLATOR_DISABLE_NEW_ERROR" +DEFAULT_DISABLE_NEW_ERROR = 0 + def attach_error_data(error, in_runtime=False): """ @@ -103,7 +112,10 @@ class ErrorData(object): # Simplify error value to improve readability if error is raised in runtime if self.in_runtime: - self._simplify_error_value() + if int( + os.getenv(SIMPLIFY_ERROR_ENV_NAME, + DEFAULT_SIMPLIFY_NEW_ERROR)): + self._simplify_error_value() message_lines.append(str(self.error_value)) return '\n'.join(message_lines) @@ -150,3 +162,22 @@ class ErrorData(object): error_value_str = '\n'.join(error_value_lines) self.error_value = self.error_type(error_value_str) + + def raise_new_exception(self): + + # Raises the origin error if disable dygraph2static error module, + if int(os.getenv(DISABLE_ERROR_ENV_NAME, DEFAULT_DISABLE_NEW_ERROR)): + raise + + new_exception = self.create_exception() + if six.PY3: + # NOTE(liym27): + # 1. Why `raise new_exception from None`? + # In Python 3, by default, an new exception is raised with trace information of the caught exception. + # This only raises new_exception and hides unwanted implementation details from tracebacks of the + # caught exception. + # 2. Use exec to bypass syntax error checking in Python 2. + + six.exec_("raise new_exception from None") + else: + raise new_exception diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/function_spec.py b/python/paddle/fluid/dygraph/dygraph_to_static/function_spec.py index 90e38bd98863ff62174bd569a483b11984480b5a..37ce8b0a152ff8e258e8aee2a54ed7215f77c146 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/function_spec.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/function_spec.py @@ -135,6 +135,11 @@ class FunctionSpec(object): input_with_spec = pack_sequence_as(args, input_with_spec) + # If without specificing name in input_spec, add default name + # according to argument name from decorated function. + input_with_spec = replace_spec_empty_name(self._arg_names, + input_with_spec) + return input_with_spec @switch_to_static_graph @@ -309,3 +314,61 @@ def convert_to_input_spec(inputs, input_spec): raise TypeError( "The type(input_spec) should be a `InputSpec` or dict/list/tuple of it, but received {}.". type_name(input_spec)) + + +def replace_spec_empty_name(args_name, input_with_spec): + """ + Adds default name according to argument name from decorated function + if without specificing InputSpec.name + + The naming rule are as followed: + 1. If InputSpec.name is not None, do nothing. + 2. If each argument `x` corresponds to an InputSpec, using the argument name like `x` + 3. If the arguments `inputs` corresponds to a list(InputSpec), using name like `inputs_0`, `inputs_1` + 4. If the arguments `input_dic` corresponds to a dict(InputSpec), using key as name. + + For example: + + # case 1: foo(x, y) + foo = to_static(foo, input_spec=[InputSpec([None, 10]), InputSpec([None])]) + print([in_var.name for in_var in foo.inputs]) # [x, y] + + # case 2: foo(inputs) where inputs is a list + foo = to_static(foo, input_spec=[[InputSpec([None, 10]), InputSpec([None])]]) + print([in_var.name for in_var in foo.inputs]) # [inputs_0, inputs_1] + + # case 3: foo(inputs) where inputs is a dict + foo = to_static(foo, input_spec=[{'x': InputSpec([None, 10]), 'y': InputSpec([None])}]) + print([in_var.name for in_var in foo.inputs]) # [x, y] + """ + input_with_spec = list(input_with_spec) + candidate_arg_names = args_name[:len(input_with_spec)] + + for i, arg_name in enumerate(candidate_arg_names): + input_spec = input_with_spec[i] + input_with_spec[i] = _replace_spec_name(arg_name, input_spec) + + return input_with_spec + + +def _replace_spec_name(name, input_spec): + """ + Replaces InputSpec.name with given `name` while not specificing it. + """ + if isinstance(input_spec, paddle.static.InputSpec): + if input_spec.name is None: + input_spec.name = name + return input_spec + elif isinstance(input_spec, (list, tuple)): + processed_specs = [] + for i, spec in enumerate(input_spec): + new_name = "{}_{}".format(name, i) + processed_specs.append(_replace_spec_name(new_name, spec)) + return processed_specs + elif isinstance(input_spec, dict): + processed_specs = {} + for key, spec in six.iteritems(input_spec): + processed_specs[key] = _replace_spec_name(key, spec) + return processed_specs + else: + return input_spec diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/origin_info.py b/python/paddle/fluid/dygraph/dygraph_to_static/origin_info.py index 13f38b0726c27566ff0eda41d6c365e6a7e4aa4b..76e732d4d37f6a2056afba72649077acf16ba30e 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/origin_info.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/origin_info.py @@ -124,8 +124,13 @@ class OriginInfoAttacher(gast.NodeTransformer): def _abs_lineno(self, node): # NOTE(liym27): - # If the first gast.FunctionDef has decorator, its lineno is 1, which - # equals to the lineno of the first decorator node. + # There are differences in ast_node.lineno between PY3.8+ and PY3.8-. + # If the first gast.FunctionDef has decorator, the lineno of gast.FunctionDef is differs. + # 1. < PY3.8 + # its lineno equals to the lineno of the first decorator node, which is not right. + # 2. >= PY3.8 + # its lineno is the actual lineno, which is right. + return self.lineno_offset + node.lineno def _abs_col_offset(self, node): diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/program_translator.py b/python/paddle/fluid/dygraph/dygraph_to_static/program_translator.py index 3d27810f1db94c4f6c273399ec93b9335f5bb03a..e5fce3e6ede1511458f8da916165738d9e842d1a 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/program_translator.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/program_translator.py @@ -32,8 +32,7 @@ from paddle.fluid.layers.utils import flatten from paddle.fluid.dygraph.base import param_guard from paddle.fluid.dygraph.base import switch_to_static_graph from paddle.fluid.dygraph.dygraph_to_static import DygraphToStaticAst -from paddle.fluid.dygraph.dygraph_to_static.error import ERROR_DATA -from paddle.fluid.dygraph.dygraph_to_static.error import attach_error_data +from paddle.fluid.dygraph.dygraph_to_static import error from paddle.fluid.dygraph.dygraph_to_static import logging_utils from paddle.fluid.dygraph.dygraph_to_static.origin_info import attach_origin_info from paddle.fluid.dygraph.dygraph_to_static.origin_info import create_and_update_origin_info_map @@ -315,6 +314,7 @@ class StaticLayer(object): # 2. trace ops from dygraph layers and cache the generated program. args, kwargs = self._function_spec.unified_args_and_kwargs(args, kwargs) + try: concrete_program, partial_program_layer = self.get_concrete_program( *args, **kwargs) @@ -324,27 +324,22 @@ class StaticLayer(object): partial_program_layer.training = self._class_instance.training # 4. return outputs. - return partial_program_layer(args) + try: + return partial_program_layer(args) + except Exception as e: + if not hasattr(e, error.ERROR_DATA): + # runtime error + error.attach_error_data(e, in_runtime=True) + raise except Exception as e: - if not hasattr(e, ERROR_DATA): - # runtime error - attach_error_data(e, in_runtime=True) - error_data = getattr(e, ERROR_DATA, None) + error_data = getattr(e, error.ERROR_DATA, None) if error_data: - new_exception = error_data.create_exception() - if six.PY3: - # NOTE(liym27): - # 1. Why `raise new_exception from None`? - # In Python 3, by default, an new exception is raised with trace information of the caught exception. - # This only raises new_exception and hides unwanted implementation details from tracebacks of the - # caught exception. - # 2. Use exec to bypass syntax error checking in Python 2. - - six.exec_("raise new_exception from None") - else: - raise new_exception + error_data.raise_new_exception() else: - raise + logging_utils.warn( + "Please file an issue at 'https://github.com/PaddlePaddle/Paddle/issues'" + " if you can't handle this {} yourself.".format(type(e))) + raise e def _call_dygraph_function(self, *args, **kwargs): """ @@ -593,7 +588,7 @@ class ConcreteProgram(object): outputs = static_func(*inputs) except BaseException as e: # NOTE: If e is raised in compile time, e should be attached to ERROR_DATA here. - attach_error_data(e) + error.attach_error_data(e) raise if not isinstance(outputs, @@ -813,28 +808,36 @@ class ProgramTranslator(object): "The ProgramTranslator.get_output doesn't work when setting ProgramTranslator.enable = False. " "We will just return dygraph output.") return dygraph_func(*args, **kwargs) - - function_spec = FunctionSpec(dygraph_func) - cache_key = CacheKey.from_func_and_args(function_spec, args, kwargs, - getattr(dygraph_func, - '__self__', None)) - _, partial_program_layer = self._program_cache[cache_key] - - if args and isinstance(args[0], layers.Layer): - # Synchronize self.training attribute. - partial_program_layer.training = args[0].training - args = args[1:] try: - return partial_program_layer(args) - + function_spec = FunctionSpec(dygraph_func) + cache_key = CacheKey.from_func_and_args(function_spec, args, kwargs, + getattr(dygraph_func, + '__self__', None)) + _, partial_program_layer = self._program_cache[cache_key] + + if args and isinstance(args[0], layers.Layer): + # Synchronize self.training attribute. + partial_program_layer.training = args[0].training + args = args[1:] + try: + return partial_program_layer(args) + except BaseException as e: + # NOTE: + # 1. If e is raised in compile time, e should have been attached to ERROR_DATA before; + # 2. If e raised in runtime, e should be attached to ERROR_DATA here. + if not hasattr(e, error.ERROR_DATA): + # runtime error + error.attach_error_data(e, in_runtime=True) + raise except BaseException as e: - # NOTE: - # 1. If e is raised in compile time, e should have been attached to ERROR_DATA before; - # 2. If e raised in runtime, e should be attached to ERROR_DATA here. - if not hasattr(e, ERROR_DATA): - # runtime error - attach_error_data(e, in_runtime=True) - raise + error_data = getattr(e, error.ERROR_DATA, None) + if error_data: + error_data.raise_new_exception() + else: + logging_utils.warn( + "Please file an issue at 'https://github.com/PaddlePaddle/Paddle/issues'" + " if you can't handle this {} yourself.".format(type(e))) + raise e def get_func(self, dygraph_func): """ diff --git a/python/paddle/fluid/dygraph/io.py b/python/paddle/fluid/dygraph/io.py index 1d2ea142c7d5f2e653e446986a39d1bc155006f0..335ac500c898085e4bf60aabdf8db95fa65db31f 100644 --- a/python/paddle/fluid/dygraph/io.py +++ b/python/paddle/fluid/dygraph/io.py @@ -479,11 +479,15 @@ def _load_persistable_vars(model_path, var_file_path = os.path.join(model_path, params_filename) else: var_file_path = os.path.join(model_path, VARIABLE_FILENAME) - framework._dygraph_tracer().trace_op( - type='load_combine', - inputs={}, - outputs={'Out': load_var_list}, - attrs={'file_path': var_file_path}) + if not os.path.exists(var_file_path): + if len(extra_var_info) != 0: + raise ValueError("The model to be loaded is incomplete.") + else: + framework._dygraph_tracer().trace_op( + type='load_combine', + inputs={}, + outputs={'Out': load_var_list}, + attrs={'file_path': var_file_path}) return load_var_dict diff --git a/python/paddle/fluid/dygraph/jit.py b/python/paddle/fluid/dygraph/jit.py index 9f4ec2b55bc6b56fc796d3124edf1ec0deb3f23e..57864efec8a9447cca0be94f0f1b433c18435376 100644 --- a/python/paddle/fluid/dygraph/jit.py +++ b/python/paddle/fluid/dygraph/jit.py @@ -17,6 +17,8 @@ from __future__ import print_function import os import pickle import warnings +import functools +from collections import OrderedDict import six import paddle @@ -36,7 +38,7 @@ from paddle.fluid.wrapped_decorator import wrap_decorator __all__ = [ 'TracedLayer', 'declarative', 'dygraph_to_static_func', 'set_code_level', - 'set_verbosity' + 'set_verbosity', 'save', 'load', 'SaveLoadConfig' ] @@ -210,7 +212,16 @@ def declarative(function=None, input_spec=None): # for usage: `declarative(foo, ...)` if function is not None: - return decorated(function) + if isinstance(function, Layer): + if isinstance(function.forward, StaticLayer): + class_name = function.__class__.__name__ + warnings.warn( + "`{}.forward` has already been decorated somewhere. It will be redecorated to replace previous one.". + format(class_name)) + function.forward = decorated(function.forward) + return function + else: + return decorated(function) # for usage: `@declarative` return decorated @@ -228,63 +239,60 @@ class SaveLoadConfig(object): .. code-block:: python - import numpy as np - import paddle.fluid as fluid - from paddle.fluid.dygraph import Linear - from paddle.fluid.dygraph import declarative + import paddle + import paddle.nn as nn + import paddle.optimizer as opt - class SimpleNet(fluid.dygraph.Layer): + class SimpleNet(nn.Layer): def __init__(self, in_size, out_size): super(SimpleNet, self).__init__() - self._linear = Linear(in_size, out_size) + self._linear = nn.Linear(in_size, out_size) - @declarative + @paddle.jit.to_static def forward(self, x): y = self._linear(x) z = self._linear(y) return z # enable dygraph mode - fluid.enable_dygraph() + paddle.disable_static() # train model net = SimpleNet(8, 8) - adam = fluid.optimizer.AdamOptimizer(learning_rate=0.1, parameter_list=net.parameters()) - x = fluid.dygraph.to_variable(np.random.random((4, 8)).astype('float32')) + adam = opt.Adam(learning_rate=0.1, parameters=net.parameters()) + x = paddle.randn([4, 8], 'float32') for i in range(10): out = net(x) - loss = fluid.layers.mean(out) + loss = paddle.tensor.mean(out) loss.backward() - adam.minimize(loss) - net.clear_gradients() + adam.step() + adam.clear_grad() # use SaveLoadconfig when saving model model_path = "simplenet.example.model" - configs = fluid.dygraph.jit.SaveLoadConfig() - configs.model_filename = "__simplenet__" - fluid.dygraph.jit.save( + config = paddle.SaveLoadConfig() + config.model_filename = "__simplenet__" + paddle.jit.save( layer=net, model_path=model_path, - input_spec=[x], - configs=configs) + config=config) 2. Using ``SaveLoadConfig`` when loading model .. code-block:: python - import numpy as np - import paddle.fluid as fluid + import paddle # enable dygraph mode - fluid.enable_dygraph() + paddle.disable_static() # use SaveLoadconfig when loading model model_path = "simplenet.example.model" - configs = fluid.dygraph.jit.SaveLoadConfig() - configs.model_filename = "__simplenet__" - infer_net = fluid.dygraph.jit.load(model_path, configs=configs) + config = paddle.SaveLoadConfig() + config.model_filename = "__simplenet__" + infer_net = paddle.jit.load(model_path, config=config) # inference - x = fluid.dygraph.to_variable(np.random.random((4, 8)).astype('float32')) + x = paddle.randn([4, 8], 'float32') pred = infer_net(x) """ @@ -324,51 +332,46 @@ class SaveLoadConfig(object): Examples: .. code-block:: python - import numpy as np - import paddle.fluid as fluid - from paddle.fluid.dygraph import Linear - from paddle.fluid.dygraph import declarative + import paddle + import paddle.nn as nn + import paddle.optimizer as opt - class SimpleNet(fluid.dygraph.Layer): + class SimpleNet(nn.Layer): def __init__(self, in_size, out_size): super(SimpleNet, self).__init__() - self._linear = Linear(in_size, out_size) + self._linear = nn.Linear(in_size, out_size) - @declarative + @paddle.jit.to_static def forward(self, x): y = self._linear(x) z = self._linear(y) - loss = fluid.layers.mean(z) + loss = paddle.tensor.mean(z) return z, loss # enable dygraph mode - fluid.enable_dygraph() + paddle.disable_static() # train model net = SimpleNet(8, 8) - adam = fluid.optimizer.AdamOptimizer(learning_rate=0.1, parameter_list=net.parameters()) - x = fluid.dygraph.to_variable(np.random.random((4, 8)).astype('float32')) + adam = opt.Adam(learning_rate=0.1, parameters=net.parameters()) + x = paddle.randn([4, 8], 'float32') for i in range(10): out, loss = net(x) loss.backward() - adam.minimize(loss) - net.clear_gradients() + adam.step() + adam.clear_grad() # use SaveLoadconfig.output_spec model_path = "simplenet.example.model.output_spec" - configs = fluid.dygraph.jit.SaveLoadConfig() - # only keep the predicted output in saved model, discard loss - configs.output_spec = [out] - - fluid.dygraph.jit.save( + config = paddle.SaveLoadConfig() + config.output_spec = [out] + paddle.jit.save( layer=net, model_path=model_path, - input_spec=[x], - configs=configs) + config=config) - infer_net = fluid.dygraph.jit.load(model_path, configs=configs) - x = fluid.dygraph.to_variable(np.random.random((4, 8)).astype('float32')) - # only have the predicted output + infer_net = paddle.jit.load(model_path) + x = paddle.randn([4, 8], 'float32') pred = infer_net(x) """ return self._output_spec @@ -395,52 +398,47 @@ class SaveLoadConfig(object): Examples: .. code-block:: python - import numpy as np - import paddle.fluid as fluid - from paddle.fluid.dygraph import Linear - from paddle.fluid.dygraph import declarative + import paddle + import paddle.nn as nn + import paddle.optimizer as opt - class SimpleNet(fluid.dygraph.Layer): + class SimpleNet(nn.Layer): def __init__(self, in_size, out_size): super(SimpleNet, self).__init__() - self._linear = Linear(in_size, out_size) + self._linear = nn.Linear(in_size, out_size) - @declarative + @paddle.jit.to_static def forward(self, x): y = self._linear(x) z = self._linear(y) return z # enable dygraph mode - fluid.enable_dygraph() + paddle.disable_static() # train model net = SimpleNet(8, 8) - adam = fluid.optimizer.AdamOptimizer(learning_rate=0.1, parameter_list=net.parameters()) - x = fluid.dygraph.to_variable(np.random.random((4, 8)).astype('float32')) + adam = opt.Adam(learning_rate=0.1, parameters=net.parameters()) + x = paddle.randn([4, 8], 'float32') for i in range(10): out = net(x) - loss = fluid.layers.mean(out) + loss = paddle.tensor.mean(out) loss.backward() - adam.minimize(loss) - net.clear_gradients() - - model_path = "simplenet.example.model.model_filename" - configs = fluid.dygraph.jit.SaveLoadConfig() - configs.model_filename = "__simplenet__" + adam.step() + adam.clear_grad() # saving with configs.model_filename - fluid.dygraph.jit.save( + model_path = "simplenet.example.model.model_filename" + config = paddle.SaveLoadConfig() + config.model_filename = "__simplenet__" + paddle.jit.save( layer=net, model_path=model_path, - input_spec=[x], - configs=configs) - # [result] the saved model directory contains: - # __simplenet__ __variables__ __variables.info__ + config=config) # loading with configs.model_filename - infer_net = fluid.dygraph.jit.load(model_path, configs=configs) - x = fluid.dygraph.to_variable(np.random.random((4, 8)).astype('float32')) + infer_net = paddle.jit.load(model_path, config=config) + x = paddle.randn([4, 8], 'float32') pred = infer_net(x) """ return self._model_filename @@ -465,52 +463,48 @@ class SaveLoadConfig(object): Examples: .. code-block:: python - import numpy as np - import paddle.fluid as fluid - from paddle.fluid.dygraph import Linear - from paddle.fluid.dygraph import declarative + import paddle + import paddle.nn as nn + import paddle.optimizer as opt - class SimpleNet(fluid.dygraph.Layer): + class SimpleNet(nn.Layer): def __init__(self, in_size, out_size): super(SimpleNet, self).__init__() - self._linear = Linear(in_size, out_size) + self._linear = nn.Linear(in_size, out_size) - @declarative + @paddle.jit.to_static def forward(self, x): y = self._linear(x) z = self._linear(y) return z # enable dygraph mode - fluid.enable_dygraph() + paddle.disable_static() # train model net = SimpleNet(8, 8) - adam = fluid.optimizer.AdamOptimizer(learning_rate=0.1, parameter_list=net.parameters()) - x = fluid.dygraph.to_variable(np.random.random((4, 8)).astype('float32')) + adam = opt.Adam(learning_rate=0.1, parameters=net.parameters()) + x = paddle.randn([4, 8], 'float32') for i in range(10): out = net(x) - loss = fluid.layers.mean(out) + loss = paddle.tensor.mean(out) loss.backward() - adam.minimize(loss) - net.clear_gradients() + adam.step() + adam.clear_grad() model_path = "simplenet.example.model.params_filename" - configs = fluid.dygraph.jit.SaveLoadConfig() - configs.params_filename = "__params__" + config = paddle.SaveLoadConfig() + config.params_filename = "__params__" # saving with configs.params_filename - fluid.dygraph.jit.save( + paddle.jit.save( layer=net, model_path=model_path, - input_spec=[x], - configs=configs) - # [result] the saved model directory contains: - # __model__ __params__ __variables.info__ + config=config) # loading with configs.params_filename - infer_net = fluid.dygraph.jit.load(model_path, configs=configs) - x = fluid.dygraph.to_variable(np.random.random((4, 8)).astype('float32')) + infer_net = paddle.jit.load(model_path, config=config) + x = paddle.randn([4, 8], 'float32') pred = infer_net(x) """ return self._params_filename @@ -544,52 +538,50 @@ class SaveLoadConfig(object): Examples: .. code-block:: python - import numpy as np - import paddle.fluid as fluid - from paddle.fluid.dygraph import Linear - from paddle.fluid.dygraph import declarative + import paddle + import paddle.nn as nn + import paddle.optimizer as opt - class SimpleNet(fluid.dygraph.Layer): + class SimpleNet(nn.Layer): def __init__(self, in_size, out_size): super(SimpleNet, self).__init__() - self._linear = Linear(in_size, out_size) + self._linear = nn.Linear(in_size, out_size) - @declarative + @paddle.jit.to_static def forward(self, x): y = self._linear(x) z = self._linear(y) return z # enable dygraph mode - fluid.enable_dygraph() + paddle.disable_static() # train model net = SimpleNet(8, 8) - adam = fluid.optimizer.AdamOptimizer(learning_rate=0.1, parameter_list=net.parameters()) - x = fluid.dygraph.to_variable(np.random.random((4, 8)).astype('float32')) + adam = opt.Adam(learning_rate=0.1, parameters=net.parameters()) + x = paddle.randn([4, 8], 'float32') for i in range(10): out = net(x) - loss = fluid.layers.mean(out) + loss = paddle.tensor.mean(out) loss.backward() - adam.minimize(loss) - net.clear_gradients() + adam.step() + adam.clear_grad() model_path = "simplenet.example.model.separate_params" - configs = fluid.dygraph.jit.SaveLoadConfig() - configs.separate_params = True + config = paddle.jit.SaveLoadConfig() + config.separate_params = True # saving with configs.separate_params - fluid.dygraph.jit.save( + paddle.jit.save( layer=net, model_path=model_path, - input_spec=[x], - configs=configs) + config=config) # [result] the saved model directory contains: # linear_0.b_0 linear_0.w_0 __model__ __variables.info__ # loading with configs.params_filename - infer_net = fluid.dygraph.jit.load(model_path, configs=configs) - x = fluid.dygraph.to_variable(np.random.random((4, 8)).astype('float32')) + infer_net = paddle.jit.load(model_path, config=config) + x = paddle.randn([4, 8], 'float32') pred = infer_net(x) """ return self._separate_params @@ -651,8 +643,88 @@ class SaveLoadConfig(object): self._keep_name_table = value +def _get_input_var_names(inputs, input_spec): + name_none_error = "The %s's name is None. " \ + "When using jit.save, please set InputSepc's name in " \ + "to_static(input_spec=[]) and jit.save(input_spec=[]) " \ + "and make sure they are consistent." + name_no_exists_error = "The tensor `%s` does not exists. " \ + "Please make sure the name of InputSpec or example Tensor " \ + "in input_spec is the same as the name of InputSpec in " \ + "`to_static` decorated on the Layer.forward method." + result_list = [] + input_var_names = [var.name for var in inputs if isinstance(var, Variable)] + if input_spec is None: + # no prune + result_list = input_var_names + elif input_spec is not None and len(input_spec) == len(input_var_names): + # no prune + result_list = input_var_names + # if input spec name not in input_var_names, only raise warning + for spec in input_spec: + if spec.name is None: + warnings.warn(name_none_error % spec) + elif spec.name not in input_var_names: + warnings.warn(name_no_exists_error % spec.name) + else: + # do nothing + pass + else: + # prune + for spec in input_spec: + if spec.name is None: + # name is None, the input_spec only can be InputSpec + raise ValueError(name_none_error % spec) + elif spec.name not in input_var_names: + # the input_spec can be `InputSpec` or `VarBase` + raise ValueError(name_no_exists_error % spec.name) + else: + result_list.append(spec.name) + + return result_list + + +def _get_output_vars(outputs, output_spec): + name_no_exists_error = "The tensor `%s` does not exists. " \ + "Please make sure the name of example Tensor " \ + "in configs.output_spec is the output tensor of " \ + "Layer.forward method." + result_list = [] + output_vars_dict = OrderedDict() + for var in outputs: + if isinstance(var, Variable): + output_vars_dict[var.name] = var + if output_spec is None: + result_list = output_vars_dict.values() + elif output_spec is not None and len(output_spec) == len(output_vars_dict): + result_list = output_vars_dict.values() + for var in output_spec: + if var.name not in output_vars_dict: + warnings.warn(name_no_exists_error % var.name) + else: + for var in output_spec: + if var.name not in output_vars_dict: + raise ValueError(name_no_exists_error % var.name) + else: + result_list.append(output_vars_dict[var.name]) + return result_list + + +# NOTE(chenweihang): change jit.save/load argument `configs` to `config` +def deprecate_save_load_configs(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + if 'configs' in kwargs: + kwargs['config'] = kwargs['configs'] + kwargs.pop('configs') + return func(*args, **kwargs) + + return wrapper + + +@deprecate_save_load_configs @switch_to_static_graph -def save(layer, model_path, input_spec=None, configs=None): +def save(layer, model_path, input_spec=None, config=None): """ Saves input declarative Layer as :ref:`api_imperative_TranslatedLayer` format model, which can be used for inference or fine-tuning after loading. @@ -677,7 +749,7 @@ def save(layer, model_path, input_spec=None, configs=None): It is the example inputs that will be passed to saved TranslatedLayer's forward function. If None, all input variables of the original Layer's forward function would be the inputs of the saved model. Default None. - configs (SaveLoadConfig, optional): :ref:`api_imperative_jit_saveLoadConfig` object + config (SaveLoadConfig, optional): :ref:`api_imperative_jit_saveLoadConfig` object that specifies additional configuration options. Default None. Returns: None @@ -686,87 +758,78 @@ def save(layer, model_path, input_spec=None, configs=None): .. code-block:: python import numpy as np - import paddle.fluid as fluid - from paddle.fluid.dygraph import Linear - from paddle.fluid.dygraph import declarative + import paddle + import paddle.nn as nn + import paddle.optimizer as opt - BATCH_SIZE = 32 - BATCH_NUM = 20 + BATCH_SIZE = 16 + BATCH_NUM = 4 + EPOCH_NUM = 4 - def random_batch_reader(): - def _get_random_images_and_labels(image_shape, label_shape): - image = np.random.random(size=image_shape).astype('float32') - label = np.random.random(size=label_shape).astype('int64') - return image, label + IMAGE_SIZE = 784 + CLASS_NUM = 10 - def __reader__(): - for _ in range(BATCH_NUM): - batch_image, batch_label = _get_random_images_and_labels( - [BATCH_SIZE, 784], [BATCH_SIZE, 1]) - yield batch_image, batch_label + # define a random dataset + class RandomDataset(paddle.io.Dataset): + def __init__(self, num_samples): + self.num_samples = num_samples - return __reader__ + def __getitem__(self, idx): + image = np.random.random([IMAGE_SIZE]).astype('float32') + label = np.random.randint(0, CLASS_NUM - 1, (1, )).astype('int64') + return image, label - class LinearNet(fluid.dygraph.Layer): - def __init__(self, in_size, out_size): + def __len__(self): + return self.num_samples + + class LinearNet(nn.Layer): + def __init__(self): super(LinearNet, self).__init__() - self._linear = Linear(in_size, out_size) + self._linear = nn.Linear(IMAGE_SIZE, CLASS_NUM) - @declarative + @paddle.jit.to_static def forward(self, x): return self._linear(x) + def train(layer, loader, loss_fn, opt): + for epoch_id in range(EPOCH_NUM): + for batch_id, (image, label) in enumerate(loader()): + out = layer(image) + loss = loss_fn(out, label) + loss.backward() + opt.step() + opt.clear_grad() + print("Epoch {} batch {}: loss = {}".format( + epoch_id, batch_id, np.mean(loss.numpy()))) + # enable dygraph mode - fluid.enable_dygraph() + place = paddle.CPUPlace() + paddle.disable_static(place) - # create network - net = LinearNet(784, 1) - adam = fluid.optimizer.AdamOptimizer(learning_rate=0.1, parameter_list=net.parameters()) - # create data loader - train_loader = fluid.io.DataLoader.from_generator(capacity=5) - train_loader.set_batch_generator(random_batch_reader()) - # train - for data in train_loader(): - img, label = data - label.stop_gradient = True + # 1. train & save model. - cost = net(img) + # create network + layer = LinearNet() + loss_fn = nn.CrossEntropyLoss() + adam = opt.Adam(learning_rate=0.001, parameters=layer.parameters()) - loss = fluid.layers.cross_entropy(cost, label) - avg_loss = fluid.layers.mean(loss) + # create data loader + dataset = RandomDataset(BATCH_NUM * BATCH_SIZE) + loader = paddle.io.DataLoader(dataset, + places=place, + batch_size=BATCH_SIZE, + shuffle=True, + drop_last=True, + num_workers=2) - avg_loss.backward() - adam.minimize(avg_loss) - net.clear_gradients() + # train + train(layer, loader, loss_fn, adam) - # save model + # save model_path = "linear.example.model" - fluid.dygraph.jit.save( - layer=net, - model_path=model_path, - input_spec=[img]) + paddle.jit.save(layer, model_path) """ - def get_inout_spec(all_vars, target_vars, return_name=False): - result_list = [] - valid_var_dict = {} - valid_vars = [var for var in all_vars if isinstance(var, Variable)] - for var in valid_vars: - valid_var_dict[var.name] = var - if target_vars: - for i, var in enumerate(target_vars): - # check target var whether exists - if var.name not in valid_var_dict: - raise RuntimeError( - "The variable to feed/fetch are not exist.") - result_list.append(valid_var_dict[var.name]) - else: - result_list = valid_vars - if return_name: - result_list = [var.name for var in result_list] - - return result_list - # 1. input check prog_translator = ProgramTranslator() if not prog_translator.enable: @@ -778,28 +841,62 @@ def save(layer, model_path, input_spec=None, configs=None): "The input layer of paddle.jit.save should be 'Layer', but received layer type is %s." % type(layer)) + configs = config if configs is None: configs = SaveLoadConfig() + # avoid change user given input_spec + inner_input_spec = None if input_spec is not None: if not isinstance(input_spec, list): raise TypeError( "The input input_spec should be 'list', but received input_spec's type is %s." % type(input_spec)) + inner_input_spec = [] for var in input_spec: - if not isinstance(var, (core.VarBase, Variable, - paddle.static.InputSpec)): + if isinstance(var, paddle.static.InputSpec): + inner_input_spec.append(var) + elif isinstance(var, (core.VarBase, Variable)): + inner_input_spec.append( + paddle.static.InputSpec.from_tensor(var)) + else: raise TypeError( "The element in input_spec list should be 'Variable' or `paddle.static.InputSpec`, but received element's type is %s." % type(var)) - # 2. get program of declarative Layer.forward - if not isinstance(layer.forward, StaticLayer): - raise RuntimeError( - "layer.forward need to be decorated by `@declarative`.") - concrete_program = layer.forward.concrete_program - - # NOTE: we maintain the mapping of variable name to + # 2. get program from Layer + # TODO(chenweihang): add support for other method, not only forward + if isinstance(layer.forward, StaticLayer): + concrete_program = layer.forward.concrete_program + else: + # transform in jit.save, if input_spec is incomplete, declarative will throw error + static_forward = declarative(layer.forward, input_spec=inner_input_spec) + concrete_program = static_forward.concrete_program + # the input_spec has been used in declarative, which is equal to + # @declarative with input_spec and jit.save without input_spec, + # avoid needless warning + inner_input_spec = None + + # 3. build input & output of save_infernece_model + # NOTE(chenweihang): [ Get input variables name ] + # There are two cases, whether to prune the inputs or not + # - not prune inputs (recommend): + # - the len(input_spec) == len((concrete_program.inputs) - 1 + # - here can use concrete_program.inputs directly + # - prune inputs: + # - the input_spec length < len((concrete_program.inputs) - 1 + # - the input_spec's name should be in concrete_program.inputs + input_var_names = _get_input_var_names(concrete_program.inputs, + inner_input_spec) + + # NOTE(chenweihang): [ Get output variables ] + # the rule is like [ Get input variables name ]. For output var, + # we only support VarBase spec, and actually, we only need the + # var name of output, and we don't recommended to use output_spec + output_vars = _get_output_vars(concrete_program.outputs, + configs.output_spec) + + # NOTE(chenweihang): we maintain the mapping of variable name to # structured name, the buffer variable (non-persistable) # saved to inference program may not need by dygraph Layer, # we only record the state_dict variable's structured name @@ -807,7 +904,7 @@ def save(layer, model_path, input_spec=None, configs=None): for structured_name, var in six.iteritems(layer.state_dict()): state_names_dict[var.name] = structured_name - # 3. share parameters from Layer to scope & record var info + # 4. share parameters from Layer to scope & record var info scope = core.Scope() extra_var_info = dict() for param_or_buffer in concrete_program.parameters: @@ -825,10 +922,6 @@ def save(layer, model_path, input_spec=None, configs=None): extra_info_dict['trainable'] = param_or_buffer.trainable extra_var_info[param_or_buffer.name] = extra_info_dict - # 4. build input & output spec - input_var_names = get_inout_spec(concrete_program.inputs, input_spec, True) - output_vars = get_inout_spec(concrete_program.outputs, configs.output_spec) - # 5. save inference model from paddle.fluid.io import save_inference_model @@ -849,7 +942,7 @@ def save(layer, model_path, input_spec=None, configs=None): export_for_deployment=configs._export_for_deployment, program_only=configs._program_only) - # NOTE: [ Save extra variable info ] + # NOTE(chenweihang): [ Save extra variable info ] # save_inference_model will lose some important variable information, including: # - Variable name and correspondence (when saved variables as one file) # - Variable.stop_gradient information @@ -869,8 +962,9 @@ def save(layer, model_path, input_spec=None, configs=None): pickle.dump(extra_var_info, f, protocol=2) +@deprecate_save_load_configs @dygraph_only -def load(model_path, configs=None): +def load(model_path, config=None): """ :api_attr: imperative @@ -887,7 +981,7 @@ def load(model_path, configs=None): Args: model_path (str): The directory path where the model is saved. - configs (SaveLoadConfig, optional): :ref:`api_imperative_jit_saveLoadConfig` object that specifies + config (SaveLoadConfig, optional): :ref:`api_imperative_jit_saveLoadConfig` object that specifies additional configuration options. Default None. Returns: @@ -899,122 +993,126 @@ def load(model_path, configs=None): .. code-block:: python import numpy as np - import paddle.fluid as fluid - from paddle.fluid.dygraph import Linear - from paddle.fluid.dygraph import declarative + import paddle + import paddle.nn as nn + import paddle.optimizer as opt - BATCH_SIZE = 32 - BATCH_NUM = 20 + BATCH_SIZE = 16 + BATCH_NUM = 4 + EPOCH_NUM = 4 - def random_batch_reader(): - def _get_random_images_and_labels(image_shape, label_shape): - image = np.random.random(size=image_shape).astype('float32') - label = np.random.random(size=label_shape).astype('int64') - return image, label + IMAGE_SIZE = 784 + CLASS_NUM = 10 - def __reader__(): - for _ in range(BATCH_NUM): - batch_image, batch_label = _get_random_images_and_labels( - [BATCH_SIZE, 784], [BATCH_SIZE, 1]) - yield batch_image, batch_label + # define a random dataset + class RandomDataset(paddle.io.Dataset): + def __init__(self, num_samples): + self.num_samples = num_samples - return __reader__ + def __getitem__(self, idx): + image = np.random.random([IMAGE_SIZE]).astype('float32') + label = np.random.randint(0, CLASS_NUM - 1, (1, )).astype('int64') + return image, label - class LinearNet(fluid.dygraph.Layer): - def __init__(self, in_size, out_size): + def __len__(self): + return self.num_samples + + class LinearNet(nn.Layer): + def __init__(self): super(LinearNet, self).__init__() - self._linear = Linear(in_size, out_size) + self._linear = nn.Linear(IMAGE_SIZE, CLASS_NUM) - @declarative + @paddle.jit.to_static def forward(self, x): return self._linear(x) + def train(layer, loader, loss_fn, opt): + for epoch_id in range(EPOCH_NUM): + for batch_id, (image, label) in enumerate(loader()): + out = layer(image) + loss = loss_fn(out, label) + loss.backward() + opt.step() + opt.clear_grad() + print("Epoch {} batch {}: loss = {}".format( + epoch_id, batch_id, np.mean(loss.numpy()))) + # enable dygraph mode - fluid.enable_dygraph() + place = paddle.CPUPlace() + paddle.disable_static(place) # 1. train & save model. + # create network - net = LinearNet(784, 1) - adam = fluid.optimizer.AdamOptimizer(learning_rate=0.1, parameter_list=net.parameters()) + layer = LinearNet() + loss_fn = nn.CrossEntropyLoss() + adam = opt.Adam(learning_rate=0.001, parameters=layer.parameters()) + # create data loader - train_loader = fluid.io.DataLoader.from_generator(capacity=5) - train_loader.set_batch_generator(random_batch_reader()) - # train - for data in train_loader(): - img, label = data - label.stop_gradient = True + dataset = RandomDataset(BATCH_NUM * BATCH_SIZE) + loader = paddle.io.DataLoader(dataset, + places=place, + batch_size=BATCH_SIZE, + shuffle=True, + drop_last=True, + num_workers=2) - cost = net(img) + # train + train(layer, loader, loss_fn, adam) - loss = fluid.layers.cross_entropy(cost, label) - avg_loss = fluid.layers.mean(loss) + # save + model_path = "linear.example.model" + paddle.jit.save(layer, model_path) - avg_loss.backward() - adam.minimize(avg_loss) - net.clear_gradients() + # 2. load model - model_path = "linear.example.model" - fluid.dygraph.jit.save( - layer=net, - model_path=model_path, - input_spec=[img]) + # load + loaded_layer = paddle.jit.load(model_path) - # 2. load model & inference - # load model - infer_net = fluid.dygraph.jit.load(model_path) # inference - x = fluid.dygraph.to_variable(np.random.random((1, 784)).astype('float32')) - pred = infer_net(x) + loaded_layer.eval() + x = paddle.randn([1, IMAGE_SIZE], 'float32') + pred = loaded_layer(x) - # 3. load model & fine-tune - # load model - train_net = fluid.dygraph.jit.load(model_path) - train_net.train() - adam = fluid.optimizer.AdamOptimizer(learning_rate=0.1, parameter_list=train_net.parameters()) - # create data loader - train_loader = fluid.io.DataLoader.from_generator(capacity=5) - train_loader.set_batch_generator(random_batch_reader()) # fine-tune - for data in train_loader(): - img, label = data - label.stop_gradient = True + loaded_layer.train() + adam = opt.Adam(learning_rate=0.001, parameters=loaded_layer.parameters()) + train(loaded_layer, loader, loss_fn, adam) - cost = train_net(img) - - loss = fluid.layers.cross_entropy(cost, label) - avg_loss = fluid.layers.mean(loss) - - avg_loss.backward() - adam.minimize(avg_loss) - train_net.clear_gradients() 2. Load model saved by :ref:`api_fluid_io_save_inference_model` then performing and fine-tune training. .. code-block:: python import numpy as np + import paddle import paddle.fluid as fluid + import paddle.nn as nn + import paddle.optimizer as opt - BATCH_SIZE = 32 - BATCH_NUM = 20 + BATCH_SIZE = 16 + BATCH_NUM = 4 + EPOCH_NUM = 4 - def random_batch_reader(): - def _get_random_images_and_labels(image_shape, label_shape): - image = np.random.random(size=image_shape).astype('float32') - label = np.random.random(size=label_shape).astype('int64') - return image, label + IMAGE_SIZE = 784 + CLASS_NUM = 10 - def __reader__(): - for _ in range(BATCH_NUM): - batch_image, batch_label = _get_random_images_and_labels( - [BATCH_SIZE, 784], [BATCH_SIZE, 1]) - yield batch_image, batch_label + # define a random dataset + class RandomDataset(paddle.io.Dataset): + def __init__(self, num_samples): + self.num_samples = num_samples + + def __getitem__(self, idx): + image = np.random.random([IMAGE_SIZE]).astype('float32') + label = np.random.randint(0, CLASS_NUM - 1, (1, )).astype('int64') + return image, label - return __reader__ + def __len__(self): + return self.num_samples - img = fluid.data(name='img', shape=[None, 784], dtype='float32') + image = fluid.data(name='image', shape=[None, 784], dtype='float32') label = fluid.data(name='label', shape=[None, 1], dtype='int64') - pred = fluid.layers.fc(input=img, size=10, act='softmax') + pred = fluid.layers.fc(input=image, size=10, act='softmax') loss = fluid.layers.cross_entropy(input=pred, label=label) avg_loss = fluid.layers.mean(loss) @@ -1025,9 +1123,15 @@ def load(model_path, configs=None): exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) - loader = fluid.io.DataLoader.from_generator( - feed_list=[img, label], capacity=5, iterable=True) - loader.set_batch_generator(random_batch_reader(), places=place) + # create data loader + dataset = RandomDataset(BATCH_NUM * BATCH_SIZE) + loader = paddle.io.DataLoader(dataset, + feed_list=[image, label], + places=place, + batch_size=BATCH_SIZE, + shuffle=True, + drop_last=True, + num_workers=2) # 1. train and save inference model for data in loader(): @@ -1038,39 +1142,42 @@ def load(model_path, configs=None): model_path = "fc.example.model" fluid.io.save_inference_model( - model_path, ["img"], [pred], exe) + model_path, ["image"], [pred], exe) + + # 2. load model # enable dygraph mode - fluid.enable_dygraph() + paddle.disable_static(place) + + # load + fc = paddle.jit.load(model_path) - # 2. load model & inference - fc = fluid.dygraph.jit.load(model_path) - x = fluid.dygraph.to_variable(np.random.random((1, 784)).astype('float32')) + # inference + fc.eval() + x = paddle.randn([1, IMAGE_SIZE], 'float32') pred = fc(x) - # 3. load model & fine-tune - fc = fluid.dygraph.jit.load(model_path) + # fine-tune fc.train() - sgd = fluid.optimizer.SGD(learning_rate=0.001, - parameter_list=fc.parameters()) - - train_loader = fluid.io.DataLoader.from_generator(capacity=5) - train_loader.set_batch_generator( - random_batch_reader(), places=place) - - for data in train_loader(): - img, label = data - label.stop_gradient = True - - cost = fc(img) - - loss = fluid.layers.cross_entropy(cost, label) - avg_loss = fluid.layers.mean(loss) - - avg_loss.backward() - sgd.minimize(avg_loss) + loss_fn = nn.CrossEntropyLoss() + adam = opt.Adam(learning_rate=0.001, parameters=fc.parameters()) + loader = paddle.io.DataLoader(dataset, + places=place, + batch_size=BATCH_SIZE, + shuffle=True, + drop_last=True, + num_workers=2) + for epoch_id in range(EPOCH_NUM): + for batch_id, (image, label) in enumerate(loader()): + out = fc(image) + loss = loss_fn(out, label) + loss.backward() + adam.step() + adam.clear_grad() + print("Epoch {} batch {}: loss = {}".format( + epoch_id, batch_id, np.mean(loss.numpy()))) """ - return TranslatedLayer._construct(model_path, configs) + return TranslatedLayer._construct(model_path, config) @dygraph_only diff --git a/python/paddle/fluid/dygraph/math_op_patch.py b/python/paddle/fluid/dygraph/math_op_patch.py index 8c4109674200bf97354444f92f00b13e053152a0..3aa7b9dfc262810686319819f717f3cfd06b5e50 100644 --- a/python/paddle/fluid/dygraph/math_op_patch.py +++ b/python/paddle/fluid/dygraph/math_op_patch.py @@ -19,7 +19,6 @@ from ..framework import Variable, convert_np_dtype_to_dtype_, _varbase_creator from ..layers.layer_function_generator import OpProtoHolder from ..layers import common_methods from . import to_variable, no_grad -import paddle import numpy as np import six @@ -163,26 +162,6 @@ def monkey_patch_math_varbase(): def _scalar_div_(var, value): return _scalar_elementwise_op_(var, 1.0 / value, 0.0) - # TODO(shenliang03): currently, it supports divide, floor_divide, remainder - # for binary operator by using the api to achieve the type promotion - def _binary_method_creator_(op_type, reverse=False): - import paddle - - def __impl__(self, other_var): - import paddle - op = getattr(paddle, op_type) - if reverse: - return op(other_var, self) - else: - return op(self, other_var) - - __impl__.__doc__ = """ - - See paddle.{}""".format(op_type) - __impl__.__name__ = op_type - - return __impl__ - # for binary operator such as elementwise, compare def _binary_creator_(method_name, op_type, @@ -281,20 +260,22 @@ def monkey_patch_math_varbase(): ## a*b == b*a. Do not need to reverse explicitly ('__rmul__', _binary_creator_('__rmul__', 'elementwise_mul', False, _scalar_mul_)), + ('__div__', _binary_creator_('__div__', 'elementwise_div', False, + _scalar_div_)), + ('__truediv__', _binary_creator_('__truediv__', 'elementwise_div', + False, _scalar_div_)), + ('__rdiv__', _binary_creator_('__rdiv__', 'elementwise_div', True, + None)), ('__rtruediv__', _binary_creator_('rtruediv__', 'elementwise_div', True, None)), ('__pow__', _binary_creator_('__pow__', 'elementwise_pow', False, None)), ('__rpow__', _binary_creator_('__rpow__', 'elementwise_pow', True, None)), - # These binary use paddle.optype - ('__div__', _binary_method_creator_('divide', False)), - ('__truediv__', _binary_method_creator_('divide', False)), - ('__rtruediv__', _binary_method_creator_('divide', True)), - ('__rdiv__', _binary_method_creator_('divide', True)), - ('__floordiv__', _binary_method_creator_('floor_divide', False)), - ('__rfloordiv__', _binary_method_creator_('floor_divide', True)), - ('__mod__', _binary_method_creator_('remainder', False)), + ('__floordiv__', _binary_creator_('__floordiv__', + 'elementwise_floordiv', False, None)), + ('__mod__', _binary_creator_('__mod__', 'elementwise_mod', False, + None)), ## for logical compare ('__eq__', _binary_creator_('__eq__', 'equal', False, None)), ('__ne__', _binary_creator_('__ne__', 'not_equal', False, None)), @@ -304,7 +285,7 @@ def monkey_patch_math_varbase(): ('__ge__', _binary_creator_('__ge__', 'greater_equal', False, None)), ('__array_ufunc__', None), ('sigmoid', _method_creator_('sigmoid', 'name=None')), - ('logsigmoid', _method_creator_('logsigmoid', 'name=None')), + ('log_sigmoid', _method_creator_('logsigmoid', 'name=None')), ('exp', _method_creator_('exp', 'name=None')), ('tanh', _method_creator_('tanh', 'name=None')), ('atan', _method_creator_('atan', 'name=None')), diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py index 5281df9ead10acea5ae8656dcc4a0eed14fb3e83..797b32f5d4768af59fa4e6aceb75e4b6d9029d91 100644 --- a/python/paddle/fluid/framework.py +++ b/python/paddle/fluid/framework.py @@ -217,7 +217,7 @@ def _dygraph_not_support_(func): def _dygraph_only_(func): def __impl__(*args, **kwargs): assert in_dygraph_mode( - ), "We Only support %s in imperative mode, please use fluid.dygraph.guard() as context to run it in imperative Mode" % func.__name__ + ), "We Only support %s in dynamic mode, please call 'paddle.disable_static()' to enter dynamic mode." % func.__name__ return func(*args, **kwargs) return __impl__ diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/ir/vars_metatools.py b/python/paddle/fluid/incubate/fleet/parameter_server/ir/vars_metatools.py index c8f3643b25be0780bbdfd1668d849ab00ece355c..c80b4a800bd149cced13c0e25322cd03ef94e468 100644 --- a/python/paddle/fluid/incubate/fleet/parameter_server/ir/vars_metatools.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/ir/vars_metatools.py @@ -12,9 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. from __future__ import print_function +from functools import reduce + from paddle.fluid.framework import Variable from paddle.fluid import core +dtype_to_size = { + core.VarDesc.VarType.FP16: 2, + core.VarDesc.VarType.FP32: 4, + core.VarDesc.VarType.FP64: 8, + core.VarDesc.VarType.INT16: 2, + core.VarDesc.VarType.INT32: 4, + core.VarDesc.VarType.INT64: 8, + core.VarDesc.VarType.BOOL: 1, + core.VarDesc.VarType.UINT8: 1, +} + class VarBlock: def __init__(self, varname, offset, size): @@ -51,11 +64,14 @@ class VarStruct(object): self.type = type self.lod_level = lod_level self.persistable = persistable + self.m_size = 1 + self.m_size = reduce(lambda x, y: x * y, shape) + self.m_size *= dtype_to_size[dtype] def __str__(self): - return "N: {}, S: {}, D: {}, T: {}, LL: {}, P: {}".format( + return "N: {}, S: {}, D: {}, T: {}, LL: {}, P: {}, M: {}".format( self.name, self.shape, self.dtype, self.type, self.lod_level, - self.persistable) + self.persistable, self.m_size) class VarDistributed(object): diff --git a/python/paddle/fluid/inference/__init__.py b/python/paddle/fluid/inference/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3013c1f2aff87fb293ea984c99d8336b418ee080 --- /dev/null +++ b/python/paddle/fluid/inference/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .wrapper import Config, DataType, PlaceType, PrecisionType, Tensor, Predictor + +from ..core import create_predictor, get_version, get_num_bytes_of_data_type, PredictorPool diff --git a/python/paddle/fluid/inference/wrapper.py b/python/paddle/fluid/inference/wrapper.py new file mode 100644 index 0000000000000000000000000000000000000000..96885edcc5e822beb5db8332f2b58d12b9c4ff63 --- /dev/null +++ b/python/paddle/fluid/inference/wrapper.py @@ -0,0 +1,23 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ..core import AnalysisConfig, PaddleDType, PaddlePlace +from ..core import PaddleInferPredictor, PaddleInferTensor + +DataType = PaddleDType +PlaceType = PaddlePlace +PrecisionType = AnalysisConfig.Precision +Config = AnalysisConfig +Tensor = PaddleInferTensor +Predictor = PaddleInferPredictor diff --git a/python/paddle/fluid/layers/detection.py b/python/paddle/fluid/layers/detection.py index ea6abe2d335e6669b27ba278c0faaca62ca0fdbb..bf87d1fc5a947e48845a3783fd71922641e28819 100644 --- a/python/paddle/fluid/layers/detection.py +++ b/python/paddle/fluid/layers/detection.py @@ -20,7 +20,8 @@ from __future__ import print_function from .layer_function_generator import generate_layer_fn from .layer_function_generator import autodoc, templatedoc from ..layer_helper import LayerHelper -from ..framework import Variable +from ..framework import Variable, in_dygraph_mode +from .. import core from .loss import softmax_with_cross_entropy from . import tensor from . import nn @@ -2893,8 +2894,8 @@ def generate_proposals(scores, nms_thresh=0.5, min_size=0.1, eta=1.0, - name=None, - return_rois_num=False): + return_rois_num=False, + name=None): """ :alias_main: paddle.nn.functional.generate_proposals :alias: paddle.nn.functional.generate_proposals,paddle.nn.functional.vision.generate_proposals @@ -2949,6 +2950,10 @@ def generate_proposals(scores, num of each image in one batch. The N is the image's num. For example, the tensor has values [4,5] that represents the first image has 4 Rois, the second image has 5 Rois. It only used in rcnn model. 'False' by default. + name(str, optional): For detailed information, please refer + to :ref:`api_guide_Name`. Usually name is no need to set and + None by default. + Returns: tuple: A tuple with format ``(rpn_rois, rpn_roi_probs)``. @@ -2969,6 +2974,14 @@ def generate_proposals(scores, im_info, anchors, variances) """ + if in_dygraph_mode(): + assert return_rois_num, "return_rois_num should be True in dygraph mode." + attrs = ('pre_nms_topN', pre_nms_top_n, 'post_nms_topN', post_nms_top_n, + 'nms_thresh', nms_thresh, 'min_size', min_size, 'eta', eta) + rpn_rois, rpn_roi_probs, rpn_rois_num = core.ops.generate_proposals( + scores, bbox_deltas, im_info, anchors, variances, *attrs) + return rpn_rois, rpn_roi_probs, rpn_rois_num + helper = LayerHelper('generate_proposals', **locals()) check_variable_and_dtype(scores, 'scores', ['float32'], @@ -2986,7 +2999,14 @@ def generate_proposals(scores, dtype=bbox_deltas.dtype) rpn_roi_probs = helper.create_variable_for_type_inference( dtype=scores.dtype) - rpn_rois_lod = helper.create_variable_for_type_inference(dtype='int32') + outputs = { + 'RpnRois': rpn_rois, + 'RpnRoiProbs': rpn_roi_probs, + } + if return_rois_num: + rpn_rois_num = helper.create_variable_for_type_inference(dtype='int32') + rpn_rois_num.stop_gradient = True + outputs['RpnRoisNum'] = rpn_rois_num helper.append_op( type="generate_proposals", @@ -3004,17 +3024,12 @@ def generate_proposals(scores, 'min_size': min_size, 'eta': eta }, - outputs={ - 'RpnRois': rpn_rois, - 'RpnRoiProbs': rpn_roi_probs, - 'RpnRoisLod': rpn_rois_lod - }) + outputs=outputs) rpn_rois.stop_gradient = True rpn_roi_probs.stop_gradient = True - rpn_rois_lod.stop_gradient = True if return_rois_num: - return rpn_rois, rpn_roi_probs, rpn_rois_lod + return rpn_rois, rpn_roi_probs, rpn_rois_num else: return rpn_rois, rpn_roi_probs @@ -3656,6 +3671,7 @@ def distribute_fpn_proposals(fpn_rois, max_level, refer_level, refer_scale, + rois_num=None, name=None): """ :alias_main: paddle.nn.functional.distribute_fpn_proposals @@ -3687,6 +3703,11 @@ def distribute_fpn_proposals(fpn_rois, come from. refer_level(int32): The referring level of FPN layer with specified scale. refer_scale(int32): The referring scale of FPN layer with specified level. + rois_num(Tensor): 1-D Tensor contains the number of RoIs in each image. + The shape is [B] and data type is int32. B is the number of images. + If it is not None then return a list of 1-D Tensor. Each element + is the output RoIs' number of each image on the corresponding level + and the shape is [B]. None by default. name(str, optional): For detailed information, please refer to :ref:`api_guide_Name`. Usually name is no need to set and None by default. @@ -3702,6 +3723,10 @@ def distribute_fpn_proposals(fpn_rois, the number of total rois. The data type is int32. It is used to restore the order of fpn_rois. + rois_num_per_level(List): A list of 1-D Tensor and each Tensor is + the RoIs' number in each image on the corresponding level. The shape + is [B] and data type of int32. B is the number of images + Examples: .. code-block:: python @@ -3716,26 +3741,52 @@ def distribute_fpn_proposals(fpn_rois, refer_level=4, refer_scale=224) """ + num_lvl = max_level - min_level + 1 + + if in_dygraph_mode(): + assert rois_num is not None, "rois_num should not be None in dygraph mode." + attrs = ('min_level', min_level, 'max_level', max_level, 'refer_level', + refer_level, 'refer_scale', refer_scale) + multi_rois, restore_ind, rois_num_per_level = core.ops.distribute_fpn_proposals( + fpn_rois, rois_num, num_lvl, num_lvl, *attrs) + return multi_rois, restore_ind, rois_num_per_level + check_variable_and_dtype(fpn_rois, 'fpn_rois', ['float32', 'float64'], 'distribute_fpn_proposals') helper = LayerHelper('distribute_fpn_proposals', **locals()) dtype = helper.input_dtype('fpn_rois') - num_lvl = max_level - min_level + 1 multi_rois = [ helper.create_variable_for_type_inference(dtype) for i in range(num_lvl) ] + restore_ind = helper.create_variable_for_type_inference(dtype='int32') + + inputs = {'FpnRois': fpn_rois} + outputs = { + 'MultiFpnRois': multi_rois, + 'RestoreIndex': restore_ind, + } + + if rois_num is not None: + inputs['RoisNum'] = rois_num + rois_num_per_level = [ + helper.create_variable_for_type_inference(dtype='int32') + for i in range(num_lvl) + ] + outputs['MultiLevelRoIsNum'] = rois_num_per_level + helper.append_op( type='distribute_fpn_proposals', - inputs={'FpnRois': fpn_rois}, - outputs={'MultiFpnRois': multi_rois, - 'RestoreIndex': restore_ind}, + inputs=inputs, + outputs=outputs, attrs={ 'min_level': min_level, 'max_level': max_level, 'refer_level': refer_level, 'refer_scale': refer_scale }) + if rois_num is not None: + return multi_rois, restore_ind, rois_num_per_level return multi_rois, restore_ind @@ -3820,6 +3871,7 @@ def collect_fpn_proposals(multi_rois, min_level, max_level, post_nms_top_n, + rois_num_per_level=None, name=None): """ :alias_main: paddle.nn.functional.collect_fpn_proposals @@ -3846,6 +3898,12 @@ def collect_fpn_proposals(multi_rois, min_level(int): The lowest level of FPN layer to collect max_level(int): The highest level of FPN layer to collect post_nms_top_n(int): The number of selected RoIs + rois_num_per_level(list, optional): The List of RoIs' numbers. + Each element is 1-D Tensor which contains the RoIs' number of each + image on each level and the shape is [B] and data type is + int32, B is the number of images. If it is not None then return + a 1-D Tensor contains the output RoIs' number of each image and + the shape is [B]. Default: None name(str, optional): For detailed information, please refer to :ref:`api_guide_Name`. Usually name is no need to set and None by default. @@ -3856,6 +3914,9 @@ def collect_fpn_proposals(multi_rois, fpn_rois(Variable): 2-D LoDTensor with shape [N, 4] and data type is float32 or float64. Selected RoIs. + rois_num(Tensor): 1-D Tensor contains the RoIs's number of each + image. The shape is [B] and data type is int32. B is the number of + images. Examples: .. code-block:: python @@ -3879,21 +3940,38 @@ def collect_fpn_proposals(multi_rois, """ check_type(multi_rois, 'multi_rois', list, 'collect_fpn_proposals') check_type(multi_scores, 'multi_scores', list, 'collect_fpn_proposals') + num_lvl = max_level - min_level + 1 + input_rois = multi_rois[:num_lvl] + input_scores = multi_scores[:num_lvl] + + if in_dygraph_mode(): + assert rois_num_per_level is not None, "rois_num_per_level should not be None in dygraph mode." + attrs = ('post_nms_topN', post_nms_top_n) + output_rois, rois_num = core.ops.collect_fpn_proposals( + input_rois, input_scores, rois_num_per_level, *attrs) + helper = LayerHelper('collect_fpn_proposals', **locals()) dtype = helper.input_dtype('multi_rois') check_dtype(dtype, 'multi_rois', ['float32', 'float64'], 'collect_fpn_proposals') - num_lvl = max_level - min_level + 1 - input_rois = multi_rois[:num_lvl] - input_scores = multi_scores[:num_lvl] output_rois = helper.create_variable_for_type_inference(dtype) output_rois.stop_gradient = True + + inputs = { + 'MultiLevelRois': input_rois, + 'MultiLevelScores': input_scores, + } + outputs = {'FpnRois': output_rois} + if rois_num_per_level is not None: + inputs['MultiLevelRoIsNum'] = rois_num_per_level + rois_num = helper.create_variable_for_type_inference(dtype='int32') + rois_num.stop_gradient = True + outputs['RoisNum'] = rois_num helper.append_op( type='collect_fpn_proposals', - inputs={ - 'MultiLevelRois': input_rois, - 'MultiLevelScores': input_scores - }, - outputs={'FpnRois': output_rois}, + inputs=inputs, + outputs=outputs, attrs={'post_nms_topN': post_nms_top_n}) + if rois_num_per_level is not None: + return output_rois, rois_num return output_rois diff --git a/python/paddle/fluid/layers/math_op_patch.py b/python/paddle/fluid/layers/math_op_patch.py index 38fc34472c8bc64338e2468bdf3f4b0bab1370ce..4595f0cf93916d71a3d0ec582af1917500d68f12 100644 --- a/python/paddle/fluid/layers/math_op_patch.py +++ b/python/paddle/fluid/layers/math_op_patch.py @@ -16,7 +16,6 @@ from __future__ import print_function import warnings import inspect -import paddle from .. import core from ..framework import Variable, unique_name @@ -46,7 +45,6 @@ EXPRESSION_MAP = { "__pow__": "A ** B", "__rpow__": "A **= B", "__floordiv__": "A //B", - "__rfloordiv__": "A //= B", "__mod__": "A % B", "__eq__": "A == B", "__ne__": "A != B", @@ -235,25 +233,6 @@ def monkey_patch_variable(): def _scalar_div_(var, value): return _scalar_op_(var, 1.0 / value, 0.0) - # TODO(shenliang03): currently, it supports divide, floor_divide, remainder - # for binary operator by using the api to achieve the type promotion - def _binary_method_creator_(op_type, reverse=False): - import paddle - - def __impl__(self, other_var): - op = getattr(paddle, op_type) - if reverse: - return op(other_var, self) - else: - return op(self, other_var) - - __impl__.__doc__ = """ - - See paddle.{}""".format(op_type) - __impl__.__name__ = op_type - - return __impl__ - def _binary_creator_(method_name, op_type, reverse=False, @@ -360,18 +339,22 @@ def monkey_patch_variable(): # a*b == b*a. Do not need to reverse explicitly ('__rmul__', _binary_creator_('__rmul__', 'elementwise_mul', False, _scalar_mul_)), + ('__div__', _binary_creator_('__div__', 'elementwise_div', False, + _scalar_div_)), + ('__truediv__', _binary_creator_('__truediv__', 'elementwise_div', + False, _scalar_div_)), + ('__rdiv__', _binary_creator_('__rdiv__', 'elementwise_div', True, + None)), + ('__rtruediv__', _binary_creator_('__rtruediv__', 'elementwise_div', + True, None)), ('__pow__', _binary_creator_('__pow__', 'elementwise_pow', False, None)), ('__rpow__', _binary_creator_('__rpow__', 'elementwise_pow', True, None)), - # These binary use paddle.optype - ('__div__', _binary_method_creator_('divide', False)), - ('__rdiv__', _binary_method_creator_('divide', True)), - ('__truediv__', _binary_method_creator_('divide', False)), - ('__rtruediv__', _binary_method_creator_('divide', True)), - ('__floordiv__', _binary_method_creator_('floor_divide', False)), - ('__rfloordiv__', _binary_method_creator_('floor_divide', True)), - ('__mod__', _binary_method_creator_('remainder', False)), + ('__floordiv__', _binary_creator_('__floordiv__', + 'elementwise_floordiv', False, None)), + ('__mod__', _binary_creator_('__mod__', 'elementwise_mod', False, + None)), # for logical compare ('__eq__', _binary_creator_('__eq__', 'equal', False, None)), ('__ne__', _binary_creator_('__ne__', 'not_equal', False, None)), diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 9313de8c64fcf4efc1e192ad2826f05f51869bbf..bc9f182d95e3b728fbc0866e1c79f5508d3a04aa 100755 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -6306,6 +6306,15 @@ def unsqueeze(input, axes, name=None): """ if in_dygraph_mode(): + if isinstance(axes, int): + axes = [axes] + elif isinstance(axes, Variable): + axes = axes.numpy().tolist() + elif isinstance(axes, (list, tuple)): + axes = [ + item.numpy().item(0) if isinstance(item, Variable) else item + for item in axes + ] out, _ = core.ops.unsqueeze2(input, 'axes', axes) return out @@ -6853,7 +6862,8 @@ def roi_pool(input, pooled_height=1, pooled_width=1, spatial_scale=1.0, - rois_lod=None): + rois_num=None, + name=None): """ :alias_main: paddle.nn.functional.roi_pool :alias: paddle.nn.functional.roi_pool,paddle.nn.functional.vision.roi_pool @@ -6873,10 +6883,14 @@ def roi_pool(input, Args: input (Variable): Input feature, 4D-Tensor with the shape of [N,C,H,W], where N is the batch size, C is the input channel, H is Height, W is weight. The data type is float32 or float64. rois (Variable): ROIs (Regions of Interest) to pool over. 2D-LoDTensor with the shape of [num_rois,4], the lod level is 1. Given as [[x1, y1, x2, y2], ...], (x1, y1) is the top left coordinates, and (x2, y2) is the bottom right coordinates. - rois_lod (Variable): The lod info of rois. Default: None pooled_height (int, optional): The pooled output height, data type is int32. Default: 1 pooled_width (int, optional): The pooled output height, data type is int32. Default: 1 spatial_scale (float, optional): Multiplicative spatial scale factor to translate ROI coords from their input scale to the scale used when pooling. Default: 1.0 + rois_num (Tensor): The number of RoIs in each image. Default: None + name(str, optional): For detailed information, please refer + to :ref:`api_guide_Name`. Usually name is no need to set and + None by default. + Returns: Variable: The pooled feature, 4D-Tensor with the shape of [num_rois, C, pooled_height, pooled_width]. @@ -6896,11 +6910,11 @@ def roi_pool(input, input_data = np.array([i for i in range(1,17)]).reshape(1,1,4,4).astype(DATATYPE) roi_data =fluid.create_lod_tensor(np.array([[1., 1., 2., 2.], [1.5, 1.5, 3., 3.]]).astype(DATATYPE),[[2]], place) - rois_lod_data = np.array([0, 2]) + rois_num_data = np.array([2]).astype('int32') x = fluid.data(name='input', shape=[None,1,4,4], dtype=DATATYPE) rois = fluid.data(name='roi', shape=[None,4], dtype=DATATYPE) - rois_lod = fluid.data(name='rois_lod', shape=[None], dtype='int64') + rois_num = fluid.data(name='rois_num', shape=[None], dtype='int32') pool_out = fluid.layers.roi_pool( input=x, @@ -6908,24 +6922,36 @@ def roi_pool(input, pooled_height=1, pooled_width=1, spatial_scale=1.0, - rois_lod=rois_lod) + rois_num=rois_num) exe = fluid.Executor(place) - out, = exe.run(feed={'input':input_data ,'roi':roi_data, 'rois_lod': rois_lod_data}, fetch_list=[pool_out.name]) + out, = exe.run(feed={'input':input_data ,'roi':roi_data, 'rois_num': rois_num_data}, fetch_list=[pool_out.name]) print(out) #array([[[[11.]]], [[[16.]]]], dtype=float32) print(np.array(out).shape) # (2, 1, 1, 1) """ + if in_dygraph_mode(): + assert rois_num is not None, "rois_num should not be None in dygraph mode." + pool_out, argmaxes = core.ops.roi_pool( + input, rois, rois_num, "pooled_height", pooled_height, + "pooled_width", pooled_width, "spatial_scale", spatial_scale) + return pool_out, argmaxes + check_variable_and_dtype(input, 'input', ['float32'], 'roi_pool') check_variable_and_dtype(rois, 'rois', ['float32'], 'roi_pool') helper = LayerHelper('roi_pool', **locals()) dtype = helper.input_dtype() pool_out = helper.create_variable_for_type_inference(dtype) argmaxes = helper.create_variable_for_type_inference(dtype='int32') + + inputs = { + "X": input, + "ROIs": rois, + } + if rois_num is not None: + inputs['RoisNum'] = rois_num helper.append_op( type="roi_pool", - inputs={"X": input, - "ROIs": rois, - "RoisLod": rois_lod}, + inputs=inputs, outputs={"Out": pool_out, "Argmax": argmaxes}, attrs={ @@ -6943,8 +6969,8 @@ def roi_align(input, pooled_width=1, spatial_scale=1.0, sampling_ratio=-1, - name=None, - rois_lod=None): + rois_num=None, + name=None): """ :alias_main: paddle.nn.functional.roi_align :alias: paddle.nn.functional.roi_align,paddle.nn.functional.vision.roi_align @@ -6959,11 +6985,11 @@ def roi_align(input, data type is float32 or float64. Given as [[x1, y1, x2, y2], ...], (x1, y1) is the top left coordinates, and (x2, y2) is the bottom right coordinates. - rois_lod (Variable): The lod info of rois. Default: None pooled_height (int32, optional): ${pooled_height_comment} Default: 1 pooled_width (int32, optional): ${pooled_width_comment} Default: 1 spatial_scale (float32, optional): ${spatial_scale_comment} Default: 1.0 sampling_ratio(int32, optional): ${sampling_ratio_comment} Default: -1 + rois_num (Tensor): The number of RoIs in each image. Default: None name(str, optional): For detailed information, please refer to :ref:`api_guide_Name`. Usually name is no need to set and None by default. @@ -6982,26 +7008,38 @@ def roi_align(input, name='data', shape=[None, 256, 32, 32], dtype='float32') rois = fluid.data( name='rois', shape=[None, 4], dtype='float32') - rois_lod = fluid.data(name='rois_lod', shape=[None], dtype='int64') + rois_num = fluid.data(name='rois_num', shape=[None], dtype='int32') align_out = fluid.layers.roi_align(input=x, rois=rois, pooled_height=7, pooled_width=7, spatial_scale=0.5, sampling_ratio=-1, - rois_lod=rois_lod) + rois_num=rois_num) """ + if in_dygraph_mode(): + assert rois_num is not None, "rois_num should not be None in dygraph mode." + align_out = core.ops.roi_align( + input, rois, rois_num, "pooled_height", pooled_height, + "pooled_width", pooled_width, "spatial_scale", spatial_scale, + "sampling_ratio", sampling_ratio) + return align_out + check_variable_and_dtype(input, 'input', ['float32', 'float64'], 'roi_align') check_variable_and_dtype(rois, 'rois', ['float32', 'float64'], 'roi_align') helper = LayerHelper('roi_align', **locals()) dtype = helper.input_dtype() align_out = helper.create_variable_for_type_inference(dtype) + inputs = { + "X": input, + "ROIs": rois, + } + if rois_num is not None: + inputs['RoisNum'] = rois_num helper.append_op( type="roi_align", - inputs={"X": input, - "ROIs": rois, - "RoisLod": rois_lod}, + inputs=inputs, outputs={"Out": align_out}, attrs={ "pooled_height": pooled_height, @@ -10841,8 +10879,7 @@ def slice(input, axes, starts, ends): result = [ [2, 3, 4], ] # result = data[0:1, 1:4] Args: input (Variable): A ``Tensor`` or ``LoDTensor`` . The data type is ``float16``, ``float32``, ``float64``, ``int32`` or ``int64``. - axes (list|tuple): The data type is ``int32`` . Axes that `starts` and `ends` apply to. - It's optional. If it is not provides, it will be treated as :math:`[0,1,...,len(starts)-1]`. + axes (list|tuple): The data type is ``int32`` . Axes that `starts` and `ends` apply to . starts (list|tuple|Variable): The data type is ``int32`` . If ``starts`` is a list or tuple, the elements of it should be integers or Tensors with shape [1]. If ``starts`` is an Variable, it should be an 1-D Tensor. It represents starting indices of corresponding axis in ``axes``. diff --git a/python/paddle/fluid/layers/ops.py b/python/paddle/fluid/layers/ops.py index 1efae3ddf1f3422a53f69c4b5b8eeec6183fae96..6cdc617a0dc17ae9f0893083285c404ca73712f7 100644 --- a/python/paddle/fluid/layers/ops.py +++ b/python/paddle/fluid/layers/ops.py @@ -20,7 +20,10 @@ from ..framework import convert_np_dtype_to_dtype_, Variable from ..data_feeder import convert_dtype, check_variable_and_dtype, check_type, check_dtype from paddle.utils import deprecated -__deprecated_func_name__ = {'tanh_shrink': 'tanhshrink', } +__deprecated_func_name__ = { + 'tanh_shrink': 'tanhshrink', + 'logsigmoid': 'log_sigmoid' +} __activations_noattr__ = [ 'sigmoid', @@ -106,7 +109,7 @@ Examples: paddle.disable_static() x = paddle.to_tensor([-0.4, -0.2, 0.1, 0.3]) - out = F.logsigmoid(x) + out = F.log_sigmoid(x) print(out.numpy()) # [-0.91301525 -0.79813887 -0.64439666 -0.55435524] diff --git a/python/paddle/fluid/layers/tensor.py b/python/paddle/fluid/layers/tensor.py index a90551c1b7b4fd45ae9a0e1cfa225a87db811295..89acfc6075be0b625da04d187cd46dd47ac699c9 100644 --- a/python/paddle/fluid/layers/tensor.py +++ b/python/paddle/fluid/layers/tensor.py @@ -36,7 +36,7 @@ __all__ = [ 'tensor_array_to_tensor', 'concat', 'sums', 'assign', 'fill_constant_batch_size_like', 'fill_constant', 'argmin', 'argmax', 'argsort', 'ones', 'zeros', 'reverse', 'has_inf', 'has_nan', 'isfinite', - 'range', 'linspace', 'zeros_like', 'ones_like', 'diag', 'eye' + 'range', 'linspace', 'zeros_like', 'ones_like', 'diag', 'eye', 'triu' ] @@ -1725,3 +1725,9 @@ def ones_like(x, out=None): attrs={'value': 1.0}, outputs={'Out': [out]}) return out + + +@deprecated(since="2.0.0", update_to="paddle.triu") +def triu(input, diagonal=0, name=None): + import paddle + return paddle.tensor.triu(x=input, diagonal=diagonal, name=name) diff --git a/python/paddle/fluid/optimizer.py b/python/paddle/fluid/optimizer.py old mode 100644 new mode 100755 index 8b37cfef3890eace0ff5141eeb91d85e78f1c964..192effd2e42dc937fbf47efdd1d772a4c078f888 --- a/python/paddle/fluid/optimizer.py +++ b/python/paddle/fluid/optimizer.py @@ -1604,7 +1604,7 @@ class LarsMomentumOptimizer(Optimizer): & local\_learning\_rate = learning\_rate * lars\_coeff * \\ \\frac{||param||}{||gradient|| + lars\_weight\_decay * ||param||} - & velocity = mu * velocity + local\_learning\_rate * (gradient + lars\_weight\_decay * param) + & velocity = mu * velocity + local\_learning\_rate * (gradient + lars\_weight\_decay * param + epsilon) & param = param - velocity @@ -1628,7 +1628,9 @@ class LarsMomentumOptimizer(Optimizer): :ref:`api_fluid_clip_GradientClipByValue` ). Default None, meaning there is no gradient clipping. name (str, optional): This parameter is used by developers to print debugging information. \ For details, please refer to :ref:`api_guide_Name`. Default is None. - + exclude_from_weight_decay (list[str], optional): Name string of layers which will be exclude from lars weight decay. Default is None. + epsilon (float, optional): Epsilon to avoid Division by Zero when calculate local lr. Default is 0. + Examples: .. code-block:: python @@ -1659,7 +1661,9 @@ class LarsMomentumOptimizer(Optimizer): parameter_list=None, regularization=None, grad_clip=None, - name=None): + name=None, + exclude_from_weight_decay=None, + epsilon=0): assert learning_rate is not None assert momentum is not None super(LarsMomentumOptimizer, self).__init__( @@ -1672,6 +1676,11 @@ class LarsMomentumOptimizer(Optimizer): self._momentum = momentum self._lars_coeff = float(lars_coeff) self._lars_weight_decay = float(lars_weight_decay) + self._epsilon = float(epsilon) + if exclude_from_weight_decay is None: + self._exclude_from_weight_decay = [] + else: + self._exclude_from_weight_decay = exclude_from_weight_decay def _create_accumulators(self, block, parameters): assert isinstance(block, framework.Block) @@ -1682,6 +1691,14 @@ class LarsMomentumOptimizer(Optimizer): def _append_optimize_op(self, block, param_and_grad): assert isinstance(block, framework.Block) + _lars_weight_decay = self._lars_weight_decay + param_name = param_and_grad[0].name + if len(self._exclude_from_weight_decay) > 0: + for name in self._exclude_from_weight_decay: + if name in param_name: + _lars_weight_decay = 0.0 + break + velocity_acc = self._get_accumulator(self._velocity_acc_str, param_and_grad[0]) # create the momentum optimize op @@ -1700,7 +1717,8 @@ class LarsMomentumOptimizer(Optimizer): attrs={ "mu": self._momentum, "lars_coeff": self._lars_coeff, - "lars_weight_decay": self._lars_weight_decay + "lars_weight_decay": _lars_weight_decay, + "epsilon": self._epsilon }, stop_gradient=True) diff --git a/python/paddle/fluid/reader.py b/python/paddle/fluid/reader.py index 76c95be75d67d60cd59efe13ecba6f01a1c1d614..f2bb567b95b01eaf9a820359acef74e1c360c7f2 100644 --- a/python/paddle/fluid/reader.py +++ b/python/paddle/fluid/reader.py @@ -85,6 +85,30 @@ def _convert_places(places): return ret +# NOTE(chenweihang): _reader_process_loop must be top level method to be pickled +def _reader_process_loop(batch_reader, data_queue): + try: + # set signal handler + core._set_process_signal_handler() + + # NOTE: [ mmap files clear ] When the child process exits unexpectedly, + # some shared memory objects may have been applied for but have not yet + # been put into the inter-process Queue. This part of the object needs + # to be cleaned up when the process ends. + CleanupFuncRegistrar.register(_cleanup_mmap) + + for batch in batch_reader(): + tensor_list = core._convert_to_tensor_list(batch) + data_queue.put(tensor_list) + core._remove_tensor_list_mmap_fds(tensor_list) + data_queue.put(None) + except KeyboardInterrupt: + # NOTE: Main process will raise KeyboardInterrupt anyways, ignore it in child process + pass + except: + six.reraise(*sys.exc_info()) + + class DataLoaderBase(object): def __init__(self): self._places = None @@ -811,7 +835,8 @@ class DygraphGeneratorLoader(DataLoaderBase): global multiprocess_queue_set multiprocess_queue_set.add(self._data_queue) self._process = multiprocessing.Process( - target=self._reader_process_loop) + target=_reader_process_loop, + args=(self._batch_reader, self._data_queue)) self._process.daemon = True self._process.start() @@ -867,28 +892,6 @@ class DygraphGeneratorLoader(DataLoaderBase): self._blocking_queue.kill() logging.error("DataLoader reader thread raised an exception!") - def _reader_process_loop(self): - try: - # set signal handler - core._set_process_signal_handler() - - # NOTE: [ mmap files clear ] When the child process exits unexpectedly, - # some shared memory objects may have been applied for but have not yet - # been put into the inter-process Queue. This part of the object needs - # to be cleaned up when the process ends. - CleanupFuncRegistrar.register(_cleanup_mmap) - - for batch in self._batch_reader(): - tensor_list = core._convert_to_tensor_list(batch) - self._data_queue.put(tensor_list) - core._remove_tensor_list_mmap_fds(tensor_list) - self._data_queue.put(None) - except KeyboardInterrupt: - # NOTE: Main process will raise KeyboardInterrupt anyways, ignore it in child process - pass - except: - six.reraise(*sys.exc_info()) - def _reader_thread_loop_for_multiprocess(self): while not self._thread_done_event.is_set(): try: diff --git a/python/paddle/fluid/tests/test_detection.py b/python/paddle/fluid/tests/test_detection.py index a1526934f4aa1415c529e79bfa8dea6c0754bea9..425c4e3c7e38cff2f892eff28428082b57b3727d 100644 --- a/python/paddle/fluid/tests/test_detection.py +++ b/python/paddle/fluid/tests/test_detection.py @@ -19,6 +19,57 @@ import paddle.fluid.layers as layers from paddle.fluid.layers import detection from paddle.fluid.framework import Program, program_guard import unittest +import contextlib +import numpy as np +from unittests.test_imperative_base import new_program_scope +from paddle.fluid.dygraph import base +from paddle.fluid import core + + +class LayerTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.seed = 111 + + @classmethod + def tearDownClass(cls): + pass + + def _get_place(self, force_to_use_cpu=False): + # this option for ops that only have cpu kernel + if force_to_use_cpu: + return core.CPUPlace() + else: + if core.is_compiled_with_cuda(): + return core.CUDAPlace(0) + return core.CPUPlace() + + @contextlib.contextmanager + def static_graph(self): + with new_program_scope(): + fluid.default_startup_program().random_seed = self.seed + fluid.default_main_program().random_seed = self.seed + yield + + def get_static_graph_result(self, + feed, + fetch_list, + with_lod=False, + force_to_use_cpu=False): + exe = fluid.Executor(self._get_place(force_to_use_cpu)) + exe.run(fluid.default_startup_program()) + return exe.run(fluid.default_main_program(), + feed=feed, + fetch_list=fetch_list, + return_numpy=(not with_lod)) + + @contextlib.contextmanager + def dynamic_graph(self, force_to_use_cpu=False): + with fluid.dygraph.guard( + self._get_place(force_to_use_cpu=force_to_use_cpu)): + fluid.default_startup_program().random_seed = self.seed + fluid.default_main_program().random_seed = self.seed + yield class TestDetection(unittest.TestCase): @@ -481,45 +532,67 @@ class TestRpnTargetAssign(unittest.TestCase): print(str(program)) -class TestGenerateProposals(unittest.TestCase): +class TestGenerateProposals(LayerTest): def test_generate_proposals(self): - program = Program() - with program_guard(program): - data_shape = [20, 64, 64] - images = fluid.layers.data( - name='images', shape=data_shape, dtype='float32') - im_info = fluid.layers.data( - name='im_info', shape=[3], dtype='float32') - anchors, variances = fluid.layers.anchor_generator( - name='anchor_generator', - input=images, - anchor_sizes=[32, 64], - aspect_ratios=[1.0], - variance=[0.1, 0.1, 0.2, 0.2], - stride=[16.0, 16.0], - offset=0.5) - num_anchors = anchors.shape[2] - scores = fluid.layers.data( - name='scores', shape=[num_anchors, 8, 8], dtype='float32') - bbox_deltas = fluid.layers.data( - name='bbox_deltas', - shape=[num_anchors * 4, 8, 8], - dtype='float32') - rpn_rois, rpn_roi_probs = fluid.layers.generate_proposals( - name='generate_proposals', - scores=scores, - bbox_deltas=bbox_deltas, - im_info=im_info, - anchors=anchors, - variances=variances, - pre_nms_top_n=6000, - post_nms_top_n=1000, - nms_thresh=0.5, - min_size=0.1, - eta=1.0) - self.assertIsNotNone(rpn_rois) - self.assertIsNotNone(rpn_roi_probs) - print(rpn_rois.shape) + scores_np = np.random.rand(2, 3, 4, 4).astype('float32') + bbox_deltas_np = np.random.rand(2, 12, 4, 4).astype('float32') + im_info_np = np.array([[8, 8, 0.5], [6, 6, 0.5]]).astype('float32') + anchors_np = np.reshape(np.arange(4 * 4 * 3 * 4), + [4, 4, 3, 4]).astype('float32') + variances_np = np.ones((4, 4, 3, 4)).astype('float32') + + with self.static_graph(): + scores = fluid.data( + name='scores', shape=[2, 3, 4, 4], dtype='float32') + bbox_deltas = fluid.data( + name='bbox_deltas', shape=[2, 12, 4, 4], dtype='float32') + im_info = fluid.data(name='im_info', shape=[2, 3], dtype='float32') + anchors = fluid.data( + name='anchors', shape=[4, 4, 3, 4], dtype='float32') + variances = fluid.data( + name='var', shape=[4, 4, 3, 4], dtype='float32') + rois, roi_probs, rois_num = fluid.layers.generate_proposals( + scores, + bbox_deltas, + im_info, + anchors, + variances, + pre_nms_top_n=10, + post_nms_top_n=5, + return_rois_num=True) + rois_stat, roi_probs_stat, rois_num_stat = self.get_static_graph_result( + feed={ + 'scores': scores_np, + 'bbox_deltas': bbox_deltas_np, + 'im_info': im_info_np, + 'anchors': anchors_np, + 'var': variances_np + }, + fetch_list=[rois, roi_probs, rois_num], + with_lod=True) + + with self.dynamic_graph(): + scores_dy = base.to_variable(scores_np) + bbox_deltas_dy = base.to_variable(bbox_deltas_np) + im_info_dy = base.to_variable(im_info_np) + anchors_dy = base.to_variable(anchors_np) + variances_dy = base.to_variable(variances_np) + rois, roi_probs, rois_num = fluid.layers.generate_proposals( + scores_dy, + bbox_deltas_dy, + im_info_dy, + anchors_dy, + variances_dy, + pre_nms_top_n=10, + post_nms_top_n=5, + return_rois_num=True) + rois_dy = rois.numpy() + roi_probs_dy = roi_probs.numpy() + rois_num_dy = rois_num.numpy() + + self.assertTrue(np.array_equal(np.array(rois_stat), rois_dy)) + self.assertTrue(np.array_equal(np.array(roi_probs_stat), roi_probs_dy)) + self.assertTrue(np.array_equal(np.array(rois_num_stat), rois_num_dy)) class TestYoloDetection(unittest.TestCase): @@ -648,30 +721,81 @@ class TestMulticlassNMS2(unittest.TestCase): self.assertIsNotNone(index) -class TestCollectFpnPropsals(unittest.TestCase): +class TestCollectFpnPropsals(LayerTest): def test_collect_fpn_proposals(self): - program = Program() - with program_guard(program): + multi_bboxes_np = [] + multi_scores_np = [] + rois_num_per_level_np = [] + for i in range(4): + bboxes_np = np.random.rand(5, 4).astype('float32') + scores_np = np.random.rand(5, 1).astype('float32') + rois_num = np.array([2, 3]).astype('int32') + multi_bboxes_np.append(bboxes_np) + multi_scores_np.append(scores_np) + rois_num_per_level_np.append(rois_num) + + with self.static_graph(): multi_bboxes = [] multi_scores = [] + rois_num_per_level = [] for i in range(4): - bboxes = layers.data( + bboxes = fluid.data( name='rois' + str(i), - shape=[10, 4], + shape=[5, 4], dtype='float32', - lod_level=1, - append_batch_size=False) - scores = layers.data( + lod_level=1) + scores = fluid.data( name='scores' + str(i), - shape=[10, 1], + shape=[5, 1], dtype='float32', - lod_level=1, - append_batch_size=False) + lod_level=1) + rois_num = fluid.data( + name='rois_num' + str(i), shape=[None], dtype='int32') + multi_bboxes.append(bboxes) multi_scores.append(scores) - fpn_rois = layers.collect_fpn_proposals(multi_bboxes, multi_scores, - 2, 5, 10) - self.assertIsNotNone(fpn_rois) + rois_num_per_level.append(rois_num) + + fpn_rois, rois_num = layers.collect_fpn_proposals( + multi_bboxes, + multi_scores, + 2, + 5, + 10, + rois_num_per_level=rois_num_per_level) + feed = {} + for i in range(4): + feed['rois' + str(i)] = multi_bboxes_np[i] + feed['scores' + str(i)] = multi_scores_np[i] + feed['rois_num' + str(i)] = rois_num_per_level_np[i] + fpn_rois_stat, rois_num_stat = self.get_static_graph_result( + feed=feed, fetch_list=[fpn_rois, rois_num], with_lod=True) + fpn_rois_stat = np.array(fpn_rois_stat) + rois_num_stat = np.array(rois_num_stat) + + with self.dynamic_graph(): + multi_bboxes_dy = [] + multi_scores_dy = [] + rois_num_per_level_dy = [] + for i in range(4): + bboxes_dy = base.to_variable(multi_bboxes_np[i]) + scores_dy = base.to_variable(multi_scores_np[i]) + rois_num_dy = base.to_variable(rois_num_per_level_np[i]) + multi_bboxes_dy.append(bboxes_dy) + multi_scores_dy.append(scores_dy) + rois_num_per_level_dy.append(rois_num_dy) + fpn_rois_dy, rois_num_dy = fluid.layers.collect_fpn_proposals( + multi_bboxes_dy, + multi_scores_dy, + 2, + 5, + 10, + rois_num_per_level=rois_num_per_level_dy) + fpn_rois_dy = fpn_rois_dy.numpy() + rois_num_dy = rois_num_dy.numpy() + + self.assertTrue(np.array_equal(fpn_rois_stat, fpn_rois_dy)) + self.assertTrue(np.array_equal(rois_num_stat, rois_num_dy)) def test_collect_fpn_proposals_error(self): def generate_input(bbox_type, score_type, name): @@ -717,20 +841,51 @@ class TestCollectFpnPropsals(unittest.TestCase): post_nms_top_n=2000) -class TestDistributeFpnProposals(unittest.TestCase): +class TestDistributeFpnProposals(LayerTest): def test_distribute_fpn_proposals(self): - program = Program() - with program_guard(program): - fpn_rois = fluid.layers.data( - name='data', shape=[4], dtype='float32', lod_level=1) - multi_rois, restore_ind = layers.distribute_fpn_proposals( - fpn_rois=fpn_rois, + rois_np = np.random.rand(10, 4).astype('float32') + rois_num_np = np.array([4, 6]).astype('int32') + with self.static_graph(): + rois = fluid.data(name='rois', shape=[10, 4], dtype='float32') + rois_num = fluid.data(name='rois_num', shape=[None], dtype='int32') + multi_rois, restore_ind, rois_num_per_level = layers.distribute_fpn_proposals( + fpn_rois=rois, min_level=2, max_level=5, refer_level=4, - refer_scale=224) - self.assertIsNotNone(multi_rois) - self.assertIsNotNone(restore_ind) + refer_scale=224, + rois_num=rois_num) + fetch_list = multi_rois + [restore_ind] + rois_num_per_level + output_stat = self.get_static_graph_result( + feed={'rois': rois_np, + 'rois_num': rois_num_np}, + fetch_list=fetch_list, + with_lod=True) + output_stat_np = [] + for output in output_stat: + output_np = np.array(output) + if len(output_np) > 0: + output_stat_np.append(output_np) + + with self.dynamic_graph(): + rois_dy = base.to_variable(rois_np) + rois_num_dy = base.to_variable(rois_num_np) + multi_rois_dy, restore_ind_dy, rois_num_per_level_dy = layers.distribute_fpn_proposals( + fpn_rois=rois_dy, + min_level=2, + max_level=5, + refer_level=4, + refer_scale=224, + rois_num=rois_num_dy) + output_dy = multi_rois_dy + [restore_ind_dy] + rois_num_per_level_dy + output_dy_np = [] + for output in output_dy: + output_np = output.numpy() + if len(output_np) > 0: + output_dy_np.append(output_np) + + for res_stat, res_dy in zip(output_stat_np, output_dy_np): + self.assertTrue(np.array_equal(res_stat, res_dy)) def test_distribute_fpn_proposals_error(self): program = Program() diff --git a/python/paddle/fluid/tests/unittests/CMakeLists.txt b/python/paddle/fluid/tests/unittests/CMakeLists.txt index a25cba029dd8bac81d6b00c1d9fb710f421ce9d0..8f3945a48e387766f77c4202957bbc4a76ee0104 100644 --- a/python/paddle/fluid/tests/unittests/CMakeLists.txt +++ b/python/paddle/fluid/tests/unittests/CMakeLists.txt @@ -47,6 +47,8 @@ list(APPEND MIXED_DIST_TEST_OPS test_fleet_dgc_meta_optimizer) list(APPEND MIXED_DIST_TEST_OPS test_fleet_private_function) list(APPEND MIXED_DIST_TEST_OPS test_fleet_graph_executor) list(APPEND MIXED_DIST_TEST_OPS test_fleet_meta_optimizer_base) +list(APPEND MIXED_DIST_TEST_OPS test_fleet_distributed_strategy) +list(APPEND MIXED_DIST_TEST_OPS test_fleet_auto) foreach(TEST_OP ${MIXED_DIST_TEST_OPS}) list(REMOVE_ITEM TEST_OPS ${TEST_OP}) endforeach() @@ -458,10 +460,12 @@ if(WITH_DISTRIBUTE) py_test_modules(test_fleet_pipeline_meta_optimizer MODULES test_fleet_pipeline_meta_optimizer ENVS ${dist_ENVS}) py_test_modules(test_fleet_private_function MODULES test_fleet_private_function ENVS ${dist_ENVS}) py_test_modules(test_fleet_meta_optimizer_base MODULES test_fleet_meta_optimizer_base ENVS ${dist_ENVS}) + py_test_modules(test_fleet_distributed_strategy MODULES test_fleet_distributed_strategy) + #py_test_modules(test_fleet_auto MODULES test_fleet_auto ENVS ${dist_ENVS}) if(NOT WIN32) py_test_modules(test_fleet_localsgd_meta_optimizer MODULES test_fleet_localsgd_meta_optimizer ENVS ${dist_ENVS}) - py_test_modules(test_fleet_lars_meta_optimizer MODULES test_fleet_lars_meta_optimizer ENVS ${dist_ENVS}) - py_test_modules(test_fleet_lamb_meta_optimizer MODULES test_fleet_lamb_meta_optimizer ENVS ${dist_ENVS}) + #py_test_modules(test_fleet_lars_meta_optimizer MODULES test_fleet_lars_meta_optimizer ENVS ${dist_ENVS}) + #py_test_modules(test_fleet_lamb_meta_optimizer MODULES test_fleet_lamb_meta_optimizer ENVS ${dist_ENVS}) endif(NOT WIN32) endif(NOT APPLE) if(WITH_DGC) @@ -555,8 +559,8 @@ endif() set_tests_properties(test_parallel_executor_test_while_train test_parallel_executor_mnist test_parallel_executor_feed_persistable_var test_buffer_shared_memory_reuse_pass_and_fuse_optimization_op_pass - test_data_norm_op test_imperative_using_non_zero_gpu test_fuse_bn_act_pass - test_optimizer_in_control_flow test_dataloader_keep_order + test_data_norm_op test_imperative_using_non_zero_gpu + test_dataloader_keep_order test_dataloader_unkeep_order test_parallel_executor_fetch_isolated_var test_parallel_executor_inference_feed_partial_data diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_declarative.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_declarative.py index 0b8df63d666b6547d5dccfc2ce0b420d653cc544..5582a65304d3e9bad2d4621e11f8a4f312189a9a 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_declarative.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_declarative.py @@ -47,8 +47,8 @@ class SimpleNet(Layer): return z @declarative(input_spec=[[InputSpec([None, 10]), InputSpec([None, 10])]]) - def func_with_list(self, l): - x, y, int_val = l + def func_with_list(self, l, int_val=1): + x, y = l z = x + y z = z + int_val return z @@ -60,10 +60,7 @@ class SimpleNet(Layer): def func_with_dict(self, d): x = d['x'] y = d['y'] - int_val = d['int_val'] - z = x + y - z = z + int_val return z @@ -131,10 +128,10 @@ class TestInputSpec(unittest.TestCase): self.assertTrue(len(net.add_func.program_cache) == 1) # 5. test input with list - out = net.func_with_list([x, y, int_val]) + out = net.func_with_list([x, y], int_val) # 6. test input with dict - out = net.func_with_dict({'x': x, 'y': y, 'int_val': int_val}) + out = net.func_with_dict({'x': x, 'y': y}) # 7. test input with lits contains dict int_np = np.ones([1]).astype('float32') @@ -293,6 +290,30 @@ class TestDifferentInputSpecCacheProgram(unittest.TestCase): foo_3.concrete_program +class TestInputDefaultName(unittest.TestCase): + def setUp(self): + paddle.disable_static() + self.net = SimpleNet() + + def assert_default_name(self, func_name, input_names): + decorated_func = getattr(self.net, func_name) + + spec_names = [x.name for x in decorated_func.inputs] + self.assertListEqual(spec_names, input_names) + + def test_common_input(self): + self.assert_default_name('forward', ['x']) + + def test_list_input(self): + self.assert_default_name('func_with_list', ['l_0', 'l_1']) + + def test_dict_input(self): + self.assert_default_name('func_with_dict', ['x', 'y']) + + def test_nest_input(self): + self.assert_default_name('func_with_list_dict', ['dl_0', 'x', 'y']) + + class TestDeclarativeAPI(unittest.TestCase): def test_error(self): func = declarative(dyfunc_to_variable) @@ -311,5 +332,31 @@ class TestDeclarativeAPI(unittest.TestCase): func(np.ones(5).astype("int32")) +class TestDecorateModelDirectly(unittest.TestCase): + def setUp(self): + paddle.disable_static() + program_trans.enable(True) + self.x = to_variable(np.ones([4, 10]).astype('float32')) + + def test_fake_input(self): + net = SimpleNet() + net = declarative(net) + y = net(self.x) + self.assertTrue(len(net.forward.program_cache) == 1) + + def test_input_spec(self): + net = SimpleNet() + net = declarative(net, input_spec=[InputSpec([None, 8, 10])]) + self.assertTrue(len(net.forward.inputs) == 1) + self.assertTrue(len(net.forward.program_cache) == 1) + input_shape = net.forward.inputs[0].shape + self.assertListEqual(list(input_shape), [-1, 8, 10]) + + # redecorate + net = declarative(net, input_spec=[InputSpec([None, 16, 10])]) + input_shape = net.forward.inputs[0].shape + self.assertListEqual(list(input_shape), [-1, 16, 10]) + + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_error.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_error.py index 586020d434519b12c6fff4cbba812a013cf45c3d..2998ba85757e7677d5f9ab39ff81682a8b315072 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_error.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_error.py @@ -14,15 +14,15 @@ from __future__ import print_function +import os import inspect import unittest - import numpy as np +import paddle import paddle.fluid as fluid from paddle.fluid.core import EnforceNotMet -from paddle.fluid.dygraph.dygraph_to_static.error import ERROR_DATA, ErrorData +from paddle.fluid.dygraph.dygraph_to_static import error from paddle.fluid.dygraph.dygraph_to_static.origin_info import unwrap -from paddle.fluid.dygraph.jit import declarative def inner_func(): @@ -30,7 +30,7 @@ def inner_func(): return -@declarative +@paddle.jit.to_static def func_error_in_compile_time(x): x = fluid.dygraph.to_variable(x) inner_func() @@ -41,14 +41,14 @@ def func_error_in_compile_time(x): return x_v -@declarative +@paddle.jit.to_static def func_error_in_compile_time_2(x): x = fluid.dygraph.to_variable(x) x = fluid.layers.reshape(x, shape=[1, 2]) return x -@declarative +@paddle.jit.to_static def func_error_in_runtime(x, iter_num=3): x = fluid.dygraph.to_variable(x) two = fluid.layers.fill_constant(shape=[1], value=2, dtype="int32") @@ -61,6 +61,9 @@ class TestErrorInCompileTime(unittest.TestCase): self.set_func() self.set_input() self.set_exception_type() + self.prog_trans = paddle.jit.ProgramTranslator() + self.simplify_error = 1 + self.disable_error = 0 def set_func(self): self.func = func_error_in_compile_time @@ -88,14 +91,38 @@ class TestErrorInCompileTime(unittest.TestCase): for m in self.expected_message: self.assertIn(m, error_message) - def test(self): - with fluid.dygraph.guard(): - with self.assertRaises(self.exception_type) as cm: - self.func(self.input) - exception = cm.exception - error_data = getattr(exception, ERROR_DATA) - self.assertIsInstance(error_data, ErrorData) - self._test_create_message(error_data) + def _test_attach_and_raise_new_exception(self, func_call): + paddle.disable_static() + with self.assertRaises(self.exception_type) as cm: + func_call() + exception = cm.exception + + error_data = getattr(exception, error.ERROR_DATA, None) + + self.assertIsInstance(error_data, error.ErrorData) + self._test_create_message(error_data) + + def test_static_layer_call(self): + # NOTE: self.func(self.input) is the StaticLayer().__call__(self.input) + call_dy2static = lambda: self.func(self.input) + + self.set_flags(0) + self._test_attach_and_raise_new_exception(call_dy2static) + + def test_program_translator_get_output(self): + call_dy2static = lambda : self.prog_trans.get_output(unwrap(self.func), self.input) + + self.set_flags(0) + self._test_attach_and_raise_new_exception(call_dy2static) + + def set_flags(self, disable_error=0, simplify_error=1): + os.environ[error.DISABLE_ERROR_ENV_NAME] = str(disable_error) + self.disable_error = int(os.getenv(error.DISABLE_ERROR_ENV_NAME, 0)) + self.assertEqual(self.disable_error, disable_error) + + os.environ[error.SIMPLIFY_ERROR_ENV_NAME] = str(simplify_error) + self.simplify_error = int(os.getenv(error.SIMPLIFY_ERROR_ENV_NAME, 1)) + self.assertEqual(self.simplify_error, simplify_error) class TestErrorInCompileTime2(TestErrorInCompileTime): @@ -143,5 +170,28 @@ class TestErrorInRuntime(TestErrorInCompileTime): self.assertIn(m, error_message) +@unwrap +@paddle.jit.to_static() +def func_decorated_by_other_1(): + return 1 + + +@paddle.jit.to_static() +@unwrap +def func_decorated_by_other_2(): + return 1 + + +class TestErrorInOther(unittest.TestCase): + def test(self): + paddle.disable_static() + prog_trans = paddle.jit.ProgramTranslator() + with self.assertRaises(NotImplementedError): + prog_trans.get_output(func_decorated_by_other_1) + + with self.assertRaises(NotImplementedError): + func_decorated_by_other_2() + + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_origin_info.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_origin_info.py index b03777b6ebc7f3cceb73cd32e6fdfea11755320e..3f77e9ade285e2c3d8452ea2171505442ee52fb0 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_origin_info.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_origin_info.py @@ -14,6 +14,7 @@ from __future__ import print_function +import sys import unittest from paddle.fluid.dygraph.dygraph_to_static.ast_transformer import DygraphToStaticAst @@ -177,8 +178,20 @@ class TestOriginInfoWithDecoratedFunc(TestOriginInfo): def set_dygraph_info(self): self.line_num = 2 - self.line_index_list = [0, 2] - self.dy_rel_lineno_list = [0, 2] + + # NOTE(liym27): + # There are differences in ast_node.lineno between PY3.8+ and PY3.8-. + # If the first gast.FunctionDef has decorator, the lineno of gast.FunctionDef is differs. + # 1. < PY3.8 + # its lineno equals to the lineno of the first decorator node, which is not right. + # 2. >= PY3.8 + # its lineno is the actual lineno, which is right. + if sys.version_info >= (3, 8): + self.line_index_list = [1, 2] + self.dy_rel_lineno_list = [1, 2] + else: + self.line_index_list = [0, 2] + self.dy_rel_lineno_list = [0, 2] self.dy_abs_col_offset = [0, 4] self.dy_func_name = [self.dygraph_func.__name__] * self.line_num @@ -199,8 +212,13 @@ class TestOriginInfoWithDecoratedFunc2(TestOriginInfo): def set_dygraph_info(self): self.line_num = 2 - self.line_index_list = [0, 3] - self.dy_rel_lineno_list = [0, 3] + + if sys.version_info >= (3, 8): + self.line_index_list = [2, 3] + self.dy_rel_lineno_list = [2, 3] + else: + self.line_index_list = [0, 3] + self.dy_rel_lineno_list = [0, 3] self.dy_abs_col_offset = [0, 4] self.dy_func_name = [self.dygraph_func.__name__] * self.line_num diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_partial_program.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_partial_program.py index f0fbe54f9dbbf93121655e784601467c13b3a70d..91067f360995e1661c200df923a698f3f146b71e 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_partial_program.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_partial_program.py @@ -14,6 +14,7 @@ from __future__ import print_function import numpy as np +import paddle import paddle.fluid as fluid from paddle.fluid.layers.utils import flatten from paddle.fluid.dygraph import declarative, ProgramTranslator @@ -151,5 +152,33 @@ class TestWithTrainAndEval(unittest.TestCase): partial_layer._train_program) +class GPT2LMHeadModel(fluid.dygraph.Layer): + def __init__(self): + super(GPT2LMHeadModel, self).__init__() + self.embedding0 = paddle.nn.Embedding(20, 16) + self.embedding1 = paddle.nn.Embedding(20, 32) + self.lm_head_weight = paddle.to_tensor( + np.random.rand(2, 3).astype('float32')) + + @declarative + def forward(self, x): + x = fluid.layers.reshape(x, shape=[-1, 6]) + x1, x2, x3 = fluid.layers.split(input=x, dim=1, num_or_sections=3) + return x1 + + +class TestPruneUnusedParamInProgram(unittest.TestCase): + def test_prune(self): + input_ids = np.array([[15, 11, 6, 3, 18, 13]]).astype("float32") + + place = fluid.CPUPlace() + with fluid.dygraph.guard(place): + model = GPT2LMHeadModel() + model.eval() + input_ids = paddle.to_tensor(input_ids) + out = model(input_ids) + self.assertTrue(np.array_equal(out.numpy(), [[15, 11]])) + + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_gru_int8_mkldnn_op.py b/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_gru_int8_mkldnn_op.py new file mode 100644 index 0000000000000000000000000000000000000000..ff4531f0e250e325f39ef69161c8d1ee751a2336 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/mkldnn/test_fusion_gru_int8_mkldnn_op.py @@ -0,0 +1,145 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import numpy as np +from paddle.fluid.tests.unittests.op_test import OpTest +from paddle.fluid.tests.unittests.test_fusion_gru_op import fusion_gru +from paddle.fluid.tests.unittests.test_fusion_lstm_op import fc, ACTIVATION + + +class TestFusionGRUINT8MKLDNNOp(OpTest): + def set_confs(self): + pass + + def setUp(self): + self.op_type = "fusion_gru" + self.lod = [[2, 4, 3]] + self.IC = 3 + self.OC = 5 + self.is_reverse = False + self.with_h0 = False + self.with_bias = True + self.act_state = 'tanh' + self.act_gate = 'sigmoid' + self.origin_mode = True + self.use_mkldnn = True + self.force_fp32_output = True + self.error_margin = 1e-5 + self.set_confs() + + # RNN dimensions + T = sum(self.lod[0]) + N = len(self.lod[0]) + + # Input data + x_f32 = np.random.rand(T, self.IC).astype('float32') * 2 - 1 + scale_data = 63 + shift_data = 64 + x_u8 = (x_f32 * scale_data + shift_data).astype(np.uint8) + + # WeightX/WeightH data + wx = np.random.rand(self.IC, 3 * self.OC).astype('float32') * 2 - 1 + wh = np.random.rand(self.OC, 3 * self.OC).astype('float32') * 2 - 1 + + # Calculating weight scales + # scales = 63 / max(abs(channel_wise(weightsX + weightsH))) + # WeightX data shape in PP: [IC, 3 * OC] + # WeightH data shape in PP: [OC, 2 * OC] + [OC, OC] + # Scales shape in oneDNN: [3, OC] + scale_ur = 63 / np.max(np.abs( + np.concatenate( + [ + wx[:, :2 * self.OC], wh.flatten()[:2 * self.OC * self.OC] + .reshape(self.OC, 2 * self.OC) + ], + axis=0)), + axis=0) + scale_o = 63 / np.max(np.abs( + np.concatenate( + [ + wx[:, 2 * self.OC:], wh.flatten()[2 * self.OC * self.OC:] + .reshape(self.OC, self.OC) + ], + axis=0)), + axis=0) + + scale_weights = np.concatenate([scale_ur, scale_o]).astype('float') + + bias = np.random.rand( + 1, 3 * self.OC).astype('float32') if self.with_bias else np.zeros( + (1, 3 * self.OC), dtype='float32') + h0 = np.random.rand( + N, self.OC).astype('float32') if self.with_h0 else np.zeros( + (N, self.OC), dtype='float32') + + _, _, _, hidden_f32 = fusion_gru(x_f32, self.lod, h0, wx, wh, bias, + self.is_reverse, self.origin_mode, + ACTIVATION[self.act_state], + ACTIVATION[self.act_gate]) + + self.inputs = {'X': (x_u8, self.lod), 'WeightX': wx, 'WeightH': wh} + + if self.with_bias: + self.inputs['Bias'] = bias + + if self.with_h0: + self.inputs['H0'] = h0 + + if self.force_fp32_output: + self.error_margin = 1e-1 + self.outputs = {'Hidden': (hidden_f32, self.lod)} + else: + self.error_margin = 1 + hidden_u8 = (hidden_f32 * scale_data + shift_data).astype(np.uint8) + self.outputs = {'Hidden': (hidden_u8, self.lod)} + + self.attrs = { + 'activation': self.act_state, + 'gate_activation': self.act_gate, + 'is_reverse': self.is_reverse, + 'origin_mode': self.origin_mode, + 'use_mkldnn': self.use_mkldnn, + 'force_fp32_output': self.force_fp32_output, + 'Scale_data': scale_data, + 'Shift_data': shift_data, + 'Scale_weights': scale_weights + } + + def test_check_output(self): + self.check_output(check_dygraph=False, atol=self.error_margin) + + +class TestFusionGRUINT8MKLDNNOp2(TestFusionGRUINT8MKLDNNOp): + def set_confs(self): + self.force_fp32_output = False + + +class TestFusionGRUINT8MKLDNNOp3(TestFusionGRUINT8MKLDNNOp): + def set_confs(self): + self.origin_mode = False + + +class TestFusionGRUINT8MKLDNNOp4(TestFusionGRUINT8MKLDNNOp): + def set_confs(self): + self.with_bias = False + + +class TestFusionGRUINT8MKLDNNOp5(TestFusionGRUINT8MKLDNNOp): + def set_confs(self): + self.with_h0 = False + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_activation_op.py b/python/paddle/fluid/tests/unittests/test_activation_op.py index ab61a5b3cfccb0e885debe9786ae91a9754e9345..f6ba03194aa909279aa2cd884fc575041b01a4cd 100755 --- a/python/paddle/fluid/tests/unittests/test_activation_op.py +++ b/python/paddle/fluid/tests/unittests/test_activation_op.py @@ -128,7 +128,7 @@ class TestLogSigmoid(TestActivation): class TestLogSigmoidAPI(unittest.TestCase): - # test paddle.nn.LogSigmoid, paddle.nn.functional.logsigmoid + # test paddle.nn.LogSigmoid, paddle.nn.functional.log_sigmoid def setUp(self): self.x_np = np.random.uniform(-1, 1, [11, 17]).astype('float32') self.place=paddle.CUDAPlace(0) if core.is_compiled_with_cuda() \ @@ -137,36 +137,45 @@ class TestLogSigmoidAPI(unittest.TestCase): def test_static_api(self): with paddle.static.program_guard(paddle.static.Program()): x = paddle.data('X', [11, 17]) - out1 = F.logsigmoid(x) + out1 = F.log_sigmoid(x) m = paddle.nn.LogSigmoid() out2 = m(x) exe = paddle.static.Executor(self.place) res = exe.run(feed={'X': self.x_np}, fetch_list=[out1, out2]) out_ref = np.log(1 / (1 + np.exp(-self.x_np))) for r in res: - self.assertEqual(np.allclose(out_ref, r), True) + self.assertTrue(np.allclose(out_ref, r)) def test_dygraph_api(self): paddle.disable_static(self.place) x = paddle.to_tensor(self.x_np) - out1 = F.logsigmoid(x) + out1 = F.log_sigmoid(x) m = paddle.nn.LogSigmoid() out2 = m(x) out_ref = np.log(1 / (1 + np.exp(-self.x_np))) for r in [out1, out2]: - self.assertEqual(np.allclose(out_ref, r.numpy()), True) + self.assertTrue(np.allclose(out_ref, r.numpy())) paddle.enable_static() + def test_fluid_api(self): + with paddle.static.program_guard(paddle.static.Program()): + x = paddle.data('X', [11, 17]) + out = paddle.fluid.layers.logsigmoid(x) + exe = paddle.static.Executor(self.place) + res = exe.run(feed={'X': self.x_np}, fetch_list=[out]) + out_ref = np.log(1 / (1 + np.exp(-self.x_np))) + self.assertTrue(np.allclose(out_ref, res[0])) + def test_errors(self): with paddle.static.program_guard(paddle.static.Program()): # The input type must be Variable. - self.assertRaises(TypeError, F.logsigmoid, 1) + self.assertRaises(TypeError, F.log_sigmoid, 1) # The input dtype must be float16, float32, float64. x_int32 = paddle.data(name='x_int32', shape=[11, 17], dtype='int32') - self.assertRaises(TypeError, F.logsigmoid, x_int32) + self.assertRaises(TypeError, F.log_sigmoid, x_int32) # support the input dtype is float16 x_fp16 = paddle.data(name='x_fp16', shape=[11, 17], dtype='float16') - F.logsigmoid(x_fp16) + F.log_sigmoid(x_fp16) class TestTanh(TestActivation, TestParameter): diff --git a/python/paddle/fluid/tests/unittests/test_adam_op.py b/python/paddle/fluid/tests/unittests/test_adam_op.py index 14e83fccd655527d8f3012365e4757d23236a445..47bf8f49e39b6451ee480d461e83324b89cacee2 100644 --- a/python/paddle/fluid/tests/unittests/test_adam_op.py +++ b/python/paddle/fluid/tests/unittests/test_adam_op.py @@ -448,7 +448,6 @@ class TestAdamOpV2(unittest.TestCase): def test_adam_op_with_state_dict(self): - import paddle paddle.disable_static() emb = paddle.nn.Embedding(10, 10) @@ -517,6 +516,20 @@ class TestAdamOpV2(unittest.TestCase): adam = paddle.optimizer.Adam( 0.1, epsilon=-1, parameters=linear.parameters()) + def test_adam_op_with_sparse_input_and_weight_decay(self): + + paddle.disable_static() + x_data = np.arange(0, 10).reshape((10, 1)).astype(np.int64) + x = paddle.to_tensor(x_data, stop_gradient=False) + emb = paddle.nn.Embedding(10, 10, sparse=True) + adam = paddle.optimizer.Adam( + 0.001, parameters=emb.parameters(), weight_decay=0.01) + + with self.assertRaises(RuntimeError): + out = emb(x) + out.backward() + adam.step() + if __name__ == "__main__": unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool1d.py b/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool1d.py index 5a135cea52903a0d896df2d446b58d99e5a18993..424406c15bb18bade54a9b11bfdd96862d4df85c 100644 --- a/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool1d.py +++ b/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool1d.py @@ -94,6 +94,10 @@ class TestPool1d_API(unittest.TestCase): result = ada_max_pool1d_dg(input) self.assertTrue(np.allclose(result.numpy(), result_np)) + result = paddle.nn.functional.common.interpolate( + input, mode="area", size=16) + self.assertTrue(np.allclose(result.numpy(), result_np)) + def check_adaptive_avg_static_results(self, place): with fluid.program_guard(fluid.Program(), fluid.Program()): input = fluid.data(name="input", shape=[2, 3, 32], dtype="float32") diff --git a/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool2d.py b/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool2d.py index 55c30e3d2ade0725e6debcdd0a69ca4eee622aec..e3c70884ebcf116feb4f5b0aa808c71e4b7f8c4e 100644 --- a/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool2d.py +++ b/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool2d.py @@ -163,6 +163,9 @@ class TestAdaptiveAvgPool2dAPI(unittest.TestCase): out_5 = paddle.nn.functional.adaptive_avg_pool2d( x=x, output_size=[None, 3]) + out_6 = paddle.nn.functional.interpolate( + x=x, mode="area", size=[2, 5]) + assert np.allclose(out_1.numpy(), self.res_1_np) assert np.allclose(out_2.numpy(), self.res_2_np) @@ -173,6 +176,8 @@ class TestAdaptiveAvgPool2dAPI(unittest.TestCase): assert np.allclose(out_5.numpy(), self.res_5_np) + assert np.allclose(out_6.numpy(), self.res_3_np) + class TestAdaptiveAvgPool2dClassAPI(unittest.TestCase): def setUp(self): diff --git a/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool3d.py b/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool3d.py index c04ee660667edaff01d7029e83b912c05429a15f..a3c9dd91a69ea83b08c3f817403620460333b5e9 100755 --- a/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool3d.py +++ b/python/paddle/fluid/tests/unittests/test_adaptive_avg_pool3d.py @@ -178,6 +178,9 @@ class TestAdaptiveAvgPool3dAPI(unittest.TestCase): out_5 = paddle.nn.functional.adaptive_avg_pool3d( x=x, output_size=[None, 3, None]) + out_6 = paddle.nn.functional.interpolate( + x=x, mode="area", size=[2, 3, 5]) + assert np.allclose(out_1.numpy(), self.res_1_np) assert np.allclose(out_2.numpy(), self.res_2_np) @@ -188,6 +191,8 @@ class TestAdaptiveAvgPool3dAPI(unittest.TestCase): assert np.allclose(out_5.numpy(), self.res_5_np) + assert np.allclose(out_6.numpy(), self.res_3_np) + class TestAdaptiveAvgPool3dClassAPI(unittest.TestCase): def setUp(self): diff --git a/python/paddle/fluid/tests/unittests/test_amp_check_finite_and_scale_op.py b/python/paddle/fluid/tests/unittests/test_amp_check_finite_and_scale_op.py index 70863d3857c43c84a583f0ccf7b9bd733fdb4fd0..fbacaa3d5ce10bdad6dd87fdfc04c1173aff18ff 100644 --- a/python/paddle/fluid/tests/unittests/test_amp_check_finite_and_scale_op.py +++ b/python/paddle/fluid/tests/unittests/test_amp_check_finite_and_scale_op.py @@ -18,9 +18,9 @@ from op_test import OpTest, skip_check_grad_ci import paddle.fluid as fluid -class TestAmpCheckFiniteAndScaleOp(OpTest): +class TestCheckFiniteAndUnscaleOp(OpTest): def setUp(self): - self.op_type = "amp_check_finite_and_scale" + self.op_type = "check_finite_and_unscale" self.init_dtype() x = np.random.random((1024, 1024)).astype(self.dtype) scale = np.random.random((1)).astype(self.dtype) @@ -28,7 +28,7 @@ class TestAmpCheckFiniteAndScaleOp(OpTest): self.inputs = {'X': [('x0', x)], 'Scale': scale} self.outputs = { 'FoundInfinite': np.array([0]), - 'Out': [('out0', x * scale)], + 'Out': [('out0', x / scale)], } def init_dtype(self): @@ -38,9 +38,9 @@ class TestAmpCheckFiniteAndScaleOp(OpTest): self.check_output() -class TestAmpCheckFiniteAndScaleOpWithNan(OpTest): +class TestCheckFiniteAndUnscaleOpWithNan(OpTest): def setUp(self): - self.op_type = "amp_check_finite_and_scale" + self.op_type = "check_finite_and_unscale" self.init_dtype() x = np.random.random((1024, 1024)).astype(self.dtype) x[128][128] = np.nan @@ -61,9 +61,9 @@ class TestAmpCheckFiniteAndScaleOpWithNan(OpTest): self.check_output(no_check_set=['Out']) -class TestAmpCheckFiniteAndScaleOpWithInf(OpTest): +class TestCheckFiniteAndUnscaleOpWithInf(OpTest): def setUp(self): - self.op_type = "amp_check_finite_and_scale" + self.op_type = "check_finite_and_unscale" self.init_dtype() x = np.random.random((1024, 1024)).astype(self.dtype) x[128][128] = np.inf diff --git a/python/paddle/fluid/tests/unittests/test_arg_min_max_v2_op.py b/python/paddle/fluid/tests/unittests/test_arg_min_max_v2_op.py index 1b1b1d7c983282974d2fa46038c35c98de4f9ec2..74f76030a29d2c9ce27278b61548c8877c1467ad 100644 --- a/python/paddle/fluid/tests/unittests/test_arg_min_max_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_arg_min_max_v2_op.py @@ -325,16 +325,16 @@ class TestArgMinMaxOpError(unittest.TestCase): def test_argmax_dtype_type(): data = paddle.static.data( name="test_argmax", shape=[10], dtype="float32") - output = paddle.argmax(x=data, dtype=1) + output = paddle.argmax(x=data, dtype=None) - self.assertRaises(TypeError, test_argmax_dtype_type) + self.assertRaises(ValueError, test_argmax_dtype_type) def test_argmin_dtype_type(): data = paddle.static.data( name="test_argmin", shape=[10], dtype="float32") - output = paddle.argmin(x=data, dtype=1) + output = paddle.argmin(x=data, dtype=None) - self.assertRaises(TypeError, test_argmin_dtype_type) + self.assertRaises(ValueError, test_argmin_dtype_type) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_bicubic_interp_v2_op.py b/python/paddle/fluid/tests/unittests/test_bicubic_interp_v2_op.py index 01daea32167d28edbb46d6854872976aed79494e..b1ec74411987a73cf2e6a7d60aecce6c87ed598e 100644 --- a/python/paddle/fluid/tests/unittests/test_bicubic_interp_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_bicubic_interp_v2_op.py @@ -53,6 +53,8 @@ def value_bound(input, w, h, x, y): def bicubic_interp_np(input, out_h, out_w, + scale_h=0, + scale_w=0, out_size=None, actual_shape=None, align_corners=True, @@ -73,13 +75,19 @@ def bicubic_interp_np(input, if (align_corners): ratio_h = (in_h - 1.0) / (out_h - 1.0) else: - ratio_h = 1.0 * in_h / out_h + if scale_h > 0: + ratio_h = 1.0 / scale_h + else: + ratio_h = 1.0 * in_h / out_h if out_w > 1: if (align_corners): ratio_w = (in_w - 1.0) / (out_w - 1.0) else: - ratio_w = 1.0 * in_w / out_w + if scale_w > 0: + ratio_w = 1.0 / scale_w + else: + ratio_w = 1.0 * in_w / out_w out = np.zeros((batch_size, channel, out_h, out_w)) @@ -128,7 +136,8 @@ class TestBicubicInterpOp(OpTest): self.init_test_case() self.op_type = "bicubic_interp_v2" input_np = np.random.random(self.input_shape).astype("float64") - + scale_h = 0 + scale_w = 0 if self.data_layout == "NCHW": in_h = self.input_shape[2] in_w = self.input_shape[3] @@ -151,9 +160,9 @@ class TestBicubicInterpOp(OpTest): out_h = self.out_h out_w = self.out_w - output_np = bicubic_interp_np(input_np, out_h, out_w, self.out_size, - self.actual_shape, self.align_corners, - self.data_layout) + output_np = bicubic_interp_np(input_np, out_h, out_w, scale_h, scale_w, + self.out_size, self.actual_shape, + self.align_corners, self.data_layout) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -480,10 +489,34 @@ class TestBicubicOpError(unittest.TestCase): out = interpolate( x, size=None, - mode='trilinear', + mode='bicubic', align_corners=False, scale_factor=[1, 2, 2]) + def test_size_and_scale(): + x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") + out = interpolate( + x, + size=None, + mode='bicubic', + align_corners=False, + scale_factor=None) + + def test_size_and_scale2(): + x = fluid.data( + name="input", shape=[2, 3, 6, 9, 4], dtype="float32") + out = interpolate( + x, + size=[2, 2, 2], + mode='trilinear', + align_corners=False, + scale_factor=2.0) + + def test_size_type(): + x = fluid.data(name="x", shape=[2, 3, 6, 6], dtype="float32") + out = interpolate( + x, size={2, 2}, mode='bicubic', align_corners=False) + self.assertRaises(ValueError, test_mode_type) self.assertRaises(ValueError, test_input_shape) self.assertRaises(TypeError, test_align_corcers) @@ -498,6 +531,9 @@ class TestBicubicOpError(unittest.TestCase): self.assertRaises(ValueError, test_align_corners_and_nearest) self.assertRaises(ValueError, test_scale_shape) self.assertRaises(ValueError, test_scale_value) + self.assertRaises(ValueError, test_size_and_scale) + self.assertRaises(ValueError, test_size_and_scale2) + self.assertRaises(TypeError, test_size_type) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_bilinear_interp_v2_op.py b/python/paddle/fluid/tests/unittests/test_bilinear_interp_v2_op.py index d139a53c7e2ccc68964457f3142b4ed890d339f2..9fc4971fec23923a40080613612d3a1843a86d2e 100755 --- a/python/paddle/fluid/tests/unittests/test_bilinear_interp_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_bilinear_interp_v2_op.py @@ -26,6 +26,8 @@ import paddle def bilinear_interp_np(input, out_h, out_w, + scale_w=0, + scale_h=0, out_size=None, actual_shape=None, align_corners=True, @@ -47,12 +49,18 @@ def bilinear_interp_np(input, if (align_corners): ratio_h = (in_h - 1.0) / (out_h - 1.0) else: - ratio_h = 1.0 * in_h / out_h + if scale_h > 0: + ratio_h = 1.0 / scale_h + else: + ratio_h = 1.0 * in_h / out_h if out_w > 1: if (align_corners): ratio_w = (in_w - 1.0) / (out_w - 1.0) else: - ratio_w = 1.0 * in_w / out_w + if scale_w > 0: + ratio_w = 1.0 / scale_w + else: + ratio_w = 1.0 * in_w / out_w out = np.zeros((batch_size, channel, out_h, out_w)) @@ -110,7 +118,8 @@ class TestBilinearInterpOp(OpTest): else: in_h = self.input_shape[1] in_w = self.input_shape[2] - + scale_h = 0 + scale_w = 0 if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): if self.scale > 0.: @@ -126,9 +135,9 @@ class TestBilinearInterpOp(OpTest): out_h = self.out_h out_w = self.out_w - output_np = bilinear_interp_np(input_np, out_h, out_w, self.out_size, - self.actual_shape, self.align_corners, - self.align_mode, self.data_layout) + output_np = bilinear_interp_np( + input_np, out_h, out_w, 0, 0, self.out_size, self.actual_shape, + self.align_corners, self.align_mode, self.data_layout) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -238,6 +247,17 @@ class TestBilinearInterpCase6(TestBilinearInterpOp): self.align_mode = 1 +class TestBilinearInterpCase7(TestBilinearInterpOp): + def init_test_case(self): + self.interp_method = 'bilinear' + self.input_shape = [1, 1, 32, 64] + self.out_h = 64 + self.out_w = 32 + self.scale = [2.0, 0.5] + self.align_corners = False + self.align_mode = 1 + + class TestBilinearInterpSame(TestBilinearInterpOp): def init_test_case(self): self.interp_method = 'bilinear' @@ -298,9 +318,9 @@ class TestBilinearInterpOpUint8(OpTest): out_h = self.out_h out_w = self.out_w - output_np = bilinear_interp_np(input_np, out_h, out_w, self.out_size, - self.actual_shape, self.align_corners, - self.align_mode) + output_np = bilinear_interp_np(input_np, out_h, out_w, 0, 0, + self.out_size, self.actual_shape, + self.align_corners, self.align_mode) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -481,8 +501,9 @@ class TestBilinearInterpOp_attr_tensor(OpTest): if isinstance(self.scale, list) and len(self.scale) == 1: self.scale = [self.scale[0], self.scale[0]] self.attrs['scale'] = self.scale - output_np = bilinear_interp_np(input_np, out_h, out_w, self.out_size, - self.actual_shape, self.align_corners) + output_np = bilinear_interp_np(input_np, out_h, out_w, 0, 0, + self.out_size, self.actual_shape, + self.align_corners) self.outputs = {'Out': output_np} def test_check_output(self): diff --git a/python/paddle/fluid/tests/unittests/test_calc_gradient.py b/python/paddle/fluid/tests/unittests/test_calc_gradient.py index 3e8c449d8995ca90401861e93f2fb987d1c6967d..fdfaf6a3113bbb9a50a79de7ef4ac4c3251d5759 100644 --- a/python/paddle/fluid/tests/unittests/test_calc_gradient.py +++ b/python/paddle/fluid/tests/unittests/test_calc_gradient.py @@ -15,7 +15,7 @@ from __future__ import print_function import unittest - +import numpy as np import paddle.fluid as fluid import paddle.fluid.layers as layers from paddle.fluid.backward import calc_gradient @@ -81,5 +81,22 @@ class TestDoubleGrad(unittest.TestCase): self.assertEqual(12, out[0]) +class TestGradientWithPrune(unittest.TestCase): + def test_prune(self): + x = fluid.data(name='x', shape=[3], dtype='float32') + x.stop_gradient = False + x1, x2, x3 = fluid.layers.split(x, dim=0, num_or_sections=3) + y = x1 * 2 + x1_grad = fluid.gradients(y, x) + + exe = fluid.Executor(fluid.CPUPlace()) + main = fluid.default_main_program() + exe.run(fluid.default_startup_program()) + out = exe.run(main, + feed={'x': np.ones([3]).astype('float32')}, + fetch_list=[x1_grad]) + self.assertTrue(np.array_equal(out[0], [2., 0., 0.])) + + if __name__ == "__main__": unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_collect_fpn_proposals_op.py b/python/paddle/fluid/tests/unittests/test_collect_fpn_proposals_op.py index 034bb7f8dc7e00a321b6c6a5a4776fa4f7398ab5..a2f56c428012c615dcf55b6832a54ca473771d08 100644 --- a/python/paddle/fluid/tests/unittests/test_collect_fpn_proposals_op.py +++ b/python/paddle/fluid/tests/unittests/test_collect_fpn_proposals_op.py @@ -33,10 +33,14 @@ class TestCollectFPNProposalstOp(OpTest): for i in range(self.num_level)] self.inputs = { 'MultiLevelRois': inputs_x, - "MultiLevelScores": self.scores_input + "MultiLevelScores": self.scores_input, + 'MultiLevelRoIsNum': [] } self.attrs = {'post_nms_topN': self.post_nms_top_n, } - self.outputs = {'FpnRois': (self.rois, [self.lod])} + self.outputs = { + 'FpnRois': (self.rois, [self.lod]), + 'RoisNum': np.array(self.lod).astype('int32') + } def init_test_case(self): self.post_nms_top_n = 20 @@ -96,5 +100,32 @@ class TestCollectFPNProposalstOp(OpTest): self.check_output(check_dygraph=False) +class TestCollectFPNProposalstOpWithRoisNum(TestCollectFPNProposalstOp): + def set_data(self): + self.init_test_case() + self.make_rois() + self.scores_input = [('y%d' % i, + (self.scores[i].reshape(-1, 1), self.rois_lod[i])) + for i in range(self.num_level)] + self.rois, self.lod = self.calc_rois_collect() + inputs_x = [('x%d' % i, (self.roi_inputs[i][:, 1:], self.rois_lod[i])) + for i in range(self.num_level)] + rois_num_per_level = [ + ('rois%d' % i, np.array(self.rois_lod[i][0]).astype('int32')) + for i in range(self.num_level) + ] + + self.inputs = { + 'MultiLevelRois': inputs_x, + "MultiLevelScores": self.scores_input, + 'MultiLevelRoIsNum': rois_num_per_level + } + self.attrs = {'post_nms_topN': self.post_nms_top_n, } + self.outputs = { + 'FpnRois': (self.rois, [self.lod]), + 'RoisNum': np.array(self.lod).astype('int32') + } + + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_compare_op.py b/python/paddle/fluid/tests/unittests/test_compare_op.py index cfad50409802d4f3d35c9da3b22597c681da91b1..25ae65aa7c968b2e6f1f9429d1a4e4e618fe7033 100644 --- a/python/paddle/fluid/tests/unittests/test_compare_op.py +++ b/python/paddle/fluid/tests/unittests/test_compare_op.py @@ -38,6 +38,7 @@ def create_test_class(op_type, typename, callback): self.check_output() def test_errors(self): + paddle.enable_static() with program_guard(Program(), Program()): x = fluid.layers.data(name='x', shape=[2], dtype='int32') y = fluid.layers.data(name='y', shape=[2], dtype='int32') @@ -80,6 +81,7 @@ def create_paddle_case(op_type, callback): self.place = paddle.CUDAPlace(0) def test_api(self): + paddle.enable_static() with program_guard(Program(), Program()): x = fluid.data(name='x', shape=[4], dtype='int64') y = fluid.data(name='y', shape=[4], dtype='int64') @@ -92,6 +94,7 @@ def create_paddle_case(op_type, callback): self.assertEqual((res == self.real_result).all(), True) def test_broadcast_api_1(self): + paddle.enable_static() with program_guard(Program(), Program()): x = paddle.static.data( name='x', shape=[1, 2, 1, 3], dtype='int32') @@ -108,6 +111,7 @@ def create_paddle_case(op_type, callback): self.assertEqual((res == real_result).all(), True) def test_attr_name(self): + paddle.enable_static() with program_guard(Program(), Program()): x = fluid.layers.data(name='x', shape=[4], dtype='int32') y = fluid.layers.data(name='y', shape=[4], dtype='int32') @@ -130,6 +134,7 @@ create_paddle_case('not_equal', lambda _a, _b: _a != _b) class TestCompareOpError(unittest.TestCase): def test_errors(self): + paddle.enable_static() with program_guard(Program(), Program()): # The input x and y of compare_op must be Variable. x = fluid.layers.data(name='x', shape=[1], dtype="float32") @@ -140,6 +145,7 @@ class TestCompareOpError(unittest.TestCase): class API_TestElementwise_Equal(unittest.TestCase): def test_api(self): + paddle.enable_static() with fluid.program_guard(fluid.Program(), fluid.Program()): label = fluid.layers.assign(np.array([3, 3], dtype="int32")) limit = fluid.layers.assign(np.array([3, 2], dtype="int32")) @@ -159,5 +165,31 @@ class API_TestElementwise_Equal(unittest.TestCase): self.assertEqual((res == np.array([True, True])).all(), True) +class TestCompareOpPlace(unittest.TestCase): + def test_place_1(self): + paddle.enable_static() + place = paddle.CPUPlace() + if core.is_compiled_with_cuda(): + place = paddle.CUDAPlace(0) + label = fluid.layers.assign(np.array([3, 3], dtype="int32")) + limit = fluid.layers.assign(np.array([3, 2], dtype="int32")) + out = fluid.layers.less_than(label, limit, force_cpu=True) + exe = fluid.Executor(place) + res, = exe.run(fetch_list=[out]) + self.assertEqual((res == np.array([False, False])).all(), True) + + def test_place_2(self): + place = paddle.CPUPlace() + data_place = place + if core.is_compiled_with_cuda(): + place = paddle.CUDAPlace(0) + data_place = paddle.CUDAPinnedPlace() + paddle.disable_static(place) + data = np.array([9], dtype="int64") + data_tensor = paddle.to_tensor(data, place=data_place) + result = data_tensor == 0 + self.assertEqual((result.numpy() == np.array([False])).all(), True) + + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto.py new file mode 100644 index 0000000000000000000000000000000000000000..5a5d8afc55bac4c0ea862e75b728c6c1a37b3188 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto.py @@ -0,0 +1,67 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import paddle +import os +import paddle.distributed.fleet.base.role_maker as role_maker +import time + + +class TestFleetGradientMergeMetaOptimizer(unittest.TestCase): + def setUp(self): + os.environ["PADDLE_PSERVER_NUMS"] = "2" + os.environ["PADDLE_TRAINERS_NUM"] = "2" + os.environ["POD_IP"] = "127.0.0.1" + os.environ["PADDLE_PORT"] = "36001" + os.environ["PADDLE_TRAINER_ID"] = "0" + os.environ["PADDLE_TRAINERS_NUM"] = "2" + os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ + "127.0.0.1:36001,127.0.0.2:36001" + + def test_a_sync_optimizer1(self): + os.environ["TRAINING_ROLE"] = "TRAINER" + import paddle.distributed.fleet as fleet + + main_program = paddle.fluid.Program() + startup_program = paddle.fluid.Program() + + paddle.fluid.framework.switch_main_program(main_program) + paddle.fluid.framework.switch_startup_program(startup_program) + + fleet.init(role_maker.PaddleCloudRoleMaker()) + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32') + input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64') + + fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') + fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') + prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax') + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y) + avg_cost = paddle.fluid.layers.mean(x=cost) + + strategy = paddle.distributed.fleet.DistributedStrategy() + strategy.auto = True + optimizer = paddle.fluid.optimizer.Adam(learning_rate=0.01) + optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy) + optimizer.minimize(avg_cost) + + self.assertTrue(optimizer.user_defined_strategy.a_sync) + a_sync_configs = optimizer.user_defined_strategy.a_sync_configs + self.assertTrue(a_sync_configs['k_steps'] == 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto_async.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto_async.py new file mode 100644 index 0000000000000000000000000000000000000000..9085556c04c356e5b703ec0b36c3884100ad73f8 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto_async.py @@ -0,0 +1,79 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import paddle +import os +import paddle.distributed.fleet.base.role_maker as role_maker +import time + + +class TestFleetGradientMergeMetaOptimizer(unittest.TestCase): + def setUp(self): + os.environ["PADDLE_PSERVER_NUMS"] = "2" + os.environ["PADDLE_TRAINERS_NUM"] = "2" + os.environ["POD_IP"] = "127.0.0.1" + os.environ["PADDLE_PORT"] = "36001" + os.environ["PADDLE_TRAINER_ID"] = "0" + os.environ["PADDLE_TRAINERS_NUM"] = "2" + os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ + "127.0.0.1:36001,127.0.0.2:36001" + + def test_a_sync_optimizer3(self): + os.environ["TRAINING_ROLE"] = "TRAINER" + import paddle.distributed.fleet as fleet + + main_program = paddle.fluid.Program() + startup_program = paddle.fluid.Program() + + paddle.fluid.framework.switch_main_program(main_program) + paddle.fluid.framework.switch_startup_program(startup_program) + + fleet.init(role_maker.PaddleCloudRoleMaker()) + input_x = paddle.fluid.layers.data( + name="x", + shape=[-1, 1], + dtype="int64", + lod_level=1, + append_batch_size=False) + x_embedding = paddle.fluid.layers.embedding( + is_distributed=False, + input=input_x, + size=[1000000000, 100000], + param_attr=paddle.fluid.ParamAttr( + name="embedding", + initializer=paddle.fluid.initializer.Constant(value=0.01)), + is_sparse=True) + input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64') + + fc_1 = paddle.fluid.layers.fc(input=x_embedding, size=64, act='tanh') + fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') + prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax') + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y) + avg_cost = paddle.fluid.layers.mean(x=cost) + + strategy = paddle.distributed.fleet.DistributedStrategy() + strategy.auto = True + optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.01) + optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy) + optimizer.minimize(avg_cost) + + self.assertTrue(optimizer.user_defined_strategy.a_sync) + a_sync_configs = optimizer.user_defined_strategy.a_sync_configs + self.assertTrue(a_sync_configs['k_steps'] == 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto_geo.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto_geo.py new file mode 100644 index 0000000000000000000000000000000000000000..4787d048bd2566fe063073867bcbd4138d25ff21 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_a_sync_optimizer_auto_geo.py @@ -0,0 +1,67 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import paddle +import os +import paddle.distributed.fleet.base.role_maker as role_maker +import time + + +class TestFleetGradientMergeMetaOptimizer(unittest.TestCase): + def setUp(self): + os.environ["PADDLE_PSERVER_NUMS"] = "2" + os.environ["PADDLE_TRAINERS_NUM"] = "2" + os.environ["POD_IP"] = "127.0.0.1" + os.environ["PADDLE_PORT"] = "36001" + os.environ["PADDLE_TRAINER_ID"] = "0" + os.environ["PADDLE_TRAINERS_NUM"] = "2" + os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ + "127.0.0.1:36001,127.0.0.2:36001" + + def test_a_sync_optimizer2(self): + os.environ["TRAINING_ROLE"] = "TRAINER" + import paddle.distributed.fleet as fleet + + main_program = paddle.fluid.Program() + startup_program = paddle.fluid.Program() + + paddle.fluid.framework.switch_main_program(main_program) + paddle.fluid.framework.switch_startup_program(startup_program) + + fleet.init(role_maker.PaddleCloudRoleMaker()) + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32') + input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64') + + fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') + fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') + prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax') + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y) + avg_cost = paddle.fluid.layers.mean(x=cost) + + strategy = paddle.distributed.fleet.DistributedStrategy() + strategy.auto = True + optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.01) + optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy) + optimizer.minimize(avg_cost) + + self.assertTrue(optimizer.user_defined_strategy.a_sync) + a_sync_configs = optimizer.user_defined_strategy.a_sync_configs + self.assertTrue(a_sync_configs['k_steps'] == 800) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_base.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_base.py index beb0069eb770f25d7834749ff9c188e5252e13c0..3a923dbf3f72e28c64c3f01d22d4d6f2d897f79b 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_base.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_base.py @@ -76,9 +76,10 @@ class FleetDistRunnerBase(object): return role def build_strategy(self, args): - self.strategy = paddle.distributed.fleet.DistributedStrategy() - self.strategy.a_sync = False - if args.mode == "async": + if args.mode == "sync": + self.strategy = paddle.distributed.fleet.DistributedStrategy() + self.strategy.a_sync = False + elif args.mode == "async": self.strategy = paddle.distributed.fleet.DistributedStrategy() self.strategy.a_sync = True elif args.mode == "geo": @@ -87,6 +88,10 @@ class FleetDistRunnerBase(object): self.strategy.a_sync_configs = { "k_steps": args.geo_sgd_need_push_nums } + elif args.mode == "auto": + self.strategy = paddle.distributed.fleet.DistributedStrategy() + self.strategy.auto = True + self.dump_param = os.getenv("dump_param", "").split(",") self.dump_fields = os.getenv("dump_fields", "").split(",") self.dump_fields_path = os.getenv("dump_fields_path", "") @@ -232,14 +237,17 @@ class TestFleetBase(unittest.TestCase): tr0_pipe = open(tempfile.gettempdir() + "/tr0_err.log", "wb+") tr1_pipe = open(tempfile.gettempdir() + "/tr1_err.log", "wb+") + tr0_out = open(tempfile.gettempdir() + "/tr0_stdout.log", "wb+") + tr1_out = open(tempfile.gettempdir() + "/tr1_stdout.log", "wb+") + tr0_proc = subprocess.Popen( tr0_cmd.strip().split(" "), - stdout=subprocess.PIPE, + stdout=tr0_out, stderr=tr0_pipe, env=required_envs) tr1_proc = subprocess.Popen( tr1_cmd.strip().split(" "), - stdout=subprocess.PIPE, + stdout=tr1_out, stderr=tr1_pipe, env=required_envs) diff --git a/python/paddle/fluid/tests/unittests/test_dist_fleet_ctr.py b/python/paddle/fluid/tests/unittests/test_dist_fleet_ctr.py index e2336caac1c07f555280b82ba8fcfa7e5ec7f5b8..02ca0588e7452d44817f6c288ea9cf77b80dbfe8 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_fleet_ctr.py +++ b/python/paddle/fluid/tests/unittests/test_dist_fleet_ctr.py @@ -52,6 +52,38 @@ class TestDistMnistSync2x2(TestFleetBase): "dist_fleet_ctr.py", delta=1e-5, check_error_log=True) +class TestDistMnistAuto2x2(TestFleetBase): + def _setup_config(self): + self._mode = "auto" + self._reader = "pyreader" + + def check_with_place(self, + model_file, + delta=1e-3, + check_error_log=False, + need_envs={}): + required_envs = { + "PATH": os.getenv("PATH", ""), + "PYTHONPATH": os.getenv("PYTHONPATH", ""), + "LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", ""), + "FLAGS_rpc_deadline": "5000", # 5sec to fail fast + "http_proxy": "", + "CPU_NUM": "2" + } + + required_envs.update(need_envs) + + if check_error_log: + required_envs["GLOG_v"] = "3" + required_envs["GLOG_logtostderr"] = "1" + + tr0_losses, tr1_losses = self._run_cluster(model_file, required_envs) + + def test_dist_train(self): + self.check_with_place( + "dist_fleet_ctr.py", delta=1e-5, check_error_log=True) + + class TestDistMnistAsync2x2(TestFleetBase): def _setup_config(self): self._mode = "async" diff --git a/python/paddle/fluid/tests/unittests/test_dist_transpiler_async_decay.py b/python/paddle/fluid/tests/unittests/test_dist_transpiler_async_decay.py index 1062123948481a4164a12a4bed818b964923006f..761d57408b9a8f9e52419331bfb0bca5b0135c30 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_transpiler_async_decay.py +++ b/python/paddle/fluid/tests/unittests/test_dist_transpiler_async_decay.py @@ -113,8 +113,8 @@ class TranspilerAsyncLRDecayTest(unittest.TestCase): ["listen_and_serv"]) # block1: sum,cast,scale,floor,fill_constant,elementwise_pow,scale self.assertEqual([op.type for op in pserver.blocks[1].ops], [ - "sum", "cast", "fill_constant", "elementwise_div", "floor", - "fill_constant", "elementwise_pow", "scale" + "sum", "cast", "scale", "floor", "fill_constant", "elementwise_pow", + "scale" ]) # block1~2: optimize pass diff --git a/python/paddle/fluid/tests/unittests/test_distribute_fpn_proposals_op.py b/python/paddle/fluid/tests/unittests/test_distribute_fpn_proposals_op.py index 55b21f1a722f822f1bfcb7bbbda645109092b8a3..ec0125b28ed1b870025adbfd2bd4ba78244bcc11 100644 --- a/python/paddle/fluid/tests/unittests/test_distribute_fpn_proposals_op.py +++ b/python/paddle/fluid/tests/unittests/test_distribute_fpn_proposals_op.py @@ -35,9 +35,10 @@ class TestDistributeFPNProposalsOp(OpTest): } output = [('out%d' % i, self.rois_fpn[i]) for i in range(len(self.rois_fpn))] + self.outputs = { 'MultiFpnRois': output, - 'RestoreIndex': self.rois_idx_restore.reshape(-1, 1) + 'RestoreIndex': self.rois_idx_restore.reshape(-1, 1), } def init_test_case(self): @@ -117,5 +118,34 @@ class TestDistributeFPNProposalsOp(OpTest): self.check_output() +class TestDistributeFPNProposalsOpWithRoisNum(TestDistributeFPNProposalsOp): + def set_data(self): + self.init_test_case() + self.make_rois() + self.rois_fpn, self.rois_idx_restore = self.calc_rois_distribute() + self.inputs = { + 'FpnRois': (self.rois[:, 1:5], self.rois_lod), + 'RoisNum': np.array(self.rois_lod[0]).astype('int32') + } + self.attrs = { + 'max_level': self.roi_max_level, + 'min_level': self.roi_min_level, + 'refer_scale': self.canonical_scale, + 'refer_level': self.canonical_level + } + output = [('out%d' % i, self.rois_fpn[i]) + for i in range(len(self.rois_fpn))] + rois_num_per_level = [ + ('rois_num%d' % i, np.array(self.rois_fpn[i][1][0]).astype('int32')) + for i in range(len(self.rois_fpn)) + ] + + self.outputs = { + 'MultiFpnRois': output, + 'RestoreIndex': self.rois_idx_restore.reshape(-1, 1), + 'MultiLevelRoIsNum': rois_num_per_level + } + + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_dygraph_weight_norm.py b/python/paddle/fluid/tests/unittests/test_dygraph_weight_norm.py index 466226c53fabbd315acd19c6421f210d0ca225c1..a963c2ece0958048b5f0c850184a0930022e6671 100644 --- a/python/paddle/fluid/tests/unittests/test_dygraph_weight_norm.py +++ b/python/paddle/fluid/tests/unittests/test_dygraph_weight_norm.py @@ -121,6 +121,9 @@ class TestDygraphWeightNorm(unittest.TestCase): before_weight = linear.weight.numpy() if self.dim == None: self.dim = -1 + + if self.dim != -1: + self.dim = (self.dim + len(before_weight)) % len(before_weight) wn = weight_norm(linear, dim=self.dim) outputs = [] for name, data in self.data.items(): @@ -158,6 +161,13 @@ class TestDygraphWeightNormCase3(TestDygraphWeightNorm): self.dim = 3 +class TestDygraphWeightNormCase4(TestDygraphWeightNorm): + def init_test_case(self): + self.batch_size = 3 + self.data_desc = (['x', [2, 3, 3]], ) + self.dim = -3 + + class TestDygraphRemoveWeightNorm(unittest.TestCase): def setUp(self): self.init_test_case() diff --git a/python/paddle/fluid/tests/unittests/test_dynrnn_static_input.py b/python/paddle/fluid/tests/unittests/test_dynrnn_static_input.py index b4359fc69ae18b45774af0d2b20c1540bd99da5c..698f914f89984d8c09619a46c6a6b292b00aac9a 100644 --- a/python/paddle/fluid/tests/unittests/test_dynrnn_static_input.py +++ b/python/paddle/fluid/tests/unittests/test_dynrnn_static_input.py @@ -25,6 +25,7 @@ import bisect import numpy as np fluid.default_startup_program().random_seed = 1 +np.random.seed(1) class TestDyRnnStaticInput(unittest.TestCase): diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_div_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_div_op.py index 9ebaf8ff9438be8c8a57815be0798b861d05caaf..3cfbac8b613c125956861f73b1bab24c34e05572 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_div_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_div_op.py @@ -240,124 +240,25 @@ class TestElementwiseDivBroadcast(unittest.TestCase): self.assertEqual((out_result == (2 / x)).all(), True) -class TestDivideAPI(unittest.TestCase): - def setUp(self): - paddle.set_default_dtype("float64") - self.places = [fluid.CPUPlace()] - if core.is_compiled_with_cuda(): - self.places.append(fluid.CUDAPlace(0)) - - def check_static_result(self, place): - # rule 1 - with fluid.program_guard(fluid.Program(), fluid.Program()): - x = fluid.data(name="x", shape=[3], dtype="float64") - y = np.array([1, 2, 3]) - self.assertRaises(TypeError, paddle.divide, x=x, y=y) - - # rule 2: both the inputs are not Tensor - with fluid.program_guard(fluid.Program(), fluid.Program()): - x = 2 - y = 4 - res = paddle.divide(x, y) - exe = fluid.Executor(place) - np_z = exe.run(fluid.default_main_program(), - feed={}, - fetch_list=[res]) - self.assertEqual(np_z[0] == 0.5, True) - - # rule 3: - with fluid.program_guard(fluid.Program(), fluid.Program()): - x = fluid.data(name="x", shape=[3], dtype="float64") - y = fluid.data(name="y", shape=[3], dtype="float32") - self.assertRaises(TypeError, paddle.divide, x=x, y=y) - - # rule 4: x is Tensor, y is scalar - with fluid.program_guard(fluid.Program(), fluid.Program()): - x = fluid.data(name="x", shape=[3], dtype="float64") - y = 2 - exe = fluid.Executor(place) - res = x / y - np_z = exe.run(fluid.default_main_program(), - feed={"x": np.array([2, 3, 4]).astype('float64')}, - fetch_list=[res]) - z_expected = np.array([1., 1.5, 2.]) - self.assertEqual((np_z[0] == z_expected).all(), True) - - # rule 5: y is Tensor, x is scalar - with fluid.program_guard(fluid.Program(), fluid.Program()): - x = fluid.data(name="x", shape=[3], dtype="float64") - y = 2 - exe = fluid.Executor(place) - res = y / x - np_z = exe.run(fluid.default_main_program(), - feed={"x": np.array([2, 8, 4]).astype('float64')}, - fetch_list=[res]) - z_expected = np.array([1., 0.25, 0.5]) - self.assertEqual((np_z[0] == z_expected).all(), True) - - # rule 6: y is Tensor, x is Tensor - with fluid.program_guard(fluid.Program(), fluid.Program()): - x = fluid.data(name="x", shape=[3], dtype="float64") - y = fluid.data(name="y", shape=[3], dtype="float64") - exe = fluid.Executor(place) - res = x / y - np_z = exe.run(fluid.default_main_program(), - feed={ - "x": np.array([2, 3, 4]).astype('float64'), - "y": np.array([1, 5, 2]).astype('float64') - }, - fetch_list=[res]) - z_expected = np.array([2., 0.6, 2.]) - self.assertEqual((np_z[0] == z_expected).all(), True) +class TestDivideOp(unittest.TestCase): + def test_name(self): + with fluid.program_guard(fluid.Program()): + x = fluid.data(name="x", shape=[2, 3], dtype="float32") + y = fluid.data(name='y', shape=[2, 3], dtype='float32') - def test_static(self): - for place in self.places: - self.check_static_result(place=place) + y_1 = paddle.divide(x, y, name='div_res') + self.assertEqual(('div_res' in y_1.name), True) def test_dygraph(self): - for place in self.places: - with fluid.dygraph.guard(place): - # rule 1 : avoid numpy.ndarray - np_x = np.array([2, 3, 4]) - np_y = np.array([1, 5, 2]) - x = paddle.to_tensor(np_x) - self.assertRaises(TypeError, paddle.divide, x=x, y=np_y) - - # rule 2: both the inputs are not Tensor - z = paddle.divide(3, 2) - self.assertEqual(z.numpy()[0] == 1.5, True) - - # rule 3: both the inputs are Tensor - np_x = np.array([2, 3, 4]) - np_y = np.array([1, 5, 2]) - x = paddle.to_tensor(np_x, dtype="float32") - y = paddle.to_tensor(np_y, dtype="float64") - self.assertRaises(TypeError, paddle.divide, x=x, y=y) - - # rule 4: x is Tensor, y is scalar - np_x = np.array([2, 3, 4]) - x = paddle.to_tensor(np_x, dtype="int32") - y = 2 - z = x / y - z_expected = np.array([1., 1.5, 2.]) - self.assertEqual((z_expected == z.numpy()).all(), True) - - # rule 5: y is Tensor, x is scalar - np_x = np.array([2, 1, 4]) - x = paddle.to_tensor(np_x, dtype="int32") - y = 2 - z = y / x - z_expected = np.array([1., 2., 0.5]) - self.assertEqual((z_expected == z.numpy()).all(), True) - - # rule 6: y is Tensor, x is Tensor - np_x = np.array([2, 3, 4]) - np_y = np.array([1, 5, 2]) - x = paddle.to_tensor(np_x) - y = paddle.to_tensor(np_y) - z = x / y - z_expected = np.array([2., 0.6, 2.]) - self.assertEqual((z_expected == z.numpy()).all(), True) + with fluid.dygraph.guard(): + np_x = np.array([2, 3, 4]).astype('float64') + np_y = np.array([1, 5, 2]).astype('float64') + x = paddle.to_tensor(np_x) + y = paddle.to_tensor(np_y) + z = paddle.divide(x, y) + np_z = z.numpy() + z_expected = np.array([2., 0.6, 2.]) + self.assertEqual((np_z == z_expected).all(), True) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_floordiv_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_floordiv_op.py index 0b6acc7615395ed99a484e0e56f9c62447a1f345..f339081e31b87b8d5584fd4f866e0aaf6f391ea7 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_floordiv_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_floordiv_op.py @@ -58,13 +58,6 @@ class TestElementwiseModOp(OpTest): pass -class TestElementwiseModOpInverse(TestElementwiseModOp): - def init_input_output(self): - self.x = np.random.uniform(0, 10000, [10]).astype(self.dtype) - self.y = np.random.uniform(0, 1000, [10, 10]).astype(self.dtype) - self.out = np.floor_divide(self.x, self.y) - - class TestElementwiseModOp_scalar(TestElementwiseModOp): def init_input_output(self): scale_x = random.randint(0, 100000000) @@ -74,124 +67,25 @@ class TestElementwiseModOp_scalar(TestElementwiseModOp): self.out = np.floor_divide(self.x, self.y) -class TestFloorDivideAPI(unittest.TestCase): - def setUp(self): - paddle.set_default_dtype("float64") - self.places = [fluid.CPUPlace()] - if core.is_compiled_with_cuda(): - self.places.append(fluid.CUDAPlace(0)) - - def check_static_result(self, place): - # rule 1 - with fluid.program_guard(fluid.Program(), fluid.Program()): - x = fluid.data(name="x", shape=[3], dtype="float64") - y = np.array([1, 2, 3]) - self.assertRaises(TypeError, paddle.floor_divide, x=x, y=y) - - # rule 2: both the inputs are not Tensor - with fluid.program_guard(fluid.Program(), fluid.Program()): - x = 2 - y = 4 - res = paddle.floor_divide(x, y) - exe = fluid.Executor(place) - np_z = exe.run(fluid.default_main_program(), - feed={}, - fetch_list=[res]) - self.assertEqual(np_z[0] == 0., True) - - # rule 3: - with fluid.program_guard(fluid.Program(), fluid.Program()): - x = fluid.data(name="x", shape=[3], dtype="float64") - y = fluid.data(name="y", shape=[3], dtype="float32") - self.assertRaises(TypeError, paddle.floor_divide, x=x, y=y) - - # rule 4: x is Tensor, y is scalar - with fluid.program_guard(fluid.Program(), fluid.Program()): - x = fluid.data(name="x", shape=[3], dtype="float64") - y = 2 - exe = fluid.Executor(place) - res = x // y - np_z = exe.run(fluid.default_main_program(), - feed={"x": np.array([2, 3, 4]).astype('float64')}, - fetch_list=[res]) - z_expected = np.array([1., 1., 2.]) - self.assertEqual((np_z[0] == z_expected).all(), True) - - # rule 5: y is Tensor, x is scalar - with fluid.program_guard(fluid.Program(), fluid.Program()): - x = fluid.data(name="x", shape=[3], dtype="float64") - y = 2 - exe = fluid.Executor(place) - res = y // x - np_z = exe.run(fluid.default_main_program(), - feed={"x": np.array([2, 8, 4]).astype('float64')}, - fetch_list=[res]) - z_expected = np.array([1., 0., 0.]) - self.assertEqual((np_z[0] == z_expected).all(), True) - - # rule 6: y is Tensor, x is Tensor - with fluid.program_guard(fluid.Program(), fluid.Program()): - x = fluid.data(name="x", shape=[3], dtype="float64") - y = fluid.data(name="y", shape=[3], dtype="float64") - exe = fluid.Executor(place) - res = x // y - np_z = exe.run(fluid.default_main_program(), - feed={ - "x": np.array([2, 3, 4]).astype('float64'), - "y": np.array([1, 5, 2]).astype('float64') - }, - fetch_list=[res]) - z_expected = np.array([2., 0., 2.]) - self.assertEqual((np_z[0] == z_expected).all(), True) - - def test_static(self): - for place in self.places: - self.check_static_result(place=place) +class TestFloorDivideOp(unittest.TestCase): + def test_name(self): + with fluid.program_guard(fluid.Program()): + x = fluid.data(name="x", shape=[2, 3], dtype="int64") + y = fluid.data(name='y', shape=[2, 3], dtype='int64') + + y_1 = paddle.floor_divide(x, y, name='div_res') + self.assertEqual(('div_res' in y_1.name), True) def test_dygraph(self): - for place in self.places: - with fluid.dygraph.guard(place): - # rule 1 : avoid numpy.ndarray - np_x = np.array([2, 3, 4]) - np_y = np.array([1, 5, 2]) - x = paddle.to_tensor(np_x) - self.assertRaises(TypeError, paddle.floor_divide, x=x, y=np_y) - - # rule 2: both the inputs are not Tensor - z = paddle.floor_divide(3, 2) - self.assertEqual(z.numpy()[0] == 1., True) - - # rule 3: both the inputs are Tensor - np_x = np.array([2, 3, 4]) - np_y = np.array([1, 5, 2]) - x = paddle.to_tensor(np_x, dtype="float32") - y = paddle.to_tensor(np_y, dtype="float64") - self.assertRaises(TypeError, paddle.floor_divide, x=x, y=y) - - # rule 4: x is Tensor, y is scalar - np_x = np.array([2, 3, 4]) - x = paddle.to_tensor(np_x, dtype="int32") - y = 2 - z = x // y - z_expected = np.array([1, 1, 2]) - self.assertEqual((z_expected == z.numpy()).all(), True) - - # rule 5: y is Tensor, x is scalar - np_x = np.array([2, 1, 4]) - x = paddle.to_tensor(np_x, dtype="int32") - y = 2 - z = y // x - z_expected = np.array([1, 2, 0]) - self.assertEqual((z_expected == z.numpy()).all(), True) - - # rule 6: y is Tensor, x is Tensor - np_x = np.array([2, 3, 4]) - np_y = np.array([1, 5, 2]) - x = paddle.to_tensor(np_x) - y = paddle.to_tensor(np_y) - z = x // y - z_expected = np.array([2., 0., 2.]) - self.assertEqual((z_expected == z.numpy()).all(), True) + with fluid.dygraph.guard(): + np_x = np.array([2, 3, 8, 7]).astype('int64') + np_y = np.array([1, 5, 3, 3]).astype('int64') + x = paddle.to_tensor(np_x) + y = paddle.to_tensor(np_y) + z = paddle.floor_divide(x, y) + np_z = z.numpy() + z_expected = np.array([2, 0, 2, 2]) + self.assertEqual((np_z == z_expected).all(), True) with fluid.dygraph.guard(fluid.CPUPlace()): # divide by zero diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_mod_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_mod_op.py index cab6160d761004877896deea8d44ca02c9de2e1e..2a8ca51693ecfad55f2239d7619e355c6dd7f3f8 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_mod_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_mod_op.py @@ -84,149 +84,41 @@ class TestElementwiseModOpDouble(TestElementwiseModOpFloat): self.dtype = np.float64 -class TestRemainderAPI(unittest.TestCase): - def setUp(self): - paddle.set_default_dtype("float64") - self.places = [fluid.CPUPlace()] - if core.is_compiled_with_cuda(): - self.places.append(fluid.CUDAPlace(0)) - - def check_static_result(self, place): - # rule 1 - with fluid.program_guard(fluid.Program(), fluid.Program()): - x = fluid.data(name="x", shape=[3], dtype="float64") - y = np.array([1, 2, 3]) - self.assertRaises(TypeError, paddle.remainder, x=x, y=y) - - # rule 3: - with fluid.program_guard(fluid.Program(), fluid.Program()): - x = fluid.data(name="x", shape=[3], dtype="float64") - y = fluid.data(name="y", shape=[3], dtype="float32") - self.assertRaises(TypeError, paddle.remainder, x=x, y=y) - - # rule 4: x is Tensor, y is scalar - with fluid.program_guard(fluid.Program(), fluid.Program()): - x = fluid.data(name="x", shape=[3], dtype="float64") - y = 2 - exe = fluid.Executor(place) - res = x % y - np_z = exe.run(fluid.default_main_program(), - feed={"x": np.array([2, 3, 4]).astype('float64')}, - fetch_list=[res]) - z_expected = np.array([0., 1., 0.]) - self.assertEqual((np_z[0] == z_expected).all(), True) - - # rule 5: y is Tensor, x is scalar - with fluid.program_guard(fluid.Program(), fluid.Program()): - x = 3 - y = fluid.data(name="y", shape=[3], dtype="float32") - self.assertRaises(TypeError, paddle.remainder, x=x, y=y) - - # rule 6: y is Tensor, x is Tensor - with fluid.program_guard(fluid.Program(), fluid.Program()): - x = fluid.data(name="x", shape=[3], dtype="float64") - y = fluid.data(name="y", shape=[1], dtype="float64") - exe = fluid.Executor(place) - res = x % y - np_z = exe.run(fluid.default_main_program(), - feed={ - "x": np.array([1., 2., 4]).astype('float64'), - "y": np.array([1.5]).astype('float64') - }, - fetch_list=[res]) - z_expected = np.array([1., 0.5, 1.0]) - self.assertEqual((np_z[0] == z_expected).all(), True) - - # rule 6: y is Tensor, x is Tensor - with fluid.program_guard(fluid.Program(), fluid.Program()): - x = fluid.data(name="x", shape=[6], dtype="float64") - y = fluid.data(name="y", shape=[1], dtype="float64") - exe = fluid.Executor(place) - res = x % y - np_z = exe.run( - fluid.default_main_program(), - feed={ - "x": np.array([-3., -2, -1, 1, 2, 3]).astype('float64'), - "y": np.array([2]).astype('float64') - }, - fetch_list=[res]) - z_expected = np.array([1., 0., 1., 1., 0., 1.]) - self.assertEqual((np_z[0] == z_expected).all(), True) - - def test_static(self): - for place in self.places: - self.check_static_result(place=place) +class TestRemainderOp(unittest.TestCase): + def test_name(self): + with fluid.program_guard(fluid.Program()): + x = fluid.data(name="x", shape=[2, 3], dtype="int64") + y = fluid.data(name='y', shape=[2, 3], dtype='int64') + + y_1 = paddle.remainder(x, y, name='div_res') + self.assertEqual(('div_res' in y_1.name), True) def test_dygraph(self): - for place in self.places: - with fluid.dygraph.guard(place): - # rule 1 : avoid numpy.ndarray - np_x = np.array([2, 3, 4]) - np_y = np.array([1, 5, 2]) - x = paddle.to_tensor(np_x) - self.assertRaises(TypeError, paddle.remainder, x=x, y=np_y) - - # rule 3: both the inputs are Tensor - np_x = np.array([2, 3, 4]) - np_y = np.array([1, 5, 2]) - x = paddle.to_tensor(np_x, dtype="float32") - y = paddle.to_tensor(np_y, dtype="float64") - self.assertRaises(TypeError, paddle.remainder, x=x, y=y) - - # rule 4: x is Tensor, y is scalar - np_x = np.array([2, 3, 4]) - x = paddle.to_tensor(np_x, dtype="int32") - y = 2 - z = x % y - z_expected = np.array([0, 1, 0]) - self.assertEqual((z_expected == z.numpy()).all(), True) - - # rule 5: y is Tensor, x is scalar - np_x = np.array([2, 3, 4]) - x = paddle.to_tensor(np_x) - self.assertRaises(TypeError, paddle.remainder, x=3, y=x) - - # rule 6: y is Tensor, x is Tensor - np_x = np.array([1., 2., 4]) - np_y = np.array([1.5]) - x = paddle.to_tensor(np_x) - y = paddle.to_tensor(np_y) - z = x % y - z_expected = np.array([1., 0.5, 1.0]) - self.assertEqual((z_expected == z.numpy()).all(), True) - - # rule 6: y is Tensor, x is Tensor - np_x = np.array([-3., -2, -1, 1, 2, 3]) - np_y = np.array([2.]) - x = paddle.to_tensor(np_x) - y = paddle.to_tensor(np_y) - z = x % y - z_expected = np.array([1., 0., 1., 1., 0., 1.]) - self.assertEqual((z_expected == z.numpy()).all(), True) - - np_x = np.array([-3.3, 11.5, -2, 3.5]) - np_y = np.array([-1.2, 2., 3.3, -2.3]) - x = paddle.to_tensor(np_x) - y = paddle.to_tensor(np_y) - z = x % y - z_expected = np.array([-0.9, 1.5, 1.3, -1.1]) - self.assertEqual(np.allclose(z_expected, z.numpy()), True) - - np_x = np.array([-3, 11, -2, 3]) - np_y = np.array([-1, 2, 3, -2]) - x = paddle.to_tensor(np_x, dtype="int64") - y = paddle.to_tensor(np_y, dtype="int64") - z = x % y - z_expected = np.array([0, 1, 1, -1]) - self.assertEqual(np.allclose(z_expected, z.numpy()), True) - - np_x = np.array([-3, 3]) - np_y = np.array([[2, 3], [-2, -1]]) - x = paddle.to_tensor(np_x, dtype="int64") - y = paddle.to_tensor(np_y, dtype="int64") - z = x % y - z_expected = np.array([[1, 0], [-1, 0]]) - self.assertEqual(np.allclose(z_expected, z.numpy()), True) + with fluid.dygraph.guard(): + np_x = np.array([2, 3, 8, 7]).astype('int64') + np_y = np.array([1, 5, 3, 3]).astype('int64') + x = paddle.to_tensor(np_x) + y = paddle.to_tensor(np_y) + z = paddle.remainder(x, y) + np_z = z.numpy() + z_expected = np.array([0, 3, 2, 1]) + self.assertEqual((np_z == z_expected).all(), True) + + np_x = np.array([-3.3, 11.5, -2, 3.5]) + np_y = np.array([-1.2, 2., 3.3, -2.3]) + x = paddle.to_tensor(np_x) + y = paddle.to_tensor(np_y) + z = x % y + z_expected = np.array([-0.9, 1.5, 1.3, -1.1]) + self.assertEqual(np.allclose(z_expected, z.numpy()), True) + + np_x = np.array([-3, 11, -2, 3]) + np_y = np.array([-1, 2, 3, -2]) + x = paddle.to_tensor(np_x, dtype="int64") + y = paddle.to_tensor(np_y, dtype="int64") + z = x % y + z_expected = np.array([0, 1, 1, -1]) + self.assertEqual(np.allclose(z_expected, z.numpy()), True) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_empty_op.py b/python/paddle/fluid/tests/unittests/test_empty_op.py new file mode 100644 index 0000000000000000000000000000000000000000..e8b1f836fcaa8d53671307d9075efd45fc88ce7b --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_empty_op.py @@ -0,0 +1,270 @@ +#Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import unittest +import numpy as np +import paddle +import paddle.fluid as fluid +from op_test import OpTest +from paddle.fluid import Program, program_guard +from paddle.fluid.framework import convert_np_dtype_to_dtype_ + + +# Situation 1: Attr(shape) is a list(without tensor) +class TestEmptyOp(OpTest): + def setUp(self): + self.op_type = "empty" + self.init_config() + + def test_check_output(self): + self.check_output_customized(self.verify_output) + + def verify_output(self, outs): + data_type = outs[0].dtype + if data_type in ['float32', 'float64', 'int32', 'int64']: + max_value = np.nanmax(outs[0]) + min_value = np.nanmin(outs[0]) + + always_full_zero = max_value == 0.0 and min_value == 0.0 + always_non_full_zero = max_value > min_value + self.assertTrue(always_full_zero or always_non_full_zero, + 'always_full_zero or always_non_full_zero.') + elif data_type in ['bool']: + total_num = outs[0].size + true_num = np.sum(outs[0] == True) + false_num = np.sum(outs[0] == False) + self.assertTrue(total_num == true_num + false_num, + 'The value should always be True or False.') + else: + self.assertTrue(False, 'invalid data type') + + def init_config(self): + shape = [500, 3] + dtype = 'float32' + dtype_inner = convert_np_dtype_to_dtype_(dtype) + self.attrs = {'shape': shape, 'dtype': dtype_inner} + self.inputs = {} + self.outputs = {'Out': np.zeros(shape).astype(dtype)} + + +class TestEmptyOp2(TestEmptyOp): + def init_config(self): + shape = [500, 3] + dtype = 'float64' + dtype_inner = convert_np_dtype_to_dtype_(dtype) + self.attrs = {'shape': shape, 'dtype': dtype_inner} + self.inputs = {} + self.outputs = {'Out': np.zeros(shape).astype(dtype)} + + +class TestEmptyOp3(TestEmptyOp): + def init_config(self): + shape = [500, 3] + dtype = 'int32' + dtype_inner = convert_np_dtype_to_dtype_(dtype) + self.attrs = {'shape': shape, 'dtype': dtype_inner} + self.inputs = {} + self.outputs = {'Out': np.zeros(shape).astype(dtype)} + + +class TestEmptyOp4(TestEmptyOp): + def init_config(self): + shape = [500, 3] + dtype = 'int64' + dtype_inner = convert_np_dtype_to_dtype_(dtype) + self.attrs = {'shape': shape, 'dtype': dtype_inner} + self.inputs = {} + self.outputs = {'Out': np.zeros(shape).astype(dtype)} + + +class TestEmptyOp5(TestEmptyOp): + def init_config(self): + shape = [500, 3] + dtype = 'bool' + dtype_inner = convert_np_dtype_to_dtype_(dtype) + self.attrs = {'shape': shape, 'dtype': dtype_inner} + self.inputs = {} + self.outputs = {'Out': np.zeros(shape).astype(dtype)} + + +# Situation 2: shape is a tensor +class TestEmptyOp_ShapeTensor(OpTest): + def setUp(self): + self.op_type = "empty" + self.init_config() + + def init_config(self): + self.shape = [500, 3] + dtype = 'float32' + dtype_inner = convert_np_dtype_to_dtype_(dtype) + self.attrs = {'dtype': dtype_inner} + self.inputs = {"ShapeTensor": np.array(self.shape).astype("int32")} + self.outputs = {'Out': np.zeros(self.shape).astype(dtype)} + + def test_check_output(self): + self.check_output_customized(self.verify_output) + + def verify_output(self, outs): + data_type = outs[0].dtype + if data_type in ['float32', 'float64', 'int32', 'int64']: + max_value = np.nanmax(outs[0]) + min_value = np.nanmin(outs[0]) + + always_full_zero = max_value == 0.0 and min_value == 0.0 + always_non_full_zero = max_value > min_value + self.assertTrue(always_full_zero or always_non_full_zero, + 'always_full_zero or always_non_full_zero.') + elif data_type in ['bool']: + total_num = outs[0].size + true_num = np.sum(outs[0] == True) + false_num = np.sum(outs[0] == False) + self.assertTrue(total_num == true_num + false_num, + 'The value should always be True or False.') + else: + self.assertTrue(False, 'invalid data type') + + +# Situation 3: Attr(shape) is a list(with tensor) +class TestEmptyOp_ShapeTensorList(OpTest): + def setUp(self): + self.op_type = "empty" + self.init_config() + + def init_config(self): + self.shape = [123, 92] + self.infer_shape = [-1, 92] + + dtype = 'float32' + dtype_inner = convert_np_dtype_to_dtype_(dtype) + + shape_tensor_list = [] + for index, ele in enumerate(self.shape): + shape_tensor_list.append(("x" + str(index), np.ones( + (1)).astype('int32') * ele)) + + self.inputs = {"ShapeTensorList": shape_tensor_list} + self.attrs = {'shape': self.infer_shape, 'dtype': dtype_inner} + self.outputs = {'Out': np.zeros(self.shape).astype(dtype)} + + def test_check_output(self): + self.check_output_customized(self.verify_output) + + def verify_output(self, outs): + data_type = outs[0].dtype + if data_type in ['float32', 'float64', 'int32', 'int64']: + max_value = np.nanmax(outs[0]) + min_value = np.nanmin(outs[0]) + + always_full_zero = max_value == 0.0 and min_value == 0.0 + always_non_full_zero = max_value > min_value + self.assertTrue(always_full_zero or always_non_full_zero, + 'always_full_zero or always_non_full_zero.') + elif data_type in ['bool']: + total_num = outs[0].size + true_num = np.sum(outs[0] == True) + false_num = np.sum(outs[0] == False) + self.assertTrue(total_num == true_num + false_num, + 'The value should always be True or False.') + else: + self.assertTrue(False, 'invalid data type') + + +class TestEmptyAPI(unittest.TestCase): + def __check_out__(self, out, dtype='float32'): + max_value = np.nanmax(np.array(out)) + min_value = np.nanmin(np.array(out)) + always_non_full_zero = max_value > min_value + always_full_zero = max_value == 0.0 and min_value == 0.0 + self.assertTrue(always_full_zero or always_non_full_zero, + 'always_full_zero or always_non_full_zero.') + + def test_dygraph_api_out(self): + paddle.disable_static() + shape = [200, 3] + out = paddle.empty(shape=shape) + self.__check_out__(out) + paddle.enable_static() + + def test_dygraph_api_out_2(self): + paddle.disable_static() + shape_data = np.array([200, 3]).astype('int32') + shape = paddle.to_tensor(shape_data) + out = paddle.empty(shape=shape) + self.__check_out__(out) + paddle.enable_static() + + def test_dygraph_api_out_3(self): + paddle.disable_static() + shape_data = np.array([200, 3]).astype('int64') + shape = paddle.to_tensor(shape_data) + out = paddle.empty(shape=shape) + self.__check_out__(out) + paddle.enable_static() + + def test_dygraph_api_attr(self): + paddle.disable_static() + shape = [200, 3] + dtype = 'float64' + out = paddle.empty(shape=shape, dtype=dtype) + self.__check_out__(out, dtype) + paddle.enable_static() + + def test_static_graph(self): + dtype = 'float64' + + positive_2_int32 = fluid.layers.fill_constant([1], "int32", 3) + positive_2_int64 = fluid.layers.fill_constant([1], "int64", 3) + + shape_tensor_int32 = fluid.data( + name="shape_tensor_int32", shape=[2], dtype="int32") + shape_tensor_int64 = fluid.data( + name="shape_tensor_int64", shape=[2], dtype="int64") + + out_1 = paddle.empty(shape=[200, 3], dtype=dtype) + out_2 = paddle.empty(shape=shape_tensor_int32, dtype=dtype) + out_3 = paddle.empty(shape=shape_tensor_int64, dtype=dtype) + out_4 = paddle.empty(shape=[200, positive_2_int32], dtype=dtype) + out_5 = paddle.empty(shape=[200, positive_2_int64], dtype=dtype) + + place = paddle.CPUPlace() + exe = paddle.static.Executor(place) + res_1, res_2, res_3, res_4, res_5 = exe.run( + fluid.default_main_program(), + feed={ + "shape_tensor_int32": np.array([200, 3]).astype("int32"), + "shape_tensor_int64": np.array([200, 3]).astype("int64"), + }, + fetch_list=[out_1, out_2, out_3, out_4, out_5]) + + self.__check_out__(res_1, dtype) + self.__check_out__(res_2, dtype) + self.__check_out__(res_3, dtype) + self.__check_out__(res_4, dtype) + self.__check_out__(res_5, dtype) + + +class TestEmptyError(unittest.TestCase): + def test_attr(self): + def test_dtype(): + shape = [200, 3] + dtype = 'uint8' + result = paddle.empty(shape=shape, dtype=dtype) + + self.assertRaises(TypeError, test_dtype) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_fetch_unmerged.py b/python/paddle/fluid/tests/unittests/test_fetch_unmerged.py index 1181272bd98b00f65e6925b44da814662f96045f..37d269e3369bfe7db00529dea5e08b287151691a 100644 --- a/python/paddle/fluid/tests/unittests/test_fetch_unmerged.py +++ b/python/paddle/fluid/tests/unittests/test_fetch_unmerged.py @@ -28,7 +28,7 @@ class TestFetchUnmerged(unittest.TestCase): conv_pool_1 = fluid.nets.simple_img_conv_pool( input=img, filter_size=5, - num_filters=20, + num_filters=8, pool_size=2, pool_stride=2, pool_type='max', @@ -37,12 +37,12 @@ class TestFetchUnmerged(unittest.TestCase): conv_pool_2 = fluid.nets.simple_img_conv_pool( input=conv_pool_1, filter_size=5, - num_filters=50, + num_filters=16, pool_size=2, pool_stride=2, pool_type='avg', act="relu") - hidden = fluid.layers.fc(input=conv_pool_2, size=100, act='relu') + hidden = fluid.layers.fc(input=conv_pool_2, size=32, act='relu') prediction = fluid.layers.fc(input=hidden, size=10, act='softmax') loss = fluid.layers.cross_entropy(input=prediction, label=label) avg_loss = fluid.layers.mean(loss) @@ -75,8 +75,8 @@ class TestFetchUnmerged(unittest.TestCase): binary = fluid.CompiledProgram(main_program).with_data_parallel( loss_name=loss.name, build_strategy=build_strategy) - iters = 3 - batch_size = 64 + iters = 2 + batch_size = 16 train_reader = paddle.batch( paddle.reader.shuffle( paddle.dataset.mnist.train(), buf_size=500), diff --git a/python/paddle/fluid/tests/unittests/test_fleet_amp_meta_optimizer.py b/python/paddle/fluid/tests/unittests/test_fleet_amp_meta_optimizer.py index 38c3903306e6e76188cdb50476d6797814c434e9..73e014b35008ff5a0539c6a338755b9dc2cf68d4 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_amp_meta_optimizer.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_amp_meta_optimizer.py @@ -57,7 +57,7 @@ class TestFleetAMPOptimizer(unittest.TestCase): ops = [op.type for op in avg_cost.block.ops] self.assertIn('cast', ops) - self.assertIn('isfinite', ops) + self.assertIn('check_finite_and_unscale', ops) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_fleet_auto.py b/python/paddle/fluid/tests/unittests/test_fleet_auto.py new file mode 100644 index 0000000000000000000000000000000000000000..020f2f4db382ef1277167d85917e8fdba9c83893 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_fleet_auto.py @@ -0,0 +1,51 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import paddle +import os +import paddle.distributed.fleet as fleet +import paddle.distributed.fleet.base.role_maker as role_maker + + +class TestDistributedStrategyAuto(unittest.TestCase): + def setUp(self): + os.environ["POD_IP"] = "127.0.0.1" + os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001" + os.environ["PADDLE_TRAINERS_NUM"] = "2" + os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ + "127.0.0.1:36001,127.0.0.2:36001" + + def test_distributed_strategy_auto(self): + fleet.init(is_collective=True) + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32') + input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64') + + fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') + fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') + prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax') + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y) + avg_cost = paddle.fluid.layers.mean(x=cost) + + strategy = paddle.distributed.fleet.DistributedStrategy() + strategy.auto = True + optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.01) + optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy) + optimizer.minimize(avg_cost) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_fleet_distributed_strategy.py b/python/paddle/fluid/tests/unittests/test_fleet_distributed_strategy.py index 8d715674cc6c9ba4f8b5c1ff4fe0cbdbe7841643..6f8af3017efcb9010b129131a01c5ee071b5bc36 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_distributed_strategy.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_distributed_strategy.py @@ -81,9 +81,10 @@ class TestStrategyConfig(unittest.TestCase): def test_localsgd_configs(self): strategy = paddle.distributed.fleet.DistributedStrategy() - configs = {"k_steps": 4} + configs = {"k_steps": 4, "begin_step": 120} strategy.localsgd_configs = configs self.assertEqual(strategy.localsgd_configs["k_steps"], 4) + self.assertEqual(strategy.localsgd_configs["begin_step"], 120) def test_dgc(self): strategy = paddle.distributed.fleet.DistributedStrategy() @@ -230,7 +231,7 @@ class TestStrategyConfig(unittest.TestCase): strategy.a_sync = True strategy.localsgd = True strategy.dgc = True - localsgd_configs = {"k_steps": 5} + localsgd_configs = {"k_steps": 5, "begin_step": 1} strategy.localsgd_configs = localsgd_configs build_strategy = paddle.fluid.BuildStrategy() build_strategy.enable_sequential_execution = True @@ -316,6 +317,14 @@ class TestStrategyConfig(unittest.TestCase): self.assertEqual(strategy.conv_workspace_size_limit, 1000) strategy._enable_env() + def test_distributed_strategy_repr(self): + strategy = paddle.distributed.fleet.DistributedStrategy() + strategy.recompute = True + strategy.recompute_configs = {"checkpoints": ["a1", "a2", "a3"]} + strategy.amp = True + strategy.localsgd = True + print(str(strategy)) + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_fleet_lamb_meta_optimizer.py b/python/paddle/fluid/tests/unittests/test_fleet_lamb_meta_optimizer.py index 3f140f53b043b1949572f3728ca8a0c556317783..ff305fb95231b96b6d8f951b2943a0ab47060ce0 100755 --- a/python/paddle/fluid/tests/unittests/test_fleet_lamb_meta_optimizer.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_lamb_meta_optimizer.py @@ -22,11 +22,9 @@ import paddle.distributed.fleet.base.role_maker as role_maker class TestFleetLambMetaOptimizer(unittest.TestCase): def setUp(self): - os.environ["POD_IP"] = "127.0.0.1" - os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001" - os.environ["PADDLE_TRAINERS_NUM"] = "2" - os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ - "127.0.0.1:36001,127.0.0.2:36001" + os.environ["PADDLE_TRAINER_ID"] = "1" + os.environ[ + "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001,127.0.0.1:36002" def net(self, main_prog, startup_prog): with fluid.program_guard(main_prog, startup_prog): @@ -97,13 +95,54 @@ class TestFleetLambMetaOptimizer(unittest.TestCase): optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy) optimizer.minimize(avg_cost) - ops_with_bias = [ + ops_without_wd = [ op for op in avg_cost.block.ops if op.type == 'lamb' and op.attr('op_role_var')[0].endswith('.b_0') ] - for op in ops_with_bias: + for op in ops_without_wd: self.assertEqual(op.attr('weight_decay'), 0) + def test_lamb_apply_with_amp(self): + role = role_maker.PaddleCloudRoleMaker(is_collective=True) + fleet.init(role) + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32') + input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64') + + fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') + fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') + prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax') + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y) + avg_cost = paddle.fluid.layers.mean(x=cost) + + strategy = paddle.distributed.fleet.DistributedStrategy() + strategy.amp = True + strategy.amp_configs = { + "init_loss_scaling": 32768, + "decr_every_n_nan_or_inf": 2, + "incr_every_n_steps": 1000, + "incr_ratio": 2.0, + "use_dynamic_loss_scaling": True, + "decr_ratio": 0.5, + "custom_white_list": ['softmax'], + "custom_black_list": ['tanh'], + } + strategy.lamb = True + strategy.lamb_configs = { + 'lamb_weight_decay': 0.01, + 'exclude_from_weight_decay': [], + } + + optimizer = paddle.fluid.optimizer.Adam(learning_rate=0.01) + optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy) + optimizer.minimize(avg_cost) + + ops = [op.type for op in avg_cost.block.ops] + self.assertIn('lamb', ops) + self.assertIn('cast', ops) + self.assertIn('isfinite', ops) + if __name__ == "__main__": unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_fleet_lars_meta_optimizer.py b/python/paddle/fluid/tests/unittests/test_fleet_lars_meta_optimizer.py index 3caa1a4eac0bf191b13e6708b1a9adffdb111ca7..34ab423e064eebb9c93010fbc869adedb42bd6fa 100755 --- a/python/paddle/fluid/tests/unittests/test_fleet_lars_meta_optimizer.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_lars_meta_optimizer.py @@ -22,11 +22,9 @@ import paddle.distributed.fleet.base.role_maker as role_maker class TestFleetLarsMetaOptimizer(unittest.TestCase): def setUp(self): - os.environ["POD_IP"] = "127.0.0.1" - os.environ["PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001" - os.environ["PADDLE_TRAINERS_NUM"] = "2" - os.environ["PADDLE_PSERVERS_IP_PORT_LIST"] = \ - "127.0.0.1:36001,127.0.0.2:36001" + os.environ["PADDLE_TRAINER_ID"] = "1" + os.environ[ + "PADDLE_TRAINER_ENDPOINTS"] = "127.0.0.1:36001,127.0.0.1:36002" def net(self, main_prog, startup_prog): with fluid.program_guard(main_prog, startup_prog): @@ -52,6 +50,8 @@ class TestFleetLarsMetaOptimizer(unittest.TestCase): strategy.lars_configs = { "lars_coeff": 0.001, "lars_weight_decay": 0.0005, + "epsilon": 0, + "exclude_from_weight_decay": ["batch_norm", ".b"], } return avg_cost, strategy @@ -83,6 +83,70 @@ class TestFleetLarsMetaOptimizer(unittest.TestCase): ops = [op.type for op in avg_cost.block.ops] self.assertNotIn('lars_momentum', ops) + def test_lars_exclude_fn(self): + role = role_maker.PaddleCloudRoleMaker(is_collective=True) + fleet.init(role) + startup_prog = fluid.Program() + train_prog = fluid.Program() + avg_cost, strategy = self.net(train_prog, startup_prog) + optimizer = paddle.fluid.optimizer.Momentum( + learning_rate=0.01, momentum=0.9) + + optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy) + optimizer.minimize(avg_cost) + + ops_without_wd = [ + op for op in avg_cost.block.ops + if op.type == 'lars_momentum' and ("batch_norm" in op.attr( + 'op_role_var')[0] or ".b" in op.attr('op_role_var')[0]) + ] + for op in ops_without_wd: + self.assertEqual(op.attr('lars_weight_decay'), 0) + + def test_lars_apply_with_amp(self): + role = role_maker.PaddleCloudRoleMaker(is_collective=True) + fleet.init(role) + input_x = paddle.fluid.layers.data( + name="x", shape=[32], dtype='float32') + input_y = paddle.fluid.layers.data(name="y", shape=[1], dtype='int64') + + fc_1 = paddle.fluid.layers.fc(input=input_x, size=64, act='tanh') + fc_2 = paddle.fluid.layers.fc(input=fc_1, size=64, act='tanh') + prediction = paddle.fluid.layers.fc(input=[fc_2], size=2, act='softmax') + cost = paddle.fluid.layers.cross_entropy( + input=prediction, label=input_y) + avg_cost = paddle.fluid.layers.mean(x=cost) + + strategy = paddle.distributed.fleet.DistributedStrategy() + strategy.amp = True + strategy.amp_configs = { + "init_loss_scaling": 32768, + "decr_every_n_nan_or_inf": 2, + "incr_every_n_steps": 1000, + "incr_ratio": 2.0, + "use_dynamic_loss_scaling": True, + "decr_ratio": 0.5, + "custom_white_list": ['softmax'], + "custom_black_list": ['tanh'], + } + strategy.lars = True + strategy.lars_configs = { + "lars_coeff": 0.001, + "lars_weight_decay": 0.0005, + "epsilon": 0, + "exclude_from_weight_decay": ["batch_norm", ".b"], + } + + optimizer = paddle.fluid.optimizer.Momentum( + learning_rate=0.01, momentum=0.9) + optimizer = fleet.distributed_optimizer(optimizer, strategy=strategy) + optimizer.minimize(avg_cost) + + ops = [op.type for op in avg_cost.block.ops] + self.assertIn('lars_momentum', ops) + self.assertIn('cast', ops) + self.assertIn('isfinite', ops) + if __name__ == "__main__": unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_fleet_localsgd_meta_optimizer.py b/python/paddle/fluid/tests/unittests/test_fleet_localsgd_meta_optimizer.py index 07b988bf8752057e68925bc42f564a72d466361d..945f5ae57454b2c4a509badb93574a6e03b607e8 100644 --- a/python/paddle/fluid/tests/unittests/test_fleet_localsgd_meta_optimizer.py +++ b/python/paddle/fluid/tests/unittests/test_fleet_localsgd_meta_optimizer.py @@ -44,6 +44,7 @@ class TestFleetLocalSGDMetaOptimizer(unittest.TestCase): strategy.auto = True config = strategy.localsgd_configs config['k_steps'] = 1 + config['begin_step'] = 1 strategy.localsgd_configs = config optimizer = paddle.fluid.optimizer.SGD(learning_rate=0.01) diff --git a/python/paddle/fluid/tests/unittests/test_fuse_bn_act_pass.py b/python/paddle/fluid/tests/unittests/test_fuse_bn_act_pass.py index 921dbdbc6d4e1b169c2c8aa199ea15f886bd0128..5bcfc8720ddd2a8b495c50f886c03047c9abdb32 100644 --- a/python/paddle/fluid/tests/unittests/test_fuse_bn_act_pass.py +++ b/python/paddle/fluid/tests/unittests/test_fuse_bn_act_pass.py @@ -25,7 +25,7 @@ class TestFuseBatchNormActPass(unittest.TestCase): hidden1 = fluid.layers.conv2d( input=x, filter_size=3, - num_filters=32, + num_filters=16, stride=1, padding=1, act=None, @@ -43,7 +43,7 @@ class TestFuseBatchNormActPass(unittest.TestCase): bias_attr=bias_attr, act='relu', data_layout='NHWC') - hidden3 = fluid.layers.fc(input=hidden2, size=128, act='relu') + hidden3 = fluid.layers.fc(input=hidden2, size=32, act='relu') hidden4 = fluid.layers.batch_norm( input=hidden3, act='relu', data_layout='NHWC') prediction = fluid.layers.fc(input=hidden4, size=10, act='softmax') @@ -63,7 +63,7 @@ class TestFuseBatchNormActPass(unittest.TestCase): startup_program = fluid.Program() x, y, loss = self.build_program(main_program, startup_program, use_cuda) exe = fluid.Executor(place) - iters = 10 + iters = 8 batch_size = 16 feeder = fluid.DataFeeder(feed_list=[x, y], place=place) diff --git a/python/paddle/fluid/tests/unittests/test_generate_proposals_op.py b/python/paddle/fluid/tests/unittests/test_generate_proposals_op.py index ce561cd317c48228c4877a2b65b67fe049a0d84a..26fc01ca04506758599ac5d6fe6842984a8d7a9c 100644 --- a/python/paddle/fluid/tests/unittests/test_generate_proposals_op.py +++ b/python/paddle/fluid/tests/unittests/test_generate_proposals_op.py @@ -34,18 +34,18 @@ def generate_proposals_in_python(scores, bbox_deltas, im_info, anchors, rpn_rois = [] rpn_roi_probs = [] - lod = [] + rois_num = [] num_images = scores.shape[0] for img_idx in range(num_images): img_i_boxes, img_i_probs = proposal_for_one_image( im_info[img_idx, :], all_anchors, variances, bbox_deltas[img_idx, :, :, :], scores[img_idx, :, :, :], pre_nms_topN, post_nms_topN, nms_thresh, min_size, eta) - lod.append(img_i_probs.shape[0]) + rois_num.append(img_i_probs.shape[0]) rpn_rois.append(img_i_boxes) rpn_roi_probs.append(img_i_probs) - return rpn_rois, rpn_roi_probs, lod + return rpn_rois, rpn_roi_probs, rois_num def proposal_for_one_image(im_info, all_anchors, variances, bbox_deltas, scores, @@ -87,6 +87,10 @@ def proposal_for_one_image(im_info, all_anchors, variances, bbox_deltas, scores, proposals = clip_tiled_boxes(proposals, im_info[:2]) # remove predicted boxes with height or width < min_size keep = filter_boxes(proposals, min_size, im_info) + if len(keep) == 0: + proposals = np.zeros((1, 4)).astype('float32') + scores = np.zeros((1, 1)).astype('float32') + return proposals, scores proposals = proposals[keep, :] scores = scores[keep, :] @@ -280,8 +284,8 @@ class TestGenerateProposalsOp(OpTest): } self.outputs = { - 'RpnRois': (self.rpn_rois[0], [self.lod]), - 'RpnRoiProbs': (self.rpn_roi_probs[0], [self.lod]), + 'RpnRois': (self.rpn_rois[0], [self.rois_num]), + 'RpnRoiProbs': (self.rpn_roi_probs[0], [self.rois_num]), } def test_check_output(self): @@ -320,7 +324,7 @@ class TestGenerateProposalsOp(OpTest): (batch_size, num_anchors * 4, layer_h, layer_w)).astype('float32') def init_test_output(self): - self.rpn_rois, self.rpn_roi_probs, self.lod = generate_proposals_in_python( + self.rpn_rois, self.rpn_roi_probs, self.rois_num = generate_proposals_in_python( self.scores, self.bbox_deltas, self.im_info, self.anchors, self.variances, self.pre_nms_topN, self.post_nms_topN, self.nms_thresh, self.min_size, self.eta) @@ -349,12 +353,21 @@ class TestGenerateProposalsOutLodOp(TestGenerateProposalsOp): } self.outputs = { - 'RpnRois': (self.rpn_rois[0], [self.lod]), - 'RpnRoiProbs': (self.rpn_roi_probs[0], [self.lod]), - 'RpnRoisLod': (np.asarray( - self.lod, dtype=np.int32)) + 'RpnRois': (self.rpn_rois[0], [self.rois_num]), + 'RpnRoiProbs': (self.rpn_roi_probs[0], [self.rois_num]), + 'RpnRoisNum': (np.asarray( + self.rois_num, dtype=np.int32)) } +class TestGenerateProposalsOpNoBoxLeft(TestGenerateProposalsOp): + def init_test_params(self): + self.pre_nms_topN = 12000 # train 12000, test 2000 + self.post_nms_topN = 5000 # train 6000, test 1000 + self.nms_thresh = 0.7 + self.min_size = 1000.0 + self.eta = 1. + + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_grid_sample_function.py b/python/paddle/fluid/tests/unittests/test_grid_sample_function.py index 4a33f32a0b6977716d8065419f8e0f88d6c4f44a..ea94a8ba69a784efb1a2a12f6f251316553cab50 100644 --- a/python/paddle/fluid/tests/unittests/test_grid_sample_function.py +++ b/python/paddle/fluid/tests/unittests/test_grid_sample_function.py @@ -100,7 +100,7 @@ def add_cases(suite): GridSampleTestCase( methodName='runTest', mode='bilinear', - padding_mode='reflect', + padding_mode='reflection', align_corners=True)) suite.addTest( GridSampleTestCase( diff --git a/python/paddle/fluid/tests/unittests/test_grid_sampler_op.py b/python/paddle/fluid/tests/unittests/test_grid_sampler_op.py index 4d1ed5aeb96ebbe064e35c1bee9d5775812440f7..bf2f9518fb0c720556b7eecdf5b286dea0fff96c 100644 --- a/python/paddle/fluid/tests/unittests/test_grid_sampler_op.py +++ b/python/paddle/fluid/tests/unittests/test_grid_sampler_op.py @@ -73,7 +73,7 @@ def unnormalizeAndClip(grid_slice, max_val, align_corners, padding_mode): if padding_mode == "border": grid_slice = clip(grid_slice, 0, max_val) - elif padding_mode == "reflect": + elif padding_mode == "reflection": double_range = 2 * max_val if align_corners else (max_val + 1) * 2 grid_abs = np.abs(grid_slice) if align_corners else np.abs(grid_slice + 0.5) @@ -211,7 +211,7 @@ class Case2(TestGridSamplerOp): self.grid_shape = (2, 8, 9, 2) self.theta_shape = (2, 2, 3) self.align_corners = False - self.padding_mode = "reflect" + self.padding_mode = "reflection" self.mode = "bilinear" @@ -221,7 +221,7 @@ class Case3(TestGridSamplerOp): self.grid_shape = (2, 8, 9, 2) self.theta_shape = (2, 2, 3) self.align_corners = True - self.padding_mode = "reflect" + self.padding_mode = "reflection" self.mode = "bilinear" @@ -231,7 +231,7 @@ class Case4(TestGridSamplerOp): self.grid_shape = (2, 8, 9, 2) self.theta_shape = (2, 2, 3) self.align_corners = False - self.padding_mode = "reflect" + self.padding_mode = "reflection" self.mode = "nearest" self.numeric_grad_delta = 0.0001 diff --git a/python/paddle/fluid/tests/unittests/test_imperative_data_loader_process.py b/python/paddle/fluid/tests/unittests/test_imperative_data_loader_process.py index 7fb2cb0090da57ae837d1f774518dd90a41df56c..9b2d71c9f907779bc9b27b51e21056496f8d4dd5 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_data_loader_process.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_data_loader_process.py @@ -18,6 +18,7 @@ import multiprocessing import numpy as np import paddle.fluid as fluid from paddle.fluid import core +from paddle.fluid.reader import _reader_process_loop if sys.version_info[0] == 2: import Queue as queue @@ -66,7 +67,7 @@ class TestDygraphDataLoaderProcess(unittest.TestCase): batch_generator_creator(self.batch_size, self.batch_num), places=fluid.CPUPlace()) loader._data_queue = queue.Queue(self.batch_num + 1) - loader._reader_process_loop() + _reader_process_loop(loader._batch_reader, loader._data_queue) # For clean memory mapped files util_queue = multiprocessing.Queue(self.batch_num + 1) for _ in range(self.batch_num): @@ -94,7 +95,7 @@ class TestDygraphDataLoaderProcess(unittest.TestCase): loader._data_queue = queue.Queue(self.batch_num + 1) exception = None try: - loader._reader_process_loop() + _reader_process_loop(loader._batch_reader, loader._data_queue) except core.EnforceNotMet as ex: exception = ex self.assertIsNotNone(exception) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_numpy_bridge.py b/python/paddle/fluid/tests/unittests/test_imperative_numpy_bridge.py index da01be8159a5c5d277a22134eb60ef37ef85fc4f..772dd913e4d20ccf51601ea620822c250cb45320 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_numpy_bridge.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_numpy_bridge.py @@ -15,18 +15,26 @@ import unittest import numpy as np import paddle.fluid as fluid +import warnings class TestImperativeNumpyBridge(unittest.TestCase): def test_tensor_from_numpy(self): data_np = np.array([[2, 3, 1]]).astype('float32') with fluid.dygraph.guard(fluid.CPUPlace()): - var = fluid.dygraph.to_variable(data_np, zero_copy=True) - self.assertTrue(np.array_equal(var.numpy(), data_np)) - data_np[0][0] = 4 - self.assertEqual(data_np[0][0], 4) - self.assertEqual(var[0][0].numpy()[0], 4) - self.assertTrue(np.array_equal(var.numpy(), data_np)) + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + var = fluid.dygraph.to_variable(data_np, zero_copy=True) + assert "Currently, zero_copy is not supported, and it will be discarded." in str( + w[-1].message) + # Temporally diable zero_copy + # var = fluid.dygraph.to_variable(data_np, zero_copy=True) + # self.assertTrue(np.array_equal(var.numpy(), data_np)) + # data_np[0][0] = 4 + # self.assertEqual(data_np[0][0], 4) + # self.assertEqual(var[0][0].numpy()[0], 4) + # self.assertTrue(np.array_equal(var.numpy(), data_np)) + var2 = fluid.dygraph.to_variable(data_np, zero_copy=False) self.assertTrue(np.array_equal(var2.numpy(), data_np)) data_np[0][0] = -1 diff --git a/python/paddle/fluid/tests/unittests/test_jit_save_load.py b/python/paddle/fluid/tests/unittests/test_jit_save_load.py index f7fcc1ff561b90dc1b78a67ffbe7c047ed06d0e9..7e6ca8076de5186def1229b58bd23df73021430e 100644 --- a/python/paddle/fluid/tests/unittests/test_jit_save_load.py +++ b/python/paddle/fluid/tests/unittests/test_jit_save_load.py @@ -23,7 +23,7 @@ from paddle.static import InputSpec import paddle.fluid as fluid from paddle.fluid.dygraph import Linear from paddle.fluid.dygraph import declarative, ProgramTranslator -from paddle.fluid.dygraph.io import EXTRA_VAR_INFO_FILENAME +from paddle.fluid.dygraph.io import EXTRA_VAR_INFO_FILENAME, VARIABLE_FILENAME BATCH_SIZE = 32 BATCH_NUM = 10 @@ -56,6 +56,16 @@ class LinearNet(fluid.dygraph.Layer): return self._linear(x) +class LinearNetWithInputSpec(fluid.dygraph.Layer): + def __init__(self, in_size, out_size): + super(LinearNetWithInputSpec, self).__init__() + self._linear = Linear(in_size, out_size) + + @declarative(input_spec=[InputSpec(shape=[None, 784], dtype='float32')]) + def forward(self, x): + return self._linear(x) + + class LinearNetNotDeclarative(fluid.dygraph.Layer): def __init__(self, in_size, out_size): super(LinearNetNotDeclarative, self).__init__() @@ -65,6 +75,23 @@ class LinearNetNotDeclarative(fluid.dygraph.Layer): return self._linear(x) +class LinerNetWithLabel(paddle.nn.Layer): + def __init__(self, in_size, out_size): + super(LinerNetWithLabel, self).__init__() + self._linear = Linear(in_size, out_size) + + @declarative(input_spec=[ + InputSpec( + shape=[None, 784], dtype='float32', name="image"), InputSpec( + shape=[None, 1], dtype='int64', name="label") + ]) + def forward(self, x, label): + out = self._linear(x) + loss = fluid.layers.cross_entropy(out, label) + avg_loss = fluid.layers.mean(loss) + return out, avg_loss + + class LinearNetReturnLoss(fluid.dygraph.Layer): def __init__(self, in_size, out_size): super(LinearNetReturnLoss, self).__init__() @@ -78,6 +105,72 @@ class LinearNetReturnLoss(fluid.dygraph.Layer): return z, loss +class LinearNetMultiInput(fluid.dygraph.Layer): + def __init__(self, in_size, out_size): + super(LinearNetMultiInput, self).__init__() + self._linear1 = Linear(in_size, out_size) + self._linear2 = Linear(in_size, out_size) + + @declarative(input_spec=[ + InputSpec( + [None, 8], dtype='float32'), InputSpec( + [None, 8], dtype='float32') + ]) + def forward(self, x, y): + x_out = self._linear1(x) + y_out = self._linear2(y) + loss = fluid.layers.mean(x_out + y_out) + return x_out, y_out, loss + + +class MultiLoadingLinearNet(fluid.dygraph.Layer): + def __init__(self, size, model_path): + super(MultiLoadingLinearNet, self).__init__() + self._linear = Linear(size, size) + self._load_linear1 = fluid.dygraph.jit.load(model_path) + self._load_linear2 = fluid.dygraph.jit.load(model_path) + + @declarative + def forward(self, x): + tmp1 = self._linear(x) + tmp2 = self._load_linear1(tmp1) + tmp3 = self._load_linear2(tmp2) + y = self._linear(tmp3) + return y + + +class LinearNetReturnHidden(fluid.dygraph.Layer): + def __init__(self, in_size, out_size): + super(LinearNetReturnHidden, self).__init__() + self._linear_1 = Linear(in_size, out_size) + self._linear_2 = Linear(in_size, out_size) + + @declarative + def forward(self, x): + y = self._linear_1(x) + z = self._linear_2(y) + loss = fluid.layers.mean(z) + return y, loss + + +class EmptyLayer(paddle.nn.Layer): + def __init__(self): + super(EmptyLayer, self).__init__() + + @paddle.jit.to_static + def forward(self, x): + return x + + +class NoParamLayer(paddle.nn.Layer): + def __init__(self): + super(NoParamLayer, self).__init__() + + @paddle.jit.to_static + def forward(self, x, y): + return x + y + + def train(layer, input_size=784, label_size=1): # create optimizer sgd = fluid.optimizer.SGDOptimizer( @@ -102,6 +195,27 @@ def train(layer, input_size=784, label_size=1): return [img], layer, avg_loss +def train_with_label(layer, input_size=784, label_size=1): + # create optimizer + sgd = fluid.optimizer.SGDOptimizer( + learning_rate=0.01, parameter_list=layer.parameters()) + # create data loader + train_loader = fluid.io.DataLoader.from_generator(capacity=5) + train_loader.set_batch_generator( + random_batch_reader(input_size, label_size)) + # train + for data in train_loader(): + img, label = data + label.stop_gradient = True + + out, avg_loss = layer(img, label) + + avg_loss.backward() + sgd.minimize(avg_loss) + layer.clear_gradients() + return out + + class TestJitSaveLoad(unittest.TestCase): def setUp(self): self.model_path = "model.test_jit_save_load" @@ -159,8 +273,11 @@ class TestJitSaveLoad(unittest.TestCase): train_layer.eval() # construct new model new_layer = LinearNet(784, 1) - model_dict, _ = fluid.dygraph.load_dygraph(self.model_path) - new_layer.set_dict(model_dict) + orig_state_dict = new_layer.state_dict() + load_state_dict, _ = fluid.dygraph.load_dygraph(self.model_path) + for structured_name in orig_state_dict: + self.assertTrue(structured_name in load_state_dict) + new_layer.set_state_dict(load_state_dict) new_layer.eval() # inference & compare x = fluid.dygraph.to_variable( @@ -168,38 +285,20 @@ class TestJitSaveLoad(unittest.TestCase): self.assertTrue( np.array_equal(train_layer(x).numpy(), new_layer(x).numpy())) - def test_save_get_program_failed(self): - layer = LinearNetNotDeclarative(784, 1) - example_inputs, layer, _ = train(layer) - with self.assertRaises(RuntimeError): - fluid.dygraph.jit.save( - layer=layer, - model_path=self.model_path, - input_spec=example_inputs) - def test_load_dygraph_no_path(self): model_path = "model.test_jit_save_load.no_path" new_layer = LinearNet(784, 1) with self.assertRaises(ValueError): model_dict, _ = fluid.dygraph.load_dygraph(model_path) - -class LinearNetMultiInput(fluid.dygraph.Layer): - def __init__(self, in_size, out_size): - super(LinearNetMultiInput, self).__init__() - self._linear1 = Linear(in_size, out_size) - # self._linear2 = Linear(in_size, out_size) - - @declarative(input_spec=[ - InputSpec( - [None, 8], dtype='float32'), InputSpec( - [None, 8], dtype='float32') - ]) - def forward(self, x, y): - x_out = self._linear1(x) - y_out = self._linear1(y) - loss = fluid.layers.mean(x_out + y_out) - return x_out, y_out, loss + def test_jit_load_model_incomplete(self): + model_path = "model.test_jit_save_load.remove_variables" + self.train_and_save_model(model_path=model_path) + # remove `__variables__` + var_path = os.path.join(model_path, VARIABLE_FILENAME) + os.remove(var_path) + with self.assertRaises(ValueError): + paddle.jit.load(model_path) class TestSaveLoadWithInputSpec(unittest.TestCase): @@ -345,22 +444,6 @@ class TestJitSaveLoadConfig(unittest.TestCase): np.array_equal(train_layer(x)[0].numpy(), infer_layer(x).numpy())) -class MultiLoadingLinearNet(fluid.dygraph.Layer): - def __init__(self, size, model_path): - super(MultiLoadingLinearNet, self).__init__() - self._linear = Linear(size, size) - self._load_linear1 = fluid.dygraph.jit.load(model_path) - self._load_linear2 = fluid.dygraph.jit.load(model_path) - - @declarative - def forward(self, x): - tmp1 = self._linear(x) - tmp2 = self._load_linear1(tmp1) - tmp3 = self._load_linear2(tmp2) - y = self._linear(tmp3) - return y - - class TestJitMultipleLoading(unittest.TestCase): def setUp(self): self.linear_size = 4 @@ -389,20 +472,6 @@ class TestJitMultipleLoading(unittest.TestCase): name_set.add(var.name) -class LinearNetReturnHidden(fluid.dygraph.Layer): - def __init__(self, in_size, out_size): - super(LinearNetReturnHidden, self).__init__() - self._linear_1 = Linear(in_size, out_size) - self._linear_2 = Linear(in_size, out_size) - - @declarative - def forward(self, x): - y = self._linear_1(x) - z = self._linear_2(y) - loss = fluid.layers.mean(z) - return y, loss - - class TestJitPruneModelAndLoad(unittest.TestCase): def setUp(self): self.linear_size = 4 @@ -461,5 +530,230 @@ class TestJitPruneModelAndLoad(unittest.TestCase): fluid.dygraph.jit.load(self.model_path) +class TestJitSaveMultiCases(unittest.TestCase): + def setUp(self): + # enable dygraph mode + fluid.enable_dygraph() + # config seed + paddle.manual_seed(SEED) + paddle.framework.random._manual_program_seed(SEED) + + def verify_inference_correctness(self, layer, model_path, with_label=False): + layer.eval() + loaded_layer = paddle.jit.load(model_path) + loaded_layer.eval() + # inference & compare + x = paddle.to_variable(np.random.random((1, 784)).astype('float32')) + if with_label: + y = paddle.to_variable(np.random.random((1, 1)).astype('int64')) + pred, _ = layer(x, y) + pred = pred.numpy() + else: + pred = layer(x).numpy() + loaded_pred = loaded_layer(x).numpy() + self.assertTrue( + np.array_equal(pred, loaded_pred), + msg="Result diff when load and inference:\nlayer result:\n{}\n" \ + "loaded layer result:\n{}".format(pred, loaded_pred)) + + def test_no_prune_to_static_after_train(self): + layer = LinearNet(784, 1) + + train(layer) + + model_path = "test_no_prune_to_static_after_train" + paddle.jit.save(layer, model_path) + + self.verify_inference_correctness(layer, model_path) + + def test_no_prune_to_static_no_train(self): + layer = LinearNetWithInputSpec(784, 1) + + model_path = "test_no_prune_to_static_no_train" + paddle.jit.save(layer, model_path) + + self.verify_inference_correctness(layer, model_path) + + def test_no_prune_no_to_static_after_train(self): + layer = LinearNetNotDeclarative(784, 1) + + train(layer) + + model_path = "test_no_prune_no_to_static_after_train" + paddle.jit.save( + layer, + model_path, + input_spec=[InputSpec( + shape=[None, 784], dtype='float32')]) + + self.verify_inference_correctness(layer, model_path) + + def test_no_prune_no_to_static_after_train_with_examples(self): + layer = LinearNetNotDeclarative(784, 1) + + example_inputs, _, _ = train(layer) + + model_path = "test_no_prune_no_to_static_after_train_with_examples" + fluid.dygraph.jit.save( + layer=layer, model_path=model_path, input_spec=example_inputs) + + self.verify_inference_correctness(layer, model_path) + + def test_no_prune_no_to_static_no_train(self): + layer = LinearNetNotDeclarative(784, 1) + + model_path = "test_no_prune_no_to_static_no_train" + paddle.jit.save( + layer, + model_path, + input_spec=[InputSpec( + shape=[None, 784], dtype='float32')]) + + self.verify_inference_correctness(layer, model_path) + + def test_prune_to_static_after_train(self): + layer = LinerNetWithLabel(784, 1) + + out = train_with_label(layer) + + model_path = "test_prune_to_static_after_train" + configs = paddle.SaveLoadConfig() + configs.output_spec = [out] + paddle.jit.save( + layer, + model_path, + input_spec=[ + InputSpec( + shape=[None, 784], dtype='float32', name="image") + ], + configs=configs) + + self.verify_inference_correctness(layer, model_path, True) + + def test_prune_to_static_no_train(self): + layer = LinerNetWithLabel(784, 1) + + model_path = "test_prune_to_static_no_train" + configs = paddle.SaveLoadConfig() + # TODO: no train, cannot get output_spec var here + # now only can use index + configs.output_spec = layer.forward.outputs[:1] + paddle.jit.save( + layer, + model_path, + input_spec=[ + InputSpec( + shape=[None, 784], dtype='float32', name="image") + ], + configs=configs) + + self.verify_inference_correctness(layer, model_path, True) + + def test_no_prune_input_spec_name_warning(self): + layer = LinearNetWithInputSpec(784, 1) + + train(layer) + + model_path = "test_no_prune_input_spec_name_warning" + paddle.jit.save( + layer, + model_path, + input_spec=[InputSpec( + shape=[None, 784], dtype='float32')]) + paddle.jit.save( + layer, + model_path, + input_spec=[ + InputSpec( + shape=[None, 784], dtype='float32', name='feed_input') + ]) + + self.verify_inference_correctness(layer, model_path) + + def test_not_prune_output_spec_name_warning(self): + layer = LinearNet(784, 1) + + train(layer) + + model_path = "test_not_prune_output_spec_name_warning" + configs = paddle.SaveLoadConfig() + out = paddle.to_variable(np.random.random((1, 1)).astype('float')) + configs.output_spec = [out] + paddle.jit.save(layer, model_path, configs=configs) + + self.verify_inference_correctness(layer, model_path) + + def test_prune_input_spec_name_error(self): + layer = LinerNetWithLabel(784, 1) + + model_path = "test_prune_input_spec_name_error" + with self.assertRaises(ValueError): + paddle.jit.save( + layer, + model_path, + input_spec=[InputSpec( + shape=[None, 784], dtype='float32')]) + with self.assertRaises(ValueError): + paddle.jit.save( + layer, + model_path, + input_spec=[ + InputSpec( + shape=[None, 784], dtype='float32', name='feed_input') + ]) + + def test_prune_output_spec_name_error(self): + layer = LinerNetWithLabel(784, 1) + + train_with_label(layer) + + model_path = "test_prune_to_static_after_train" + configs = paddle.SaveLoadConfig() + out = paddle.to_variable(np.random.random((1, 1)).astype('float')) + configs.output_spec = [out] + with self.assertRaises(ValueError): + paddle.jit.save( + layer, + model_path, + input_spec=[ + InputSpec( + shape=[None, 784], dtype='float32', name="image") + ], + configs=configs) + + +class TestJitSaveLoadEmptyLayer(unittest.TestCase): + def setUp(self): + self.model_path = "model.jit_save_load_empty_layer" + # enable dygraph mode + paddle.disable_static() + + def test_save_load_empty_layer(self): + layer = EmptyLayer() + x = paddle.to_variable(np.random.random((10)).astype('float32')) + out = layer(x) + paddle.jit.save(layer, self.model_path) + load_layer = paddle.jit.load(self.model_path) + load_out = load_layer(x) + self.assertTrue(np.array_equal(out, load_out)) + + +class TestJitSaveLoadNoParamLayer(unittest.TestCase): + def setUp(self): + self.model_path = "model.jit_save_load_no_param_layer" + # enable dygraph mode + paddle.disable_static() + + def test_save_load_no_param_layer(self): + layer = NoParamLayer() + x = paddle.to_variable(np.random.random((5)).astype('float32')) + y = paddle.to_variable(np.random.random((5)).astype('float32')) + out = layer(x, y) + paddle.jit.save(layer, self.model_path) + load_layer = paddle.jit.load(self.model_path) + load_out = load_layer(x, y) + self.assertTrue(np.array_equal(out, load_out)) + + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_kldiv_loss_op.py b/python/paddle/fluid/tests/unittests/test_kldiv_loss_op.py index 8780727e4cb276a989a8d04d05c6419a4874e7f5..041fe4e9043d60852fcaab42bc233b63b39609ce 100644 --- a/python/paddle/fluid/tests/unittests/test_kldiv_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_kldiv_loss_op.py @@ -24,7 +24,10 @@ def kldiv_loss(x, target, reduction): loss = np.where(target >= 0, output, np.zeros_like(x)) if reduction == "batchmean": - return loss.sum() / x.shape[0] + if len(x.shape) > 0: + return loss.sum() / x.shape[0] + else: + return loss.sum() if reduction == "mean": return loss.mean() if reduction == "sum": @@ -93,6 +96,9 @@ class TestKLDivLossDygraph(unittest.TestCase): def test_kl_loss_batchmean(self): self.run_kl_loss('batchmean') + def test_kl_loss_batchmean_shape(self): + self.run_kl_loss('batchmean', ()) + def test_kl_loss_mean(self): self.run_kl_loss('mean') diff --git a/python/paddle/fluid/tests/unittests/test_layers.py b/python/paddle/fluid/tests/unittests/test_layers.py index b76887f0965ca64b2b40bf9c0ce6e82b44fdad2f..26073f49bdd3d494da7b39346c5bafb2aefba56a 100644 --- a/python/paddle/fluid/tests/unittests/test_layers.py +++ b/python/paddle/fluid/tests/unittests/test_layers.py @@ -2677,13 +2677,6 @@ class TestBook(LayerTest): out = layers.sigmoid(input, name='sigmoid') return (out) - def make_logsigmoid(self): - with program_guard(fluid.default_main_program(), - fluid.default_startup_program()): - input = self._get_data(name="input", shape=[16], dtype="float32") - out = layers.logsigmoid(input, name='logsigmoid') - return (out) - def make_exp(self): with program_guard(fluid.default_main_program(), fluid.default_startup_program()): @@ -3318,15 +3311,29 @@ class TestBook(LayerTest): return (out) def test_roi_pool(self): - # TODO(minqiyang): dygraph do not support lod now + x_np = np.random.rand(2, 3, 8, 8).astype('float32') + rois_np = np.random.rand(3, 4).astype('float32') + rois_num_np = np.array([1, 2]).astype('int32') + with self.static_graph(): - x = layers.data(name="x", shape=[256, 30, 30], dtype="float32") - rois = layers.data( - name="rois", shape=[4], dtype="float32", lod_level=1) - rois_lod = layers.data( - name="rois_lod", shape=[None, ], dtype="int", lod_level=1) - output = layers.roi_pool(x, rois, 7, 7, 0.6, rois_lod) - return (output) + x = layers.data(name="x", shape=[3, 8, 8], dtype="float32") + rois = layers.data(name="rois", shape=[4], dtype="float32") + rois_num = fluid.data(name="rois_num", shape=[None], dtype="int32") + output = layers.roi_pool(x, rois, 4, 4, 0.5, rois_num=rois_num) + static_res = self.get_static_graph_result( + feed={'x': x_np, + 'rois': rois_np, + 'rois_num': rois_num_np}, + fetch_list=[output])[0] + + with self.dynamic_graph(): + x_dy = base.to_variable(x_np) + rois_dy = base.to_variable(rois_np) + rois_num_dy = base.to_variable(rois_num_np) + dy_res = layers.roi_pool( + x_dy, rois_dy, 4, 4, 0.5, rois_num=rois_num_dy) + dy_res_value = dy_res[0].numpy() + self.assertTrue(np.array_equal(static_res, dy_res_value)) def test_sequence_enumerate(self): # TODO(minqiyang): dygraph do not support lod now @@ -3335,16 +3342,29 @@ class TestBook(LayerTest): out = layers.sequence_enumerate(input=x, win_size=2, pad_value=0) def test_roi_align(self): - # TODO(minqiyang): dygraph do not support lod now + x_np = np.random.rand(2, 3, 8, 8).astype('float32') + rois_np = np.random.rand(3, 4).astype('float32') + rois_num_np = np.array([1, 2]).astype('int32') + with self.static_graph(): - x = layers.data(name="x", shape=[256, 30, 30], dtype="float32") - rois = layers.data( - name="rois", shape=[4], dtype="float32", lod_level=1) - rois_lod = layers.data( - name="rois_lod", shape=[None, ], dtype="int", lod_level=1) - output = layers.roi_align(x, rois, 14, 14, 0.5, 2, 'roi_align', - rois_lod) - return (output) + x = layers.data(name="x", shape=[3, 8, 8], dtype="float32") + rois = layers.data(name="rois", shape=[4], dtype="float32") + rois_num = fluid.data(name="rois_num", shape=[None], dtype="int32") + output = layers.roi_align(x, rois, 4, 4, 0.5, 2, rois_num=rois_num) + static_res = self.get_static_graph_result( + feed={'x': x_np, + 'rois': rois_np, + 'rois_num': rois_num_np}, + fetch_list=[output])[0] + + with self.dynamic_graph(): + x_dy = base.to_variable(x_np) + rois_dy = base.to_variable(rois_np) + rois_num_dy = base.to_variable(rois_num_np) + dy_res = layers.roi_align( + x_dy, rois_dy, 4, 4, 0.5, 2, rois_num=rois_num_dy) + dy_res_value = dy_res.numpy() + self.assertTrue(np.array_equal(static_res, dy_res_value)) def test_roi_perspective_transform(self): # TODO(minqiyang): dygraph do not support lod now diff --git a/python/paddle/fluid/tests/unittests/test_linear_interp_op.py b/python/paddle/fluid/tests/unittests/test_linear_interp_op.py index 53e8b02081ae3acf8a7fb5dd2bc6e05cbc3be901..c9948edad0061012cf028bec674a4bb713364541 100755 --- a/python/paddle/fluid/tests/unittests/test_linear_interp_op.py +++ b/python/paddle/fluid/tests/unittests/test_linear_interp_op.py @@ -293,7 +293,7 @@ class TestLinearInterpOpAPI2_0(unittest.TestCase): # dygraph x_data = np.random.random((1, 3, 128)).astype("float32") - us_1 = paddle.nn.UpSample( + us_1 = paddle.nn.Upsample( size=[64, ], mode='linear', align_mode=1, @@ -385,19 +385,19 @@ class TestLinearInterpOpError(unittest.TestCase): def input_shape_error(): x1 = fluid.data(name="x1", shape=[1], dtype="float32") - out1 = paddle.nn.UpSample( + out1 = paddle.nn.Upsample( size=[256, ], data_format='NCW', mode='linear') out1_res = out1(x1) def data_format_error(): x2 = fluid.data(name="x2", shape=[1, 3, 128], dtype="float32") - out2 = paddle.nn.UpSample( + out2 = paddle.nn.Upsample( size=[256, ], data_format='NHWCD', mode='linear') out2_res = out2(x2) def out_shape_error(): x3 = fluid.data(name="x3", shape=[1, 3, 128], dtype="float32") - out3 = paddle.nn.UpSample( + out3 = paddle.nn.Upsample( size=[ 256, 256, diff --git a/python/paddle/fluid/tests/unittests/test_linear_interp_v2_op.py b/python/paddle/fluid/tests/unittests/test_linear_interp_v2_op.py index 04b56677fc158583fe79ec0dc1276210bd2ebbdc..b34989f5f5c79dfd27158f120175824389ac9731 100755 --- a/python/paddle/fluid/tests/unittests/test_linear_interp_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_linear_interp_v2_op.py @@ -26,6 +26,7 @@ from paddle.nn.functional import interpolate def linear_interp_np(input, out_w, + scale_w=0, out_size=None, actual_shape=None, align_corners=True, @@ -44,7 +45,10 @@ def linear_interp_np(input, if (align_corners): ratio_w = (in_w - 1.0) / (out_w - 1.0) else: - ratio_w = 1.0 * in_w / out_w + if scale_w > 0: + ratio_w = 1.0 / scale_w + else: + ratio_w = 1.0 * in_w / out_w out = np.zeros((batch_size, channel, out_w)) @@ -81,6 +85,7 @@ class TestLinearInterpOp(OpTest): self.op_type = "linear_interp_v2" input_np = np.random.random(self.input_shape).astype("float64") + scale_w = 0 if self.data_layout == "NCHW": in_w = self.input_shape[2] else: @@ -95,7 +100,7 @@ class TestLinearInterpOp(OpTest): else: out_w = self.out_w - output_np = linear_interp_np(input_np, out_w, self.out_size, + output_np = linear_interp_np(input_np, out_w, self.scale, self.out_size, self.actual_shape, self.align_corners, self.align_mode, self.data_layout) self.inputs = {'X': input_np} @@ -195,7 +200,7 @@ class TestLinearInterpOpSizeTensor(TestLinearInterpOp): else: out_w = self.out_w - output_np = linear_interp_np(input_np, out_w, self.out_size, + output_np = linear_interp_np(input_np, out_w, 0, self.out_size, self.actual_shape, self.align_corners, self.align_mode, self.data_layout) @@ -309,7 +314,7 @@ class TestLinearInterpOpAPI2_0(unittest.TestCase): # dygraph x_data = np.random.random((1, 3, 128)).astype("float32") - us_1 = paddle.nn.UpSample( + us_1 = paddle.nn.Upsample( size=[64, ], mode='linear', align_mode=1, @@ -342,7 +347,7 @@ class TestResizeLinearOpUint8(OpTest): else: out_w = self.out_w - output_np = linear_interp_np(input_np, out_w, self.out_size, + output_np = linear_interp_np(input_np, out_w, 0, self.out_size, self.actual_shape, self.align_corners, self.align_mode) self.inputs = {'X': input_np} @@ -410,19 +415,19 @@ class TestLinearInterpOpError(unittest.TestCase): def input_shape_error(): x1 = fluid.data(name="x1", shape=[1], dtype="float32") - out1 = paddle.nn.UpSample( + out1 = paddle.nn.Upsample( size=[256, ], data_format='NCW', mode='linear') out1_res = out1(x1) def data_format_error(): x2 = fluid.data(name="x2", shape=[1, 3, 128], dtype="float32") - out2 = paddle.nn.UpSample( + out2 = paddle.nn.Upsample( size=[256, ], data_format='NHWCD', mode='linear') out2_res = out2(x2) def out_shape_error(): x3 = fluid.data(name="x3", shape=[1, 3, 128], dtype="float32") - out3 = paddle.nn.UpSample( + out3 = paddle.nn.Upsample( size=[ 256, 256, diff --git a/python/paddle/fluid/tests/unittests/test_logsumexp.py b/python/paddle/fluid/tests/unittests/test_logsumexp.py index c2201a52605bc87246fb9c8734494b19f83ff180..cf9203dffcbaa5da641b3f7cb8925ac9efcbe115 100644 --- a/python/paddle/fluid/tests/unittests/test_logsumexp.py +++ b/python/paddle/fluid/tests/unittests/test_logsumexp.py @@ -46,8 +46,8 @@ class TestLogsumexp(OpTest): self.inputs = {'X': x} self.outputs = {'Out': out} self.attrs = { - 'dim': self.axis, - 'keep_dim': self.keepdim, + 'axis': self.axis, + 'keepdim': self.keepdim, 'reduce_all': self.reduce_all } diff --git a/python/paddle/fluid/tests/unittests/test_math_op_patch.py b/python/paddle/fluid/tests/unittests/test_math_op_patch.py index 00137f63e244a0e166047e89f9ef436da158ed16..f6eff22d6ce5f06d8853d6244f79b4b07b3fa4f5 100644 --- a/python/paddle/fluid/tests/unittests/test_math_op_patch.py +++ b/python/paddle/fluid/tests/unittests/test_math_op_patch.py @@ -189,15 +189,15 @@ class TestMathOpPatches(unittest.TestCase): @prog_scope() def test_integer_div(self): a = fluid.layers.data(name="a", shape=[1], dtype='int64') - b = a / 2 + b = a / 7 place = fluid.CPUPlace() exe = fluid.Executor(place) - a_np = numpy.array([3, 4, 10, 14, 9, 18]) + a_np = numpy.array([3, 4, 10, 14, 9, 18]).astype('int64') b_np, = exe.run(fluid.default_main_program(), feed={"a": a_np}, fetch_list=[b]) - # for paddle2.0, use true_divide - b_np_actual = (a_np / 2.0) + + b_np_actual = (a_np / 7).astype('int64') self.assertTrue(numpy.array_equal(b_np, b_np_actual)) @prog_scope() diff --git a/python/paddle/fluid/tests/unittests/test_math_op_patch_var_base.py b/python/paddle/fluid/tests/unittests/test_math_op_patch_var_base.py index 9bb12d546550a821e8a133dd9c91d5d41a50b1b2..a70862f40197c513a0cd04753553264708ee2a1c 100644 --- a/python/paddle/fluid/tests/unittests/test_math_op_patch_var_base.py +++ b/python/paddle/fluid/tests/unittests/test_math_op_patch_var_base.py @@ -307,7 +307,7 @@ class TestMathOpPatchesVarBase(unittest.TestCase): np.array_equal(x.sigmoid().numpy(), fluid.layers.sigmoid(x).numpy( ))) self.assertTrue( - np.array_equal(x.logsigmoid().numpy(), + np.array_equal(x.log_sigmoid().numpy(), fluid.layers.logsigmoid(x).numpy())) self.assertTrue(np.array_equal(x.exp().numpy(), paddle.exp(x).numpy())) self.assertTrue( diff --git a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_exception.py b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_exception.py index 3a8867f6bd29f5bc0e512f9c8b22ecf192253fc7..6fd14b40bc9108b6075a0ac1f40cbefd79b8f0d9 100644 --- a/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_exception.py +++ b/python/paddle/fluid/tests/unittests/test_multiprocess_dataloader_exception.py @@ -27,6 +27,7 @@ import paddle.fluid.core as core from paddle.io import Dataset, IterableDataset, BatchSampler, DataLoader from paddle.fluid.dygraph.nn import Linear from paddle.fluid.dygraph.base import to_variable +from paddle.fluid.dataloader.dataloader_iter import _worker_loop class RandomDataset(Dataset): @@ -185,9 +186,10 @@ class TestDataLoaderWorkerLoop(unittest.TestCase): for i in range(10): indices_queue.put([i, i + 10]) indices_queue.put(None) - loader._worker_loop( - loader._dataset, 0, indices_queue, loader._data_queue, - loader._workers_done_event, _collate_fn, _init_fn, 0, 1) + _worker_loop(loader._dataset, 0, indices_queue, + loader._data_queue, loader._workers_done_event, + _collate_fn, _init_fn, 0, 1, + loader._use_shared_memory) self.assertTrue(False) except AssertionError: pass @@ -228,9 +230,10 @@ class TestDataLoaderWorkerLoop(unittest.TestCase): indices_queue.put([i, i + 10]) indices_queue.put(None) loader._workers_done_event.set() - loader._worker_loop( - loader._dataset, 0, indices_queue, loader._data_queue, - loader._workers_done_event, _collate_fn, _init_fn, 0, 1) + _worker_loop(loader._dataset, 0, indices_queue, + loader._data_queue, loader._workers_done_event, + _collate_fn, _init_fn, 0, 1, + loader._use_shared_memory) self.assertTrue(True) except AssertionError: pass diff --git a/python/paddle/fluid/tests/unittests/test_nearest_interp_v2_op.py b/python/paddle/fluid/tests/unittests/test_nearest_interp_v2_op.py index 19da09a463f3cc6224a22eb90278abae9ec59b91..2feca1c30689cec20e1d696cc672516414786038 100755 --- a/python/paddle/fluid/tests/unittests/test_nearest_interp_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_nearest_interp_v2_op.py @@ -26,6 +26,8 @@ import paddle def nearest_neighbor_interp_np(X, out_h, out_w, + scale_h=0, + scale_w=0, out_size=None, actual_shape=None, align_corners=True, @@ -46,13 +48,18 @@ def nearest_neighbor_interp_np(X, if (align_corners): ratio_h = (in_h - 1.0) / (out_h - 1.0) else: - ratio_h = 1.0 * in_h / out_h + if scale_h > 0: + ratio_h = 1.0 / scale_h + else: + ratio_h = 1.0 * in_h / out_h if (out_w > 1): if (align_corners): ratio_w = (in_w - 1.0) / (out_w - 1.0) else: - ratio_w = 1.0 * in_w / out_w - + if scale_w > 0: + ratio_w = 1.0 / scale_w + else: + ratio_w = 1.0 * in_w / out_w out = np.zeros((n, c, out_h, out_w)) if align_corners: @@ -89,7 +96,8 @@ class TestNearestInterpOp(OpTest): else: in_h = self.input_shape[1] in_w = self.input_shape[2] - + scale_h = 0 + scale_w = 0 if self.scale: if isinstance(self.scale, float) or isinstance(self.scale, int): if self.scale > 0: @@ -106,8 +114,8 @@ class TestNearestInterpOp(OpTest): out_w = self.out_w output_np = nearest_neighbor_interp_np( - input_np, out_h, out_w, self.out_size, self.actual_shape, - self.align_corners, self.data_layout) + input_np, out_h, out_w, scale_h, scale_w, self.out_size, + self.actual_shape, self.align_corners, self.data_layout) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -265,7 +273,7 @@ class TestNearestInterpOpUint8(OpTest): out_h = self.out_h out_w = self.out_w - output_np = nearest_neighbor_interp_np(input_np, out_h, out_w, + output_np = nearest_neighbor_interp_np(input_np, out_h, out_w, 0, 0, self.out_size, self.actual_shape, self.align_corners) self.inputs = {'X': input_np} @@ -408,7 +416,7 @@ class TestNearestInterpOp_attr_tensor(OpTest): if isinstance(self.scale, list) and len(self.scale) == 1: self.scale = [self.scale[0], self.scale[0]] self.attrs['scale'] = self.scale - output_np = nearest_neighbor_interp_np(input_np, out_h, out_w, + output_np = nearest_neighbor_interp_np(input_np, out_h, out_w, 0, 0, self.out_size, self.actual_shape, self.align_corners) self.outputs = {'Out': output_np} diff --git a/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_dygraph.py b/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_dygraph.py index e0edf9019356f38eb3c74b9cadfa6ae575e9b823..43a0d481b28fdc47dec52fe9763dd920fd5a76a2 100644 --- a/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_dygraph.py +++ b/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_dygraph.py @@ -16,20 +16,49 @@ from __future__ import print_function import unittest +import paddle +import paddle.nn as nn +import numpy as np + +paddle.disable_static() + class EmbeddingDygraph(unittest.TestCase): def test_1(self): - import paddle - import paddle.nn as nn - import numpy as np - paddle.disable_static() + x_data = np.arange(3, 6).reshape((3, 1)).astype(np.int64) + y_data = np.arange(6, 12).reshape((3, 2)).astype(np.float32) + paddle.disable_static(paddle.CPUPlace()) + x = paddle.to_tensor(x_data, stop_gradient=False) + y = paddle.to_tensor(y_data, stop_gradient=False) + + embedding = paddle.nn.Embedding(10, 3, sparse=True) + + w0 = np.full(shape=(10, 3), fill_value=2).astype(np.float32) + embedding.weight.set_value(w0) + + adam = paddle.optimizer.Adam( + parameters=[embedding.weight], learning_rate=0.01) + adam.clear_grad() + + out = embedding(x) + out.backward() + adam.step() + + def test_2(self): + x_data = np.arange(3, 6).reshape((3, 1)).astype(np.int64) + y_data = np.arange(6, 12).reshape((3, 2)).astype(np.float32) + paddle.disable_static(paddle.CPUPlace()) + x = paddle.to_tensor(x_data, stop_gradient=False) + y = paddle.to_tensor(y_data, stop_gradient=False) + + with self.assertRaises(ValueError): + embedding = paddle.nn.Embedding(10, 3, padding_idx=11, sparse=True) - # example 1 - inp_word = np.array([[2, 3, 5], [4, 2, 1]]).astype('int64') - inp_word.shape # [2, 3] - dict_size = 20 + with self.assertRaises(ValueError): + embedding = paddle.nn.Embedding(-1, 3, sparse=True) - emb = nn.Embedding(dict_size, 32, weight_attr='emb.w', sparse=False) + with self.assertRaises(ValueError): + embedding = paddle.nn.Embedding(10, -3, sparse=True) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_static.py b/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_static.py index c9c91ceb39de42c44f9ce81658aa79b896999552..4af0cce12b7334857c54ae9e8e9418848275ff32 100644 --- a/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_static.py +++ b/python/paddle/fluid/tests/unittests/test_nn_functional_embedding_static.py @@ -73,8 +73,13 @@ class EmbeddingStatic(unittest.TestCase): dtype="int32") emb = functional.embedding( - x=label, weight=weight, sparse=True, name="embedding") + x=label, + weight=weight, + padding_idx=129, + sparse=True, + name="embedding") + with self.assertRaises(ValueError): test_bad_x() diff --git a/python/paddle/fluid/tests/unittests/test_nn_grad.py b/python/paddle/fluid/tests/unittests/test_nn_grad.py index 0c39dc5e731d25720149af4480020a7ab3ac5bb9..5d1e016287e07a8505336e6cb447c0e1b29a2ec2 100644 --- a/python/paddle/fluid/tests/unittests/test_nn_grad.py +++ b/python/paddle/fluid/tests/unittests/test_nn_grad.py @@ -17,6 +17,7 @@ from __future__ import print_function import unittest import numpy as np +import paddle import paddle.fluid as fluid import paddle.fluid.layers as layers import paddle.fluid.core as core @@ -153,6 +154,30 @@ class TestMulDoubleGradCheck(unittest.TestCase): class TestReshapeDoubleGradCheck(unittest.TestCase): + @prog_scope() + def func(self, place): + x_shape = [3, 12] + expand_times = [4, 9] + eps = 0.005 + dtype = np.float64 + + x = layers.data('x', x_shape, False, dtype) + x.persistable = True + out = layers.expand(x, expand_times) + x_arr = np.random.uniform(-1, 1, x_shape).astype(dtype) + + gradient_checker.double_grad_check( + [x], out, x_init=x_arr, place=place, eps=eps) + + def test_grad(self): + places = [fluid.CPUPlace()] + if core.is_compiled_with_cuda(): + places.append(fluid.CUDAPlace(0)) + for p in places: + self.func(p) + + +class TestExpandDoubleGradCheck(unittest.TestCase): @prog_scope() def func(self, place): x_shape = [3, 12] @@ -176,5 +201,53 @@ class TestReshapeDoubleGradCheck(unittest.TestCase): self.func(p) +class TestTileDoubleGradCheck(unittest.TestCase): + @prog_scope() + def func(self, place): + x_shape = [3, 12] + repeat_times = [4, 9] + eps = 0.005 + dtype = np.float64 + + x = layers.data('x', x_shape, False, dtype) + x.persistable = True + out = paddle.tile(x, repeat_times) + x_arr = np.random.uniform(-1, 1, x_shape).astype(dtype) + + gradient_checker.double_grad_check( + [x], out, x_init=x_arr, place=place, eps=eps) + + def test_grad(self): + places = [fluid.CPUPlace()] + if core.is_compiled_with_cuda(): + places.append(fluid.CUDAPlace(0)) + for p in places: + self.func(p) + + +class TestExpandV2DoubleGradCheck(unittest.TestCase): + @prog_scope() + def func(self, place): + x_shape = [1, 12] + new_shape = [4, 12] + eps = 0.005 + dtype = np.float64 + + x = layers.data('x', x_shape, False, dtype) + x.persistable = True + out = paddle.expand(x, new_shape) + x_arr = np.random.uniform(-1, 1, x_shape).astype(dtype) + + gradient_checker.double_grad_check( + [x], out, x_init=x_arr, place=place, eps=eps) + + def test_grad(self): + places = [fluid.CPUPlace()] + if core.is_compiled_with_cuda(): + places.append(fluid.CUDAPlace(0)) + for p in places: + self.func(p) + + if __name__ == "__main__": unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_norm_all.py b/python/paddle/fluid/tests/unittests/test_norm_all.py index c047cf6ddff78641b918de75a284574175bb3bca..352089e1fb75fa4c3423d29012fd85c3d611c81b 100644 --- a/python/paddle/fluid/tests/unittests/test_norm_all.py +++ b/python/paddle/fluid/tests/unittests/test_norm_all.py @@ -26,11 +26,11 @@ def p_norm(x, axis, porder, keepdims=False): if axis is None: x = x.flatten() if porder == np.inf: - r = np.amax(np.abs(x)) + r = np.amax(np.abs(x), keepdims=keepdims) elif porder == -np.inf: - r = np.amin(np.abs(x)) + r = np.amin(np.abs(x), keepdims=keepdims) else: - r = np.linalg.norm(x, ord=porder) + r = np.linalg.norm(x, ord=porder, keepdims=keepdims) elif isinstance(axis, list or tuple) and len(axis) == 2: if porder == np.inf: axis = tuple(axis) @@ -41,10 +41,10 @@ def p_norm(x, axis, porder, keepdims=False): elif porder == 0: axis = tuple(axis) r = x.astype(bool) - r = np.sum(r, axis) + r = np.sum(r, axis, keepdims=keepdims) elif porder == 1: axis = tuple(axis) - r = np.sum(np.abs(x), axis) + r = np.sum(np.abs(x), axis, keepdims=keepdims) else: axis = tuple(axis) xp = np.power(np.abs(x), porder) @@ -61,7 +61,7 @@ def p_norm(x, axis, porder, keepdims=False): def frobenius_norm(x, axis=None, keepdims=False): if isinstance(axis, list): axis = tuple(axis) - if axis is None: axis = (-2, -1) + if axis is None: x = x.reshape(1, x.size) r = np.linalg.norm( x, ord='fro', axis=axis, keepdims=keepdims).astype(x.dtype) return r @@ -217,28 +217,37 @@ class TestPnormOp5(TestPnormOp): self.check_grad(['X'], 'Out', user_defined_grads=self.gradient) -def run_fro(self, p, axis, shape_x, dtype): +def run_fro(self, p, axis, shape_x, dtype, keep_dim, check_dim=False): with fluid.program_guard(fluid.Program()): data = fluid.data(name="X", shape=shape_x, dtype=dtype) - out = paddle.norm(x=data, p=p, axis=axis) + out = paddle.norm(x=data, p=p, axis=axis, keepdim=keep_dim) place = fluid.CPUPlace() exe = fluid.Executor(place) np_input = (np.random.rand(*shape_x) + 1.0).astype(dtype) - expected_result = frobenius_norm(np_input, axis=axis) + expected_result = frobenius_norm(np_input, axis=axis, keepdims=keep_dim) result, = exe.run(feed={"X": np_input}, fetch_list=[out]) self.assertEqual((np.abs(result - expected_result) < 1e-6).all(), True) + if keep_dim and check_dim: + self.assertEqual( + (np.abs(np.array(result.shape) - np.array(expected_result.shape)) < + 1e-6).all(), True) -def run_pnorm(self, p, axis, shape_x, dtype): +def run_pnorm(self, p, axis, shape_x, dtype, keep_dim, check_dim=False): with fluid.program_guard(fluid.Program()): data = fluid.data(name="X", shape=shape_x, dtype=dtype) - out = paddle.norm(x=data, p=p, axis=axis) + out = paddle.norm(x=data, p=p, axis=axis, keepdim=keep_dim) place = fluid.CPUPlace() exe = fluid.Executor(place) np_input = (np.random.rand(*shape_x) + 1.0).astype(dtype) - expected_result = p_norm(np_input, porder=p, axis=axis).astype(dtype) + expected_result = p_norm( + np_input, porder=p, axis=axis, keepdims=keep_dim).astype(dtype) result, = exe.run(feed={"X": np_input}, fetch_list=[out]) - self.assertEqual((np.abs(result - expected_result) < 1e-6).all(), True) + self.assertEqual((np.abs(result - expected_result) < 1e-6).all(), True) + if keep_dim and check_dim: + self.assertEqual( + (np.abs(np.array(result.shape) - np.array(expected_result.shape)) < + 1e-6).all(), True) def run_graph(self, p, axis, shape_x, dtype): @@ -253,6 +262,7 @@ def run_graph(self, p, axis, shape_x, dtype): # compute frobenius norm along last two dimensions. out_fro = paddle.norm(x, p='fro') + out_fro = paddle.norm(x, p='fro', axis=0) out_fro = paddle.norm(x, p='fro', axis=[0, 1]) # compute 2-order norm along [0,1] dimension. out_pnorm = paddle.norm(x, p=2, axis=[0, 1]) @@ -274,27 +284,133 @@ def run_graph(self, p, axis, shape_x, dtype): class API_NormTest(unittest.TestCase): def test_basic(self): - run_fro(self, p='fro', axis=None, shape_x=[2, 3, 4], dtype="float32") - run_fro(self, p='fro', axis=[0, 1], shape_x=[2, 3, 4], dtype="float64") - run_pnorm(self, p=2, axis=None, shape_x=[3, 4], dtype="float32") - run_pnorm(self, p=2, axis=1, shape_x=[3, 4], dtype="float64") - run_pnorm(self, p=np.inf, axis=0, shape_x=[2, 3, 4], dtype="float32") - run_pnorm(self, p=np.inf, axis=None, shape_x=[2, 3, 4], dtype="float32") - run_pnorm(self, p=-np.inf, axis=0, shape_x=[2, 3, 4], dtype="float64") - run_pnorm( - self, p=-np.inf, axis=None, shape_x=[2, 3, 4], dtype="float64") - run_pnorm(self, p=0, axis=1, shape_x=[3, 4], dtype="float64") - - run_pnorm(self, p=1, axis=1, shape_x=[3, 4], dtype="float64") - run_pnorm(self, p=0, axis=None, shape_x=[3, 4], dtype="float64") - run_pnorm(self, p=2, axis=[0, 1], shape_x=[2, 3, 4], dtype="float64") - run_pnorm(self, p=2, axis=-1, shape_x=[2, 3, 4], dtype="float64") - run_pnorm(self, p=1, axis=[0, 1], shape_x=[2, 3, 4], dtype="float64") - run_pnorm(self, p=0, axis=[0, 1], shape_x=[2, 3, 4], dtype="float64") - run_pnorm( - self, p=np.inf, axis=[0, 1], shape_x=[2, 3, 4], dtype="float64") - run_pnorm( - self, p=-np.inf, axis=[0, 1], shape_x=[2, 3, 4], dtype="float64") + keep_dims = {False, True} + for keep in keep_dims: + run_fro( + self, + p='fro', + axis=None, + shape_x=[2, 3, 4], + dtype="float32", + keep_dim=keep) + run_fro( + self, + p='fro', + axis=[0, 1], + shape_x=[2, 3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True) + run_pnorm( + self, + p=2, + axis=None, + shape_x=[3, 4], + dtype="float32", + keep_dim=keep) + run_pnorm( + self, + p=2, + axis=1, + shape_x=[3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True) + run_pnorm( + self, + p=np.inf, + axis=0, + shape_x=[2, 3, 4], + dtype="float32", + keep_dim=keep, + check_dim=True) + run_pnorm( + self, + p=np.inf, + axis=None, + shape_x=[2, 3, 4], + dtype="float32", + keep_dim=keep) + run_pnorm( + self, + p=-np.inf, + axis=0, + shape_x=[2, 3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True) + run_pnorm( + self, + p=-np.inf, + axis=None, + shape_x=[2, 3, 4], + dtype="float64", + keep_dim=keep) + run_pnorm( + self, + p=0, + axis=1, + shape_x=[3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True) + + run_pnorm( + self, + p=1, + axis=1, + shape_x=[3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True) + run_pnorm( + self, + p=0, + axis=None, + shape_x=[3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True) + run_pnorm( + self, + p=2, + axis=[0, 1], + shape_x=[2, 3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True) + run_pnorm( + self, + p=2, + axis=-1, + shape_x=[2, 3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True) + run_pnorm( + self, + p=1, + axis=[0, 1], + shape_x=[2, 3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True) + run_pnorm( + self, + p=np.inf, + axis=[0, 1], + shape_x=[2, 3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True) + run_pnorm( + self, + p=-np.inf, + axis=[0, 1], + shape_x=[2, 3, 4], + dtype="float64", + keep_dim=keep, + check_dim=True) def test_dygraph(self): run_graph(self, p='fro', axis=None, shape_x=[2, 3, 4], dtype="float32") @@ -315,6 +431,7 @@ class API_NormTest(unittest.TestCase): paddle.norm(data, p=p, out=out) self.assertRaises(TypeError, err_dtype, "fro", [2, 2], "int64") + self.assertRaises(ValueError, paddle.norm, "inf", [2], "int64") out = fluid.data(name="out", shape=[1], dtype="int64") self.assertRaises(TypeError, err_dtype, "fro", [2, 2], "float64", out) @@ -325,6 +442,7 @@ class API_NormTest(unittest.TestCase): self.assertRaises(ValueError, paddle.norm, data, p="unsupport norm") self.assertRaises(ValueError, paddle.norm, data, p=[1]) self.assertRaises(ValueError, paddle.norm, data, p=[1], axis=-1) + self.assertRaises(ValueError, paddle.norm, 0, [1, 0], "float64") data = fluid.data(name="data_3d", shape=[2, 2, 2], dtype="float64") self.assertRaises( ValueError, paddle.norm, data, p='unspport', axis=[-3, -2, -1]) diff --git a/python/paddle/fluid/tests/unittests/test_optimizer_in_control_flow.py b/python/paddle/fluid/tests/unittests/test_optimizer_in_control_flow.py index 4b2914c223a08c52444e085f0ef9e41518694593..c1992d0d539a5c6499b9b8d022b88997729ef782 100644 --- a/python/paddle/fluid/tests/unittests/test_optimizer_in_control_flow.py +++ b/python/paddle/fluid/tests/unittests/test_optimizer_in_control_flow.py @@ -261,7 +261,13 @@ class TestMultiOptimizersMultiCardsError(unittest.TestCase): exe.run(startup_program) np.random.seed(SEED) + + # NOTE(liym27): + # This test needs to run in multi cards to test NotImplementedError. + # Here, move this test from RUN_TYPE=DIST in tests/unittests/CMakeList.txt, + # to use multi cards ** only on CPU ** not GPU to reduce CI time. os.environ['CPU_NUM'] = str(2) + pe_exe = fluid.ParallelExecutor( use_cuda=use_cuda, main_program=main_program, diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py index 6671a2def3cccd2acd76025e73486b06b4bb1471..ea59a7f584a2dd5a06d37ede160ace130fc93580 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py @@ -176,7 +176,7 @@ class TestCRFModel(unittest.TestCase): place=fluid.CPUPlace()) data = train_data() - for i in range(10): + for i in range(4): cur_batch = next(data) print(exe.run(train_cp, feed=feeder.feed(cur_batch), diff --git a/python/paddle/fluid/tests/unittests/test_pool3d_api.py b/python/paddle/fluid/tests/unittests/test_pool3d_api.py index a77f1cdd57d7bade92e2a4f914dc3d91624d4845..505a1c738384194032329f66c33fa27e3ed3045c 100644 --- a/python/paddle/fluid/tests/unittests/test_pool3d_api.py +++ b/python/paddle/fluid/tests/unittests/test_pool3d_api.py @@ -165,7 +165,6 @@ class TestPool3d_API(unittest.TestCase): self.assertTrue(np.allclose(result.numpy(), result_np)) def check_max_dygraph_ndhwc_results(self, place): - print("run ndchw max pool3d") with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32, 32, 32]).astype("float32") input = fluid.dygraph.to_variable( @@ -190,7 +189,6 @@ class TestPool3d_API(unittest.TestCase): np.transpose(result.numpy(), [0, 4, 1, 2, 3]), result_np)) def check_max_dygraph_ceilmode_results(self, place): - print("run ceil mode max pool3d") with fluid.dygraph.guard(place): input_np = np.random.random([2, 3, 32, 32, 32]).astype("float32") input = fluid.dygraph.to_variable(input_np) diff --git a/python/paddle/fluid/tests/unittests/test_rnn_decode_api.py b/python/paddle/fluid/tests/unittests/test_rnn_decode_api.py index 7e2ef36c1a7fda5c31049ec9c752c5226bfb89dc..6ca194b2694b6c7537ceb94e11eb1a1a0aeb8d8d 100644 --- a/python/paddle/fluid/tests/unittests/test_rnn_decode_api.py +++ b/python/paddle/fluid/tests/unittests/test_rnn_decode_api.py @@ -248,8 +248,7 @@ class PolicyGradient(object): func=reward_func, x=[action, length], out=reward) neg_log_prob = layers.cross_entropy(act_prob, action) cost = neg_log_prob * reward - cost = (layers.reduce_sum(cost) / - layers.cast(layers.reduce_sum(length), "float32") + cost = (layers.reduce_sum(cost) / layers.reduce_sum(length) ) if length is not None else layers.reduce_mean(cost) optimizer = fluid.optimizer.Adam(self.lr) optimizer.minimize(cost) diff --git a/python/paddle/fluid/tests/unittests/test_roi_align_op.py b/python/paddle/fluid/tests/unittests/test_roi_align_op.py index b01863880866e247f2aee4b94ae3121c9d891f92..fb8a090b80700d9b884a72f7f430723754523a13 100644 --- a/python/paddle/fluid/tests/unittests/test_roi_align_op.py +++ b/python/paddle/fluid/tests/unittests/test_roi_align_op.py @@ -181,16 +181,11 @@ class TestROIAlignInLodOp(TestROIAlignOp): self.calc_roi_align() seq_len = self.rois_lod[0] - cur_len = 0 - lod = [cur_len] - for l in seq_len: - cur_len += l - lod.append(cur_len) self.inputs = { 'X': self.x, 'ROIs': (self.rois[:, 1:5], self.rois_lod), - 'RoisLod': np.asarray(lod).astype('int64') + 'RoisNum': np.asarray(seq_len).astype('int32') } self.attrs = { diff --git a/python/paddle/fluid/tests/unittests/test_roi_pool_op.py b/python/paddle/fluid/tests/unittests/test_roi_pool_op.py index 1200b0e3470f650dce4365ee46458c8184281292..c6622cf8d9ce8ae655a6b2e5c130ed9990fd2a5b 100644 --- a/python/paddle/fluid/tests/unittests/test_roi_pool_op.py +++ b/python/paddle/fluid/tests/unittests/test_roi_pool_op.py @@ -174,16 +174,11 @@ class TestROIPoolInLodOp(TestROIPoolOp): self.calc_roi_pool() seq_len = self.rois_lod[0] - cur_len = 0 - lod = [cur_len] - for l in seq_len: - cur_len += l - lod.append(cur_len) self.inputs = { 'X': self.x, 'ROIs': (self.rois[:, 1:5], self.rois_lod), - 'RoisLod': np.asarray(lod).astype('int64') + 'RoisNum': np.asarray(seq_len).astype('int32') } self.attrs = { diff --git a/python/paddle/fluid/tests/unittests/test_sync_batch_norm_op.py b/python/paddle/fluid/tests/unittests/test_sync_batch_norm_op.py index 09cd40d9cc59914c82cc343bb78b72fbc2b29e59..1c11e831b0ad31a3c450c70e7f7c258455409d05 100644 --- a/python/paddle/fluid/tests/unittests/test_sync_batch_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_sync_batch_norm_op.py @@ -227,14 +227,15 @@ class TestConvertSyncBatchNorm(unittest.TestCase): return with program_guard(Program(), Program()): + compare_model = paddle.nn.Sequential( + paddle.nn.Conv2d(3, 5, 3), paddle.nn.BatchNorm2d(5)) model = paddle.nn.Sequential( paddle.nn.Conv2d(3, 5, 3), paddle.nn.BatchNorm2d(5)) - sync_model = paddle.nn.SyncBatchNorm.convert_sync_batchnorm(model) - for idx, sublayer in enumerate(model.sublayers()): + model = paddle.nn.SyncBatchNorm.convert_sync_batchnorm(model) + for idx, sublayer in enumerate(compare_model.sublayers()): if isinstance(sublayer, paddle.nn.BatchNorm2d): self.assertEqual( - isinstance(sync_model[idx], paddle.nn.SyncBatchNorm), - True) + isinstance(model[idx], paddle.nn.SyncBatchNorm), True) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_tril_triu_op.py b/python/paddle/fluid/tests/unittests/test_tril_triu_op.py index aed265b21b5781d88da0380b04872061e893d736..2cd2599f2ea2f4fb26b2d2730ca45384a3b664a7 100644 --- a/python/paddle/fluid/tests/unittests/test_tril_triu_op.py +++ b/python/paddle/fluid/tests/unittests/test_tril_triu_op.py @@ -142,6 +142,18 @@ class TestTrilTriuOpAPI(unittest.TestCase): self.assertTrue(np.allclose(tril_out, np.tril(data))) self.assertTrue(np.allclose(triu_out, np.triu(data))) + def test_fluid_api(self): + data = np.random.random([1, 9, 9, 4]).astype('float32') + x = fluid.data(shape=[1, 9, -1, 4], dtype='float32', name='x') + triu_out = fluid.layers.triu(x) + + place = fluid.CUDAPlace(0) if fluid.core.is_compiled_with_cuda( + ) else fluid.CPUPlace() + exe = fluid.Executor(place) + triu_out = exe.run(fluid.default_main_program(), + feed={"x": data}, + fetch_list=[triu_out]) + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_trilinear_interp_v2_op.py b/python/paddle/fluid/tests/unittests/test_trilinear_interp_v2_op.py index 49924b44441aa9ae323f0d7921d71bf58b8c2cf2..245c2623b869af30acfb5d0379c7597813645031 100755 --- a/python/paddle/fluid/tests/unittests/test_trilinear_interp_v2_op.py +++ b/python/paddle/fluid/tests/unittests/test_trilinear_interp_v2_op.py @@ -26,6 +26,9 @@ def trilinear_interp_np(input, out_d, out_h, out_w, + scale_d=0, + scale_h=0, + scale_w=0, out_size=None, actual_shape=None, align_corners=True, @@ -49,17 +52,26 @@ def trilinear_interp_np(input, if (align_corners): ratio_d = (in_d - 1.0) / (out_d - 1.0) else: - ratio_d = 1.0 * in_d / out_d + if scale_d > 0: + ratio_d = 1.0 / scale_d + else: + ratio_d = 1.0 * in_d / out_d if out_h > 1: if (align_corners): ratio_h = (in_h - 1.0) / (out_h - 1.0) else: - ratio_h = 1.0 * in_h / out_h + if scale_h > 0: + ratio_h = 1.0 / scale_h + else: + ratio_h = 1.0 * in_h / out_h if out_w > 1: if (align_corners): ratio_w = (in_w - 1.0) / (out_w - 1.0) else: - ratio_w = 1.0 * in_w / out_w + if scale_w > 0: + ratio_w = 1.0 / scale_w + else: + ratio_w = 1.0 * in_w / out_w out = np.zeros((batch_size, channel, out_d, out_h, out_w)) @@ -133,6 +145,9 @@ class TestTrilinearInterpOp(OpTest): self.op_type = "trilinear_interp_v2" input_np = np.random.random(self.input_shape).astype("float32") + scale_w = 0 + scale_h = 0 + scale_d = 0 if self.data_layout == "NCDHW": in_d = self.input_shape[2] in_h = self.input_shape[3] @@ -159,9 +174,10 @@ class TestTrilinearInterpOp(OpTest): out_h = self.out_h out_w = self.out_w - output_np = trilinear_interp_np( - input_np, out_d, out_h, out_w, self.out_size, self.actual_shape, - self.align_corners, self.align_mode, self.data_layout) + output_np = trilinear_interp_np(input_np, out_d, out_h, out_w, scale_d, + scale_h, scale_w, self.out_size, + self.actual_shape, self.align_corners, + self.align_mode, self.data_layout) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size @@ -359,7 +375,7 @@ class TestTrilinearInterpOpUint8(OpTest): out_h = self.out_h out_w = self.out_w - output_np = trilinear_interp_np(input_np, out_d, out_h, out_w, + output_np = trilinear_interp_np(input_np, out_d, out_h, out_w, 0, 0, 0, self.out_size, self.actual_shape, self.align_corners, self.align_mode) self.inputs = {'X': input_np} @@ -482,7 +498,7 @@ class TestTrilinearInterpZero(TestTrilinearInterpOp): self.out_d = 60 self.out_h = 40 self.out_w = 25 - self.scale = 0.2 + self.scale = 0.0 self.align_corners = False self.align_mode = 0 @@ -541,7 +557,7 @@ class TestTrilinearInterpOp_attr_tensor(OpTest): if isinstance(self.scale, list) and len(self.scale) == 1: self.scale = [self.scale[0], self.scale[0], self.scale[0]] self.attrs['scale'] = self.scale - output_np = trilinear_interp_np(input_np, out_d, out_h, out_w, + output_np = trilinear_interp_np(input_np, out_d, out_h, out_w, 0, 0, 0, self.out_size, self.actual_shape, self.align_corners, self.align_mode) self.outputs = {'Out': output_np} diff --git a/python/paddle/fluid/tests/unittests/test_unsqueeze_op.py b/python/paddle/fluid/tests/unittests/test_unsqueeze_op.py index 9382d53e7fec6ba9e1217f99ba5006b3dfe5c150..1975e4306026ee459aa585c47afa74fce6a6aeed 100644 --- a/python/paddle/fluid/tests/unittests/test_unsqueeze_op.py +++ b/python/paddle/fluid/tests/unittests/test_unsqueeze_op.py @@ -134,29 +134,61 @@ class API_TestUnsqueeze3(unittest.TestCase): result1, = exe.run(feed={"data1": input, "data2": input2}, fetch_list=[result_squeeze]) - self.assertTrue(np.allclose(input1, result1)) + self.assertTrue(np.array_equal(input1, result1)) + self.assertEqual(input1.shape, result1.shape) class API_TestDyUnsqueeze(unittest.TestCase): def test_out(self): with fluid.dygraph.guard(): input_1 = np.random.random([5, 1, 10]).astype("int32") - input1 = np.squeeze(input_1, axis=1) + input1 = np.expand_dims(input_1, axis=1) input = fluid.dygraph.to_variable(input_1) output = paddle.unsqueeze(input, axis=[1]) out_np = output.numpy() - self.assertTrue(np.allclose(input1, out_np)) + self.assertTrue(np.array_equal(input1, out_np)) + self.assertEqual(input1.shape, out_np.shape) class API_TestDyUnsqueeze2(unittest.TestCase): def test_out(self): with fluid.dygraph.guard(): - input_1 = np.random.random([5, 1, 10]).astype("int32") - input1 = np.squeeze(input_1, axis=1) - input = fluid.dygraph.to_variable(input_1) + input1 = np.random.random([5, 10]).astype("int32") + out1 = np.expand_dims(input1, axis=1) + input = fluid.dygraph.to_variable(input1) output = paddle.unsqueeze(input, axis=1) out_np = output.numpy() - self.assertTrue(np.allclose(input1, out_np)) + self.assertTrue(np.array_equal(out1, out_np)) + self.assertEqual(out1.shape, out_np.shape) + + +class API_TestDyUnsqueezeAxisTensor(unittest.TestCase): + def test_out(self): + with fluid.dygraph.guard(): + input1 = np.random.random([5, 10]).astype("int32") + out1 = np.expand_dims(input1, axis=1) + out1 = np.expand_dims(out1, axis=2) + input = fluid.dygraph.to_variable(input1) + output = paddle.unsqueeze(input, axis=paddle.to_tensor([1, 2])) + out_np = output.numpy() + self.assertTrue(np.array_equal(out1, out_np)) + self.assertEqual(out1.shape, out_np.shape) + + +class API_TestDyUnsqueezeAxisTensorList(unittest.TestCase): + def test_out(self): + with fluid.dygraph.guard(): + input1 = np.random.random([5, 10]).astype("int32") + # Actually, expand_dims supports tuple since version 1.18.0 + out1 = np.expand_dims(input1, axis=1) + out1 = np.expand_dims(out1, axis=2) + input = fluid.dygraph.to_variable(input1) + output = paddle.unsqueeze( + fluid.dygraph.to_variable(input1), + axis=[paddle.to_tensor([1]), paddle.to_tensor([2])]) + out_np = output.numpy() + self.assertTrue(np.array_equal(out1, out_np)) + self.assertEqual(out1.shape, out_np.shape) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_update_loss_scaling_op.py b/python/paddle/fluid/tests/unittests/test_update_loss_scaling_op.py new file mode 100644 index 0000000000000000000000000000000000000000..fb93334415c3046362090a143f6c15069793709a --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_update_loss_scaling_op.py @@ -0,0 +1,250 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import numpy as np +from op_test import OpTest +import paddle.fluid as fluid +import paddle.fluid.contrib.mixed_precision.amp_nn as amp_nn + + +class TestUpdateLossScalingOp(OpTest): + def setUp(self): + self.op_type = "update_loss_scaling" + self.init() + found_inf = np.array([False], dtype=np.bool) + x = np.random.random((1024, 1024)).astype(self.dtype) + + self.inputs = { + 'X': [('x0', x)], + 'FoundInfinite': found_inf, + 'PrevLossScaling': self.prev_loss_scaling, + 'InGoodSteps': self.num_good_steps, + 'InBadSteps': self.num_bad_steps + } + + self.outputs = { + 'Out': [('out0', np.zeros_like(x))], + 'LossScaling': self.prev_loss_scaling * self.incr_ratio, + 'OutGoodSteps': self.zero_steps, + 'OutBadSteps': self.zero_steps + } + + def init(self): + self.incr_ratio = 2.0 + self.decr_ratio = 0.8 + self.dtype = np.float32 + self.prev_loss_scaling = np.array([2048]).astype(self.dtype) + self.num_good_steps = np.array([999], dtype=np.int32) + self.num_bad_steps = np.array([1], dtype=np.int32) + self.zero_steps = np.array([0], dtype=np.int32) + self.attrs = { + 'incr_every_n_steps': 1000, + 'decr_every_n_nan_or_inf': 2, + 'incr_ratio': self.incr_ratio, + 'decr_ratio': self.decr_ratio, + } + + def test_check_output(self): + self.check_output(no_check_set=['Out']) + + +class TestUpdateLossScalingOpBad(TestUpdateLossScalingOp): + def setUp(self): + self.op_type = "update_loss_scaling" + self.init() + found_inf = np.array([True], dtype=np.bool) + x = np.random.random((1024, 1024)).astype(self.dtype) + i = np.random.randint(0, 1024, 1) + j = np.random.randint(0, 1024, 1) + x[i[0]][j[0]] = np.inf + + self.inputs = { + 'X': [('x0', x)], + 'FoundInfinite': found_inf, + 'PrevLossScaling': self.prev_loss_scaling, + 'InGoodSteps': self.num_good_steps, + 'InBadSteps': self.num_bad_steps + } + + self.outputs = { + 'Out': [('out0', np.zeros_like(x))], + 'LossScaling': self.prev_loss_scaling * self.decr_ratio, + 'OutGoodSteps': self.zero_steps, + 'OutBadSteps': self.zero_steps + } + + def test_check_output(self): + self.check_output() + + +class TestUpdateLossScalingLayer(unittest.TestCase): + def loss_scaling_check(self, use_cuda=True, scope=fluid.Scope()): + a = fluid.data(name="a", shape=[1024, 1024], dtype='float32') + b = fluid.data(name="b", shape=[512, 128], dtype='float32') + x = [a, b] + found_inf = fluid.data(name="found_inf", shape=[1], dtype='bool') + prev_loss_scaling = fluid.data( + name="prev_loss_scaling", shape=[1], dtype='float32') + num_good_steps = fluid.data( + name="num_good_steps", shape=[1], dtype='int32') + num_bad_steps = fluid.data( + name="num_bad_steps", shape=[1], dtype='int32') + + a_v = np.random.random([1024, 1024]).astype('float32') + b_v = np.random.random([512, 128]).astype('float32') + found_inf_v = np.array([False]).astype('bool') + prev_loss_scaling_v = np.array([2048]).astype('float32') + num_good_steps_v = np.array([999], dtype=np.int32) + num_bad_steps_v = np.array([1], dtype=np.int32) + + incr_every_n_steps = 1000 + decr_every_n_nan_or_inf = 2 + incr_ratio = 2 + decr_ratio = 0.8 + + result = amp_nn.update_loss_scaling( + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + incr_every_n_steps, + decr_every_n_nan_or_inf, + incr_ratio, + decr_ratio, + name="update_loss_scaling") + + place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() + exe = fluid.Executor(place) + with fluid.scope_guard(scope): + exe.run(fluid.default_startup_program()) + result_v = exe.run(feed={ + 'a': a_v, + 'b': b_v, + 'found_inf': found_inf_v, + 'prev_loss_scaling': prev_loss_scaling_v, + 'num_good_steps': num_good_steps_v, + 'num_bad_steps': num_bad_steps_v + }, + fetch_list=[ + result, x, found_inf, prev_loss_scaling, + num_good_steps, num_bad_steps + ]) + assert np.array_equal(result_v[0], a_v) + assert np.array_equal(result_v[1], b_v) + assert np.array_equal(result_v[0], result_v[2]) + assert np.array_equal(result_v[1], result_v[3]) + assert np.array_equal(result_v[4], found_inf_v) + assert np.array_equal(result_v[5], prev_loss_scaling_v * incr_ratio) + assert np.array_equal(result_v[6], np.zeros_like(num_good_steps_v)) + assert np.array_equal(result_v[7], np.zeros_like(num_bad_steps_v)) + + def loss_scaling_check_inf(self, use_cuda=True, scope=fluid.Scope()): + a = fluid.data(name="a", shape=[1024, 1024], dtype='float32') + b = fluid.data(name="b", shape=[512, 128], dtype='float32') + x = [a, b] + found_inf = fluid.data(name="found_inf", shape=[1], dtype='bool') + prev_loss_scaling = fluid.data( + name="prev_loss_scaling", shape=[1], dtype='float32') + num_good_steps = fluid.data( + name="num_good_steps", shape=[1], dtype='int32') + num_bad_steps = fluid.data( + name="num_bad_steps", shape=[1], dtype='int32') + + a_v = np.random.random([1024, 1024]).astype('float32') + b_v = np.random.random([512, 128]).astype('float32') + i = np.random.randint(0, 1024, 1) + j = np.random.randint(0, 1024, 1) + a_v[i[0]][j[0]] = np.inf + found_inf_v = np.array([True]).astype('bool') + prev_loss_scaling_v = np.array([2048]).astype('float32') + num_good_steps_v = np.array([999], dtype=np.int32) + num_bad_steps_v = np.array([1], dtype=np.int32) + + incr_every_n_steps = 1000 + decr_every_n_nan_or_inf = 2 + incr_ratio = 2 + decr_ratio = 0.8 + + result = amp_nn.update_loss_scaling( + x, + found_inf, + prev_loss_scaling, + num_good_steps, + num_bad_steps, + incr_every_n_steps, + decr_every_n_nan_or_inf, + incr_ratio, + decr_ratio, + name="update_loss_scaling") + + place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() + exe = fluid.Executor(place) + with fluid.scope_guard(scope): + exe.run(fluid.default_startup_program()) + result_v = exe.run(feed={ + 'a': a_v, + 'b': b_v, + 'found_inf': found_inf_v, + 'prev_loss_scaling': prev_loss_scaling_v, + 'num_good_steps': num_good_steps_v, + 'num_bad_steps': num_bad_steps_v + }, + fetch_list=[ + result, x, found_inf, prev_loss_scaling, + num_good_steps, num_bad_steps + ]) + assert np.array_equal(result_v[0], np.zeros_like(a_v)) + assert np.array_equal(result_v[1], np.zeros_like(b_v)) + assert np.array_equal(result_v[2], np.zeros_like(a_v)) + assert np.array_equal(result_v[3], np.zeros_like(b_v)) + assert np.array_equal(result_v[4], found_inf_v) + assert np.array_equal(result_v[5], prev_loss_scaling_v * decr_ratio) + assert np.array_equal(result_v[6], np.zeros_like(num_good_steps_v)) + assert np.array_equal(result_v[7], np.zeros_like(num_bad_steps_v)) + + def test_loss_scaling_cpu(self): + main = fluid.Program() + startup = fluid.Program() + with fluid.unique_name.guard(): + with fluid.program_guard(main, startup): + self.loss_scaling_check(use_cuda=False) + + def test_loss_scaling_cpu_inf(self): + main = fluid.Program() + startup = fluid.Program() + with fluid.unique_name.guard(): + with fluid.program_guard(main, startup): + self.loss_scaling_check_inf(use_cuda=False) + + def test_loss_scaling_gpu(self): + if fluid.core.is_compiled_with_cuda(): + main = fluid.Program() + startup = fluid.Program() + with fluid.unique_name.guard(): + with fluid.program_guard(main, startup): + self.loss_scaling_check(use_cuda=True) + + def test_loss_scaling_gpu_inf(self): + if fluid.core.is_compiled_with_cuda(): + main = fluid.Program() + startup = fluid.Program() + with fluid.unique_name.guard(): + with fluid.program_guard(main, startup): + self.loss_scaling_check_inf(use_cuda=True) + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/white_list/no_check_set_white_list.py b/python/paddle/fluid/tests/unittests/white_list/no_check_set_white_list.py index 0de0eeb464ad700abb2144e49a822582b8653589..afd3414943e9c94799aba5e5e747182623b0a095 100644 --- a/python/paddle/fluid/tests/unittests/white_list/no_check_set_white_list.py +++ b/python/paddle/fluid/tests/unittests/white_list/no_check_set_white_list.py @@ -25,6 +25,7 @@ no_check_set_white_list = [ 'unsqueeze2', 'cross_entropy2', 'seed', - 'amp_check_finite_and_scale', + 'check_finite_and_unscale', + 'update_loss_scaling', 'cudnn_lstm', ] diff --git a/python/paddle/hapi/model_summary.py b/python/paddle/hapi/model_summary.py index 716be1b539809ea3f90885b512f51ac45d85cd37..d388ba62f2a244f84497810739e5fd6b50f669d2 100644 --- a/python/paddle/hapi/model_summary.py +++ b/python/paddle/hapi/model_summary.py @@ -12,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import warnings import numpy as np +import numbers import paddle import paddle.nn as nn @@ -107,6 +109,11 @@ def summary(net, input_size, batch_size=None, dtypes=None): if batch_size is None: batch_size = -1 + if not paddle.in_dynamic_mode(): + warnings.warn( + "Your model was created in static mode, this may not get correct summary information!" + ) + result, params_info = summary_string(net, _input_size, batch_size, dtypes) print(result) @@ -121,16 +128,16 @@ def summary_string(model, input_size, batch_size=-1, dtypes=None): depth = len(list(model.sublayers())) - def register_hook(module): - def hook(module, input, output): - class_name = str(module.__class__).split(".")[-1].split("'")[0] + def register_hook(layer): + def hook(layer, input, output): + class_name = str(layer.__class__).split(".")[-1].split("'")[0] try: - module_idx = int(module._full_name.split('_')[-1]) + layer_idx = int(layer._full_name.split('_')[-1]) except: - module_idx = len(summary) + layer_idx = len(summary) - m_key = "%s-%i" % (class_name, module_idx + 1) + m_key = "%s-%i" % (class_name, layer_idx + 1) summary[m_key] = OrderedDict() summary[m_key]["input_shape"] = list(input[0].shape) summary[m_key]["input_shape"][0] = batch_size @@ -142,23 +149,50 @@ def summary_string(model, input_size, batch_size=-1, dtypes=None): summary[m_key]["output_shape"][0] = batch_size params = 0 - if hasattr(module, "weight") and hasattr(module.weight, "shape"): - params += np.prod(module.weight.shape) - summary[m_key]["trainable"] = module.weight.trainable or ( - not module.weight.stop_gradient) - if hasattr(module, "bias") and hasattr(module.bias, "shape"): - params += np.prod(module.bias.shape) + + if paddle.in_dynamic_mode(): + layer_state_dict = layer._parameters + else: + layer_state_dict = layer.state_dict() + + for k, v in layer_state_dict.items(): + params += np.prod(v.shape) + + try: + if (getattr(getattr(layer, k), 'trainable')) and ( + not getattr(getattr(layer, k), 'stop_gradient')): + summary[m_key]["trainable"] = True + else: + summary[m_key]["trainable"] = False + except: + summary[m_key]["trainable"] = True + summary[m_key]["nb_params"] = params - if (not isinstance(module, nn.Sequential) and - not isinstance(module, nn.LayerList) and - (not (module == model) or depth < 1)): + if (not isinstance(layer, nn.Sequential) and + not isinstance(layer, nn.LayerList) and + (not (layer == model) or depth < 1)): + + hooks.append(layer.register_forward_post_hook(hook)) + + def _check_input_size(input_sizes): + for input_size in input_sizes: + for item in input_size: + if not isinstance(item, numbers.Number): + raise TypeError( + "Expected item in input size be a number, but got {}". + format(type(item))) - hooks.append(module.register_forward_post_hook(hook)) + if item <= 0: + raise ValueError( + "Expected item in input size greater than zero, but got {}". + format(item)) if isinstance(input_size, tuple): input_size = [input_size] + _check_input_size(input_size) + x = [ paddle.rand( [2] + list(in_size), dtype=dtype) @@ -197,7 +231,12 @@ def summary_string(model, input_size, batch_size=-1, dtypes=None): "{0:,}".format(summary[layer]["nb_params"]), ) total_params += summary[layer]["nb_params"] - total_output += np.prod(summary[layer]["output_shape"]) + try: + total_output += np.prod(summary[layer]["output_shape"]) + except: + for output_shape in summary[layer]["output_shape"]: + total_output += np.prod(output_shape) + if "trainable" in summary[layer]: if summary[layer]["trainable"] == True: trainable_params += summary[layer]["nb_params"] diff --git a/python/paddle/nn/__init__.py b/python/paddle/nn/__init__.py index 66caba540f2fed8c035d0f1af14f9e40a329bca5..79583f344f0c1f642586c4a8ecc08f2aa4e24008 100644 --- a/python/paddle/nn/__init__.py +++ b/python/paddle/nn/__init__.py @@ -89,7 +89,7 @@ from .layer.common import CosineSimilarity #DEFINE_ALIAS from .layer.common import Embedding #DEFINE_ALIAS from .layer.common import Linear #DEFINE_ALIAS from .layer.common import Flatten #DEFINE_ALIAS -from .layer.common import UpSample #DEFINE_ALIAS +from .layer.common import Upsample #DEFINE_ALIAS from .layer.common import UpsamplingNearest2d #DEFINE_ALIAS from .layer.common import UpsamplingBilinear2d #DEFINE_ALIAS from .layer.common import Bilinear #DEFINE_ALIAS diff --git a/python/paddle/nn/functional/__init__.py b/python/paddle/nn/functional/__init__.py index 325eaa64d5ca4bd3d65bf266ff0a42226a3199e6..163c249ab37457d7d4566553c71e3231f384a8b1 100644 --- a/python/paddle/nn/functional/__init__.py +++ b/python/paddle/nn/functional/__init__.py @@ -39,7 +39,7 @@ from .activation import hard_sigmoid #DEFINE_ALIAS from .activation import hard_swish #DEFINE_ALIAS from .activation import hsigmoid #DEFINE_ALIAS from .activation import leaky_relu #DEFINE_ALIAS -from .activation import logsigmoid #DEFINE_ALIAS +from .activation import log_sigmoid #DEFINE_ALIAS from .activation import maxout #DEFINE_ALIAS from .activation import prelu #DEFINE_ALIAS from .activation import relu #DEFINE_ALIAS @@ -72,6 +72,7 @@ from .common import unfold #DEFINE_ALIAS # from .common import bilinear_tensor_product #DEFINE_ALIAS from .common import assign #DEFINE_ALIAS from .common import interpolate #DEFINE_ALIAS +from .common import upsample #DEFINE_ALIAS from .common import bilinear #DEFINE_ALIAS from .conv import conv1d #DEFINE_ALIAS from .conv import conv_transpose1d #DEFINE_ALIAS diff --git a/python/paddle/nn/functional/activation.py b/python/paddle/nn/functional/activation.py index ffedb027330bda94db86dc0943a5c4a7281f254f..f7bbe0c94e03dc48ebfb21a62aeded9f446afc63 100644 --- a/python/paddle/nn/functional/activation.py +++ b/python/paddle/nn/functional/activation.py @@ -35,7 +35,7 @@ __all__ = [ 'hard_swish', 'hsigmoid', 'leaky_relu', - 'logsigmoid', + 'log_sigmoid', 'maxout', 'prelu', 'relu', @@ -552,13 +552,13 @@ def relu(x, name=None): return out -def logsigmoid(x, name=None): +def log_sigmoid(x, name=None): """ - logsigmoid activation. + log_sigmoid activation. .. math:: - logsigmoid(x) = log \\frac{1}{1 + e^{-x}} + log\\_sigmoid(x) = log \\frac{1}{1 + e^{-x}} Parameters: x (Tensor): The input Tensor with data type float32, float64. @@ -573,20 +573,19 @@ def logsigmoid(x, name=None): import paddle import paddle.nn.functional as F - import numpy as np paddle.disable_static() - x = paddle.to_tensor(np.array([1.0, 2.0, 3.0, 4.0])) - out = F.logsigmoid(x) # [-0.313262 -0.126928 -0.0485874 -0.0181499] + x = paddle.to_tensor([1.0, 2.0, 3.0, 4.0]) + out = F.log_sigmoid(x) # [-0.313262 -0.126928 -0.0485874 -0.0181499] """ if in_dygraph_mode(): return core.ops.logsigmoid(x) check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], - 'logsigmoid') - helper = LayerHelper("logsigmoid", **locals()) + 'log_sigmoid') + helper = LayerHelper("log_sigmoid", **locals()) out = helper.create_variable_for_type_inference(x.dtype) helper.append_op(type='logsigmoid', inputs={'X': x}, outputs={'Out': out}) return out diff --git a/python/paddle/nn/functional/common.py b/python/paddle/nn/functional/common.py index ad84a32186e8baeabbe8eea7d14e2b7391332944..9f7fb0185133f580deba64634b62d82955670641 100644 --- a/python/paddle/nn/functional/common.py +++ b/python/paddle/nn/functional/common.py @@ -80,6 +80,8 @@ def interpolate(x, The input must be a 3-D Tensor of the shape (num_batches, channels, in_w) or 4-D (num_batches, channels, in_h, in_w), or a 5-D Tensor of the shape (num_batches, channels, in_d, in_h, in_w) or (num_batches, in_d, in_h, in_w, channels), + Where in_w is width of the input tensor, in_h is the height of the input tensor, + in_d is the depth of the intput tensor. and the resizing only applies on the three dimensions(depth, height and width). Supporting resample methods: @@ -88,6 +90,7 @@ def interpolate(x, 'trilinear' : Trilinear interpolation 'nearest' : Nearest neighbor interpolation 'bicubic' : Bicubic interpolation + 'area': Area interpolation Linear interpolation is the method of using a line connecting two known quantities to determine the value of an unknown quantity between the two known quantities. @@ -114,6 +117,12 @@ def interpolate(x, smoother than corresponding surfaces obtained by bilinear interpolation or nearest-neighbor interpolation. + Area interpolation is to perform area interpolation + in both the 3rd dimension(in height direction) , the 4th dimension(in width + direction) and the 5th dimension(in depth direction) on input tensor. Set to + area will directly call `paddle.nn.functional.adaptive_avg_pool1d` or + `paddle.nn.functional.adaptive_avg_pool2d` or `paddle.nn.functional.adaptive_avg_pool3d`. + Example: .. code-block:: text @@ -207,11 +216,11 @@ def interpolate(x, when input is a 4-D Tensor and is (out_d, out_h, out_w) when input is a 5-D Tensor. Default: None. If a list, each element can be an integer or a Tensor Variable of shape: [1]. If a Tensor Variable, its dimensions size should be a 1. - scale_factor (float|Tensor|list|None): The multiplier for the input height or width. At - least one of :attr:`out_shape` or :attr:`scale_factor` must be set. - And :attr:`out_shape` has a higher priority than :attr:`scale_factor`.Has to match input size if it is a list. + scale_factor (float|Tensor|list|tuple|None): The multiplier for the input height or width. At + least one of :attr:`size` or :attr:`scale_factor` must be set. + And :attr:`size` has a higher priority than :attr:`scale_factor`.Has to match input size if it is either a list or a tuple or a Tensor. Default: None. - mode (str): The resample method. It supports 'linear', 'nearest', 'bilinear', + mode (str): The resample method. It supports 'linear', 'area', 'nearest', 'bilinear', 'bicubic' and 'trilinear' currently. Default: 'nearest' align_corners(bool) : An optional bool, If True, the centers of the 4 corner pixels of the input and output tensors are aligned, preserving the values at the @@ -235,7 +244,7 @@ def interpolate(x, Raises: TypeError: size should be a list or tuple or Tensor. ValueError: The 'mode' of image_resize can only be 'linear', 'bilinear', - 'trilinear', 'bicubic', or 'nearest' currently. + 'trilinear', 'bicubic', 'area' or 'nearest' currently. ValueError: 'linear' only support 3-D tensor. ValueError: 'bilinear', 'bicubic' and 'nearest' only support 4-D tensor. ValueError: 'trilinear' only support 5-D tensor. @@ -283,10 +292,11 @@ def interpolate(x, 'TRILINEAR', 'NEAREST', 'BICUBIC', + 'AREA', ] if resample not in resample_methods: raise ValueError( - "The 'resample' of image_resize can only be 'linaer', 'bilinear', 'trilinear', " + "The 'resample' of image_resize can only be 'area', 'linear', 'bilinear', 'trilinear', " " 'bicubic' or 'nearest' currently.") if resample in ['LINEAR'] and len(x.shape) != 3: @@ -310,8 +320,17 @@ def interpolate(x, raise ValueError( "align_corners option can only be set with the interpolating modes: linear | bilinear | bicubic | trilinear" ) + + if resample == 'AREA' and len(x.shape) == 3: + return paddle.nn.functional.adaptive_avg_pool1d(x, size) + + if resample == 'AREA' and len(x.shape) == 4: + return paddle.nn.functional.adaptive_avg_pool2d(x, size) + if resample == 'AREA' and len(x.shape) == 5: + return paddle.nn.functional.adaptive_avg_pool3d(x, size) + helper = LayerHelper('{}_interp_v2'.format(resample_type), **locals()) - dtype = helper.input_dtype() + dtype = helper.input_dtype(input_param_name='x') if len(x.shape) == 3 and data_format not in ['NCW', 'NWC']: raise ValueError( "Got wrong value for param `data_format`: " + data_format + @@ -349,14 +368,15 @@ def interpolate(x, out_shape = size scale = scale_factor + if out_shape is not None and scale is not None: + raise ValueError("Only one of size or scale_factor should be defined.") if out_shape is not None: if isinstance(out_shape, Variable): out_shape.stop_gradient = True inputs['OutSize'] = out_shape else: if not (_is_list_or_turple_(out_shape)): - raise TypeError( - "out_shape should be a list or tuple or Variable.") + raise TypeError("size should be a list or tuple or Variable.") # Validate the shape contain_var = False for dim_idx, dim_size in enumerate(out_shape): @@ -388,7 +408,7 @@ def interpolate(x, if len(x.shape) == 3: if len(out_shape) != 1: raise ValueError( - "out_shape length should be 2 for input 3-D tensor") + "size length should be 2 for input 3-D tensor") if contain_var: attrs['out_w'] = size_list[0] else: @@ -396,7 +416,7 @@ def interpolate(x, attrs['out_w'] = out_shape[0] if len(x.shape) == 4: if len(out_shape) != 2: - raise ValueError("out_shape length should be 2 for " + raise ValueError("size length should be 2 for " "input 4-D tensor.") if contain_var: attrs['out_h'] = size_list[0] @@ -407,7 +427,7 @@ def interpolate(x, attrs['out_w'] = out_shape[1] if len(x.shape) == 5: if len(out_shape) != 3: - raise ValueError("out_shape length should be 3 for " + raise ValueError("size length should be 3 for " "input 5-D tensor.") if contain_var: attrs['out_d'] = size_list[0] @@ -430,7 +450,7 @@ def interpolate(x, for i in range(len(x.shape) - 2): scale_list.append(scale) attrs['scale'] = list(map(float, scale_list)) - elif isinstance(scale, list): + elif isinstance(scale, list) or isinstance(scale, tuple): if len(scale) != len(x.shape) - 2: raise ValueError("scale_shape length should be {} for " "input {}-D tensor.".format( @@ -441,7 +461,8 @@ def interpolate(x, attrs['scale'] = list(map(float, scale)) else: raise TypeError( - "Attr(scale)'s type should be float, int, list or Tensor.") + "Attr(scale)'s type should be float, int, list, tuple, or Tensor." + ) if in_dygraph_mode(): attr_list = [] @@ -480,9 +501,12 @@ def upsample(x, name=None): """ This op resizes a batch of images. + The input must be a 3-D Tensor of the shape (num_batches, channels, in_w) or 4-D (num_batches, channels, in_h, in_w), or a 5-D Tensor of the shape (num_batches, channels, in_d, in_h, in_w) or (num_batches, in_d, in_h, in_w, channels), + Where in_w is width of the input tensor, in_h is the height of the input tensor, + in_d is the depth of the intput tensor. and the resizing only applies on the three dimensions(depth, height and width). Supporting resample methods: @@ -507,12 +531,21 @@ def upsample(x, data points on a two-dimensional regular grid. The interpolated surface is smoother than corresponding surfaces obtained by bilinear interpolation or nearest-neighbor interpolation. + Trilinear interpolation is an extension of linear interpolation for interpolating functions of three variables (e.g. D-direction, H-direction and W-direction in this op) on a rectilinear 3D grid. + The linear interpolation is performed on three directions. align_corners and align_mode are optional parameters,the calculation method of interpolation can be selected by them. + + Area interpolation is to perform area interpolation + in both the 3rd dimension(in height direction) , the 4th dimension(in width + direction) and the 5th dimension(in depth direction) on input tensor. Set to + area will directly call `paddle.nn.functional.adaptive_avg_pool1d` or + `paddle.nn.functional.adaptive_avg_pool2d` or `paddle.nn.functional.adaptive_avg_pool3d`. + Example: .. code-block:: text For scale_factor: @@ -605,9 +638,10 @@ def upsample(x, when input is a 4-D Tensor and is (out_d, out_h, out_w) when input is a 5-D Tensor. Default: None. If a list, each element can be an integer or a Tensor Variable of shape: [1]. If a Tensor Variable, its dimensions size should be a 1. - scale_factor (float|Tensor|list|None): The multiplier for the input height or width. At - least one of :attr:`out_shape` or :attr:`scale_factor` must be set. - And :attr:`out_shape` has a higher priority than :attr:`scale_factor`. + scale_factor (float|Tensor|list|tuple|None): The multiplier for the input height or width. At + least one of :attr:`size` or :attr:`scale_factor` must be set. + And :attr:`size` has a higher priority than :attr:`scale_factor`.Has to match input size if + it is either a list or a tuple or a Tensor. Default: None. mode (str): The resample method. It supports 'linear', 'nearest', 'bilinear', 'bicubic' and 'trilinear' currently. Default: 'nearest' diff --git a/python/paddle/nn/functional/conv.py b/python/paddle/nn/functional/conv.py index 3c1482e69c3c36232ee5d70f2156a8d16c2d212a..5cf4953933242292c6a732513dbee2164811dd35 100644 --- a/python/paddle/nn/functional/conv.py +++ b/python/paddle/nn/functional/conv.py @@ -267,8 +267,8 @@ def conv1d(x, dilation = utils.convert_to_list(dilation, 1, 'dilation') + [1] l_type = "conv2d" - if (num_channels == groups and num_filters % num_channels == 0 and - not use_cudnn): + if (num_channels == groups and num_channels != 1 and + num_filters % num_channels == 0 and not use_cudnn): l_type = 'depthwise_conv2d' use_cudnn = False @@ -491,7 +491,8 @@ def conv2d(x, dilation = utils.convert_to_list(dilation, 2, 'dilation') l_type = "conv2d" - if (num_channels == groups and num_filters % num_channels == 0): + if (num_channels == groups and num_channels != 1 and + num_filters % num_channels == 0): l_type = 'depthwise_conv2d' use_cudnn = False @@ -761,7 +762,8 @@ def conv_transpose1d(x, op_type = 'conv2d_transpose' num_filters = weight.shape[1] - if (num_channels == groups and num_filters == 1 and not use_cudnn): + if (num_channels == groups and num_channels != 1 and num_filters == 1 and + not use_cudnn): op_type = 'depthwise_conv2d_transpose' use_cudnn = False @@ -1010,7 +1012,7 @@ def conv_transpose2d(x, op_type = 'conv2d_transpose' num_filters = weight.shape[1] - if (num_channels == groups and num_filters == 1): + if (num_channels == groups and num_channels != 1 and num_filters == 1): op_type = 'depthwise_conv2d_transpose' use_cudnn = False diff --git a/python/paddle/nn/functional/input.py b/python/paddle/nn/functional/input.py index bc48cc21c29e6683602f37fb3eab6c9485fe4977..0794b95c801011da6845eaf82c32a5428e0d5f41 100644 --- a/python/paddle/nn/functional/input.py +++ b/python/paddle/nn/functional/input.py @@ -113,17 +113,18 @@ def one_hot(x, num_classes, name=None): def embedding(x, weight, padding_idx=None, sparse=False, name=None): """ - The operator is used to lookup embeddings vector of ids provided by :attr:`input` . + The operator is used to lookup embeddings vector of ids provided by :attr:`x` . The shape of output Tensor is generated by appending the last dimension of the input Tensor shape with embedding size. - **Note:** The id in :attr:`input` must satisfy :math:`0 =< id < weight.shape[0]` , + + **Note:** The id in :attr:`x` must satisfy :math:`0 =< id < weight.shape[0]` , otherwise the program will throw an exception and exit. .. code-block:: text Case 1: - input is a Tensor. + x is a Tensor. padding_idx = -1 x.data = [[1, 3], [2, 4], [4, 127]] x.shape = [3, 2] @@ -138,7 +139,7 @@ def embedding(x, weight, padding_idx=None, sparse=False, name=None): [0.0, 0.0, ..., 0.0 ]]] # padding data The input padding_idx is less than 0, it is automatically converted to padding_idx = -1 + 128 = 127 - It will pad all-zero data when ids is 127. + It will pad all-zero data when id is 127. Args: x(Tensor): A Tensor with type int32/int64, which contains the id information. The value of the input id should @@ -151,10 +152,10 @@ def embedding(x, weight, padding_idx=None, sparse=False, name=None): such as :ref:`api_optimizer_AdadeltaOptimizer` , :ref:`api_optimizer_AdamaxOptimizer` , :ref:`api_optimizer_DecayedAdagradOptimizer` , :ref:`api_optimizer_FtrlOptimizer` , :ref:`api_optimizer_LambOptimizer` and :ref:`api_optimizer_LarsMomentumOptimizer` . - In these cases, is_sparse must be False. Default: False. - padding_idx(int|long|None): padding_idx needs to be in the interval [-vocab_size, vocab_size). + In these cases, sparse must be False. Default: False. + padding_idx(int|long|None): padding_idx needs to be in the interval [-weight.shape[0], weight.shape[0]). If :math:`padding\_idx < 0`, the :math:`padding\_idx` will automatically be converted - to :math:`vocab\_size + padding\_idx` . It will output all-zero padding data whenever lookup + to :math:`weight.shape[0] + padding\_idx` . It will output all-zero padding data whenever lookup encounters :math:`padding\_idx` in id. And the padding data will not be updated while training. If set None, it makes no effect to output. Default: None. name(str|None): For detailed information, please refer @@ -162,7 +163,7 @@ def embedding(x, weight, padding_idx=None, sparse=False, name=None): None by default. Returns: - Tensor: Embedding Tensor mapped by input. The data type is the same as :attr:`weight`. + Tensor: Embedding Tensor mapped by x. The data type is the same as :attr:`weight`. Examples: @@ -209,6 +210,10 @@ def embedding(x, weight, padding_idx=None, sparse=False, name=None): padding_idx = -1 if padding_idx is None else padding_idx if padding_idx >= 0 else ( weight.shape[0] + padding_idx) + if padding_idx >= weight.shape[0] or padding_idx < -weight.shape[0]: + raise ValueError("padding_idx must be within [-{}, {})".format( + weight.shape[0], weight.shape[0])) + helper.append_op( type='lookup_table_v2', inputs={'Ids': x, diff --git a/python/paddle/nn/functional/loss.py b/python/paddle/nn/functional/loss.py index 3d5894064c44cb72259472fc638d46b67c5703fc..da086c0955e849619ccbce17a297ca4615a3f3d0 100644 --- a/python/paddle/nn/functional/loss.py +++ b/python/paddle/nn/functional/loss.py @@ -780,10 +780,10 @@ def kl_div(input, label, reduction='mean', name=None): input = np.random.uniform(-10, 10, shape).astype('float32') target = np.random.uniform(-10, 10, shape).astype('float32') - # 'batchmean' reduction, loss shape will be [N] + # 'batchmean' reduction, loss shape will be [1] pred_loss = F.kl_div(paddle.to_tensor(input), paddle.to_tensor(target), reduction='batchmean') - # shape=[5] + # shape=[1] # 'mean' reduction, loss shape will be [1] pred_loss = F.kl_div(paddle.to_tensor(input), @@ -1009,8 +1009,7 @@ def ctc_loss(log_probs, loss_out = fluid.layers.squeeze(loss_out, [-1]) assert reduction in ['mean', 'sum', 'none'] if reduction == 'mean': - loss_out = paddle.mean(loss_out / paddle.cast(label_lengths, - loss_out.dtype)) + loss_out = paddle.mean(loss_out / label_lengths) elif reduction == 'sum': loss_out = paddle.sum(loss_out) return loss_out diff --git a/python/paddle/nn/functional/pooling.py b/python/paddle/nn/functional/pooling.py index 662205ab69550255406ff5edfda4556b73b98843..042625a3dbd6b07487d6f77442621959f7492af6 100755 --- a/python/paddle/nn/functional/pooling.py +++ b/python/paddle/nn/functional/pooling.py @@ -389,7 +389,7 @@ def avg_pool3d(x, stride=None, padding=0, ceil_mode=False, - count_include_pad=False, + count_include_pad=True, divisor_override=None, data_format="NCDHW", name=None): diff --git a/python/paddle/nn/functional/vision.py b/python/paddle/nn/functional/vision.py index 1dfdac26e990851ac5f192742acd47fb92633d0d..a74a98d5ed45b9f613b0f2f6d5f04544ffae3d2a 100644 --- a/python/paddle/nn/functional/vision.py +++ b/python/paddle/nn/functional/vision.py @@ -249,7 +249,7 @@ def grid_sample(x, mode(str, optional): The interpolation method which can be 'bilinear' or 'nearest'. Default: 'bilinear'. padding_mode(str, optional) The padding method used when source index - is out of input images. It can be 'zeros', 'reflect' and 'border'. + is out of input images. It can be 'zeros', 'reflection' and 'border'. Default: zeros. align_corners(bool, optional): If `align_corners` is true, it will projects -1 and 1 to the centers of the corner pixels. Otherwise, it will @@ -312,7 +312,7 @@ def grid_sample(x, if not isinstance(grid, Variable): raise ValueError("The grid should be a Variable") _modes = ['bilinear', 'nearest'] - _padding_modes = ['zeros', 'reflect', 'border'] + _padding_modes = ['zeros', 'reflection', 'border'] if mode not in _modes: raise ValueError( "The mode of grid sample function should be in {}, but got: {}". diff --git a/python/paddle/nn/layer/__init__.py b/python/paddle/nn/layer/__init__.py index 7d7a392ebe80c3af8c991dbff746d0f8f216b18b..760af09f1f2f5af066058572f681ec21f9a93180 100644 --- a/python/paddle/nn/layer/__init__.py +++ b/python/paddle/nn/layer/__init__.py @@ -59,7 +59,7 @@ from .common import CosineSimilarity #DEFINE_ALIAS from .common import Embedding #DEFINE_ALIAS from .common import Linear #DEFINE_ALIAS from .common import Flatten #DEFINE_ALIAS -from .common import UpSample #DEFINE_ALIAS +from .common import Upsample #DEFINE_ALIAS from .common import UpsamplingNearest2d #DEFINE_ALIAS from .common import UpsamplingBilinear2d #DEFINE_ALIAS from .common import Dropout #DEFINE_ALIAS diff --git a/python/paddle/nn/layer/activation.py b/python/paddle/nn/layer/activation.py index c38d6018a2500111280a482aa60d072e65e27742..585d369c607e5b6eb6a2a3bcb28bd8999a2e0dca 100644 --- a/python/paddle/nn/layer/activation.py +++ b/python/paddle/nn/layer/activation.py @@ -860,11 +860,10 @@ class LogSigmoid(layers.Layer): .. code-block:: python import paddle - import numpy as np paddle.disable_static() - x = paddle.to_tensor(np.array([1.0, 2.0, 3.0, 4.0])) + x = paddle.to_tensor([1.0, 2.0, 3.0, 4.0]) m = paddle.nn.LogSigmoid() out = m(x) # [-0.313262 -0.126928 -0.0485874 -0.0181499] """ @@ -874,7 +873,7 @@ class LogSigmoid(layers.Layer): self._name = name def forward(self, x): - return F.logsigmoid(x, self._name) + return F.log_sigmoid(x, self._name) class Softmax(layers.Layer): diff --git a/python/paddle/nn/layer/common.py b/python/paddle/nn/layer/common.py index d8e1d03b02840e76ff865986d8b90ca9d6cdd9f8..433443fee1765a3ecd4cf0bbe53a960bbeaefc71 100644 --- a/python/paddle/nn/layer/common.py +++ b/python/paddle/nn/layer/common.py @@ -26,7 +26,7 @@ __all__ = [ 'Pool2D', 'Embedding', 'Linear', - 'UpSample', + 'Upsample', 'Pad2D', 'UpsamplingNearest2d', 'UpsamplingBilinear2d', @@ -131,12 +131,15 @@ class Linear(layers.Layer): return out -class UpSample(layers.Layer): +class Upsample(layers.Layer): """ This op resizes a batch of images. + The input must be a 3-D Tensor of the shape (num_batches, channels, in_w) or 4-D (num_batches, channels, in_h, in_w), or a 5-D Tensor of the shape (num_batches, channels, in_d, in_h, in_w) or (num_batches, in_d, in_h, in_w, channels), + Where in_w is width of the input tensor, in_h is the height of the input tensor, + in_d is the depth of the intput tensor. and the resizing only applies on the three dimensions(depth, height and width). Supporting resample methods: @@ -171,6 +174,12 @@ class UpSample(layers.Layer): align_corners and align_mode are optional parameters,the calculation method of interpolation can be selected by them. + Area interpolation is to perform area interpolation + in both the 3rd dimension(in height direction) , the 4th dimension(in width + direction) and the 5th dimension(in depth direction) on input tensor. Set to + area will directly call `paddle.nn.functional.adaptive_avg_pool1d` or + `paddle.nn.functional.adaptive_avg_pool2d` or `paddle.nn.functional.adaptive_avg_pool3d`. + Example: .. code-block:: text @@ -273,9 +282,9 @@ class UpSample(layers.Layer): when input is a 4-D Tensor and is (out_d, out_h, out_w) when input is a 5-D Tensor. Default: None. If a list, each element can be an integer or a Tensor Variable of shape: [1]. If a Tensor Variable, its dimensions size should be a 1. - scale_factor (float|Tensor|list|None): The multiplier for the input height or width. At - least one of :attr:`out_shape` or :attr:`scale_factor` must be set. - And :attr:`out_shape` has a higher priority than :attr:`scale_factor`.Has to match input size if it is a list. + scale_factor (float|Tensor|list|tuple|None): The multiplier for the input height or width. At + least one of :attr:`size` or :attr:`scale_factor` must be set. + And :attr:`size` has a higher priority than :attr:`scale_factor`. Has to match input size if it is either a list or a tuple or a Tensor. Default: None. mode (str): The resample method. It supports 'linear', 'nearst', 'bilinear', 'bicubic' and 'trilinear' currently. Default: 'nearest' @@ -322,7 +331,7 @@ class UpSample(layers.Layer): paddle.disable_static() input_data = np.random.rand(2,3,6,10).astype("float32") - upsample_out = paddle.nn.UpSample(size=[12,12]) + upsample_out = paddle.nn.Upsample(size=[12,12]) input = paddle.to_tensor(input_data) output = upsample_out(x=input) @@ -339,7 +348,7 @@ class UpSample(layers.Layer): align_mode=0, data_format='NCHW', name=None): - super(UpSample, self).__init__() + super(Upsample, self).__init__() self.size = size self.scale_factor = scale_factor self.mode = mode.lower() @@ -366,7 +375,8 @@ class UpsamplingNearest2d(layers.Layer): """ This op upsamples a batch of images, using nearest neighbours' pixel values. The input must be a 4-D Tensor of the shape (num_batches, channels, in_h, in_w), - and the upsampling only applies on the two dimensions(height and width). + where in_w is width of the input tensor, in_h is the height of the input tensor. + And the upsampling only applies on the two dimensions(height and width). Nearest neighbor interpolation is to perform nearest neighbor interpolation in both the 3rd dimension(in height direction) and the 4th dimension(in width @@ -381,10 +391,11 @@ class UpsamplingNearest2d(layers.Layer): layer, the shape is (out_h, out_w) when input is a 4-D Tensor. Default: None. If a list, each element can be an integer or a Tensor Variable of shape: [1]. If a Tensor Variable, its dimensions size should be a 1. - scale_factor (float|int|list|Tensor|None): The multiplier for the input height or width. At - least one of :attr:`out_shape` or :attr:`scale_factor` must be set. - And :attr:`out_shape` has a higher priority than :attr:`scale_factor`. - Default: None. Has to match input size if it is a list. + scale_factor (float|int|list|tuple|Tensor|None): The multiplier for the input height or width. At + least one of :attr:`size` or :attr:`scale_factor` must be set. + And :attr:`size` has a higher priority than :attr:`scale_factor`. + Has to match input size if it is either a list or a tuple or a Tensor. + Default: None. data_format (str, optional): Specify the data format of the input, and the data format of the output will be consistent with that of the input. An optional string from:`NCW`, `NWC`, `"NCHW"`, `"NHWC"`, `"NCDHW"`, `"NDHWC"`. The default is `"NCHW"`. When it is `"NCHW"`, the data is stored in the order of: @@ -449,7 +460,8 @@ class UpsamplingBilinear2d(layers.Layer): """ This op upsamples a batch of images, using bilinear' pixel values. The input must be a 4-D Tensor of the shape (num_batches, channels, in_h, in_w), - and the upsampling only applies on the two dimensions(height and width). + where in_w is width of the input tensor, in_h is the height of the input tensor. + And the upsampling only applies on the two dimensions(height and width). Bilinear interpolation is an extension of linear interpolation for interpolating functions of two variables (e.g. H-direction and @@ -466,10 +478,11 @@ class UpsamplingBilinear2d(layers.Layer): layer, the shape is (out_h, out_w) when input is a 4-D Tensor. Default: None. If a list, each element can be an integer or a Tensor Variable of shape: [1]. If a Tensor Variable, its dimensions size should be a 1. - scale_factor (float|int|list|Tensor|None): The multiplier for the input height or width. At - least one of :attr:`out_shape` or :attr:`scale_factor` must be set. - And :attr:`out_shape` has a higher priority than :attr:`scale_factor`. - Default: None. Has to match input size if it is a list. + scale_factor (float|int|list|tuple|Tensor|None): The multiplier for the input height or width. At + least one of :attr:`size` or :attr:`scale_factor` must be set. + And :attr:`size` has a higher priority than :attr:`scale_factor`. + Has to match input size if it is either a list or a tuple or a Tensor. + Default: None. data_format (str, optional): Specify the data format of the input, and the data format of the output will be consistent with that of the input. An optional string from:`NCW`, `NWC`, `"NCHW"`, `"NHWC"`, `"NCDHW"`, `"NDHWC"`. The default is `"NCHW"`. When it is `"NCHW"`, the data is stored in the order of: @@ -1551,22 +1564,18 @@ class CosineSimilarity(layers.Layer): class Embedding(layers.Layer): """ - :alias_main: paddle.nn.Embedding - :alias: paddle.nn.Embedding,paddle.nn.layer.Embedding,paddle.nn.layer.common.Embedding - :old_api: paddle.fluid.dygraph.Embedding - **Embedding Layer** This interface is used to construct a callable object of the ``Embedding`` class. For specific usage, refer to code examples. It implements the function of the Embedding Layer. - This layer is used to lookup embeddings vector of ids provided by :attr:`input` . + This layer is used to lookup embeddings vector of ids provided by :attr:`x` . It automatically constructs a 2D embedding matrix based on the - input :attr:`size` (vocab_size, emb_size) and :attr:`dtype` . + input :attr:`num_embeddings` and attr:`embedding_dim`. The shape of output Tensor is generated by appending an emb_size dimension to the last dimension of the input Tensor shape. - **Note:** The id in :attr:`input` must satisfy :math:`0 =< id < size[0]` , + **Note:** The id in :attr:`x` must satisfy :math:`0 =< id < num_embeddings` , otherwise the program will throw an exception and exit. .. code-block:: text @@ -1594,7 +1603,7 @@ class Embedding(layers.Layer): num_embeddings (int): Just one element which indicate the size of the dictionary of embeddings. embedding_dim: Just one element which indicate the size of each embedding vector respectively. - padding_idx(int|long|None): padding_idx needs to be in the interval [-vocab_size, vocab_size). + padding_idx(int|long|None): padding_idx needs to be in the interval [-num_embeddings, num_embeddings). If :math:`padding\_idx < 0`, the :math:`padding\_idx` will automatically be converted to :math:`vocab\_size + padding\_idx` . It will output all-zero padding data whenever lookup encounters :math:`padding\_idx` in id. And the padding data will not be updated while training. @@ -1605,13 +1614,13 @@ class Embedding(layers.Layer): such as :ref:`api_optimizer_AdadeltaOptimizer` , :ref:`api_optimizer_AdamaxOptimizer` , :ref:`api_optimizer_DecayedAdagradOptimizer` , :ref:`api_optimizer_FtrlOptimizer` , :ref:`api_optimizer_LambOptimizer` and :ref:`api_optimizer_LarsMomentumOptimizer` . - In these case, is_sparse must be False. Default: False. + In these case, sparse must be False. Default: False. weight_attr(ParamAttr): To specify the weight parameter property. Default: None, which means the - default weight parameter property is used. See usage for details in :ref:`api_fluid_ParamAttr` . In addition, + default weight parameter property is used. See usage for details in :ref:`api_ParamAttr` . In addition, user-defined or pre-trained word vectors can be loaded with the :attr:`param_attr` parameter. The local word vector needs to be transformed into numpy format, and the shape of local word - vector should be consistent with :attr:`size` . Then :ref:`api_fluid_initializer_NumpyArrayInitializer` - is used to load custom or pre-trained word vectors. See code example 2 for details. + vector should be consistent with :attr:`num_embeddings` . Then :ref:`api_initializer_NumpyArrayInitializer` + is used to load custom or pre-trained word vectors. See code example for details. name(str|None): For detailed information, please refer to :ref:`api_guide_Name`. Usually name is no need to set and None by default. @@ -1626,20 +1635,34 @@ class Embedding(layers.Layer): .. code-block:: python - import paddle - import paddle.nn as nn - import numpy as np - paddle.disable_static() + import paddle + import numpy as np + + x_data = np.arange(3, 6).reshape((3, 1)).astype(np.int64) + y_data = np.arange(6, 12).reshape((3, 2)).astype(np.float32) + paddle.disable_static(paddle.CPUPlace()) + x = paddle.to_tensor(x_data, stop_gradient=False) + y = paddle.to_tensor(y_data, stop_gradient=False) - # example 1 - inp_word = np.array([[2, 3, 5], [4, 2, 1]]).astype('int64') - inp_word.shape # [2, 3] - dict_size = 20 + embedding = paddle.nn.Embedding(10, 3, sparse=True) + + w0=np.full(shape=(10, 3), fill_value=2).astype(np.float32) + embedding.weight.set_value(w0) + + adam = paddle.optimizer.Adam(parameters=[embedding.weight], learning_rate=0.01) + adam.clear_grad() + + # weight.shape = [10, 3] + + # x.data = [[3],[4],[5]] + # x.shape = [3, 1] + + # out.data = [[2,2,2], [2,2,2], [2,2,2]] + # out.shape = [3, 1, 3] + out=embedding(x) + out.backward() + adam.step() - emb = nn.Embedding( - dict_size, - 32, - sparse=False) """ def __init__(self, @@ -1656,13 +1679,24 @@ class Embedding(layers.Layer): self._is_distributed = False self._padding_idx = -1 if padding_idx is None else padding_idx if padding_idx >= 0 else ( num_embeddings + padding_idx) + + if self._num_embeddings <= 0: + raise ValueError("num_embeddings must be gather than 0") + + if self._embedding_dim <= 0: + raise ValueError("embedding_dim must be gather than 0") + + if self._padding_idx >= num_embeddings or self._padding_idx < -num_embeddings: + raise ValueError("padding_idx must be within [-{}, {})".format( + num_embeddings, num_embeddings)) + self._dtype = self._helper.get_default_dtype() self._size = [self._num_embeddings, self._embedding_dim] self._weight_attr = weight_attr self._remote_prefetch = False self._name = name - self._weight = self.create_parameter( + self.weight = self.create_parameter( attr=self._weight_attr, shape=self._size, dtype=self._dtype, @@ -1671,7 +1705,7 @@ class Embedding(layers.Layer): def forward(self, x): return F.embedding( x, - weight=self._weight, + weight=self.weight, padding_idx=self._padding_idx, sparse=self._sparse, name=self._name) diff --git a/python/paddle/nn/layer/loss.py b/python/paddle/nn/layer/loss.py index a60e615d5064bf4ef2229dd67193774030383888..271dc9b4e685ce06cdb12ccdcb6bb0704a5ef2a1 100644 --- a/python/paddle/nn/layer/loss.py +++ b/python/paddle/nn/layer/loss.py @@ -627,10 +627,13 @@ class KLDivLoss(fluid.dygraph.Layer): $$l(x, y) = y * (\log(y) - x)$$ Parameters: - reduction (str, optional): Indicate how to average the loss, - the candicates are ``'none'`` | ``'mean'`` | ``'sum'``. - If :attr:`reduction` is ``'mean'``, the reduced mean loss is returned; - Default is ``'mean'``. + reduction (Tensor): Indicate how to average the loss, + the candicates are ``'none'`` | ``'batchmean'`` | ``'mean'`` | ``'sum'``. + If `reduction` is ``'mean'``, the reduced mean loss is returned; + If `reduction` is ``'batchmean'``, the sum loss divided by batch size is returned; + if `reduction` is ``'sum'``, the reduced sum loss is returned; + if `reduction` is ``'none'``, no reduction will be apllied. + Default is ``'mean'``. Shape: @@ -654,11 +657,11 @@ class KLDivLoss(fluid.dygraph.Layer): x = np.random.uniform(-10, 10, shape).astype('float32') target = np.random.uniform(-10, 10, shape).astype('float32') - # 'batchmean' reduction, loss shape will be [N] + # 'batchmean' reduction, loss shape will be [1] kldiv_criterion = nn.KLDivLoss(reduction='batchmean') pred_loss = kldiv_criterion(paddle.to_tensor(x), paddle.to_tensor(target)) - # shape=[5] + # shape=[1] # 'mean' reduction, loss shape will be [1] kldiv_criterion = nn.KLDivLoss(reduction='mean') @@ -684,7 +687,7 @@ class KLDivLoss(fluid.dygraph.Layer): self.reduction = reduction def forward(self, input, label): - out = paddle.nn.functional.kl_div(input, label, self.reduction) + out = F.kl_div(input, label, self.reduction) return out diff --git a/python/paddle/nn/layer/norm.py b/python/paddle/nn/layer/norm.py index d13bf66ba5bfe483284e78dbcd2a42f8f3397210..2000fbf388f88d1da7119402104706a433cebf06 100644 --- a/python/paddle/nn/layer/norm.py +++ b/python/paddle/nn/layer/norm.py @@ -1130,10 +1130,10 @@ class SyncBatchNorm(_BatchNormBase): """ layer_output = layer if isinstance(layer, _BatchNormBase): - layer_output = SyncBatchNorm(layer._num_features, layer._epsilon, - layer._momentum, layer._weight_attr, - layer._bias_attr, layer._data_format, - layer._name) + layer_output = SyncBatchNorm( + layer._num_features, layer._momentum, layer._epsilon, + layer._weight_attr, layer._bias_attr, layer._data_format, + layer._track_running_stats, layer._name) if layer._weight_attr != False and layer._bias_attr != False: with no_grad(): diff --git a/python/paddle/nn/utils/weight_norm_hook.py b/python/paddle/nn/utils/weight_norm_hook.py index ad53bf394660f3a7e0e48fdbd5eb530abd0852bb..7a21e7661d4e78d0004996ee67c80ddc35006bc3 100644 --- a/python/paddle/nn/utils/weight_norm_hook.py +++ b/python/paddle/nn/utils/weight_norm_hook.py @@ -112,6 +112,14 @@ class WeightNorm(object): if dim is None: dim = -1 + # support dim is negative numeber, (dim = -1) == (dim = None) + weight_dim = len(layer._parameters[name].shape) + assert ( + dim < weight_dim and dim >= -1 * weight_dim + ), "dim must set between [-R, R), R means the dimension of weight." + if dim != -1: + dim = (dim + weight_dim) % weight_dim + fn = WeightNorm(name, dim) w = getattr(layer, name) diff --git a/python/paddle/optimizer/adam.py b/python/paddle/optimizer/adam.py index 3150b8c2d0363274dfb6fd3465110c89339cd4c9..708aaa788f60d56a2adb41c8a571079354b3c192 100644 --- a/python/paddle/optimizer/adam.py +++ b/python/paddle/optimizer/adam.py @@ -250,3 +250,47 @@ class Adam(Optimizer): stop_gradient=True) return adam_op + + @framework.dygraph_only + def step(self): + """ + Execute the optimizer and update parameters once. + + Returns: + None + + Examples: + .. code-block:: python + + import paddle + import numpy as np + paddle.disable_static() + value = np.arange(26).reshape(2, 13).astype("float32") + a = paddle.to_tensor(value) + linear = paddle.nn.Linear(13, 5) + # This can be any optimizer supported by dygraph. + adam = paddle.optimizer.Adam(learning_rate = 0.01, + parameters = linear.parameters()) + out = linear(a) + out.backward() + adam.step() + adam.clear_grad() + """ + parameter_list = self._parameter_list + self._dtype = None + params_grads = [] + for param in self._parameter_list: + if not param.trainable: + continue + if hasattr( + param, "_is_sparse" + ) and param._is_sparse and self.regularization is not None: + raise RuntimeError( + "Adam don't support weight_decay with sparse parameters, please set it to None." + ) + if param._grad_ivar() is not None: + grad_var = param._grad_ivar() + params_grads.append((param, grad_var)) + + optimize_ops = self._apply_optimize( + loss=None, startup_program=None, params_grads=params_grads) diff --git a/python/paddle/tensor/__init__.py b/python/paddle/tensor/__init__.py index 0fed32a1676759bd94961af0a8949d035ec48c8f..8bb584be2362e7b02bc5b7c5603b148d37499c2d 100755 --- a/python/paddle/tensor/__init__.py +++ b/python/paddle/tensor/__init__.py @@ -40,6 +40,7 @@ from .creation import full_like #DEFINE_ALIAS from .creation import triu #DEFINE_ALIAS from .creation import tril #DEFINE_ALIAS from .creation import meshgrid #DEFINE_ALIAS +from .creation import empty #DEFINE_ALIAS from .io import save #DEFINE_ALIAS from .io import load #DEFINE_ALIAS from .linalg import matmul #DEFINE_ALIAS diff --git a/python/paddle/tensor/creation.py b/python/paddle/tensor/creation.py index 9eece1240d7d3c0b8a863091367e993047bd4527..8011b92964b7e21fd930f19cec954b27f470e0c6 100644 --- a/python/paddle/tensor/creation.py +++ b/python/paddle/tensor/creation.py @@ -48,6 +48,7 @@ __all__ = [ 'eye', 'full', 'full_like', + 'empty', 'triu', 'tril', 'meshgrid' @@ -62,8 +63,7 @@ def to_tensor(data, dtype=None, place=None, stop_gradient=True): If the ``data`` is already a tensor, and ``dtype`` or ``place`` does't change, no copy will be performed and return origin tensor, otherwise a new tensor will be constructed - and returned. Similarly, if the data is an numpy\.ndarray of with the same ``dtype`` - and the current place is cpu, no copy will be performed. + and returned. The ``ComplexTensor`` is a unique type of paddle. If x is ``ComplexTensor``, then ``x.real`` is the real part, and ``x.imag`` is the imaginary part. @@ -208,20 +208,20 @@ def to_tensor(data, dtype=None, place=None, stop_gradient=True): value=data, place=place, persistable=False, - zero_copy=True, + zero_copy=False, stop_gradient=stop_gradient) else: name = unique_name.generate('generated_tensor') real_tensor = paddle.Tensor( value=data.real, place=place, - zero_copy=True, + zero_copy=False, name=name + ".real", stop_gradient=stop_gradient) imag_tensor = paddle.Tensor( value=data.imag, place=place, - zero_copy=True, + zero_copy=False, name=name + ".imag", stop_gradient=stop_gradient) return paddle.ComplexTensor(real_tensor, imag_tensor) @@ -981,3 +981,90 @@ def diag(x, offset=0, padding_value=0, name=None): out.stop_gradient = True return out + + +def empty(shape, dtype=None, name=None): + """ + This Op returns a Tensor with uninitialized data which size is same as ``shape``. + + Args: + shape(list|tuple|Tensor): Shape of the Tensor to be created. + The data type of dimension of shape is ``int32`` or ``int64`` . If ``shape`` is a list or tuple, + the elements of it should be integers or Tensors with shape [1]. + If ``shape`` is an Tensor, it should be an 1-D Tensor. + dtype(np.dtype|str, optional): Data type of the output Tensor + which can be bool, float16, float32, float64, int32, int64, if dytpe is `None`, the data + type of created Tensor use global default dtype (see ``get_default_dtype`` + for details). + name(str, optional): The default value is None. Normally there is no need for user to set this + property. For more information, please refer to :ref:`api_guide_Name`. + + Returns: + Tensor: Tensor which is created according to ``shape`` and ``dtype``, and is uninitialized. + + Examples: + .. code-block:: python + + import paddle + import numpy as np + + paddle.disable_static() # Now we are in imperative mode + paddle.set_device("cpu") # and use cpu device + + # example 1: argument ``shape`` is a list which doesn't contain Tensor. + data1 = paddle.empty(shape=[2,3], dtype='float32') + #[[4.3612203e+27 1.8176809e+31 1.3555911e-19] # uninitialized + # [1.1699684e-19 1.3563156e-19 3.6408321e-11]] # uninitialized + + # example 2: argument ``shape`` is a Tensor, the data type must be int64 or int32. + shape_data = np.array([2, 3]).astype('int32') + shape = paddle.to_tensor(shape_data) + data2 = paddle.empty(shape=shape, dtype='float32') + #[[1.7192326e-37 4.8125365e-38 1.9866003e-36] # uninitialized + # [1.3284029e-40 7.1117408e-37 2.5353012e+30]] # uninitialized + + # example 3: argument ``shape`` is a list which contains Tensor. + dim2_data = np.array([3]).astype('int32') + dim2 = paddle.to_tensor(dim2_data) + data3 = paddle.empty(shape=[2, dim2], dtype='float32') + #[[1.1024214e+24 7.0379409e+22 6.5737699e-34] # uninitialized + # [7.5563101e+31 7.7130405e+31 2.8020654e+20]] # uninitialized + """ + + if dtype is None: + dtype = paddle.get_default_dtype() + + dtype = convert_dtype(dtype) + + if in_dygraph_mode(): + shape = utils.convert_shape_to_list(shape) + out = core.ops.empty('shape', shape, 'dtype', + convert_np_dtype_to_dtype_(dtype)) + out.stop_gradient = True + return out + + helper = LayerHelper("empty", **locals()) + inputs = {} + + check_dtype(dtype, 'dtype', + ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'], + 'empty') + check_type(shape, 'shape', (Variable, list, tuple), 'empty') + + if isinstance(shape, Variable): + check_dtype(shape.dtype, 'shape', ['int32', 'int64'], 'empty') + + attrs = {} + utils.get_shape_tensor_inputs( + inputs=inputs, attrs=attrs, shape=shape, op_type='empty') + + out = helper.create_variable_for_type_inference(dtype=dtype) + attrs['dtype'] = convert_np_dtype_to_dtype_(dtype) + helper.append_op( + type='empty', + inputs=inputs, + outputs={'Out': [out]}, + attrs=attrs, + stop_gradient=True) + out.stop_gradient = True + return out diff --git a/python/paddle/tensor/linalg.py b/python/paddle/tensor/linalg.py index 7ddda5091a0a260f56b29bcedfdcb0786e82ddd6..67e3ce21ffba0c312eb01163cdf32f87c6433ee1 100644 --- a/python/paddle/tensor/linalg.py +++ b/python/paddle/tensor/linalg.py @@ -183,12 +183,13 @@ def norm(x, p='fro', axis=None, keepdim=False, name=None): x (Tensor): The input tensor could be N-D tensor, and the input data type could be float32 or float64. p (float|string, optional): Order of the norm. Supported values are `fro`, `0`, `1`, `2`, - `inf`,`-inf` and any positive real number yielding the corresponding p-norm. - Not supported: ord < 0, nuclear norm. + `inf`, `-inf` and any positive real number yielding the corresponding p-norm. Not supported: ord < 0 and nuclear norm. + Default value is `fro`. axis (int|list|tuple, optional): The axis on which to apply norm operation. If axis is int or list(int)/tuple(int) with only one element, the vector norm is computed over the axis. If `axis < 0`, the dimension to norm operation is rank(input) + axis. If axis is a list(int)/tuple(int) with two elements, the matrix norm is computed over the axis. + Defalut value is `None`. keepdim (bool, optional): Whether to reserve the reduced dimension in the output Tensor. The result tensor will have fewer dimension than the :attr:`input` unless :attr:`keepdim` is true, default @@ -197,13 +198,9 @@ def norm(x, p='fro', axis=None, keepdim=False, name=None): user to set this property. For more information, please refer to :ref:`api_guide_Name`. Returns: - Variable: Tensor, results of norm operation on the specified axis of input tensor, + Tensor: results of norm operation on the specified axis of input tensor, it's data type is the same as input's Tensor. - Raises: - TypeError, if out data type is different with the input data type. - ValueError, If `p` or `axis` is invalid. - Examples: .. code-block:: python @@ -256,15 +253,13 @@ def norm(x, p='fro', axis=None, keepdim=False, name=None): "The dim of frobenius norm op should be None or two elements list!" ) if in_dygraph_mode(): - if dim is None: dim = [-1] - return core.ops.frobenius_norm(input, 'dim', dim, 'keepdim', - keepdim) - attrs = { - 'dim': dim if dim != None else [-2, -1], - 'keep_dim': keepdim, - 'reduce_all': False - } - if len(attrs['dim']) == len(input.shape): + if dim is None: + return core.ops.frobenius_norm(input, 'keep_dim', keepdim, + 'reduce_all', True) + return core.ops.frobenius_norm(input, 'dim', dim, 'keep_dim', + keepdim, 'reduce_all', False) + attrs = {'dim': dim, 'keep_dim': keepdim, 'reduce_all': False} + if dim is None: attrs['reduce_all'] = True check_variable_and_dtype(input, 'input', ['float32', 'float64'], 'frobenius_norm') @@ -351,42 +346,6 @@ def norm(x, p='fro', axis=None, keepdim=False, name=None): return reduce_out - def p0_matrix_norm(input, porder=0., axis=axis, keepdim=False, name=None): - block = LayerHelper('norm', **locals()) - out = block.create_variable_for_type_inference( - dtype=block.input_dtype()) - - cast_out = block.create_variable_for_type_inference(dtype=bool) - block.append_op( - type='cast', - inputs={'X': input}, - outputs={'Out': cast_out}, - attrs={ - 'in_dtype': input.dtype, - 'out_dtype': int(core.VarDesc.VarType.BOOL) - }) - cast_out2 = block.create_variable_for_type_inference(dtype=bool) - block.append_op( - type='cast', - inputs={'X': cast_out}, - outputs={'Out': cast_out2}, - attrs={ - 'in_dtype': cast_out.dtype, - 'out_dtype': int(core.VarDesc.VarType.FP32) - }) - sum_out = block.create_variable_for_type_inference( - dtype=block.input_dtype()) - block.append_op( - type='reduce_sum', - inputs={'X': cast_out2}, - outputs={'Out': sum_out}, - attrs={ - 'dim': axis, - 'keep_dim': keepdim, - 'reduce_all': True if axis is None else False - }) - return sum_out - def p_matrix_norm(input, porder=1., axis=axis, keepdim=False, name=None): block = LayerHelper('norm', **locals()) out = block.create_variable_for_type_inference( @@ -448,7 +407,20 @@ def norm(x, p='fro', axis=None, keepdim=False, name=None): #calculate vector norm, where axis is int or list with only one integer if isinstance(axis, int): - if isinstance(p, (int, float)): + if isinstance(p, str): + if p == "fro": + return vector_norm( + x, + porder=2, + axis=axis, + keepdim=keepdim, + asvector=False, + name=name) + + else: + raise ValueError( + "only valid string values are 'fro', found {}".format(p)) + elif isinstance(p, (int, float)): return vector_norm( x, axis=axis, @@ -464,10 +436,12 @@ def norm(x, p='fro', axis=None, keepdim=False, name=None): elif isinstance(axis, list) and len(axis) == 2: if p == "fro": return frobenius_norm(x, dim=axis, keepdim=keepdim, name=name) - elif p == 0: - return p0_matrix_norm(x, axis=axis, keepdim=keepdim, name=name) elif p == np.inf or p == -np.inf: return inf_norm(x, porder=p, axis=axis, keepdim=keepdim, name=name) + elif p == 0: + raise ValueError( + "just suport axis type int or list (length of list <=1) if p = 0, found {}". + format(axis)) else: return p_matrix_norm( x, porder=p, axis=axis, keepdim=keepdim, name=name) diff --git a/python/paddle/tensor/manipulation.py b/python/paddle/tensor/manipulation.py index 363c3ffceb85ef6168dc8c33b81185cac08083fb..9de407841fb461713d00f997afdf33a38a531245 100644 --- a/python/paddle/tensor/manipulation.py +++ b/python/paddle/tensor/manipulation.py @@ -21,6 +21,7 @@ from ..fluid.data_feeder import convert_dtype, check_variable_and_dtype, check_t from ..fluid.layers.tensor import fill_constant from ..fluid.layers import utils import numpy as np +import six # TODO: define functions to manipulate a tensor from ..fluid.layers import cast #DEFINE_ALIAS from ..fluid.layers import slice #DEFINE_ALIAS @@ -746,8 +747,6 @@ def unsqueeze(x, axis, name=None): print(out3.shape) # [1, 1, 1, 5, 10] """ - if isinstance(axis, int): - axis = [axis] return layers.unsqueeze(x, axis, name) @@ -1001,7 +1000,7 @@ def chunk(x, chunks, axis=0, name=None): x_np = np.random.random([3, 9, 5]).astype("int32") x = paddle.to_tensor(x_np) - out0, out1, out22 = paddle.chunk(x, chunks=3, axis=1) + out0, out1, out2 = paddle.chunk(x, chunks=3, axis=1) # out0.shape [3, 3, 5] # out1.shape [3, 3, 5] # out2.shape [3, 3, 5] @@ -1058,10 +1057,25 @@ def tile(x, repeat_times, name=None): """ if in_dygraph_mode(): return core.ops.tile(x, 'repeat_times', repeat_times) + check_type(repeat_times, 'repeat_times', (list, tuple, Variable), 'tile') + if isinstance(repeat_times, Variable): + assert len(repeat_times.shape) == 1, ( + 'repeat_times must be an 1-D Tensor.') + else: + for elem in repeat_times: + if isinstance(elem, Variable): + assert len(elem.shape) == 1, ( + 'Elements in repeat_times must be 1-D Tensors or integers.') + else: + if six.PY3: + type_tuple = (int, np.int32, np.int64) + elif six.PY2: + type_tuple = (int, long, np.int32, np.int64) + assert isinstance(elem, type_tuple), ( + 'Elements in repeat_times must be 1-D Tensors or integers.') check_variable_and_dtype( x, 'x', ['bool', 'float32', 'float64', 'int32', 'int64'], 'tile') - check_type(repeat_times, 'repeat_times', (list, tuple, Variable), 'tile') if convert_dtype(x.dtype) == 'bool' and x.stop_gradient == False: raise ValueError( "When the date type is bool for the input 'x' of tile op, you " @@ -1183,18 +1197,33 @@ def expand(x, shape, name=None): if in_dygraph_mode(): return core.ops.expand_v2(x, 'shape', shape) + if isinstance(shape, Variable): + assert len(shape.shape) == 1, ('shape must be an 1-D Tensor.') + else: + for elem in shape: + if isinstance(elem, Variable): + assert len(elem.shape) == 1, ( + 'Elements in shape must be 1-D Tensors or integers.') + else: + if six.PY3: + type_tuple = (int, np.int32, np.int64) + elif six.PY2: + type_tuple = (int, long, np.int32, np.int64) + assert isinstance(elem, type_tuple), ( + 'Elements in shape must be 1-D Tensors or integers.') + check_variable_and_dtype( x, 'x', ['bool', 'float32', 'float64', 'int32', 'int64'], 'expand') check_type(shape, 'shape', (list, tuple, Variable), 'expand') - - inputs = {"X": [x]} - attrs = {} if convert_dtype(x.dtype) == 'bool' and x.stop_gradient == False: raise ValueError("When the data type of input 'x' for expand is bool, " "you must set its stop_gradient to be False by " "some_var.stop_gradient = True, supporting " "some_var as the input.") + inputs = {"X": [x]} + attrs = {} + helper = LayerHelper('expand', **locals()) def get_attr_expand_shape(list_expand_shape): diff --git a/python/paddle/tensor/math.py b/python/paddle/tensor/math.py index ed2bbe03a366054dfe7d798310c7fa5d419b44a8..966544c7abb54ae7de163aa322890a55ee94d3d8 100755 --- a/python/paddle/tensor/math.py +++ b/python/paddle/tensor/math.py @@ -64,7 +64,6 @@ from ..fluid.layers import increment #DEFINE_ALIAS from ..fluid.layers import multiplex #DEFINE_ALIAS from ..fluid.layers import sums #DEFINE_ALIAS from ..fluid import layers -import paddle __all__ = [ @@ -343,69 +342,9 @@ def divide(x, y, name=None): axis = -1 act = None if in_dygraph_mode(): - # rule 1 : avoid numpy.ndarray - if isinstance(x, numpy.ndarray) or isinstance(y, numpy.ndarray): - raise TypeError("divide(): arguments must be Tensor or scalar, not numpy.ndarray.") - - # rule 2: both the inputs are not Tensor - elif not isinstance(x, paddle.Tensor) and not isinstance(y, paddle.Tensor): - x = paddle.full(shape=[1], dtype=paddle.get_default_dtype(), fill_value=x) - y = paddle.full(shape=[1], dtype=paddle.get_default_dtype(), fill_value=y) - - # rule 3: both the inputs are Tensor - elif isinstance(x, paddle.Tensor) and isinstance(y, paddle.Tensor): - if y.dtype != x.dtype: - raise TypeError("divide(): argument position 1 and argument position 2 must have the same dtype." - "But x is {}, y is {}".format(x.dtype, y.dtype)) - elif x.dtype in _supported_int_dtype_: - x = x.astype(paddle.get_default_dtype()) - y = y.astype(paddle.get_default_dtype()) - - # rule 4: x is Tensor, y is scalar - elif isinstance(x, paddle.Tensor) and not isinstance(y, paddle.Tensor): - if x.dtype in _supported_int_dtype_: - x = x.astype(paddle.get_default_dtype()) - y = paddle.full(shape=[1], dtype=x.dtype, fill_value=y) - - # rule 5: x is scalar, y is Tensor - elif not isinstance(x, paddle.Tensor) and isinstance(y, paddle.Tensor): - if y.dtype in _supported_int_dtype_: - y = y.astype(paddle.get_default_dtype()) - x = paddle.full(shape=[1], dtype=y.dtype, fill_value=x) - return _elementwise_op_in_dygraph( x, y, axis=axis, act=act, op_name=op_type) - # rule 1 : avoid numpy.ndarray - if isinstance(x, numpy.ndarray) or isinstance(y, numpy.ndarray): - raise TypeError("divide(): arguments must be Tensor or scalar, not numpy.ndarray.") - - # rule 2: both the inputs are not Tensor - elif not isinstance(x, Variable) and not isinstance(y, Variable): - x = paddle.fill_constant(shape=[1], dtype=paddle.get_default_dtype(), value=x) - y = paddle.fill_constant(shape=[1], dtype=paddle.get_default_dtype(), value=y) - - # rule 3: both the inputs are Tensor - elif isinstance(x, Variable) and isinstance(y, Variable): - if y.dtype != x.dtype: - raise TypeError("divide(): argument position 1 and argument position 2 must have the same dtype." - "But x is {}, y is {}".format(x.dtype, y.dtype)) - elif x.dtype in _supported_int_dtype_: - x = paddle.cast(x, paddle.get_default_dtype()) - y = paddle.cast(y, paddle.get_default_dtype()) - - # rule 4: x is Tensor, y is scalar - elif isinstance(x, Variable) and not isinstance(y, Variable): - if x.dtype in _supported_int_dtype_: - x = paddle.cast(x, paddle.get_default_dtype()) - y = paddle.fill_constant(shape=[1], dtype=x.dtype, value=y) - - # rule 5: x is scalar, y is Tensor - elif not isinstance(x, Variable) and isinstance(y, Variable): - if y.dtype in _supported_int_dtype_: - y = paddle.cast(y, paddle.get_default_dtype()) - x = paddle.fill_constant(shape=[1], dtype=y.dtype, value=x) - return _elementwise_op(LayerHelper(op_type, **locals())) @@ -444,55 +383,9 @@ def floor_divide(x, y, name=None): op_type = 'elementwise_floordiv' axis = -1 if in_dygraph_mode(): - # rule 1 : avoid numpy.ndarray - if isinstance(x, numpy.ndarray) or isinstance(y, numpy.ndarray): - raise TypeError("floor_divide(): arguments must be Tensor or scalar, not numpy.ndarray.") - - # rule 2: both the inputs are not Tensor - elif not isinstance(x, paddle.Tensor) and not isinstance(y, paddle.Tensor): - x = paddle.full(shape=[1], dtype=paddle.get_default_dtype(), fill_value=x) - y = paddle.full(shape=[1], dtype=paddle.get_default_dtype(), fill_value=y) - - # rule 3: both the inputs are Tensor - elif isinstance(x, paddle.Tensor) and isinstance(y, paddle.Tensor): - if y.dtype != x.dtype: - raise TypeError("floor_divide(): argument position 1 and argument position 2 must have the same dtype." - "But x is {}, y is {}".format(x.dtype, y.dtype)) - - # rule 4: x is Tensor, y is scalar - elif isinstance(x, paddle.Tensor) and not isinstance(y, paddle.Tensor): - y = paddle.full(shape=[1], dtype=x.dtype, fill_value=y) - - # rule 5: x is scalar, y is Tensor - elif not isinstance(x, paddle.Tensor) and isinstance(y, paddle.Tensor): - x = paddle.full(shape=[1], dtype=y.dtype, fill_value=x) - return _elementwise_op_in_dygraph( x, y, axis=axis, op_name=op_type) - # rule 1 : avoid numpy.ndarray - if isinstance(x, numpy.ndarray) or isinstance(y, numpy.ndarray): - raise TypeError("divide(): arguments must be Tensor or scalar, not numpy.ndarray.") - - # rule 2: both the inputs are not Tensor - elif not isinstance(x, Variable) and not isinstance(y, Variable): - x = paddle.fill_constant(shape=[1], dtype=paddle.get_default_dtype(), value=x) - y = paddle.fill_constant(shape=[1], dtype=paddle.get_default_dtype(), value=y) - - # rule 3: both the inputs are Tensor - elif isinstance(x, Variable) and isinstance(y, Variable): - if y.dtype != x.dtype: - raise TypeError("divide(): argument position 1 and argument position 2 must have the same dtype." - "But x is {}, y is {}".format(x.dtype, y.dtype)) - - # rule 4: x is Tensor, y is scalar - elif isinstance(x, Variable) and not isinstance(y, Variable): - y = paddle.fill_constant(shape=[1], dtype=x.dtype, value=y) - - # rule 5: x is scalar, y is Tensor - elif not isinstance(x, Variable) and isinstance(y, Variable): - x = paddle.fill_constant(shape=[1], dtype=y.dtype, value=x) - return _elementwise_op(LayerHelper(op_type, **locals())) @@ -531,43 +424,9 @@ def remainder(x, y, name=None): op_type = 'elementwise_mod' axis = -1 if in_dygraph_mode(): - # rule 1 : avoid numpy.ndarray - if isinstance(x, numpy.ndarray) or isinstance(y, numpy.ndarray): - raise TypeError("remainder(): arguments must be Tensor or scalar, not numpy.ndarray.") - - elif not isinstance(x, paddle.Tensor): - raise TypeError("remainder(): arguments position 1 must be Tensor, not {}".format(type(x))) - - # rule 3: both the inputs are Tensor - elif isinstance(y, paddle.Tensor): - if y.dtype != x.dtype: - raise TypeError("remainder(): argument position 1 and argument position 2 must have the same dtype." - "But x is {}, y is {}".format(x.dtype, y.dtype)) - - # rule 4: x is Tensor, y is scalar - elif not isinstance(y, paddle.Tensor): - y = paddle.full(shape=[1], dtype=x.dtype, fill_value=y) - return _elementwise_op_in_dygraph( x, y, axis=axis, op_name=op_type) - # rule 1 : avoid numpy.ndarray - if isinstance(x, numpy.ndarray) or isinstance(y, numpy.ndarray): - raise TypeError("remainder(): arguments must be Tensor or scalar, not numpy.ndarray.") - - elif not isinstance(x, Variable): - raise TypeError("remainder(): arguments position 1 must be Tensor, not {}".format(type(x))) - - # rule 3: both the inputs are Tensor - elif isinstance(y, Variable): - if y.dtype != x.dtype: - raise TypeError("remainder(): argument position 1 and argument position 2 must have the same dtype." - "But x is {}, y is {}".format(x.dtype, y.dtype)) - - # rule 4: x is Tensor, y is scalar - elif not isinstance(y, paddle.Tensor): - y = paddle.fill_constant(shape=[1], dtype=x.dtype, value=y) - return _elementwise_op(LayerHelper(op_type, **locals())) @@ -1194,15 +1053,14 @@ def logsumexp(x, axis=None, keepdim=False, name=None): axis = [0] if in_dygraph_mode(): - return core.ops.logsumexp(x, 'dim', axis, 'keep_dim', keepdim, - 'reduce_all', reduce_all) + return core.ops.logsumexp(x, 'axis', axis, 'keepdim', keepdim, 'reduce_all', reduce_all) check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'logsumexp') helper = LayerHelper('logsumexp', **locals()) - attrs = {'dim': axis, 'keep_dim': keepdim, 'reduce_all': reduce_all} + attrs = {'axis': axis, 'keepdim': keepdim, 'reduce_all':reduce_all} out = helper.create_variable_for_type_inference(x.dtype) helper.append_op( type='logsumexp', inputs={'X': x}, outputs={'Out': out}, attrs=attrs) diff --git a/python/paddle/tensor/search.py b/python/paddle/tensor/search.py index ce03d0ef15f0f80f4e01cf57bc8cc449186c2560..f55d285586f0ec6959573af64e720bea5de10c8d 100644 --- a/python/paddle/tensor/search.py +++ b/python/paddle/tensor/search.py @@ -167,10 +167,10 @@ def argmax(x, axis=None, keepdim=False, dtype="int64", name=None): "The type of 'axis' must be int or None in argmax, but received %s." % (type(axis))) - if not (isinstance(dtype, str) or isinstance(dtype, np.dtype)): - raise TypeError( - "the type of 'dtype' in argmax must be str or np.dtype, but received {}". - format(type(dtype))) + if dtype is None: + raise ValueError( + "the value of 'dtype' in argmax could not be None, but received None" + ) var_dtype = convert_np_dtype_to_dtype_(dtype) check_dtype(var_dtype, 'dtype', ['int32', 'int64'], 'argmin') @@ -245,10 +245,10 @@ def argmin(x, axis=None, keepdim=False, dtype="int64", name=None): "The type of 'axis' must be int or None in argmin, but received %s." % (type(axis))) - if not (isinstance(dtype, str) or isinstance(dtype, np.dtype)): - raise TypeError( - "the type of 'dtype' in argmin must be str or np.dtype, but received {}". - format(dtype(dtype))) + if dtype is None: + raise ValueError( + "the value of 'dtype' in argmin could not be None, but received None" + ) var_dtype = convert_np_dtype_to_dtype_(dtype) check_dtype(var_dtype, 'dtype', ['int32', 'int64'], 'argmin') diff --git a/python/paddle/tests/test_model.py b/python/paddle/tests/test_model.py index b7b5d44650f8d62926241a57feedfd5b932a37f5..5c4e98feaa686217bc78ad3915423593ad4fcdce 100644 --- a/python/paddle/tests/test_model.py +++ b/python/paddle/tests/test_model.py @@ -523,6 +523,24 @@ class TestModelFunction(unittest.TestCase): model.summary(input_size=[(20)]) model.summary(input_size=(20), batch_size=2) + def test_summary_nlp(self): + paddle.enable_static() + nlp_net = paddle.nn.GRU(input_size=2, hidden_size=3, num_layers=3) + paddle.summary(nlp_net, (1, 2)) + + def test_summary_error(self): + with self.assertRaises(TypeError): + nlp_net = paddle.nn.GRU(input_size=2, hidden_size=3, num_layers=3) + paddle.summary(nlp_net, (1, '2')) + + with self.assertRaises(ValueError): + nlp_net = paddle.nn.GRU(input_size=2, hidden_size=3, num_layers=3) + paddle.summary(nlp_net, (-1, -1)) + + paddle.disable_static() + nlp_net = paddle.nn.GRU(input_size=2, hidden_size=3, num_layers=3) + paddle.summary(nlp_net, (1, 2)) + def test_export_deploy_model(self): for dynamic in [True, False]: fluid.enable_dygraph() if dynamic else None diff --git a/python/setup.py.in b/python/setup.py.in index 64ac2b9b9a4d210c59193e117c6000986bfb07a0..773166400347ab550f82e4fabcb0d89b90818fc2 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -156,6 +156,7 @@ packages=['paddle', 'paddle.framework', 'paddle.jit', 'paddle.fluid', + 'paddle.fluid.inference', 'paddle.fluid.dygraph', 'paddle.fluid.dygraph.dygraph_to_static', 'paddle.fluid.dygraph.amp', diff --git a/python/unittest_py/requirements.txt b/python/unittest_py/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3b6a7546616678c21d4241da42f0965d4b85f2e6 --- /dev/null +++ b/python/unittest_py/requirements.txt @@ -0,0 +1,4 @@ +PyGithub +coverage +pycrypto +mock diff --git a/tools/check_file_diff_approvals.sh b/tools/check_file_diff_approvals.sh index 1e5179d0282d7f35c4232d9b9783cb831e83f462..84254cc89bb8eef12a95189416cd29cce828f5ca 100644 --- a/tools/check_file_diff_approvals.sh +++ b/tools/check_file_diff_approvals.sh @@ -287,12 +287,19 @@ fi pip install PyGithub # For getting PR related data wget https://paddle-ci.gz.bcebos.com/blk/block.txt --no-check-certificate +wget https://sys-p0.bj.bcebos.com/bk-ci/bk.txt --no-check-certificate HASUTFIXED=`python ${PADDLE_ROOT}/tools/check_ut.py | grep "has unit-test to be fixed" || true` if [ "${HASUTFIXED}" != "" ]; then echo_line="${HASUTFIXED} You must have one RD (chalsliu (Recommend) or kolinwei) approval.\n" check_approval 1 45041955 22165420 fi +HASUTFIXED=`python ${PADDLE_ROOT}/tools/check_ut.py | grep "has benchmark issue to be fixed" || true` +if [ "${HASUTFIXED}" != "" ]; then + echo_line="${HASUTFIXED} You must have one RD (hysunflower or xiegegege or Xreki) approval.\n" + check_approval 1 52739577 46314656 12538138 +fi + if [ -n "${echo_list}" ];then echo "****************" echo -e "${echo_list[@]}" diff --git a/tools/check_ut.py b/tools/check_ut.py index 7b5e5a4f1c55574edc3f28dac76ebf1d932748d7..f5fe4c687dd7828f001ddbab744d66931e37f532 100644 --- a/tools/check_ut.py +++ b/tools/check_ut.py @@ -27,9 +27,12 @@ class PRChecker(object): self.github = Github(os.getenv('GITHUB_API_TOKEN'), timeout=60) self.repo = None - def check(self): - """ check pr. """ - filename = 'block.txt' + def check(self, filename, msg): + """ + Args: + filename (str): File to get block names. + msg (str): Error message. + """ pr_id = os.getenv('GIT_PR_ID') if not pr_id: print('No PR ID') @@ -44,12 +47,10 @@ class PRChecker(object): with open(filename) as f: for l in f: if l.rstrip('\r\n') == user: - print('{} has unit-test to be fixed, so CI failed.'.format( - user)) - exit(1) - exit(0) + print('{} {}'.format(user, msg)) if __name__ == '__main__': pr_checker = PRChecker() - pr_checker.check() + pr_checker.check('block.txt', 'has unit-test to be fixed, so CI failed.') + pr_checker.check('bk.txt', 'has benchmark issue to be fixed, so CI failed.') diff --git a/tools/coverage/coverage_diff.py b/tools/coverage/coverage_diff.py index 051348d358f459a67d39dd9ca798721a82aa2233..38f671fe4089d1f94caafaf26640e4df75870f55 100644 --- a/tools/coverage/coverage_diff.py +++ b/tools/coverage/coverage_diff.py @@ -90,12 +90,12 @@ def get_info_file_lines(info_file, diff_file): continue elif line.startswith('LF:'): - print 'LF:{}'.format(current_lf) + print('LF:{}'.format(current_lf)) continue elif line.startswith('LH:'): - print 'LH:{}'.format(current_lh) + print('LH:{}'.format(current_lh)) continue diff --git a/tools/coverage/coverage_diff_list.py b/tools/coverage/coverage_diff_list.py index 57222da4d9818ebbcb0b9aeea6937494038eecdf..8975185edadfbd567a428bbd90523923f5ab675d 100644 --- a/tools/coverage/coverage_diff_list.py +++ b/tools/coverage/coverage_diff_list.py @@ -40,7 +40,7 @@ def filter_by(list_file, max_rate): except: pass - print name, rate + print(name, rate) if __name__ == '__main__': diff --git a/tools/coverage/coverage_lines.py b/tools/coverage/coverage_lines.py index eb846cc9f249a3ff177fde437362a1122f4409a5..cdec5b8b1bb1873f8b9ef761e9d8575c89fee234 100644 --- a/tools/coverage/coverage_lines.py +++ b/tools/coverage/coverage_lines.py @@ -33,7 +33,7 @@ def get_lines(info_file): hits += 1 if total == 0: - print 'no data found' + print('no data found') exit() return hits / total @@ -47,17 +47,17 @@ if __name__ == '__main__': expected = float(sys.argv[2]) if not os.path.isfile(info_file): - print 'info file {} is not exists, ignored'.format(info_file) + print('info file {} is not exists, ignored'.format(info_file)) exit() actual = get_lines(info_file) actual = round(actual, 3) if actual < expected: - print 'expected >= {} %, actual {} %, failed'.format( - round(expected * 100, 1), round(actual * 100, 1)) + print('expected >= {} %, actual {} %, failed'.format( + round(expected * 100, 1), round(actual * 100, 1))) exit(1) - print 'expected >= {} %, actual {} %, passed'.format( - round(expected * 100, 1), round(actual * 100, 1)) + print('expected >= {} %, actual {} %, passed'.format( + round(expected * 100, 1), round(actual * 100, 1))) diff --git a/tools/coverage/paddle_coverage.sh b/tools/coverage/paddle_coverage.sh index d54434b738db5b5e6192f7d3bf9e48edbebc5b7c..008b35d01ca565a6d32265f595dd2d6aa55be707 100644 --- a/tools/coverage/paddle_coverage.sh +++ b/tools/coverage/paddle_coverage.sh @@ -5,7 +5,7 @@ set -xe PADDLE_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}")/../../" && pwd )" # install lcov -curl -o /lcov-1.14.tar.gz -x "" -s https://paddle-ci.gz.bcebos.com/coverage/lcov-1.14.tar.gz +curl -o /lcov-1.14.tar.gz -x "" -s https://paddle-ci.gz.bcebos.com/coverage/lcov-1.14.tar.gz || exit 101 tar -xf /lcov-1.14.tar.gz -C / cd /lcov-1.14 make install @@ -14,7 +14,7 @@ make install cd /paddle/build -python ${PADDLE_ROOT}/tools/coverage/gcda_clean.py ${GIT_PR_ID} +python3 ${PADDLE_ROOT}/tools/coverage/gcda_clean.py ${GIT_PR_ID} lcov --capture -d ./ -o coverage.info --rc lcov_branch_coverage=0 @@ -53,9 +53,9 @@ gen_full_html_report || true function gen_diff_html_report() { if [ "${GIT_PR_ID}" != "" ]; then - COVERAGE_DIFF_PATTERN="`python ${PADDLE_ROOT}/tools/coverage/pull_request.py files ${GIT_PR_ID}`" + COVERAGE_DIFF_PATTERN="`python3 ${PADDLE_ROOT}/tools/coverage/pull_request.py files ${GIT_PR_ID}`" - python ${PADDLE_ROOT}/tools/coverage/pull_request.py diff ${GIT_PR_ID} > git-diff.out + python3 ${PADDLE_ROOT}/tools/coverage/pull_request.py diff ${GIT_PR_ID} > git-diff.out fi lcov --extract coverage-full.info \ @@ -63,7 +63,7 @@ function gen_diff_html_report() { -o coverage-diff.info \ --rc lcov_branch_coverage=0 - python ${PADDLE_ROOT}/tools/coverage/coverage_diff.py coverage-diff.info git-diff.out > coverage-diff.tmp + python3 ${PADDLE_ROOT}/tools/coverage/coverage_diff.py coverage-diff.info git-diff.out > coverage-diff.tmp mv -f coverage-diff.tmp coverage-diff.info @@ -82,7 +82,7 @@ set -x coverage xml -i -o python-coverage.xml -python ${PADDLE_ROOT}/tools/coverage/python_coverage.py > python-coverage.info +python3 ${PADDLE_ROOT}/tools/coverage/python_coverage.py > python-coverage.info # python full html report # @@ -143,5 +143,6 @@ echo "Assert Python Diff Coverage" python ${PADDLE_ROOT}/tools/coverage/coverage_lines.py python-coverage-diff.info 0.9 || PYTHON_COVERAGE_LINES_ASSERT=1 if [ "$COVERAGE_LINES_ASSERT" = "1" ] || [ "$PYTHON_COVERAGE_LINES_ASSERT" = "1" ]; then + echo "exit 9" > /tmp/paddle_coverage.result exit 9 fi diff --git a/tools/coverage/pull_request.py b/tools/coverage/pull_request.py index 979f476d2a1f3fbf28d43e2b717deb7b2d1b0fff..105460032f7db538eaf7a193776bf8085e2837a1 100644 --- a/tools/coverage/pull_request.py +++ b/tools/coverage/pull_request.py @@ -40,7 +40,7 @@ def get_files(args): pull = get_pull(args.pull_id) for file in pull.get_files(): - print '/paddle/{}'.format(file.filename) + print('/paddle/{}'.format(file.filename)) def diff(args): @@ -55,8 +55,8 @@ def diff(args): pull = get_pull(args.pull_id) for file in pull.get_files(): - print '+++ {}'.format(file.filename) - print file.patch + print('+++ {}'.format(file.filename)) + print(file.patch) if __name__ == '__main__': diff --git a/tools/coverage/python_coverage.py b/tools/coverage/python_coverage.py index ba67e12249bb0ccee608ea120321eaa3a2ccbc91..8ad9d85c1bf6b5ed542fb8469173c4f1815050a4 100644 --- a/tools/coverage/python_coverage.py +++ b/tools/coverage/python_coverage.py @@ -12,10 +12,7 @@ root = tree.getroot() sources = root.findall('sources/source') -if len(sources) > 1: - exit(1) - -source = sources[0].text +source = sources[-1].text for clazz in root.findall('packages/package/classes/class'): clazz_filename = clazz.attrib.get('filename') @@ -28,8 +25,8 @@ for clazz in root.findall('packages/package/classes/class'): if not path.exists(clazz_filename): continue - print 'TN:' - print 'SF:{}'.format(clazz_filename) + print('TN:') + print('SF:{}'.format(clazz_filename)) branch_index = 0 @@ -50,16 +47,16 @@ for clazz in root.findall('packages/package/classes/class'): taken = int(taken) for _ in range(taken): - print 'BRDA:{},{},{},{}'.format(line_number, 0, branch_index, - line_hits) + print('BRDA:{},{},{},{}'.format(line_number, 0, branch_index, + line_hits)) branch_index += 1 if line_missing_branches: for missing_branch in line_missing_branches.split(','): - print 'BRDA:{},{},{},{}'.format(line_number, 0, - branch_index, 0) + print('BRDA:{},{},{},{}'.format(line_number, 0, + branch_index, 0)) branch_index += 1 - print 'DA:{},{}'.format(line_number, line_hits) + print('DA:{},{}'.format(line_number, line_hits)) - print 'end_of_record' + print('end_of_record') diff --git a/tools/enforce/count_all_enforce.sh b/tools/enforce/count_all_enforce.sh index c1b7508de0361b7a9036557f88fd0b10f326dcc6..683b73614d29bb42871c63dc94d365626d0375ad 100644 --- a/tools/enforce/count_all_enforce.sh +++ b/tools/enforce/count_all_enforce.sh @@ -39,7 +39,7 @@ # Valid PADDLE_ENFORCE{_**} & PADDLE_THROW Count: 1706 # Invalid PADDLE_ENFORCE{_**} & PADDLE_THROW Count: 4572 -ROOT_DIR=../paddle/fluid +ROOT_DIR=../../paddle/fluid ALL_PADDLE_CHECK_CNT=0 VALID_PADDLE_CHECK_CNT=0 diff --git a/tools/enforce/count_enforce_by_dir.sh b/tools/enforce/count_enforce_by_dir.sh index 03233d417ac88eef775e1ca6a77d0600a4faa361..3cb13edf7cc27f6a0de45080a7b90e4b4e24b6b5 100644 --- a/tools/enforce/count_enforce_by_dir.sh +++ b/tools/enforce/count_enforce_by_dir.sh @@ -59,7 +59,7 @@ . ./count_all_enforce.sh --source-only -ROOT_DIR=../paddle/fluid +ROOT_DIR=../../paddle/fluid function count_dir_independently(){ local sub_dir_total_check_cnt=0