diff --git a/CMakeLists.txt b/CMakeLists.txt index 9b77659f6142da3c8b6bb4913a8219683b723a76..26d94384a9150735aa8341fd8a18cb039895ff91 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -71,11 +71,11 @@ option(ANAKIN_BUILD_CROSS_PLANTFORM "Build anakin lib for any nvidia device plan option(WITH_GRPC "Use grpc as the default rpc framework" ${WITH_DISTRIBUTE}) option(WITH_BRPC_RDMA "Use brpc rdma as the rpc protocal" OFF) option(ON_INFER "Turn on inference optimization." OFF) -option(WITH_INFERENCE_API_TEST "Test fluid inference high-level api interface" OFF) +option(WITH_INFERENCE_API_TEST "Test fluid inference C++ high-level api interface" OFF) +option(WITH_HIGH_LEVEL_API_TEST "Test fluid python high-level api interface" OFF) option(WITH_SYSTEM_BLAS "Use system blas library" OFF) option(PY_VERSION "Compile PaddlePaddle with python3 support" ${PY_VERSION}) option(WITH_FAST_MATH "Make use of fast math library, might affect the precision to some extent" ON) -option(WITH_WBAES "Compile PaddlePaddle with WBAES support" ON) # PY_VERSION if(NOT PY_VERSION) @@ -149,7 +149,6 @@ include(external/dlpack) include(external/snappy) # download snappy include(external/snappystream) # download snappystream include(external/warpctc) # download, build, install warpctc -include(external/wbaes) # download wbaes if (NOT WIN32) # there is no official support of nccl, cupti in windows diff --git a/cmake/configure.cmake b/cmake/configure.cmake index 283845541b8e303babeed7ed9f9ece2d51a6a2fc..93d74bb0a8f726ad31685cbfc7831b5441cd5108 100644 --- a/cmake/configure.cmake +++ b/cmake/configure.cmake @@ -157,7 +157,3 @@ endif(WITH_BRPC_RDMA) if(ON_INFER) add_definitions(-DPADDLE_ON_INFERENCE) endif(ON_INFER) - -if(WITH_WBAES) - add_definitions(-DPADDLE_WITH_WBAES) -endif(WITH_WBAES) diff --git a/cmake/external/protobuf.cmake b/cmake/external/protobuf.cmake index 69da9b98198de358348621ecdb444f2f81c7757f..09eb437aede4364f8aa285d5296f21cd8460fca1 100644 --- a/cmake/external/protobuf.cmake +++ b/cmake/external/protobuf.cmake @@ -221,6 +221,7 @@ FUNCTION(build_protobuf TARGET_NAME BUILD_FOR_HOST) -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${PROTOBUF_INSTALL_DIR} -DCMAKE_INSTALL_LIBDIR=lib + -DBUILD_SHARED_LIBS=OFF CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${PROTOBUF_INSTALL_DIR} -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE} diff --git a/cmake/external/wbaes.cmake b/cmake/external/wbaes.cmake deleted file mode 100644 index feda5cb367aeb532702c9ab8560388d1207c201c..0000000000000000000000000000000000000000 --- a/cmake/external/wbaes.cmake +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -IF(NOT ${WITH_WBAES}) - return() -ENDIF(NOT ${WITH_WBAES}) - -INCLUDE(ExternalProject) -SET(WBAES_DST_DIR "wbaes") -SET(WBAES_INSTALL_ROOT "${THIRD_PARTY_PATH}/install") -SET(WBAES_INSTALL_DIR ${WBAES_INSTALL_ROOT}/${WBAES_DST_DIR}) -SET(WBAES_ROOT ${WBAES_INSTALL_DIR}) -SET(WBAES_INC_DIR ${WBAES_ROOT}/include) -SET(WBAES_LIB_DIR ${WBAES_ROOT}/lib) - -SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}" "${WBAES_ROOT}/lib") -SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) - -IF(APPLE) - SET(WBAES_TAG "v1.0.0" CACHE STRING "" FORCE) - SET(WBAES_URL "http://paddlepaddledeps.bj.bcebos.com/wbaes-sdk.mac.${WBAES_TAG}.tgz" CACHE STRING "" FORCE) - SET(WBAES_LIB ${WBAES_LIB_DIR}/libwbaes.dylib) - SET(WBAES_SHARED_LIB ${WBAES_LIB_DIR}/libwbaes.dylib) -ELSEIF(WIN32) - SET(WBAES_TAG "v1.0.0" CACHE STRING "" FORCE) - SET(WBAES_URL "http://paddlepaddledeps.bj.bcebos.com/wbaes-sdk.windows-x64.${WBAES_TAG}.tgz" CACHE STRING "" FORCE) - SET(WBAES_LIB ${WBAES_LIB_DIR}/libwbaes.lib) - SET(WBAES_SHARED_LIB ${WBAES_LIB_DIR}/libwbaes.dll) -ELSE() - SET(WBAES_TAG "v1.0.2" CACHE STRING "" FORCE) - SET(WBAES_URL "http://paddlepaddledeps.bj.bcebos.com/wbaes-sdk.linux-x86_64.${WBAES_TAG}.tgz" CACHE STRING "" FORCE) - SET(WBAES_LIB ${WBAES_LIB_DIR}/libwbaes.so) - SET(WBAES_SHARED_LIB ${WBAES_LIB_DIR}/libwbaes.so) -ENDIF() - -SET(WBAES_PROJECT "extern_wbaes") -MESSAGE(STATUS "WBAES_URL: ${WBAES_URL}, WBAES_LIB: ${WBAES_LIB}") -SET(WBAES_SOURCE_DIR "${THIRD_PARTY_PATH}/wbaes") -SET(WBAES_DOWNLOAD_DIR "${WBAES_SOURCE_DIR}/src/${WBAES_PROJECT}") - -ExternalProject_Add( - ${WBAES_PROJECT} - ${EXTERNAL_PROJECT_LOG_ARGS} - PREFIX ${WBAES_SOURCE_DIR} - URL ${WBAES_URL} - DOWNLOAD_DIR ${WBAES_DOWNLOAD_DIR} - DOWNLOAD_NO_PROGRESS 1 - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" - ${CMAKE_COMMAND} -E copy_directory ${WBAES_DOWNLOAD_DIR}/include ${WBAES_INC_DIR} && - ${CMAKE_COMMAND} -E copy_directory ${WBAES_DOWNLOAD_DIR}/lib ${WBAES_LIB_DIR} -) - -INCLUDE_DIRECTORIES(${WBAES_INC_DIR}) - -ADD_LIBRARY(wbaes SHARED IMPORTED GLOBAL) -SET_PROPERTY(TARGET wbaes PROPERTY IMPORTED_LOCATION ${WBAES_LIB}) -SET_PROPERTY(TARGET wbaes PROPERTY IMPORTED_NO_SONAME 1) -ADD_DEPENDENCIES(wbaes ${WBAES_PROJECT}) diff --git a/cmake/generic.cmake b/cmake/generic.cmake index 19110812c240db4cbe3ba73a3a42ab0f1511a115..6679a09dfc9dd00cfe3b5c5da3e12bd1c1389432 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -264,14 +264,6 @@ function(cc_library TARGET_NAME) list(REMOVE_ITEM cc_library_DEPS warpctc) add_dependencies(${TARGET_NAME} warpctc) endif() - # Only deps libwbaes.so, not link - if("${cc_library_DEPS};" MATCHES "wbaes;") - list(REMOVE_ITEM cc_library_DEPS wbaes) - if(NOT "${TARGET_NAME}" MATCHES "dynload_wbaes") - list(APPEND cc_library_DEPS dynload_wbaes) - endif() - add_dependencies(${TARGET_NAME} wbaes) - endif() # Only deps libmklml.so, not link if("${cc_library_DEPS};" MATCHES "mklml;") list(REMOVE_ITEM cc_library_DEPS mklml) diff --git a/cmake/inference_lib.cmake b/cmake/inference_lib.cmake index 2f558bffbd11a59699e050e6c8a53bca4cbb0884..b7c32f80db0dcb826f3f67ffb55da1c715785add 100644 --- a/cmake/inference_lib.cmake +++ b/cmake/inference_lib.cmake @@ -170,14 +170,6 @@ copy(snappystream_lib DSTS ${dst_dir} ${dst_dir}/lib DEPS snappystream) -if (WITH_WBAES) - set(dst_dir "${FLUID_INSTALL_DIR}/third_party/install/wbaes") - copy(wbaes_lib - SRCS ${WBAES_INC_DIR} ${WBAES_LIB} - DSTS ${dst_dir} ${dst_dir}/lib - DEPS wbaes) -endif () - set(dst_dir "${FLUID_INSTALL_DIR}/third_party/install/zlib") copy(zlib_lib SRCS ${ZLIB_INCLUDE_DIR} ${ZLIB_LIBRARIES} diff --git a/paddle/fluid/API.spec b/paddle/fluid/API.spec index b19d50a6ad6afa312f5e695583174e56bf490755..bf39325cc9bfb258051ec1a7fc7f5eb139c60133 100644 --- a/paddle/fluid/API.spec +++ b/paddle/fluid/API.spec @@ -13,6 +13,7 @@ paddle.fluid.name_scope (ArgSpec(args=['prefix'], varargs=None, keywords=None, d paddle.fluid.cuda_places (ArgSpec(args=['device_ids'], varargs=None, keywords=None, defaults=(None,)), ('document', '7d9a51fc9cf3c5245b5227080a8064c3')) paddle.fluid.cpu_places (ArgSpec(args=['device_count'], varargs=None, keywords=None, defaults=(None,)), ('document', '4c0cd83f0b401fc2ff84c70974e5d210')) paddle.fluid.cuda_pinned_places (ArgSpec(args=['device_count'], varargs=None, keywords=None, defaults=(None,)), ('document', 'd0c3ebd813c39958c92b78e3eef7e912')) +paddle.fluid.in_dygraph_mode (ArgSpec(args=[], varargs=None, keywords=None, defaults=None), ('document', 'f06314a1cb30c96b5808dde2219c2dae')) paddle.fluid.Executor.__init__ (ArgSpec(args=['self', 'place'], varargs=None, keywords=None, defaults=None), ('document', '6adf97f83acf6453d4a6a4b1070f3754')) paddle.fluid.Executor.close (ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None), ('document', 'f5369953dd0c443961cf79f7a00e1a03')) paddle.fluid.Executor.infer_from_dataset (ArgSpec(args=['self', 'program', 'dataset', 'scope', 'thread', 'debug', 'fetch_list', 'fetch_info', 'print_period'], varargs=None, keywords=None, defaults=(None, None, None, 0, False, None, None, 100)), ('document', '9c7decb955b9c4f718114179c8985581')) @@ -117,6 +118,8 @@ paddle.fluid.layers.reduce_mean (ArgSpec(args=['input', 'dim', 'keep_dim', 'name paddle.fluid.layers.reduce_max (ArgSpec(args=['input', 'dim', 'keep_dim', 'name'], varargs=None, keywords=None, defaults=(None, False, None)), ('document', '66a622db727551761ce4eb73eaa7f6a4')) paddle.fluid.layers.reduce_min (ArgSpec(args=['input', 'dim', 'keep_dim', 'name'], varargs=None, keywords=None, defaults=(None, False, None)), ('document', 'd50ac552b5d131468ed466d08bb2d38c')) paddle.fluid.layers.reduce_prod (ArgSpec(args=['input', 'dim', 'keep_dim', 'name'], varargs=None, keywords=None, defaults=(None, False, None)), ('document', 'fcd8301a0ce15f219c7a4bcd0c1e8eca')) +paddle.fluid.layers.reduce_all (ArgSpec(args=['input', 'dim', 'keep_dim', 'name'], varargs=None, keywords=None, defaults=(None, False, None)), ('document', '646ca4d4a2cc16084f59de44b6927eca')) +paddle.fluid.layers.reduce_any (ArgSpec(args=['input', 'dim', 'keep_dim', 'name'], varargs=None, keywords=None, defaults=(None, False, None)), ('document', 'f36661060aeeaf6c6b1331e41b3726fa')) paddle.fluid.layers.sequence_first_step (ArgSpec(args=['input'], varargs=None, keywords=None, defaults=None), ('document', '2b290d3d77882bfe9bb8d331cac8cdd3')) paddle.fluid.layers.sequence_last_step (ArgSpec(args=['input'], varargs=None, keywords=None, defaults=None), ('document', 'c16a892f44f7fe71bfa5afc32d3f34ce')) paddle.fluid.layers.sequence_slice (ArgSpec(args=['input', 'offset', 'length', 'name'], varargs=None, keywords=None, defaults=(None,)), ('document', 'fdcea0e8b5bc7d8d4b1b072c521014e6')) @@ -124,7 +127,7 @@ paddle.fluid.layers.dropout (ArgSpec(args=['x', 'dropout_prob', 'is_test', 'seed paddle.fluid.layers.split (ArgSpec(args=['input', 'num_or_sections', 'dim', 'name'], varargs=None, keywords=None, defaults=(-1, None)), ('document', '652625345c2acb900029c78cc75f8aa6')) paddle.fluid.layers.ctc_greedy_decoder (ArgSpec(args=['input', 'blank', 'name'], varargs=None, keywords=None, defaults=(None,)), ('document', 'ebbf2adbd79683dc93db03454dfa18c2')) paddle.fluid.layers.edit_distance (ArgSpec(args=['input', 'label', 'normalized', 'ignored_tokens'], varargs=None, keywords=None, defaults=(True, None)), ('document', '97f0262f97602644c83142789d784571')) -paddle.fluid.layers.l2_normalize (ArgSpec(args=['x', 'axis', 'epsilon', 'name'], varargs=None, keywords=None, defaults=(1e-12, None)), ('document', '6e428384ce6a77207fa2c70d9f011990')) +paddle.fluid.layers.l2_normalize (ArgSpec(args=['x', 'axis', 'epsilon', 'name'], varargs=None, keywords=None, defaults=(1e-12, None)), ('document', '35c6a241bcc1a1fc89508860d82ad62b')) paddle.fluid.layers.matmul (ArgSpec(args=['x', 'y', 'transpose_x', 'transpose_y', 'alpha', 'name'], varargs=None, keywords=None, defaults=(False, False, 1.0, None)), ('document', 'b4cbe1ac451005df6dad12e9ffdccca9')) paddle.fluid.layers.topk (ArgSpec(args=['input', 'k', 'name'], varargs=None, keywords=None, defaults=(None,)), ('document', 'd3570c02f71bcd78e60b3f31dc8f5b32')) paddle.fluid.layers.warpctc (ArgSpec(args=['input', 'label', 'blank', 'norm_by_times', 'use_cudnn'], varargs=None, keywords=None, defaults=(0, False, False)), ('document', 'aaba49c038ba927f0a8e45c0c9a686ab')) @@ -155,10 +158,10 @@ paddle.fluid.layers.label_smooth (ArgSpec(args=['label', 'prior_dist', 'epsilon' paddle.fluid.layers.roi_pool (ArgSpec(args=['input', 'rois', 'pooled_height', 'pooled_width', 'spatial_scale'], varargs=None, keywords=None, defaults=(1, 1, 1.0)), ('document', 'c317aa595deb31649083c8faa91cdb97')) paddle.fluid.layers.roi_align (ArgSpec(args=['input', 'rois', 'pooled_height', 'pooled_width', 'spatial_scale', 'sampling_ratio', 'name'], varargs=None, keywords=None, defaults=(1, 1, 1.0, -1, None)), ('document', '12c5bbb8b38c42e623fbc47611d766e1')) paddle.fluid.layers.dice_loss (ArgSpec(args=['input', 'label', 'epsilon'], varargs=None, keywords=None, defaults=(1e-05,)), ('document', '1ba0508d573f65feecf3564dce22aa1d')) -paddle.fluid.layers.image_resize (ArgSpec(args=['input', 'out_shape', 'scale', 'name', 'resample', 'actual_shape', 'align_corners', 'align_mode'], varargs=None, keywords=None, defaults=(None, None, None, 'BILINEAR', None, True, 1)), ('document', '7a1966d7c3a48f1fc0881cdaf5d83b0b')) +paddle.fluid.layers.image_resize (ArgSpec(args=['input', 'out_shape', 'scale', 'name', 'resample', 'actual_shape', 'align_corners', 'align_mode'], varargs=None, keywords=None, defaults=(None, None, None, 'BILINEAR', None, True, 1)), ('document', 'd1b08c11bb9277386fcf6ae70b6622d1')) paddle.fluid.layers.image_resize_short (ArgSpec(args=['input', 'out_short_len', 'resample'], varargs=None, keywords=None, defaults=('BILINEAR',)), ('document', '06211aefc50c5a3e940d7204d859cdf7')) -paddle.fluid.layers.resize_bilinear (ArgSpec(args=['input', 'out_shape', 'scale', 'name', 'actual_shape', 'align_corners', 'align_mode'], varargs=None, keywords=None, defaults=(None, None, None, None, True, 1)), ('document', 'e4fb4ed511b2293b8f04f7e872afbfd7')) -paddle.fluid.layers.resize_nearest (ArgSpec(args=['input', 'out_shape', 'scale', 'name', 'actual_shape', 'align_corners'], varargs=None, keywords=None, defaults=(None, None, None, None, True)), ('document', '735fa9758a6d7ff3b47d7b827f961c1d')) +paddle.fluid.layers.resize_bilinear (ArgSpec(args=['input', 'out_shape', 'scale', 'name', 'actual_shape', 'align_corners', 'align_mode'], varargs=None, keywords=None, defaults=(None, None, None, None, True, 1)), ('document', 'c45591fbc4f64a178fbca219e1546a58')) +paddle.fluid.layers.resize_nearest (ArgSpec(args=['input', 'out_shape', 'scale', 'name', 'actual_shape', 'align_corners'], varargs=None, keywords=None, defaults=(None, None, None, None, True)), ('document', 'ae6d73cdc7f3a138d8a338ecdb33c1ae')) paddle.fluid.layers.gather (ArgSpec(args=['input', 'index'], varargs=None, keywords=None, defaults=None), ('document', '98f1c86716b9b7f4dda83f20e2adeee2')) paddle.fluid.layers.scatter (ArgSpec(args=['input', 'index', 'updates', 'name'], varargs=None, keywords=None, defaults=(None,)), ('document', '65f8e9d8ddfd0b412f940579c4faa342')) paddle.fluid.layers.sequence_scatter (ArgSpec(args=['input', 'index', 'updates', 'name'], varargs=None, keywords=None, defaults=(None,)), ('document', '15b522457dfef103f0c20ca9d397678b')) @@ -203,6 +206,7 @@ paddle.fluid.layers.gaussian_random_batch_size_like (ArgSpec(args=['input', 'sha paddle.fluid.layers.sum (ArgSpec(args=['x'], varargs=None, keywords=None, defaults=None), ('document', 'a418e3ccb5e2ac21bd60f5cc221d5860')) paddle.fluid.layers.slice (ArgSpec(args=['input', 'axes', 'starts', 'ends'], varargs=None, keywords=None, defaults=None), ('document', '01dbb91e7c74cb11336cd531013de51a')) paddle.fluid.layers.shape (ArgSpec(args=['input'], varargs=None, keywords=None, defaults=None), ('document', '17db0f814eb7bb5a3fac1ca6e60e16d8')) +paddle.fluid.layers.rank (ArgSpec(args=['input'], varargs=None, keywords=None, defaults=None), ('document', 'ee1386c42ecc8f424fe3fb21862fefc2')) paddle.fluid.layers.logical_and (ArgSpec(args=['x', 'y', 'out', 'name'], varargs=None, keywords=None, defaults=(None, None)), ('document', 'cdcf20c494c92060d10feb9374532f42')) paddle.fluid.layers.logical_or (ArgSpec(args=['x', 'y', 'out', 'name'], varargs=None, keywords=None, defaults=(None, None)), ('document', '0eae3f726a4afe590757552fa3ced012')) paddle.fluid.layers.logical_xor (ArgSpec(args=['x', 'y', 'out', 'name'], varargs=None, keywords=None, defaults=(None, None)), ('document', 'b0daaa3fa4a0aa62f9b58c43d959eb25')) @@ -235,6 +239,7 @@ paddle.fluid.layers.huber_loss (ArgSpec(args=['input', 'label', 'delta'], vararg paddle.fluid.layers.kldiv_loss (ArgSpec(args=['x', 'target', 'reduction', 'name'], varargs=None, keywords=None, defaults=('mean', None)), ('document', '776d536cac47c89073abc7ee524d5aec')) paddle.fluid.layers.tree_conv (ArgSpec(args=['nodes_vector', 'edge_set', 'output_size', 'num_filters', 'max_depth', 'act', 'param_attr', 'bias_attr', 'name'], varargs=None, keywords=None, defaults=(1, 2, 'tanh', None, None, None)), ('document', '34ea12ac9f10a65dccbc50100d12e607')) paddle.fluid.layers.npair_loss (ArgSpec(args=['anchor', 'positive', 'labels', 'l2_reg'], varargs=None, keywords=None, defaults=(0.002,)), ('document', '46994d10276dd4cb803b4062b5d14329')) +paddle.fluid.layers.pixel_shuffle (ArgSpec(args=['x', 'upscale_factor'], varargs=None, keywords=None, defaults=None), ('document', '731b21c62a4add60a33bd76d802ffc5c')) paddle.fluid.layers.fsp_matrix (ArgSpec(args=['x', 'y'], varargs=None, keywords=None, defaults=None), ('document', 'b76ccca3735bea4a58a0dbf0d77c5393')) paddle.fluid.layers.data (ArgSpec(args=['name', 'shape', 'append_batch_size', 'dtype', 'lod_level', 'type', 'stop_gradient'], varargs=None, keywords=None, defaults=(True, 'float32', 0, VarType.LOD_TENSOR, True)), ('document', '33bbd42027d872b3818b3d64ec52e139')) paddle.fluid.layers.open_files (ArgSpec(args=['filenames', 'shapes', 'lod_levels', 'dtypes', 'thread_num', 'buffer_size', 'pass_num', 'is_test'], varargs=None, keywords=None, defaults=(None, None, 1, None)), ('document', 'b1ae2e1cc0750e58726374061ea90ecc')) @@ -270,6 +275,7 @@ paddle.fluid.layers.has_inf (ArgSpec(args=['x'], varargs=None, keywords=None, de paddle.fluid.layers.has_nan (ArgSpec(args=['x'], varargs=None, keywords=None, defaults=None), ('document', '2e53e83127dbfd86e7098bdfe9a549e8')) paddle.fluid.layers.isfinite (ArgSpec(args=['x'], varargs=None, keywords=None, defaults=None), ('document', '0a437011c3906079fd8947ed3e52d292')) paddle.fluid.layers.range (ArgSpec(args=['start', 'end', 'step', 'dtype'], varargs=None, keywords=None, defaults=None), ('document', '2ec937ede953ded2fdff2675883900bb')) +paddle.fluid.layers.linspace (ArgSpec(args=['start', 'stop', 'num', 'dtype'], varargs=None, keywords=None, defaults=None), ('document', '495e21e9a848c2d075a102802fc67756')) paddle.fluid.layers.While.__init__ (ArgSpec(args=['self', 'cond', 'is_test', 'name'], varargs=None, keywords=None, defaults=(False, None)), ('document', '6adf97f83acf6453d4a6a4b1070f3754')) paddle.fluid.layers.While.block (ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None), ('document', '6adf97f83acf6453d4a6a4b1070f3754')) paddle.fluid.layers.Switch.__init__ (ArgSpec(args=['self', 'name'], varargs=None, keywords=None, defaults=(None,)), ('document', '6adf97f83acf6453d4a6a4b1070f3754')) @@ -295,12 +301,12 @@ paddle.fluid.layers.DynamicRNN.static_input (ArgSpec(args=['self', 'x'], varargs paddle.fluid.layers.DynamicRNN.step_input (ArgSpec(args=['self', 'x', 'level'], varargs=None, keywords=None, defaults=(0,)), ('document', '7568c5ac7622a10288d3307a94134655')) paddle.fluid.layers.DynamicRNN.update_memory (ArgSpec(args=['self', 'ex_mem', 'new_mem'], varargs=None, keywords=None, defaults=None), ('document', '5d83987da13b98363d6a807a52d8024f')) paddle.fluid.layers.StaticRNN.__init__ (ArgSpec(args=['self', 'name'], varargs=None, keywords=None, defaults=(None,)), ('document', '6adf97f83acf6453d4a6a4b1070f3754')) -paddle.fluid.layers.StaticRNN.memory (ArgSpec(args=['self', 'init', 'shape', 'batch_ref', 'init_value', 'init_batch_dim_idx', 'ref_batch_dim_idx'], varargs=None, keywords=None, defaults=(None, None, None, 0.0, 0, 1)), ('document', 'c24e368e23afac1ed91a78a639d7a9c7')) -paddle.fluid.layers.StaticRNN.output (ArgSpec(args=['self'], varargs='outputs', keywords=None, defaults=None), ('document', '6adf97f83acf6453d4a6a4b1070f3754')) -paddle.fluid.layers.StaticRNN.step (ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None), ('document', '6adf97f83acf6453d4a6a4b1070f3754')) -paddle.fluid.layers.StaticRNN.step_input (ArgSpec(args=['self', 'x'], varargs=None, keywords=None, defaults=None), ('document', '6adf97f83acf6453d4a6a4b1070f3754')) -paddle.fluid.layers.StaticRNN.step_output (ArgSpec(args=['self', 'o'], varargs=None, keywords=None, defaults=None), ('document', '6adf97f83acf6453d4a6a4b1070f3754')) -paddle.fluid.layers.StaticRNN.update_memory (ArgSpec(args=['self', 'mem', 'var'], varargs=None, keywords=None, defaults=None), ('document', '6adf97f83acf6453d4a6a4b1070f3754')) +paddle.fluid.layers.StaticRNN.memory (ArgSpec(args=['self', 'init', 'shape', 'batch_ref', 'init_value', 'init_batch_dim_idx', 'ref_batch_dim_idx'], varargs=None, keywords=None, defaults=(None, None, None, 0.0, 0, 1)), ('document', '72530f299d6451a567cf4a12dc3fb1ff')) +paddle.fluid.layers.StaticRNN.output (ArgSpec(args=['self'], varargs='outputs', keywords=None, defaults=None), ('document', 'df6ceab6e6c9bd31e97914d7e7538137')) +paddle.fluid.layers.StaticRNN.step (ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None), ('document', '6d3e0a5d9aa519a9773a36e1620ea9b7')) +paddle.fluid.layers.StaticRNN.step_input (ArgSpec(args=['self', 'x'], varargs=None, keywords=None, defaults=None), ('document', '903387ec11f3d0bf46821d31a68cffa5')) +paddle.fluid.layers.StaticRNN.step_output (ArgSpec(args=['self', 'o'], varargs=None, keywords=None, defaults=None), ('document', '252890d4c3199a7623ab8667e13fd837')) +paddle.fluid.layers.StaticRNN.update_memory (ArgSpec(args=['self', 'mem', 'var'], varargs=None, keywords=None, defaults=None), ('document', '7a0000520f179f35239956a5ba55119f')) paddle.fluid.layers.reorder_lod_tensor_by_rank (ArgSpec(args=['x', 'rank_table'], varargs=None, keywords=None, defaults=None), ('document', '3545f529ef04e8f6ecb76b47fa3df01a')) paddle.fluid.layers.Print (ArgSpec(args=['input', 'first_n', 'message', 'summarize', 'print_tensor_name', 'print_tensor_type', 'print_tensor_shape', 'print_tensor_lod', 'print_phase'], varargs=None, keywords=None, defaults=(-1, None, -1, True, True, True, True, 'both')), ('document', '5fef91b0e21c93610785f2b1f7161732')) paddle.fluid.layers.is_empty (ArgSpec(args=['x', 'cond'], varargs=None, keywords=None, defaults=(None,)), ('document', 'bbe578dbb49ad13e15b014e98c22b519')) @@ -359,8 +365,7 @@ paddle.fluid.layers.inverse_time_decay (ArgSpec(args=['learning_rate', 'decay_st paddle.fluid.layers.polynomial_decay (ArgSpec(args=['learning_rate', 'decay_steps', 'end_learning_rate', 'power', 'cycle'], varargs=None, keywords=None, defaults=(0.0001, 1.0, False)), ('document', '882634f420f626642f0874481263da40')) paddle.fluid.layers.piecewise_decay (ArgSpec(args=['boundaries', 'values'], varargs=None, keywords=None, defaults=None), ('document', 'c717d9d1d78a53c809d01b8bc56f3cae')) paddle.fluid.layers.noam_decay (ArgSpec(args=['d_model', 'warmup_steps'], varargs=None, keywords=None, defaults=None), ('document', 'd9a95746353fd574be36dc28d8726c28')) -paddle.fluid.layers.append_LARS (ArgSpec(args=['params_grads', 'learning_rate', 'weight_decay'], varargs=None, keywords=None, defaults=None), ('document', 'd24fa1e7d62ac8a534fc6a86002f84f8')) -paddle.fluid.layers.cosine_decay (ArgSpec(args=['learning_rate', 'step_each_epoch', 'epochs'], varargs=None, keywords=None, defaults=None), ('document', '9588c64c26ffaef3c466e404a6af9d9b')) +paddle.fluid.layers.cosine_decay (ArgSpec(args=['learning_rate', 'step_each_epoch', 'epochs'], varargs=None, keywords=None, defaults=None), ('document', 'f8b2727bccf0f368c997d7cf05847e49')) paddle.fluid.layers.linear_lr_warmup (ArgSpec(args=['learning_rate', 'warmup_steps', 'start_lr', 'end_lr'], varargs=None, keywords=None, defaults=None), ('document', '2ef3f5ca5cd71ea4217c418e5a7a0565')) paddle.fluid.contrib.InitState.__init__ (ArgSpec(args=['self', 'init', 'shape', 'value', 'init_boot', 'need_reorder', 'dtype'], varargs=None, keywords=None, defaults=(None, None, 0.0, None, False, 'float32')), ('document', '6adf97f83acf6453d4a6a4b1070f3754')) paddle.fluid.contrib.StateCell.__init__ (ArgSpec(args=['self', 'inputs', 'states', 'out_state', 'name'], varargs=None, keywords=None, defaults=(None,)), ('document', '6adf97f83acf6453d4a6a4b1070f3754')) diff --git a/paddle/fluid/framework/data_feed.cc b/paddle/fluid/framework/data_feed.cc index b5f7e6c22405d6928f0e423458d6cd720f2d09a8..365c80da34eb287f50d2f0dcbf3844001ab43ec8 100644 --- a/paddle/fluid/framework/data_feed.cc +++ b/paddle/fluid/framework/data_feed.cc @@ -72,7 +72,6 @@ bool DataFeed::PickOneFile(std::string* filename) { } VLOG(3) << "file_idx_=" << *file_idx_; *filename = filelist_[(*file_idx_)++]; - // LOG(ERROR) << "pick file:" << *filename; return true; } @@ -466,6 +465,17 @@ void MultiSlotDataFeed::Init( if (slot.is_used()) { use_slots_.push_back(all_slots_[i]); use_slots_is_dense_.push_back(slot.is_dense()); + std::vector local_shape; + if (slot.is_dense()) { + // for batch size holder if is_dense + if (slot.shape(0) > 0) { + local_shape.push_back(0); + } + } + for (size_t i = 0; i < slot.shape_size(); ++i) { + local_shape.push_back(slot.shape(i)); + } + use_slots_shape_.push_back(local_shape); } } feed_vec_.resize(use_slots_.size()); @@ -752,8 +762,8 @@ void MultiSlotDataFeed::PutToFeedVec( LoD data_lod{offset}; feed_vec_[i]->set_lod(data_lod); if (use_slots_is_dense_[i]) { - int dim = total_instance / batch_size_; - feed_vec_[i]->Resize({batch_size_, dim}); + use_slots_shape_[i][0] = batch_size_; + feed_vec_[i]->Resize(framework::make_ddim(use_slots_shape_[i])); } } #endif @@ -785,6 +795,16 @@ void MultiSlotInMemoryDataFeed::Init( if (slot.is_used()) { use_slots_.push_back(all_slots_[i]); use_slots_is_dense_.push_back(slot.is_dense()); + std::vector local_shape; + if (slot.is_dense()) { + if (slot.shape(0) > 0) { + local_shape.push_back(0); + } + } + for (size_t i = 0; i < slot.shape_size(); ++i) { + local_shape.push_back(slot.shape(i)); + } + use_slots_shape_.push_back(local_shape); } } feed_vec_.resize(use_slots_.size()); @@ -940,8 +960,8 @@ void MultiSlotInMemoryDataFeed::PutToFeedVec( LoD data_lod{offset}; feed_vec_[i]->set_lod(data_lod); if (use_slots_is_dense_[i]) { - int dim = total_instance / batch_size_; - feed_vec_[i]->Resize({batch_size_, dim}); + use_slots_shape_[i][0] = batch_size_; + feed_vec_[i]->Resize(framework::make_ddim(use_slots_shape_[i])); } } #endif diff --git a/paddle/fluid/framework/data_feed.h b/paddle/fluid/framework/data_feed.h index 648c874a0b8763b18118e18adf3b3e93acfd104b..d098c7858a98c644bd3cad78d3cf1e3b35ca026b 100644 --- a/paddle/fluid/framework/data_feed.h +++ b/paddle/fluid/framework/data_feed.h @@ -142,6 +142,7 @@ class DataFeed { // object) std::vector all_slots_; std::vector all_slots_type_; + std::vector> use_slots_shape_; std::vector use_slots_index_; // -1: not used; >=0: the index of use_slots_ diff --git a/paddle/fluid/framework/data_feed.proto b/paddle/fluid/framework/data_feed.proto index 77911306299b77748a2ad9437d49680748885003..03996e0e20a1729ee300a5ad37abc325876930b7 100644 --- a/paddle/fluid/framework/data_feed.proto +++ b/paddle/fluid/framework/data_feed.proto @@ -19,6 +19,7 @@ message Slot { required string type = 2; optional bool is_dense = 3 [ default = false ]; optional bool is_used = 4 [ default = false ]; + repeated int32 shape = 5; // we can define N-D Tensor } message MultiSlotDesc { repeated Slot slots = 1; } diff --git a/paddle/fluid/framework/details/all_reduce_op_handle.cc b/paddle/fluid/framework/details/all_reduce_op_handle.cc index ed75b48090b27a9c430afe067467a1a39d711938..61276efedeeca76a8818c15ddab73b3c53725c4b 100644 --- a/paddle/fluid/framework/details/all_reduce_op_handle.cc +++ b/paddle/fluid/framework/details/all_reduce_op_handle.cc @@ -53,6 +53,10 @@ AllReduceOpHandle::AllReduceOpHandle(ir::Node *node, this->SetDeviceContext(p, nccl_ctxs_->DevCtx(p)); } } + // TODO(gongwb) :polish them! + if (is_encoded) { + VLOG(1) << "Use dgc allreduce mode"; + } } #else AllReduceOpHandle::AllReduceOpHandle(ir::Node *node, @@ -86,7 +90,7 @@ void AllReduceOpHandle::RunImplEncoded() { paddle::framework::GradOriginalVarName(in_var_handles[i]->name()); auto encode_var_name = original_name + g_dgc_encoded; auto *in_var = local_scope->FindVar(encode_var_name); - PADDLE_ENFORCE_NOT_NULL(in_var); + PADDLE_ENFORCE_NOT_NULL(in_var, "%s should not be null", encode_var_name); auto &in = in_var->Get(); ins.emplace_back(&in); diff --git a/paddle/fluid/framework/details/alloc_continuous_space_for_grad_pass.cc b/paddle/fluid/framework/details/alloc_continuous_space_for_grad_pass.cc index 8e8258ffb124e5008954a455264f5c0bc5cabc37..58ec427859e9f0ec4d29cc419f5bfe382e245852 100644 --- a/paddle/fluid/framework/details/alloc_continuous_space_for_grad_pass.cc +++ b/paddle/fluid/framework/details/alloc_continuous_space_for_grad_pass.cc @@ -12,17 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "paddle/fluid/framework/details/alloc_continuous_space_for_grad_pass.h" #include #include +#include #include #include - #include "paddle/fluid/framework/details/build_strategy.h" #include "paddle/fluid/framework/details/multi_devices_helper.h" #include "paddle/fluid/framework/ir/graph_helper.h" #include "paddle/fluid/framework/op_registry.h" -DEFINE_uint32(fuse_parameter_memory_size, 0, // 0 KB +DEFINE_uint64(fuse_parameter_memory_size, 0, // 0 KB "fuse_parameter_memory_size is up limited memory size " "of one group parameters' gradient which is the input " "of communication calling(e.g NCCLAllReduce). " @@ -40,355 +41,365 @@ DEFINE_int32( namespace paddle { namespace framework { namespace details { +// SetFuseParameterGroupsSize and SetFuseParameterMemorySize are used in unit +// test, because it is invalid that seting 'FLAGS_fuse_parameter_memory_size' +// and 'FLAGS_fuse_parameter_groups_size' in unit test. +void SetFuseParameterGroupsSize(int group_size) { + FLAGS_fuse_parameter_groups_size = group_size; +} -static const char kUnKnow[] = "@UNKNOW@"; -static framework::proto::VarType::Type kDefaultDtype = - framework::proto::VarType::Type::VarType_Type_BOOL; +int GetFuseParameterGroupsSize() { return FLAGS_fuse_parameter_groups_size; } -class AllocContinuousSpaceForGradPass : public ir::Pass { - protected: - void ApplyImpl(ir::Graph *graph) const override { - ir::Graph &result = *graph; +void SetFuseParameterMemorySize(uint64_t memory_size) { + FLAGS_fuse_parameter_memory_size = memory_size; +} - auto &places = Get>(kPlaces); - auto &local_scopes = Get>(kLocalScopes); +uint64_t GetFuseParameterMemorySize() { + return FLAGS_fuse_parameter_memory_size; +} - ResetAttribute(kParamsAndGrads, &result); - ResetAttribute(kGroupGradsAndParams, &result); +static const char kUnKnow[] = "@UNKNOW@"; +static framework::proto::VarType::Type kDefaultDtype = + framework::proto::VarType::Type::VarType_Type_BOOL; - // NOTE: The operator nodes should be in topology order. - std::vector topo_nodes = ir::TopologySortOperations(result); - auto ¶ms_grads = result.Get(kParamsAndGrads); - for (auto &node : topo_nodes) { - RecordParamsAndGrads(node, ¶ms_grads); - } +void AllocContinuousSpaceForGradPass::ApplyImpl(ir::Graph *graph) const { + ir::Graph &result = *graph; - if (params_grads.size() == 0) { - VLOG(10) << "Doesn't find gradients"; - return; - } + auto &places = Get>(kPlaces); + auto &local_scopes = Get>(kLocalScopes); - std::unordered_map vars; - for (ir::Node *node : result.Nodes()) { - if (node->IsVar() && node->Var()) { - // Note: The graph may have the same name node. For example, parameter - // is the input of operator and it also is the output of optimizer; - vars.emplace(node->Var()->Name(), node); - } - } + ResetAttribute(kParamsAndGrads, &result); + ResetAttribute(kGroupGradsAndParams, &result); - auto &group_grads_params = - result.Get(kGroupGradsAndParams); + // NOTE: The operator nodes should be in topology order. + std::vector topo_nodes = ir::TopologySortOperations(result); + auto ¶ms_grads = result.Get(kParamsAndGrads); + for (auto &node : topo_nodes) { + RecordParamsAndGrads(node, ¶ms_grads); + } - // Note: the order of params_grads may be changed by SetGroupGradsAndParams. - SetGroupGradsAndParams(vars, params_grads, &group_grads_params); + if (params_grads.size() == 0) { + VLOG(10) << "Doesn't find gradients"; + return; + } - params_grads.clear(); - for (auto &group_p_g : group_grads_params) { - params_grads.insert(params_grads.begin(), group_p_g.begin(), - group_p_g.end()); - } - for (auto &p_g : params_grads) { - std::swap(p_g.first, p_g.second); + std::unordered_map vars; + for (ir::Node *node : result.Nodes()) { + if (node->IsVar() && node->Var()) { + // Note: The graph may have the same name node. For example, parameter + // is the input of operator and it also is the output of optimizer; + vars.emplace(node->Var()->Name(), node); } + } - // Set Gradients as Persistable to prevent this var becoming reusable. - auto dtype = kDefaultDtype; - for (auto &p_g : params_grads) { - // Get gradient var - auto iter = vars.find(p_g.second); - PADDLE_ENFORCE(iter != vars.end(), "%s is not found.", p_g.second); - iter->second->Var()->SetPersistable(true); - - PADDLE_ENFORCE(IsSupportedVarType(iter->second->Var()->GetType())); + auto &group_grads_params = + result.Get(kGroupGradsAndParams); - // Get Dtype - auto ele_dtype = iter->second->Var()->GetDataType(); - if (dtype == kDefaultDtype) { - dtype = ele_dtype; - PADDLE_ENFORCE_NE(ele_dtype, kDefaultDtype, - "The data type should not be bool."); - } - PADDLE_ENFORCE_EQ(ele_dtype, dtype, - "The data type of input is not consistent."); - } + // Note: the order of params_grads may be changed by SetGroupGradsAndParams. + SetGroupGradsAndParams(vars, params_grads, &group_grads_params); - // Create a FusedVarsSet to avoid duplicating names for fused_var in other - // pass. - if (!result.Has(kFusedVars)) { - result.Set(kFusedVars, new FusedVars); - } - // the kFusedGrads is used be fuse_optimizer_op_pass. - result.Set(kFusedGrads, new FusedGrads); - - // the fused_var_name should be unique, so it appends - // params_grads.begin()->second. - auto fused_var_name = std::string(kFusedVarNamePrefix) + "@GRAD@" + - params_grads.begin()->second; - result.Get(kFusedGrads) = fused_var_name; - auto &fused_var_set = result.Get(kFusedVars); - PADDLE_ENFORCE_EQ(fused_var_set.count(fused_var_name), 0, - "%s is duplicate in FusedVars.", fused_var_name); - fused_var_set.insert(fused_var_name); - - InitFusedVarsAndAllocSpaceForVars(places, local_scopes, vars, - fused_var_name, params_grads); + params_grads.clear(); + for (auto &group_p_g : group_grads_params) { + params_grads.insert(params_grads.begin(), group_p_g.begin(), + group_p_g.end()); + } + for (auto &p_g : params_grads) { + std::swap(p_g.first, p_g.second); } - template - void ResetAttribute(const std::string &attr_name, ir::Graph *graph) const { - if (graph->Has(attr_name)) { - VLOG(10) << attr_name << " is reset."; - graph->Erase(attr_name); + // Set Gradients as Persistable to prevent this var becoming reusable. + auto dtype = kDefaultDtype; + for (auto &p_g : params_grads) { + // Get gradient var + auto iter = vars.find(p_g.second); + PADDLE_ENFORCE(iter != vars.end(), "%s is not found.", p_g.second); + iter->second->Var()->SetPersistable(true); + + PADDLE_ENFORCE(IsSupportedVarType(iter->second->Var()->GetType())); + + // Get Dtype + auto ele_dtype = iter->second->Var()->GetDataType(); + if (dtype == kDefaultDtype) { + dtype = ele_dtype; + PADDLE_ENFORCE_NE(ele_dtype, kDefaultDtype, + "The data type should not be bool."); } - graph->Set(attr_name, new AttrType); + PADDLE_ENFORCE_EQ(ele_dtype, dtype, + "The data type of input is not consistent."); } - void SetGroupGradsAndParams( - const std::unordered_map &var_nodes, - const ParamsAndGrads ¶ms_grads, - GroupGradsAndParams *group_grads_params) const { - SetGroupAccordingToLayers(var_nodes, params_grads, group_grads_params); - SetGroupAccordingToMemorySize(var_nodes, group_grads_params); - SetGroupAccordingToGroupSize(var_nodes, group_grads_params); + // Create a FusedVarsSet to avoid duplicating names for fused_var in other + // pass. + if (!result.Has(kFusedVars)) { + result.Set(kFusedVars, new FusedVars); } - - void SetGroupAccordingToLayers( - const std::unordered_map &var_nodes, - const ParamsAndGrads ¶ms_grads, - GroupGradsAndParams *group_grads_params) const { - std::unordered_map> layer_params; - - for (size_t i = 0; i < params_grads.size(); ++i) { - auto pos = params_grads[i].first.find_first_of("."); - if (pos == std::string::npos) { - layer_params[std::string(kUnKnow)].emplace_back(i); - } else { - layer_params[params_grads[i].first.substr(0, pos)].emplace_back(i); - } + // the kFusedGrads is used be fuse_optimizer_op_pass. + result.Set(kFusedGrads, new FusedGrads); + + // the fused_var_name should be unique, so it appends + // params_grads.begin()->second. + auto fused_var_name = std::string(kFusedVarNamePrefix) + "@GRAD@" + + params_grads.begin()->second; + result.Get(kFusedGrads) = fused_var_name; + auto &fused_var_set = result.Get(kFusedVars); + PADDLE_ENFORCE_EQ(fused_var_set.count(fused_var_name), 0, + "%s is duplicate in FusedVars.", fused_var_name); + fused_var_set.insert(fused_var_name); + + InitFusedVarsAndAllocSpaceForVars(places, local_scopes, vars, fused_var_name, + params_grads); +} + +template +void AllocContinuousSpaceForGradPass::ResetAttribute( + const std::string &attr_name, ir::Graph *graph) const { + if (graph->Has(attr_name)) { + VLOG(10) << attr_name << " is reset."; + graph->Erase(attr_name); + } + graph->Set(attr_name, new AttrType); +} + +void AllocContinuousSpaceForGradPass::SetGroupGradsAndParams( + const std::unordered_map &var_nodes, + const ParamsAndGrads ¶ms_grads, + GroupGradsAndParams *group_grads_params) const { + SetGroupAccordingToLayers(var_nodes, params_grads, group_grads_params); + SetGroupAccordingToMemorySize(var_nodes, group_grads_params); + SetGroupAccordingToGroupSize(var_nodes, group_grads_params); +} + +void AllocContinuousSpaceForGradPass::SetGroupAccordingToLayers( + const std::unordered_map &var_nodes, + const ParamsAndGrads ¶ms_grads, + GroupGradsAndParams *group_grads_params) const { + std::unordered_map> layer_params; + + for (size_t i = 0; i < params_grads.size(); ++i) { + auto pos = params_grads[i].first.find_first_of("."); + if (pos == std::string::npos) { + layer_params[std::string(kUnKnow)].emplace_back(i); + } else { + layer_params[params_grads[i].first.substr(0, pos)].emplace_back(i); } + } - group_grads_params->reserve(layer_params.size()); - for (size_t i = 0; i < params_grads.size(); ++i) { - auto pos = params_grads[i].first.find_first_of("."); - std::string key = kUnKnow; - if (pos != std::string::npos) { - key = params_grads[i].first.substr(0, pos); - } - auto iter = layer_params.find(key); - if (iter == layer_params.end()) continue; - - group_grads_params->emplace_back(); - auto &local_group_grads_params = group_grads_params->back(); - for (auto &idx : iter->second) { - local_group_grads_params.emplace_back( - std::make_pair(params_grads[idx].second, params_grads[idx].first)); - } - layer_params.erase(iter); + group_grads_params->reserve(layer_params.size()); + for (size_t i = 0; i < params_grads.size(); ++i) { + auto pos = params_grads[i].first.find_first_of("."); + std::string key = kUnKnow; + if (pos != std::string::npos) { + key = params_grads[i].first.substr(0, pos); } - - VLOG(10) << "SetGroupAccordingToLayers: "; - for (size_t i = 0; i < group_grads_params->size(); ++i) { - VLOG(10) << "group " << i; - std::stringstream out; - for (auto &p_g : group_grads_params->at(i)) { - out << "(" << p_g.second << ", " << p_g.first << "), "; - } - VLOG(10) << out.str(); + auto iter = layer_params.find(key); + if (iter == layer_params.end()) continue; + + group_grads_params->emplace_back(); + auto &local_group_grads_params = group_grads_params->back(); + for (auto &idx : iter->second) { + local_group_grads_params.emplace_back( + std::make_pair(params_grads[idx].second, params_grads[idx].first)); } + layer_params.erase(iter); } - void SetGroupAccordingToMemorySize( - const std::unordered_map &var_nodes, - GroupGradsAndParams *group_grads_params) const { - if (FLAGS_fuse_parameter_memory_size == 0) { - return; + VLOG(10) << "SetGroupAccordingToLayers: "; + for (size_t i = 0; i < group_grads_params->size(); ++i) { + VLOG(10) << "group " << i; + std::stringstream out; + for (auto &p_g : group_grads_params->at(i)) { + out << "(" << p_g.second << ", " << p_g.first << "), "; } - size_t group_memory_size = - static_cast(FLAGS_fuse_parameter_memory_size); - GroupGradsAndParams local_group_grads_params; - - size_t j = 0; + VLOG(10) << out.str(); + } +} + +void AllocContinuousSpaceForGradPass::SetGroupAccordingToMemorySize( + const std::unordered_map &var_nodes, + GroupGradsAndParams *group_grads_params) const { + const uint64_t group_memory_size = GetFuseParameterMemorySize(); + if (group_memory_size == 0) { + return; + } + GroupGradsAndParams local_group_grads_params; + size_t j = 0; + while (j < group_grads_params->size()) { + local_group_grads_params.emplace_back(); + auto &group_p_g = local_group_grads_params.back(); + size_t local_group_memory_size = 0; while (j < group_grads_params->size()) { - local_group_grads_params.emplace_back(); - auto &group_p_g = local_group_grads_params.back(); - size_t local_group_memory_size = 0; - while (j < group_grads_params->size()) { - std::for_each( - group_grads_params->at(j).begin(), group_grads_params->at(j).end(), - [&local_group_memory_size, - &var_nodes](const std::pair &g_p) { - auto iter = var_nodes.find(g_p.second); - PADDLE_ENFORCE(iter != var_nodes.end(), "%s is not found.", - g_p.second); - auto shape = iter->second->Var()->GetShape(); - size_t size = - framework::SizeOfType(iter->second->Var()->GetDataType()); - std::for_each(shape.begin(), shape.end(), - [&size](const int64_t &n) { size *= n; }); - local_group_memory_size += size; - }); - group_p_g.insert(group_p_g.end(), group_grads_params->at(j).begin(), - group_grads_params->at(j).end()); - ++j; - if (local_group_memory_size >= group_memory_size) { - break; - } - } - } - - std::swap(*group_grads_params, local_group_grads_params); - - VLOG(10) << string::Sprintf( - "SetGroupAccordingToMemorySize(memory_size: %d):", - FLAGS_fuse_parameter_memory_size); - for (size_t i = 0; i < group_grads_params->size(); ++i) { - VLOG(10) << "group " << i; - std::stringstream out; - for (auto &g_p : group_grads_params->at(i)) { - auto iter = var_nodes.find(g_p.second); - PADDLE_ENFORCE(iter != var_nodes.end(), "%s is not found.", g_p.second); - auto shape = iter->second->Var()->GetShape(); - size_t size = framework::SizeOfType(iter->second->Var()->GetDataType()); - std::for_each(shape.begin(), shape.end(), - [&size](const int64_t &n) { size *= n; }); - out << string::Sprintf("(%s(%d), %s)", g_p.second, size, g_p.first); + std::for_each( + group_grads_params->at(j).begin(), group_grads_params->at(j).end(), + [&local_group_memory_size, + &var_nodes](const std::pair &g_p) { + auto iter = var_nodes.find(g_p.second); + PADDLE_ENFORCE(iter != var_nodes.end(), "%s is not found.", + g_p.second); + auto shape = iter->second->Var()->GetShape(); + size_t size = + framework::SizeOfType(iter->second->Var()->GetDataType()); + std::for_each(shape.begin(), shape.end(), + [&size](const int64_t &n) { size *= n; }); + local_group_memory_size += size; + }); + group_p_g.insert(group_p_g.end(), group_grads_params->at(j).begin(), + group_grads_params->at(j).end()); + ++j; + if (local_group_memory_size >= group_memory_size) { + break; } - VLOG(10) << out.str(); } } - void SetGroupAccordingToGroupSize( - const std::unordered_map &var_nodes, - GroupGradsAndParams *group_grads_params) const { - if (FLAGS_fuse_parameter_groups_size == 1) { - return; - } - size_t group_size = static_cast(FLAGS_fuse_parameter_groups_size); - if (FLAGS_fuse_parameter_groups_size == -1) { - group_size = group_grads_params->size(); - } - PADDLE_ENFORCE_GT(group_size, 1); - size_t groups = (group_grads_params->size() + group_size - 1) / group_size; - GroupGradsAndParams local_group_grads_params; - local_group_grads_params.reserve(groups); - - size_t j = 0; - for (size_t i = 0; i < groups; ++i) { - local_group_grads_params.emplace_back(); - auto &group_p_g = local_group_grads_params.back(); - group_p_g.reserve(group_size); - while (j < group_grads_params->size()) { - group_p_g.insert(group_p_g.end(), group_grads_params->at(j).begin(), - group_grads_params->at(j).end()); - ++j; - if (j % group_size == 0) break; - } - } - std::swap(*group_grads_params, local_group_grads_params); - - VLOG(10) << "SetGroupAccordingToGroupSize(group_size: " << group_size - << "): "; - for (size_t i = 0; i < group_grads_params->size(); ++i) { - VLOG(10) << "group " << i; - std::stringstream out; - for (auto &p_g : group_grads_params->at(i)) { - out << "(" << p_g.second << ", " << p_g.first << "), "; - } - VLOG(10) << out.str(); + std::swap(*group_grads_params, local_group_grads_params); + + VLOG(10) << string::Sprintf("SetGroupAccordingToMemorySize(memory_size: %d):", + group_memory_size); + for (size_t i = 0; i < group_grads_params->size(); ++i) { + VLOG(10) << "group " << i; + std::stringstream out; + for (auto &g_p : group_grads_params->at(i)) { + auto iter = var_nodes.find(g_p.second); + PADDLE_ENFORCE(iter != var_nodes.end(), "%s is not found.", g_p.second); + auto shape = iter->second->Var()->GetShape(); + size_t size = framework::SizeOfType(iter->second->Var()->GetDataType()); + std::for_each(shape.begin(), shape.end(), + [&size](const int64_t &n) { size *= n; }); + out << string::Sprintf("(%s(%d), %s)", g_p.second, size, g_p.first); } + VLOG(10) << out.str(); } +} - private: - bool IsSupportedVarType(const proto::VarType::Type &type) const { - // Current only support LOD_TENSOR. - return type == proto::VarType::LOD_TENSOR; +void AllocContinuousSpaceForGradPass::SetGroupAccordingToGroupSize( + const std::unordered_map &var_nodes, + GroupGradsAndParams *group_grads_params) const { + if (GetFuseParameterGroupsSize() == 1) { + return; } - - void RecordParamsAndGrads(ir::Node *node, - ParamsAndGrads *params_grads) const { - try { - bool is_bk_op = - static_cast(boost::get(node->Op()->GetAttr( - OpProtoAndCheckerMaker::OpRoleAttrName())) & - static_cast(OpRole::kBackward)); - if (!is_bk_op) return; - - // Currently, we assume that once gradient is generated, it can be - // broadcast, and each gradient is only broadcast once. - auto backward_vars = - boost::get>(node->Op()->GetNullableAttr( - OpProtoAndCheckerMaker::OpRoleVarAttrName())); - PADDLE_ENFORCE_EQ(backward_vars.size() % 2, static_cast(0)); - - for (size_t i = 0; i < backward_vars.size(); i += 2) { - VLOG(10) << "Trainable parameter: " << backward_vars[i] - << ", gradient: " << backward_vars[i + 1]; - - params_grads->emplace_back(std::make_pair( - backward_vars[i] /*param*/, backward_vars[i + 1] /*grad*/)); - } - } catch (boost::bad_get e) { + const int group_size = GetFuseParameterGroupsSize() == -1 + ? static_cast(group_grads_params->size()) + : GetFuseParameterGroupsSize(); + PADDLE_ENFORCE_GT(group_size, 1); + size_t groups = (group_grads_params->size() + group_size - 1) / group_size; + GroupGradsAndParams local_group_grads_params; + local_group_grads_params.reserve(groups); + + size_t j = 0; + for (size_t i = 0; i < groups; ++i) { + local_group_grads_params.emplace_back(); + auto &group_p_g = local_group_grads_params.back(); + group_p_g.reserve(group_size); + while (j < group_grads_params->size()) { + group_p_g.insert(group_p_g.end(), group_grads_params->at(j).begin(), + group_grads_params->at(j).end()); + ++j; + if (j % group_size == 0) break; } } - - void InitFusedVarsAndAllocSpaceForVars( - const std::vector &places, - const std::vector &local_scopes, - const std::unordered_map &vars, - const std::string &fused_var_name, - const ParamsAndGrads ¶ms_grads) const { - // Init Gradients and FusedVars - VLOG(10) << "Init FusedVars and Gradients."; - for (auto it = local_scopes.rbegin(); it != local_scopes.rend(); ++it) { - auto &scope = *it; - - PADDLE_ENFORCE(scope->FindVar(fused_var_name) == nullptr, - "%s has existed in scope.", fused_var_name); - scope->Var(fused_var_name)->GetMutable(); - - for (auto &p_g : params_grads) { - auto iter = vars.find(p_g.second); - PADDLE_ENFORCE(iter != vars.end()); - PADDLE_ENFORCE_NOT_NULL(iter->second->Var()); - PADDLE_ENFORCE_EQ(iter->second->Var()->GetType(), - proto::VarType::LOD_TENSOR); - scope->Var(p_g.second)->GetMutable(); - } + std::swap(*group_grads_params, local_group_grads_params); + + VLOG(10) << string::Sprintf("SetGroupAccordingToGroupSize(group_size: %d):", + group_size); + for (size_t i = 0; i < group_grads_params->size(); ++i) { + VLOG(10) << "group " << i; + std::stringstream out; + for (auto &p_g : group_grads_params->at(i)) { + out << "(" << p_g.second << ", " << p_g.first << "), "; + } + VLOG(10) << out.str(); + } +} + +bool AllocContinuousSpaceForGradPass::IsSupportedVarType( + const proto::VarType::Type &type) const { + // Current only support LOD_TENSOR. + return type == proto::VarType::LOD_TENSOR; +} + +void AllocContinuousSpaceForGradPass::RecordParamsAndGrads( + ir::Node *node, ParamsAndGrads *params_grads) const { + try { + bool is_bk_op = + static_cast(boost::get(node->Op()->GetAttr( + OpProtoAndCheckerMaker::OpRoleAttrName())) & + static_cast(OpRole::kBackward)); + if (!is_bk_op) return; + + // Currently, we assume that once gradient is generated, it can be + // broadcast, and each gradient is only broadcast once. + auto backward_vars = + boost::get>(node->Op()->GetNullableAttr( + OpProtoAndCheckerMaker::OpRoleVarAttrName())); + PADDLE_ENFORCE_EQ(backward_vars.size() % 2, static_cast(0)); + + for (size_t i = 0; i < backward_vars.size(); i += 2) { + VLOG(10) << "Trainable parameter: " << backward_vars[i] + << ", gradient: " << backward_vars[i + 1]; + + params_grads->emplace_back(std::make_pair(backward_vars[i] /*param*/, + backward_vars[i + 1] /*grad*/)); } + } catch (boost::bad_get e) { + } +} + +void AllocContinuousSpaceForGradPass::InitFusedVarsAndAllocSpaceForVars( + const std::vector &places, + const std::vector &local_scopes, + const std::unordered_map &vars, + const std::string &fused_var_name, + const ParamsAndGrads ¶ms_grads) const { + // Init Gradients and FusedVars + VLOG(10) << "Init FusedVars and Gradients."; + for (auto it = local_scopes.rbegin(); it != local_scopes.rend(); ++it) { + auto &scope = *it; + + PADDLE_ENFORCE(scope->FindVar(fused_var_name) == nullptr, + "%s has existed in scope.", fused_var_name); + scope->Var(fused_var_name)->GetMutable(); - // Alloc continuous space for vars. - std::vector grads_name; - std::vector params_name; - grads_name.reserve(params_grads.size()); - params_name.reserve(params_grads.size()); for (auto &p_g : params_grads) { - params_name.emplace_back(p_g.first); - grads_name.emplace_back(p_g.second); - } - framework::ProgramDesc program_desc; - AppendAllocSpaceForVarsOp(params_name, grads_name, fused_var_name, - program_desc.MutableBlock(0)); - - for (size_t i = 0; i < local_scopes.size(); ++i) { - for (auto &op_desc : program_desc.Block(0).AllOps()) { - auto op = OpRegistry::CreateOp(*op_desc); - op->Run(*local_scopes[i], places[i]); - } + auto iter = vars.find(p_g.second); + PADDLE_ENFORCE(iter != vars.end()); + PADDLE_ENFORCE_NOT_NULL(iter->second->Var()); + PADDLE_ENFORCE_EQ(iter->second->Var()->GetType(), + proto::VarType::LOD_TENSOR); + scope->Var(p_g.second)->GetMutable(); } } - void AppendAllocSpaceForVarsOp(const std::vector ¶ms_name, - const std::vector &grads_name, - const std::string &fused_var_name, - BlockDesc *global_block) const { - auto op_desc = global_block->AppendOp(); - op_desc->SetType("alloc_continuous_space"); - op_desc->SetInput("Input", params_name); - op_desc->SetOutput("Output", grads_name); - op_desc->SetOutput("FusedOutput", {fused_var_name}); + // Alloc continuous space for vars. + std::vector grads_name; + std::vector params_name; + grads_name.reserve(params_grads.size()); + params_name.reserve(params_grads.size()); + for (auto &p_g : params_grads) { + params_name.emplace_back(p_g.first); + grads_name.emplace_back(p_g.second); + } + framework::ProgramDesc program_desc; + AppendAllocSpaceForVarsOp(params_name, grads_name, fused_var_name, + program_desc.MutableBlock(0)); + + for (size_t i = 0; i < local_scopes.size(); ++i) { + for (auto &op_desc : program_desc.Block(0).AllOps()) { + auto op = OpRegistry::CreateOp(*op_desc); + op->Run(*local_scopes[i], places[i]); + } } -}; +} + +void AllocContinuousSpaceForGradPass::AppendAllocSpaceForVarsOp( + const std::vector ¶ms_name, + const std::vector &grads_name, + const std::string &fused_var_name, BlockDesc *global_block) const { + auto op_desc = global_block->AppendOp(); + op_desc->SetType("alloc_continuous_space"); + op_desc->SetInput("Input", params_name); + op_desc->SetOutput("Output", grads_name); + op_desc->SetOutput("FusedOutput", {fused_var_name}); +} } // namespace details } // namespace framework diff --git a/paddle/fluid/framework/details/alloc_continuous_space_for_grad_pass.h b/paddle/fluid/framework/details/alloc_continuous_space_for_grad_pass.h new file mode 100644 index 0000000000000000000000000000000000000000..e6d56f17cc4ef7e07500aae8067211a7b9ac04b0 --- /dev/null +++ b/paddle/fluid/framework/details/alloc_continuous_space_for_grad_pass.h @@ -0,0 +1,79 @@ +// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once +#include +#include +#include +#include +#include +#include "paddle/fluid/framework/details/build_strategy.h" +#include "paddle/fluid/framework/details/multi_devices_helper.h" +#include "paddle/fluid/framework/ir/graph_helper.h" + +namespace paddle { +namespace framework { +namespace details { + +void SetFuseParameterGroupsSize(int group_size); +int GetFuseParameterGroupsSize(); + +void SetFuseParameterMemorySize(uint64_t memory_size); +uint64_t GetFuseParameterMemorySize(); + +class AllocContinuousSpaceForGradPass : public ir::Pass { + protected: + void ApplyImpl(ir::Graph *graph) const override; + + template + void ResetAttribute(const std::string &attr_name, ir::Graph *graph) const; + + void SetGroupGradsAndParams( + const std::unordered_map &var_nodes, + const ParamsAndGrads ¶ms_grads, + GroupGradsAndParams *group_grads_params) const; + + void SetGroupAccordingToLayers( + const std::unordered_map &var_nodes, + const ParamsAndGrads ¶ms_grads, + GroupGradsAndParams *group_grads_params) const; + + void SetGroupAccordingToMemorySize( + const std::unordered_map &var_nodes, + GroupGradsAndParams *group_grads_params) const; + + void SetGroupAccordingToGroupSize( + const std::unordered_map &var_nodes, + GroupGradsAndParams *group_grads_params) const; + + private: + bool IsSupportedVarType(const proto::VarType::Type &type) const; + + void RecordParamsAndGrads(ir::Node *node, ParamsAndGrads *params_grads) const; + + void InitFusedVarsAndAllocSpaceForVars( + const std::vector &places, + const std::vector &local_scopes, + const std::unordered_map &vars, + const std::string &fused_var_name, + const ParamsAndGrads ¶ms_grads) const; + + void AppendAllocSpaceForVarsOp(const std::vector ¶ms_name, + const std::vector &grads_name, + const std::string &fused_var_name, + BlockDesc *global_block) const; +}; + +} // namespace details +} // namespace framework +} // namespace paddle diff --git a/paddle/fluid/framework/details/build_strategy.cc b/paddle/fluid/framework/details/build_strategy.cc index f8bf43bcb48226b4d1317a78ade7179741097378..196603bbff1db79e46ebbe8b18f1092fcbaac7f9 100644 --- a/paddle/fluid/framework/details/build_strategy.cc +++ b/paddle/fluid/framework/details/build_strategy.cc @@ -101,8 +101,6 @@ class ParallelExecutorPassBuilder : public ir::PassBuilder { "mode."; strategy_.fuse_all_optimizer_ops_ = false; } else { - VLOG(10) << "Add alloc_continuous_space_for_grad_pass"; - AppendPass("alloc_continuous_space_for_grad_pass"); // NOTE: fuse_all_xx_ops will count the number of xx operator first, // if the number is zero, fuse_all_reduce_ops will do nothing. // Currently, only one type of optimization algorithm can be fused. @@ -142,6 +140,19 @@ class ParallelExecutorPassBuilder : public ir::PassBuilder { AppendPass("memory_optimize_pass"); } + // runtime_context_cache pass should be the last pass to enable the attr of + // all original and fused operators. But no operators can be enabled this + // attr if putting it after MultiDevPass. + if (strategy_.cache_runtime_context_) { + VLOG(10) << "Add runtime_context_cache_pass"; + AppendPass("runtime_context_cache_pass"); + } + + if (strategy_.cache_expected_kernel_) { + VLOG(10) << "Add expected_kernel_cache_pass"; + AppendPass("expected_kernel_cache_pass"); + } + AppendMultiDevPass(strategy_); if (strategy_.fuse_all_reduce_ops_) { @@ -243,7 +254,7 @@ ir::Graph *BuildStrategy::Apply(ir::Graph *graph, CreatePassesFromStrategy(false); for (std::shared_ptr &pass : pass_builder_->AllPasses()) { - VLOG(3) << "apply " << pass->Type(); + VLOG(3) << "BuildStrategy::Apply pass:" << pass->Type(); if (IsMultiDevPass(pass->Type())) { pass->Erase(kPlaces); pass->SetNotOwned>(kPlaces, &places); @@ -328,3 +339,5 @@ USE_PASS(graph_to_program_pass); USE_PASS(fuse_adam_op_pass); USE_PASS(fuse_sgd_op_pass); USE_PASS(fuse_all_reduce_op_pass); +USE_PASS(runtime_context_cache_pass); +USE_PASS(expected_kernel_cache_pass); diff --git a/paddle/fluid/framework/details/build_strategy.h b/paddle/fluid/framework/details/build_strategy.h index cc48c51e924039d93b2e1e18bea752611e7bef92..b1601cfbcd5e9c66f1bbecd1f6fe10bc279cea26 100644 --- a/paddle/fluid/framework/details/build_strategy.h +++ b/paddle/fluid/framework/details/build_strategy.h @@ -83,11 +83,11 @@ struct BuildStrategy { bool sync_batch_norm_{false}; - bool memory_optimize_{true}; - // TODO(dzhwinter): - // make enable_inplace, memory_optimize_ - // memory_early_delete_ true by default - bool enable_inplace_{true}; + // FIXME(liuwei1031) disable memory_optimzie and enable_inplace in 1.4 + // to open them by default, we need to solve the fetch variable issue + bool memory_optimize_{false}; + + bool enable_inplace_{false}; bool enable_sequential_execution_{false}; @@ -107,6 +107,9 @@ struct BuildStrategy { std::vector trainers_endpoints_; bool remove_unnecessary_lock_{true}; + bool cache_runtime_context_{false}; + bool cache_expected_kernel_{true}; + // NOTE: // Before you add new options, think if it's a general strategy that works // with other strategy. If not, the strategy should be created through diff --git a/paddle/fluid/framework/details/fuse_adam_op_pass.cc b/paddle/fluid/framework/details/fuse_adam_op_pass.cc index 0ef75e319244e2ccc63dfa3f93f0cd764cf67633..f95d93fd5575ae538274c4c0322cf661c631849a 100644 --- a/paddle/fluid/framework/details/fuse_adam_op_pass.cc +++ b/paddle/fluid/framework/details/fuse_adam_op_pass.cc @@ -24,7 +24,7 @@ namespace details { const std::string FuseAdamOpPass::GetOpType() const { return "adam"; } const std::vector FuseAdamOpPass::GetAuxiliaryVarNames() const { - return {"Param", "Moment1", "Moment2", "Beta1Pow", "Beta2Pow"}; + return {"Moment1", "Moment2", "Beta1Pow", "Beta2Pow"}; } void FuseAdamOpPass::FuseOptimizerOps( @@ -77,16 +77,16 @@ void FuseAdamOpPass::FuseAdamOps( VLOG(10) << "Insert adam to graph "; OpDesc adam_desc(adam_ops[0]->Op()->Block()); adam_desc.SetType("adam"); - adam_desc.SetInput("Param", {fused_vars_name.at("Param")}); - adam_desc.SetInput("Grad", {fused_vars_name.at("Grad")}); + adam_desc.SetInput(kParam, {fused_vars_name.at(kParam)}); + adam_desc.SetInput(kGrad, {fused_vars_name.at(kGrad)}); adam_desc.SetInput("Moment1", {fused_vars_name.at("Moment1")}); adam_desc.SetInput("Moment2", {fused_vars_name.at("Moment2")}); // TODO(zcd): The LearningRate, Beta1Pow, Beta2Pow should be equal. - adam_desc.SetInput("LearningRate", adam_ops[0]->Op()->Input("LearningRate")); + adam_desc.SetInput(kLearningRate, adam_ops[0]->Op()->Input(kLearningRate)); adam_desc.SetInput("Beta1Pow", adam_ops[0]->Op()->Input("Beta1Pow")); adam_desc.SetInput("Beta2Pow", adam_ops[0]->Op()->Input("Beta2Pow")); - adam_desc.SetOutput("ParamOut", {fused_vars_name.at("Param")}); + adam_desc.SetOutput("ParamOut", {fused_vars_name.at(kParam)}); adam_desc.SetOutput("Moment1Out", {fused_vars_name.at("Moment1")}); adam_desc.SetOutput("Moment2Out", {fused_vars_name.at("Moment2")}); adam_desc.SetAttr("beta1", beta1); diff --git a/paddle/fluid/framework/details/fuse_optimizer_op_pass.cc b/paddle/fluid/framework/details/fuse_optimizer_op_pass.cc index b49f095d428a017dd1a3bed2788a048af9afa6bb..25aa3019d102293725d836cf1f8e9fce8462408b 100644 --- a/paddle/fluid/framework/details/fuse_optimizer_op_pass.cc +++ b/paddle/fluid/framework/details/fuse_optimizer_op_pass.cc @@ -29,7 +29,9 @@ void FuseOptimizerOpPass::ApplyImpl(ir::Graph *graph) const { auto &local_scopes = Get>(kLocalScopes); const std::string fuse_op_type = GetOpType(); - const std::vector aux_var_names = GetAuxiliaryVarNames(); + std::vector aux_var_names = GetAuxiliaryVarNames(); + aux_var_names.emplace_back(kParam); + aux_var_names.emplace_back(kGrad); // Step 1: Get the specified op and auxiliary variables. std::vector topo_nodes = ir::TopologySortOperations(result); @@ -61,7 +63,7 @@ void FuseOptimizerOpPass::ApplyImpl(ir::Graph *graph) const { result.Set(kFusedVars, new FusedVars); } std::unordered_map fused_vars_name; - fused_vars_name.reserve(aux_var_names.size() + 1); + fused_vars_name.reserve(aux_var_names.size()); auto &fused_var_set = result.Get(kFusedVars); const std::string prefix(kFusedVarNamePrefix); // NOTE: the fused_var_name should be unique. @@ -75,39 +77,103 @@ void FuseOptimizerOpPass::ApplyImpl(ir::Graph *graph) const { } // Step 3: Get the fused Gradient's name - auto ¶ms_grads = result.Get(kParamsAndGrads); - if (!result.Has(kFusedGrads)) { - PADDLE_THROW( - "The alloc_continuous_space_for_grad_pass should be called before this " - "pass."); - } - auto &fused_grad = result.Get(kFusedGrads); - auto &fused_vars = result.Get(kFusedVars); - auto iter = std::find(fused_vars.begin(), fused_vars.end(), fused_grad); - PADDLE_ENFORCE(iter != fused_vars.end(), "Not find the fused_grad."); - fused_vars_name.emplace("Grad", fused_grad); - - // Step 4: Sort the parameters and auxiliary variables according - // to parameters' name to make variables' name correspond correctly. - PADDLE_ENFORCE(result.Has(kParamsAndGrads), "Does't find kParamsAndGrads."); - PADDLE_ENFORCE_EQ(params_grads.size(), aux_var_set.begin()->second.size(), - "The size of params_grads and aux_var_set are not equal."); - SortParametersAndAuxVars(params_grads, &aux_var_set, &opt_ops); - - // Step 5: Alloc continuous space for Parameters and AuxiliaryVar(e.g. + bool grad_fused = false; + if (result.Has(kParamsAndGrads)) { + auto ¶ms_grads = result.Get(kParamsAndGrads); + PADDLE_ENFORCE_EQ( + params_grads.size(), aux_var_set.at(kGrad).size(), + "The number of gradients and optimizer ops is not equal."); + std::unordered_set opt_grad_set(aux_var_set.at(kGrad).begin(), + aux_var_set.at(kGrad).end()); + size_t same_grad_num = 0; + for (auto &p_g : params_grads) { + if (opt_grad_set.count(p_g.second)) { + ++same_grad_num; + } + } + + // NOTE(zcd): the gradient of kParamsAndGrads may be different with the + // kGrad. + if (same_grad_num == aux_var_set.at(kGrad).size()) { + if (!result.Has(kFusedGrads)) { + PADDLE_THROW( + "The alloc_continuous_space_for_grad_pass should be called before " + "this pass."); + } + auto &fused_grad = result.Get(kFusedGrads); + auto &fused_vars = result.Get(kFusedVars); + auto iter = std::find(fused_vars.begin(), fused_vars.end(), fused_grad); + PADDLE_ENFORCE(iter != fused_vars.end(), "Not find the fused_grad."); + fused_vars_name[kGrad] = fused_grad; + + // Sort the parameters and auxiliary variables according + // to parameters' name to make variables' name correspond correctly. + SortParametersAndAuxVars(params_grads, &aux_var_set, &opt_ops); + grad_fused = true; + } + } + + // Step 4: Alloc continuous space for Parameters and AuxiliaryVar(e.g. // Moment1, Moment2, Beta1Pow, Beta2Pow) of all the optimizer ops separately. + aux_var_names.pop_back(); + if (!grad_fused) { + InitFusedGradsAndAllocSpaceForGrads( + places, local_scopes, aux_var_set.at(kParam), aux_var_set.at(kGrad), + fused_vars_name.at(kGrad), &result); + } InitFusedVarsAndAllocSpaceForVars(places, local_scopes, aux_var_names, aux_var_set, fused_vars_name); - // Step 6: Fuse optimizer Ops and Scale Ops + // Step 5: Fuse optimizer Ops and Scale Ops FuseOptimizerOps(aux_var_set, fused_vars_name, opt_ops, &result); - // Step 7: Remove optimizer Ops + // Step 6: Remove optimizer Ops for (auto &opt_op : opt_ops) { graph->RemoveNode(opt_op); } } +void FuseOptimizerOpPass::InitFusedGradsAndAllocSpaceForGrads( + const std::vector &places, + const std::vector &local_scopes, + const std::vector ¶ms, + const std::vector &grads, const std::string &fused_grad_name, + ir::Graph *result) const { + // Get Var Nodes + std::unordered_map vars; + for (ir::Node *node : result->Nodes()) { + if (node->IsVar() && node->Var()) { + // Note: The graph may have the same name node. For example, parameter + // is the input of operator and it also is the output of optimizer; + vars.emplace(node->Var()->Name(), node); + } + } + // Init Grads + for (auto it = local_scopes.rbegin(); it != local_scopes.rend(); ++it) { + auto &scope = *it; + VLOG(10) << "Init " << fused_grad_name; + PADDLE_ENFORCE(scope->FindVar(fused_grad_name) == nullptr, + "%s has existed in scope.", fused_grad_name); + scope->Var(fused_grad_name)->GetMutable(); + + for (auto &grad_var_name : grads) { + auto iter = vars.find(grad_var_name); + PADDLE_ENFORCE(iter != vars.end()); + PADDLE_ENFORCE_NOT_NULL(iter->second->Var()); + PADDLE_ENFORCE_EQ(iter->second->Var()->GetType(), + proto::VarType::LOD_TENSOR); + scope->Var(grad_var_name)->GetMutable(); + } + } + // Define Ops + ProgramDesc program_desc; + auto *global_block = program_desc.MutableBlock(0); + AppendAllocContinuousSpace(params, grads, fused_grad_name, global_block, + false, false); + // Run Ops + RunInitOps(places, local_scopes, *global_block); +} + void FuseOptimizerOpPass::InitFusedVarsAndAllocSpaceForVars( const std::vector &places, const std::vector &local_scopes, @@ -115,37 +181,49 @@ void FuseOptimizerOpPass::InitFusedVarsAndAllocSpaceForVars( const std::unordered_map> &aux_var_set, const std::unordered_map &fused_vars_name) const { - VLOG(10) << "Init FusedVars."; - // Alloc parameters and auxiliary vars in the respective scope. - size_t idx = local_scopes.size(); - for (auto iter = local_scopes.rbegin(); iter != local_scopes.rend(); - ++iter, --idx) { - auto &scope = *iter; - for (auto &var_name : aux_var_names) { - auto fused_var_name = fused_vars_name.at(var_name); - VLOG(10) << "Init " << fused_var_name; - PADDLE_ENFORCE(scope->FindVar(fused_var_name) == nullptr, - "%s has exist in scope[%d]", fused_var_name, idx); - scope->Var(fused_var_name)->GetMutable(); - } + // Init Vars + for (auto &var_name : aux_var_names) { + auto &fused_var_name = fused_vars_name.at(var_name); + InitVars(local_scopes, fused_var_name); } - + // Define Ops ProgramDesc program_desc; auto *global_block = program_desc.MutableBlock(0); for (auto &var_name : aux_var_names) { - AppendAllocContinuousSpace(aux_var_set.at(var_name), - fused_vars_name.at(var_name), true, - global_block); + AppendAllocContinuousSpace( + aux_var_set.at(var_name), aux_var_set.at(var_name), + fused_vars_name.at(var_name), global_block, true); } + // Run Ops + RunInitOps(places, local_scopes, *global_block); +} +void FuseOptimizerOpPass::RunInitOps(const std::vector &places, + const std::vector &local_scopes, + const BlockDesc &global_block) const { for (size_t i = 0; i < local_scopes.size(); ++i) { - for (auto &op_desc : global_block->AllOps()) { + for (auto &op_desc : global_block.AllOps()) { auto op = OpRegistry::CreateOp(*op_desc); op->Run(*local_scopes[i], places[i]); } } } +void FuseOptimizerOpPass::InitVars(const std::vector &local_scopes, + const std::string &fused_var_name) const { + VLOG(10) << "Init FusedVars."; + // Alloc parameters and auxiliary vars in the respective scope. + size_t idx = local_scopes.size(); + for (auto iter = local_scopes.rbegin(); iter != local_scopes.rend(); + ++iter, --idx) { + auto &scope = *iter; + VLOG(10) << "Init " << fused_var_name; + PADDLE_ENFORCE(scope->FindVar(fused_var_name) == nullptr, + "%s has exist in scope[%d]", fused_var_name, idx); + scope->Var(fused_var_name)->GetMutable(); + } +} + void FuseOptimizerOpPass::SortParametersAndAuxVars( const std::vector> ¶ms_grads, std::unordered_map> *aux_vars_set, @@ -203,15 +281,16 @@ void FuseOptimizerOpPass::GetSpecifiedOpsAndVars( } void FuseOptimizerOpPass::AppendAllocContinuousSpace( - const std::vector &args, const std::string &out_arg, - bool copy_data, BlockDesc *global_block) const { + const std::vector &in_args, + const std::vector &out_args, const std::string &fused_out_arg, + BlockDesc *global_block, bool copy_data, bool check_name) const { auto op_desc = global_block->AppendOp(); op_desc->SetType("alloc_continuous_space"); - op_desc->SetInput("Input", args); - op_desc->SetOutput("Output", args); - op_desc->SetOutput("FusedOutput", {out_arg}); + op_desc->SetInput("Input", in_args); + op_desc->SetOutput("Output", out_args); + op_desc->SetOutput("FusedOutput", {fused_out_arg}); op_desc->SetAttr("copy_data", copy_data); - op_desc->SetAttr("check_name", true); + op_desc->SetAttr("check_name", check_name); } void FuseOptimizerOpPass::InserInputAndOutputForOptOps( diff --git a/paddle/fluid/framework/details/fuse_optimizer_op_pass.h b/paddle/fluid/framework/details/fuse_optimizer_op_pass.h index 0240f1594d7ef9d855eb6e96e8e8a32ee1d957ba..47efc1693dd31ca88787da3a9d6d06aa7ef65786 100644 --- a/paddle/fluid/framework/details/fuse_optimizer_op_pass.h +++ b/paddle/fluid/framework/details/fuse_optimizer_op_pass.h @@ -27,6 +27,10 @@ namespace paddle { namespace framework { namespace details { +constexpr char kGrad[] = "Grad"; +constexpr char kParam[] = "Param"; +constexpr char kLearningRate[] = "LearningRate"; + class FuseOptimizerOpPass : public ir::Pass { protected: void ApplyImpl(ir::Graph *graph) const override; @@ -56,9 +60,18 @@ class FuseOptimizerOpPass : public ir::Pass { std::unordered_map> *aux_args_name) const; - void AppendAllocContinuousSpace(const std::vector &args, - const std::string &out_arg, bool copy_data, - BlockDesc *global_block) const; + void AppendAllocContinuousSpace(const std::vector &in_args, + const std::vector &out_args, + const std::string &fused_out_arg, + BlockDesc *global_block, bool copy_data, + bool check_name = true) const; + + void InitFusedGradsAndAllocSpaceForGrads( + const std::vector &places, + const std::vector &local_scopes, + const std::vector ¶ms, + const std::vector &grads, const std::string &fused_grad_name, + ir::Graph *result) const; void InitFusedVarsAndAllocSpaceForVars( const std::vector &places, @@ -68,6 +81,13 @@ class FuseOptimizerOpPass : public ir::Pass { &aux_var_set, const std::unordered_map &fused_vars_name) const; + + void RunInitOps(const std::vector &places, + const std::vector &local_scopes, + const BlockDesc &global_block) const; + + void InitVars(const std::vector &local_scopes, + const std::string &fused_var_name) const; }; } // namespace details diff --git a/paddle/fluid/framework/details/fuse_sgd_op_pass.cc b/paddle/fluid/framework/details/fuse_sgd_op_pass.cc index f91c21e3cc869de1a6d67146eb99f27a2ca5497c..2219f3209f77de5cb34abfb9edb8bdea6a8eebb0 100644 --- a/paddle/fluid/framework/details/fuse_sgd_op_pass.cc +++ b/paddle/fluid/framework/details/fuse_sgd_op_pass.cc @@ -24,7 +24,7 @@ namespace details { const std::string FuseSgdOpPass::GetOpType() const { return "sgd"; } const std::vector FuseSgdOpPass::GetAuxiliaryVarNames() const { - return {"Param"}; + return {}; } void FuseSgdOpPass::FuseOptimizerOps( @@ -50,12 +50,12 @@ void FuseSgdOpPass::FuseSgdOps( // Add fused scale OpDesc Sgd_desc(sgd_ops[0]->Op()->Block()); Sgd_desc.SetType("sgd"); - Sgd_desc.SetInput("Param", {fused_vars_name.at("Param")}); - Sgd_desc.SetInput("Grad", {fused_vars_name.at("Grad")}); - Sgd_desc.SetOutput("ParamOut", {fused_vars_name.at("Param")}); + Sgd_desc.SetInput(kParam, {fused_vars_name.at(kParam)}); + Sgd_desc.SetInput(kGrad, {fused_vars_name.at(kGrad)}); + Sgd_desc.SetOutput("ParamOut", {fused_vars_name.at(kParam)}); // TODO(zcd): The LearningRate, Beta1Pow, Beta2Pow should be equal. - Sgd_desc.SetInput("LearningRate", sgd_ops[0]->Op()->Input("LearningRate")); + Sgd_desc.SetInput(kLearningRate, sgd_ops[0]->Op()->Input(kLearningRate)); // NOTE: multi_devices_pass requires that every op should have a role. Sgd_desc.SetAttr(OpProtoAndCheckerMaker::OpRoleAttrName(), op_role); diff --git a/paddle/fluid/framework/details/inplace_op_pass.cc b/paddle/fluid/framework/details/inplace_op_pass.cc index 79150f719e379ca4e2b87d2e7db1b2daeee9aa67..84c9e4a379a5e07dc3a8e85409c804eebc390c73 100644 --- a/paddle/fluid/framework/details/inplace_op_pass.cc +++ b/paddle/fluid/framework/details/inplace_op_pass.cc @@ -305,6 +305,12 @@ void InplacePass::TryInplaceOpInputOutput(ir::Node* op, VLOG(4) << "Try to inplace " << in_var_name << " with " << out_var_name; + if (var_nodes_[in_var_name].back() != in_node) { + VLOG(4) << "SKIP since " << in_var_name + << " is also used as output by other ops"; + continue; + } + bool can_replace = true; if (in_var_name == out_var_name) { can_replace = false; @@ -527,6 +533,9 @@ void GraphView::Build(ir::Graph* g) { }; for (auto& node : g->Nodes()) { if (!node->IsOp()) continue; + // avoid optimize the variable used in sub-blocks + if (OpHasSubBlock(node->Op())) update_skip_set(node); + if (node->Name() == "send") update_skip_set(node); if (node->Name() == "recv") update_skip_set(node); if (node->Name() == "prefetch") update_skip_set(node); diff --git a/paddle/fluid/framework/details/op_registry.h b/paddle/fluid/framework/details/op_registry.h index a9a4fb08a2ca4689e8b6a6f10f83d065332ac192..18de595983f52e56dba4f5069257f354132db51b 100644 --- a/paddle/fluid/framework/details/op_registry.h +++ b/paddle/fluid/framework/details/op_registry.h @@ -233,6 +233,12 @@ struct OpInfoFiller { } }; +// A fake OpInfoFiller of void +template <> +struct OpInfoFiller { + void operator()(const char* op_type, OpInfo* info) const {} +}; + } // namespace details } // namespace framework diff --git a/paddle/fluid/framework/details/parallel_ssa_graph_executor.cc b/paddle/fluid/framework/details/parallel_ssa_graph_executor.cc index 137e0dd7708dcc77c3a927979cfb357249f1fdc9..1bd27263f7dad5f733c553c202444ba7cacd2510 100644 --- a/paddle/fluid/framework/details/parallel_ssa_graph_executor.cc +++ b/paddle/fluid/framework/details/parallel_ssa_graph_executor.cc @@ -106,7 +106,7 @@ ParallelSSAGraphExecutor::ParallelSSAGraphExecutor( VLOG(1) << "set num_threads: " << strategy_.num_threads_ << " to run the operators of the graph on each device."; for (size_t i = 0; i < places.size(); ++i) { - executors_.emplace_back(new details::ThreadedSSAGraphExecutor( + executors_.emplace_back(new details::FastThreadedSSAGraphExecutor( strategy_, local_scopes_, {places_[i]}, graphs_.at(i).get())); } } diff --git a/paddle/fluid/framework/details/parallel_ssa_graph_executor.h b/paddle/fluid/framework/details/parallel_ssa_graph_executor.h index 1e421f2a3a51363fe368859f7a34593c8c894077..faf071b05306a49c0049421bc72e4981c0bfc84c 100644 --- a/paddle/fluid/framework/details/parallel_ssa_graph_executor.h +++ b/paddle/fluid/framework/details/parallel_ssa_graph_executor.h @@ -14,12 +14,12 @@ #pragma once +#include #include #include - #include "ThreadPool.h" +#include "paddle/fluid/framework/details/fast_threaded_ssa_graph_executor.h" #include "paddle/fluid/framework/details/multi_devices_helper.h" -#include "paddle/fluid/framework/details/threaded_ssa_graph_executor.h" #include "paddle/fluid/framework/ir/graph.h" namespace paddle { @@ -48,7 +48,8 @@ class ParallelSSAGraphExecutor : public SSAGraphExecutor { std::vector places_; std::vector> graphs_; - std::vector> executors_; + std::vector> + executors_; ExceptionHolder exception_holder_; }; diff --git a/paddle/fluid/framework/downpour_worker.cc b/paddle/fluid/framework/downpour_worker.cc index 4ca7842fa261a1b8178438d35ca5d626146663d4..386ffd84c57063e950cd8b0d57304c66190be4c4 100644 --- a/paddle/fluid/framework/downpour_worker.cc +++ b/paddle/fluid/framework/downpour_worker.cc @@ -21,40 +21,40 @@ namespace framework { void DownpourWorker::Initialize(const TrainerDesc& desc) { param_ = desc.downpour_param(); - for (size_t i = 0; i < param_.sparse_table_size(); ++i) { + for (int i = 0; i < param_.sparse_table_size(); ++i) { uint64_t table_id = static_cast(param_.sparse_table(i).table_id()); TableParameter table = param_.sparse_table(i); sparse_key_names_[table_id].resize(table.sparse_key_name_size()); - for (size_t j = 0; j < table.sparse_key_name_size(); ++j) { + for (int j = 0; j < table.sparse_key_name_size(); ++j) { sparse_key_names_[table_id][j] = table.sparse_key_name(j); } sparse_value_names_[table_id].resize(table.sparse_value_name_size()); - for (size_t j = 0; j < table.sparse_value_name_size(); ++j) { + for (int j = 0; j < table.sparse_value_name_size(); ++j) { sparse_value_names_[table_id][j] = table.sparse_value_name(j); } sparse_grad_names_[table_id].resize(table.sparse_grad_name_size()); - for (size_t j = 0; j < table.sparse_grad_name_size(); ++j) { + for (int j = 0; j < table.sparse_grad_name_size(); ++j) { sparse_grad_names_[table_id][j] = table.sparse_grad_name(j); } label_var_name_[table_id] = table.label_var_name(); } - for (size_t i = 0; i < param_.dense_table_size(); ++i) { + for (int i = 0; i < param_.dense_table_size(); ++i) { uint64_t table_id = static_cast(param_.dense_table(i).table_id()); auto table = param_.dense_table(i); dense_value_names_[table_id].resize(table.dense_value_name_size()); - for (size_t j = 0; j < table.dense_value_name_size(); ++j) { + for (int j = 0; j < table.dense_value_name_size(); ++j) { dense_value_names_[table_id][j] = table.dense_value_name(j); } dense_grad_names_[table_id].resize(table.dense_grad_name_size()); - for (size_t j = 0; j < table.dense_grad_name_size(); ++j) { + for (int j = 0; j < table.dense_grad_name_size(); ++j) { dense_grad_names_[table_id][j] = table.dense_grad_name(j); } } skip_ops_.resize(param_.skip_ops_size()); - for (size_t i = 0; i < param_.skip_ops_size(); ++i) { + for (int i = 0; i < param_.skip_ops_size(); ++i) { skip_ops_[i] = param_.skip_ops(i); } @@ -83,14 +83,14 @@ void DownpourWorker::CollectLabelInfo(size_t table_idx) { LoDTensor* tensor = var->GetMutable(); int64_t* label_ptr = tensor->data(); - int global_index = 0; + size_t global_index = 0; for (size_t i = 0; i < sparse_key_names_[table_id].size(); ++i) { VLOG(3) << "sparse_key_names_[" << i << "]: " << sparse_key_names_[table_id][i]; Variable* fea_var = thread_scope_->FindVar(sparse_key_names_[table_id][i]); LoDTensor* tensor = fea_var->GetMutable(); int64_t* ids = tensor->data(); - int fea_idx = 0; + size_t fea_idx = 0; // tensor->lod()[0].size() == batch_size + 1 for (auto lod_idx = 1u; lod_idx < tensor->lod()[0].size(); ++lod_idx) { for (; fea_idx < tensor->lod()[0][lod_idx]; ++fea_idx) { @@ -138,7 +138,7 @@ void DownpourWorker::FillSparseValue(size_t table_idx) { auto& tensor_lod = tensor->lod()[0]; LoD data_lod{tensor_lod}; tensor_emb->set_lod(data_lod); - for (auto index = 0u; index < len; ++index) { + for (int index = 0; index < len; ++index) { if (ids[index] == 0u) { memcpy(ptr + table.emb_dim() * index, init_value.data() + 2, sizeof(float) * table.emb_dim()); @@ -192,7 +192,7 @@ void DownpourWorker::TrainFilesWithProfiler() { read_time += timeline.ElapsedSec(); total_time += timeline.ElapsedSec(); VLOG(3) << "program config size: " << param_.program_config_size(); - for (size_t i = 0; i < param_.program_config(0).pull_sparse_table_id_size(); + for (int i = 0; i < param_.program_config(0).pull_sparse_table_id_size(); ++i) { uint64_t tid = static_cast( param_.program_config(0).pull_sparse_table_id(i)); @@ -244,8 +244,8 @@ void DownpourWorker::TrainFilesWithProfiler() { } if (need_to_push_sparse_) { - for (size_t i = 0; - i < param_.program_config(0).push_sparse_table_id_size(); ++i) { + for (int i = 0; i < param_.program_config(0).push_sparse_table_id_size(); + ++i) { uint64_t tid = static_cast( param_.program_config(0).push_sparse_table_id(i)); TableParameter table; @@ -268,8 +268,8 @@ void DownpourWorker::TrainFilesWithProfiler() { if (need_to_push_dense_) { timeline.Start(); - for (size_t i = 0; - i < param_.program_config(0).push_dense_table_id_size(); ++i) { + for (int i = 0; i < param_.program_config(0).push_dense_table_id_size(); + ++i) { uint64_t tid = static_cast( param_.program_config(0).push_dense_table_id(i)); fleet_ptr_->PushDenseVarsAsync( @@ -315,8 +315,8 @@ void DownpourWorker::TrainFilesWithProfiler() { } if (need_to_push_dense_) { - for (size_t i = 0; - i < param_.program_config(0).push_dense_table_id_size(); ++i) { + for (int i = 0; i < param_.program_config(0).push_dense_table_id_size(); + ++i) { uint64_t tid = static_cast( param_.program_config(0).push_dense_table_id(i)); pull_dense_worker_->IncreaseThreadVersion(thread_id_, tid); @@ -362,7 +362,7 @@ void DownpourWorker::TrainFiles() { int cur_batch; while ((cur_batch = device_reader_->Next()) > 0) { // pull sparse here - for (size_t i = 0; i < param_.program_config(0).pull_sparse_table_id_size(); + for (int i = 0; i < param_.program_config(0).pull_sparse_table_id_size(); ++i) { uint64_t tid = static_cast( param_.program_config(0).pull_sparse_table_id(i)); @@ -397,8 +397,8 @@ void DownpourWorker::TrainFiles() { if (need_to_push_sparse_) { // push gradients here - for (size_t i = 0; - i < param_.program_config(0).push_sparse_table_id_size(); ++i) { + for (int i = 0; i < param_.program_config(0).push_sparse_table_id_size(); + ++i) { uint64_t tid = static_cast( param_.program_config(0).push_sparse_table_id(i)); TableParameter table; @@ -416,8 +416,8 @@ void DownpourWorker::TrainFiles() { } if (need_to_push_dense_) { - for (size_t i = 0; - i < param_.program_config(0).push_dense_table_id_size(); ++i) { + for (int i = 0; i < param_.program_config(0).push_dense_table_id_size(); + ++i) { uint64_t tid = static_cast( param_.program_config(0).push_dense_table_id(i)); fleet_ptr_->PushDenseVarsAsync( @@ -461,8 +461,8 @@ void DownpourWorker::TrainFiles() { } if (need_to_push_dense_) { - for (size_t i = 0; - i < param_.program_config(0).push_dense_table_id_size(); ++i) { + for (int i = 0; i < param_.program_config(0).push_dense_table_id_size(); + ++i) { uint64_t tid = static_cast( param_.program_config(0).push_dense_table_id(i)); pull_dense_worker_->IncreaseThreadVersion(thread_id_, tid); diff --git a/paddle/fluid/framework/ir/CMakeLists.txt b/paddle/fluid/framework/ir/CMakeLists.txt index ba1d7379c56d953a0f37d03deed6c47e46cbf129..16fc1721eb6f5d2517ad45289f2415ef41749df2 100644 --- a/paddle/fluid/framework/ir/CMakeLists.txt +++ b/paddle/fluid/framework/ir/CMakeLists.txt @@ -68,6 +68,7 @@ pass_library(transpose_flatten_concat_fuse_pass inference) pass_library(identity_scale_op_clean_pass base) pass_library(sync_batch_norm_pass base) pass_library(runtime_context_cache_pass base) +pass_library(expected_kernel_cache_pass base) pass_library(quant_conv2d_dequant_fuse_pass inference) pass_library(fillconstant_elementwisemul_fuse inference) diff --git a/paddle/fluid/framework/ir/expected_kernel_cache_pass.cc b/paddle/fluid/framework/ir/expected_kernel_cache_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..4a99d4c1a9c0f0bd973097d281e380341fe88515 --- /dev/null +++ b/paddle/fluid/framework/ir/expected_kernel_cache_pass.cc @@ -0,0 +1,37 @@ +/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/framework/ir/expected_kernel_cache_pass.h" +#include +#include "paddle/fluid/framework/operator.h" + +namespace paddle { +namespace framework { +namespace ir { + +void ExpectedKernelCachePass::ApplyImpl(ir::Graph* graph) const { + VLOG(3) << "Applies Expected Kernel Cache strategy."; + for (const Node* n : graph->Nodes()) { + if (n->IsOp() && n->Op()) { + n->Op()->SetAttr(kEnableCacheExpectedKernel, true); + } + } +} + +} // namespace ir +} // namespace framework +} // namespace paddle + +REGISTER_PASS(expected_kernel_cache_pass, + paddle::framework::ir::ExpectedKernelCachePass); diff --git a/paddle/fluid/platform/dynload/wbaes.cc b/paddle/fluid/framework/ir/expected_kernel_cache_pass.h similarity index 66% rename from paddle/fluid/platform/dynload/wbaes.cc rename to paddle/fluid/framework/ir/expected_kernel_cache_pass.h index 37387b202aadddef859b0eecca55cb9c99d826ee..bf0907d3feb7bccd163363da65505e0af3fb9bf6 100644 --- a/paddle/fluid/platform/dynload/wbaes.cc +++ b/paddle/fluid/framework/ir/expected_kernel_cache_pass.h @@ -12,23 +12,20 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#ifdef PADDLE_WITH_WBAES +#pragma once -#include "paddle/fluid/platform/dynload/wbaes.h" +#include +#include "paddle/fluid/framework/ir/pass.h" namespace paddle { -namespace platform { -namespace dynload { +namespace framework { +namespace ir { -std::once_flag wbaes_dso_flag; -void *wbaes_dso_handle = nullptr; +class ExpectedKernelCachePass : public Pass { + protected: + void ApplyImpl(ir::Graph* graph) const override; +}; -#define DEFINE_WRAP(__name) DynLoad__##__name __name - -WBAES_ROUTINE_EACH(DEFINE_WRAP); - -} // namespace dynload -} // namespace platform +} // namespace ir +} // namespace framework } // namespace paddle - -#endif diff --git a/paddle/fluid/framework/ir/graph_helper.cc b/paddle/fluid/framework/ir/graph_helper.cc index 28a37f331c100695f0ffec7288db84f4493d68a0..12ce99c8788625e2aae6e07abdea565bb2c2ebb9 100644 --- a/paddle/fluid/framework/ir/graph_helper.cc +++ b/paddle/fluid/framework/ir/graph_helper.cc @@ -31,10 +31,10 @@ namespace paddle { namespace framework { namespace ir { namespace { -void SortHelper( - const std::map> &adj_list, - ir::Node *node, std::unordered_set *visited, - std::vector *ret) { +void SortHelper(const std::map, + ir::NodeComp> &adj_list, + ir::Node *node, std::unordered_set *visited, + std::vector *ret) { visited->insert(node); for (auto adj : adj_list.at(node)) { @@ -50,7 +50,8 @@ void SortHelper( bool HasCircleHelper( ir::Node *node, - const std::map> &adj_list, + const std::map, ir::NodeComp> + &adj_list, std::unordered_set *visited, std::unordered_set *in_trace, std::vector> *circles) { @@ -84,7 +85,8 @@ bool HasCircleHelper( } bool HasCircleInternal( - const std::map> &adj_list, + const std::map, ir::NodeComp> + &adj_list, std::vector> *circles) { std::unordered_set visited; std::unordered_set in_trace; @@ -107,8 +109,8 @@ bool FindCircleSubGraph(const Graph &graph, } std::vector TopologySortOperations(const Graph &graph) { - std::map> adj_list = - BuildOperationAdjList(graph); + std::map, ir::NodeComp> + adj_list = BuildOperationAdjList(graph); PADDLE_ENFORCE(!HasCircleInternal(adj_list, nullptr)); std::unordered_set visited; std::vector ret; @@ -117,34 +119,30 @@ std::vector TopologySortOperations(const Graph &graph) { SortHelper(adj_list, adj.first, &visited, &ret); } } + return ret; } // Build operator inlink edge table. -std::map> BuildOperationAdjList( - const Graph &graph) { - std::map> adj_list; +std::map, ir::NodeComp> +BuildOperationAdjList(const Graph &graph) { + std::map, ir::NodeComp> + adj_list; for (auto &n : graph.Nodes()) { if (!n->IsOp()) continue; if (adj_list.find(n) == adj_list.end()) { - adj_list[n] = std::unordered_set(); + adj_list[n] = std::set(); } - std::vector nodes; for (auto &var : n->inputs) { for (auto &adj_n : var->inputs) { PADDLE_ENFORCE(adj_n->NodeType() == ir::Node::Type::kOperation); VLOG(4) << "adj " << adj_n->Name() << reinterpret_cast(adj_n) << " -> " << n->Name() << reinterpret_cast(n) << " via " << var->Name() << reinterpret_cast(var); - nodes.push_back(adj_n); + adj_list[n].insert(adj_n); } } - std::sort(nodes.begin(), nodes.end(), [](ir::Node *node1, ir::Node *node2) { - return node1->id() > node2->id(); - }); - adj_list[n].insert(std::make_move_iterator(nodes.begin()), - std::make_move_iterator(nodes.end())); } return adj_list; } diff --git a/paddle/fluid/framework/ir/graph_helper.h b/paddle/fluid/framework/ir/graph_helper.h index 214de9ec7d85aee6021b18866295777e317aa79d..849a9c3be6904f3f9c3669d8fc9d750154863031 100644 --- a/paddle/fluid/framework/ir/graph_helper.h +++ b/paddle/fluid/framework/ir/graph_helper.h @@ -16,6 +16,7 @@ limitations under the License. */ #include #include +#include #include #include "paddle/fluid/framework/ir/graph.h" @@ -25,6 +26,13 @@ namespace paddle { namespace framework { namespace ir { +// Compare nodes via node id. +struct NodeComp { + bool operator()(ir::Node *const &node1, ir::Node *const &node2) const { + return node1->id() < node2->id(); + } +}; + // Test if the graph contains circle. bool HasCircle(const Graph &graph); @@ -57,8 +65,8 @@ std::vector TopologyVarientSort(const Graph &graph, SortKind sort_kind); void CleanIndividualNodes(Graph *graph); // Build an adjacency list of operations for the `graph`. -std::map> BuildOperationAdjList( - const Graph &graph); +std::map, ir::NodeComp> +BuildOperationAdjList(const Graph &graph); template std::vector FilterByNodeWrapper(const Graph &graph) { diff --git a/paddle/fluid/framework/ir/runtime_context_cache_pass.cc b/paddle/fluid/framework/ir/runtime_context_cache_pass.cc index c7cf9b0dc342bbfaa80b622d7dcd0f6348f78d42..566b654f237cbd71e1983c971374ee13d7b36805 100644 --- a/paddle/fluid/framework/ir/runtime_context_cache_pass.cc +++ b/paddle/fluid/framework/ir/runtime_context_cache_pass.cc @@ -23,7 +23,7 @@ namespace ir { void RuntimeContextCachePass::ApplyImpl(ir::Graph* graph) const { VLOG(3) << "Applies Runtime Context Cache strategy."; for (const Node* n : graph->Nodes()) { - if (n->IsOp()) { + if (n->IsOp() && n->Op()) { n->Op()->SetAttr(kEnableCacheRuntimeContext, true); } } diff --git a/paddle/fluid/framework/op_desc.cc b/paddle/fluid/framework/op_desc.cc index e6f5b15af8cd440a9304235acfe62787c5f1b134..1ea93b7638a85e67bcc85a0c0e130d636938d6c5 100644 --- a/paddle/fluid/framework/op_desc.cc +++ b/paddle/fluid/framework/op_desc.cc @@ -241,6 +241,7 @@ OpDesc::OpDesc(const std::string &type, const VariableNameMap &inputs, outputs_ = outputs; attrs_ = attrs; need_update_ = true; + block_ = nullptr; } OpDesc::OpDesc(const OpDesc &other, BlockDesc *block) { diff --git a/paddle/fluid/framework/operator.cc b/paddle/fluid/framework/operator.cc index 168f287a455c644695b6eaff426ce31ded8d38a5..1723a9a78a0da6e3eac7f823f79fe802a916e5b3 100644 --- a/paddle/fluid/framework/operator.cc +++ b/paddle/fluid/framework/operator.cc @@ -880,7 +880,16 @@ std::vector* OperatorWithKernel::GetKernelConfig( void OperatorWithKernel::RunImpl(const Scope& scope, const platform::Place& place) const { - if (!HasAttr(kEnableCacheRuntimeContext)) { + // To reduce the elapsed time of HasAttr, we use bool variable to record the + // result of HasAttr. + if (!enable_cache_runtime_context && HasAttr(kEnableCacheRuntimeContext)) + enable_cache_runtime_context = true; + if (!enable_cache_expected_kernel && HasAttr(kEnableCacheExpectedKernel)) + enable_cache_expected_kernel = true; + if (!all_kernels_must_compute_runtime_shape && + HasAttr(kAllKernelsMustComputeRuntimeShape)) + all_kernels_must_compute_runtime_shape = true; + if (!enable_cache_runtime_context) { RuntimeContext ctx(Inputs(), Outputs(), scope); RunImpl(scope, place, &ctx); } else { @@ -899,60 +908,33 @@ void OperatorWithKernel::RunImpl(const Scope& scope, platform::DeviceContextPool& pool = platform::DeviceContextPool::Instance(); auto* dev_ctx = pool.Get(place); - // check if op[type] has kernel registered. - auto& all_op_kernels = AllOpKernels(); - auto kernels_iter = all_op_kernels.find(type_); - if (kernels_iter == all_op_kernels.end()) { - PADDLE_THROW( - "There are no kernels which are registered in the %s operator.", type_); + if (!enable_cache_expected_kernel || !kernel_type_) { + ChooseKernel(*runtime_ctx, scope, place); } - OpKernelMap& kernels = kernels_iter->second; - - auto expected_kernel_key = this->GetExpectedKernelType( - ExecutionContext(*this, scope, *dev_ctx, *runtime_ctx, nullptr)); - VLOG(3) << "expected_kernel_key:" << expected_kernel_key; - - auto kernel_iter = kernels.find(expected_kernel_key); -#ifdef PADDLE_WITH_MKLDNN - // workaround for missing MKLDNN kernel when FLAGS_use_mkldnn env var is set - if (kernel_iter == kernels.end() && - expected_kernel_key.library_type_ == LibraryType::kMKLDNN) { - VLOG(3) << "missing MKLDNN kernel: fallbacking to PLAIN one"; - expected_kernel_key.library_type_ = LibraryType::kPlain; - expected_kernel_key.data_layout_ = DataLayout::kAnyLayout; - kernel_iter = kernels.find(expected_kernel_key); - } -#endif - if (kernel_iter == kernels.end()) { - PADDLE_THROW("op %s does not have kernel for %s", type_, - KernelTypeToString(expected_kernel_key)); - } - - std::vector* kernel_configs = - GetKernelConfig(expected_kernel_key); + std::vector* kernel_configs = GetKernelConfig(*kernel_type_); // do data transformScope &transfer_scope; std::vector transfered_inplace_vars; - auto* transfer_scope = PrepareData(scope, expected_kernel_key, - &transfered_inplace_vars, runtime_ctx); + auto* transfer_scope = + PrepareData(scope, *kernel_type_, &transfered_inplace_vars, runtime_ctx); // exec scope is the scope that kernel actually executed on. const Scope& exec_scope = (transfer_scope == nullptr ? scope : *transfer_scope); - if (!(expected_kernel_key.place_ == dev_ctx->GetPlace())) { - dev_ctx = pool.Get(expected_kernel_key.place_); + if (!(kernel_type_->place_ == dev_ctx->GetPlace())) { + dev_ctx = pool.Get(kernel_type_->place_); } - if (!HasAttr(kAllKernelsMustComputeRuntimeShape)) { + if (!all_kernels_must_compute_runtime_shape) { RuntimeInferShapeContext infer_shape_ctx(*this, exec_scope, *runtime_ctx); this->InferShape(&infer_shape_ctx); } // TODO(panyx0718): ExecutionContext should only depend on RuntimeContext // not Scope. Imperative mode only pass inputs and get outputs. - kernel_iter->second(ExecutionContext(*this, exec_scope, *dev_ctx, - *runtime_ctx, kernel_configs)); + (*kernel_func_)(ExecutionContext(*this, exec_scope, *dev_ctx, *runtime_ctx, + kernel_configs)); if (!transfered_inplace_vars.empty()) { // there is inplace variable has been transfered. @@ -978,6 +960,46 @@ void OperatorWithKernel::RunImpl(const Scope& scope, } } +void OperatorWithKernel::ChooseKernel(const RuntimeContext& ctx, + const Scope& scope, + const platform::Place& place) const { + platform::DeviceContextPool& pool = platform::DeviceContextPool::Instance(); + auto* dev_ctx = pool.Get(place); + + // check if op[type] has kernel registered. + auto& all_op_kernels = AllOpKernels(); + auto kernels_iter = all_op_kernels.find(type_); + if (kernels_iter == all_op_kernels.end()) { + PADDLE_THROW( + "There are no kernels which are registered in the %s operator.", type_); + } + + OpKernelMap& kernels = kernels_iter->second; + + auto expected_kernel_key = this->GetExpectedKernelType( + ExecutionContext(*this, scope, *dev_ctx, ctx, nullptr)); + VLOG(3) << "expected_kernel_key:" << expected_kernel_key; + + auto kernel_iter = kernels.find(expected_kernel_key); +#ifdef PADDLE_WITH_MKLDNN + // workaround for missing MKLDNN kernel when FLAGS_use_mkldnn env var is set + if (kernel_iter == kernels.end() && + expected_kernel_key.library_type_ == LibraryType::kMKLDNN) { + VLOG(3) << "missing MKLDNN kernel: fallbacking to PLAIN one"; + expected_kernel_key.library_type_ = LibraryType::kPlain; + expected_kernel_key.data_layout_ = DataLayout::kAnyLayout; + kernel_iter = kernels.find(expected_kernel_key); + } +#endif + if (kernel_iter == kernels.end()) { + PADDLE_THROW("op %s does not have kernel for %s", type_, + KernelTypeToString(expected_kernel_key)); + } + + kernel_type_.reset(new OpKernelType(expected_kernel_key)); + kernel_func_.reset(new OpKernelFunc(kernel_iter->second)); +} + void OperatorWithKernel::TransferInplaceVarsBack( const Scope& scope, const std::vector& inplace_vars, const Scope& transfer_scope) const { diff --git a/paddle/fluid/framework/operator.h b/paddle/fluid/framework/operator.h index a02e53dcf764368601646a900833ac650c5bb31a..489b66099658d522fe1f1adaad763b66bdd22c91 100644 --- a/paddle/fluid/framework/operator.h +++ b/paddle/fluid/framework/operator.h @@ -70,6 +70,12 @@ constexpr char kNewGradSuffix[] = "@NEWGRAD@"; /// this Op's execution to save the elapsed time. constexpr char kEnableCacheRuntimeContext[] = "@ENABLE_CACHE_RUNTIME_CONTEXT@"; +/// If an Op has attribtue kEnableCacheExpectedKernel, it means that in a same +/// name scope and same place, since the expected kerenl of this Op does not +/// change in the execution, it could be recorded only at the first iteration of +/// this Op's execution to save the elapsed time. +constexpr char kEnableCacheExpectedKernel[] = "@ENABLE_CACHE_EXPECTED_KERNEL@"; + /// If an Op has this attribute, all its kernels should calculate output /// variable's shape in the corresponding Compute() function. And /// OperatorWithKernel::RunImpl() would skip call this Op's InferShape() @@ -491,10 +497,18 @@ class OperatorWithKernel : public OperatorBase { const std::vector& inplace_vars, const Scope& exec_scope) const; + void ChooseKernel(const RuntimeContext& ctx, const Scope& scope, + const platform::Place& place) const; + protected: mutable OpKernelConfigsMap kernel_configs_map_; + mutable std::unique_ptr kernel_type_; + mutable std::unique_ptr kernel_func_; mutable std::unique_ptr runtime_ctx_; mutable const Scope* pre_scope_ = nullptr; + mutable bool enable_cache_runtime_context = false; + mutable bool enable_cache_expected_kernel = false; + mutable bool all_kernels_must_compute_runtime_shape = false; }; extern bool OpSupportGPU(const std::string& op_type); diff --git a/paddle/fluid/framework/parallel_executor.cc b/paddle/fluid/framework/parallel_executor.cc index 4245caf1689c76d72b410c742488c55562c8b998..c4bf2b7e8c017b22f917c9f9bd40e75b8cde08b2 100644 --- a/paddle/fluid/framework/parallel_executor.cc +++ b/paddle/fluid/framework/parallel_executor.cc @@ -221,7 +221,7 @@ ParallelExecutor::ParallelExecutor(const std::vector &places, PADDLE_ENFORCE(!member_->use_cuda_, "gpu mode does not support async_mode_ now!"); graphs.push_back(graph); - for (int i = 1; i < places.size(); ++i) { + for (size_t i = 1; i < places.size(); ++i) { auto *tmp_graph = new ir::Graph(graph->OriginProgram()); async_graphs_.emplace_back(tmp_graph); graphs.push_back(tmp_graph); @@ -315,7 +315,7 @@ ParallelExecutor::ParallelExecutor(const std::vector &places, graph = build_strategy.Apply(graph, {member_->places_[0]}, loss_var_name, {member_->local_scopes_[0]}, 1, member_->use_cuda_, member_->nccl_ctxs_.get()); - for (int i = 1; i < member_->places_.size(); ++i) { + for (size_t i = 1; i < member_->places_.size(); ++i) { graphs[i] = build_strategy.Apply(graphs[i], {member_->places_[i]}, loss_var_name, {member_->local_scopes_[i]}, 1, diff --git a/paddle/fluid/framework/trainer_desc.proto b/paddle/fluid/framework/trainer_desc.proto index 389c1a870fb54ad28806ad49632323b1c93676f4..4fc05ccf5c9be37e80b4ae7263166ad76eb6d6a7 100644 --- a/paddle/fluid/framework/trainer_desc.proto +++ b/paddle/fluid/framework/trainer_desc.proto @@ -76,7 +76,7 @@ message PullDenseWorkerParameter { message TableParameter { // dense table only - optional int64 table_id = 1; + optional uint64 table_id = 1; repeated string dense_value_name = 2; repeated string dense_grad_name = 3; repeated int32 push_dense_wait_times = 5; diff --git a/paddle/fluid/framework/var_type_inference.h b/paddle/fluid/framework/var_type_inference.h index 2e9c64d3e6854bf70c0aee06128b9f1b7c8c7439..66e6ac81623a1cd1c79981c1e4a97d974e9c2426 100644 --- a/paddle/fluid/framework/var_type_inference.h +++ b/paddle/fluid/framework/var_type_inference.h @@ -45,12 +45,16 @@ class InferVarTypeContext { virtual bool HasInput(const std::string& name) const { PADDLE_ENFORCE_NOT_NULL(op_); - return op_->Inputs().count(name) > 0; + auto& inputs = op_->Inputs(); + auto input = inputs.find(name); + return input != inputs.end() && !input->second.empty(); } virtual bool HasOutput(const std::string& name) const { PADDLE_ENFORCE_NOT_NULL(op_); - return op_->Outputs().count(name) > 0; + auto& outputs = op_->Outputs(); + auto output = outputs.find(name); + return output != outputs.end() && !output->second.empty(); } virtual const std::vector& Input(const std::string& name) const { diff --git a/paddle/fluid/imperative/CMakeLists.txt b/paddle/fluid/imperative/CMakeLists.txt index 0d116a6495477ca69c10c130e63247a4f6c03b23..e52a0283f726640eb56b24a2978af6ee44e658ff 100644 --- a/paddle/fluid/imperative/CMakeLists.txt +++ b/paddle/fluid/imperative/CMakeLists.txt @@ -3,4 +3,7 @@ cc_library(layer SRCS layer.cc DEPS proto_desc operator device_context blas pybi cc_library(tracer SRCS tracer.cc DEPS proto_desc device_context pybind) cc_library(engine SRCS engine.cc) cc_library(imperative_profiler SRCS profiler.cc) +cc_library(nccl_context SRCS nccl_context.cc DEPS device_context) + +cc_test(nccl_context_test SRCS nccl_context_test.cc DEPS nccl_context) endif() diff --git a/paddle/fluid/imperative/nccl_context.cc b/paddle/fluid/imperative/nccl_context.cc new file mode 100644 index 0000000000000000000000000000000000000000..f96c83936df590e5bd3abe89b7e7c2a6ddf92d01 --- /dev/null +++ b/paddle/fluid/imperative/nccl_context.cc @@ -0,0 +1,133 @@ +// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/imperative/nccl_context.h" + +namespace paddle { +namespace imperative { +#if defined(PADDLE_WITH_CUDA) && !defined(_WIN32) +void NCCLParallelContext::RecvNCCLID(const std::string &ep, + ncclUniqueId *nccl_id) { + auto addr = paddle::string::Split(ep, ':'); + PADDLE_ENFORCE_EQ(addr.size(), 2UL, + "The endpoint should contain host and port: %s", ep); + std::string host = addr[0]; + int port = std::stoi(addr[1]); + + int server_fd, new_socket; + struct sockaddr_in address; + int addrlen = sizeof(address); + char buffer[1024] = {0}; + int opt = 0; + // creating socket fd + if ((server_fd = socket(AF_INET, SOCK_STREAM, 0)) == 0) + PADDLE_THROW("create server fd failed"); + if (setsockopt(server_fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt))) + PADDLE_THROW("set socket opt failed"); + + address.sin_family = AF_INET; + address.sin_addr.s_addr = INADDR_ANY; + address.sin_port = htons(port); + + if (bind(server_fd, (struct sockaddr *)&address, sizeof(address)) < 0) + PADDLE_THROW("binding failed on ep: %s", ep); + VLOG(3) << "listening on: " << ep; + if (listen(server_fd, 3) < 0) PADDLE_THROW("listen on server fd failed"); + + if ((new_socket = + accept(server_fd, reinterpret_cast(&address), + reinterpret_cast(&addrlen))) < 0) + PADDLE_THROW("accept the new socket fd failed"); + + if (read(new_socket, buffer, 1024) < 0) + PADDLE_THROW("reading the ncclUniqueId from socket failed"); + VLOG(3) << "recevived the ncclUniqueId"; + memcpy(nccl_id, buffer, NCCL_UNIQUE_ID_BYTES); + + VLOG(3) << "closing the socket server: " << ep; + close(server_fd); +} + +void NCCLParallelContext::SendNCCLID(const std::string &ep, + ncclUniqueId *nccl_id) { + auto addr = paddle::string::Split(ep, ':'); + PADDLE_ENFORCE_EQ(addr.size(), 2UL, + "The endpoint should contain host and port: %s", ep); + std::string host = addr[0]; + int port = std::stoi(addr[1]); + // struct sockaddr_in address; + int sock = 0; + struct sockaddr_in serv_addr; + char buffer[1024] = {0}; + + memcpy(buffer, nccl_id, NCCL_UNIQUE_ID_BYTES); + if ((sock = socket(AF_INET, SOCK_STREAM, 0)) < 0) + PADDLE_THROW("create socket failed"); + + memset(&serv_addr, '0', sizeof(serv_addr)); + serv_addr.sin_family = AF_INET; + serv_addr.sin_port = htons(port); + + if (inet_pton(AF_INET, host.c_str(), &serv_addr.sin_addr) <= 0) + PADDLE_THROW("invalied address: %s", ep); + + while (true) { + if (connect(sock, (struct sockaddr *)&serv_addr, sizeof(serv_addr)) < 0) { + VLOG(0) << "worker: " << ep + << " is not ready, will retry after 3 seconds..."; + std::this_thread::sleep_for(std::chrono::seconds(3)); + continue; + } + VLOG(3) << "sending the ncclUniqueId to " << ep; + send(sock, buffer, NCCL_UNIQUE_ID_BYTES, 0); + break; + } +} + +void NCCLParallelContext::BcastNCCLId(ncclUniqueId *nccl_id, int root) { + if (strategy_.local_rank_ == root) { + for (auto ep : strategy_.trainer_endpoints_) { + if (ep != strategy_.current_endpoint_) SendNCCLID(ep, nccl_id); + } + } else { + RecvNCCLID(strategy_.current_endpoint_, nccl_id); + } +} + +void NCCLParallelContext::Init() { + ncclUniqueId nccl_id; + ncclComm_t comm; + if (strategy_.local_rank_ == 0) { + // generate the unique ncclid on the root worker + platform::dynload::ncclGetUniqueId(&nccl_id); + BcastNCCLId(&nccl_id, 0); + } else { + BcastNCCLId(&nccl_id, 0); + } + int gpu_id = boost::get(place_).device; + VLOG(0) << "init nccl context nranks: " << strategy_.nranks_ + << " local rank: " << strategy_.local_rank_ << " gpu id: " << gpu_id; + + PADDLE_ENFORCE(cudaSetDevice(gpu_id)); + PADDLE_ENFORCE(platform::dynload::ncclCommInitRank( + &comm, strategy_.nranks_, nccl_id, strategy_.local_rank_)); + + platform::DeviceContextPool &pool = platform::DeviceContextPool::Instance(); + auto *dev_ctx = static_cast(pool.Get(place_)); + dev_ctx->set_nccl_comm(comm); +} +#endif + +} // namespace imperative +} // namespace paddle diff --git a/paddle/fluid/imperative/nccl_context.h b/paddle/fluid/imperative/nccl_context.h new file mode 100644 index 0000000000000000000000000000000000000000..b4f44e56405a51082e60afd69fb6f011dab44b86 --- /dev/null +++ b/paddle/fluid/imperative/nccl_context.h @@ -0,0 +1,81 @@ +// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +// network header files +#if defined(PADDLE_WITH_CUDA) && !defined(_WIN32) +#include +#include +#include +#include +#endif + +#include +#include + +#include "paddle/fluid/framework/variable.h" +#include "paddle/fluid/platform/device_context.h" +#if defined(PADDLE_WITH_CUDA) && !defined(_WIN32) +#include "paddle/fluid/platform/dynload/nccl.h" +#endif +#include "paddle/fluid/platform/place.h" +#include "paddle/fluid/string/split.h" + +namespace paddle { +namespace imperative { + +struct ParallelStrategy { + int nranks_{1}; + int local_rank_{0}; + std::vector trainer_endpoints_{}; + std::string current_endpoint_{""}; +}; + +class ParallelContext { + public: + explicit ParallelContext(const ParallelStrategy& strategy, + const platform::Place& place) + : strategy_(strategy), place_(place) {} + + virtual ~ParallelContext() {} + + virtual void Init() = 0; + + protected: + ParallelStrategy strategy_; + platform::Place place_; +}; + +#if defined(PADDLE_WITH_CUDA) && !defined(_WIN32) +class NCCLParallelContext : ParallelContext { + public: + explicit NCCLParallelContext(const ParallelStrategy& strategy, + const platform::Place& place) + : ParallelContext(strategy, place) {} + + ~NCCLParallelContext() {} + + void BcastNCCLId(ncclUniqueId* nccl_id, int root); + + void Init() override; + + protected: + void RecvNCCLID(const std::string& endpoint, ncclUniqueId* nccl_id); + + void SendNCCLID(const std::string& endpoint, ncclUniqueId* nccl_id); +}; +#endif + +} // namespace imperative +} // namespace paddle diff --git a/paddle/fluid/imperative/nccl_context_test.cc b/paddle/fluid/imperative/nccl_context_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..74a74ebe921378e2994a6a4cb2087d0acde950b1 --- /dev/null +++ b/paddle/fluid/imperative/nccl_context_test.cc @@ -0,0 +1,52 @@ +// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/imperative/nccl_context.h" +#include "gtest/gtest.h" +#include "paddle/fluid/platform/device_context.h" + +namespace imperative = paddle::imperative; +namespace platform = paddle::platform; + +imperative::ParallelStrategy GetStrategy(int local_rank) { + std::vector eps = {"127.0.0.1:9866", "127.0.0.1:9867"}; + imperative::ParallelStrategy strategy; + strategy.trainer_endpoints_ = eps; + strategy.current_endpoint_ = eps[local_rank]; + strategy.nranks_ = 2; + strategy.local_rank_ = local_rank; + return strategy; +} + +#if defined(PADDLE_WITH_CUDA) && !defined(_WIN32) +void BcastNCCLId(int local_rank, ncclUniqueId *nccl_id) { + auto strategy = GetStrategy(local_rank); + platform::CUDAPlace gpu(local_rank); + imperative::NCCLParallelContext ctx(strategy, gpu); + ctx.BcastNCCLId(nccl_id, 0); +} + +TEST(BcastNCCLId, Run) { + ncclUniqueId nccl_id; + platform::dynload::ncclGetUniqueId(&nccl_id); + std::thread t(BcastNCCLId, 0, &nccl_id); + + ncclUniqueId recv_nccl_id; + BcastNCCLId(1, &recv_nccl_id); + + t.join(); + EXPECT_EQ(0, std::memcmp(nccl_id.internal, recv_nccl_id.internal, + NCCL_UNIQUE_ID_BYTES)); +} +#endif diff --git a/paddle/fluid/imperative/tracer.cc b/paddle/fluid/imperative/tracer.cc index 7c9d0af3ecd647604ab46ee6239fc352e5fd8d85..7c495ddd68221acfed8537fd72e9a582e891f8db 100644 --- a/paddle/fluid/imperative/tracer.cc +++ b/paddle/fluid/imperative/tracer.cc @@ -177,7 +177,7 @@ std::set Tracer::Trace(OpBase* op, const VarBasePtrMap& inputs, current_vars_map[out->Name()] = out; } - VLOG(3) << "input var name: " << out->Name() + VLOG(3) << "output var name: " << out->Name() << " inited: " << out->var_->IsInitialized() << " stop_grad: " << out->IsStopGradient(); } @@ -215,6 +215,7 @@ std::set Tracer::Trace(OpBase* op, const VarBasePtrMap& inputs, framework::Scope scope; op->place_ = GetExpectedPlace(expected_place, inputs); + PreparedOp prepared_op = PreparedOp::Prepare(ctx, *op_kernel, op->place_); prepared_op.op.RuntimeInferShape(scope, op->place_, ctx); prepared_op.func( diff --git a/paddle/fluid/inference/api/analysis_config.cc b/paddle/fluid/inference/api/analysis_config.cc index 0109b4a4fa7617880f642f5a39639bca38475515..b54ea269ff250f02b6331807237e10ee65b0b0b4 100644 --- a/paddle/fluid/inference/api/analysis_config.cc +++ b/paddle/fluid/inference/api/analysis_config.cc @@ -231,6 +231,7 @@ void AnalysisConfig::Update() { pass_builder()->InsertPass(3, "tensorrt_subgraph_pass"); } pass_builder()->DeletePass("runtime_context_cache_pass"); + pass_builder()->DeletePass("expected_kernel_cache_pass"); } if (use_mkldnn_) { diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc index 6942604b0723f8665f0e8b058d48a5356a1a01f4..0155609a029664da2c3d4c90a152ec556927c32d 100644 --- a/paddle/fluid/inference/api/analysis_predictor.cc +++ b/paddle/fluid/inference/api/analysis_predictor.cc @@ -259,6 +259,9 @@ bool AnalysisPredictor::SetFeed(const std::vector &inputs, return false; } + PADDLE_ENFORCE_NOT_NULL(input_ptr); + PADDLE_ENFORCE_NOT_NULL(inputs[i].data.data()); + if (platform::is_cpu_place(place_)) { // TODO(panyx0718): Init LoDTensor from existing memcpy to save a copy. std::memcpy(static_cast(input_ptr), inputs[i].data.data(), diff --git a/paddle/fluid/inference/api/api.cc b/paddle/fluid/inference/api/api.cc index 7d57b6ec74468dbdb0519f85140629a0ac01c18d..fc2d7b48c2a1f89232dcb96d1899667230e2ddda 100644 --- a/paddle/fluid/inference/api/api.cc +++ b/paddle/fluid/inference/api/api.cc @@ -54,6 +54,7 @@ PaddleBuf &PaddleBuf::operator=(const PaddleBuf &other) { memory_owned_ = other.memory_owned_; } else { Resize(other.length()); + PADDLE_ENFORCE(!(other.length() > 0 && other.data() == nullptr)); memcpy(data_, other.data(), other.length()); length_ = other.length(); memory_owned_ = true; diff --git a/paddle/fluid/inference/api/api_impl.cc b/paddle/fluid/inference/api/api_impl.cc index 54f40563c3662af24e794422be4d3262d86c76a7..56996c5cff88f5b4a9094291a09996f8b8d70a23 100644 --- a/paddle/fluid/inference/api/api_impl.cc +++ b/paddle/fluid/inference/api/api_impl.cc @@ -169,6 +169,7 @@ std::unique_ptr NativePaddlePredictor::Clone() { std::unique_ptr cls(new NativePaddlePredictor(config_)); // Hot fix the bug that result diff in multi-thread. // TODO(Superjomn) re-implement a real clone here. + PADDLE_ENFORCE_NOT_NULL(dynamic_cast(cls.get())); if (!dynamic_cast(cls.get())->Init(nullptr)) { LOG(ERROR) << "fail to call Init"; return nullptr; @@ -210,6 +211,8 @@ bool NativePaddlePredictor::SetFeed(const std::vector &inputs, return false; } + PADDLE_ENFORCE_NOT_NULL(input_ptr); + PADDLE_ENFORCE_NOT_NULL(inputs[i].data.data()); if (platform::is_cpu_place(place_)) { // TODO(panyx0718): Init LoDTensor from existing memcpy to save a copy. std::memcpy(static_cast(input_ptr), inputs[i].data.data(), @@ -316,6 +319,8 @@ std::unique_ptr CreatePaddlePredictor< } std::unique_ptr predictor(new NativePaddlePredictor(config)); + PADDLE_ENFORCE_NOT_NULL( + dynamic_cast(predictor.get())); if (!dynamic_cast(predictor.get())->Init(nullptr)) { return nullptr; } diff --git a/paddle/fluid/inference/api/paddle_pass_builder.cc b/paddle/fluid/inference/api/paddle_pass_builder.cc index 87e02a02caebd93d701dfd9e51c35fb974c770ed..9b0873aecb545067180723c363a38bed1552fb2a 100644 --- a/paddle/fluid/inference/api/paddle_pass_builder.cc +++ b/paddle/fluid/inference/api/paddle_pass_builder.cc @@ -86,7 +86,8 @@ const std::vector kAnakinSubgraphPasses({ GpuPassStrategy::GpuPassStrategy() : PassStrategy({}) { passes_.assign({ - "infer_clean_graph_pass", // + "infer_clean_graph_pass", // + "runtime_context_cache_pass", // // "identity_scale_op_clean_pass", // "conv_affine_channel_fuse_pass", // "conv_eltwiseadd_affine_channel_fuse_pass", // @@ -96,9 +97,9 @@ GpuPassStrategy::GpuPassStrategy() : PassStrategy({}) { "conv_elementwise_add_act_fuse_pass", // "conv_elementwise_add2_act_fuse_pass", // "conv_elementwise_add_fuse_pass", // - "runtime_context_cache_pass", // #endif // "transpose_flatten_concat_fuse_pass", + "expected_kernel_cache_pass", // }); use_gpu_ = true; @@ -116,10 +117,14 @@ CpuPassStrategy::CpuPassStrategy() : PassStrategy({}) { // NOTE the large fusions should be located in the front, so that they will // not be damaged by smaller ones. passes_.assign({ - "infer_clean_graph_pass", // + "infer_clean_graph_pass", // + // TODO(luotao): runtime_context_cache_pass should be located in the + // front, see https://github.com/PaddlePaddle/Paddle/issues/16609, + // will enhance this pass later. + "runtime_context_cache_pass", // "attention_lstm_fuse_pass", // - "seqpool_concat_fuse_pass", // "seqconv_eltadd_relu_fuse_pass", // + // "seqpool_concat_fuse_pass", // // "embedding_fc_lstm_fuse_pass", // "fc_lstm_fuse_pass", // "mul_lstm_fuse_pass", // @@ -132,9 +137,9 @@ CpuPassStrategy::CpuPassStrategy() : PassStrategy({}) { "conv_bn_fuse_pass", // "conv_eltwiseadd_bn_fuse_pass", // "is_test_pass", // - "identity_scale_op_clean_pass", // - "runtime_context_cache_pass", // + "expected_kernel_cache_pass", // }); + use_gpu_ = false; } diff --git a/paddle/fluid/inference/tests/api/analyzer_int8_image_classification_tester.cc b/paddle/fluid/inference/tests/api/analyzer_int8_image_classification_tester.cc index ece094717b8076321c68d7fdd29f07c4da6b0ed4..fbf67d933786e3ee2baab7a20911da2837cdce4d 100644 --- a/paddle/fluid/inference/tests/api/analyzer_int8_image_classification_tester.cc +++ b/paddle/fluid/inference/tests/api/analyzer_int8_image_classification_tester.cc @@ -23,18 +23,11 @@ namespace analysis { void SetConfig(AnalysisConfig *cfg) { cfg->SetModel(FLAGS_infer_model); - cfg->SetProgFile("__model__"); cfg->DisableGpu(); cfg->SwitchIrOptim(); - cfg->SwitchSpecifyInputNames(false); + cfg->SwitchSpecifyInputNames(); cfg->SetCpuMathLibraryNumThreads(FLAGS_paddle_num_threads); cfg->EnableMKLDNN(); - cfg->pass_builder()->SetPasses( - {"infer_clean_graph_pass", "mkldnn_placement_pass", - "depthwise_conv_mkldnn_pass", "conv_bn_fuse_pass", - "conv_eltwiseadd_bn_fuse_pass", "conv_bias_mkldnn_fuse_pass", - "conv_elementwise_add_mkldnn_fuse_pass", "conv_relu_mkldnn_fuse_pass", - "fc_fuse_pass", "is_test_pass"}); } template @@ -84,13 +77,13 @@ std::shared_ptr> GetWarmupData( std::to_string(num_images) + " is bigger than all test data size."); PaddleTensor images; - images.name = "input"; + images.name = "image"; images.shape = {num_images, 3, 224, 224}; images.dtype = PaddleDType::FLOAT32; images.data.Resize(sizeof(float) * num_images * 3 * 224 * 224); PaddleTensor labels; - labels.name = "labels"; + labels.name = "label"; labels.shape = {num_images, 1}; labels.dtype = PaddleDType::INT64; labels.data.Resize(sizeof(int64_t) * num_images); @@ -132,7 +125,7 @@ void SetInput(std::vector> *inputs, images_offset_in_file + sizeof(float) * total_images * 3 * 224 * 224; TensorReader image_reader(file, images_offset_in_file, - image_batch_shape, "input"); + image_batch_shape, "image"); TensorReader label_reader(file, labels_offset_in_file, label_batch_shape, "label"); diff --git a/paddle/fluid/inference/tests/api/analyzer_seq_conv1_tester.cc b/paddle/fluid/inference/tests/api/analyzer_seq_conv1_tester.cc index 9f23b9f037bcaeb758312d011067ae29c82e73cd..5ee848c3cfa2117b2adeab5e563c5d07ce1d76ca 100644 --- a/paddle/fluid/inference/tests/api/analyzer_seq_conv1_tester.cc +++ b/paddle/fluid/inference/tests/api/analyzer_seq_conv1_tester.cc @@ -47,6 +47,7 @@ struct DataRecord { num_lines++; std::vector data; split(line, '\t', &data); + PADDLE_ENFORCE(data.size() >= 4); // load title1 data std::vector title1_data; split_to_int64(data[0], ' ', &title1_data); diff --git a/paddle/fluid/inference/tests/api/analyzer_seq_pool1_tester.cc b/paddle/fluid/inference/tests/api/analyzer_seq_pool1_tester.cc index d6f7f468a6c83bd6c4ac087931d0c6b7cac3cc1c..3cebf8e96984fad0de8d8c6775990f7c6a6cabe5 100644 --- a/paddle/fluid/inference/tests/api/analyzer_seq_pool1_tester.cc +++ b/paddle/fluid/inference/tests/api/analyzer_seq_pool1_tester.cc @@ -150,6 +150,9 @@ void SetConfig(AnalysisConfig *cfg, bool use_mkldnn = false) { if (use_mkldnn) { cfg->EnableMKLDNN(); } + // Enable seqpool_concat_fuse_pass, disabled by default since it takes much + // time + cfg->pass_builder()->InsertPass(2, "seqpool_concat_fuse_pass"); } void profile(bool use_mkldnn = false) { diff --git a/paddle/fluid/inference/tests/api/analyzer_transformer_tester.cc b/paddle/fluid/inference/tests/api/analyzer_transformer_tester.cc index bd4f1b61973fb0de06dcc288e329c94756d5ed47..a23297f29cf65d891f530850ffd184aa58e10886 100644 --- a/paddle/fluid/inference/tests/api/analyzer_transformer_tester.cc +++ b/paddle/fluid/inference/tests/api/analyzer_transformer_tester.cc @@ -214,28 +214,23 @@ TEST(Analyzer_Transformer, fuse_statis) { } // Compare result of NativeConfig and AnalysisConfig -// void compare(bool use_mkldnn = false) { -// AnalysisConfig cfg; -// SetConfig(&cfg); -// if (use_mkldnn) { -// cfg.EnableMKLDNN(); -// } -// -// std::vector> input_slots_all; -// SetInput(&input_slots_all); -// CompareNativeAndAnalysis( -// reinterpret_cast(&cfg), -// input_slots_all); -// } - -// TODO(yihuaxu): -// Disable compare and compare_mkldnn temporary, see -// https://github.com/paddlePaddle/Paddle/issues/16316 for details. -// TEST(Analyzer_Transformer, compare) { compare(); } -// #ifdef PADDLE_WITH_MKLDNN -// TEST(Analyzer_Transformer, compare_mkldnn) { compare(true /* use_mkldnn */); -// } -// #endif +void compare(bool use_mkldnn = false) { + AnalysisConfig cfg; + SetConfig(&cfg); + if (use_mkldnn) { + cfg.EnableMKLDNN(); + } + + std::vector> input_slots_all; + SetInput(&input_slots_all); + CompareNativeAndAnalysis( + reinterpret_cast(&cfg), input_slots_all); +} + +TEST(Analyzer_Transformer, compare) { compare(); } +#ifdef PADDLE_WITH_MKLDNN +TEST(Analyzer_Transformer, compare_mkldnn) { compare(true /* use_mkldnn */); } +#endif } // namespace inference } // namespace paddle diff --git a/paddle/fluid/inference/tests/api/int8_mkldnn_quantization.md b/paddle/fluid/inference/tests/api/int8_mkldnn_quantization.md new file mode 100644 index 0000000000000000000000000000000000000000..cbeef5fb9da42388eade6fa90344abf77cb59bd6 --- /dev/null +++ b/paddle/fluid/inference/tests/api/int8_mkldnn_quantization.md @@ -0,0 +1,70 @@ +# INT8 MKL-DNN quantization + +This document describes how to use Paddle inference Engine to convert the FP32 model to INT8 model on ResNet-50 and MobileNet-V1. We provide the instructions on enabling INT8 MKL-DNN quantization in Paddle inference and show the ResNet-50 and MobileNet-V1 results in accuracy and performance. + +## 0. Install PaddlePaddle +Follow PaddlePaddle [installation instruction](https://github.com/PaddlePaddle/models/tree/develop/fluid/PaddleCV/image_classification#installation) to install PaddlePaddle. If you build PaddlePaddle yourself, please use the following cmake arguments. +``` +cmake .. -DWITH_TESTING=ON -WITH_FLUID_ONLY=ON -DWITH_GPU=OFF -DWITH_MKL=ON -WITH_SWIG_PY=OFF -DWITH_INFERENCE_API_TEST=ON -DON_INFER=ON + +``` +Note: MKL-DNN and MKL are required. + +## 1. Enable INT8 MKL-DNN quantization +For reference, please examine the code of unit test enclosed in [analyzer_int8_image_classification_tester.cc](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/fluid/inference/tests/api/analyzer_int8_image_classification_tester.cc). + +* ### Create Analysis config +INT8 quantization is one of the optimizations in analysis config. More information about analysis config can be found [here](https://github.com/PaddlePaddle/FluidDoc/blob/develop/doc/fluid/advanced_usage/deploy/inference/native_infer_en.md#upgrade-performance-based-on-contribanalysisconfig-prerelease) + +* ### Create quantize config by analysis config +We enable the MKL-DNN quantization procedure by calling an appropriate method from analysis config. Afterwards, all the required quantization parameters (quantization op names, quantization strategies etc.) can be set through quantizer config which is present in the analysis config. It is also necessary to specify a pre-processed warmup dataset and desired batch size. + +```cpp +//Enable MKL-DNN quantization +cfg.EnableMkldnnQuantizer(); + +//use analysis config to call the MKL-DNN quantization config +cfg.mkldnn_quantizer_config()->SetWarmupData(warmup_data); +cfg.mkldnn_quantizer_config()->SetWarmupBatchSize(100); +``` + +## 2. Accuracy and Performance benchmark + +We provide the results of accuracy and performance measured on Intel(R) Xeon(R) Gold 6271 on single core. + + >**I. Top-1 Accuracy on Intel(R) Xeon(R) Gold 6271** + +| Model | Dataset | FP32 Accuracy | INT8 Accuracy | Accuracy Diff | +| :------------: | :------------: | :------------: | :------------: | :------------: | +| ResNet-50 | Full ImageNet Val | 76.63% | 76.48% | 0.15% | +| MobileNet-V1 | Full ImageNet Val | 70.78% | 70.36% | 0.42% | + + >**II. Throughput on Intel(R) Xeon(R) Gold 6271 (batch size 1 on single core)** + +| Model | Dataset | FP32 Throughput | INT8 Throughput | Ratio(INT8/FP32) | +| :------------: | :------------: | :------------: | :------------: | :------------: | +| ResNet-50 | Full ImageNet Val | 13.17 images/s | 49.84 images/s | 3.78 | +| MobileNet-V1 | Full ImageNet Val | 75.49 images/s | 232.38 images/s | 3.07 | + +Notes: +* Measurement of accuracy requires a model which accepts two inputs: data and labels. +* Different sampling batch size data may cause slight difference on INT8 top accuracy. +* CAPI performance data is better than python API performance data because of the python overhead. Especially for the small computational model, python overhead will be more obvious. + + +## 3. Commands to reproduce the above accuracy and performance benchmark +* #### Full dataset (Single core) + * ##### Download full ImageNet Validation Dataset +```bash +cd /PATH/TO/PADDLE/build +python ../paddle/fluid/inference/tests/api/full_ILSVRC2012_val_preprocess.py +``` +The converted data binary file is saved by default in ~/.cache/paddle/dataset/int8/download/int8_full_val.bin + * ##### ResNet50 Full dataset benchmark +```bash +./paddle/fluid/inference/tests/api/test_analyzer_int8_resnet50 --infer_model=third_party/inference_demo/int8v2/resnet50/model --infer_data=/path/to/converted/int8_full_val.bin --batch_size=1 --paddle_num_threads=1 +``` + * ##### Mobilenet-v1 Full dataset benchmark +```bash +./paddle/fluid/inference/tests/api/test_analyzer_int8_mobilenet --infer_model=third_party/inference_demo/int8v2/mobilenet/model --infer_data=/path/to/converted/int8_full_val.bin --batch_size=1 --paddle_num_threads=1 +``` diff --git a/paddle/fluid/inference/tests/api/tester_helper.h b/paddle/fluid/inference/tests/api/tester_helper.h index 9a0dcc722cf00984b8c0e3ac20f13849e2904102..d13469a8482304d04b99c96e70bac5c8b90e4043 100644 --- a/paddle/fluid/inference/tests/api/tester_helper.h +++ b/paddle/fluid/inference/tests/api/tester_helper.h @@ -55,6 +55,9 @@ DEFINE_bool(record_benchmark, false, DEFINE_double(accuracy, 1e-3, "Result Accuracy."); DEFINE_double(quantized_accuracy, 1e-2, "Result Quantized Accuracy."); DEFINE_bool(zero_copy, false, "Use ZeroCopy to speedup Feed/Fetch."); +DEFINE_bool(warmup, false, + "Use warmup to calculate elapsed_time more accurately. " + "To reduce CI time, it sets false in default."); DECLARE_bool(profile); DECLARE_int32(paddle_num_threads); @@ -316,7 +319,8 @@ void PredictionRun(PaddlePredictor *predictor, int num_threads, int tid) { int num_times = FLAGS_repeat; int iterations = inputs.size(); // process the whole dataset ... - if (FLAGS_iterations > 0 && FLAGS_iterations < inputs.size()) + if (FLAGS_iterations > 0 && + FLAGS_iterations < static_cast(inputs.size())) iterations = FLAGS_iterations; // ... unless the number of iterations is set outputs->resize(iterations); @@ -329,14 +333,14 @@ void PredictionRun(PaddlePredictor *predictor, #endif if (!FLAGS_zero_copy) { run_timer.tic(); - for (size_t i = 0; i < iterations; i++) { + for (int i = 0; i < iterations; i++) { for (int j = 0; j < num_times; j++) { predictor->Run(inputs[i], &(*outputs)[i], FLAGS_batch_size); } } elapsed_time = run_timer.toc(); } else { - for (size_t i = 0; i < iterations; i++) { + for (int i = 0; i < iterations; i++) { ConvertPaddleTensorToZeroCopyTensor(predictor, inputs[i]); run_timer.tic(); for (int j = 0; j < num_times; j++) { @@ -366,9 +370,10 @@ void TestOneThreadPrediction( const std::vector> &inputs, std::vector> *outputs, bool use_analysis = true) { auto predictor = CreateTestPredictor(config, use_analysis); - PredictionWarmUp(predictor.get(), inputs, outputs, FLAGS_paddle_num_threads, - 0); - PredictionRun(predictor.get(), inputs, outputs, FLAGS_paddle_num_threads, 0); + if (FLAGS_warmup) { + PredictionWarmUp(predictor.get(), inputs, outputs, 1, 0); + } + PredictionRun(predictor.get(), inputs, outputs, 1, 0); } void TestMultiThreadPrediction( @@ -395,7 +400,10 @@ void TestMultiThreadPrediction( ->SetMkldnnThreadID(static_cast(tid) + 1); } #endif - PredictionWarmUp(predictor.get(), inputs, &outputs_tid, num_threads, tid); + if (FLAGS_warmup) { + PredictionWarmUp(predictor.get(), inputs, &outputs_tid, num_threads, + tid); + } PredictionRun(predictor.get(), inputs, &outputs_tid, num_threads, tid); }); } diff --git a/paddle/fluid/op_use_default_grad_op_maker.spec b/paddle/fluid/op_use_default_grad_op_maker.spec index f0e3d3e86f24cb68f6e9d41f48c9698b43bca13e..63eaa676a43fc784dce2437ca15bc85e2295dbb7 100644 --- a/paddle/fluid/op_use_default_grad_op_maker.spec +++ b/paddle/fluid/op_use_default_grad_op_maker.spec @@ -1,14 +1,7 @@ -abs -acos -asin -atan attention_lstm -brelu conv_shift -cos cos_sim dequantize -elu fc flatten fsp @@ -21,14 +14,8 @@ fusion_seqconv_eltadd_relu fusion_seqexpand_concat_fc fusion_seqpool_concat fusion_squared_mat_sub -gelu gru -hard_shrink hierarchical_sigmoid -leaky_relu -log -logsigmoid -lookup_table lrn lstm_unit lstmp @@ -39,10 +26,11 @@ modified_huber_loss nce pool2d pool3d -pow prelu quantize rank_loss +reduce_all +reduce_any reduce_max reduce_mean reduce_min @@ -51,26 +39,10 @@ reduce_sum requantize reshape rnn_memory_helper -round sequence_softmax -sin -softplus -softshrink -softsign -space_to_depth spp -square -squared_l2_distance -squared_l2_norm squeeze -stanh -swish -tanh_shrink -teacher_student_sigmoid_loss tensor_array_to_tensor -thresholded_relu transpose -tree_conv unpool unsqueeze -warpctc diff --git a/paddle/fluid/operators/activation_cudnn_op.cu.cc b/paddle/fluid/operators/activation_cudnn_op.cu.cc index a382414d5c473a9c36f92a9af56837da819e96a4..f03355eb441f99b54d78fe90bcb3bea116db58f1 100644 --- a/paddle/fluid/operators/activation_cudnn_op.cu.cc +++ b/paddle/fluid/operators/activation_cudnn_op.cu.cc @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include +#include +#include #include "paddle/fluid/framework/op_registry.h" #include "paddle/fluid/operators/activation_op.h" #include "paddle/fluid/platform/cudnn_desc.h" @@ -82,6 +85,8 @@ template struct CudnnReluGradFunctor : public CudnnActivationGradFunctor { explicit CudnnReluGradFunctor(const CUDADeviceContext& ctx) : CudnnActivationGradFunctor(ctx, 0.0, CUDNN_ACTIVATION_RELU) {} + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepOut; } }; template @@ -94,6 +99,8 @@ struct CudnnRelu6GradFunctor : public CudnnActivationGradFunctor { explicit CudnnRelu6GradFunctor(const CUDADeviceContext& ctx) : CudnnActivationGradFunctor(ctx, 6.0, CUDNN_ACTIVATION_CLIPPED_RELU) { } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepOut; } }; template @@ -105,6 +112,8 @@ template struct CudnnSigmoidGradFunctor : public CudnnActivationGradFunctor { explicit CudnnSigmoidGradFunctor(const CUDADeviceContext& ctx) : CudnnActivationGradFunctor(ctx, 0.0, CUDNN_ACTIVATION_SIGMOID) {} + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepOut; } }; template @@ -116,6 +125,8 @@ template struct CudnnTanhGradFunctor : public CudnnActivationGradFunctor { explicit CudnnTanhGradFunctor(const CUDADeviceContext& ctx) : CudnnActivationGradFunctor(ctx, 0.0, CUDNN_ACTIVATION_TANH) {} + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepOut; } }; template @@ -140,10 +151,13 @@ class CudnnActivationGradKernel public: using T = typename Functor::ELEMENT_TYPE; void Compute(const framework::ExecutionContext& context) const override { + static_assert(Functor::FwdDeps() == kDepOut, "Forward deps must be Out."); + const framework::Tensor *X, *Out, *dOut; X = Out = dOut = nullptr; framework::Tensor* dX = nullptr; - ExtractActivationGradTensor(context, &X, &Out, &dOut, &dX); + ExtractActivationGradTensor(context, &X, &Out, &dOut, + &dX); dX->mutable_data(context.GetPlace()); auto& dev_ctx = context.template device_context(); Functor functor(dev_ctx); diff --git a/paddle/fluid/operators/activation_op.cc b/paddle/fluid/operators/activation_op.cc index c87e4b22b37027efd1293e74f72598283946e62d..1e5d63fc11d1d81350525e2b3390a3ae44f00f8d 100644 --- a/paddle/fluid/operators/activation_op.cc +++ b/paddle/fluid/operators/activation_op.cc @@ -15,7 +15,9 @@ limitations under the License. */ #include "paddle/fluid/operators/activation_op.h" #include #include +#include #include +#include #include "paddle/fluid/operators/mkldnn/mkldnn_activation_op.h" #include "paddle/fluid/platform/port.h" #ifdef PADDLE_WITH_CUDA @@ -27,6 +29,25 @@ namespace operators { using paddle::framework::Tensor; +template +static constexpr bool CanInplaceAct() { + return GradFunctor::FwdDeps() == kDepOut || GradFunctor::FwdDeps() == kNoDeps; +} + +std::unique_ptr> GetInplaceOpSet() { + std::unique_ptr> ret( + new std::unordered_set()); +#define INSERT_INTO_INPLACE_OP_SET(op_type, __omitted, fwd_functor, \ + bwd_functor) \ + if (CanInplaceAct>()) { \ + ret->insert(#op_type); \ + } + + FOR_EACH_ACTIVATION_OP(INSERT_INTO_INPLACE_OP_SET); +#undef INSERT_INTO_INPLACE_OP_SET + return ret; +} + #define REGISTER_ACTIVATION_OP_MAKER(OP_NAME, OP_COMMENT) \ class OP_NAME##OpMaker \ : public ::paddle::framework::OpProtoAndCheckerMaker { \ @@ -50,26 +71,32 @@ using paddle::framework::Tensor; } \ } -#define REGISTER_ACTIVATION_OP_GRAD_MAKER(OP_NAME, KERNEL_TYPE) \ - class OP_NAME##GradMaker \ - : public ::paddle::framework::SingleGradOpDescMaker { \ - public: \ - using ::paddle::framework::SingleGradOpDescMaker::SingleGradOpDescMaker; \ - \ - protected: \ - std::unique_ptr<::paddle::framework::OpDesc> Apply() const override { \ - auto* op = new ::paddle::framework::OpDesc(); \ - op->SetType(#KERNEL_TYPE "_grad"); \ - op->SetInput("Out", Output("Out")); \ - op->SetInput(::paddle::framework::GradVarName("Out"), \ - OutputGrad("Out")); \ - \ - op->SetAttrMap(Attrs()); \ - \ - op->SetOutput(::paddle::framework::GradVarName("X"), InputGrad("X")); \ - return std::unique_ptr<::paddle::framework::OpDesc>(op); \ - } \ +template +class ActivationGradOpDescMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + std::unique_ptr Apply() const override { + std::unique_ptr op(new framework::OpDesc()); + op->SetType(ForwardOpType() + "_grad"); + op->SetInput(framework::GradVarName("Out"), OutputGrad("Out")); + op->SetOutput(framework::GradVarName("X"), InputGrad("X")); + op->SetAttrMap(Attrs()); + + if (static_cast(kDepValue) & + static_cast(ActBwdOpFwdDeps::kDepX)) { + op->SetInput("X", Input("X")); + } + + if (static_cast(kDepValue) & + static_cast(ActBwdOpFwdDeps::kDepOut)) { + op->SetInput("Out", Output("Out")); + } + + return op; } +}; framework::OpKernelType GetKernelType(const framework::ExecutionContext& ctx, const framework::OperatorWithKernel& oper, @@ -129,14 +156,15 @@ class ActivationOpGrad : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; void InferShape(framework::InferShapeContext* ctx) const override { - ctx->ShareDim("Out", framework::GradVarName("X")); - ctx->ShareLoD("Out", framework::GradVarName("X")); + auto out_grad_name = framework::GradVarName("Out"); + ctx->ShareDim(out_grad_name, framework::GradVarName("X")); + ctx->ShareLoD(out_grad_name, framework::GradVarName("X")); } protected: framework::OpKernelType GetExpectedKernelType( const framework::ExecutionContext& ctx) const override { - return GetKernelType(ctx, *this, "Out"); + return GetKernelType(ctx, *this, framework::GradVarName("Out")); } }; @@ -558,79 +586,27 @@ REGISTER_ACTIVATION_OP_MAKER(Log, LogDoc); REGISTER_ACTIVATION_OP_MAKER(Square, SquareDoc); REGISTER_ACTIVATION_OP_MAKER(Softplus, SoftplusDoc); REGISTER_ACTIVATION_OP_MAKER(Softsign, SoftsignDoc); - -REGISTER_ACTIVATION_OP_GRAD_MAKER(Sigmoid, sigmoid); -REGISTER_ACTIVATION_OP_GRAD_MAKER(Relu, relu); -REGISTER_ACTIVATION_OP_GRAD_MAKER(Gelu, gelu); -REGISTER_ACTIVATION_OP_GRAD_MAKER(Exp, exp); -REGISTER_ACTIVATION_OP_GRAD_MAKER(Tanh, tanh); -REGISTER_ACTIVATION_OP_GRAD_MAKER(Ceil, ceil); -REGISTER_ACTIVATION_OP_GRAD_MAKER(Floor, floor); -REGISTER_ACTIVATION_OP_GRAD_MAKER(Sqrt, sqrt); -REGISTER_ACTIVATION_OP_GRAD_MAKER(SoftRelu, soft_relu); -REGISTER_ACTIVATION_OP_GRAD_MAKER(Relu6, relu6); -REGISTER_ACTIVATION_OP_GRAD_MAKER(Reciprocal, reciprocal); -REGISTER_ACTIVATION_OP_GRAD_MAKER(HardSigmoid, hard_sigmoid); } // namespace operators } // namespace paddle namespace ops = paddle::operators; -#define FOR_EACH_INPLACE_OP_FUNCTOR(__macro) \ - __macro(Sigmoid, sigmoid); \ - __macro(Relu, relu); \ - __macro(Exp, exp); \ - __macro(Tanh, tanh); \ - __macro(Ceil, ceil); \ - __macro(Floor, floor); \ - __macro(Sqrt, sqrt); \ - __macro(SoftRelu, soft_relu); \ - __macro(Relu6, relu6); \ - __macro(Reciprocal, reciprocal); \ - __macro(HardSigmoid, hard_sigmoid); - -#define FOR_EACH_OP_FUNCTOR(__macro) \ - __macro(LogSigmoid, logsigmoid); \ - __macro(SoftShrink, softshrink); \ - __macro(Abs, abs); \ - __macro(Cos, cos); \ - __macro(Acos, acos); \ - __macro(Sin, sin); \ - __macro(Asin, asin); \ - __macro(Atan, atan); \ - __macro(Round, round); \ - __macro(Log, log); \ - __macro(Square, square); \ - __macro(Gelu, gelu); \ - __macro(BRelu, brelu); \ - __macro(Pow, pow); \ - __macro(STanh, stanh); \ - __macro(Softplus, softplus); \ - __macro(Softsign, softsign); \ - __macro(LeakyRelu, leaky_relu); \ - __macro(TanhShrink, tanh_shrink); \ - __macro(ELU, elu); \ - __macro(HardShrink, hard_shrink); \ - __macro(Swish, swish); \ - __macro(ThresholdedRelu, thresholded_relu); - -#define REGISTER_INPLACE_ACTIVATION_OP(OP_NAME, KERNEL_TYPE) \ - REGISTER_OPERATOR(KERNEL_TYPE, ::paddle::operators::ActivationOp, \ - ::paddle::operators::OP_NAME##OpMaker, \ - ::paddle::operators::ActivationOpInferVarType, \ - ::paddle::operators::OP_NAME##GradMaker, \ - ::paddle::framework::SingleOpInplaceInToOut); \ - REGISTER_OPERATOR(KERNEL_TYPE##_grad, ::paddle::operators::ActivationOpGrad, \ - ::paddle::framework::SingleOpInplaceInToOut) - -#define REGISTER_ACTIVATION_OP(OP_NAME, KERNEL_TYPE) \ - REGISTER_OPERATOR(KERNEL_TYPE, ::paddle::operators::ActivationOp, \ - ::paddle::operators::OP_NAME##OpMaker, \ - ::paddle::operators::ActivationOpInferVarType, \ - ::paddle::framework::DefaultGradOpDescMaker); \ - REGISTER_OPERATOR(KERNEL_TYPE##_grad, ::paddle::operators::ActivationOpGrad) - -#define REGISTER_ACTIVATION_CPU_KERNEL(act_type, functor, grad_functor) \ +#define REGISTER_ACTIVATION_OP(KERNEL_TYPE, OP_NAME, functor, grad_functor) \ + REGISTER_OPERATOR( \ + KERNEL_TYPE, ops::ActivationOp, ops::OP_NAME##OpMaker, \ + ops::ActivationOpInferVarType, \ + ops::ActivationGradOpDescMaker::FwdDeps()>, \ + std::conditional>(), \ + ::paddle::framework::SingleOpInplaceInToOut, \ + void>::type); \ + REGISTER_OPERATOR( \ + KERNEL_TYPE##_grad, ops::ActivationOpGrad, \ + std::conditional>(), \ + ::paddle::framework::SingleOpInplaceInToOut, \ + void>::type) + +#define REGISTER_ACTIVATION_CPU_KERNEL(act_type, op_name, functor, \ + grad_functor) \ REGISTER_OP_CPU_KERNEL( \ act_type, ops::ActivationKernel>, \ @@ -643,6 +619,5 @@ namespace ops = paddle::operators; ops::ActivationGradKernel>); -FOR_EACH_OP_FUNCTOR(REGISTER_ACTIVATION_OP); -FOR_EACH_INPLACE_OP_FUNCTOR(REGISTER_INPLACE_ACTIVATION_OP); -FOR_EACH_KERNEL_FUNCTOR(REGISTER_ACTIVATION_CPU_KERNEL); +FOR_EACH_ACTIVATION_OP(REGISTER_ACTIVATION_OP); +FOR_EACH_ACTIVATION_OP(REGISTER_ACTIVATION_CPU_KERNEL); diff --git a/paddle/fluid/operators/activation_op.cu b/paddle/fluid/operators/activation_op.cu index d3a7ceed466a9b5e4d773f1531d198adff97eac2..9c7a8d8971cba4090db1bbc32c7eabf2285e7eff 100644 --- a/paddle/fluid/operators/activation_op.cu +++ b/paddle/fluid/operators/activation_op.cu @@ -15,7 +15,8 @@ limitations under the License. */ namespace ops = paddle::operators; namespace plat = paddle::platform; -#define REGISTER_ACTIVATION_CUDA_KERNEL(act_type, functor, grad_functor) \ +#define REGISTER_ACTIVATION_CUDA_KERNEL(act_type, op_name, functor, \ + grad_functor) \ REGISTER_OP_CUDA_KERNEL( \ act_type, \ ops::ActivationKernel>, \ @@ -30,4 +31,4 @@ namespace plat = paddle::platform; ops::ActivationGradKernel>); -FOR_EACH_KERNEL_FUNCTOR(REGISTER_ACTIVATION_CUDA_KERNEL); +FOR_EACH_ACTIVATION_OP(REGISTER_ACTIVATION_CUDA_KERNEL); diff --git a/paddle/fluid/operators/activation_op.h b/paddle/fluid/operators/activation_op.h index ff7e623f6f383ed2a8b8a40b3186d9c439ff1d86..915632a328feb99c021ec062a9b22a04623eff4a 100644 --- a/paddle/fluid/operators/activation_op.h +++ b/paddle/fluid/operators/activation_op.h @@ -12,6 +12,7 @@ limitations under the License. */ #pragma once #include #include +#include #include #include #include @@ -35,21 +36,29 @@ limitations under the License. */ namespace paddle { namespace operators { -/* Use ugly global variable, for the using in python layer side - Please refer to the layer_helper.py and get the details. - */ -static std::unordered_set InplaceOpSet = { - "sigmoid", "exp", "relu", "tanh", "sqrt", "ceil", - "floor", "reciprocal", "relu6", "soft_relu", "hard_sigmoid"}; +enum ActBwdOpFwdDeps { + kNoDeps = 0x00, // Do not need any forward input/output + kDepX = 0x01, // Only need forward input X + kDepOut = 0x02, // Only need forward output Out + + // Never add kDepXOut, because Out can be always calculated + // by forward input X in backward part. + // FIXME(zjl): but in MKLDNN abs, X and Out are all needed... + // Developers should not rely on this enum value! + kDepXOut = 0x03 +}; + +std::unique_ptr> GetInplaceOpSet(); static bool IsInplace(const std::string& op) { - bool inplace = InplaceOpSet.count(op); + static auto InplaceOpSet = GetInplaceOpSet(); + bool inplace = InplaceOpSet->count(op); // for op_grad const int kGradSuffixLen = 4; if (op.size() > kGradSuffixLen && op.compare(op.size() - kGradSuffixLen - 1, kGradSuffixLen, "grad")) { inplace = - InplaceOpSet.count(op.substr(0, op.size() - (kGradSuffixLen + 1))); + InplaceOpSet->count(op.substr(0, op.size() - (kGradSuffixLen + 1))); } return inplace; } @@ -85,16 +94,21 @@ inline void ExtractActivationTensor(const framework::ExecutionContext& context, context.op().Output("Out")); } +template inline void ExtractActivationGradTensor( const framework::ExecutionContext& context, const framework::Tensor** X, const framework::Tensor** Out, const framework::Tensor** dOut, framework::Tensor** dX) { - auto out_var = context.InputVar("Out"); auto out_grad_var = context.InputVar(framework::GradVarName("Out")); auto x_grad_var = context.OutputVar(framework::GradVarName("X")); - PADDLE_ENFORCE(out_var != nullptr, - "Cannot get input Variable Out, variable name = %s", - context.op().Input("Out")); + const framework::Variable* out_var = nullptr; + + if (static_cast(kDepValue) & static_cast(kDepOut)) { + out_var = context.InputVar("Out"); + PADDLE_ENFORCE(out_var != nullptr, + "Cannot get input Variable Out, variable name = %s", + context.op().Input("Out")); + } PADDLE_ENFORCE(out_grad_var != nullptr, "Cannot get input Variable %s, variable name = %s", framework::GradVarName("Out"), @@ -105,23 +119,36 @@ inline void ExtractActivationGradTensor( context.op().Output(framework::GradVarName("X"))); if (CanBeUsedBySelectedRows.count(context.op().Type())) { - *Out = paddle::framework::GetLoDTensorOrSelectedRowsValueFromVar(*out_var); *dOut = paddle::framework::GetLoDTensorOrSelectedRowsValueFromVar( *out_grad_var); *dX = paddle::framework::GetMutableLoDTensorOrSelectedRowsValueFromVar( x_grad_var); + + if (out_var) { + *Out = + paddle::framework::GetLoDTensorOrSelectedRowsValueFromVar(*out_var); + } else { + *Out = *dOut; // fake out + } + } else { *Out = context.Input("Out"); *dOut = context.Input(framework::GradVarName("Out")); *dX = context.Output(framework::GradVarName("X")); + + if (out_var) { + *Out = &(out_var->Get()); + } else { + *Out = *dOut; // fake out + } } + PADDLE_ENFORCE(*dX != nullptr, "Cannot get output tensor %s, variable name = %s", framework::GradVarName("X"), context.op().Output(framework::GradVarName("X"))); - bool inplace = IsInplace(context.op().Type()); - if (!inplace) { + if (static_cast(kDepValue) & static_cast(kDepX)) { auto x_var = context.InputVar("X"); PADDLE_ENFORCE(x_var != nullptr, "Cannot get input tensor X, variable name = %s", @@ -172,7 +199,8 @@ class ActivationGradKernel const framework::Tensor *X, *Out, *dOut; framework::Tensor* dX = nullptr; X = Out = dOut = nullptr; - ExtractActivationGradTensor(context, &X, &Out, &dOut, &dX); + ExtractActivationGradTensor(context, &X, &Out, &dOut, + &dX); dX->mutable_data(context.GetPlace()); auto dout = framework::EigenVector::Flatten(detail::Ref(dOut)); auto out = framework::EigenVector::Flatten(detail::Ref(Out)); @@ -222,6 +250,8 @@ struct SigmoidGradFunctor : public BaseActivationFunctor { void operator()(Device d, X x, Out out, dOut dout, dX dx) const { dx.device(d) = dout * out * (static_cast(1) - out); } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepOut; } }; // Originally: logsigmoid(x) = -log (1 + exp(-x)) @@ -258,6 +288,8 @@ struct LogSigmoidGradFunctor : public BaseActivationFunctor { dx.device(d) = dout * ((-x - temp).exp() / ((-temp).exp() + (-x - temp).exp())); } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepX; } }; // exp(x) = e^x @@ -276,6 +308,8 @@ struct ExpGradFunctor : public BaseActivationFunctor { void operator()(Device d, X x, Out out, dOut dout, dX dx) const { dx.device(d) = dout * out; } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepOut; } }; // relu(x) = max(x, 0) @@ -294,6 +328,8 @@ struct ReluGradFunctor : public BaseActivationFunctor { void operator()(Device d, X x, Out out, dOut dout, dX dx) const { dx.device(d) = dout * (out > static_cast(0)).template cast(); } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepOut; } }; // gelu(x) = 0.5 * x * (1 + erf(x / sqrt(2))) @@ -338,6 +374,8 @@ struct GeluGradFunctor : BaseActivationFunctor { (-static_cast(0.5) * x.square()).exp(); dx.device(d) = dout * (first + second); } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepX; } }; // tanh(x) = (exp(x) - exp(-x)) / (exp(x) + exp(-x)) @@ -356,6 +394,8 @@ struct TanhGradFunctor : public BaseActivationFunctor { void operator()(Device d, X x, Out out, dOut dout, dX dx) const { dx.device(d) = dout * (static_cast(1) - out * out); } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepOut; } }; // tanhshrink(x) = x - tanh(x) @@ -375,6 +415,8 @@ struct TanhShrinkGradFunctor : public BaseActivationFunctor { void operator()(Device d, X x, Out out, dOut dout, dX dx) const { dx.device(d) = dout * (x.tanh() * x.tanh()); } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepX; } }; // tanhshrink(x) = x - tanh(x) @@ -409,6 +451,8 @@ struct HardShrinkGradFunctor : public BaseActivationFunctor { auto temp2 = (x > static_cast(threshold)).template cast().eval(); dx.device(d) = dout * (temp1 + temp2).template cast(); } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepX; } }; // softshrink(x) = x - lambda, if x > lambda; x + lambda, if x < -lambda; 0 @@ -443,6 +487,8 @@ struct SoftShrinkGradFunctor : public BaseActivationFunctor { auto temp2 = (x < -lambdaT).template cast().eval(); dx.device(d) = dout * (temp1 + temp2).template cast(); } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepX; } }; // sqrt(x) = x^(1/2) @@ -461,6 +507,8 @@ struct SqrtGradFunctor : public BaseActivationFunctor { void operator()(Device d, X x, Out out, dOut dout, dX dx) const { dx.device(d) = static_cast(0.5) * dout / out; } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepOut; } }; // ceil(x) = ceiling(x) @@ -479,6 +527,8 @@ struct ZeroGradFunctor : public BaseActivationFunctor { void operator()(Device d, X x, Out out, dOut dout, dX dx) const { dx.device(d) = static_cast(0) / out; } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kNoDeps; } }; // floor(x) = flooring(x) @@ -522,6 +572,8 @@ struct CosGradFunctor : public BaseActivationFunctor { void operator()(Device d, X x, Out out, dOut dout, dX dx) const { dx.device(d) = -dout * x.unaryExpr(Sine()); } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepX; } }; // cosine(x) = cos(x) @@ -541,6 +593,8 @@ struct SinGradFunctor : public BaseActivationFunctor { void operator()(Device d, X x, Out out, dOut dout, dX dx) const { dx.device(d) = dout * x.unaryExpr(Cosine()); } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepX; } }; // sine(x) = sin(x) @@ -582,6 +636,8 @@ struct AcosGradFunctor : public BaseActivationFunctor { dx.device(d) = -dout * static_cast(1) / (static_cast(1) - x.square()).sqrt(); } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepX; } }; template @@ -614,6 +670,8 @@ struct AsinGradFunctor : public BaseActivationFunctor { dx.device(d) = dout * static_cast(1) / (static_cast(1) - x.square()).sqrt(); } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepX; } }; template @@ -645,6 +703,8 @@ struct AtanGradFunctor : public BaseActivationFunctor { void operator()(Device d, X x, Out out, dOut dout, dX dx) const { dx.device(d) = dout * static_cast(1) / (static_cast(1) + x.square()); } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepX; } }; // round(x) = [x] @@ -672,6 +732,8 @@ struct AbsGradFunctor : public BaseActivationFunctor { void operator()(Device d, X x, Out out, dOut dout, dX dx) const { dx.device(d) = dout * x.sign(); } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepXOut; } }; // reciprocal(x) = 1 / x @@ -690,6 +752,8 @@ struct ReciprocalGradFunctor : public BaseActivationFunctor { void operator()(Device d, X x, Out out, dOut dout, dX dx) const { dx.device(d) = dout * static_cast(-1) * out * out; } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepOut; } }; // log(x) = natural logarithm of x @@ -708,6 +772,8 @@ struct LogGradFunctor : public BaseActivationFunctor { void operator()(Device d, X x, Out out, dOut dout, dX dx) const { dx.device(d) = dout * (static_cast(1) / x); } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepX; } }; // square(x) = x^2 @@ -726,6 +792,8 @@ struct SquareGradFunctor : public BaseActivationFunctor { void operator()(Device d, X x, Out out, dOut dout, dX dx) const { dx.device(d) = dout * static_cast(2) * x; } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepX; } }; template @@ -760,6 +828,8 @@ struct BReluGradFunctor : public BaseActivationFunctor { ((x > static_cast(t_min)) * (x < static_cast(t_max))) .template cast(); } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepX; } }; // relu6(x) = min(max(0, x), 6) @@ -792,6 +862,8 @@ struct Relu6GradFunctor : public BaseActivationFunctor { ((out > static_cast(0)) * (out < static_cast(threshold))) .template cast(); } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepOut; } }; // softplus(x) = log(1 + exp(x)) @@ -821,6 +893,8 @@ struct SoftplusGradFunctor : public BaseActivationFunctor { dx.device(d) = dout * ((x - temp).exp() / ((-temp).exp() + (x - temp).exp())); } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepX; } }; // softsign(x) = x / (1 + |x|) @@ -842,6 +916,8 @@ struct SoftsignGradFunctor : public BaseActivationFunctor { dx.device(d) = dout * (static_cast(1) / (static_cast(1) + x.abs()).square()); } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepX; } }; template @@ -872,6 +948,8 @@ struct SoftReluGradFunctor : public BaseActivationFunctor { auto temp = ((out > -tmp) * (out < tmp)).template cast().eval(); dx.device(d) = dout * (static_cast(1) - (-out).exp()) * temp; } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepOut; } }; template @@ -901,6 +979,8 @@ struct LeakyReluGradFunctor : public BaseActivationFunctor { auto temp2 = (x >= static_cast(0)).template cast().eval(); dx.device(d) = dout * (temp1 + temp2).template cast(); } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepX; } }; template @@ -928,9 +1008,11 @@ struct ELUGradFunctor : public BaseActivationFunctor { typename dX> void operator()(Device d, X x, Out out, dOut dout, dX dx) const { dx.device(d) = dout * (x > static_cast(0)).template cast() + - dout * (out + static_cast(alpha)) * + dout * static_cast(alpha) * x.exp() * (x < static_cast(0)).template cast(); } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepX; } }; // FIXME(qijun) https://github.com/PaddlePaddle/Paddle/issues/5198 @@ -958,6 +1040,8 @@ struct PowGradFunctor : public BaseActivationFunctor { dx.device(d) = dout * static_cast(factor) * x.pow(static_cast(factor) - static_cast(1)); } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepX; } }; template @@ -991,6 +1075,8 @@ struct STanhGradFunctor : public BaseActivationFunctor { auto temp = (a * x).tanh() * (a * x).tanh(); dx.device(d) = dout * a * b * (static_cast(1) - temp); } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepX; } }; template @@ -1020,6 +1106,8 @@ struct ThresholdedReluGradFunctor : public BaseActivationFunctor { auto th = static_cast(threshold); dx.device(d) = dout * (x > th).template cast(); } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepX; } }; template @@ -1053,6 +1141,8 @@ struct HardSigmoidGradFunctor : public BaseActivationFunctor { .template cast() * static_cast(slope); } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepOut; } }; template @@ -1077,49 +1167,54 @@ struct SwishGradFunctor : public BaseActivationFunctor { template - void operator()(Device d, X x, Out out, dOut dout, dX dx) const { + void operator()(Device d, X x, Out fake_out, dOut dout, dX dx) const { auto temp1 = static_cast(1) / (static_cast(1) + (static_cast(-beta) * x).exp()); + auto out = x * temp1; auto temp2 = temp1 * (static_cast(1) - (static_cast(beta) * out)); dx.device(d) = dout * ((static_cast(beta) * out) + temp2); } + + static constexpr ActBwdOpFwdDeps FwdDeps() { return kDepX; } }; } // namespace operators } // namespace paddle -#define FOR_EACH_KERNEL_FUNCTOR(__macro) \ - __macro(sigmoid, SigmoidFunctor, SigmoidGradFunctor); \ - __macro(logsigmoid, LogSigmoidFunctor, LogSigmoidGradFunctor); \ - __macro(exp, ExpFunctor, ExpGradFunctor); \ - __macro(relu, ReluFunctor, ReluGradFunctor); \ - __macro(gelu, GeluFunctor, GeluGradFunctor); \ - __macro(tanh, TanhFunctor, TanhGradFunctor); \ - __macro(atan, AtanFunctor, AtanGradFunctor); \ - __macro(softshrink, SoftShrinkFunctor, SoftShrinkGradFunctor); \ - __macro(sqrt, SqrtFunctor, SqrtGradFunctor); \ - __macro(abs, AbsFunctor, AbsGradFunctor); \ - __macro(ceil, CeilFunctor, ZeroGradFunctor); \ - __macro(floor, FloorFunctor, ZeroGradFunctor); \ - __macro(cos, CosFunctor, CosGradFunctor); \ - __macro(acos, AcosFunctor, AcosGradFunctor); \ - __macro(sin, SinFunctor, SinGradFunctor); \ - __macro(asin, AsinFunctor, AsinGradFunctor); \ - __macro(round, RoundFunctor, ZeroGradFunctor); \ - __macro(reciprocal, ReciprocalFunctor, ReciprocalGradFunctor); \ - __macro(log, LogFunctor, LogGradFunctor); \ - __macro(square, SquareFunctor, SquareGradFunctor); \ - __macro(brelu, BReluFunctor, BReluGradFunctor); \ - __macro(soft_relu, SoftReluFunctor, SoftReluGradFunctor); \ - __macro(pow, PowFunctor, PowGradFunctor); \ - __macro(stanh, STanhFunctor, STanhGradFunctor); \ - __macro(softplus, SoftplusFunctor, SoftplusGradFunctor); \ - __macro(softsign, SoftsignFunctor, SoftsignGradFunctor); \ - __macro(relu6, Relu6Functor, Relu6GradFunctor); \ - __macro(leaky_relu, LeakyReluFunctor, LeakyReluGradFunctor); \ - __macro(tanh_shrink, TanhShrinkFunctor, TanhShrinkGradFunctor); \ - __macro(elu, ELUFunctor, ELUGradFunctor); \ - __macro(hard_shrink, HardShrinkFunctor, HardShrinkGradFunctor); \ - __macro(hard_sigmoid, HardSigmoidFunctor, HardSigmoidGradFunctor); \ - __macro(swish, SwishFunctor, SwishGradFunctor); \ - __macro(thresholded_relu, ThresholdedReluFunctor, ThresholdedReluGradFunctor); +#define FOR_EACH_ACTIVATION_OP(__macro) \ + __macro(sigmoid, Sigmoid, SigmoidFunctor, SigmoidGradFunctor); \ + __macro(logsigmoid, LogSigmoid, LogSigmoidFunctor, LogSigmoidGradFunctor); \ + __macro(exp, Exp, ExpFunctor, ExpGradFunctor); \ + __macro(relu, Relu, ReluFunctor, ReluGradFunctor); \ + __macro(gelu, Gelu, GeluFunctor, GeluGradFunctor); \ + __macro(tanh, Tanh, TanhFunctor, TanhGradFunctor); \ + __macro(atan, Atan, AtanFunctor, AtanGradFunctor); \ + __macro(softshrink, SoftShrink, SoftShrinkFunctor, SoftShrinkGradFunctor); \ + __macro(sqrt, Sqrt, SqrtFunctor, SqrtGradFunctor); \ + __macro(abs, Abs, AbsFunctor, AbsGradFunctor); \ + __macro(ceil, Ceil, CeilFunctor, ZeroGradFunctor); \ + __macro(floor, Floor, FloorFunctor, ZeroGradFunctor); \ + __macro(cos, Cos, CosFunctor, CosGradFunctor); \ + __macro(acos, Acos, AcosFunctor, AcosGradFunctor); \ + __macro(sin, Sin, SinFunctor, SinGradFunctor); \ + __macro(asin, Asin, AsinFunctor, AsinGradFunctor); \ + __macro(round, Round, RoundFunctor, ZeroGradFunctor); \ + __macro(reciprocal, Reciprocal, ReciprocalFunctor, ReciprocalGradFunctor); \ + __macro(log, Log, LogFunctor, LogGradFunctor); \ + __macro(square, Square, SquareFunctor, SquareGradFunctor); \ + __macro(brelu, BRelu, BReluFunctor, BReluGradFunctor); \ + __macro(soft_relu, SoftRelu, SoftReluFunctor, SoftReluGradFunctor); \ + __macro(pow, Pow, PowFunctor, PowGradFunctor); \ + __macro(stanh, STanh, STanhFunctor, STanhGradFunctor); \ + __macro(softplus, Softplus, SoftplusFunctor, SoftplusGradFunctor); \ + __macro(softsign, Softsign, SoftsignFunctor, SoftsignGradFunctor); \ + __macro(relu6, Relu6, Relu6Functor, Relu6GradFunctor); \ + __macro(leaky_relu, LeakyRelu, LeakyReluFunctor, LeakyReluGradFunctor); \ + __macro(tanh_shrink, TanhShrink, TanhShrinkFunctor, TanhShrinkGradFunctor); \ + __macro(elu, ELU, ELUFunctor, ELUGradFunctor); \ + __macro(hard_shrink, HardShrink, HardShrinkFunctor, HardShrinkGradFunctor); \ + __macro(hard_sigmoid, HardSigmoid, HardSigmoidFunctor, \ + HardSigmoidGradFunctor); \ + __macro(swish, Swish, SwishFunctor, SwishGradFunctor); \ + __macro(thresholded_relu, ThresholdedRelu, ThresholdedReluFunctor, \ + ThresholdedReluGradFunctor); diff --git a/paddle/fluid/operators/affine_grid_op.h b/paddle/fluid/operators/affine_grid_op.h index 87d23831486e658374d4c011412fdef57be1b994..73df8a38b96c30196a7e39d2cf1e348f2a7722ec 100644 --- a/paddle/fluid/operators/affine_grid_op.h +++ b/paddle/fluid/operators/affine_grid_op.h @@ -121,9 +121,11 @@ class AffineGridOpKernel : public framework::OpKernel { // TODO(wanghaoshuang): Refine batched matrix multiply auto blas = math::GetBlas(ctx); for (int i = 0; i < n; ++i) { - Tensor sliced_grid = grid.Slice(i, i + 1).Resize({h * w, 3}); + Tensor sliced_grid = grid.Slice(i, i + 1).Resize( + {static_cast(h) * static_cast(w), 3}); Tensor sliced_theta = theta->Slice(i, i + 1).Resize({2, 3}); - Tensor sliced_out = output->Slice(i, i + 1).Resize({h * w, 2}); + Tensor sliced_out = output->Slice(i, i + 1).Resize( + {static_cast(h) * static_cast(w), 2}); blas.MatMul(sliced_grid, false, sliced_theta, true, T(1), &sliced_out, T(0)); } @@ -161,8 +163,10 @@ class AffineGridGradOpKernel : public framework::OpKernel { // TODO(wanghaoshuang): Refine batched matrix multiply auto blas = math::GetBlas(ctx); for (int i = 0; i < n; ++i) { - Tensor sliced_grid = grid.Slice(i, i + 1).Resize({h * w, 3}); - Tensor sliced_out_grad = output_grad->Slice(i, i + 1).Resize({h * w, 2}); + Tensor sliced_grid = grid.Slice(i, i + 1).Resize( + {static_cast(h) * static_cast(w), 3}); + Tensor sliced_out_grad = output_grad->Slice(i, i + 1).Resize( + {static_cast(h) * static_cast(w), 2}); Tensor sliced_theta_grad = theta_grad->Slice(i, i + 1).Resize({2, 3}); blas.MatMul(sliced_out_grad, true, sliced_grid, false, T(1), &sliced_theta_grad, T(0)); diff --git a/paddle/fluid/operators/controlflow/conditional_block_op.cc b/paddle/fluid/operators/controlflow/conditional_block_op.cc index dd28f82b65403550c67418cae535bbfeeef4476e..f0dc718195506e89bf9fecc0eb5e0d5117275a33 100644 --- a/paddle/fluid/operators/controlflow/conditional_block_op.cc +++ b/paddle/fluid/operators/controlflow/conditional_block_op.cc @@ -12,6 +12,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include +#include +#include +#include #include "paddle/fluid/framework/executor.h" #include "paddle/fluid/framework/op_registry.h" #include "paddle/fluid/framework/var_type.h" @@ -174,24 +177,41 @@ class ConditionalBlockGradOp : public ConditionalOp { framework::Executor exec(dev_place); auto *block = Attr("sub_block"); - exec.Run(*block->Program(), &cur_scope, block->ID(), false); - AssignLocalGradientToGlobal(dev_place, cur_scope, Inputs("Input"), - Outputs(framework::GradVarName("Input"))); + const auto &ins = Inputs("Input"); + const auto &d_ins = Outputs(framework::GradVarName("Input")); + const auto &conds = Inputs("Cond"); + const auto &d_conds = Outputs(framework::GradVarName("Cond")); + + std::vector ins_conds_grads; + ins_conds_grads.reserve(ins.size() + conds.size()); + for (auto &in : ins) { + ins_conds_grads.emplace_back(framework::GradVarName(in)); + } + for (auto &cond : conds) { + ins_conds_grads.emplace_back(framework::GradVarName(cond)); + } + + exec.Run(*block->Program(), &cur_scope, block->ID(), false, true, + ins_conds_grads); + + AssignLocalGradientToGlobal(dev_place, cur_scope, ins_conds_grads.data(), + ins.size(), d_ins); - AssignLocalGradientToGlobal(dev_place, cur_scope, Inputs("Cond"), - Outputs(framework::GradVarName("Cond"))); + AssignLocalGradientToGlobal(dev_place, cur_scope, + ins_conds_grads.data() + ins.size(), + conds.size(), d_conds); } } private: void AssignLocalGradientToGlobal( const platform::Place &place, const framework::Scope &cur_scope, - const std::vector &p_names, + const std::string *p_grad_names, size_t p_grad_names_num, const std::vector &pg_names) const { - for (size_t i = 0; i < p_names.size(); ++i) { + for (size_t i = 0; i < p_grad_names_num; ++i) { auto out_grad_name = pg_names[i]; - auto in_grad_name = framework::GradVarName(p_names[i]); + const auto &in_grad_name = p_grad_names[i]; auto *in_var = cur_scope.FindVar(in_grad_name); if (in_var == nullptr) { continue; diff --git a/paddle/fluid/operators/detection/gpc.cc b/paddle/fluid/operators/detection/gpc.cc index 7c0823c0487d39eece5be08322e7d182b931ba3c..f46aaf7d0a7b2d48f18ba6cccb555bbb691ad353 100644 --- a/paddle/fluid/operators/detection/gpc.cc +++ b/paddle/fluid/operators/detection/gpc.cc @@ -24,6 +24,7 @@ **/ #include "paddle/fluid/operators/detection/gpc.h" +#include "paddle/fluid/platform/enforce.h" namespace gpc { @@ -689,6 +690,7 @@ static bbox *create_contour_bboxes(gpc_polygon *p) { gpc_malloc(box, p->num_contours * sizeof(bbox), const_cast("Bounding box creation")); + PADDLE_ENFORCE_NOT_NULL(box); /* Construct contour bounding boxes */ for (c = 0; c < p->num_contours; c++) { @@ -852,6 +854,7 @@ void gpc_add_contour(gpc_polygon *p, gpc_vertex_list *new_contour, int hole) { /* Create an extended hole array */ gpc_malloc(extended_hole, (p->num_contours + 1) * sizeof(int), const_cast("contour hole addition")); + PADDLE_ENFORCE_NOT_NULL(extended_hole); /* Create an extended contour array */ gpc_malloc(extended_contour, @@ -969,6 +972,7 @@ void gpc_polygon_clip(gpc_op op, gpc_polygon *subj, gpc_polygon *clip, /* Build scanbeam table from scanbeam tree */ gpc_malloc(sbt, sbt_entries * sizeof(double), const_cast("sbt creation")); + PADDLE_ENFORCE_NOT_NULL(sbt); build_sbt(&scanbeam, sbt, sbtree); scanbeam = 0; free_sbtree(&sbtree); @@ -1604,6 +1608,7 @@ void gpc_tristrip_clip(gpc_op op, gpc_polygon *subj, gpc_polygon *clip, /* Build scanbeam table from scanbeam tree */ gpc_malloc(sbt, sbt_entries * sizeof(double), const_cast("sbt creation")); + PADDLE_ENFORCE_NOT_NULL(sbt); build_sbt(&scanbeam, sbt, sbtree); scanbeam = 0; free_sbtree(&sbtree); diff --git a/paddle/fluid/operators/distributed/grpc/grpc_serde.cc b/paddle/fluid/operators/distributed/grpc/grpc_serde.cc index 6e65aa5fae83536d229be63fbaf7874bd45f967d..91c398d0c84db1fc67740cd2368d178610ef0841 100644 --- a/paddle/fluid/operators/distributed/grpc/grpc_serde.cc +++ b/paddle/fluid/operators/distributed/grpc/grpc_serde.cc @@ -16,6 +16,7 @@ limitations under the License. */ #include #endif #include +#include #include // NOLINT #include "google/protobuf/io/coded_stream.h" @@ -104,8 +105,10 @@ void SerializeToByteBuffer(const std::string& name, framework::Variable* var, e.WriteVarlengthBeginning(VarMsg::kSerializedFieldNumber, payload->memory_size()); if (payload->memory_size() >= std::numeric_limits::max()) { - LOG(FATAL) << "AppendZeroCopy varname:" << name - << ", vlen:" << payload->memory_size(); + LOG(FATAL) << "FATAL error: varname:" << name + << ", vlen:" << payload->memory_size() + << " >= std::numeric_limits::max():" + << std::numeric_limits::max() << ", so exit!"; } // steal reference of tensor data ::grpc::Slice slices[4]; // metadata, tensor, rows meta, rows diff --git a/paddle/fluid/operators/interpolate_op.cc b/paddle/fluid/operators/interpolate_op.cc index edee8c08d070742d54f761083592466658a445c9..9f2e3ad4a5ac1786096c67154d5a9ef5ea62855c 100644 --- a/paddle/fluid/operators/interpolate_op.cc +++ b/paddle/fluid/operators/interpolate_op.cc @@ -37,10 +37,19 @@ class InterpolateOp : public framework::OperatorWithKernel { "Interpolation method can only be \"bilinear\" or \"nearest\"."); auto dim_x = ctx->GetInputDim("X"); // NCHW format - int out_h = ctx->Attrs().Get("out_h"); - int out_w = ctx->Attrs().Get("out_w"); PADDLE_ENFORCE_EQ(dim_x.size(), 4, "X's dimension must be 4"); + int out_h, out_w; + float scale = ctx->Attrs().Get("scale"); + if (scale > 0) { + // round down + out_h = static_cast(dim_x[2] * scale); + out_w = static_cast(dim_x[3] * scale); + } else { + out_h = ctx->Attrs().Get("out_h"); + out_w = ctx->Attrs().Get("out_w"); + } + if (ctx->HasInput("OutSize") && ctx->IsRuntime()) { auto out_size_dim = ctx->GetInputDim("OutSize"); PADDLE_ENFORCE_EQ(out_size_dim.size(), 1, @@ -77,6 +86,7 @@ class InterpolateOpMaker : public framework::OpProtoAndCheckerMaker { AddAttr("out_h", "output height of interpolate op."); AddAttr("out_w", "output width of interpolate op."); + AddAttr("scale", "scale factor of interpolate op.").SetDefault(0.); AddAttr("interp_method", "(string, default \"bilinear\"), interpolation " "method, can be \"bilinear\" for " diff --git a/paddle/fluid/operators/interpolate_op.cu b/paddle/fluid/operators/interpolate_op.cu index b887878ea2291d6c56fec91738784e338606b84f..35177a4e9ade26831f50de84bbb943d856cb98d9 100644 --- a/paddle/fluid/operators/interpolate_op.cu +++ b/paddle/fluid/operators/interpolate_op.cu @@ -192,9 +192,21 @@ class InterpolateOpCUDAKernel : public framework::OpKernel { auto* output = ctx.Output("Out"); auto* input_data = input->data(); + int n = input->dims()[0]; + int c = input->dims()[1]; + int in_h = input->dims()[2]; + int in_w = input->dims()[3]; + auto interp_method = ctx.Attr("interp_method"); int out_h = ctx.Attr("out_h"); int out_w = ctx.Attr("out_w"); + + float scale = ctx.Attr("scale"); + if (scale > 0) { + out_h = in_h * scale; + out_w = in_w * scale; + } + auto out_size = ctx.Input("OutSize"); if (out_size != nullptr) { Tensor sizes; @@ -207,11 +219,6 @@ class InterpolateOpCUDAKernel : public framework::OpKernel { bool align_corners = ctx.Attr("align_corners"); int align_mode = ctx.Attr("align_mode"); - int n = input->dims()[0]; - int c = input->dims()[1]; - int in_h = input->dims()[2]; - int in_w = input->dims()[3]; - auto* output_data = output->mutable_data({n, c, out_h, out_w}, ctx.GetPlace()); @@ -268,14 +275,20 @@ class InterpolateGradOpCUDAKernel : public framework::OpKernel { math::SetConstant zero; zero(device_ctx, input_grad, static_cast(0.0)); + int n = input_grad->dims()[0]; + int c = input_grad->dims()[1]; + int in_h = input_grad->dims()[2]; + int in_w = input_grad->dims()[3]; + auto interp_method = ctx.Attr("interp_method"); int out_h = ctx.Attr("out_h"); int out_w = ctx.Attr("out_w"); + float scale = ctx.Attr("scale"); + if (scale > 0) { + out_h = in_h * scale; + out_w - in_w* scale; + } auto out_size = ctx.Input("OutSize"); - - bool align_corners = ctx.Attr("align_corners"); - int align_mode = ctx.Attr("align_mode"); - if (out_size != nullptr) { Tensor sizes; framework::TensorCopy(*out_size, platform::CPUPlace(), &sizes); @@ -284,10 +297,8 @@ class InterpolateGradOpCUDAKernel : public framework::OpKernel { out_w = size_data[1]; } - int n = input_grad->dims()[0]; - int c = input_grad->dims()[1]; - int in_h = input_grad->dims()[2]; - int in_w = input_grad->dims()[3]; + bool align_corners = ctx.Attr("align_corners"); + int align_mode = ctx.Attr("align_mode"); int in_hw = in_h * in_w; int out_hw = out_h * out_w; diff --git a/paddle/fluid/operators/interpolate_op.h b/paddle/fluid/operators/interpolate_op.h index c631ad1dd158ce114169602f073d69b2291b5b3b..5fd42809dfec6dd821c9b27bc97d61de94b5d326 100644 --- a/paddle/fluid/operators/interpolate_op.h +++ b/paddle/fluid/operators/interpolate_op.h @@ -163,9 +163,21 @@ class InterpolateKernel : public framework::OpKernel { auto* input = ctx.Input("X"); auto* output = ctx.Output("Out"); + const int n = input->dims()[0]; + const int c = input->dims()[1]; + const int in_h = input->dims()[2]; + const int in_w = input->dims()[3]; + std::string interp_method = ctx.Attr("interp_method"); int out_h = ctx.Attr("out_h"); int out_w = ctx.Attr("out_w"); + + float scale = ctx.Attr("scale"); + if (scale > 0) { + out_h = static_cast(in_h * scale); + out_w = static_cast(in_w * scale); + } + auto out_size = ctx.Input("OutSize"); if (out_size != nullptr) { auto out_size_data = out_size->data(); @@ -175,11 +187,6 @@ class InterpolateKernel : public framework::OpKernel { bool align_corners = ctx.Attr("align_corners"); int align_mode = ctx.Attr("align_mode"); - const int n = input->dims()[0]; - const int c = input->dims()[1]; - const int in_h = input->dims()[2]; - const int in_w = input->dims()[3]; - output->mutable_data({n, c, out_h, out_w}, ctx.GetPlace()); auto& device_ctx = ctx.template device_context(); @@ -221,23 +228,31 @@ class InterpolateGradKernel : public framework::OpKernel { auto* input_grad = ctx.Output(framework::GradVarName("X")); auto* output_grad = ctx.Input(framework::GradVarName("Out")); + const int n = input->dims()[0]; + const int c = input->dims()[1]; + const int in_h = input->dims()[2]; + const int in_w = input->dims()[3]; + std::string interp_method = ctx.Attr("interp_method"); int out_h = ctx.Attr("out_h"); int out_w = ctx.Attr("out_w"); + + float scale = ctx.Attr("scale"); + if (scale > 0) { + out_h = static_cast(in_h * scale); + out_w = static_cast(in_w * scale); + } + auto out_size = ctx.Input("OutSize"); if (out_size != nullptr) { auto out_size_data = out_size->data(); out_h = out_size_data[0]; out_w = out_size_data[1]; } + bool align_corners = ctx.Attr("align_corners"); int align_mode = ctx.Attr("align_mode"); - const int n = input->dims()[0]; - const int c = input->dims()[1]; - const int in_h = input->dims()[2]; - const int in_w = input->dims()[3]; - input_grad->mutable_data({n, c, in_h, in_w}, ctx.GetPlace()); auto& device_ctx = ctx.template device_context(); diff --git a/paddle/fluid/operators/jit/test.cc b/paddle/fluid/operators/jit/test.cc index d30fa014ed5fbac9ed71f3185ce0443d33f4a281..875d4f864353c131ca4d72b5176adcae8aff724a 100644 --- a/paddle/fluid/operators/jit/test.cc +++ b/paddle/fluid/operators/jit/test.cc @@ -991,15 +991,17 @@ TEST(JITKernel_pool, jitpool) { TEST(JITKernel_pool, more) { const auto& kers = jit::KernelPool::Instance().AllKernels(); -#if defined(__APPLE__) || defined(__OSX__) - EXPECT_EQ(kers.size(), 10UL); -#else -#ifdef PADDLE_WITH_MKLML - EXPECT_EQ(kers.size(), 22UL); -#else - EXPECT_EQ(kers.size(), 8UL); + size_t target_num = 8; + +#ifdef __AVX__ + target_num += 2; #endif + +#ifdef PADDLE_WITH_MKLML + target_num += 12; #endif + + EXPECT_EQ(kers.size(), target_num); } TEST(JITKernel_pool, refer) { diff --git a/paddle/fluid/operators/linspace_op.cc b/paddle/fluid/operators/linspace_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..f4aeb062d8dfae31a72b8ebccb3d377276662da6 --- /dev/null +++ b/paddle/fluid/operators/linspace_op.cc @@ -0,0 +1,84 @@ +/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/operators/linspace_op.h" + +namespace paddle { +namespace operators { + +class LinspaceOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("Start"), + "Input(Start) of LinspaceOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Stop"), + "Input(Stop) of LinspaceOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Num"), + "Input(Num) of LinspaceOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(OUt) of LinspaceOp should not be null."); + + auto s_dims = ctx->GetInputDim("Start"); + PADDLE_ENFORCE((s_dims.size() == 1) && (s_dims[0] == 1), + "The shape of Input(Start) should be [1]."); + + auto e_dims = ctx->GetInputDim("Stop"); + PADDLE_ENFORCE((e_dims.size() == 1) && (e_dims[0] == 1), + "The shape of Input(Stop) should be [1]."); + + auto step_dims = ctx->GetInputDim("Num"); + PADDLE_ENFORCE((step_dims.size() == 1) && (step_dims[0] == 1), + "The shape of Input(Num) should be [1]."); + + ctx->SetOutputDim("Out", {-1}); + } + + protected: + framework::OpKernelType GetExpectedKernelType( + const framework::ExecutionContext& ctx) const override { + framework::LibraryType library_{framework::LibraryType::kPlain}; + framework::DataLayout layout_ = framework::DataLayout::kAnyLayout; + return framework::OpKernelType( + ctx.Input("Start")->type(), ctx.device_context(), + layout_, library_); + } +}; + +class LinspaceOpMaker : public framework::OpProtoAndCheckerMaker { + public: + void Make() override { + AddInput("Start", + "First entry in the sequence. It is a tensor of shape [1], should " + "be of type float32 or float64."); + AddInput("Stop", + "Last entry in the sequence. It is a tensor of shape [1], should " + "be of type float32 or float64."); + AddInput("Num", + "Number of entry in the sequence. It is a tensor of shape [1], " + "should be of type int32."); + AddOutput("Out", "A sequence of numbers."); + AddComment(R"DOC( + Return fixed number of evenly spaced values within a given interval. First entry is start, and last entry is stop. In the case when Num is 1, only Start is returned. Like linspace function of numpy. +)DOC"); + } +}; +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP_WITHOUT_GRADIENT(linspace, ops::LinspaceOp, ops::LinspaceOpMaker); +REGISTER_OP_CPU_KERNEL(linspace, ops::CPULinspaceKernel, + ops::CPULinspaceKernel); diff --git a/paddle/fluid/operators/linspace_op.cu b/paddle/fluid/operators/linspace_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..90bd17cda0e0d1f78810233537bb502f9115fbd0 --- /dev/null +++ b/paddle/fluid/operators/linspace_op.cu @@ -0,0 +1,75 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/framework/op_registry.h" +#include "paddle/fluid/operators/linspace_op.h" +#include "paddle/fluid/platform/cuda_primitives.h" + +namespace paddle { +namespace operators { + +#define CUDA_1D_KERNEL_LOOP(i, n) \ + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < (n); \ + i += blockDim.x * gridDim.x) + +template +__global__ void LinspaceKernel(T start, T step, int64_t size, T* out) { + CUDA_1D_KERNEL_LOOP(index, size) { out[index] = start + step * index; } +} + +template +__global__ void LinspaceSpecialKernel(T start, T* out) { + out[0] = start; +} + +template +class CUDALinspaceKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* start_t = context.Input("Start"); + auto* stop_t = context.Input("Stop"); + auto* num_t = context.Input("Num"); + auto* out = context.Output("Out"); + + framework::Tensor n; + framework::TensorCopy(*start_t, platform::CPUPlace(), &n); + T start = n.data()[0]; + framework::TensorCopy(*stop_t, platform::CPUPlace(), &n); + T stop = n.data()[0]; + framework::TensorCopy(*num_t, platform::CPUPlace(), &n); + int32_t num = n.data()[0]; + + PADDLE_ENFORCE(num > 0, "The num of linspace op should be larger than 0."); + + out->Resize(framework::make_ddim({num})); + T* out_data = out->mutable_data(context.GetPlace()); + + T step = 0; + if (num != 1) { + step = (stop - start) / (num - 1); + } + + auto stream = context.cuda_device_context().stream(); + int block = 512; + int grid = (num + block - 1) / block; + LinspaceKernel<<>>(start, step, num, out_data); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP_CUDA_KERNEL(linspace, ops::CUDALinspaceKernel, + ops::CUDALinspaceKernel); diff --git a/paddle/fluid/operators/linspace_op.h b/paddle/fluid/operators/linspace_op.h new file mode 100644 index 0000000000000000000000000000000000000000..b1fcac73b0ad249aa19859bde770a8554cdb7408 --- /dev/null +++ b/paddle/fluid/operators/linspace_op.h @@ -0,0 +1,51 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once +#include +#include "paddle/fluid/framework/op_registry.h" +#include "paddle/fluid/operators/math/math_function.h" + +namespace paddle { +namespace operators { + +template +class CPULinspaceKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + T start = context.Input("Start")->data()[0]; + T stop = context.Input("Stop")->data()[0]; + int32_t num = context.Input("Num")->data()[0]; + auto* out = context.Output("Out"); + PADDLE_ENFORCE(num > 0, "The num of linspace op should be larger than 0."); + + out->Resize(framework::make_ddim({num})); + + T* out_data = out->mutable_data(context.GetPlace()); + + if (num > 1) { + T step = (stop - start) / (num - 1); + T value = start; + for (int i = 0; i < num; ++i) { + out_data[i] = value; + value += step; + } + } else { + out_data[0] = start; + } + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/fluid/operators/load_op.cc b/paddle/fluid/operators/load_op.cc index 656728c609eb19f90390d9dec72d9e30fd3040fd..435c755df3642ae0ba5144a89ed30ed6e0b63258 100644 --- a/paddle/fluid/operators/load_op.cc +++ b/paddle/fluid/operators/load_op.cc @@ -29,7 +29,7 @@ class LoadOp : public framework::OperatorWithKernel { framework::OpKernelType GetExpectedKernelType( const framework::ExecutionContext &ctx) const override { framework::OpKernelType kt = framework::OpKernelType( - framework::proto::VarType::FP32, platform::CPUPlace()); + framework::proto::VarType::FP32, ctx.GetPlace()); return kt; } }; diff --git a/paddle/fluid/operators/lod_reset_op.cc b/paddle/fluid/operators/lod_reset_op.cc index e0ab02cd90cdee848250a6aba882b0cb0c17abd7..458037c5aca6af4c8c97b2da630c35929770c156 100644 --- a/paddle/fluid/operators/lod_reset_op.cc +++ b/paddle/fluid/operators/lod_reset_op.cc @@ -30,10 +30,10 @@ class LoDResetOp : public framework::OperatorWithKernel { if (!ctx->HasInput("Y")) { auto level0 = ctx->Attrs().Get>("target_lod"); - PADDLE_ENFORCE_GT(level0.size(), 1, + PADDLE_ENFORCE_GT(level0.size(), 0, "If Input(Y) not provided, the target lod should be " "specified by attribute `target_lod`."); - } else { + } else if (ctx->IsRuntime()) { ctx->ShareLoD("Y", "Out"); } @@ -48,6 +48,23 @@ class LoDResetOp : public framework::OperatorWithKernel { } }; +class LoDResetOpVarTypeInference : public framework::VarTypeInference { + public: + void operator()(framework::InferVarTypeContext *ctx) const override { + auto x_var_name = ctx->Input("X").front(); + auto out_var_name = ctx->Output("Out").front(); + if (ctx->HasInput("Y")) { + auto y_var_name = ctx->Input("Y").front(); + auto y_lod_level = std::max(ctx->GetLoDLevel(y_var_name), 1); + ctx->SetLoDLevel(out_var_name, y_lod_level); + } else { + ctx->SetLoDLevel(out_var_name, 1); + } + ctx->SetDataType(out_var_name, ctx->GetDataType(x_var_name)); + ctx->SetType(out_var_name, paddle::framework::proto::VarType::LOD_TENSOR); + } +}; + class LoDResetOpMaker : public framework::OpProtoAndCheckerMaker { public: void Make() override { @@ -177,9 +194,10 @@ DECLARE_NO_NEED_BUFFER_VARS_INFERENCE(LoDResetGradNoNeedBufferVarInference, namespace ops = paddle::operators; REGISTER_OPERATOR(lod_reset, ops::LoDResetOp, ops::LoDResetOpMaker, - ops::LoDResetGradDescMaker); + ops::LoDResetGradDescMaker, ops::LoDResetOpVarTypeInference); REGISTER_OPERATOR(lod_reset_grad, ops::LoDResetGradOp, ops::LoDResetGradNoNeedBufferVarInference); + REGISTER_OP_CPU_KERNEL( lod_reset, ops::LoDResetKernel, ops::LoDResetKernel, diff --git a/paddle/fluid/operators/lod_reset_op.h b/paddle/fluid/operators/lod_reset_op.h index d36aa0ce025a1c0f717913131fcc75040d16afac..1c2f0b0ac8ab4be35e4716acc7be3f05b9d63805 100644 --- a/paddle/fluid/operators/lod_reset_op.h +++ b/paddle/fluid/operators/lod_reset_op.h @@ -63,7 +63,7 @@ class LoDResetKernel : public framework::OpKernel { "Target LoD should be a vector end with the " "first dimension of Input(X)."); for (size_t i = 0; i < level0.size() - 1; ++i) { - PADDLE_ENFORCE(level0[i + 1] > level0[i], + PADDLE_ENFORCE(level0[i + 1] >= level0[i], "Target LoD should be an ascending vector."); } diff --git a/paddle/fluid/operators/lookup_table_op.cc b/paddle/fluid/operators/lookup_table_op.cc index 04323eee02c8dbed6eeffef67ef75b18f351e46b..8b7d7a52704d5452487373d38d75626ea2b239c8 100644 --- a/paddle/fluid/operators/lookup_table_op.cc +++ b/paddle/fluid/operators/lookup_table_op.cc @@ -13,6 +13,10 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/fluid/operators/lookup_table_op.h" + +#include + +#include "paddle/fluid/framework/no_need_buffer_vars_inference.h" #include "paddle/fluid/framework/var_type_inference.h" namespace paddle { @@ -119,6 +123,29 @@ or not. And the output only shares the LoD information with input Ids. } }; +DECLARE_NO_NEED_BUFFER_VARS_INFERENCE(LookupTableGradOpNoBuffer, "W"); + +class LookupTableGradOpDescMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + std::unique_ptr Apply() const override { + std::unique_ptr op(new framework::OpDesc()); + + op->SetType("lookup_table_grad"); + + op->SetInput("W", Input("W")); + op->SetInput("Ids", Input("Ids")); + op->SetInput(framework::GradVarName("Out"), OutputGrad("Out")); + + op->SetOutput(framework::GradVarName("W"), InputGrad("W")); + + op->SetAttrMap(Attrs()); + return op; + } +}; + class LookupTableOpGrad : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; @@ -131,7 +158,8 @@ class LookupTableOpGrad : public framework::OperatorWithKernel { protected: framework::OpKernelType GetExpectedKernelType( const framework::ExecutionContext& ctx) const override { - auto data_type = framework::GetDataTypeOfVar(ctx.InputVar("Out")); + auto data_type = framework::GetDataTypeOfVar( + ctx.InputVar(framework::GradVarName("Out"))); return framework::OpKernelType(data_type, ctx.device_context()); } }; @@ -159,10 +187,11 @@ class LookupTableOpGradVarTypeInference : public framework::VarTypeInference { } // namespace paddle namespace ops = paddle::operators; -REGISTER_OPERATOR(lookup_table, ops::LookupTableOp, - paddle::framework::DefaultGradOpDescMaker, - ops::LookupTableOpMaker); +REGISTER_OPERATOR(lookup_table, ops::LookupTableOp, ops::LookupTableOpMaker, + ops::LookupTableGradOpDescMaker); + REGISTER_OPERATOR(lookup_table_grad, ops::LookupTableOpGrad, + ops::LookupTableGradOpNoBuffer, ops::LookupTableOpGradVarTypeInference); REGISTER_OP_CPU_KERNEL(lookup_table, ops::LookupTableKernel, diff --git a/paddle/fluid/operators/ngraph/ngraph_engine.cc b/paddle/fluid/operators/ngraph/ngraph_engine.cc index 9f73bbc1fdc72766a0b57bc72c62d208277c2f20..5ef385d2fcbaf01dce5c9b85321b41c103e5655a 100644 --- a/paddle/fluid/operators/ngraph/ngraph_engine.cc +++ b/paddle/fluid/operators/ngraph/ngraph_engine.cc @@ -75,6 +75,7 @@ std::vector NgraphEngine::feed_vars = {}; std::vector NgraphEngine::fetch_vars = {}; framework::Variable* NgraphEngine::pre_var_ptr = nullptr; const framework::BlockDesc* NgraphEngine::p_bdesc = nullptr; +bool NgraphEngine::is_training = false; std::unordered_map NgraphEngine::engine_cache = {}; std::unordered_map> NgraphOpIntervals( int size = ops->size(); int left = 0; while (left < size && ops->at(left)->Type() != framework::kFeedOpType && + ops->at(left)->Type() != "read" && ops->at(left)->Type() != framework::kFetchOpType) { ++left; } - while (left < size && ops->at(left)->Type() == framework::kFeedOpType) { + while (left < size && (ops->at(left)->Type() == framework::kFeedOpType || + ops->at(left)->Type() == "read")) { for (auto& var_name_item : ops->at(left)->Outputs()) { for (auto& var_name : var_name_item.second) { NgraphEngine::feed_vars.emplace_back(var_name); @@ -270,6 +273,7 @@ void NgraphEngine::Prepare(const std::vector& interval) { for (auto op_desc : ops_desc) { if (op_desc->Type().find("_grad") != std::string::npos) { + is_training = true; this->is_test_ = false; break; } @@ -590,7 +594,7 @@ void NgraphEngine::Run(const framework::Scope& scope, } bool is_persistable = (p_persistables->find(vi) != p_persistables->end()) ? true : false; - if (is_test && is_persistable) { + if (!is_training && is_test && is_persistable) { ti->set_stale(false); } (*p_t_in).emplace_back(ti); diff --git a/paddle/fluid/operators/ngraph/ngraph_engine.h b/paddle/fluid/operators/ngraph/ngraph_engine.h index b6532519e947bc59f0605c4f2008270f5e51b0e0..19400ac5b0ecd9d3254583b8db9889fc6cf8bc0f 100644 --- a/paddle/fluid/operators/ngraph/ngraph_engine.h +++ b/paddle/fluid/operators/ngraph/ngraph_engine.h @@ -57,6 +57,7 @@ class NgraphEngine { void Run(const framework::Scope& scope, const platform::Place& place) const; + static bool is_training; static const framework::BlockDesc* p_bdesc; static std::vector feed_vars, fetch_vars; diff --git a/paddle/fluid/operators/pixel_shuffle_op.cc b/paddle/fluid/operators/pixel_shuffle_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..59ba660af79bff02cd350afb3eb7675bfe8ac498 --- /dev/null +++ b/paddle/fluid/operators/pixel_shuffle_op.cc @@ -0,0 +1,135 @@ +/*Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/operators/pixel_shuffle_op.h" +#include + +namespace paddle { +namespace operators { + +class PixelShuffleOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), + "Input(X) of PixelShuffleOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of PixelShuffleOp should not be null."); + + auto input_dims = ctx->GetInputDim("X"); + PADDLE_ENFORCE(input_dims.size() == 4, "The layout of input is NCHW."); + auto upscale_factor = ctx->Attrs().Get("upscale_factor"); + + PADDLE_ENFORCE(input_dims[1] % (upscale_factor * upscale_factor) == 0, + "Upscale_factor should devide the number of channel"); + + auto output_dims = input_dims; + output_dims[0] = input_dims[0]; + output_dims[1] = input_dims[1] / (upscale_factor * upscale_factor); + output_dims[2] = input_dims[2] * upscale_factor; + output_dims[3] = input_dims[3] * upscale_factor; + ctx->SetOutputDim("Out", output_dims); + } +}; + +class PixelShuffleOpMaker : public framework::OpProtoAndCheckerMaker { + public: + void Make() override { + AddInput( + "X", + "(Tensor, default Tensor), " + "the input feature data of PixelShuffleOp, the layout is [N C H W]."); + AddOutput( + "Out", + "(Tensor, default Tensor), the output of " + "PixelShuffleOp. The layout is [N,C/factor^2,H*factor,W*factor]."); + AddAttr("upscale_factor", + "the factor to increase spatial resolution by.") + .SetDefault(1) + .AddCustomChecker([](const int& upscale_factor) { + PADDLE_ENFORCE_GE(upscale_factor, 1, + "upscale_factor should be larger than 0."); + }); + + AddComment(R"DOC( + Pixel Shuffle operator + This operator rearranges elements in a tensor of shape :math:`(*, C \times r^2, H, W)` + to a tensor of shape :math:`(C, H \times r, W \times r)`. + + This is useful for implementing efficient sub-pixel convolution + with a stride of :math:`1/r`. + + Please refer to the paper: + `Real-Time Single Image and Video Super-Resolution Using an Efficient + Sub-Pixel Convolutional Neural Network `_ + by Shi et. al (2016) for more details. + + )DOC"); + } +}; + +class PixelShuffleGradMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + std::unique_ptr Apply() const override { + auto* op = new framework::OpDesc(); + op->SetType("pixel_shuffle_grad"); + op->SetInput(framework::GradVarName("Out"), OutputGrad("Out")); + op->SetAttrMap(Attrs()); + op->SetOutput(framework::GradVarName("X"), InputGrad("X")); + return std::unique_ptr(op); + } +}; + +class PixelShuffleGradOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")), + "Input(Out@Grad) should not be null"); + PADDLE_ENFORCE(ctx->HasOutput(framework::GradVarName("X")), + "Output(X@Grad) should not be null"); + + auto do_dims = ctx->GetInputDim(framework::GradVarName("Out")); + PADDLE_ENFORCE(do_dims.size() == 4, "The layout of input is NCHW."); + + auto upscale_factor = ctx->Attrs().Get("upscale_factor"); + + auto dx_dims = do_dims; + dx_dims[0] = do_dims[0]; + dx_dims[1] = do_dims[1] * (upscale_factor * upscale_factor); + dx_dims[2] = do_dims[2] / upscale_factor; + dx_dims[3] = do_dims[3] / upscale_factor; + ctx->SetOutputDim(framework::GradVarName("X"), dx_dims); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OPERATOR(pixel_shuffle, ops::PixelShuffleOp, ops::PixelShuffleOpMaker, + ops::PixelShuffleGradMaker); + +REGISTER_OPERATOR(pixel_shuffle_grad, ops::PixelShuffleGradOp); + +REGISTER_OP_CPU_KERNEL( + pixel_shuffle, + ops::PixelShuffleOpKernel, + ops::PixelShuffleOpKernel); + +REGISTER_OP_CPU_KERNEL( + pixel_shuffle_grad, + ops::PixelShuffleGradOpKernel, + ops::PixelShuffleGradOpKernel); diff --git a/paddle/fluid/operators/pixel_shuffle_op.cu b/paddle/fluid/operators/pixel_shuffle_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..6faf91079e1dac00b3516ccde8dc82cec73a79e6 --- /dev/null +++ b/paddle/fluid/operators/pixel_shuffle_op.cu @@ -0,0 +1,26 @@ +/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/operators/pixel_shuffle_op.h" + +namespace ops = paddle::operators; +namespace plat = paddle::platform; + +REGISTER_OP_CUDA_KERNEL( + pixel_shuffle, ops::PixelShuffleOpKernel, + ops::PixelShuffleOpKernel); +REGISTER_OP_CUDA_KERNEL( + pixel_shuffle_grad, + ops::PixelShuffleGradOpKernel, + ops::PixelShuffleGradOpKernel); diff --git a/paddle/fluid/operators/pixel_shuffle_op.h b/paddle/fluid/operators/pixel_shuffle_op.h new file mode 100644 index 0000000000000000000000000000000000000000..1ae1c7e9d50cb9d701fd0e79337a1906f2f5d545 --- /dev/null +++ b/paddle/fluid/operators/pixel_shuffle_op.h @@ -0,0 +1,82 @@ +/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once +#include +#include +#include "paddle/fluid/framework/op_registry.h" +#include "paddle/fluid/operators/math/math_function.h" + +namespace paddle { +namespace operators { + +template +class PixelShuffleOpKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + auto* in = ctx.Input("X"); + auto* out = ctx.Output("Out"); + out->mutable_data(ctx.GetPlace()); + + int factor = ctx.Attr("upscale_factor"); + + auto in_dims = in->dims(); + auto o_dims = out->dims(); + + framework::Tensor t; + t.ShareDataWith(*in); + t.Resize({in_dims[0], o_dims[1], factor, factor, in_dims[2], in_dims[3]}); + + std::vector axis = {0, 1, 4, 2, 5, 3}; + + framework::Tensor o; + o.ShareDataWith(*out); + o.Resize({in_dims[0], o_dims[1], in_dims[2], factor, in_dims[3], factor}); + + math::Transpose trans; + auto& dev_ctx = ctx.template device_context(); + trans(dev_ctx, t, &o, axis); + out->Resize(o_dims); + } +}; + +template +class PixelShuffleGradOpKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + auto* dout = ctx.Input(framework::GradVarName("Out")); + auto* dx = ctx.Output(framework::GradVarName("X")); + dx->mutable_data(ctx.GetPlace()); + + int factor = ctx.Attr("upscale_factor"); + + auto do_dims = dout->dims(); + auto dx_dims = dx->dims(); + + framework::Tensor t; + t.ShareDataWith(*dout); + t.Resize({do_dims[0], do_dims[1], dx_dims[2], factor, dx_dims[3], factor}); + + std::vector axis = {0, 1, 3, 5, 2, 4}; + + framework::Tensor o; + o.ShareDataWith(*dx); + o.Resize({do_dims[0], do_dims[1], factor, factor, dx_dims[2], dx_dims[3]}); + + math::Transpose trans; + auto& dev_ctx = ctx.template device_context(); + trans(dev_ctx, t, &o, axis); + dx->Resize(dx_dims); + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/fluid/operators/recurrent_op.cc b/paddle/fluid/operators/recurrent_op.cc index 2898a62ddbac524ceb212cac5f34aeda3b1e01cb..1a2feee11c951cd4a55958df58f3756472f64769 100644 --- a/paddle/fluid/operators/recurrent_op.cc +++ b/paddle/fluid/operators/recurrent_op.cc @@ -23,6 +23,7 @@ constexpr char kInitialStates[] = "initial_states"; constexpr char kParameters[] = "parameters"; constexpr char kOutputs[] = "outputs"; constexpr char kStepScopes[] = "step_scopes"; +constexpr char kHasStates[] = "has_states"; constexpr char kExStates[] = "ex_states"; constexpr char kStates[] = "states"; constexpr char kStepBlock[] = "sub_block"; @@ -241,11 +242,16 @@ class RecurrentOp : public RecurrentBase { private: void RunImpl(const framework::Scope &scope, const platform::Place &place) const override { + bool has_state = Attr(kHasStates); auto seq_len = static_cast(this->GetSequenceLength(scope)); VLOG(3) << "Static RNN input sequence length = " << seq_len; StepScopes scopes = CreateStepScopes(scope, seq_len); auto reverse = Attr(kReverse); + // get device context from pool + platform::DeviceContextPool &pool = platform::DeviceContextPool::Instance(); + auto &dev_ctx = *pool.Get(place); + framework::Executor executor(place); auto *block = Attr(kStepBlock); @@ -269,15 +275,17 @@ class RecurrentOp : public RecurrentBase { inside->Resize(framework::make_ddim(dims)); }); - if (i == 0) { - // Link initial states --> ex_states - LinkTensor(scope, Inputs(kInitialStates), &cur_scope, - Attr>(kExStates)); - } else { - auto &ex_scope = scopes.ExScope(); - // Link ex_scope::state --> cur_scope::ex_state - LinkTensor(ex_scope, Attr>(kStates), - &cur_scope, Attr>(kExStates)); + if (has_state) { + if (i == 0) { + // Link initial states --> ex_states + LinkTensor(scope, Inputs(kInitialStates), &cur_scope, + Attr>(kExStates)); + } else { + auto &ex_scope = scopes.ExScope(); + // Link ex_scope::state --> cur_scope::ex_state + LinkTensor(ex_scope, Attr>(kStates), + &cur_scope, Attr>(kExStates)); + } } // Every inputs are linked now, execute! @@ -286,11 +294,6 @@ class RecurrentOp : public RecurrentBase { std::vector() /*skip_ref_cnt_vars*/, true /*force_disable_gc*/); - // get device context from pool - platform::DeviceContextPool &pool = - platform::DeviceContextPool::Instance(); - auto &dev_ctx = *pool.Get(place); - // Copy inside::output -> outside::output // outside::output[seq_offset: seq_offset + 1] = inside::output this->LinkTensorWithCallback( @@ -333,13 +336,13 @@ class RecurrentGradOp : public RecurrentBase { private: void RunImpl(const framework::Scope &scope, const platform::Place &place) const override { - auto seq_len = static_cast(GetSequenceLength(scope)); + bool has_state = Attr(kHasStates); + const size_t seq_len = static_cast(GetSequenceLength(scope)); StepScopes scopes = CreateStepScopes(scope, seq_len); auto reverse = Attr(kReverse); framework::Executor executor(place); auto *block = Attr(kStepBlock); - auto *program = block->Program(); // get device context from pool @@ -350,6 +353,7 @@ class RecurrentGradOp : public RecurrentBase { size_t seq_offset = reverse ? step_id : seq_len - step_id - 1; VLOG(3) << "Recurrent backward operate at the time step " << seq_offset; auto &cur_scope = scopes.CurScope(); + // Link outside::output_grads --> inside::output_grads // inside::output_grad = outside::output_grad[seq_offset:seq_offset+1] LinkTensorWithCallback( @@ -370,30 +374,32 @@ class RecurrentGradOp : public RecurrentBase { VLOG(10) << " RNN output gradients = [" << sout.str() << "]"; } - // Link states - // if cur_scope::cur_state_grad in out_grads: - // cur_scope::cur_state_grad += ex_scope::ex_state_grad - // else: - // ex_scope::ex_state_grad --> cur_scope::cur_state_grad - if (step_id != 0) { // not at beginning - auto &ex_scope = scopes.ExScope(); - auto ex_state_grads = - GradVarLists(Attr>(kExStates)); - auto cur_state_grads = - GradVarLists(Attr>(kStates)); - - PADDLE_ENFORCE_EQ(ex_state_grads.size(), cur_state_grads.size()); - for (size_t i = 0; i < ex_state_grads.size(); ++i) { - auto &cur_grad = cur_state_grads[i]; - auto &ex_grad = ex_state_grads[i]; - auto &ex_tensor = - ex_scope.FindVar(ex_grad)->Get(); - - VLOG(10) << " RNN link " << cur_grad << " from " << ex_grad; - auto *cur_grad_var = cur_scope.Var(cur_grad); - auto cur_grad_tensor = - cur_grad_var->GetMutable(); - framework::TensorCopy(ex_tensor, place, dev_ctx, cur_grad_tensor); + if (has_state) { + // Link states + // if cur_scope::cur_state_grad in out_grads: + // cur_scope::cur_state_grad += ex_scope::ex_state_grad + // else: + // ex_scope::ex_state_grad --> cur_scope::cur_state_grad + if (step_id != 0) { // not at beginning + auto &ex_scope = scopes.ExScope(); + auto ex_state_grads = + GradVarLists(Attr>(kExStates)); + auto cur_state_grads = + GradVarLists(Attr>(kStates)); + + PADDLE_ENFORCE_EQ(ex_state_grads.size(), cur_state_grads.size()); + for (size_t i = 0; i < ex_state_grads.size(); ++i) { + auto &cur_grad = cur_state_grads[i]; + auto &ex_grad = ex_state_grads[i]; + auto &ex_tensor = + ex_scope.FindVar(ex_grad)->Get(); + + VLOG(10) << " RNN link " << cur_grad << " from " << ex_grad; + auto *cur_grad_var = cur_scope.Var(cur_grad); + auto cur_grad_tensor = + cur_grad_var->GetMutable(); + framework::TensorCopy(ex_tensor, place, dev_ctx, cur_grad_tensor); + } } } @@ -442,8 +448,8 @@ class RecurrentGradOp : public RecurrentBase { } auto new_inside_name = cur_scope.Rename(inside_grad_name); - // sum gradient + // sum gradient auto sum_op = framework::OpRegistry::CreateOp( "sum", {{"X", {pg_names[param_id], new_inside_name}}}, {{"Out", {pg_names[param_id]}}}, @@ -475,22 +481,33 @@ class RecurrentGradOp : public RecurrentBase { true /*is_backward*/); VLOG(5) << "Link outside gradient finished "; - if (step_id + 1 == seq_len) { // at_end - // copy initialize states gradient from inside to outside - LinkTensorWithCallback( - cur_scope, GradVarLists(Attr>(kExStates)), - scope, Outputs(kInitStateGrads), - [&](const framework::LoDTensor &inside, - framework::LoDTensor *outside) { - outside->Resize(inside.dims()); - outside->mutable_data(place, inside.type()); - framework::TensorCopy(inside, place, dev_ctx, outside); - }, - true /*is_backward*/); - VLOG(5) << "Link initialize state gradient finished "; + if (has_state) { + if (step_id + 1 == seq_len) { // at_end + // copy initialize states gradient from inside to outside + LinkTensorWithCallback( + cur_scope, + GradVarLists(Attr>(kExStates)), scope, + Outputs(kInitStateGrads), + [&](const framework::LoDTensor &inside, + framework::LoDTensor *outside) { + outside->Resize(inside.dims()); + outside->mutable_data(place, inside.type()); + framework::TensorCopy(inside, place, dev_ctx, outside); + }, + true /*is_backward*/); + VLOG(5) << "Link initialize state gradient finished "; + } } scopes.Next(); } + // Delete the scope of StepScopes + dev_ctx.Wait(); + auto *var = scope.FindVar(Input(kStepScopes)); + PADDLE_ENFORCE(var != nullptr); + auto step_scopes = var->GetMutable(); + for (auto *sub_scope : *step_scopes) { + const_cast(scope).DeleteScope(sub_scope); + } } private: @@ -541,6 +558,7 @@ class RecurrentOpProtoMaker : public framework::OpProtoAndCheckerMaker { .AsDuplicable(); AddOutput(kStepScopes, "StepScopes contain all local variables in each time step."); + AddAttr(kHasStates, "Whether has states.").SetDefault(false); AddAttr>(kExStates, string::Sprintf( R"DOC(The ex-state variable names. @@ -624,20 +642,44 @@ class RecurrentGradOpDescMaker : public framework::SingleGradOpDescMaker { class RecurrentGradOpShapeInference : public framework::InferShapeBase { public: void operator()(framework::InferShapeContext *ctx) const override { - std::vector input{kInputs, kInitialStates}; std::vector output{kOutputs}; - for (auto &s : input) { - // NOTE(zcd): In some case, some of kInputs doesn't have gradient. - PADDLE_ENFORCE(ctx->HasInputs(s)); - } - for (auto &s : output) { - PADDLE_ENFORCE(ctx->HasInputs(s)); + + // In some case the kInitialStates is empty. + // If the kInitialStates is empty, all the states should be empty. + if (!ctx->HasInputs(kInitialStates)) { + PADDLE_ENFORCE_EQ( + ctx->Attrs().Get>(kExStates).size(), 0, + "The Attr(%s) should be empty.", kExStates); + PADDLE_ENFORCE_EQ( + ctx->Attrs().Get>(kStates).size(), 0, + "The Attr(%s) should be empty.", kStates); } - for (auto &s : input) { - ctx->SetOutputsDim(framework::GradVarName(s), ctx->GetInputsDim(s)); + + PADDLE_ENFORCE(ctx->HasInputs(kInputs), + "The input(%s) should not be empty.", kInputs); + PADDLE_ENFORCE(ctx->HasInputs(kOutputs), + "The input(%s) should not be empty.", kOutputs); + + // In some case the kInitialStates is empty. + if (ctx->HasInputs(kInitialStates)) { + PADDLE_ENFORCE(ctx->HasOutputs(framework::GradVarName(kInitialStates)), + "The output of(%s) should not be empty.", + framework::GradVarName(kInitialStates)); + ctx->SetOutputsDim(framework::GradVarName(kInitialStates), + ctx->GetInputsDim(kInitialStates)); } + + PADDLE_ENFORCE(ctx->HasOutputs(framework::GradVarName(kInputs)), + "The output of(%s) should not be empty.", + framework::GradVarName(kInputs)); + ctx->SetOutputsDim(framework::GradVarName(kInputs), + ctx->GetInputsDim(kInputs)); + + // In some case the kParameters is empty. if (ctx->HasInputs(kParameters)) { - PADDLE_ENFORCE(ctx->HasOutputs(framework::GradVarName(kParameters))); + PADDLE_ENFORCE(ctx->HasOutputs(framework::GradVarName(kParameters)), + "The output of(%s) should not be empty.", + framework::GradVarName(kParameters)); ctx->SetOutputsDim(framework::GradVarName(kParameters), ctx->GetInputsDim(kParameters)); } diff --git a/paddle/fluid/operators/reduce_ops/reduce_all_op.cc b/paddle/fluid/operators/reduce_ops/reduce_all_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..b087fbbb94c7ba2f7449f6bda56010dee1c38ea6 --- /dev/null +++ b/paddle/fluid/operators/reduce_ops/reduce_all_op.cc @@ -0,0 +1,20 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/operators/reduce_ops/reduce_all_op.h" + +REGISTER_REDUCE_OP(reduce_all); +REGISTER_OP_CPU_KERNEL(reduce_all, + ops::ReduceKernel); diff --git a/paddle/fluid/operators/reduce_ops/reduce_all_op.cu b/paddle/fluid/operators/reduce_ops/reduce_all_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..bd94ba263d957d0d65506ecd802bf43add6e2fb4 --- /dev/null +++ b/paddle/fluid/operators/reduce_ops/reduce_all_op.cu @@ -0,0 +1,19 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/operators/reduce_ops/reduce_all_op.h" + +REGISTER_OP_CUDA_KERNEL(reduce_all, + ops::ReduceKernel); diff --git a/paddle/fluid/operators/reduce_ops/reduce_all_op.h b/paddle/fluid/operators/reduce_ops/reduce_all_op.h new file mode 100644 index 0000000000000000000000000000000000000000..ba159dd703c8904784546eda262bf7be77967d48 --- /dev/null +++ b/paddle/fluid/operators/reduce_ops/reduce_all_op.h @@ -0,0 +1,29 @@ +// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include "paddle/fluid/operators/reduce_ops/reduce_op.h" + +namespace paddle { +namespace operators { + +struct AllFunctor { + template + void operator()(const DeviceContext& place, X* x, Y* y, const Dim& dim) { + y->device(place) = x->all(dim); + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/fluid/operators/reduce_ops/reduce_any_op.cc b/paddle/fluid/operators/reduce_ops/reduce_any_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..d865dcb3c935b76b8da25d723a5f780fb4de255b --- /dev/null +++ b/paddle/fluid/operators/reduce_ops/reduce_any_op.cc @@ -0,0 +1,20 @@ +// Copyright (c) 2018 PaddlePaddle Authors. Any Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/operators/reduce_ops/reduce_any_op.h" + +REGISTER_REDUCE_OP(reduce_any); +REGISTER_OP_CPU_KERNEL(reduce_any, + ops::ReduceKernel); diff --git a/paddle/fluid/operators/reduce_ops/reduce_any_op.cu b/paddle/fluid/operators/reduce_ops/reduce_any_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..66f0c9997ea1e27cf172a6839a68d2eb23395c4d --- /dev/null +++ b/paddle/fluid/operators/reduce_ops/reduce_any_op.cu @@ -0,0 +1,19 @@ +// Copyright (c) 2018 PaddlePaddle Authors. Any Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/operators/reduce_ops/reduce_any_op.h" + +REGISTER_OP_CUDA_KERNEL(reduce_any, + ops::ReduceKernel); diff --git a/paddle/fluid/operators/reduce_ops/reduce_any_op.h b/paddle/fluid/operators/reduce_ops/reduce_any_op.h new file mode 100644 index 0000000000000000000000000000000000000000..b36bad9cada259932d2bd77c2426fbb46790de76 --- /dev/null +++ b/paddle/fluid/operators/reduce_ops/reduce_any_op.h @@ -0,0 +1,29 @@ +// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include "paddle/fluid/operators/reduce_ops/reduce_op.h" + +namespace paddle { +namespace operators { + +struct AnyFunctor { + template + void operator()(const DeviceContext& place, X* x, Y* y, const Dim& dim) { + y->device(place) = x->any(dim); + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/fluid/operators/rnn_memory_helper_op.cc b/paddle/fluid/operators/rnn_memory_helper_op.cc index 834dd1eabd68db6c8b571071f8043589c66f8671..b00cc07dea920a6d7caa8b70c99d84b72a785a99 100644 --- a/paddle/fluid/operators/rnn_memory_helper_op.cc +++ b/paddle/fluid/operators/rnn_memory_helper_op.cc @@ -40,9 +40,12 @@ class RNNMemoryHelperOp : public framework::OperatorBase { "Cannot find out_var in scope, out_var_name is %s", out_name); + platform::DeviceContextPool &pool = platform::DeviceContextPool::Instance(); + auto &dev_ctx = *pool.Get(dev_place); + auto *out_tensor = out_var->GetMutable(); auto &mem_tensor = mem_var->Get(); - framework::TensorCopySync(mem_tensor, dev_place, out_tensor); + framework::TensorCopy(mem_tensor, dev_place, dev_ctx, out_tensor); out_tensor->set_lod(mem_tensor.lod()); } }; @@ -92,6 +95,9 @@ class RNNMemoryHelperGradOp : public framework::OperatorBase { "Cannot find in_grad_var in scope, name is %s", in_grad_var_name); + platform::DeviceContextPool &pool = platform::DeviceContextPool::Instance(); + auto &dev_ctx = *pool.Get(dev_place); + if (out_grad_var == nullptr) { VLOG(5) << "Using fill constant 0 as starting gradient"; auto in_var_name = Input("X"); @@ -109,7 +115,8 @@ class RNNMemoryHelperGradOp : public framework::OperatorBase { } else { auto &out_grad_tensor = out_grad_var->Get(); auto *in_grad_tensor = in_grad_var->GetMutable(); - framework::TensorCopySync(out_grad_tensor, dev_place, in_grad_tensor); + framework::TensorCopy(out_grad_tensor, dev_place, dev_ctx, + in_grad_tensor); in_grad_tensor->set_lod(out_grad_tensor.lod()); } } diff --git a/paddle/fluid/operators/space_to_depth_op.cc b/paddle/fluid/operators/space_to_depth_op.cc index b579244673fa1618c282c4d4fedf2ba6d1726a82..a286fea3eff0f7ee5592707be697ef35ee93dffa 100644 --- a/paddle/fluid/operators/space_to_depth_op.cc +++ b/paddle/fluid/operators/space_to_depth_op.cc @@ -13,12 +13,18 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/fluid/operators/space_to_depth_op.h" + +#include #include #include +#include "paddle/fluid/framework/no_need_buffer_vars_inference.h" + namespace paddle { namespace operators { +using Tensor = framework::Tensor; + class SpaceToDepthOp : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; @@ -100,6 +106,28 @@ class SpaceToDepthOpMaker : public framework::OpProtoAndCheckerMaker { } }; +DECLARE_NO_NEED_BUFFER_VARS_INFERENCE(SpaceToDepthGradOpNoBuffer, "X"); + +class SpaceToDepthGradOpDescMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + std::unique_ptr Apply() const override { + std::unique_ptr op(new framework::OpDesc()); + + op->SetType("space_to_depth_grad"); + + op->SetInput(framework::GradVarName("Out"), OutputGrad("Out")); + op->SetInput("X", Input("X")); + + op->SetOutput(framework::GradVarName("X"), InputGrad("X")); + + op->SetAttrMap(Attrs()); + return op; + } +}; + class SpaceToDepthGradOp : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; @@ -110,6 +138,14 @@ class SpaceToDepthGradOp : public framework::OperatorWithKernel { "Input(Out@GRAD) shouldn't be null."); ctx->SetOutputDim(framework::GradVarName("X"), ctx->GetInputDim("X")); } + + protected: + framework::OpKernelType GetExpectedKernelType( + const framework::ExecutionContext& ctx) const override { + return framework::OpKernelType( + ctx.Input(framework::GradVarName("Out"))->type(), + ctx.GetPlace()); + } }; } // namespace operators } // namespace paddle @@ -117,8 +153,9 @@ class SpaceToDepthGradOp : public framework::OperatorWithKernel { namespace ops = paddle::operators; REGISTER_OPERATOR(space_to_depth, ops::SpaceToDepthOp, ops::SpaceToDepthOpMaker, - paddle::framework::DefaultGradOpDescMaker); -REGISTER_OPERATOR(space_to_depth_grad, ops::SpaceToDepthGradOp); + ops::SpaceToDepthGradOpDescMaker); +REGISTER_OPERATOR(space_to_depth_grad, ops::SpaceToDepthGradOp, + ops::SpaceToDepthGradOpNoBuffer); REGISTER_OP_CPU_KERNEL( space_to_depth, ops::SpaceToDepthKernel, diff --git a/paddle/fluid/operators/squared_l2_distance_op.cc b/paddle/fluid/operators/squared_l2_distance_op.cc index 42532a294b2ef9ffdb240fac8596278047daf7fe..0652c163f71709c66b2b9c1cedcbfd3ce9061bea 100644 --- a/paddle/fluid/operators/squared_l2_distance_op.cc +++ b/paddle/fluid/operators/squared_l2_distance_op.cc @@ -14,6 +14,10 @@ limitations under the License. */ #include "paddle/fluid/operators/squared_l2_distance_op.h" +#include + +#include "paddle/fluid/framework/no_need_buffer_vars_inference.h" + namespace paddle { namespace operators { @@ -54,6 +58,34 @@ class SquaredL2DistanceOp : public framework::OperatorWithKernel { } }; +DECLARE_NO_NEED_BUFFER_VARS_INFERENCE(SquaredL2DistanceGradOpNoBuffer, "X", + "Y"); + +class SquaredL2DistanceGradOpDescMaker + : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + std::unique_ptr Apply() const override { + std::unique_ptr op(new framework::OpDesc()); + + op->SetType("squared_l2_distance_grad"); + + op->SetInput(framework::GradVarName("Out"), OutputGrad("Out")); + op->SetInput("sub_result", Output("sub_result")); + op->SetInput("X", Input("X")); + op->SetInput("Y", Input("Y")); + + op->SetOutput(framework::GradVarName("X"), InputGrad("X")); + op->SetOutput(framework::GradVarName("Y"), InputGrad("Y")); + + op->SetAttrMap(Attrs()); + + return op; + } +}; + class SquaredL2DistanceOpMaker : public framework::OpProtoAndCheckerMaker { public: void Make() override { @@ -88,6 +120,7 @@ class SquaredL2DistanceGradOp : public framework::OperatorWithKernel { void InferShape(framework::InferShapeContext* ctx) const override { PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")), "Gradient of Out should not be null"); + PADDLE_ENFORCE(ctx->HasInput("sub_result"), "SubResult should not be null"); auto out_dims = ctx->GetInputDim(framework::GradVarName("Out")); auto x_dims = ctx->GetInputDim("X"); auto y_dims = ctx->GetInputDim("Y"); @@ -102,6 +135,13 @@ class SquaredL2DistanceGradOp : public framework::OperatorWithKernel { if (ctx->HasOutput(x_grad_name)) ctx->SetOutputDim(x_grad_name, x_dims); if (ctx->HasOutput(y_grad_name)) ctx->SetOutputDim(y_grad_name, y_dims); } + + protected: + framework::OpKernelType GetExpectedKernelType( + const framework::ExecutionContext& ctx) const override { + return framework::OpKernelType(ctx.Input("sub_result")->type(), + ctx.GetPlace()); + } }; } // namespace operators @@ -110,8 +150,9 @@ class SquaredL2DistanceGradOp : public framework::OperatorWithKernel { namespace ops = paddle::operators; REGISTER_OPERATOR(squared_l2_distance, ops::SquaredL2DistanceOp, ops::SquaredL2DistanceOpMaker, - paddle::framework::DefaultGradOpDescMaker); -REGISTER_OPERATOR(squared_l2_distance_grad, ops::SquaredL2DistanceGradOp); + ops::SquaredL2DistanceGradOpDescMaker); +REGISTER_OPERATOR(squared_l2_distance_grad, ops::SquaredL2DistanceGradOp, + ops::SquaredL2DistanceGradOpNoBuffer); REGISTER_OP_CPU_KERNEL( squared_l2_distance, ops::SquaredL2DistanceKernel); diff --git a/paddle/fluid/operators/squared_l2_distance_op.h b/paddle/fluid/operators/squared_l2_distance_op.h index e0133d33e6a840d2d06832393a064df978cb9cbc..12a8f05b5a603417ead8ebd250ff7951f928f4a1 100644 --- a/paddle/fluid/operators/squared_l2_distance_op.h +++ b/paddle/fluid/operators/squared_l2_distance_op.h @@ -77,6 +77,9 @@ class SquaredL2DistanceGradKernel : public framework::OpKernel { auto* x_g = context.Output(framework::GradVarName("X")); auto* y_g = context.Output(framework::GradVarName("Y")); + PADDLE_ENFORCE_NOT_NULL(x_g); + PADDLE_ENFORCE_NOT_NULL(y_g); + auto sub_result = EigenMatrix::From(*in0); auto out_grad = EigenMatrix::From(*in1); @@ -92,31 +95,28 @@ class SquaredL2DistanceGradKernel : public framework::OpKernel { // propagate back to input auto& eigen_place = *context.template device_context().eigen_device(); - if (x_g) { - x_g->mutable_data(context.GetPlace()); - // eigen matrix - auto x_grad = - EigenMatrix::From(*x_g, framework::make_ddim({x_dims[0], cols})); - // dimensions are same with subResult - x_grad.device(eigen_place) = grad_mat; - } - if (y_g) { - y_g->mutable_data(context.GetPlace()); - - PADDLE_ENFORCE_GE(sub_result.dimensions()[0], y_dims[0], - "First dimension of gradient must be greater or " - "equal than first dimension of target."); - - if (sub_result.dimensions()[0] == y_dims[0]) { - auto y_grad = - EigenMatrix::From(*y_g, framework::make_ddim({y_dims[0], cols})); - y_grad.device(eigen_place) = -1 * grad_mat; - } else { - auto col_sum_res = -1 * (grad_mat.sum(Eigen::array({{0}}))); - auto y_grad = EigenVector::Flatten(*y_g); - y_grad.device(eigen_place) = col_sum_res; - } + x_g->mutable_data(context.GetPlace()); + // eigen matrix + auto x_grad = + EigenMatrix::From(*x_g, framework::make_ddim({x_dims[0], cols})); + // dimensions are same with subResult + x_grad.device(eigen_place) = grad_mat; + + y_g->mutable_data(context.GetPlace()); + + PADDLE_ENFORCE_GE(sub_result.dimensions()[0], y_dims[0], + "First dimension of gradient must be greater or " + "equal than first dimension of target."); + + if (sub_result.dimensions()[0] == y_dims[0]) { + auto y_grad = + EigenMatrix::From(*y_g, framework::make_ddim({y_dims[0], cols})); + y_grad.device(eigen_place) = -1 * grad_mat; + } else { + auto col_sum_res = -1 * (grad_mat.sum(Eigen::array({{0}}))); + auto y_grad = EigenVector::Flatten(*y_g); + y_grad.device(eigen_place) = col_sum_res; } } }; diff --git a/paddle/fluid/operators/squared_l2_norm_op.cc b/paddle/fluid/operators/squared_l2_norm_op.cc index 7bd82e0ce4add6d4434e1defaee43da178a6f309..9d2deb678ecf714421f507af88e7eabade7ecb68 100644 --- a/paddle/fluid/operators/squared_l2_norm_op.cc +++ b/paddle/fluid/operators/squared_l2_norm_op.cc @@ -14,6 +14,8 @@ limitations under the License. */ #include "paddle/fluid/operators/squared_l2_norm_op.h" +#include + namespace paddle { namespace operators { @@ -31,6 +33,26 @@ class SquaredL2NormOp : public framework::OperatorWithKernel { } }; +class SquaredL2NormGradOpDescMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + std::unique_ptr Apply() const override { + std::unique_ptr op(new framework::OpDesc()); + + op->SetType("squared_l2_norm_grad"); + + op->SetInput(framework::GradVarName("Out"), OutputGrad("Out")); + op->SetInput("X", Input("X")); + + op->SetOutput(framework::GradVarName("X"), InputGrad("X")); + + op->SetAttrMap(Attrs()); + return op; + } +}; + class SquaredL2NormGradOp : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; @@ -67,8 +89,7 @@ $$Out = \sum_{i} X_{i}^2$$ namespace ops = paddle::operators; REGISTER_OPERATOR(squared_l2_norm, ops::SquaredL2NormOp, - ops::SquaredL2NormOpMaker, - paddle::framework::DefaultGradOpDescMaker); + ops::SquaredL2NormOpMaker, ops::SquaredL2NormGradOpDescMaker); REGISTER_OPERATOR(squared_l2_norm_grad, ops::SquaredL2NormGradOp); REGISTER_OP_CPU_KERNEL( squared_l2_norm, diff --git a/paddle/fluid/operators/teacher_student_sigmoid_loss_op.cc b/paddle/fluid/operators/teacher_student_sigmoid_loss_op.cc index 640644a94690d9682a5e6b1aa788a9ebdc5d2a54..6a4bea94376bb66fcabc1fa9872f9dc9b6febac2 100644 --- a/paddle/fluid/operators/teacher_student_sigmoid_loss_op.cc +++ b/paddle/fluid/operators/teacher_student_sigmoid_loss_op.cc @@ -13,6 +13,9 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/fluid/operators/teacher_student_sigmoid_loss_op.h" + +#include + #include "paddle/fluid/operators/math/math_function.h" namespace paddle { @@ -55,6 +58,28 @@ class TeacherStudentSigmoidLossOp : public framework::OperatorWithKernel { } }; +class TeacherStudentSigmoidLossGradOpDescMaker + : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + std::unique_ptr Apply() const override { + std::unique_ptr op(new framework::OpDesc()); + + op->SetType("teacher_student_sigmoid_loss_grad"); + + op->SetInput("X", Input("X")); + op->SetInput("Label", Input("Label")); + op->SetInput(framework::GradVarName("Y"), OutputGrad("Y")); + + op->SetOutput(framework::GradVarName("X"), InputGrad("X")); + + op->SetAttrMap(Attrs()); + return op; + } +}; + class TeacherStudentSigmoidLossGradientOp : public framework::OperatorWithKernel { public: @@ -148,7 +173,7 @@ namespace ops = paddle::operators; REGISTER_OPERATOR(teacher_student_sigmoid_loss, ops::TeacherStudentSigmoidLossOp, ops::TeacherStudentSigmoidLossOpMaker, - paddle::framework::DefaultGradOpDescMaker); + ops::TeacherStudentSigmoidLossGradOpDescMaker); REGISTER_OPERATOR(teacher_student_sigmoid_loss_grad, ops::TeacherStudentSigmoidLossGradientOp); diff --git a/paddle/fluid/operators/tree_conv_op.cc b/paddle/fluid/operators/tree_conv_op.cc index 615ea285e54b97a8fb81acfef9bf0d18ac4e914d..159e59494648d6107dc4854089f27c42ab369b4a 100644 --- a/paddle/fluid/operators/tree_conv_op.cc +++ b/paddle/fluid/operators/tree_conv_op.cc @@ -13,6 +13,8 @@ // limitations under the License. #include "paddle/fluid/operators/tree_conv_op.h" + +#include #include namespace paddle { @@ -86,6 +88,30 @@ class TreeConvOp : public framework::OperatorWithKernel { } }; +class TreeConvGradOpDescMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + std::unique_ptr Apply() const override { + std::unique_ptr op(new framework::OpDesc()); + + op->SetType("tree_conv_grad"); + + op->SetInput(framework::GradVarName("Out"), OutputGrad("Out")); + op->SetInput("Filter", Input("Filter")); + op->SetInput("EdgeSet", Input("EdgeSet")); + op->SetInput("NodesVector", Input("NodesVector")); + + op->SetOutput(framework::GradVarName("NodesVector"), + InputGrad("NodesVector")); + op->SetOutput(framework::GradVarName("Filter"), InputGrad("Filter")); + + op->SetAttrMap(Attrs()); + return op; + } +}; + class TreeConvGradOp : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; @@ -115,7 +141,7 @@ class TreeConvGradOp : public framework::OperatorWithKernel { namespace ops = paddle::operators; REGISTER_OPERATOR(tree_conv, ops::TreeConvOp, ops::TreeConvOpMaker, - paddle::framework::DefaultGradOpDescMaker); + ops::TreeConvGradOpDescMaker); REGISTER_OPERATOR(tree_conv_grad, ops::TreeConvGradOp); diff --git a/paddle/fluid/operators/warpctc_op.cc b/paddle/fluid/operators/warpctc_op.cc index e2ae7caae1ebe46b30c811ae4537f718ca587939..217d400bb3c20b4b9e6117074cebbb35161017fd 100644 --- a/paddle/fluid/operators/warpctc_op.cc +++ b/paddle/fluid/operators/warpctc_op.cc @@ -14,6 +14,8 @@ limitations under the License. */ #include "paddle/fluid/operators/warpctc_op.h" +#include + #ifdef PADDLE_WITH_CUDA #include "paddle/fluid/platform/cudnn_helper.h" #endif @@ -118,6 +120,27 @@ http://machinelearning.wustl.edu/mlpapers/paper_files/icml2006_GravesFGS06.pdf). } }; +class WarpCTCGradOpDescMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + std::unique_ptr Apply() const override { + std::unique_ptr op(new framework::OpDesc()); + + op->SetType("warpctc_grad"); + + op->SetInput("WarpCTCGrad", Output("WarpCTCGrad")); + op->SetInput("Logits", Input("Logits")); + op->SetInput(framework::GradVarName("Loss"), OutputGrad("Loss")); + + op->SetOutput(framework::GradVarName("Logits"), InputGrad("Logits")); + + op->SetAttrMap(Attrs()); + return op; + } +}; + class WarpCTCGradOp : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; @@ -145,7 +168,7 @@ class WarpCTCGradOp : public framework::OperatorWithKernel { namespace ops = paddle::operators; REGISTER_OPERATOR(warpctc, ops::WarpCTCOp, ops::WarpCTCOpMaker, - paddle::framework::DefaultGradOpDescMaker); + ops::WarpCTCGradOpDescMaker); REGISTER_OPERATOR(warpctc_grad, ops::WarpCTCGradOp); REGISTER_OP_CPU_KERNEL( warpctc, ops::WarpCTCKernel); diff --git a/paddle/fluid/platform/dynload/CMakeLists.txt b/paddle/fluid/platform/dynload/CMakeLists.txt index 1697343790d13c37d63505acfe471b379bf897d9..07159d4a12ef4b628f7705ed206d3334be46dfc8 100644 --- a/paddle/fluid/platform/dynload/CMakeLists.txt +++ b/paddle/fluid/platform/dynload/CMakeLists.txt @@ -17,9 +17,6 @@ if (CUPTI_FOUND) endif(CUPTI_FOUND) nv_library(dynload_cuda SRCS ${CUDA_SRCS} DEPS dynamic_loader) cc_library(dynload_warpctc SRCS warpctc.cc DEPS dynamic_loader warpctc) -if (WITH_WBAES) - cc_library(dynload_wbaes SRCS wbaes.cc DEPS dynamic_loader wbaes) -endif() if (WITH_MKLML) cc_library(dynload_mklml SRCS mklml.cc DEPS dynamic_loader mklml) endif() diff --git a/paddle/fluid/platform/dynload/dynamic_loader.cc b/paddle/fluid/platform/dynload/dynamic_loader.cc index 8ac9393787324d3a8a17ac5a800bcf69638a4fed..15d516836652ea4ea4d1bcdf35022e6b79cc3b52 100644 --- a/paddle/fluid/platform/dynload/dynamic_loader.cc +++ b/paddle/fluid/platform/dynload/dynamic_loader.cc @@ -48,8 +48,6 @@ DEFINE_string( DEFINE_string(mklml_dir, "", "Specify path for loading libmklml_intel.so."); -DEFINE_string(wbaes_dir, "", "Specify path for loading libwbaes.so."); - namespace paddle { namespace platform { namespace dynload { @@ -248,16 +246,6 @@ void* GetMKLMLDsoHandle() { #endif } -void* GetWBAESDsoHandle() { -#if defined(__APPLE__) || defined(__OSX__) - return GetDsoHandleFromSearchPath(FLAGS_wbaes_dir, "libwbaes.dylib"); -#elif defined(_WIN32) - return GetDsoHandleFromSearchPath(FLAGS_wbaes_dir, "libwbaes.dll"); -#else - return GetDsoHandleFromSearchPath(FLAGS_wbaes_dir, "libwbaes.so"); -#endif -} - } // namespace dynload } // namespace platform } // namespace paddle diff --git a/paddle/fluid/platform/dynload/dynamic_loader.h b/paddle/fluid/platform/dynload/dynamic_loader.h index 5a642967c7666f5d5943214f557786c87491d740..edb4c649addfaf941a00588395d9191038217979 100644 --- a/paddle/fluid/platform/dynload/dynamic_loader.h +++ b/paddle/fluid/platform/dynload/dynamic_loader.h @@ -32,7 +32,6 @@ void* GetWarpCTCDsoHandle(); void* GetNCCLDsoHandle(); void* GetTensorRtDsoHandle(); void* GetMKLMLDsoHandle(); -void* GetWBAESDsoHandle(); } // namespace dynload } // namespace platform diff --git a/paddle/fluid/platform/dynload/wbaes.h b/paddle/fluid/platform/dynload/wbaes.h deleted file mode 100644 index 22400d44e4ca5568f1d74e4e194e45e81cbdfefe..0000000000000000000000000000000000000000 --- a/paddle/fluid/platform/dynload/wbaes.h +++ /dev/null @@ -1,63 +0,0 @@ -/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ -#pragma once - -#ifdef PADDLE_WITH_WBAES - -#include -#include // NOLINT - -#include "paddle/fluid/platform/dynload/dynamic_loader.h" -#include "paddle/fluid/platform/port.h" - -namespace paddle { -namespace platform { -namespace dynload { - -extern std::once_flag wbaes_dso_flag; -extern void *wbaes_dso_handle; - -/** - * The following macro definition can generate structs - * (for each function) to dynamic load wbaes routine - * via operator overloading. - */ - -#define DYNAMIC_LOAD_WBAES_WRAP(__name) \ - struct DynLoad__##__name { \ - template \ - auto operator()(Args... args) -> DECLARE_TYPE(__name, args...) { \ - using wbaesFunc = decltype(&::__name); \ - std::call_once(wbaes_dso_flag, []() { \ - wbaes_dso_handle = paddle::platform::dynload::GetWBAESDsoHandle(); \ - }); \ - static void *p_##__name = dlsym(wbaes_dso_handle, #__name); \ - return reinterpret_cast(p_##__name)(args...); \ - } \ - }; \ - extern DynLoad__##__name __name - -#define DECLARE_DYNAMIC_LOAD_WBAES_WRAP(__name) DYNAMIC_LOAD_WBAES_WRAP(__name) - -#define WBAES_ROUTINE_EACH(__macro) __macro(GSECF); - -WBAES_ROUTINE_EACH(DECLARE_DYNAMIC_LOAD_WBAES_WRAP); - -#undef DYNAMIC_LOAD_WBAES_WRAP - -} // namespace dynload -} // namespace platform -} // namespace paddle - -#endif diff --git a/paddle/fluid/platform/lodtensor_printer.cc b/paddle/fluid/platform/lodtensor_printer.cc index a5aa1a4148686b032c52f99497252fde4867438f..07eaf42d2d3bc20e7f7dc56bb0f4e0cc2fbac5e3 100644 --- a/paddle/fluid/platform/lodtensor_printer.cc +++ b/paddle/fluid/platform/lodtensor_printer.cc @@ -52,16 +52,26 @@ void PrintVar(framework::Scope* scope, const std::string& var_name, return; } -#define PrintLoDTensorCallback(cpp_type, proto_type) \ - do { \ - if (tensor->type() == proto_type) { \ - print_lod_tensor(var_name, *tensor, print_info); \ - return; \ - } \ + framework::LoDTensor printed_tensor; + printed_tensor.set_lod(tensor->lod()); + printed_tensor.Resize(tensor->dims()); + if (platform::is_cpu_place(tensor->place())) { + printed_tensor.ShareDataWith(*tensor); + } else { + platform::CPUPlace place; + framework::TensorCopy(*tensor, place, &printed_tensor); + } + +#define PrintLoDTensorCallback(cpp_type, proto_type) \ + do { \ + if (tensor->type() == proto_type) { \ + print_lod_tensor(var_name, printed_tensor, print_info); \ + return; \ + } \ } while (0) _ForEachDataType_(PrintLoDTensorCallback); - VLOG(1) << "PrintVar: unrecognized data type:" << tensor->type(); + VLOG(1) << "PrintVar: unrecognized data type:" << printed_tensor.type(); } } // end namespace platform diff --git a/paddle/fluid/pybind/CMakeLists.txt b/paddle/fluid/pybind/CMakeLists.txt index c8a0aa58859cca06375ce578e5a7097179e23107..16365c1fd0b0adb914cdfd08e3f6542fca952e06 100644 --- a/paddle/fluid/pybind/CMakeLists.txt +++ b/paddle/fluid/pybind/CMakeLists.txt @@ -1,6 +1,6 @@ set(PYBIND_DEPS pybind python proto_desc memory executor async_executor fleet_wrapper prune feed_fetch_method pass_builder parallel_executor profiler layer scope_pool - tracer analysis_predictor imperative_profiler) + tracer analysis_predictor imperative_profiler nccl_context) if(WITH_PYTHON) list(APPEND PYBIND_DEPS py_func_op) diff --git a/paddle/fluid/pybind/imperative.cc b/paddle/fluid/pybind/imperative.cc index e9ed4e16443eba481143bd2095f9970bcb167d71..265707f1bccdabd37b9a7248755d0b81339418c3 100644 --- a/paddle/fluid/pybind/imperative.cc +++ b/paddle/fluid/pybind/imperative.cc @@ -29,7 +29,7 @@ namespace paddle { namespace pybind { // Bind Methods -void BindTracer(pybind11::module* m) { +void BindImperative(pybind11::module* m) { pybind11::class_(*m, "Tracer", "") .def("__init__", [](imperative::Tracer& self, framework::BlockDesc* root_block) { @@ -59,6 +59,47 @@ void BindTracer(pybind11::module* m) { }) .def("py_trace", &imperative::Tracer::PyTrace, pybind11::return_value_policy::take_ownership); + + // define parallel context + pybind11::class_ parallel_strategy( + *m, "ParallelStrategy", ""); + parallel_strategy.def(pybind11::init()) + .def_property( + "nranks", + [](const imperative::ParallelStrategy& self) { return self.nranks_; }, + [](imperative::ParallelStrategy& self, int nranks) { + self.nranks_ = nranks; + }) + .def_property("local_rank", + [](const imperative::ParallelStrategy& self) { + return self.local_rank_; + }, + [](imperative::ParallelStrategy& self, int local_rank) { + self.local_rank_ = local_rank; + }) + .def_property( + "trainer_endpoints", + [](const imperative::ParallelStrategy& self) { + return self.trainer_endpoints_; + }, + [](imperative::ParallelStrategy& self, std::vector eps) { + self.trainer_endpoints_ = eps; + }) + .def_property("current_endpoint", + [](const imperative::ParallelStrategy& self) { + return self.current_endpoint_; + }, + [](imperative::ParallelStrategy& self, + const std::string& ep) { self.current_endpoint_ = ep; }); +#if defined(PADDLE_WITH_CUDA) && !defined(_WIN32) + pybind11::class_ nccl_ctx( + *m, "NCCLParallelContext"); + + nccl_ctx + .def(pybind11::init()) + .def("init", [](imperative::NCCLParallelContext& self) { self.Init(); }); +#endif } } // namespace pybind diff --git a/paddle/fluid/pybind/imperative.h b/paddle/fluid/pybind/imperative.h index 8496cbfcb18798ee8ce1714431b7877bb2b7d377..f9d4a7c990e23b30eb7f5086fe56587f7c38bd22 100644 --- a/paddle/fluid/pybind/imperative.h +++ b/paddle/fluid/pybind/imperative.h @@ -17,6 +17,7 @@ limitations under the License. */ #include #include #include "paddle/fluid/imperative/layer.h" +#include "paddle/fluid/imperative/nccl_context.h" #include "pybind11/pybind11.h" #include "pybind11/stl.h" @@ -46,7 +47,7 @@ class PyVarBase : public imperative::VarBase { using imperative::VarBase::VarBase; // Inherit constructors }; -void BindTracer(pybind11::module* m); +void BindImperative(pybind11::module* m); } // namespace pybind } // namespace paddle diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index 044677fb756e0368c65b84f15fdf2540abbd14b8..a8a2a94d473b18fdcd78771063ef4565c7fe0e42 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -21,6 +21,7 @@ limitations under the License. */ #include #include +#include "paddle/fluid/framework/details/alloc_continuous_space_for_grad_pass.h" #include "paddle/fluid/framework/executor.h" #include "paddle/fluid/framework/feed_fetch_method.h" #include "paddle/fluid/framework/framework.pb.h" @@ -165,6 +166,11 @@ PYBIND11_MODULE(core, m) { // to enable eager deletion mode in unittest. m.def("_set_eager_deletion_mode", &paddle::framework::SetEagerDeletionMode); + m.def("_set_fuse_parameter_group_size", + &paddle::framework::details::SetFuseParameterGroupsSize); + m.def("_set_fuse_parameter_memory_size", + &paddle::framework::details::SetFuseParameterMemorySize); + m.add_object("_cleanup", py::capsule([]() { ScopePool::Instance().Clear(); })); @@ -288,7 +294,7 @@ PYBIND11_MODULE(core, m) { }) .def_static("num_funcs", &imperative::PyLayer::NumFuncs); - BindTracer(&m); + BindImperative(&m); py::class_(m, "Tensor", py::buffer_protocol()) .def_buffer( @@ -1356,6 +1362,14 @@ All parameter, weight, gradient are variables in Paddle. "fuse_all_reduce_ops", [](const BuildStrategy &self) { return self.fuse_all_reduce_ops_; }, [](BuildStrategy &self, bool b) { self.fuse_all_reduce_ops_ = b; }) + .def_property( + "cache_runtime_context", + [](const BuildStrategy &self) { return self.cache_runtime_context_; }, + [](BuildStrategy &self, bool b) { self.cache_runtime_context_ = b; }) + .def_property( + "cache_expected_kernel", + [](const BuildStrategy &self) { return self.cache_expected_kernel_; }, + [](BuildStrategy &self, bool b) { self.cache_expected_kernel_ = b; }) .def("_finalize_strategy_and_create_passes", [](BuildStrategy &self) -> std::shared_ptr { return self.CreatePassesFromStrategy(true); diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index fc52c281c4f0de2b05ab2b58aa81cdbf1216e6a7..7bb713493182239b2fd17f7b7fb496afdc9b8e6c 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -202,6 +202,7 @@ function cmake_gen() { -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DWITH_CONTRIB=${WITH_CONTRIB:-ON} -DWITH_INFERENCE_API_TEST=${WITH_INFERENCE_API_TEST:-ON} + -DWITH_HIGH_LEVEL_API_TEST=${WITH_HIGH_LEVEL_API_TEST:-OFF} -DINFERENCE_DEMO_INSTALL_DIR=${INFERENCE_DEMO_INSTALL_DIR} -DWITH_ANAKIN=${WITH_ANAKIN:-OFF} -DANAKIN_BUILD_FAT_BIN=${ANAKIN_BUILD_FAT_BIN:OFF} @@ -234,6 +235,7 @@ EOF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ -DWITH_CONTRIB=${WITH_CONTRIB:-ON} \ -DWITH_INFERENCE_API_TEST=${WITH_INFERENCE_API_TEST:-ON} \ + -DWITH_HIGH_LEVEL_API_TEST=${WITH_HIGH_LEVEL_API_TEST:-OFF} \ -DINFERENCE_DEMO_INSTALL_DIR=${INFERENCE_DEMO_INSTALL_DIR} \ -DWITH_ANAKIN=${WITH_ANAKIN:-OFF} \ -DANAKIN_BUILD_FAT_BIN=${ANAKIN_BUILD_FAT_BIN:OFF}\ @@ -291,8 +293,12 @@ function build() { Building in /paddle/build ... ============================================ EOF + parallel_number=`nproc` + if [[ "$1" != "" ]]; then + parallel_number=$1 + fi make clean - make -j `nproc` + make -j ${parallel_number} make install -j `nproc` } @@ -737,9 +743,13 @@ function gen_fluid_lib() { Generating fluid library for train and inference ... ======================================== EOF + parallel_number=`nproc` + if [[ "$1" != "" ]]; then + parallel_number=$1 + fi cmake .. -DWITH_DISTRIBUTE=OFF -DON_INFER=ON - make -j `nproc` fluid_lib_dist - make -j `nproc` inference_lib_dist + make -j ${parallel_number} fluid_lib_dist + make -j ${parallel_number} inference_lib_dist } function tar_fluid_lib() { @@ -770,11 +780,22 @@ EOF function main() { local CMD=$1 + local parallel_number=$2 init case $CMD in + build_only) + cmake_gen ${PYTHON_ABI:-""} + build ${parallel_number} + ;; + build_and_check) + cmake_gen ${PYTHON_ABI:-""} + build ${parallel_number} + assert_api_not_changed ${PYTHON_ABI:-""} + assert_api_spec_approvals + ;; build) cmake_gen ${PYTHON_ABI:-""} - build + build ${parallel_number} gen_dockerfile ${PYTHON_ABI:-""} ;; test) @@ -797,7 +818,7 @@ function main() { ;; fluid_inference_lib) cmake_gen ${PYTHON_ABI:-""} - gen_fluid_lib + gen_fluid_lib ${parallel_number} tar_fluid_lib test_fluid_lib ;; @@ -806,16 +827,16 @@ function main() { ;; cicheck) cmake_gen ${PYTHON_ABI:-""} - build + build ${parallel_number} assert_api_not_changed ${PYTHON_ABI:-""} run_test - gen_fluid_lib + gen_fluid_lib ${parallel_number} test_fluid_lib assert_api_spec_approvals ;; cicheck_brpc) cmake_gen ${PYTHON_ABI:-""} - build + build ${parallel_number} run_brpc_test ;; assert_api) @@ -823,7 +844,7 @@ function main() { assert_api_spec_approvals ;; test_inference) - gen_fluid_lib + gen_fluid_lib ${parallel_number} test_fluid_lib ;; assert_api_approvals) @@ -840,7 +861,7 @@ function main() { ;; cicheck_py35) cmake_gen ${PYTHON_ABI:-""} - build + build ${parallel_number} run_test assert_api_not_changed ${PYTHON_ABI:-""} ;; @@ -848,7 +869,7 @@ function main() { cmake_gen ${PYTHON_ABI:-""} ;; gen_fluid_lib) - gen_fluid_lib + gen_fluid_lib ${parallel_number} ;; test_fluid_lib) test_fluid_lib diff --git a/python/paddle/distributed/launch.py b/python/paddle/distributed/launch.py index 03c4078775d455fdb19aaf78ace4dcb98c8dd66a..d8153fa00267b00eedc52aa043af9ba7dc090f7d 100644 --- a/python/paddle/distributed/launch.py +++ b/python/paddle/distributed/launch.py @@ -32,6 +32,7 @@ default_envs = { "NCCL_SOCKET_IFNAME": "eth0", "NCCL_IB_GID_INDEX": "3", "NCCL_IB_RETRY_CNT": "0", + "PYTHONPATH": os.getenv("PYTHONPATH", ""), } GPUS = 8 diff --git a/python/paddle/fluid/__init__.py b/python/paddle/fluid/__init__.py index 0af883764e157db24e17a1a4ef1bff27f9b39b0f..983d8243b1d8aa6c8d01855d6dbeab76c335f70c 100644 --- a/python/paddle/fluid/__init__.py +++ b/python/paddle/fluid/__init__.py @@ -66,6 +66,8 @@ from . import compiler from .compiler import * from paddle.fluid.layers.math_op_patch import monkey_patch_variable from . import install_check +from .dygraph.nn import * +from .dygraph.layers import * Tensor = LoDTensor diff --git a/python/paddle/fluid/dataset.py b/python/paddle/fluid/dataset.py index 9e899a2a6cf217c1ba0848ad8648581530b449cf..d539940133881d5aa632eae6e3975ae57a385ebf 100644 --- a/python/paddle/fluid/dataset.py +++ b/python/paddle/fluid/dataset.py @@ -136,6 +136,7 @@ class DatasetBase(object): slot_var.name = var.name if var.lod_level == 0: slot_var.is_dense = True + slot_var.shape.extend(var.shape) if var.dtype == core.VarDesc.VarType.FP32: slot_var.type = "float" elif var.dtype == core.VarDesc.VarType.INT64: diff --git a/python/paddle/fluid/device_worker.py b/python/paddle/fluid/device_worker.py index 7fc72191884020f4cc57c9269b636161635f06d0..0998f779acfea23f3a494a25b43a6fa824b985f1 100644 --- a/python/paddle/fluid/device_worker.py +++ b/python/paddle/fluid/device_worker.py @@ -26,8 +26,8 @@ class DeviceWorker(object): """ Init. """ - self.program_ = None - self.infer_ = None + self._program = None + self._infer = None def _set_infer(self, infer=False): """ @@ -36,7 +36,7 @@ class DeviceWorker(object): Args: infer(bool): whether to do inference """ - self.infer_ = infer + self._infer = infer def _set_fleet_desc(self, fleet_desc): """ @@ -45,7 +45,7 @@ class DeviceWorker(object): Args: fleet_desc(PSParameter): pslib.PSParameter object """ - self.fleet_desc_ = fleet_desc + self._fleet_desc = fleet_desc def _set_program(self, program): """ @@ -54,7 +54,7 @@ class DeviceWorker(object): Args: program(Program): a Program object """ - self.program_ = program + self._program = program def _gen_worker_desc(self, trainer_desc): """ @@ -88,7 +88,7 @@ class Hogwild(DeviceWorker): trainer_desc(TrainerDesc): a TrainerDesc object """ trainer_desc.device_worker_name = "HogwildWorker" - if self.infer_: + if self._infer: # just ignore feed op for inference model trainer_desc.hogwild_param.skip_ops.extend(["feed"]) @@ -113,11 +113,11 @@ class DownpourSGD(DeviceWorker): trainer_desc(TrainerDesc): a TrainerDesc object """ dense_table_set = set() - program_id = str(id(self.program_)) - if self.program_ == None: + program_id = str(id(self._program)) + if self._program == None: print("program of current device worker is not configured") exit(-1) - opt_info = self.program_._fleet_opt + opt_info = self._program._fleet_opt program_configs = opt_info["program_configs"] downpour = trainer_desc.downpour_param @@ -140,7 +140,7 @@ class DownpourSGD(DeviceWorker): trainer_desc.device_worker_name = "DownpourWorker" pull_thread = trainer_desc.pull_dense_param pull_thread.device_num = trainer_desc.thread_num - for i in self.fleet_desc_.trainer_param.dense_table: + for i in self._fleet_desc.trainer_param.dense_table: if i.table_id in dense_table_set: dense_table = pull_thread.dense_table.add() dense_table.dense_value_name.extend(i.dense_variable_name) @@ -148,29 +148,29 @@ class DownpourSGD(DeviceWorker): i.table_id sparse_table = downpour.sparse_table.add() sparse_table.table_id = \ - self.fleet_desc_.trainer_param.sparse_table[0].table_id + self._fleet_desc.trainer_param.sparse_table[0].table_id sparse_table.sparse_key_name.extend( - self.fleet_desc_.trainer_param.sparse_table[0].slot_key) + self._fleet_desc.trainer_param.sparse_table[0].slot_key) sparse_table.sparse_value_name.extend( - self.fleet_desc_.trainer_param.sparse_table[0].slot_value) + self._fleet_desc.trainer_param.sparse_table[0].slot_value) sparse_table.sparse_grad_name.extend( - self.fleet_desc_.trainer_param.sparse_table[0].slot_gradient) + self._fleet_desc.trainer_param.sparse_table[0].slot_gradient) sparse_table.emb_dim = \ - self.fleet_desc_.server_param.downpour_server_param.downpour_table_param[ + self._fleet_desc.server_param.downpour_server_param.downpour_table_param[ 0].accessor.fea_dim - 2 sparse_table.fea_dim = sparse_table.emb_dim + 2 # TODO(guru4elephant): hard code here, need to improve sparse_table.label_var_name = "click" - for i in self.fleet_desc_.trainer_param.dense_table: + for i in self._fleet_desc.trainer_param.dense_table: if i.table_id in dense_table_set: dense_table = downpour.dense_table.add() dense_table.table_id = i.table_id dense_table.dense_value_name.extend(i.dense_variable_name) dense_table.dense_grad_name.extend( i.dense_gradient_variable_name) - downpour.skip_ops.extend(self.fleet_desc_.trainer_param.skip_op) - if self.infer_: + downpour.skip_ops.extend(self._fleet_desc.trainer_param.skip_op) + if self._infer: downpour.push_dense = False downpour.push_sparse = False diff --git a/python/paddle/fluid/dygraph/__init__.py b/python/paddle/fluid/dygraph/__init__.py index 2d0c7b7ddaacee28da599d5850e9b3381c01de5c..9bb72ede304dbde732153bac980f24a74bcd126d 100644 --- a/python/paddle/fluid/dygraph/__init__.py +++ b/python/paddle/fluid/dygraph/__init__.py @@ -29,6 +29,9 @@ from .tracer import * from . import profiler from .profiler import * +from . import parallel +from .parallel import * + from . import checkpoint from .checkpoint import * @@ -41,5 +44,6 @@ __all__ += base.__all__ __all__ += nn.__all__ __all__ += tracer.__all__ __all__ += profiler.__all__ +__all__ += parallel.__all__ __all__ += checkpoint.__all__ __all__ += learning_rate_scheduler.__all__ diff --git a/python/paddle/fluid/dygraph/base.py b/python/paddle/fluid/dygraph/base.py index d55dbbb9c72cb887e169849c3a3e32a13c202a7b..bf484b35c7bf9a2b17126789ff247bd73095fe7b 100644 --- a/python/paddle/fluid/dygraph/base.py +++ b/python/paddle/fluid/dygraph/base.py @@ -22,7 +22,7 @@ __all__ = ['enabled', 'guard', 'to_variable'] def enabled(): - return framework._in_dygraph_mode() + return framework.in_dygraph_mode() @signature_safe_contextmanager diff --git a/python/paddle/fluid/dygraph/checkpoint.py b/python/paddle/fluid/dygraph/checkpoint.py index f992ae0576c81ed98a3e9f7a446b0c2e808622ea..f2b01aece7bf86b1a195296ba49a626721213b7a 100644 --- a/python/paddle/fluid/dygraph/checkpoint.py +++ b/python/paddle/fluid/dygraph/checkpoint.py @@ -97,20 +97,12 @@ def load_persistables(vardict, dirname, filename=None): Examples: .. code-block:: python - my_layer = layer(fluid.dygraph.Layer) + my_layer = layer(fluid.Layer) param_path = "./my_paddle_model" param_dict = fluid.dygraph.load_persistables(my_layer.parameters(), param_path) param_1 = param_dict['PtbModel_0.w_1'] - or: - my_layer = layer(fluid.dygraph.Layer) - param_path = "./my_paddle_model" - filename = "model.file" - param_dict = fluid.dygraph.load_persistables(my_layer.state_dict(), param_path, - filename=filename) - param_1 = param_dict['PtbModel_0.w_1'] - """ if isinstance(vardict, collections.OrderedDict): return _load_var_from_file(vardict, dirname, filename) diff --git a/python/paddle/fluid/dygraph/layer_object_helper.py b/python/paddle/fluid/dygraph/layer_object_helper.py index f0be5ff3bf2394f1f7da8fbcc341a0d2dfacdab3..9fd1e392791f2bf7a19942749eae87001ec3ede8 100644 --- a/python/paddle/fluid/dygraph/layer_object_helper.py +++ b/python/paddle/fluid/dygraph/layer_object_helper.py @@ -16,7 +16,7 @@ from __future__ import print_function import copy import six -from ..framework import Parameter, _in_dygraph_mode +from ..framework import Parameter, in_dygraph_mode from ..param_attr import ParamAttr from .. import core from six.moves import zip diff --git a/python/paddle/fluid/dygraph/layers.py b/python/paddle/fluid/dygraph/layers.py index 014ee41f4c5aa280fb5b366d8f1704290cc067d4..39e06e3486cd5479f69cbdb67811f03bd9646123 100644 --- a/python/paddle/fluid/dygraph/layers.py +++ b/python/paddle/fluid/dygraph/layers.py @@ -139,14 +139,14 @@ class Layer(core.Layer): def clear_gradients(self): for p in self.parameters(): - p._clear_gradient() + p.clear_gradient() - def _build_once(self, *args): + def build_once(self, *args): pass def __call__(self, *inputs): if not self._built: - self._build_once(*inputs) + self.build_once(*inputs) outputs = self.forward(*inputs) self._built = True diff --git a/python/paddle/fluid/dygraph/nn.py b/python/paddle/fluid/dygraph/nn.py index 04da8561a370056a40b374887ef08a4c2110e6cc..0ab981518beb4cc48e18c17e4f0f91c22b60dbb7 100644 --- a/python/paddle/fluid/dygraph/nn.py +++ b/python/paddle/fluid/dygraph/nn.py @@ -15,23 +15,127 @@ from __future__ import print_function from six.moves import reduce -import numpy as np from .. import core from ..layers import utils from . import layers -from ..framework import Variable, OpProtoHolder, Parameter -from ..layers import layer_function_generator +from ..framework import Variable, in_dygraph_mode, OpProtoHolder, Parameter from ..param_attr import ParamAttr from ..initializer import Normal, Constant, NumpyArrayInitializer +import numpy as np __all__ = [ - 'Conv2D', 'Pool2D', 'FC', 'BatchNorm', 'Embedding', 'GRUUnit', 'LayerNorm', - 'NCE', 'PRelu', 'BilinearTensorProduct', 'Conv2DTranspose', 'SequenceConv' + 'Conv2D', 'Conv3D', 'Pool2D', 'FC', 'BatchNorm', 'Embedding', 'GRUUnit', + 'LayerNorm', 'NCE', 'PRelu', 'BilinearTensorProduct', 'Conv2DTranspose', + 'Conv3DTranspose', 'SequenceConv', 'RowConv', 'GroupNorm', 'SpectralNorm', + 'TreeConv' ] class Conv2D(layers.Layer): + """ + The convolution2D layer calculates the output based on the input, filter + and strides, paddings, dilations, groups parameters. Input and + Output are in NCHW format, where N is batch size, C is the number of + channels, H is the height of the feature, and W is the width of the feature. + Filter is in MCHW format, where M is the number of output image channels, + C is the number of input image channels, H is the height of the filter, + and W is the width of the filter. If the groups is greater than 1, + C will equal the number of input image channels divided by the groups. + Please refer to UFLDL's `convolution + `_ + for more detials. + If bias attribution and activation type are provided, bias is added to the + output of the convolution, and the corresponding activation function is + applied to the final result. + + For each input :math:`X`, the equation is: + + .. math:: + + Out = \sigma (W \\ast X + b) + + Where: + + * :math:`X`: Input value, a tensor with NCHW format. + * :math:`W`: Filter value, a tensor with MCHW format. + * :math:`\\ast`: Convolution operation. + * :math:`b`: Bias value, a 2-D tensor with shape [M, 1]. + * :math:`\\sigma`: Activation function. + * :math:`Out`: Output value, the shape of :math:`Out` and :math:`X` may be different. + + Example: + + - Input: + + Input shape: :math:`(N, C_{in}, H_{in}, W_{in})` + + Filter shape: :math:`(C_{out}, C_{in}, H_f, W_f)` + + - Output: + + Output shape: :math:`(N, C_{out}, H_{out}, W_{out})` + + Where + + .. math:: + + H_{out}&= \\frac{(H_{in} + 2 * paddings[0] - (dilations[0] * (H_f - 1) + 1))}{strides[0]} + 1 \\\\ + W_{out}&= \\frac{(W_{in} + 2 * paddings[1] - (dilations[1] * (W_f - 1) + 1))}{strides[1]} + 1 + + Args: + input (Variable): The input image with [N, C, H, W] format. + num_filters(int): The number of filter. It is as same as the output + image channel. + filter_size (int|tuple|None): The filter size. If filter_size is a tuple, + it must contain two integers, (filter_size_H, filter_size_W). + Otherwise, the filter will be a square. + stride (int|tuple): The stride size. If stride is a tuple, it must + contain two integers, (stride_H, stride_W). Otherwise, the + stride_H = stride_W = stride. Default: stride = 1. + padding (int|tuple): The padding size. If padding is a tuple, it must + contain two integers, (padding_H, padding_W). Otherwise, the + padding_H = padding_W = padding. Default: padding = 0. + dilation (int|tuple): The dilation size. If dilation is a tuple, it must + contain two integers, (dilation_H, dilation_W). Otherwise, the + dilation_H = dilation_W = dilation. Default: dilation = 1. + groups (int): The groups number of the Conv2d Layer. According to grouped + convolution in Alex Krizhevsky's Deep CNN paper: when group=2, + the first half of the filters is only connected to the first half + of the input channels, while the second half of the filters is only + connected to the second half of the input channels. Default: groups=1. + param_attr (ParamAttr|None): The parameter attribute for learnable parameters/weights + of conv2d. If it is set to None or one attribute of ParamAttr, conv2d + will create ParamAttr as param_attr. If the Initializer of the param_attr + is not set, the parameter is initialized with :math:`Normal(0.0, std)`, + and the :math:`std` is :math:`(\\frac{2.0 }{filter\_elem\_num})^{0.5}`. Default: None. + bias_attr (ParamAttr|bool|None): The parameter attribute for the bias of conv2d. + If it is set to False, no bias will be added to the output units. + If it is set to None or one attribute of ParamAttr, conv2d + will create ParamAttr as bias_attr. If the Initializer of the bias_attr + is not set, the bias is initialized zero. Default: None. + use_cudnn (bool): Use cudnn kernel or not, it is valid only when the cudnn + library is installed. Default: True + act (str): Activation type, if it is set to None, activation is not appended. + Default: None + name (str|None): A name for this layer(optional). If set None, the layer + will be named automatically. Default: None + + Returns: + Variable: The tensor variable storing the convolution and \ + non-linearity activation result. + + Raises: + ValueError: If the shapes of input, filter_size, stride, padding and + groups mismatch. + + Examples: + .. code-block:: python + + data = fluid.layers.data(name='data', shape=[3, 32, 32], dtype='float32') + conv2d = fluid.layers.conv2d(input=data, num_filters=2, filter_size=3, act="relu") + """ + def __init__(self, name_scope, num_channels, @@ -47,7 +151,7 @@ class Conv2D(layers.Layer): bias_attr=None, dtype=core.VarDesc.VarType.FP32): assert param_attr is not False, "param_attr should not be False here." - super(Conv2D, self).__init__(name_scope) + super(Conv2D, self).__init__(name_scope, dtype) self._groups = groups self._stride = utils.convert_to_list(stride, 2, 'stride') self._padding = utils.convert_to_list(padding, 2, 'padding') @@ -119,25 +223,480 @@ class Conv2D(layers.Layer): 'paddings': self._padding, 'dilations': self._dilation, 'groups': self._groups if self._groups else 1, - 'use_cudnn': self._use_cudnn, - 'use_mkldnn': False, + 'use_cudnn': self._use_cudnn, + 'use_mkldnn': False, + }) + + pre_act = self._helper.create_variable_for_type_inference( + dtype=self._dtype) + + self._helper.append_op( + type='elementwise_add', + inputs={'X': [pre_bias], + 'Y': [self._bias_param]}, + outputs={'Out': [pre_act]}, + attrs={'axis': 1}) + + # Currently, we don't support inplace in dygraph mode + return self._helper.append_activation(pre_act, act=self._act) + + +class Conv3D(layers.Layer): + """ + **Convlution3D Layer** + + The convolution3D layer calculates the output based on the input, filter + and strides, paddings, dilations, groups parameters. Input(Input) and + Output(Output) are in NCDHW format. Where N is batch size C is the number of + channels, D is the depth of the feature, H is the height of the feature, + and W is the width of the feature. Convlution3D is similar with Convlution2D + but adds one dimension(depth). If bias attribution and activation type are + provided, bias is added to the output of the convolution, and the + corresponding activation function is applied to the final result. + + For each input :math:`X`, the equation is: + + .. math:: + + Out = \sigma (W \\ast X + b) + + In the above equation: + + * :math:`X`: Input value, a tensor with NCDHW format. + * :math:`W`: Filter value, a tensor with MCDHW format. + * :math:`\\ast`: Convolution operation. + * :math:`b`: Bias value, a 2-D tensor with shape [M, 1]. + * :math:`\\sigma`: Activation function. + * :math:`Out`: Output value, the shape of :math:`Out` and :math:`X` may be different. + + Example: + + - Input: + + Input shape: :math:`(N, C_{in}, D_{in}, H_{in}, W_{in})` + + Filter shape: :math:`(C_{out}, C_{in}, D_f, H_f, W_f)` + + - Output: + Output shape: :math:`(N, C_{out}, D_{out}, H_{out}, W_{out})` + + Where + + .. math:: + + D_{out}&= \\frac{(D_{in} + 2 * paddings[0] - (dilations[0] * (D_f - 1) + 1))}{strides[0]} + 1 \\\\ + H_{out}&= \\frac{(H_{in} + 2 * paddings[1] - (dilations[1] * (H_f - 1) + 1))}{strides[1]} + 1 \\\\ + W_{out}&= \\frac{(W_{in} + 2 * paddings[2] - (dilations[2] * (W_f - 1) + 1))}{strides[2]} + 1 + + Args: + input (Variable): The input image with [N, C, D, H, W] format. + num_filters(int): The number of filter. It is as same as the output + image channel. + filter_size (int|tuple|None): The filter size. If filter_size is a tuple, + it must contain three integers, (filter_size_D, filter_size_H, filter_size_W). + Otherwise, the filter will be a square. + stride (int|tuple): The stride size. If stride is a tuple, it must + contain three integers, (stride_D, stride_H, stride_W). Otherwise, the + stride_D = stride_H = stride_W = stride. Default: stride = 1. + padding (int|tuple): The padding size. If padding is a tuple, it must + contain three integers, (padding_D, padding_H, padding_W). Otherwise, the + padding_D = padding_H = padding_W = padding. Default: padding = 0. + dilation (int|tuple): The dilation size. If dilation is a tuple, it must + contain three integers, (dilation_D, dilation_H, dilation_W). Otherwise, the + dilation_D = dilation_H = dilation_W = dilation. Default: dilation = 1. + groups (int): The groups number of the Conv3d Layer. According to grouped + convolution in Alex Krizhevsky's Deep CNN paper: when group=2, + the first half of the filters is only connected to the first half + of the input channels, while the second half of the filters is only + connected to the second half of the input channels. Default: groups=1 + param_attr (ParamAttr|None): The parameter attribute for learnable parameters/weights + of conv3d. If it is set to None or one attribute of ParamAttr, conv3d + will create ParamAttr as param_attr. If it is set to None, the parameter + is initialized with :math:`Normal(0.0, std)`, and the :math:`std` is + :math:`(\\frac{2.0 }{filter\_elem\_num})^{0.5}`. Default: None. + bias_attr (ParamAttr|bool|None): The parameter attribute for the bias of conv3d. + If it is set to False, no bias will be added to the output units. + If it is set to None or one attribute of ParamAttr, conv3d + will create ParamAttr as bias_attr. If the Initializer of the bias_attr + is not set, the bias is initialized zero. Default: None. + use_cudnn (bool): Use cudnn kernel or not, it is valid only when the cudnn + library is installed. Default: True + act (str): Activation type, if it is set to None, activation is not appended. + Default: None. + name (str|None): A name for this layer(optional). If set None, the layer + will be named automatically. Default: None. + + Returns: + Variable: The tensor variable storing the convolution and \ + non-linearity activation result. + + Raises: + ValueError: If the shapes of input, filter_size, stride, padding and + groups mismatch. + + Examples: + .. code-block:: python + + data = fluid.layers.data(name='data', shape=[3, 12, 32, 32], dtype='float32') + conv3d = fluid.layers.conv3d(input=data, num_filters=2, filter_size=3, act="relu") + """ + + def __init__(self, + name_scope, + num_filters, + filter_size, + stride=1, + padding=0, + dilation=1, + groups=None, + param_attr=None, + bias_attr=None, + use_cudnn=True, + act=None): + assert param_attr is not False, "param_attr should not be False here." + super(Conv3D, self).__init__(name_scope) + self._groups = groups + self._stride = utils.convert_to_list(stride, 3, 'stride') + self._padding = utils.convert_to_list(padding, 3, 'padding') + self._dilation = utils.convert_to_list(dilation, 3, 'dilation') + self._act = act + if not isinstance(use_cudnn, bool): + raise ValueError("use_cudnn should be True or False") + self._use_cudnn = use_cudnn + self._filter_size = filter_size + self._num_filters = num_filters + self._param_attr = param_attr + self._bias_attr = bias_attr + + def build_once(self, input): + num_channels = input.shape[1] + self._dtype = self._helper.input_dtype(input) + + if self._groups is None: + num_filter_channels = num_channels + else: + if num_channels % self._groups != 0: + raise ValueError("num_channels must be divisible by groups.") + num_filter_channels = num_channels // self._groups + + filter_size = utils.convert_to_list(self._filter_size, 3, 'filter_size') + + filter_shape = [self._num_filters, num_filter_channels] + filter_size + + def _get_default_param_initializer(): + filter_elem_num = filter_size[0] * filter_size[1] * filter_size[ + 2] * num_channels + std = (2.0 / filter_elem_num)**0.5 + return Normal(0.0, std, 0) + + self._filter_param = self.create_parameter( + attr=self._param_attr, + shape=filter_shape, + dtype=self._dtype, + default_initializer=_get_default_param_initializer()) + + self._bias_param = self.create_parameter( + attr=self._bias_attr, + shape=[self._num_filters], + dtype=self._dtype, + is_bias=True) + + def forward(self, input): + pre_bias = self._helper.create_variable_for_type_inference( + dtype=self._dtype) + + self._helper.append_op( + type='conv3d', + inputs={ + 'Input': input, + 'Filter': self._filter_param, + }, + outputs={"Output": pre_bias}, + attrs={ + 'strides': self._stride, + 'paddings': self._padding, + 'dilations': self._dilation, + 'groups': self._groups if self._groups else 1, + 'use_cudnn': self._use_cudnn, + 'use_mkldnn': False + }) + + pre_act = self._helper.create_variable_for_type_inference( + dtype=self._dtype) + + self._helper.append_op( + type='elementwise_add', + inputs={'X': [pre_bias], + 'Y': [self._bias_param]}, + outputs={'Out': [pre_act]}, + attrs={'axis': 1}) + + return self._helper.append_activation(pre_act, act=self._act) + + +class Conv3DTranspose(layers.Layer): + """ + **Convlution3D transpose layer** + + The convolution3D transpose layer calculates the output based on the input, + filter, and dilations, strides, paddings. Input(Input) and output(Output) + are in NCDHW format. Where N is batch size, C is the number of channels, + D is the depth of the feature, H is the height of the feature, and W + is the width of the feature. Parameters(dilations, strides, paddings) are + two elements. These two elements represent height and width, respectively. + The details of convolution transpose layer, please refer to the following + explanation and references `therein `_. + If bias attribution and activation type are provided, bias is added to + the output of the convolution, and the corresponding activation function + is applied to the final result. + + For each input :math:`X`, the equation is: + + .. math:: + + Out = \sigma (W \\ast X + b) + + In the above equation: + + * :math:`X`: Input value, a tensor with NCDHW format. + * :math:`W`: Filter value, a tensor with MCDHW format. + * :math:`\\ast`: Convolution operation. + * :math:`b`: Bias value, a 2-D tensor with shape [M, 1]. + * :math:`\\sigma`: Activation function. + * :math:`Out`: Output value, the shape of :math:`Out` and :math:`X` may be different. + + Example: + + - Input: + + Input shape: :math:`(N, C_{in}, D_{in}, H_{in}, W_{in})` + + Filter shape: :math:`(C_{in}, C_{out}, D_f, H_f, W_f)` + + - Output: + + Output shape: :math:`(N, C_{out}, D_{out}, H_{out}, W_{out})` + + Where + + .. math:: + + D_{out} &= (D_{in} - 1) * strides[0] - 2 * paddings[0] + dilations[0] * (D_f - 1) + 1 \\\\ + H_{out} &= (H_{in} - 1) * strides[1] - 2 * paddings[1] + dilations[1] * (H_f - 1) + 1 \\\\ + W_{out} &= (W_{in} - 1) * strides[2] - 2 * paddings[2] + dilations[2] * (W_f - 1) + 1 + + Args: + input(Variable): The input image with [N, C, D, H, W] format. + num_filters(int): The number of the filter. It is as same as the output + image channel. + output_size(int|tuple|None): The output image size. If output size is a + tuple, it must contain three integers, (image_D, image_H, image_W). This + parameter only works when filter_size is None. + filter_size(int|tuple|None): The filter size. If filter_size is a tuple, + it must contain three integers, (filter_size_D, filter_size_H, filter_size_W). + Otherwise, the filter will be a square. None if use output size to + calculate filter_size. + padding(int|tuple): The padding size. If padding is a tuple, it must + contain three integers, (padding_D, padding_H, padding_W). Otherwise, the + padding_D = padding_H = padding_W = padding. Default: padding = 0. + stride(int|tuple): The stride size. If stride is a tuple, it must + contain three integers, (stride_D, stride_H, stride_W). Otherwise, the + stride_D = stride_H = stride_W = stride. Default: stride = 1. + dilation(int|tuple): The dilation size. If dilation is a tuple, it must + contain three integers, (dilation_D, dilation_H, dilation_W). Otherwise, the + dilation_D = dilation_H = dilation_W = dilation. Default: dilation = 1. + groups(int): The groups number of the Conv3d transpose layer. Inspired by + grouped convolution in Alex Krizhevsky's Deep CNN paper, in which + when group=2, the first half of the filters is only connected to the + first half of the input channels, while the second half of the + filters is only connected to the second half of the input channels. + Default: groups=1 + param_attr (ParamAttr|None): The parameter attribute for learnable parameters/weights + of conv3d_transpose. If it is set to None or one attribute of ParamAttr, conv3d_transpose + will create ParamAttr as param_attr. If the Initializer of the param_attr + is not set, the parameter is initialized with Xavier. Default: None. + bias_attr (ParamAttr|bool|None): The parameter attribute for the bias of conv3d_transpose. + If it is set to False, no bias will be added to the output units. + If it is set to None or one attribute of ParamAttr, conv3d_transpose + will create ParamAttr as bias_attr. If the Initializer of the bias_attr + is not set, the bias is initialized zero. Default: None. + use_cudnn(bool): Use cudnn kernel or not, it is valid only when the cudnn + library is installed. Default: True + act (str): Activation type, if it is set to None, activation is not appended. + Default: None. + name(str|None): A name for this layer(optional). If set None, the layer + will be named automatically. + + Returns: + Variable: The tensor variable storing the convolution transpose result. + + Raises: + ValueError: If the shapes of input, filter_size, stride, padding and + groups mismatch. + + Examples: + .. code-block:: python + + conv3d_transpose = nn.Conv3DTranspose( + 'Conv3DTranspose', + num_filters=12, + filter_size=12, + use_cudnn=False) + transpose_res = conv3d_transpose(base.to_variable(input_array)) + """ + + def __init__(self, + name_scope, + num_filters, + output_size=None, + filter_size=None, + padding=0, + stride=1, + dilation=1, + groups=None, + param_attr=None, + bias_attr=None, + use_cudnn=True, + act=None, + name=None): + super(Conv3DTranspose, self).__init__(name_scope) + if not isinstance(use_cudnn, bool): + raise ValueError("use_cudnn should be True or False") + assert param_attr is not False, "param_attr should not be False in conv3d_transpose." + self._padding = utils.convert_to_list(padding, 3, 'padding') + self._stride = utils.convert_to_list(stride, 3, 'stride') + self._dilation = utils.convert_to_list(dilation, 3, 'dilation') + self._param_attr = param_attr + self._filter_size = filter_size + self._output_size = output_size + self._groups = 1 if groups is None else groups + self._num_filters = num_filters + self._use_cudnn = use_cudnn + self._bias_attr = bias_attr + self._act = act + + def build_once(self, input): + self._dtype = self._helper.input_dtype(input) + self._input_channel = input.shape[1] + + if self._filter_size is None: + if self._output_size is None: + raise ValueError( + "output_size must be set when filter_size is None") + if isinstance(self._output_size, int): + self._output_size = [self._output_size, self._output_size] + + d_in = input.shape[2] + h_in = input.shape[3] + w_in = input.shape[4] + + filter_size_d = (self._output_size[0] - + (d_in - 1) * self._stride[0] + 2 * self._padding[0] + - 1) // self._dilation[0] + 1 + filter_size_h = (self._output_size[1] - + (h_in - 1) * self._stride[1] + 2 * self._padding[1] + - 1) // self._dilation[1] + 1 + filter_size_w = (self._output_size[2] - + (w_in - 1) * self._stride[2] + 2 * self._padding[2] + - 1) // self._dilation[2] + 1 + self._filter_size = [filter_size_d, filter_size_h, filter_size_w] + else: + self._filter_size = utils.convert_to_list( + self._filter_size, 3, 'conv3d_transpose.filter_size') + + filter_shape = [ + self._input_channel, self._num_filters // self._groups + ] + self._filter_size + self._img_filter = self.create_parameter( + dtype=self._dtype, shape=filter_shape, attr=self._param_attr) + if self._bias_attr: + self._bias_param = self.create_parameter( + attr=self._bias_attr, + shape=[self._num_filters], + dtype=self._dtype, + is_bias=True) + + def forward(self, input): + pre_bias = self._helper.create_variable_for_type_inference( + dtype=self._dtype) + self._helper.append_op( + type="conv3d_transpose", + inputs={'Input': [input], + 'Filter': [self._img_filter]}, + outputs={'Output': pre_bias}, + attrs={ + 'strides': self._stride, + 'paddings': self._padding, + 'dilations': self._dilation, + 'groups': self._groups if self._groups else 1, + 'use_cudnn': self._use_cudnn }) - pre_act = self._helper.create_variable_for_type_inference( - dtype=self._dtype) - - self._helper.append_op( - type='elementwise_add', - inputs={'X': [pre_bias], - 'Y': [self._bias_param]}, - outputs={'Out': [pre_act]}, - attrs={'axis': 1}) + if self._bias_attr: + pre_act = self._helper.create_variable_for_type_inference( + dtype=self._dtype) + self._helper.append_op( + type='elementwise_add', + inputs={'X': [pre_bias], + 'Y': [self._bias_param]}, + outputs={'Out': [pre_act]}, + attrs={'axis': 1}) + else: + pre_act = pre_bias - # Currently, we don't support inplace in dygraph mode + # Currently, we don't support inplace in imperative mode return self._helper.append_activation(pre_act, act=self._act) class Pool2D(layers.Layer): + """ + ${comment} + + Args: + input (Variable): The input tensor of pooling operator. The format of + input tensor is NCHW, where N is batch size, C is + the number of channels, H is the height of the + feature, and W is the width of the feature. + pool_size (int|list|tuple): The pool kernel size. If pool kernel size is a tuple or list, + it must contain two integers, (pool_size_Height, pool_size_Width). + Otherwise, the pool kernel size will be a square of an int. + pool_type: ${pooling_type_comment} + pool_stride (int|list|tuple): The pool stride size. If pool stride size is a tuple or list, + it must contain two integers, (pool_stride_Height, pool_stride_Width). + Otherwise, the pool stride size will be a square of an int. + pool_padding (int|list|tuple): The pool padding size. If pool padding size is a tuple, + it must contain two integers, (pool_padding_on_Height, pool_padding_on_Width). + Otherwise, the pool padding size will be a square of an int. + global_pooling (bool): ${global_pooling_comment} + use_cudnn (bool): ${use_cudnn_comment} + ceil_mode (bool): ${ceil_mode_comment} + name (str|None): A name for this layer(optional). If set None, the + layer will be named automatically. + exclusive (bool): Whether to exclude padding points in average pooling + mode, default is true + + Returns: + Variable: The pooling result. + + Raises: + ValueError: If 'pool_type' is not "max" nor "avg" + ValueError: If 'global_pooling' is False and 'pool_size' is -1 + ValueError: If 'use_cudnn' is not a bool value. + + Examples: + + .. code-block:: python + + data = fluid.layers.data( + name='data', shape=[3, 32, 32], dtype='float32') + pool2d = fluid.Pool2D("pool2d",pool_size=2, + pool_type='max', + pool_stride=1, + global_pooling=False) + + pool2d_res = pool2d(data) + """ + def __init__(self, name_scope, pool_size=-1, @@ -197,6 +756,102 @@ class Pool2D(layers.Layer): class FC(layers.Layer): + """ + **Fully Connected Layer** + + This function creates a fully connected layer in the network. It can take + one or multiple tensors as its inputs(input can be a list of Variable, see + Args in detail). It creates a variable called weights for each input tensor, + which represents a fully connected weight matrix from each input unit to + each output unit. The fully connected layer multiplies each input tensor + with its corresponding weight to produce an output Tensor with shape [M, `size`], + where M is batch size. If multiple input tensors are given, the results of + multiple output tensors with shape [M, `size`] will be summed up. If bias_attr + is not None, a bias variable will be created and added to the output. + Finally, if activation is not None, it will be applied to the output as well. + + When the input is single tensor: + + .. math:: + + Out = Act({XW + b}) + + When the input are multiple tensors: + + .. math:: + + Out = Act({\sum_{i=0}^{N-1}X_iW_i + b}) + + In the above equation: + + * :math:`N`: Number of the input. N equals to len(input) if input is list of Variable. + * :math:`X_i`: The i-th input tensor. + * :math:`W_i`: The i-th weights matrix corresponding i-th input tensor. + * :math:`b`: The bias parameter created by this layer (if needed). + * :math:`Act`: The activation function. + * :math:`Out`: The output tensor. + + See below for an example. + + .. code-block:: text + + Given: + data_1.data = [[[0.1, 0.2], + [0.3, 0.4]]] + data_1.shape = (1, 2, 2) # 1 is batch_size + + data_2 = [[[0.1, 0.2, 0.3]]] + data_2.shape = (1, 1, 3) + + out = fluid.layers.fc(input=[data_1, data_2], size=2) + + Then: + out.data = [[0.18669507, 0.1893476]] + out.shape = (1, 2) + + Args: + input (Variable|list of Variable): The input tensor(s) of this layer, and the dimension of + the input tensor(s) is at least 2. + size(int): The number of output units in this layer. + num_flatten_dims (int, default 1): The fc layer can accept an input tensor with more than + two dimensions. If this happens, the multidimensional tensor will first be flattened + into a 2-dimensional matrix. The parameter `num_flatten_dims` determines how the input + tensor is flattened: the first `num_flatten_dims` (inclusive, index starts from 1) + dimensions will be flatten to form the first dimension of the final matrix (height of + the matrix), and the rest `rank(X) - num_flatten_dims` dimensions are flattened to + form the second dimension of the final matrix (width of the matrix). For example, suppose + `X` is a 5-dimensional tensor with a shape [2, 3, 4, 5, 6], and `num_flatten_dims` = 3. + Then, the flattened matrix will have a shape [2 x 3 x 4, 5 x 6] = [24, 30]. + param_attr (ParamAttr|list of ParamAttr, default None): The parameter attribute for learnable + parameters/weights of this layer. + bias_attr (ParamAttr|list of ParamAttr, default None): The parameter attribute for the bias + of this layer. If it is set to False, no bias will be added to the output units. + If it is set to None, the bias is initialized zero. Default: None. + act (str, default None): Activation to be applied to the output of this layer. + is_test(bool): A flag indicating whether execution is in test phase. + name (str, default None): The name of this layer. + + Returns: + Variable: The transformation result. + + Raises: + ValueError: If rank of the input tensor is less than 2. + + Examples: + .. code-block:: python + + # when input is single tensor + data = fluid.layers.data(name="data", shape=[32, 32], dtype="float32") + fc = fluid.FC("fc", size=1000, act="tanh") + fc_res = fc(data) + + # when input are multiple tensors + data_1 = fluid.layers.data(name="data_1", shape=[32, 32], dtype="float32") + data_2 = fluid.layers.data(name="data_2", shape=[24, 36], dtype="float32") + fc = fluid.FC("fc", size=1000, act="tanh") + fc_res = fc([data_1, data_2]) + """ + def __init__(self, name_scope, size, @@ -205,7 +860,7 @@ class FC(layers.Layer): num_flatten_dims=1, dtype=core.VarDesc.VarType.FP32, act=None): - super(FC, self).__init__(name_scope) + super(FC, self).__init__(name_scope, dtype) self._size = size self._num_flatten_dims = num_flatten_dims @@ -224,7 +879,7 @@ class FC(layers.Layer): assert isinstance(value, Parameter) self.__w[i] = value - def _build_once(self, input): + def build_once(self, input): i = 0 for inp, param in self._helper.iter_inputs_and_params(input, self._param_attr): @@ -293,6 +948,91 @@ class FC(layers.Layer): class BatchNorm(layers.Layer): + """ + **Batch Normalization Layer** + + Can be used as a normalizer function for conv2d and fully_connected operations. + The required data format for this layer is one of the following: + + 1. NHWC `[batch, in_height, in_width, in_channels]` + + 2. NCHW `[batch, in_channels, in_height, in_width]` + + Refer to `Batch Normalization: Accelerating Deep Network Training by Reducing + Internal Covariate Shift `_ + for more details. + + :math:`input` is the input features over a mini-batch. + + .. math:: + + \\mu_{\\beta} &\\gets \\frac{1}{m} \\sum_{i=1}^{m} x_i \\qquad &//\\ + \ mini-batch\ mean \\\\ + \\sigma_{\\beta}^{2} &\\gets \\frac{1}{m} \\sum_{i=1}^{m}(x_i - \\ + \\mu_{\\beta})^2 \\qquad &//\ mini-batch\ variance \\\\ + \\hat{x_i} &\\gets \\frac{x_i - \\mu_\\beta} {\\sqrt{\\ + \\sigma_{\\beta}^{2} + \\epsilon}} \\qquad &//\ normalize \\\\ + y_i &\\gets \\gamma \\hat{x_i} + \\beta \\qquad &//\ scale\ and\ shift + + + When use_global_stats = True, the :math:`\\mu_{\\beta}` + and :math:`\\sigma_{\\beta}^{2}` are not the statistics of one mini-batch. + They are global (or running) statistics. (It usually got from the + pre-trained model.) + The training and testing (or inference) have the same behavior: + + .. math:: + + \\hat{x_i} &\\gets \\frac{x_i - \\mu_\\beta} {\\sqrt{\\ + \\sigma_{\\beta}^{2} + \\epsilon}} \\\\ + y_i &\\gets \\gamma \\hat{x_i} + \\beta + + Args: + input(variable): The rank of input variable can be 2, 3, 4, 5. + act(string, Default None): Activation type, linear|relu|prelu|... + is_test (bool, Default False): A flag indicating whether it is in + test phrase or not. + momentum(float, Default 0.9): The value used for the moving_mean and + moving_var computation. The updated formula is: + :math:`moving\_mean = moving\_mean * momentum + new\_mean * (1. - momentum)` + :math:`moving\_var = moving\_var * momentum + new\_var * (1. - momentum)` + Default is 0.9. + epsilon(float, Default 1e-05): A value added to the denominator for + numerical stability. Default is 1e-5. + param_attr(ParamAttr|None): The parameter attribute for Parameter `scale` + of batch_norm. If it is set to None or one attribute of ParamAttr, batch_norm + will create ParamAttr as param_attr. If the Initializer of the param_attr + is not set, the parameter is initialized with Xavier. Default: None. + bias_attr(ParamAttr|None): The parameter attribute for the bias of batch_norm. + If it is set to None or one attribute of ParamAttr, batch_norm + will create ParamAttr as bias_attr. If the Initializer of the bias_attr + is not set, the bias is initialized zero. Default: None. + data_layout(string, default NCHW): NCHW|NHWC + in_place(bool, Default False): Make the input and output of batch norm reuse memory. + name(string, Default None): A name for this layer(optional). If set None, the layer + will be named automatically. + moving_mean_name(string, Default None): The name of moving_mean which store the global Mean. + moving_variance_name(string, Default None): The name of the moving_variance which store the global Variance. + do_model_average_for_mean_and_var(bool, Default False): Do model average for mean and variance or not. + fuse_with_relu (bool): if True, this OP performs relu after batch norm. + use_global_stats(bool, Default False): Whether to use global mean and + variance. In inference or test mode, set use_global_stats to true + or is_test to true, and the behavior is equivalent. + In train mode, when setting use_global_stats True, the global mean + and variance are also used during train period. + + Returns: + Variable: A tensor variable which is the result after applying batch normalization on the input. + + Examples: + + .. code-block:: python + fc = fluid.FC('fc', size=200, param_attr='fc1.w') + hidden1 = fc(x) + batch_norm = fluid.BatchNorm("batch_norm", 10) + hidden2 = batch_norm(hidden1) + """ + def __init__(self, name_scope, num_channels, @@ -310,7 +1050,7 @@ class BatchNorm(layers.Layer): do_model_average_for_mean_and_var=False, fuse_with_relu=False, use_global_stats=False): - super(BatchNorm, self).__init__(name_scope) + super(BatchNorm, self).__init__(name_scope, dtype) self._param_attr = param_attr self._param_attr = bias_attr self._act = act @@ -331,7 +1071,7 @@ class BatchNorm(layers.Layer): dtype=self._dtype, default_initializer=Constant(1.0)) if use_global_stats and self._param_attr.learning_rate == 0.: - self._scale._stop_gradient = True + self._scale.stop_gradient = True self._bias = self.create_parameter( attr=self._param_attr, @@ -339,7 +1079,7 @@ class BatchNorm(layers.Layer): dtype=self._dtype, is_bias=True) if use_global_stats and self._param_attr.learning_rate == 0.: - self._bias._stop_gradient = True + self._bias.stop_gradient = True self._mean = self.create_parameter( attr=ParamAttr( @@ -349,7 +1089,7 @@ class BatchNorm(layers.Layer): do_model_average=do_model_average_for_mean_and_var), shape=param_shape, dtype=self._dtype) - self._mean._stop_gradient = True + self._mean.stop_gradient = True self._variance = self.create_parameter( attr=ParamAttr( @@ -359,7 +1099,7 @@ class BatchNorm(layers.Layer): do_model_average=do_model_average_for_mean_and_var), shape=param_shape, dtype=self._dtype) - self._variance._stop_gradient = True + self._variance.stop_gradient = True self._in_place = in_place self._momentum = momentum @@ -368,7 +1108,7 @@ class BatchNorm(layers.Layer): self._fuse_with_relu = fuse_with_relu self._use_global_stats = use_global_stats - def _build_once(self, input): + def build_once(self, input): pass def forward(self, input): @@ -449,7 +1189,7 @@ class Embedding(layers.Layer): dict_size = len(dataset.ids) input = fluid.layers.data(name='ids', shape=[32, 32], dtype='float32') - embedding = fluid.dygraph.Embedding(size=[dict_size, 16]) + embedding = fluid.Embedding(size=[dict_size, 16]) fc = embedding(input) """ @@ -462,7 +1202,7 @@ class Embedding(layers.Layer): param_attr=None, dtype='float32'): - super(Embedding, self).__init__(name_scope) + super(Embedding, self).__init__(name_scope, dtype) self._size = size self._is_sparse = is_sparse self._is_distributed = is_distributed @@ -499,70 +1239,70 @@ class Embedding(layers.Layer): class LayerNorm(layers.Layer): - def __init__(self, - name_scope, - scale=True, - shift=True, - begin_norm_axis=1, - epsilon=1e-05, - param_attr=None, - bias_attr=None, - act=None): - """ - ${comment} + """ + ${comment} - The formula is as follows: + The formula is as follows: - .. math:: + .. math:: - \\mu & = \\frac{1}{H}\\sum_{i=1}^{H} a_i + \\mu & = \\frac{1}{H}\\sum_{i=1}^{H} a_i - \\sigma & = \\sqrt{\\frac{1}{H}\sum_{i=1}^{H}(a_i - \\mu)^2} + \\sigma & = \\sqrt{\\frac{1}{H}\sum_{i=1}^{H}(a_i - \\mu)^2} - h & = f(\\frac{g}{\\sigma}(a - \\mu) + b) + h & = f(\\frac{g}{\\sigma}(a - \\mu) + b) - * :math:`a`: the vector representation of the summed inputs to the neurons - in that layer. + * :math:`a`: the vector representation of the summed inputs to the neurons + in that layer. - * :math:`H`: the number of hidden units in a layers + * :math:`H`: the number of hidden units in a layers - * :math:`g`: the trainable scale parameter. + * :math:`g`: the trainable scale parameter. - * :math:`b`: the trainable bias parameter. + * :math:`b`: the trainable bias parameter. - Args: - input(Variable): The input tensor variable. - scale(bool): Whether to learn the adaptive gain :math:`g` after - normalization. Default True. - shift(bool): Whether to learn the adaptive bias :math:`b` after - normalization. Default True. - begin_norm_axis(int): The normalization will be performed along - dimensions from :attr:`begin_norm_axis` to :attr:`rank(input)`. - Default 1. - epsilon(float): The small value added to the variance to prevent - division by zero. Default 1e-05. - param_attr(ParamAttr|None): The parameter attribute for the learnable - gain :math:`g`. If :attr:`scale` is False, :attr:`param_attr` is - omitted. If :attr:`scale` is True and :attr:`param_attr` is None, - a default :code:`ParamAttr` would be added as scale. The - :attr:`param_attr` is initialized as 1 if it is added. Default None. - bias_attr(ParamAttr|None): The parameter attribute for the learnable - bias :math:`b`. If :attr:`shift` is False, :attr:`bias_attr` is - omitted. If :attr:`shift` is True and :attr:`param_attr` is None, - a default :code:`ParamAttr` would be added as bias. The - :attr:`bias_attr` is initialized as 0 if it is added. Default None. - act(str): Activation to be applied to the output of layer normalizaiton. - Default None. - Returns: - ${y_comment} + Args: + input(Variable): The input tensor variable. + scale(bool): Whether to learn the adaptive gain :math:`g` after + normalization. Default True. + shift(bool): Whether to learn the adaptive bias :math:`b` after + normalization. Default True. + begin_norm_axis(int): The normalization will be performed along + dimensions from :attr:`begin_norm_axis` to :attr:`rank(input)`. + Default 1. + epsilon(float): The small value added to the variance to prevent + division by zero. Default 1e-05. + param_attr(ParamAttr|None): The parameter attribute for the learnable + gain :math:`g`. If :attr:`scale` is False, :attr:`param_attr` is + omitted. If :attr:`scale` is True and :attr:`param_attr` is None, + a default :code:`ParamAttr` would be added as scale. The + :attr:`param_attr` is initialized as 1 if it is added. Default None. + bias_attr(ParamAttr|None): The parameter attribute for the learnable + bias :math:`b`. If :attr:`shift` is False, :attr:`bias_attr` is + omitted. If :attr:`shift` is True and :attr:`param_attr` is None, + a default :code:`ParamAttr` would be added as bias. The + :attr:`bias_attr` is initialized as 0 if it is added. Default None. + act(str): Activation to be applied to the output of layer normalizaiton. + Default None. + Returns: + ${y_comment} - Examples: + Examples: - >>> data = fluid.layers.data(name='data', shape=[3, 32, 32], - >>> dtype='float32') - >>> x = fluid.layers.layer_norm(input=data, begin_norm_axis=1) - """ + >>> data = fluid.layers.data(name='data', shape=[3, 32, 32], + >>> dtype='float32') + >>> x = fluid.layers.layer_norm(input=data, begin_norm_axis=1) + """ + def __init__(self, + name_scope, + scale=True, + shift=True, + begin_norm_axis=1, + epsilon=1e-05, + param_attr=None, + bias_attr=None, + act=None): super(LayerNorm, self).__init__(name_scope) self._scale = scale self._shift = shift @@ -572,7 +1312,7 @@ class LayerNorm(layers.Layer): self._bias_attr = bias_attr self._act = act - def _build_once(self, input): + def build_once(self, input): self._dtype = self._helper.input_dtype(input) input_shape = input.shape param_shape = [ @@ -710,7 +1450,7 @@ class GRUUnit(layers.Layer): gate_activation='sigmoid', origin_mode=False, dtype='float32'): - super(GRUUnit, self).__init__(name_scope) + super(GRUUnit, self).__init__(name_scope, dtype) activation_dict = dict( identity=0, @@ -934,7 +1674,7 @@ class NCE(layers.Layer): 'remote_prefetch': remote_prefetch } - def _build_once(self, input, label, sample_weight=None): + def build_once(self, input, label, sample_weight=None): assert isinstance(input, Variable) assert isinstance(label, Variable) @@ -1020,7 +1760,7 @@ class PRelu(layers.Layer): raise ValueError('mode should be one of all, channel, element.') self._alpha_shape = [1] - def _build_once(self, input): + def build_once(self, input): if self._mode == 'channel': self._alpha_shape = [1, input.shape[1], 1, 1] elif self._mode == 'element': @@ -1098,7 +1838,7 @@ class BilinearTensorProduct(layers.Layer): self._name = name self._inputs = dict() - def _build_once(self, x, y): + def build_once(self, x, y): self._dtype = self._helper.input_dtype(x) param_shape = [self._size, x.shape[1], y.shape[1]] @@ -1274,7 +2014,7 @@ class Conv2DTranspose(layers.Layer): self._output_size = output_size self._op_type = 'conv2d_transpose' - def _build_once(self, input): + def build_once(self, input): input_channel = input.shape[1] if (input_channel == self._groups and self._num_filters == input_channel and not self._use_cudnn): @@ -1388,6 +2128,8 @@ class SequenceConv(layers.Layer): bias_attr=None, param_attr=None, act=None): + assert not in_dygraph_mode( + ), "SequenceConv is not supported by dynamic graph mode yet!" super(SequenceConv, self).__init__(name_scope) self._num_filters = num_filters self._filter_size = filter_size @@ -1396,13 +2138,11 @@ class SequenceConv(layers.Layer): self._bias_attr = bias_attr self._param_attr = param_attr - def _build_once(self, input): - + def build_once(self, input): self._dtype = self._helper.input_dtype(input) - print(self._filter_size) filter_shape = [self._filter_size * input.shape[1], self._num_filters] self._filter_param = self.create_parameter( - attr=self.param_attr, shape=filter_shape, dtype=self._dtype) + attr=self._param_attr, shape=filter_shape, dtype=self._dtype) def forward(self, input): pre_bias = self._helper.create_variable_for_type_inference(self._dtype) @@ -1420,3 +2160,237 @@ class SequenceConv(layers.Layer): }) pre_act = self._helper.append_bias_op(pre_bias) return self._helper.append_activation(pre_act) + + +class RowConv(layers.Layer): + def __init__(self, + name_scope, + future_context_size, + param_attr=None, + act=None): + assert not in_dygraph_mode( + ), "RowConv is not supported by dynamic graph mode yet!" + super(RowConv, self).__init__(name_scope) + self._act = act + self._param_attr = param_attr + self._future_context_size = future_context_size + + def build_once(self, input): + self._dtype = self._helper.input_dtype(input) + filter_shape = [self._future_context_size + 1, input.shape[1]] + self._filter_param = self.create_parameter( + attr=self._param_attr, + shape=filter_shape, + dtype=self._dtype, + is_bias=False) + + def forward(self, input): + out = self._helper.create_variable_for_type_inference(self._dtype) + self._helper.append_op( + type='row_conv', + inputs={'X': [input], + 'Filter': [self._filter_param]}, + outputs={'Out': [out]}) + return self._helper.append_activation(out, act=self._act) + + +class GroupNorm(layers.Layer): + """ + **Group Normalization Layer** + + Refer to `Group Normalization `_ . + + Args: + name_scope (str): See base class. + groups(int): The number of groups that divided from channels. + epsilon(float): The small value added to the variance to prevent + division by zero. + param_attr(ParamAttr|None): The parameter attribute for the learnable + scale :math:`g`. If it is set to False, no scale will be added to the output units. + If it is set to None, the bias is initialized one. Default: None. + bias_attr(ParamAttr|None): The parameter attribute for the learnable + bias :math:`b`. If it is set to False, no bias will be added to the output units. + If it is set to None, the bias is initialized zero. Default: None. + act(str): Activation to be applied to the output of group normalizaiton. + data_layout(string|NCHW): Only NCHW is supported. + dtype(np.dtype|core.VarDesc.VarType|str): The type of data : float32, float_16, int etc + + Returns: + Variable: A tensor variable which is the result after applying group normalization on the input. + + + """ + + def __init__(self, + name_scope, + groups, + epsilon=1e-05, + param_attr=None, + bias_attr=None, + act=None, + data_layout='NCHW'): + super(GroupNorm, self).__init__(name_scope) + self._param_attr = param_attr + self._bias_attr = bias_attr + self._epsilon = epsilon + self._groups = groups + self._act = act + if data_layout != 'NCHW': + raise ValueError("unsupported data layout:" + data_layout) + + def build_once(self, input): + self._dtype = self._helper.input_dtype(input) + param_shape = [input.shape[1]] + if self._bias_attr: + self._bias = self.create_parameter( + attr=self._bias_attr, + shape=param_shape, + dtype=self._dtype, + is_bias=True) + + if self._param_attr: + self._scale = self.create_parameter( + attr=self._param_attr, + shape=param_shape, + dtype=self._dtype, + default_initializer=Constant(1.0)) + + def forward(self, input): + inputs = {'X': input} + if self._bias: + inputs['Bias'] = self._bias + if self._scale: + inputs['Scale'] = self._scale + + # create output + mean_out = self._helper.create_variable_for_type_inference( + dtype=self._dtype, stop_gradient=True) + variance_out = self._helper.create_variable_for_type_inference( + dtype=self._dtype, stop_gradient=True) + group_norm_out = self._helper.create_variable_for_type_inference( + dtype=self._dtype) + + self._helper.append_op( + type="group_norm", + inputs=inputs, + outputs={ + "Y": group_norm_out, + "Mean": mean_out, + "Variance": variance_out, + }, + attrs={"epsilon": self._epsilon, + "groups": self._groups}) + + return self._helper.append_activation(group_norm_out, self._act) + + +class SpectralNorm(layers.Layer): + def __init__(self, name_scope, dim=0, power_iters=1, eps=1e-12, name=None): + super(SpectralNorm, self).__init__(name_scope) + self._power_iters = power_iters + self._eps = eps + self._dim = dim + + def build_once(self, weight): + self._dtype = self._helper.input_dtype(weight) + input_shape = weight.shape + h = input_shape[self._dim] + w = np.prod(input_shape) // h + + self.u = self.create_parameter( + attr=ParamAttr(), + shape=[h], + dtype=self._dtype, + default_initializer=Normal(0., 1.)) + self.u.stop_gradient = True + + self.v = self.create_parameter( + attr=ParamAttr(), + shape=[w], + dtype=self._dtype, + default_initializer=Normal(0., 1.)) + self.v.stop_gradient = True + + def forward(self, weight): + inputs = {'Weight': weight, 'U': self.u, 'V': self.v} + out = self._helper.create_variable_for_type_inference(self._dtype) + self._helper.append_op( + type="spectral_norm", + inputs=inputs, + outputs={"Out": out, }, + attrs={ + "dim": self._dim, + "power_iters": self._power_iters, + "eps": self._eps, + }) + + return out + + +class TreeConv(layers.Layer): + def __init__(self, + name_scope, + output_size, + num_filters=1, + max_depth=2, + act='tanh', + param_attr=None, + bias_attr=None, + name=None): + super(TreeConv, self).__init__(name_scope) + self._name = name + self._output_size = output_size + self._act = act + self._max_depth = max_depth + self._num_filters = num_filters + self._bias_attr = bias_attr + self._param_attr = param_attr + + def build_once(self, nodes_vector, edge_set): + assert isinstance(nodes_vector, Variable) + assert isinstance(edge_set, Variable) + self._dtype = self._helper.input_dtype(nodes_vector) + + feature_size = nodes_vector.shape[2] + w_shape = [feature_size, 3, self._output_size, self._num_filters] + if self._bias_attr: + self._bias_param = self.create_parameter( + attr=self._bias_attr, + shape=[self._num_filters], + dtype=self._dtype, + is_bias=True) + self.W = self.create_parameter( + attr=self._param_attr, + shape=w_shape, + dtype=self._dtype, + is_bias=False) + + def forward(self, nodes_vector, edge_set): + if self._name: + out = self.create_variable( + name=self._name, dtype=self._dtype, persistable=False) + else: + out = self._helper.create_variable_for_type_inference( + dtype=self._dtype) + + self._helper.append_op( + type='tree_conv', + inputs={ + 'NodesVector': nodes_vector, + 'EdgeSet': edge_set, + 'Filter': self.W + }, + outputs={'Out': out, }, + attrs={'max_depth': self._max_depth}) + if self._bias_attr: + pre_activation = self._helper.create_variable_for_type_inference( + dtype=self._dtype) + self._helper.append_op( + type='elementwise_add', + inputs={'X': [out], + 'Y': [self._bias_param]}, + outputs={'Out': [pre_activation]}, + attrs={'axis': 1}) + else: + pre_activation = out + return self._helper.append_activation(pre_activation, act=self._act) diff --git a/python/paddle/fluid/dygraph/parallel.py b/python/paddle/fluid/dygraph/parallel.py new file mode 100644 index 0000000000000000000000000000000000000000..f7decac963f47ba1dcc33e9c8eab7900e745d1df --- /dev/null +++ b/python/paddle/fluid/dygraph/parallel.py @@ -0,0 +1,60 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except jin compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from .. import core + +__all__ = ["prepare_context"] + +ParallelStrategy = core.ParallelStrategy + +__parallel_ctx__clz__ = None + + +def prepare_context(parallel_strategy, place): + global __parallel_ctx__clz__ + assert __parallel_ctx__clz__ is None, "ParallelContext can only be initialized once." + + if isinstance(place, core.CUDAPlace): + __parallel_ctx__clz__ = core.NCCLParallelContext(parallel_strategy, + place) + else: + # TODO(Yancey1989): add Gloo Parallel Context to support CPU parallel computation + assert ("Only support CUDAPlace for now.") + __parallel_ctx__clz__.init() + + +class Env(object): + def __init__(self): + self._nranks = int(os.getenv("PADDLE_TRAINERS_NUM", "1")) + self._local_rank = int(os.getenv("PADDLE_TRAINER_ID", "0")) + self._dev_id = int(os.getenv("FLAGS_selected_gpus", "0")) + self._trainer_endpoints = os.getenv("PADDLE_TRAINER_ENDPOINTS", + "").split(",") + self._current_endpoint = os.getenv("PADDLE_CURRENT_ENDPOINT", "") + + @property + def nranks(self): + return self._nranks + + @property + def local_rank(self): + return self._local_rank + + @property + def dev_id(self): + return self._dev_id + + @property + def current_endpoint(self): + return self._current_endpoint diff --git a/python/paddle/fluid/executor.py b/python/paddle/fluid/executor.py index e15197037e1d901855883919b02a1574b7bc9a29..fa8b49a021294e8555e979459615b1956d9b2b55 100644 --- a/python/paddle/fluid/executor.py +++ b/python/paddle/fluid/executor.py @@ -712,10 +712,6 @@ class Executor(object): if dataset == None: raise RuntimeError("dataset is needed and should be initialized") - if not isinstance(self.place, core.CPUPlace): - raise RuntimeError("infer_from_dataset is verified on CPUPlace" - "We will open CUDAPlace in the future") - scope, trainer = self._prepare_trainer( program=program, dataset=dataset, @@ -796,10 +792,6 @@ class Executor(object): if dataset == None: raise RuntimeError("dataset is need and should be initialized") - if not isinstance(self.place, core.CPUPlace): - raise RuntimeError("train_from_dataset is verified on CPUPlace" - "We will open CUDAPlace in the future") - scope, trainer = self._prepare_trainer( program=program, dataset=dataset, diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py index 7953d98bcbb826267fa21f6503e55049c8aff5ba..c05e5fb9e3a46e721c20fd9288b89009e32afcbe 100644 --- a/python/paddle/fluid/framework.py +++ b/python/paddle/fluid/framework.py @@ -67,6 +67,7 @@ __all__ = [ 'cuda_places', 'cpu_places', 'cuda_pinned_places', + 'in_dygraph_mode', ] EMPTY_VAR_NAME = core.kEmptyVarName() @@ -79,7 +80,10 @@ _dygraph_tracer_ = None _dygraph_current_expected_place_ = None -def _in_dygraph_mode(): +def in_dygraph_mode(): + ''' + Returns(bool): True if the program is running in dynamic graph mode + ''' return _dygraph_tracer_ is not None @@ -396,7 +400,7 @@ class Variable(object): if not isinstance(dtype, core.VarDesc.VarType): dtype = convert_np_dtype_to_dtype_(dtype) - if _in_dygraph_mode(): + if in_dygraph_mode(): # record vars in tracer rather than blocks self._ivar = kwargs.get("ivar", None) if not self._ivar: @@ -482,21 +486,21 @@ class Variable(object): self.block.vars[name] = self self.op = None - self.stop_gradient = stop_gradient + self._stop_gradient = stop_gradient self.is_data = is_data - def _numpy(self): + def numpy(self): new_ivar = self._ivar._copy_to(core.CPUPlace(), True) return np.array(new_ivar.value().get_tensor()) - def _backward(self): + def backward(self): self._ivar._run_backward() - def _gradient(self): + def gradient(self): new_ivar = self._ivar._grad_ivar()._copy_to(core.CPUPlace(), True) return np.array(new_ivar.value().get_tensor()) - def _clear_gradient(self): + def clear_gradient(self): self._ivar._clear_gradient() def __str__(self): @@ -516,7 +520,7 @@ class Variable(object): Returns: str: The debug string. """ - if _in_dygraph_mode(): + if in_dygraph_mode(): # TODO(panyx0718): add more dygraph debug info. return 'name %s, dtype: %s shape: %s' % (self.name, self.dtype, self.shape) @@ -535,7 +539,7 @@ class Variable(object): __repr__ = __str__ - def _set_desc(self, input): + def set_desc(self, input): """ Set the variable description. @@ -548,43 +552,43 @@ class Variable(object): self.desc = input @property - def _stop_gradient(self): - if _in_dygraph_mode(): + def stop_gradient(self): + if in_dygraph_mode(): return self._ivar.stop_gradient else: - return self.stop_gradient + return self._stop_gradient - @_stop_gradient.setter - def _stop_gradient(self, s): - if _in_dygraph_mode(): + @stop_gradient.setter + def stop_gradient(self, s): + if in_dygraph_mode(): self._ivar.stop_gradient = s else: - self.stop_gradient = s + self._stop_gradient = s @property def persistable(self): - if _in_dygraph_mode(): + if in_dygraph_mode(): return self._ivar.persistable else: return self.desc.persistable() @persistable.setter def persistable(self, p): - if _in_dygraph_mode(): + if in_dygraph_mode(): return self._ivar.persistable else: self.desc.set_persistable(p) @property def name(self): - if _in_dygraph_mode(): + if in_dygraph_mode(): return self._ivar.name else: return cpt.to_text(self.desc.name()) @name.setter def name(self, new_name): - if _in_dygraph_mode(): + if in_dygraph_mode(): self._ivar.name = new_name else: self.desc.set_name(new_name) @@ -592,14 +596,14 @@ class Variable(object): @property def shape(self): # convert to tuple, make it as same as numpy API. - if _in_dygraph_mode(): + if in_dygraph_mode(): return self._ivar.shape else: return tuple(self.desc.shape()) @property def dtype(self): - if _in_dygraph_mode(): + if in_dygraph_mode(): return self._ivar.dtype else: return self.desc.dtype() @@ -611,7 +615,7 @@ class Variable(object): @property def type(self): - if _in_dygraph_mode(): + if in_dygraph_mode(): return self._ivar.dtype else: return self.desc.type() @@ -721,7 +725,7 @@ class Variable(object): name=unique_name.generate(".".join(self.name)), dtype=self.dtype, persistable=self.persistable, - stop_gradient=self._stop_gradient, ) + stop_gradient=self.stop_gradient, ) else: return self @@ -930,7 +934,7 @@ class Operator(object): inputs=None, outputs=None, attrs=None): - if _in_dygraph_mode(): + if in_dygraph_mode(): if type is None: raise ValueError( "`type` to initialized an Operator can not be None.") @@ -1049,7 +1053,7 @@ class Operator(object): for arg in out_args: out_arg_names.append(cpt.to_text(arg.name)) # TODO(minqiyang): could we remove variable's op in static mode? - if not _in_dygraph_mode(): + if not in_dygraph_mode(): arg.op = self self.desc.set_output(out_proto.name, out_arg_names) @@ -1095,7 +1099,7 @@ class Operator(object): @property def type(self): - if _in_dygraph_mode(): + if in_dygraph_mode(): return self.iop.type else: return self.desc.type() @@ -1638,7 +1642,7 @@ class Block(object): Returns: Operator: the append Operator. """ - if _in_dygraph_mode(): + if in_dygraph_mode(): op = Operator( block=self, desc=None, @@ -1710,7 +1714,7 @@ class Block(object): return self.ops[start:end] def _prepend_op(self, *args, **kwargs): - if _in_dygraph_mode(): + if in_dygraph_mode(): op = Operator( self, None, diff --git a/python/paddle/fluid/incubate/fleet/base/role_maker.py b/python/paddle/fluid/incubate/fleet/base/role_maker.py index 506a38059c33b94f8dc421c1e2e3b800de35fac7..ffc7ae0172e26191264625d0a8bdd28dab69c833 100644 --- a/python/paddle/fluid/incubate/fleet/base/role_maker.py +++ b/python/paddle/fluid/incubate/fleet/base/role_maker.py @@ -23,10 +23,10 @@ class RoleMakerBase(object): """ def __init__(self): - self.role_maker_name_ = "" - self.trainer_endpoints_ = [] - self.pserver_endpoints_ = [] - self.role_is_generated_ = False + self._role_maker_name = "" + self._trainer_endpoints = [] + self._pserver_endpoints = [] + self._role_is_generated = False def _is_worker(self): """ @@ -45,20 +45,20 @@ class RoleMakerBase(object): return get local ip """ import socket - self.ip_ = socket.gethostbyname(socket.gethostname()) - return self.ip_ + self._ip = socket.gethostbyname(socket.gethostname()) + return self._ip def _get_trainer_endpoints(self): """ return trainer endpoints """ - return self.trainer_endpoints_ + return self._trainer_endpoints def _get_pserver_endpoints(self): """ return pserver endpoints """ - return self.pserver_endpoints_ + return self._pserver_endpoints def _generate_role(self): """ @@ -76,53 +76,53 @@ class MPIRoleMaker(RoleMakerBase): def __init__(self): super(MPIRoleMaker, self).__init__() from mpi4py import MPI - self.comm_ = MPI.COMM_WORLD + self._comm = MPI.COMM_WORLD self.MPI = MPI - self.ips_ = None + self._ips = None def _get_rank(self): """ return rank """ - self.rank_ = self.comm_.Get_rank() - return self.rank_ + self._rank = self._comm.Get_rank() + return self._rank def _get_size(self): """ return size """ - self.size_ = self.comm_.Get_size() - return self.size_ + self._size = self._comm.Get_size() + return self._size def _all_gather(self, obj): """ all_gather(obj) will call MPI's allgather function """ self._barrier_all() - return self.comm_.allgather(obj) + return self._comm.allgather(obj) def _worker_gather(self, obj): """ worker_gather(obj) will call MPI's allgather function """ if self._is_worker(): - self.node_type_comm_.barrier() - return self.node_type_comm_.allgather(obj) + self._node_type_comm.barrier() + return self._node_type_comm.allgather(obj) return None def _barrier_all(self): """ barrier_all() will call MPI's barrier_all function """ - self.comm_.barrier() + self._comm.barrier() def _get_ips(self): """ collect current distributed job's ip list """ - if self.ips_ == None: - self.ips_ = self.comm_.allgather(self._get_local_ip()) - return self.ips_ + if self._ips == None: + self._ips = self._comm.allgather(self._get_local_ip()) + return self._ips def _finalize(self): """ @@ -140,11 +140,11 @@ class MPISymetricRoleMaker(MPIRoleMaker): def __init__(self): super(MPISymetricRoleMaker, self).__init__() - self.node_type_ = None - self.proc_per_node_ = 2 + self._node_type = None + self._proc_per_node = 2 def _check_role_generation(self): - if not self.role_is_generated_: + if not self._role_is_generated: sys.stderr.write("generate_role() should be called first") sys.exit(-1) return False @@ -163,7 +163,7 @@ class MPISymetricRoleMaker(MPIRoleMaker): return whether current process is worker assigned by role maker """ if self._check_role_generation(): - return self.node_type_ == 1 + return self._node_type == 1 return False def _is_server(self): @@ -171,7 +171,7 @@ class MPISymetricRoleMaker(MPIRoleMaker): return whether current process is server assigned by role maker """ if self._check_role_generation(): - return self.node_type_ == 0 + return self._node_type == 0 return False def _worker_num(self): @@ -197,7 +197,7 @@ class MPISymetricRoleMaker(MPIRoleMaker): return the index of worker """ if self._check_role_generation(): - return self.rank_ / self.proc_per_node_ + return self._rank / self._proc_per_node return 0 def _server_index(self): @@ -205,7 +205,7 @@ class MPISymetricRoleMaker(MPIRoleMaker): return the index of server """ if self._check_role_generation(): - return self.rank_ / self.proc_per_node_ + return self._rank / self._proc_per_node return 0 def _barrier_worker(self): @@ -214,7 +214,7 @@ class MPISymetricRoleMaker(MPIRoleMaker): """ if self._check_role_generation(): if self._is_worker(): - self.node_type_comm_.barrier() + self._node_type_comm.barrier() def _barrier_server(self): """ @@ -222,20 +222,20 @@ class MPISymetricRoleMaker(MPIRoleMaker): """ if self._check_role_generation(): if self._is_server(): - self.node_type_comm_.barrier() + self._node_type_comm.barrier() def _generate_role(self): """ generate currently process's role """ - if not self.role_is_generated_: + if not self._role_is_generated: # TODO(guru4elephant): only allow to be called once - self.trainer_endpoints_ = self._get_ips() - self.pserver_endpoints_ = self._get_ips() + self._trainer_endpoints = self._get_ips() + self._pserver_endpoints = self._get_ips() - if 0 == self._get_rank() % self.proc_per_node_ % 2: - self.node_type_ = 0 + if 0 == self._get_rank() % self._proc_per_node % 2: + self._node_type = 0 else: - self.node_type_ = 1 - self.node_type_comm_ = self.comm_.Split(self.node_type_) - self.role_is_generated_ = True + self._node_type = 1 + self._node_type_comm = self._comm.Split(self._node_type) + self._role_is_generated = True diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/__init__.py b/python/paddle/fluid/incubate/fleet/parameter_server/__init__.py index 5eefb6e94d124447ef61aa018da5011195f2bbf1..4a7665b9bced9df5f3fb8a82bbcfd7a8feb6a24a 100644 --- a/python/paddle/fluid/incubate/fleet/parameter_server/__init__.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/__init__.py @@ -64,9 +64,9 @@ class Fleet(object): def __init__(self): self._opt_info = None # for fleet only - self.role_maker_ = None - self.local_ip_ = 0 - self.is_initialized_ = False + self._role_maker = None + self._local_ip = 0 + self._is_initialized = False def init(self): # TODO(guru4elephant) @@ -78,22 +78,22 @@ class Fleet(object): current node's role, e.g. worker, server, etc. """ if not self.is_initialized_: - self.role_maker_ = MPISymetricRoleMaker() - self.role_maker_._generate_role() + self._role_maker = MPISymetricRoleMaker() + self._role_maker._generate_role() self._fleet_ptr = fluid.core.Fleet() - self.is_initialized_ = True + self._is_initialized = True def stop(self): """ stop(): will be called after a user finishes his/her training task. Fleet instance will be destroyed when stop() is called. """ - self.role_maker_._barrier_worker() - if self.role_maker_._is_first_worker(): + self._role_maker._barrier_worker() + if self._role_maker._is_first_worker(): self._fleet_ptr.stop_server() - self.role_maker_._barrier_worker() - self.role_maker_._barrier_all() - self.role_maker_._finalize() + self._role_maker._barrier_worker() + self._role_maker._barrier_all() + self._role_maker._finalize() def init_pserver(self): """ @@ -110,15 +110,15 @@ class Fleet(object): sys.exit(-1) self._fleet_ptr.init_server(self._dist_desc_str, self.role_maker_._get_rank()) - self.local_ip_ = self._fleet_ptr.run_server() + self._local_ip = self._fleet_ptr.run_server() # barrier_all for init_server - self.role_maker_._barrier_all() - self.all_ips_ = self.role_maker_._all_gather(self.local_ip_) + self._role_maker._barrier_all() + self._all_ips = self._role_maker._all_gather(self.local_ip_) - self._fleet_ptr.gather_servers(self.all_ips_, - self.role_maker_._get_size()) + self._fleet_ptr.gather_servers(self._all_ips, + self._role_maker._get_size()) # barrier_all for init_worker, wait all workers start - self.role_maker_._barrier_all() + self._role_maker._barrier_all() else: print("You should run DistributedOptimizer.minimize() first") sys.exit(-1) @@ -151,21 +151,21 @@ class Fleet(object): print("You should run DistributedOptimizer.minimize() first") sys.exit(-1) # barrier_all for init_server, wait for server starts - self.role_maker_._barrier_all() - self.all_ips_ = self.role_maker_._all_gather(self.local_ip_) - self._fleet_ptr.init_worker(self._dist_desc_str, self.all_ips_, - self.role_maker_._get_size(), - self.role_maker_._get_rank()) + self._role_maker._barrier_all() + self._all_ips = self._role_maker._all_gather(self.local_ip_) + self._fleet_ptr.init_worker(self._dist_desc_str, self._all_ips, + self._role_maker._get_size(), + self._role_maker._get_rank()) # barrier_all for init_worker - self.role_maker_._barrier_all() + self._role_maker._barrier_all() # prepare for client to client communication info = self._fleet_ptr.get_clients_info() - all_info = self.role_maker_._worker_gather(info[0]) + all_info = self._role_maker._worker_gather(info[0]) self._fleet_ptr.gather_clients(all_info) self._fleet_ptr.create_client2client_connection() # barrier for init model - self.role_maker_._barrier_worker() - if self.role_maker_._is_first_worker(): + self._role_maker._barrier_worker() + if self._role_maker._is_first_worker(): tables = self._dist_desc.trainer_param.dense_table for prog, scope in zip(programs, scopes): prog_id = str(id(prog)) @@ -192,7 +192,7 @@ class Fleet(object): int(table.table_id), var_name_list) # barrier for init model done - self.role_maker_._barrier_worker() + self._role_maker._barrier_worker() else: print("You should run DistributedOptimizer.minimize() first") sys.exit(-1) @@ -201,39 +201,39 @@ class Fleet(object): """ return the number of current job's worker num """ - return self.role_maker_._worker_num() + return self._role_maker._worker_num() def get_server_num(self): """ return the number of current job's server num """ - return self.role_maker_._server_num() + return self._role_maker._server_num() def get_worker_index(self): """ return the mpi rank of current worker """ - return self.role_maker_._worker_index() + return self._role_maker._worker_index() def is_worker(self): """ return whether current node is a worker """ - return self.role_maker_._is_worker() + return self._role_maker._is_worker() def is_server(self): """ return whether current node is pserver """ - return self.role_maker_._is_server() + return self._role_maker._is_server() def init_pserver_model(self): """ init pserver model called from pserver """ - if self.role_maker_._is_first_worker(): + if self._role_maker._is_first_worker(): self._fleet_ptr.init_model() - self.role_maker_._barrier_worker() + self._role_maker._barrier_worker() def save_pserver_model(self, save_path): """ diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/node.py b/python/paddle/fluid/incubate/fleet/parameter_server/node.py index 60035b6e8da3e40158f8be0bafdd911f6bd6f543..641c294c4a6edeb3d9823b4152b0ea158c8faa80 100644 --- a/python/paddle/fluid/incubate/fleet/parameter_server/node.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/node.py @@ -42,13 +42,13 @@ class DownpourServer(Server): """ def __init__(self): - self.server_ = pslib.ServerParameter() - self.server_.downpour_server_param.service_param.start_server_port = 0 - self.server_.downpour_server_param.service_param.server_class = "DownpourBrpcPsServer" - self.server_.downpour_server_param.service_param.client_class = "DownpourBrpcPsClient" - self.server_.downpour_server_param.service_param.service_class = "DownpourPsService" - self.server_.downpour_server_param.service_param.start_server_port = 0 - self.server_.downpour_server_param.service_param.server_thread_num = 12 + self._server = pslib.ServerParameter() + self._server.downpour_server_param.service_param.start_server_port = 0 + self._server.downpour_server_param.service_param.server_class = "DownpourBrpcPsServer" + self._server.downpour_server_param.service_param.client_class = "DownpourBrpcPsClient" + self._server.downpour_server_param.service_param.service_class = "DownpourPsService" + self._server.downpour_server_param.service_param.start_server_port = 0 + self._server.downpour_server_param.service_param.server_thread_num = 12 def add_sparse_table(self, table_id, learning_rate, slot_key_vars, slot_value_var): @@ -62,7 +62,7 @@ class DownpourServer(Server): Returns: return None """ - table = self.server_.downpour_server_param.downpour_table_param.add() + table = self._server.downpour_server_param.downpour_table_param.add() table.table_id = table_id table.table_class = "DownpourSparseTable" table.type = pslib.PS_SPARSE_TABLE @@ -123,7 +123,7 @@ class DownpourServer(Server): Returns: return None """ - table = self.server_.downpour_server_param.downpour_table_param.add() + table = self._server.downpour_server_param.downpour_table_param.add() table.table_id = table_id table.table_class = "DownpourDenseTable" table.type = pslib.PS_DENSE_TABLE @@ -140,7 +140,7 @@ class DownpourServer(Server): """ Return downpour server program_desc """ - return self.server_ + return self._server class DownpourWorker(Worker): @@ -155,7 +155,7 @@ class DownpourWorker(Worker): def __init__(self, window): self.window = window - self.worker_ = pslib.DownpourTrainerParameter() + self._worker = pslib.DownpourTrainerParameter() def add_sparse_table(self, table_id, learning_rate, slot_key_vars, slot_value_vars): @@ -187,7 +187,7 @@ class DownpourWorker(Worker): Returns: return None """ - table = self.worker_.dense_table.add() + table = self._worker.dense_table.add() table.table_id = table_id table.dense_variable_name.extend( filter(lambda x: x.find("embedding") == -1, @@ -200,4 +200,4 @@ class DownpourWorker(Worker): """ Return downpour worker program_desc """ - return self.worker_ + return self._worker diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/optimizer_factory.py b/python/paddle/fluid/incubate/fleet/parameter_server/optimizer_factory.py index 94f79e77e72bfa2d0a09502722ef36d474b610b2..ba1f2c8f6ba43bcdb8d4240e33210370e5a454f6 100644 --- a/python/paddle/fluid/incubate/fleet/parameter_server/optimizer_factory.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/optimizer_factory.py @@ -24,9 +24,9 @@ from .node import DownpourWorker, DownpourServer class DistributedOptimizerImplBase(object): def __init__(self, optimizer): - self.optimizer_ = optimizer - self.learning_rate_ = optimizer._learning_rate - self.regularization_ = optimizer.regularization + self._optimizer = optimizer + self._learning_rate = optimizer._learning_rate + self._regularization = optimizer.regularization def minimize(self, losses, @@ -41,7 +41,7 @@ class DistributedAdam(DistributedOptimizerImplBase): # todo(guru4elephant): add more optimizers here as argument # todo(guru4elephant): make learning_rate as a variable super(DistributedAdam, self).__init__(optimizer) - self.window_ = 1 + self._window = 1 self.type = "downpour" self.data_norm_name = [ ".batch_size", ".batch_square_sum", ".batch_sum", @@ -79,9 +79,9 @@ class DistributedAdam(DistributedOptimizerImplBase): server = DownpourServer() worker = DownpourWorker(self.window_) sparse_table_index = 0 - server.add_sparse_table(sparse_table_index, self.learning_rate_, + server.add_sparse_table(sparse_table_index, self._learning_rate, prefetch_slots, prefetch_slots_emb) - worker.add_sparse_table(sparse_table_index, self.learning_rate_, + worker.add_sparse_table(sparse_table_index, self._learning_rate, prefetch_slots, prefetch_slots_emb) dense_table_index = 1 program_configs = {} @@ -124,9 +124,9 @@ class DistributedAdam(DistributedOptimizerImplBase): data_norm_grads.append(i[1]) if not is_data_norm_data: grads.append(i[1]) - server.add_dense_table(dense_table_index, self.learning_rate_, + server.add_dense_table(dense_table_index, self._learning_rate, params, grads) - worker.add_dense_table(dense_table_index, self.learning_rate_, + worker.add_dense_table(dense_table_index, self._learning_rate, params, grads) program_configs[program_id]["pull_dense"] = [dense_table_index] program_configs[program_id]["push_dense"] = [dense_table_index] @@ -135,9 +135,9 @@ class DistributedAdam(DistributedOptimizerImplBase): if len(data_norm_params) != 0 and len(data_norm_grads) != 0: dense_table_index += 1 server.add_data_norm_table(dense_table_index, - self.learning_rate_, + self._learning_rate, data_norm_params, data_norm_grads) - worker.add_dense_table(dense_table_index, self.learning_rate_, + worker.add_dense_table(dense_table_index, self._learning_rate, data_norm_params, data_norm_grads) #program_config.pull_dense_table_id.extend([dense_table_index]) #program_config.push_dense_table_id.extend([dense_table_index]) diff --git a/python/paddle/fluid/initializer.py b/python/paddle/fluid/initializer.py index 6aff93dceaf5cfd299bdc9f68246ed579f248f3c..da2591b98058a2283275cc222194e89240e87ae1 100644 --- a/python/paddle/fluid/initializer.py +++ b/python/paddle/fluid/initializer.py @@ -165,7 +165,7 @@ class ConstantInitializer(Initializer): 'force_cpu': self._force_cpu or force_init_on_cpu() }, stop_gradient=True) - if not framework._in_dygraph_mode(): + if not framework.in_dygraph_mode(): var.op = op return op @@ -245,7 +245,7 @@ class UniformInitializer(Initializer): attrs={"in_dtype": out_var.dtype, "out_dtype": var.dtype}) - if not framework._in_dygraph_mode(): + if not framework.in_dygraph_mode(): var.op = op return op @@ -324,7 +324,7 @@ class NormalInitializer(Initializer): outputs={"Out": var}, attrs={"in_dtype": out_var.dtype, "out_dtype": var.dtype}) - if not framework._in_dygraph_mode(): + if not framework.in_dygraph_mode(): var.op = op return op @@ -403,7 +403,7 @@ class TruncatedNormalInitializer(Initializer): outputs={"Out": var}, attrs={"in_dtype": out_var.dtype, "out_dtype": var.dtype}) - if not framework._in_dygraph_mode(): + if not framework.in_dygraph_mode(): var.op = op return op @@ -509,7 +509,7 @@ class XavierInitializer(Initializer): "seed": self._seed }, stop_gradient=True) - if not framework._in_dygraph_mode(): + if not framework.in_dygraph_mode(): var.op = op return op @@ -610,7 +610,7 @@ class MSRAInitializer(Initializer): "seed": self._seed }, stop_gradient=True) - if not framework._in_dygraph_mode(): + if not framework.in_dygraph_mode(): var.op = op return op @@ -709,7 +709,7 @@ class BilinearInitializer(Initializer): 'shape': list(shape), value_name: values }) - if not framework._in_dygraph_mode(): + if not framework.in_dygraph_mode(): var.op = op return op @@ -768,7 +768,7 @@ class NumpyArrayInitializer(Initializer): value_name: values }, stop_gradient=True) - if not framework._in_dygraph_mode(): + if not framework.in_dygraph_mode(): var.op = op return op diff --git a/python/paddle/fluid/layer_helper.py b/python/paddle/fluid/layer_helper.py index 7eb912645e5077d35a2d11d7d09a033d28345e15..11e3c4938bef4a3c97a724798e2f7273c25f06ed 100644 --- a/python/paddle/fluid/layer_helper.py +++ b/python/paddle/fluid/layer_helper.py @@ -17,7 +17,7 @@ from __future__ import print_function import copy import six -from .framework import Parameter, dtype_is_floating, _in_dygraph_mode +from .framework import Parameter, dtype_is_floating, in_dygraph_mode from . import unique_name from paddle.fluid.initializer import Constant, Xavier from .param_attr import ParamAttr diff --git a/python/paddle/fluid/layer_helper_base.py b/python/paddle/fluid/layer_helper_base.py index 869a5f54e9cdf5740c5e216917d92880d7d61e2d..9eed00b16185d00f30dfd75f03e31fb45cf9567c 100644 --- a/python/paddle/fluid/layer_helper_base.py +++ b/python/paddle/fluid/layer_helper_base.py @@ -17,7 +17,7 @@ from __future__ import print_function import copy import numpy as np -from .framework import Variable, default_main_program, default_startup_program, _in_dygraph_mode, _current_expected_place +from .framework import Variable, default_main_program, default_startup_program, in_dygraph_mode, _current_expected_place from . import unique_name from .param_attr import ParamAttr, WeightNormParamAttr from . import core @@ -54,7 +54,7 @@ class LayerHelperBase(object): Return Variable construct from value """ if isinstance(value, np.ndarray): - assert _in_dygraph_mode( + assert in_dygraph_mode( ), "to_variable could only be called in dygraph mode" if not block: @@ -302,7 +302,7 @@ class LayerHelperBase(object): param = self._create_weight_normalize(attr, shape, dtype) WeightNormParamAttr.params_with_weight_norm.append(param) return param - if _in_dygraph_mode(): + if in_dygraph_mode(): # In dygraph mode, we want the returned parameter to be # initialized so that it can be used imperatively. return self.main_program.global_block().create_parameter( @@ -370,7 +370,7 @@ class LayerHelperBase(object): initializer: initializer to use """ assert isinstance(var, Variable) - if _in_dygraph_mode(): + if in_dygraph_mode(): initializer(var, var.block) else: self.startup_program.global_block().create_var( diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index a5e513ed5e35d530dd07c49339995461da8454a1..f8f461853f34a09eb2317f6ac93ad385cca3609f 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -267,8 +267,44 @@ class StaticRNN(object): """ StaticRNN class. - StaticRNN class is used to create a StaticRNN. The RNN will have its - own parameters like inputs, outputs, memories, status and length. + The StaticRNN can process a batch of sequence data. The length of each + sample sequence must be equal. The StaticRNN will have its own parameters + like inputs, outputs, memories. **Note that the first dimension of inputs + represents sequence length, and all the sequence length of inputs must be + the same. And the meaning of each axis of input and output are the same.** + + Examples: + >>> import paddle.fluid as fluid + >>> import paddle.fluid.layers as layers + >>> + >>> vocab_size, hidden_size=10000, 200 + >>> x = layers.data(name="x", shape=[-1, 1, 1], dtype='int64') + >>> x_emb = layers.embedding( + >>> input=x, + >>> size=[vocab_size, hidden_size], + >>> dtype='float32', + >>> is_sparse=False) + >>> x_emb = layers.transpose(x_emb, perm=[1, 0, 2]) + >>> + >>> rnn = fluid.layers.StaticRNN() + >>> with rnn.step(): + >>> word = rnn.step_input(x_emb) + >>> prev = rnn.memory(shape=[-1, hidden_size], batch_ref = word) + >>> hidden = fluid.layers.fc(input=[word, prev], size=hidden_size, act='relu') + >>> rnn.update_memory(prev, hidden) # set prev to hidden + >>> rnn.step_output(hidden) + >>> + >>> result = rnn() + + The StaticRNN will unfold sequence into time steps. Users need to define + how to process each time step during the :code:`with` step. + + The :code:`memory` is used as a staging data cross time step. The initial + value of memory can be a variable that is filled with a constant value or + a specified variable. + + The StaticRNN can mark multiple variables as its output. Use `rnn()` to + get the output sequence. """ BEFORE_RNN_BLOCK = 0 IN_RNN_BLOCK = 1 @@ -284,6 +320,9 @@ class StaticRNN(object): self.seq_len = None def step(self): + """ + The block for user to define operators in RNN. + """ return BlockGuardWithCompletion(self) def _assert_in_rnn_block_(self, method): @@ -298,13 +337,28 @@ class StaticRNN(object): init_batch_dim_idx=0, ref_batch_dim_idx=1): """ + Create a memory variable for static rnn. + + If the :code:`init` is not None, :code:`memory` will be initialized by + this Variable. If the :code:`init` is None, :code:`shape` and :code:`batch_ref` + must be set, and this function will initialize a :code:`init` Variable. + Args: - init: boot memory, if not set, a shape, batch_ref must be provided - shape: shape of the boot memory - batch_ref: batch size reference variable - init_value: the init value of boot memory - init_batch_dim_idx: the index of batch size in init's dimension - ref_batch_dim_idx: the index of batch size in batch_ref's dimension + init(Variable|None): The initialized variable. If it is not set, + :code:`shape` and :code:`batch_ref` must be provided. + Default: None. + shape(list|tuple): The shape of the boot memory. NOTE the shape + does not contain batch_size. Default: None. + batch_ref(Variable|None): The batch size reference Variable. + Default: None. + init_value(float): the init value of boot memory. Default: 0.0. + init_batch_dim_idx(int): the batch_size axis of the + :code:`init` Variable. Default: 0. + ref_batch_dim_idx(int): the batch_size axis of the + :code:`batch_ref` Variable. Default: 1. + + Returns: + The memory variable. """ self._assert_in_rnn_block_('memory') if init is None: @@ -343,6 +397,16 @@ class StaticRNN(object): return pre_mem def step_input(self, x): + """ + Mark a sequence as a StaticRNN input. + + Args: + x(Variable): The input sequence, the shape of x + should be [seq_len, ...]. + + Returns: + The current time step in the input sequence. + """ self._assert_in_rnn_block_('step_input') if not isinstance(x, Variable): raise TypeError("step input takes a Variable") @@ -357,6 +421,15 @@ class StaticRNN(object): return ipt def step_output(self, o): + """ + Mark a sequence as a StaticRNN output. + + Args: + o(Variable): The output sequence. + + Returns: + None. + """ self._assert_in_rnn_block_('step_output') if not isinstance(o, Variable): raise TypeError("step output takes a Variable") @@ -376,10 +449,30 @@ class StaticRNN(object): self.outputs.append(out_var) def output(self, *outputs): + """ + Mark the StaticRNN output variables. + + Args: + outputs: The output Variables. + + Returns: + None + """ for each in outputs: self.step_output(each) def update_memory(self, mem, var): + """ + Update the memory from ex_mem to new_mem. NOTE that the shape and data + type of :code:`ex_mem` and :code:`new_mem` must be same. + + Args: + mem(Variable): the memory variable. + var(Variable): the plain variable generated in RNN block. + + Returns: + None + """ if not isinstance(mem, Variable) or not isinstance(var, Variable): raise TypeError("update memory should take variables") self.memories[mem.name].mem = var @@ -419,6 +512,9 @@ class StaticRNN(object): for m in self.memories: local_inputs.add(m) + # NOTE(zcd): the params have two categories of variables. + # - the variables that are the out of StaticRnn. + # - the variables that are the parameters of some layers, for example, conv2d. params = list() for op in rnn_block.ops: assert isinstance(op, Operator) @@ -435,17 +531,19 @@ class StaticRNN(object): inlinks = [parent_block.var(i.name) for i in self.inputs] outlinks = self.outputs + # NOTE(zcd): the states maybe empty in some case. boot_memories = [] pre_memories = [] memories = [] for _, mem in six.iteritems(self.memories): boot_memories.append(mem.init) pre_memories.append(mem.pre_mem.name) + assert mem.mem is not None, "%s should be updated in every step." % ( + mem.init.name) mem_var = rnn_block.var(mem.mem.name) assert isinstance(mem_var, Variable) new_mem = self.helper.create_variable_for_type_inference( dtype=mem_var.dtype) - rnn_block.append_op( type='rnn_memory_helper', inputs={'X': [mem_var]}, @@ -464,6 +562,7 @@ class StaticRNN(object): outputs={'outputs': outlinks, 'step_scopes': [step_scope]}, attrs={ + 'has_states': len(pre_memories) > 0, 'ex_states': pre_memories, 'states': memories, 'sub_block': rnn_block diff --git a/python/paddle/fluid/layers/learning_rate_scheduler.py b/python/paddle/fluid/layers/learning_rate_scheduler.py index b7d1eeba80d93d549a019455087bb7cc1d2a1083..a67c8058f2c42713738420e81316452e15acb697 100644 --- a/python/paddle/fluid/layers/learning_rate_scheduler.py +++ b/python/paddle/fluid/layers/learning_rate_scheduler.py @@ -35,8 +35,8 @@ from ..dygraph import learning_rate_scheduler as imperate_lr __all__ = [ 'exponential_decay', 'natural_exp_decay', 'inverse_time_decay', - 'polynomial_decay', 'piecewise_decay', 'noam_decay', 'append_LARS', - 'cosine_decay', 'linear_lr_warmup' + 'polynomial_decay', 'piecewise_decay', 'noam_decay', 'cosine_decay', + 'linear_lr_warmup' ] @@ -349,24 +349,26 @@ def cosine_decay(learning_rate, step_each_epoch, epochs): training progresses. By using this function, the learning rate will be decayed by following cosine decay strategy. - decayed_lr = learning_rate * 0.5 * (math.cos(epoch * math.pi / epochs) + 1) + .. math:: + + decayed\_lr = learning\_rate * 0.5 * (math.cos * (epoch * \\frac{math.pi}{epochs} ) + 1) Args: learning_rate(Variable|float): The initial learning rate. step_each_epoch(int): the number of steps in an epoch. epochs(int): the number of epochs. - Returns: - Variable: The decayed learning rate. - - Examples: + Returns: + Variable: The decayed learning rate. - ..code-block:: python + Examples: + .. code-block:: python - base_lr = 0.1 - lr = fluid.layers.cosine_decay( - learning_rate = base_lr, step_each_epoch=10000, epochs=120) + base_lr = 0.1 + lr = fluid.layers.cosine_decay( + learning_rate = base_lr, step_each_epoch=10000, epochs=120) """ + with default_main_program()._lr_schedule_guard(): if imperative_base.enabled(): decay = imperate_lr.CosineDecay(learning_rate, step_each_epoch, @@ -381,50 +383,6 @@ def cosine_decay(learning_rate, step_each_epoch, epochs): return decayed_lr -def append_LARS(params_grads, learning_rate, weight_decay): - """ - Applies LARS (LAYER-WISE ADAPTIVE RATE SCALING) to learning rate for - each layer. - - Args: - learning_rate: A learning rate Variable. This - is the global learning rate for LARS. - weight_decay: A Python `float` number. - - Returns: - The decayed learning rate - Examples: - .. code-block:: python - - learning_rate *= local_gw_ratio * sqrt(sumsq(param)) - / (sqrt(sumsq(gradient))+ weight_decay * sqrt(sumsq(param))) - """ - - assert not imperative_base.enabled( - ), "append_LARS is NOT supported in dygraph mode now" - - def _balanced_weight(param_norm, grad_norm): - if weight_decay == 1.0: - return grad_norm + param_norm - else: - return grad_norm + weight_decay * param_norm - - for param, grad in params_grads: - with param.block.program.optimized_guard( - [param, grad]), name_scope("optimizer"): - param_lr = param.optimize_attr['learning_rate'] - param_norm = ops.sqrt(nn.reduce_sum(input=ops.square(param))) - grad_norm = ops.sqrt(nn.reduce_sum(input=ops.square(grad))) - if type(param_lr) == float and param_lr == 1.0: - decayed_lr = learning_rate * param_norm \ - / _balanced_weight(param_norm, grad_norm) - else: - decayed_lr = learning_rate * param_lr * param_norm \ - / _balanced_weight(param_norm, grad_norm) - # set back param local learning rate - param.optimize_attr['learning_rate'] = decayed_lr - - def linear_lr_warmup(learning_rate, warmup_steps, start_lr, end_lr): """ Applies linear learning rate warmup before the normal learning rate diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 91414fdeb207781afd5e28afa5a3fa6e1018efb1..93e46eef16fb177169db679a8437d9a33ed38e99 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -23,7 +23,7 @@ import os import inspect from ..layer_helper import LayerHelper from ..initializer import Normal, Constant, NumpyArrayInitializer -from ..framework import Variable, OpProtoHolder, _in_dygraph_mode +from ..framework import Variable, OpProtoHolder, in_dygraph_mode from ..dygraph import base from ..param_attr import ParamAttr from .layer_function_generator import autodoc, templatedoc, _generate_doc_string_ @@ -73,6 +73,8 @@ __all__ = [ 'reduce_max', 'reduce_min', 'reduce_prod', + 'reduce_all', + 'reduce_any', 'sequence_first_step', 'sequence_last_step', 'sequence_slice', @@ -159,6 +161,7 @@ __all__ = [ 'sum', 'slice', 'shape', + 'rank', 'logical_and', 'logical_or', 'logical_xor', @@ -191,6 +194,7 @@ __all__ = [ 'kldiv_loss', 'tree_conv', 'npair_loss', + 'pixel_shuffle', 'fsp_matrix', ] @@ -480,6 +484,8 @@ def dynamic_lstm(input, forward, _ = fluid.layers.dynamic_lstm( input=forward_proj, size=hidden_dim * 4, use_peepholes=False) """ + assert in_dygraph_mode( + ) is not True, "please use lstm instead of dynamic_lstm in dygraph mode!" assert bias_attr is not False, "bias_attr should not be False in dynamic_lstmp." helper = LayerHelper('lstm', **locals()) size = size // 4 @@ -864,6 +870,9 @@ def dynamic_lstmp(input, proj_activation="tanh") """ + assert in_dygraph_mode( + ) is not True, "please use lstm instead of dynamic_lstmp in dygraph mode!" + assert bias_attr is not False, "bias_attr should not be False in dynamic_lstmp." helper = LayerHelper('lstmp', **locals()) size = size // 4 @@ -1035,6 +1044,9 @@ def dynamic_gru(input, hidden = fluid.layers.dynamic_gru(input=x, size=hidden_dim) """ + assert in_dygraph_mode( + ) is not True, "please use gru instead of dynamic_gru in dygraph mode!" + helper = LayerHelper('gru', **locals()) dtype = helper.input_dtype() @@ -1751,6 +1763,8 @@ def sequence_conv(input, Variable: output of sequence_conv """ + assert not in_dygraph_mode(), ( + "sequence layer is not supported in dygraph mode yet.") helper = LayerHelper('sequence_conv', **locals()) dtype = helper.input_dtype() filter_shape = [filter_size * input.shape[1], num_filters] @@ -1810,6 +1824,8 @@ def sequence_softmax(input, use_cudnn=False, name=None): dtype='float32', lod_level=1) x_sequence_softmax = fluid.layers.sequence_softmax(input=x) """ + assert not in_dygraph_mode(), ( + "sequence layer is not supported in dygraph mode yet.") helper = LayerHelper('sequence_softmax', **locals()) dtype = helper.input_dtype() softmax_out = helper.create_variable_for_type_inference(dtype) @@ -2302,6 +2318,8 @@ def sequence_pool(input, pool_type, is_test=False): last_x = fluid.layers.sequence_pool(input=x, pool_type='last') first_x = fluid.layers.sequence_pool(input=x, pool_type='first') """ + assert not in_dygraph_mode(), ( + "sequence layer is not supported in dygraph mode yet.") helper = LayerHelper('sequence_pool', **locals()) dtype = helper.input_dtype() pool_out = helper.create_variable_for_type_inference(dtype) @@ -2341,6 +2359,8 @@ def sequence_concat(input, name=None): out = fluid.layers.sequence_concat(input=[seq1, seq2, seq3]) """ + assert not in_dygraph_mode(), ( + "sequence layer is not supported in dygraph mode yet.") helper = LayerHelper('sequence_concat', **locals()) out = helper.create_variable_for_type_inference(dtype=helper.input_dtype()) helper.append_op( @@ -2468,6 +2488,8 @@ def sequence_slice(input, offset, length, name=None): subseqs = fluid.layers.sequence_slice(input=seqs, offset=offset, length=length) """ + assert not in_dygraph_mode(), ( + "sequence layer is not supported in dygraph mode yet.") helper = LayerHelper("sequence_slice", **locals()) dtype = helper.input_dtype() out = helper.create_variable_for_type_inference(dtype) @@ -3288,7 +3310,7 @@ def layer_norm(input, >>> dtype='float32') >>> x = fluid.layers.layer_norm(input=data, begin_norm_axis=1) """ - assert _in_dygraph_mode( + assert in_dygraph_mode( ) is not True, "please use FC instead of fc in dygraph mode!" helper = LayerHelper('layer_norm', **locals()) dtype = helper.input_dtype() @@ -3927,6 +3949,8 @@ def sequence_expand(x, y, ref_level=-1, name=None): dtype='float32', lod_level=1) out = layers.sequence_expand(x=x, y=y, ref_level=0) """ + assert not in_dygraph_mode(), ( + "sequence layer is not supported in dygraph mode yet.") helper = LayerHelper('sequence_expand', input=x, **locals()) dtype = helper.input_dtype() tmp = helper.create_variable_for_type_inference(dtype) @@ -3993,6 +4017,8 @@ def sequence_expand_as(x, y, name=None): dtype='float32', lod_level=1) out = layers.sequence_expand_as(x=x, y=y) """ + assert not in_dygraph_mode(), ( + "sequence layer is not supported in dygraph mode yet.") helper = LayerHelper('sequence_expand_as', input=x, **locals()) dtype = helper.input_dtype() tmp = helper.create_variable_for_type_inference(dtype) @@ -4039,6 +4065,8 @@ def sequence_pad(x, pad_value, maxlen=None, name=None): out = fluid.layers.sequence_pad(x=x, pad_value=pad_value) """ + assert not in_dygraph_mode(), ( + "sequence layer is not supported in dygraph mode yet.") helper = LayerHelper('sequence_pad', input=x, **locals()) dtype = helper.input_dtype() out = helper.create_variable_for_type_inference(dtype) @@ -4105,6 +4133,8 @@ def sequence_unpad(x, length, name=None): out = fluid.layers.sequence_unpad(x=x, length=len) """ + assert not in_dygraph_mode(), ( + "sequence layer is not supported in dygraph mode yet.") helper = LayerHelper('sequence_unpad', input=x, **locals()) dtype = helper.input_dtype() out = helper.create_variable_for_type_inference(dtype) @@ -4711,6 +4741,106 @@ def reduce_prod(input, dim=None, keep_dim=False, name=None): return out +def reduce_all(input, dim=None, keep_dim=False, name=None): + """ + Computes the ``logical and`` of tensor elements over the given dimension. + + Args: + input (Variable): The input variable which is a Tensor or LoDTensor. + dim (list|int|None): The dimension along which the logical and is computed. + If :attr:`None`, compute the logical and over all elements of + :attr:`input` and return a Tensor variable with a single element, + otherwise must be in the range :math:`[-rank(input), rank(input))`. + If :math:`dim[i] < 0`, the dimension to reduce is :math:`rank + dim[i]`. + keep_dim (bool): Whether to reserve the reduced dimension in the + output Tensor. The result tensor will have one fewer dimension + than the :attr:`input` unless :attr:`keep_dim` is true. + name(str|None): A name for this layer(optional). If set None, the layer + will be named automatically. + + Returns: + Variable: The reduced Tensor variable. + + Examples: + .. code-block:: python + + # x is a bool Tensor variable with following elements: + # [[True, False] + # [True, True]] + # Each example is followed by the correspending output tensor. + fluid.layers.reduce_all(x) # False + fluid.layers.reduce_all(x, dim=0) # [True, False] + fluid.layers.reduce_all(x, dim=-1) # [False, True] + fluid.layers.reduce_all(x, dim=1, + keep_dim=True) # [[False], [True]] + + """ + helper = LayerHelper('reduce_all', **locals()) + out = helper.create_variable_for_type_inference(dtype=helper.input_dtype()) + if dim is not None and not isinstance(dim, list): + dim = [dim] + helper.append_op( + type='reduce_all', + inputs={'X': input}, + outputs={'Out': out}, + attrs={ + 'dim': dim if dim != None else [0], + 'keep_dim': keep_dim, + 'reduce_all': True if dim == None else False + }) + return out + + +def reduce_any(input, dim=None, keep_dim=False, name=None): + """ + Computes the ``logical or`` of tensor elements over the given dimension. + + Args: + input (Variable): The input variable which is a Tensor or LoDTensor. + dim (list|int|None): The dimension along which the logical or is computed. + If :attr:`None`, compute the logical or over all elements of + :attr:`input` and return a Tensor variable with a single element, + otherwise must be in the range :math:`[-rank(input), rank(input))`. + If :math:`dim[i] < 0`, the dimension to reduce is :math:`rank + dim[i]`. + keep_dim (bool): Whether to reserve the reduced dimension in the + output Tensor. The result tensor will have one fewer dimension + than the :attr:`input` unless :attr:`keep_dim` is true. + name(str|None): A name for this layer(optional). If set None, the layer + will be named automatically. + + Returns: + Variable: The reduced Tensor variable. + + Examples: + .. code-block:: python + + # x is a bool Tensor variable with following elements: + # [[True, False] + # [False, False]] + # Each example is followed by the correspending output tensor. + fluid.layers.reduce_any(x) # True + fluid.layers.reduce_any(x, dim=0) # [True, False] + fluid.layers.reduce_any(x, dim=-1) # [True, False] + fluid.layers.reduce_any(x, dim=1, + keep_dim=True) # [[True], [False]] + + """ + helper = LayerHelper('reduce_any', **locals()) + out = helper.create_variable_for_type_inference(dtype=helper.input_dtype()) + if dim is not None and not isinstance(dim, list): + dim = [dim] + helper.append_op( + type='reduce_any', + inputs={'X': input}, + outputs={'Out': out}, + attrs={ + 'dim': dim if dim != None else [0], + 'keep_dim': keep_dim, + 'reduce_all': True if dim == None else False + }) + return out + + def split(input, num_or_sections, dim=-1, name=None): """ Split the input tensor into multiple sub-tensors. @@ -4792,7 +4922,7 @@ def l2_normalize(x, axis, epsilon=1e-12, name=None): the dimension to normalization is rank(X) + axis. -1 is the last dimension. epsilon(float): The epsilon value is used to avoid division by zero, \ - the defalut value is 1e-10. + the defalut value is 1e-12. name(str|None): A name for this layer(optional). If set None, the layer \ will be named automatically. @@ -5278,6 +5408,8 @@ def sequence_reshape(input, new_dim): x = fluid.layers.data(shape=[5, 20], dtype='float32', lod_level=1) x_reshaped = fluid.layers.sequence_reshape(input=x, new_dim=10) """ + assert not in_dygraph_mode(), ( + "sequence layer is not supported in dygraph mode yet.") helper = LayerHelper('sequence_reshape', **locals()) out = helper.create_variable_for_type_inference(helper.input_dtype()) helper.append_op( @@ -5812,6 +5944,8 @@ def im2sequence(input, input=layer, stride=[1, 1], filter_size=[2, 2]) """ + assert not in_dygraph_mode(), ( + "sequence layer is not supported in dygraph mode yet.") if isinstance(filter_size, int): filter_size = [filter_size, filter_size] @@ -6228,7 +6362,7 @@ def smooth_l1(x, y, inside_weight=None, outside_weight=None, sigma=None): }, outputs={'Diff': diff, 'Out': loss}, - attrs={'sigma': sigma}) + attrs={'sigma': sigma if sigma is not None else 1.0}) return loss @@ -6454,7 +6588,7 @@ def squeeze(input, axes, name=None): x = layers.data(name='x', shape=[5, 1, 10]) y = layers.sequeeze(input=x, axes=[1]) """ - assert not _in_dygraph_mode(), ( + assert not in_dygraph_mode(), ( "squeeze layer is not supported in dygraph mode yet.") helper = LayerHelper("squeeze", **locals()) out = helper.create_variable_for_type_inference(dtype=input.dtype) @@ -7107,10 +7241,10 @@ def image_resize(input, out_shape(list|tuple|Variable|None): Output shape of image resize layer, the shape is (out_h, out_w). Default: None - scale(float|None): The multiplier for the input height or width. - At least one of out_shape or scale must be set. - And out_shape has a higher priority than scale. - Default: None + scale(float|None): The multiplier for the input height or width. At + least one of :attr:`out_shape` or :attr:`scale` must be set. + And :attr:`out_shape` has a higher priority than :attr:`scale`. + Default: None. name(str|None): A name for this layer(optional). If set None, the layer will be named automatically. resample(str): The resample method. It supports 'BILINEAR' and 'NEAREST' @@ -7148,6 +7282,7 @@ def image_resize(input, or 'NEAREST' currently. ValueError: One of out_shape and scale must not be None. ValueError: out_shape length should be 2. + ValueError: scale should be greater than zero. TypeError: align_corners shoule be a bool value ValueError: align_mode can only be '0' or '1' @@ -7179,26 +7314,36 @@ def image_resize(input, def _is_list_or_turple_(data): return (isinstance(data, list) or isinstance(data, tuple)) - out_h = 0 - out_w = 0 inputs = {"X": input} + attrs = { + "out_h": 0, + "out_w": 0, + "interp_method": resample_type, + "align_corners": align_corners, + "align_mode": align_mode + } + if out_shape is not None: if isinstance(out_shape, Variable): warnings.warn("out_shape as Variable type is deprecated, \ it is recommended to use actual_shape instead of \ out_shape to specify output shape dynamically.") inputs['OutSize'] = out_shape - elif not (_is_list_or_turple_(out_shape)): - raise TypeError("out_shape should be a list or tuple or Variable.") - elif len(out_shape) != 2: - raise ValueError("out_shape length should be 2.") - - out_shape = list(map(int, out_shape)) - out_h = out_shape[0] - out_w = out_shape[1] + else: + if not (_is_list_or_turple_(out_shape)): + raise TypeError( + "out_shape should be a list or tuple or Variable.") + if len(out_shape) != 2: + raise ValueError("out_shape length should be 2.") + + out_shape = list(map(int, out_shape)) + attrs['out_h'] = out_shape[0] + attrs['out_w'] = out_shape[1] + else: - out_h = int(input.shape[2] * scale) - out_w = int(input.shape[3] * scale) + if scale <= 0: + raise ValueError("scale should be greater than zero.") + attrs['scale'] = float(scale) if isinstance(actual_shape, Variable): inputs["OutSize"] = actual_shape @@ -7210,13 +7355,7 @@ def image_resize(input, type='{}_interp'.format(resample_type), inputs=inputs, outputs={"Out": out}, - attrs={ - "out_h": out_h, - "out_w": out_w, - "interp_method": resample_type, - "align_corners": align_corners, - "align_mode": align_mode - }) + attrs=attrs) return out @@ -7284,11 +7423,14 @@ def resize_bilinear(input, Args: input(${x_type}): ${x_comment}. - out_shape(${out_size_type}): ${out_size_comment}. + out_shape(list|tuple|Variable|None): Output shape of resize bilinear + layer, the shape is (out_h, out_w). + Default: None scale(float|None): The multiplier for the input height or width. At - least one of out_shape or scale must be set. And out_shape has - a higher priority than scale. Default: None. + least one of :attr:`out_shape` or :attr:`scale` must be set. + And :attr:`out_shape` has a higher priority than :attr:`scale`. + Default: None. name(str|None): The output variable name. actual_shape(Variable): An optional input to specify output shape @@ -7375,11 +7517,14 @@ def resize_nearest(input, Args: input(${x_type}): ${x_comment}. - out_shape(${out_size_type}): ${out_size_comment}. + out_shape(list|tuple|Variable|None): Output shape of resize nearest + layer, the shape is (out_h, out_w). + Default: None scale(float|None): The multiplier for the input height or width. At - least one of out_shape or scale must be set. And out_shape has - a higher priority than scale. Default: None. + least one of :attr:`out_shape` or :attr:`scale` must be set. + And :attr:`out_shape` has a higher priority than :attr:`scale`. + Default: None. name(str|None): The output variable name. actual_shape(Variable): An optional input to specify output shape @@ -7589,6 +7734,8 @@ def sequence_scatter(input, index, updates, name=None): output = fluid.layers.sequence_scatter(input, index, updates) """ + assert not in_dygraph_mode(), ( + "sequence layer is not supported in dygraph mode yet.") helper = LayerHelper('sequence_scatter', **locals()) dtype = helper.input_dtype() out = helper.create_variable_for_type_inference(dtype) @@ -8677,6 +8824,8 @@ def sequence_enumerate(input, win_size, pad_value=0, name=None): x = fluid.layers.data(shape[30, 1], dtype='int32', lod_level=1) out = fluid.layers.sequence_enumerate(input=x, win_size=3, pad_value=0) """ + assert not in_dygraph_mode(), ( + "sequence layer is not supported in dygraph mode yet.") helper = LayerHelper('sequence_enumerate', **locals()) out = helper.create_variable_for_type_inference( helper.input_dtype(), stop_gradient=True) @@ -8716,6 +8865,8 @@ def sequence_mask(x, maxlen=None, dtype='int64', name=None): Variable: The output sequence mask. """ + assert not in_dygraph_mode(), ( + "sequence layer is not supported in dygraph mode yet.") helper = LayerHelper('sequence_mask', **locals()) if name is None: @@ -9189,11 +9340,37 @@ def shape(input): return out +def rank(input): + """ + **Rank Layer** + + Returns the number of dimensions for a tensor, which is a 0-D int32 Tensor. + + Args: + input (Variable): The input variable. + + Returns: + Variable: The rank of the input variable. + + Examples: + .. code-block:: python + + input = layers.data( + name="input", shape=[3, 100, 100], dtype="float32") + rank = layers.rank(input) # 4 + """ + + ndims = len(input.shape) + out = assign(np.array(ndims, 'int32')) + + return out + + def _elementwise_op(helper): op_type = helper.layer_type x = helper.kwargs.get('x', None) y = helper.kwargs.get('y', None) - if _in_dygraph_mode(): + if in_dygraph_mode(): x = base.to_variable(x) y = base.to_variable(y) @@ -9766,6 +9943,8 @@ def sequence_reverse(x, name=None): Returns: out(${y_type}): ${y_comment} """ + assert not in_dygraph_mode(), ( + "sequence layer is not supported in dygraph mode yet.") helper = LayerHelper("sequence_reverse", **locals()) if name is None: out = helper.create_variable_for_type_inference(dtype=x.dtype) @@ -10923,6 +11102,65 @@ def npair_loss(anchor, positive, labels, l2_reg=0.002): return l2loss + celoss +def pixel_shuffle(x, upscale_factor): + """ + + **Pixel Shuffle Layer** + + This layer rearranges elements in a tensor of shape [N, C, H, W] + to a tensor of shape [N, C/r**2, H*r, W*r]. + This is useful for implementing efficient sub-pixel convolution + with a stride of 1/r. + Please refer to the paper: `Real-Time Single Image and Video Super-Resolution + Using an Efficient Sub-Pixel Convolutional Neural Network `_ . + by Shi et. al (2016) for more details. + + .. code-block:: text + + Given a 4-D tensor with the shape: + x.shape = [1, 9, 4, 4] + Given upscale_factor: + upscale_factor= 3 + output shape is: + [1, 1, 12, 12] + + Args: + + x(Variable): The input tensor variable. + upscale_factor(int): factor to increase spatial resolution + + Returns: + + Out(Variable): Reshaped tensor according to the new dimension. + + Raises: + + ValueError: If the square of upscale_factor cannot divide the channels of input. + + Examples: + + .. code-block:: python + + input = fluid.layers.data(shape=[9,4,4]) + output = fluid.layers.pixel_shuffle(x=input, upscale_factor=3) + + """ + + helper = LayerHelper("pixel_shuffle", **locals()) + + out = helper.create_variable_for_type_inference(dtype=x.dtype) + + if not isinstance(upscale_factor, int): + raise TypeError("upscale factor must be int type") + + helper.append_op( + type="pixel_shuffle", + inputs={"X": x}, + outputs={"Out": out}, + attrs={"upscale_factor": upscale_factor}) + return out + + def fsp_matrix(x, y): """ diff --git a/python/paddle/fluid/layers/tensor.py b/python/paddle/fluid/layers/tensor.py index 80450119f44e93aae4b483983484ea18be5b2035..03ebd41fa00c69bfce66d325e32fc9aeb25a2486 100644 --- a/python/paddle/fluid/layers/tensor.py +++ b/python/paddle/fluid/layers/tensor.py @@ -24,26 +24,11 @@ from .layer_function_generator import templatedoc import numpy __all__ = [ - 'create_tensor', - 'create_parameter', - 'create_global_var', - 'cast', - 'tensor_array_to_tensor', - 'concat', - 'sums', - 'assign', - 'fill_constant_batch_size_like', - 'fill_constant', - 'argmin', - 'argmax', - 'argsort', - 'ones', - 'zeros', - 'reverse', - 'has_inf', - 'has_nan', - 'isfinite', - 'range', + 'create_tensor', 'create_parameter', 'create_global_var', 'cast', + 'tensor_array_to_tensor', 'concat', 'sums', 'assign', + 'fill_constant_batch_size_like', 'fill_constant', 'argmin', 'argmax', + 'argsort', 'ones', 'zeros', 'reverse', 'has_inf', 'has_nan', 'isfinite', + 'range', 'linspace' ] @@ -826,3 +811,45 @@ def range(start, end, step, dtype): 'Step': step}, outputs={'Out': [out]}) return out + + +def linspace(start, stop, num, dtype): + """ + Return fixed number of evenly spaced values within a given interval. + + First entry is start, and last entry is stop. In the case when Num is 1, only Start is returned. Like linspace function of numpy. + + Args: + start(float|Variable): First entry in the sequence. It is a float scalar, or a tensor of shape [1] with type 'float32'|'float64'. + stop(float|Variable): Last entry in the sequence. It is a float scalar, or a tensor of shape [1] with type 'float32'|'float64'. + num(int|Variable): Number of entry in the sequence. It is an int scalar, or a tensor of shape [1] with type int32. + dtype(string): 'float32'|'float64', the data type of the output tensor. + + Returns: + Variable: The tensor variable storing a 1-D tensor. + + Examples: + .. code-block:: python + + data = fluid.layers.linspace(0, 10, 5, 'float32') # [0.0, 2.5, 5.0, 7.5, 10.0] + data = fluid.layers.linspace(0, 10, 1, 'float32') # [0.0] + + """ + helper = LayerHelper("linspace", **locals()) + + if not isinstance(start, Variable): + start = fill_constant([1], dtype, start) + if not isinstance(stop, Variable): + stop = fill_constant([1], dtype, stop) + if not isinstance(num, Variable): + num = fill_constant([1], 'int32', num) + + out = helper.create_variable_for_type_inference(dtype=start.dtype) + + helper.append_op( + type='linspace', + inputs={'Start': start, + 'Stop': stop, + 'Num': num}, + outputs={'Out': [out]}) + return out diff --git a/python/paddle/fluid/optimizer.py b/python/paddle/fluid/optimizer.py index 7e6e37116fe23f26eb14dd0573dbe031aec98dd8..a375ba657a6152c6e9fb67b8990ea85925e6670a 100644 --- a/python/paddle/fluid/optimizer.py +++ b/python/paddle/fluid/optimizer.py @@ -55,7 +55,7 @@ class Optimizer(object): """ def __init__(self, learning_rate, regularization=None, name=None): - if framework._in_dygraph_mode(): + if framework.in_dygraph_mode(): if not isinstance(learning_rate, float) and \ not isinstance(learning_rate, LearningRateDecay): raise TypeError( @@ -205,7 +205,7 @@ class Optimizer(object): name = self._name + "_" + name if (name in self._accumulators and param.name in self._accumulators[name]): - if framework._in_dygraph_mode(): + if framework.in_dygraph_mode(): return self._accumulators[name][param.name] raise Exception("Accumulator {} already exists for parameter {}". format(name, param.name)) @@ -363,7 +363,7 @@ class Optimizer(object): See examples in `apply_gradients`. """ self._dtype = loss.dtype - if framework._in_dygraph_mode(): + if framework.in_dygraph_mode(): if parameter_list is not None: parameters = parameter_list else: @@ -448,7 +448,7 @@ class Optimizer(object): Returns: list: A list of operators appended to the current program. """ - if framework._in_dygraph_mode(): + if framework.in_dygraph_mode(): with program_guard(framework.default_main_program(), framework.default_startup_program()): optimize_ops = self._create_optimization_pass(params_grads) @@ -628,16 +628,16 @@ class DGCMomentumOptimizer(MomentumOptimizer): Original paper is https://arxiv.org/abs/1712.01887 - DGC reduce the communication bandwidth by sending only the important gradients (sparse update):\ + DGC reduces the communication bandwidth by sending only the important gradients (sparse update):\ only gradients larger than a threshold are transmitted. - To avoid losing information, DGC accumulate the rest of the gradients locally. + To avoid losing information, DGC accumulates the rest of the gradients locally. Eventually, these gradients become large enough to be transmitted. - Thus, DGC send the large gradients immediately but eventually send all of the gradients over time. + Thus, DGC sends the large gradients immediately but eventually send all of the gradients over time. - To ensure no loss of accuracy, DGC employs momentum correc-tionandlocal gradient clipping on top of the gradient sparsification to maintain model performance. + To ensure no loss of accuracy, DGC employs momentum correction and local gradient clipping on top of the gradient sparsification to maintain model performance. DGC also uses momentum factor masking and warmup training to overcome the staleness problem caused by reduced communication. @@ -652,7 +652,7 @@ class DGCMomentumOptimizer(MomentumOptimizer): learning_rate (float|Variable): the learning rate used to update parameters. \ Can be a float value or a Variable with one float value as data element. momentum (float): Momentum factor. - rampup_begin_step (int): The begining step from which gradient compression is implemented. + rampup_begin_step (int): The beginning step from which gradient compression is implemented. rampup_step (int): How long it use the sparsity periods. Default is 1. for example: If the sparsity is [0.75, 0.9375, 0.984375, 0.996, 0.999], and the rampup_step is 5, \ it will use 0.75 at 0 step, and 0.9375 at 1 step, and so on. And when reach sparsity array ends, \ @@ -660,9 +660,9 @@ class DGCMomentumOptimizer(MomentumOptimizer): sparsity (list[float]): Get top important element from gradient tensor, the ratio is (1 - current sparsity). use_nesterov (bool): Enables Nesterov momentum. True means use nesterov. local_grad_clip_norm (float): Clip norm value if needed. - num_trainers: The number of training node. + num_trainers: The number of training nodes. regularization: A Regularizer, such as fluid.regularizer.L2DecayRegularizer. - name: A optional name prefix. + name: An optional name prefix. Examples: .. code-block:: python @@ -752,7 +752,7 @@ class DGCMomentumOptimizer(MomentumOptimizer): force_cpu=True) for param_var, grad_var in param_and_grads: - var_numel = reduce(lambda x, y: x * y, param_var.shape) + var_numel = abs(reduce(lambda x, y: x * y, param_var.shape)) if var_numel < 16384 or \ param_var.type == core.VarDesc.VarType.SELECTED_ROWS or \ grad_var.type == core.VarDesc.VarType.SELECTED_ROWS or \ diff --git a/python/paddle/fluid/parallel_executor.py b/python/paddle/fluid/parallel_executor.py index 6b88e7a99fd78f6a7670ba55bc678e85d229ddf4..092cd5aea7d2f3ae7e5ba927261921fbe28f51bf 100644 --- a/python/paddle/fluid/parallel_executor.py +++ b/python/paddle/fluid/parallel_executor.py @@ -104,6 +104,7 @@ class ParallelExecutor(object): self._scope = scope if scope is not None else executor.global_scope() if main_program is not None and main_program._enable_dgc: + assert num_trainers > 1 assert build_strategy.reduce_strategy == BuildStrategy.ReduceStrategy.AllReduce assert num_trainers * len( self._places) > 1, "dgc is not useful for single card training" @@ -123,6 +124,11 @@ class ParallelExecutor(object): exec_strategy=exec_strategy, share_vars_from=share_vars_from._compiled_program if share_vars_from else None) + + # FIXME(gongwb): I will move dgc from dist mode to allreduce mode in next pr. + if main_program._enable_dgc: + self._compiled_program._build_strategy.is_distribution = True + self._place = core.CUDAPlace(0) if use_cuda else core.CPUPlace() self._exe = executor.Executor(self._place) self._compiled_program._compile(place=self._place, scope=self._scope) diff --git a/python/paddle/fluid/tests/book/CMakeLists.txt b/python/paddle/fluid/tests/book/CMakeLists.txt index ee734f3c782adb5196a03aca5718377009a5b4e7..999a765b6dc32323a24f9069f11134360dbadcb8 100644 --- a/python/paddle/fluid/tests/book/CMakeLists.txt +++ b/python/paddle/fluid/tests/book/CMakeLists.txt @@ -6,4 +6,6 @@ foreach(src ${TEST_OPS}) py_test(${src} SRCS ${src}.py) endforeach() -add_subdirectory(high-level-api) +if(WITH_HIGH_LEVEL_API_TEST) + add_subdirectory(high-level-api) +endif() diff --git a/python/paddle/fluid/tests/book/high-level-api/CMakeLists.txt b/python/paddle/fluid/tests/book/high-level-api/CMakeLists.txt index efa5ee2d06af3d31e7d84122dd7eea37d6dcf3a3..c034709fbdc2aa315ca995a42c278b261e6283a4 100644 --- a/python/paddle/fluid/tests/book/high-level-api/CMakeLists.txt +++ b/python/paddle/fluid/tests/book/high-level-api/CMakeLists.txt @@ -1,16 +1,28 @@ -file(GLOB TEST_OPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "test_*.py") +file(GLOB TEST_OPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "test_*_new_api.py") string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}") -# default test -foreach(src ${TEST_OPS}) - py_test(${src} SRCS ${src}.py) -endforeach() +# This test is buggy +# py_test(test_understand_sentiment_dynamic_rnn SRCS +# test_understand_sentiment_dynamic_rnn.py SERIAL) +LIST(REMOVE_ITEM TEST_OPS test_understand_sentiment_dynamic_rnn_new_api) -add_subdirectory(fit_a_line) -add_subdirectory(recognize_digits) -add_subdirectory(image_classification) -add_subdirectory(understand_sentiment) -add_subdirectory(label_semantic_roles) -add_subdirectory(word2vec) -add_subdirectory(recommender_system) -add_subdirectory(machine_translation) +if(NOT APPLE) + # default test + foreach(src ${TEST_OPS}) + py_test(${src} SRCS ${src}.py) + endforeach() +else() + foreach(src ${TEST_OPS}) + if(${src} STREQUAL "test_image_classification_vgg_new_api") + message(WARNING "These tests has been disabled in OSX for random fail: \n" ${src}) + elseif(${src} STREQUAL "test_image_classification_resnet_new_api") + message(WARNING "These tests has been disabled in OSX for random fail: \n" ${src}) + elseif(${src} STREQUAL "test_recognize_digits_conv_new_api") + message(WARNING "These tests has been disabled in OSX for random fail: \n" ${src}) + elseif(${src} STREQUAL "test_recognize_digits_mlp_new_api") + message(WARNING "These tests has been disabled in OSX for random fail: \n" ${src}) + elseif() + py_test(${src} SRCS ${src}.py) + endif() + endforeach() +endif() diff --git a/python/paddle/fluid/tests/book/high-level-api/image_classification/cifar10_small_test_set.py b/python/paddle/fluid/tests/book/high-level-api/cifar10_small_test_set.py similarity index 100% rename from python/paddle/fluid/tests/book/high-level-api/image_classification/cifar10_small_test_set.py rename to python/paddle/fluid/tests/book/high-level-api/cifar10_small_test_set.py diff --git a/python/paddle/fluid/tests/book/high-level-api/fit_a_line/CMakeLists.txt b/python/paddle/fluid/tests/book/high-level-api/fit_a_line/CMakeLists.txt deleted file mode 100644 index 673c965b662a022739f8d489c331f4de9455a926..0000000000000000000000000000000000000000 --- a/python/paddle/fluid/tests/book/high-level-api/fit_a_line/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -file(GLOB TEST_OPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "test_*.py") -string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}") - -# default test -foreach(src ${TEST_OPS}) - py_test(${src} SRCS ${src}.py) -endforeach() diff --git a/python/paddle/fluid/tests/book/high-level-api/image_classification/CMakeLists.txt b/python/paddle/fluid/tests/book/high-level-api/image_classification/CMakeLists.txt deleted file mode 100644 index 91c1d17eb5391ea37a41a886594cc71c6e6c56bd..0000000000000000000000000000000000000000 --- a/python/paddle/fluid/tests/book/high-level-api/image_classification/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -file(GLOB TEST_OPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "test_*.py") -string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}") - -if(NOT APPLE) - # default test - foreach(src ${TEST_OPS}) - py_test(${src} SRCS ${src}.py) - endforeach() -else() - foreach(src ${TEST_OPS}) - if(${src} STREQUAL "test_image_classification_vgg") - message(WARNING "These tests has been disabled in OSX for random fail: \n" ${src}) - elseif(${src} STREQUAL "test_image_classification_resnet") - message(WARNING "These tests has been disabled in OSX for random fail: \n" ${src}) - elseif() - py_test(${src} SRCS ${src}.py) - endif() - endforeach() -endif() diff --git a/python/paddle/fluid/tests/book/high-level-api/label_semantic_roles/CMakeLists.txt b/python/paddle/fluid/tests/book/high-level-api/label_semantic_roles/CMakeLists.txt deleted file mode 100644 index 673c965b662a022739f8d489c331f4de9455a926..0000000000000000000000000000000000000000 --- a/python/paddle/fluid/tests/book/high-level-api/label_semantic_roles/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -file(GLOB TEST_OPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "test_*.py") -string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}") - -# default test -foreach(src ${TEST_OPS}) - py_test(${src} SRCS ${src}.py) -endforeach() diff --git a/python/paddle/fluid/tests/book/high-level-api/machine_translation/CMakeLists.txt b/python/paddle/fluid/tests/book/high-level-api/machine_translation/CMakeLists.txt deleted file mode 100644 index 673c965b662a022739f8d489c331f4de9455a926..0000000000000000000000000000000000000000 --- a/python/paddle/fluid/tests/book/high-level-api/machine_translation/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -file(GLOB TEST_OPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "test_*.py") -string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}") - -# default test -foreach(src ${TEST_OPS}) - py_test(${src} SRCS ${src}.py) -endforeach() diff --git a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/CMakeLists.txt b/python/paddle/fluid/tests/book/high-level-api/recognize_digits/CMakeLists.txt deleted file mode 100644 index f9c6d60540fcb6f8a73fdc4e68471448e16cbdc2..0000000000000000000000000000000000000000 --- a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -file(GLOB TEST_OPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "test_*.py") -string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}") - -# default test -if(NOT APPLE) - foreach(src ${TEST_OPS}) - py_test(${src} SRCS ${src}.py) - endforeach() -else() - foreach(src ${TEST_OPS}) - if(${src} STREQUAL "test_recognize_digits_conv") - message(WARNING "These tests has been disabled in OSX for random fail: \n" ${src}) - elseif(${src} STREQUAL "test_recognize_digits_mlp") - message(WARNING "These tests has been disabled in OSX for random fail: \n" ${src}) - else() - py_test(${src} SRCS ${src}.py) - endif() - endforeach() -endif() diff --git a/python/paddle/fluid/tests/book/high-level-api/recommender_system/CMakeLists.txt b/python/paddle/fluid/tests/book/high-level-api/recommender_system/CMakeLists.txt deleted file mode 100644 index 673c965b662a022739f8d489c331f4de9455a926..0000000000000000000000000000000000000000 --- a/python/paddle/fluid/tests/book/high-level-api/recommender_system/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -file(GLOB TEST_OPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "test_*.py") -string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}") - -# default test -foreach(src ${TEST_OPS}) - py_test(${src} SRCS ${src}.py) -endforeach() diff --git a/python/paddle/fluid/tests/book/high-level-api/fit_a_line/test_fit_a_line.py b/python/paddle/fluid/tests/book/high-level-api/test_fit_a_line_new_api.py similarity index 100% rename from python/paddle/fluid/tests/book/high-level-api/fit_a_line/test_fit_a_line.py rename to python/paddle/fluid/tests/book/high-level-api/test_fit_a_line_new_api.py diff --git a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_resnet.py b/python/paddle/fluid/tests/book/high-level-api/test_image_classification_resnet_new_api.py similarity index 100% rename from python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_resnet.py rename to python/paddle/fluid/tests/book/high-level-api/test_image_classification_resnet_new_api.py diff --git a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py b/python/paddle/fluid/tests/book/high-level-api/test_image_classification_vgg_new_api.py similarity index 100% rename from python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py rename to python/paddle/fluid/tests/book/high-level-api/test_image_classification_vgg_new_api.py diff --git a/python/paddle/fluid/tests/book/high-level-api/label_semantic_roles/test_label_semantic_roles_newapi.py b/python/paddle/fluid/tests/book/high-level-api/test_label_semantic_roles_new_api.py similarity index 100% rename from python/paddle/fluid/tests/book/high-level-api/label_semantic_roles/test_label_semantic_roles_newapi.py rename to python/paddle/fluid/tests/book/high-level-api/test_label_semantic_roles_new_api.py diff --git a/python/paddle/fluid/tests/book/high-level-api/machine_translation/test_machine_translation.py b/python/paddle/fluid/tests/book/high-level-api/test_machine_translation_new_api.py similarity index 100% rename from python/paddle/fluid/tests/book/high-level-api/machine_translation/test_machine_translation.py rename to python/paddle/fluid/tests/book/high-level-api/test_machine_translation_new_api.py diff --git a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_conv.py b/python/paddle/fluid/tests/book/high-level-api/test_recognize_digits_conv_new_api.py similarity index 100% rename from python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_conv.py rename to python/paddle/fluid/tests/book/high-level-api/test_recognize_digits_conv_new_api.py diff --git a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py b/python/paddle/fluid/tests/book/high-level-api/test_recognize_digits_mlp_new_api.py similarity index 100% rename from python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py rename to python/paddle/fluid/tests/book/high-level-api/test_recognize_digits_mlp_new_api.py diff --git a/python/paddle/fluid/tests/book/high-level-api/recommender_system/test_recommender_system_newapi.py b/python/paddle/fluid/tests/book/high-level-api/test_recommender_system_new_api.py similarity index 100% rename from python/paddle/fluid/tests/book/high-level-api/recommender_system/test_recommender_system_newapi.py rename to python/paddle/fluid/tests/book/high-level-api/test_recommender_system_new_api.py diff --git a/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_conv.py b/python/paddle/fluid/tests/book/high-level-api/test_understand_sentiment_conv_new_api.py similarity index 100% rename from python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_conv.py rename to python/paddle/fluid/tests/book/high-level-api/test_understand_sentiment_conv_new_api.py diff --git a/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_dynamic_rnn.py b/python/paddle/fluid/tests/book/high-level-api/test_understand_sentiment_dynamic_rnn_new_api.py similarity index 100% rename from python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_dynamic_rnn.py rename to python/paddle/fluid/tests/book/high-level-api/test_understand_sentiment_dynamic_rnn_new_api.py diff --git a/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_stacked_lstm.py b/python/paddle/fluid/tests/book/high-level-api/test_understand_sentiment_stacked_lstm_new_api.py similarity index 100% rename from python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_stacked_lstm.py rename to python/paddle/fluid/tests/book/high-level-api/test_understand_sentiment_stacked_lstm_new_api.py diff --git a/python/paddle/fluid/tests/book/high-level-api/word2vec/test_word2vec_new_api.py b/python/paddle/fluid/tests/book/high-level-api/test_word2vec_new_api.py similarity index 100% rename from python/paddle/fluid/tests/book/high-level-api/word2vec/test_word2vec_new_api.py rename to python/paddle/fluid/tests/book/high-level-api/test_word2vec_new_api.py diff --git a/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/CMakeLists.txt b/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/CMakeLists.txt deleted file mode 100644 index d71147a85e77ea6dc5b6391aa169abd9b02a0aa1..0000000000000000000000000000000000000000 --- a/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -file(GLOB TEST_OPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "test_*.py") -string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}") - -# This test is buggy -# py_test(test_understand_sentiment_dynamic_rnn SRCS -# test_understand_sentiment_dynamic_rnn.py SERIAL) -LIST(REMOVE_ITEM TEST_OPS test_understand_sentiment_dynamic_rnn) - -# default test -foreach(src ${TEST_OPS}) - py_test(${src} SRCS ${src}.py) -endforeach() diff --git a/python/paddle/fluid/tests/book/high-level-api/word2vec/CMakeLists.txt b/python/paddle/fluid/tests/book/high-level-api/word2vec/CMakeLists.txt deleted file mode 100644 index 673c965b662a022739f8d489c331f4de9455a926..0000000000000000000000000000000000000000 --- a/python/paddle/fluid/tests/book/high-level-api/word2vec/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -file(GLOB TEST_OPS RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "test_*.py") -string(REPLACE ".py" "" TEST_OPS "${TEST_OPS}") - -# default test -foreach(src ${TEST_OPS}) - py_test(${src} SRCS ${src}.py) -endforeach() diff --git a/python/paddle/fluid/tests/unittests/CMakeLists.txt b/python/paddle/fluid/tests/unittests/CMakeLists.txt index 1390e759d7e309674a2ecb61c59043b0f5032400..65045a4ab20338817908e481e5f990706cc88e71 100644 --- a/python/paddle/fluid/tests/unittests/CMakeLists.txt +++ b/python/paddle/fluid/tests/unittests/CMakeLists.txt @@ -25,7 +25,6 @@ endif() list(REMOVE_ITEM TEST_OPS test_seq_concat_op) # FIXME(helin): https://github.com/PaddlePaddle/Paddle/issues/8290 list(REMOVE_ITEM TEST_OPS test_modified_huber_loss_op) # FIXME(qijun) https://github.com/PaddlePaddle/Paddle/issues/5184 list(REMOVE_ITEM TEST_OPS test_lstm_unit_op) # # FIXME(qijun) https://github.com/PaddlePaddle/Paddle/issues/5185 -list(REMOVE_ITEM TEST_OPS test_recurrent_op) # FIXME(qijun) https://github.com/PaddlePaddle/Paddle/issues/6152 list(REMOVE_ITEM TEST_OPS test_cond_op) # FIXME(qijun): https://github.com/PaddlePaddle/Paddle/issues/5101#issuecomment-339814957 list(REMOVE_ITEM TEST_OPS op_test) # op_test is a helper python file, not a test @@ -74,13 +73,13 @@ list(REMOVE_ITEM TEST_OPS test_dgc_op) list(REMOVE_ITEM TEST_OPS test_dist_se_resnext_nccl) list(REMOVE_ITEM TEST_OPS test_dist_transformer) list(REMOVE_ITEM TEST_OPS test_parallel_executor_transformer) -list(REMOVE_ITEM TEST_OPS test_image_classification_resnet) list(REMOVE_ITEM TEST_OPS test_bilinear_interp_op) list(REMOVE_ITEM TEST_OPS test_nearest_interp_op) list(REMOVE_ITEM TEST_OPS test_imperative_resnet) list(REMOVE_ITEM TEST_OPS test_imperative_se_resnext) list(REMOVE_ITEM TEST_OPS test_imperative_mnist) list(REMOVE_ITEM TEST_OPS test_ir_memory_optimize_transformer) +list(REMOVE_ITEM TEST_OPS test_layers) foreach(TEST_OP ${TEST_OPS}) py_test_modules(${TEST_OP} MODULES ${TEST_OP}) endforeach(TEST_OP) @@ -117,16 +116,13 @@ endif() py_test_modules(test_parallel_executor_crf MODULES test_parallel_executor_crf SERIAL) py_test_modules(test_parallel_executor_fetch_feed MODULES test_parallel_executor_fetch_feed SERIAL) set_tests_properties(test_parallel_executor_fetch_feed PROPERTIES TIMEOUT 450) +set_tests_properties(test_parallel_executor_seresnext PROPERTIES TIMEOUT 740) py_test_modules(test_parallel_executor_transformer MODULES test_parallel_executor_transformer SERIAL) - +py_test_modules(test_layers MODULES test_layers ENVS FLAGS_cudnn_deterministic=1) if(NOT WIN32) py_test_modules(test_ir_memory_optimize_transformer MODULES test_ir_memory_optimize_transformer SERIAL) endif() -if(NOT APPLE) - py_test_modules(test_image_classification_resnet MODULES test_image_classification_resnet SERIAL) -endif() - if(CMAKE_BUILD_TYPE STREQUAL "Debug") # change the timeout from 600 to 2200, because in debug mode, this test need more time. set_tests_properties(test_parallel_executor_seresnext PROPERTIES TIMEOUT 2200) diff --git a/python/paddle/fluid/tests/unittests/fake_reader.py b/python/paddle/fluid/tests/unittests/fake_reader.py new file mode 100644 index 0000000000000000000000000000000000000000..34a256e15dd2f3a8a83aaba4e178efe52c8d8547 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/fake_reader.py @@ -0,0 +1,34 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import six + + +def fake_imdb_reader(word_dict_size, + sample_num, + lower_seq_len=100, + upper_seq_len=200, + class_dim=2): + def __reader__(): + for _ in six.moves.range(sample_num): + length = np.random.random_integers( + low=lower_seq_len, high=upper_seq_len, size=[1])[0] + ids = np.random.random_integers( + low=0, high=word_dict_size - 1, size=[length]).astype('int64') + label = np.random.random_integers( + low=0, high=class_dim - 1, size=[1]).astype('int64')[0] + yield ids, label + + return __reader__ diff --git a/python/paddle/fluid/tests/unittests/parallel_executor_test_base.py b/python/paddle/fluid/tests/unittests/parallel_executor_test_base.py index 18ed02a72275437fa6106e57c0383e17647d9700..723aafb171271ed248c93665a21089029a30a836 100644 --- a/python/paddle/fluid/tests/unittests/parallel_executor_test_base.py +++ b/python/paddle/fluid/tests/unittests/parallel_executor_test_base.py @@ -29,7 +29,8 @@ __all__ = ['TestParallelExecutorBase'] class TestParallelExecutorBase(unittest.TestCase): - def check_network_convergence(self, + @classmethod + def check_network_convergence(cls, method, use_cuda=True, memory_opt=True, diff --git a/python/paddle/fluid/tests/unittests/simple_nets.py b/python/paddle/fluid/tests/unittests/simple_nets.py new file mode 100644 index 0000000000000000000000000000000000000000..20ec6c34c3d5fd4d62e5ffed3bdfe4734f9587ca --- /dev/null +++ b/python/paddle/fluid/tests/unittests/simple_nets.py @@ -0,0 +1,66 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import paddle.fluid as fluid +import numpy as np + + +def simple_fc_net(use_feed=None): + img = fluid.layers.data(name='image', shape=[784], dtype='float32') + label = fluid.layers.data(name='label', shape=[1], dtype='int64') + + hidden = img + for _ in range(4): + hidden = fluid.layers.fc( + hidden, + size=200, + act='relu', + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=1.0))) + prediction = fluid.layers.fc(hidden, size=10, act='softmax') + loss = fluid.layers.cross_entropy(input=prediction, label=label) + loss = fluid.layers.mean(loss) + return loss + + +def fc_with_batchnorm(use_feed=None): + img = fluid.layers.data(name='image', shape=[784], dtype='float32') + label = fluid.layers.data(name='label', shape=[1], dtype='int64') + + hidden = img + for _ in range(2): + hidden = fluid.layers.fc( + hidden, + size=200, + act='relu', + bias_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant(value=1.0))) + + hidden = fluid.layers.batch_norm(input=hidden) + + prediction = fluid.layers.fc(hidden, size=10, act='softmax') + loss = fluid.layers.cross_entropy(input=prediction, label=label) + loss = fluid.layers.mean(loss) + return loss + + +def init_data(batch_size=32, img_shape=[784], label_range=9): + np.random.seed(5) + assert isinstance(img_shape, list) + input_shape = [batch_size] + img_shape + img = np.random.random(size=input_shape).astype(np.float32) + label = np.array( + [np.random.randint(0, label_range) for _ in range(batch_size)]).reshape( + (-1, 1)).astype("int64") + return img, label diff --git a/python/paddle/fluid/tests/unittests/test_base_layer.py b/python/paddle/fluid/tests/unittests/test_base_layer.py index 9cb88d4a8553f3b750f6cf3b24115b4d188ed1d6..04a36f7cafe7b4445125c4e9bd58f6d30d6c71aa 100644 --- a/python/paddle/fluid/tests/unittests/test_base_layer.py +++ b/python/paddle/fluid/tests/unittests/test_base_layer.py @@ -18,7 +18,7 @@ import numpy as np import paddle.fluid as fluid -class L1(fluid.dygraph.Layer): +class L1(fluid.Layer): def __init__(self, prefix): super(L1, self).__init__(prefix) self._param_attr = fluid.ParamAttr( @@ -32,7 +32,7 @@ class L1(fluid.dygraph.Layer): return self.w1 + self.w2 -class L2(fluid.dygraph.Layer): +class L2(fluid.Layer): def __init__(self, prefix): super(L2, self).__init__(prefix) self.layer1 = L1(self.full_name()) @@ -42,7 +42,7 @@ class L2(fluid.dygraph.Layer): return self.layer1() + self.layer2() -class L3(fluid.dygraph.Layer): +class L3(fluid.Layer): def __init__(self, prefix): super(L3, self).__init__(prefix) self.layer1 = L2(self.full_name()) @@ -59,7 +59,7 @@ class TestBaseLayer(unittest.TestCase): ret = l() self.assertEqual(l.w1.name, "test_one_level/L1_0.w_0") self.assertEqual(l.w2.name, "test_one_level/L1_0.w_1") - self.assertTrue(np.allclose(ret._numpy(), 0.2 * np.ones([2, 2]))) + self.assertTrue(np.allclose(ret.numpy(), 0.2 * np.ones([2, 2]))) def test_three_level(self): with fluid.dygraph.guard(): @@ -72,7 +72,7 @@ class TestBaseLayer(unittest.TestCase): self.assertEqual(names[3], "test_three_level/L3_0/L2_0/L1_1.w_1") self.assertEqual(names[4], "test_three_level/L3_0/L2_1/L1_0.w_0") self.assertEqual(names[5], "test_three_level/L3_0/L2_1/L1_0.w_1") - self.assertTrue(np.allclose(ret._numpy(), 0.8 * np.ones([2, 2]))) + self.assertTrue(np.allclose(ret.numpy(), 0.8 * np.ones([2, 2]))) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_bilinear_interp_op.py b/python/paddle/fluid/tests/unittests/test_bilinear_interp_op.py index f60ed1d79ae5778f751d6101fde386ae3a90c0f7..963a17e7d697512e871a97ef24cb1c4ba37a7547 100644 --- a/python/paddle/fluid/tests/unittests/test_bilinear_interp_op.py +++ b/python/paddle/fluid/tests/unittests/test_bilinear_interp_op.py @@ -91,17 +91,26 @@ class TestBilinearInterpOp(OpTest): self.op_type = "bilinear_interp" input_np = np.random.random(self.input_shape).astype("float32") - output_np = bilinear_interp_np(input_np, self.out_h, self.out_w, - self.out_size, self.actual_shape, - self.align_corners, self.align_mode) + if self.scale > 0: + out_h = int(self.input_shape[2] * self.scale) + out_w = int(self.input_shape[3] * self.scale) + else: + out_h = self.out_h + out_w = self.out_w + + output_np = bilinear_interp_np(input_np, out_h, out_w, self.out_size, + self.actual_shape, self.align_corners, + self.align_mode) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size if self.actual_shape is not None: self.inputs['OutSize'] = self.actual_shape + self.attrs = { 'out_h': self.out_h, 'out_w': self.out_w, + 'scale': self.scale, 'interp_method': self.interp_method, 'align_corners': self.align_corners, 'align_mode': self.align_mode @@ -119,6 +128,7 @@ class TestBilinearInterpOp(OpTest): self.input_shape = [2, 3, 4, 4] self.out_h = 2 self.out_w = 2 + self.scale = 0. self.out_size = np.array([3, 3]).astype("int32") self.align_corners = True self.align_mode = 1 @@ -130,6 +140,7 @@ class TestBilinearInterpCase1(TestBilinearInterpOp): self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 + self.scale = 0. self.align_corners = True self.align_mode = 1 @@ -140,6 +151,7 @@ class TestBilinearInterpCase2(TestBilinearInterpOp): self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 + self.scale = 0. self.align_corners = True self.align_mode = 1 @@ -150,6 +162,7 @@ class TestBilinearInterpCase3(TestBilinearInterpOp): self.input_shape = [1, 1, 128, 64] self.out_h = 64 self.out_w = 128 + self.scale = 0. self.align_corners = True self.align_mode = 1 @@ -160,6 +173,7 @@ class TestBilinearInterpCase4(TestBilinearInterpOp): self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 + self.scale = 0. self.out_size = np.array([2, 2]).astype("int32") self.align_corners = True self.align_mode = 1 @@ -171,6 +185,7 @@ class TestBilinearInterpCase5(TestBilinearInterpOp): self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 + self.scale = 0. self.out_size = np.array([11, 11]).astype("int32") self.align_corners = True self.align_mode = 1 @@ -182,6 +197,7 @@ class TestBilinearInterpCase6(TestBilinearInterpOp): self.input_shape = [1, 1, 128, 64] self.out_h = 64 self.out_w = 128 + self.scale = 0. self.out_size = np.array([65, 129]).astype("int32") self.align_corners = True self.align_mode = 1 @@ -193,6 +209,7 @@ class TestBilinearInterpActualShape(TestBilinearInterpOp): self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 + self.scale = 0. self.out_size = np.array([66, 40]).astype("int32") self.align_corners = True self.align_mode = 1 @@ -206,15 +223,25 @@ class TestBilinearInterpOpUint8(OpTest): self.op_type = "bilinear_interp" input_np = np.random.randint( low=0, high=256, size=self.input_shape).astype("uint8") - output_np = bilinear_interp_np(input_np, self.out_h, self.out_w, - self.out_size, self.actual_shape, - self.align_corners, self.align_mode) + + if self.scale > 0: + out_h = int(self.input_shape[2] * self.scale) + out_w = int(self.input_shape[3] * self.scale) + else: + out_h = self.out_h + out_w = self.out_w + + output_np = bilinear_interp_np(input_np, out_h, out_w, self.out_size, + self.actual_shape, self.align_corners, + self.align_mode) self.inputs = {'X': input_np} if self.out_size is not None: self.inputs['OutSize'] = self.out_size + self.attrs = { 'out_h': self.out_h, 'out_w': self.out_w, + 'scale': self.scale, 'interp_method': self.interp_method, 'align_corners': self.align_corners, 'align_mode': self.align_mode @@ -229,6 +256,7 @@ class TestBilinearInterpOpUint8(OpTest): self.input_shape = [1, 3, 9, 6] self.out_h = 10 self.out_w = 9 + self.scale = 0. self.align_corners = True self.align_mode = 1 @@ -239,6 +267,7 @@ class TestBilinearInterpCase1Uint8(TestBilinearInterpOpUint8): self.input_shape = [2, 3, 128, 64] self.out_h = 120 self.out_w = 50 + self.scale = 0. self.align_corners = True self.align_mode = 1 @@ -249,6 +278,7 @@ class TestBilinearInterpCase2Uint8(TestBilinearInterpOpUint8): self.input_shape = [4, 1, 7, 8] self.out_h = 5 self.out_w = 13 + self.scale = 0. self.out_size = np.array([6, 15]).astype("int32") self.align_corners = True self.align_mode = 1 @@ -272,5 +302,38 @@ class TestBilinearInterpWithMethod3(TestBilinearInterpOp): self.align_mode = 0 +class TestBilinearInterpScale1(TestBilinearInterpOp): + def init_test_case(self): + self.interp_method = 'bilinear' + self.input_shape = [2, 3, 16, 32] + self.out_h = 60 + self.out_w = 25 + self.scale = 2. + self.align_corners = True + self.align_mode = 1 + + +class TestBilinearInterpScale2(TestBilinearInterpOp): + def init_test_case(self): + self.interp_method = 'bilinear' + self.input_shape = [2, 3, 16, 32] + self.out_h = 60 + self.out_w = 25 + self.scale = 1. + self.align_corners = True + self.align_mode = 1 + + +class TestBilinearInterpScale3(TestBilinearInterpOp): + def init_test_case(self): + self.interp_method = 'bilinear' + self.input_shape = [2, 3, 16, 32] + self.out_h = 60 + self.out_w = 25 + self.scale = 1.5 + self.align_corners = True + self.align_mode = 1 + + if __name__ == "__main__": unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_decoupled_py_reader.py b/python/paddle/fluid/tests/unittests/test_decoupled_py_reader.py index 377014510b55633f697ef7bf2f5f597281e5f5a5..0fbf0d42f5dcc34947235d9bd1db6f8b1c07d59a 100644 --- a/python/paddle/fluid/tests/unittests/test_decoupled_py_reader.py +++ b/python/paddle/fluid/tests/unittests/test_decoupled_py_reader.py @@ -19,7 +19,7 @@ import time import six import unittest -EPOCH_NUM = 60 +EPOCH_NUM = 20 BATCH_SIZE = 32 CLASS_NUM = 10 diff --git a/python/paddle/fluid/tests/unittests/test_dyn_rnn.py b/python/paddle/fluid/tests/unittests/test_dyn_rnn.py index 48fb93ec529bee32b9652a89ba7da3dc77f7853a..4b0195d307dc83f77ff04e89544d7bc751b8c011 100644 --- a/python/paddle/fluid/tests/unittests/test_dyn_rnn.py +++ b/python/paddle/fluid/tests/unittests/test_dyn_rnn.py @@ -24,15 +24,15 @@ from paddle.fluid.layers.control_flow import max_sequence_len from paddle.fluid.layers.control_flow import lod_tensor_to_array from paddle.fluid.layers.control_flow import array_to_lod_tensor from paddle.fluid.layers.control_flow import shrink_memory +from fake_reader import fake_imdb_reader class TestDynRNN(unittest.TestCase): def setUp(self): - self.word_dict = paddle.dataset.imdb.word_dict() + self.word_dict_len = 5147 self.BATCH_SIZE = 2 - self.train_data = paddle.batch( - paddle.dataset.imdb.train(self.word_dict), - batch_size=self.BATCH_SIZE) + reader = fake_imdb_reader(self.word_dict_len, self.BATCH_SIZE * 100) + self.train_data = paddle.batch(reader, batch_size=self.BATCH_SIZE) def test_plain_while_op(self): main_program = fluid.Program() @@ -42,7 +42,7 @@ class TestDynRNN(unittest.TestCase): sentence = fluid.layers.data( name='word', shape=[1], dtype='int64', lod_level=1) sent_emb = fluid.layers.embedding( - input=sentence, size=[len(self.word_dict), 32], dtype='float32') + input=sentence, size=[self.word_dict_len, 32], dtype='float32') label = fluid.layers.data(name='label', shape=[1], dtype='float32') @@ -109,7 +109,7 @@ class TestDynRNN(unittest.TestCase): sentence = fluid.layers.data( name='word', shape=[1], dtype='int64', lod_level=1) sent_emb = fluid.layers.embedding( - input=sentence, size=[len(self.word_dict), 32], dtype='float32') + input=sentence, size=[self.word_dict_len, 32], dtype='float32') rnn = fluid.layers.DynamicRNN() diff --git a/python/paddle/fluid/tests/unittests/test_eager_deletion_conditional_block.py b/python/paddle/fluid/tests/unittests/test_eager_deletion_conditional_block.py new file mode 100644 index 0000000000000000000000000000000000000000..95cae1c2029c472c5a34b37a79739e2ff088feb2 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_eager_deletion_conditional_block.py @@ -0,0 +1,23 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import paddle.fluid as fluid +import unittest + +fluid.core._set_eager_deletion_mode(0.0, 1.0, True) + +from test_conditional_block import * + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_eager_deletion_dynamic_rnn_base.py b/python/paddle/fluid/tests/unittests/test_eager_deletion_dynamic_rnn_base.py index d4c043d9c76f21482f17b9bb20c4fde5ce7cc6e7..eb3832ca9ffb7ac9b4261de1036c85c93c6d0a81 100644 --- a/python/paddle/fluid/tests/unittests/test_eager_deletion_dynamic_rnn_base.py +++ b/python/paddle/fluid/tests/unittests/test_eager_deletion_dynamic_rnn_base.py @@ -22,6 +22,8 @@ import paddle import paddle.fluid.core as core import paddle.fluid as fluid from paddle.fluid import compiler +import numpy as np +from fake_reader import fake_imdb_reader def train(network, use_cuda, use_parallel_executor, batch_size=32, pass_num=2): @@ -35,16 +37,16 @@ def train(network, use_cuda, use_parallel_executor, batch_size=32, pass_num=2): ) return - word_dict = paddle.dataset.imdb.word_dict() - train_reader = paddle.batch( - paddle.dataset.imdb.train(word_dict), batch_size=batch_size) + word_dict_size = 5147 + reader = fake_imdb_reader(word_dict_size, batch_size * 40) + train_reader = paddle.batch(reader, batch_size=batch_size) data = fluid.layers.data( name="words", shape=[1], dtype="int64", lod_level=1) label = fluid.layers.data(name="label", shape=[1], dtype="int64") - cost = network(data, label, len(word_dict)) + cost = network(data, label, word_dict_size) cost.persistable = True optimizer = fluid.optimizer.Adagrad(learning_rate=0.2) optimizer.minimize(cost) diff --git a/python/paddle/fluid/tests/unittests/test_eager_deletion_no_need_buffer_vars_inference.py b/python/paddle/fluid/tests/unittests/test_eager_deletion_no_need_buffer_vars_inference.py index a84ff1fd6d46c30ad7aa72f1b29a8ae668b90e20..3fd582e4d5cb7cec1db0719160a4a795a30e54f1 100644 --- a/python/paddle/fluid/tests/unittests/test_eager_deletion_no_need_buffer_vars_inference.py +++ b/python/paddle/fluid/tests/unittests/test_eager_deletion_no_need_buffer_vars_inference.py @@ -18,20 +18,21 @@ import importlib fluid.core._set_eager_deletion_mode(0.0, 1.0, True) +from test_bilinear_interp_op import * +from test_concat_op import * from test_elementwise_add_op import * from test_elementwise_sub_op import * -from test_concat_op import * +from test_fill_constant_batch_size_like_op import * +from test_fill_zeros_like2_op import * from test_gather_op import * from test_gaussian_random_batch_size_like_op import * -from test_uniform_random_batch_size_like_op import * -from test_fill_constant_batch_size_like_op import * +from test_linear_chain_crf_op import * from test_lod_reset_op import * -from test_scatter_op import * +from test_lookup_table_op import * from test_mean_op import * -from test_slice_op import * -from test_linear_chain_crf_op import * -from test_bilinear_interp_op import * from test_nearest_interp_op import * +from test_pad2d_op import * +from test_scatter_op import * from test_sequence_concat import * from test_seq_conv import * from test_seq_pool import * @@ -41,8 +42,10 @@ from test_sequence_pad_op import * from test_sequence_unpad_op import * from test_sequence_scatter_op import * from test_sequence_slice_op import * -from test_pad2d_op import * -from test_fill_zeros_like2_op import * +from test_slice_op import * +from test_space_to_depth_op import * +from test_squared_l2_distance_op import * +from test_uniform_random_batch_size_like_op import * if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_fuse_all_reduce_pass.py b/python/paddle/fluid/tests/unittests/test_fuse_all_reduce_pass.py index ca8669bbc6f3ea7b3f3340793712a221b0bf8c6a..0990045a8fd8775b90ddb6569c5c269ff57d6e38 100644 --- a/python/paddle/fluid/tests/unittests/test_fuse_all_reduce_pass.py +++ b/python/paddle/fluid/tests/unittests/test_fuse_all_reduce_pass.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +from simple_nets import simple_fc_net, fc_with_batchnorm, init_data from parallel_executor_test_base import TestParallelExecutorBase import paddle.fluid as fluid import paddle.fluid.core as core @@ -22,45 +22,6 @@ import unittest import os -def simple_fc_net(use_feed): - img = fluid.layers.data(name='image', shape=[784], dtype='float32') - label = fluid.layers.data(name='label', shape=[1], dtype='int64') - - hidden = img - for _ in range(4): - hidden = fluid.layers.fc( - hidden, - size=200, - act='relu', - bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=1.0))) - prediction = fluid.layers.fc(hidden, size=10, act='softmax') - loss = fluid.layers.cross_entropy(input=prediction, label=label) - loss = fluid.layers.mean(loss) - return loss - - -def fc_with_batchnorm(use_feed): - img = fluid.layers.data(name='image', shape=[784], dtype='float32') - label = fluid.layers.data(name='label', shape=[1], dtype='int64') - - hidden = img - for _ in range(2): - hidden = fluid.layers.fc( - hidden, - size=200, - act='relu', - bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=1.0))) - - hidden = fluid.layers.batch_norm(input=hidden) - - prediction = fluid.layers.fc(hidden, size=10, act='softmax') - loss = fluid.layers.cross_entropy(input=prediction, label=label) - loss = fluid.layers.mean(loss) - return loss - - class TestMNIST(TestParallelExecutorBase): @classmethod def setUpClass(cls): @@ -75,10 +36,10 @@ class TestMNIST(TestParallelExecutorBase): label = np.ones(shape=[32, 1], dtype='int64') return img, label - def _compare_fuse_all_reduce_ops(self, model, use_cuda, random_data=True): + def _compare_fuse_all_reduce_ops(self, model, use_cuda): if use_cuda and not core.is_compiled_with_cuda(): return - img, label = self._init_data(random_data) + img, label = init_data() def _optimizer(learning_rate=1e-6): optimizer = fluid.optimizer.SGD( diff --git a/python/paddle/fluid/tests/unittests/test_fuse_elewise_add_act_pass.py b/python/paddle/fluid/tests/unittests/test_fuse_elewise_add_act_pass.py index 763dfa2160d22c2d89cce834a839b5e2b5eaff55..552f94e769e5a8764dd8426d130fd879dc718b20 100644 --- a/python/paddle/fluid/tests/unittests/test_fuse_elewise_add_act_pass.py +++ b/python/paddle/fluid/tests/unittests/test_fuse_elewise_add_act_pass.py @@ -12,108 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. +from simple_nets import simple_fc_net, fc_with_batchnorm, init_data from parallel_executor_test_base import TestParallelExecutorBase import paddle.fluid as fluid import paddle.fluid.core as core -import numpy as np -import paddle -import paddle.dataset.mnist as mnist import unittest import os -MNIST_RECORDIO_FILE = "./mnist_test_pe.recordio" - - -def simple_fc_net(use_feed): - if use_feed: - img = fluid.layers.data(name='image', shape=[784], dtype='float32') - label = fluid.layers.data(name='label', shape=[1], dtype='int64') - else: - reader = fluid.layers.open_files( - filenames=[MNIST_RECORDIO_FILE], - shapes=[[-1, 784], [-1, 1]], - lod_levels=[0, 0], - dtypes=['float32', 'int64']) - reader = fluid.layers.io.double_buffer(reader) - img, label = fluid.layers.read_file(reader) - hidden = img - for _ in range(4): - hidden = fluid.layers.fc( - hidden, - size=200, - act='relu', - bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=1.0))) - prediction = fluid.layers.fc(hidden, size=10, act='softmax') - loss = fluid.layers.cross_entropy(input=prediction, label=label) - loss = fluid.layers.mean(loss) - return loss - - -def fc_with_batchnorm(use_feed): - if use_feed: - img = fluid.layers.data(name='image', shape=[784], dtype='float32') - label = fluid.layers.data(name='label', shape=[1], dtype='int64') - else: - reader = fluid.layers.open_files( - filenames=[MNIST_RECORDIO_FILE], - shapes=[[-1, 784], [-1, 1]], - lod_levels=[0, 0], - dtypes=['float32', 'int64']) - reader = fluid.layers.io.double_buffer(reader) - img, label = fluid.layers.read_file(reader) - - hidden = img - for _ in range(2): - hidden = fluid.layers.fc( - hidden, - size=200, - act='relu', - bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=1.0))) - - hidden = fluid.layers.batch_norm(input=hidden) - - prediction = fluid.layers.fc(hidden, size=10, act='softmax') - loss = fluid.layers.cross_entropy(input=prediction, label=label) - loss = fluid.layers.mean(loss) - return loss - class TestMNIST(TestParallelExecutorBase): @classmethod def setUpClass(cls): os.environ['CPU_NUM'] = str(4) - # Convert mnist to recordio file - with fluid.program_guard(fluid.Program(), fluid.Program()): - reader = paddle.batch(mnist.train(), batch_size=4) - feeder = fluid.DataFeeder( - feed_list=[ # order is image and label - fluid.layers.data( - name='image', shape=[784]), - fluid.layers.data( - name='label', shape=[1], dtype='int64'), - ], - place=fluid.CPUPlace()) - fluid.recordio_writer.convert_reader_to_recordio_file( - MNIST_RECORDIO_FILE, reader, feeder) - - def _init_data(self, random=True): - np.random.seed(5) - if random: - img = np.random.random(size=[32, 784]).astype(np.float32) - else: - img = np.ones(shape=[32, 784], dtype='float32') - label = np.ones(shape=[32, 1], dtype='int64') - return img, label - def _compare_fuse_elewise_add_act_ops(self, - model, - use_cuda, - random_data=True): + def _compare_fuse_elewise_add_act_ops(self, model, use_cuda): if use_cuda and not core.is_compiled_with_cuda(): return - img, label = self._init_data(random_data) + img, label = init_data() def _optimizer(learning_rate=1e-6): optimizer = fluid.optimizer.SGD( diff --git a/python/paddle/fluid/tests/unittests/test_fuse_optimizer_pass.py b/python/paddle/fluid/tests/unittests/test_fuse_optimizer_pass.py index 93e67deaf3c9f7fe17296049137fbbe00374c6f1..510be19af406ba821ab8159abf071440ae3d1831 100644 --- a/python/paddle/fluid/tests/unittests/test_fuse_optimizer_pass.py +++ b/python/paddle/fluid/tests/unittests/test_fuse_optimizer_pass.py @@ -11,78 +11,26 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +from simple_nets import simple_fc_net, fc_with_batchnorm, init_data from parallel_executor_test_base import TestParallelExecutorBase import paddle.fluid as fluid import paddle.fluid.core as core -import numpy as np -import paddle -import paddle.dataset.mnist as mnist import unittest import os -def simple_fc_net(use_feed): - img = fluid.layers.data(name='image', shape=[784], dtype='float32') - label = fluid.layers.data(name='label', shape=[1], dtype='int64') - - hidden = img - for _ in range(4): - hidden = fluid.layers.fc( - hidden, - size=200, - act='relu', - bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=1.0))) - prediction = fluid.layers.fc(hidden, size=10, act='softmax') - loss = fluid.layers.cross_entropy(input=prediction, label=label) - loss = fluid.layers.mean(loss) - return loss - - -def fc_with_batchnorm(use_feed): - img = fluid.layers.data(name='image', shape=[784], dtype='float32') - label = fluid.layers.data(name='label', shape=[1], dtype='int64') - - hidden = img - for _ in range(2): - hidden = fluid.layers.fc( - hidden, - size=200, - act='relu', - bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=1.0))) - - hidden = fluid.layers.batch_norm(input=hidden) - - prediction = fluid.layers.fc(hidden, size=10, act='softmax') - loss = fluid.layers.cross_entropy(input=prediction, label=label) - loss = fluid.layers.mean(loss) - return loss - - class TestFuseAdamOps(TestParallelExecutorBase): @classmethod def setUpClass(cls): os.environ['CPU_NUM'] = str(4) - def _init_data(self, random=True): - np.random.seed(5) - if random: - img = np.random.random(size=[32, 784]).astype(np.float32) - else: - img = np.ones(shape=[32, 784], dtype='float32') - label = np.ones(shape=[32, 1], dtype='int64') - return img, label - def _compare_fused_optimizer_ops(self, model, use_cuda, - random_data=True, optimizer=fluid.optimizer.Adam): if use_cuda and not core.is_compiled_with_cuda(): return - img, label = self._init_data(random_data) + img, label = init_data() not_fuse_op_first_loss, not_fuse_op_last_loss = self.check_network_convergence( model, feed_dict={"image": img, @@ -111,7 +59,7 @@ class TestFuseAdamOps(TestParallelExecutorBase): def test_batchnorm_fc_with_fuse_op(self): self._compare_fused_optimizer_ops(fc_with_batchnorm, True) - # self._compare_fused_optimizer_ops(fc_with_batchnorm, False) + self._compare_fused_optimizer_ops(fc_with_batchnorm, False) class TestFuseSGDOps(TestFuseAdamOps): diff --git a/python/paddle/fluid/tests/unittests/test_gradient_clip.py b/python/paddle/fluid/tests/unittests/test_gradient_clip.py index e49239da6d3918211fbbc302d2c56818460b6d51..470187e6421173d1cb1213d06660331c164859c4 100644 --- a/python/paddle/fluid/tests/unittests/test_gradient_clip.py +++ b/python/paddle/fluid/tests/unittests/test_gradient_clip.py @@ -19,6 +19,8 @@ import numpy as np import paddle import paddle.fluid.core as core import paddle.fluid as fluid +import six +from fake_reader import fake_imdb_reader def bow_net(data, @@ -48,11 +50,10 @@ def bow_net(data, class TestGradientClip(unittest.TestCase): def setUp(self): - self.word_dict = paddle.dataset.imdb.word_dict() + self.word_dict_len = 5147 self.BATCH_SIZE = 2 - self.train_data = paddle.batch( - paddle.dataset.imdb.train(self.word_dict), - batch_size=self.BATCH_SIZE) + reader = fake_imdb_reader(self.word_dict_len, self.BATCH_SIZE * 100) + self.train_data = paddle.batch(reader, batch_size=self.BATCH_SIZE) def get_places(self): places = [core.CPUPlace()] @@ -131,7 +132,7 @@ class TestGradientClip(unittest.TestCase): data = fluid.layers.data( name="words", shape=[1], dtype="int64", lod_level=1) label = fluid.layers.data(name="label", shape=[1], dtype="int64") - cost = bow_net(data, label, len(self.word_dict)) + cost = bow_net(data, label, self.word_dict_len) fluid.clip.set_gradient_clip( clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=5.0)) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_basic.py b/python/paddle/fluid/tests/unittests/test_imperative_basic.py index 13f2d662178c7e1474ec43fdeadf7046516eb8e5..8404a57eb85a30edda6889150e588cab783be685 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_basic.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_basic.py @@ -18,11 +18,11 @@ import numpy as np import paddle.fluid as fluid from paddle.fluid import core -from paddle.fluid.dygraph.nn import FC +from paddle.fluid import FC from test_imperative_base import new_program_scope -class MyLayer(fluid.dygraph.Layer): +class MyLayer(fluid.Layer): def __init__(self, name_scope): super(MyLayer, self).__init__(name_scope) @@ -34,7 +34,7 @@ class MyLayer(fluid.dygraph.Layer): return [x] -class MyPyLayer(fluid.dygraph.PyLayer): +class MyPyLayer(fluid.PyLayer): def __init__(self): super(MyPyLayer, self).__init__() @@ -48,7 +48,7 @@ class MyPyLayer(fluid.dygraph.PyLayer): return np.array(dout) * (1 - np.square(np.array(out))) -class MLP(fluid.dygraph.Layer): +class MLP(fluid.Layer): def __init__(self, name_scope): super(MLP, self).__init__(name_scope) self._fc1 = FC(self.full_name(), @@ -71,7 +71,7 @@ class MLP(fluid.dygraph.Layer): return x -class SimpleRNNCell(fluid.dygraph.Layer): +class SimpleRNNCell(fluid.Layer): def __init__(self, name_scope, step_input_size, hidden_size, output_size, param_attr): super(SimpleRNNCell, self).__init__(name_scope) @@ -81,7 +81,7 @@ class SimpleRNNCell(fluid.dygraph.Layer): self._dtype = core.VarDesc.VarType.FP32 self.param_attr = param_attr - def _build_once(self, inputs, pre_hidden): + def build_once(self, inputs, pre_hidden): i2h_param_shape = [self.step_input_size, self.hidden_size] h2h_param_shape = [self.hidden_size, self.hidden_size] h2o_param_shape = [self.output_size, self.hidden_size] @@ -159,7 +159,7 @@ class SimpleRNNCell(fluid.dygraph.Layer): return reduce_out, hidden -class SimpleRNN(fluid.dygraph.Layer): +class SimpleRNN(fluid.Layer): def __init__(self, name_scope): super(SimpleRNN, self).__init__(name_scope) self.seq_len = 4 @@ -200,22 +200,22 @@ class TestImperative(unittest.TestCase): inputs.append(fluid.dygraph.base.to_variable(x)) ret = fluid.layers.sums(inputs) loss = fluid.layers.reduce_sum(ret) - loss._backward() - self.assertTrue(np.allclose(ret._numpy(), x * 10)) - self.assertTrue(np.allclose(inputs[0]._gradient(), x)) + loss.backward() + self.assertTrue(np.allclose(ret.numpy(), x * 10)) + self.assertTrue(np.allclose(inputs[0].gradient(), x)) def test_layer(self): with fluid.dygraph.guard(): cl = core.Layer() cl.forward([]) - l = fluid.dygraph.Layer("l") + l = fluid.Layer("l") self.assertRaises(NotImplementedError, l.forward, []) def test_pylayer_func_id(self): with fluid.dygraph.guard(): - class PyLayer1(fluid.dygraph.PyLayer): + class PyLayer1(fluid.PyLayer): def __init__(self): super(PyLayer1, self).__init__() @@ -227,7 +227,7 @@ class TestImperative(unittest.TestCase): def backward(input): return input - class PyLayer2(fluid.dygraph.PyLayer): + class PyLayer2(fluid.PyLayer): def __init__(self): super(PyLayer2, self).__init__() @@ -257,9 +257,9 @@ class TestImperative(unittest.TestCase): my_py_layer = MyPyLayer() var_inp = fluid.dygraph.base.to_variable(np_inp) outs = my_py_layer(var_inp) - dy_out = np.sum(outs[0]._numpy()) - outs[0]._backward() - dy_grad = var_inp._gradient() + dy_out = np.sum(outs[0].numpy()) + outs[0].backward() + dy_grad = var_inp.gradient() with new_program_scope(): inp = fluid.layers.data( @@ -287,9 +287,9 @@ class TestImperative(unittest.TestCase): l = MyLayer("my_layer") x = l(var_inp)[0] self.assertIsNotNone(x) - dy_out = x._numpy() - x._backward() - dy_grad = l._x_for_debug._gradient() + dy_out = x.numpy() + x.backward() + dy_grad = l._x_for_debug.gradient() with new_program_scope(): inp = fluid.layers.data( @@ -314,9 +314,9 @@ class TestImperative(unittest.TestCase): var_inp = fluid.dygraph.base.to_variable(np_inp) mlp = MLP("mlp") out = mlp(var_inp) - dy_out = out._numpy() - out._backward() - dy_grad = mlp._fc1._w._gradient() + dy_out = out.numpy() + out.backward() + dy_grad = mlp._fc1._w.gradient() with new_program_scope(): inp = fluid.layers.data( @@ -348,6 +348,55 @@ class TestImperative(unittest.TestCase): self.assertEqual(mlp._fc2, sublayers[1]) self.assertEqual(len(sublayers), 2) + def test_dygraph_vs_static(self): + inp1 = np.random.rand(4, 3, 3) + inp2 = np.random.rand(4, 3, 3) + + # dynamic graph + with fluid.dygraph.guard(): + if np.sum(inp1) < np.sum(inp2): + x = fluid.layers.elementwise_add(inp1, inp2) + else: + x = fluid.layers.elementwise_sub(inp1, inp2) + dygraph_result = x.numpy() + + # static graph + with new_program_scope(): + inp_data1 = fluid.layers.data( + name='inp1', shape=[3, 3], dtype=np.float32) + inp_data2 = fluid.layers.data( + name='inp2', shape=[3, 3], dtype=np.float32) + + a = fluid.layers.expand( + fluid.layers.reshape( + fluid.layers.reduce_sum(inp_data1), [1, 1]), [4, 1]) + b = fluid.layers.expand( + fluid.layers.reshape( + fluid.layers.reduce_sum(inp_data2), [1, 1]), [4, 1]) + cond = fluid.layers.less_than(x=a, y=b) + + ie = fluid.layers.IfElse(cond) + with ie.true_block(): + d1 = ie.input(inp_data1) + d2 = ie.input(inp_data2) + d3 = fluid.layers.elementwise_add(d1, d2) + ie.output(d3) + + with ie.false_block(): + d1 = ie.input(inp_data1) + d2 = ie.input(inp_data2) + d3 = fluid.layers.elementwise_sub(d1, d2) + ie.output(d3) + out = ie() + + exe = fluid.Executor(fluid.CPUPlace( + ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) + static_result = exe.run(fluid.default_main_program(), + feed={'inp1': inp1, + 'inp2': inp2}, + fetch_list=out)[0] + self.assertTrue(np.allclose(dygraph_result, static_result)) + def test_rnn(self): np_inp = np.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0], [10.0, 11.0, 12.0]]) @@ -358,11 +407,11 @@ class TestImperative(unittest.TestCase): var_inp = fluid.layers.reshape(var_inp, shape=[1, 4, 3]) simple_rnn = SimpleRNN("simple_rnn") outs, pre_hiddens = simple_rnn.forward(var_inp) - dy_out = outs[3]._numpy() - outs[3]._backward() - dy_grad_h2o = simple_rnn._cell._h2o_w._gradient() - dy_grad_h2h = simple_rnn._cell._h2h_w._gradient() - dy_grad_i2h = simple_rnn._cell._i2h_w._gradient() + dy_out = outs[3].numpy() + outs[3].backward() + dy_grad_h2o = simple_rnn._cell._h2o_w.gradient() + dy_grad_h2h = simple_rnn._cell._h2h_w.gradient() + dy_grad_i2h = simple_rnn._cell._i2h_w.gradient() with new_program_scope(): inp = fluid.layers.data( diff --git a/python/paddle/fluid/tests/unittests/test_imperative_checkpoint.py b/python/paddle/fluid/tests/unittests/test_imperative_checkpoint.py index a92b7d62fa598a3ec9b53bade2805cc033f4b9d9..c28058100a43eb4f7da8331d9ac75db9c090bdf9 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_checkpoint.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_checkpoint.py @@ -18,11 +18,11 @@ import numpy as np import paddle import paddle.fluid as fluid from paddle.fluid.optimizer import SGDOptimizer -from paddle.fluid.dygraph.nn import Conv2D, Pool2D, FC +from paddle.fluid import Conv2D, Pool2D, FC from paddle.fluid.dygraph.base import to_variable -class SimpleImgConvPool(fluid.dygraph.Layer): +class SimpleImgConvPool(fluid.Layer): def __init__(self, name_scope, num_channels, @@ -71,7 +71,7 @@ class SimpleImgConvPool(fluid.dygraph.Layer): return x -class MNIST(fluid.dygraph.Layer): +class MNIST(fluid.Layer): def __init__(self, name_scope): super(MNIST, self).__init__(name_scope) @@ -125,21 +125,21 @@ class TestDygraphCheckpoint(unittest.TestCase): img = to_variable(dy_x_data) label = to_variable(y_data) - label._stop_gradient = True + label.stop_gradient = True cost = mnist(img) loss = fluid.layers.cross_entropy(cost, label) avg_loss = fluid.layers.mean(loss) - dy_out = avg_loss._numpy() + dy_out = avg_loss.numpy() - avg_loss._backward() + avg_loss.backward() sgd.minimize(avg_loss) fluid.dygraph.save_persistables(mnist, "save_dir") mnist.clear_gradients() for param in mnist.parameters(): - dy_param_init_value[param.name] = param._numpy() + dy_param_init_value[param.name] = param.numpy() mnist.load_dict( fluid.dygraph.load_persistables(mnist, "save_dir")) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_deepcf.py b/python/paddle/fluid/tests/unittests/test_imperative_deepcf.py index ccebd4a54727f383bd4e46ff57bfdc9381577d05..ca2cffa9c75cc851f0911cb0063f4e82bb2a41eb 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_deepcf.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_deepcf.py @@ -32,11 +32,11 @@ NUM_BATCHES = int(os.environ.get('NUM_BATCHES', 5)) NUM_EPOCHES = int(os.environ.get('NUM_EPOCHES', 1)) -class DMF(fluid.dygraph.Layer): +class DMF(fluid.Layer): def __init__(self, name_scope): super(DMF, self).__init__(name_scope) - self._user_latent = fluid.dygraph.FC(self.full_name(), 256) - self._item_latent = fluid.dygraph.FC(self.full_name(), 256) + self._user_latent = fluid.FC(self.full_name(), 256) + self._item_latent = fluid.FC(self.full_name(), 256) self._user_layers = [] self._item_layers = [] @@ -45,13 +45,11 @@ class DMF(fluid.dygraph.Layer): self._user_layers.append( self.add_sublayer( 'user_layer_%d' % i, - fluid.dygraph.FC( - self.full_name(), self._hid_sizes[i], act='relu'))) + fluid.FC(self.full_name(), self._hid_sizes[i], act='relu'))) self._item_layers.append( self.add_sublayer( 'item_layer_%d' % i, - fluid.dygraph.FC( - self.full_name(), self._hid_sizes[i], act='relu'))) + fluid.FC(self.full_name(), self._hid_sizes[i], act='relu'))) def forward(self, users, items): users = self._user_latent(users) @@ -63,19 +61,18 @@ class DMF(fluid.dygraph.Layer): return fluid.layers.elementwise_mul(users, items) -class MLP(fluid.dygraph.Layer): +class MLP(fluid.Layer): def __init__(self, name_scope): super(MLP, self).__init__(name_scope) - self._user_latent = fluid.dygraph.FC(self.full_name(), 256) - self._item_latent = fluid.dygraph.FC(self.full_name(), 256) + self._user_latent = fluid.FC(self.full_name(), 256) + self._item_latent = fluid.FC(self.full_name(), 256) self._match_layers = [] self._hid_sizes = [128, 64] for i in range(len(self._hid_sizes)): self._match_layers.append( self.add_sublayer( 'match_layer_%d' % i, - fluid.dygraph.FC( - self.full_name(), self._hid_sizes[i], act='relu'))) + fluid.FC(self.full_name(), self._hid_sizes[i], act='relu'))) self._mat def forward(self, users, items): @@ -88,7 +85,7 @@ class MLP(fluid.dygraph.Layer): return match_vec -class DeepCF(fluid.dygraph.Layer): +class DeepCF(fluid.Layer): def __init__(self, name_scope, num_users, num_items, matrix): super(DeepCF, self).__init__(name_scope) self._num_users = num_users @@ -99,11 +96,11 @@ class DeepCF(fluid.dygraph.Layer): matrix.dtype, is_bias=False, default_initializer=fluid.initializer.NumpyArrayInitializer(matrix)) - self._rating_matrix._stop_gradient = True + self._rating_matrix.stop_gradient = True self._mlp = MLP(self.full_name()) self._dmf = DMF(self.full_name()) - self._match_fc = fluid.dygraph.FC(self.full_name(), 1, act='sigmoid') + self._match_fc = fluid.FC(self.full_name(), 1, act='sigmoid') def forward(self, users, items): # users_emb = self._user_emb(users) @@ -255,10 +252,10 @@ class TestDygraphDeepCF(unittest.TestCase): fluid.layers.log_loss(prediction, to_variable(labels_np[ slice:slice + BATCH_SIZE]))) - loss._backward() + loss.backward() adam.minimize(loss) deepcf.clear_gradients() - dy_loss = loss._numpy() + dy_loss = loss.numpy() sys.stderr.write('dynamic loss: %s %s\n' % (slice, dy_loss)) self.assertEqual(static_loss, dy_loss) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_gan.py b/python/paddle/fluid/tests/unittests/test_imperative_gan.py index 58faa1cb85af9cedb70f3a12244cfeb44e0f4f52..5d773ec1c9db160cd63a28c634043037260e0b82 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_gan.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_gan.py @@ -22,12 +22,12 @@ import paddle import paddle.fluid as fluid import paddle.fluid.core as core from paddle.fluid.optimizer import SGDOptimizer -from paddle.fluid.dygraph.nn import Conv2D, Pool2D, FC +from paddle.fluid import Conv2D, Pool2D, FC from test_imperative_base import new_program_scope from paddle.fluid.dygraph.base import to_variable -class Discriminator(fluid.dygraph.Layer): +class Discriminator(fluid.Layer): def __init__(self, name_scope): super(Discriminator, self).__init__(name_scope) self._fc1 = FC(self.full_name(), size=32, act='elu') @@ -38,7 +38,7 @@ class Discriminator(fluid.dygraph.Layer): return self._fc2(x) -class Generator(fluid.dygraph.Layer): +class Generator(fluid.Layer): def __init__(self, name_scope): super(Generator, self).__init__(name_scope) self._fc1 = FC(self.full_name(), size=64, act='elu') @@ -150,7 +150,7 @@ class TestDygraphGAN(unittest.TestCase): x=d_fake, label=to_variable(np.zeros([2, 1], np.float32)))) d_loss = d_loss_real + d_loss_fake - d_loss._backward() + d_loss.backward() sgd.minimize(d_loss) discriminator.clear_gradients() generator.clear_gradients() @@ -160,15 +160,15 @@ class TestDygraphGAN(unittest.TestCase): g_loss = fluid.layers.reduce_mean( fluid.layers.sigmoid_cross_entropy_with_logits( x=d_fake, label=to_variable(np.ones([2, 1], np.float32)))) - g_loss._backward() + g_loss.backward() sgd.minimize(g_loss) for p in discriminator.parameters(): - dy_params[p.name] = p._numpy() + dy_params[p.name] = p.numpy() for p in generator.parameters(): - dy_params[p.name] = p._numpy() + dy_params[p.name] = p.numpy() - dy_g_loss = g_loss._numpy() - dy_d_loss = d_loss._numpy() + dy_g_loss = g_loss.numpy() + dy_d_loss = d_loss.numpy() self.assertEqual(dy_g_loss, static_g_loss) self.assertEqual(dy_d_loss, static_d_loss) diff --git a/python/paddle/fluid/tests/unittests/test_imperative_gnn.py b/python/paddle/fluid/tests/unittests/test_imperative_gnn.py index a8fb9ecfe4be16b73ac2144259f25ed3859ece7e..234fcd60404286977309083257c24d941db77449 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_gnn.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_gnn.py @@ -15,14 +15,12 @@ import contextlib import unittest import numpy as np -import six import sys import paddle import paddle.fluid as fluid import paddle.fluid.core as core from paddle.fluid.optimizer import AdamOptimizer -from paddle.fluid.dygraph.nn import Conv2D, Pool2D, FC from test_imperative_base import new_program_scope from paddle.fluid.dygraph.base import to_variable @@ -31,7 +29,7 @@ def gen_data(): pass -class GraphConv(fluid.dygraph.Layer): +class GraphConv(fluid.Layer): def __init__(self, name_scope, in_features, out_features): super(GraphConv, self).__init__(name_scope) @@ -50,7 +48,7 @@ class GraphConv(fluid.dygraph.Layer): return fluid.layers.matmul(adj, support) + self.bias -class GCN(fluid.dygraph.Layer): +class GCN(fluid.Layer): def __init__(self, name_scope, num_hidden): super(GCN, self).__init__(name_scope) self.gc = GraphConv(self.full_name(), num_hidden, 32) @@ -134,10 +132,9 @@ class TestDygraphGNN(unittest.TestCase): loss = fluid.layers.reduce_sum(loss) adam = AdamOptimizer(learning_rate=1e-3) adam.minimize(loss) - self.assertEqual(static_loss, loss._numpy()) - self.assertTrue( - np.allclose(static_weight, model.gc.weight._numpy())) - sys.stderr.write('%s %s\n' % (static_loss, loss._numpy())) + self.assertEqual(static_loss, loss.numpy()) + self.assertTrue(np.allclose(static_weight, model.gc.weight.numpy())) + sys.stderr.write('%s %s\n' % (static_loss, loss.numpy())) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_imperative_mnist.py b/python/paddle/fluid/tests/unittests/test_imperative_mnist.py index 5ab01839fbc20bbd3c242878c4ea23a00f7b0dca..76b8d3aa3943e44a17ab822618d8d1cb85aaa551 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_mnist.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_mnist.py @@ -128,25 +128,25 @@ class TestImperativeMnist(unittest.TestCase): img = to_variable(dy_x_data) label = to_variable(y_data) - label._stop_gradient = True + label.stop_gradient = True cost = mnist(img) loss = fluid.layers.cross_entropy(cost, label) avg_loss = fluid.layers.mean(loss) - dy_out = avg_loss._numpy() + dy_out = avg_loss.numpy() if epoch == 0 and batch_id == 0: for param in mnist.parameters(): - dy_param_init_value[param.name] = param._numpy() + dy_param_init_value[param.name] = param.numpy() - avg_loss._backward() + avg_loss.backward() sgd.minimize(avg_loss) mnist.clear_gradients() dy_param_value = {} for param in mnist.parameters(): - dy_param_value[param.name] = param._numpy() + dy_param_value[param.name] = param.numpy() with new_program_scope(): fluid.default_startup_program().random_seed = seed diff --git a/python/paddle/fluid/tests/unittests/test_imperative_optimizer.py b/python/paddle/fluid/tests/unittests/test_imperative_optimizer.py index 8b659a3e08e381dd6f55b666d9f5f1b172a51930..b9f93119e83159c5bc3052b0292168a9ef641d3e 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_optimizer.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_optimizer.py @@ -28,7 +28,7 @@ from paddle.fluid.dygraph.base import to_variable from test_imperative_base import new_program_scope -class MLP(fluid.dygraph.Layer): +class MLP(fluid.Layer): def __init__(self, name_scope, param_attr=None, bias_attr=None): super(MLP, self).__init__(name_scope) @@ -75,18 +75,18 @@ class TestImperativeOptimizerBase(unittest.TestCase): cost = mlp(img) avg_loss = fluid.layers.reduce_mean(cost) - dy_out = avg_loss._numpy() + dy_out = avg_loss.numpy() if batch_id == 0: for param in mlp.parameters(): - dy_param_init_value[param.name] = param._numpy() + dy_param_init_value[param.name] = param.numpy() - avg_loss._backward() + avg_loss.backward() optimizer.minimize(avg_loss) mlp.clear_gradients() dy_param_value = {} for param in mlp.parameters(): - dy_param_value[param.name] = param._numpy() + dy_param_value[param.name] = param.numpy() with new_program_scope(): fluid.default_startup_program().random_seed = seed diff --git a/python/paddle/fluid/tests/unittests/test_imperative_ptb_rnn.py b/python/paddle/fluid/tests/unittests/test_imperative_ptb_rnn.py index 552eb019500b1e43ee54a3dd4ec90b292f0a24a5..088d36be2327a91da0efc639d7f970ed9e43d151 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_ptb_rnn.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_ptb_rnn.py @@ -16,6 +16,7 @@ from __future__ import print_function import unittest import paddle.fluid as fluid +import paddle.fluid.core as core from paddle.fluid.dygraph.nn import Embedding import paddle.fluid.framework as framework from paddle.fluid.optimizer import SGDOptimizer @@ -23,10 +24,9 @@ from paddle.fluid.dygraph.base import to_variable from test_imperative_base import new_program_scope import numpy as np import six -from paddle.fluid.backward import append_backward -class SimpleLSTMRNN(fluid.dygraph.Layer): +class SimpleLSTMRNN(fluid.Layer): def __init__(self, name_scope, hidden_size, @@ -44,7 +44,7 @@ class SimpleLSTMRNN(fluid.dygraph.Layer): self.cell_array = [] self.hidden_array = [] - def _build_once(self, input_embedding, init_hidden=None, init_cell=None): + def build_once(self, input_embedding, init_hidden=None, init_cell=None): self.weight_1_arr = [] self.weight_2_arr = [] self.bias_arr = [] @@ -131,7 +131,7 @@ class SimpleLSTMRNN(fluid.dygraph.Layer): return real_res, last_hidden, last_cell -class PtbModel(fluid.dygraph.Layer): +class PtbModel(fluid.Layer): def __init__(self, name_scope, hidden_size, @@ -176,7 +176,7 @@ class PtbModel(fluid.dygraph.Layer): default_initializer=fluid.initializer.UniformInitializer( low=-self.init_scale, high=self.init_scale)) - def _build_once(self, input, label, init_hidden, init_cell): + def build_once(self, input, label, init_hidden, init_cell): pass def forward(self, input, label, init_hidden, init_cell): @@ -259,13 +259,13 @@ class TestDygraphPtbRnn(unittest.TestCase): init_cell) if i == 0: for param in ptb_model.parameters(): - dy_param_init[param.name] = param._numpy() - dy_loss._backward() + dy_param_init[param.name] = param.numpy() + dy_loss.backward() sgd.minimize(dy_loss) ptb_model.clear_gradients() if i == batch_num - 1: for param in ptb_model.parameters(): - dy_param_updated[param.name] = param._numpy() + dy_param_updated[param.name] = param.numpy() with new_program_scope(): fluid.default_startup_program().random_seed = seed @@ -278,7 +278,8 @@ class TestDygraphPtbRnn(unittest.TestCase): num_steps=num_steps, init_scale=init_scale) - exe = fluid.Executor(fluid.CPUPlace()) + exe = fluid.Executor(fluid.CPUPlace( + ) if not core.is_compiled_with_cuda() else fluid.CUDAPlace(0)) sgd = SGDOptimizer(learning_rate=1e-3) x = fluid.layers.data( name="x", shape=[-1, num_steps, 1], dtype='int64') @@ -331,18 +332,16 @@ class TestDygraphPtbRnn(unittest.TestCase): for k in range(3, len(out)): static_param_updated[static_param_name_list[k - 3]] = out[k] - self.assertTrue(np.allclose(static_loss_value, dy_loss._numpy())) - self.assertTrue(np.allclose(static_last_cell_value, last_cell._numpy())) + + self.assertTrue(np.array_equal(static_loss_value, dy_loss.numpy())) + self.assertTrue( + np.array_equal(static_last_cell_value, last_cell.numpy())) self.assertTrue( - np.allclose(static_last_hidden_value, last_hidden._numpy())) + np.array_equal(static_last_hidden_value, last_hidden.numpy())) for key, value in six.iteritems(static_param_init): - # print("static_init name: {}, value {}".format(key, value)) - # print("dy_init name: {}, value {}".format(key, dy_param_init[key])) - self.assertTrue(np.allclose(value, dy_param_init[key])) + self.assertTrue(np.array_equal(value, dy_param_init[key])) for key, value in six.iteritems(static_param_updated): - # print("static name: {}, value {}".format(key, value)) - # print("dy name: {}, value {}".format(key, dy_param_updated[key])) - self.assertTrue(np.allclose(value, dy_param_updated[key])) + self.assertTrue(np.array_equal(value, dy_param_updated[key])) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_imperative_resnet.py b/python/paddle/fluid/tests/unittests/test_imperative_resnet.py index 1d786d584632769e4318bcdeb24ef7ef8ea18597..d9ef08b3c491b24323bb1469165ed5482737013a 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_resnet.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_resnet.py @@ -21,7 +21,7 @@ import paddle import paddle.fluid as fluid from paddle.fluid import core from paddle.fluid.layer_helper import LayerHelper -from paddle.fluid.dygraph.nn import Conv2D, Pool2D, BatchNorm, FC +from paddle.fluid import Conv2D, Pool2D, BatchNorm, FC from paddle.fluid.dygraph.base import to_variable from test_imperative_base import new_program_scope @@ -68,7 +68,7 @@ def optimizer_setting(params): return optimizer -class ConvBNLayer(fluid.dygraph.Layer): +class ConvBNLayer(fluid.Layer): def __init__(self, name_scope, num_channels, @@ -99,7 +99,7 @@ class ConvBNLayer(fluid.dygraph.Layer): return y -class BottleneckBlock(fluid.dygraph.Layer): +class BottleneckBlock(fluid.Layer): def __init__(self, name_scope, num_channels, @@ -156,7 +156,7 @@ class BottleneckBlock(fluid.dygraph.Layer): return layer_helper.append_activation(y) -class ResNet(fluid.dygraph.Layer): +class ResNet(fluid.Layer): def __init__(self, name_scope, layers=50, class_dim=102): super(ResNet, self).__init__(name_scope) @@ -247,7 +247,7 @@ class TestDygraphResnet(unittest.TestCase): dy_param_init_value = {} for param in resnet.parameters(): - dy_param_init_value[param.name] = param._numpy() + dy_param_init_value[param.name] = param.numpy() for batch_id, data in enumerate(train_reader()): if batch_id >= batch_num: @@ -260,20 +260,20 @@ class TestDygraphResnet(unittest.TestCase): img = to_variable(dy_x_data) label = to_variable(y_data) - label._stop_gradient = True + label.stop_gradient = True out = resnet(img) loss = fluid.layers.cross_entropy(input=out, label=label) avg_loss = fluid.layers.mean(x=loss) - dy_out = avg_loss._numpy() + dy_out = avg_loss.numpy() if batch_id == 0: for param in resnet.parameters(): if param.name not in dy_param_init_value: - dy_param_init_value[param.name] = param._numpy() + dy_param_init_value[param.name] = param.numpy() - avg_loss._backward() + avg_loss.backward() dy_grad_value = {} for param in resnet.parameters(): @@ -288,7 +288,7 @@ class TestDygraphResnet(unittest.TestCase): dy_param_value = {} for param in resnet.parameters(): - dy_param_value[param.name] = param._numpy() + dy_param_value[param.name] = param.numpy() with new_program_scope(): fluid.default_startup_program().random_seed = seed diff --git a/python/paddle/fluid/tests/unittests/test_imperative_se_resnext.py b/python/paddle/fluid/tests/unittests/test_imperative_se_resnext.py index 69931f0849480b2569a31d04c7b0b0f9db0d61a3..3f3f92cde57c80fa4ba3d2f1389cc47efd74ca5b 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_se_resnext.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_se_resnext.py @@ -56,7 +56,7 @@ def optimizer_setting(params): #bd = [step * e for e in ls["epochs"]] #base_lr = params["lr"] #lr = [base_lr * (0.1**i) for i in range(len(bd) + 1)] - optimizer = fluid.optimizer.SGD(learning_rate=0.1) + optimizer = fluid.optimizer.SGD(learning_rate=0.01) return optimizer @@ -109,7 +109,7 @@ class SqueezeExcitation(fluid.dygraph.Layer): size=num_channels, param_attr=fluid.ParamAttr( initializer=fluid.initializer.Constant(value=0.05)), - act='relu') + act='sigmoid') def forward(self, input): y = self._pool(input) @@ -316,6 +316,7 @@ class TestImperativeResneXt(unittest.TestCase): batch_size = train_parameters["batch_size"] batch_num = 2 + epoch_num = 1 with fluid.dygraph.guard(): fluid.default_startup_program().random_seed = seed fluid.default_main_program().random_seed = seed @@ -327,52 +328,54 @@ class TestImperativeResneXt(unittest.TestCase): random.seed = seed train_reader = paddle.batch( paddle.dataset.flowers.train(use_xmap=False), - batch_size=batch_size) + batch_size=batch_size, + drop_last=True) dy_param_init_value = {} for param in se_resnext.parameters(): - dy_param_init_value[param.name] = param._numpy() - - for batch_id, data in enumerate(train_reader()): - if batch_id >= batch_num: - break - - dy_x_data = np.array( - [x[0].reshape(3, 224, 224) for x in data]).astype('float32') - y_data = np.array([x[1] for x in data]).astype('int64').reshape( - batch_size, 1) - - img = to_variable(dy_x_data) - label = to_variable(y_data) - label._stop_gradient = True - - out = se_resnext(img) - loss = fluid.layers.cross_entropy(input=out, label=label) - avg_loss = fluid.layers.mean(x=loss) - - dy_out = avg_loss._numpy() - - if batch_id == 0: + dy_param_init_value[param.name] = param.numpy() + for epoch_id in range(epoch_num): + for batch_id, data in enumerate(train_reader()): + + if batch_id >= batch_num and batch_num != -1: + break + + dy_x_data = np.array( + [x[0].reshape(3, 224, 224) + for x in data]).astype('float32') + y_data = np.array( + [x[1] for x in data]).astype('int64').reshape( + batch_size, 1) + + img = to_variable(dy_x_data) + label = to_variable(y_data) + label.stop_gradient = True + + out = se_resnext(img) + loss = fluid.layers.cross_entropy(input=out, label=label) + avg_loss = fluid.layers.mean(x=loss) + + dy_out = avg_loss.numpy() + + if batch_id == 0: + for param in se_resnext.parameters(): + if param.name not in dy_param_init_value: + dy_param_init_value[param.name] = param.numpy() + avg_loss.backward() + + #dy_grad_value = {} + #for param in se_resnext.parameters(): + # if param.trainable: + # np_array = np.array(param._ivar._grad_ivar().value() + # .get_tensor()) + # dy_grad_value[param.name + core.grad_var_suffix()] = np_array + + optimizer.minimize(avg_loss) + se_resnext.clear_gradients() + + dy_param_value = {} for param in se_resnext.parameters(): - if param.name not in dy_param_init_value: - dy_param_init_value[param.name] = param._numpy() - - avg_loss._backward() - - dy_grad_value = {} - for param in se_resnext.parameters(): - if param.trainable: - np_array = np.array(param._ivar._grad_ivar().value() - .get_tensor()) - dy_grad_value[param.name + core.grad_var_suffix( - )] = np_array - - optimizer.minimize(avg_loss) - se_resnext.clear_gradients() - - dy_param_value = {} - for param in se_resnext.parameters(): - dy_param_value[param.name] = param._numpy() + dy_param_value[param.name] = param.numpy() with new_program_scope(): fluid.default_startup_program().random_seed = seed @@ -389,7 +392,8 @@ class TestImperativeResneXt(unittest.TestCase): random.seed = seed train_reader = paddle.batch( paddle.dataset.flowers.train(use_xmap=False), - batch_size=batch_size) + batch_size=batch_size, + drop_last=True) img = fluid.layers.data( name='pixel', shape=[3, 224, 224], dtype='float32') @@ -415,37 +419,42 @@ class TestImperativeResneXt(unittest.TestCase): for i in range(len(static_param_name_list)): static_param_init_value[static_param_name_list[i]] = out[i] - - for batch_id, data in enumerate(train_reader()): - if batch_id >= batch_num: - break - - static_x_data = np.array( - [x[0].reshape(3, 224, 224) for x in data]).astype('float32') - y_data = np.array([x[1] for x in data]).astype('int64').reshape( - [batch_size, 1]) - - fetch_list = [avg_loss.name] - fetch_list.extend(static_param_name_list) - fetch_list.extend(static_grad_name_list) - out = exe.run(fluid.default_main_program(), - feed={"pixel": static_x_data, - "label": y_data}, - fetch_list=fetch_list) - - static_param_value = {} - static_grad_value = {} - static_out = out[0] - param_start_pos = 1 - grad_start_pos = len(static_param_name_list) + param_start_pos - for i in range(param_start_pos, - len(static_param_name_list) + param_start_pos): - static_param_value[static_param_name_list[ - i - param_start_pos]] = out[i] - for i in range(grad_start_pos, - len(static_grad_name_list) + grad_start_pos): - static_grad_value[static_grad_name_list[ - i - grad_start_pos]] = out[i] + for epoch_id in range(epoch_num): + for batch_id, data in enumerate(train_reader()): + if batch_id >= batch_num and batch_num != -1: + break + + static_x_data = np.array( + [x[0].reshape(3, 224, 224) + for x in data]).astype('float32') + y_data = np.array( + [x[1] for x in data]).astype('int64').reshape( + [batch_size, 1]) + + fetch_list = [avg_loss.name] + fetch_list.extend(static_param_name_list) + fetch_list.extend(static_grad_name_list) + out = exe.run( + fluid.default_main_program(), + feed={"pixel": static_x_data, + "label": y_data}, + fetch_list=fetch_list) + + static_param_value = {} + static_grad_value = {} + static_out = out[0] + param_start_pos = 1 + grad_start_pos = len( + static_param_name_list) + param_start_pos + for i in range( + param_start_pos, + len(static_param_name_list) + param_start_pos): + static_param_value[static_param_name_list[ + i - param_start_pos]] = out[i] + for i in range(grad_start_pos, + len(static_grad_name_list) + grad_start_pos): + static_grad_value[static_grad_name_list[ + i - grad_start_pos]] = out[i] self.assertTrue(np.allclose(static_out, dy_out)) self.assertEqual(len(dy_param_init_value), len(static_param_init_value)) @@ -454,12 +463,12 @@ class TestImperativeResneXt(unittest.TestCase): self.assertTrue(np.allclose(value, dy_param_init_value[key])) self.assertTrue(np.isfinite(value.all())) self.assertFalse(np.isnan(value.any())) - - self.assertEqual(len(dy_grad_value), len(static_grad_value)) - for key, value in six.iteritems(static_grad_value): - self.assertTrue(np.allclose(value, dy_grad_value[key])) - self.assertTrue(np.isfinite(value.all())) - self.assertFalse(np.isnan(value.any())) + # FIXME(Yancey1989): np.array(_ivar.value().get_tensor()) leads to memory lake + #self.assertEqual(len(dy_grad_value), len(static_grad_value)) + #for key, value in six.iteritems(static_grad_value): + # self.assertTrue(np.allclose(value, dy_grad_value[key])) + # self.assertTrue(np.isfinite(value.all())) + # self.assertFalse(np.isnan(value.any())) self.assertEqual(len(dy_param_value), len(static_param_value)) for key, value in six.iteritems(static_param_value): diff --git a/python/paddle/fluid/tests/unittests/test_imperative_transformer.py b/python/paddle/fluid/tests/unittests/test_imperative_transformer.py index 732f0681c4e65006628d51e083a400c0b5bd3d92..b24bab210a15528f308804c71732bd71eb6105a4 100644 --- a/python/paddle/fluid/tests/unittests/test_imperative_transformer.py +++ b/python/paddle/fluid/tests/unittests/test_imperative_transformer.py @@ -16,7 +16,8 @@ from __future__ import print_function import unittest import paddle.fluid as fluid -from paddle.fluid.dygraph import Embedding, LayerNorm, FC, to_variable, Layer, guard +from paddle.fluid import Embedding, LayerNorm, FC, Layer +from paddle.fluid.dygraph import to_variable, guard from test_imperative_base import new_program_scope from paddle.fluid import core import numpy as np @@ -116,7 +117,7 @@ class ModelHyperParams(object): # to process after each sub-layer postprocess_cmd = "da" # dropout + residual connection # random seed used in dropout for CE. - dropout_seed = 1 + dropout_seed = None # the flag indicating whether to share embedding and softmax weights. # vocabularies in source and target should be same for weight sharing. weight_sharing = True @@ -166,15 +167,21 @@ def create_data(is_static=False): ] else: enc_inputs = [ - to_variable(src_word_np), to_variable(src_pos_np), - to_variable(src_slf_attn_bias_np) + to_variable( + src_word_np, name='src_word'), to_variable( + src_pos_np, name='src_pos'), to_variable( + src_slf_attn_bias_np, name='src_slf_attn_bias') ] dec_inputs = [ - to_variable(trg_word_np), to_variable(trg_pos_np), - to_variable(trg_slf_attn_bias_np), to_variable(trg_src_attn_bias_np) + to_variable( + trg_word_np, name='trg_word'), to_variable( + trg_pos_np, name='trg_pos'), to_variable( + trg_slf_attn_bias_np, name='trg_slf_attn_bias'), + to_variable( + trg_src_attn_bias_np, name='trg_src_attn_bias') ] - label = to_variable(lbl_word_np) - weight = to_variable(lbl_weight_np) + label = to_variable(lbl_word_np, name='lbl_word') + weight = to_variable(lbl_weight_np, name='lbl_weight') return enc_inputs, dec_inputs, label, weight @@ -211,7 +218,7 @@ def make_all_inputs(input_fields): # The placeholder for batch_size in compile time. Must be -1 currently to be # consistent with some ops' infer-shape output in compile time, such as the # sequence_expand op used in beamsearch decoder. -batch_size = 32 +batch_size = -1 # The placeholder for squence length in compile time. seq_len = ModelHyperParams.max_length # Here list the data shapes and data types of all inputs. @@ -303,56 +310,42 @@ use_py_reader = False sync = False # how many batches we use -batch_num = 50 +batch_num = 5 -np.random.seed = 1 +np.random.seed = 90 src_word_np = np.random.randint( 1, ModelHyperParams.src_vocab_size - 1, - size=(batch_size, seq_len, 1), + size=(TrainTaskConfig.batch_size, seq_len, 1), dtype='int64') src_pos_np = np.random.randint( - 1, seq_len, size=(batch_size, seq_len, 1), dtype='int64') -src_slf_attn_bias_np = np.random.randn(batch_size, ModelHyperParams.n_head, - seq_len, seq_len).astype('float32') + 1, seq_len, size=(TrainTaskConfig.batch_size, seq_len, 1), dtype='int64') +src_slf_attn_bias_np = np.random.randn(TrainTaskConfig.batch_size, + ModelHyperParams.n_head, seq_len, + seq_len).astype('float32') trg_word_np = np.random.randint( 1, ModelHyperParams.src_vocab_size - 1, - size=(batch_size, seq_len, 1), + size=(TrainTaskConfig.batch_size, seq_len, 1), dtype='int64') trg_pos_np = np.random.randint( - 1, seq_len, size=(batch_size, seq_len, 1), dtype='int64') -trg_slf_attn_bias_np = np.random.randn(batch_size, ModelHyperParams.n_head, - seq_len, seq_len).astype('float32') -trg_src_attn_bias_np = np.random.randn(batch_size, ModelHyperParams.n_head, - seq_len, seq_len).astype('float32') + 1, seq_len, size=(TrainTaskConfig.batch_size, seq_len, 1), dtype='int64') +trg_slf_attn_bias_np = np.random.randn(TrainTaskConfig.batch_size, + ModelHyperParams.n_head, seq_len, + seq_len).astype('float32') +trg_src_attn_bias_np = np.random.randn(TrainTaskConfig.batch_size, + ModelHyperParams.n_head, seq_len, + seq_len).astype('float32') lbl_word_np = np.random.randint( 1, ModelHyperParams.src_vocab_size - 1, - size=(batch_size * seq_len, 1), + size=(TrainTaskConfig.batch_size * seq_len, 1), dtype='int64') -lbl_weight_np = np.random.randn(batch_size * seq_len, 1).astype('float32') - -# np.random.seed = 1 -# src_word_np = np.arange(0, 10).reshape([batch_size, seq_len, 1]).astype('int64') -# src_pos_np = np.random.randint( -# 1, seq_len, size=(batch_size, seq_len, 1), dtype='int64') -# src_slf_attn_bias_np = np.random.randn(batch_size, ModelHyperParams.n_head, -# seq_len, seq_len).astype('float32') -# -# trg_word_np = np.arange(0, 10).reshape([batch_size, seq_len, 1]).astype('int64') -# trg_pos_np = np.random.randint( -# 1, seq_len, size=(batch_size, seq_len, 1), dtype='int64') -# trg_slf_attn_bias_np = np.random.randn(batch_size, ModelHyperParams.n_head, -# seq_len, seq_len).astype('float32') -# trg_src_attn_bias_np = np.random.randn(batch_size, ModelHyperParams.n_head, -# seq_len, seq_len).astype('float32') -# -# lbl_word_np = np.arange(0, 10).reshape([batch_size * seq_len, 1]).astype('int64') -# lbl_weight_np = np.random.randn(batch_size * seq_len, 1).astype('float32') -# +lbl_weight_np = np.random.randn(TrainTaskConfig.batch_size * seq_len, + 1).astype('float32') + pos_inp1 = position_encoding_init(ModelHyperParams.max_length, ModelHyperParams.d_model) pos_inp2 = position_encoding_init(ModelHyperParams.max_length, @@ -466,7 +459,7 @@ class MultiHeadAttentionLayer(Layer): x=v, shape=[0, 0, self._n_head, self._d_value], inplace=False) transpose_v = fluid.layers.transpose(x=reshaped_v, perm=[0, 2, 1, 3]) - #scale dot product attention + # scale dot product attention product = fluid.layers.matmul( x=transpose_q, y=transpose_k, @@ -739,7 +732,7 @@ class DecoderSubLayer(Layer): enc_attn_output_pp = self._multihead_attention_layer2( pre_process_rlt2, enc_output, enc_output, dec_enc_attn_bias) enc_attn_output = self._post_process_layer2( - slf_attn_output, enc_attn_output_pp, self._postprocess_cmd, + slf_attn_output_pp, enc_attn_output_pp, self._postprocess_cmd, self._prepostprcess_dropout) pre_process_rlt3 = self._pre_process_layer3(None, enc_attn_output, self._preprocess_cmd, @@ -990,16 +983,18 @@ class TestDygraphTransformer(unittest.TestCase): enc_inputs, dec_inputs, label, weights = create_data() dy_sum_cost, dy_avg_cost, dy_predict, dy_token_num = transformer( enc_inputs, dec_inputs, label, weights) + if i == 0: for param in transformer.parameters(): - dy_param_init[param.name] = param._numpy() + dy_param_init[param.name] = param.numpy() - dy_avg_cost._backward() + dy_avg_cost.backward() optimizer.minimize(dy_avg_cost) transformer.clear_gradients() + if i == batch_num - 1: for param in transformer.parameters(): - dy_param_updated[param.name] = param._numpy() + dy_param_updated[param.name] = param.numpy() with new_program_scope(): fluid.default_startup_program().random_seed = seed @@ -1043,7 +1038,6 @@ class TestDygraphTransformer(unittest.TestCase): static_param_name_list = list() static_sum_cost, static_avg_cost, static_predict, static_token_num = transformer( enc_inputs, dec_inputs, label, weights) - optimizer.minimize(static_avg_cost) for param in transformer.parameters(): static_param_name_list.append(param.name) @@ -1061,8 +1055,8 @@ class TestDygraphTransformer(unittest.TestCase): static_sum_cost, static_avg_cost, static_predict, static_token_num ] - fetch_list.extend(static_param_name_list) + fetch_list.extend(static_param_name_list) out = exe.run(fluid.default_main_program(), feed=feed_dict, fetch_list=fetch_list) @@ -1076,13 +1070,14 @@ class TestDygraphTransformer(unittest.TestCase): 4]] = out[k] self.assertTrue( - np.array_equal(static_avg_cost_value, dy_avg_cost._numpy())) + np.array_equal(static_avg_cost_value, dy_avg_cost.numpy())) self.assertTrue( - np.array_equal(static_sum_cost_value, dy_sum_cost._numpy())) + np.array_equal(static_sum_cost_value, dy_sum_cost.numpy())) self.assertTrue( - np.array_equal(static_predict_value, dy_predict._numpy())) + np.array_equal(static_predict_value, dy_predict.numpy())) self.assertTrue( - np.array_equal(static_token_num_value, dy_token_num._numpy())) + np.array_equal(static_token_num_value, dy_token_num.numpy())) + for key, value in six.iteritems(static_param_init): self.assertTrue(np.array_equal(value, dy_param_init[key])) for key, value in six.iteritems(static_param_updated): diff --git a/python/paddle/fluid/tests/unittests/test_layers.py b/python/paddle/fluid/tests/unittests/test_layers.py index 674965882d76e142e4dc818374768ae7549120e0..91f8bc5fd0a510dcc05cb7ba2397cad52be16af5 100644 --- a/python/paddle/fluid/tests/unittests/test_layers.py +++ b/python/paddle/fluid/tests/unittests/test_layers.py @@ -18,6 +18,8 @@ import unittest import contextlib import numpy as np import decorators +import inspect +from six.moves import filter import paddle import paddle.fluid as fluid @@ -58,8 +60,12 @@ class LayerTest(unittest.TestCase): fluid.default_main_program().random_seed = self.seed yield - def get_static_graph_result(self, feed, fetch_list, with_lod=False): - exe = fluid.Executor(self._get_place()) + def get_static_graph_result(self, + feed, + fetch_list, + with_lod=False, + force_to_use_cpu=False): + exe = fluid.Executor(self._get_place(force_to_use_cpu)) exe.run(fluid.default_startup_program()) return exe.run(fluid.default_main_program(), feed=feed, @@ -77,7 +83,6 @@ class LayerTest(unittest.TestCase): class TestLayer(LayerTest): def test_fc(self): - # pdb.set_trace() inp = np.ones([3, 32, 32], dtype='float32') with self.static_graph(): t = layers.data( @@ -109,7 +114,7 @@ class TestLayer(LayerTest): dy_ret = fc2(ret) self.assertTrue(np.array_equal(static_ret, static_ret2)) - self.assertTrue(np.array_equal(static_ret, dy_ret._numpy())) + self.assertTrue(np.array_equal(static_ret, dy_ret.numpy())) def test_layer_norm(self): inp = np.ones([3, 32, 32], dtype='float32') @@ -137,7 +142,7 @@ class TestLayer(LayerTest): dy_ret = lm(base.to_variable(inp)) self.assertTrue(np.allclose(static_ret, static_ret2)) - self.assertTrue(np.allclose(dy_ret._numpy(), static_ret2)) + self.assertTrue(np.allclose(dy_ret.numpy(), static_ret2)) def test_relu(self): with self.static_graph(): @@ -151,7 +156,7 @@ class TestLayer(LayerTest): t = np.ones([3, 3], dtype='float32') dy_ret = layers.relu(base.to_variable(t)) - self.assertTrue(np.allclose(static_ret, dy_ret._numpy())) + self.assertTrue(np.allclose(static_ret, dy_ret.numpy())) def test_matmul(self): with self.static_graph(): @@ -172,7 +177,7 @@ class TestLayer(LayerTest): t2 = np.ones([3, 3], dtype='float32') dy_ret = layers.matmul(base.to_variable(t), base.to_variable(t2)) - self.assertTrue(np.allclose(static_ret, dy_ret._numpy())) + self.assertTrue(np.allclose(static_ret, dy_ret.numpy())) def test_conv2d(self): with self.static_graph(): @@ -199,7 +204,7 @@ class TestLayer(LayerTest): 'conv2d', num_channels=3, num_filters=3, filter_size=[2, 2]) dy_ret = conv2d(base.to_variable(images)) - self.assertTrue(np.allclose(static_ret, dy_ret._numpy())) + self.assertTrue(np.allclose(static_ret, dy_ret.numpy())) self.assertTrue(np.allclose(static_ret, static_ret2)) def test_gru_unit(self): @@ -241,7 +246,7 @@ class TestLayer(LayerTest): for i in range(len(static_ret)): self.assertTrue(np.allclose(static_ret[i], static_ret2[i])) - self.assertTrue(np.allclose(static_ret[i], dy_ret[i]._numpy())) + self.assertTrue(np.allclose(static_ret[i], dy_ret[i].numpy())) def test_elementwise_math(self): n = np.ones([3, 3], dtype='float32') @@ -283,8 +288,8 @@ class TestLayer(LayerTest): ret = layers.elementwise_sub(ret, n5) dy_ret = layers.elementwise_mul(ret, n6) self.assertTrue( - np.allclose(static_ret, dy_ret._numpy()), - '%s vs %s' % (static_ret, dy_ret._numpy())) + np.allclose(static_ret, dy_ret.numpy()), + '%s vs %s' % (static_ret, dy_ret.numpy())) def test_elementwise_minmax(self): n = np.ones([3, 3], dtype='float32') @@ -294,8 +299,8 @@ class TestLayer(LayerTest): min_ret = layers.elementwise_min(n, n2) max_ret = layers.elementwise_max(n, n2) - self.assertTrue(np.allclose(n, min_ret._numpy())) - self.assertTrue(np.allclose(n2, max_ret._numpy())) + self.assertTrue(np.allclose(n, min_ret.numpy())) + self.assertTrue(np.allclose(n2, max_ret.numpy())) def test_sequence_conv(self): inp_np = np.arange(12).reshape([3, 4]).astype('float32') @@ -362,7 +367,7 @@ class TestLayer(LayerTest): 'conv2d_transpose', num_filters=10, output_size=28) dy_rlt = conv2d_transpose(base.to_variable(inp_np)) self.assertTrue(np.allclose(static_rlt2, static_rlt)) - self.assertTrue(np.allclose(dy_rlt._numpy(), static_rlt)) + self.assertTrue(np.allclose(dy_rlt.numpy(), static_rlt)) def test_bilinear_tensor_product(self): inp_np_x = np.array([[1, 2, 3]]).astype('float32') @@ -405,7 +410,7 @@ class TestLayer(LayerTest): dy_rlt = btp(base.to_variable(inp_np_x), base.to_variable(inp_np_y)) self.assertTrue(np.allclose(static_rlt2, static_rlt)) - self.assertTrue(np.allclose(dy_rlt._numpy(), static_rlt)) + self.assertTrue(np.allclose(dy_rlt.numpy(), static_rlt)) def test_prelu(self): inp_np = np.ones([5, 200, 100, 100]).astype('float32') @@ -446,7 +451,7 @@ class TestLayer(LayerTest): dy_rlt = prelu(base.to_variable(inp_np)) self.assertTrue(np.allclose(static_rlt2, static_rlt)) - self.assertTrue(np.allclose(dy_rlt._numpy(), static_rlt)) + self.assertTrue(np.allclose(dy_rlt.numpy(), static_rlt)) def test_embeding(self): inp_word = np.array([[[1]]]).astype('int64') @@ -479,7 +484,7 @@ class TestLayer(LayerTest): static_rlt3 = emb2(base.to_variable(inp_word)) self.assertTrue(np.allclose(static_rlt2, static_rlt)) - self.assertTrue(np.allclose(static_rlt3._numpy(), static_rlt)) + self.assertTrue(np.allclose(static_rlt3.numpy(), static_rlt)) def test_nce(self): window_size = 5 @@ -593,28 +598,379 @@ class TestLayer(LayerTest): nce_loss3 = nce(embs3, words[label_word]) self.assertTrue(np.allclose(static_rlt2, static_rlt)) - self.assertTrue(np.allclose(nce_loss3._numpy(), static_rlt)) + self.assertTrue(np.allclose(nce_loss3.numpy(), static_rlt)) + def test_conv3d(self): + with self.static_graph(): + images = layers.data( + name='pixel', shape=[3, 6, 6, 6], dtype='float32') + ret = layers.conv3d(input=images, num_filters=3, filter_size=2) + static_ret = self.get_static_graph_result( + feed={'pixel': np.ones( + [2, 3, 6, 6, 6], dtype='float32')}, + fetch_list=[ret])[0] -class TestBook(unittest.TestCase): - def test_fit_a_line(self): - program = Program() - with program_guard(program, startup_program=Program()): - x = layers.data(name='x', shape=[13], dtype='float32') + with self.static_graph(): + images = layers.data( + name='pixel', shape=[3, 6, 6, 6], dtype='float32') + conv3d = nn.Conv3D('conv3d', num_filters=3, filter_size=2) + ret = conv3d(images) + static_ret2 = self.get_static_graph_result( + feed={'pixel': np.ones( + [2, 3, 6, 6, 6], dtype='float32')}, + fetch_list=[ret])[0] + + with self.dynamic_graph(): + images = np.ones([2, 3, 6, 6, 6], dtype='float32') + conv3d = nn.Conv3D('conv3d', num_filters=3, filter_size=2) + dy_ret = conv3d(base.to_variable(images)) + + self.assertTrue(np.allclose(static_ret, dy_ret.numpy())) + self.assertTrue(np.allclose(static_ret, static_ret2)) + + def test_row_conv(self): + input = np.arange(15).reshape([3, 5]).astype('float32') + if core.is_compiled_with_cuda(): + place = core.CUDAPlace(0) + else: + place = core.CPUPlace() + + with self.static_graph(): + x = layers.data( + name='X', + shape=[3, 5], + dtype='float32', + lod_level=1, + append_batch_size=False) + ret = layers.row_conv(input=x, future_context_size=2) + static_ret = self.get_static_graph_result( + feed={ + 'X': fluid.create_lod_tensor( + data=input, recursive_seq_lens=[[1, 1, 1]], place=place) + }, + fetch_list=[ret], + with_lod=True)[0] + + with self.static_graph(): + x = layers.data( + name='X', + shape=[3, 5], + dtype='float32', + lod_level=1, + append_batch_size=False) + rowConv = nn.RowConv('RowConv', future_context_size=2) + ret = rowConv(x) + static_ret2 = self.get_static_graph_result( + feed={ + 'X': fluid.create_lod_tensor( + data=input, recursive_seq_lens=[[1, 1, 1]], place=place) + }, + fetch_list=[ret], + with_lod=True)[0] + + # TODO: dygraph can't support LODTensor + + self.assertTrue(np.allclose(static_ret, static_ret2)) + + def test_group_norm(self): + if core.is_compiled_with_cuda(): + place = core.CUDAPlace(0) + else: + place = core.CPUPlace() + + shape = (2, 4, 3, 3) + + input = np.random.random(shape).astype('float32') + + with self.static_graph(): + X = fluid.layers.data( + name='X', + shape=shape, + dtype='float32', + lod_level=1, + append_batch_size=False) + ret = layers.group_norm(input=X, groups=2) + static_ret = self.get_static_graph_result( + feed={ + 'X': fluid.create_lod_tensor( + data=input, recursive_seq_lens=[[1, 1]], place=place) + }, + fetch_list=[ret], + with_lod=True)[0] + + with self.static_graph(): + X = fluid.layers.data( + name='X', + shape=shape, + dtype='float32', + lod_level=1, + append_batch_size=False) + groupNorm = nn.GroupNorm('GroupNorm', groups=2) + ret = groupNorm(X) + static_ret2 = self.get_static_graph_result( + feed={ + 'X': fluid.create_lod_tensor( + data=input, recursive_seq_lens=[[1, 1]], place=place) + }, + fetch_list=[ret], + with_lod=True)[0] + + with self.dynamic_graph(): + groupNorm = nn.GroupNorm('GroupNorm', groups=2) + dy_ret = groupNorm(base.to_variable(input)) + + self.assertTrue(np.allclose(static_ret, dy_ret.numpy())) + self.assertTrue(np.allclose(static_ret, static_ret2)) + + def test_spectral_norm(self): + if core.is_compiled_with_cuda(): + place = core.CUDAPlace(0) + else: + place = core.CPUPlace() + + shape = (2, 4, 3, 3) + + input = np.random.random(shape).astype('float32') + + with self.static_graph(): + Weight = fluid.layers.data( + name='Weight', + shape=shape, + dtype='float32', + lod_level=1, + append_batch_size=False) + ret = layers.spectral_norm(weight=Weight, dim=1, power_iters=2) + static_ret = self.get_static_graph_result( + feed={ + 'Weight': fluid.create_lod_tensor( + data=input, recursive_seq_lens=[[1, 1]], place=place), + }, + fetch_list=[ret], + with_lod=True)[0] + + with self.static_graph(): + Weight = fluid.layers.data( + name='Weight', + shape=shape, + dtype='float32', + lod_level=1, + append_batch_size=False) + spectralNorm = nn.SpectralNorm('SpectralNorm', dim=1, power_iters=2) + ret = spectralNorm(Weight) + static_ret2 = self.get_static_graph_result( + feed={ + 'Weight': fluid.create_lod_tensor( + data=input, recursive_seq_lens=[[1, 1]], place=place) + }, + fetch_list=[ret], + with_lod=True)[0] + + with self.dynamic_graph(): + spectralNorm = nn.SpectralNorm('SpectralNorm', dim=1, power_iters=2) + dy_ret = spectralNorm(base.to_variable(input)) + + self.assertTrue(np.allclose(static_ret, dy_ret.numpy())) + self.assertTrue(np.allclose(static_ret, static_ret2)) + + def test_tree_conv(self): + if core.is_compiled_with_cuda(): + place = core.CUDAPlace(0) + else: + place = core.CPUPlace() + adj_array = [1, 2, 1, 3, 1, 4, 1, 5, 2, 6, 2, 7, 2, 8, 4, 9, 4, 10] + adj = np.array(adj_array).reshape((1, 9, 2)).astype('int32') + adj = np.tile(adj, (1, 1, 1)) + vectors = np.random.random((1, 10, 5)).astype('float32') + with self.static_graph(): + NodesVector = fluid.layers.data( + name='NodesVector', + shape=(1, 10, 5), + dtype='float32', + lod_level=1, + append_batch_size=False) + EdgeSet = fluid.layers.data( + name='EdgeSet', + shape=(1, 9, 2), + dtype='int32', + lod_level=1, + append_batch_size=False) + ret = layers.tree_conv( + nodes_vector=NodesVector, + edge_set=EdgeSet, + output_size=6, + num_filters=1, + max_depth=2) + static_ret = self.get_static_graph_result( + feed={ + 'NodesVector': fluid.create_lod_tensor( + data=vectors, recursive_seq_lens=[[1]], place=place), + 'EdgeSet': fluid.create_lod_tensor( + data=adj, recursive_seq_lens=[[1]], place=place) + }, + fetch_list=[ret], + with_lod=False)[0] + + with self.static_graph(): + NodesVector = fluid.layers.data( + name='NodesVector', + shape=(1, 10, 5), + dtype='float32', + lod_level=1, + append_batch_size=False) + EdgeSet = fluid.layers.data( + name='EdgeSet', + shape=(1, 9, 2), + dtype='int32', + lod_level=1, + append_batch_size=False) + treeConv = nn.TreeConv( + 'TreeConv', output_size=6, num_filters=1, max_depth=2) + ret = treeConv(NodesVector, EdgeSet) + static_ret2 = self.get_static_graph_result( + feed={ + 'NodesVector': fluid.create_lod_tensor( + data=vectors, recursive_seq_lens=[[1]], place=place), + 'EdgeSet': fluid.create_lod_tensor( + data=adj, recursive_seq_lens=[[1]], place=place) + }, + fetch_list=[ret], + with_lod=False)[0] + + with self.dynamic_graph(): + treeConv = nn.TreeConv( + 'SpectralNorm', output_size=6, num_filters=1, max_depth=2) + dy_ret = treeConv(base.to_variable(vectors), base.to_variable(adj)) + + self.assertTrue(np.allclose(static_ret, static_ret2)) + self.assertTrue(np.allclose(static_ret, dy_ret.numpy())) + + def test_conv3d_transpose(self): + input_array = np.arange(0, 48).reshape( + [2, 3, 2, 2, 2]).astype('float32') + + with self.static_graph(): + img = layers.data(name='pixel', shape=[3, 2, 2, 2], dtype='float32') + out = layers.conv3d_transpose( + input=img, num_filters=12, filter_size=12, use_cudnn=False) + static_rlt = self.get_static_graph_result( + feed={'pixel': input_array}, fetch_list=[out])[0] + with self.static_graph(): + img = layers.data(name='pixel', shape=[3, 2, 2, 2], dtype='float32') + conv3d_transpose = nn.Conv3DTranspose( + 'Conv3DTranspose', + num_filters=12, + filter_size=12, + use_cudnn=False) + out = conv3d_transpose(img) + static_rlt2 = self.get_static_graph_result( + feed={'pixel': input_array}, fetch_list=[out])[0] + with self.dynamic_graph(): + conv3d_transpose = nn.Conv3DTranspose( + 'Conv3DTranspose', + num_filters=12, + filter_size=12, + use_cudnn=False) + dy_rlt = conv3d_transpose(base.to_variable(input_array)) + self.assertTrue(np.allclose(static_rlt2, static_rlt)) + self.assertTrue(np.allclose(dy_rlt.numpy(), static_rlt)) + + +class TestBook(LayerTest): + def test_all_layers(self): + attrs = (getattr(self, name) for name in dir(self)) + methods = filter(inspect.ismethod, attrs) + for method in methods: + if not method.__name__.startswith('make_'): + continue + self._low_data_bound = 0 + self._high_data_bound = 2 + self._batch_size = 2 + self._feed_dict = {} + self._force_to_use_cpu = False + with self.static_graph(): + static_var = method() + if isinstance(static_var, tuple): + static_var = static_var[0] + + if static_var is not None: + fetch_list = [static_var.name] + static_result = self.get_static_graph_result( + feed=self._feed_dict, + fetch_list=fetch_list, + force_to_use_cpu=self._force_to_use_cpu) + else: + assert method.__name__ in ('make_get_places') + continue + + with self.dynamic_graph(self._force_to_use_cpu): + dy_result = method() + if isinstance(dy_result, tuple): + dy_result = dy_result[0] + + self.assertTrue(np.array_equal(static_result[0], dy_result.numpy())) + + def _get_np_data(self, shape, dtype, append_batch_size=True): + np.random.seed(self.seed) + if append_batch_size: + shape = [self._batch_size] + shape + if dtype == 'float32': + return np.random.random(shape).astype(dtype) + elif dtype == 'float64': + return np.random.random(shape).astype(dtype) + elif dtype == 'int32': + return np.random.randint(self._low_data_bound, + self._high_data_bound, shape).astype(dtype) + elif dtype == 'int64': + return np.random.randint(self._low_data_bound, + self._high_data_bound, shape).astype(dtype) + + def _get_data(self, + name, + shape, + dtype, + set_feed_dict=True, + append_batch_size=True): + if base.enabled(): + return base.to_variable( + value=self._get_np_data(shape, dtype, append_batch_size), + name=name) + else: + if set_feed_dict: + self._feed_dict[name] = self._get_np_data(shape, dtype, + append_batch_size) + return layers.data( + name=name, + shape=shape, + dtype=dtype, + append_batch_size=append_batch_size) + + def make_sampled_softmax_with_cross_entropy(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + logits = self._get_data(name='Logits', shape=[256], dtype='float32') + label = self._get_data(name='Label', shape=[1], dtype='int64') + num_samples = 25 + output = layers.sampled_softmax_with_cross_entropy(logits, label, + num_samples) + return (output) + + def make_fit_a_line(self): + with program_guard( + fluid.default_main_program(), + startup_program=fluid.default_startup_program()): + x = self._get_data(name='x', shape=[13], dtype='float32') y_predict = layers.fc(input=x, size=1, act=None) - y = layers.data(name='y', shape=[1], dtype='float32') + y = self._get_data(name='y', shape=[1], dtype='float32') cost = layers.square_error_cost(input=y_predict, label=y) avg_cost = layers.mean(cost) - self.assertIsNotNone(avg_cost) - - print(str(program)) + return (avg_cost) - def test_recognize_digits_mlp(self): - program = Program() - with program_guard(program, startup_program=Program()): + def make_recognize_digits_mlp(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): # Change g_program, so the rest layers use `g_program` - images = layers.data(name='pixel', shape=[784], dtype='float32') - label = layers.data(name='label', shape=[1], dtype='int32') + images = self._get_data(name='pixel', shape=[784], dtype='float32') + label = self._get_data(name='label', shape=[1], dtype='int64') hidden1 = layers.fc(input=images, size=128, act='relu') hidden2 = layers.fc(input=hidden1, size=64, act='relu') predict = layers.fc(input=[hidden2, hidden1], @@ -623,32 +979,21 @@ class TestBook(unittest.TestCase): param_attr=["sftmax.w1", "sftmax.w2"]) cost = layers.cross_entropy(input=predict, label=label) avg_cost = layers.mean(cost) - self.assertIsNotNone(avg_cost) - - print(str(program)) - - def test_simple_conv2d(self): - program = Program() - with program_guard(program, startup_program=Program()): - images = layers.data( - name='pixel', shape=[3, 48, 48], dtype='float32') - layers.conv2d(input=images, num_filters=3, filter_size=[4, 4]) - - print(str(program)) + return (avg_cost) - def test_conv2d_transpose(self): - program = Program() - with program_guard(program): - img = layers.data(name='pixel', shape=[3, 2, 2], dtype='float32') - layers.conv2d_transpose(input=img, num_filters=10, output_size=28) - print(str(program)) + def make_conv2d_transpose(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + img = self._get_data(name='pixel', shape=[3, 2, 2], dtype='float32') + return layers.conv2d_transpose( + input=img, num_filters=10, output_size=28) - def test_recognize_digits_conv(self): - program = Program() - with program_guard(program, startup_program=Program()): - images = layers.data( + def make_recognize_digits_conv(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + images = self._get_data( name='pixel', shape=[1, 28, 28], dtype='float32') - label = layers.data(name='label', shape=[1], dtype='int32') + label = self._get_data(name='label', shape=[1], dtype='int64') conv_pool_1 = nets.simple_img_conv_pool( input=images, filter_size=5, @@ -667,19 +1012,19 @@ class TestBook(unittest.TestCase): predict = layers.fc(input=conv_pool_2, size=10, act="softmax") cost = layers.cross_entropy(input=predict, label=label) avg_cost = layers.mean(cost) + return avg_cost - print(str(program)) - - def test_word_embedding(self): - program = Program() - with program_guard(program, startup_program=Program()): + def make_word_embedding(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): dict_size = 10000 embed_size = 32 - first_word = layers.data(name='firstw', shape=[1], dtype='int64') - second_word = layers.data(name='secondw', shape=[1], dtype='int64') - third_word = layers.data(name='thirdw', shape=[1], dtype='int64') - forth_word = layers.data(name='forthw', shape=[1], dtype='int64') - next_word = layers.data(name='nextw', shape=[1], dtype='int64') + first_word = self._get_data(name='firstw', shape=[1], dtype='int64') + second_word = self._get_data( + name='secondw', shape=[1], dtype='int64') + third_word = self._get_data(name='thirdw', shape=[1], dtype='int64') + forth_word = self._get_data(name='forthw', shape=[1], dtype='int64') + next_word = self._get_data(name='nextw', shape=[1], dtype='int64') embed_first = layers.embedding( input=first_word, @@ -713,257 +1058,126 @@ class TestBook(unittest.TestCase): act='softmax') cost = layers.cross_entropy(input=predict_word, label=next_word) avg_cost = layers.mean(cost) - self.assertIsNotNone(avg_cost) - - print(str(program)) - - def test_linear_chain_crf(self): - program = Program() - with program_guard(program, startup_program=Program()): - label_dict_len = 10 - images = layers.data(name='pixel', shape=[784], dtype='float32') - label = layers.data(name='label', shape=[1], dtype='int32') - hidden = layers.fc(input=images, size=128) - crf = layers.linear_chain_crf( - input=hidden, label=label, param_attr=ParamAttr(name="crfw")) - crf_decode = layers.crf_decoding( - input=hidden, param_attr=ParamAttr(name="crfw")) - layers.chunk_eval( - input=crf_decode, - label=label, - chunk_scheme="IOB", - num_chunk_types=(label_dict_len - 1) // 2) - self.assertFalse(crf is None) - self.assertFalse(crf_decode is None) - - print(str(program)) + return (avg_cost) - def test_sigmoid_cross_entropy(self): - program = Program() - with program_guard(program): - dat = layers.data(name='data', shape=[10], dtype='float32') - lbl = layers.data(name='label', shape=[10], dtype='float32') + def make_sigmoid_cross_entropy(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + dat = self._get_data(name='data', shape=[10], dtype='float32') + lbl = self._get_data(name='label', shape=[10], dtype='float32') ignore_index = -1 - self.assertIsNotNone( - layers.sigmoid_cross_entropy_with_logits( - x=dat, label=lbl, ignore_index=ignore_index)) - print(str(program)) + return (layers.sigmoid_cross_entropy_with_logits( + x=dat, label=lbl, ignore_index=ignore_index)) - def test_hsigmoid(self): - program = Program() - with program_guard(program): - x = layers.data(name='x', shape=[2], dtype='float32') - y = layers.data(name='y', shape=[2], dtype='int64') - self.assertIsNotNone( - layers.hsigmoid( - input=x, label=y, num_classes=2)) - print(str(program)) + def make_hsigmoid(self): + self._force_to_use_cpu = True + with fluid.framework._dygraph_place_guard(place=fluid.CPUPlace()): + x = self._get_data(name='x', shape=[2], dtype='float32') + y = self._get_data(name='y', shape=[2], dtype='int64') + return (layers.hsigmoid(input=x, label=y, num_classes=2)) # test hsigmod with custom tree structure program2 = Program() with program_guard(program2): - x2 = layers.data(name='x2', shape=[4, 8], dtype='float32') - y2 = layers.data(name='y2', shape=[4], dtype='int64') - path_table = layers.data( + x2 = self._get_data(name='x2', shape=[4, 8], dtype='float32') + y2 = self._get_data(name='y2', shape=[4], dtype='int64') + path_table = self._get_data( name='path_table', shape=[4, 6], dtype='int64') - path_code = layers.data( + path_code = self._get_data( name='path_code', shape=[4, 6], dtype='int64') - self.assertIsNotNone( - layers.hsigmoid( - input=x2, - label=y2, - num_classes=6, - path_table=path_table, - path_code=path_code, - is_custom=True)) - print(str(program2)) - - def test_sequence_expand(self): - program = Program() - with program_guard(program): - x = layers.data(name='x', shape=[10], dtype='float32') - y = layers.data( - name='y', shape=[10, 20], dtype='float32', lod_level=2) - self.assertIsNotNone(layers.sequence_expand(x=x, y=y, ref_level=1)) - print(str(program)) - - def test_sequence_unpad(self): - program = Program() - with program_guard(program): - x = layers.data(name='x', shape=[10, 5], dtype='float32') - length = layers.data(name='length', shape=[1], dtype='int64') - self.assertIsNotNone(layers.sequence_unpad(x=x, length=length)) - print(str(program)) - - def test_pool2d(self): - program = Program() - with program_guard(program): - x = layers.data(name='x', shape=[3, 224, 224], dtype='float32') - self.assertIsNotNone( - layers.pool2d( - x, - pool_size=[5, 3], - pool_stride=[1, 2], - pool_padding=(2, 1))) - - def test_adaptive_pool2d(self): - program = Program() - with program_guard(program): - x = layers.data(name='x', shape=[3, 224, 224], dtype='float32') - self.assertIsNotNone( - layers.adaptive_pool2d( - x, [3, 3], pool_type='avg')) + return (layers.hsigmoid( + input=x2, + label=y2, + num_classes=6, + path_table=path_table, + path_code=path_code, + is_custom=True)) + + def make_pool2d(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + x = self._get_data(name='x', shape=[3, 224, 224], dtype='float32') + return (layers.pool2d( + x, pool_size=[5, 3], pool_stride=[1, 2], pool_padding=(2, 1))) + + def make_adaptive_pool2d(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + x = self._get_data(name='x', shape=[3, 224, 224], dtype='float32') + return (layers.adaptive_pool2d(x, [3, 3], pool_type='avg')) pool, mask = layers.adaptive_pool2d(x, [3, 3], require_index=True) - self.assertIsNotNone(pool) - self.assertIsNotNone(mask) - self.assertIsNotNone(layers.adaptive_pool2d(x, 3, pool_type='avg')) + return (pool) + return (mask) + return (layers.adaptive_pool2d(x, 3, pool_type='avg')) pool, mask = layers.adaptive_pool2d(x, 3, require_index=True) - self.assertIsNotNone(pool) - self.assertIsNotNone(mask) - - def test_adaptive_pool3d(self): - program = Program() - with program_guard(program): - x = layers.data(name='x', shape=[3, 244, 224, 224], dtype='float32') - self.assertIsNotNone( - layers.adaptive_pool3d( - x, [3, 3, 3], pool_type='avg')) + return (pool) + return (mask) + + def make_adaptive_pool3d(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + x = self._get_data( + name='x', shape=[3, 244, 224, 224], dtype='float32') + return (layers.adaptive_pool3d(x, [3, 3, 3], pool_type='avg')) pool, mask = layers.adaptive_pool3d( x, [3, 3, 3], require_index=True) - self.assertIsNotNone(pool) - self.assertIsNotNone(mask) - self.assertIsNotNone(layers.adaptive_pool3d(x, 3, pool_type='avg')) + return (pool) + return (mask) + return (layers.adaptive_pool3d(x, 3, pool_type='avg')) pool, mask = layers.adaptive_pool3d(x, 3, require_index=True) - self.assertIsNotNone(pool) - self.assertIsNotNone(mask) + return (pool) + return (mask) - def test_lstm_unit(self): - program = Program() - with program_guard(program): - x_t_data = layers.data( + def make_lstm_unit(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + x_t_data = self._get_data( name='x_t_data', shape=[10, 10], dtype='float32') x_t = layers.fc(input=x_t_data, size=10) - prev_hidden_data = layers.data( + prev_hidden_data = self._get_data( name='prev_hidden_data', shape=[10, 30], dtype='float32') prev_hidden = layers.fc(input=prev_hidden_data, size=30) - prev_cell_data = layers.data( + prev_cell_data = self._get_data( name='prev_cell', shape=[10, 30], dtype='float32') prev_cell = layers.fc(input=prev_cell_data, size=30) - self.assertIsNotNone( - layers.lstm_unit( - x_t=x_t, hidden_t_prev=prev_hidden, cell_t_prev=prev_cell)) - print(str(program)) + return (layers.lstm_unit( + x_t=x_t, hidden_t_prev=prev_hidden, cell_t_prev=prev_cell)) - def test_dynamic_lstmp(self): - program = Program() - with program_guard(program): - hidden_dim, proj_dim = 16, 8 - seq_data = layers.data( - name='seq_data', shape=[10, 10], dtype='float32', lod_level=1) - fc_out = layers.fc(input=seq_data, size=4 * hidden_dim) - self.assertIsNotNone( - layers.dynamic_lstmp( - input=fc_out, size=4 * hidden_dim, proj_size=proj_dim)) - print(str(program)) + def make_softmax(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + data = self._get_data(name='data', shape=[10], dtype='float32') + hid = layers.fc(input=data, size=20) + return (layers.softmax(hid, axis=1)) - def test_sequence_softmax(self): - program = Program() - with program_guard(program): - seq_data = layers.data( - name='seq_data', shape=[10, 10], dtype='float32', lod_level=1) - seq = layers.fc(input=seq_data, size=20) - self.assertIsNotNone(layers.sequence_softmax(seq)) - print(str(program)) - - def test_softmax(self): - program = Program() - with program_guard(program): - data = layers.data(name='data', shape=[10], dtype='float32') - hid = layers.fc(input=data, size=20) - self.assertIsNotNone(layers.softmax(hid, axis=1)) - print(str(program)) - - def test_space_to_depth(self): - program = Program() - with program_guard(program): - data = layers.data( + def make_space_to_depth(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + data = self._get_data( name='data', shape=[32, 9, 6, 6], append_batch_size=False, dtype='float32') - self.assertIsNotNone(layers.space_to_depth(data, 3)) - print(str(program)) - - def test_sequence_unsqueeze(self): - program = Program() - with program_guard(program): - x = layers.data(name='x', shape=[8, 2], dtype='float32') - out = layers.unsqueeze(input=x, axes=[1]) - self.assertIsNotNone(out) - print(str(program)) - - def test_squeeze(self): - program = Program() - with program_guard(program): - x = layers.data(name='x', shape=[1, 1, 4], dtype='float32') - out = layers.squeeze(input=x, axes=[2]) - self.assertIsNotNone(out) - print(str(program)) - - def test_lrn(self): - program = Program() - with program_guard(program): - data = layers.data(name='data', shape=[6, 2, 2], dtype='float32') - self.assertIsNotNone(layers.lrn(data)) - print(str(program)) - - def test_get_places(self): - program = Program() - with program_guard(program): - x = get_places(device_count=4) - self.assertIsNotNone(x) - print(str(program)) - - def test_sequence_reshape(self): - program = Program() - with program_guard(program): - x = layers.data(name='x', shape=[8], dtype='float32', lod_level=1) - out = layers.sequence_reshape(input=x, new_dim=16) - self.assertIsNotNone(out) - print(str(program)) + return (layers.space_to_depth(data, 3)) - def test_im2sequence(self): - program = Program() - with program_guard(program): - x = layers.data(name='x', shape=[3, 128, 128], dtype='float32') - y = layers.data(name='y', shape=[], dtype='float32') - output = layers.im2sequence( - input=x, - input_image_size=y, - stride=[1, 1], - filter_size=[2, 2], - out_stride=[1, 1]) - self.assertIsNotNone(output) - print(str(program)) + def make_lrn(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + data = self._get_data(name='data', shape=[6, 2, 2], dtype='float32') + return (layers.lrn(data)) - def test_sampled_softmax_with_cross_entropy(self): - program = Program() - with program_guard(program): - logits = layers.data(name='Logits', shape=[256], dtype='float64') - label = layers.data(name='Label', shape=[1], dtype='int64') - num_samples = 25 - output = layers.sampled_softmax_with_cross_entropy(logits, label, - num_samples) - self.assertIsNotNone(output) - print(str(program)) + def make_get_places(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + get_places(device_count=1) @decorators.prog_scope() - def test_nce(self): + def make_nce(self): window_size = 5 words = [] for i in range(window_size): words.append( - layers.data( + self._get_data( name='word_{0}'.format(i), shape=[1], dtype='int64')) dict_size = 10000 @@ -989,278 +1203,168 @@ class TestBook(unittest.TestCase): param_attr='nce.w', bias_attr='nce.b') avg_loss = layers.mean(loss) - self.assertIsNotNone(avg_loss) - print(str(default_main_program())) - - def test_row_conv(self): - program = Program() - with program_guard(program): - x = layers.data(name='x', shape=[16], dtype='float32', lod_level=1) - out = layers.row_conv(input=x, future_context_size=2) - self.assertIsNotNone(out) - print(str(program)) - - def test_multiplex(self): - program = Program() - with program_guard(program): - x1 = layers.data(name='x1', shape=[4], dtype='float32') - x2 = layers.data(name='x2', shape=[4], dtype='float32') - index = layers.data(name='index', shape=[1], dtype='int32') + return (avg_loss) + + def make_multiplex(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + x1 = self._get_data(name='x1', shape=[4], dtype='float32') + x2 = self._get_data(name='x2', shape=[4], dtype='float32') + index = self._get_data(name='index', shape=[1], dtype='int32') out = layers.multiplex(inputs=[x1, x2], index=index) - self.assertIsNotNone(out) - print(str(program)) + return (out) - def test_softmax_with_cross_entropy(self): - program = Program() - with program_guard(program): - x = layers.data(name='x', shape=[16], dtype='float32') - y = layers.data(name='label', shape=[1], dtype='int64') + def make_softmax_with_cross_entropy(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + x = self._get_data(name='x', shape=[16], dtype='float32') + y = self._get_data(name='label', shape=[1], dtype='int64') loss, softmax = layers.softmax_with_cross_entropy( x, y, return_softmax=True) - self.assertIsNotNone(loss) - self.assertIsNotNone(softmax) + return (loss) + return (softmax) loss = layers.softmax_with_cross_entropy(x, y) - self.assertIsNotNone(loss) - print(str(program)) + return (loss) - def test_smooth_l1(self): - program = Program() - with program_guard(program): - x = layers.data(name='x', shape=[4], dtype='float32') - y = layers.data(name='label', shape=[4], dtype='float32') + def make_smooth_l1(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + x = self._get_data(name='x', shape=[4], dtype='float32') + y = self._get_data(name='label', shape=[4], dtype='float32') loss = layers.smooth_l1(x, y) - self.assertIsNotNone(loss) - print(str(program)) + return (loss) - def test_scatter(self): - program = Program() - with program_guard(program): - x = layers.data( + def make_scatter(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + x = self._get_data( name='x', shape=[3, 3], append_batch_size=False, dtype='float32') - idx = layers.data( + idx = self._get_data( name='idx', shape=[2], append_batch_size=False, dtype='int32') - updates = layers.data( + updates = self._get_data( name='updates', shape=[2, 3], append_batch_size=False, dtype='float32') out = layers.scatter(input=x, index=idx, updates=updates) - self.assertIsNotNone(out) - print(str(program)) - - def test_sequence_scatter(self): - program = Program() - with program_guard(program): - x = layers.data( - name='x', - shape=[3, 6], - append_batch_size=False, - dtype='float32') - idx = layers.data( - name='idx', - shape=[12, 1], - append_batch_size=False, - dtype='int32', - lod_level=1) - updates = layers.data( - name='updates', - shape=[12, 1], - append_batch_size=False, - dtype='float32', - lod_level=1) - out = layers.sequence_scatter(input=x, index=idx, updates=updates) - self.assertIsNotNone(out) - print(str(program)) - - def test_sequence_slice(self): - program = Program() - with program_guard(program): - import numpy as np - seqs = layers.data( - name='x', shape=[10, 5], dtype='float32', lod_level=1) - offset = layers.assign(input=np.array([[0, 1]]).astype('int32')) - length = layers.assign(input=np.array([[2, 1]]).astype('int32')) - out = layers.sequence_slice( - input=seqs, offset=offset, length=length) - self.assertIsNotNone(out) - print(str(program)) - - def test_lod_reset(self): - program = Program() - with program_guard(program): - x = layers.data(name='x', shape=[10], dtype='float32') - y = layers.data( - name='y', shape=[10, 20], dtype='float32', lod_level=2) - print(layers.lod_reset(x=x, y=y)) - print(str(program)) + return (out) - def test_label_smooth(self): - program = Program() - with program_guard(program): - label = layers.data(name="label", shape=[1], dtype="float32") + def make_label_smooth(self): + # TODO(minqiyang): support gpu ut + self._force_to_use_cpu = True + with fluid.framework._dygraph_place_guard(place=fluid.CPUPlace()): + label = self._get_data(name="label", shape=[1], dtype="int32") one_hot_label = layers.one_hot(input=label, depth=10) smooth_label = layers.label_smooth( - label=one_hot_label, epsilon=0.1, dtype="float32") - self.assertIsNotNone(smooth_label) - print(str(program)) + label=one_hot_label, epsilon=0.1, dtype="int32") + return (smooth_label) - def test_topk(self): - program = Program() - with program_guard(program): - data = layers.data(name="label", shape=[200], dtype="float32") + def make_topk(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + data = self._get_data(name="label", shape=[200], dtype="float32") values, indices = layers.topk(data, k=5) - self.assertIsNotNone(values) - self.assertIsNotNone(indices) - print(str(program)) - - def test_roi_pool(self): - program = Program() - with program_guard(program): - x = layers.data(name="x", shape=[256, 30, 30], dtype="float32") - rois = layers.data( - name="rois", shape=[4], dtype="float32", lod_level=1) - output = layers.roi_pool(x, rois, 7, 7, 0.6) - self.assertIsNotNone(output) - print(str(program)) - - def test_psroi_pool(self): - program = Program() - with program_guard(program): - x = layers.data(name="x", shape=[245, 30, 30], dtype="float32") - rois = layers.data( - name="rois", shape=[4], dtype="float32", lod_level=1) - output = layers.psroi_pool(x, rois, 5, 0.25, 7, 7) - self.assertIsNotNone(output) - print(str(program)) - - def test_roi_align(self): - program = Program() - with program_guard(program): - x = layers.data(name="x", shape=[256, 30, 30], dtype="float32") - rois = layers.data( - name="rois", shape=[4], dtype="float32", lod_level=1) - output = layers.roi_align(x, rois, 14, 14, 0.5, 2) - self.assertIsNotNone(output) - print(str(program)) + return (values) + return (indices) - def test_resize_bilinear(self): - program = Program() - with program_guard(program): - x = layers.data(name='x', shape=[3, 9, 6], dtype="float32") + def make_resize_bilinear(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + x = self._get_data(name='x', shape=[3, 9, 6], dtype="float32") output = layers.resize_bilinear(x, out_shape=[12, 12]) - self.assertIsNotNone(output) + return (output) output = layers.resize_bilinear(x, scale=3) - self.assertIsNotNone(output) - print(str(program)) + return (output) - def test_resize_nearest(self): - program = Program() - with program_guard(program): - x = layers.data(name='x', shape=[3, 9, 6], dtype="float32") + def make_resize_nearest(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + x = self._get_data(name='x', shape=[3, 9, 6], dtype="float32") output = layers.resize_nearest(x, out_shape=[12, 12]) - self.assertIsNotNone(output) + return (output) output = layers.resize_nearest(x, scale=3) - self.assertIsNotNone(output) - print(str(program)) + return (output) - def test_polygon_box_transform(self): - program = Program() - with program_guard(program): - x = layers.data(name='x', shape=[8, 4, 4], dtype="float32") + def make_polygon_box_transform(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + x = self._get_data(name='x', shape=[8, 4, 4], dtype="float32") output = layers.polygon_box_transform(input=x) - self.assertIsNotNone(output) - print(str(program)) + return (output) - def test_l2_normalize(self): - program = Program() - with program_guard(program): - x = layers.data(name='x', shape=[8, 7, 10], dtype="float32") + def make_l2_normalize(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + x = self._get_data(name='x', shape=[8, 7, 10], dtype="float32") output = layers.l2_normalize(x, axis=1) + return output - def test_maxout(self): - program = Program() - with program_guard(program): - data = layers.data(name='x', shape=[8, 6, 6], dtype="float32") + def make_maxout(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + data = self._get_data(name='x', shape=[8, 6, 6], dtype="float32") output = layers.maxout(x=data, groups=2) - self.assertIsNotNone(output) - print(str(program)) + return (output) - def test_crop(self): - program = Program() - with program_guard(program): - x = layers.data(name='x', shape=[3, 5], dtype="float32") - y = layers.data(name='y', shape=[2, 3], dtype="float32") + def make_crop(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + x = self._get_data(name='x', shape=[3, 5], dtype="float32") + y = self._get_data(name='y', shape=[2, 3], dtype="float32") output = layers.crop(x, shape=y) - self.assertIsNotNone(output) - print(str(program)) - - def test_mean_iou(self): - program = Program() - with program_guard(program): - x = layers.data(name='x', shape=[16], dtype='float32') - y = layers.data(name='label', shape=[1], dtype='int64') - iou = layers.mean_iou(x, y, 2) - self.assertIsNotNone(iou) - print(str(program)) - - def test_argsort(self): - program = Program() - with program_guard(program): - data = layers.data(name='x', shape=[2, 3, 3], dtype="float32") + return (output) + + def make_mean_iou(self): + with fluid.framework._dygraph_place_guard(place=fluid.CPUPlace()): + x = self._get_data(name='x', shape=[16], dtype='int32') + y = self._get_data(name='label', shape=[16], dtype='int32') + iou = layers.mean_iou(x, y, self._high_data_bound) + return (iou) + + def make_argsort(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + data = self._get_data(name='x', shape=[2, 3, 3], dtype="float32") out, ids = layers.argsort(input=data, axis=1) - self.assertIsNotNone(out) - self.assertIsNotNone(ids) - print(str(program)) + return (out) + return (ids) - def test_rank_loss(self): - program = Program() - with program_guard(program): - label = layers.data( + def make_rank_loss(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + label = self._get_data( name='label', append_batch_size=False, shape=[16, 1], dtype="float32") - left = layers.data( + left = self._get_data( name='left', append_batch_size=False, shape=[16, 1], dtype="float32") - right = layers.data( + right = self._get_data( name='right', append_batch_size=False, shape=[16, 1], dtype="float32") out = layers.rank_loss(label, left, right, name="rank_loss") - self.assertIsNotNone(out) - print(str(program)) - - def test_flatten(self): - program = Program() - with program_guard(program): - x = layers.data( - name='x', - append_batch_size=False, - shape=[4, 4, 3], - dtype="float32") - out = layers.flatten(x, axis=1, name="flatten") - self.assertIsNotNone(out) + return (out) - def test_shape(self): - program = Program() - with program_guard(program): - input = layers.data( + def make_shape(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + input = self._get_data( name="input", shape=[3, 100, 100], dtype="float32") out = layers.shape(input) - self.assertIsNotNone(out) - print(str(program)) + return (out) - def test_pad2d(self): - program = Program() - with program_guard(program): - input = layers.data( + def make_pad2d(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + input = self._get_data( name="input", shape=[3, 100, 100], dtype="float32") paddings = layers.fill_constant(shape=[4], dtype='int32', value=1) out = layers.pad2d( @@ -1275,14 +1379,13 @@ class TestBook(unittest.TestCase): mode='reflect', data_format='NCHW', name="shape") - self.assertIsNotNone(out) - self.assertIsNotNone(out_1) - print(str(program)) + return (out) + return (out_1) - def test_prelu(self): - program = Program() - with program_guard(program): - input = layers.data( + def make_prelu(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + input = self._get_data( name="input", shape=[5, 200, 100, 100], dtype="float32") mode = 'channel' out = layers.prelu( @@ -1290,291 +1393,389 @@ class TestBook(unittest.TestCase): mode, param_attr=ParamAttr(initializer=Constant(1.0)), name='prelu') - self.assertIsNotNone(out) - print(str(program)) + return (out) - def test_brelu(self): - program = Program() - with program_guard(program): - input = layers.data(name="input", shape=[16], dtype="float32") + def make_brelu(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.brelu(input, t_min=1.0, t_max=20.0, name='brelu') - self.assertIsNotNone(out) - print(str(program)) + return (out) - def test_leaky_relu(self): - program = Program() - with program_guard(program): - input = layers.data(name="input", shape=[16], dtype="float32") + def make_leaky_relu(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.leaky_relu(input, alpha=0.1, name='leaky_relu') - self.assertIsNotNone(out) - print(str(program)) + return (out) - def test_soft_relu(self): - program = Program() - with program_guard(program): - input = layers.data(name="input", shape=[16], dtype="float32") + def make_soft_relu(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.soft_relu(input, threshold=30.0, name='soft_relu') - self.assertIsNotNone(out) - print(str(program)) + return (out) - def test_sigmoid(self): - program = Program() - with program_guard(program): - input = layers.data(name="input", shape=[16], dtype="float32") + def make_sigmoid(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.sigmoid(input, name='sigmoid') - self.assertIsNotNone(out) - print(str(program)) + return (out) - def test_logsigmoid(self): - program = Program() - with program_guard(program): - input = layers.data(name="input", shape=[16], dtype="float32") + def make_logsigmoid(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.logsigmoid(input, name='logsigmoid') - self.assertIsNotNone(out) - print(str(program)) + return (out) - def test_exp(self): - program = Program() - with program_guard(program): - input = layers.data(name="input", shape=[16], dtype="float32") + def make_exp(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.exp(input, name='exp') - self.assertIsNotNone(out) - print(str(program)) + return (out) - def test_tanh(self): - program = Program() - with program_guard(program): - input = layers.data(name="input", shape=[16], dtype="float32") + def make_tanh(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.tanh(input, name='tanh') - self.assertIsNotNone(out) - print(str(program)) + return (out) - def test_tanh_shrink(self): - program = Program() - with program_guard(program): - input = layers.data(name="input", shape=[16], dtype="float32") + def make_tanh_shrink(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.tanh_shrink(input, name='tanh_shrink') - self.assertIsNotNone(out) - print(str(program)) + return (out) - def test_sqrt(self): - program = Program() - with program_guard(program): - input = layers.data(name="input", shape=[16], dtype="float32") + def make_sqrt(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.sqrt(input, name='sqrt') - self.assertIsNotNone(out) - print(str(program)) + return (out) - def test_abs(self): - program = Program() - with program_guard(program): - input = layers.data(name="input", shape=[16], dtype="float32") + def make_abs(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.abs(input, name='abs') - self.assertIsNotNone(out) - print(str(program)) + return (out) - def test_ceil(self): - program = Program() - with program_guard(program): - input = layers.data(name="input", shape=[16], dtype="float32") + def make_ceil(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.ceil(input, name='ceil') - self.assertIsNotNone(out) - print(str(program)) + return (out) - def test_floor(self): - program = Program() - with program_guard(program): - input = layers.data(name="input", shape=[16], dtype="float32") + def make_floor(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.floor(input, name='floor') - self.assertIsNotNone(out) - print(str(program)) + return (out) - def test_cos(self): - program = Program() - with program_guard(program): - input = layers.data(name="input", shape=[16], dtype="float32") + def make_cos(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.cos(input, name='cos') - self.assertIsNotNone(out) - print(str(program)) + return (out) - def test_sin(self): - program = Program() - with program_guard(program): - input = layers.data(name="input", shape=[16], dtype="float32") + def make_sin(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.sin(input, name='sin') - self.assertIsNotNone(out) - print(str(program)) + return (out) - def test_round(self): - program = Program() - with program_guard(program): - input = layers.data(name="input", shape=[16], dtype="float32") + def make_round(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.round(input, name='round') - self.assertIsNotNone(out) - print(str(program)) + return (out) - def test_reciprocal(self): - program = Program() - with program_guard(program): - input = layers.data(name="input", shape=[16], dtype="float32") + def make_reciprocal(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.reciprocal(input, name='reciprocal') - self.assertIsNotNone(out) - print(str(program)) + return (out) - def test_square(self): - program = Program() - with program_guard(program): - input = layers.data(name="input", shape=[16], dtype="float32") + def make_square(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.square(input, name='square') - self.assertIsNotNone(out) - print(str(program)) + return (out) - def test_softplus(self): - program = Program() - with program_guard(program): - input = layers.data(name="input", shape=[16], dtype="float32") + def make_softplus(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.softplus(input, name='softplus') - self.assertIsNotNone(out) - print(str(program)) + return (out) - def test_softsign(self): - program = Program() - with program_guard(program): - input = layers.data(name="input", shape=[16], dtype="float32") + def make_softsign(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.softsign(input, name='softsign') - self.assertIsNotNone(out) - print(str(program)) - - def test_roi_perspective_transform(self): - program = Program() - with program_guard(program): - x = layers.data(name="x", shape=[256, 30, 30], dtype="float32") - rois = layers.data( - name="rois", shape=[8], dtype="float32", lod_level=1) - output = layers.roi_perspective_transform(x, rois, 7, 7, 0.6) - self.assertIsNotNone(output) - print(str(program)) - - def test_sequence_enumerate(self): - program = Program() - with program_guard(program): - x = layers.data(name="input", shape=[1], dtype='int32', lod_level=1) - out = layers.sequence_enumerate(input=x, win_size=2, pad_value=0) - print(str(program)) + return (out) - def test_cross_entropy(self): - program = Program() - with program_guard(program): - x = layers.data(name="x", shape=[30, 10], dtype="float32") - label = layers.data(name="label", shape=[30, 1], dtype="int32") + def make_cross_entropy(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + x = self._get_data(name="x", shape=[30, 10], dtype="float32") + label = self._get_data(name="label", shape=[30, 1], dtype="int64") mode = 'channel' out = layers.cross_entropy(x, label, False, 4) - self.assertIsNotNone(out) + return (out) - def test_bpr_loss(self): - program = Program() - with program_guard(program): - x = layers.data(name="x", shape=[30, 10], dtype="float32") - label = layers.data(name="label", shape=[30, 1], dtype="int32") + def make_bpr_loss(self): + self._force_to_use_cpu = True + with fluid.framework._dygraph_place_guard(place=fluid.CPUPlace()): + x = self._get_data(name="x", shape=[30, 10], dtype="float32") + label = self._get_data(name="label", shape=[30, 1], dtype="int64") out = layers.bpr_loss(x, label) - self.assertIsNotNone(out) - print(str(program)) + return (out) - def test_expand(self): - program = Program() - with program_guard(program): - x = layers.data(name="input", shape=[10], dtype='int32') + def make_expand(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + x = self._get_data(name="input", shape=[10], dtype='int32') out = layers.expand(x, [1, 2]) - print(str(program)) + return out - def test_uniform_random_batch_size_like(self): - program = Program() - with program_guard(program): - input = layers.data(name="input", shape=[13, 11], dtype='float32') + def make_uniform_random_batch_size_like(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + input = self._get_data( + name="input", shape=[13, 11], dtype='float32') out = layers.uniform_random_batch_size_like(input, [-1, 11]) - self.assertIsNotNone(out) - print(str(program)) + return (out) - def test_gaussian_random(self): - program = Program() - with program_guard(program): + def make_gaussian_random(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): out = layers.gaussian_random(shape=[20, 30]) - self.assertIsNotNone(out) - print(str(program)) + return (out) - def test_sampling_id(self): - program = Program() - with program_guard(program): - x = layers.data( + def make_sampling_id(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + x = self._get_data( name="X", shape=[13, 11], dtype='float32', append_batch_size=False) out = layers.sampling_id(x) - self.assertIsNotNone(out) - print(str(program)) + return (out) - def test_gaussian_random_batch_size_like(self): - program = Program() - with program_guard(program): - input = layers.data(name="input", shape=[13, 11], dtype='float32') + def make_gaussian_random_batch_size_like(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + input = self._get_data( + name="input", shape=[13, 11], dtype='float32') out = layers.gaussian_random_batch_size_like( input, shape=[-1, 11], mean=1.0, std=2.0) - self.assertIsNotNone(out) - print(str(program)) + return (out) - def test_sum(self): - program = Program() - with program_guard(program): - input = layers.data(name="input", shape=[13, 11], dtype='float32') + def make_sum(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + input = self._get_data( + name="input", shape=[13, 11], dtype='float32') out = layers.sum(input) - self.assertIsNotNone(out) - print(str(program)) + return (out) - def test_slice(self): + def make_slice(self): starts = [1, 0, 2] ends = [3, 3, 4] axes = [0, 1, 2] - program = Program() - with program_guard(program): - input = layers.data( + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + input = self._get_data( name="input", shape=[3, 4, 5, 6], dtype='float32') out = layers.slice(input, axes=axes, starts=starts, ends=ends) + return out - def test_softshrink(self): - program = Program() - with program_guard(program): - input = layers.data(name="input", shape=[16], dtype="float32") + def make_softshrink(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + input = self._get_data(name="input", shape=[16], dtype="float32") out = layers.softshrink(input, name='softshrink') - self.assertIsNotNone(out) - print(str(program)) + return (out) - def iou_similarity(self): - program = Program() - with program_guard(program): - x = layers.data(name="x", shape=[16], dtype="float32") - y = layers.data(name="y", shape=[16], dtype="float32") + def make_iou_similarity(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + x = self._get_data(name="x", shape=[4], dtype="float32") + y = self._get_data(name="y", shape=[4], dtype="float32") out = layers.iou_similarity(x, y, name='iou_similarity') - self.assertIsNotNone(out) - print(str(program)) + return (out) - def test_grid_sampler(self): - program = Program() - with program_guard(program): - x = layers.data(name='x', shape=[3, 5, 7], dtype='float32') - grid = layers.data(name='grid', shape=[5, 7, 2], dtype='float32') + def make_grid_sampler(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + x = self._get_data(name='x', shape=[3, 5, 7], dtype='float32') + grid = self._get_data(name='grid', shape=[5, 7, 2], dtype='float32') out = layers.grid_sampler(x, grid) - self.assertIsNotNone(out) - print(str(program)) + return (out) + + def make_bilinear_tensor_product_layer(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + data = self._get_data(name='data', shape=[4], dtype="float32") + + theta = self._get_data(name="theta", shape=[5], dtype="float32") + out = layers.bilinear_tensor_product(data, theta, 6) + return (out) + + def make_batch_norm(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + data = self._get_data( + name='data', shape=[32, 128, 128], dtype="float32") + out = layers.batch_norm(data) + return (out) + + def make_range(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + layers.range(0, 10, 2, 'int32') + y = layers.range(0.1, 10.0, 0.2, 'float32') + return y + + def make_spectral_norm(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + weight = self._get_data( + name='weight', + shape=[2, 3, 32, 32], + dtype="float32", + append_batch_size=False) + out = layers.spectral_norm(weight, dim=1, power_iters=1) + return (out) + + def make_kldiv_loss(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + x = self._get_data( + name='x', + shape=[32, 128, 128], + dtype="float32", + append_batch_size=False) + target = self._get_data( + name='target', + shape=[32, 128, 128], + dtype="float32", + append_batch_size=False) + loss = layers.kldiv_loss(x=x, target=target, reduction='batchmean') + return (loss) + + def make_temporal_shift(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + x = self._get_data(name="X", shape=[16, 4, 4], dtype="float32") + out = layers.temporal_shift(x, seg_num=2, shift_ratio=0.2) + return (out) + + def make_shuffle_channel(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + x = self._get_data(name="X", shape=[16, 4, 4], dtype="float32") + out = layers.shuffle_channel(x, group=4) + return (out) + + def make_fsp_matrix(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + x = self._get_data(name="X", shape=[16, 4, 4], dtype="float32") + y = self._get_data(name="Y", shape=[8, 4, 4], dtype="float32") + out = layers.fsp_matrix(x, y) + return (out) + + def make_pixel_shuffle(self): + with program_guard(fluid.default_main_program(), + fluid.default_startup_program()): + x = self._get_data(name="X", shape=[9, 4, 4], dtype="float32") + out = layers.pixel_shuffle(x, upscale_factor=3) + return (out) + + def test_dynamic_lstmp(self): + # TODO(minqiyang): dygraph do not support lod now + with self.static_graph(): + hidden_dim, proj_dim = 16, 8 + seq_data = layers.data( + name='seq_data', shape=[10, 10], dtype='float32', lod_level=1) + fc_out = layers.fc(input=seq_data, size=4 * hidden_dim) + self.assertIsNotNone( + layers.dynamic_lstmp( + input=fc_out, size=4 * hidden_dim, proj_size=proj_dim)) + + def test_linear_chain_crf(self): + # TODO(minqiyang): dygraph do not support lod now + with self.static_graph(): + label_dict_len = 10 + images = layers.data(name='pixel', shape=[784], dtype='float32') + label = layers.data(name='label', shape=[1], dtype='int32') + hidden = layers.fc(input=images, size=2) + crf = layers.linear_chain_crf( + input=hidden, label=label, param_attr=ParamAttr(name="crfw")) + crf_decode = layers.crf_decoding( + input=hidden, param_attr=ParamAttr(name="crfw")) + self.assertFalse(crf is None) + self.assertFalse(crf_decode is None) + return layers.chunk_eval( + input=crf_decode, + label=label, + chunk_scheme="IOB", + num_chunk_types=(label_dict_len - 1) // 2) + + def test_im2sequence(self): + # TODO(minqiyang): dygraph do not support lod now + with self.static_graph(): + x = layers.data(name='x', shape=[3, 128, 128], dtype='float32') + y = layers.data(name='y', shape=[], dtype='float32') + output = layers.im2sequence( + input=x, + input_image_size=y, + stride=[1, 1], + filter_size=[2, 2], + out_stride=[1, 1]) + return (output) + + def test_lod_reset(self): + # TODO(minqiyang): dygraph do not support lod now + with self.static_graph(): + # case 1 + x = layers.data(name='x', shape=[10], dtype='float32') + y = layers.data( + name='y', shape=[10, 20], dtype='float32', lod_level=2) + z = layers.lod_reset(x=x, y=y) + self.assertTrue(z.lod_level == 2) + # case 2 + lod_tensor_in = layers.data(name='lod_in', shape=[1], dtype='int64') + z = layers.lod_reset(x=x, y=lod_tensor_in) + self.assertTrue(z.lod_level == 1) + # case 3 + z = layers.lod_reset(x=x, target_lod=[1, 2, 3]) + self.assertTrue(z.lod_level == 1) + return z def test_affine_grid(self): - program = Program() - with program_guard(program): + with self.static_graph(): data = layers.data(name='data', shape=[2, 3, 3], dtype="float32") out, ids = layers.argsort(input=data, axis=1) @@ -1586,79 +1787,158 @@ class TestBook(unittest.TestCase): self.assertIsNotNone(data_0) self.assertIsNotNone(data_1) - print(str(program)) - def test_bilinear_tensor_product_layer(self): - program = Program() - with program_guard(program): - data = layers.data(name='data', shape=[4], dtype="float32") + def test_psroi_pool(self): + # TODO(minqiyang): dygraph do not support lod now + with self.static_graph(): + x = layers.data(name="x", shape=[245, 30, 30], dtype="float32") + rois = layers.data( + name="rois", shape=[4], dtype="float32", lod_level=1) + output = layers.psroi_pool(x, rois, 5, 0.25, 7, 7) + return (output) - theta = layers.data(name="theta", shape=[5], dtype="float32") - out = layers.bilinear_tensor_product(data, theta, 6) + def test_sequence_expand(self): + # TODO(minqiyang): dygraph do not support lod now + with self.static_graph(): + x = layers.data(name='x', shape=[10], dtype='float32') + y = layers.data( + name='y', shape=[10, 20], dtype='float32', lod_level=2) + return (layers.sequence_expand(x=x, y=y, ref_level=1)) - print(str(program)) + def test_sequence_reshape(self): + # TODO(minqiyang): dygraph do not support lod now + with self.static_graph(): + x = layers.data(name='x', shape=[8], dtype='float32', lod_level=1) + out = layers.sequence_reshape(input=x, new_dim=16) + return (out) - def test_batch_norm(self): - program = Program() - with program_guard(program): - data = layers.data( - name='data', shape=[32, 128, 128], dtype="float32") - out = layers.batch_norm(data) + def test_sequence_unpad(self): + # TODO(minqiyang): dygraph do not support lod now + with self.static_graph(): + x = layers.data(name='x', shape=[10, 5], dtype='float32') + length = layers.data(name='length', shape=[1], dtype='int64') + return (layers.sequence_unpad(x=x, length=length)) - print(str(program)) + def test_sequence_softmax(self): + # TODO(minqiyang): dygraph do not support lod now + with self.static_graph(): + seq_data = layers.data( + name='seq_data', shape=[10, 10], dtype='float32', lod_level=1) + seq = layers.fc(input=seq_data, size=20) + return (layers.sequence_softmax(seq)) - def test_range(self): - program = Program() - with program_guard(program): - layers.range(0, 10, 2, 'int32') - layers.range(0.1, 10.0, 0.2, 'float32') + def test_sequence_unsqueeze(self): + # TODO(minqiyang): dygraph do not support lod now + with self.static_graph(): + x = layers.data(name='x', shape=[8, 2], dtype='float32') + out = layers.unsqueeze(input=x, axes=[1]) + return (out) - print(str(program)) + def test_sequence_scatter(self): + # TODO(minqiyang): dygraph do not support lod now + with self.static_graph(): + x = layers.data( + name='x', + shape=[3, 6], + append_batch_size=False, + dtype='float32') + idx = layers.data( + name='idx', + shape=[12, 1], + append_batch_size=False, + dtype='int32', + lod_level=1) + updates = layers.data( + name='updates', + shape=[12, 1], + append_batch_size=False, + dtype='float32', + lod_level=1) + out = layers.sequence_scatter(input=x, index=idx, updates=updates) + return (out) - def test_spectral_norm(self): - program = Program() - with program_guard(program): - weight = layers.data( - name='weight', - shape=[2, 3, 32, 32], - dtype="float32", - append_batch_size=False) - out = layers.spectral_norm(weight, dim=1, power_iters=1) - self.assertIsNotNone(out) + def test_sequence_slice(self): + # TODO(minqiyang): dygraph do not support lod now + with self.static_graph(): + import numpy as np + seqs = layers.data( + name='x', shape=[10, 5], dtype='float32', lod_level=1) + offset = layers.assign(input=np.array([[0, 1]]).astype('int32')) + length = layers.assign(input=np.array([[2, 1]]).astype('int32')) + out = layers.sequence_slice( + input=seqs, offset=offset, length=length) + return (out) - def test_kldiv_loss(self): - program = Program() - with program_guard(program): - x = layers.data(name='x', shape=[32, 128, 128], dtype="float32") - target = layers.data( - name='target', shape=[32, 128, 128], dtype="float32") - loss = layers.kldiv_loss(x=x, target=target, reduction='batchmean') - self.assertIsNotNone(loss) + def test_roi_pool(self): + # TODO(minqiyang): dygraph do not support lod now + with self.static_graph(): + x = layers.data(name="x", shape=[256, 30, 30], dtype="float32") + rois = layers.data( + name="rois", shape=[4], dtype="float32", lod_level=1) + output = layers.roi_pool(x, rois, 7, 7, 0.6) + return (output) - print(str(program)) + def test_sequence_enumerate(self): + # TODO(minqiyang): dygraph do not support lod now + with self.static_graph(): + x = layers.data(name="input", shape=[1], dtype='int32', lod_level=1) + out = layers.sequence_enumerate(input=x, win_size=2, pad_value=0) - def test_temporal_shift(self): - program = Program() - with program_guard(program): - x = layers.data(name="X", shape=[16, 4, 4], dtype="float32") - out = layers.temporal_shift(x, seg_num=4, shift_ratio=0.2) - self.assertIsNotNone(out) - print(str(program)) + def test_roi_align(self): + # TODO(minqiyang): dygraph do not support lod now + with self.static_graph(): + x = layers.data(name="x", shape=[256, 30, 30], dtype="float32") + rois = layers.data( + name="rois", shape=[4], dtype="float32", lod_level=1) + output = layers.roi_align(x, rois, 14, 14, 0.5, 2) + return (output) - def test_shuffle_channel(self): - program = Program() - with program_guard(program): - x = layers.data(name="X", shape=[16, 4, 4], dtype="float32") - out = layers.shuffle_channel(x, group=4) - self.assertIsNotNone(out) - print(str(program)) + def test_roi_perspective_transform(self): + # TODO(minqiyang): dygraph do not support lod now + with self.static_graph(): + x = layers.data(name="x", shape=[256, 30, 30], dtype="float32") + rois = layers.data( + name="rois", shape=[8], dtype="float32", lod_level=1) + output = layers.roi_perspective_transform(x, rois, 7, 7, 0.6) + return (output) + + def test_row_conv(self): + # TODO(minqiyang): dygraph do not support lod now + with self.static_graph(): + x = layers.data(name='x', shape=[16], dtype='float32', lod_level=1) + out = layers.row_conv(input=x, future_context_size=2) + return (out) + + def test_simple_conv2d(self): + # TODO(minqiyang): dygraph do not support layers with param now + with self.static_graph(): + images = layers.data( + name='pixel', shape=[3, 48, 48], dtype='float32') + return layers.conv2d( + input=images, num_filters=3, filter_size=[4, 4]) + + def test_squeeze(self): + # TODO(minqiyang): dygraph do not support layers with param now + with self.static_graph(): + x = layers.data(name='x', shape=[1, 1, 4], dtype='float32') + out = layers.squeeze(input=x, axes=[2]) + return (out) - def test_fsp(self): + def test_flatten(self): + # TODO(minqiyang): dygraph do not support op without kernel now + with self.static_graph(): + x = layers.data( + name='x', + append_batch_size=False, + shape=[4, 4, 3], + dtype="float32") + out = layers.flatten(x, axis=1, name="flatten") + return (out) + + def test_linspace(self): program = Program() with program_guard(program): - x = layers.data(name="X", shape=[16, 4, 4], dtype="float32") - y = layers.data(name="Y", shape=[8, 4, 4], dtype="float32") - out = layers.fsp_matrix(x, y) + out = layers.linspace(20, 10, 5, 'float64') self.assertIsNotNone(out) print(str(program)) diff --git a/python/paddle/fluid/tests/unittests/test_linspace.py b/python/paddle/fluid/tests/unittests/test_linspace.py new file mode 100644 index 0000000000000000000000000000000000000000..eeecf178320327cc251f32bfe46c1622200339f4 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_linspace.py @@ -0,0 +1,71 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import unittest +import numpy as np +from op_test import OpTest + + +class TestLinspaceOpCommonCase(OpTest): + def setUp(self): + self.op_type = "linspace" + dtype = 'float32' + self.inputs = { + 'Start': np.array([0]).astype(dtype), + 'Stop': np.array([10]).astype(dtype), + 'Num': np.array([11]).astype('int32') + } + + self.outputs = {'Out': np.arange(0, 11).astype(dtype)} + + def test_check_output(self): + self.check_output() + + +class TestLinspaceOpReverseCase(OpTest): + def setUp(self): + self.op_type = "linspace" + dtype = 'float32' + self.inputs = { + 'Start': np.array([10]).astype(dtype), + 'Stop': np.array([0]).astype(dtype), + 'Num': np.array([11]).astype('int32') + } + + self.outputs = {'Out': np.arange(10, -1, -1).astype(dtype)} + + def test_check_output(self): + self.check_output() + + +class TestLinspaceOpNumOneCase(OpTest): + def setUp(self): + self.op_type = "linspace" + dtype = 'float32' + self.inputs = { + 'Start': np.array([10]).astype(dtype), + 'Stop': np.array([0]).astype(dtype), + 'Num': np.array([1]).astype('int32') + } + + self.outputs = {'Out': np.array(10, dtype=dtype)} + + def test_check_output(self): + self.check_output() + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_nearest_interp_op.py b/python/paddle/fluid/tests/unittests/test_nearest_interp_op.py index 5bb2260ef7a143670dd75fc88769603d1437173d..eb82af75e4a2bf834c010aede79d50b0d73c98bc 100644 --- a/python/paddle/fluid/tests/unittests/test_nearest_interp_op.py +++ b/python/paddle/fluid/tests/unittests/test_nearest_interp_op.py @@ -73,7 +73,14 @@ class TestNearestInterpOp(OpTest): self.op_type = "nearest_interp" input_np = np.random.random(self.input_shape).astype("float32") - output_np = nearest_neighbor_interp_np(input_np, self.out_h, self.out_w, + if self.scale > 0: + out_h = int(self.input_shape[2] * self.scale) + out_w = int(self.input_shape[3] * self.scale) + else: + out_h = self.out_h + out_w = self.out_w + + output_np = nearest_neighbor_interp_np(input_np, out_h, out_w, self.out_size, self.actual_shape, self.align_corners) self.inputs = {'X': input_np} @@ -84,6 +91,7 @@ class TestNearestInterpOp(OpTest): self.attrs = { 'out_h': self.out_h, 'out_w': self.out_w, + 'scale': self.scale, 'interp_method': self.interp_method, 'align_corners': self.align_corners, } @@ -100,6 +108,7 @@ class TestNearestInterpOp(OpTest): self.input_shape = [2, 3, 4, 4] self.out_h = 2 self.out_w = 2 + self.scale = 0. self.out_size = np.array([3, 3]).astype("int32") self.align_corners = True @@ -110,6 +119,7 @@ class TestNearestNeighborInterpCase1(TestNearestInterpOp): self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 + self.scale = 0. self.align_corners = True @@ -119,6 +129,7 @@ class TestNearestNeighborInterpCase2(TestNearestInterpOp): self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 + self.scale = 0. self.align_corners = True @@ -128,6 +139,7 @@ class TestNearestNeighborInterpCase3(TestNearestInterpOp): self.input_shape = [1, 1, 128, 64] self.out_h = 64 self.out_w = 128 + self.scale = 0. self.align_corners = True @@ -137,6 +149,7 @@ class TestNearestNeighborInterpCase4(TestNearestInterpOp): self.input_shape = [4, 1, 7, 8] self.out_h = 1 self.out_w = 1 + self.scale = 0. self.out_size = np.array([2, 2]).astype("int32") self.align_corners = True @@ -147,6 +160,7 @@ class TestNearestNeighborInterpCase5(TestNearestInterpOp): self.input_shape = [3, 3, 9, 6] self.out_h = 12 self.out_w = 12 + self.scale = 0. self.out_size = np.array([11, 11]).astype("int32") self.align_corners = True @@ -157,6 +171,7 @@ class TestNearestNeighborInterpCase6(TestNearestInterpOp): self.input_shape = [1, 1, 128, 64] self.out_h = 64 self.out_w = 128 + self.scale = 0. self.out_size = np.array([65, 129]).astype("int32") self.align_corners = True @@ -167,6 +182,7 @@ class TestNearestNeighborInterpActualShape(TestNearestInterpOp): self.input_shape = [3, 2, 32, 16] self.out_h = 64 self.out_w = 32 + self.scale = 0. self.out_size = np.array([66, 40]).astype("int32") self.align_corners = True @@ -179,7 +195,15 @@ class TestNearestInterpOpUint8(OpTest): self.op_type = "nearest_interp" input_np = np.random.randint( low=0, high=256, size=self.input_shape).astype("uint8") - output_np = nearest_neighbor_interp_np(input_np, self.out_h, self.out_w, + + if self.scale > 0: + out_h = int(self.input_shape[2] * self.scale) + out_w = int(self.input_shape[3] * self.scale) + else: + out_h = self.out_h + out_w = self.out_w + + output_np = nearest_neighbor_interp_np(input_np, out_h, out_w, self.out_size, self.actual_shape, self.align_corners) self.inputs = {'X': input_np} @@ -188,6 +212,7 @@ class TestNearestInterpOpUint8(OpTest): self.attrs = { 'out_h': self.out_h, 'out_w': self.out_w, + 'scale': self.scale, 'interp_method': self.interp_method, 'align_corners': self.align_corners } @@ -201,6 +226,7 @@ class TestNearestInterpOpUint8(OpTest): self.input_shape = [1, 3, 9, 6] self.out_h = 10 self.out_w = 9 + self.scale = 0. self.align_corners = True @@ -210,6 +236,7 @@ class TestNearestNeighborInterpCase1Uint8(TestNearestInterpOpUint8): self.input_shape = [2, 3, 128, 64] self.out_h = 120 self.out_w = 50 + self.scale = 0. self.align_corners = True @@ -219,6 +246,7 @@ class TestNearestNeighborInterpCase2Uint8(TestNearestInterpOpUint8): self.input_shape = [4, 1, 7, 8] self.out_h = 5 self.out_w = 13 + self.scale = 0. self.out_size = np.array([6, 15]).astype("int32") self.align_corners = True @@ -228,5 +256,38 @@ class TestNearestInterpWithoutCorners(TestNearestInterpOp): self.align_corners = False +class TestNearestNeighborInterpScale1(TestNearestInterpOp): + def init_test_case(self): + self.interp_method = 'nearest' + self.input_shape = [3, 2, 32, 16] + self.out_h = 64 + self.out_w = 32 + self.scale = 2. + self.out_size = np.array([66, 40]).astype("int32") + self.align_corners = True + + +class TestNearestNeighborInterpScale2(TestNearestInterpOp): + def init_test_case(self): + self.interp_method = 'nearest' + self.input_shape = [3, 2, 32, 16] + self.out_h = 64 + self.out_w = 32 + self.scale = 1.5 + self.out_size = np.array([66, 40]).astype("int32") + self.align_corners = True + + +class TestNearestNeighborInterpScale3(TestNearestInterpOp): + def init_test_case(self): + self.interp_method = 'nearest' + self.input_shape = [3, 2, 32, 16] + self.out_h = 64 + self.out_w = 32 + self.scale = 1. + self.out_size = np.array([66, 40]).astype("int32") + self.align_corners = True + + if __name__ == "__main__": unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_pg.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_pg.py index 041c56fce11e6f6abb0a941a9e9c9ad1cb60ab42..e1b3c2cb6dca1149e0a0b995d35977d74e04e4fe 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_pg.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_pg.py @@ -21,25 +21,8 @@ import os os.environ['FLAGS_enable_parallel_graph'] = str(1) import paddle.fluid.core as core import os -import paddle.fluid as fluid from parallel_executor_test_base import TestParallelExecutorBase - - -def simple_fc_net(use_feed): - img = fluid.layers.data(name='image', shape=[784], dtype='float32') - label = fluid.layers.data(name='label', shape=[1], dtype='int64') - hidden = img - for _ in range(4): - hidden = fluid.layers.fc( - hidden, - size=200, - act='tanh', - bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=1.0))) - prediction = fluid.layers.fc(hidden, size=10, act='softmax') - loss = fluid.layers.cross_entropy(input=prediction, label=label) - loss = fluid.layers.mean(loss) - return loss +from simple_nets import simple_fc_net, init_data class TestMNIST(TestParallelExecutorBase): @@ -47,19 +30,12 @@ class TestMNIST(TestParallelExecutorBase): def setUpClass(cls): os.environ['CPU_NUM'] = str(4) - def _init_data(self): - np.random.seed(5) - img = np.random.random(size=[32, 784]).astype(np.float32) - label = np.ones(shape=[32, 1], dtype='int64') - return img, label - # simple_fc def check_simple_fc_convergence(self, use_cuda, use_reduce=False): if use_cuda and not core.is_compiled_with_cuda(): return - img, label = self._init_data() - + img, label = init_data() self.check_network_convergence( simple_fc_net, feed_dict={"image": img, @@ -75,8 +51,7 @@ class TestMNIST(TestParallelExecutorBase): if use_cuda and not core.is_compiled_with_cuda(): return - img, label = self._init_data() - + img, label = init_data() single_first_loss, single_last_loss = self.check_network_convergence( method=simple_fc_net, seed=1, diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext.py index 1f23fae92c9d8148efb25facb602cdc4d485865b..92a5c58c11773e97ca0bb5ff2c21cbc8df612d58 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext.py @@ -14,19 +14,22 @@ from __future__ import print_function import os -os.environ['FLAGS_fuse_parameter_memory_size'] = "131072" -os.environ['FLAGS_fuse_parameter_groups_size'] = "3" import paddle.fluid as fluid +fluid.core._set_fuse_parameter_group_size(3) +fluid.core._set_fuse_parameter_memory_size(131072) + import paddle.fluid.layers.ops as ops from paddle.fluid.initializer import init_on_cpu from paddle.fluid.layers.learning_rate_scheduler import _decay_step_counter import paddle.fluid.core as core from parallel_executor_test_base import TestParallelExecutorBase +from simple_nets import init_data import unittest import math import numpy as np - +from functools import partial +os.environ['CPU_NUM'] = str(4) # FIXME(zcd): If the neural net has dropout_op, the output of ParallelExecutor # and Executor is different. Because, for ParallelExecutor, the dropout_op of # the neural net will be copied N copies(N is the number of device). This will @@ -110,7 +113,6 @@ def bottleneck_block(input, num_filters, stride, cardinality, reduction_ratio): return fluid.layers.elementwise_add(x=short, y=scale, act='relu') -batch_size = 12 img_shape = [3, 224, 224] @@ -178,53 +180,84 @@ def optimizer(learning_rate=0.01): return optimizer +def _batch_size(): + return 12 + + +def _iter(use_cuda): + if use_cuda: + return 10 + return 2 + + +gpu_img, gpu_label = init_data( + batch_size=_batch_size(), img_shape=img_shape, label_range=999) +cpu_img, cpu_label = init_data( + batch_size=_batch_size(), img_shape=img_shape, label_range=999) +feed_dict_gpu = {"image": gpu_img, "label": gpu_label} +feed_dict_cpu = {"image": cpu_img, "label": cpu_label} +model = SE_ResNeXt50Small + + +def _feed_dict(use_cuda): + if use_cuda: + return feed_dict_gpu + return feed_dict_cpu + + +def _get_result_of_origin_model(use_cuda): + global remove_bn + global remove_dropout + remove_bn = True + remove_dropout = True + first_loss, last_loss = TestParallelExecutorBase.check_network_convergence( + model, + feed_dict=_feed_dict(use_cuda), + iter=_iter(use_cuda), + batch_size=_batch_size(), + use_cuda=use_cuda, + use_reduce=False, + optimizer=optimizer) + + return first_loss, last_loss + + +origin_cpu_first_loss, origin_cpu_last_loss = _get_result_of_origin_model(False) +if core.is_compiled_with_cuda(): + origin_gpu_first_loss, origin_gpu_last_loss = _get_result_of_origin_model( + True) + + +def _get_origin_result(use_cuda): + if use_cuda: + assert core.is_compiled_with_cuda(), "Doesn't compiled with CUDA." + return origin_gpu_first_loss, origin_gpu_last_loss + return origin_cpu_first_loss, origin_cpu_last_loss + + class TestResnet(TestParallelExecutorBase): - @classmethod - def setUpClass(cls): - os.environ['CPU_NUM'] = str(4) - global remove_dropout - global remove_bn - remove_dropout = False - remove_bn = False - - def _init_data(self, batch_size=2, random=True): - np.random.seed(5) - if random: - img = np.random.random( - size=[batch_size] + img_shape).astype(np.float32) - else: - img = np.ones(shape=[batch_size] + img_shape, dtype='float32') - label = [np.random.randint(0, 999) for _ in range(batch_size)] - label = np.array(label).astype(np.int64).reshape(-1, 1) - return img, label - - def _compare_reduce_and_allreduce(self, - model, - use_cuda, - iter=20, - delta2=1e-5): + def _compare_reduce_and_allreduce(self, use_cuda, delta2=1e-5): if use_cuda and not core.is_compiled_with_cuda(): return global remove_bn + global remove_dropout remove_bn = True + remove_dropout = True - img, label = self._init_data(batch_size=batch_size) all_reduce_first_loss, all_reduce_last_loss = self.check_network_convergence( model, - feed_dict={"image": img, - "label": label}, - iter=iter, - batch_size=batch_size, + feed_dict=_feed_dict(use_cuda), + iter=_iter(use_cuda), + batch_size=_batch_size(), use_cuda=use_cuda, use_reduce=False, optimizer=optimizer) reduce_first_loss, reduce_last_loss = self.check_network_convergence( model, - feed_dict={"image": img, - "label": label}, - iter=iter, - batch_size=batch_size, + feed_dict=_feed_dict(use_cuda), + iter=_iter(use_cuda), + batch_size=_batch_size(), use_cuda=use_cuda, use_reduce=True, optimizer=optimizer) @@ -239,10 +272,9 @@ class TestResnet(TestParallelExecutorBase): all_reduce_first_loss_seq, all_reduce_last_loss_seq = self.check_network_convergence( model, - feed_dict={"image": img, - "label": label}, - iter=iter, - batch_size=batch_size, + feed_dict=_feed_dict(use_cuda), + iter=_iter(use_cuda), + batch_size=_batch_size(), use_cuda=use_cuda, use_reduce=False, optimizer=optimizer, @@ -250,10 +282,9 @@ class TestResnet(TestParallelExecutorBase): reduce_first_loss_seq, reduce_last_loss_seq = self.check_network_convergence( model, - feed_dict={"image": img, - "label": label}, - iter=iter, - batch_size=batch_size, + feed_dict=_feed_dict(use_cuda), + iter=_iter(use_cuda), + batch_size=_batch_size(), use_cuda=use_cuda, use_reduce=True, optimizer=optimizer, @@ -274,98 +305,91 @@ class TestResnet(TestParallelExecutorBase): for loss in zip(all_reduce_last_loss_seq, reduce_last_loss_seq): self.assertAlmostEquals(loss[0], loss[1], delta=delta2) - def _check_resnet_convergence(self, - model, - use_cuda=True, - use_reduce=False, - iter=20, - delta2=1e-5): + def _compare_result_with_origin_model(self, + get_origin_result, + check_func_2, + use_cuda, + delta2=1e-5, + compare_seperately=True, + rm_drop_out=False, + rm_bn=False): if use_cuda and not core.is_compiled_with_cuda(): return - global remove_dropout global remove_bn - remove_dropout = True - remove_bn = True - - img, label = self._init_data(batch_size=batch_size) - single_first_loss, single_last_loss = self.check_network_convergence( - model, - feed_dict={"image": img, - "label": label}, - iter=iter, - batch_size=batch_size, - use_cuda=use_cuda, - use_reduce=use_reduce, - optimizer=optimizer, - use_parallel_executor=False) - parallel_first_loss, parallel_last_loss = self.check_network_convergence( - model, - feed_dict={"image": img, - "label": label}, - iter=iter, - batch_size=batch_size, - use_cuda=use_cuda, - use_reduce=use_reduce, - optimizer=optimizer) - - self.assertAlmostEquals( - np.mean(parallel_first_loss), single_first_loss[0], delta=1e-5) - self.assertAlmostEquals( - np.mean(parallel_last_loss), single_last_loss[0], delta=delta2) - - def _compare_with_fused_all_reduce(self, - model, - use_cuda, - iter=20, - delta2=1e-5): - if use_cuda and not core.is_compiled_with_cuda(): - return - - global remove_bn - remove_bn = True + global remove_dropout + remove_bn = rm_bn or use_cuda + remove_dropout = rm_drop_out - img, label = self._init_data(batch_size=batch_size) - all_reduce_first_loss, all_reduce_last_loss = self.check_network_convergence( + func_1_first_loss, func_1_last_loss = get_origin_result(use_cuda) + func_2_first_loss, func_2_last_loss = check_func_2( model, - feed_dict={"image": img, - "label": label}, - iter=iter, - batch_size=batch_size, - use_cuda=use_cuda, - fuse_all_reduce_ops=False, - optimizer=optimizer) - reduce_first_loss, reduce_last_loss = self.check_network_convergence( - model, - feed_dict={"image": img, - "label": label}, - iter=iter, - batch_size=batch_size, - use_cuda=use_cuda, - fuse_all_reduce_ops=True, - optimizer=optimizer) + feed_dict=_feed_dict(use_cuda), + iter=_iter(use_cuda), + batch_size=_batch_size(), + use_cuda=use_cuda) + + if compare_seperately: + for loss in zip(func_1_first_loss, func_2_first_loss): + self.assertAlmostEquals(loss[0], loss[1], delta=1e-5) + for loss in zip(func_1_last_loss, func_2_last_loss): + self.assertAlmostEquals(loss[0], loss[1], delta=delta2) + else: + self.assertAlmostEquals( + np.mean(func_1_first_loss), func_2_first_loss[0], delta=1e-5) + self.assertAlmostEquals( + np.mean(func_1_last_loss), func_2_last_loss[0], delta=delta2) - for loss in zip(all_reduce_first_loss, reduce_first_loss): - self.assertAlmostEquals(loss[0], loss[1], delta=1e-5) - for loss in zip(all_reduce_last_loss, reduce_last_loss): - self.assertAlmostEquals(loss[0], loss[1], delta=delta2) + def test_seresnext_with_reduce(self): + self._compare_reduce_and_allreduce(use_cuda=False, delta2=1e-3) + self._compare_reduce_and_allreduce(use_cuda=True, delta2=1e-2) def test_seresnext_with_learning_rate_decay(self): - self._check_resnet_convergence(model=SE_ResNeXt50Small, use_cuda=True) - self._check_resnet_convergence( - model=SE_ResNeXt50Small, use_cuda=False, iter=2, delta2=1e-3) - - def test_seresnext_with_reduce(self): - self._compare_reduce_and_allreduce( - model=SE_ResNeXt50Small, use_cuda=True, delta2=1e-2) - self._compare_reduce_and_allreduce( - model=SE_ResNeXt50Small, use_cuda=False, iter=5) + # NOTE(zcd): This test is compare the result of use parallel_executor and executor, + # and the result of drop_out op and batch_norm op in this two executor + # have diff, so the two ops should be removed from the model. + check_func_1 = _get_origin_result + check_func_2 = partial( + self.check_network_convergence, + optimizer=optimizer, + use_parallel_executor=False) + self._compare_result_with_origin_model( + check_func_1, + check_func_2, + use_cuda=False, + rm_drop_out=True, + rm_bn=True, + compare_seperately=False, + delta2=1e-3) + self._compare_result_with_origin_model( + check_func_1, + check_func_2, + use_cuda=True, + rm_drop_out=True, + rm_bn=True, + compare_seperately=False) def test_seresnext_with_fused_all_reduce(self): - self._compare_with_fused_all_reduce( - model=SE_ResNeXt50Small, use_cuda=True, delta2=1e-3) - self._compare_with_fused_all_reduce( - model=SE_ResNeXt50Small, use_cuda=False, iter=2, delta2=1e-3) + # NOTE(zcd): In order to make the program faster, + # this unit test remove drop_out and batch_norm. + check_func_1 = _get_origin_result + check_func_2 = partial( + self.check_network_convergence, + optimizer=optimizer, + fuse_all_reduce_ops=True) + self._compare_result_with_origin_model( + check_func_1, + check_func_2, + use_cuda=False, + rm_drop_out=True, + rm_bn=True) + self._compare_result_with_origin_model( + check_func_1, + check_func_2, + use_cuda=True, + rm_drop_out=True, + rm_bn=True, + delta2=1e-3) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_test_while_train.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_test_while_train.py index d89fd87a38be460c561dbff656cdaa069ffbbd53..eaf9e484df922051ca503c4a8cd679fc243a0fe8 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_test_while_train.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_test_while_train.py @@ -13,7 +13,7 @@ # limitations under the License. from __future__ import print_function - +from simple_nets import simple_fc_net import paddle.fluid as fluid from paddle.fluid import compiler import paddle.fluid.core as core @@ -24,23 +24,6 @@ import sys import math -def simple_fc_net(): - img = fluid.layers.data(name='image', shape=[784], dtype='float32') - label = fluid.layers.data(name='label', shape=[1], dtype='int64') - hidden = img - for _ in range(4): - hidden = fluid.layers.fc( - hidden, - size=200, - act='tanh', - bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=1.0))) - prediction = fluid.layers.fc(hidden, size=10, act='softmax') - loss = fluid.layers.cross_entropy(input=prediction, label=label) - loss = fluid.layers.mean(loss) - return loss - - class ParallelExecutorTestingDuringTraining(unittest.TestCase): def check_network_convergence(self, use_cuda, build_strategy=None): os.environ['CPU_NUM'] = str(4) diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_transformer.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_transformer.py index aacc1c3ecda8c25dec9f08827a856d38c37b1b2f..8960cbcdd2f574a647229894c44c2b6ea188b7d4 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_transformer.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_transformer.py @@ -175,7 +175,7 @@ class TestTransformer(TestParallelExecutorBase): self.check_network_convergence(transformer, use_cuda=True) self.check_network_convergence( transformer, use_cuda=True, enable_sequential_execution=True) - self.check_network_convergence(transformer, use_cuda=False, iter=5) + self.check_network_convergence(transformer, use_cuda=False, iter=2) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_pass_builder.py b/python/paddle/fluid/tests/unittests/test_pass_builder.py index a96cb624f52303f05e40f572ccda858d1e329941..497bea43567774f356de379acced2544c8302d46 100644 --- a/python/paddle/fluid/tests/unittests/test_pass_builder.py +++ b/python/paddle/fluid/tests/unittests/test_pass_builder.py @@ -14,6 +14,7 @@ from __future__ import print_function +from simple_nets import simple_fc_net import paddle.fluid as fluid import paddle.fluid.core as core from paddle.fluid import compiler @@ -24,23 +25,6 @@ import sys import math -def simple_fc_net(): - img = fluid.layers.data(name='image', shape=[784], dtype='float32') - label = fluid.layers.data(name='label', shape=[1], dtype='int64') - hidden = img - for _ in range(4): - hidden = fluid.layers.fc( - hidden, - size=200, - act='tanh', - bias_attr=fluid.ParamAttr( - initializer=fluid.initializer.Constant(value=1.0))) - prediction = fluid.layers.fc(hidden, size=10, act='softmax') - loss = fluid.layers.cross_entropy(input=prediction, label=label) - loss = fluid.layers.mean(loss) - return loss - - class TestPassBuilder(unittest.TestCase): def check_network_convergence(self, use_cuda, build_strategy=None): os.environ['CPU_NUM'] = str(4) diff --git a/python/paddle/fluid/tests/unittests/test_pixel_shuffle.py b/python/paddle/fluid/tests/unittests/test_pixel_shuffle.py new file mode 100644 index 0000000000000000000000000000000000000000..cc3ae2b3b9d4c40a7ee992c04cac79f518acac6d --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_pixel_shuffle.py @@ -0,0 +1,50 @@ +# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import unittest +import numpy as np +from op_test import OpTest + + +class TestPixelShuffle(OpTest): + def setUp(self): + self.op_type = "pixel_shuffle" + n, c, h, w = 2, 9, 4, 4 + up_factor = 3 + shape = [n, c, h, w] + x = np.random.random(shape).astype("float32") + new_shape = (n, c // (up_factor * up_factor), up_factor, up_factor, h, + w) + # reshape to (num,output_channel,upscale_factor,upscale_factor,h,w) + npresult = np.reshape(x, new_shape) + # transpose to (num,output_channel,h,upscale_factor,w,upscale_factor) + npresult = npresult.transpose(0, 1, 4, 2, 5, 3) + oshape = [n, c // (up_factor * up_factor), h * up_factor, w * up_factor] + npresult = np.reshape(npresult, oshape) + + self.inputs = {'X': x} + self.outputs = {'Out': npresult} + self.attrs = {'upscale_factor': up_factor} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Out') + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_recurrent_op.py b/python/paddle/fluid/tests/unittests/test_recurrent_op.py index 6dfc85e301a2eda66bade09a8b6dd0004155f385..cf86ebf0a81c5c6cd36a5edb5d61a11cdd98ae11 100644 --- a/python/paddle/fluid/tests/unittests/test_recurrent_op.py +++ b/python/paddle/fluid/tests/unittests/test_recurrent_op.py @@ -15,7 +15,7 @@ from __future__ import print_function import unittest - +import paddle.fluid as fluid import paddle.fluid.layers as layers from paddle.fluid.framework import Program, grad_var_name from paddle.fluid.executor import Executor @@ -115,10 +115,6 @@ class RecurrentOpTest1(unittest.TestCase): def setup_program(self): self.main_program = Program() self.startup_program = Program() - self.p_info = { - "main_program": self.main_program, - "startup_program": self.startup_program - } self.place = core.CPUPlace() def setUp(self): @@ -129,33 +125,29 @@ class RecurrentOpTest1(unittest.TestCase): self.output_shape = (self.sent_len, self.batch_size, self.input_dim) self.py_rnn = PySimpleRNN1(self.input_shape, self.output_shape) - self.output = layers.mean(self.create_rnn_op(), **self.p_info) + with fluid.program_guard(self.main_program, self.startup_program): + self.output = layers.mean(self.create_rnn_op()) def create_rnn_op(self): x = layers.data( shape=[self.sent_len, self.batch_size, self.input_dim], dtype='float32', name='x', - append_batch_size=False, - **self.p_info) + append_batch_size=False) x.stop_gradient = False h_boot = layers.data( - shape=[self.input_dim], - dtype='float32', - name='h_boot', - **self.p_info) + shape=[self.input_dim], dtype='float32', name='h_boot') h_boot.stop_gradient = False - rnn = layers.StaticRNN(main_program=self.main_program) + rnn = layers.StaticRNN() with rnn.step(): h_pre = rnn.memory(init=h_boot) x_t = rnn.step_input(x) h = layers.scale( x=layers.elementwise_add( - x=h_pre, y=x_t, **self.p_info), - scale=self.py_rnn.scale, - **self.p_info) + x=h_pre, y=x_t), + scale=self.py_rnn.scale) rnn.update_memory(h_pre, h) rnn.output(h) @@ -193,7 +185,8 @@ class RecurrentOpTest1(unittest.TestCase): def test_backward(self): self.check_forward() - append_backward(self.output) + with fluid.program_guard(self.main_program, self.startup_program): + append_backward(self.output) ana_grad = [np.array(x) for x in self.backward()] @@ -205,12 +198,8 @@ class RecurrentOpTest1(unittest.TestCase): num_grad[idx], ana_grad[idx], rtol=0.1).all()) def check_forward(self): - print('test recurrent op forward') pd_output = self.forward() py_output = self.py_rnn.forward() - print('pd_output', pd_output) - print - print('py_output', py_output) self.assertEqual(pd_output.shape, py_output.shape) self.assertTrue(np.isclose(pd_output, py_output, rtol=0.1).all()) @@ -263,24 +252,21 @@ class RecurrentOpTest2(RecurrentOpTest1): self.output_shape = (self.sent_len, self.batch_size, self.input_dim) self.py_rnn = PySimpleRNN2(self.input_shape, self.output_shape) - self.output = layers.mean(self.create_rnn_op(), **self.p_info) + with fluid.program_guard(self.main_program, self.startup_program): + self.output = layers.mean(self.create_rnn_op()) def create_rnn_op(self): x = layers.data( shape=[self.sent_len, self.batch_size, self.input_dim], dtype='float32', name='x', - append_batch_size=False, - **self.p_info) + append_batch_size=False) x.stop_gradient = False h_boot = layers.data( - shape=[self.input_dim], - dtype='float32', - name='h_boot', - **self.p_info) + shape=[self.input_dim], dtype='float32', name='h_boot') h_boot.stop_gradient = False - rnn = layers.StaticRNN(main_program=self.main_program) + rnn = layers.StaticRNN() with rnn.step(): h_pre = rnn.memory(init=h_boot) x_t = rnn.step_input(x) @@ -288,18 +274,13 @@ class RecurrentOpTest2(RecurrentOpTest1): temp_l = layers.fc(input=x_t, size=self.input_dim, param_attr='W', - bias_attr=False, - **self.p_info) + bias_attr=False) temp_r = layers.fc(input=h_pre, size=self.input_dim, param_attr='U', - bias_attr=False, - **self.p_info) + bias_attr=False) - h = layers.sigmoid( - x=layers.elementwise_add( - x=temp_l, y=temp_r, **self.p_info), - **self.p_info) + h = layers.sigmoid(x=layers.elementwise_add(x=temp_l, y=temp_r)) rnn.update_memory(h_pre, h) rnn.output(h) @@ -362,40 +343,38 @@ class RecurrentOpMultipleMemoryTest(RecurrentOpTest1): self.py_rnn = RecurrentOpMultipleMemoryTest.PySimpleRNN3( self.input_shape, self.output_shape) - self.output = layers.mean(self.create_rnn_op(), **self.p_info) + with fluid.program_guard(self.main_program, self.startup_program): + self.output = layers.mean(self.create_rnn_op()) def create_rnn_op(self): x = layers.data( shape=[self.sent_len, self.batch_size, self.input_dim], dtype='float32', name='x', - append_batch_size=False, - **self.p_info) + append_batch_size=False) x.stop_gradient = False h_boot1 = layers.data( shape=[self.batch_size, self.input_dim], dtype='float32', name='h_boot1', - append_batch_size=False, - **self.p_info) + append_batch_size=False) h_boot1.stop_gradient = False h_boot2 = layers.data( shape=[self.batch_size, self.input_dim], dtype='float32', name='h_boot2', - append_batch_size=False, - **self.p_info) + append_batch_size=False) h_boot2.stop_gradient = False - rnn = layers.StaticRNN(main_program=self.main_program) + rnn = layers.StaticRNN() with rnn.step(): h_pre1 = rnn.memory(init=h_boot1) h_pre2 = rnn.memory(init=h_boot2) x_t = rnn.step_input(x) - mem1 = layers.scale(x=h_pre1, scale=1.0, **self.p_info) - mem2 = layers.scale(x=h_pre2, scale=1.0, **self.p_info) - out = layers.sums(input=[mem1, x_t, mem2], **self.p_info) + mem1 = layers.scale(x=h_pre1, scale=1.0) + mem2 = layers.scale(x=h_pre2, scale=1.0) + out = layers.sums(input=[mem1, x_t, mem2]) rnn.update_memory(h_pre1, mem1) rnn.update_memory(h_pre2, mem2) @@ -446,23 +425,23 @@ class RecurrentOpNoMemBootTest(RecurrentOpTest1): self.output_shape = (self.sent_len, self.batch_size, self.input_dim) self.py_rnn = RecurrentOpNoMemBootTest.PySimpleRNN4(self.input_shape, self.output_shape) - self.output = layers.mean(self.create_rnn_op(), **self.p_info) - print(self.main_program) + + with fluid.program_guard(self.main_program, self.startup_program): + self.output = layers.mean(self.create_rnn_op()) def create_rnn_op(self): x = layers.data( shape=[self.sent_len, self.batch_size, self.input_dim], dtype='float32', name='x', - append_batch_size=False, - **self.p_info) + append_batch_size=False) x.stop_gradient = False - rnn = layers.StaticRNN(main_program=self.main_program) + rnn = layers.StaticRNN() with rnn.step(): mem_pre = rnn.memory(shape=[-1, self.input_dim], batch_ref=x) x_t = rnn.step_input(x) - mem = layers.elementwise_add(x=mem_pre, y=x_t, **self.p_info) + mem = layers.elementwise_add(x=mem_pre, y=x_t) rnn.update_memory(mem_pre, mem) rnn.output(mem) diff --git a/python/paddle/fluid/tests/unittests/test_reduce_op.py b/python/paddle/fluid/tests/unittests/test_reduce_op.py index 8fc8125a773543eea768783155ad152c475535b5..65fc1453d8db13ad9c85746c3bf148f898e8f788 100644 --- a/python/paddle/fluid/tests/unittests/test_reduce_op.py +++ b/python/paddle/fluid/tests/unittests/test_reduce_op.py @@ -91,6 +91,78 @@ class TestProdOp(OpTest): self.check_grad(['X'], 'Out') +class TestAllOp(OpTest): + def setUp(self): + self.op_type = "reduce_all" + self.inputs = {'X': np.random.randint(0, 2, (5, 6, 10)).astype("bool")} + self.outputs = {'Out': self.inputs['X'].all()} + self.attrs = {'reduce_all': True} + + def test_check_output(self): + self.check_output() + + +class TestAllOpWithDim(OpTest): + def setUp(self): + self.op_type = "reduce_all" + self.inputs = {'X': np.random.randint(0, 2, (5, 6, 10)).astype("bool")} + self.attrs = {'dim': [1]} + self.outputs = {'Out': self.inputs['X'].all(axis=1)} + + def test_check_output(self): + self.check_output() + + +class TestAllOpWithKeepDim(OpTest): + def setUp(self): + self.op_type = "reduce_all" + self.inputs = {'X': np.random.randint(0, 2, (5, 6, 10)).astype("bool")} + self.attrs = {'dim': [1], 'keep_dim': True} + self.outputs = { + 'Out': np.expand_dims( + self.inputs['X'].all(axis=1), axis=1) + } + + def test_check_output(self): + self.check_output() + + +class TestAnyOp(OpTest): + def setUp(self): + self.op_type = "reduce_any" + self.inputs = {'X': np.random.randint(0, 2, (5, 6, 10)).astype("bool")} + self.outputs = {'Out': self.inputs['X'].any()} + self.attrs = {'reduce_all': True} + + def test_check_output(self): + self.check_output() + + +class TestAnyOpWithDim(OpTest): + def setUp(self): + self.op_type = "reduce_any" + self.inputs = {'X': np.random.randint(0, 2, (5, 6, 10)).astype("bool")} + self.attrs = {'dim': [1]} + self.outputs = {'Out': self.inputs['X'].any(axis=1)} + + def test_check_output(self): + self.check_output() + + +class TestAnyOpWithKeepDim(OpTest): + def setUp(self): + self.op_type = "reduce_any" + self.inputs = {'X': np.random.randint(0, 2, (5, 6, 10)).astype("bool")} + self.attrs = {'dim': [1], 'keep_dim': True} + self.outputs = { + 'Out': np.expand_dims( + self.inputs['X'].any(axis=1), axis=1) + } + + def test_check_output(self): + self.check_output() + + class Test1DReduce(OpTest): def setUp(self): self.op_type = "reduce_sum" diff --git a/python/paddle/fluid/trainer_desc.py b/python/paddle/fluid/trainer_desc.py index 380c404fb2d6a36bf3732ebbff4b6cef22f71362..c742ee002aa6c470c41d46978a4e08fc774c3152 100644 --- a/python/paddle/fluid/trainer_desc.py +++ b/python/paddle/fluid/trainer_desc.py @@ -28,10 +28,10 @@ class TrainerDesc(object): import multiprocessing as mp # set default thread num == cpu count self.proto_desc.thread_num = mp.cpu_count() - self.fleet_desc_ = None - self.device_worker_ = None - self.program_ = None - self.infer_ = False + self._fleet_desc = None + self._device_worker = None + self._program = None + self._infer = False def _set_fetch_var_and_info(self, fetch_vars, fetch_info, print_period): for i, v in enumerate(fetch_vars): @@ -47,19 +47,19 @@ class TrainerDesc(object): self.proto_desc.thread_num = thread_num def _set_device_worker(self, device_worker): - self.device_worker_ = device_worker + self._device_worker = device_worker def _set_infer(self, infer): - self.infer_ = infer + self._infer = infer def _set_fleet_desc(self, fleet_desc): - self.fleet_desc_ = fleet_desc + self._fleet_desc = fleet_desc def _gen_trainer_desc(self): pass def _set_program(self, program): - self.program_ = program + self._program = program def _desc(self): from google.protobuf import text_format @@ -73,13 +73,13 @@ class MultiTrainer(TrainerDesc): def _set_program(self, program): super(MultiTrainer, self)._set_program(program) - self.program_ = program + self._program = program def _gen_trainer_desc(self): super(MultiTrainer, self)._gen_trainer_desc() self.proto_desc.class_name = "MultiTrainer" - self.device_worker_._set_infer(self.infer_) - self.device_worker_._gen_worker_desc(self.proto_desc) + self._device_worker._set_infer(self._infer) + self._device_worker._gen_worker_desc(self.proto_desc) class DistMultiTrainer(TrainerDesc): @@ -89,13 +89,13 @@ class DistMultiTrainer(TrainerDesc): def _set_program(self, program): super(DistMultiTrainer, self)._set_program(program) - self.program_ = program + self._program = program def _gen_trainer_desc(self): super(DistMultiTrainer, self)._gen_trainer_desc() self.proto_desc.class_name = "DistMultiTrainer" - if self.program_ == None: + if self._program == None: raise RuntimeError("None Program") - self.device_worker_._set_infer(self.infer_) - self.device_worker_._set_program(self.program_) - self.device_worker_._gen_worker_desc(self.proto_desc) + self._device_worker._set_infer(self._infer) + self._device_worker._set_program(self._program) + self._device_worker._gen_worker_desc(self.proto_desc) diff --git a/python/setup.py.in b/python/setup.py.in index 9ab4e9742cfbaf4e2d08e7c27b6ba231c85c4ec2..eef8afac65225e78f1f5bff35d74311e6450191c 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -157,10 +157,6 @@ package_data['paddle.libs']= [] package_data['paddle.libs']=[('libwarpctc' if os.name != 'nt' else 'warpctc') + ext_name] shutil.copy('${WARPCTC_LIBRARIES}', libs_path) -if '${WITH_WBAES}' == 'ON': - package_data['paddle.libs'] += ['libwbaes' + ext_name] - shutil.copy('${WBAES_SHARED_LIB}', libs_path) - if '${WITH_MKL}' == 'ON': shutil.copy('${MKLML_SHARED_LIB}', libs_path) shutil.copy('${MKLML_SHARED_IOMP_LIB}', libs_path)