Skip to content

P
Paddle

BaiXuePrincess / Paddle
与 Fork 源项目一致

Fork自 PaddlePaddle / Paddle

通知 1
Star 1
Fork 0
P
Paddle
提交 745634a1 编写于 作者: Y Yang Yang(Tony) 提交者： GitHub
浏览文件
操作

Merge branch 'develop' into update-doc-pybind

上级 62b56e76 064c3695
展开全部 隐藏空白更改
内联 并排
Showing with 19282 addition and 6021 deletion
.pre-commit-config.yaml
浏览文件 @ 745634a1
......@@ -31,6 +31,3 @@
- id: go-fmt
types:
- go
- id: gometalinter
types:
- go
CMakeLists.txt
浏览文件 @ 745634a1
......@@ -86,6 +86,14 @@ if(ANDROID OR IOS)
"Disable MKLDNN when cross-compiling for Android and iOS" FORCE)
set(WITH_MKLML OFF CACHE STRING
"Disable MKLML package when cross-compiling for Android and iOS" FORCE)
# Compile PaddlePaddle mobile inference library
if (NOT WITH_C_API)
set(WITH_C_API ON CACHE STRING
"Always compile the C_API when cross-compiling for Android and iOS" FORCE)
endif()
set(MOBILE_INFERENCE ON)
add_definitions(-DPADDLE_MOBILE_INFERENCE)
endif()
set(THIRD_PARTY_PATH "${CMAKE_BINARY_DIR}/third_party" CACHE STRING
......@@ -97,6 +105,12 @@ if (WITH_C_API AND WITH_PYTHON)
"different Python interpreter from compiling.")
endif()
if(MOBILE_INFERENCE)
set(THIRD_PARTY_BUILD_TYPE MinSizeRel)
else()
set(THIRD_PARTY_BUILD_TYPE Release)
endif()
########################################################################################
include(external/mklml) # download mklml package
......@@ -160,9 +174,11 @@ endif(USE_NNPACK)
add_subdirectory(proto)
# "add_subdirectory(go)" should be placed after the following loine,
# because it depends on paddle/optimizer.
add_subdirectory(paddle/optimizer)
if(NOT MOBILE_INFERENCE)
# "add_subdirectory(go)" should be placed after the following loine,
# because it depends on paddle/optimizer.
add_subdirectory(paddle/optimizer)
endif()
# "add_subdirectory(paddle)" and "add_subdirectory(python)" should be
# placed after this block, because they depends on it.
......
cmake/configure.cmake
浏览文件 @ 745634a1
......@@ -24,6 +24,10 @@ if(WITH_DOUBLE)
add_definitions(-DPADDLE_TYPE_DOUBLE)
endif(WITH_DOUBLE)
if(WITH_TESTING)
add_definitions(-DPADDLE_WITH_TESTING)
endif(WITH_TESTING)
if(NOT WITH_TIMER)
add_definitions(-DPADDLE_DISABLE_TIMER)
endif(NOT WITH_TIMER)
......@@ -49,11 +53,12 @@ if(NOT WITH_GOLANG)
endif(NOT WITH_GOLANG)
if(NOT WITH_GPU)
add_definitions(-DPADDLE_ONLY_CPU)
add_definitions(-DHPPL_STUB_FUNC)
list(APPEND CMAKE_CXX_SOURCE_FILE_EXTENSIONS cu)
else()
add_definitions(-DPADDLE_WITH_CUDA)
FIND_PACKAGE(CUDA REQUIRED)
if(${CUDA_VERSION_MAJOR} VERSION_LESS 7)
......
cmake/external/eigen.cmake
浏览文件 @ 745634a1
......@@ -8,7 +8,7 @@ ExternalProject_Add(
extern_eigen3
${EXTERNAL_PROJECT_LOG_ARGS}
GIT_REPOSITORY "https://github.com/RLovelett/eigen.git"
GIT_TAG "master"
GIT_TAG 4e79cb69b9425f5f8c3a84be4350d4ab75b5fd9d
PREFIX ${EIGEN_SOURCE_DIR}
UPDATE_COMMAND ""
CONFIGURE_COMMAND ""
......
cmake/external/gflags.cmake
浏览文件 @ 745634a1
......@@ -36,6 +36,7 @@ ExternalProject_Add(
# change this back to the official Github repo once my PR is
# merged.
GIT_REPOSITORY "https://github.com/wangkuiyi/gflags.git"
GIT_TAG 986964c07427ecb9cdb5bd73f73ebbd40e54dadb
PREFIX ${GFLAGS_SOURCES_DIR}
UPDATE_COMMAND ""
CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
......@@ -45,11 +46,11 @@ ExternalProject_Add(
-DCMAKE_INSTALL_PREFIX=${GFLAGS_INSTALL_DIR}
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
-DBUILD_TESTING=OFF
-DCMAKE_BUILD_TYPE=Release
-DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
${EXTERNAL_OPTIONAL_ARGS}
CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GFLAGS_INSTALL_DIR}
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
-DCMAKE_BUILD_TYPE:STRING=Release
-DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
)
ADD_LIBRARY(gflags STATIC IMPORTED GLOBAL)
......
cmake/external/glog.cmake
浏览文件 @ 745634a1
......@@ -31,6 +31,7 @@ ExternalProject_Add(
${EXTERNAL_PROJECT_LOG_ARGS}
DEPENDS gflags
GIT_REPOSITORY "https://github.com/google/glog.git"
GIT_TAG v0.3.5
PREFIX ${GLOG_SOURCES_DIR}
UPDATE_COMMAND ""
CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
......@@ -43,12 +44,12 @@ ExternalProject_Add(
-DWITH_GFLAGS=ON
-Dgflags_DIR=${GFLAGS_INSTALL_DIR}/lib/cmake/gflags
-DBUILD_TESTING=OFF
-DCMAKE_BUILD_TYPE=Release
-DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
${EXTERNAL_OPTIONAL_ARGS}
CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GLOG_INSTALL_DIR}
-DCMAKE_INSTALL_LIBDIR:PATH=${GLOG_INSTALL_DIR}/lib
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
-DCMAKE_BUILD_TYPE:STRING=Release
-DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
)
ADD_LIBRARY(glog STATIC IMPORTED GLOBAL)
......
cmake/external/gtest.cmake
浏览文件 @ 745634a1
......@@ -56,11 +56,11 @@ IF(WITH_TESTING)
-DBUILD_GMOCK=ON
-Dgtest_disable_pthreads=ON
-Dgtest_force_shared_crt=ON
-DCMAKE_BUILD_TYPE=Release
-DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
${EXTERNAL_OPTIONAL_ARGS}
CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GTEST_INSTALL_DIR}
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
-DCMAKE_BUILD_TYPE:STRING=Release
-DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
)
ADD_LIBRARY(gtest STATIC IMPORTED GLOBAL)
......
cmake/external/protobuf.cmake
浏览文件 @ 745634a1
......@@ -191,12 +191,12 @@ FUNCTION(build_protobuf TARGET_NAME BUILD_FOR_HOST)
${OPTIONAL_ARGS}
-Dprotobuf_BUILD_TESTS=OFF
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
-DCMAKE_BUILD_TYPE=Release
-DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
-DCMAKE_INSTALL_PREFIX=${PROTOBUF_INSTALL_DIR}
-DCMAKE_INSTALL_LIBDIR=lib
CMAKE_CACHE_ARGS
-DCMAKE_INSTALL_PREFIX:PATH=${PROTOBUF_INSTALL_DIR}
-DCMAKE_BUILD_TYPE:STRING=Release
-DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
-DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
${OPTIONAL_CACHE_ARGS}
......
cmake/external/warpctc.cmake
浏览文件 @ 745634a1
......@@ -35,6 +35,7 @@ ExternalProject_Add(
extern_warpctc
${EXTERNAL_PROJECT_LOG_ARGS}
GIT_REPOSITORY "https://github.com/gangliao/warp-ctc.git"
GIT_TAG b63a0644654a3e0ed624c85a1767bc8193aead09
PREFIX ${WARPCTC_SOURCES_DIR}
UPDATE_COMMAND ""
CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
......@@ -48,9 +49,9 @@ ExternalProject_Add(
-DCMAKE_DISABLE_FIND_PACKAGE_Torch=ON
-DBUILD_SHARED=ON
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
-DCMAKE_BUILD_TYPE=Release
-DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
${EXTERNAL_OPTIONAL_ARGS}
CMAKE_CACHE_ARGS -DCMAKE_BUILD_TYPE:STRING=Release
CMAKE_CACHE_ARGS -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
-DCMAKE_INSTALL_PREFIX:PATH=${WARPCTC_INSTALL_DIR}
)
......
cmake/external/zlib.cmake
浏览文件 @ 745634a1
......@@ -42,11 +42,11 @@ ExternalProject_Add(
-DBUILD_SHARED_LIBS=OFF
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
-DCMAKE_MACOSX_RPATH=ON
-DCMAKE_BUILD_TYPE=Release
-DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
${EXTERNAL_OPTIONAL_ARGS}
CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${ZLIB_INSTALL_DIR}
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
-DCMAKE_BUILD_TYPE:STRING=Release
-DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
)
LIST(APPEND external_project_dependencies zlib)
......
cmake/generic.cmake
浏览文件 @ 745634a1
......@@ -389,13 +389,60 @@ function(go_test TARGET_NAME)
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
endfunction(go_test)
# Modification of standard 'protobuf_generate_cpp()' with protobuf-lite support
# Usage:
# paddle_protobuf_generate_cpp(<proto_srcs> <proto_hdrs> <proto_files>)
function(paddle_protobuf_generate_cpp SRCS HDRS)
if(NOT ARGN)
message(SEND_ERROR "Error: paddle_protobuf_generate_cpp() called without any proto files")
return()
endif()
set(${SRCS})
set(${HDRS})
if (MOBILE_INFERENCE)
set(EXTRA_FLAG "lite:")
else()
set(EXTRA_FLAG "")
endif()
foreach(FIL ${ARGN})
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
get_filename_component(FIL_WE ${FIL} NAME_WE)
set(_protobuf_protoc_src "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
set(_protobuf_protoc_hdr "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
list(APPEND ${SRCS} "${_protobuf_protoc_src}")
list(APPEND ${HDRS} "${_protobuf_protoc_hdr}")
add_custom_command(
OUTPUT "${_protobuf_protoc_src}"
"${_protobuf_protoc_hdr}"
COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_CURRENT_BINARY_DIR}"
COMMAND ${PROTOBUF_PROTOC_EXECUTABLE}
-I${CMAKE_CURRENT_SOURCE_DIR}
--cpp_out "${EXTRA_FLAG}${CMAKE_CURRENT_BINARY_DIR}" ${ABS_FIL}
DEPENDS ${ABS_FIL} protoc
COMMENT "Running C++ protocol buffer compiler on ${FIL}"
VERBATIM )
endforeach()
set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
set(${SRCS} ${${SRCS}} PARENT_SCOPE)
set(${HDRS} ${${HDRS}} PARENT_SCOPE)
endfunction()
function(proto_library TARGET_NAME)
set(oneValueArgs "")
set(multiValueArgs SRCS DEPS)
cmake_parse_arguments(proto_library "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
set(proto_srcs)
set(proto_hdrs)
protobuf_generate_cpp(proto_srcs proto_hdrs ${proto_library_SRCS})
paddle_protobuf_generate_cpp(proto_srcs proto_hdrs ${proto_library_SRCS})
cc_library(${TARGET_NAME} SRCS ${proto_srcs} DEPS ${proto_library_DEPS} protobuf)
endfunction()
......
cmake/util.cmake
浏览文件 @ 745634a1
......@@ -73,25 +73,43 @@ function(link_paddle_exe TARGET_NAME)
generate_rdma_links()
endif()
target_circle_link_libraries(${TARGET_NAME}
ARCHIVE_START
paddle_gserver
paddle_function
ARCHIVE_END
paddle_pserver
paddle_trainer_lib
paddle_network
paddle_math
paddle_utils
paddle_parameter
paddle_proto
paddle_cuda
paddle_optimizer
${EXTERNAL_LIBS}
${CMAKE_THREAD_LIBS_INIT}
${CMAKE_DL_LIBS}
${RDMA_LD_FLAGS}
${RDMA_LIBS})
if(MOBILE_INFERENCE)
target_circle_link_libraries(${TARGET_NAME}
ARCHIVE_START
paddle_gserver
paddle_function
ARCHIVE_END
paddle_math
paddle_utils
paddle_parameter
paddle_proto
paddle_cuda
${EXTERNAL_LIBS}
${CMAKE_THREAD_LIBS_INIT}
${CMAKE_DL_LIBS}
${RDMA_LD_FLAGS}
${RDMA_LIBS})
else()
target_circle_link_libraries(${TARGET_NAME}
ARCHIVE_START
paddle_gserver
paddle_function
ARCHIVE_END
paddle_pserver
paddle_trainer_lib
paddle_network
paddle_math
paddle_utils
paddle_parameter
paddle_proto
paddle_cuda
paddle_optimizer
${EXTERNAL_LIBS}
${CMAKE_THREAD_LIBS_INIT}
${CMAKE_DL_LIBS}
${RDMA_LD_FLAGS}
${RDMA_LIBS})
endif()
if(ANDROID)
target_link_libraries(${TARGET_NAME} log)
......
doc/api/v1/index_cn.rst
浏览文件 @ 745634a1
......@@ -21,7 +21,7 @@ Model Config API
trainer_config_helpers/optimizers.rst
trainer_config_helpers/data_sources.rst
trainer_config_helpers/layers.rst
trainer_config_helpers/activations.rst
trainer_config_helpers/activations.rst
trainer_config_helpers/poolings.rst
trainer_config_helpers/networks.rst
trainer_config_helpers/evaluators.rst
......
doc/api/v2/config/layer.rst
浏览文件 @ 745634a1
......@@ -345,6 +345,11 @@ clip
.. autoclass:: paddle.v2.layer.clip
:noindex:
resize
------
.. autoclass:: paddle.v2.layer.resize
:noindex:
slope_intercept
---------------
.. autoclass:: paddle.v2.layer.slope_intercept
......
doc/design/block.md
浏览文件 @ 745634a1
......@@ -5,12 +5,12 @@
Both deep learning systems and programming languages help users describe computation procedures. These systems use various representations of computation:
- Caffe, Torch, and Paddle: sequences of layers.
- TensorFlow, Caffe2, Mxnet: graphs of operators.
- TensorFlow, Caffe2, Mxnet: graph of operators.
- PaddlePaddle: nested blocks, like C++ and Java programs.
## Block in Programming Languages and Deep Learning
In programming languages, a block is a pair of curly braces that includes local variables definitions and a sequence of instructions, or operators.
In programming languages, a block is a pair of curly braces that includes local variables definitions and a sequence of instructions or operators.
Blocks work with control flow structures like `if`, `else`, and `for`, which have equivalents in deep learning:
......@@ -24,14 +24,14 @@ A key difference is that a C++ program describes a one pass computation, whereas
## Stack Frames and the Scope Hierarchy
The existence of the backward makes the execution of a block of traditional programs and PaddlePaddle different to each other:
The existence of the backward pass makes the execution of a block of PaddlePaddle different from traditional programs:
| programming languages | PaddlePaddle |
|-----------------------|-------------------------------|
| stack | scope hierarchy |
| stack frame | scope |
| push at entering block| push at entering block |
| pop at leaving block | destroy at minibatch completes|
| programming languages | PaddlePaddle |
|-----------------------|---------------------------------|
| stack | scope hierarchy |
| stack frame | scope |
| push at entering block| push at entering block |
| pop at leaving block | destroy when minibatch completes|
1. In traditional programs:
......@@ -42,9 +42,9 @@ The existence of the backward makes the execution of a block of traditional prog
1. In PaddlePaddle
- When the execution enters a block, PaddlePaddle adds a new scope, where it realizes variables.
- PaddlePaddle doesn't pop a scope after the execution of the block because variables therein are to be used by the backward pass. So it has a stack forest known as a *scope hierarchy*.
- PaddlePaddle doesn't pop a scope after the execution of the block because variables therein are used by the backward pass. So it has a stack forest known as a *scope hierarchy*.
- The height of the highest tree is the maximum depth of nested blocks.
- After the process of a minibatch, PaddlePaddle destroys the scope hierarchy.
- After the processing of a minibatch, PaddlePaddle destroys the scope hierarchy.
## Use Blocks in C++ and PaddlePaddle Programs
......@@ -55,17 +55,23 @@ Let us consolidate the discussion by presenting some examples.
The following C++ programs shows how blocks are used with the `if-else` structure:
```c++
namespace pd = paddle;
int x = 10;
int y = 20;
int out;
int y = 1;
int z = 10;
bool cond = false;
int o1, o2;
if (cond) {
int z = x + y;
out = softmax(z);
o1 = z;
o2 = pd::layer::softmax(z);
} else {
int z = fc(x);
out = z;
int d = pd::layer::fc(z);
o1 = d;
o2 = d+1;
}
```
An equivalent PaddlePaddle program from the design doc of the [IfElseOp operator](./if_else_op.md) is as follows:
......@@ -73,57 +79,55 @@ An equivalent PaddlePaddle program from the design doc of the [IfElseOp operator
```python
import paddle as pd
x = var(10)
y = var(20)
cond = var(false)
ie = pd.create_ifelseop(inputs=[x], output_num=1)
x = minibatch([10, 20, 30]) # shape=[None, 1]
y = var(1) # shape=[1], value=1
z = minibatch([10, 20, 30]) # shape=[None, 1]
cond = larger_than(x, 15) # [false, true, true]
ie = pd.ifelse()
with ie.true_block():
x = ie.inputs(true, 0)
z = operator.add(x, y)
ie.set_output(true, 0, operator.softmax(z))
d = pd.layer.add_scalar(x, y)
ie.output(d, pd.layer.softmax(d))
with ie.false_block():
x = ie.inputs(false, 0)
z = layer.fc(x)
ie.set_output(true, 0, operator.softmax(z))
out = b(cond)
d = pd.layer.fc(z)
ie.output(d, d+1)
o1, o2 = ie(cond)
```
In both examples, the left branch computes `softmax(x+y)` and the right branch computes `fc(x)`.
In both examples, the left branch computes `x+y` and `softmax(x+y)`, the right branch computes `fc(x)` and `x+1` .
The difference is that variables in the C++ program contain scalar values, whereas those in the PaddlePaddle programs are mini-batches of instances.
A difference is that variables in the C++ program contain scalar values, whereas those in the PaddlePaddle programs are mini-batches of instances. The `ie.input(true, 0)` invocation returns instances in the 0-th input, `x`, that corresponds to true values in `cond` as the local variable `x`, where `ie.input(false, 0)` returns instances corresponding to false values.
### Blocks with `for` and `RNNOp`
The following RNN model from the [RNN design doc](./rnn.md)
The following RNN model in PaddlePaddle from the [RNN design doc](./rnn.md) :
```python
x = sequence([10, 20, 30])
m = var(0)
W = tensor()
U = tensor()
rnn = create_rnn(inputs=[input])
with rnn.stepnet() as net:
x = net.set_inputs(0)
h = net.add_memory(init=m)
fc_out = pd.matmul(W, x)
hidden_out = pd.matmul(U, h.pre(n=1))
sum = pd.add_two(fc_out, hidden_out)
act = pd.sigmoid(sum)
h.update(act) # update memory with act
net.set_outputs(0, act, hidden_out) # two outputs
x = sequence([10, 20, 30]) # shape=[None, 1]
m = var(0) # shape=[1]
W = var(0.314, param=true) # shape=[1]
U = var(0.375, param=true) # shape=[1]
rnn = pd.rnn()
with rnn.step():
h = rnn.memory(init = m)
h_prev = rnn.previous_memory(h)
a = layer.fc(W, x)
b = layer.fc(U, h_prev)
s = pd.add(a, b)
act = pd.sigmoid(s)
rnn.update_memory(h, act)
rnn.output(a, b)
o1, o2 = rnn()
print o1, o2
```
has its equivalent C++ program as follows
```c++
int* x = {10, 20, 30};
int m = 0;
int W = some_value();
int U = some_other_value();
int* m = {0};
int* W = {0.314};
int* U = {0.375};
int mem[sizeof(x) / sizeof(x[0]) + 1];
int o1[sizeof(x) / sizeof(x[0]) + 1];
......@@ -131,25 +135,21 @@ int o2[sizeof(x) / sizeof(x[0]) + 1];
for (int i = 1; i <= sizeof(x)/sizeof(x[0]); ++i) {
int x = x[i-1];
if (i == 1) mem[0] = m;
int fc_out = W * x;
int hidden_out = Y * mem[i-1];
int sum = fc_out + hidden_out;
int a = W * x;
int b = Y * mem[i-1];
int s = fc_out + hidden_out;
int act = sigmoid(sum);
mem[i] = act;
o1[i] = act;
o2[i] = hidden_out;
}
print_array(o1);
print_array(o2);
```
## Compilation and Execution
Like TensorFlow programs, a PaddlePaddle program is written in Python. The first part describes a neural network as a protobuf message, and the rest part executes the message for training or inference.
Like TensorFlow, a PaddlePaddle program is written in Python. The first part describes a neural network as a protobuf message, and the rest executes the message for training or inference.
The generation of this protobuf message is like what a compiler generates a binary executable file. The execution of the message that the OS executes the binary file.
The generation of this protobuf message is similar to how a compiler generates a binary executable file. The execution of the message is similar to how the OS executes the binary file.
## The "Binary Executable File Format"
......@@ -186,8 +186,8 @@ Also, the RNN operator in above example is serialized into a protobuf message of
```
OpDesc {
inputs = {0} // the index of x
outputs = {5, 3} // indices of act and hidden_out
inputs = {0} // the index of x in vars of BlockDesc above
outputs = {5, 3} // indices of act and hidden_out in vars of BlockDesc above
attrs {
"memories" : {1} // the index of h
"step_net" : <above step net>
......@@ -203,32 +203,32 @@ This `OpDesc` value is in the `ops` field of the `BlockDesc` value representing
During the generation of the Protobuf message, the Block should store VarDesc (the Protobuf message which describes Variable) and OpDesc (the Protobuf message which describes Operator).
VarDesc in a block should have its name scope to avoid local variables affect parent block's name scope.
Child block's name scopes should inherit the parent's so that OpDesc in child block can reference a VarDesc that stored in parent block. For example
Child block's name scopes should inherit the parent's so that OpDesc in child block can reference a VarDesc that stored in parent block. For example:
```python
a = pd.Varaible(shape=[20, 20])
a = pd.Variable(shape=[20, 20])
b = pd.fc(a, params=["fc.w", "fc.b"])
rnn = pd.create_rnn()
with rnn.stepnet() as net:
x = net.set_inputs(a)
with rnn.stepnet():
x = a.as_step_input()
# reuse fc's parameter
fc_without_b = pd.get_variable("fc.w")
net.set_outputs(fc_without_b)
rnn.output(fc_without_b)
out = rnn()
```
the method `pd.get_variable` can help retrieve a Variable by a name, a Variable may store in a parent block, but might be retrieved in a child block, so block should have a variable scope that supports inheritance.
The method `pd.get_variable` can help retrieve a Variable by the name. The Variable may be stored in a parent block, but might be retrieved in a child block, so block should have a variable scope that supports inheritance.
In compiler design, the symbol table is a data structure created and maintained by compilers to store information about the occurrence of various entities such as variable names, function names, classes, etc.
To store the definition of variables and operators, we define a C++ class `SymbolTable`, like the one used in compilers.
`SymbolTable` can do the following stuff:
`SymbolTable` can do the following:
- store the definitions (some names and attributes) of variables and operators,
- to verify if a variable was declared,
- to make it possible to implement type checking (offer Protobuf message pointers to `InferShape` handlers).
- verify if a variable was declared,
- make it possible to implement type checking (offer Protobuf message pointers to `InferShape` handlers).
```c++
......@@ -240,19 +240,18 @@ class SymbolTable {
OpDesc* NewOp(const string& name="");
// TODO determine whether name is generated by python or C++
// currently assume that a unique name will be generated by C++ if the
// argument name left default.
VarDesc* NewVar(const string& name="");
// TODO determine whether name is generated by python or C++.
// Currently assume that a unique name will be generated by C++ if the
// argument name is left default.
VarDesc* Var(const string& name="");
// find a VarDesc by name, if recursive true, find parent's SymbolTable
// find a VarDesc by name, if recursive is true, find parent's SymbolTable
// recursively.
// this interface is introduced to support InferShape, find protobuf messages
// of variables and operators, pass pointers into InferShape.
// operator
//
// NOTE maybe some C++ classes such as VarDescBuilder and OpDescBuilder should
// be proposed and embedded into pybind to enable python operate on C++ pointers.
// be proposed and embedded into pybind to enable python operation on C++ pointers.
VarDesc* FindVar(const string& name, bool recursive=true);
OpDesc* FindOp(const string& name);
......@@ -270,7 +269,7 @@ class SymbolTable {
After all the description of variables and operators is added into SymbolTable,
the block has enough information to run.
The `Block` class takes a `BlockDesc` as input, and provide `Run` and `InferShape` functions.
The `Block` class takes a `BlockDesc` as input, and provides `Run` and `InferShape` functions.
```c++
......@@ -302,7 +301,7 @@ public:
void CreateVariables(const framework::Scope& scope);
void CreateOperators();
// some other necessary interfaces of NetOp are list below
// some other necessary interfaces of NetOp are listed below
// ...
private:
......@@ -316,15 +315,14 @@ private:
Block inherits from OperatorBase, which has a Run method.
Block's Run method will run its operators sequentially.
There is another important interface called `Eval`, which take some arguments called targets, and generate a minimal graph which takes targets as the end points and creates a new Block,
after `Run`, `Eval` will get the latest value and return the targets.
There is another important interface called `Eval`, which takes some arguments called targets and generates a minimal graph which treats targets as the end points and creates a new Block. After `Run`, `Eval` will get the latest value and return the targets.
The definition of Eval is as follows:
```c++
// clean a block description by targets using the corresponding dependency graph.
// return a new BlockDesc with minimal number of operators.
// NOTE not return a Block but the block's description so that this can be distributed
// NOTE: The return type is not a Block but the block's description so that this can be distributed
// to a cluster.
BlockDesc Prune(const BlockDesc& desc, vector<string> targets);
......
doc/design/executor.md 0 → 100644
浏览文件 @ 745634a1
# Executor Design Doc
## Motivation
We use executor to do the runtime evaluation of a `ProgramDesc`.
## Overview
An executor takes a `ProgramDesc`, a `block_id` and a `Scope`. The `ProgramDesc` is a list of blocks and each block contains the protobuf definition of all the parameters and operators. The `block_id` specifies the entrance block. And the `Scope` is the container of all the variable instance, which is persistent throughout different runs.
### What does executor do?
It evaluates all the operators in the `block_id`th block of a `ProgramDesc`.
### What does executor NOT do?
It does not do runtime optimization, meaning intelligently parse the dependency of each op a choose which one to be run and in which order they should be run.
It does not do graph partitioning, meaning dividing the `ProgramDesc` into several small pieces and executing them on different devices.
## Implementation
`Executor` evaluates a `ProgramDesc`. Essentially, it instantiates Variables and Operators, then run all the operators in sequence. [[code]](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/executor.cc)
doc/design/gan_api.md 0 → 100644
浏览文件 @ 745634a1
# Design for GAN
GAN (General Adversarial Net [https://arxiv.org/abs/1406.2661]) is an important model for unsupervised learning and widely used in many areas.
It applies several important concepts in machine learning system design, including building and running subgraphs, dependency tracing, different optimizers in one executor and so forth.
In our GAN design, we wrap it as a user-friendly easily customized python API to design different models. We take the conditional DC-GAN (Unsupervised Representation Learning with Deep Convolutional Generative Adversarial Networks [https://arxiv.org/abs/1511.06434]) as an example due to its good performance on image generation.
<p align="center">
<img src="./test.dot.png" width = "35%" align="center"/><br/>
Figure 1. The overall running logic of GAN. The black solid arrows indicate the forward pass; the green dashed arrows indicate the backward pass of generator training; the red dashed arrows indicate the backward pass of the discriminator training. The BP pass of the green (red) arrow should only update the parameters in the green (red) boxes. The diamonds indicate the data providers. d\_loss and g\_loss marked in red and green are the two targets we would like to run.
</p>
The operators, layers and functions required/optional to build a GAN demo is summarized in https://github.com/PaddlePaddle/Paddle/issues/4563.
<p align="center">
<img src="./dcgan.png" width = "90%" align="center"/><br/>
Figure 2. Photo borrowed from the original DC-GAN paper.
</p>
## The Conditional-GAN might be a class.
This design we adopt the popular open source design in https://github.com/carpedm20/DCGAN-tensorflow and https://github.com/rajathkmp/DCGAN. It contains following data structure:
- DCGAN(object): which contains everything required to build a GAN model. It provides following member functions methods as API:
- __init__(...): Initialize hyper-parameters (like conv dimension and so forth), and declare model parameters of discriminator and generator as well.
- generator(z, y=None): Generate a fake image from input noise z. If the label y is provided, the conditional GAN model will be chosen.
Returns a generated image.
- discriminator(image):
Given an image, decide if it is from a real source or a fake one.
Returns a 0/1 binary label.
- build_model(self):
build the whole GAN model, define training loss for both generator and discrimator.
## Discussion on Engine Functions required to build GAN
- Trace the tensor and variable dependency in the engine executor. (Very critical, otherwise GAN can'be be trained correctly)
- Different optimizers responsible for optimizing different loss.
To be more detailed, we introduce our design of DCGAN as following:
### Class member Function: Initializer
- Set up hyper-parameters, including condtional dimension, noise dimension, batch size and so forth.
- Declare and define all the model variables. All the discriminator parameters are included in the list self.theta_D and all the generator parameters are included in the list self.theta_G.
```python
class DCGAN(object):
def __init__(self, y_dim=None):
# hyper parameters
self.y_dim = y_dim # conditional gan or not
self.batch_size = 100
self.z_dim = z_dim # input noise dimension
# define parameters of discriminators
self.D_W0 = pd.Variable(shape=[3,3, 1, 128], data=pd.gaussian_normal_randomizer())
self.D_b0 = pd.Variable(np.zeros(128)) # variable also support initialization using a numpy data
self.D_W1 = pd.Variable(shape=[784, 128], data=pd.gaussian_normal_randomizer())
self.D_b1 = pd.Variable(np.zeros(128)) # variable also support initialization using a numpy data
self.D_W2 = pd.Varialble(np.random.rand(128, 1))
self.D_b2 = pd.Variable(np.zeros(128))
self.theta_D = [self.D_W0, self.D_b0, self.D_W1, self.D_b1, self.D_W2, self.D_b2]
# define parameters of generators
self.G_W0 = pd.Variable(shape=[784, 128], data=pd.gaussian_normal_randomizer())
self.G_b0 = pd.Variable(np.zeros(128)) # variable also support initialization using a numpy data
self.G_W1 = pd.Variable(shape=[784, 128], data=pd.gaussian_normal_randomizer())
self.G_b1 = pd.Variable(np.zeros(128)) # variable also support initialization using a numpy data
self.G_W2 = pd.Varialble(np.random.rand(128, 1))
self.G_b2 = pd.Variable(np.zeros(128))
self.theta_G = [self.G_W0, self.G_b0, self.G_W1, self.G_b1, self.G_W2, self.G_b2]
```
### Class member Function: Generator
- Given a noisy input z, returns a fake image.
- Concatenation, batch-norm, FC operations required;
- Deconv layer required, which is missing now...
```python
class DCGAN(object):
def generator(self, z, y = None):
# input z: the random noise
# input y: input data label (optional)
# output G_im: generated fake images
if not self.y_dim:
z = pd.layer.concat(1, [z, y])
G_h0 = pd.layer.fc(z, self.G_w0, self.G_b0)
G_h0_bn = pd.layer.batch_norm(G_h0)
G_h0_relu = pd.layer.relu(G_h0_bn)
G_h1 = pd.layer.deconv(G_h0_relu, self.G_w1, self.G_b1)
G_h1_bn = pd.layer.batch_norm(G_h1)
G_h1_relu = pd.layer.relu(G_h1_bn)
G_h2 = pd.layer.deconv(G_h1_relu, self.G_W2, self.G_b2))
G_im = pd.layer.tanh(G_im)
return G_im
```
### Class member function: Discriminator
- Given a noisy input z, returns a fake image.
- Concatenation, Convolution, batch-norm, FC, Leaky-ReLU operations required;
```python
class DCGAN(object):
def discriminator(self, image):
# input image: either generated images or real ones
# output D_h2: binary logit of the label
D_h0 = pd.layer.conv2d(image, w=self.D_w0, b=self.D_b0)
D_h0_bn = pd.layer.batchnorm(h0)
D_h0_relu = pd.layer.lrelu(h0_bn)
D_h1 = pd.layer.conv2d(D_h0_relu, w=self.D_w1, b=self.D_b1)
D_h1_bn = pd.layer.batchnorm(D_h1)
D_h1_relu = pd.layer.lrelu(D_h1_bn)
D_h2 = pd.layer.fc(D_h1_relu, w=self.D_w2, b=self.D_b2)
return D_h2
```
### Class member function: Build the model
- Define data readers as placeholders to hold the data;
- Build generator and discriminators;
- Define two training losses for discriminator and generator, respectively.
If we have execution dependency engine to back-trace all tensors, the module building our GAN model will be like this:
```python
class DCGAN(object):
def build_model(self):
if self.y_dim:
self.y = pd.data(pd.float32, [self.batch_size, self.y_dim])
self.images = pd.data(pd.float32, [self.batch_size, self.im_size, self.im_size])
self.faked_images = pd.data(pd.float32, [self.batch_size, self.im_size, self.im_size])
self.z = pd.data(tf.float32, [None, self.z_size])
# step 1: generate images by generator, classify real/fake images with discriminator
if self.y_dim: # if conditional GAN, includes label
self.G = self.generator(self.z, self.y)
self.D_t = self.discriminator(self.images)
# generated fake images
self.sampled = self.sampler(self.z, self.y)
self.D_f = self.discriminator(self.G)
else: # original version of GAN
self.G = self.generator(self.z)
self.D_t = self.discriminator(self.images)
# generate fake images
self.sampled = self.sampler(self.z)
self.D_f = self.discriminator(self.images)
# step 2: define the two losses
self.d_loss_real = pd.reduce_mean(pd.cross_entropy(self.D_t, np.ones(self.batch_size))
self.d_loss_fake = pd.reduce_mean(pd.cross_entropy(self.D_f, np.zeros(self.batch_size))
self.d_loss = self.d_loss_real + self.d_loss_fake
self.g_loss = pd.reduce_mean(pd.cross_entropy(self.D_f, np.ones(self.batch_szie))
```
If we do not have dependency engine but blocks, the module building our GAN model will be like this:
```python
class DCGAN(object):
def build_model(self, default_block):
# input data in the default block
if self.y_dim:
self.y = pd.data(pd.float32, [self.batch_size, self.y_dim])
self.images = pd.data(pd.float32, [self.batch_size, self.im_size, self.im_size])
# self.faked_images = pd.data(pd.float32, [self.batch_size, self.im_size, self.im_size])
self.z = pd.data(tf.float32, [None, self.z_size])
# step 1: generate images by generator, classify real/fake images with discriminator
with pd.default_block().g_block():
if self.y_dim: # if conditional GAN, includes label
self.G = self.generator(self.z, self.y)
self.D_g = self.discriminator(self.G, self.y)
else: # original version of GAN
self.G = self.generator(self.z)
self.D_g = self.discriminator(self.G, self.y)
self.g_loss = pd.reduce_mean(pd.cross_entropy(self.D_g, np.ones(self.batch_szie))
with pd.default_block().d_block():
if self.y_dim: # if conditional GAN, includes label
self.D_t = self.discriminator(self.images, self.y)
self.D_f = self.discriminator(self.G, self.y)
else: # original version of GAN
self.D_t = self.discriminator(self.images)
self.D_f = self.discriminator(self.G)
# step 2: define the two losses
self.d_loss_real = pd.reduce_mean(pd.cross_entropy(self.D_t, np.ones(self.batch_size))
self.d_loss_fake = pd.reduce_mean(pd.cross_entropy(self.D_f, np.zeros(self.batch_size))
self.d_loss = self.d_loss_real + self.d_loss_fake
```
Some small confusion and problems with this design:
- D\_g and D\_f are actually the same thing, but has to be written twice; i.e., if we want to run two sub-graphs conceptually, the same codes have to be written twice if they are shared by the graph.
- Requires ability to create a block anytime, rather than in if-else or rnn only;
## Main function for the demo:
Generally, the user of GAN just need to the following things:
- Define an object as DCGAN class;
- Build the DCGAN model;
- Specify two optimizers for two different losses with respect to different parameters.
```python
# pd for short, should be more concise.
from paddle.v2 as pd
import numpy as np
import logging
if __name__ == "__main__":
# dcgan class in the default graph/block
# if we use dependency engine as tensorflow
# the codes, will be slightly different like:
# dcgan = DCGAN()
# dcgan.build_model()
with pd.block() as def_block:
dcgan = DCGAN()
dcgan.build_model(def_block)
# load mnist data
data_X, data_y = self.load_mnist()
# Two subgraphs required!!!
with pd.block().d_block():
d_optim = pd.train.Adam(lr = .001, beta= .1)
d_step = d_optim.minimize(dcgan.d_loss, dcgan.theta_D)
with pd.block.g_block():
g_optim = pd.train.Adam(lr = .001, beta= .1)
g_step = pd.minimize(dcgan.g_loss, dcgan.theta_G)
# executor
sess = pd.executor()
# training
for epoch in xrange(10000):
for batch_id in range(N / batch_size):
idx = ...
# sample a batch
batch_im, batch_label = data_X[idx:idx+batch_size], data_y[idx:idx+batch_size]
# sample z
batch_z = np.random.uniform(-1., 1., [batch_size, z_dim])
if batch_id % 2 == 0:
sess.run(d_step,
feed_dict = {dcgan.images: batch_im,
dcgan.y: batch_label,
dcgan.z: batch_z})
else:
sess.run(g_step,
feed_dict = {dcgan.z: batch_z})
```
# More thinking about dependency engine v.s. block design:
- What if we just want to run an intermediate result? Do we need to run the whole block/graph?
- Should we call eval() to get the fake images in the first stage? And then train the discriminator in the second stage?
doc/design/if_else_op.md
浏览文件 @ 745634a1
IfOp should have only one branch. An IfOp operator takes a `cond` variable whose value must be a vector of N boolean elements. Its return value has N instances. If cond[i] == True, input instance input[i] will go through true_block() and generate output[i]; otherwise it will produce output from false_bloack().
# The `IfElse` Operator
```python
import paddle as pd
PaddlePaddle's `IfElse` operator differs from TensorFlow's:
x = var()
y = var()
cond = var()
default_value = var()
b = pd.create_ifelseop(inputs=[x], output_num=1)
with b.true_block():
x = b.inputs(0)
z = operator.add(x, y)
b.set_output(0, operator.softmax(z))
with b.false_block():
x = b.inputs(0)
z = layer.fc(x)
b.set_output(0, operator.softmax(z))
out = b(cond)
```
- the TensorFlow version takes a scalar boolean value as the condition so that the whole mini-batch goes to either the true or the false branch, whereas
- the PaddlePaddle version takes a vector of boolean value as the condition, and instances corresponding to true values go to the true branch, those corresponding to false values go to the false branch.
## Example
The following PaddlePaddle program shows the usage of the IfElse operator:
If only true_block is set in an IfElseOp, a special case is that we can have a default value for false as:
```python
import paddle as pd
x = var()
y = var()
cond = var()
default_value = var()
b = pd.create_ifelseop(inputs=[x], output_num=1, default_value)
with b.true_block():
x = b.inputs(0)
z = operator.add(x, y)
b.set_output(0, operator.softmax(z))
x = minibatch([10, 20, 30]) # shape=[None, 1]
y = var(1) # shape=[1], value=1
z = minibatch([10, 20, 30]) # shape=[None, 1]
cond = larger_than(x, 15) # [false, true, true]
ie = pd.ifelse()
with ie.true_block():
d = pd.layer.add(x, y)
ie.output(d, pd.layer.softmax(d))
with ie.false_block():
d = pd.layer.fc(z)
ie.output(d, d+1)
o1, o2 = ie(cond)
```
out = b(cond)
A challenge to implement the `IfElse` operator is to infer those variables to be split, or, say, to identify the variable of the mini-batch or those derived from the mini-batch.
An equivalent C++ program is as follows:
```c++
namespace pd = paddle;
int x = 10;
int y = 1;
int z = 10;
bool cond = false;
int o1, o2;
if (cond) {
int d = x + y;
o1 = z;
o2 = pd::layer::softmax(z);
} else {
int d = pd::layer::fc(z);
o1 = d;
o2 = d+1;
}
```
where default_value is a list of vars for `cond` == False.
doc/design/images/graph_construction_example.dot
浏览文件 @ 745634a1
......@@ -33,7 +33,6 @@ digraph ImageClassificationGraph {
cost -> MSE_Grad [color=red];
d_cost -> MSE_Grad [color=red];
x -> MSE_Grad [color=red];
l -> MSE_Grad [color=red];
y -> MSE_Grad -> d_y [color=red];
......
doc/design/infer_var_type.md 0 → 100644
浏览文件 @ 745634a1
# Design Doc: InferVarType
## The Problem Posed
The variable in our design can hold variant types. Such as `LoDTensor` and `SelectedRows`. An operator should be able to inference the variable types of its output.
For example, a `lookup table` operator takes two `LoDTensor`; one is a float tensor as the embedding table, the other is an int tensor as word ID. The gradient operator of `lookup table` will generate a `SelectedRows` as its output. A `sum` operator can take both `LoDTensor` and `SelectedRows` as its inputs and will generate a `LoDTensor` if any of its inputs is `LoDTensor`, otherwise, the `sum` operator will generate `SelectedRows` as its output.
The variable type will be constant at runtime. Every variable's type can either be set by the user (input data and parameter) or be inferred by the operator in compile time.
## Proposed Solution
The `InferVarType` is a compile-time function which is registered to each operator. The inferface of that function is:
```c++
using InferVarTypeFN = std::function<
void (const OpDescBind& /*op_desc*/, BlockDescBind* /*block*/)>;
```
It takes an operator description as its input and will write the output variable type and store them in block description.
The `InferVarTypeFN` will be registered in `OpInfo`, to replace `infer_var_type_` field. The `OpInfo` should be
```cpp
struct OpInfo {
InferVarTypeFN infer_var_type_;
...
};
```
The default `InferVarType` will set output type as `LoDTensor`. It can be done by `GetInferVarType()`.
```cpp
void DefaultInferVarType(const OpDescBind& op_desc, BlockDescBind* block) {
// set the output type of variable as `LoDTensor`.
// ...
}
struct OpInfo {
InferVarTypeFN infer_var_type_;
InferVarTypeFN GetInferVarType() const {
if (infer_var_type_) {
return infer_var_type_;
} else {
return DefaultInferVarType;
}
}
};
```
## Register InferVarType
We provide a thin base class for registering an `InferVarTypeFN`. To use a base class will ease the implementation of registry since we can detect the registry entry is an `InferVarTypeFN` or not.
```cpp
class VarTypeInferer {
public:
virtual void operator()(const OpDescBind& op_desc, BlockDescBind* block) const = 0;
}
```
Operator developers can write the specialize `VarTypeInferer` as follow.
```cpp
class SpecialVarTypeInferer : public VarTypeInferer {
public:
virtual void operator()(const OpDescBind& op_desc, BlockDescBind* block) const {
// .. own logic
}
}
```
Then user can register the `InferVarType` just like `GradOpDescMaker` and `OpInfoMaker`.
```
REGISTER_OPERATOR(some_op, OpType, SpecialVarTypeInferer, ...);
```
doc/design/optimizer.md 0 → 100644
浏览文件 @ 745634a1
## Optimizer Design
### The Problem
A PaddlePaddle program, or a block, is a sequence of operators operating variables. A training program needs to do three kinds of works:
1. the forward pass, which computes intermediate results and the cost(s),
1. the backward pass, which derives gradients from intermediate results and costs, and
1. the optimization pass, which update model parameters to optimize the cost(s).
These works rely on three kinds of operators:
1. forward operators,
1. gradient operators, and
1. optimization operators.
It's true that users should be able to create all these operators manually by calling some low-level API, but it would be much more convenient if they could only describe the forward pass and let PaddlePaddle create the backward and optimization operators automatically.
In this design, we propose a high-level API that automatically derives the optimisation pass and operators from the forward pass.
### High-level Python API to describe the training process
1. User write code to describe the network:
```python
images = layer.data("images")
labels = layer.data("labels")
w1 = pd.var("w1")
b1 = pd.var("b1")
hidden = layer.fc(images, w=w1, b=b1)
cost = layer.mse(hidden, labels)
```
The above code snippet will create forward operators in [Block](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/block.md).
2. Users create a certain kind of Optimizer with some argument.
```python
optimizer = AdagradOptimizer(learing_rate=0.001)
```
3. Users use the optimizer to `minimize` a certain `cost` through updating parameters in parameter_list.
```python
opt_op_list = optimizer.minimize(cost, parameter_list=[w1, b1])
```
The above code snippet will create gradient and optimization operators in Block. The return value of `minimize()` is list of optimization operators that will be run by session.
4. Users use Session/Executor to run this opt_op_list as target to do training.
```python
sess.run(target= opt_op_list, ...)
```
#### Optimizer Python interface:
```python
class Optimizer(object):
"""Optimizer Base class.
"""
def __init__(self):
pass
def create_backward_pass(self, loss, parameter_list=None):
"""
create and add gradient Operators in BlockDesc to Compute gradients of `loss`
for parameters in parameter_list
Args:
loss: an variable generated by cost function.
parameter_list: parameters that need to compute gradient and update to optimize the lost.
Returns:
list of (parameters, gradients) pair.
"""
return None
def create_optimization_pass(self, parameters_and_grads):
"""Add optimization operators to update gradients to variables.
Args:
parameters_and_grads: a list of (variable, gradient) pair to update.
Returns:
optmization_op_list: a list of optimization operator that will update parameter using gradient.
"""
return None
def minimize(self, loss, parameter_list):
"""Add operations to minimize `loss` by updating `parameter_list`.
This method combines interface `create_backward_pass()` and
`create_optimization_pass()` into one.
"""
params_grads = self.create_backward_pass(loss, parameter_list)
update_ops = self.create_optimization_pass(params_grads)
return update_ops
```
Users can inherit the Optimizer above to create their own Optimizer with some special logic, such as AdagradOptimizer.
doc/design/program.md
浏览文件 @ 745634a1
# Design Doc: ProgramDesc
# Design Doc: PaddlePaddle Programs
The basic structure of a PaddlePaddle program is some nested blocks, as a C++ or Java program.
## Compile and Execution
A PaddlePaddle program consists of two parts -- the first generates a `ProgramDesc` protobuf message that describes the program, and the second runs this message using a C++ class `Executor`.
As described in [graph.md](./graph.md), the first five lines of the following PaddlePaddle program
A simple example PaddlePaddle program can be found in [graph.md](./graph.md):
```python
x = layer.data("images")
......@@ -13,36 +15,112 @@ optimize(cost)
train(cost, reader=mnist.train())
```
generates, or compiles, a PaddelPaddle program, which is represented by the following protobuf message:
The first five lines of the following PaddlePaddle program generates, or, compiles, the `ProgramDesc` message. The last line runs it.
```protobuf
message ProgramDesc {
repeated BlockDesc blocks = 1;
## Programs and Blocks
The basic structure of a PaddlePaddle program is some nested blocks, as a C++ or Java program.
- program: some nested blocks
- [block](./block.md):
- some local variable definitions, and
- a sequence of operators
The concept of block comes from usual programs. For example, the following C++ program has three blocks:
```c++
int main() { // block 0
int i = 0;
if (i < 10) { // block 1
for (int j = 0; j < 10; j++) { // block 2
}
}
return 0;
}
```
The following PaddlePaddle program has three blocks:
```python
import paddle as pd // block 0
x = minibatch([10, 20, 30]) # shape=[None, 1]
y = var(1) # shape=[1], value=1
z = minibatch([10, 20, 30]) # shape=[None, 1]
cond = larger_than(x, 15) # [false, true, true]
ie = pd.ifelse()
with ie.true_block(): // block 1
d = pd.layer.add_scalar(x, y)
ie.output(d, pd.layer.softmax(d))
with ie.false_block(): // block 2
d = pd.layer.fc(z)
ie.output(d, d+1)
o1, o2 = ie(cond)
```
## `BlockDesc` and `ProgramDesc`
All protobuf messages are defined in `framework.proto`.
`BlockDesc` is straight-forward -- it includes local variable definitions, `vars`, and a sequence of operators, `ops`.
```protobuf
message BlockDesc {
required int32 parent = 1;
repeated VarDesc vars = 2;
repeated OpDesc ops = 3;
}
```
The parent ID indicates the parent block so that operators in a block can refer to variables defined locally and also those defined in their ancestor blocks.
All hierarchical blocks in a program are flattened and stored in an array. The block ID is the index of the block in this array.
```protobuf
message ProgramDesc {
repeated BlockDesc blocks = 1;
}
```
### Global Block
The global block is the first one in the above array.
## Operators that Use Blocks
In the above example, the operator `IfElseOp` has two blocks -- the true branch and the false branch.
The definition of `OpDesc` shows that an operator could have some attributes:
```protobuf
message OpDesc {
AttrDesc attrs = 1;
...
}
```
and an attribute could be of type block, which is, in fact, a block ID as described above:
```
message AttrDesc {
required AttrType type = 1;
required string name = 1;
// index into ProgramDesc::blocks when type==BLOCK
optional int32 block = 2;
enum AttrType {
INT = 1,
STRING = 2,
...
BLOCK = ...
}
required AttrType type = 2;
optional int32 block = 10; // when type == BLOCK
...
}
```
When each of the first five lines runs, related Python function, e.g., `layer.fc`, calls C++ InferShape functions. This InferShape function needs to access the properties of VarDesc's accessed by the current OpDesc. These VarDesc's might not be defined in the current block, but in some ancestor blocks. This requires that we can trace the parent of a block.
A nested block is often an attribute of an operator, most likely, an IfElseOp or a WhileOp. In above solution, all blocks are in `ProgramDesc::blocks`, this implicitly assigns a zero-based ID to each block -- the index of the block in `ProgramDesc::blocks`. So that `AttrDesc::block` could be an integer block ID.
## InferShape
With this design, the InferShape function should take the following parameters:
......
doc/design/python_api.md 0 → 100644
浏览文件 @ 745634a1
# Design Doc: Python API
Due to the refactorization of the PaddlePaddle core, we need Python classes to construct corresponding protobuf messages that describe a DL program.
| Python classes | Protobuf messages |
| --- | --- |
| Program | ProgramDesc |
| Block | BlockDesc |
| Operator | OpDesc |
| Variable | VarDesc |
Please be aware that these Python classes need to maintain some construction-time information, which are not part of the protobuf messages.
## Core Concepts
### Program
A `ProgramDesc` describes a [DL program](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/program.md), which is composed of an array of `BlockDesc`s. The `BlockDesc`s in a `ProgramDesc` can have a tree-like hierarchical structure. However, the `ProgramDesc` onlys stores a flattened array of `BlockDesc`s. A `BlockDesc` refers to its parent block by its index in the array. For example, operators in the step block of an RNN operator need to be able to access variables in its ancestor blocks.
Whenever we create a block, we need to set its parent block to the current block, hence the Python class `Program` needs to maintain a data member `current_block`.
```python
class Program(objects):
def __init__(self):
self.desc = core.NewProgram() # a C++ ProgramDesc pointer.
self.blocks = vector<Block>()
self.blocks.append(Block(self, -1)) # the global block
self.current_block = 0 # initialized to the global block
def global_block():
return self.blocks[0]
def current_block():
return self.get_block(self.current_block)
def rollback():
self.current_block = self.current_block().parent_idx
def create_block():
new_block_idx = len(self.block)
self.blocks.append(Block(self, self.current_block))
self.current_block = new_block_idx
return current_block()
```
`Program` is an accessor to the protobuf message `ProgramDesc`, which is created in C++ space, because the InferShape function is in C++, which manipulates `VarDesc` messages, which are in turn members of `BlockDesc`, which is a member of `ProgramDesc`.
`Program` creates the first block as the global block in its constructor. All parameters and their initializer operators are in the global block.
### Block
A [Block](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/block.md) includes
1. a map from variable names to an instance of the Python `Variable` class, and
1. a list of `Operator` instances.
```python
class Block(objects):
def __init__(self, program, parent_idx):
self.desc = core.NewBlock(program.desc)
self.program = program
self.vars = map<string, Variable>()
self.ops = vector<Operator>()
self.parent_idx = parent_idx
def create_var(self, ...):
return Variable(self, ...)
def _create_global_var(self, ...):
program.global_block().create_var(...)
def create_parameter(self, name, ...):
# Parameter is a subclass of variable. See Parameter section for details.
self.vars[name] = Parameter(self._create_global_var(...), ...)
return self.vars[name]
def append_operator(self, ...):
self.ops.append(Operator(self, ...))
def prepend_operator(self, ...): # Parameter's ctor prepands initialize operators.
self.ops.prepend(Operator(self, ...))
```
`create_parameter` is necessary because parameters are global variables, defined in the global block, but can be created in some sub-blocks. For example, an FC layer in the step block of an RNN operator.
`prepend_operator` is necessary because the constructor of `Parameter` needs to create the initialize (or load) operator of the parameter, and would like to put it in the *preamble* of the global block.
### Operator
The `Operator` class fills in the `OpDesc` message and calls the C++ function `InferShape` to infer the output shapes from the input shapes.
```python
class Operator(object):
def __init__(self,
block, # Block
type, # string
inputs, # dict<string, Variable>
outputs,# dict<stirng, Variable>
attrs # dict<string, Any>
):
self.desc = core.NewOpDesc(block.desc, type, inputs, outputs, attrs)
core.infer_shape(self.desc, inputs, outputs)
def type(self):
return self.desc.type()
```
`Operator` creates the `OpDesc` message in C++ space, so that it can call the `InferShape` function, which is in C++.
### Variable
Operators take Variables as its inputs and outputs.
```python
class Variable(object):
def __init__(self,
block=None, # Block
name=None, # string
shape, # tuple
dtype="float32", # string
lod_level=None # int
):
if name is None:
name = unique_name_generator()
self.name = name
self.block = block
self.desc = core.NewVarDesc(block.desc, name, shape, lod_level)
self.writer = None
```
Please be aware of `self.writer`, that tracks operator who creates the variable. It possible that there are more than one operators who write a variable, but in Python space, each write to a variable is represented by a Variable class. This is guaranteed by the fact that **`core.NewVarDesc` must NOT create a new `VarDesc` message if its name already exists in the specified block**.
### Parameter
A parameter is a global variable with an initializer (or load) operator.
```python
class Parameter(Variable):
def __init__(self,
block=None, # Block
name=None, # string
shape, # tuple
dtype="float32", # string
lod_level=None # int
trainable, # bool
initialize_op_attrs,
optimize_op_attrs):
super(Parameter, self).__init__(block, name, shape, dtype, lod_level)
self.trainable = trainable
self.optimize_op_attrs = optimize_op_attrs
block.prepend(Operator(block, # Block
initialize_op_attrs['type'], # string
None, # no inputs
self, # output is the parameter
initialize_op_attrs)
```
When users create a parameter, they can call
```python
program.create_parameter(
...,
init_attr={
type: "uniform_random",
min: -1.0,
max: 1.0,
})
)
```
In above example, `init_attr.type` names an initialize operator. It can also name the load operator
```python
init_attr={
type: "load",
filename: "something.numpy",
}
```
`optimize_op_attrs` is not in the `VarDesc` message, but kept in the Python instance, as it will be used in the Python space when creating the optimize operator's `OpDesc`, and will be in the `OpDesc` message.
## Layer Function
A layer is a Python function that creates some operators and variables. Layers simplify the work of application programmers.
Layer functions take `Variable` and configuration parameters as its input and return the output variable(s).
For example, `FullyConnected` take one or more variable as its input. The input could be input data or another layer's output. There are many configuration options for a `FullyConnected` layer, such as layer size, activation, parameter names, initialization strategies of parameters, and so on. The `FullyConnected` layer will return an output variable.
### Necessity for reusing code between layer functions
There are a lot of code that can be reused. Such as
* Give the default value of configuration. e.g., default initialize strategy for parameters is uniform random with `min = -1.0`, `max = 1.0`. and default initialize strategy for bias is to fill zero.
* Append the activation operator.
* Create a temporary variable.
* Create parameter.
* Generate a unique name.
* Add a bias.
* ...
A mechanism to reuse code between layer functions is necessary. It will be around [150 lines of code](https://github.com/PaddlePaddle/Paddle/pull/4724/files#diff-823b27e07e93914ada859232ae23f846R12) if we write a `FullyConnected` layer without any helper functions.
### Comparision between global functions and helper class
The `FullyConnected` layer will be as follow when we provide global functions:
```python
def fc_layer(input, size, param_attr=None, bias_attr=None, act=None, name=None):
if name is None:
name = unique_name("fc")
input = multiple_input(input)
param_attr = default_param_attr(param_attr)
param_attr = multiple_param_attr(param_attr, len(input))
# mul
mul_results = []
for ipt, attr in zip(input, param_attr):
shape = ipt.shape[1:] + [size]
w = g_program.global_block().create_parameter(shape, ipt.dtype, name, attr)
tmp = create_tmp_var(name)
g_program.current_block().append_op("mul", {ipt, w}, {tmp})
mul_results.append(tmp)
# add sum
...
# add bias
...
# add activation
...
return out
```
We can provide many helpers functions for layer developers. However, there are several disadvantages for global helper functions:
1. We need a namespace for these methods, then layer developers can quickly figure out what method they can use.
2. Global functions will force layer developers to pass its parameter time by time.
So we provide a helper class, `LayerHelper`, to share code between layer functions. The `FullyConnected` Layer will be as follow.
```python
def fc_layer(input, size, param_attr=None, bias_attr=None, act=None, name=None):
helper = LayerHelper(locals()) # pass all parameter to LayerHelper
mul_results = []
for ipt, param in helper.iter_multiple_input_and_param():
w = helper.create_parameter(shape=ipt.shape[1:] + [size], dtype = ipt.dtype)
tmp = helper.create_tmp_variable()
helper.append_op('mul', {ipt, w}, {tmp})
mul_results.append(tmp)
pre_bias = helper.add_sum(mul_results)
pre_activation = helper.add_bias(pre_bias)
return helper.add_activation(pre_activation)
```
We not only use the fewer lines of code to write `fc_layer` but also make the code clearer to understand. At the same time, layer developers can figure out what function they can invoke by typing `helper.` in a python editor.
### Implementation of layer helper
We just keep all parameters of a layer function as a dictionary in layer helper as a private data member. Every method of layer helper will look up the dictionary after it is invoked. In that way, we can implement a layer helper for all layer functions even some layer does not contain some operator. For example, The `activation` is used by the FullyConnected layer or convolution layers, but a cross-entropy layer does not use it. The example code of `add_activation` are:
```python
class LayerHelper(object):
def __init__(self, **kwargs): # kwargs is short for `keyword arguments`
self.kwargs = kwargs
def add_activation(self, input_var):
act = self.kwargs.get("act", None) # default value is None
if act is None: # do nothing if no act
return input_var
tmp = self.create_tmp_var(self)
self.append_op(type=act, input=input_var, output=tmp)
return tmp
```
## Optimizer
[Optimizer Design Doc](./optimizer.md)
doc/design/refactor/session.md 0 → 100644
浏览文件 @ 745634a1
# Design Doc: Session
## Abstract
The *session* object encapsulates the environment in which the
computation graph is executed.
We will have the *local* session and *remote* session, they offer the
same [interface](#interface). The local session encapsulates the local
runtime environment and the remote session encapsulates the cluster
runtime environment.
The local runtime environment contains:
1. computation devices (i.e., CPU, GPU) handles, and
1. the [scope](../scope.md) which holds all variables.
The remote runtime environment contains:
1. computation devices (i.e., CPU and GPU on node 0, 1) in a cluster,
and
1. the distributed [scope](../scope.md) in a cluster which holds all
variables.
The user can create a remote session on Paddle Cloud and evaluate the
computation graph with it. In this way, the user can control the
remote computation resource in a cluster from his local computer.
## Background
The current design has an implicit global session in which
`paddle.eval()` is executed. The pain point is:
Since the user is not able to explicitly switch between runtime
environments, the user cannot run a topology in two independent
environments.
For example, in reinforcement learning, the user may want to have a
stale model for inference and a fresh model for training, and only
replace the stale model with the fresh model periodically.
Furthermore, we have no concept that encapsulates a remote environment
that executes a computation graph.
We need the session object to address above issues.
## Session
A session is an object that owns the runtime environment. All
computations are executed through `session.eval()`.
### Interface
```python
eval(
targets,
feed_dict=None,
)
```
Evaluates the target Operations or Variables in `targets`.
- *targets*: the evaluation targets. Can be a single Operation or
Variable, or a list with the Operations or Variables as
elements. The value returned by `eval()` has the same shape as the
`target` argument.
The PaddlePaddle program is represented by
the [ProgramDesc](../design/program.md), `eval()` will infer the
ProgramDesc from the given targets and run the PaddlePaddle
program. Please
see
[this graph](./distributed_architecture.md#local-training-architecture) for
the detailed illustration for the local session
and
[this graph](./distributed_architecture.md#distributed-training-architecture) for
the detailed illustration for the remote session.
- *feed_dict*: a dictionary that contains the tensors which override
the edges of the computation graph.
feed_dict not only can provide the input data, it can override any
OP's input as well:
```python
a = pd.constant(2.0, name="a")
b = pd.variable(name="b")
c = pd.mul(a,b)
sess.eval(targets=c, feed_dict={"b":3.0}) # returns 6.0
```
```python
close()
```
Closes the session and releases the scope that the session owns.
### Create a Local Session
```python
session(
devices=None
)
```
Creates a new session. One session owns one global scope, so creating
multiple sessions will create different scopes.
- *devices*: a single `string` or a list of `string` of device names,
the corresponding devices will be the computation devices for
`eval()`. If not specified, all available devices (e.g., all GPUs)
will be used. The user doesn't need to specify the CPU device since
it will be always used. Multiple sessions can use the same device.
#### Example
```Python
a = paddle.constant(1.0)
b = paddle.constant(2.0)
c = a + b
sess = paddle.session(devices=["gpu:0", "gpu:1", "fpga:0"])
sess.eval(c)
sess.close()
```
### Create a Remote Session
```python
create_cloud_job(
name,
num_trainer,
mem_per_trainer,
gpu_per_trainer,
cpu_per_trainer,
num_ps,
mem_per_ps,
cpu_per_ps,
)
```
Creates a Paddle Cloud job. Fails if the job name exists.
```python
get_cloud_job(
name
)
```
Gets a Paddle Cloud job.
```python
remote_session(
job
)
```
- *job*: the Paddle Cloud job.
#### Example
```Python
reader = paddle.reader.recordio("/pfs/home/peter/mnist-train-*") # data stored on Paddle Cloud
image = reader.column(0)
label = reader.column(1)
fc1 = paddle.op.fc(image, size=256, act="sigmoid")
fc2 = paddle.op.fc(fc1, size=10, act="softmax")
cost = paddle.op.cross_entropy(fc2, label)
opt = paddle.optimizer.sgd(cost)
job = paddle.create_cloud_job("test", 3, "1G", 1, 1, 2, "1G", 1)
sess = paddle.remote_ession(job)
for i in range(1000):
sess.eval(opt)
sess.close()
```
doc/design/refactorization.md
浏览文件 @ 745634a1
......@@ -17,22 +17,22 @@ The goals of refactoring include:
1. A graph is composed of *variables* and *operators*.
1. The description of graphs must be capable of being serialized/deserialized, so that:
1. The description of graphs must be serializable/deserializable, so that:
1. It can to be sent to the cloud for distributed execution, and
1. It can be sent to the cloud for distributed execution, and
1. It can be sent to clients for mobile or enterprise deployment.
1. The Python program does the following steps
1. The Python program does two things
1. *compilation*: run a Python program to generate a protobuf message representation of the graph and send it to
1. *Compilation* runs a Python program to generate a protobuf message representation of the graph and send it to
1. the C++ library `libpaddle.so` for local execution,
1. the master process of a distributed training job for training, or
1. the server process of a Kubernetes serving job for distributed serving.
1. *execution*: execute the graph by constructing instances of class [`Variable`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/variable.h#L24) and [`OperatorBase`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/operator.h#L70), according to the protobuf message.
1. *Execution* executes the graph by constructing instances of class [`Variable`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/variable.h#L24) and [`OperatorBase`](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/operator.h#L70), according to the protobuf message.
## Description and Realization of Computation Graph
At compile time, the Python program generates a protobuf message representation of the graph, or the description of the graph.
At compile time, the Python program generates a protobuf message representation of the graph, or a description of the graph.
At runtime, the C++ program realizes the graph and runs it.
......@@ -42,11 +42,11 @@ At runtime, the C++ program realizes the graph and runs it.
|Operation|[OpDesc](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/framework.proto#L35)|[Operator](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/operator.h#L64)|
|Block|BlockDesc|Block|
The word *graph* is interchangeable with *block* in this document. A graph represents computation steps and local variables similar to a C++/Java program block, or a pair of parentheses(`{` and `}`).
The word *graph* is interchangeable with *block* in this document. A graph consists of computation steps and local variables similar to a C++/Java program block, or a pair of parentheses(`{` and `}`).
## Compilation and Execution
1. Run an application Python program to describe the graph. In particular, the Python application program does the following:
1. Run a Python program to describe the graph. In particular, the Python application program does the following:
1. Create `VarDesc` to represent local/intermediate variables,
1. Create operators and set attributes,
......@@ -54,10 +54,10 @@ The word *graph* is interchangeable with *block* in this document. A graph repr
1. Infer the type and the shape of variables,
1. Plan memory-reuse for variables,
1. Generate the backward graph
1. Optimize the computation graph.
1. Potentially, split the graph for distributed training.
1. Add optimization operators to the computation graph.
1. Optionally, split the graph for distributed training.
1. The invocation of `train` or [`infer`](https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/v2/inference.py#L108) methods in the application Python program does the following:
1. The invocation of `train` or [`infer`](https://github.com/PaddlePaddle/Paddle/blob/develop/python/paddle/v2/inference.py#L108) methods in the Python program does the following:
1. Create a new Scope instance in the [scope hierarchy](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/scope.md) for each run of a block,
1. realize local variables defined in the BlockDesc message in the new scope,
......@@ -107,8 +107,8 @@ Compile Time -> IR -> Runtime
![class_diagram](http://api.paddlepaddle.org/graphviz?dot=https://gist.githubusercontent.com/reyoung/53df507f6749762675dff3e7ce53372f/raw/dd598e8f1976f5759f58af5e5ef94738a6b2e661/op.dot)
* `Operator` is the fundamental building block of the user interface.
* Operator stores input/output variable names, and attributes.
* The `InferShape` interface is used to infer the shape of the output variable shapes based on the shapes of the input variables.
* Operator stores input/output variable names and attributes.
* The `InferShape` interface is used to infer the shape of the output variables based on the shapes of the input variables.
* Use `Run` to compute the `output` variables from the `input` variables.
---
......@@ -139,7 +139,7 @@ Compile Time -> IR -> Runtime
* Limit the number of `tensor.device(dev) = ` in your code.
* `thrust::transform` and `std::transform`.
* `thrust` has the same API as C++ standard library. Using `transform`, one can quickly implement customized element-wise kernels.
* `thrust` also has more complex APIs, like `scan`, `reduce`, `reduce_by_key`.
* `thrust`, in addition, supports more complex APIs, like `scan`, `reduce`, `reduce_by_key`.
* Hand-writing `GPUKernel` and `CPU` code
* Do not write in header (`.h`) files. CPU Kernel should be in cpp source (`.cc`) and GPU kernels should be in cuda (`.cu`) files. (GCC cannot compile GPU code.)
---
......@@ -182,10 +182,12 @@ REGISTER_OP_WITHOUT_GRADIENT(op_type, op_class, op_maker_class)
1. Write an Op class and its gradient Op class, if required.
2. Write an Op maker class. In the constructor of this class, describe the inputs, outputs and attributes of the operator.
3. Invoke the macro `REGISTER_OP`. This macro will
1. Call maker class to complete the `proto` and the `checker`
1. Call maker class to complete `proto` and `checker`
2. Using the completed `proto` and `checker`, it will add a new key-value pair to the `OpInfoMap`
4. Invoke the `USE` macro in which the Op is used to make sure that it is linked.
---
# Backward Module (1/2)
### Create Backward Operator
......@@ -195,13 +197,14 @@ REGISTER_OP_WITHOUT_GRADIENT(op_type, op_class, op_maker_class)
---
# Backward Module (2/2)
### Build Backward Network
- **Input**: graph of forward operators
- **Output**: graph of backward operators
- **Input**: a graph of forward operators
- **Output**: a graph of backward operators
- **Corner cases in construction**
- Shared Variables => insert an `Add` operator to combine gradients
- No Gradient => insert a `fill_zero_grad` operator
- Recursive NetOp => call `Backward` recursively
- RNN Op => recursively call `Backward` on stepnet
- RNN Op => recursively call `Backward` on stepnet
---
......@@ -211,10 +214,10 @@ REGISTER_OP_WITHOUT_GRADIENT(op_type, op_class, op_maker_class)
* Only dims and data pointers are stored in `Tensor`.
* All operations on `Tensor` are written in `Operator` or global functions.
* Variable length Tensor design [LoDTensor](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/framework/lod_tensor.md)
* `Variable` instances are the inputs and the outputs of an operator. Not just `Tensor`.
* `Variable` instances are the inputs and the outputs of an operator, not just `Tensor`.
* `step_scopes` in RNN is a variable and not a tensor.
* `Scope` is where variables are stores.
* map<string `variable_name`, Variable>
* `Scope` is where variables are stored.
* map<string `var name`, Variable>
* `Scope` has a hierarchical structure. The local scope can get variables from its parent scope.
---
......@@ -242,7 +245,7 @@ REGISTER_OP_WITHOUT_GRADIENT(op_type, op_class, op_maker_class)
---
# Control the migration quality
- Compare the performance of migrated models with old ones.
- Follow the google C++ style
- Follow the google C++ style guide.
- Build the automatic workflow of generating Python/C++ documentations.
- The documentation of layers and ops should be written inside the code.
- Take the documentation quality into account when submitting pull requests.
......
doc/design/register_grad_op.md 0 → 100644
浏览文件 @ 745634a1
# Design Doc: Gradient Operators Registration
## The Problem Posed
Currently, for each C++ operator class definition, there registers a *gradient operator creator* function, which takes a C++ operator instance and returns the corresponding gradient operator instance.
However, we noticed two problems with the current deisgn:
1. As we decided to separate the *compilation* and *execution* phases, we need to change the creator to take an `OpDesc` protobuf message in a `ProgramDesc` and inserts corresponding `OpDesc` messages into the `ProgramDesc` message.
1. Some operator's gradient computation requires more than one gradient operators. For example, the gradient of *minus* consists of two operators -- an identity operaotr and a scale operator. So we need to make the registration mechanism to support the mapping from an operator to a set of operators for gradient computation.
## The Current Implementation
The C++ class `OpInfos` store in a association map which key is the operator type. The `grad_op_type` indicate associated gradient operator type. Operator can create gradient operator by `OpInfo::creator_` of gradient. The pseudo code is
```cpp
struct OpInfo {
std::function<OperatorBase*(...)> creator_;
std::string grad_op_type_;
...
};
map<string, OpInfo> OpInfoMap;
OperatorBase* CreateGradientOperator(const OperatorBase& op) {
return OpInfoMap.at(op.Type()).creator_(...);
}
```
## Proposed Solution
The mapping relationship between an operator and its gradient operators is a function. The interface of that function is:
```cpp
// (OpDesc) --> vector<OpDesc>
std::function<std::vector<OpDescBind>(const OpDescBind&)>;
```
The function takes an `OpDescBind` of the forward operator and returns one or many gradient operator descriptions. `OpDescBind` is a C++ wrapper for protobuf message `OpDesc` to manipulate `OpDesc` fast.
The `GradOpDescMaker` will be registered in `OpInfo`, to replace `grad_op_type_` field. The `OpInfo` should be
```cpp
struct OpInfo {
std::function<std::vector<std::unique_ptr<OpDescBind>>(const OpDescBind&)> grad_op_maker_;
...
};
```
The `grad_op_maker_ ` is `nullptr` if the operator does not have associated gradient operators.
We propose a base class called `GradOpDescMakerBase` to let operator developers generate `Gradient Operators` easily. The public interface of that class is
```cpp
class GradOpDescMakerBase {
public:
GradOpDescMakerBase(const OpDescBind& );
virtual std::vector<std::unique_ptr<OpDescBind>> operator()()const = 0;
};
```
We can convert `GradOpDescMakerBase` to `std::function<std::vector<std::unique_ptr<OpDescBind>>(const OpDescBind&)>` by
```cpp
using GradOpMaker = ...;
std::function<std::vector<OpDescBind>(const OpDescBind&)> func;
func = [] (const OpDescBind& fwd_op) {
GradOpMaker maker(fwd_op);
return maker();
};
```
We can write many helper functions since the `GradOpDescMakerBase` is a class now. The basic helper functions get the variables of `Input`, `Output`, `InputGradient` and `OutputGradient` in the forwarding operator.
We should chagne register macros at the same time. In the current solution, there is no difference between forwarding operators and backward operators. So `REGISTER_OP` just register one operator. If the `REGISTER_OPERATOR ` contains `OpProtoAndCheckerMaker` and `GradOpDescMaker`, we just list them in the same macro. It can be done by a macro contains `__VA_ARGS__`.
The user interface should be
```cpp
vector<OpDesc> MinusOpGradMaker(OpDesc) {...}
REGISTER_OPERATOR(minus, MinusOp, MinusOpProtoAndCheckerMaker, SumOpGradMaker);
// Developers can still manually implement gradient operator.
REGISTER_OPERATOR(minus_grad, MinusGradOp);
```
The interface of current `REGISTER_OP` macro could not be changed. In `REGISTER_OP`, it will invoke `REGISTER_OPERATOR` two times and generate GradOpDescMaker inside.
```cpp
REGISTER_OP(minus, MinusOp, MinusOpProtoAndCheckerMaker, minus_grad, MinusGradOp);
```
doc/design/scope.md
浏览文件 @ 745634a1
......@@ -37,7 +37,7 @@ Scope is an association of a name to variable. All variables belong to `Scope`.
```cpp
class Scope {
public:
Variable* NewVar(const std::string& name);
Variable* Var(const std::string& name);
const Variable* FindVar(const std::string& name) const;
private:
......@@ -98,7 +98,7 @@ class Scope {
Variable* FindVar(const std::string& name) const;
// return if already contains same name variable.
Variable* NewVar(const std::string& name);
Variable* Var(const std::string& name);
private:
std::shared_ptr<Scope> parent_;
......@@ -107,7 +107,7 @@ class Scope {
```
## Only scope can create a variable
To ensure `only scope can create a variable`, we should mark `Variable`'s constructor as a private member function, and Scope is a friend class of Variable. And then only `NewVar` can construct `Variable`.
To ensure `only scope can create a variable`, we should mark `Variable`'s constructor as a private member function, and Scope is a friend class of Variable. And then only `Var` can construct `Variable`.
## When scope destroyed, all variables inside this scope should be destroyed together
......@@ -121,4 +121,4 @@ Also, as the parent scope is a `shared_ptr`, we can only `Create()` a scope shar
## Orthogonal interface
`FindVar` will return `nullptr` when `name` is not found. It can be used as `Contains` method. `NewVar` will return an `Error` when there is a name conflict locally. Combine `FindVar` and `NewVar`, we can implement `NewVar` easily.
`FindVar` will return `nullptr` when `name` is not found. It can be used as `Contains` method. `Var` will return an `Error` when there is a name conflict locally. Combine `FindVar` and `Var`, we can implement `Var` easily.
doc/design/selected_rows.md 0 → 100644
浏览文件 @ 745634a1
# Design Doc: Selected Rows
`SelectedRows` is a kind of sparse tensor data type, which is designed to support `embedding` operators. The gradient of embedding table is a sparse tensor. Only a few rows are non-zero values in that tensor. It is straightforward to represent the sparse tensor by the following sparse tensor data structure:
```cpp
class SelectedRows {
private:
vector<int> rows_;
Tensor value_;
int height_;
};
```
The field `height_` shows the first dimension of `SelectedRows`. The `rows` are the indices of which rows of `SelectedRows` are non-zeros. The `value_` field is an N-dim tensor and shape is `[rows.size() /* NUM_ROWS */, ...]`, which supplies values for each row. The dimension of `SelectedRows` satisfies `[height_] + value_.shape[1:]`.
Suppose that a SelectedRows-typed variable `x` has many rows, but only two of them have values -- row 73 is `[1, 2]` and row 84 is `[3, 4]`, the `SelectedRows` representation would be:
```
x = SelectedRow {
rows = [73, 84],
value = [[1, 2], [3,4]]
}
```
## SelectedRows in Protobuf
`SelectedRows` is a kind of `Variable`. `VarDesc` in protobuf should describe the `SelectedRows` information. Only the tensor dimension of a `SelectedRows` will be described in compile-time since the `rows_` and `value_` are related to training data.
So we use `TensorDesc` to unify `data_type` and `dims`. A LodTensorDesc contains a `TensorDesc` and `lod_level`. The description of `SelectedRows` is a Tensor description.
```proto
message TensorDesc {
required DataType data_type = 1;
repeated int64 dims = 2; // [UNK, 640, 480] is saved as [-1, 640, 480]
}
message LodTensorDesc {
required TensorDesc tensor = 1;
optional int lod_level = 2;
}
message VarDesc {
required string name = 1;
enum VarType {
LOD_TENSOR = 0;
SELECTED_ROWS = 1;
}
required VarType type = 2;
optional LodTensorDesc lod_desc = 3;
optional TensorDesc selected_rows_desc = 4;
optional bool persistable = 5 [ default = false ];
}
```
## InferShape for Selected Rows
Just like `LoD` information, `InferShape` method will inference output tensor type as well. The operator should decide whether its output is a `SelectedRows` or `Dense` tensor.
For example, the gradient operator of `TableLookup` will always generate `SelectedRows`. Its `InferShape` method should be like following
```cpp
void TableLookupGrad::InferShape(context) {
...
context.SetDataType("Embedding.Grad", kSelectedRows);
}
```
## Sparse Operators
There are several operators should be written to support `SelectedRows`. They are:
1. Operators which generates `SelectedRows` gradient. e.g. Gradient of `TableLookupOp`.
2. Optimize operators which support `SelectedRows` gradient. e.g. `SGD` or `AdaGrad` for `SelectedRows`. However, there should be only one `SGD` operator. `OpWithKernel::Run` should select a suitable kernel for both `dense` tensor or `SelectedRows`.
doc/design/tensor_array.md
浏览文件 @ 745634a1
# Design for TensorArray
This design doc presents the necessity of a new C++ class `TensorArray`.
In addition to the very simple C++ implementation
```c++
class TensorArray {
public:
explicit TensorArray(const LoDTensor&);
explicit TensorArray(size_t size);
private:
vector<LoDTensor> values_;
};
```
We also need to expose it to PaddlePaddle's Python API,
because users would want to use it with our very flexible operators `WhileLoop`.
An example for a RNN based on dynamic operators is
```python
input = pd.data(...)
num_steps = Var(12)
TensorArray states(size=num_steps)
TensorArray step_inputs(unstack_from=input)
TensorArray step_outputs(size=num_steps)
W = Tensor(...)
U = Tensor(...)
default_state = some_op()
step = Var(1)
wloop = paddle.create_whileloop(loop_vars=[step])
with wloop.frame():
wloop.break_if(pd.equal(step, num_steps)
pre_state = states.read(step-1, default_state)
step_input = step_inputs.read(step)
state = pd.sigmoid(pd.matmul(U, pre_state) + pd.matmul(W, step_input))
states.write(step, state)
step_outputs.write(step, state) # output state
step.update(state+1)
output = step_outputs.stack()
```
## Background
Steps are one of the core concepts of RNN. In each time step of RNN, there should be several input segments, states, and output segments; all these components act like arrays, for example, call `states[step_id]` will get the state in `step_id`th time step.
An RNN can be implemented with the following pseudocode
```c++
Array states;
Array input_segments;
Array output_segments;
Parameter W, U;
step = 1
seq_len = 12
while_loop {
if (step == seq_len) break;
states[step] = sigmoid(W * states[step-1] + U * input_segments[step]);
output_segments[step] = states[step] // take state as output
step++;
}
```
According to the [RNN roadmap](https://github.com/PaddlePaddle/Paddle/issues/4561), there are several different RNNs that PaddlePaddle will eventually support.
Currently, the basic RNN implementation supported by PaddlePaddle is the `recurrent_op` which takes tensors as input and splits them into `input_segments`.
Since a tensor cannot store variable-length sequences directly, PaddlePaddle implements the tensor with level of details (`LoDTensor` for short).
Segmenting the `LoDTensor` is much more complicated than splitting a tensor, that makes it necessary to refactor the `recurrent_op` with `LoDTensor` segmenting support.
As the next step in RNN support, `dynamic_recurrent_op` should be introduced to handle inputs with variable-length sequences.
The implementation is similar to `recurrent_op`.
The key difference is the way **the original input `LoDTensors` and outupts are split to get the `input_segments` and the `output_segments`.**
Though it can't be built over `recurrent_op` or `dynamic_recurrent_op` directly,
the logic behind splitting a tensor or a LoD tensor into `input_segments` remains the same.
## Why `TensorArray`
The logic behind splitting the inputs to segments, states and outputs is similar and can be shared in a seperate module.
The array of `states`, `input_segments` and `output_segments` would be exposed to users when writing a dynamic RNN model similar to the above pseudo codes.
So there should be an array-like container, which can store the segments of a tensor or LoD tensor.
**This container can store an array of tensors and provides several methods to split a tensor or a LoD tensor** .
This is where the notion of `TensorArray` comes from.
## Introduce TensorArray to uniform all the three RNNs
TensorArray as a new concept is borrowed from TensorFlow,
it is meant to be used with dynamic iteration primitives such as `while_loop` and `map_fn`.
This concept can be used to support our new design of dynamic operations, and help to refactor some existing variant-sentence-related layers,
such as `RecurrentGradientMachine`.
such as `recurrent_op`, `RecurrentGradientMachine`.
In [our design for dynamic RNN](https://github.com/PaddlePaddle/Paddle/pull/4401),
`TensorArray` is used to segment inputs and store states in all time steps.
By providing some methods similar to a C++ array,
the definition of some state-based dynamic models such as RNN could be more natural and highly flexible.
## Dynamic-Related Methods
Some basic methods should be proposed as follows:
### stack()
Pack the values in a `TensorArray` into a tensor with rank one higher than each tensor in `values`.
### unstack(axis=0)
Unpacks the given dimension of a rank-`R` tensor into rank-`(R-1)` tensors.
### concat()
Return the values in the `TensorArray` as a concatenated Tensor.
### write(index, value, data_shared=true)
Write value into index of the TensorArray.
### read(index)
Read the value at location `index` in the `TensorArray`.
### size()
Return the number of values.
the definition of some state-based dynamic models such as RNN can be more natural and highly flexible.
## Dynamic-operations on TensorArray
`TensorArray` will be used directly when defining dynamic models, so some operators listed below should be implemented
```python
# several helper operators for TensorArray
def tensor_array_stack(ta, tensor):
'''
get a tensor array `ta`, return a packed `tensor`.
'''
pass
def tensor_array_unstack(tensor, ta):
'''
get a `tensor`, unstack it and get a tensor array `ta`.
'''
pass
def tensor_array_write(ta, index, tensor, data_shared):
'''
get a `tensor` and a scalar tensor `index`, write `tensor` into index-th
value of the tensor array `ta`.
`data_shared` is an attribute that specifies whether to copy or reference the tensors.
'''
pass
def tensor_array_read(ta, index, tensor):
'''
get a tensor array `ta`, a scalar tensor `index`, read the index-th value of
`ta` and return as the `tensor`.
'''
pass
def tensor_array_size(ta, tensor):
'''
get a tensor array `ta`, return the size of `ta` and return as the scalar `tensor`.
'''
pass
```
It is trivial for users to use so many low-level operators, so some helper methods should be proposed in python wrapper to make `TensorArray` easier to use,
for example
```python
class TensorArray:
def __init__(self, name):
self.name = name
self.desc = TensorArrayDesc()
def stack(self, name=None):
'''
Pack the values in a `TensorArray` into a tensor with rank one higher
than each tensor in `values`.
`stack` can be used to split tensor into time steps for RNN or whileloop.
@name: str
the name of the variable to output.
'''
tensor = Var(name)
tensor_array_stack(self.name, tensor)
return tensor
def unstack(self, input):
'''
Unpacks the given dimension of a rank-`R` tensor into rank-`(R-1)` tensors.
`unstack` can be used to concatenate all the time steps for RNN or whileloop.
@input: str
the name of input tensor
'''
tensor_array_unstack(tensor, self.name)
def write(self, index, value, data_shared=True):
'''
Write value into index of the TensorArray.
If `data_shared` is set to True, than the index-th value in TensorArray will
be shared with the tensor passed in.
@index: str
name of a scalar tensor
@value: str
name of a tensor
@data_shared: bool
'''
tensor_array_write(self.name, index, value, data_shared)
def read(self, index, output):
'''
Read the value at location `index` in the `TensorArray`.
@index: str
name of a scalar tensor
@output:
name of a output variable
'''
tensor_array_read(self.name, index, output)
def size(self, output):
'''
Return the number of values.
@output: str
name of a scalar tensor
'''
tensor_array_size(self.name, output)
```
## LoDTensor-related Supports
The `RecurrentGradientMachine` in Paddle serves as a flexible RNN layer; it takes variant length sequences as input,
because each step of RNN could only take a tensor-represented batch of data as input,
The `RecurrentGradientMachine` in Paddle serves as a flexible RNN layer; it takes varience-length sequences as input, and output sequences too.
Since each step of RNN can only take a tensor-represented batch of data as input,
some preprocess should be taken on the inputs such as sorting the sentences by their length in descending order and cut each word and pack to new batches.
Such cut-like operations can be embedded into `TensorArray` as general methods called `unpack` and `pack`.
Such cut-like operations can be embedded into `TensorArray` as general methods called `unpack` and `pack`,
these two operations are similar to `stack` and `unstack` except that they operate on variable-length sequences formated as a LoD tensor rather than a tensor.
Some definitions are like
```python
def unpack(level):
'''
Split LodTensor in some `level` and generate batches, if set `sort_by_length`,
will sort by length.
With these two methods, a variant-sentence-RNN can be implemented like
Returns:
- a new `TensorArray`, whose values are LodTensors and represents batches
of data.
- an int32 Tensor, which stores the map from the new batch's indices to
original LoDTensor
'''
pass
def pack(level, indices_map):
'''
Recover the original LoD-arranged LoDTensor with the values in a `TensorArray`
and `level` and `indices_map`.
'''
pass
```
With these two methods, a varience-length sentence supported RNN can be implemented like
```c++
// input is the varient-length data
......@@ -58,16 +269,3 @@ LoDTensor rnn_output = ta.pack(ta, indice_map);
```
the code above shows that by embedding the LoDTensor-related preprocess operations into `TensorArray`,
the implementation of a RNN that supports varient-length sentences is far more concise than `RecurrentGradientMachine` because the latter mixes all the codes together, hard to read and extend.
some details are as follows.
### unpack(level, sort_by_length)
Split LodTensor in some `level` and generate batches, if set `sort_by_length`, will sort by length.
Returns:
- a new `TensorArray`, whose values are LodTensors and represents batches of data.
- an int32 Tensor, which stores the map from the new batch's indices to original LoDTensor
### pack(level, indices_map)
Recover the original LoD-arranged LoDTensor with the values in a `TensorArray` and `level` and `indices_map`.
doc/design/test.dot 0 → 100644
浏览文件 @ 745634a1
digraph Test {
z -> generator -> G_img;
G_img -> discriminator -> D_f -> d_loss_f;
label0 -> d_loss_f -> d_loss;
img -> discriminator -> D_t -> d_loss_t;
label1 -> d_loss_t -> d_loss;
d_loss -> d_loss_t[color=red, style=dashed];
d_loss -> d_loss_f[color=red, style=dashed];
d_loss_t -> D_t[color=red, style=dashed];
d_loss_f -> D_f[color=red, style=dashed];
D_t -> discriminator[color=red, style=dashed];
D_f -> discriminator[color=red, style=dashed];
D_f -> g_loss;
label2 -> g_loss;
g_loss -> D_f[color=green, style=dashed];
D_f -> discriminator[color=green, style=dashed];
discriminator -> G_img[color=green, style=dashed];
G_img -> generator[color=green, style=dashed];
discriminator [color=red, shape=box];
generator [color=green, shape=box];
z [shape=diamond];
img [shape=diamond];
label0 [shape=diamond];
label1 [shape=diamond];
label2 [shape=diamond];
d_loss [color=red];
g_loss [color=green];
}
doc/design/var_desc.md
浏览文件 @ 745634a1
......@@ -16,16 +16,23 @@ The computation graph is constructed by Data Node and Operation Node. The concep
## Definition of VarDesc
A VarDesc should have a name and value, in PaddlePaddle, the value will always be a tensor. Since we use LoDTensor most of the time. We add a LoDTesnorDesc to represent it.
A VarDesc should have a name, and value. The are two kinds of variable type in compile time, they are `LoDTensor` and `SelectedRows`.
```proto
message VarDesc {
required string name = 1;
optional LoDTensorDesc lod_tensor = 2;
enum VarType {
LOD_TENSOR = 0;
SELECTED_ROWS = 1;
}
required VarType type = 2;
optional LoDTensorDesc lod_desc = 3;
optional TensorDesc selected_rows_desc = 4;
optional bool persistable = 5 [ default = false ];
}
```
## Definition of LodTensorDesc
## Definition of TensorDesc
```proto
enum DataType {
......@@ -38,87 +45,25 @@ enum DataType {
FP64 = 6;
}
message LoDTensorDesc {
message TensorDesc {
required DataType data_type = 1;
repeated int32 dims = 2; // [UNK, 640, 480] is saved as [-1, 640, 480]
optional int32 lod_level = 3 [default=0];
repeated int64 dims = 2; // [UNK, 640, 480] is saved as [-1, 640, 480]
}
```
## Definition of Variable in Python
In Python API, layer will take Variable as Input, and return Variable as Output. There should be a class `Variable` in python to help create and manage Variable.
```python
image = Variable(dims=[-1, 640, 480])
# fc1 and fc2 are both Variable
fc1 = layer.fc(input=image, output_size=10)
fc2 = layer.fc(input=fc1, output_size=20)
```
### what should class `Variable` Have
1. `name`.a name of string type is used to mark the value of the Variable.
1. `initializer`. Since our Tensor does not have value. we will always use some Operator to fullfill it when run. So we should have a initialize method to help add the init operator.
1. `operator`. Variable should record which operator produce itself. The reaon is:
- we use pd.eval(targets=[var1, var2]) to run the related ops to get the value of var1 and var2. var.op is used to trace the dependency of the current variable.
In PaddlePaddle, we use Block to describe Computation Graph, so in the code we will use Block but not Graph.
```python
import VarDesc
import LoDTensorDesc
import framework
def AddInitialOperator(variable, initializer):
# add an initialize Operator to block to init this Variable
class Variable(object):
def __init__(self, name, dims, type, initializer):
self._block = get_default_block()
self._name = name
self.op = None
tensor_desc = LoDTensorDesc(data_type=type, dims=dims)
_var_desc = VarDesc(name=name, lod_tensor=tensor_desc)
self._var = framework.CreateVar(_var_desc)
self._block.add_var(self)
A TensorDesc describes `SelectedRows` and `LoDTensor`. For details of `SelectedRows`, please reference [`SelectedRows`](./selected_rows.md).
# add initial op according to initializer
if initializer is not None:
AddInitialOperator(self, initializer)
def dims(self):
return self._var.dims()
def data_type(self):
return self._var.data_type()
## Definition of LodTensorDesc
def to_proto(self):
pass
```proto
message LoDTensorDesc {
required TensorDesc tensor = 1;
optional int lod_level = 2;
}
```
Then we can use this Variable to create a fc layer in Python.
A LoDTensorDesc contains a tensor and a lod_level.
```python
import paddle as pd
def flatten_size(X, num_flatten_dims):
prod = 1 # of last num_flatten_dims
for i in xrange(num_flatten_dims):
prod = prod * X.dims[-i-1]
return prod
def layer.fc(X, output_size, num_flatten_dims):
W = Variable(pd.random_uniform(), type=FP32, dims=[flatten_size(X, num_flatten_dims), output_size])
b = Variable(pd.random_uniform(), type=FP32, dims=[output_size])
out = Variable(type=FP32)
y = operator.fc(X, W, b, output=out) # fc will put fc op input into out
pd.InferShape(y)
return out
x = Variable(dims=[-1, 640, 480])
y = layer.fc(x, output_size=100)
z = layer.fc(y, output_size=200)
## Definition of Variable in Python
paddle.eval(targets=[z], ...)
print(z)
```
For Variable in Python, please reference [`Python API`](./python_api.md).
doc/howto/deep_model/rnn/rnn_config_cn.rst
浏览文件 @ 745634a1
......@@ -21,7 +21,7 @@ wmt14数据的提供文件在 `python/paddle/v2/dataset/wmt14.py <https://github
循环神经网络在每个时间步骤顺序地处理序列。下面列出了 LSTM 的架构的示例。
.. image:: ../../../tutorials/sentiment_analysis/bi_lstm.jpg
.. image:: src/bi_lstm.jpg
:align: center
一般来说，循环网络从 :math:`t=1` 到 :math:`t=T` 或者反向地从 :math:`t=T` 到 :math:`t=1` 执行以下操作。
......@@ -96,7 +96,7 @@ Sequence to Sequence Model with Attention
我们将使用 sequence to sequence model with attention
作为例子演示如何配置复杂的循环神经网络模型。该模型的说明如下图所示。
.. image:: ../../../tutorials/text_generation/encoder-decoder-attention-model.png
.. image:: src/encoder-decoder-attention-model.png
:align: center
在这个模型中，源序列 :math:`S = \{s_1, \dots, s_T\}`
......
doc/howto/deep_model/rnn/rnn_config_en.rst
浏览文件 @ 745634a1
......@@ -19,7 +19,7 @@ Simple Gated Recurrent Neural Network
Recurrent neural network process a sequence at each time step sequentially. An example of the architecture of LSTM is listed below.
.. image:: ../../../tutorials/sentiment_analysis/src/bi_lstm.jpg
.. image:: src/bi_lstm.jpg
:align: center
Generally speaking, a recurrent network perform the following operations from :math:`t=1` to :math:`t=T`, or reversely from :math:`t=T` to :math:`t=1`.
......@@ -78,7 +78,7 @@ Sequence to Sequence Model with Attention
-----------------------------------------
We will use the sequence to sequence model with attention as an example to demonstrate how you can configure complex recurrent neural network models. An illustration of the sequence to sequence model with attention is shown in the following figure.
.. image:: ../../../tutorials/text_generation/encoder-decoder-attention-model.png
.. image:: src/encoder-decoder-attention-model.png
:align: center
In this model, the source sequence :math:`S = \{s_1, \dots, s_T\}` is encoded with a bidirectional gated recurrent neural networks. The hidden states of the bidirectional gated recurrent neural network :math:`H_S = \{H_1, \dots, H_T\}` is called *encoder vector* The decoder is a gated recurrent neural network. When decoding each token :math:`y_t`, the gated recurrent neural network generates a set of weights :math:`W_S^t = \{W_1^t, \dots, W_T^t\}`, which are used to compute a weighted sum of the encoder vector. The weighted sum of the encoder vector is utilized to condition the generation of the token :math:`y_t`.
......
doc/howto/dev/new_op_cn.md
浏览文件 @ 745634a1
......@@ -206,7 +206,7 @@ MulOp(const std::string &type, const framework::VariableNameMap &inputs,
- `REGISTER_OP` ： 注册`ops::MulOp`类，类型名为`mul`，该类的`ProtoMaker`为`ops::MulOpMaker`，注册`ops::MulOpGrad`，类型名为`mul_grad`。
- `REGISTER_OP_WITHOUT_GRADIENT` ： 用于注册没有反向的Op。
- `REGISTER_OP_CPU_KERNEL` ：注册`ops::MulKernel`类，并特化模板参数为`paddle::platform::CPUPlace`和`float`类型，同理，注册`ops::MulKernel`类。
- `REGISTER_OP_CPU_KERNEL` ：注册`ops::MulKernel`类，并特化模板参数为`paddle::platform::CPUPlace`和`float`类型，同理，注册`ops::MulGradKernel`类。
- 在 `.cu`文件中注册GPU Kernel。
......
doc/howto/dev/new_op_en.md
浏览文件 @ 745634a1
......@@ -205,7 +205,7 @@ The definition of its corresponding backward operator, if applicable, is similar
- `REGISTER_OP` registers the `ops::MulOp` class, type named `mul`, its type `ProtoMaker` is `ops::MulOpMaker`, registering `ops::MulOpGrad` as `mul_grad`.
- `REGISTER_OP_WITHOUT_GRADIENT` registers an operator without gradient.
- `REGISTER_OP_CPU_KERNEL` registers `ops::MulKernel` class and specialized template types `paddle::platform::CPUPlace` and `float`, which also registers `ops::MulKernel`.
- `REGISTER_OP_CPU_KERNEL` registers `ops::MulKernel` class and specialized template types `paddle::platform::CPUPlace` and `float`, which also registers `ops::MulGradKernel`.
- Registering GPU Kernel in `.cu` files
......
doc/tutorials/image_classification/index_cn.md 已删除 100644 → 0
浏览文件 @ 62b56e76
图像分类教程
==========
在本教程中，我们将使用CIFAR-10数据集训练一个卷积神经网络，并使用这个神经网络来对图片进行分类。如下图所示，卷积神经网络可以辨识图片中的主体，并给出分类结果。
<center>![Image Classification](./image_classification.png)</center>
## 数据准备
首先下载CIFAR-10数据集。下面是CIFAR-10数据集的官方网址：
<https://www.cs.toronto.edu/~kriz/cifar.html>
我们准备了一个脚本，可以用于从官方网站上下载CIFAR-10数据集，转为jpeg文件并存入特定的目录。使用这个脚本前请确认已经安装了pillow及相关依赖模块。可以参照下面的命令进行安装：
1. 安装pillow
```bash
sudo apt-get install libjpeg-dev
pip install pillow
```
2. 下载数据集
```bash
cd demo/image_classification/data/
sh download_cifar.sh
```
CIFAR-10数据集包含60000张32x32的彩色图片。图片分为10类，每个类包含6000张。其中50000张图片作为训练集，10000张作为测试集。
下图展示了所有的图片类别，每个类别中随机抽取了10张图片。
<center>![Image Classification](./cifar.png)</center>
脚本运行完成后，我们应当会得到一个名为cifar-out的文件夹，其下子文件夹的结构如下
```
train
---airplane
---automobile
---bird
---cat
---deer
---dog
---frog
---horse
---ship
---truck
test
---airplane
---automobile
---bird
---cat
---deer
---dog
---frog
---horse
---ship
---truck
```
cifar-out下包含`train`和`test`两个文件夹，其中分别包含了CIFAR-10中的训练集和测试集。这两个文件夹下各自有10个子文件夹，每个子文件夹下存储相应分类的图片。将图片按照上述结构存储好之后，我们就可以着手对分类模型进行训练了。
## 预处理
数据下载之后，还需要进行预处理，将数据转换为Paddle的格式。我们可以通过如下命令进行预处理工作：
```
cd demo/image_classification/
sh preprocess.sh
```
其中`preprocess.sh` 调用 `./demo/image_classification/preprocess.py` 对图片进行预处理
```sh
export PYTHONPATH=$PYTHONPATH:../../
data_dir=./data/cifar-out
python preprocess.py -i $data_dir -s 32 -c 1
```
`./demo/image_classification/preprocess.py` 使用如下参数：
- `-i` 或 `--input` 给出输入数据所在路径；
- `-s` 或 `--size` 给出图片尺寸；
- `-c` 或 `--color` 标示图片是彩色图或灰度图
## 模型训练
在开始训练之前，我们需要先创建一个模型配置文件。下面我们给出了一个配置示例。**注意**，这里的列出的和`vgg_16_cifar.py`文件稍有差别，因为该文件可适用于预测。
```python
from paddle.trainer_config_helpers import *
data_dir='data/cifar-out/batches/'
meta_path=data_dir+'batches.meta'
args = {'meta':meta_path, 'mean_img_size': 32,
'img_size': 32, 'num_classes': 10,
'use_jpeg': 1, 'color': "color"}
define_py_data_sources2(train_list=data_dir+"train.list",
test_list=data_dir+'test.list',
module='image_provider',
obj='processData',
args=args)
settings(
batch_size = 128,
learning_rate = 0.1 / 128.0,
learning_method = MomentumOptimizer(0.9),
regularization = L2Regularization(0.0005 * 128))
img = data_layer(name='image', size=3*32*32)
lbl = data_layer(name="label", size=10)
# small_vgg is predined in trainer_config_helpers.network
predict = small_vgg(input_image=img, num_channels=3)
outputs(classification_cost(input=predict, label=lbl))
```
在第一行中我们载入用于定义网络的函数。
```python
from paddle.trainer_config_helpers import *
```
之后定义的`define_py_data_sources2`使用Python数据提供器，其中 `args`将在`image_provider.py`进行使用，该文件负责产生图片数据并传递给Paddle系统
- `meta`: 训练集平均值。
- `mean_img_size`: 平均特征图的高度及宽度。
- `img_size`：输入图片的高度及宽度。
- `num_classes`：类别个数。
- `use_jpeg`：处理过程中数据存储格式。
- `color`：标示是否为彩色图片。
`settings`用于设置训练算法。在下面的例子中，learning rate被设置为0.1除以batch size，而weight decay则为0.0005乘以batch size。
```python
settings(
batch_size = 128,
learning_rate = 0.1 / 128.0,
learning_method = MomentumOptimizer(0.9),
regularization = L2Regularization(0.0005 * 128)
)
```
`small_vgg`定义了网络结构。这里我们使用的是一个小的VGG网络。关于VGG卷积神经网络的描述可以参考：[http://www.robots.ox.ac.uk/~vgg/research/very_deep/](http://www.robots.ox.ac.uk/~vgg/research/very_deep/)。
```python
# small_vgg is predined in trainer_config_helpers.network
predict = small_vgg(input_image=img, num_channels=3)
```
配置创建完毕后，可以运行脚本train.sh来训练模型。
```bash
config=vgg_16_cifar.py
output=./cifar_vgg_model
log=train.log
paddle train \
--config=$config \
--dot_period=10 \
--log_period=100 \
--test_all_data_in_one_period=1 \
--use_gpu=1 \
--save_dir=$output \
2>&1 | tee $log
python -m paddle.utils.plotcurve -i $log > plot.png
```
- 这里我们使用的是GPU模式进行训练。如果你没有GPU环境，可以设置`use_gpu=0`。
- `./demo/image_classification/vgg_16_cifar.py`是网络和数据配置文件。各项参数的详细说明可以在命令行参数相关文档中找到。
- 脚本`plotcurve.py`依赖于python的`matplotlib`模块。因此如果这个脚本运行失败，也许是因为需要安装`matplotlib`。
在训练完成后，训练及测试误差曲线图会被`plotcurve.py`脚本保存在 `plot.png`中。下面是一个误差曲线图的示例：
<center>![Training and testing curves.](./plot.png)</center>
## 预测
在训练完成后，模型及参数会被保存在路径`./cifar_vgg_model/pass-%05d`下。例如第300个pass的模型会被保存在`./cifar_vgg_model/pass-00299`。
要对一个图片的进行分类预测，我们可以使用`predict.sh`，该脚本将输出预测分类的标签：
```
sh predict.sh
```
predict.sh:
```
model=cifar_vgg_model/pass-00299/
image=data/cifar-out/test/airplane/seaplane_s_000978.png
use_gpu=1
python prediction.py $model $image $use_gpu
```
## 练习
在CUB-200数据集上使用VGG模型训练一个鸟类图片分类模型。相关的鸟类数据集可以从如下地址下载，其中包含了200种鸟类的照片（主要来自北美洲）。
<http://www.vision.caltech.edu/visipedia/CUB-200.html>
## 细节探究
### 卷积神经网络
卷积神经网络是一种使用卷积层的前向神经网络，很适合构建用于理解图片内容的模型。一个典型的神经网络如下图所示：
![Convolutional Neural Network](./lenet.png)
一个卷积神经网络包含如下层：
- 卷积层：通过卷积操作从图片或特征图中提取特征
- 池化层：使用max-pooling对特征图下采样
- 全连接层：使输入层到隐藏层的神经元是全部连接的。
卷积神经网络在图片分类上有着惊人的性能，这是因为它发掘出了图片的两类重要信息：局部关联性质和空间不变性质。通过交替使用卷积和池化处理， 卷积神经网络能够很好的表示这两类信息。
关于如何定义网络中的层，以及如何在层之间进行连接，请参考Layer文档。
doc/tutorials/image_classification/index_en.md 已删除 100644 → 0
浏览文件 @ 62b56e76
Image Classification Tutorial
==============================
This tutorial will guide you through training a convolutional neural network to classify objects using the CIFAR-10 image classification dataset.
As shown in the following figure, the convolutional neural network can recognize the main object in images, and output the classification result.
<center>![Image Classification](./image_classification.png)</center>
## Data Preparation
First, download CIFAR-10 dataset. CIFAR-10 dataset can be downloaded from its official website.
<https://www.cs.toronto.edu/~kriz/cifar.html>
We have prepared a script to download and process CIFAR-10 dataset. The script will download CIFAR-10 dataset from the official dataset.
It will convert it to jpeg images and organize them into a directory with the required structure for the tutorial. Make sure that you have installed pillow and its dependents.
Consider the following commands:
1. install pillow dependents
```bash
sudo apt-get install libjpeg-dev
pip install pillow
```
2. download data and preparation
```bash
cd demo/image_classification/data/
sh download_cifar.sh
```
The CIFAR-10 dataset consists of 60000 32x32 color images in 10 classes, with 6000 images per class. There are 50000 training images and 10000 test images.
Here are the classes in the dataset, as well as 10 random images from each:
<center>![Image Classification](./cifar.png)</center>
After downloading and converting, we should find a directory (cifar-out) containing the dataset in the following format:
```
train
---airplane
---automobile
---bird
---cat
---deer
---dog
---frog
---horse
---ship
---truck
test
---airplane
---automobile
---bird
---cat
---deer
---dog
---frog
---horse
---ship
---truck
```
It has two directories:`train` and `test`. These two directories contain training data and testing data of CIFAR-10, respectively. Each of these two folders contains 10 sub-folders, ranging from `airplane` to `truck`. Each sub-folder contains images with the corresponding label. After the images are organized into this structure, we are ready to train an image classification model.
## Preprocess
After the data has been downloaded, it needs to be pre-processed into the Paddle format. We can run the following command for preprocessing.
```
cd demo/image_classification/
sh preprocess.sh
```
`preprocess.sh` calls `./demo/image_classification/preprocess.py` to preprocess image data.
```sh
export PYTHONPATH=$PYTHONPATH:../../
data_dir=./data/cifar-out
python preprocess.py -i $data_dir -s 32 -c 1
```
`./demo/image_classification/preprocess.py` has the following arguments
- `-i` or `--input` specifes the input data directory.
- `-s` or `--size` specifies the processed size of images.
- `-c` or `--color` specifes whether images are color images or gray images.
## Model Training
We need to create a model config file before training the model. An example of the config file (vgg_16_cifar.py) is listed below. **Note**, it is slightly different from the `vgg_16_cifar.py` which also applies to the prediction.
```python
from paddle.trainer_config_helpers import *
data_dir='data/cifar-out/batches/'
meta_path=data_dir+'batches.meta'
args = {'meta':meta_path, 'mean_img_size': 32,
'img_size': 32, 'num_classes': 10,
'use_jpeg': 1, 'color': "color"}
define_py_data_sources2(train_list=data_dir+"train.list",
test_list=data_dir+'test.list',
module='image_provider',
obj='processData',
args=args)
settings(
batch_size = 128,
learning_rate = 0.1 / 128.0,
learning_method = MomentumOptimizer(0.9),
regularization = L2Regularization(0.0005 * 128))
img = data_layer(name='image', size=3*32*32)
lbl = data_layer(name="label", size=10)
# small_vgg is predined in trainer_config_helpers.network
predict = small_vgg(input_image=img, num_channels=3)
outputs(classification_cost(input=predict, label=lbl))
```
The first line imports python functions for defining networks.
```python
from paddle.trainer_config_helpers import *
```
Then define an `define_py_data_sources2` which use python data provider
interface. The arguments in `args` are used in `image_provider.py` which
yeilds image data and transform them to Paddle.
- `meta`: the mean value of training set.
- `mean_img_size`: the size of mean feature map.
- `img_size`: the height and width of input image.
- `num_classes`: the number of classes.
- `use_jpeg`: the data storage type when preprocessing.
- `color`: specify color image.
`settings` specifies the training algorithm. In the following example,
it specifies learning rate as 0.1, but divided by batch size, and the weight decay
is 0.0005 and multiplied by batch size.
```python
settings(
batch_size = 128,
learning_rate = 0.1 / 128.0,
learning_method = MomentumOptimizer(0.9),
regularization = L2Regularization(0.0005 * 128)
)
```
The `small_vgg` specifies the network. We use a small version of VGG convolutional network as our network
for classification. A description of VGG network can be found here [http://www.robots.ox.ac.uk/~vgg/research/very_deep/](http://www.robots.ox.ac.uk/~vgg/research/very_deep/).
```python
# small_vgg is predined in trainer_config_helpers.network
predict = small_vgg(input_image=img, num_channels=3)
```
After writing the config, we can train the model by running the script train.sh.
```bash
config=vgg_16_cifar.py
output=./cifar_vgg_model
log=train.log
paddle train \
--config=$config \
--dot_period=10 \
--log_period=100 \
--test_all_data_in_one_period=1 \
--use_gpu=1 \
--save_dir=$output \
2>&1 | tee $log
python -m paddle.utils.plotcurve -i $log > plot.png
```
- Here we use GPU mode to train. If you have no gpu environment, just set `use_gpu=0`.
- `./demo/image_classification/vgg_16_cifar.py` is the network and data configuration file. The meaning of the other flags can be found in the documentation of the command line flags.
- The script `plotcurve.py` requires the python module of `matplotlib`, so if it fails, maybe you need to install `matplotlib`.
After training finishes, the training and testing error curves will be saved to `plot.png` using `plotcurve.py` script. An example of the plot is shown below:
<center>![Training and testing curves.](./plot.png)</center>
## Prediction
After we train the model, the model file as well as the model parameters are stored in path `./cifar_vgg_model/pass-%05d`. For example, the model of the 300-th pass is stored at `./cifar_vgg_model/pass-00299`.
To make a prediction for an image, one can run `predict.sh` as follows. The script will output the label of the classfiication.
```
sh predict.sh
```
predict.sh:
```
model=cifar_vgg_model/pass-00299/
image=data/cifar-out/test/airplane/seaplane_s_000978.png
use_gpu=1
python prediction.py $model $image $use_gpu
```
## Exercise
Train a image classification of birds using VGG model and CUB-200 dataset. The birds dataset can be downloaded here. It contains an image dataset with photos of 200 bird species (mostly North American).
<http://www.vision.caltech.edu/visipedia/CUB-200.html>
## Delve into Details
### Convolutional Neural Network
A Convolutional Neural Network is a feedforward neural network that uses convolution layers. It is very suitable for building neural networks that process and understand images. A standard convolutional neural network is shown below:
![Convolutional Neural Network](./lenet.png)
Convolutional Neural Network contains the following layers:
- Convolutional layer: It uses convolution operation to extract features from an image or a feature map.
- Pooling layer: It uses max-pooling to downsample feature maps.
- Fully Connected layer: It uses fully connected connections to transform features.
Convolutional Neural Network achieves amazing performance for image classification because it exploits two important characteristics of images: *local correlation* and *spatial invariance*. By iteratively applying convolution and max-pooing operations, convolutional neural network can well represent these two characteristics of images.
For more details of how to define layers and their connections, please refer to the documentation of layers.
doc/tutorials/index_cn.md 已删除 100644 → 0
浏览文件 @ 62b56e76
# 完整教程
* [快速入门](quick_start/index_cn.rst)
* [个性化推荐](rec/ml_regression_cn.rst)
* [图像分类](image_classification/index_cn.md)
* [情感分析](sentiment_analysis/index_cn.md)
* [语义角色标注](semantic_role_labeling/index_cn.md)
* [机器翻译](text_generation/index_cn.md)
## 常用模型
* [ResNet模型](imagenet_model/resnet_model_cn.md)
* [词向量模型](embedding_model/index_cn.md)
doc/tutorials/index_en.md 已删除 100644 → 0
浏览文件 @ 62b56e76
# TUTORIALS
There are several examples and demos here.
* [Quick Start](quick_start/index_en.md)
* [MovieLens Regression](rec/ml_regression_en.rst)
* [Image Classification](image_classification/index_en.md)
* [Sentiment Analysis](sentiment_analysis/index_en.md)
* [Semantic Role Labeling](semantic_role_labeling/index_en.md)
* [Text Generation](text_generation/index_en.md)
* [Image Auto-Generation](gan/index_en.md)
## Model Zoo
* [ImageNet: ResNet](imagenet_model/resnet_model_en.md)
* [Embedding: Chinese Word](embedding_model/index_en.md)
doc/tutorials/rec/ml_dataset_cn.md 已删除 100644 → 0
浏览文件 @ 62b56e76
```eval_rst
.. _demo_ml_dataset:
```
# MovieLens数据集
[MovieLens 数据集](http://grouplens.org/datasets/movielens/)由GroupLens Research实验室搜集整理。
该数据集包含一些用户信息、电影信息以及电影评分\[1-5\]。根据数据量规模，该数据及有很多不同的版本。
我们用[MovieLens 百万数据集](http://files.grouplens.org/datasets/movielens/ml-1m.zip)作为示例数据
集，其中包含6,000位用户对4,000部电影的1,000,000条评价。该数据集于2003年2月发布。
## 数据集特征
在[ml-1m 数据集](http://files.grouplens.org/datasets/movielens/ml-1m.zip)中有许多的特征。在[ml-1m 数据集]
(http://files.grouplens.org/datasets/movielens/ml-1m.zip)中的这些数据文件(含有".dat"的后缀)实际上是CSV文件，
分隔符为"::"。以下我们翻译数据集网站中README文件的描述:
### 评分文件描述(ratings.dat)
所有的评分数据都包含在"ratings.dat"文件中，遵循如下的格式:
用户ID::电影ID::评分::时间戳
- 用户ID范围从1到6040
- 电影ID范围从1到3952
- 评分被调整为5星的规模(只允许整数的星级)
- 时间戳表示为从1970-01-01(UTC)来的秒数，与time(2)的返回值一致
- 每位用户至少有20条评分
### 用户文件描述(users.dat)
所有的用户信息都包含在"users.dat"文件中，遵循如下的格式:
用户ID::性别::年龄::职业::邮编
所有的人口统计学信息由用户自愿提供，没有进行正确性的检查。只有含有人
口统计学信息的用户才被包含在数据集中。
- 性别，用"M"表示男性，"F"表示女性
- 年龄从下列列表范围中选取:
* 1: "18岁以下"
* 18: "18-24岁"
* 25: "25-34岁"
* 35: "35-44岁"
* 45: "45-49岁"
* 50: "50-55岁"
* 56: "56+"
- 职业从下面所列中选择:
* 0: "其他"或不确定
* 1: "学术/教育工作者"
* 2: "艺术家"
* 3: "文书工作/管理员"
* 4: "大学生/研究生"
* 5: "客户服务"
* 6: "医生/医疗保健"
* 7: "行政工作/管理人员"
* 8: "农民"
* 9: "操持家务者"
* 10: "高中毕业生"
* 11: "律师"
* 12: "程序员"
* 13: "退休人员"
* 14: "销售/市场"
* 15: "科学家"
* 16: "自由职业者"
* 17: "技术员/工程师"
* 18: "推销员/手工艺者"
* 19: "无业人士"
* 20: "作家"
### 电影文件描述(movies.dat)
所有的电影信息都包含在"movies.dat"文件中，遵循如下的格式:
电影ID::电影名称::电影类型
- 电影名称（包括发行时间）与IMDB网站提供的一致
- 电影类型如符合多种用管道符号|分割，选自下列类型:
* 动作片
* 冒险片
* 动画片
* 儿童片
* 喜剧片
* 犯罪片
* 纪录片
* 戏剧
* 奇幻片
* 黑色电影
* 恐怖片
* 音乐剧
* 悬疑片
* 浪漫片
* 科幻片
* 惊险电影
* 战争片
* 西部片
- 由于意外的副本记录和测试记录，有些电影ID可能与实际电影不相符合
- 电影大部分是手工输入数据，因此可能会有一些错误和不一致发生
doc/tutorials/rec/ml_dataset_en.md 已删除 100644 → 0
浏览文件 @ 62b56e76
```eval_rst
.. _demo_ml_dataset:
```
# MovieLens Dataset
The [MovieLens Dataset](http://grouplens.org/datasets/movielens/) was collected by GroupLens Research.
The data set contains some user information, movie information, and many movie ratings from \[1-5\].
The data sets have many version depending on the size of set.
We use [MovieLens 1M Dataset](http://files.grouplens.org/datasets/movielens/ml-1m.zip) as a demo dataset, which contains
1 million ratings from 6000 users on 4000 movies. Released 2/2003.
## Dataset Features
In [ml-1m Dataset](http://files.grouplens.org/datasets/movielens/ml-1m.zip), there are many features in these dataset.
The data files (which have ".dat" extension) in [ml-1m Dataset](http://files.grouplens.org/datasets/movielens/ml-1m.zip)
is basically CSV file that delimiter is "::". The description in README we quote here.
### RATINGS FILE DESCRIPTION(ratings.dat)
All ratings are contained in the file "ratings.dat" and are in the
following format:
UserID::MovieID::Rating::Timestamp
- UserIDs range between 1 and 6040
- MovieIDs range between 1 and 3952
- Ratings are made on a 5-star scale (whole-star ratings only)
- Timestamp is represented in seconds since the epoch as returned by time(2)
- Each user has at least 20 ratings
### USERS FILE DESCRIPTION(users.dat)
User information is in the file "users.dat" and is in the following
format:
UserID::Gender::Age::Occupation::Zip-code
All demographic information is provided voluntarily by the users and is
not checked for accuracy. Only users who have provided some demographic
information are included in this data set.
- Gender is denoted by a "M" for male and "F" for female
- Age is chosen from the following ranges:
* 1: "Under 18"
* 18: "18-24"
* 25: "25-34"
* 35: "35-44"
* 45: "45-49"
* 50: "50-55"
* 56: "56+"
- Occupation is chosen from the following choices:
* 0: "other" or not specified
* 1: "academic/educator"
* 2: "artist"
* 3: "clerical/admin"
* 4: "college/grad student"
* 5: "customer service"
* 6: "doctor/health care"
* 7: "executive/managerial"
* 8: "farmer"
* 9: "homemaker"
* 10: "K-12 student"
* 11: "lawyer"
* 12: "programmer"
* 13: "retired"
* 14: "sales/marketing"
* 15: "scientist"
* 16: "self-employed"
* 17: "technician/engineer"
* 18: "tradesman/craftsman"
* 19: "unemployed"
* 20: "writer"
### MOVIES FILE DESCRIPTION(movies.dat)
Movie information is in the file "movies.dat" and is in the following
format:
MovieID::Title::Genres
- Titles are identical to titles provided by the IMDB (including
year of release)
- Genres are pipe-separated and are selected from the following genres:
* Action
* Adventure
* Animation
* Children's
* Comedy
* Crime
* Documentary
* Drama
* Fantasy
* Film-Noir
* Horror
* Musical
* Mystery
* Romance
* Sci-Fi
* Thriller
* War
* Western
- Some MovieIDs do not correspond to a movie due to accidental duplicate
entries and/or test entries
- Movies are mostly entered by hand, so errors and inconsistencies may exist
doc/tutorials/rec/ml_regression_cn.rst 已删除 100644 → 0
浏览文件 @ 62b56e76
MovieLens数据集评分回归模型
===========================
这里我们在MovieLens数据集描述一种 **余弦相似度回归** 任务。
该示例将展示paddle如何进行词向量嵌入，处理相似度回归，针对文本
的单词级别的卷积神经网络，以及paddle如何处理多种类型的输入。
需要注意的是，该模型网络只是用于进行demo展示paddle如何工作，而
没有进行结构的微调。
**我们非常欢迎您用PADDLEPADDLE构建更好的示例，如果您有好的建议来
让这个示例变得更好，希望能让我们知晓。**
数据准备
`````````
下载并解压数据集
'''''''''''''''''
这里我们使用 :ref:`demo_ml_dataset` 。
要下载和解压数据集，只需要简单的运行下面的命令即可。
.. code-block:: bash
cd demo/recommendation/data
./ml_data.sh
:code:`demo/recommendation/data/ml-1m` 的目录结构为:
.. code-block:: text
+--ml-1m
+--- movies.dat # 电影特征
+--- ratings.dat # 评分
+--- users.dat # 用户特征
+--- README # 数据集描述
字段配置文件
'''''''''''''
**字段配置文件** 用来具体说明数据集的字段和文件格式，
例如，说明每个特征文件具体字段是 **什么** 类型。
ml-1m的字段配置文件在目录 :code:`demo/recommendation/data/config.json` 中。
其具体说明了字段类型和文件名称:
1) 用户文件中有四种类型的字段\: 编号，性别，年龄和职业；
2) 文件名称为"users.dat"，文件的分隔符为"::"。
.. include:: ../../../demo/recommendation/data/config.json
:code: json
:literal:
准备数据
`````````
你需要安装python的第三方库。
**强烈推荐使用VIRTUALENV来创造一个干净的python环境。**
.. code-block:: bash
pip install -r requirements.txt
预处理数据一般的命令为:
.. code-block:: bash
cd demo/recommendation
./preprocess.sh
下面介绍预处理过程具体的步骤。
提取电影或用户的特征并生成python对象
'''''''''''''''''''''''''''''''''''''
在movielens 1m数据集中，电影和用户有许多的特征。
评分文件的每一行仅仅提供电影或用户的编号来代表相应的电影或用户。
我们首先处理电影或用户的特征文件，然后用pickle命令将特征( **Meta** )对象存储为文件。
Meta配置文件
.............
**Meta配置文件** 用来具体描述 **如何** 解析数据集中的每一个字段。
该文件可以从字段配置文件生成，或是手动编辑生成。文件的格式可以
为json或yaml格式。解析器能通过文件的扩展名自动识别文件的格式。
要将字段配置文件转化为meta配置文件，只需要运行：
.. code-block:: bash
cd demo/recommendation/data
python config_generator.py config.json > meta_config.json
生成的meta配置文件如下所示：
.. include:: ../../../demo/recommendation/data/meta_config.json
:code: json
:literal:
在meta文件中有两种特征\: 电影和用户。
* 在电影文件movies.dat中
* 我们仅用"::"来分隔每一行
* pos 0 代表编号
* pos 1 特征：
* name是电影名
* 利用正则表达式来解析该特征
* 基于字母的词嵌入特征
* 是序列
* pos 2 特征：
* name是体裁
* type是one hot稠密向量
* dictionary由解析自动生成，每一个key由'|'分隔
* 在用户文件users.dat中
* 我们仅用"::"来分隔每一行
* pos 0 代表编号
* pos 1 特征：
* name是性别
* 简单的基于字母的词嵌入
* pos 2 特征：
* name是年龄
* 是整个的词嵌入
* 嵌入编号会根据单词排序
* pos 3 特征：
* name是职业
* 简单的整个词嵌入
Meta文件
''''''''
有了meta配置文件之后，我们可以生成 **Meta文件** ，该文件是python的pickle对象，
存储着电影或用户信息。可以运行下面的命令来生成。
.. code-block:: bash
python meta_generator.py ml-1m meta.bin --config=meta_config.json
meta文件 :code:`meta.bin` 的结构如下：
.. code-block:: text
+--+ movie
| +--+ __meta__
| | +--+ raw_meta # 每个特征的meta配置。列表
| | | +
| | | | # 编号字段，我们用编号作为key
| | | +--+ {'count': 3883, 'max': 3952, 'is_key': True, 'type': 'id', 'min': 1}
| | | |
| | | | # 电影名字段，嵌入特征字典
| | | +--+ {'dict': [ ... ], 'type': 'embedding', 'name': 'title', 'seq': 'sequence'}
| | | |
| | | | # 体裁字段，体裁字典
| | | +--+ {'dict': [ ... ], 'type': 'one_hot_dense', 'name': 'genres'}
| | |
| | +--+ feature_map [1, 2] # a list for raw_meta index for feature field.
| | # it means there are 2 features for each key.
| | # * 0 offset of feature is raw_meta[1], Title.
| | # * 1 offset of feature is raw_meta[2], Genres.
| |
| +--+ 1 # 电影1的特征
| | +
| | +---+ [[...], [...]] # title ids, genres dense vector
| |
| +--+ 2
| |
| +--+ ...
|
+--- user
+--+ __meta__
| +
| +--+ raw_meta
| | +
| | +--+ id field as user
| | |
| | +--+ {'dict': ['F', 'M'], 'type': 'embedding', 'name': 'gender', 'seq': 'no_sequence'}
| | |
| | +--+ {'dict': ['1', '18', '25', '35', '45', '50', '56'], 'type': 'embedding', 'name': 'age', 'seq': 'no_sequence'}
| | |
| | +--+ {'dict': [...], 'type': 'embedding', 'name': 'occupation', 'seq': 'no_sequence'}
| |
| +--+ feature_map [1, 2, 3]
|
+--+ 1 # 用户1的特征
|
+--+ 2
+--+ ...
分割训练/测试文件
''''''''''''''''''
我们将 :code:`ml-1m/ratings.dat` 文件分割为训练和测试文件。分割文件的方法是：对于每位用户，我们将评分分成两部分。
这样的话每位用户在测试文件中将与训练文件含有同样的信息。
用 :code:`separate.py` 来分离训练和测试文件。
.. code-block:: bash
python split.py ml-1m/ratings.dat --delimiter="::" --test_ratio=0.1
这样就会生成两个文件：:code:`ml-1m/ratings.dat.train` 和 :code:`ml-1m/ratings.data.test` 。
将他们移动到目录 :code:`data` ，然后进行随机打乱，再为paddle的训练过程提供文件列表。
.. code-block:: bash
shuf ml-1m/ratings.dat.train > ratings.dat.train
cp ml-1m/ratings.dat.test .
echo "./data/ratings.dat.train" > train.list
echo "./data/ratings.dat.test" > test.list
神经网络结构配置
`````````````````
训练器配置文件
'''''''''''''''
网络结构如下图所示：
.. image:: rec_regression_network.png
:align: center
:alt: rec_regression_network
该示例的神经网络配置文件 :code:`trainer_config.py` 如下所示：
.. literalinclude:: ../../../demo/recommendation/trainer_config.py
:language: python
:lines: 15-
在文件 :code:`trainer_config.py` 中，我们仅仅是将每个特征种类映射到一个特征向量中，以下
展示了如何将每个特征映射到一个向量。
* :code:`id` \: 仅仅是简单的嵌入，然后添加一个全连接层。
* :code:`embedding` \:
- 如果是序列，则先做嵌入，然后再做一次文本卷积网络操作，
然后得到平均采样的结果。
- 如果不是序列，则先做嵌入，然后添加一个全连接层。
* :code:`one_host_dense` \:
- 仅仅是两个全连接层。
然后我们利用多输入的:code:`fc_layer` 全连接层将电影的每个特征结合成一个电影特征，
并且对用户的特征做同样的操作，也得到一个用户特征。然后我们求这两个特征的余弦相似度。
在这些网络中，我们用以下的一些:ref:`api_trainer_config` 中的接口。
* 数据层， :ref:`api_trainer_config_helpers_layers_data_layer`
* 全连接层， :ref:`api_trainer_config_helpers_layers_fc_layer`
* 嵌入层， :ref:`api_trainer_config_helpers_layers_embedding_layer`
* 文本投影层， :ref:`api_trainer_config_helpers_layers_context_projection`
* 采样层， :ref:`api_trainer_config_helpers_layers_pooling_layer`
* 余弦相似度层， :ref:`api_trainer_config_helpers_layers_cos_sim`
* 文本卷积采样层， :ref:`api_trainer_config_helpers_network_text_conv_pool`
* 声明Python数据源， :ref:`api_trainer_config_helpers_data_sources`
数据提供脚本
'''''''''''''
.. literalinclude:: ../../../demo/recommendation/dataprovider.py
:language: python
:lines: 15-
数据提供脚本仅仅是读取meta.bin和评分文件，生成训练需要的样本。
在脚本 :code:`dataprovider.py` 中，我们需要设置：
* obj.slots\: 特征的类型和维度。
* use_seq\: :code:`dataprovider.py` 中的数据是否为序列模式。
* process\: 返回数据的每一条样本给 :code:`paddle` 。
数据提供脚本的细节文档可以参考 :ref:`api_pydataprovider2` 。
训练
````
准备好数据，配置了网络，编写好数据提供脚本后，现在我们可以开始paddle训练了。
代码 :code:`run.sh` 如下：
.. literalinclude:: ../../../demo/recommendation/run.sh
:language: bash
:lines: 16-
该脚本仅仅是开始一个paddle训练过程，将日志写入文件 :code:`log.txt` ，然后
打印在屏幕上。
脚本 :code:`run.sh` 中的每一行命令，请参考页面 :ref:`cmd_line_index` 。
这些参数的简短介绍如下：
* config\: 告诉paddle哪个文件是神经网络的配置文件。
* save_dir\: 告诉paddle将模型保存在: code:`./output` 中。
* use_gpu\: 是否使用GPU，默认为不使用。
* trainer_count\: 一台机器上面的线程数量。
* test_all_data_in_one_period\: 每一个测试周期测试一次所有数据。否则，
每个测试周期测试: code:`batch_size` 批次的数据。
* log_period\: 在训练了: code:`log_period` 批次后打印日志。
* dot_period\: 在每训练: code:`dot_period` 个批次后打印一个 :code:`.` 。
* num_passes\: 训练至多: code:`num_passes` 轮。
如果训练过程启动成功的话，输出应该类似如下：
.. code-block:: text
I0601 08:07:22.832059 10549 TrainerInternal.cpp:157] Batch=100 samples=160000 AvgCost=4.13494 CurrentCost=4.13494 Eval: CurrentEval:
I0601 08:07:50.672627 10549 TrainerInternal.cpp:157] Batch=200 samples=320000 AvgCost=3.80957 CurrentCost=3.48421 Eval: CurrentEval:
I0601 08:08:18.877369 10549 TrainerInternal.cpp:157] Batch=300 samples=480000 AvgCost=3.68145 CurrentCost=3.42519 Eval: CurrentEval:
I0601 08:08:46.863963 10549 TrainerInternal.cpp:157] Batch=400 samples=640000 AvgCost=3.6007 CurrentCost=3.35847 Eval: CurrentEval:
I0601 08:09:15.413025 10549 TrainerInternal.cpp:157] Batch=500 samples=800000 AvgCost=3.54811 CurrentCost=3.33773 Eval: CurrentEval:
I0601 08:09:36.058670 10549 TrainerInternal.cpp:181] Pass=0 Batch=565 samples=902826 AvgCost=3.52368 Eval:
I0601 08:09:46.215489 10549 Tester.cpp:101] Test samples=97383 cost=3.32155 Eval:
I0601 08:09:46.215966 10549 GradientMachine.cpp:132] Saving parameters to ./output/model/pass-00000
I0601 08:09:46.233397 10549 ParamUtil.cpp:99] save dir ./output/model/pass-00000
I0601 08:09:46.233438 10549 Util.cpp:209] copy trainer_config.py to ./output/model/pass-00000
I0601 08:09:46.233541 10549 ParamUtil.cpp:147] fileName trainer_config.py
模型被保存在 :code:`output/` 目录中。你可以在任何时候用 :code:`Ctrl-C` 来停止训练。
模型评估和预测
```````````````
在训练了几个轮次以后，你可以对模型进行评估，得到最好轮次下的模型。运行下面命令即可：
.. code-block:: bash
./evaluate.sh
你将看到如下的信息：
.. code-block:: text
Best pass is 00009, error is 3.06949, which means predict get error as 0.875998002281
evaluating from pass output/pass-00009
然后，你可以预测任何用户对于任何一部电影的评价，运行下面命令即可：
.. code-block:: bash
python prediction.py 'output/pass-00009/'
预测程序将读取用户的输入，然后输出预测分数。用户预测的命令行界面如下：
.. code-block:: text
Input movie_id: 9
Input user_id: 4
Prediction Score is 2.56
Input movie_id: 8
Input user_id: 2
Prediction Score is 3.13
doc/tutorials/rec/ml_regression_en.rst 已删除 100644 → 0
浏览文件 @ 62b56e76
Regression MovieLens Ratting
============================
Here we demonstrate a **Cosine Similarity Regression** job in movie lens dataset.
This demo will show how paddle does (word) embedding job,
handles the similarity regression,
the character-level convolutional networks for text, and how does paddle handle
multiple types of inputs.
Note that the model structure is not fine-tuned and just a demo to show how paddle works.
YOU ARE WELCOME TO BUILD A BETTER DEMO
BY USING PADDLEPADDLE, AND LET US KNOW TO MAKE THIS DEMO BETTER.
Data Preparation
````````````````
Download and extract dataset
''''''''''''''''''''''''''''
We use :ref:`demo_ml_dataset` here.
To download and unzip the dataset, simply run the following commands.
.. code-block:: bash
cd demo/recommendation/data
./ml_data.sh
And the directory structure of :code:`demo/recommendation/data/ml-1m` is:
.. code-block:: text
+--ml-1m
+--- movies.dat # movie features
+--- ratings.dat # ratings
+--- users.dat # user features
+--- README # dataset description
Field config file
'''''''''''''''''
**Field config file** is used to specify the fields of the dataset and the file format,
i.e, specific **WHAT** type it is in each feature file.
The field config file of ml-1m shows in :code:`demo/recommendation/data/config.json`.
It specifics the field types and file names: 1) there are four types of field for user file\: id, gender, age and occupation;
2) the filename is "users.dat", and the delimiter of file is "::".
.. include:: ../../../demo/recommendation/data/config.json
:code: json
:literal:
Preprocess Data
```````````````
You need to install python 3rd party libraries.
IT IS HIGHLY RECOMMEND TO USE VIRTUALENV MAKE A CLEAN PYTHON ENVIRONMENT.
.. code-block:: bash
pip install -r requirements.txt
The general command for preprocessing the dataset is:
.. code-block:: bash
cd demo/recommendation
./preprocess.sh
And the detail steps are introduced as follows.
Extract Movie/User features to python object
'''''''''''''''''''''''''''''''''''''''''''''
There are many features in movie or user in movielens 1m dataset.
Each line of rating file just provides a Movie/User id to refer each movie or user.
We process the movie/user feature file first, and pickle the feature (**Meta**) object as a file.
Meta config file
................
**Meta config file** is used to specific **HOW** to parse each field in dataset.
It could be translated from field config file, or written by hand.
Its file format could be either json or yaml syntax file. Parser will automatically choose the file format by extension name.
To convert Field config file to meta config file, just run:
.. code-block:: bash
cd demo/recommendation/data
python config_generator.py config.json > meta_config.json
The meta config file shows below:
.. include:: ../../../demo/recommendation/data/meta_config.json
:code: json
:literal:
There are two kinds of features in meta\: movie and user.
* in movie file, whose name is movies.dat
* we just split each line by "::"
* pos 0 is id.
* pos 1 feature:
* name is title.
* it uses regex to parse this feature.
* it is a char based word embedding feature.
* it is a sequence.
* pos 2 feature:
* name is genres.
* type is one hot dense vector.
* dictionary is auto generated by parsing, each key is split by '|'
* in user file, whose name is users.dat
* we just split each line by "::"
* pos 0 is id.
* pos 1 feature:
* name is gender
* just simple char based embedding.
* pos 2 feature:
* name is age
* just whole word embedding.
* embedding id will be sort by word.
* pos 3 feature:
* name is occupation.
* just simple whole word embedding.
Meta file
'''''''''
After having meta config file, we can generate **Meta file**, a python pickle object which stores movie/user information.
The following commands could be run to generate it.
.. code-block:: bash
python meta_generator.py ml-1m meta.bin --config=meta_config.json
And the structure of the meta file :code:`meta.bin` is:
.. code-block:: text
+--+ movie
| +--+ __meta__
| | +--+ raw_meta # each feature meta config. list
| | | +
| | | | # ID Field, we use id as key
| | | +--+ {'count': 3883, 'max': 3952, 'is_key': True, 'type': 'id', 'min': 1}
| | | |
| | | | # Titile field, the dictionary list of embedding.
| | | +--+ {'dict': [ ... ], 'type': 'embedding', 'name': 'title', 'seq': 'sequence'}
| | | |
| | | | # Genres field, the genres dictionary
| | | +--+ {'dict': [ ... ], 'type': 'one_hot_dense', 'name': 'genres'}
| | |
| | +--+ feature_map [1, 2] # a list for raw_meta index for feature field.
| | # it means there are 2 features for each key.
| | # * 0 offset of feature is raw_meta[1], Title.
| | # * 1 offset of feature is raw_meta[2], Genres.
| |
| +--+ 1 # movie 1 features
| | +
| | +---+ [[...], [...]] # title ids, genres dense vector
| |
| +--+ 2
| |
| +--+ ...
|
+--- user
+--+ __meta__
| +
| +--+ raw_meta
| | +
| | +--+ id field as user
| | |
| | +--+ {'dict': ['F', 'M'], 'type': 'embedding', 'name': 'gender', 'seq': 'no_sequence'}
| | |
| | +--+ {'dict': ['1', '18', '25', '35', '45', '50', '56'], 'type': 'embedding', 'name': 'age', 'seq': 'no_sequence'}
| | |
| | +--+ {'dict': [...], 'type': 'embedding', 'name': 'occupation', 'seq': 'no_sequence'}
| |
| +--+ feature_map [1, 2, 3]
|
+--+ 1 # user 1 features
|
+--+ 2
+--+ ...
Split Training/Testing files
''''''''''''''''''''''''''''
We split :code:`ml-1m/ratings.dat` into a training and testing file. The way to split file is for each user, we split the
rating by two parts. So each user in testing file will have some rating information in training file.
Use :code:`separate.py` to separate the training and testing file.
.. code-block:: bash
python split.py ml-1m/ratings.dat --delimiter="::" --test_ratio=0.1
Then two files will be generated\: :code:`ml-1m/ratings.dat.train` and :code:`ml-1m/rating.data.test`.
Move them to workspace :code:`data`, shuffle the train file, and prepare the file list for paddle train.
.. code-block:: bash
shuf ml-1m/ratings.dat.train > ratings.dat.train
cp ml-1m/ratings.dat.test .
echo "./data/ratings.dat.train" > train.list
echo "./data/ratings.dat.test" > test.list
Neural Network Configuration
````````````````````````````
Trainer Config File
'''''''''''''''''''
The network structure shows below.
.. image:: rec_regression_network.png
:align: center
:alt: rec_regression_network
The demo's neural network config file :code:`trainer_config.py` show as below.
.. literalinclude:: ../../../demo/recommendation/trainer_config.py
:language: python
:lines: 15-
In this :code:`trainer_config.py`, we just map each feature type to
a feature vector, following shows how to map each feature to a vector shows below.
* :code:`id`\: Just simple embedding, and then add to fully connected layer.
* :code:`embedding`\:
- if is_sequence, get the embedding and do a text convolutional operation,
get the average pooling result.
- if not sequence, get the embedding and add to fully connected layer.
* :code:`one_host_dense`\:
- just two fully connected layer.
Then we combine each features of movie into one movie feature by a
:code:`fc_layer` with multiple inputs, and do the same thing to user features,
get one user feature. Then we calculate the cosine similarity of these two
features.
In these networks, we use several APIs in :ref:`api_trainer_config` . There are
* Data Layer, :ref:`api_trainer_config_helpers_layers_data_layer`
* Fully Connected Layer, :ref:`api_trainer_config_helpers_layers_fc_layer`
* Embedding Layer, :ref:`api_trainer_config_helpers_layers_embedding_layer`
* Context Projection Layer, :ref:`api_trainer_config_helpers_layers_context_projection`
* Pooling Layer, :ref:`api_trainer_config_helpers_layers_pooling_layer`
* Cosine Similarity Layer, :ref:`api_trainer_config_helpers_layers_cos_sim`
* Text Convolution Pooling Layer, :ref:`api_trainer_config_helpers_network_text_conv_pool`
* Declare Python Data Sources :ref:`api_trainer_config_helpers_data_sources`.
Data Provider
'''''''''''''
.. literalinclude:: ../../../demo/recommendation/dataprovider.py
:language: python
:lines: 15-
The data provider just read the meta.bin and rating file, yield each sample for training.
In this :code:`dataprovider.py`, we should set\:
* obj.slots\: The feature types and dimension.
* use_seq\: Whether this :code:`dataprovider.py` in sequence mode or not.
* process\: Return each sample of data to :code:`paddle`.
The data provider details document see :ref:`api_pydataprovider2`.
Train
`````
After prepare data, config network, writting data provider, now we can run paddle training.
The :code:`run.sh` is shown as follow:
.. literalinclude:: ../../../demo/recommendation/run.sh
:language: bash
:lines: 16-
It just start a paddle training process, write the log to :code:`log.txt`,
then print it on screen.
Each command line argument in :code:`run.sh`, please refer to the :ref:`cmd_line_index` page. The short description of these arguments is shown as follow.
* config\: Tell paddle which file is neural network configuration.
* save_dir\: Tell paddle save model into :code:`./output`.
* use_gpu\: Use gpu or not. Default is false.
* trainer_count\: The compute thread in one machine.
* test_all_data_in_one_period\: Test All Data during one test period. Otherwise,
will test a :code:`batch_size` data in one test period.
* log_period\: Print log after train :code:`log_period` batches.
* dot_period\: Print a :code:`.` after train :code:`dot_period` batches.
* num_passes\: Train at most :code:`num_passes`.
If training process starts successfully, the output likes follow:
.. code-block:: text
I0601 08:07:22.832059 10549 TrainerInternal.cpp:157] Batch=100 samples=160000 AvgCost=4.13494 CurrentCost=4.13494 Eval: CurrentEval:
I0601 08:07:50.672627 10549 TrainerInternal.cpp:157] Batch=200 samples=320000 AvgCost=3.80957 CurrentCost=3.48421 Eval: CurrentEval:
I0601 08:08:18.877369 10549 TrainerInternal.cpp:157] Batch=300 samples=480000 AvgCost=3.68145 CurrentCost=3.42519 Eval: CurrentEval:
I0601 08:08:46.863963 10549 TrainerInternal.cpp:157] Batch=400 samples=640000 AvgCost=3.6007 CurrentCost=3.35847 Eval: CurrentEval:
I0601 08:09:15.413025 10549 TrainerInternal.cpp:157] Batch=500 samples=800000 AvgCost=3.54811 CurrentCost=3.33773 Eval: CurrentEval:
I0601 08:09:36.058670 10549 TrainerInternal.cpp:181] Pass=0 Batch=565 samples=902826 AvgCost=3.52368 Eval:
I0601 08:09:46.215489 10549 Tester.cpp:101] Test samples=97383 cost=3.32155 Eval:
I0601 08:09:46.215966 10549 GradientMachine.cpp:132] Saving parameters to ./output/model/pass-00000
I0601 08:09:46.233397 10549 ParamUtil.cpp:99] save dir ./output/model/pass-00000
I0601 08:09:46.233438 10549 Util.cpp:209] copy trainer_config.py to ./output/model/pass-00000
I0601 08:09:46.233541 10549 ParamUtil.cpp:147] fileName trainer_config.py
The model is saved in :code:`output/` directory. You can use :code:`Ctrl-C` to stop training whenever you want.
Evaluate and Predict
````````````````````
After training several passes, you can evaluate them and get the best pass. Just run
.. code-block:: bash
./evaluate.sh
You will see messages like this:
.. code-block:: text
Best pass is 00009, error is 3.06949, which means predict get error as 0.875998002281
evaluating from pass output/pass-00009
Then, you can predict what any user will rate a movie. Just run
.. code-block:: bash
python prediction.py 'output/pass-00009/'
Predictor will read user input, and predict scores. It has a command-line user interface as follows:
.. code-block:: text
Input movie_id: 9
Input user_id: 4
Prediction Score is 2.56
Input movie_id: 8
Input user_id: 2
Prediction Score is 3.13
doc/tutorials/semantic_role_labeling/index_cn.md 已删除 100644 → 0
浏览文件 @ 62b56e76
# 语义角色标注教程 #
语义角色标注（Semantic role labeling, SRL）是浅层语义解析的一种形式，其目的是在给定的输入句子中发现每个谓词的谓词论元结构。 SRL作为很多自然语言处理任务中的中间步骤是很有用的，如信息提取、文档自动分类和问答。 实例如下 [1]:
[ <sub>A0</sub> He ] [ <sub>AM-MOD</sub> would ][ <sub>AM-NEG</sub> n’t ] [ <sub>V</sub> accept] [ <sub>A1</sub> anything of value ] from [<sub>A2</sub> those he was writing about ].
- V: 动词
- A0: 接受者
- A1: 接受的东西
- A2: 从……接受
- A3: 属性
- AM-MOD: 情态动词
- AM-NEG: 否定
给定动词“accept”，句子中的组块将会扮演某些语义角色。这里，标签方案来自 Penn Proposition Bank。
到目前为止，大多数成功的SRL系统是建立在某种形式的句法分析结果之上的，使用了基于句法结构的预定义特征模板。 本教程将介绍使用深度双向长短期记忆（DB-LSTM）模型[2]的端到端系统来解决SRL任务，这在很大程度上优于先前的最先进的系统。 这个系统将SRL任务视为序列标注问题。
## 数据描述
相关论文[2]采用 CoNLL-2005＆2012 共享任务中设置的数据进行训练和测试。由于数据许可的原因，演示采用 CoNLL-2005 的测试数据集，可以在网站上找到。
用户只需执行以下命令就可以下载并处理原始数据：
```bash
cd data
./get_data.sh
```
`data `目录会出现如下几个新的文件：
```bash
conll05st-release：the test data set of CoNll-2005 shared task
test.wsj.words：the Wall Street Journal data sentences
test.wsj.props: the propositional arguments
feature: the extracted features from data set
```
## 训练
### DB-LSTM
请参阅情感分析的演示以了解有关长期短期记忆单元的更多信息。
与在 Sentiment Analysis 演示中使用的 Bidirectional-LSTM 不同，DB-LSTM 采用另一种方法来堆叠LSTM层。首先，标准LSTM以正向处理该序列。该 LSTM 层的输入和输出作为下一个 LSTM 层的输入，并被反向处理。这两个标准 LSTM 层组成一对 LSTM。然后我们堆叠一对对的 LSTM 层后得到深度 LSTM 模型。
下图展示了时间扩展的2层 DB-LSTM 网络。
<center>
![pic](./network_arch.png)
</center>
### 特征
两个输入特征在这个流程中起着至关重要的作用：predicate（pred）和argument（arguments）。 还采用了两个其他特征：谓词上下文（ctx-p）和区域标记（mr）。 因为单个谓词不能精确地描述谓词信息，特别是当相同的词在句子中出现多于一次时。 使用谓词上下文，可以在很大程度上消除歧义。类似地，如果它位于谓词上下文区域中，则使用区域标记 m<sub>r</sub> = 1 来表示参数位置，反之则 m<sub>r</sub> = 0。这四个简单的特征是我们的SRL系统所需要的。上下文大小设置为1的一个样本的特征如下[2]所示：
<center>
![pic](./feature.jpg)
</center>
在这个示例中，相应的标记句子是：
[ <sub>A1</sub> A record date ] has [ <sub>AM-NEG</sub> n't ] been [ <sub>V</sub> set ] .
在演示中, 我们采用上面的特征模板, 包括： `argument`, `predicate`, `ctx-p (p=-1,0,1)`, `mark` 并使用 `B/I/O` 方案来标记每个参数。这些特征和标签存储在 `feature` 文件中, 用`\t`分割。
### 数据提供
`dataprovider.py` 是一个包装数据的 Python 文件。 函数 `hook()` 定义了网络的数据槽。六个特征和标签都是索引槽。
```
def hook(settings, word_dict, label_dict, **kwargs):
settings.word_dict = word_dict
settings.label_dict = label_dict
#all inputs are integral and sequential type
settings.slots = [
integer_value_sequence(len(word_dict)),
integer_value_sequence(len(predicate_dict)),
integer_value_sequence(len(word_dict)),
integer_value_sequence(len(word_dict)),
integer_value_sequence(len(word_dict)),
integer_value_sequence(len(word_dict)),
integer_value_sequence(len(word_dict)),
integer_value_sequence(2),
integer_value_sequence(len(label_dict))]
```
相应的数据迭代器如下：
```
@provider(init_hook=hook, should_shuffle=True, calc_batch_size=get_batch_size,
can_over_batch_size=False, cache=CacheType.CACHE_PASS_IN_MEM)
def process(settings, file_name):
with open(file_name, 'r') as fdata:
for line in fdata:
sentence, predicate, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, mark, label = \
line.strip().split('\t')
words = sentence.split()
sen_len = len(words)
word_slot = [settings.word_dict.get(w, UNK_IDX) for w in words]
predicate_slot = [settings.predicate_dict.get(predicate)] * sen_len
ctx_n2_slot = [settings.word_dict.get(ctx_n2, UNK_IDX)] * sen_len
ctx_n1_slot = [settings.word_dict.get(ctx_n1, UNK_IDX)] * sen_len
ctx_0_slot = [settings.word_dict.get(ctx_0, UNK_IDX)] * sen_len
ctx_p1_slot = [settings.word_dict.get(ctx_p1, UNK_IDX)] * sen_len
ctx_p2_slot = [settings.word_dict.get(ctx_p2, UNK_IDX)] * sen_len
marks = mark.split()
mark_slot = [int(w) for w in marks]
label_list = label.split()
label_slot = [settings.label_dict.get(w) for w in label_list]
yield word_slot, predicate_slot, ctx_n2_slot, ctx_n1_slot, \
ctx_0_slot, ctx_p1_slot, ctx_p2_slot, mark_slot, label_slot
```
函数 `process` 返回8个特征list和1个标签list。
### 神经网络配置
`db_lstm.py` 是在训练过程中加载字典并定义数据提供程序模块和网络架构的神经网络配置文件。
九个 `data_layer` 从数据提供程序加载实例。八个特征分别转换为向量，并由`mixed_layer`混合。 深度双向LSTM层提取softmax层的特征。目标函数是标签的交叉熵。
### 训练
训练的脚本是 `train.sh`，用户只需执行:
```bash
./train.sh
```
`train.sh` 中的内容：
```
paddle train \
--config=./db_lstm.py \
--use_gpu=0 \
--log_period=5000 \
--trainer_count=1 \
--show_parameter_stats_period=5000 \
--save_dir=./output \
--num_passes=10000 \
--average_test_period=10000000 \
--init_model_path=./data \
--load_missing_parameter_strategy=rand \
--test_all_data_in_one_period=1 \
2>&1 | tee 'train.log'
```
- \--config=./db_lstm.py : 网络配置文件
- \--use_gpu=false: 使用 CPU 训练（如果已安装 PaddlePaddle GPU版本并想使用 GPU 训练可以设置为true，目前 crf_layer 不支持 GPU）
- \--log_period=500: 每20个batch输出日志
- \--trainer_count=1: 设置线程数（或 GPU 数）
- \--show_parameter_stats_period=5000: 每100个batch显示参数统计
- \--save_dir=./output: 模型输出路径
- \--num_passes=10000: 设置数据遍历次数，一个pass意味着PaddlePaddle训练数据集中的所有样本被遍历一次
- \--average_test_period=10000000: 每个 average_test_period 批次对平均参数进行测试
- \--init_model_path=./data: 参数初始化路径
- \--load_missing_parameter_strategy=rand: 随机初始不存在的参数
- \--test_all_data_in_one_period=1: 在一个周期内测试所有数据
训练后，模型将保存在目录`output`中。 我们的训练曲线如下：
<center>
![pic](./src/curve.jpg)
</center>
### 测试
测试脚本是 `test.sh`, 执行:
```bash
./test.sh
```
`tesh.sh` 的主要部分：
```
paddle train \
--config=./db_lstm.py \
--model_list=$model_list \
--job=test \
--config_args=is_test=1 \
```
- \--config=./db_lstm.py: 网络配置文件
- \--model_list=$model_list.list: 模型列表文件
- \--job=test: 指示测试任务
- \--config_args=is_test=1: 指示测试任务的标记
- \--test_all_data_in_one_period=1: 在一个周期内测试所有数据
### 预测
预测脚本是 `predict.sh`，用户只需执行：
```bash
./predict.sh
```
在`predict.sh`中，用户应该提供网络配置文件，模型路径，标签文件，字典文件，特征文件。
```
python predict.py
-c $config_file \
-w $best_model_path \
-l $label_file \
-p $predicate_dict_file \
-d $dict_file \
-i $input_file \
-o $output_file
```
`predict.py` 是主要的可执行python脚本，其中包括函数：加载模型，加载数据，数据预测。网络模型将输出标签的概率分布。 在演示中，我们使用最大概率的标签作为结果。用户还可以根据概率分布矩阵实现柱搜索或维特比解码。
预测后，结果保存在 `predict.res` 中。
## 引用
[1] Martha Palmer, Dan Gildea, and Paul Kingsbury. The Proposition Bank: An Annotated Corpus of Semantic Roles , Computational Linguistics, 31(1), 2005.
[2] Zhou, Jie, and Wei Xu. "End-to-end learning of semantic role labeling using recurrent neural networks." Proceedings of the Annual Meeting of the Association for Computational Linguistics. 2015.
doc/tutorials/semantic_role_labeling/index_en.md 已删除 100644 → 0
浏览文件 @ 62b56e76
```eval_rst
.. _semantic_role_labeling:
```
# Semantic Role labeling Tutorial #
Semantic role labeling (SRL) is a form of shallow semantic parsing whose goal is to discover the predicate-argument structure of each predicate in a given input sentence. SRL is useful as an intermediate step in a wide range of natural language processing tasks, such as information extraction. automatic document categorization and question answering. An instance is as following [1]:
[ <sub>A0</sub> He ] [ <sub>AM-MOD</sub> would ][ <sub>AM-NEG</sub> n’t ] [ <sub>V</sub> accept] [ <sub>A1</sub> anything of value ] from [<sub>A2</sub> those he was writing about ].
- V: verb
- A0: acceptor
- A1: thing accepted
- A2: accepted-from
- A3: Attribute
- AM-MOD: modal
- AM-NEG: negation
Given the verb "accept", the chunks in sentence would play certain semantic roles. Here, the label scheme is from Penn Proposition Bank.
To this date, most of the successful SRL systems are built on top of some form of parsing results where pre-defined feature templates over the syntactic structure are used. This tutorial will present an end-to-end system using deep bidirectional long short-term memory (DB-LSTM)[2] for solving the SRL task, which largely outperforms the previous state-of-the-art systems. The system regards SRL task as the sequence labelling problem.
## Data Description
The relevant paper[2] takes the data set in CoNLL-2005&2012 Shared Task for training and testing. Accordingto data license, the demo adopts the test data set of CoNLL-2005, which can be reached on website.
To download and process the original data, user just need to execute the following command:
```bash
cd data
./get_data.sh
```
Several new files appear in the `data `directory as follows.
```bash
conll05st-release：the test data set of CoNll-2005 shared task
test.wsj.words：the Wall Street Journal data sentences
test.wsj.props: the propositional arguments
feature: the extracted features from data set
```
## Training
### DB-LSTM
Please refer to the Sentiment Analysis demo to learn more about the long short-term memory unit.
Unlike Bidirectional-LSTM that used in Sentiment Analysis demo, the DB-LSTM adopts another way to stack LSTM layer. First a standard LSTM processes the sequence in forward direction. The input and output of this LSTM layer are taken by the next LSTM layer as input, processed in reversed direction. These two standard LSTM layers compose a pair of LSTM. Then we stack LSTM layers pair after pair to obtain the deep LSTM model.
The following figure shows a temporal expanded 2-layer DB-LSTM network.
<center>
![pic](./src/network_arch.png)
</center>
### Features
Two input features play an essential role in this pipeline: predicate (pred) and argument (argu). Two other features: predicate context (ctx-p) and region mark (mr) are also adopted. Because a single predicate word can not exactly describe the predicate information, especially when the same words appear more than one times in a sentence. With the predicate context, the ambiguity can be largely eliminated. Similarly, we use region mark m<sub>r</sub> = 1 to denote the argument position if it locates in the predicate context region, or m<sub>r</sub> = 0 if does not. These four simple features are all we need for our SRL system. Features of one sample with context size set to 1 is showed as following[2]:
<center>
![pic](./src/feature.jpg)
</center>
In this sample, the coresponding labelled sentence is:
[ <sub>A1</sub> A record date ] has [ <sub>AM-NEG</sub> n't ] been [ <sub>V</sub> set ] .
In the demo, we adopt the feature template as above, consists of : `argument`, `predicate`, `ctx-p (p=-1,0,1)`, `mark` and use `B/I/O` scheme to label each argument. These features and labels are stored in `feature` file, and separated by `\t`.
### Data Provider
`dataprovider.py` is the python file to wrap data. `hook()` function is to define the data slots for network. The Six features and label are all IndexSlots.
```
def hook(settings, word_dict, label_dict, **kwargs):
settings.word_dict = word_dict
settings.label_dict = label_dict
#all inputs are integral and sequential type
settings.slots = [
integer_value_sequence(len(word_dict)),
integer_value_sequence(len(predicate_dict)),
integer_value_sequence(len(word_dict)),
integer_value_sequence(len(word_dict)),
integer_value_sequence(len(word_dict)),
integer_value_sequence(len(word_dict)),
integer_value_sequence(len(word_dict)),
integer_value_sequence(2),
integer_value_sequence(len(label_dict))]
```
The corresponding data iterator is as following:
```
@provider(init_hook=hook, should_shuffle=True, calc_batch_size=get_batch_size,
can_over_batch_size=False, cache=CacheType.CACHE_PASS_IN_MEM)
def process(settings, file_name):
with open(file_name, 'r') as fdata:
for line in fdata:
sentence, predicate, ctx_n2, ctx_n1, ctx_0, ctx_p1, ctx_p2, mark, label = \
line.strip().split('\t')
words = sentence.split()
sen_len = len(words)
word_slot = [settings.word_dict.get(w, UNK_IDX) for w in words]
predicate_slot = [settings.predicate_dict.get(predicate)] * sen_len
ctx_n2_slot = [settings.word_dict.get(ctx_n2, UNK_IDX)] * sen_len
ctx_n1_slot = [settings.word_dict.get(ctx_n1, UNK_IDX)] * sen_len
ctx_0_slot = [settings.word_dict.get(ctx_0, UNK_IDX)] * sen_len
ctx_p1_slot = [settings.word_dict.get(ctx_p1, UNK_IDX)] * sen_len
ctx_p2_slot = [settings.word_dict.get(ctx_p2, UNK_IDX)] * sen_len
marks = mark.split()
mark_slot = [int(w) for w in marks]
label_list = label.split()
label_slot = [settings.label_dict.get(w) for w in label_list]
yield word_slot, predicate_slot, ctx_n2_slot, ctx_n1_slot, \
ctx_0_slot, ctx_p1_slot, ctx_p2_slot, mark_slot, label_slot
```
The `process`function yield 9 lists which are 8 features and label.
### Neural Network Config
`db_lstm.py` is the neural network config file to load the dictionaries and define the data provider module and network architecture during the training procedure.
Nine `data_layer` load instances from data provider. Eight features are transformed into embedddings respectively, and mixed by `mixed_layer` . Deep bidirectional LSTM layers extract features for the softmax layer. The objective function is cross entropy of labels.
### Run Training
The script for training is `train.sh`, user just need to execute:
```bash
./train.sh
```
The content in `train.sh`:
```
paddle train \
--config=./db_lstm.py \
--use_gpu=0 \
--log_period=5000 \
--trainer_count=1 \
--show_parameter_stats_period=5000 \
--save_dir=./output \
--num_passes=10000 \
--average_test_period=10000000 \
--init_model_path=./data \
--load_missing_parameter_strategy=rand \
--test_all_data_in_one_period=1 \
2>&1 | tee 'train.log'
```
- \--config=./db_lstm.py : network config file.
- \--use_gpu=false: use CPU to train, set true, if you install GPU version of PaddlePaddle and want to use GPU to train, until now crf_layer do not support GPU
- \--log_period=500: print log every 20 batches.
- \--trainer_count=1: set thread number (or GPU count).
- \--show_parameter_stats_period=5000: show parameter statistic every 100 batches.
- \--save_dir=./output: output path to save models.
- \--num_passes=10000: set pass number, one pass in PaddlePaddle means training all samples in dataset one time.
- \--average_test_period=10000000: do test on average parameter every average_test_period batches
- \--init_model_path=./data: parameter initialization path
- \--load_missing_parameter_strategy=rand: random initialization unexisted parameters
- \--test_all_data_in_one_period=1: test all data in one period
After training, the models will be saved in directory `output`. Our training curve is as following:
<center>
![pic](./src/curve.jpg)
</center>
### Run testing
The script for testing is `test.sh`, user just need to execute:
```bash
./test.sh
```
The main part in `tesh.sh`
```
paddle train \
--config=./db_lstm.py \
--model_list=$model_list \
--job=test \
--config_args=is_test=1 \
```
- \--config=./db_lstm.py: network config file
- \--model_list=$model_list.list: model list file
- \--job=test: indicate the test job
- \--config_args=is_test=1: flag to indicate test
- \--test_all_data_in_one_period=1: test all data in 1 period
### Run prediction
The script for prediction is `predict.sh`, user just need to execute:
```bash
./predict.sh
```
In `predict.sh`, user should offer the network config file, model path, label file, word dictionary file, feature file
```
python predict.py
-c $config_file \
-w $best_model_path \
-l $label_file \
-p $predicate_dict_file \
-d $dict_file \
-i $input_file \
-o $output_file
```
`predict.py` is the main executable python script, which includes functions: load model, load data, data prediction. The network model will output the probability distribution of labels. In the demo, we take the label with maximum probability as result. User can also implement the beam search or viterbi decoding upon the probability distribution matrix.
After prediction, the result is saved in `predict.res`.
## Reference
[1] Martha Palmer, Dan Gildea, and Paul Kingsbury. The Proposition Bank: An Annotated Corpus of Semantic Roles , Computational Linguistics, 31(1), 2005.
[2] Zhou, Jie, and Wei Xu. "End-to-end learning of semantic role labeling using recurrent neural networks." Proceedings of the Annual Meeting of the Association for Computational Linguistics. 2015.
doc/tutorials/sentiment_analysis/index_cn.md 已删除 100644 → 0
浏览文件 @ 62b56e76
# 情感分析教程
情感分析有许多应用场景。 一个基本的应用场景是区分给定文本的褒贬两极性，给定的文本可以是一个文档、句子、或者是一个小的文本片段。 一个简单的例子如：把用户在购物网站、旅游网站、团购网站（亚马逊、天猫、淘宝等）上发表的评论分成正面评论和负面评论两类。
情感分析也常用于基于大量评论和个人博客来监控社会媒体。 例如，研究人员分析了几个关于消费者信心和政治观点的调查，结果发现它们与同时期的Twitter消息中的情绪词频率相关 [1]。 另一个例子是通过分析每日Twitter博客的文本内容来预测股票变动 [2]。
另一方面，抓取产品的用户评论并分析他们的情感，有助于理解用户对不同公司，不同产品，甚至不同竞争对手产品的偏好。
本教程将指导您完成长期短期记忆（LSTM）网络的训练过程，以分类来自[大型电影评论数据集](http://ai.stanford.edu/~amaas/data/sentiment/)（有时称为[互联网电影数据库 (IMDB)](http://ai.stanford.edu/~amaas/papers/wvSent_acl2011.pdf)）的句子的情感 。 此数据集包含电影评论及其相关联的类别标签，即正面和负面。
## 数椐准备
### IMDB 数椐介绍
训练模型之前, 我们需要预处理数椐并构建一个字典。 首先, 你可以使用下面的脚本下载 IMDB 数椐集和[Moses](http://www.statmt.org/moses/)工具, 这是一个基于统计的机器翻译系统. 我们提供了一个数据预处理脚本，它不仅能够处理IMDB数据，还能处理其他用户自定义的数据。 为了使用提前编写的脚本，需要将标记的训练和测试样本移动到另一个路径，这已经在`get_imdb.sh`中完成。
```
cd demo/sentiment/data
./get_imdb.sh
```
如果数椐获取成功，你将在目录```./demo/sentiment/data```中看到下面的文件：
```
aclImdb get_imdb.sh imdb mosesdecoder-master
```
* aclImdb: 从外部网站上下载的原始数椐集。
* imdb: 仅包含训练和测试数椐集。
* mosesdecoder-master: Moses 工具。
IMDB数据集包含25,000个已标注过的高极性电影评论用于训练，25,000个用于测试。负面的评论的得分小于等于4,正面的评论的得大于等于7，总评分10分。 运行完脚本 `./get_imdb.sh`后, 我们可以看到在目录 `aclImdb`中的数椐集的结构如下：
```
imdbEr.txt imdb.vocab README test train
```
* train: 训练数椐集。
* test : 测试数椐集。
* imdb.vocab: 字典文件。
* imdbEr.txt: 字典imdb.vocab中每个切分单词的预期评级。
* README: 数椐说明文档。
测试集和训练集目录包含下面的文件:
```
labeledBow.feat neg pos unsup unsupBow.feat urls_neg.txt urls_pos.txt urls_unsup.txt
```
* pos: 正面评价样本，包含12,500个txt文件，每个文件是一个电影评论。
* neg: 负面评价样本，包含12,500个txt文件，每个文件是一个电影评论。
* unsup: 未标记的评价样本，包含50,000个txt文件。
* urls_xx.txt: 每个评论的网址。
* xxBow.feat: 用于统计词频的Bow模型特征。
### IMDB 数椐准备
在这个例子中，我们只使用已经标注过的训练集和测试集，且默认在训练集上构建字典，而不使用IMDB数椐集中的imdb.vocab做为字典。训练集已经做了随机打乱排序而测试集没有。 Moses 工具中的脚本`tokenizer.perl` 用于切分单单词和标点符号。执行下面的命令就可以预处理数椐。
```
cd demo/sentiment/
./preprocess.sh
```
preprocess.sh:
```
data_dir="./data/imdb"
python preprocess.py -i data_dir
```
* data_dir: 输入数椐所在目录。
* preprocess.py: 预处理脚本。
运行成功后目录`demo/sentiment/data/pre-imdb` 结构如下:
```
dict.txt labels.list test.list test_part_000 train.list train_part_000
```
* test\_part\_000 and train\_part\_000: 所有标记的测试集和训练集， 训练集已经随机打乱。
* train.list and test.list: 训练集和测试集文件列表。
* dict.txt: 利用训练集生成的字典。
* labels.txt: neg 0, pos 1, 含义：标签0表示负面的评论，标签1表示正面的评论。
### 用户自定义数椐预处理
如果你执行其它的用情感分析来分类文本的任务，可以按如下的结构来准备数椐. 我们提供了脚本来构建字典和预处理数椐。所以你只用按下面的结构来组织数椐就行了。
```
dataset
|----train
| |----class1
| | |----text_files
| |----class2
| | |----text_files
| | ...
|----test
| |----class1
| | |----text_files
| |----class2
| | |----text_files
| | ...
```
* dataset: 一级目录。
* train, test: 二级目录。
* class1,class2,...: 三级目录。
* text_files: 文本格式的实例文件。
所有同目录下的文本实例文件都是同级别的。 每个文本文件包含一个或者多个实例，每一行表示一个实例。 为了充分的随机打乱训练集, 在预处理含有多行数椐的文本文件时参数设置稍有不同, 执行`preprocess.sh`脚本时需要加上`-m True`参数。 tokenizer.perl 默认用来切分单记和标点符号，如果你不需要这个操作，在运行`preprocess.sh`时加上`-t False`参数即可。
## 训练模型
在这步任务中,我们使用了循环神经网络（RNN）的 LSTM 架构来训练情感分析模型。 引入LSTM模型主要是为了克服消失梯度的问题。 LSTM网络类似于具有隐藏层的标准循环神经网络, 但是隐藏层中的每个普通节点被一个记忆单元替换。 每个记忆单元包含四个主要的元素: 输入门, 具有自循环连接的神经元，忘记门和输出门。 更多的细节可以在文献中找到[4]。 LSTM架构的最大优点是它可以在长时间间隔内记忆信息，而没有短时记忆的损失。在有新的单词来临的每一个时间步骤内，存储在记忆单元区块的历史信息被更新用来迭代的学习单词以合理的序列程现。
<center>![LSTM](src/lstm.png)</center>
<center>图表 1. LSTM [3]</center>
情感分析是自然语言理解中最典型的问题之一。 它的目的是预测在一个序列中表达的情感态度。 通常, ，仅仅是一些关键词，如形容词和副词，在预测序列或段落的情感中起主要作用。然而有些评论上下文非常长，例如 IMDB的数椐集。 我们只所以使用LSTM来执行这个任务是因为其改进的设计并且具有门机制。 首先，它能够从词级到具有可变上下文长度的上下文级别来总结表示。 第二，它可以在句子级别利用可扩展的上下文, 而大多数方法只是利用n-gram级别的知识。第三，它直接学习段落表示，而不是组合上下文级别信息。
在本演示中，我们提供两个网络，即双向LSTM和三层堆叠LSTM。
#### 双向LSTM
图2是双向LSTM网络，后面连全连接层和softmax层。
<center>![BiLSTM](src/bi_lstm.jpg)</center>
<center>图 2. Bidirectional-LSTM </center>
#### Stacked-LSTM
图3是三层LSTM结构。图的底部是word embedding(对文档处理后形成的单词向量)。 接下来，连接三个LSTM隐藏层，并且第二个是反向LSTM。然后提取隐藏LSTM层的所有时间步长的最大词向量作为整个序列的表示。 最后，使用具有softmax激活的全连接前馈层来执行分类任务。 更多内容可查看参考文献 [5]。
<center>![StackedLSTM](src/stacked_lstm.jpg)</center>
<center>图 3. Stacked-LSTM for sentiment analysis </center>
**配置**
进入`demo/sentiment` 目录 , `trainer_config.py` 是一个配置文件的例子, 其中包含算法和网络配置。第一行从`sentiment_net.py`中导出预定义的网络。
trainer_config.py:
```python
from sentiment_net import *
data_dir = "./data/pre-imdb"
# whether this config is used for test
is_test = get_config_arg('is_test', bool, False)
# whether this config is used for prediction
is_predict = get_config_arg('is_predict', bool, False)
dict_dim, class_dim = sentiment_data(data_dir, is_test, is_predict)
################## Algorithm Config #####################
settings(
batch_size=128,
learning_rate=2e-3,
learning_method=AdamOptimizer(),
regularization=L2Regularization(8e-4),
gradient_clipping_threshold=25
)
#################### Network Config ######################
stacked_lstm_net(dict_dim, class_dim=class_dim,
stacked_num=3, is_predict=is_predict)
#bidirectional_lstm_net(dict_dim, class_dim=class_dim, is_predict=is_predict)
```
* **数椐定义**:
* get\_config\_arg(): 获取通过 `--config_args=xx` 设置的命令行参数。
* 定义训练数椐和测试数椐提供者, 这里使用了PaddlePaddle的Python接口来加载数椐。想了解更多细节可以参考PyDataProvider部分的文档
* **算法配置**:
* 使用随机梯度下降（sgd）算法。
* 使用 adam 优化。
* 设置batch size大小为128。
* 设置平均sgd窗口。
* 设置全局学习率。
* **网络配置**:
* dict_dim: 获取字典维度。
* class_dim: 设置类别数，IMDB有两个标签，即正面评价标签和负面评价标签。
* `stacked_lstm_net`: 预定义网络如图3所示，默认情况下使用此网络
* `bidirectional_lstm_net`: 预定义网络，如图2所示。
**训练**
首先安装PaddlePaddle。 然后使用下面的脚本 `train.sh` 来开启本地的训练。
```
cd demo/sentiment/
./train.sh
```
train.sh:
```
config=trainer_config.py
output=./model_output
paddle train --config=$config \
--save_dir=$output \
--job=train \
--use_gpu=false \
--trainer_count=4 \
--num_passes=10 \
--log_period=20 \
--dot_period=20 \
--show_parameter_stats_period=100 \
--test_all_data_in_one_period=1 \
2>&1 | tee 'train.log'
```
* \--config=$config: 设置网络配置。
* \--save\_dir=$output: 设置输出路径以保存训练完成的模型。
* \--job=train: 设置工作模式为训练。
* \--use\_gpu=false: 使用CPU训练，如果你安装GPU版本的PaddlePaddle，并想使用GPU来训练设置为true。
* \--trainer\_count=4:设置线程数（或GPU个数）。
* \--num\_passes=15: 设置pass，PaddlePaddle中的一个pass意味着对数据集中的所有样本进行一次训练。
* \--log\_period=20: 每20个batch打印一次日志。
* \--show\_parameter\_stats\_period=100: 每100个batch打印一次统计信息。
* \--test\_all_data\_in\_one\_period=1: 每次测试都测试所有数据。
如果运行成功，输出日志保存在路径 `demo/sentiment/train.log`中，模型保存在目录`demo/sentiment/model_output/`中。 输出日志说明如下：
```
Batch=20 samples=2560 AvgCost=0.681644 CurrentCost=0.681644 Eval: classification_error_evaluator=0.36875 CurrentEval: classification_error_evaluator=0.36875
...
Pass=0 Batch=196 samples=25000 AvgCost=0.418964 Eval: classification_error_evaluator=0.1922
Test samples=24999 cost=0.39297 Eval: classification_error_evaluator=0.149406
```
- Batch=xx: 表示训练了xx个Batch。
- samples=xx: 表示训练了xx个样本。。
- AvgCost=xx: 从第0个batch到当前batch的平均损失。
- CurrentCost=xx: 最新log_period个batch处理的当前损失。
- Eval: classification\_error\_evaluator=xx: 表示第0个batch到当前batch的分类错误。
- CurrentEval: classification\_error\_evaluator: 最新log_period个batch的分类错误。
- Pass=0: 通过所有训练集一次称为一遍。 0表示第一次经过训练集。
默认情况下，我们使用`stacked_lstm_net`网络，当传递相同的样本数时，它的收敛速度比`bidirectional_lstm_net`快。如果要使用双向LSTM，只需删除最后一行中的注释并把“stacked_lstm_net”注释掉。
## 测试模型
测试模型是指使用训练出的模型评估已标记的验证集。
```
cd demo/sentiment
./test.sh
```
test.sh:
```bash
function get_best_pass() {
cat $1 | grep -Pzo 'Test .*\n.*pass-.*' | \
sed -r 'N;s/Test.* error=([0-9]+\.[0-9]+).*\n.*pass-([0-9]+)/\1 \2/g' | \
sort | head -n 1
}
log=train.log
LOG=`get_best_pass $log`
LOG=(${LOG})
evaluate_pass="model_output/pass-${LOG[1]}"
echo 'evaluating from pass '$evaluate_pass
model_list=./model.list
touch $model_list | echo $evaluate_pass > $model_list
net_conf=trainer_config.py
paddle train --config=$net_conf \
--model_list=$model_list \
--job=test \
--use_gpu=false \
--trainer_count=4 \
--config_args=is_test=1 \
2>&1 | tee 'test.log'
```
函数`get_best_pass`依据分类错误率获得最佳模型进行测试。 在本示例中，我们默认使用IMDB的测试数据集作为验证。 与训练不同，它需要在这里指定`--job = test`和模型路径，即`--model_list = $model_list`。如果运行成功，日志将保存在“demo / sentiment / test.log”的路径中。例如，在我们的测试中，最好的模型是`model_output / pass-00002`，分类误差是0.115645，如下：
```
Pass=0 samples=24999 AvgCost=0.280471 Eval: classification_error_evaluator=0.115645
```
## 预测
`predict.py`脚本提供了一个预测接口。在使用它之前请安装PaddlePaddle的python api。 预测IMDB的未标记评论的一个实例如下：
```
cd demo/sentiment
./predict.sh
```
predict.sh:
```
#Note the default model is pass-00002, you shold make sure the model path
#exists or change the mode path.
model=model_output/pass-00002/
config=trainer_config.py
label=data/pre-imdb/labels.list
cat ./data/aclImdb/test/pos/10007_10.txt | python predict.py \
--tconf=$config\
--model=$model \
--label=$label \
--dict=./data/pre-imdb/dict.txt \
--batch_size=1
```
* `cat ./data/aclImdb/test/pos/10007_10.txt` : 输入预测样本。
* `predict.py` : 预测接口脚本。
* `--tconf=$config` : 设置网络配置。
* `--model=$model` : 设置模型路径。
* `--label=$label` : 设置标签类别字典，这个字典是整数标签和字符串标签的一个对应。
* `--dict=data/pre-imdb/dict.txt` : 设置字典文件。
* `--batch_size=1` : 设置batch size。
注意应该确保默认模型路径`model_output / pass-00002`存在或更改为其它模型路径。
本示例的预测结果：
```
Loading parameters from model_output/pass-00002/
./data/aclImdb/test/pos/10014_7.txt: predicting label is pos
```
我们真诚地感谢您的关注，并欢迎您来参与贡献。
## 参考文档
[1] Brendan O'Connor, Ramnath Balasubramanyan, Bryan R. Routledge, and Noah A. Smith. 2010. [From Tweets to Polls: Linking Text Sentiment to Public Opinion Time Series](http://homes.cs.washington.edu/~nasmith/papers/oconnor+balasubramanyan+routledge+smith.icwsm10.pdf). In ICWSM-2010. <br>
[2] Johan Bollen, Huina Mao, Xiaojun Zeng. 2011. [Twitter mood predicts the stock market](http://arxiv.org/abs/1010.3003), Journal of Computational Science.<br>
[3] Alex Graves, Marcus Liwicki, Santiago Fernan- dez, Roman Bertolami, Horst Bunke, and Ju ̈rgen Schmidhuber. 2009. [A novel connectionist system for unconstrained handwriting recognition. IEEE Transactions on Pattern Analysis and Machine In- telligence](http://www.cs.toronto.edu/~graves/tpami_2009.pdf), 31(5):855–868.<br>
[4] Zachary C. Lipton, [A Critical Review of Recurrent Neural Networks for Sequence Learning](http://arxiv.org/abs/1506.00019v1), arXiv:1506.00019. <br>
[5] Jie Zhou and Wei Xu; [End-to-end Learning of Semantic Role Labeling Using Recurrent Neural Networks](http://www.aclweb.org/anthology/P/P15/P15-1109.pdf); ACL-IJCNLP 2015. <br>
doc/tutorials/sentiment_analysis/index_en.md 已删除 100644 → 0
浏览文件 @ 62b56e76
此差异已折叠。
doc/tutorials/text_generation/index_cn.md 已删除 100644 → 0
浏览文件 @ 62b56e76
此差异已折叠。
doc/tutorials/text_generation/index_en.md 已删除 100644 → 0
浏览文件 @ 62b56e76
此差异已折叠。
doc/v1_api_tutorials/README.md 0 → 100644
浏览文件 @ 745634a1
The tutorials in v1_api_tutorials are using v1_api currently, and will be upgraded to v2_api later.
Thus, v1_api_tutorials is a temporary directory. We decide not to maintain it and will delete it in future.
Please go to [PaddlePaddle/book](https://github.com/PaddlePaddle/book) and
[PaddlePaddle/models](https://github.com/PaddlePaddle/models) to learn PaddlePaddle.
go/glide.lock
浏览文件 @ 745634a1
此差异已折叠。
go/glide.yaml
浏览文件 @ 745634a1
此差异已折叠。
paddle/CMakeLists.txt
浏览文件 @ 745634a1
此差异已折叠。
paddle/api/CMakeLists.txt
浏览文件 @ 745634a1
此差异已折叠。
paddle/api/Util.cpp
浏览文件 @ 745634a1
此差异已折叠。
paddle/capi/CMakeLists.txt
浏览文件 @ 745634a1
此差异已折叠。
paddle/capi/Matrix.cpp
浏览文件 @ 745634a1
此差异已折叠。
paddle/capi/tests/CMakeLists.txt
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/CMakeLists.txt
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/attribute.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/backward.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/backward.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/backward_test.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/block_desc.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/block_desc.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/data_type.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/details/op_registry.h 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/executor.cc 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/executor.h 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/feed_fetch_method.h 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/feed_fetch_type.h 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/framework.proto
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/grad_op_builder.cc 已删除 100644 → 0
浏览文件 @ 62b56e76
此差异已折叠。
paddle/framework/grad_op_builder_test.cc 已删除 100644 → 0
浏览文件 @ 62b56e76
此差异已折叠。
paddle/framework/grad_op_desc_maker.h 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/lod_tensor.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/lod_tensor.md
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/op_desc.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/op_desc.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/op_info.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/op_proto_maker.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/op_registry.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/op_registry.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/operator.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/operator.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/operator_test.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/program_desc.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/program_desc.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/scope.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/scope.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/scope_test.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/selected_rows.h 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/selected_rows_test.cc 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/shape_inference.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/tensor.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/tensor_array.cc 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/tensor_array.h 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/tensor_array_test.cc 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/tensor_impl.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/tensor_test.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/type_defs.h 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/var_desc.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/var_desc.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/var_type_inference.h 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/framework/var_type_inference_test.cc 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/function/BlockExpandOp.cpp
浏览文件 @ 745634a1
此差异已折叠。
paddle/function/CosSimOp.cpp
浏览文件 @ 745634a1
此差异已折叠。
paddle/function/CropOp.cpp
浏览文件 @ 745634a1
此差异已折叠。
paddle/function/DepthwiseConvOp.cpp
浏览文件 @ 745634a1
此差异已折叠。
paddle/function/GemmConvOp.cpp
浏览文件 @ 745634a1
此差异已折叠。
paddle/function/GemmConvOpTest.cpp
浏览文件 @ 745634a1
此差异已折叠。
paddle/function/Im2ColTest.cpp
浏览文件 @ 745634a1
此差异已折叠。
paddle/function/MulOp.cpp
浏览文件 @ 745634a1
此差异已折叠。
paddle/function/PadOp.cpp
浏览文件 @ 745634a1
此差异已折叠。
paddle/function/RowConvOp.cpp
浏览文件 @ 745634a1
此差异已折叠。
paddle/function/SwitchOp.cpp
浏览文件 @ 745634a1
此差异已折叠。
paddle/gserver/CMakeLists.txt
浏览文件 @ 745634a1
此差异已折叠。
paddle/gserver/layers/CostLayer.cpp
浏览文件 @ 745634a1
此差异已折叠。
paddle/gserver/layers/Layer.cpp
浏览文件 @ 745634a1
此差异已折叠。
paddle/gserver/layers/Layer.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/gserver/layers/MKLDNNLayer.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/gserver/layers/PoolLayer.cpp
浏览文件 @ 745634a1
此差异已折叠。
paddle/gserver/tests/CMakeLists.txt
浏览文件 @ 745634a1
此差异已折叠。
paddle/math/Matrix.cpp
浏览文件 @ 745634a1
此差异已折叠。
paddle/math/SparseMatrix.cpp
浏览文件 @ 745634a1
此差异已折叠。
paddle/math/Vector.cpp
浏览文件 @ 745634a1
此差异已折叠。
paddle/math/tests/test_Matrix.cpp
浏览文件 @ 745634a1
此差异已折叠。
paddle/math/tests/test_Tensor.cu
浏览文件 @ 745634a1
此差异已折叠。
paddle/memory/memcpy.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/memory/memcpy.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/memory/memory.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/memory/memory_test.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/CMakeLists.txt
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/accuracy_op.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/activation_op.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/activation_op.cu
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/activation_op.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/adadelta_op.cc 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/adadelta_op.cu 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/adadelta_op.h 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/adagrad_op.cc 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/adagrad_op.cu 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/adagrad_op.h 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/adam_op.cc 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/adam_op.cu 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/adam_op.h 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/adamax_op.cc 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/adamax_op.h 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/clip_op.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/concat_op.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/cond_op.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/cond_op.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/conv2d_op.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/conv2d_op.cu
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/conv_cudnn_op.cc 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/conv_cudnn_op.cu 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/conv_shift_op.cc 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/conv_shift_op.cu 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/conv_shift_op.h 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/cos_sim_op.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/crop_op.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/cross_entropy_op.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/decayed_adagrad_op.cc 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/decayed_adagrad_op.cu 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/decayed_adagrad_op.h 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/dropout_op.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/dynamic_recurrent_op.cc 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/dynamic_recurrent_op.h 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/dynamic_recurrent_op_test.cc 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/elementwise_op.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/feed_op.cc 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/fetch_op.cc 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/fill_constant_op.cu 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/gather.cu.h 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/gather.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/gather_op.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/gather_op.cu 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/gather_op.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/gather_test.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/gru_unit_op.cc 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/gru_unit_op.cu 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/gru_unit_op.h 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/interp_op.cc 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/lookup_table_op.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/lstm_unit_op.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/lstm_unit_op.cu
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/lstm_unit_op.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/margin_rank_loss_op.cc 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/margin_rank_loss_op.cu 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/margin_rank_loss_op.h 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/math/math_function_test.cu 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/math/pooling.cc 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/math/pooling.cu 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/math/pooling.h 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/math/selected_rows_functor.cc 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/math/selected_rows_functor.cu 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/math/selected_rows_functor.h 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/math/selected_rows_functor_test.cc 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/math/selected_rows_functor_test.cu 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/math/vol2col.cc 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/math/vol2col.cu 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/math/vol2col.h 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/math/vol2col_test.cc 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/mean_op.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/minus_op.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/mul_op.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/multiplex_op.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/multiplex_op.cu
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/name_convention.md
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/net_op.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/pad_op.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/pool_op.cc 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/pool_op.cu 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/pool_op.h 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/pool_with_index_op.cc 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/pool_with_index_op.cu 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/pool_with_index_op.h 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/prelu_op.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/rank_loss_op.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/recurrent_op.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/recurrent_op.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/reduce_op.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/reduce_op.cu
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/reduce_op.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/reshape_op.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/reshape_op.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/rmsprop_op.cc 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/rmsprop_op.cu 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/rmsprop_op.h 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/scale_op.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/scatter.cu.h 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/scatter.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/scatter_op.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/scatter_op.cu 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/scatter_op.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/scatter_test.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/sequence_concat_op.cc 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/sequence_concat_op.cu 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/sequence_concat_op.h 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/sequence_pool_op.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/sgd_op.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/sgd_op.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/softmax_op.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/split_op.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/sum_op.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/sum_op.cu
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/sum_op.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/top_k_op.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/operators/transpose_op.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/platform/cudnn_helper.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/platform/device_context.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/platform/device_context.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/platform/enforce.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/platform/enforce_test.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/platform/gpu_info.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/platform/gpu_info.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/platform/hostdevice.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/platform/macros.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/platform/variant.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/pserver/LightNetwork.cpp
浏览文件 @ 745634a1
此差异已折叠。
paddle/pserver/test/SocketTest.cpp
浏览文件 @ 745634a1
此差异已折叠。
paddle/pybind/CMakeLists.txt
浏览文件 @ 745634a1
此差异已折叠。
paddle/pybind/protobuf.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/pybind/pybind.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/pybind/tensor_py.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/string/to_string_test.cc
浏览文件 @ 745634a1
此差异已折叠。
paddle/trainer/MergeModel.cpp
浏览文件 @ 745634a1
此差异已折叠。
paddle/trainer/tests/CMakeLists.txt
浏览文件 @ 745634a1
此差异已折叠。
paddle/utils/Flags.cpp
浏览文件 @ 745634a1
此差异已折叠。
paddle/utils/Util.h
浏览文件 @ 745634a1
此差异已折叠。
paddle/utils/Version.h
浏览文件 @ 745634a1
此差异已折叠。
proto/CMakeLists.txt
浏览文件 @ 745634a1
此差异已折叠。
python/paddle/v2/event.py
浏览文件 @ 745634a1
此差异已折叠。
python/paddle/v2/framework/framework.py 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
python/paddle/v2/framework/layer_helper.py 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
python/paddle/v2/framework/layers.py 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
python/paddle/v2/framework/op.py
浏览文件 @ 745634a1
此差异已折叠。
python/paddle/v2/framework/tests/test_adadelta_op.py 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
python/paddle/v2/framework/tests/test_adagrad_op.py 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
python/paddle/v2/framework/tests/test_adam_op.py 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
python/paddle/v2/framework/tests/test_adamax_op.py 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
python/paddle/v2/framework/tests/test_conv_shift_op.py 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
python/paddle/v2/framework/tests/test_gru_unit_op.py 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
python/paddle/v2/framework/tests/test_infer_shape.py 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
python/paddle/v2/framework/tests/test_interp_op.py 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
python/paddle/v2/framework/tests/test_layers.py 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
python/paddle/v2/framework/tests/test_operator_desc.py 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
python/paddle/v2/framework/tests/test_parameter.py 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
python/paddle/v2/framework/tests/test_pool2d_op.py 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
python/paddle/v2/framework/tests/test_pool3d_op.py 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
python/paddle/v2/framework/tests/test_pool_max_op.py 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
python/paddle/v2/framework/tests/test_program.py 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
python/paddle/v2/framework/tests/test_rmsprop_op.py 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
python/paddle/v2/framework/tests/test_selected_rows.py 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
python/paddle/v2/framework/tests/test_seq_concat_op.py 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
python/paddle/v2/framework/tests/test_tensor_array.py 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
python/paddle/v2/framework/tests/test_variable.py 0 → 100644
浏览文件 @ 745634a1
此差异已折叠。
python/paddle/v2/trainer.py
浏览文件 @ 745634a1
此差异已折叠。
v1_api_demo/README.md
浏览文件 @ 745634a1
此差异已折叠。
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或