diff --git a/.travis.yml b/.travis.yml index a406841f6abf01f15826f34fe4c63b4c24486ccd..361136ac2c8d899a0d7a4d7945083fcc489551b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,15 +27,6 @@ script: # 43min timeout paddle/scripts/paddle_docker_build.sh ${JOB} if [ $? -eq 0 ] || [ $? -eq 142 ]; then true; else exit 1; fi; - - | - if [[ "$JOB" != "doc" ]]; then exit 0; fi; - # For document only - if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then exit 0; fi; - if [[ "$TRAVIS_BRANCH" != "develop" && ! "$TRAVIS_BRANCH" =~ ^v|release/[[:digit:]]+\.[[:digit:]]+(\.[[:digit:]]+)?(-\S*)?$ ]]; then exit 0; fi; - export DEPLOY_DOCS_SH=https://raw.githubusercontent.com/PaddlePaddle/PaddlePaddle.org/master/scripts/deploy/deploy_docs.sh - export DOCS_DIR=`pwd` - cd .. - curl $DEPLOY_DOCS_SH | bash -s $CONTENT_DEC_PASSWD $TRAVIS_BRANCH $DOCS_DIR $DOCS_DIR/build/doc/ notifications: email: on_success: change diff --git a/CMakeLists.txt b/CMakeLists.txt index bdd48565edeca051f54e8fe4eb51cd1dbd5e836a..f2514d44acf230762cf8a2d2d40e5d715c0ca9fb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -65,6 +65,7 @@ option(REPLACE_ENFORCE_GLOG "Replace PADDLE_ENFORCE with glog/CHECK for better d option(WITH_ANAKIN "Compile with Anakin library" OFF) option(WITH_GRPC "Use grpc as the default rpc framework" ${WITH_DISTRIBUTE}) option(WITH_BRPC_RDMA "Use brpc rdma as the rpc protocal" OFF) +option(WITH_INFERENCE "Compile fluid inference library" ON) option(WITH_SYSTEM_BLAS "Use system blas library" OFF) option(PY_VERSION "Compile PaddlePaddle with python3 support" ${PY_VERSION}) @@ -72,6 +73,7 @@ option(PY_VERSION "Compile PaddlePaddle with python3 support" ${PY_VER if(NOT PY_VERSION) set(PY_VERSION 2.7) endif() +set(PYBIND11_PYTHON_VERSION ${PY_VERSION}) # CMAKE_BUILD_TYPE if(NOT CMAKE_BUILD_TYPE) @@ -174,6 +176,7 @@ include(external/any) # download libn::any include(external/eigen) # download eigen3 include(external/pybind11) # download pybind11 include(external/cares) +include(external/cub) if(WITH_DISTRIBUTE) if(WITH_GRPC) diff --git a/cmake/external/cub.cmake b/cmake/external/cub.cmake new file mode 100644 index 0000000000000000000000000000000000000000..c94849cf4b96746e6c507db2a6310c2f305dacf5 --- /dev/null +++ b/cmake/external/cub.cmake @@ -0,0 +1,35 @@ +if(NOT WITH_GPU) + return() +endif() + +include(ExternalProject) + +set(CUB_SOURCE_DIR ${THIRD_PARTY_PATH}/cub) +set(CUB_INCLUDE_DIR ${CUB_SOURCE_DIR}/src/extern_cub) + +include_directories(${CUB_INCLUDE_DIR}) + +ExternalProject_Add( + extern_cub + ${EXTERNAL_PROJECT_LOG_ARGS} + GIT_REPOSITORY "https://github.com/NVlabs/cub.git" + GIT_TAG "v1.8.0" + PREFIX ${CUB_SOURCE_DIR} + UPDATE_COMMAND "" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +) + +if(${CMAKE_VERSION} VERSION_LESS "3.3.0") + set(dummyfile ${CMAKE_CURRENT_BINARY_DIR}/cub_dummy.c) + file(WRITE ${dummyfile} "const char *dummy = \"${dummyfile}\";") + add_library(cub STATIC ${dummyfile}) +else() + add_library(cub INTERFACE) +endif() + +add_dependencies(cub extern_cub) + +LIST(APPEND externl_project_dependencies cub) diff --git a/cmake/generic.cmake b/cmake/generic.cmake index 07bab994d354df834d0667c69f307b2d7684fb22..82c958073cba92f00a341121e36ba45531b22aec 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -264,7 +264,10 @@ function(cc_test TARGET_NAME) WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) if (${cc_test_SERIAL}) set_property(TEST ${TARGET_NAME} PROPERTY RUN_SERIAL 1) + + set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT FLAGS_cpu_deterministic=true) set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT FLAGS_init_allocated_mem=true) + set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT FLAGS_cudnn_deterministic=true) endif() endif() endfunction(cc_test) @@ -329,7 +332,10 @@ function(nv_test TARGET_NAME) add_test(${TARGET_NAME} ${TARGET_NAME}) if (nv_test_SERIAL) set_property(TEST ${TARGET_NAME} PROPERTY RUN_SERIAL 1) + + set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT FLAGS_cpu_deterministic=true) set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT FLAGS_init_allocated_mem=true) + set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT FLAGS_cudnn_deterministic=true) endif() endif() endfunction(nv_test) @@ -577,7 +583,9 @@ function(py_test TARGET_NAME) set(multiValueArgs SRCS DEPS ARGS ENVS) cmake_parse_arguments(py_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) add_test(NAME ${TARGET_NAME} - COMMAND env FLAGS_init_allocated_mem=true PYTHONPATH=${PADDLE_BINARY_DIR}/python ${py_test_ENVS} + COMMAND env FLAGS_init_allocated_mem=true FLAGS_cudnn_deterministic=true + FLAGS_cpu_deterministic=true + PYTHONPATH=${PADDLE_BINARY_DIR}/python ${py_test_ENVS} ${PYTHON_EXECUTABLE} -u ${py_test_SRCS} ${py_test_ARGS} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) endif() diff --git a/doc/survey/op_fusion_design.md b/doc/survey/op_fusion_design.md new file mode 100644 index 0000000000000000000000000000000000000000..d6e48f4f58269b67450cb012f6dcc59e1083abba --- /dev/null +++ b/doc/survey/op_fusion_design.md @@ -0,0 +1,20 @@ +# Operator fusion +Fusing multiple operators together is an important method to optimize the program execution, particularly for GPU or other specialized accelerators. An obvious benefit is to avoid the overhead of saving the intermediate result back into global memory. + +There are generally two ways to fuse operators, fusing directly connected operators and fusing non directly connected operators. The first method is mainly used by [NNVM Compiler](https://github.com/dmlc/tvm/) and [XLA](https://www.tensorflow.org/performance/xla/). The second method is mainly used by Dynet and TensorFlow Fold to do auto-batching. The principle of fusing operator is according to some rules to combine multiple operations into one, for example, `Y = X * W` and `Z = Y + B` can be fused to `Z = X * W + B`, and `Y1 = X1 * W` and `Y2 = X2 * W` can be fused to `[Y1;Y2] = [X1;X2] * W`. In order to get a short-term profit, we decided to try to manually specify these rules. + +## Challenge +The challenge of fusing operators is: + - how to make the rules. + - how to implement these rules efficiently. + +### How to make the rules? + +The problem of determining the best single location for a fusion operator is an NP-hard combinatorial problem. After analysis the operators of the DL model, we found there are two group of operators can be fused explicitly, one is the simple and adjacent operations, for example, `tmp = x + y` and `z = Relu(tmp)`, and the other is the operators that have the same function, for example, a serials of `SGD` or `Momentum`. They usually appear in the model in a large number. So we should think about how to fuse them separately first. + +### How to implement these rules efficiently? +#### How to fuse the adjacent operations efficiently? +Here we use a template function to represent the fused operations. The pros of using a template function are that it is simple and efficient, and the cons are that it is not easy to expand, and it can only be used to express some simple operations. So taking into account our current needs, the template function is more appropriate. + +#### How to fuse the operators that have the same function efficiently? +We take SGD operator as an example, the training model may have hundreds of parameters and correspondingly have the same number of SGD operators. The expression(`w = w - lr*w_g`) of those operators is the same, so during of training, the executor will execute this expression hundreds time in CPU or other specialized accelerators. If we can fuse them and make the address of all `w` and all `w_g` continuous respectively, we only need execute one time. For some accelerators, the time of launching kernel is not neglected, so the time of hundreds of times of launching and executing kernel may be larger than launching and executing only once. There usually are many operators that similar to `SGD` in the DL model, such as `AllReduce` and `FC`. diff --git a/paddle/fluid/API.spec b/paddle/fluid/API.spec index 3ef317bb7a1c25c5738342f34ae7994b0184a7de..dd172ff9c97814c089ddb2e5bf729880cf0c9cdb 100644 --- a/paddle/fluid/API.spec +++ b/paddle/fluid/API.spec @@ -336,6 +336,7 @@ paddle.fluid.contrib.BeamSearchDecoder.decode ArgSpec(args=['self'], varargs=Non paddle.fluid.contrib.BeamSearchDecoder.early_stop ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None) paddle.fluid.contrib.BeamSearchDecoder.read_array ArgSpec(args=['self', 'init', 'is_ids', 'is_scores'], varargs=None, keywords=None, defaults=(False, False)) paddle.fluid.contrib.BeamSearchDecoder.update_array ArgSpec(args=['self', 'array', 'value'], varargs=None, keywords=None, defaults=None) +paddle.fluid.contrib.memory_usage ArgSpec(args=['program', 'batch_size'], varargs=None, keywords=None, defaults=None) paddle.fluid.transpiler.DistributeTranspiler.__init__ ArgSpec(args=['self', 'config'], varargs=None, keywords=None, defaults=(None,)) paddle.fluid.transpiler.DistributeTranspiler.create_splited_vars ArgSpec(args=['self', 'source_var', 'block', 'tag'], varargs=None, keywords=None, defaults=None) paddle.fluid.transpiler.DistributeTranspiler.get_pserver_program ArgSpec(args=['self', 'endpoint'], varargs=None, keywords=None, defaults=None) diff --git a/paddle/fluid/CMakeLists.txt b/paddle/fluid/CMakeLists.txt index d274d96c29bdbf5973d568d783369c3975bdc436..2577e59d9cf24c26b7c04aa00cdde6cde17f7206 100644 --- a/paddle/fluid/CMakeLists.txt +++ b/paddle/fluid/CMakeLists.txt @@ -5,5 +5,7 @@ add_subdirectory(operators) add_subdirectory(pybind) add_subdirectory(string) add_subdirectory(recordio) -# NOTE: please add subdirectory inference at last. -add_subdirectory(inference) +if(WITH_INFERENCE) + # NOTE: please add subdirectory inference at last. + add_subdirectory(inference) +endif() diff --git a/paddle/fluid/framework/details/build_strategy.h b/paddle/fluid/framework/details/build_strategy.h index b2e5399e2376a86c1cd310b29c768832665af87f..8714a42162bda3d5ad12e7925fe8cc4e693f51b1 100644 --- a/paddle/fluid/framework/details/build_strategy.h +++ b/paddle/fluid/framework/details/build_strategy.h @@ -21,6 +21,26 @@ namespace framework { namespace details { struct BuildStrategy { + // ParallelExecutor supports two modes of ReduceStrategy, kAllReduce and + // kReduce, for CPU and GPU. If you use kAllReduce, different threads + // optimize their parameters separately. If you use kReduce, the optimizations + // of parameters are distributed to different threads. + // For example, a model has 100 parameters and is running with four threads, + // if you choose kAllReduce, every thread is to optimize 100 parameters + // separately, if you choose kReduce, every thread is to optimize 25 + // parameters. + // Of particular note is, if you use kReduce when using CPU training, + // all the parameters are shared between different threads. This feature will + // save memory. + // FIXME(zcd): The result of the two modes(kAllReduce and kReduce) maybe not + // equal for GPU. Because, the result of the different order of summing maybe + // different, for example, the result of `a+b+c+d` may be different with the + // result of `c+a+b+d`. + // For GPU, the implementation of kAllReduce and kReduce is adopted NCCL, + // so the result of kAllReduce and kReduce maybe not equal. + // For CPU, if you want to fix the order of summing to make the result + // of kAllReduce and kReduce no diff, you can add + // `FLAGS_cpu_deterministic=true` to env. enum class ReduceStrategy { kAllReduce = 0, kReduce = 1 }; enum class GradientScaleStrategy { diff --git a/paddle/fluid/framework/details/multi_devices_graph_builder.cc b/paddle/fluid/framework/details/multi_devices_graph_builder.cc index 5ca2ed8f96244a11925dfa6af8e48458cf334ecd..a4fdbcb26d1d0cfb05edebff5419d9559c336b3a 100644 --- a/paddle/fluid/framework/details/multi_devices_graph_builder.cc +++ b/paddle/fluid/framework/details/multi_devices_graph_builder.cc @@ -275,7 +275,8 @@ std::unique_ptr MultiDevSSAGraphBuilder::ApplyImpl( if (strategy_.gradient_scale_ != BuildStrategy::GradientScaleStrategy::kCustomized) { // TODO(paddle-dev): Why is there no input for this op_handle? - CreateScaleLossGradOp(&result); + auto loss_grad_name = node->Op()->OutputArgumentNames()[0]; + CreateScaleLossGradOp(&result, loss_grad_name); } // This assumes the backward generating code will ensure IsScaleLossOp // is true only for the op that scale the final scalar loss. @@ -535,7 +536,8 @@ int MultiDevSSAGraphBuilder::GetVarDeviceID(const ir::Graph &graph, return got == sharded_var_device.end() ? -1 : got->second; } -void MultiDevSSAGraphBuilder::CreateScaleLossGradOp(ir::Graph *result) const { +void MultiDevSSAGraphBuilder::CreateScaleLossGradOp( + ir::Graph *result, const std::string &loss_grad_name) const { for (size_t i = 0; i < places_.size(); ++i) { // Insert ScaleCost OpHandle #ifdef PADDLE_WITH_CUDA @@ -558,10 +560,10 @@ void MultiDevSSAGraphBuilder::CreateScaleLossGradOp(ir::Graph *result) const { // loss->pending_ops_.emplace_back(op_handle); // op_handle->inputs_.emplace_back(loss); - CreateOpOutput(result, op_handle, - result->CreateEmptyNode(GradVarName(loss_var_name_), - ir::Node::Type::kVariable), - places_[i], i); + CreateOpOutput( + result, op_handle, + result->CreateEmptyNode(loss_grad_name, ir::Node::Type::kVariable), + places_[i], i); } } diff --git a/paddle/fluid/framework/details/multi_devices_graph_builder.h b/paddle/fluid/framework/details/multi_devices_graph_builder.h index 099dbe5abef6458c4613c9f680440734f59cb6e2..f2cb6bb1c861e07f1034f1742ad4f3cfbb0d8837 100644 --- a/paddle/fluid/framework/details/multi_devices_graph_builder.h +++ b/paddle/fluid/framework/details/multi_devices_graph_builder.h @@ -75,7 +75,9 @@ class MultiDevSSAGraphBuilder : public SSAGraphBuilder { void CreateComputationalOps(ir::Graph *result, ir::Node *node, size_t num_places) const; - void CreateScaleLossGradOp(ir::Graph *result) const; + void CreateScaleLossGradOp(ir::Graph *result, + const std::string &loss_grad_name) const; + VarHandle *CreateReduceOp(ir::Graph *result, const std::string &og, int dst_dev_id) const; void CreateComputationalOp(ir::Graph *result, ir::Node *node, diff --git a/paddle/fluid/framework/details/reduce_op_handle.cc b/paddle/fluid/framework/details/reduce_op_handle.cc index 68bdfbaf52120d19d05d156529626f42adda630d..6c7e5c1fb06620b1c071b00fcfcc1b4a29bf8d62 100644 --- a/paddle/fluid/framework/details/reduce_op_handle.cc +++ b/paddle/fluid/framework/details/reduce_op_handle.cc @@ -18,6 +18,10 @@ #include "paddle/fluid/framework/details/variable_visitor.h" #include "paddle/fluid/platform/profiler.h" +DEFINE_bool( + cpu_deterministic, false, + "Whether to make the result of computation deterministic in CPU side."); + namespace paddle { namespace framework { namespace details { @@ -91,11 +95,33 @@ void ReduceOpHandle::RunImpl() { } else { std::vector lod_tensors = GetInputValues(in_var_handles, var_scopes); + if (paddle::platform::is_cpu_place(lod_tensors[0]->place())) { this->RunAndRecordEvent([&] { - ReduceLoDTensor func(lod_tensors, - out_var->GetMutable()); - VisitDataType(ToDataType(lod_tensors[0]->type()), func); + // FIXME(zcd): The order of summing is important, + // especially when the type of data is float or double. + // For example, the result of `a+b+c+d` may be different + // with the result of `c+a+b+d`, so the summing order should be fixed. + if (!FLAGS_cpu_deterministic) { + ReduceLoDTensor func(lod_tensors, + out_var->GetMutable()); + VisitDataType(ToDataType(lod_tensors[0]->type()), func); + } else { + // We sum lod_tensors to reduce_sum_trg which is in local_scopes_0 + // here, but it doesn't mean reduce_sum_trg must be in local_scopes_0. + auto &reduce_sum_trg = *this->local_scopes_[0] + ->FindVar(kLocalExecScopeName) + ->Get() + ->FindVar(out_var_handle->name_) + ->GetMutable(); + ReduceLoDTensor func(lod_tensors, &reduce_sum_trg); + VisitDataType(ToDataType(lod_tensors[0]->type()), func); + + auto trg = out_var->GetMutable(); + if (reduce_sum_trg.data() != trg->data()) { + TensorCopy(reduce_sum_trg, platform::CPUPlace(), trg); + } + } }); } else if (paddle::platform::is_gpu_place(lod_tensors[0]->place())) { #ifdef PADDLE_WITH_CUDA diff --git a/paddle/fluid/framework/executor.cc b/paddle/fluid/framework/executor.cc index c2800c972a5501859672fbfd6921499e84d09cb0..dad170ed78c64202b5c812bd8682887fe3b736d6 100644 --- a/paddle/fluid/framework/executor.cc +++ b/paddle/fluid/framework/executor.cc @@ -330,12 +330,7 @@ void Executor::RunPreparedContext(ExecutorPrepareContext* ctx, Scope* scope, } for (auto& op : ctx->ops_) { - VLOG(4) << place_ << " " << op->DebugStringEx(local_scope); op->Run(*local_scope, place_); - // NOTE! Please do not delete this line, it's usefull because the debug - // string before and after op.run are different, after run the output - // will have right shape which is usefull for debug. - VLOG(3) << place_ << " " << op->DebugStringEx(local_scope); if (FLAGS_benchmark) { VLOG(2) << "Memory used after operator " + op->Type() + " running: " diff --git a/paddle/fluid/framework/operator.cc b/paddle/fluid/framework/operator.cc index 0c8acf71bfa0814e66560258ad6131c743ebc81b..d04f7744961b2561977f4d36d0073a97557043da 100644 --- a/paddle/fluid/framework/operator.cc +++ b/paddle/fluid/framework/operator.cc @@ -127,7 +127,7 @@ static LoD GetLoD(const Scope& scope, const std::string& name) { } void OperatorBase::Run(const Scope& scope, const platform::Place& place) { - VLOG(10) << "- " << DebugStringEx(&scope); + VLOG(4) << place << " " << DebugStringEx(&scope); if (platform::is_gpu_place(place)) { #ifndef PADDLE_WITH_CUDA PADDLE_THROW("Cannot run operator on place %s", place); @@ -139,7 +139,7 @@ void OperatorBase::Run(const Scope& scope, const platform::Place& place) { platform::DeviceContextPool& pool = platform::DeviceContextPool::Instance(); platform::RecordEvent record_event(Type(), pool.Get(place)); RunImpl(scope, place); - VLOG(10) << "+ " << DebugStringEx(&scope); + VLOG(3) << place << " " << DebugStringEx(&scope); } bool OperatorBase::HasInputs(const std::string& name) const { @@ -778,6 +778,7 @@ proto::VarType::Type OperatorWithKernel::IndicateDataType( const ExecutionContext& ctx) const { auto& scope = ctx.scope(); int data_type = -1; + std::string last_input_name; for (auto& input : this->inputs_) { for (auto& ipt_name : input.second) { auto* var = scope.FindVar(ipt_name); @@ -794,9 +795,10 @@ proto::VarType::Type OperatorWithKernel::IndicateDataType( int tmp = static_cast(ToDataType(t->type())); PADDLE_ENFORCE( tmp == data_type || data_type == -1, - "DataType of Paddle Op %s must be the same. Get %d != %d", Type(), - data_type, tmp); + "DataType of Paddle Op %s must be the same. Get %s(%d) != %s(%d)", + Type(), last_input_name, data_type, ipt_name, tmp); data_type = tmp; + last_input_name = ipt_name; } } } diff --git a/paddle/fluid/inference/analysis/analyzer.cc b/paddle/fluid/inference/analysis/analyzer.cc index 98bdfcc00b9f0e8f40dfc92e4021b2bd6fb19313..c4ab26a2288bb9d8f3cd54a797d2062e0606b219 100644 --- a/paddle/fluid/inference/analysis/analyzer.cc +++ b/paddle/fluid/inference/analysis/analyzer.cc @@ -24,7 +24,7 @@ namespace paddle { -DEFINE_bool(inference_analysis_enable_tensorrt_subgraph_engine, false, +DEFINE_bool(inference_analysis_enable_tensorrt_subgraph_engine, true, "Enable subgraph to TensorRT engine for acceleration"); DEFINE_string(inference_analysis_graphviz_log_root, "./", @@ -42,10 +42,19 @@ class DfgPassManagerImpl final : public DfgPassManager { // TODO(Superjomn) set the key with pass reprs. AddPass("fluid-to-data-flow-graph", new FluidToDataFlowGraphPass); if (FLAGS_inference_analysis_enable_tensorrt_subgraph_engine) { - auto trt_teller = [](const Node* node) { + auto trt_teller = [&](const Node* node) { + std::unordered_set teller_set( + {"elementwise_add", "mul", "conv2d", "pool2d", "relu"}); if (!node->IsFunction()) return false; - return static_cast(node)->func_type() == "mul"; + + const auto* func = static_cast(node); + if (teller_set.count(func->func_type())) + return true; + else { + return false; + } }; + AddPass("tensorrt-subgraph-marker", new TensorRTSubgraphNodeMarkPass(trt_teller)); AddPass("tensorrt-subgraph", new TensorRTSubGraphPass(trt_teller)); diff --git a/paddle/fluid/inference/analysis/data_flow_graph.cc b/paddle/fluid/inference/analysis/data_flow_graph.cc index 8a3af0a8ebd5bad7be7046fa399cca4920da3d71..7f64bc75ae8ad40a268739cdc36051e76af9f49a 100644 --- a/paddle/fluid/inference/analysis/data_flow_graph.cc +++ b/paddle/fluid/inference/analysis/data_flow_graph.cc @@ -337,6 +337,34 @@ ExtractInputAndOutputOfSubGraph(std::vector &graph) { // NOLINT std::vector(outputs.begin(), outputs.end())); } +void FilterRedundantOutputOfSubGraph(DataFlowGraph *graph) { + std::vector op_nodes; + for (auto &node : GraphTraits(graph).nodes_in_TS()) { + if (node.type() == Node::Type::kValue || node.deleted()) { + continue; + } + op_nodes.push_back(&node); + } + size_t op_num = op_nodes.size(); + for (size_t i = 0; i < op_num; i++) { + if (op_nodes[i]->type() == Node::Type::kFunction) continue; + std::unordered_set follow_up_input_names; + for (size_t j = i + 1; j < op_num; j++) { + for (auto *in : op_nodes[j]->inlinks) { + follow_up_input_names.insert(in->name()); + } + } + std::vector filtered_subgraph_outlinks; + for (auto *out : op_nodes[i]->outlinks) { + if (follow_up_input_names.count(out->name())) { + filtered_subgraph_outlinks.push_back(out); + } + } + PADDLE_ENFORCE_GE(filtered_subgraph_outlinks.size(), 1UL); + op_nodes[i]->outlinks = filtered_subgraph_outlinks; + } +} + } // namespace analysis } // namespace inference } // namespace paddle diff --git a/paddle/fluid/inference/analysis/data_flow_graph.h b/paddle/fluid/inference/analysis/data_flow_graph.h index 16aeae4d35e7bd54646053190da7f47eaca69aa0..bb3ec6bbc1d9555386aba8837b019d2511653258 100644 --- a/paddle/fluid/inference/analysis/data_flow_graph.h +++ b/paddle/fluid/inference/analysis/data_flow_graph.h @@ -178,6 +178,7 @@ struct GraphTraits { std::pair, std::vector> ExtractInputAndOutputOfSubGraph(std::vector &graph); // NOLINT +void FilterRedundantOutputOfSubGraph(DataFlowGraph *graph); } // namespace analysis } // namespace inference } // namespace paddle diff --git a/paddle/fluid/inference/analysis/data_flow_graph_to_fluid_pass.cc b/paddle/fluid/inference/analysis/data_flow_graph_to_fluid_pass.cc index 2328d870422c5a31c22d7b09980aae35e01b2b25..18c32fa09199003f17183207828cdfe4e627ae1a 100644 --- a/paddle/fluid/inference/analysis/data_flow_graph_to_fluid_pass.cc +++ b/paddle/fluid/inference/analysis/data_flow_graph_to_fluid_pass.cc @@ -23,7 +23,7 @@ namespace paddle { namespace inference { -DEFINE_int32(tensorrt_max_batchsize, 300, "TensorRT maximum batch size"); +DEFINE_int32(tensorrt_max_batchsize, 3, "TensorRT maximum batch size"); DEFINE_int32(tensorrt_workspace_size, 2048, "TensorRT workspace size"); namespace analysis { @@ -52,6 +52,7 @@ bool DataFlowGraphToFluidPass::Initialize(Argument *argument) { bool DataFlowGraphToFluidPass::Finalize() { return true; } void DataFlowGraphToFluidPass::Run(DataFlowGraph *graph) { + FilterRedundantOutputOfSubGraph(graph); LOG(INFO) << "graph.inputs " << graph->inputs.size(); for (auto &node : GraphTraits(graph).nodes_in_TS()) { if (node.deleted()) continue; @@ -87,34 +88,113 @@ void DataFlowGraphToFluidPass::AddFluidOp(Node *node) { } void CreateTrtEngineOp(Node *node, const DataFlowGraph &graph, - const framework::proto::BlockDesc &block) { + framework::proto::BlockDesc *block) { static int counter{0}; PADDLE_ENFORCE(node->IsFunctionBlock()); framework::OpDesc desc; auto *func = static_cast(node); // collect inputs - std::vector io; + std::unordered_set input_names; for (auto *x : func->inlinks) { - io.push_back(x->name()); + input_names.insert(x->name()); } - desc.SetInput("Xs", io); + desc.SetInput( + "Xs", std::vector(input_names.begin(), input_names.end())); - // collect outputs - io.clear(); + std::unordered_set output_names; for (auto *x : func->outlinks) { - io.push_back(x->name()); + output_names.insert(x->name()); } - desc.SetOutput("Ys", io); + + std::vector output_temp(output_names.begin(), + output_names.end()); + desc.SetOutput("Ys", output_temp); desc.SetType("tensorrt_engine"); - PADDLE_ENFORCE(!block.vars().empty(), "the block has no var-desc"); + std::unordered_map output_name_map; + + // The following procedure is used to rename all the intermediate + // variables and the output variables of the subgraph. + // Why we do this? + // During the transition from fluid OP to tensorrt OP, we map + // the input and output Tensor(fluid data structure) of fluid OP + // to the correspondin ITensor (trt data structure) through the + // Tensor name. When we set up ITensor for an variable, we must + // ensure that it has not been set before. + // If there is variable in the fluid graph, which is not only the + // input of a OP, but also the output of a Op, there will be problems. + // So we have to rename the variable in the subgraph to make sure + // it is either an OP's input or an OP's output. + + auto subgraph_nodes = func->subgraph; + for (int index = 0; index < block->ops_size(); index++) { + framework::proto::OpDesc *op = block->mutable_ops(index); + auto correspond_node = subgraph_nodes[index]; + PADDLE_ENFORCE_EQ(correspond_node->name(), op->type()); + + std::unordered_map var2id; + for (auto *in_var : correspond_node->inlinks) { + var2id[in_var->name()] = in_var->id(); + } + // rename for the input variables of op inside subgraph + for (int i = 0; i < op->inputs_size(); i++) { + framework::proto::OpDesc_Var *in_var = op->mutable_inputs(i); + std::vector replaced_names; + for (int k = 0; k < in_var->arguments_size(); k++) { + std::string arg_value = in_var->arguments(k); + if (input_names.count(arg_value)) { + replaced_names.push_back(arg_value); + } else { + replaced_names.push_back(arg_value + + std::to_string(var2id[arg_value])); + } + } + in_var->clear_arguments(); + for (size_t k = 0; k < replaced_names.size(); k++) { + in_var->add_arguments(replaced_names[k]); + } + } + var2id.clear(); + for (auto out_var : correspond_node->outlinks) { + var2id[out_var->name()] = out_var->id(); + } + + // rename for the output variables of op inside subgraph + for (int i = 0; i < op->outputs_size(); i++) { + framework::proto::OpDesc_Var *out_var = op->mutable_outputs(i); + std::vector replaced_names; + for (int k = 0; k < out_var->arguments_size(); k++) { + std::string arg_value = out_var->arguments(k); + if (output_names.count(arg_value)) { + output_name_map[arg_value] = + arg_value + std::to_string(var2id[arg_value]); + } + replaced_names.push_back(arg_value + std::to_string(var2id[arg_value])); + } + out_var->clear_arguments(); + for (size_t k = 0; k < replaced_names.size(); k++) { + out_var->add_arguments(replaced_names[k]); + } + } + } + // When tensorrt engine runs at the end of the operation, + // output_mapping help us copy the data from the renamed ITensor + // to Tensor. + std::vector output_mapping; + for (auto name : output_names) { + PADDLE_ENFORCE(output_name_map.count(name) != 0); + output_mapping.push_back(output_name_map[name]); + } + + PADDLE_ENFORCE(!block->vars().empty(), "the block has no var-desc"); // Set attrs - SetAttr(desc.Proto(), "subgraph", block.SerializeAsString()); + SetAttr(desc.Proto(), "subgraph", block->SerializeAsString()); SetAttr(desc.Proto(), "engine_uniq_key", "trt-" + std::to_string(counter++)); SetAttr(desc.Proto(), "max_batch", FLAGS_tensorrt_max_batchsize); SetAttr(desc.Proto(), "max_workspace", FLAGS_tensorrt_workspace_size); SetAttr(desc.Proto(), "parameters", ExtractParameters(graph.nodes.nodes())); + SetAttr(desc.Proto(), "output_name_mapping", output_mapping); node->SetPbMsg(desc.Proto()->SerializeAsString()); } @@ -146,15 +226,17 @@ void DataFlowGraphToFluidPass::AddEngineOp(Node *node) { LOG(INFO) << "transformed variable size: " << block_desc.Proto()->vars().size(); // copy ops. + for (auto *node : block_node->subgraph) { auto *op = block_desc.AppendOp(); PADDLE_ENFORCE(!node->pb_msg().empty()); op->Proto()->ParseFromString(node->pb_msg()); } + *block_desc.Proto()->mutable_vars() = argument_->origin_program_desc->blocks(0).vars(); PADDLE_ENFORCE(!block_desc.Proto()->vars().empty()); - CreateTrtEngineOp(node, *argument_->main_dfg, *block_desc.Proto()); + CreateTrtEngineOp(node, *argument_->main_dfg, block_desc.Proto()); auto *main_block = desc_->mutable_blocks(framework::kRootBlockIndex); auto *op = main_block->add_ops(); PADDLE_ENFORCE(!node->pb_msg().empty(), "failed to set desc for block"); diff --git a/paddle/fluid/inference/analysis/dfg_graphviz_draw_pass.cc b/paddle/fluid/inference/analysis/dfg_graphviz_draw_pass.cc index a6f85484756417e103cbb60bcb664e8b800b9f28..c05b0e5d4690d0a447edf63a149903704bc2c9be 100644 --- a/paddle/fluid/inference/analysis/dfg_graphviz_draw_pass.cc +++ b/paddle/fluid/inference/analysis/dfg_graphviz_draw_pass.cc @@ -46,9 +46,9 @@ std::string DFG_GraphvizDrawPass::Draw(DataFlowGraph *graph) { for (size_t i = 0; i < graph->nodes.size(); i++) { const Node &node = graph->nodes.Get(i); if (!config_.display_deleted_node && node.deleted()) continue; - for (auto &in : node.inlinks) { - if (!config_.display_deleted_node && in->deleted()) continue; - dot.AddEdge(in->repr(), node.repr(), {}); + for (auto &out : node.outlinks) { + if (!config_.display_deleted_node && out->deleted()) continue; + dot.AddEdge(node.repr(), out->repr(), {}); } } return dot.Build(); diff --git a/paddle/fluid/inference/analysis/subgraph_splitter.cc b/paddle/fluid/inference/analysis/subgraph_splitter.cc index 389f9e1a9148a4daf0e5b751cce5cb6325252a4e..80809d4c43ca08298bad25cf614dcb4117d3f99a 100644 --- a/paddle/fluid/inference/analysis/subgraph_splitter.cc +++ b/paddle/fluid/inference/analysis/subgraph_splitter.cc @@ -76,7 +76,7 @@ void UnionFindCombine(const node_map_t &node_map, size_t a, size_t b) { std::vector> SubGraphSplitter::ExtractSubGraphs() { std::vector marked_nodes; - for (auto &node : GraphTraits(graph_).nodes()) { + for (auto &node : GraphTraits(graph_).nodes_in_TS()) { if (node.attr(kMarkerAttrName).Bool()) { marked_nodes.push_back(&node); } diff --git a/paddle/fluid/inference/api/api.cc b/paddle/fluid/inference/api/api.cc index e74f23ff969f5a8f58a71da337c16dcbc14f10c0..63c3f0d7b3f5c2b9246e2b041796caf5eb562826 100644 --- a/paddle/fluid/inference/api/api.cc +++ b/paddle/fluid/inference/api/api.cc @@ -12,6 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ +#include #include "paddle/fluid/inference/api/paddle_inference_api.h" namespace paddle { @@ -40,19 +41,36 @@ PaddleBuf::PaddleBuf(PaddleBuf&& other) PaddleBuf::PaddleBuf(const PaddleBuf& other) { *this = other; } PaddleBuf& PaddleBuf::operator=(const PaddleBuf& other) { + if (!other.memory_owned_) { + data_ = other.data_; + length_ = other.length_; + memory_owned_ = other.memory_owned_; + } else { + Resize(other.length()); + memcpy(data_, other.data(), other.length()); + length_ = other.length(); + memory_owned_ = true; + } + return *this; +} + +PaddleBuf& PaddleBuf::operator=(PaddleBuf&& other) { // only the buffer with external memory can be copied - assert(!other.memory_owned_); data_ = other.data_; length_ = other.length_; memory_owned_ = other.memory_owned_; + other.data_ = nullptr; + other.length_ = 0; + other.memory_owned_ = false; return *this; } void PaddleBuf::Resize(size_t length) { // Only the owned memory can be reset, the external memory can't be changed. if (length_ == length) return; - assert(memory_owned_); - Free(); + if (memory_owned_) { + Free(); + } data_ = new char[length]; length_ = length; memory_owned_ = true; @@ -68,7 +86,7 @@ void PaddleBuf::Reset(void* data, size_t length) { void PaddleBuf::Free() { if (memory_owned_ && data_) { assert(length_ > 0); - delete static_cast(data_); + delete[] static_cast(data_); data_ = nullptr; length_ = 0; } diff --git a/paddle/fluid/inference/api/paddle_inference_api.h b/paddle/fluid/inference/api/paddle_inference_api.h index 59b0df7968cce137843ba8cad38a62fdb8d3bfc1..b24414e8245b1a4d90acce4fa1ad5690e06b47dd 100644 --- a/paddle/fluid/inference/api/paddle_inference_api.h +++ b/paddle/fluid/inference/api/paddle_inference_api.h @@ -40,11 +40,12 @@ class PaddleBuf { // Copy only available when memory is managed externally. explicit PaddleBuf(const PaddleBuf&); PaddleBuf& operator=(const PaddleBuf&); + PaddleBuf& operator=(PaddleBuf&&); // Do not own the memory. PaddleBuf(void* data, size_t length) : data_(data), length_(length), memory_owned_{false} {} // Own memory. - explicit PaddleBuf(size_t length) + PaddleBuf(size_t length) : data_(new char[length]), length_(length), memory_owned_(true) {} // Resize to `length` bytes. void Resize(size_t length); diff --git a/paddle/fluid/inference/tensorrt/convert/CMakeLists.txt b/paddle/fluid/inference/tensorrt/convert/CMakeLists.txt index d86c046f2e5b08a4c00cf6cad19627e6a196c798..8f42a37cd3f8978b917b42e8f45a128b8422aa57 100644 --- a/paddle/fluid/inference/tensorrt/convert/CMakeLists.txt +++ b/paddle/fluid/inference/tensorrt/convert/CMakeLists.txt @@ -1,6 +1,7 @@ # Add TRT tests nv_library(tensorrt_converter SRCS mul_op.cc conv2d_op.cc fc_op.cc pool2d_op.cc elementwise_op.cc +activation_op.cc DEPS tensorrt_engine operator scope framework_proto op_registry) nv_test(test_op_converter SRCS test_op_converter.cc DEPS diff --git a/paddle/fluid/inference/tensorrt/convert/op_converter.h b/paddle/fluid/inference/tensorrt/convert/op_converter.h index 1b6a0ad82f3ceb00cec15c28c8121adc22271b7a..41faaf7212accaaec238062b1340e8da8fa6be33 100644 --- a/paddle/fluid/inference/tensorrt/convert/op_converter.h +++ b/paddle/fluid/inference/tensorrt/convert/op_converter.h @@ -55,7 +55,6 @@ class OpConverter { it = Registry::Lookup("fc"); } } - if (op_desc.Type().find("elementwise") != std::string::npos) { static std::unordered_set add_tensor_op_set{ "add", "mul", "sub", "div", "max", "min", "pow"}; @@ -72,6 +71,8 @@ class OpConverter { "Unsupported elementwise type" + op_type); it = Registry::Lookup("elementwise_" + op_type + "_weight"); + PADDLE_ENFORCE_NOT_NULL(it, "no OpConverter for optype [%s]", + op_desc.Type()); } else { PADDLE_ENFORCE(add_tensor_op_set.count(op_type) > 0, "Unsupported elementwise type" + op_type); diff --git a/paddle/fluid/operators/conv_mkldnn_op.cc b/paddle/fluid/operators/conv_mkldnn_op.cc index 0511eb42a073ac305634110a71a35e501f062132..f07ab5a33b87d7945e5fcdf8f3644f0711ce643b 100644 --- a/paddle/fluid/operators/conv_mkldnn_op.cc +++ b/paddle/fluid/operators/conv_mkldnn_op.cc @@ -280,12 +280,16 @@ class ConvMKLDNNOpKernel : public paddle::framework::OpKernel { * ('any') which lets a primitive (convolution in this case) choose * the memory format preferred for best performance */ + std::string data_format = ctx.Attr("data_format"); + auto chosen_memory_format = + platform::data_format_to_memory_format(data_format); + auto src_md = platform::MKLDNNMemDesc( - src_tz, platform::MKLDNNGetDataType(), memory::format::any); + src_tz, platform::MKLDNNGetDataType(), chosen_memory_format); auto weights_md = platform::MKLDNNMemDesc( - weights_tz, platform::MKLDNNGetDataType(), memory::format::any); + weights_tz, platform::MKLDNNGetDataType(), chosen_memory_format); auto dst_md = platform::MKLDNNMemDesc( - dst_tz, platform::MKLDNNGetDataType(), memory::format::any); + dst_tz, platform::MKLDNNGetDataType(), chosen_memory_format); // create a conv primitive descriptor and save it for usage in backward std::shared_ptr conv_pd = @@ -423,16 +427,20 @@ class ConvMKLDNNGradOpKernel : public paddle::framework::OpKernel { * ('any') which lets a primitive (conv backward in this case) choose * the memory format preferred for best performance */ + std::string data_format = ctx.Attr("data_format"); + auto chosen_memory_format = + platform::data_format_to_memory_format(data_format); + auto src_md = platform::MKLDNNMemDesc( - src_tz, platform::MKLDNNGetDataType(), memory::format::any); + src_tz, platform::MKLDNNGetDataType(), chosen_memory_format); auto diff_src_md = platform::MKLDNNMemDesc( - src_tz, platform::MKLDNNGetDataType(), memory::format::any); + src_tz, platform::MKLDNNGetDataType(), chosen_memory_format); auto weights_md = platform::MKLDNNMemDesc( - weights_tz, platform::MKLDNNGetDataType(), memory::format::any); + weights_tz, platform::MKLDNNGetDataType(), chosen_memory_format); auto diff_weights_md = platform::MKLDNNMemDesc( - weights_tz, platform::MKLDNNGetDataType(), memory::format::any); + weights_tz, platform::MKLDNNGetDataType(), chosen_memory_format); auto diff_dst_md = platform::MKLDNNMemDesc( - dst_tz, platform::MKLDNNGetDataType(), memory::format::any); + dst_tz, platform::MKLDNNGetDataType(), chosen_memory_format); // Retrieve conv_pd from device context auto conv_pd = diff --git a/paddle/fluid/operators/elementwise_op_function.h b/paddle/fluid/operators/elementwise_op_function.h index eb8272e90c32c3a0be2c0ce1bc679571af876317..bc3e95e904f8b6c2cdd2ae6685bf67580178e6b6 100644 --- a/paddle/fluid/operators/elementwise_op_function.h +++ b/paddle/fluid/operators/elementwise_op_function.h @@ -534,8 +534,8 @@ void ElemwiseGradCompute(const framework::ExecutionContext& ctx, const framework::Tensor& dout, int axis, framework::Tensor* dx, framework::Tensor* dy, DX_OP dx_op, DY_OP dy_op) { - const framework::DDim x_dim = x.dims(); - const framework::DDim y_dim = y.dims(); + const framework::DDim& x_dim = x.dims(); + const framework::DDim& y_dim = y.dims(); if (x.dims() == y.dims()) { ElemwiseGradComputeNoBroadcast( ctx, x_dim, y_dim, x, y, out, dout, axis, dx, dy, dx_op, dy_op); @@ -558,19 +558,19 @@ void ElemwiseExplicitGradCompute(const framework::ExecutionContext& ctx, framework::Tensor* dx, framework::Tensor* dy, DX_OP dx_op, DY_OP dy_op) { if (dy == nullptr) { - const framework::DDim dx_dims = dout.dims(); + const framework::DDim& dx_dims = dout.dims(); auto dy_dims = dx_dims; ElemwiseGradComputeNoBroadcast( ctx, dx_dims, dy_dims, x, y, out, dout, axis, dx, dy, dx_op, dy_op); } else { if (dout.dims() == dy->dims()) { - const framework::DDim dx_dims = dout.dims(); - const framework::DDim dy_dims = dy->dims(); + const framework::DDim& dx_dims = dout.dims(); + const framework::DDim& dy_dims = dy->dims(); ElemwiseGradComputeNoBroadcast( ctx, dx_dims, dy_dims, x, y, out, dout, axis, dx, dy, dx_op, dy_op); } else { // Y is a scalar auto dx_dims = dout.dims(); - const framework::DDim dy_dims = dy->dims(); + const framework::DDim& dy_dims = dy->dims(); ElemwiseGradComputeWithBroadcast( ctx, dx_dims, dy_dims, x, y, out, dout, axis, dx, dy, dx_op, dy_op); } diff --git a/paddle/fluid/operators/fused_elemwise_activation_op.cc b/paddle/fluid/operators/fused_elemwise_activation_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..a6fd0aeb021dce40339c32251af130d5984dccd2 --- /dev/null +++ b/paddle/fluid/operators/fused_elemwise_activation_op.cc @@ -0,0 +1,221 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include +#include + +#include "paddle/fluid/operators/fused_elemwise_activation_op.h" + +namespace paddle { +namespace operators { + +class FusedElemwiseActivationOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext *ctx) const override { + PADDLE_ENFORCE( + ctx->HasInput("X"), + "Input(X) of FusedElemwiseActivationOp op should not be null."); + PADDLE_ENFORCE( + ctx->HasInput("Y"), + "Input(Y) of FusedElemwiseActivationOp op should not be null."); + PADDLE_ENFORCE( + ctx->HasOutput("Out"), + "Output(Out) of FusedElemwiseActivationOp op should not be null."); + + auto x_dim = ctx->GetInputDim("X"); + auto y_dim = ctx->GetInputDim("Y"); + PADDLE_ENFORCE_GE(x_dim.size(), y_dim.size(), + "Rank of first input must >= rank of second input."); + + ctx->SetOutputDim("Out", x_dim); + ctx->ShareLoD("X", /*->*/ "Out"); + } + + protected: + framework::OpKernelType GetExpectedKernelType( + const framework::ExecutionContext &ctx) const override { + PADDLE_ENFORCE_EQ(ctx.Input("X")->type(), + ctx.Input("Y")->type(), + "The element's type of input should be the same."); + auto input_data_type = + framework::ToDataType(ctx.Input("X")->type()); + return framework::OpKernelType(input_data_type, ctx.GetPlace()); + } +}; + +class FusedElemwiseActivationMaker : public framework::OpProtoAndCheckerMaker { + public: + void Make() override { + AddInput("X", "(vector)"); + AddInput("Y", "(vector)"); + AddOutput("Out", "vector"); + AddAttr("axis", + "axis is used by elementwise_op, the default value is -1.") + .SetDefault(-1); + AddAttr("scale", + "scale is used by scale_op, the default value is 0.0.") + .SetDefault(0.0); + AddAttr("recomputation", + "Whether to recompute the Out." + "fused_elemwise_activation_grad has two methods to get the " + "dx and dy, one " + "is to use the 'Out', and the other is not to use it. " + "The former method will save the time of recomputing the " + "'Out', but it must occupy the memory to store the 'out'. " + "While, the later method can avoid occupying the memory, " + "but it must recompute the 'Out'. The default value is true.") + .SetDefault(true); + AddAttr>("functor_list", + "The functors that should be fused.") + .AddCustomChecker([&](const std::vector &functor_list) { + PADDLE_ENFORCE(ValidCheck(functor_list)); + }); + + AddComment(R"DOC( +FusedElemwiseActivation Operator. + +At present, FusedElemwiseActivation only supports Two kinds of compound +operators (elementwise_op and activation_op): + + Z = Binary(X, Unary(Y)) + Z = Unary(Binary(X, Y)) + +The attributions of activation_op can be get from fused_elemwise_activation_op's +attributions. functor_list records the functors to be fused, for example +"scale,elementwise_add". + +)DOC"); + } + + private: + bool ValidCheck(const std::vector &functors) { + std::unordered_set unary_fun = {"scale", "relu"}; + std::unordered_set binary_fun = {"elementwise_add"}; + + std::string unary_fun_str; + if (binary_fun.count(functors[0])) { + unary_fun_str = functors[1]; + } else if (binary_fun.count(functors[1])) { + unary_fun_str = functors[0]; + } else { + PADDLE_THROW("%s and %s are not included in fused_list.", functors[0], + functors[1]); + } + PADDLE_ENFORCE_EQ(unary_fun.count(unary_fun_str), 1, + "%s is not included in fused_list.", unary_fun_str); + return true; + } +}; + +class FusedElemwiseActivationGradMaker + : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + std::unique_ptr Apply() const override { + auto *op_desc_ptr = new framework::OpDesc(); + op_desc_ptr->SetType(this->ForwardOpType() + "_grad"); + + for (auto &input_param : this->InputNames()) { + op_desc_ptr->SetInput(input_param, this->Input(input_param)); + op_desc_ptr->SetOutput(framework::GradVarName(input_param), + this->InputGrad(input_param, true)); + } + + for (auto &output_param : this->OutputNames()) { + op_desc_ptr->SetInput(output_param, this->Output(output_param)); + op_desc_ptr->SetInput(framework::GradVarName(output_param), + this->OutputGrad(output_param)); + } + op_desc_ptr->SetAttrMap(this->Attrs()); + + std::vector functor_names = + boost::get>( + op_desc_ptr->GetAttr("functor_list")); + functor_names[0] += "_grad"; + functor_names[1] += "_grad"; + op_desc_ptr->SetAttr("functor_list", functor_names); + return std::unique_ptr(op_desc_ptr); + } +}; + +class FusedElemwiseActivationOpGrad : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext *ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should not be null"); + PADDLE_ENFORCE(ctx->HasInput("Y"), "Input(Y) should not be null"); + PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")), + "Input(Out@GRAD) should not be null"); + + auto x_dims = ctx->GetInputDim("X"); + auto y_dims = ctx->GetInputDim("Y"); + auto out_dims = ctx->GetInputDim(framework::GradVarName("Out")); + + PADDLE_ENFORCE_GE(x_dims.size(), y_dims.size(), + "Rank of first input must >= rank of second input."); + + auto x_grad_name = framework::GradVarName("X"); + auto y_grad_name = framework::GradVarName("Y"); + if (ctx->HasOutput(x_grad_name)) { + ctx->SetOutputDim(x_grad_name, x_dims); + } + if (ctx->HasOutput(y_grad_name)) { + ctx->SetOutputDim(y_grad_name, y_dims); + } + } + + protected: + framework::OpKernelType GetExpectedKernelType( + const framework::ExecutionContext &ctx) const override { + auto input_data_type_index = ctx.Input("X")->type(); + PADDLE_ENFORCE_EQ(input_data_type_index, + ctx.Input("Y")->type(), + "The element's type of input should be the same."); + PADDLE_ENFORCE_EQ( + input_data_type_index, + ctx.Input(framework::GradVarName("Out"))->type(), + "The element's type of input should be the same."); + + auto input_data_type = framework::ToDataType(input_data_type_index); + return framework::OpKernelType(input_data_type, ctx.GetPlace()); + } +}; +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OPERATOR(fused_elemwise_activation, ops::FusedElemwiseActivationOp, + ops::FusedElemwiseActivationMaker, + ops::FusedElemwiseActivationGradMaker); +REGISTER_OPERATOR(fused_elemwise_activation_grad, + ops::FusedElemwiseActivationOpGrad); + +REGISTER_OP_CPU_KERNEL( + fused_elemwise_activation, + ops::FusedElemwiseActivationKernel, + ops::FusedElemwiseActivationKernel); + +REGISTER_OP_CPU_KERNEL( + fused_elemwise_activation_grad, + ops::FusedElemwiseActivationGradKernel, + ops::FusedElemwiseActivationGradKernel); diff --git a/paddle/fluid/operators/fused_elemwise_activation_op.cu b/paddle/fluid/operators/fused_elemwise_activation_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..e1d2b16b4b5e3a480777f834c2cbeb6d00a755e4 --- /dev/null +++ b/paddle/fluid/operators/fused_elemwise_activation_op.cu @@ -0,0 +1,30 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/operators/fused_elemwise_activation_op.h" + +namespace ops = paddle::operators; +REGISTER_OP_CUDA_KERNEL( + fused_elemwise_activation, + ops::FusedElemwiseActivationKernel, + ops::FusedElemwiseActivationKernel); + +REGISTER_OP_CUDA_KERNEL( + fused_elemwise_activation_grad, + ops::FusedElemwiseActivationGradKernel, + ops::FusedElemwiseActivationGradKernel); diff --git a/paddle/fluid/operators/fused_elemwise_activation_op.h b/paddle/fluid/operators/fused_elemwise_activation_op.h new file mode 100644 index 0000000000000000000000000000000000000000..fe0017b824532b1210d0ae3e51983d63d081f12a --- /dev/null +++ b/paddle/fluid/operators/fused_elemwise_activation_op.h @@ -0,0 +1,425 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include +#include +#include "paddle/fluid/framework/op_desc.h" +#include "paddle/fluid/framework/op_registry.h" +#include "paddle/fluid/operators/detail/safe_ref.h" +#include "paddle/fluid/operators/elementwise_op_function.h" +#include "paddle/fluid/operators/math/functors.h" + +namespace math = paddle::operators::math; + +namespace paddle { +namespace operators { + +// CompoundFunctors +// For example: Z = Binary(X, Unary(Y)) +template +struct BinaryCompoundFunctor { + BinaryCompoundFunctor(const BinaryFun &binary_fun, const UnaryFun &unary_fun) + : binary_fun_(binary_fun), unary_fun_(unary_fun) {} + + inline HOSTDEVICE T operator()(T x, T y) { + return binary_fun_(x, unary_fun_(y)); + } + + private: + BinaryFun binary_fun_; + UnaryFun unary_fun_; +}; + +// For example: Z = Unary(Binary(X, Y)) +template +struct UnaryCompoundFunctor { + UnaryCompoundFunctor(const UnaryFun &unary_fun, const BinaryFun &binary_fun) + : unary_fun_(unary_fun), binary_fun_(binary_fun) {} + + inline HOSTDEVICE T operator()(T x, T y) { + return unary_fun_(binary_fun_(x, y)); + } + + private: + UnaryFun unary_fun_; + BinaryFun binary_fun_; +}; + +// FIXME(zcd): DBinaryFun and DUnaryFun have to method to get +// the dx, one is to use the 'out', and the other is not to use it. +// the former method will save the time of recomputing the +// 'out', but it must occupy the memory to store the 'out'. +// While the later method can avoid occupying this memory, +// but it must recompute the 'out'. + +template +struct BinaryCompoundGradDxFunctor { + BinaryCompoundGradDxFunctor(const DBinaryFun &d_binary_fun, + const UnaryFun &unary_fun) + : d_binary_fun_(d_binary_fun), unary_fun_(unary_fun) {} + + inline HOSTDEVICE T operator()(T x, T y, T out, T dout) { + if (Recomputation) { + return dout * d_binary_fun_(x, unary_fun_(y)); + } else { + return dout * d_binary_fun_(x, unary_fun_(y), out); + } + } + + private: + DBinaryFun d_binary_fun_; + UnaryFun unary_fun_; +}; + +template +struct BinaryCompoundGradDyFunctor { + BinaryCompoundGradDyFunctor(const DBinaryFun &d_binary_fun, + const UnaryFun &unary_fun, + const DUnaryFun &d_unary_fun) + : d_binary_fun_(d_binary_fun), + unary_fun_(unary_fun), + d_unary_fun_(d_unary_fun) {} + + inline HOSTDEVICE T operator()(T x, T y, T out, T dout) { + if (Recomputation) { + return dout * d_binary_fun_(unary_fun_(y), x) * d_unary_fun_(y); + } else { + return dout * d_binary_fun_(unary_fun_(y), x, out) * d_unary_fun_(y); + } + } + + private: + DBinaryFun d_binary_fun_; + UnaryFun unary_fun_; + DUnaryFun d_unary_fun_; +}; + +template +struct UnaryCompoundGradDxFunctor { + UnaryCompoundGradDxFunctor(const DUnaryFun &d_unary_fun, + const BinaryFun &binary_fun, + const DBinaryFun &d_binary_fun) + : d_unary_fun_(d_unary_fun), + binary_fun_(binary_fun), + d_binary_fun_(d_binary_fun) {} + + inline HOSTDEVICE T operator()(T x, T y, T out, T dout) { + T base; + if (Recomputation) { + base = dout * d_unary_fun_(binary_fun_(x, y)); + } else { + base = dout * d_unary_fun_(binary_fun_(x, y), out); + } + return base * d_binary_fun_(x, y); + } + + private: + DUnaryFun d_unary_fun_; + BinaryFun binary_fun_; + DBinaryFun d_binary_fun_; +}; + +template +struct UnaryCompoundGradDyFunctor { + UnaryCompoundGradDyFunctor(const DUnaryFun &d_unary_fun, + const BinaryFun &binary_fun, + const DBinaryFun &d_binary_fun) + : d_unary_fun_(d_unary_fun), + binary_fun_(binary_fun), + d_binary_fun_(d_binary_fun) {} + + inline HOSTDEVICE T operator()(T x, T y, T out, T dout) { + T base; + if (Recomputation) { + base = dout * d_unary_fun_(binary_fun_(x, y)); + } else { + base = dout * d_unary_fun_(binary_fun_(x, y), out); + } + return base * d_binary_fun_(y, x); + } + + private: + DUnaryFun d_unary_fun_; + BinaryFun binary_fun_; + DBinaryFun d_binary_fun_; +}; + +template +static void RunBinaryCompoundFunctor(const framework::ExecutionContext &ctx, + const BinaryFunctor &binary_functor, + const UnaryFunctor &unary_functor, + const framework::Tensor *in_x, + const framework::Tensor *in_y, + framework::Tensor *output) { + int axis = ctx.Attr("axis"); + using BinaryCompoundFunctor = + BinaryCompoundFunctor; + + ElementwiseComputeEx( + ctx, in_x, in_y, axis, + BinaryCompoundFunctor(binary_functor, unary_functor), output); +} + +template +static void RunUnaryCompoundFunctors(const framework::ExecutionContext &ctx, + const UnaryFunctor &unary_functor, + const BinaryFunctor &binary_functor, + const framework::Tensor *in_x, + const framework::Tensor *in_y, + framework::Tensor *output) { + int axis = ctx.Attr("axis"); + + using UnaryCompoundFunctor = + UnaryCompoundFunctor; + + ElementwiseComputeEx( + ctx, in_x, in_y, axis, + UnaryCompoundFunctor(unary_functor, binary_functor), output); +} + +template +static void RunBinaryCompoundGradFunctors( + const framework::ExecutionContext &ctx, + const BinaryGradFunctor &binary_grad_functor, + const UnaryFunctor &unary_functor, + const UnaryGradFunctor &unary_grad_functor, const framework::Tensor *in_x, + const framework::Tensor *in_y, const framework::Tensor *in_out, + const framework::Tensor *in_out_grad, framework::Tensor *x_grad, + framework::Tensor *y_grad) { + int axis = ctx.Attr("axis"); + + using BinaryCompoundDxFunctor = + BinaryCompoundGradDxFunctor; + using BinaryCompoundDyFunctor = + BinaryCompoundGradDyFunctor; + + ElemwiseGradCompute( + ctx, *in_x, *in_y, *in_out, *in_out_grad, axis, x_grad, y_grad, + BinaryCompoundDxFunctor(binary_grad_functor, unary_functor), + BinaryCompoundDyFunctor(binary_grad_functor, unary_functor, + unary_grad_functor)); +} + +template +static void RunUnaryCompoundGradFunctors( + const framework::ExecutionContext &ctx, + const UnaryGradFunctor &unary_grad_functor, + const BinaryFunctor &binary_functor, + const BinaryGradFunctor &binary_grad_functor, const framework::Tensor *in_x, + const framework::Tensor *in_y, const framework::Tensor *in_out, + const framework::Tensor *in_out_grad, framework::Tensor *x_grad, + framework::Tensor *y_grad) { + int axis = ctx.Attr("axis"); + + using UnaryCompoundDxFunctor = + UnaryCompoundGradDxFunctor; + using UnaryCompoundDyFunctor = + UnaryCompoundGradDyFunctor; + + ElemwiseGradCompute( + ctx, *in_x, *in_y, *in_out, *in_out_grad, axis, x_grad, y_grad, + UnaryCompoundDxFunctor(unary_grad_functor, binary_functor, + binary_grad_functor), + UnaryCompoundDyFunctor(unary_grad_functor, binary_functor, + binary_grad_functor)); +} + +template +static void RunFunctors(const framework::ExecutionContext &ctx, + const framework::Tensor *in_x, + const framework::Tensor *in_y, + framework::Tensor *output) { + auto &functors = ctx.Attr>("functor_list"); + auto funcs_str = functors[0] + "," + functors[1]; + // TODO(zcd): The following code can be refined. + if (funcs_str == "elementwise_add,scale") { + // Z = Binary(X, Unary(Y)) + T scale = static_cast(ctx.Attr("scale")); + RunBinaryCompoundFunctor, + math::ScaleFunctor>( + ctx, math::AddFunctor(), math::ScaleFunctor(scale), in_x, in_y, + output); + } else if (funcs_str == "scale,elementwise_add") { + // Z = Unary(Binary(X, Y)) + T scale = static_cast(ctx.Attr("scale")); + RunUnaryCompoundFunctors, + math::AddFunctor>( + ctx, math::ScaleFunctor(scale), math::AddFunctor(), in_x, in_y, + output); + } else if (funcs_str == "elementwise_add,relu") { + RunBinaryCompoundFunctor, + math::ReluFunctor>( + ctx, math::AddFunctor(), math::ReluFunctor(), in_x, in_y, output); + } else if (funcs_str == "relu,elementwise_add") { + RunUnaryCompoundFunctors, + math::AddFunctor>( + ctx, math::ReluFunctor(), math::AddFunctor(), in_x, in_y, output); + } else { + PADDLE_THROW("%s has not been implemented.", funcs_str); + } +} + +template +static void RunGradFunctors(const framework::ExecutionContext &ctx, + const framework::Tensor *in_x, + const framework::Tensor *in_y, + const framework::Tensor *in_out, + const framework::Tensor *in_out_grad, + framework::Tensor *x_grad, + framework::Tensor *y_grad) { + auto &functors = ctx.Attr>("functor_list"); + auto funcs_str = functors[0] + "," + functors[1]; + + bool recomputation = ctx.Attr("recomputation"); + + // TODO(zcd): The following code can be refined. for example, use registion + if (funcs_str == "elementwise_add_grad,scale_grad") { + // The backward of Z = Binary(X, Unary(Y)) + T scale = static_cast(ctx.Attr("scale")); + if (recomputation) { + RunBinaryCompoundGradFunctors, + math::ScaleFunctor, + math::ScaleGradFunctor, true>( + ctx, math::AddGradFunctor(), math::ScaleFunctor(scale), + math::ScaleGradFunctor(scale), in_x, in_y, in_out, in_out_grad, + x_grad, y_grad); + } else { + RunBinaryCompoundGradFunctors, + math::ScaleFunctor, + math::ScaleGradFunctor, false>( + ctx, math::AddGradFunctor(), math::ScaleFunctor(scale), + math::ScaleGradFunctor(scale), in_x, in_y, in_out, in_out_grad, + x_grad, y_grad); + } + } else if (funcs_str == "scale_grad,elementwise_add_grad") { + // The backward of Z = Unary(Binary(X, Y)) + T scale = static_cast(ctx.Attr("scale")); + if (recomputation) { + RunUnaryCompoundGradFunctors, + math::AddFunctor, math::AddGradFunctor, + true>(ctx, math::ScaleGradFunctor(scale), + math::AddFunctor(), + math::AddGradFunctor(), in_x, in_y, + in_out, in_out_grad, x_grad, y_grad); + } else { + RunUnaryCompoundGradFunctors, + math::AddFunctor, math::AddGradFunctor, + false>(ctx, math::ScaleGradFunctor(scale), + math::AddFunctor(), + math::AddGradFunctor(), in_x, in_y, + in_out, in_out_grad, x_grad, y_grad); + } + } else if (funcs_str == "elementwise_add_grad,relu_grad") { + if (recomputation) { + RunBinaryCompoundGradFunctors, + math::ReluFunctor, + math::ReluGradFunctor, true>( + ctx, math::AddGradFunctor(), math::ReluFunctor(), + math::ReluGradFunctor(), in_x, in_y, in_out, in_out_grad, x_grad, + y_grad); + } else { + RunBinaryCompoundGradFunctors, + math::ReluFunctor, + math::ReluGradFunctor, false>( + ctx, math::AddGradFunctor(), math::ReluFunctor(), + math::ReluGradFunctor(), in_x, in_y, in_out, in_out_grad, x_grad, + y_grad); + } + } else if (funcs_str == "relu_grad,elementwise_add_grad") { + if (recomputation) { + RunUnaryCompoundGradFunctors, + math::AddFunctor, math::AddGradFunctor, + true>(ctx, math::ReluGradFunctor(), + math::AddFunctor(), + math::AddGradFunctor(), in_x, in_y, + in_out, in_out_grad, x_grad, y_grad); + } else { + RunUnaryCompoundGradFunctors, + math::AddFunctor, math::AddGradFunctor, + false>(ctx, math::ReluGradFunctor(), + math::AddFunctor(), + math::AddGradFunctor(), in_x, in_y, + in_out, in_out_grad, x_grad, y_grad); + } + } else { + PADDLE_THROW("%s has not been implemented.", funcs_str); + } +} + +template +class FusedElemwiseActivationKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext &ctx) const override { + auto &in_x = detail::Ref(ctx.Input("X"), + "Cannot get input tensor %s, variable name = %s", + "X", ctx.op().Input("X")); + auto &in_y = detail::Ref(ctx.Input("Y"), + "Cannot get input tensor %s, variable name = %s", + "Y", ctx.op().Input("Y")); + auto &output = detail::Ref(ctx.Output("Out"), + "Cannot get input tensor %s, variable name = %s", + "Out", ctx.op().Output("Out")); + + RunFunctors(ctx, &in_x, &in_y, &output); + } +}; + +template +class FusedElemwiseActivationGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext &ctx) const override { + auto &in_x = detail::Ref(ctx.Input("X"), + "Cannot get input tensor %s, variable name = %s", + "X", ctx.op().Input("X")); + auto &in_y = detail::Ref(ctx.Input("Y"), + "Cannot get input tensor %s, variable name = %s", + "Y", ctx.op().Input("Y")); + auto &in_out = detail::Ref(ctx.Input("Out"), + "Cannot get input tensor %s, variable name = %s", + "Out", ctx.op().Input("Out")); + auto &in_out_grad = + detail::Ref(ctx.Input(framework::GradVarName("Out")), + "Cannot get input tensor %s, variable name = %s", + framework::GradVarName("Out"), + ctx.op().Input(framework::GradVarName("Out"))); + + framework::Tensor *x_grad = + ctx.Output(framework::GradVarName("X")); + framework::Tensor *y_grad = + ctx.Output(framework::GradVarName("Y")); + + RunGradFunctors(ctx, &in_x, &in_y, &in_out, &in_out_grad, + x_grad, y_grad); + } +}; +} // namespace operators +} // namespace paddle diff --git a/paddle/fluid/operators/math/functors.h b/paddle/fluid/operators/math/functors.h new file mode 100644 index 0000000000000000000000000000000000000000..ad2f49ccbf5ff37d33cc9e71c1a683571f4f8137 --- /dev/null +++ b/paddle/fluid/operators/math/functors.h @@ -0,0 +1,71 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +namespace paddle { +namespace operators { +namespace math { + +// AddFunctor +template +struct AddFunctor { + // out = x + y; + inline HOSTDEVICE T operator()(T x, T y) { return x + y; } +}; + +template +struct AddGradFunctor { + inline HOSTDEVICE T operator()(T x, T y) { return 1; } + + inline HOSTDEVICE T operator()(T x, T y, T out) const { return 1; } +}; + +template +struct ScaleFunctor { + explicit ScaleFunctor(const T coeff) : coeff_(coeff) {} + + inline HOSTDEVICE T operator()(T ele) { return ele * coeff_; } + + private: + T coeff_; +}; + +template +struct ScaleGradFunctor { + explicit ScaleGradFunctor(T coeff) : coeff_(coeff) {} + + inline HOSTDEVICE T operator()(T x) { return coeff_; } + + inline HOSTDEVICE T operator()(T x, T out) { return coeff_; } + + private: + T coeff_; +}; + +template +struct ReluFunctor { + inline HOSTDEVICE T operator()(T x) { return x * (x > 0); } +}; + +template +struct ReluGradFunctor { + inline HOSTDEVICE T operator()(T x) { return x > 0 ? 1 : 0; } + + inline HOSTDEVICE T operator()(T x, T out) { return x > 0 ? 1 : 0; } +}; + +} // namespace math +} // namespace operators +} // namespace paddle diff --git a/paddle/fluid/operators/parallel_do_op.cc b/paddle/fluid/operators/parallel_do_op.cc index 916cdad3fd288d1f3ffb19bc769ab827dd1e9103..eb09470f37eabb5524f774bc289fc68f5884c540 100644 --- a/paddle/fluid/operators/parallel_do_op.cc +++ b/paddle/fluid/operators/parallel_do_op.cc @@ -163,12 +163,11 @@ class ParallelDoOp : public framework::OperatorBase { auto &place = places[place_idx]; auto *cur_scope = sub_scopes[place_idx]; - workers.emplace_back( - framework::Async([program, cur_scope, place, block, place_idx] { - framework::Executor executor(place); - executor.Run(*program, cur_scope, block->ID(), - false /*create_local_scope*/); - })); + workers.emplace_back(framework::Async([program, cur_scope, place, block] { + framework::Executor executor(place); + executor.Run(*program, cur_scope, block->ID(), + false /*create_local_scope*/); + })); } for (auto &worker : workers) { worker.wait(); @@ -239,12 +238,11 @@ class ParallelDoGradOp : public framework::OperatorBase { auto *cur_scope = sub_scopes[i]; // execute - workers.emplace_back( - framework::Async([program, cur_scope, place, block, i] { - framework::Executor executor(place); - executor.Run(*program, cur_scope, block->ID(), - false /*create_local_scope*/); - })); + workers.emplace_back(framework::Async([program, cur_scope, place, block] { + framework::Executor executor(place); + executor.Run(*program, cur_scope, block->ID(), + false /*create_local_scope*/); + })); } for (auto &worker : workers) { worker.wait(); diff --git a/paddle/fluid/operators/read_op.cc b/paddle/fluid/operators/read_op.cc index 65fcce8bb019965a805ad09d50be0aba64e4f24e..a0d640b2020958af53a4405ae886eadb2a1e117e 100644 --- a/paddle/fluid/operators/read_op.cc +++ b/paddle/fluid/operators/read_op.cc @@ -15,6 +15,7 @@ #include "paddle/fluid/framework/op_registry.h" #include "paddle/fluid/framework/reader.h" #include "paddle/fluid/operators/detail/safe_ref.h" +#include "paddle/fluid/platform/profiler.h" namespace paddle { namespace operators { @@ -65,6 +66,12 @@ class ReadOp : public framework::OperatorBase { .GetMutable(); std::vector out_arg_names = Outputs("Out"); std::vector ins; + + // For profiling + platform::DeviceContextPool& pool = platform::DeviceContextPool::Instance(); + auto& ctx = *pool.Get(dev_place); + platform::RecordEvent record_event(Type(), &ctx); + reader->ReadNext(&ins); if (ins.empty()) { if (Attr("throw_eof_exp")) { diff --git a/paddle/fluid/operators/softmax_with_cross_entropy_op.cu b/paddle/fluid/operators/softmax_with_cross_entropy_op.cu index 8f7840cee1dd95a828fd4ac8815e335a5db47e3d..a559b01ed32a48e3befb37c2ae8935b4f3a4acb0 100644 --- a/paddle/fluid/operators/softmax_with_cross_entropy_op.cu +++ b/paddle/fluid/operators/softmax_with_cross_entropy_op.cu @@ -1,4 +1,4 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,6 +14,8 @@ limitations under the License. */ #define EIGEN_USE_GPU +#include +#include "paddle/fluid/operators/math/cross_entropy.h" #include "paddle/fluid/operators/softmax_with_cross_entropy_op.h" namespace paddle { @@ -53,8 +55,196 @@ __global__ void SoftCrossEntropyGradientKernel(T* logit_grad, logit_grad[ids] = loss_grad[row_ids] * (logit_grad[ids] - labels[ids]); } } + } // namespace +static __device__ __forceinline__ float real_exp(float x) { return expf(x); } +static __device__ __forceinline__ double real_exp(double x) { return exp(x); } +static __device__ __forceinline__ float real_log(float x) { + return math::TolerableValue()(logf(x)); +} +static __device__ __forceinline__ double real_log(double x) { + return math::TolerableValue()(log(x)); +} + +/** In the following codes, 3 CUDA kernels are implemented to calculate softmax + * and loss **/ +/* + Supposing the x is `logits` and y is `labels`, the equations are as +followings: + + cross\_entropy_i = \sum_{j}[- y_i_j * log({e^{x_i_j}/\sum_{j}e^{x_i_j}})] + = \sum_{j}[- y_i_j * log({e^{x_i_j - max_i}/\sum_{j}e^{x_i_j-max_i}})] + = \sum_{j}[-y_i_j * (x_i_j - max_i - log\sum_{j}e^{x_i_j - max_i})] + = \sum_{j}[-y_i_j * (x_i_j - max_i - logDiffMaxSum_i)] + = \sum_{j}(-y_i_j * tmp_i_j) + + softmax_i_j = e^{tmp_i_j} + +where: + max_i = \max_{j}{x_i_j} + logDiffMaxSum_i = log\sum_{j}e^{x_i_j - max_i} + tmp_i_j = x_i_j - max_i - logDiffMaxSum_i + +Therefore, the calculation can be separated into 3 steps: +Step 1: row-wise operation to calculate max_i +Step 2: row-wise operation to calculate logDiffMaxSum_i +Step 3: caculate tmp_i_j, and finally get softmax_i_j and cross\_entropy_i + +To save memory, we can share memory among max_i, logDiffMaxSum_i and +cross\_entropy_i. +In this way, the 3 steps should be changed to: +Step 1 (RowReductionForMax): row-wise operation to calculate max_i +Step 2 (RowReductionForDiffMaxSum): calculate immediate result of softmax'_i_j = +x_i_j - max_i, and row-wise operation to calculate logDiffMaxSum_i +Step 3 (RowReductionForSoftmaxAndCrossEntropy): calculate tmp_i_j = softmax'_i_j +- logDiffMaxSum_i, and finally get softmax_i_j and cross\_entropy_i +*/ + +// There are 3 kinds of reduce algorithms in cub: +// BLOCK_REDUCE_RAKING_COMMUTATIVE_ONLY +// BLOCK_REDUCE_RAKING +// BLOCK_REDUCE_WARP_REDUCTIONS (default) +template +using BlockReduce = + cub::BlockReduce; + +template +using BlockReduceTempStorage = typename BlockReduce::TempStorage; + +// Make sure that BlockDim <= feature_size +// This kernel is used to calculate the max element of each row +template +__global__ void RowReductionForMax(const T* logits_data, T* max_data, + int feature_size) { + __shared__ BlockReduceTempStorage temp_storage; + + auto beg_idx = feature_size * blockIdx.x + threadIdx.x; + auto end_idx = feature_size * (blockIdx.x + 1); + + T cur_max = logits_data[beg_idx]; + beg_idx += BlockDim; + while (beg_idx < end_idx) { + if (cur_max < logits_data[beg_idx]) { + cur_max = logits_data[beg_idx]; + } + beg_idx += BlockDim; + } + + cur_max = BlockReduce(temp_storage).Reduce(cur_max, cub::Max()); + + if (threadIdx.x == 0) { + max_data[blockIdx.x] = cur_max < -64 ? -64 : cur_max; + } +} + +// Make sure that BlockDim <= feature_size +template +__global__ void RowReductionForDiffMaxSum(const T* logits_data, T* max_data, + T* softmax, int feature_size) { + __shared__ BlockReduceTempStorage temp_storage; + + auto beg_idx = feature_size * blockIdx.x + threadIdx.x; + auto end_idx = feature_size * (blockIdx.x + 1); + + auto block_max = max_data[blockIdx.x]; + + softmax[beg_idx] = logits_data[beg_idx] - block_max; + T diff_max_sum = real_exp(softmax[beg_idx]); + beg_idx += BlockDim; + while (beg_idx < end_idx) { + softmax[beg_idx] = logits_data[beg_idx] - block_max; + diff_max_sum += real_exp(softmax[beg_idx]); + beg_idx += BlockDim; + } + + diff_max_sum = + BlockReduce(temp_storage).Reduce(diff_max_sum, cub::Sum()); + if (threadIdx.x == 0) max_data[blockIdx.x] = real_log(diff_max_sum); +} + +// Make sure that BlockDim <= feature_size +template +__global__ void RowReductionForSoftmaxAndCrossEntropy(const T* logits_data, + const T* labels_data, + T* loss_data, T* softmax, + int feature_size) { + __shared__ BlockReduceTempStorage temp_storage; + + auto beg_idx = feature_size * blockIdx.x + threadIdx.x; + auto end_idx = feature_size * (blockIdx.x + 1); + + // log_diff_max_sum shares memory with loss + auto block_log_diff_max_sum = loss_data[blockIdx.x]; + auto tmp = softmax[beg_idx] - block_log_diff_max_sum; + softmax[beg_idx] = real_exp(tmp); + auto loss = -labels_data[beg_idx] * tmp; + beg_idx += BlockDim; + while (beg_idx < end_idx) { + tmp = softmax[beg_idx] - block_log_diff_max_sum; + softmax[beg_idx] = real_exp(tmp); + loss -= (labels_data[beg_idx] * tmp); + beg_idx += BlockDim; + } + + loss = BlockReduce(temp_storage).Reduce(loss, cub::Sum()); + if (threadIdx.x == 0) loss_data[blockIdx.x] = loss; +} + +template +__global__ void SetSoftmaxToOneWhenFeatureSizeIsOne(T* out, int batch_size) { + auto idx = threadIdx.x + blockIdx.x * blockDim.x; + if (idx < batch_size) out[idx] = static_cast(1); +} + +template +static void SoftmaxWithCrossEntropyFusedKernel(const T* logits_data, + const T* labels_data, + T* softmax_data, T* loss_data, + int batch_size, int feature_size, + cudaStream_t stream) { + constexpr int kMaxBlockDim = 512; + int block_dim = feature_size >= kMaxBlockDim + ? kMaxBlockDim + : (1 << static_cast(std::log2(feature_size))); + +#define CALL_SOFTMAX_WITH_CROSS_ENTROPY_FUSED_KERNEL(BlockDim) \ + case BlockDim: \ + RowReductionForMax<<>>( \ + logits_data, loss_data, feature_size); \ + RowReductionForDiffMaxSum<<>>( \ + logits_data, loss_data, softmax_data, feature_size); \ + RowReductionForSoftmaxAndCrossEntropy< \ + T, BlockDim><<>>( \ + logits_data, labels_data, loss_data, softmax_data, feature_size); \ + break + + switch (block_dim) { + CALL_SOFTMAX_WITH_CROSS_ENTROPY_FUSED_KERNEL(512); + CALL_SOFTMAX_WITH_CROSS_ENTROPY_FUSED_KERNEL(256); + CALL_SOFTMAX_WITH_CROSS_ENTROPY_FUSED_KERNEL(128); + CALL_SOFTMAX_WITH_CROSS_ENTROPY_FUSED_KERNEL(64); + CALL_SOFTMAX_WITH_CROSS_ENTROPY_FUSED_KERNEL(32); + CALL_SOFTMAX_WITH_CROSS_ENTROPY_FUSED_KERNEL(16); + CALL_SOFTMAX_WITH_CROSS_ENTROPY_FUSED_KERNEL(8); + CALL_SOFTMAX_WITH_CROSS_ENTROPY_FUSED_KERNEL(4); + CALL_SOFTMAX_WITH_CROSS_ENTROPY_FUSED_KERNEL(2); + case 1: + SetSoftmaxToOneWhenFeatureSizeIsOne<<<(batch_size + kMaxBlockDim - 1) / + kMaxBlockDim, + kMaxBlockDim, 0, stream>>>( + softmax_data, batch_size); + cudaMemsetAsync(loss_data, 0, batch_size, stream); + break; + default: + PADDLE_THROW("BlockDim must be 2^n in softmax_with_cross_entropy_op"); + break; + } + +#undef CALL_SOFTMAX_WITH_CROSS_ENTROPY_FUSED_KERNEL +} + template class SoftmaxWithCrossEntropyCUDAKernel : public framework::OpKernel { public: @@ -66,14 +256,24 @@ class SoftmaxWithCrossEntropyCUDAKernel : public framework::OpKernel { Tensor* softmax = context.Output("Softmax"); Tensor* loss = context.Output("Loss"); - softmax->mutable_data(context.GetPlace()); - loss->mutable_data(context.GetPlace()); - - math::SoftmaxFunctor()( - context.cuda_device_context(), logits, softmax); - math::CrossEntropyFunctor()( - context.cuda_device_context(), loss, softmax, labels, - context.Attr("soft_label")); + auto* softmax_data = softmax->mutable_data(context.GetPlace()); + auto* loss_data = loss->mutable_data(context.GetPlace()); + + auto soft_label = context.Attr("soft_label"); + if (soft_label) { + int batch_size = logits->dims()[0]; + int feature_size = logits->dims()[1]; + auto* logits_data = logits->data(); + auto* labels_data = labels->data(); + SoftmaxWithCrossEntropyFusedKernel( + logits_data, labels_data, softmax_data, loss_data, batch_size, + feature_size, context.cuda_device_context().stream()); + } else { + math::SoftmaxCUDNNFunctor()(context.cuda_device_context(), logits, + softmax); + math::CrossEntropyFunctor()( + context.cuda_device_context(), loss, softmax, labels, false); + } } }; diff --git a/paddle/fluid/operators/tensorrt_engine_op.cc b/paddle/fluid/operators/tensorrt_engine_op.cc index 1172822e12222ded219104e3bad2613d30f891b8..ee3078876c15b06a887064f08dc0c05d450b5f77 100644 --- a/paddle/fluid/operators/tensorrt_engine_op.cc +++ b/paddle/fluid/operators/tensorrt_engine_op.cc @@ -55,18 +55,8 @@ nvinfer1::Dims Vec2TRT_Dims(const std::vector &shape) { "TensorRT' tensor input requires at least 2 dimensions"); PADDLE_ENFORCE_LE(shape.size(), 4UL, "TensorRT' tensor input requires at most 4 dimensions"); - - switch (shape.size()) { - case 2: - return nvinfer1::Dims2(1, shape[1]); - case 3: - return nvinfer1::Dims3(1, shape[1], shape[2]); - case 4: - return nvinfer1::Dims4(1, shape[1], shape[2], shape[3]); - default: - return nvinfer1::Dims(); - } - return nvinfer1::Dims(); + PADDLE_ENFORCE_EQ(shape.size(), 4UL); + return nvinfer1::DimsCHW(shape[1], shape[2], shape[3]); } } // namespace @@ -86,6 +76,9 @@ void TensorRTEngineKernel::Prepare( parameters.insert(param); } + std::vector output_maps = + context.Attr>("output_name_mapping"); + // TODO(Superjomn) replace this with a different stream auto *engine = Singleton::Global().Create( max_batch, max_workspace, nullptr /*engine hold its own stream*/, @@ -97,6 +90,7 @@ void TensorRTEngineKernel::Prepare( // Add inputs VLOG(4) << "declare inputs"; for (auto &input : context.Inputs("Xs")) { + if (parameters.count(input)) continue; VLOG(4) << "declare input " << input; auto *var = block.FindVar(input); // TensorRT engine need to create parameters. The parameter's description @@ -122,7 +116,7 @@ void TensorRTEngineKernel::Prepare( block_desc, parameters, context.scope(), engine); // Add outputs - for (auto &output : context.Outputs("Ys")) { + for (auto &output : output_maps) { engine->DeclareOutput(output); } diff --git a/paddle/fluid/operators/tensorrt_engine_op.h b/paddle/fluid/operators/tensorrt_engine_op.h index 32d10fd8a5687ebaae1d7d75af531cbc45ef4245..2cbe1213a2f428a3ce56b06f97636baeb4b66c26 100644 --- a/paddle/fluid/operators/tensorrt_engine_op.h +++ b/paddle/fluid/operators/tensorrt_engine_op.h @@ -66,8 +66,17 @@ class TensorRTEngineKernel : public framework::OpKernel { PADDLE_ENFORCE_LE(FLAGS_tensorrt_engine_batch_size, context.Attr("max_batch")); + std::vector output_maps = + context.Attr>("output_name_mapping"); + + auto params = context.Attr>("parameters"); + std::unordered_set parameters; + for (const auto& param : params) { + parameters.insert(param); + } // Convert input tensor from fluid to engine. for (const auto& x : context.Inputs("Xs")) { + if (parameters.count(x)) continue; // convert input and copy to TRT engine's buffer auto& t = inference::analysis::GetFromScope( context.scope(), x); @@ -82,10 +91,12 @@ class TensorRTEngineKernel : public framework::OpKernel { // Execute the engine. PADDLE_ENFORCE_GT(FLAGS_tensorrt_engine_batch_size, 0); engine->Execute(FLAGS_tensorrt_engine_batch_size); + // Convert output tensor from engine to fluid + int output_index = 0; for (const auto& y : context.Outputs("Ys")) { // convert output and copy to fluid. - nvinfer1::ITensor* trt_t = engine->GetITensor(y); + nvinfer1::ITensor* trt_t = engine->GetITensor(output_maps[output_index]); auto dims = trt_t->getDimensions(); // Use the output ITensor's dims to reshape the Fluid Tensor. std::vector ddim(dims.d, dims.d + dims.nbDims); @@ -102,7 +113,7 @@ class TensorRTEngineKernel : public framework::OpKernel { // TODO(Superjomn) change this float to dtype size. auto size = inference::analysis::AccuDims(dims.d, dims.nbDims) * FLAGS_tensorrt_engine_batch_size; - engine->GetOutputInCPU(y, + engine->GetOutputInCPU(output_maps[output_index], fluid_t->mutable_data(platform::CPUPlace()), size * sizeof(float)); //} else { @@ -110,6 +121,7 @@ class TensorRTEngineKernel : public framework::OpKernel { // y, fluid_t->mutable_data(platform::CUDAPlace()), // size * sizeof(float)); //} + output_index += 1; } cudaStreamSynchronize(*engine->stream()); diff --git a/paddle/fluid/operators/tensorrt_engine_op_test.cc b/paddle/fluid/operators/tensorrt_engine_op_test.cc index 7cb1e47a1516c32fb31a7818e7203b498e31e431..37657fa0b0498986fe67027415279af1775e58b9 100644 --- a/paddle/fluid/operators/tensorrt_engine_op_test.cc +++ b/paddle/fluid/operators/tensorrt_engine_op_test.cc @@ -103,6 +103,9 @@ TEST(TensorRTEngineOp, manual) { SetAttr(engine_op_desc.Proto(), "engine_uniq_key", "a_engine"); SetAttr>(engine_op_desc.Proto(), "parameters", std::vector({})); + SetAttr>(engine_op_desc.Proto(), + "output_name_mapping", + std::vector({"z0"})); LOG(INFO) << "create engine op"; auto engine_op = framework::OpRegistry::CreateOp(*engine_op_desc.Proto()); @@ -196,6 +199,10 @@ void Execute(int batch_size, int input_dim, int output_dim, int nlayers = 1) { std::vector({"y0", "y1", "y2", "y3"})); SetAttr(engine_op_desc.Proto(), "engine_uniq_key", "b_engine"); + SetAttr>(engine_op_desc.Proto(), + "output_name_mapping", + std::vector({"z3"})); + auto engine_op = framework::OpRegistry::CreateOp(*engine_op_desc.Proto()); // Execute them. diff --git a/paddle/fluid/platform/mkldnn_helper.h b/paddle/fluid/platform/mkldnn_helper.h index a8f93e6848a1db1f5aa0ee266a076af2b5d0c964..10a3ad256b17ba41380cdc0377905d03188cbaa3 100644 --- a/paddle/fluid/platform/mkldnn_helper.h +++ b/paddle/fluid/platform/mkldnn_helper.h @@ -223,7 +223,7 @@ class MKLDNNHandler { static std::string GetHash(mkldnn::memory::dims& operand_dims, // NOLINT const std::string& suffix) { return dims2str(operand_dims) + suffix; - }; + } protected: static std::string dims2str(const mkldnn::memory::dims& operand_dims) { @@ -251,5 +251,17 @@ inline mkldnn::memory::format MKLDNNFormatForSize( return data_format; } +inline mkldnn::memory::format data_format_to_memory_format( + const std::string& data_format) { + switch (framework::StringToDataLayout(data_format)) { + case framework::DataLayout::kNHWC: + return mkldnn::memory::format::nhwc; + case framework::DataLayout::kNCHW: + return mkldnn::memory::format::nchw; + default: + return mkldnn::memory::format::any; + } +} + } // namespace platform } // namespace paddle diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index ee1c8d46ddfb4f0c09591bb78dc720555dc735b4..2320f3e4dbc5d98698670fee19ed983411a802a9 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -394,8 +394,10 @@ All parameter, weight, gradient are variables in Paddle. InferenceOptimize(*(origin.Proto()), &pruned_desc); return new ProgramDesc(pruned_desc); }); - m.def("empty_var_name", []() { return framework::kEmptyVarName; }); - m.def("grad_var_suffix", []() { return framework::kGradVarSuffix; }); + m.def("empty_var_name", + []() { return std::string(framework::kEmptyVarName); }); + m.def("grad_var_suffix", + []() { return std::string(framework::kGradVarSuffix); }); m.def_submodule( "var_names", "The module will return special predefined variable name in Paddle") diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index a8bc16f1b5b9b624e88e355d8ce4741fcec34bc3..8460f93b841fe136db138e0dc7576f3aacdbeb5f 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -419,6 +419,25 @@ EOF linkchecker doc/v2/en/html/index.html linkchecker doc/v2/cn/html/index.html linkchecker doc/v2/api/en/html/index.html + + if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then exit 0; fi; + + # Deploy to the the content server if its a "develop" or "release/version" branch + # The "develop_doc" branch is reserved to test full deploy process without impacting the real content. + if [ "$TRAVIS_BRANCH" == "develop_doc" ]; then + PPO_SCRIPT_BRANCH=develop + elif [[ "$TRAVIS_BRANCH" == "develop" || "$TRAVIS_BRANCH" =~ ^v|release/[[:digit:]]+\.[[:digit:]]+(\.[[:digit:]]+)?(-\S*)?$ ]]; then + PPO_SCRIPT_BRANCH=master + else + # Early exit, this branch doesn't require documentation build + return 0; + fi + # Fetch the paddlepaddle.org deploy_docs.sh from the appopriate branch + export DEPLOY_DOCS_SH=https://raw.githubusercontent.com/PaddlePaddle/PaddlePaddle.org/$PPO_SCRIPT_BRANCH/scripts/deploy/deploy_docs.sh + export PYTHONPATH=$PYTHONPATH:${PADDLE_ROOT}/build/python:/paddle/build/python + cd .. + curl $DEPLOY_DOCS_SH | bash -s $CONTENT_DEC_PASSWD $TRAVIS_BRANCH ${PADDLE_ROOT} ${PADDLE_ROOT}/build/doc/ ${PPO_SCRIPT_BRANCH} + cd - } function gen_html() { diff --git a/paddle/scripts/paddle_docker_build.sh b/paddle/scripts/paddle_docker_build.sh index 3462deb9c2f88b6da643d6aa833449ed5f4a9b34..174c2a12f007b282a5182c0aec9b0a6bec9e55fa 100755 --- a/paddle/scripts/paddle_docker_build.sh +++ b/paddle/scripts/paddle_docker_build.sh @@ -52,6 +52,9 @@ EOL ${DOCKER_CMD} run -it \ ${DOCKER_ENV} \ -e SCRIPT_NAME=$0 \ + -e CONTENT_DEC_PASSWD=$CONTENT_DEC_PASSWD \ + -e TRAVIS_BRANCH=$TRAVIS_BRANCH \ + -e TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST \ -v $PADDLE_ROOT:/paddle \ -v ${HOME}/.ccache:/root/.ccache \ -w /paddle \ diff --git a/python/paddle/dataset/cifar.py b/python/paddle/dataset/cifar.py index 79ddd8b7e6f31383fa531f398ef37315b92a9807..f6b4ff8fbd0f83b1d652d37c1b2d04efd3c73cbb 100644 --- a/python/paddle/dataset/cifar.py +++ b/python/paddle/dataset/cifar.py @@ -28,11 +28,12 @@ images per class. """ -import cPickle import itertools import numpy import paddle.dataset.common import tarfile +from six.moves import zip +from six.moves import cPickle as pickle __all__ = ['train100', 'test100', 'train10', 'test10', 'convert'] @@ -48,7 +49,7 @@ def reader_creator(filename, sub_name, cycle=False): data = batch['data'] labels = batch.get('labels', batch.get('fine_labels', None)) assert labels is not None - for sample, label in itertools.izip(data, labels): + for sample, label in zip(data, labels): yield (sample / 255.0).astype(numpy.float32), int(label) def reader(): @@ -58,7 +59,7 @@ def reader_creator(filename, sub_name, cycle=False): while True: for name in names: - batch = cPickle.load(f.extractfile(name)) + batch = pickle.load(f.extractfile(name)) for item in read_batch(batch): yield item if not cycle: diff --git a/python/paddle/dataset/common.py b/python/paddle/dataset/common.py index 68660601c161d2332b17b448fae089506238ba78..6195cc50df338e83bea1f4ad416529464636a33e 100644 --- a/python/paddle/dataset/common.py +++ b/python/paddle/dataset/common.py @@ -20,9 +20,8 @@ import shutil import sys import importlib import paddle.dataset -import cPickle +import six.moves.cPickle as pickle import glob -import cPickle as pickle __all__ = [ 'DATA_HOME', @@ -75,13 +74,13 @@ def download(url, module_name, md5sum, save_name=None): retry_limit = 3 while not (os.path.exists(filename) and md5file(filename) == md5sum): if os.path.exists(filename): - print "file md5", md5file(filename), md5sum + print("file md5", md5file(filename), md5sum) if retry < retry_limit: retry += 1 else: raise RuntimeError("Cannot download {0} within retry limit {1}". format(url, retry_limit)) - print "Cache file %s not found, downloading %s" % (filename, url) + print("Cache file %s not found, downloading %s" % (filename, url)) r = requests.get(url, stream=True) total_length = r.headers.get('content-length') @@ -104,8 +103,9 @@ def download(url, module_name, md5sum, save_name=None): def fetch_all(): - for module_name in filter(lambda x: not x.startswith("__"), - dir(paddle.dataset)): + for module_name in [ + x for x in dir(paddle.dataset) if not x.startswith("__") + ]: if "fetch" in dir( importlib.import_module("paddle.dataset.%s" % module_name)): getattr( @@ -114,8 +114,9 @@ def fetch_all(): def fetch_all_recordio(path): - for module_name in filter(lambda x: not x.startswith("__"), - dir(paddle.dataset)): + for module_name in [ + x for x in dir(paddle.dataset) if not x.startswith("__") + ]: if "convert" in dir( importlib.import_module("paddle.dataset.%s" % module_name)) and \ not module_name == "common": @@ -126,7 +127,7 @@ def fetch_all_recordio(path): "convert")(ds_path) -def split(reader, line_count, suffix="%05d.pickle", dumper=cPickle.dump): +def split(reader, line_count, suffix="%05d.pickle", dumper=pickle.dump): """ you can call the function as: @@ -167,7 +168,7 @@ def split(reader, line_count, suffix="%05d.pickle", dumper=cPickle.dump): def cluster_files_reader(files_pattern, trainer_count, trainer_id, - loader=cPickle.load): + loader=pickle.load): """ Create a reader that yield element from the given files, select a file set according trainer count and trainer_id @@ -188,7 +189,7 @@ def cluster_files_reader(files_pattern, my_file_list = [] for idx, fn in enumerate(file_list): if idx % trainer_count == trainer_id: - print "append file: %s" % fn + print("append file: %s" % fn) my_file_list.append(fn) for fn in my_file_list: with open(fn, "r") as f: @@ -221,7 +222,7 @@ def convert(output_path, reader, line_count, name_prefix): for l in lines: # FIXME(Yancey1989): # dumps with protocol: pickle.HIGHEST_PROTOCOL - writer.write(cPickle.dumps(l)) + writer.write(pickle.dumps(l)) writer.close() lines = [] diff --git a/python/paddle/dataset/conll05.py b/python/paddle/dataset/conll05.py index 4e94ce89892f8e6822c15fdc510805e75dfca988..a97c95d067b876a87f0aa19b2ddd0702a848bd4a 100644 --- a/python/paddle/dataset/conll05.py +++ b/python/paddle/dataset/conll05.py @@ -24,18 +24,19 @@ import tarfile import gzip import itertools import paddle.dataset.common +from six.moves import zip __all__ = ['test, get_dict', 'get_embedding', 'convert'] DATA_URL = 'http://www.cs.upc.edu/~srlconll/conll05st-tests.tar.gz' DATA_MD5 = '387719152ae52d60422c016e92a742fc' -WORDDICT_URL = 'http://paddlepaddle.bj.bcebos.com/demo/srl_dict_and_embedding/wordDict.txt' +WORDDICT_URL = 'http://paddlemodels.bj.bcebos.com/conll05st%2FwordDict.txt' WORDDICT_MD5 = 'ea7fb7d4c75cc6254716f0177a506baa' -VERBDICT_URL = 'http://paddlepaddle.bj.bcebos.com/demo/srl_dict_and_embedding/verbDict.txt' +VERBDICT_URL = 'http://paddlemodels.bj.bcebos.com/conll05st%2FverbDict.txt' VERBDICT_MD5 = '0d2977293bbb6cbefab5b0f97db1e77c' -TRGDICT_URL = 'http://paddlepaddle.bj.bcebos.com/demo/srl_dict_and_embedding/targetDict.txt' +TRGDICT_URL = 'http://paddlemodels.bj.bcebos.com/conll05st%2FtargetDict.txt' TRGDICT_MD5 = 'd8c7f03ceb5fc2e5a0fa7503a4353751' -EMB_URL = 'http://paddlepaddle.bj.bcebos.com/demo/srl_dict_and_embedding/emb' +EMB_URL = 'http://paddlemodels.bj.bcebos.com/conll05st%2Femb' EMB_MD5 = 'bf436eb0faa1f6f9103017f8be57cdb7' UNK_IDX = 0 @@ -87,12 +88,12 @@ def corpus_reader(data_path, words_name, props_name): sentences = [] labels = [] one_seg = [] - for word, label in itertools.izip(words_file, props_file): + for word, label in zip(words_file, props_file): word = word.strip() label = label.strip().split() if len(label) == 0: # end of sentence - for i in xrange(len(one_seg[0])): + for i in range(len(one_seg[0])): a_kind_lable = [x[i] for x in one_seg] labels.append(a_kind_lable) diff --git a/python/paddle/dataset/flowers.py b/python/paddle/dataset/flowers.py index 2354987d20b908a32209f9ac22a2065ee43c3dfd..914dae348bc94d061072543aa14aba2219f4b52d 100644 --- a/python/paddle/dataset/flowers.py +++ b/python/paddle/dataset/flowers.py @@ -28,10 +28,9 @@ Graphics and Image Processing (2008) http://www.robots.ox.ac.uk/~vgg/publications/papers/nilsback08.{pdf,ps.gz}. """ -import cPickle import itertools import functools -from common import download +from .common import download import tarfile import scipy.io as scio from paddle.dataset.image import * @@ -39,6 +38,8 @@ from paddle.reader import * import os import numpy as np from multiprocessing import cpu_count +from six.moves import cPickle as pickle +from six.moves import zip __all__ = ['train', 'test', 'valid'] DATA_URL = 'http://www.robots.ox.ac.uk/~vgg/data/flowers/102/102flowers.tgz' @@ -116,10 +117,10 @@ def reader_creator(data_file, file = file.strip() batch = None with open(file, 'r') as f: - batch = cPickle.load(f) + batch = pickle.load(f) data = batch['data'] labels = batch['label'] - for sample, label in itertools.izip(data, batch['label']): + for sample, label in zip(data, batch['label']): yield sample, int(label) - 1 if not cycle: break diff --git a/python/paddle/dataset/image.py b/python/paddle/dataset/image.py index 9235c41e9eb95b25a0dc53a494a203e7a4525981..3b3d89c93c48d611dccf6f14958c310a6cac1a7b 100644 --- a/python/paddle/dataset/image.py +++ b/python/paddle/dataset/image.py @@ -36,7 +36,7 @@ except ImportError: cv2 = None import os import tarfile -import cPickle +import six.moves.cPickle as pickle __all__ = [ "load_image_bytes", "load_image", "resize_short", "to_chw", "center_crop", @@ -86,10 +86,10 @@ def batch_images_from_tar(data_file, output = {} output['label'] = labels output['data'] = data - cPickle.dump( + pickle.dump( output, open('%s/batch_%d' % (out_path, file_id), 'w'), - protocol=cPickle.HIGHEST_PROTOCOL) + protocol=pickle.HIGHEST_PROTOCOL) file_id += 1 data = [] labels = [] @@ -97,10 +97,10 @@ def batch_images_from_tar(data_file, output = {} output['label'] = labels output['data'] = data - cPickle.dump( + pickle.dump( output, open('%s/batch_%d' % (out_path, file_id), 'w'), - protocol=cPickle.HIGHEST_PROTOCOL) + protocol=pickle.HIGHEST_PROTOCOL) with open(meta_file, 'a') as meta: for file in os.listdir(out_path): diff --git a/python/paddle/dataset/imdb.py b/python/paddle/dataset/imdb.py index 5ff05b1e9b7f4c42909370a21beb140ecdcd6868..e7fe4e0b7e5832c2bc7ca1307725936a70292c39 100644 --- a/python/paddle/dataset/imdb.py +++ b/python/paddle/dataset/imdb.py @@ -42,13 +42,13 @@ def tokenize(pattern): # sequential access of member files, other than # tarfile.extractfile, which does random access and might # destroy hard disks. - tf = tarf.next() + tf = next(tarf) while tf != None: if bool(pattern.match(tf.name)): # newline and punctuations removal and ad-hoc tokenization. yield tarf.extractfile(tf).read().rstrip("\n\r").translate( None, string.punctuation).lower().split() - tf = tarf.next() + tf = next(tarf) def build_dict(pattern, cutoff): @@ -62,11 +62,11 @@ def build_dict(pattern, cutoff): word_freq[word] += 1 # Not sure if we should prune less-frequent words here. - word_freq = filter(lambda x: x[1] > cutoff, word_freq.items()) + word_freq = [x for x in list(word_freq.items()) if x[1] > cutoff] dictionary = sorted(word_freq, key=lambda x: (-x[1], x[0])) words, _ = list(zip(*dictionary)) - word_idx = dict(zip(words, xrange(len(words)))) + word_idx = dict(list(zip(words, list(range(len(words)))))) word_idx[''] = len(words) return word_idx diff --git a/python/paddle/dataset/imikolov.py b/python/paddle/dataset/imikolov.py index c6c0a0f54373dd068b2c493f6fbc9c8578593aef..bc007c9d3c8e2f1e4ff091f7c2c93eacbbe8d0e0 100644 --- a/python/paddle/dataset/imikolov.py +++ b/python/paddle/dataset/imikolov.py @@ -64,11 +64,11 @@ def build_dict(min_word_freq=50): # remove for now, since we will set it as last index del word_freq[''] - word_freq = filter(lambda x: x[1] > min_word_freq, word_freq.items()) + word_freq = [x for x in list(word_freq.items()) if x[1] > min_word_freq] word_freq_sorted = sorted(word_freq, key=lambda x: (-x[1], x[0])) words, _ = list(zip(*word_freq_sorted)) - word_idx = dict(zip(words, xrange(len(words)))) + word_idx = dict(list(zip(words, list(range(len(words)))))) word_idx[''] = len(words) return word_idx diff --git a/python/paddle/dataset/mnist.py b/python/paddle/dataset/mnist.py index 6259cc35b4f7bb781886bb5da9d16924831d7246..ffa9008c80129b80b3807dbab37bc198e59cf5a2 100644 --- a/python/paddle/dataset/mnist.py +++ b/python/paddle/dataset/mnist.py @@ -65,7 +65,7 @@ def reader_creator(image_filename, label_filename, buffer_size): images = images / 255.0 * 2.0 - 1.0 - for i in xrange(buffer_size): + for i in range(buffer_size): yield images[i, :], int(labels[i]) finally: try: diff --git a/python/paddle/dataset/movielens.py b/python/paddle/dataset/movielens.py index ab11716202a8298c182e23b661eb1d2ac74bf4da..056ec2178607329dd6daa1764820c2312bbaed59 100644 --- a/python/paddle/dataset/movielens.py +++ b/python/paddle/dataset/movielens.py @@ -16,7 +16,7 @@ Movielens 1-M dataset. Movielens 1-M dataset contains 1 million ratings from 6000 users on 4000 movies, which was collected by GroupLens Research. This module will download -Movielens 1-M dataset from +Movielens 1-M dataset from http://files.grouplens.org/datasets/movielens/ml-1m.zip and parse training set and test set into paddle reader creators. @@ -187,7 +187,7 @@ def max_movie_id(): Get the maximum value of movie id. """ __initialize_meta_info__() - return reduce(__max_index_info__, MOVIE_INFO.viewvalues()).index + return reduce(__max_index_info__, list(MOVIE_INFO.values())).index def max_user_id(): @@ -195,7 +195,7 @@ def max_user_id(): Get the maximum value of user id. """ __initialize_meta_info__() - return reduce(__max_index_info__, USER_INFO.viewvalues()).index + return reduce(__max_index_info__, list(USER_INFO.values())).index def __max_job_id_impl__(a, b): @@ -210,7 +210,7 @@ def max_job_id(): Get the maximum value of job id. """ __initialize_meta_info__() - return reduce(__max_job_id_impl__, USER_INFO.viewvalues()).job_id + return reduce(__max_job_id_impl__, list(USER_INFO.values())).job_id def movie_categories(): @@ -243,7 +243,7 @@ def unittest(): for test_count, _ in enumerate(test()()): pass - print train_count, test_count + print(train_count, test_count) def fetch(): diff --git a/python/paddle/dataset/mq2007.py b/python/paddle/dataset/mq2007.py index d3b3dd524c34be660c5f2d4fc5ce2fa0420efbc1..cc4d088316dfd490dc9d6b247c66c2495cedf2c3 100644 --- a/python/paddle/dataset/mq2007.py +++ b/python/paddle/dataset/mq2007.py @@ -26,7 +26,7 @@ http://research.microsoft.com/en-us/um/beijing/projects/letor/LETOR4.0/Data/MQ20 import os import functools import rarfile -from common import download +from .common import download import numpy as np # URL = "http://research.microsoft.com/en-us/um/beijing/projects/letor/LETOR4.0/Data/MQ2007.rar" @@ -53,7 +53,7 @@ class Query(object): ---------- query_id : int query_id in dataset, mapping from query to relevance documents - relevance_score : int + relevance_score : int relevance score of query and document pair feature_vector : array, dense feature feature in vector format @@ -92,7 +92,7 @@ class Query(object): sys.stdout.write("expect 48 space split parts, get %d" % (len(parts))) return None - # format : 0 qid:10 1:0.000272 2:0.000000 .... + # format : 0 qid:10 1:0.000272 2:0.000000 .... self.relevance_score = int(parts[0]) self.query_id = int(parts[1].split(':')[1]) for p in parts[2:]: @@ -295,7 +295,7 @@ def __reader__(filepath, format="pairwise", shuffle=False, fill_missing=-1): -------- filename : string fill_missing : fill the missing value. default in MQ2007 is -1 - + Returns ------ yield @@ -330,4 +330,4 @@ if __name__ == "__main__": mytest = functools.partial( __reader__, filepath="MQ2007/MQ2007/Fold1/sample", format="listwise") for label, query in mytest(): - print label, query + print(label, query) diff --git a/python/paddle/dataset/sentiment.py b/python/paddle/dataset/sentiment.py index f5461164fe6b816356e42fc7b7dcf388eccfadfb..953ada057bc114ebbfe39011d2fd3b5b7a2b0d37 100644 --- a/python/paddle/dataset/sentiment.py +++ b/python/paddle/dataset/sentiment.py @@ -43,11 +43,11 @@ def download_data_if_not_yet(): nltk.data.path.append(paddle.dataset.common.DATA_HOME) movie_reviews.categories() except LookupError: - print "Downloading movie_reviews data set, please wait....." + print("Downloading movie_reviews data set, please wait.....") nltk.download( 'movie_reviews', download_dir=paddle.dataset.common.DATA_HOME) - print "Download data set success....." - print "Path is " + nltk.data.find('corpora/movie_reviews').path + print("Download data set success.....") + print("Path is " + nltk.data.find('corpora/movie_reviews').path) def get_word_dict(): @@ -64,7 +64,7 @@ def get_word_dict(): for field in movie_reviews.fileids(category): for words in movie_reviews.words(field): word_freq_dict[words] += 1 - words_sort_list = word_freq_dict.items() + words_sort_list = list(word_freq_dict.items()) words_sort_list.sort(cmp=lambda a, b: b[1] - a[1]) for index, word in enumerate(words_sort_list): words_freq_sorted.append((word[0], index)) @@ -80,7 +80,8 @@ def sort_files(): files_list = list() neg_file_list = movie_reviews.fileids('neg') pos_file_list = movie_reviews.fileids('pos') - files_list = list(chain.from_iterable(zip(neg_file_list, pos_file_list))) + files_list = list( + chain.from_iterable(list(zip(neg_file_list, pos_file_list)))) return files_list diff --git a/python/paddle/dataset/tests/common_test.py b/python/paddle/dataset/tests/common_test.py index e7cc02aa83061599ffefa18de6cb02ac0fc9e9b7..777cd06a19726f8ad73774c958c8cb512808a3aa 100644 --- a/python/paddle/dataset/tests/common_test.py +++ b/python/paddle/dataset/tests/common_test.py @@ -36,7 +36,7 @@ class TestCommon(unittest.TestCase): def test_split(self): def test_reader(): def reader(): - for x in xrange(10): + for x in range(10): yield x return reader @@ -49,7 +49,7 @@ class TestCommon(unittest.TestCase): def test_cluster_file_reader(self): _, temp_path = tempfile.mkstemp() - for x in xrange(5): + for x in range(5): with open(temp_path + '/%05d.test' % x) as f: f.write('%d\n' % x) reader = paddle.dataset.common.cluster_files_reader( @@ -63,7 +63,7 @@ class TestCommon(unittest.TestCase): def test_reader(): def reader(): - for x in xrange(record_num): + for x in range(record_num): yield x return reader diff --git a/python/paddle/dataset/tests/imikolov_test.py b/python/paddle/dataset/tests/imikolov_test.py index 233fd9fc8cea4cd0b5cd052580030fc8c993693c..50f50d947d221686d6308a6ed44cbcff3b10c6f5 100644 --- a/python/paddle/dataset/tests/imikolov_test.py +++ b/python/paddle/dataset/tests/imikolov_test.py @@ -59,7 +59,7 @@ class TestMikolov(unittest.TestCase): self.assertEqual(first_line, read_line) def test_total(self): - _, idx = zip(*WORD_DICT.items()) + _, idx = list(zip(*list(WORD_DICT.items()))) self.assertEqual(sorted(idx)[-1], len(WORD_DICT) - 1) diff --git a/python/paddle/dataset/tests/test_sentiment.py b/python/paddle/dataset/tests/test_sentiment.py index 543f4b7378b583ea3857bf785cf330c43e535c2a..37326517f7b39fb74c694684eb8a547d5f021946 100644 --- a/python/paddle/dataset/tests/test_sentiment.py +++ b/python/paddle/dataset/tests/test_sentiment.py @@ -24,9 +24,8 @@ from nltk.corpus import movie_reviews class TestSentimentMethods(unittest.TestCase): def test_get_word_dict(self): word_dict = st.get_word_dict()[0:10] - test_word_list = [(u',', 0), (u'the', 1), (u'.', 2), (u'a', 3), - (u'and', 4), (u'of', 5), (u'to', 6), (u"'", 7), - (u'is', 8), (u'in', 9)] + test_word_list = [(',', 0), ('the', 1), ('.', 2), ('a', 3), ('and', 4), + ('of', 5), ('to', 6), ("'", 7), ('is', 8), ('in', 9)] for idx, each in enumerate(word_dict): self.assertEqual(each, test_word_list[idx]) self.assertTrue("/root/.cache/paddle/dataset" in nltk.data.path) diff --git a/python/paddle/dataset/uci_housing.py b/python/paddle/dataset/uci_housing.py index fbfa477d055eb5f484989eacce38cee8d617d729..410ca7af0d6d1dc26acbf92fce5e49fce7d3a3bb 100644 --- a/python/paddle/dataset/uci_housing.py +++ b/python/paddle/dataset/uci_housing.py @@ -49,9 +49,12 @@ def feature_range(maximums, minimums): import matplotlib.pyplot as plt fig, ax = plt.subplots() feature_num = len(maximums) - ax.bar(range(feature_num), maximums - minimums, color='r', align='center') + ax.bar(list(range(feature_num)), + maximums - minimums, + color='r', + align='center') ax.set_title('feature scale') - plt.xticks(range(feature_num), feature_names) + plt.xticks(list(range(feature_num)), feature_names) plt.xlim([-1, feature_num]) fig.set_figheight(6) fig.set_figwidth(10) @@ -71,7 +74,7 @@ def load_data(filename, feature_num=14, ratio=0.8): maximums, minimums, avgs = data.max(axis=0), data.min(axis=0), data.sum( axis=0) / data.shape[0] feature_range(maximums[:-1], minimums[:-1]) - for i in xrange(feature_num - 1): + for i in range(feature_num - 1): data[:, i] = (data[:, i] - avgs[i]) / (maximums[i] - minimums[i]) offset = int(data.shape[0] * ratio) UCI_TRAIN_DATA = data[:offset] diff --git a/python/paddle/dataset/wmt14.py b/python/paddle/dataset/wmt14.py index f0908c737874fa7335cca5b5f0cba83190c9f90f..c9a257ba3f86aac79bfe891d78fdaf36d2df8a50 100644 --- a/python/paddle/dataset/wmt14.py +++ b/python/paddle/dataset/wmt14.py @@ -40,7 +40,7 @@ URL_TRAIN = ('http://paddlepaddle.cdn.bcebos.com/demo/' 'wmt_shrinked_data/wmt14.tgz') MD5_TRAIN = '0791583d57d5beb693b9414c5b36798c' # BLEU of this trained model is 26.92 -URL_MODEL = 'http://paddlepaddle.bj.bcebos.com/demo/wmt_14/wmt14_model.tar.gz' +URL_MODEL = 'http://paddlemodels.bj.bcebos.com/wmt%2Fwmt14.tgz' MD5_MODEL = '0cb4a5366189b6acba876491c8724fa3' START = "" @@ -154,8 +154,8 @@ def get_dict(dict_size, reverse=True): tar_file = paddle.dataset.common.download(URL_TRAIN, 'wmt14', MD5_TRAIN) src_dict, trg_dict = __read_to_dict(tar_file, dict_size) if reverse: - src_dict = {v: k for k, v in src_dict.items()} - trg_dict = {v: k for k, v in trg_dict.items()} + src_dict = {v: k for k, v in list(src_dict.items())} + trg_dict = {v: k for k, v in list(trg_dict.items())} return src_dict, trg_dict diff --git a/python/paddle/dataset/wmt16.py b/python/paddle/dataset/wmt16.py index 540d43b692e0f65460f558dd74a52715ff4db68d..4e3c466c38e402cc574e93ef3a5935edf8f9dd3b 100644 --- a/python/paddle/dataset/wmt16.py +++ b/python/paddle/dataset/wmt16.py @@ -70,7 +70,9 @@ def __build_dict(tar_file, dict_size, save_path, lang): fout.write("%s\n%s\n%s\n" % (START_MARK, END_MARK, UNK_MARK)) for idx, word in enumerate( sorted( - word_dict.iteritems(), key=lambda x: x[1], reverse=True)): + iter(list(word_dict.items())), + key=lambda x: x[1], + reverse=True)): if idx + 3 == dict_size: break fout.write("%s\n" % (word[0])) diff --git a/python/paddle/fluid/__init__.py b/python/paddle/fluid/__init__.py index 956e3c43485b36aaeb2d366d6145edd3d4535122..2ab176ec272a9f250d5229ae67faf350e5eda72c 100644 --- a/python/paddle/fluid/__init__.py +++ b/python/paddle/fluid/__init__.py @@ -14,49 +14,49 @@ from __future__ import print_function # import all class inside framework into fluid module -import framework -from framework import * +from . import framework +from .framework import * # import all class inside executor into fluid module -import executor -from executor import * - -import trainer -from trainer import Trainer -from trainer import BeginEpochEvent -from trainer import EndEpochEvent -from trainer import BeginStepEvent -from trainer import EndStepEvent -from trainer import CheckpointConfig - -import inferencer -from inferencer import Inferencer - -import io -import evaluator -import initializer -import layers -import contrib -import nets -import optimizer -import backward -import regularizer -import average -import metrics -import transpiler -from param_attr import ParamAttr, WeightNormParamAttr -from data_feeder import DataFeeder -from core import LoDTensor, LoDTensorArray, CPUPlace, CUDAPlace, CUDAPinnedPlace, Scope -from transpiler import DistributeTranspiler, InferenceTranspiler, \ +from . import executor +from .executor import * + +from . import trainer +from .trainer import Trainer +from .trainer import BeginEpochEvent +from .trainer import EndEpochEvent +from .trainer import BeginStepEvent +from .trainer import EndStepEvent +from .trainer import CheckpointConfig + +from . import inferencer +from .inferencer import Inferencer + +from . import io +from . import evaluator +from . import initializer +from . import layers +from . import contrib +from . import nets +from . import optimizer +from . import backward +from . import regularizer +from . import average +from . import metrics +from . import transpiler +from .param_attr import ParamAttr, WeightNormParamAttr +from .data_feeder import DataFeeder +from .core import LoDTensor, LoDTensorArray, CPUPlace, CUDAPlace, CUDAPinnedPlace, Scope +from .transpiler import DistributeTranspiler, InferenceTranspiler, \ memory_optimize, release_memory, DistributeTranspilerConfig -from concurrency import (Go, make_channel, channel_send, channel_recv, - channel_close, Select) -from lod_tensor import create_lod_tensor, create_random_int_lodtensor -import clip -import profiler -import unique_name -import recordio_writer -import parallel_executor -from parallel_executor import * +from .concurrency import (Go, make_channel, channel_send, channel_recv, + channel_close, Select) +from .lod_tensor import create_lod_tensor, create_random_int_lodtensor +from . import clip +from . import profiler +from . import unique_name +from . import recordio_writer +from . import parallel_executor +from .parallel_executor import * from paddle.fluid.layers.math_op_patch import monkey_patch_variable Tensor = LoDTensor @@ -99,8 +99,8 @@ def __bootstrap__(): None """ import sys - import core import os + from . import core in_test = 'unittest' in sys.modules @@ -123,7 +123,8 @@ def __bootstrap__(): read_env_flags = [ 'use_pinned_memory', 'check_nan_inf', 'benchmark', 'warpctc_dir', 'eager_delete_scope', 'use_mkldnn', 'initial_cpu_memory_in_mb', - 'init_allocated_mem', 'free_idle_memory', 'paddle_num_threads' + 'init_allocated_mem', 'free_idle_memory', 'paddle_num_threads', + 'cpu_deterministic' ] if core.is_compiled_with_dist(): read_env_flags.append('rpc_deadline') diff --git a/python/paddle/fluid/annotations.py b/python/paddle/fluid/annotations.py index bb8756a4664013643c278c013ca21bb237a6b4a7..15e7976354f2a22065f1723bfa696d056181dac2 100644 --- a/python/paddle/fluid/annotations.py +++ b/python/paddle/fluid/annotations.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function import functools import sys @@ -28,7 +29,7 @@ def deprecated(since, instead, extra_message=""): @functools.wraps(func) def wrapper(*args, **kwargs): - print >> sys.stderr, err_msg + print(err_msg, file=sys.stderr) return func(*args, **kwargs) wrapper.__doc__ += "\n " diff --git a/python/paddle/fluid/backward.py b/python/paddle/fluid/backward.py index 812f68bdd849544456b2e0ebf0b739f4f92b09ea..6b739745119836a80518d7bc6138b37101e74431 100644 --- a/python/paddle/fluid/backward.py +++ b/python/paddle/fluid/backward.py @@ -16,7 +16,8 @@ from paddle.fluid import framework as framework from . import core import collections import copy -import unique_name +import six +from . import unique_name __all__ = ['append_backward'] @@ -44,17 +45,25 @@ def _create_op_desc_(op_type, inputs, outputs, attrs): """ op_desc = core.OpDesc() op_desc.set_type(op_type) - for para, args in inputs.iteritems(): - op_desc.set_input(para, args) - for para, args in outputs.iteritems(): - op_desc.set_output(para, args) + for para, args in list(inputs.items()): + op_desc.set_input( + para, + list( + map(lambda arg: arg.decode() if isinstance(arg, six.binary_type) else arg, + args))) + for para, args in list(outputs.items()): + op_desc.set_output( + para, + list( + map(lambda arg: arg.decode() if isinstance(arg, six.binary_type) else arg, + args))) op_role_attr_name = core.op_proto_and_checker_maker.kOpRoleAttrName() if op_role_attr_name not in attrs: attrs[ op_role_attr_name] = core.op_proto_and_checker_maker.OpRole.Backward - for name, val in attrs.iteritems(): + for name, val in list(attrs.items()): if isinstance(val, framework.Block): op_desc.set_block_attr(name, val.desc) else: @@ -105,7 +114,9 @@ def _strip_grad_suffix_(name): e.g. x@GRAD ==> x y@GRAD@RENAME@1 ==> y """ - pos = name.find(core.grad_var_suffix()) + if isinstance(name, six.text_type): + name = name.encode() + pos = name.find(six.b(core.grad_var_suffix())) return name[:pos] if pos != -1 else name @@ -114,7 +125,9 @@ def _append_grad_suffix_(name): Append grad suffix to the given variable name e.g. x ==> x@GRAD """ - return name + core.grad_var_suffix() + if isinstance(name, six.text_type): + name = name.encode() + return name + six.b(core.grad_var_suffix()) def _addup_repetitive_outputs_(op_descs): @@ -174,7 +187,7 @@ def _addup_repetitive_outputs_(op_descs): op_desc.set_output(param_name, arg_names) renamed_vars[var_name].append(new_name) - for var_name, inputs in renamed_vars.iteritems(): + for var_name, inputs in list(renamed_vars.items()): if len(inputs) > 1: pending_sum_ops.append( (_create_op_desc_("sum", {"X": inputs}, {"Out": [var_name]}, @@ -198,16 +211,19 @@ def _remove_no_grad_branch_(op_descs, no_grad_set): out_arg_names = op_desc.output_arg_names() if len(out_arg_names) == 0 or _all_in_set_(out_arg_names, no_grad_set): return True - if _all_in_set_( - filter(lambda name: name.find(core.grad_var_suffix()) != -1, - op_desc.input_arg_names()), no_grad_set): + if _all_in_set_([ + name for name in op_desc.input_arg_names() + if name.find(core.grad_var_suffix()) != -1 + ], no_grad_set): no_grad_set.update(out_arg_names) return True return False # Remove ops whose outputs are all in no_grad_dict - op_descs = filter( - lambda op_desc: not _op_can_be_removed_(op_desc, no_grad_set), op_descs) + op_descs = [ + op_desc for op_desc in op_descs + if not _op_can_be_removed_(op_desc, no_grad_set) + ] # Insert fill_zeros_like_op to_insert = [] for idx, op_desc in enumerate(op_descs): @@ -217,12 +233,12 @@ def _remove_no_grad_branch_(op_descs, no_grad_set): "X": [_strip_grad_suffix_(arg)] }, {"Out": [arg]}, {}), idx)) - map(lambda p: op_descs.insert(p[1], p[0]), reversed(to_insert)) + list([op_descs.insert(p[1], p[0]) for p in reversed(to_insert)]) return op_descs -import proto.framework_pb2 as framework_pb2 +from .proto import framework_pb2 def serialize_op_decs(op_desc): @@ -244,8 +260,10 @@ def _callback_lookup_(op): if op.type == 'parallel_do' and op.attr('use_nccl'): all_vars = op.block.vars param_names = set(op.input('parameters')) - param_names = filter(lambda name: all_vars[name].stop_gradient is False, - param_names) + param_names = [ + name for name in param_names + if all_vars[name].stop_gradient is False + ] param_grad_names = [n + "@GRAD" for n in param_names] class ParallelDoCallBack(object): @@ -399,7 +417,7 @@ def _append_backward_vars_(block, start_op_idx, grad_to_var, grad_info_map): continue block.desc.var(grad_var_name) new_vars.add(grad_var_name) - if not grad_to_var.has_key(grad_var_name): + if grad_var_name not in grad_to_var: continue grad_info_map[grad_to_var[grad_var_name]] = (grad_var_name, block) # infer_shape and infer_type @@ -427,7 +445,7 @@ def _rename_grad_(block, start_op_idx, grad_to_var, target_grad_map): op_desc.rename_output(name, new_name) var_map[name] = new_name - for g, ng in var_map.iteritems(): + for g, ng in list(var_map.items()): if g in grad_to_var: grad_to_var[ng] = grad_to_var[g] grad_to_var.pop(g) @@ -439,7 +457,7 @@ def _get_stop_gradients_(program): for block in program.blocks: assert isinstance(block, framework.Block) block_no_grad_set = set() - for var in block.vars.itervalues(): + for var in list(block.vars.values()): assert isinstance(var, framework.Variable) if var.stop_gradient: block_no_grad_set.add(_append_grad_suffix_(var.name)) @@ -452,51 +470,51 @@ def append_backward(loss, parameter_list=None, no_grad_set=None, """ Append backward part to main_program. - A complete neural network training is made up of forward and backward - propagation. However, when we configure a network, we only need to - specify its forwrd part. The backward part is generated automatically + A complete neural network training is made up of forward and backward + propagation. However, when we configure a network, we only need to + specify its forwrd part. The backward part is generated automatically according to the forward part by this function. - In most cases, users do not need to invoke this function manually. It + In most cases, users do not need to invoke this function manually. It will be automatically invoked by the optimizer's `minimize` function. Args: loss(Variable): The loss variable of the network. - parameter_list(list[string]|None): Names of parameters that need - to be updated by optimizers. - If it is None, all parameters + parameter_list(list[string]|None): Names of parameters that need + to be updated by optimizers. + If it is None, all parameters will be updated. Default: None - no_grad_set(set|None): Variables in the Block 0 whose gradients - should be ignored. All variables with - `step_gradient=True` from all blocks will + no_grad_set(set|None): Variables in the Block 0 whose gradients + should be ignored. All variables with + `step_gradient=True` from all blocks will be automatically added into this set. Default: None - callbacks(list[callable object]|None): The callbacks are used for - doing some custom jobs during - backward part building. All - callable objects in it will - be invoked once each time a - new gradient operator is added - into the program. The callable - object must has two input - parameters: 'block' and 'context'. - The 'block' is the block which - the new gradient operator will - be added to. The 'context' is a - map, whose keys are gradient - variable names and values are + callbacks(list[callable object]|None): The callbacks are used for + doing some custom jobs during + backward part building. All + callable objects in it will + be invoked once each time a + new gradient operator is added + into the program. The callable + object must has two input + parameters: 'block' and 'context'. + The 'block' is the block which + the new gradient operator will + be added to. The 'context' is a + map, whose keys are gradient + variable names and values are corresponding original variables. - In addition to this, the 'context' - has another special key-value pair: - the key is string '__current_op_desc__' - and the value is the op_desc of the - gradient operator who has just - triggered the callable object. + In addition to this, the 'context' + has another special key-value pair: + the key is string '__current_op_desc__' + and the value is the op_desc of the + gradient operator who has just + triggered the callable object. Returns: - list[(Variable,Variable)]: Pairs of parameter and its - corresponding gradients. The key is the parameter and the + list[(Variable,Variable)]: Pairs of parameter and its + corresponding gradients. The key is the parameter and the value is gradient variable. Raises: @@ -535,7 +553,7 @@ def append_backward(loss, parameter_list=None, no_grad_set=None, no_grad_set = set() no_grad_set = copy.copy(no_grad_set) no_grad_dict = _get_stop_gradients_(program) - no_grad_dict[0].update(map(_append_grad_suffix_, no_grad_set)) + no_grad_dict[0].update(list(map(_append_grad_suffix_, no_grad_set))) grad_info_map = dict() root_block = program.block(0) @@ -558,7 +576,7 @@ def append_backward(loss, parameter_list=None, no_grad_set=None, block_no_grad_set = set(map(_strip_grad_suffix_, no_grad_dict[0])) op_path = _find_op_path_(root_block, [loss], [], block_no_grad_set) - no_grad_dict[0].update(map(_append_grad_suffix_, block_no_grad_set)) + no_grad_dict[0].update(list(map(_append_grad_suffix_, block_no_grad_set))) _append_backward_ops_(root_block, op_path, root_block, no_grad_dict, grad_to_var, callbacks) @@ -572,8 +590,6 @@ def append_backward(loss, parameter_list=None, no_grad_set=None, program.current_block_idx = current_block_idx program._sync_with_cpp() - # FIXME(zcd): prevent loss.grad optimized by mem_opt. - loss.block.var(_append_grad_suffix_(loss.name)).persistable = True if parameter_list is not None: parameters = parameter_list @@ -699,7 +715,7 @@ def calc_gradient(targets, inputs, target_gradients=None, no_grad_set=None): no_grad_set = set() no_grad_set = copy.copy(no_grad_set) no_grad_dict = _get_stop_gradients_(prog) - no_grad_dict[0].update(map(_append_grad_suffix_, no_grad_set)) + no_grad_dict[0].update(list(map(_append_grad_suffix_, no_grad_set))) fwd_op_num = block.desc.op_size() @@ -733,7 +749,7 @@ def calc_gradient(targets, inputs, target_gradients=None, no_grad_set=None): block_no_grad_set = set(map(_strip_grad_suffix_, no_grad_dict[0])) op_path = _find_op_path_(block, targets, inputs, block_no_grad_set) - no_grad_dict[0].update(map(_append_grad_suffix_, block_no_grad_set)) + no_grad_dict[0].update(list(map(_append_grad_suffix_, block_no_grad_set))) grad_to_var = dict() grad_info_map = dict() _append_backward_ops_(block, op_path, block, no_grad_dict, grad_to_var) diff --git a/python/paddle/fluid/clip.py b/python/paddle/fluid/clip.py index c029662ebc1b7e7f7d1ea44b4ebd4b08b812a579..4b0a792f784fffcce3f911d3e7448b472d39f8e1 100644 --- a/python/paddle/fluid/clip.py +++ b/python/paddle/fluid/clip.py @@ -13,10 +13,11 @@ # limitations under the License. import copy +import six import functools -import layers -import framework +from . import layers +from . import framework from . import core __all__ = [ @@ -80,8 +81,7 @@ def error_clip_callback(block, context): # the context is a grad_to_var map grad_to_var = context op_desc = block.desc.op(block.desc.op_size() - 1) - for grad_n in filter(lambda n: grad_to_var.has_key(n), - op_desc.output_arg_names()): + for grad_n in [n for n in op_desc.output_arg_names() if n in grad_to_var]: fwd_var = block._var_recursive(grad_to_var[grad_n]) error_clip = getattr(fwd_var, "error_clip", None) if not (error_clip is None or isinstance(error_clip, @@ -247,8 +247,8 @@ class GradientClipByGlobalNorm(BaseGradientClipAttr): """ def __init__(self, clip_norm, group_name="default_group"): - if not isinstance(group_name, basestring): - raise TypeError("'group_name' must be a basestring.") + if not isinstance(group_name, six.string_types): + raise TypeError("'group_name' must be a %s." % (six.string_types)) self.clip_norm = clip_norm self.group_name = group_name @@ -284,7 +284,7 @@ class GradientClipByGlobalNorm(BaseGradientClipAttr): x=clip_var, y=layers.elementwise_max( x=clip_var, y=group_norm_var)) - assert group_scale_var.shape == (1L, ) + assert group_scale_var.shape == (1, ) self.context[group_scale_name] = group_scale_var new_grad = layers.elementwise_mul( @@ -313,7 +313,7 @@ def set_gradient_clip(clip, param_list=None, program=None): program = framework.default_main_program() if param_list is None: param_list = program.block(0).all_parameters() - if all(isinstance(elem, basestring) for elem in param_list): + if all(isinstance(elem, six.string_types) for elem in param_list): param_list = [program.block(0).var(elem) for elem in param_list] if not all(isinstance(elem, framework.Parameter) for elem in param_list): raise TypeError( diff --git a/python/paddle/fluid/concurrency.py b/python/paddle/fluid/concurrency.py index b8fe9bd4c1988dd3f6fa82df391c3059dfbfcf93..a8c4d66720d6eda857e5960d86fc3b8ec8f11ade 100644 --- a/python/paddle/fluid/concurrency.py +++ b/python/paddle/fluid/concurrency.py @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from layers.control_flow import BlockGuard, equal +from .layers.control_flow import BlockGuard, equal from .framework import Operator -from layer_helper import LayerHelper, unique_name -from layers import fill_constant -import core +from .layer_helper import LayerHelper, unique_name +from .layers import fill_constant +from . import core __all__ = [ 'Go', 'make_channel', 'channel_send', 'channel_recv', 'channel_close', diff --git a/python/paddle/fluid/contrib/__init__.py b/python/paddle/fluid/contrib/__init__.py index 12cd5d918e93181c6b7e328e6aee4ad941b0a0da..58f2da1c3ba2f84602e7a18c7b1c78d1f0d2ede1 100644 --- a/python/paddle/fluid/contrib/__init__.py +++ b/python/paddle/fluid/contrib/__init__.py @@ -12,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import decoder -from decoder import * +from . import decoder +from .decoder import * +from . import memory_usage_calc +from .memory_usage_calc import * -__all__ = decoder.__all__ +__all__ = decoder.__all__ + memory_usage_calc.__all__ diff --git a/python/paddle/fluid/contrib/decoder/__init__.py b/python/paddle/fluid/contrib/decoder/__init__.py index 22cfe692690a686f32eba34ee34b9193f0d5ba35..6343c1543d206f82e605c5c986fa91d70c467113 100644 --- a/python/paddle/fluid/contrib/decoder/__init__.py +++ b/python/paddle/fluid/contrib/decoder/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import beam_search_decoder -from beam_search_decoder import * +from . import beam_search_decoder +from .beam_search_decoder import * __all__ = beam_search_decoder.__all__ diff --git a/python/paddle/fluid/contrib/decoder/beam_search_decoder.py b/python/paddle/fluid/contrib/decoder/beam_search_decoder.py index ba6e13878291ad9f30e92f998767df6d8c6f32c3..d268a948f7a2cf038a419c95521b81088ed8215f 100644 --- a/python/paddle/fluid/contrib/decoder/beam_search_decoder.py +++ b/python/paddle/fluid/contrib/decoder/beam_search_decoder.py @@ -22,6 +22,7 @@ This API is still under active development and may change drastically. import contextlib import numpy as np +import six from ... import layers from ...framework import Variable @@ -191,7 +192,7 @@ class StateCell(object): self._helper = LayerHelper('state_cell', name=name) self._cur_states = {} self._state_names = [] - for state_name, state in states.items(): + for state_name, state in six.iteritems(states): if not isinstance(state, InitState): raise ValueError('state must be an InitState object.') self._cur_states[state_name] = state @@ -346,7 +347,7 @@ class StateCell(object): if self._in_decoder and not self._switched_decoder: self._switch_decoder() - for input_name, input_value in inputs.items(): + for input_name, input_value in six.iteritems(inputs): if input_name not in self._inputs: raise ValueError('Unknown input %s. ' 'Please make sure %s in input ' @@ -361,7 +362,7 @@ class StateCell(object): if self._in_decoder and not self._switched_decoder: self._switched_decoder() - for state_name, decoder_state in self._states_holder.items(): + for state_name, decoder_state in six.iteritems(self._states_holder): if id(self._cur_decoder_obj) not in decoder_state: raise ValueError('Unknown decoder object, please make sure ' 'switch_decoder been invoked.') @@ -671,7 +672,7 @@ class BeamSearchDecoder(object): feed_dict = {} update_dict = {} - for init_var_name, init_var in self._input_var_dict.items(): + for init_var_name, init_var in six.iteritems(self._input_var_dict): if init_var_name not in self.state_cell._inputs: raise ValueError('Variable ' + init_var_name + ' not found in StateCell!\n') @@ -721,7 +722,8 @@ class BeamSearchDecoder(object): self.state_cell.update_states() self.update_array(prev_ids, selected_ids) self.update_array(prev_scores, selected_scores) - for update_name, var_to_update in update_dict.items(): + for update_name, var_to_update in six.iteritems( + update_dict): self.update_array(var_to_update, feed_dict[update_name]) def read_array(self, init, is_ids=False, is_scores=False): diff --git a/python/paddle/fluid/contrib/memory_usage_calc.py b/python/paddle/fluid/contrib/memory_usage_calc.py new file mode 100644 index 0000000000000000000000000000000000000000..5da846edb63c28efd791fdfac4046cfa56c24181 --- /dev/null +++ b/python/paddle/fluid/contrib/memory_usage_calc.py @@ -0,0 +1,102 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +This module privides a memory usage calculate function for user. +The purpose of this API is to allow users to estimate memory usage of +a program under a special batch size, then user can set appropriate +batch size to fully utilize a GPU. + +This API is still under active development and may change drastically. +""" + +from .. import core +from ..framework import Program, Variable + +__all__ = ['memory_usage'] + +dtype_to_size = { + core.VarDesc.VarType.FP16: 2, + core.VarDesc.VarType.FP32: 4, + core.VarDesc.VarType.FP64: 8, + core.VarDesc.VarType.INT16: 2, + core.VarDesc.VarType.INT32: 4, + core.VarDesc.VarType.INT64: 8, + core.VarDesc.VarType.BOOL: 1, + core.VarDesc.VarType.UINT8: 1, +} + +DEBUG = False + + +def memory_usage(program, batch_size): + """ + Get the estimate memory usage of program with input batch size. + + Args: + program(Program): The current Program. + batch_size(int): The current input data batch_size. + + Returns: + min_total_memory(float): the estimate memory usage lower bound. + max_total_memory(float): the estimate memory usage upper bound. + unit_str(string): the unit of estimate usage result. + + Examples: + + >>> import paddle.fluid as fluid + >>> lower_usage, upper_usage, unit = fluid.contrib.memory_usage( + fluid.default_main_program(), batch_size=10) + >>> print "memory usage is about %.3f - %.3f %s" % \ + (lower_usage, upper_usage, unit) + + """ + + # Parameters check + if not isinstance(program, Program): + raise TypeError( + "Calculating Memory Usage requires Program as its Parameter." + "But you passed in %s" % (type(prgram))) + if batch_size <= 0: + raise ValueError("The batch size need to be positive.") + + # Get the var_name list of first block and calculate + total_memory = 0.0 + for var in program.global_block().vars.itervalues(): + data_count = 1 + for x in var.shape: + if x == -1: + data_count *= batch_size + else: + data_count *= x + var_memory = data_count * dtype_to_size[var.dtype] + if DEBUG: + print "%s memory usage: %d" % (var.name, var_memory) + total_memory += var_memory + if DEBUG: + print "total memory usage: %.2f" % (total_memory) + + # Convert appropriate unit + unit_str = "B" + if total_memory > 1024: + total_memory /= 1024 + unit_str = "KB" + if total_memory > 1024: + total_memory /= 1024 + unit_str = "MB" + + # Append extra memory consumption (5% - 10%) + min_total_memory = total_memory * 1.05 + max_total_memory = total_memory * 1.1 + + return min_total_memory, max_total_memory, unit_str diff --git a/python/paddle/fluid/data_feeder.py b/python/paddle/fluid/data_feeder.py index c859778b3757f638ac531620f241e684522add57..9452cf0e2a3a2eddb761149466bfc1ee3d23dfd9 100644 --- a/python/paddle/fluid/data_feeder.py +++ b/python/paddle/fluid/data_feeder.py @@ -12,14 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function -import core +from . import core import numpy import os -import six.moves as six +import six +from six.moves import zip, range, xrange import multiprocessing -from framework import Variable, default_main_program +from .framework import Variable, default_main_program __all__ = ['DataFeeder'] @@ -53,7 +53,7 @@ class DataToLoDTensorConverter(object): self.data = [] self.lod = [] - for i in six.range(lod_level): + for i in six.moves.range(lod_level): self.lod.append([]) def feed(self, data): @@ -142,7 +142,7 @@ class DataFeeder(object): if program is None: program = default_main_program() for each_var in feed_list: - if isinstance(each_var, basestring): + if isinstance(each_var, six.string_types): each_var = program.block(0).var(each_var) if not isinstance(each_var, Variable): raise TypeError("Feed list should contain a list of variable") @@ -174,7 +174,7 @@ class DataFeeder(object): dict: the result of conversion. """ converter = [] - for lod_level, shape, dtype in six.zip( + for lod_level, shape, dtype in six.moves.zip( self.feed_lod_level, self.feed_shapes, self.feed_dtypes): converter.append( DataToLoDTensorConverter( @@ -187,10 +187,12 @@ class DataFeeder(object): assert len(each_sample) == len(converter), ( "The number of fields in data (%s) does not match " + "len(feed_list) (%s)") % (len(each_sample), len(converter)) - for each_converter, each_slot in six.zip(converter, each_sample): + for each_converter, each_slot in six.moves.zip(converter, + each_sample): each_converter.feed(each_slot) ret_dict = {} - for each_name, each_converter in six.zip(self.feed_names, converter): + for each_name, each_converter in six.moves.zip(self.feed_names, + converter): ret_dict[each_name] = each_converter.done() return ret_dict @@ -212,12 +214,14 @@ class DataFeeder(object): if isinstance(self.place, core.CUDAPlace): places = [ core.CUDAPlace(i) - for i in six.xrange(self._get_number_of_places_(num_places)) + for i in six.moves.xrange( + self._get_number_of_places_(num_places)) ] else: places = [ core.CPUPlace() - for _ in six.xrange(self._get_number_of_places_(num_places)) + for _ in six.moves.xrange( + self._get_number_of_places_(num_places)) ] if len(iterable) != len(places): @@ -227,7 +231,7 @@ class DataFeeder(object): "must be same.") place = self.place - for p, batch in six.zip(places, iterable): + for p, batch in six.moves.zip(places, iterable): self.place = p yield self.feed(batch) self.place = place diff --git a/python/paddle/fluid/debugger.py b/python/paddle/fluid/debugger.py index 1c56064a1e8bdc5d975837cb5a75a40d557765ad..b7a92cf044900acdd41ede378dd68aa2d9c6b2dc 100644 --- a/python/paddle/fluid/debugger.py +++ b/python/paddle/fluid/debugger.py @@ -14,8 +14,8 @@ import sys import re -from graphviz import GraphPreviewGenerator -import proto.framework_pb2 as framework_pb2 +from .graphviz import GraphPreviewGenerator +from .proto import framework_pb2 from google.protobuf import text_format _vartype2str_ = [ diff --git a/python/paddle/fluid/evaluator.py b/python/paddle/fluid/evaluator.py index 00ba1a0457583d1cc1fa7136ebd51e9ced167832..c0671cce9a1f169f02ba03a839c45b6e4df2c47a 100644 --- a/python/paddle/fluid/evaluator.py +++ b/python/paddle/fluid/evaluator.py @@ -15,11 +15,11 @@ import warnings import numpy as np -import layers -from framework import Program, Variable, program_guard -import unique_name -from layer_helper import LayerHelper -from initializer import Constant +from . import layers +from .framework import Program, Variable, program_guard +from . import unique_name +from .layer_helper import LayerHelper +from .initializer import Constant __all__ = [ 'ChunkEvaluator', diff --git a/python/paddle/fluid/executor.py b/python/paddle/fluid/executor.py index 4178971398c953236bf8de4d5cb6e93d0e33380c..35da1d06a2c1da8ba663ea0f0b9a0e58ea7c4470 100644 --- a/python/paddle/fluid/executor.py +++ b/python/paddle/fluid/executor.py @@ -14,7 +14,8 @@ import numpy as np import contextlib -from framework import Program, default_main_program, Variable +import six +from .framework import Program, default_main_program, Variable from . import core __all__ = [ @@ -204,19 +205,19 @@ def fetch_var(name, scope=None, return_numpy=True): def _get_program_cache_key(feed, fetch_list): - feed_var_names = feed.keys() + feed_var_names = list(feed.keys()) def to_name_str(var): if isinstance(var, Variable): return var.desc.name() elif isinstance(var, str): return var - elif isinstance(var, basestring): + elif isinstance(var, six.string_types): return str(var) else: raise TypeError(str(var) + " should be Variable or str") - fetch_var_names = map(to_name_str, fetch_list) + fetch_var_names = list(map(to_name_str, fetch_list)) return str(feed_var_names + fetch_var_names) @@ -229,8 +230,8 @@ class Executor(object): to feed map and fetch_list. Feed map provides input data for the program. fetch_list provides the variables(or names) that user want to get after program run. Note: the executor will run all operators in the program but not only the operators dependent by the fetch_list. - It store the global variables into the global scope, and create a local scope for the temporary - variables. The local scope contents will be discarded after every minibatch forward/backward finished. + It store the global variables into the global scope, and create a local scope for the temporary + variables. The local scope contents will be discarded after every minibatch forward/backward finished. But the global scope variables will be persistent through different runs. All of ops in program will be running in sequence. @@ -345,7 +346,7 @@ class Executor(object): def _fetch_data(self, fetch_list, fetch_var_name, scope): outs = [ core.get_fetch_variable(scope, fetch_var_name, i) - for i in xrange(len(fetch_list)) + for i in range(len(fetch_list)) ] return outs diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py index e10f8325e46ee52e98f9d31caddaf9ec7d188d67..3d7c29c6ea33ebc49b071681c5f9746b97445aa3 100644 --- a/python/paddle/fluid/framework.py +++ b/python/paddle/fluid/framework.py @@ -15,21 +15,22 @@ import collections import contextlib import re +import six import numpy as np -import proto.framework_pb2 as framework_pb2 +from .proto import framework_pb2 try: from . import core -except ImportError, e: +except ImportError as e: raise ImportError( """NOTE: You may need to run \"export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH\" if you encounters \"libmkldnn.so not found\" errors. If you have python installed in other directory, replace \"/usr/local/lib\" with your own directory. The original error is: \n""" + e.message) -except Exception, e: +except Exception as e: raise e -import unique_name +from . import unique_name __all__ = [ 'Program', @@ -86,7 +87,7 @@ def convert_np_dtype_to_dtype_(np_dtype): elif dtype == np.uint8: return core.VarDesc.VarType.UINT8 else: - raise ValueError("Not supported numpy dtype " + str(dtype)) + raise ValueError("Not supported numpy dtype " + six.binary_type(dtype)) def dtype_is_floating(dtype): @@ -129,15 +130,15 @@ def _debug_string_(proto, throw_on_error=True): class Variable(object): """ - In Fluid, every input and output of an operator is a variable. In most - cases, variables are used for holding different kinds of data or training - labels. A variable belongs to a block. All variable has its own name and + In Fluid, every input and output of an operator is a variable. In most + cases, variables are used for holding different kinds of data or training + labels. A variable belongs to a block. All variable has its own name and two variables in different blocks could have the same name. - There are many kinds of variables. Each kind of them has its own attributes - and usages. Please reference the framework.proto for details. + There are many kinds of variables. Each kind of them has its own attributes + and usages. Please reference the framework.proto for details. - Most of a Variable's member variables can be setted to be None. It mean + Most of a Variable's member variables can be setted to be None. It mean it is not available or will be specified later. Args: @@ -197,6 +198,7 @@ class Variable(object): if name is None: name = unique_name.generate('_generated_var') is_new_var = False + name = name if isinstance(name, six.binary_type) else name.encode() self.desc = self.block.desc.find_var(name) if self.desc is None: @@ -290,13 +292,13 @@ class Variable(object): assert isinstance(throw_on_error, bool) and isinstance(with_details, bool) protostr = self.desc.serialize_to_string() - proto = framework_pb2.VarDesc.FromString(str(protostr)) + proto = framework_pb2.VarDesc.FromString(six.binary_type(protostr)) res_str = _debug_string_(proto, throw_on_error) if with_details: additional_attr = ("error_clip", "stop_gradient") for attr_name in additional_attr: - res_str += "%s: %s\n" % (attr_name, - str(getattr(self, attr_name))) + res_str += "%s: %s\n" % ( + attr_name, six.binary_type(getattr(self, attr_name))) return res_str __repr__ = __str__ @@ -369,7 +371,7 @@ def get_all_op_protos(): protostrs = core.get_all_op_protos() ret_values = [] for pbstr in protostrs: - op_proto = framework_pb2.OpProto.FromString(str(pbstr)) + op_proto = framework_pb2.OpProto.FromString(six.binary_type(pbstr)) ret_values.append(op_proto) return ret_values @@ -472,7 +474,6 @@ class Operator(object): inputs=None, outputs=None, attrs=None): - self.block = block self.desc = desc self.attrs = attrs @@ -523,10 +524,19 @@ class Operator(object): % (in_proto.name, len(in_args))) in_arg_names = [] for arg in in_args: - if isinstance(arg, basestring): + if isinstance(arg, six.string_types): in_arg_names.append(arg) + elif isinstance(arg, six.binary_type): + in_arg_names.append(arg.decode()) else: - in_arg_names.append(arg.name) + if isinstance(arg.name, six.string_types): + in_arg_names.append(arg.name) + elif isinstance(arg.name, six.binary_type): + in_arg_names.append(arg.name.decode()) + else: + raise TypeError( + "arguments require unicode, str or bytes, but get %s instead." + % (type(arg.name))) self.desc.set_input(in_proto.name, in_arg_names) else: self.desc.set_input(in_proto.name, []) @@ -541,8 +551,9 @@ class Operator(object): if not given == need: raise ValueError(("Incorrect setting for output(s) of " "operator \"%s\". Need: [%s] Given: [%s]") % - (type, ", ".join(str(e) for e in need), - ", ".join(str(e) for e in given))) + (type, + ", ".join(six.binary_type(e) for e in need), + ", ".join(six.binary_type(e) for e in given))) for out_proto in proto.outputs: out_args = outputs[out_proto.name] @@ -554,7 +565,14 @@ class Operator(object): (out_proto.name, len(out_args))) out_arg_names = [] for arg in out_args: - out_arg_names.append(arg.name) + if isinstance(arg.name, six.string_types): + out_arg_names.append(arg.name) + elif isinstance(arg.name, six.binary_type): + out_arg_names.append(arg.name.decode()) + else: + raise TypeError( + "arguments require unicode, str or bytes, but get %s instead." + % (type(arg.name))) arg.op = self self.desc.set_output(out_proto.name, out_arg_names) @@ -590,7 +608,7 @@ class Operator(object): """ protostr = self.desc.serialize_to_string() - proto = framework_pb2.OpDesc.FromString(str(protostr)) + proto = framework_pb2.OpDesc.FromString(six.binary_type(protostr)) return _debug_string_(proto, throw_on_error) def __str__(self): @@ -845,7 +863,7 @@ class Block(object): re_add_indent = re.compile(r"\n(.)") res_str = "blocks {\n idx: %d\n parent_idx: %d" % ( self.idx, self.parent_idx) - for var in self.vars.itervalues(): + for var in list(self.vars.values()): res_str += "\n vars {\n %s }" % re_add_indent.sub( r"\n \1", var.to_string(throw_on_error, with_details)) for op in self.ops: @@ -854,7 +872,8 @@ class Block(object): res_str += "\n}" else: protostr = self.desc.serialize_to_string() - proto = framework_pb2.BlockDesc.FromString(str(protostr)) + proto = framework_pb2.BlockDesc.FromString( + six.binary_type(protostr)) res_str = _debug_string_(proto, throw_on_error) return res_str @@ -898,10 +917,11 @@ class Block(object): Returns: Variable: the Variable with the giving name. """ - if not isinstance(name, basestring): - raise TypeError( - "var require string as parameter, but get %s instead." % - (type(name))) + if not isinstance(name, six.string_types): + if not isinstance(name, six.binary_type): + raise TypeError( + "var require string as parameter, but get %s instead." % + (type(name))) v = self.vars.get(name, None) if v is None: raise ValueError("var %s not in this block" % name) @@ -949,10 +969,10 @@ class Block(object): raise ValueError("Var {0} is not found recursively".format(name)) def all_parameters(self): - return list(self._iter_parameters()) + return list(self.iter_parameters()) - def _iter_parameters(self): - return (item[1] for item in self.vars.iteritems() + def iter_parameters(self): + return (item[1] for item in list(self.vars.items()) if isinstance(item[1], Parameter)) def create_var(self, *args, **kwargs): @@ -1038,7 +1058,26 @@ class Block(object): global_block = self.program.global_block() param = Parameter(global_block, *args, **kwargs) if 'initializer' in kwargs: - kwargs['initializer'](param, self) + + def _is_inited_by(block, var): + init_ops = [] + for op in block.ops: + if var.name in op.output_arg_names: + init_ops.append(op) + return init_ops + + initializer = kwargs['initializer'] + init_ops = _is_inited_by(global_block, param) + init_ops_len = len(init_ops) + if init_ops_len > 1: + raise RuntimeError("param " + param.name + + " is inited by multiple init ops " + str( + init_ops)) + elif init_ops_len == 1: + #TODO already inited, do nothing, should log a warning + pass + else: + initializer(param, self) return param def append_op(self, *args, **kwargs): @@ -1113,7 +1152,7 @@ class Block(object): self.create_var(name=var.name(), desc=var, type=var.type()) # sync variables removed from c++ end - for var in self.vars.keys(): + for var in list(self.vars.keys()): if not self.desc.find_var(var): self.vars.pop(var) @@ -1185,7 +1224,7 @@ class Block(object): if not isinstance(other, Block): raise TypeError( "_copy_param_info_from should be invoked with Block") - for p in other._iter_parameters(): + for p in other.iter_parameters(): assert isinstance(p, Parameter) v = self.vars.get(p.name, None) if v is None: @@ -1384,7 +1423,8 @@ class Program(object): res_str += block.to_string(throw_on_error, with_details) else: protostr = self.desc.serialize_to_string() - proto = framework_pb2.ProgramDesc.FromString(str(protostr)) + proto = framework_pb2.ProgramDesc.FromString( + six.binary_type(protostr)) res_str = _debug_string_(proto, throw_on_error) return res_str @@ -1482,7 +1522,7 @@ class Program(object): else: p = Program() p.desc = core.ProgramDesc(self.desc) - p.blocks = [Block(p, i) for i in xrange(self.desc.num_blocks())] + p.blocks = [Block(p, i) for i in range(self.desc.num_blocks())] p._sync_with_cpp() p._copy_param_info_from(self) @@ -1534,7 +1574,7 @@ class Program(object): targets_idx.append([t.block.idx, t.idx]) res = Program() res.desc = core.prune(self.desc, targets_idx) - res.blocks = [Block(res, i) for i in xrange(res.desc.num_blocks())] + res.blocks = [Block(res, i) for i in range(res.desc.num_blocks())] res._sync_with_cpp() return res @@ -1545,7 +1585,7 @@ class Program(object): 2. Remove the :code:`read_op` if exists. - 3. change the :code:`is_test` + 3. change the :code:`is_test` attribute of operators to :code:`True`. All the :code:`Parameter` information will be lost. @@ -1575,13 +1615,13 @@ class Program(object): root_block._remove_var(var.name()) # change all `is_test` attributes to True - for i in xrange(res.desc.num_blocks()): + for i in range(res.desc.num_blocks()): block = res.desc.block(i) - for j in xrange(block.op_size()): + for j in range(block.op_size()): op = block.op(j) if op.has_attr('is_test'): op.set_attr('is_test', True) - res.blocks = [Block(res, i) for i in xrange(res.desc.num_blocks())] + res.blocks = [Block(res, i) for i in range(res.desc.num_blocks())] res._sync_with_cpp() return res @@ -1594,14 +1634,14 @@ class Program(object): and deserialization. Args: - binary_str(str): The binary prootbuf string. + binary_str_type(str): The binary prootbuf string. Returns: Program: A deserialized program desc. """ p = Program() p.desc = core.ProgramDesc(binary_str) - p.blocks = [Block(p, i) for i in xrange(p.desc.num_blocks())] + p.blocks = [Block(p, i) for i in range(p.desc.num_blocks())] p._sync_with_cpp() return p @@ -1629,7 +1669,7 @@ class Program(object): self._seed = seed def __repr__(self): - return str(self) + return self.__str__() def global_block(self): """ @@ -1740,7 +1780,7 @@ class Program(object): if len(self.blocks) != len(other.blocks): raise ValueError("_copy_param_info_from should be invoked with two " "program, with represent the same topology") - for var in other.global_block().vars.itervalues(): + for var in list(other.global_block().vars.values()): if var.is_data: self.global_block().var(var.name).is_data = True @@ -1752,15 +1792,15 @@ class Program(object): iterable: The generator will yield every variable in this program. """ for each_block in self.blocks: - for each_var in each_block.vars.itervalues(): + for each_var in list(each_block.vars.values()): yield each_var class Parameter(Variable): """ - Parameter is derived from Variable. A parameter is a persistable + Parameter is derived from Variable. A parameter is a persistable Variable, and will be updated by optimizers after each iteration. - The training of a neural network is essentially the updating of + The training of a neural network is essentially the updating of its parameters. Relative to a general Variable, a Parameter has several its own @@ -1826,8 +1866,8 @@ class Parameter(Variable): additional_attr = ("trainable", "optimize_attr", "regularizer", "gradient_clip_attr", "do_model_average") for attr_name in additional_attr: - res_str += "%s: %s\n" % (attr_name, - str(getattr(self, attr_name))) + res_str += "%s: %s\n" % ( + attr_name, six.binary_type(getattr(self, attr_name))) else: res_str = Variable.to_string(self, throw_on_error, False) return res_str diff --git a/python/paddle/fluid/graphviz.py b/python/paddle/fluid/graphviz.py index 125b4efa9d476e561bd78d0365cd92bbf7e66605..ba67bf5ae6fe44ea23414d444a270c436c195326 100644 --- a/python/paddle/fluid/graphviz.py +++ b/python/paddle/fluid/graphviz.py @@ -14,12 +14,13 @@ import os import random +import six import subprocess import logging def crepr(v): - if type(v) is str or type(v) is unicode: + if isinstance(v, six.string_types): return '"%s"' % v return str(v) @@ -104,7 +105,7 @@ class Graph(object): def _rank_repr(self): ranks = sorted( - self.rank_groups.items(), + list(self.rank_groups.items()), cmp=lambda a, b: a[1].priority > b[1].priority) repr = [] for x in ranks: @@ -148,7 +149,7 @@ class Node(object): name=self.name, label=self.label, extra=',' + ','.join("%s=%s" % (key, crepr(value)) - for key, value in self.attrs.items()) + for key, value in list(self.attrs.items())) if self.attrs else "") return reprs @@ -172,7 +173,7 @@ class Edge(object): target=self.target.name, extra="" if not self.attrs else "[" + ','.join("{}={}".format(attr[0], crepr(attr[1])) - for attr in self.attrs.items()) + "]") + for attr in list(self.attrs.items())) + "]") return repr diff --git a/python/paddle/fluid/inferencer.py b/python/paddle/fluid/inferencer.py index a81e39695b78f235d6ae896d90117dd392692634..ff382d8b832b4b2bc6779dbb28d3fd95c8a0984e 100644 --- a/python/paddle/fluid/inferencer.py +++ b/python/paddle/fluid/inferencer.py @@ -14,14 +14,14 @@ import contextlib -import core - -import executor -import framework -import io -import parallel_executor -import unique_name -from trainer import check_and_get_place +from . import core + +from . import executor +from . import framework +from . import io +from . import parallel_executor +from . import unique_name +from .trainer import check_and_get_place __all__ = ['Inferencer', ] diff --git a/python/paddle/fluid/initializer.py b/python/paddle/fluid/initializer.py index 0e640bf280d396504deec1183821da3e8a156530..83290ac60839b855a0348696ad6898af7335e2fc 100644 --- a/python/paddle/fluid/initializer.py +++ b/python/paddle/fluid/initializer.py @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import framework +from . import framework import numpy as np import contextlib -from framework import convert_np_dtype_to_dtype_ -from core import VarDesc +from .framework import convert_np_dtype_to_dtype_ +from .core import VarDesc __all__ = [ 'Constant', 'Uniform', 'Normal', 'Xavier', 'Bilinear', 'MSRA', diff --git a/python/paddle/fluid/io.py b/python/paddle/fluid/io.py index 1ec670de07062057ba09e15ac1e4da026d035a53..55e517f1f4a48beddf62fe875c2064da09b2f6b0 100644 --- a/python/paddle/fluid/io.py +++ b/python/paddle/fluid/io.py @@ -16,6 +16,7 @@ import os import errno import time import shutil +import six from paddle.fluid.evaluator import Evaluator from paddle.fluid.framework import Program, Parameter, default_main_program, default_startup_program, Variable @@ -92,34 +93,34 @@ def save_vars(executor, """ Save variables to the given directory by executor. - There are two ways to specify variables to be saved: The first way, list - variables in a list and assign it to the `vars`. The second way, assign the - `main_program` with an existing program, then all variables in the program - will be saved. The first way has a higher priority. In other words, if `vars` + There are two ways to specify variables to be saved: The first way, list + variables in a list and assign it to the `vars`. The second way, assign the + `main_program` with an existing program, then all variables in the program + will be saved. The first way has a higher priority. In other words, if `vars` are assigned, the `main_program` and the `predicate` will be ignored. - The `dirname` are used to specify the folder where to save variables. - If you prefer to save variables in separate files in the folder `dirname`, - set `filename` None; if you prefer to save all variables in a single file, + The `dirname` are used to specify the folder where to save variables. + If you prefer to save variables in separate files in the folder `dirname`, + set `filename` None; if you prefer to save all variables in a single file, use `filename` to specify it. Args: executor(Executor): The executor to run for saving variables. dirname(str): The directory path. - main_program(Program|None): The program whose variables will be saved. - If it is None, the default main program will + main_program(Program|None): The program whose variables will be saved. + If it is None, the default main program will be used automatically. Default: None - vars(list[Variable]|None): The list that contains all variables to save. + vars(list[Variable]|None): The list that contains all variables to save. It has a higher priority than the `main_program`. Default: None - predicate(function|None): If it is not None, only variables in the - `main_program` that makes predicate(variable)==True - will be saved. It only works when we are using the - `main_program` to specify variables (In other words + predicate(function|None): If it is not None, only variables in the + `main_program` that makes predicate(variable)==True + will be saved. It only works when we are using the + `main_program` to specify variables (In other words `vars` is None). Default: None - filename(str|None): The file which to save all variables. If you prefer to save + filename(str|None): The file which to save all variables. If you prefer to save variables separately, set it to None. Default: None @@ -149,7 +150,7 @@ def save_vars(executor, # The second usage: using `vars` to specify variables var_list = [var_a, var_b, var_c] - fluid.io.save_vars(executor=exe, dirname=path, vars=var_list, + fluid.io.save_vars(executor=exe, dirname=path, vars=var_list, filename="vars_file") # var_a, var_b and var_c will be saved. And they are going to be # saved in the same file named 'var_file' in the path "./my_paddle_model". @@ -163,7 +164,7 @@ def save_vars(executor, save_vars( executor, dirname=dirname, - vars=filter(predicate, main_program.list_vars()), + vars=list(filter(predicate, main_program.list_vars())), filename=filename) else: save_program = Program() @@ -203,14 +204,14 @@ def save_params(executor, dirname, main_program=None, filename=None): This function filters out all parameters from the give `main_program` and then save them to the folder `dirname` or the file `filename`. - Use the `dirname` to specify the saving folder. If you would like to - save parameters in separate files, set `filename` None; if you would - like to save all parameters in a single file, use `filename` to specify + Use the `dirname` to specify the saving folder. If you would like to + save parameters in separate files, set `filename` None; if you would + like to save all parameters in a single file, use `filename` to specify the file name. - NOTICE: Some variables are not Parameter while they are necessary for - training. So you can NOT save and continue your training just by - `save_params()` and `load_params()`. Please use `save_persistables()` + NOTICE: Some variables are not Parameter while they are necessary for + training. So you can NOT save and continue your training just by + `save_params()` and `load_params()`. Please use `save_persistables()` and `load_persistables()` instead. Args: @@ -220,8 +221,8 @@ def save_params(executor, dirname, main_program=None, filename=None): saved. If it is None, the default main program will be used automatically. Default: None - filename(str|None): The file to save all parameters. If you prefer - to save parameters in differnet files, set it + filename(str|None): The file to save all parameters. If you prefer + to save parameters in differnet files, set it to None. Default: None @@ -234,7 +235,7 @@ def save_params(executor, dirname, main_program=None, filename=None): exe = fluid.Executor(fluid.CPUPlace()) param_path = "./my_paddle_model" prog = fluid.default_main_program() - fluid.io.save_params(executor=exe, dirname=param_path, + fluid.io.save_params(executor=exe, dirname=param_path, main_program=None) """ save_vars( @@ -248,23 +249,23 @@ def save_params(executor, dirname, main_program=None, filename=None): def save_persistables(executor, dirname, main_program=None, filename=None): """ - This function filters out all variables with `persistable==True` from the - give `main_program` and then saves these variables to the folder `dirname` + This function filters out all variables with `persistable==True` from the + give `main_program` and then saves these variables to the folder `dirname` or file `filename`. - The `dirname` is used to specify the folder where persistable variables - are going to be saved. If you would like to save variables in separate - files, set `filename` None; if you would like to save all variables in a + The `dirname` is used to specify the folder where persistable variables + are going to be saved. If you would like to save variables in separate + files, set `filename` None; if you would like to save all variables in a single file, use `filename` to specify the file name. Args: executor(Executor): The executor to run for saving persistable variables. dirname(str): The directory path. - main_program(Program|None): The program whose persistbale variables will - be saved. If it is None, the default main + main_program(Program|None): The program whose persistbale variables will + be saved. If it is None, the default main program will be used automatically. Default: None - filename(str|None): The file to saved all variables. If you prefer to + filename(str|None): The file to saved all variables. If you prefer to save variables in differnet files, set it to None. Default: None @@ -277,7 +278,7 @@ def save_persistables(executor, dirname, main_program=None, filename=None): exe = fluid.Executor(fluid.CPUPlace()) param_path = "./my_paddle_model" prog = fluid.default_main_program() - fluid.io.save_persistables(executor=exe, dirname=param_path, + fluid.io.save_persistables(executor=exe, dirname=param_path, main_program=None) """ save_vars( @@ -298,34 +299,34 @@ def load_vars(executor, """ Load variables from the given directory by executor. - There are two ways to specify variables to be loaded: The first way, list - variables in a list and assign it to the `vars`. The second way, assign the - `main_program` with an existing program, then all variables in the program - will be loaded. The first way has a higher priority. In other words if `vars` + There are two ways to specify variables to be loaded: The first way, list + variables in a list and assign it to the `vars`. The second way, assign the + `main_program` with an existing program, then all variables in the program + will be loaded. The first way has a higher priority. In other words if `vars` are assigned, the `main_program` and the `predicate` will be ignored. - The `dirname` are used to specify the folder where to load variables. - If variables were saved in separate files in the folder `dirname`, - set `filename` None; if all variables were saved in a single file, + The `dirname` are used to specify the folder where to load variables. + If variables were saved in separate files in the folder `dirname`, + set `filename` None; if all variables were saved in a single file, use `filename` to specify it. Args: executor(Executor): The executor to run for loading variables. dirname(str): The directory path. - main_program(Program|None): The program whose variables will be loaded. - If it is None, the default main program will + main_program(Program|None): The program whose variables will be loaded. + If it is None, the default main program will be used automatically. Default: None - vars(list[Variable]|None): The list that contains all variables to load. + vars(list[Variable]|None): The list that contains all variables to load. It has a higher priority than the `main_program`. Default: None - predicate(function|None): If it is not None, only variables in the - `main_program` that makes predicate(variable)==True - will be loaded. It only works when we are using the - `main_program` to specify variables (In other words + predicate(function|None): If it is not None, only variables in the + `main_program` that makes predicate(variable)==True + will be loaded. It only works when we are using the + `main_program` to specify variables (In other words `vars` is None). Default: None - filename(str|None): The file which saved all required variables. If variables + filename(str|None): The file which saved all required variables. If variables were saved in differnet files, set it to None. Default: None @@ -355,9 +356,9 @@ def load_vars(executor, # The second usage: using `vars` to specify variables var_list = [var_a, var_b, var_c] - fluid.io.load_vars(executor=exe, dirname=path, vars=var_list, + fluid.io.load_vars(executor=exe, dirname=path, vars=var_list, filename="vars_file") - # var_a, var_b and var_c will be loaded. And they are supposed to haven + # var_a, var_b and var_c will be loaded. And they are supposed to haven # been saved in the same file named 'var_file' in the path "./my_paddle_model". """ if vars is None: @@ -369,7 +370,7 @@ def load_vars(executor, load_vars( executor, dirname=dirname, - vars=filter(predicate, main_program.list_vars()), + vars=list(filter(predicate, main_program.list_vars())), filename=filename) else: load_prog = Program() @@ -410,15 +411,15 @@ def load_params(executor, dirname, main_program=None, filename=None): and then trys to load these parameters from the folder `dirname` or the file `filename`. - Use the `dirname` to specify the folder where parameters were saved. If - parameters were saved in separate files in the folder `dirname`, set - `filename` None; if all parameters were saved in a single file, use + Use the `dirname` to specify the folder where parameters were saved. If + parameters were saved in separate files in the folder `dirname`, set + `filename` None; if all parameters were saved in a single file, use `filename` to specify the file name. - NOTICE: Some variables are not Parameter while they are necessary for - training. So you can NOT save and continue your training just by - `save_params()` and `load_params()`. Please use `save_persistables()` - and `load_persistables()` instead. + NOTICE: Some variables are not Parameter while they are necessary for + training. So you can NOT save and continue your training just by + `save_params()` and `load_params()`. Please use `save_persistables()` + and `load_persistables()` instead. Args: executor(Executor): The executor to run for loading parameters. @@ -427,7 +428,7 @@ def load_params(executor, dirname, main_program=None, filename=None): loaded. If it is None, the default main program will be used automatically. Default: None - filename(str|None): The file which saved all parameters. If parameters + filename(str|None): The file which saved all parameters. If parameters were saved in differnet files, set it to None. Default: None @@ -440,7 +441,7 @@ def load_params(executor, dirname, main_program=None, filename=None): exe = fluid.Executor(fluid.CPUPlace()) param_path = "./my_paddle_model" prog = fluid.default_main_program() - fluid.io.load_params(executor=exe, dirname=param_path, + fluid.io.load_params(executor=exe, dirname=param_path, main_program=None) """ load_vars( @@ -453,23 +454,23 @@ def load_params(executor, dirname, main_program=None, filename=None): def load_persistables(executor, dirname, main_program=None, filename=None): """ - This function filters out all variables with `persistable==True` from the - give `main_program` and then trys to load these variables from the folder + This function filters out all variables with `persistable==True` from the + give `main_program` and then trys to load these variables from the folder `dirname` or the file `filename`. - Use the `dirname` to specify the folder where persistable variables were - saved. If variables were saved in separate files, set `filename` None; - if all variables were saved in a single file, use `filename` to specify + Use the `dirname` to specify the folder where persistable variables were + saved. If variables were saved in separate files, set `filename` None; + if all variables were saved in a single file, use `filename` to specify the file name. Args: executor(Executor): The executor to run for loading persistable variables. dirname(str): The directory path. - main_program(Program|None): The program whose persistbale variables will - be loaded. If it is None, the default main + main_program(Program|None): The program whose persistbale variables will + be loaded. If it is None, the default main program will be used automatically. Default: None - filename(str|None): The file which saved all variables. If variables were + filename(str|None): The file which saved all variables. If variables were saved in differnet files, set it to None. Default: None @@ -482,7 +483,7 @@ def load_persistables(executor, dirname, main_program=None, filename=None): exe = fluid.Executor(fluid.CPUPlace()) param_path = "./my_paddle_model" prog = fluid.default_main_program() - fluid.io.load_persistables(executor=exe, dirname=param_path, + fluid.io.load_persistables(executor=exe, dirname=param_path, main_program=None) """ load_vars( @@ -561,20 +562,20 @@ def save_inference_model(dirname, Args: dirname(str): The directory path to save the inference model. - feeded_var_names(list[str]): Names of variables that need to be feeded data + feeded_var_names(list[str]): Names of variables that need to be feeded data during inference. - target_vars(list[Variable]): Variables from which we can get inference + target_vars(list[Variable]): Variables from which we can get inference results. executor(Executor): The executor that saves the inference model. - main_program(Program|None): The original program, which will be pruned to - build the inference model. If is setted None, + main_program(Program|None): The original program, which will be pruned to + build the inference model. If is setted None, the default main program will be used. Default: None. - model_filename(str|None): The name of file to save the inference program - itself. If is setted None, a default filename + model_filename(str|None): The name of file to save the inference program + itself. If is setted None, a default filename `__model__` will be used. - params_filename(str|None): The name of file to save all related parameters. - If it is setted None, parameters will be saved + params_filename(str|None): The name of file to save all related parameters. + If it is setted None, parameters will be saved in separate files . Returns: @@ -592,20 +593,32 @@ def save_inference_model(dirname, fluid.io.save_inference_model(dirname=path, feeded_var_names=['img'], target_vars=[predict_var], executor=exe) - # In this exsample, the function will prune the default main program - # to make it suitable for infering the `predict_var`. The pruned - # inference program is going to be saved in the "./infer_model/__model__" + # In this exsample, the function will prune the default main program + # to make it suitable for infering the `predict_var`. The pruned + # inference program is going to be saved in the "./infer_model/__model__" # and parameters are going to be saved in separate files under folder - # "./infer_model". + # "./infer_model". """ - if isinstance(feeded_var_names, basestring): + if isinstance(feeded_var_names, six.binary_type): feeded_var_names = [feeded_var_names] + elif isinstance(feeded_var_names, six.text_type): + feeded_var_names = [feeded_var_names.encode()] else: if len(feeded_var_names) > 0: + # TODO(paddle-dev): polish these code blocks if not (bool(feeded_var_names) and all( - isinstance(name, basestring) for name in feeded_var_names)): - raise ValueError("'feed_var_names' should be a list of str.") + isinstance(name, six.binary_type) + for name in feeded_var_names)): + if not (all( + isinstance(name, six.text_type) + for name in feeded_var_names)): + raise ValueError( + "'feed_var_names' should be a list of str.") + else: + feeded_var_names = [ + name.encode() for name in feeded_var_names + ] if isinstance(target_vars, Variable): target_vars = [target_vars] @@ -662,22 +675,22 @@ def load_inference_model(dirname, dirname(str): The directory path executor(Executor): The executor to run for loading inference model. model_filename(str|None): The name of file to load inference program. - If it is None, the default filename + If it is None, the default filename '__model__' will be used. Default: None params_filename(str|None): The name of file to load all parameters. - It is only used for the case that all - parameters were saved in a single binary - file. If parameters were saved in separate + It is only used for the case that all + parameters were saved in a single binary + file. If parameters were saved in separate files, set it as 'None'. Returns: tuple: The return of this function is a tuple with three elements: - (program, feed_target_names, fetch_targets). The `program` is a - Program, it's the program for inference. The `feed_target_names` is - a list of str, it contains Names of variables that need to feed - data in the inference program. The `fetch_targets` is a list of - Variable. It contains variables from which we can get inference + (program, feed_target_names, fetch_targets). The `program` is a + Program, it's the program for inference. The `feed_target_names` is + a list of str, it contains Names of variables that need to feed + data in the inference program. The `fetch_targets` is a list of + Variable. It contains variables from which we can get inference results. Raises: @@ -688,17 +701,17 @@ def load_inference_model(dirname, exe = fluid.Executor(fluid.CPUPlace()) path = "./infer_model" - [inference_program, feed_target_names, fetch_targets] = + [inference_program, feed_target_names, fetch_targets] = fluid.io.load_inference_model(dirname=path, executor=exe) results = exe.run(inference_program, feed={feed_target_names[0]: tensor_img}, fetch_list=fetch_targets) - # In this exsample, the inference program was saved in the - # "./infer_model/__model__" and parameters were saved in - # separate files in ""./infer_model". - # After getting inference program, feed target names and - # fetch targets, we can use an Executor to run the inference + # In this exsample, the inference program was saved in the + # "./infer_model/__model__" and parameters were saved in + # separate files in ""./infer_model". + # After getting inference program, feed target names and + # fetch targets, we can use an Executor to run the inference # program to get the inference result. """ diff --git a/python/paddle/fluid/layer_helper.py b/python/paddle/fluid/layer_helper.py index de752d1daeb6bc725cf6eff1bb74a786e2ad6b95..0c2b1eb795860373220eb254612161f7dc816ffd 100644 --- a/python/paddle/fluid/layer_helper.py +++ b/python/paddle/fluid/layer_helper.py @@ -14,12 +14,14 @@ import copy import itertools +import six -from framework import Variable, Parameter, default_main_program, default_startup_program, dtype_is_floating -import unique_name +from .framework import Variable, Parameter, default_main_program, default_startup_program, dtype_is_floating +from . import unique_name from paddle.fluid.initializer import Constant, Xavier -from param_attr import ParamAttr, WeightNormParamAttr -import core +from .param_attr import ParamAttr, WeightNormParamAttr +from . import core +from six.moves import zip class LayerHelper(object): @@ -83,7 +85,7 @@ class LayerHelper(object): raise ValueError("parameter number mismatch") elif len(param_attr) == 1 and length != 1: tmp = [None] * length - for i in xrange(length): + for i in range(length): tmp[i] = copy.deepcopy(param_attr[0]) param_attr = tmp return param_attr @@ -91,7 +93,7 @@ class LayerHelper(object): def iter_inputs_and_params(self, input_param_name='input'): inputs = self.multiple_input(input_param_name) param_attrs = self.multiple_param_attr(len(inputs)) - for ipt, param_attr in itertools.izip(inputs, param_attrs): + for ipt, param_attr in zip(inputs, param_attrs): yield ipt, param_attr def input_dtype(self, input_param_name='input'): @@ -218,7 +220,7 @@ class LayerHelper(object): norm = __norm_op(reshape, dim=0, block=block) __reshape_op(norm, out=out, shape=out_shape, block=block) else: - perm = range(len(x.shape)) + perm = list(range(len(x.shape))) perm[0], perm[dim] = dim, 0 transpose = __transpose_op(x, perm, block=block) norm = __norm_op(transpose, dim=0, block=block) @@ -397,8 +399,10 @@ class LayerHelper(object): act = self.kwargs.get('act', None) if act is None: return input_var - if isinstance(act, basestring): + if isinstance(act, six.string_types): act = {'type': act} + else: + raise TypeError(str(act) + " should be unicode or str") if 'use_cudnn' in self.kwargs and self.kwargs.get('use_cudnn'): act['use_cudnn'] = self.kwargs.get('use_cudnn') diff --git a/python/paddle/fluid/layers/__init__.py b/python/paddle/fluid/layers/__init__.py index 4917e67de0d20ff9e8f9a27f38e1bd2abef5c503..a48e360463456ab7e00534dc0684aa153c8205cd 100644 --- a/python/paddle/fluid/layers/__init__.py +++ b/python/paddle/fluid/layers/__init__.py @@ -12,25 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -import ops -from ops import * -import nn -from nn import * -import io -from io import * -import tensor -from tensor import * -import control_flow -from control_flow import * -import device -from device import * -import math_op_patch -from math_op_patch import * -import detection -from detection import * -import metric_op -from metric_op import * -from learning_rate_scheduler import * +from . import ops +from .ops import * +from . import nn +from .nn import * +from . import io +from .io import * +from . import tensor +from .tensor import * +from . import control_flow +from .control_flow import * +from . import device +from .device import * +from . import math_op_patch +from .math_op_patch import * +from . import detection +from .detection import * +from . import metric_op +from .metric_op import * +from .learning_rate_scheduler import * __all__ = [] __all__ += nn.__all__ diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index 3ee1c636ace504e14cf7d6c106df1bc3e864d660..9fb7b4d0cad67db2d2d4b56e43d8837b8160cdb0 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -13,15 +13,16 @@ # limitations under the License. import contextlib -from layer_function_generator import autodoc, templatedoc -from tensor import assign, fill_constant +from .layer_function_generator import autodoc, templatedoc +from .tensor import assign, fill_constant from .. import core from ..framework import Program, Variable, Operator from ..layer_helper import LayerHelper, unique_name from ..initializer import force_init_on_cpu -from ops import logical_and, logical_not, logical_or +from .ops import logical_and, logical_not, logical_or import numpy import warnings +from functools import reduce __all__ = [ 'While', @@ -276,7 +277,7 @@ class ParallelDo(object): avg_cost = fluid.layers.mean(x=cost) .. warning:: - + It will be soon deprecated, please use ParallelExecutor instead. """ @@ -601,7 +602,7 @@ class StaticRNN(object): boot_memories = [] pre_memories = [] memories = [] - for _, mem in self.memories.iteritems(): + for _, mem in list(self.memories.items()): boot_memories.append(mem.init) pre_memories.append(mem.pre_mem.name) mem_var = rnn_block.var(mem.mem.name) @@ -819,21 +820,21 @@ def max_sequence_len(rank_table): def lod_tensor_to_array(x, table): - """ + """ Convert a LoDTensor to a LoDTensorArray. - This function split a LoDTesnor to a LoDTensorArray according to its LoD - information. LoDTensorArray is an alias of C++ std::vector in - PaddlePaddle. The generated LoDTensorArray of this function can be further read - or written by `read_from_array()` and `write_to_array()` operators. However, - this function is generally an internal component of PaddlePaddle `DynamicRNN`. + This function split a LoDTesnor to a LoDTensorArray according to its LoD + information. LoDTensorArray is an alias of C++ std::vector in + PaddlePaddle. The generated LoDTensorArray of this function can be further read + or written by `read_from_array()` and `write_to_array()` operators. However, + this function is generally an internal component of PaddlePaddle `DynamicRNN`. Users should not use it directly. Args: x (Variable|list): The LoDTensor to be converted to a LoDTensorArray. table (ParamAttr|list): The variable that stores the level of lod which is ordered by sequence length in - descending order. It is generally generated + descending order. It is generally generated by `layers.lod_rank_table()` API. Returns: @@ -1067,9 +1068,9 @@ def array_read(array, i): Given: array = [0.6, 0.1, 0.3, 0.1] - + And: - + i = 2 Then: @@ -1176,9 +1177,9 @@ def array_length(array): class ConditionalBlockGuard(BlockGuard): """ - ConditionalBlockGuard is derived from BlockGuard. It is dedicated for - holding a ConditionalBlock, and helping users entering and exiting the - ConditionalBlock via Python's 'with' keyword. However, ConditionalBlockGuard + ConditionalBlockGuard is derived from BlockGuard. It is dedicated for + holding a ConditionalBlock, and helping users entering and exiting the + ConditionalBlock via Python's 'with' keyword. However, ConditionalBlockGuard is generally an internal component of IfElse, users should not use it directly. """ @@ -1512,7 +1513,7 @@ class IfElse(object): def __call__(self): if self.status != self.OUT_IF_ELSE_BLOCKS: raise ValueError("IfElse::__call__ must be out of sub-block") - false_len, true_len = map(len, self.output_table) + false_len, true_len = list(map(len, self.output_table)) if false_len == 0 and true_len == 0: raise ValueError("Must invoke true_block/false_block before " "__call__") @@ -1932,7 +1933,7 @@ def is_empty(x, cond=None, **ignored): Args: x (Variable): The Variable to be tested. - cond (Variable|None): Output parameter. Returns the test result + cond (Variable|None): Output parameter. Returns the test result of given 'x'. Default: None Returns: diff --git a/python/paddle/fluid/layers/detection.py b/python/paddle/fluid/layers/detection.py index 60a12686f8ff43f5ee7e30650a208296963bda3d..9fae96d9bc7f5e6e89f8ee3894d28ae306e10ca9 100644 --- a/python/paddle/fluid/layers/detection.py +++ b/python/paddle/fluid/layers/detection.py @@ -15,12 +15,13 @@ All layers just related to the detection neural network. """ -from layer_function_generator import generate_layer_fn -from layer_function_generator import autodoc, templatedoc +from .layer_function_generator import generate_layer_fn +from .layer_function_generator import autodoc, templatedoc from ..layer_helper import LayerHelper -import tensor -import nn +from . import tensor +from . import nn import math +from functools import reduce __all__ = [ 'prior_box', @@ -1032,7 +1033,7 @@ def multi_box_head(inputs, min_sizes = [] max_sizes = [] step = int(math.floor(((max_ratio - min_ratio)) / (num_layer - 2))) - for ratio in xrange(min_ratio, max_ratio + 1, step): + for ratio in range(min_ratio, max_ratio + 1, step): min_sizes.append(base_size * ratio / 100.) max_sizes.append(base_size * (ratio + step) / 100.) min_sizes = [base_size * .10] + min_sizes diff --git a/python/paddle/fluid/layers/device.py b/python/paddle/fluid/layers/device.py index 384d302a709eeec220864b9e8c9210ed028470f6..bb1fb7fd571a56acf367e663af0cf9431211bcea 100644 --- a/python/paddle/fluid/layers/device.py +++ b/python/paddle/fluid/layers/device.py @@ -15,7 +15,7 @@ All util layers. """ -from layer_function_generator import autodoc +from .layer_function_generator import autodoc from ..framework import unique_name from ..layer_helper import LayerHelper from ..annotations import deprecated diff --git a/python/paddle/fluid/layers/io.py b/python/paddle/fluid/layers/io.py index fab4a92a0ac5ab28508fb52a84aefdba19ac6dde..327ae309816344a0bcebfe70ffb59a00eab1d86f 100644 --- a/python/paddle/fluid/layers/io.py +++ b/python/paddle/fluid/layers/io.py @@ -16,8 +16,8 @@ import multiprocessing import threading from ..data_feeder import DataFeeder -from control_flow import BlockGuard -from layer_function_generator import templatedoc +from .control_flow import BlockGuard +from .layer_function_generator import templatedoc from .. import core from ..executor import global_scope from ..framework import convert_np_dtype_to_dtype_, default_main_program, \ @@ -69,7 +69,7 @@ def data(name, """ helper = LayerHelper('data', **locals()) shape = list(shape) - for i in xrange(len(shape)): + for i in range(len(shape)): if shape[i] is None: shape[i] = -1 append_batch_size = False @@ -387,9 +387,9 @@ def random_data_generator(low, high, shapes, lod_levels, for_parallel=True): Create a uniform random data generator This layer returns a Reader Variable. - Instead of opening a file and reading data from it, this - Reader Variable generates float uniform random data by itself. - It can be used as a dummy reader to test a network without + Instead of opening a file and reading data from it, this + Reader Variable generates float uniform random data by itself. + It can be used as a dummy reader to test a network without opening a real file. Args: @@ -707,9 +707,9 @@ def open_files(filenames, """ Open files - This layer takes a list of files to read from and returns a Reader Variable. - Via the Reader Variable, we can get data from given files. All files must - have name suffixs to indicate their formats, e.g., '*.recordio'. + This layer takes a list of files to read from and returns a Reader Variable. + Via the Reader Variable, we can get data from given files. All files must + have name suffixs to indicate their formats, e.g., '*.recordio'. Args: filenames(list): The list of file names. @@ -825,9 +825,9 @@ def shuffle(reader, buffer_size): def batch(reader, batch_size): """ - This layer is a reader decorator. It takes a reader and adds - 'batching' decoration on it. When reading with the result - decorated reader, output data will be automatically organized + This layer is a reader decorator. It takes a reader and adds + 'batching' decoration on it. When reading with the result + decorated reader, output data will be automatically organized to the form of batches. Args: @@ -852,11 +852,11 @@ def batch(reader, batch_size): # If we read data with the raw_reader: # data = fluid.layers.read_file(raw_reader) # We can only get data instance by instance. - # + # # However, if we read data with the batch_reader: # data = fluid.layers.read_file(batch_reader) - # Each 5 adjacent instances will be automatically combined together - # to become a batch. So what we get('data') is a batch data instead + # Each 5 adjacent instances will be automatically combined together + # to become a batch. So what we get('data') is a batch data instead # of an instance. """ return __create_unshared_decorated_reader__( @@ -903,8 +903,8 @@ def read_file(reader): """ Execute the given reader and get data via it. - A reader is also a Variable. It can be a raw reader generated by - `fluid.layers.open_files()` or a decorated one generated by + A reader is also a Variable. It can be a raw reader generated by + `fluid.layers.open_files()` or a decorated one generated by `fluid.layers.double_buffer()` and so on. Args: @@ -1005,7 +1005,7 @@ class Preprocessor(object): source_lod_levels = self.underlying_reader.desc.lod_levels() self.source_var_names = [ unique_name("preprocessor_source") - for _ in xrange(len(source_shapes)) + for _ in range(len(source_shapes)) ] source_vars = [] for var_name, shape, dtype, lod_level in zip( diff --git a/python/paddle/fluid/layers/layer_function_generator.py b/python/paddle/fluid/layers/layer_function_generator.py index 3096389101a5e5b302c78145b8bc9f1d71f6b8cb..c0d72620b1ddb183f43ebce766688518b5a737ac 100644 --- a/python/paddle/fluid/layers/layer_function_generator.py +++ b/python/paddle/fluid/layers/layer_function_generator.py @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. import re -import cStringIO import functools import warnings import string +from six.moves import cStringIO from ..proto import framework_pb2 from ..framework import OpProtoHolder, Variable from ..layer_helper import LayerHelper @@ -70,7 +70,7 @@ def _generate_doc_string_(op_proto): if not isinstance(op_proto, framework_pb2.OpProto): raise TypeError("OpProto should be `framework_pb2.OpProto`") - buf = cStringIO.StringIO() + buf = cStringIO() buf.write(escape_math(op_proto.comment)) buf.write('\nArgs:\n') for each_input in op_proto.inputs: @@ -119,9 +119,9 @@ def generate_layer_fn(op_type): """ op_proto = OpProtoHolder.instance().get_op_proto(op_type) not_intermediate_outputs = \ - filter(lambda output: not output.intermediate, op_proto.outputs) + [output for output in op_proto.outputs if not output.intermediate] intermediate_outputs = \ - filter(lambda output: output.intermediate, op_proto.outputs) + [output for output in op_proto.outputs if output.intermediate] if len(not_intermediate_outputs) != 1: raise ValueError("Only one non intermediate output operator can be", diff --git a/python/paddle/fluid/layers/learning_rate_scheduler.py b/python/paddle/fluid/layers/learning_rate_scheduler.py index c7966e36f15ef0e3f30f8a96ad71df04aece0fa1..daf91a40f7ad7935d355a287819ad1dbcdd84eb8 100644 --- a/python/paddle/fluid/layers/learning_rate_scheduler.py +++ b/python/paddle/fluid/layers/learning_rate_scheduler.py @@ -20,10 +20,10 @@ User can also implement their own learning_rate_decay strategy according to this module. """ -import control_flow -import nn -import ops -import tensor +from . import control_flow +from . import nn +from . import ops +from . import tensor from ..initializer import init_on_cpu from ..framework import default_main_program, Parameter diff --git a/python/paddle/fluid/layers/math_op_patch.py b/python/paddle/fluid/layers/math_op_patch.py index f814c41633fbac76eb9411e2f418f521e8e9679d..0e10a91d25877984396f9bcf9aae6438707eeab1 100644 --- a/python/paddle/fluid/layers/math_op_patch.py +++ b/python/paddle/fluid/layers/math_op_patch.py @@ -13,7 +13,7 @@ # limitations under the License. from ..framework import Variable, unique_name -from layer_function_generator import OpProtoHolder +from .layer_function_generator import OpProtoHolder from ..initializer import force_init_on_cpu diff --git a/python/paddle/fluid/layers/metric_op.py b/python/paddle/fluid/layers/metric_op.py index e7d7a9e826de95514b6f2e04e7408075ab0b8cb6..49bae1e8af768d93294120e1d13ef0242313aa3c 100644 --- a/python/paddle/fluid/layers/metric_op.py +++ b/python/paddle/fluid/layers/metric_op.py @@ -20,7 +20,7 @@ from ..layer_helper import LayerHelper from ..initializer import Normal, Constant from ..framework import Variable from ..param_attr import ParamAttr -import nn +from . import nn __all__ = ['accuracy', 'auc'] diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 12e7170fc3da83071f4a23b6c39463d8c2543391..0960b54123d36b0ddcced6841384399cab816f48 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -33,11 +33,12 @@ from ..layer_helper import LayerHelper from ..initializer import Normal, Constant from ..framework import Variable from ..param_attr import ParamAttr -from layer_function_generator import autodoc, templatedoc -from tensor import concat -import utils +from .layer_function_generator import autodoc, templatedoc +from .tensor import concat +from . import utils import random from .. import unique_name +from functools import reduce __all__ = [ 'fc', @@ -949,6 +950,10 @@ def dropout(x, dropout_prob, is_test=False, seed=None, name=None): helper = LayerHelper('dropout', **locals()) out = helper.create_tmp_variable(dtype=x.dtype) mask = helper.create_tmp_variable(dtype=x.dtype, stop_gradient=True) + + if (seed is None or seed == 0) and helper.main_program.random_seed != 0: + seed = helper.main_program.random_seed + helper.append_op( type='dropout', inputs={'X': [x]}, @@ -4845,7 +4850,7 @@ def dice_loss(input, label, epsilon=0.00001): loss = fluid.layers.dice_loss(input=predictions, label=label, 2) """ label = one_hot(label, depth=input.shape[-1]) - reduce_dim = range(1, len(input.shape)) + reduce_dim = list(range(1, len(input.shape))) inse = reduce_sum(input * label, dim=reduce_dim) dice_denominator = reduce_sum( input, dim=reduce_dim) + reduce_sum( diff --git a/python/paddle/fluid/layers/ops.py b/python/paddle/fluid/layers/ops.py index 01db8645b3aff77371f01c3dec51c85f99065552..f70c7f2258ce588444cf46d6c8affc4c9555203e 100644 --- a/python/paddle/fluid/layers/ops.py +++ b/python/paddle/fluid/layers/ops.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from layer_function_generator import generate_layer_fn +from .layer_function_generator import generate_layer_fn __activations__ = [ 'sigmoid', diff --git a/python/paddle/fluid/layers/tensor.py b/python/paddle/fluid/layers/tensor.py index b6614ecf3bc16e73683f4991779769049c6800ed..b93d721c12cb6ead044dc790f2f2af8a61a63b60 100644 --- a/python/paddle/fluid/layers/tensor.py +++ b/python/paddle/fluid/layers/tensor.py @@ -18,7 +18,7 @@ from ..framework import convert_np_dtype_to_dtype_ from ..framework import Variable from ..initializer import Constant, force_init_on_cpu from ..core import VarDesc -from layer_function_generator import templatedoc +from .layer_function_generator import templatedoc import numpy __all__ = [ diff --git a/python/paddle/fluid/lod_tensor.py b/python/paddle/fluid/lod_tensor.py index b2b3186c1e8dd84e1527ff18744bd611f1f74c5f..53c33616f55be5f5ef7068a6e94418e17d739e3c 100644 --- a/python/paddle/fluid/lod_tensor.py +++ b/python/paddle/fluid/lod_tensor.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import core +from . import core import numpy as np __all__ = ['create_lod_tensor', 'create_random_int_lodtensor'] @@ -24,7 +24,7 @@ def create_lod_tensor(data, recursive_seq_lens, place): Create a lod tensor by doing the following: - 1. Check that the length-based level of detail (LoD) also known as + 1. Check that the length-based level of detail (LoD) also known as recursive_sequence_lengths of the input is valid. 2. Convert recursive_sequence_lengths to a offset-based LoD. @@ -33,7 +33,7 @@ def create_lod_tensor(data, recursive_seq_lens, place): CPU or GPU device (based on input place). 4. Set the level of detail (LoD) using the offset-based LoD. - + Examples: Suppose we want LoDTensor to hold data for sequences of word, where each @@ -51,7 +51,7 @@ def create_lod_tensor(data, recursive_seq_lens, place): Args: data(numpy.ndarray|list|LoDTensor): a numpy array or a LoDTensor or a list holding the data to be copied. - recursive_seq_lens(list): a list of lists indicating the length-based level of detail + recursive_seq_lens(list): a list of lists indicating the length-based level of detail info specified by the user. place(Place): CPU or GPU place indicating where the data in the new LoDTensor will be stored. @@ -62,10 +62,10 @@ def create_lod_tensor(data, recursive_seq_lens, place): if isinstance(data, core.LoDTensor): return create_lod_tensor(np.array(data), recursive_seq_lens, place) elif isinstance(data, list): - # When input data is a list, it only deal with the case where the base element - # is an index of shape [1] and dtype int64 (e.g., word id). Hence, the generated - # LoDTensor will be of shape [n, 1] and dtype int64, where `n` is the total number - # of words or other indexes in the sequence. + # When input data is a list, it only deal with the case where the base element + # is an index of shape [1] and dtype int64 (e.g., word id). Hence, the generated + # LoDTensor will be of shape [n, 1] and dtype int64, where `n` is the total number + # of words or other indexes in the sequence. new_recursive_seq_lens = [] for seq in data: new_recursive_seq_lens.append(len(seq)) @@ -109,12 +109,12 @@ def create_random_int_lodtensor(recursive_seq_lens, base_shape, place, low, Suppose we want LoDTensor to hold data for sequences of word, where each word is represented by an integer. If we want to create a LoDTensor to represent two sentences, one of 2 words, and one of 3 words. Then - 'base_shape' is [1], input length-based 'recursive_seq_lens' is [[2, 3]]. - Then the overall shape of the LoDTensor would be [5, 1], holding 5 words + 'base_shape' is [1], input length-based 'recursive_seq_lens' is [[2, 3]]. + Then the overall shape of the LoDTensor would be [5, 1], holding 5 words for two sentences. Args: - recursive_seq_lens(list): a list of lists indicating the length-based + recursive_seq_lens(list): a list of lists indicating the length-based level of detail info specified by the user. base_shape(list): the shape of the basic element to be held by the LoDTensor. @@ -124,11 +124,11 @@ def create_random_int_lodtensor(recursive_seq_lens, base_shape, place, low, high(int): the upper bound of the random integers. Returns: - A fluid LoDTensor object with tensor data and recursive_seq_lens info. + A fluid LoDTensor object with tensor data and recursive_seq_lens info. """ assert isinstance(base_shape, list), "base_shape should be a list" # append the total number of basic elements to the front of its shape overall_shape = [sum(recursive_seq_lens[-1])] + base_shape - # the range of integer data elements is [low, high] + # the range of integer data elements is [low, high] data = np.random.random_integers(low, high, overall_shape).astype("int64") return create_lod_tensor(data, recursive_seq_lens, place) diff --git a/python/paddle/fluid/metrics.py b/python/paddle/fluid/metrics.py index b37b09ac81687882443c948569d9c4fca9310f78..cd8934522755691217a99a2cca271badda55368e 100644 --- a/python/paddle/fluid/metrics.py +++ b/python/paddle/fluid/metrics.py @@ -79,10 +79,10 @@ class MetricBase(object): """ states = { attr: value - for attr, value in self.__dict__.iteritems() + for attr, value in list(self.__dict__.items()) if not attr.startswith("_") } - for attr, value in states.iteritems(): + for attr, value in list(states.items()): if isinstance(value, int): setattr(self, attr, 0) elif isinstance(value, float): @@ -105,7 +105,7 @@ class MetricBase(object): """ states = { attr: value - for attr, value in self.__dict__.iteritems() + for attr, value in list(self.__dict__.items()) if not attr.startswith("_") } config = {} diff --git a/python/paddle/fluid/net_drawer.py b/python/paddle/fluid/net_drawer.py index 73946a0721dc4a6d03074a4708cf574951412e66..623a7d3fd05567a26bb6923550f597a0e1e27e32 100644 --- a/python/paddle/fluid/net_drawer.py +++ b/python/paddle/fluid/net_drawer.py @@ -24,7 +24,7 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) try: - from graphviz import Digraph + from .graphviz import Digraph except ImportError: logger.info( 'Cannot import graphviz, which is required for drawing a network. This ' @@ -77,7 +77,7 @@ def parse_graph(program, graph, var_dict, **kwargs): # fill the known variables for block in program.blocks: for var in block.vars: - if not var_dict.has_key(var): + if var not in var_dict: var_dict[var] = "Feed" temp_id = 0 @@ -93,17 +93,17 @@ def parse_graph(program, graph, var_dict, **kwargs): var_dict[arg] = op.type for e in op.inputs: for arg in e.arguments: - if var_dict.has_key(arg): + if arg in var_dict: graph.edge(**draw_edge(var_dict, op, e, arg)) break # only plot the first block def draw_graph(startup_program, main_program, **kwargs): - if kwargs.has_key("graph_attr"): + if "graph_attr" in kwargs: GRAPH_STYLE.update(kwargs[graph_attr]) - if kwargs.has_key("node_attr"): + if "node_attr" in kwargs: OP_STYLE.update(kwargs[node_attr]) - if kwargs.has_key("edge_attr"): + if "edge_attr" in kwargs: VAR_STYLE.update(kwargs[edge_attr]) graph_id = unique_id() diff --git a/python/paddle/fluid/nets.py b/python/paddle/fluid/nets.py index 9b3f2aebee73e56ee820dc8ff4c9cfabd1456aaa..08480671d8a5c50bbec97930c451cbcdc241e1fe 100644 --- a/python/paddle/fluid/nets.py +++ b/python/paddle/fluid/nets.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import layers +from . import layers __all__ = [ "simple_img_conv_pool", @@ -210,7 +210,7 @@ def img_conv_group(input, conv_with_batchnorm = __extend_list__(conv_with_batchnorm) conv_batchnorm_drop_rate = __extend_list__(conv_batchnorm_drop_rate) - for i in xrange(len(conv_num_filter)): + for i in range(len(conv_num_filter)): local_conv_act = conv_act if conv_with_batchnorm[i]: local_conv_act = None @@ -488,10 +488,11 @@ def scaled_dot_product_attention(queries, trans_x = layers.transpose(x, perm=[0, 2, 1, 3]) return layers.reshape( x=trans_x, - shape=map(int, [ - trans_x.shape[0], trans_x.shape[1], - trans_x.shape[2] * trans_x.shape[3] - ])) + shape=list( + map(int, [ + trans_x.shape[0], trans_x.shape[1], trans_x.shape[2] * + trans_x.shape[3] + ]))) q, k, v = __compute_qkv(queries, keys, values, num_heads) diff --git a/python/paddle/fluid/op.py b/python/paddle/fluid/op.py index 0b76e94157e378b40baff641c466968e239d8a83..93f021a360ac61f64e769d057df188d79f6f2bb6 100644 --- a/python/paddle/fluid/op.py +++ b/python/paddle/fluid/op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import six + import paddle.fluid.core as core import paddle.fluid.proto.framework_pb2 as framework_pb2 @@ -24,13 +26,13 @@ def get_all_op_protos(): protostrs = core.get_all_op_protos() ret_values = [] for pbstr in protostrs: - op_proto = framework_pb2.OpProto.FromString(str(pbstr)) + op_proto = framework_pb2.OpProto.FromString(six.binary_type(pbstr)) ret_values.append(op_proto) return ret_values def is_str(s): - return isinstance(s, str) or isinstance(s, unicode) + return isinstance(s, six.string_types) class OpDescCreationMethod(object): @@ -189,7 +191,7 @@ class OperatorFactory(object): return self.get_op_info(t).method(**kwargs) def types(self): - return self.op_methods.keys() + return list(self.op_methods.keys()) def get_op_info(self, t): if t not in self.op_methods: @@ -197,13 +199,13 @@ class OperatorFactory(object): return self.op_methods.get(t) def get_op_input_names(self, type): - return map(lambda x: x[0], self.get_op_info(type).inputs) + return [x[0] for x in self.get_op_info(type).inputs] def get_op_inputs(self, type): return self.get_op_info(type).inputs def get_op_output_names(self, type): - return map(lambda x: x[0], self.get_op_info(type).outputs) + return [x[0] for x in self.get_op_info(type).outputs] def get_op_outputs(self, type): return self.get_op_info(type).outputs diff --git a/python/paddle/fluid/optimizer.py b/python/paddle/fluid/optimizer.py index 3fe99f55011ab7f745c3ad98ec44dfe277a13e05..a07325f46a2892222c2d1dcd74aa7cb01f6760a1 100644 --- a/python/paddle/fluid/optimizer.py +++ b/python/paddle/fluid/optimizer.py @@ -14,15 +14,15 @@ import re from collections import defaultdict from paddle.fluid.framework import Program, Variable -import framework -import layers -from backward import append_backward -from framework import program_guard -import unique_name -from initializer import Constant -from layer_helper import LayerHelper -from regularizer import append_regularization_ops -from clip import append_gradient_clip_ops, error_clip_callback +from . import framework +from . import layers +from .backward import append_backward +from .framework import program_guard +from . import unique_name +from .initializer import Constant +from .layer_helper import LayerHelper +from .regularizer import append_regularization_ops +from .clip import append_gradient_clip_ops, error_clip_callback from contextlib import contextmanager __all__ = [ diff --git a/python/paddle/fluid/parallel_executor.py b/python/paddle/fluid/parallel_executor.py index 3bbd11d9836a62cdf9f2a84fc75e933001e12159..eabe6bb901ad1b7afe71717eb3f5f5765f261201 100644 --- a/python/paddle/fluid/parallel_executor.py +++ b/python/paddle/fluid/parallel_executor.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import core +from __future__ import print_function import multiprocessing -import framework -import executor +from . import core +from . import framework +from . import executor import warnings import sys import os @@ -94,7 +95,7 @@ class ParallelExecutor(object): self._places = [] self._act_places = [] if use_cuda: - for i in xrange(core.get_cuda_device_count()): + for i in range(core.get_cuda_device_count()): p = core.Place() self._act_places.append(core.CUDAPlace(i)) p.set_place(self._act_places[-1]) @@ -102,7 +103,7 @@ class ParallelExecutor(object): else: cpu_num = int( os.environ.get('CPU_NUM', multiprocessing.cpu_count())) - for i in xrange(cpu_num): + for i in range(cpu_num): p = core.Place() self._act_places.append(core.CPUPlace()) p.set_place(self._act_places[-1]) @@ -143,16 +144,16 @@ class ParallelExecutor(object): ) if share_vars_from else [] self.persistable_vars = [ - v.name - for v in filter( - lambda var: var.persistable and var.type != core.VarDesc.VarType.RAW, - main.list_vars()) + v.name for v in [ + var for var in main.list_vars() + if var.persistable and var.type != core.VarDesc.VarType.RAW + ] ] self.executor = core.ParallelExecutor( self._places, set([ - p.name for p in main.global_block()._iter_parameters() + p.name for p in main.global_block().iter_parameters() if not p.stop_gradient ]), set(self.persistable_vars), main.desc, loss_name @@ -227,7 +228,9 @@ class ParallelExecutor(object): """ if feed is None and feed_dict is not None: feed = feed_dict - print >> sys.stderr, "`feed_dict` is deprecated. Please use `feed=`" + print( + "`feed_dict` is deprecated. Please use `feed=`", + file=sys.stderr) if isinstance(feed, dict): feed_tensor_dict = dict() diff --git a/python/paddle/fluid/param_attr.py b/python/paddle/fluid/param_attr.py index 4a61f85ec4b5c5108ded31632af75dbbdaaaba71..afae577656c8970338f3b02208fcb4c738628ab6 100644 --- a/python/paddle/fluid/param_attr.py +++ b/python/paddle/fluid/param_attr.py @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from initializer import Initializer, Xavier, Constant -from regularizer import WeightDecayRegularizer +import six + +from .initializer import Initializer, Xavier, Constant +from .regularizer import WeightDecayRegularizer __all__ = [ 'ParamAttr', @@ -134,7 +136,7 @@ class ParamAttr(object): return [ParamAttr._to_attr(a) for a in arg] elif isinstance(arg, ParamAttr): return arg - elif isinstance(arg, str) or isinstance(arg, unicode): + elif isinstance(arg, six.string_types): return ParamAttr(name=arg) elif isinstance(arg, Initializer): return ParamAttr(initializer=arg) diff --git a/python/paddle/fluid/profiler.py b/python/paddle/fluid/profiler.py index 6a321ae024dcb50452bc4d96d7e7e70f590a42c6..60e9215457e2a7867d5d9ec69f65dd70bcba9745 100644 --- a/python/paddle/fluid/profiler.py +++ b/python/paddle/fluid/profiler.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import core +from . import core from contextlib import contextmanager import os @@ -224,7 +224,7 @@ def profiler(state, sorted_key=None, profile_path='/tmp/profile'): If the state == 'All', a profile proto file will be written to `profile_path`. This file records timeline information during the execution. - Then users can visualize this file to see the timeline, please refer + Then users can visualize this file to see the timeline, please refer https://github.com/PaddlePaddle/Paddle/blob/develop/doc/fluid/howto/optimization/timeline.md Args: diff --git a/python/paddle/fluid/recordio_writer.py b/python/paddle/fluid/recordio_writer.py index bd57772713057f12b876942de58ee43527e94834..93b38ad3fa37bd4bff04c529cd5518a8138e55ea 100644 --- a/python/paddle/fluid/recordio_writer.py +++ b/python/paddle/fluid/recordio_writer.py @@ -13,8 +13,8 @@ # limitations under the License. import os -import core import contextlib +from . import core __all__ = [ 'convert_reader_to_recordio_file', 'convert_reader_to_recordio_files' ] diff --git a/python/paddle/fluid/regularizer.py b/python/paddle/fluid/regularizer.py index 3712955b3b32de457a0d47120a00ab7d4ecd5a66..6eaac4432d4df1288f37607a01484434542f1138 100644 --- a/python/paddle/fluid/regularizer.py +++ b/python/paddle/fluid/regularizer.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import framework +from . import framework from . import core __all__ = ['L1Decay', 'L2Decay', 'L1DecayRegularizer', 'L2DecayRegularizer'] diff --git a/python/paddle/fluid/tests/book/high-level-api/fit_a_line/test_fit_a_line.py b/python/paddle/fluid/tests/book/high-level-api/fit_a_line/test_fit_a_line.py index ad28c9eff560507e5b326451159be3949353f58f..36a1a223cfd7c69aff3e8648da990d23e4e75202 100644 --- a/python/paddle/fluid/tests/book/high-level-api/fit_a_line/test_fit_a_line.py +++ b/python/paddle/fluid/tests/book/high-level-api/fit_a_line/test_fit_a_line.py @@ -63,7 +63,7 @@ def train(use_cuda, train_program, params_dirname): if event.step == 10: test_metrics = trainer.test( reader=test_reader, feed_order=['x', 'y']) - print test_metrics + print(test_metrics) ''' ... ['25.768919467926025'] diff --git a/python/paddle/fluid/tests/book/high-level-api/image_classification/cifar10_small_test_set.py b/python/paddle/fluid/tests/book/high-level-api/image_classification/cifar10_small_test_set.py index 7fed6d914f75b690e34411aa154359c93b6ca989..9e4c384d92943227c2d68da829e6019e649a35fb 100644 --- a/python/paddle/fluid/tests/book/high-level-api/image_classification/cifar10_small_test_set.py +++ b/python/paddle/fluid/tests/book/high-level-api/image_classification/cifar10_small_test_set.py @@ -28,11 +28,12 @@ images per class. """ -import cPickle import itertools import numpy import paddle.v2.dataset.common import tarfile +from six.moves import cPickle as pickle +from six.moves import zip __all__ = ['train10'] @@ -46,7 +47,7 @@ def reader_creator(filename, sub_name, batch_size=None): data = batch['data'] labels = batch.get('labels', batch.get('fine_labels', None)) assert labels is not None - for sample, label in itertools.izip(data, labels): + for sample, label in zip(data, labels): yield (sample / 255.0).astype(numpy.float32), int(label) def reader(): @@ -56,7 +57,7 @@ def reader_creator(filename, sub_name, batch_size=None): batch_count = 0 for name in names: - batch = cPickle.load(f.extractfile(name)) + batch = pickle.load(f.extractfile(name)) for item in read_batch(batch): if isinstance(batch_size, int) and batch_count > batch_size: break diff --git a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_resnet.py b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_resnet.py index 8e222d26907e8fe697b596a67e62cc9df84afe0e..a1f62db093904b617f0e37dc20d586ccea7eacd2 100644 --- a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_resnet.py +++ b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_resnet.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import paddle import paddle.fluid as fluid import numpy diff --git a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py index dbc7bc06c93157f271c79e85b6925468e861e57f..8429551765740e7db0eda82ce0b17cff129359b0 100644 --- a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py +++ b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import paddle import paddle.fluid as fluid import numpy diff --git a/python/paddle/fluid/tests/book/high-level-api/label_semantic_roles/test_label_semantic_roles_newapi.py b/python/paddle/fluid/tests/book/high-level-api/label_semantic_roles/test_label_semantic_roles_newapi.py index 67aa21e8c5699f1cb568dad23cd13f4cb51a6ec9..e3602e2d5643c233b2575d1adb7f181127f60287 100755 --- a/python/paddle/fluid/tests/book/high-level-api/label_semantic_roles/test_label_semantic_roles_newapi.py +++ b/python/paddle/fluid/tests/book/high-level-api/label_semantic_roles/test_label_semantic_roles_newapi.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import paddle import paddle.fluid as fluid import numpy as np @@ -178,14 +176,15 @@ def train(use_cuda, train_program, params_dirname): if float(avg_cost) < 100.0: # Large value to increase CI speed trainer.save_params(params_dirname) else: - print('BatchID {0}, Test Loss {1:0.2}'.format(event.epoch + 1, - float(avg_cost))) + print( + ('BatchID {0}, Test Loss {1:0.2}'.format(event.epoch + 1, + float(avg_cost)))) if math.isnan(float(avg_cost)): sys.exit("got NaN loss, training failed.") elif isinstance(event, fluid.EndStepEvent): print("Step {0}, Epoch {1} Metrics {2}".format( - event.step, event.epoch, map(np.array, event.metrics))) + event.step, event.epoch, list(map(np.array, event.metrics)))) if event.step == 1: # Run 2 iterations to speed CI trainer.save_params(params_dirname) trainer.stop() @@ -207,14 +206,14 @@ def infer(use_cuda, inference_program, params_dirname): inference_program, param_path=params_dirname, place=place) # Setup input by creating LoDTensor to represent sequence of words. - # Here each word is the basic element of the LoDTensor and the shape of - # each word (base_shape) should be [1] since it is simply an index to + # Here each word is the basic element of the LoDTensor and the shape of + # each word (base_shape) should be [1] since it is simply an index to # look up for the corresponding word vector. # Suppose the recursive_sequence_lengths info is set to [[3, 4, 2]], - # which has only one level of detail. Then the created LoDTensor will have only - # one higher level structure (sequence of words, or sentence) than the basic - # element (word). Hence the LoDTensor will hold data for three sentences of - # length 3, 4 and 2, respectively. + # which has only one level of detail. Then the created LoDTensor will have only + # one higher level structure (sequence of words, or sentence) than the basic + # element (word). Hence the LoDTensor will hold data for three sentences of + # length 3, 4 and 2, respectively. # Note that recursive_sequence_lengths should be a list of lists. recursive_seq_lens = [[3, 4, 2]] base_shape = [1] diff --git a/python/paddle/fluid/tests/book/high-level-api/machine_translation/test_machine_translation.py b/python/paddle/fluid/tests/book/high-level-api/machine_translation/test_machine_translation.py index 8becd2404b0201c44b587a28e88995958082cd28..6fb0c85a8be2b4560ea1fdb32f01146a9206ee78 100644 --- a/python/paddle/fluid/tests/book/high-level-api/machine_translation/test_machine_translation.py +++ b/python/paddle/fluid/tests/book/high-level-api/machine_translation/test_machine_translation.py @@ -250,7 +250,7 @@ def decode_main(use_cuda, is_sparse): feeder = fluid.DataFeeder(feed_list, place) for data in train_data(): - feed_dict = feeder.feed(map(lambda x: [x[0]], data)) + feed_dict = feeder.feed([[x[0]] for x in data]) feed_dict['init_ids'] = init_ids feed_dict['init_scores'] = init_scores @@ -259,7 +259,7 @@ def decode_main(use_cuda, is_sparse): feed=feed_dict, fetch_list=[translation_ids, translation_scores], return_numpy=False) - print result_ids.recursive_sequence_lengths() + print(result_ids.recursive_sequence_lengths()) break diff --git a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_conv.py b/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_conv.py index fd278f45f1c1b71a1653c3b28ace8bca8e4b1545..898807db6f343cbefcc877e0f03ed6c5b82dd669 100644 --- a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_conv.py +++ b/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_conv.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function + import argparse import paddle.fluid as fluid import paddle.fluid.core as core @@ -89,8 +89,10 @@ def train(use_cuda, train_program, params_dirname): if math.isnan(avg_cost): sys.exit("got NaN loss, training failed.") elif isinstance(event, fluid.EndStepEvent): - print("Step {0}, Epoch {1} Metrics {2}".format( - event.step, event.epoch, map(numpy.array, event.metrics))) + print( + ("Step {0}, Epoch {1} Metrics {2}".format( + event.step, event.epoch, + list(map(numpy.array, event.metrics))))) train_reader = paddle.batch( paddle.reader.shuffle( diff --git a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py b/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py index b2b544e791d7ea35ff7d2c9a2dce7ce7f5680f38..6dd64be315159f1835244fa027e578434e6cb038 100644 --- a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py +++ b/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function + import argparse import paddle.fluid as fluid import paddle diff --git a/python/paddle/fluid/tests/book/high-level-api/recommender_system/test_recommender_system_newapi.py b/python/paddle/fluid/tests/book/high-level-api/recommender_system/test_recommender_system_newapi.py index c860f1641708d947fd2a8008d3d3ccd0a231f6c2..60f3d8e105209938360487d963b0328d95e7b1f0 100644 --- a/python/paddle/fluid/tests/book/high-level-api/recommender_system/test_recommender_system_newapi.py +++ b/python/paddle/fluid/tests/book/high-level-api/recommender_system/test_recommender_system_newapi.py @@ -186,8 +186,9 @@ def train(use_cuda, train_program, params_dirname): trainer.save_params(params_dirname) trainer.stop() else: - print('BatchID {0}, Test Loss {1:0.2}'.format(event.epoch + 1, - float(avg_cost))) + print( + ('BatchID {0}, Test Loss {1:0.2}'.format(event.epoch + 1, + float(avg_cost)))) if math.isnan(float(avg_cost)): sys.exit("got NaN loss, training failed.") diff --git a/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_conv.py b/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_conv.py index 1668ae83d3581125b799508c8c3115a038e93d5a..24e65d1bd54cff7ad64453a3a61f50351d32ef08 100644 --- a/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_conv.py +++ b/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_conv.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import paddle import paddle.fluid as fluid from functools import partial @@ -98,7 +96,7 @@ def train(use_cuda, train_program, params_dirname): sys.exit("got NaN loss, training failed.") elif isinstance(event, fluid.EndStepEvent): print("Step {0}, Epoch {1} Metrics {2}".format( - event.step, event.epoch, map(np.array, event.metrics))) + event.step, event.epoch, list(map(np.array, event.metrics)))) if event.step == 1: # Run 2 iterations to speed CI trainer.save_params(params_dirname) trainer.stop() @@ -125,14 +123,14 @@ def infer(use_cuda, inference_program, params_dirname=None): place=place) # Setup input by creating LoDTensor to represent sequence of words. - # Here each word is the basic element of the LoDTensor and the shape of - # each word (base_shape) should be [1] since it is simply an index to + # Here each word is the basic element of the LoDTensor and the shape of + # each word (base_shape) should be [1] since it is simply an index to # look up for the corresponding word vector. # Suppose the recursive_sequence_lengths info is set to [[3, 4, 2]], - # which has only one level of detail. Then the created LoDTensor will have only - # one higher level structure (sequence of words, or sentence) than the basic - # element (word). Hence the LoDTensor will hold data for three sentences of - # length 3, 4 and 2, respectively. + # which has only one level of detail. Then the created LoDTensor will have only + # one higher level structure (sequence of words, or sentence) than the basic + # element (word). Hence the LoDTensor will hold data for three sentences of + # length 3, 4 and 2, respectively. # Note that recursive_sequence_lengths should be a list of lists. recursive_seq_lens = [[3, 4, 2]] base_shape = [1] diff --git a/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_dynamic_rnn.py b/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_dynamic_rnn.py index 8da89d82cb8e00853eebfd794602a0e1e1020e7c..b3b1505a0fad07144f3f53c22abd5553054d8c51 100644 --- a/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_dynamic_rnn.py +++ b/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_dynamic_rnn.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import paddle import paddle.fluid as fluid from functools import partial @@ -113,7 +111,7 @@ def train(use_cuda, train_program, params_dirname): sys.exit("got NaN loss, training failed.") elif isinstance(event, fluid.EndStepEvent): print("Step {0}, Epoch {1} Metrics {2}".format( - event.step, event.epoch, map(np.array, event.metrics))) + event.step, event.epoch, list(map(np.array, event.metrics)))) if event.step == 1: # Run 2 iterations to speed CI trainer.save_params(params_dirname) trainer.stop() @@ -140,14 +138,14 @@ def infer(use_cuda, inference_program, params_dirname=None): place=place) # Setup input by creating LoDTensor to represent sequence of words. - # Here each word is the basic element of the LoDTensor and the shape of - # each word (base_shape) should be [1] since it is simply an index to + # Here each word is the basic element of the LoDTensor and the shape of + # each word (base_shape) should be [1] since it is simply an index to # look up for the corresponding word vector. # Suppose the recursive_sequence_lengths info is set to [[3, 4, 2]], - # which has only one level of detail. Then the created LoDTensor will have only - # one higher level structure (sequence of words, or sentence) than the basic - # element (word). Hence the LoDTensor will hold data for three sentences of - # length 3, 4 and 2, respectively. + # which has only one level of detail. Then the created LoDTensor will have only + # one higher level structure (sequence of words, or sentence) than the basic + # element (word). Hence the LoDTensor will hold data for three sentences of + # length 3, 4 and 2, respectively. # Note that recursive_sequence_lengths should be a list of lists. recursive_seq_lens = [[3, 4, 2]] base_shape = [1] diff --git a/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_stacked_lstm.py b/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_stacked_lstm.py index 74faa2e8aa734cd644dfcc38127fd12df1fb1092..25f99ff0fd2d1050bb62338a6bf87aa29f913fb6 100644 --- a/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_stacked_lstm.py +++ b/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_stacked_lstm.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import paddle import paddle.fluid as fluid from functools import partial @@ -107,7 +105,7 @@ def train(use_cuda, train_program, params_dirname): sys.exit("got NaN loss, training failed.") elif isinstance(event, fluid.EndStepEvent): print("Step {0}, Epoch {1} Metrics {2}".format( - event.step, event.epoch, map(np.array, event.metrics))) + event.step, event.epoch, list(map(np.array, event.metrics)))) if event.step == 1: # Run 2 iterations to speed CI trainer.save_params(params_dirname) trainer.stop() @@ -135,14 +133,14 @@ def infer(use_cuda, inference_program, params_dirname=None): place=place) # Setup input by creating LoDTensor to represent sequence of words. - # Here each word is the basic element of the LoDTensor and the shape of - # each word (base_shape) should be [1] since it is simply an index to + # Here each word is the basic element of the LoDTensor and the shape of + # each word (base_shape) should be [1] since it is simply an index to # look up for the corresponding word vector. # Suppose the recursive_sequence_lengths info is set to [[3, 4, 2]], - # which has only one level of detail. Then the created LoDTensor will have only - # one higher level structure (sequence of words, or sentence) than the basic - # element (word). Hence the LoDTensor will hold data for three sentences of - # length 3, 4 and 2, respectively. + # which has only one level of detail. Then the created LoDTensor will have only + # one higher level structure (sequence of words, or sentence) than the basic + # element (word). Hence the LoDTensor will hold data for three sentences of + # length 3, 4 and 2, respectively. # Note that recursive_sequence_lengths should be a list of lists. recursive_seq_lens = [[3, 4, 2]] base_shape = [1] diff --git a/python/paddle/fluid/tests/book/notest_understand_sentiment.py b/python/paddle/fluid/tests/book/notest_understand_sentiment.py index 95002aa7f9bb639828b47eb1e86e4ef954fb85ff..ce6342c2dad0b33e57d0ea90fc6ef1660ae4e68b 100644 --- a/python/paddle/fluid/tests/book/notest_understand_sentiment.py +++ b/python/paddle/fluid/tests/book/notest_understand_sentiment.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function + from paddle.fluid.layers.device import get_places import unittest import paddle.fluid as fluid @@ -175,7 +175,7 @@ def train(word_dict, def train_loop(main_program): exe.run(fluid.default_startup_program()) - for pass_id in xrange(PASS_NUM): + for pass_id in range(PASS_NUM): for data in train_data(): cost_val, acc_val = exe.run(main_program, feed=feeder.feed(data), @@ -235,14 +235,14 @@ def infer(word_dict, use_cuda, save_dirname=None): word_dict_len = len(word_dict) # Setup input by creating LoDTensor to represent sequence of words. - # Here each word is the basic element of the LoDTensor and the shape of - # each word (base_shape) should be [1] since it is simply an index to + # Here each word is the basic element of the LoDTensor and the shape of + # each word (base_shape) should be [1] since it is simply an index to # look up for the corresponding word vector. # Suppose the recursive_sequence_lengths info is set to [[3, 4, 2]], - # which has only one level of detail. Then the created LoDTensor will have only - # one higher level structure (sequence of words, or sentence) than the basic - # element (word). Hence the LoDTensor will hold data for three sentences of - # length 3, 4 and 2, respectively. + # which has only one level of detail. Then the created LoDTensor will have only + # one higher level structure (sequence of words, or sentence) than the basic + # element (word). Hence the LoDTensor will hold data for three sentences of + # length 3, 4 and 2, respectively. # Note that recursive_sequence_lengths should be a list of lists. recursive_seq_lens = [[3, 4, 2]] base_shape = [1] diff --git a/python/paddle/fluid/tests/book/test_fit_a_line.py b/python/paddle/fluid/tests/book/test_fit_a_line.py index 71bf5f8b3a9b17f24ce35220a9348bb871852623..37b64fa94a9aad7042e153e414ed29de3142db5a 100644 --- a/python/paddle/fluid/tests/book/test_fit_a_line.py +++ b/python/paddle/fluid/tests/book/test_fit_a_line.py @@ -114,7 +114,7 @@ def infer(use_cuda, save_dirname=None): test_reader = paddle.batch( paddle.dataset.uci_housing.test(), batch_size=batch_size) - test_data = test_reader().next() + test_data = next(test_reader()) test_feat = numpy.array( [data[0] for data in test_data]).astype("float32") test_label = numpy.array( diff --git a/python/paddle/fluid/tests/book/test_image_classification.py b/python/paddle/fluid/tests/book/test_image_classification.py index a2fb186b86c9706ac1aff0de49defbfb06e2eb0f..de6fe5f140a86545e3291db165af824739a814ef 100644 --- a/python/paddle/fluid/tests/book/test_image_classification.py +++ b/python/paddle/fluid/tests/book/test_image_classification.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import paddle import paddle.fluid as fluid import contextlib @@ -121,7 +119,7 @@ def train(net_type, use_cuda, save_dirname, is_local): avg_cost = fluid.layers.mean(cost) acc = fluid.layers.accuracy(input=predict, label=label) - # Test program + # Test program test_program = fluid.default_main_program().clone(for_test=True) optimizer = fluid.optimizer.Adam(learning_rate=0.001) diff --git a/python/paddle/fluid/tests/book/test_label_semantic_roles.py b/python/paddle/fluid/tests/book/test_label_semantic_roles.py index d489feae9c568ec1d9e3a230766d10d1ced0200a..b7ac911cafdc751f38c7f66bc48263a17a84dc08 100644 --- a/python/paddle/fluid/tests/book/test_label_semantic_roles.py +++ b/python/paddle/fluid/tests/book/test_label_semantic_roles.py @@ -181,7 +181,7 @@ def train(use_cuda, save_dirname=None, is_local=True): start_time = time.time() batch_id = 0 - for pass_id in xrange(PASS_NUM): + for pass_id in range(PASS_NUM): for data in train_data(): cost = exe.run(main_program, feed=feeder.feed(data), @@ -248,14 +248,14 @@ def infer(use_cuda, save_dirname=None): fetch_targets] = fluid.io.load_inference_model(save_dirname, exe) # Setup input by creating LoDTensor to represent sequence of words. - # Here each word is the basic element of the LoDTensor and the shape of - # each word (base_shape) should be [1] since it is simply an index to + # Here each word is the basic element of the LoDTensor and the shape of + # each word (base_shape) should be [1] since it is simply an index to # look up for the corresponding word vector. # Suppose the recursive_sequence_lengths info is set to [[3, 4, 2]], - # which has only one level of detail. Then the created LoDTensor will have only - # one higher level structure (sequence of words, or sentence) than the basic - # element (word). Hence the LoDTensor will hold data for three sentences of - # length 3, 4 and 2, respectively. + # which has only one level of detail. Then the created LoDTensor will have only + # one higher level structure (sequence of words, or sentence) than the basic + # element (word). Hence the LoDTensor will hold data for three sentences of + # length 3, 4 and 2, respectively. # Note that recursive_sequence_lengths should be a list of lists. recursive_seq_lens = [[3, 4, 2]] base_shape = [1] diff --git a/python/paddle/fluid/tests/book/test_machine_translation.py b/python/paddle/fluid/tests/book/test_machine_translation.py index 90c301a66105d8d872ee531556c5060b5d727515..462faad3e1cb7108f3bd6934017efe25fb9a4276 100644 --- a/python/paddle/fluid/tests/book/test_machine_translation.py +++ b/python/paddle/fluid/tests/book/test_machine_translation.py @@ -199,7 +199,7 @@ def train_main(use_cuda, is_sparse, is_local=True): feeder = fluid.DataFeeder(feed_list, place) batch_id = 0 - for pass_id in xrange(1): + for pass_id in range(1): for data in train_data(): outs = exe.run(main_program, feed=feeder.feed(data), @@ -273,7 +273,7 @@ def decode_main(use_cuda, is_sparse): feeder = fluid.DataFeeder(feed_list, place) for data in train_data(): - feed_dict = feeder.feed(map(lambda x: [x[0]], data)) + feed_dict = feeder.feed([[x[0]] for x in data]) feed_dict['init_ids'] = init_ids feed_dict['init_scores'] = init_scores @@ -282,7 +282,7 @@ def decode_main(use_cuda, is_sparse): feed=feed_dict, fetch_list=[translation_ids, translation_scores], return_numpy=False) - print result_ids.recursive_sequence_lengths() + print(result_ids.recursive_sequence_lengths()) break diff --git a/python/paddle/fluid/tests/book/test_recognize_digits.py b/python/paddle/fluid/tests/book/test_recognize_digits.py index c471863920999a28cbede93a7965f07ee784f96d..3e5f76d12d41d016c995e5c85feda3c1847e356f 100644 --- a/python/paddle/fluid/tests/book/test_recognize_digits.py +++ b/python/paddle/fluid/tests/book/test_recognize_digits.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function import paddle.fluid.core as core import math diff --git a/python/paddle/fluid/tests/book/test_recommender_system.py b/python/paddle/fluid/tests/book/test_recommender_system.py index 6548766ef5d0162b50d4dd072e8e91dd95dc5d2b..b30c8771fcf267260c4c5aa7076bedc89e3b7e8b 100644 --- a/python/paddle/fluid/tests/book/test_recommender_system.py +++ b/python/paddle/fluid/tests/book/test_recommender_system.py @@ -260,15 +260,15 @@ def infer(use_cuda, save_dirname=None): # Use the first data from paddle.dataset.movielens.test() as input assert feed_target_names[0] == "user_id" - # Use create_lod_tensor(data, recursive_sequence_lengths, place) API - # to generate LoD Tensor where `data` is a list of sequences of index - # numbers, `recursive_sequence_lengths` is the length-based level of detail + # Use create_lod_tensor(data, recursive_sequence_lengths, place) API + # to generate LoD Tensor where `data` is a list of sequences of index + # numbers, `recursive_sequence_lengths` is the length-based level of detail # (lod) info associated with `data`. # For example, data = [[10, 2, 3], [2, 3]] means that it contains # two sequences of indexes, of length 3 and 2, respectively. - # Correspondingly, recursive_sequence_lengths = [[3, 2]] contains one - # level of detail info, indicating that `data` consists of two sequences - # of length 3 and 2, respectively. + # Correspondingly, recursive_sequence_lengths = [[3, 2]] contains one + # level of detail info, indicating that `data` consists of two sequences + # of length 3 and 2, respectively. user_id = fluid.create_lod_tensor([[1]], [[1]], place) assert feed_target_names[1] == "gender_id" diff --git a/python/paddle/fluid/tests/book/test_rnn_encoder_decoder.py b/python/paddle/fluid/tests/book/test_rnn_encoder_decoder.py index 467282624154086a874b0e73736ed5b1358915ff..2e79be2bd0fc7a368df86e188b7fa616055bb3e7 100644 --- a/python/paddle/fluid/tests/book/test_rnn_encoder_decoder.py +++ b/python/paddle/fluid/tests/book/test_rnn_encoder_decoder.py @@ -175,7 +175,7 @@ def train(use_cuda, save_dirname=None): feeder = fluid.DataFeeder(feed_list, place) batch_id = 0 - for pass_id in xrange(2): + for pass_id in range(2): for data in train_data(): outs = exe.run(framework.default_main_program(), feed=feeder.feed(data), @@ -213,14 +213,14 @@ def infer(use_cuda, save_dirname=None): fetch_targets] = fluid.io.load_inference_model(save_dirname, exe) # Setup input by creating LoDTensor to represent sequence of words. - # Here each word is the basic element of the LoDTensor and the shape of - # each word (base_shape) should be [1] since it is simply an index to + # Here each word is the basic element of the LoDTensor and the shape of + # each word (base_shape) should be [1] since it is simply an index to # look up for the corresponding word vector. # Suppose the recursive_sequence_lengths info is set to [[4, 6]], - # which has only one level of detail. Then the created LoDTensor will have only - # one higher level structure (sequence of words, or sentence) than the basic - # element (word). Hence the LoDTensor will hold data for two sentences of - # length 4 and 6, respectively. + # which has only one level of detail. Then the created LoDTensor will have only + # one higher level structure (sequence of words, or sentence) than the basic + # element (word). Hence the LoDTensor will hold data for two sentences of + # length 4 and 6, respectively. # Note that recursive_sequence_lengths should be a list of lists. recursive_seq_lens = [[4, 6]] base_shape = [1] diff --git a/python/paddle/fluid/tests/book/test_word2vec.py b/python/paddle/fluid/tests/book/test_word2vec.py index 3b957508ca1f11fea3bbc182dca7eaa938594cb6..e761e05795313da23a9d984263ac2e202939b1e7 100644 --- a/python/paddle/fluid/tests/book/test_word2vec.py +++ b/python/paddle/fluid/tests/book/test_word2vec.py @@ -85,9 +85,11 @@ def train(use_cuda, is_sparse, is_parallel, save_dirname, is_local=True): pd = fluid.layers.ParallelDo(places) with pd.do(): avg_cost, predict_word = __network__( - map(pd.read_input, [ - first_word, second_word, third_word, forth_word, next_word - ])) + list( + map(pd.read_input, [ + first_word, second_word, third_word, forth_word, + next_word + ]))) pd.write_output(avg_cost) avg_cost = fluid.layers.mean(pd()) @@ -167,11 +169,11 @@ def infer(use_cuda, save_dirname=None): word_dict = paddle.dataset.imikolov.build_dict() dict_size = len(word_dict) - # Setup inputs by creating 4 LoDTensors representing 4 words. Here each word - # is simply an index to look up for the corresponding word vector and hence - # the shape of word (base_shape) should be [1]. The recursive_sequence_lengths, - # which is length-based level of detail (lod) of each LoDTensor, should be [[1]] - # meaning there is only one level of detail and there is only one sequence of + # Setup inputs by creating 4 LoDTensors representing 4 words. Here each word + # is simply an index to look up for the corresponding word vector and hence + # the shape of word (base_shape) should be [1]. The recursive_sequence_lengths, + # which is length-based level of detail (lod) of each LoDTensor, should be [[1]] + # meaning there is only one level of detail and there is only one sequence of # one word on this level. # Note that recursive_sequence_lengths should be a list of lists. recursive_seq_lens = [[1]] diff --git a/python/paddle/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py b/python/paddle/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py index bec9f8594ff7c1aff8ae5ed55c9623754d9ea091..ccc62b442f62fa9fa175de031b0732febe38ee9a 100644 --- a/python/paddle/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py +++ b/python/paddle/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py @@ -78,7 +78,7 @@ for pass_id in range(PASS_NUM): if avg_loss_value[0] < 10.0: exit(0) # if avg cost less than 10.0, we think our code is good. - print avg_loss_value[0] + print(avg_loss_value[0]) if math.isnan(float(avg_loss_value)): sys.exit("got NaN loss, training failed.") exit(1) diff --git a/python/paddle/fluid/tests/book_memory_optimization/test_memopt_image_classification_train.py b/python/paddle/fluid/tests/book_memory_optimization/test_memopt_image_classification_train.py index dfebb9a06ea4f290f128c486dcaccaeccdcef8c4..b2a59d27da9b3348b581d51a68d769bbf3b90d35 100644 --- a/python/paddle/fluid/tests/book_memory_optimization/test_memopt_image_classification_train.py +++ b/python/paddle/fluid/tests/book_memory_optimization/test_memopt_image_classification_train.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import sys import paddle diff --git a/python/paddle/fluid/tests/book_memory_optimization/test_memopt_machine_translation.py b/python/paddle/fluid/tests/book_memory_optimization/test_memopt_machine_translation.py index fa696acdfa9058af14f0bd34ce1a2980db5aeafc..323ddfb6911fdd57b32344933373189370005126 100644 --- a/python/paddle/fluid/tests/book_memory_optimization/test_memopt_machine_translation.py +++ b/python/paddle/fluid/tests/book_memory_optimization/test_memopt_machine_translation.py @@ -118,7 +118,7 @@ def main(): feeder = fluid.DataFeeder(feed_list, place) batch_id = 0 - for pass_id in xrange(10): + for pass_id in range(10): for data in train_data(): outs = exe.run(fluid.default_main_program(), feed=feeder.feed(data), diff --git a/python/paddle/fluid/tests/demo/fc_gan.py b/python/paddle/fluid/tests/demo/fc_gan.py index 8ea1b2b15cc0c0eb5bca67a9c5a6ac6c6774e7e2..3d92f50f0adeca79adefc291cdfba6a012fc2118 100644 --- a/python/paddle/fluid/tests/demo/fc_gan.py +++ b/python/paddle/fluid/tests/demo/fc_gan.py @@ -137,7 +137,7 @@ def main(): generated_img = exe.run(g_program, feed={'noise': n}, fetch_list={g_img})[0] - real_data = numpy.array(map(lambda x: x[0], data)).astype('float32') + real_data = numpy.array([x[0] for x in data]).astype('float32') real_data = real_data.reshape(num_true, 784) total_data = numpy.concatenate([real_data, generated_img]) total_label = numpy.concatenate([ @@ -150,7 +150,7 @@ def main(): feed={'img': total_data, 'label': total_label}, fetch_list={d_loss})[0] - for _ in xrange(NUM_TRAIN_TIMES_OF_DG): + for _ in range(NUM_TRAIN_TIMES_OF_DG): n = numpy.random.uniform( low=-1.0, high=1.0, size=[2 * num_true * NOISE_SIZE]).astype('float32').reshape( diff --git a/python/paddle/fluid/tests/demo/file_reader/convert_data_to_recordio.py b/python/paddle/fluid/tests/demo/file_reader/convert_data_to_recordio.py index b839e14889884bca8d27586aa8c1d76fba3458c1..a00325d79be2eba4d7f770b5316c5857952fe272 100644 --- a/python/paddle/fluid/tests/demo/file_reader/convert_data_to_recordio.py +++ b/python/paddle/fluid/tests/demo/file_reader/convert_data_to_recordio.py @@ -36,7 +36,7 @@ if len(sys.argv) == 1: else: word_dict = load_vocab(sys.argv[1]) word_dict[""] = len(word_dict) -print "Dict dim = ", len(word_dict) +print("Dict dim = ", len(word_dict)) # input text data data = fluid.layers.data(name="words", shape=[1], dtype="int64", lod_level=1) diff --git a/python/paddle/fluid/tests/no_test_concurrency.py b/python/paddle/fluid/tests/no_test_concurrency.py index e8f6cfb4a907b2c01e9662e7e9bf2cb0fbd6cb1b..3bc0c9808e2345b610dea79abc56cfb0065ea46f 100644 --- a/python/paddle/fluid/tests/no_test_concurrency.py +++ b/python/paddle/fluid/tests/no_test_concurrency.py @@ -194,7 +194,7 @@ class TestRoutineOp(unittest.TestCase): quit_ch = fluid.make_channel(dtype=core.VarDesc.VarType.LOD_TENSOR) with fluid.Go(): - for i in xrange(10): + for i in range(10): fluid.channel_recv(ch1, result) Print(result) diff --git a/python/paddle/fluid/tests/test_beam_search_decoder.py b/python/paddle/fluid/tests/test_beam_search_decoder.py index 7a2502fa2f9733a7280e8e8d884b61b419719492..8bf750940d570cdad5e110168afc5f632202e869 100644 --- a/python/paddle/fluid/tests/test_beam_search_decoder.py +++ b/python/paddle/fluid/tests/test_beam_search_decoder.py @@ -155,7 +155,7 @@ def train_main(use_cuda): ] feeder = fluid.DataFeeder(feed_list, place) - for pass_id in xrange(1): + for pass_id in range(1): for batch_id, data in enumerate(train_reader()): outs = exe.run(main_program, feed=feeder.feed(data), @@ -204,8 +204,8 @@ def decode_main(use_cuda): ] feeder = fluid.DataFeeder(feed_list, place) - data = train_reader().next() - feed_dict = feeder.feed(map(lambda x: [x[0]], data)) + data = next(train_reader()) + feed_dict = feeder.feed([[x[0]] for x in data]) feed_dict['init_ids'] = init_ids feed_dict['init_scores'] = init_scores @@ -214,7 +214,7 @@ def decode_main(use_cuda): feed=feed_dict, fetch_list=[translation_ids, translation_scores], return_numpy=False) - print result_ids.lod() + print(result_ids.lod()) class TestBeamSearchDecoder(unittest.TestCase): diff --git a/python/paddle/fluid/tests/test_detection.py b/python/paddle/fluid/tests/test_detection.py index 2d70c986b1b6c42ff709e9cf3b4234cf4fc26836..fd45abd0a77cb54a3ca8e60cf80a1efe9f9d2060 100644 --- a/python/paddle/fluid/tests/test_detection.py +++ b/python/paddle/fluid/tests/test_detection.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function import paddle.fluid as fluid import paddle.fluid.layers as layers from paddle.fluid.framework import Program, program_guard diff --git a/python/paddle/fluid/tests/test_error_clip.py b/python/paddle/fluid/tests/test_error_clip.py index 3dc858971c584cca947cd958680dbdcf25df9e99..e8edd7fbbb31b1a6ecbf2a25a7d39e7b3f66363a 100644 --- a/python/paddle/fluid/tests/test_error_clip.py +++ b/python/paddle/fluid/tests/test_error_clip.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function import numpy as np import paddle import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/test_if_else_op.py b/python/paddle/fluid/tests/test_if_else_op.py index 799c31dfe5161ff6aef47601f1b6f6e38885760b..082f64c146f65eee4be0757d07495c33764fa841 100644 --- a/python/paddle/fluid/tests/test_if_else_op.py +++ b/python/paddle/fluid/tests/test_if_else_op.py @@ -76,15 +76,15 @@ class TestMNISTIfElseOp(unittest.TestCase): PASS_NUM = 100 for pass_id in range(PASS_NUM): for data in train_reader(): - x_data = np.array(map(lambda x: x[0], data)).astype("float32") - y_data = np.array(map(lambda x: x[1], data)).astype("int64") + x_data = np.array([x[0] for x in data]).astype("float32") + y_data = np.array([x[1] for x in data]).astype("int64") y_data = np.expand_dims(y_data, axis=1) outs = exe.run(prog, feed={'x': x_data, 'y': y_data}, fetch_list=[avg_loss]) - print outs[0] + print(outs[0]) if outs[0] < 1.0: return self.assertFalse(True) @@ -131,15 +131,15 @@ class TestMNISTIfElseOp(unittest.TestCase): PASS_NUM = 100 for pass_id in range(PASS_NUM): for data in train_reader(): - x_data = np.array(map(lambda x: x[0], data)).astype("float32") - y_data = np.array(map(lambda x: x[1], data)).astype("int64") + x_data = np.array([x[0] for x in data]).astype("float32") + y_data = np.array([x[1] for x in data]).astype("int64") y_data = y_data.reshape((y_data.shape[0], 1)) outs = exe.run(prog, feed={'x': x_data, 'y': y_data}, fetch_list=[avg_loss]) - print outs[0] + print(outs[0]) if outs[0] < 1.0: return self.assertFalse(True) diff --git a/python/paddle/fluid/tests/unittests/benchmark.py b/python/paddle/fluid/tests/unittests/benchmark.py index e891ee932f1440001eb25b222f1f4613e97dfcb1..b98a92dcbe5626c6cca93b3f5894302399793bf9 100644 --- a/python/paddle/fluid/tests/unittests/benchmark.py +++ b/python/paddle/fluid/tests/unittests/benchmark.py @@ -16,6 +16,7 @@ import numpy as np import unittest import time import itertools +import six import paddle.fluid as fluid import paddle.fluid.core as core @@ -40,8 +41,8 @@ class BenchmarkSuite(OpTest): expect_t = np.array(item_cpu_out) actual = item_gpu_out actual_t = np.array(item_gpu_out) - var_name = variable if isinstance(variable, - basestring) else variable.name + var_name = variable if isinstance( + variable, six.string_types) else variable.name self.assertTrue( np.allclose( actual_t, expect_t, atol=atol), @@ -53,7 +54,7 @@ class BenchmarkSuite(OpTest): def _get_input_names(self): inputs = [] - for name, value in self.inputs.iteritems(): + for name, value in list(self.inputs.items()): if isinstance(value, list): inputs.extend([sub_name for sub_name, _ in value]) inputs.append(name) @@ -61,7 +62,7 @@ class BenchmarkSuite(OpTest): def _get_output_names(self): outputs = [] - for var_name, var in self.outputs.iteritems(): + for var_name, var in list(self.outputs.items()): if isinstance(var, list): for sub_var_name, sub_var in var: outputs.append(sub_var_name) diff --git a/python/paddle/fluid/tests/unittests/dist_se_resnext.py b/python/paddle/fluid/tests/unittests/dist_se_resnext.py index bf7816b2466edd7db836c738da90f5f97b631843..62ca67ae4c041c0fe6c0836c79ebe6c23248e43e 100644 --- a/python/paddle/fluid/tests/unittests/dist_se_resnext.py +++ b/python/paddle/fluid/tests/unittests/dist_se_resnext.py @@ -14,6 +14,7 @@ import numpy as np import argparse +import six import time import math @@ -174,6 +175,9 @@ class SE_ResNeXt(): padding=(filter_size - 1) / 2, groups=groups, act=None, + # avoid pserver CPU init differs from GPU + param_attr=fluid.ParamAttr( + initializer=fluid.initializer.Constant()), bias_attr=False) return fluid.layers.batch_norm(input=conv, act=act) @@ -194,10 +198,8 @@ class SE_ResNeXt(): def get_model(batch_size): # Input data - image = fluid.layers.fill_constant( - shape=[batch_size, 3, 224, 224], dtype='float32', value=0.0) - label = fluid.layers.fill_constant( - shape=[batch_size, 1], dtype='int64', value=0.0) + image = fluid.layers.data(name="data", shape=[3, 224, 224], dtype='float32') + label = fluid.layers.data(name="int64", shape=[1], dtype='int64') # Train program model = SE_ResNeXt(layers=50) @@ -222,8 +224,10 @@ def get_model(batch_size): lr = [base_lr * (0.1**i) for i in range(len(bd) + 1)] optimizer = fluid.optimizer.Momentum( - learning_rate=fluid.layers.piecewise_decay( - boundaries=bd, values=lr), + # FIXME(typhoonzero): add back LR decay once ParallelExecutor fixed. + #learning_rate=fluid.layers.piecewise_decay( + # boundaries=bd, values=lr), + learning_rate=base_lr, momentum=0.9, regularization=fluid.regularizer.L2Decay(1e-4)) optimizer.minimize(avg_cost) @@ -232,7 +236,7 @@ def get_model(batch_size): train_reader = paddle.batch( paddle.dataset.flowers.train(), batch_size=batch_size) test_reader = paddle.batch( - paddle.dataset.flowers.test(), batch_size=batch_size) + paddle.dataset.flowers.test(use_xmap=False), batch_size=batch_size) return test_program, avg_cost, train_reader, test_reader, acc_top1, out @@ -256,7 +260,6 @@ class DistSeResneXt2x2: trainers) pserver_prog = t.get_pserver_program(current_endpoint) startup_prog = t.get_startup_program(current_endpoint, pserver_prog) - place = fluid.CPUPlace() exe = fluid.Executor(place) exe.run(startup_prog) @@ -297,17 +300,24 @@ class DistSeResneXt2x2: True, loss_name=avg_cost.name, exec_strategy=strategy) feed_var_list = [ - var for var in trainer_prog.global_block().vars.itervalues() + var for var in trainer_prog.global_block().vars.values() if var.is_data ] feeder = fluid.DataFeeder(feed_var_list, place) - reader_generator = train_reader() - first_loss, = exe.run(fetch_list=[avg_cost.name]) + reader_generator = test_reader() + + data = next(reader_generator) + first_loss, = exe.run(fetch_list=[avg_cost.name], + feed=feeder.feed(data)) print(first_loss) - for i in xrange(5): - loss, = exe.run(fetch_list=[avg_cost.name]) - last_loss, = exe.run(fetch_list=[avg_cost.name]) + + for i in six.moves.xrange(5): + data = next(reader_generator) + loss, = exe.run(fetch_list=[avg_cost.name], feed=feeder.feed(data)) + + data = next(reader_generator) + last_loss, = exe.run(fetch_list=[avg_cost.name], feed=feeder.feed(data)) print(last_loss) diff --git a/python/paddle/fluid/tests/unittests/op_test.py b/python/paddle/fluid/tests/unittests/op_test.py index 2ddfd47fe0c33b0e9771fe6f502b90eb77161100..b27d773f09d9a6daad5a10b65e683f4e11881de1 100644 --- a/python/paddle/fluid/tests/unittests/op_test.py +++ b/python/paddle/fluid/tests/unittests/op_test.py @@ -26,13 +26,15 @@ from paddle.fluid.op import Operator from paddle.fluid.executor import Executor from paddle.fluid.framework import Program, OpProtoHolder, Variable from testsuite import create_op, set_input, append_input_output, append_loss_ops +from functools import reduce +from six.moves import zip def randomize_probability(batch_size, class_num, dtype='float32'): prob = np.random.uniform( 0.1, 1.0, size=(batch_size, class_num)).astype(dtype) prob_sum = prob.sum(axis=1) - for i in xrange(len(prob)): + for i in range(len(prob)): prob[i] /= prob_sum[i] return prob @@ -101,7 +103,7 @@ def get_numeric_gradient(place, # we only compute gradient of one element each time. # we use a for loop to compute the gradient of every element. - for i in xrange(tensor_size): + for i in range(tensor_size): if in_place: set_input(scope, op, inputs, place) @@ -159,7 +161,7 @@ class OpTest(unittest.TestCase): assert isinstance( numpy_dict, dict), "self.inputs, self.outputs must be numpy_dict" - for var_name, var_value in numpy_dict.iteritems(): + for var_name, var_value in numpy_dict.items(): if isinstance(var_value, (np.ndarray, np.generic)): self.try_call_once(var_value.dtype) elif isinstance(var_value, (list, tuple)): @@ -223,7 +225,7 @@ class OpTest(unittest.TestCase): def _get_io_vars(self, block, numpy_inputs): inputs = {} - for name, value in numpy_inputs.iteritems(): + for name, value in numpy_inputs.items(): if isinstance(value, list): var_list = [ block.var(sub_name) for sub_name, sub_value in value @@ -266,7 +268,7 @@ class OpTest(unittest.TestCase): # if the fetch_list is customized by user, we use it directly. # if not, fill the fetch_list by the user configured outputs in test. if len(fetch_list) == 0: - for var_name, var in outputs.iteritems(): + for var_name, var in outputs.items(): if isinstance(var, list): for v in var: fetch_list.append(v) @@ -278,7 +280,7 @@ class OpTest(unittest.TestCase): fetch_list.append(str(out_name)) # fetch_list = map(block.var, fetch_list) if not isinstance(fetch_list[0], fluid.framework.Variable): - fetch_list = map(block.var, fetch_list) + fetch_list = list(map(block.var, fetch_list)) outs = executor.run(program, feed=feed_map, fetch_list=fetch_list, @@ -369,7 +371,7 @@ class OpTest(unittest.TestCase): def __assert_is_close(self, numeric_grads, analytic_grads, names, max_relative_error, msg_prefix): - for a, b, name in itertools.izip(numeric_grads, analytic_grads, names): + for a, b, name in zip(numeric_grads, analytic_grads, names): abs_a = np.abs(a) abs_a[abs_a < 1e-3] = 1 @@ -510,6 +512,6 @@ class OpTest(unittest.TestCase): use_cuda=use_cuda, loss_name=loss.name, main_program=prog) else: executor = Executor(place) - return map(np.array, - executor.run(prog, feed_dict, fetch_list, - return_numpy=False)) + return list( + map(np.array, + executor.run(prog, feed_dict, fetch_list, return_numpy=False))) diff --git a/python/paddle/fluid/tests/unittests/parallel_executor_test_base.py b/python/paddle/fluid/tests/unittests/parallel_executor_test_base.py index fcf86cc5839113b75855ce97459b2ee4881238cd..67c35e9de7e83699bf30ca946856bb907152cbdd 100644 --- a/python/paddle/fluid/tests/unittests/parallel_executor_test_base.py +++ b/python/paddle/fluid/tests/unittests/parallel_executor_test_base.py @@ -91,7 +91,7 @@ class TestParallelExecutorBase(unittest.TestCase): first_loss, = run_executor( exe=exe, feed=feed_dict, fetch_list=[loss.name]) - for i in xrange(iter): + for i in range(iter): run_executor(exe=exe, feed=feed_dict, fetch_list=[]) last_loss, = run_executor( @@ -99,8 +99,8 @@ class TestParallelExecutorBase(unittest.TestCase): end = time.time() if batch_size is not None: - print "%.4f Instance per second" % ( - (batch_size * iter + 2) / (end - begin)) + print("%.4f Instance per second" % ( + (batch_size * iter + 2) / (end - begin))) avg_last_loss_val = np.array(last_loss).mean() avg_first_loss_val = np.array(first_loss).mean() @@ -108,6 +108,6 @@ class TestParallelExecutorBase(unittest.TestCase): float(avg_first_loss_val)): sys.exit("got NaN loss, training failed.") - print first_loss, last_loss + print(first_loss, last_loss) # self.assertGreater(first_loss[0], last_loss[0]) return first_loss, last_loss diff --git a/python/paddle/fluid/tests/unittests/test_accuracy_op.py b/python/paddle/fluid/tests/unittests/test_accuracy_op.py index 212a87e529da83c40ba8852e81bdf43d4611897b..db1861fd10e371ebe631a16380af591875886769 100644 --- a/python/paddle/fluid/tests/unittests/test_accuracy_op.py +++ b/python/paddle/fluid/tests/unittests/test_accuracy_op.py @@ -26,7 +26,7 @@ class TestAccuracyOp(OpTest): label = np.random.randint(0, 2, (n, 1)) self.inputs = {'Out': infer, 'Indices': indices, "Label": label} num_correct = 0 - for rowid in xrange(n): + for rowid in range(n): for ele in indices[rowid]: if ele == label[rowid]: num_correct += 1 diff --git a/python/paddle/fluid/tests/unittests/test_activation_op.py b/python/paddle/fluid/tests/unittests/test_activation_op.py index 5ed387fb1247f1a91147cb6981f1adc7c2eeb8a2..34f9cf0620fd1351111e93e16ed5f7e765d7078b 100644 --- a/python/paddle/fluid/tests/unittests/test_activation_op.py +++ b/python/paddle/fluid/tests/unittests/test_activation_op.py @@ -313,9 +313,9 @@ class TestAbs(OpTest): self.init_dtype() x = np.random.uniform(-1, 1, [4, 4]).astype(self.dtype) - # Because we set delta = 0.005 in caculating numeric gradient, + # Because we set delta = 0.005 in calculating numeric gradient, # if x is too small, such as 0.002, x_neg will be -0.003 - # x_pos will be 0.007, so the numeric gradient is unaccurate. + # x_pos will be 0.007, so the numeric gradient is inaccurate. # we should avoid this x[np.abs(x) < 0.005] = 0.02 out = np.abs(x) diff --git a/python/paddle/fluid/tests/unittests/test_adam_op.py b/python/paddle/fluid/tests/unittests/test_adam_op.py index 3c65f3d44adcebdca92f78f7834d4878a9fa3dfe..fa4b39879c0ede569b6802502b2c71a93b163373 100644 --- a/python/paddle/fluid/tests/unittests/test_adam_op.py +++ b/python/paddle/fluid/tests/unittests/test_adam_op.py @@ -273,7 +273,7 @@ class TestSparseAdamOp(unittest.TestCase): self.setup(scope, place) op_args = dict() - for key, np_array in self.dense_inputs.iteritems(): + for key, np_array in self.dense_inputs.items(): var = scope.var(key).get_tensor() var.set(np_array, place) op_args[key] = key @@ -290,7 +290,7 @@ class TestSparseAdamOp(unittest.TestCase): adam_op = Operator("adam", **op_args) adam_op.run(scope, place) - for key, np_array in self.outputs.iteritems(): + for key, np_array in self.outputs.items(): out_var = scope.var(key).get_tensor() actual = np.array(out_var) actual = actual.reshape([actual.size]) diff --git a/python/paddle/fluid/tests/unittests/test_array_read_write_op.py b/python/paddle/fluid/tests/unittests/test_array_read_write_op.py index a49e9035a43e04fc1d1b2328d7562c053320b24b..0000fb0958a129e9e1098de1fad888c503cfbdc5 100644 --- a/python/paddle/fluid/tests/unittests/test_array_read_write_op.py +++ b/python/paddle/fluid/tests/unittests/test_array_read_write_op.py @@ -80,8 +80,9 @@ class TestArrayReadWrite(unittest.TestCase): append_backward(total_sum_scaled) - g_vars = map(default_main_program().global_block().var, - [each_x.name + "@GRAD" for each_x in x]) + g_vars = list( + map(default_main_program().global_block().var, + [each_x.name + "@GRAD" for each_x in x])) g_out = [ item.sum() for item in exe.run( diff --git a/python/paddle/fluid/tests/unittests/test_batch_norm_op.py b/python/paddle/fluid/tests/unittests/test_batch_norm_op.py index fcb2612326e74cf6417aa93f2691154c79b5e44c..f805fdc35f624bf6e9d94d66839dcb2a0143a29b 100644 --- a/python/paddle/fluid/tests/unittests/test_batch_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_batch_norm_op.py @@ -415,7 +415,7 @@ class TestBatchNormOpTraining(unittest.TestCase): self.__assert_close(scale_grad, out[6], "scale_grad") self.__assert_close(bias_grad, out[7], "bias_grad") - print "op test forward passed: ", str(place), data_layout + print("op test forward passed: ", str(place), data_layout) places = [core.CPUPlace()] diff --git a/python/paddle/fluid/tests/unittests/test_beam_search_op.py b/python/paddle/fluid/tests/unittests/test_beam_search_op.py index 167451edd8c46c006c8019678a304a38f18cb946..e8283fc9422d93af5735aaec1a165b46ac1ef78e 100644 --- a/python/paddle/fluid/tests/unittests/test_beam_search_op.py +++ b/python/paddle/fluid/tests/unittests/test_beam_search_op.py @@ -59,8 +59,7 @@ class BeamSearchOpTester(unittest.TestCase): np.allclose( np.array(selected_scores), np.array([0.5, 0.6, 0.9, 0.7])[:, np.newaxis])) - self.assertEqual(selected_ids.lod(), - [[0L, 2L, 4L], [0L, 1L, 2L, 3L, 4L]]) + self.assertEqual(selected_ids.lod(), [[0, 2, 4], [0, 1, 2, 3, 4]]) def _create_pre_ids(self): np_data = np.array([[1, 2, 3, 4]], dtype='int64') diff --git a/python/paddle/fluid/tests/unittests/test_bipartite_match_op.py b/python/paddle/fluid/tests/unittests/test_bipartite_match_op.py index d5bd726c4a82ee839703c69a933100bb056cb736..ceeca25b74d85ed2874d672e402e3186c4ce7d47 100644 --- a/python/paddle/fluid/tests/unittests/test_bipartite_match_op.py +++ b/python/paddle/fluid/tests/unittests/test_bipartite_match_op.py @@ -48,7 +48,7 @@ def bipartite_match(distance, match_indices, match_dist): def argmax_match(distance, match_indices, match_dist, threshold): r, c = distance.shape - for j in xrange(c): + for j in range(c): if match_indices[j] != -1: continue col_dist = distance[:, j] diff --git a/python/paddle/fluid/tests/unittests/test_chunk_eval_op.py b/python/paddle/fluid/tests/unittests/test_chunk_eval_op.py index 23932194f0ca97954ec9ade3fdcaebd7a32749a0..354110f1f96f6b4aad1a4866c8d1337dec3acd16 100644 --- a/python/paddle/fluid/tests/unittests/test_chunk_eval_op.py +++ b/python/paddle/fluid/tests/unittests/test_chunk_eval_op.py @@ -63,7 +63,7 @@ class TestChunkEvalOp(OpTest): # generate chunk beginnings chunk_begins = sorted( np.random.choice( - range(starts[-1]), num_chunks, replace=False)) + list(range(starts[-1])), num_chunks, replace=False)) seq_chunk_begins = [] begin_idx = 0 # divide chunks into sequences @@ -93,7 +93,7 @@ class TestChunkEvalOp(OpTest): self.num_infer_chunks + self.num_label_chunks - self.num_correct_chunks) correct_chunks = np.random.choice( - range(len(chunks)), self.num_correct_chunks, replace=False) + list(range(len(chunks))), self.num_correct_chunks, replace=False) infer_chunks = np.random.choice( [x for x in range(len(chunks)) if x not in correct_chunks], self.num_infer_chunks - self.num_correct_chunks, @@ -138,7 +138,8 @@ class TestChunkEvalOp(OpTest): infer.fill(self.num_chunk_types * self.num_tag_types) label = np.copy(infer) starts = np.random.choice( - range(1, self.batch_size), self.num_sequences - 1, + list(range(1, self.batch_size)), + self.num_sequences - 1, replace=False).tolist() starts.extend([0, self.batch_size]) starts = sorted(starts) diff --git a/python/paddle/fluid/tests/unittests/test_conditional_block.py b/python/paddle/fluid/tests/unittests/test_conditional_block.py index d9f83905e6135e22f74e749857f9b0fbe464d3f4..77869a1242e08d348bfb1031b8f5b1ab5c81d868 100644 --- a/python/paddle/fluid/tests/unittests/test_conditional_block.py +++ b/python/paddle/fluid/tests/unittests/test_conditional_block.py @@ -39,7 +39,7 @@ class ConditionalBlockTest(unittest.TestCase): x = numpy.random.random(size=(10, 1)).astype('float32') outs = exe.run(feed={'X': x}, fetch_list=[out])[0] - print outs + print(outs) loss = layers.mean(out) append_backward(loss=loss) outs = exe.run( @@ -47,7 +47,7 @@ class ConditionalBlockTest(unittest.TestCase): fetch_list=[ default_main_program().block(0).var(data.name + "@GRAD") ])[0] - print outs + print(outs) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_conv2d_mkldnn_op.py b/python/paddle/fluid/tests/unittests/test_conv2d_mkldnn_op.py index db6be21baaa54d33af9f5c44d1815e4b389eb884..d0de7ad52c8a851c16cbbbf544d479f696dee136 100644 --- a/python/paddle/fluid/tests/unittests/test_conv2d_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv2d_mkldnn_op.py @@ -20,16 +20,19 @@ from test_conv2d_op import TestConv2dOp, TestWithPad, TestWithStride class TestMKLDNN(TestConv2dOp): def init_kernel_type(self): self.use_mkldnn = True + self.data_format = "NCHW" class TestMKLDNNWithPad(TestWithPad): def init_kernel_type(self): self.use_mkldnn = True + self.data_format = "NCHW" class TestMKLDNNWithStride(TestWithStride): def init_kernel_type(self): self.use_mkldnn = True + self.data_format = "NCHW" if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_conv2d_op.py b/python/paddle/fluid/tests/unittests/test_conv2d_op.py index a478649541ba9828e55c4239090d5aee554223ac..bb1cd87d615fa341b7244e9f3e113b9fb4765ac2 100644 --- a/python/paddle/fluid/tests/unittests/test_conv2d_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv2d_op.py @@ -66,6 +66,7 @@ class TestConv2dOp(OpTest): self.op_type = "conv2d" self.use_cudnn = False self.use_mkldnn = False + self.data_format = "AnyLayout" self.dtype = np.float32 self.init_kernel_type() self.init_group() @@ -93,7 +94,8 @@ class TestConv2dOp(OpTest): 'groups': self.groups, 'dilations': self.dilations, 'use_cudnn': self.use_cudnn, - 'use_mkldnn': self.use_mkldnn + 'use_mkldnn': self.use_mkldnn, + 'data_format': self.data_format } self.outputs = {'Output': output} @@ -101,59 +103,35 @@ class TestConv2dOp(OpTest): return core.is_compiled_with_cuda() and self.use_cudnn def test_check_output(self): - if self.testcudnn(): - place = core.CUDAPlace(0) - self.check_output_with_place(place, atol=1e-5) - else: - self.check_output() + place = core.CUDAPlace(0) if self.testcudnn() else core.CPUPlace() + self.check_output_with_place(place, atol=1e-5) def test_check_grad(self): if self.dtype == np.float16: return - if self.testcudnn(): - place = core.CUDAPlace(0) - self.check_grad_with_place( - place, - set(['Input', 'Filter']), - 'Output', - max_relative_error=0.02) - else: - self.check_grad( - set(['Input', 'Filter']), 'Output', max_relative_error=0.02) + place = core.CUDAPlace(0) if self.testcudnn() else core.CPUPlace() + self.check_grad_with_place( + place, set(['Input', 'Filter']), 'Output', max_relative_error=0.02) def test_check_grad_no_filter(self): if self.dtype == np.float16: return - if self.testcudnn(): - place = core.CUDAPlace(0) - self.check_grad_with_place( - place, ['Input'], - 'Output', - max_relative_error=0.02, - no_grad_set=set(['Filter'])) - else: - self.check_grad( - ['Input'], - 'Output', - max_relative_error=0.02, - no_grad_set=set(['Filter'])) + place = core.CUDAPlace(0) if self.testcudnn() else core.CPUPlace() + self.check_grad_with_place( + place, ['Input'], + 'Output', + max_relative_error=0.02, + no_grad_set=set(['Filter'])) def test_check_grad_no_input(self): if self.dtype == np.float16: return - if self.testcudnn(): - place = core.CUDAPlace(0) - self.check_grad_with_place( - place, ['Filter'], - 'Output', - max_relative_error=0.02, - no_grad_set=set(['Input'])) - else: - self.check_grad( - ['Filter'], - 'Output', - max_relative_error=0.02, - no_grad_set=set(['Input'])) + place = core.CUDAPlace(0) if self.testcudnn() else core.CPUPlace() + self.check_grad_with_place( + place, ['Filter'], + 'Output', + max_relative_error=0.02, + no_grad_set=set(['Input'])) def init_test_case(self): self.pad = [0, 0] diff --git a/python/paddle/fluid/tests/unittests/test_conv_shift_op.py b/python/paddle/fluid/tests/unittests/test_conv_shift_op.py index 5d4d244f439a671d895f9237b793e6c6bbf2895b..9fdb7baa90d2184c3c439e76b6bb5f0668f5f9ee 100644 --- a/python/paddle/fluid/tests/unittests/test_conv_shift_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv_shift_op.py @@ -22,8 +22,8 @@ def conv_shift_forward(x, y): M = x.shape[1] N = y.shape[1] y_half_width = (N - 1) / 2 - for i in xrange(M): - for j in xrange(N): + for i in range(M): + for j in range(N): out[:, i] += x[:, (i + j + M - y_half_width) % M] * y[:, j] return out diff --git a/python/paddle/fluid/tests/unittests/test_create_op_doc_string.py b/python/paddle/fluid/tests/unittests/test_create_op_doc_string.py index 5e6f9a20a93e467980f5a4f23fbcb6118317fe44..07c89eefc32fab37ce093e91d96fbe4471ecddc6 100644 --- a/python/paddle/fluid/tests/unittests/test_create_op_doc_string.py +++ b/python/paddle/fluid/tests/unittests/test_create_op_doc_string.py @@ -18,7 +18,7 @@ import paddle.fluid.layers as layers class TestDocString(unittest.TestCase): def test_layer_doc_string(self): - print layers.dropout.__doc__ + print(layers.dropout.__doc__) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_data_balance.py b/python/paddle/fluid/tests/unittests/test_data_balance.py index aa09b0ea445adccae3f741b53850f8182f3270cc..951282e8bab5018204c0d31caa10f8f84a8f3d6c 100644 --- a/python/paddle/fluid/tests/unittests/test_data_balance.py +++ b/python/paddle/fluid/tests/unittests/test_data_balance.py @@ -21,7 +21,7 @@ import numpy as np class TestDataBalance(unittest.TestCase): def prepare_data(self): def fake_data_generator(): - for n in xrange(self.total_ins_num): + for n in range(self.total_ins_num): yield np.ones((3, 4)) * n, n # Prepare data @@ -41,7 +41,7 @@ class TestDataBalance(unittest.TestCase): def prepare_lod_data(self): def fake_data_generator(): - for n in xrange(1, self.total_ins_num + 1): + for n in range(1, self.total_ins_num + 1): d1 = (np.ones((n, 3)) * n).astype('float32') d2 = (np.array(n).reshape((1, 1))).astype('int32') yield d1, d2 @@ -58,9 +58,9 @@ class TestDataBalance(unittest.TestCase): (0, 1)) ] lod = [0] - for _ in xrange(self.batch_size): + for _ in range(self.batch_size): try: - ins = generator.next() + ins = next(generator) except StopIteration: eof = True break diff --git a/python/paddle/fluid/tests/unittests/test_default_scope_funcs.py b/python/paddle/fluid/tests/unittests/test_default_scope_funcs.py index a3bf7b544b91c70ffe3894219c118ec9887aba81..868bcca881a65dad7d0ecabb1e388818cdd0997e 100644 --- a/python/paddle/fluid/tests/unittests/test_default_scope_funcs.py +++ b/python/paddle/fluid/tests/unittests/test_default_scope_funcs.py @@ -39,7 +39,7 @@ class TestDefaultScopeFuncs(unittest.TestCase): self.assertTrue(i.is_int()) self.assertEqual(10, i.get_int()) - for _ in xrange(10): + for _ in range(10): scoped_function(__new_scope__) diff --git a/python/paddle/fluid/tests/unittests/test_detection_map_op.py b/python/paddle/fluid/tests/unittests/test_detection_map_op.py index 05d3367ad8ec2bc3df794015a7c25e943a26c68c..8b66d1b270980a18fd1bbd068917e982a450ad6f 100644 --- a/python/paddle/fluid/tests/unittests/test_detection_map_op.py +++ b/python/paddle/fluid/tests/unittests/test_detection_map_op.py @@ -176,7 +176,7 @@ class TestDetectionMAPOp(OpTest): true_pos[label].append([score, tp]) false_pos[label].append([score, fp]) - for (label, label_pos_num) in label_count.items(): + for (label, label_pos_num) in list(label_count.items()): if label_pos_num == 0 or label not in true_pos: continue label_true_pos = true_pos[label] label_false_pos = false_pos[label] diff --git a/python/paddle/fluid/tests/unittests/test_dist_base.py b/python/paddle/fluid/tests/unittests/test_dist_base.py index 1aaab6f906ef6482bc515bb3c42d82431902e1d8..58cfd4e1fd958d8d59e49c87fbbabd0182975add 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_base.py +++ b/python/paddle/fluid/tests/unittests/test_dist_base.py @@ -63,7 +63,8 @@ class TestDistBase(unittest.TestCase): "PATH": os.getenv("PATH"), "PYTHONPATH": os.getenv("PYTHONPATH"), "LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH"), - "FLAGS_fraction_of_gpu_memory_to_use": "0.15" + "FLAGS_fraction_of_gpu_memory_to_use": "0.15", + "FLAGS_cudnn_deterministic": "1" } # Run local to get a base line env_local = {"CUDA_VISIBLE_DEVICES": "0"} diff --git a/python/paddle/fluid/tests/unittests/test_dist_mnist.py b/python/paddle/fluid/tests/unittests/test_dist_mnist.py index ad2d57f7c5f127be87e963508e1dd150fdd30225..a6fcbd977f1af9d452dfd8367efef706cd566149 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_mnist.py +++ b/python/paddle/fluid/tests/unittests/test_dist_mnist.py @@ -25,6 +25,7 @@ import unittest from multiprocessing import Process import os import signal +from functools import reduce SEED = 1 DTYPE = "float32" @@ -172,12 +173,12 @@ class TestDistMnist(unittest.TestCase): exe.run(fluid.default_startup_program()) feed_var_list = [ - var for var in trainer_prog.global_block().vars.itervalues() + var for var in trainer_prog.global_block().vars.values() if var.is_data ] feeder = fluid.DataFeeder(feed_var_list, place) - for pass_id in xrange(10): + for pass_id in range(10): for batch_id, data in enumerate(train_reader()): exe.run(trainer_prog, feed=feeder.feed(data)) diff --git a/python/paddle/fluid/tests/unittests/test_dist_se_resnext.py b/python/paddle/fluid/tests/unittests/test_dist_se_resnext.py index 04671d079731ce414561b0ede6bc2b195b07d82a..f3a5fd6985bab1d04f6e1484534367548f383dfb 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_se_resnext.py +++ b/python/paddle/fluid/tests/unittests/test_dist_se_resnext.py @@ -17,8 +17,7 @@ from test_dist_base import TestDistBase class TestDistSeResneXt2x2(TestDistBase): def test_se_resnext(self): - # TODO(paddle-dev): Is the delta too large? - self.check_with_place("dist_se_resnext.py", delta=0.2) + self.check_with_place("dist_se_resnext.py") if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_dist_transpiler.py b/python/paddle/fluid/tests/unittests/test_dist_transpiler.py index 9dbef0693bb129186dfc50f6efdd0896deedda81..3bc5e6ada5dabfcad38278db6e14d0b115c5050d 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_transpiler.py +++ b/python/paddle/fluid/tests/unittests/test_dist_transpiler.py @@ -51,17 +51,17 @@ class TranspilerTest(unittest.TestCase): self.origin_prog = main.clone() return main - def get_trainer(self, config=None): - t = self._transpiler_instance(config) + def get_trainer(self, config=None, sync_mode=True): + t = self._transpiler_instance(config, sync_mode) return t.get_trainer_program() - def get_pserver(self, ep, config=None): - t = self._transpiler_instance(config) + def get_pserver(self, ep, config=None, sync_mode=True): + t = self._transpiler_instance(config, sync_mode) pserver = t.get_pserver_program(ep) startup = t.get_startup_program(ep, pserver) return pserver, startup - def _transpiler_instance(self, config=None): + def _transpiler_instance(self, config=None, sync_mode=True): if not self.transpiler: main = self.get_main_program() self.transpiler = fluid.DistributeTranspiler(config=config) @@ -69,13 +69,23 @@ class TranspilerTest(unittest.TestCase): self.trainer_id, program=main, pservers=self.pserver_eps, - trainers=self.trainers) + trainers=self.trainers, + sync_mode=sync_mode) return self.transpiler + def transpiler_test_impl(self): + pass -class TestBasicModel(TranspilerTest): def test_transpiler(self): + main = fluid.Program() + startup = fluid.Program() + with fluid.program_guard(main, startup): + self.transpiler_test_impl() + + +class TestBasicModel(TranspilerTest): + def transpiler_test_impl(self): pserver, startup = self.get_pserver(self.pserver1_ep) pserver2, startup2 = self.get_pserver(self.pserver2_ep) @@ -123,7 +133,7 @@ class TestBasicModel(TranspilerTest): class TestBasicModelWithLargeBlockSize(TranspilerTest): - def test_transpiler(self): + def transpiler_test_impl(self): config = fluid.DistributeTranspilerConfig() config.min_block_size = 1048576 @@ -148,10 +158,10 @@ class TestBasicModelWithLargeBlockSize(TranspilerTest): ["sum", "scale", "sgd"]) # confirm startup program self.assertEqual([op.type for op in startup.global_block().ops], - ["fill_constant", "fill_constant", "fill_constant"]) + ["fill_constant", "fill_constant"]) # the variable #fc_w will be split into two blocks fc_w_var = startup2.global_block().var("fc_w") - self.assertEqual(fc_w_var.shape, (1000L, 1000L)) + self.assertEqual(fc_w_var.shape, (1000, 1000)) # all parameters should be optimized on pserver pserver_params = [] @@ -177,16 +187,16 @@ class TestNoSliceVar(TranspilerTest): def setUp(self): super(TestNoSliceVar, self).setUp() - def test_transpiler(self): + def transpiler_test_impl(self): config = fluid.DistributeTranspilerConfig() config.slice_var_up = False _, startup = self.get_pserver(self.pserver1_ep, config) _, startup2 = self.get_pserver(self.pserver2_ep, config) - if startup.global_block().vars.has_key("fc_w"): + if "fc_w" in startup.global_block().vars: fc_w_var = startup.global_block().vars["fc_w"] - elif startup2.global_block().vars.has_key("fc_w"): + elif "fc_w" in startup2.global_block().vars: fc_w_var = startup2.global_block().vars["fc_w"] self.assertEqual(fc_w_var.shape, (1000, 1000)) @@ -212,7 +222,7 @@ class TestLRDecay(TranspilerTest): sgd_optimizer.minimize(avg_cost) return - def test_transpiler(self): + def transpiler_test_impl(self): pserver, startup = self.get_pserver(self.pserver1_ep) trainer = self.get_trainer() @@ -242,7 +252,7 @@ class TestLRDecayConditional(TranspilerTest): sgd_optimizer.minimize(avg_cost) return - def test_transpiler(self): + def transpiler_test_impl(self): pserver, startup = self.get_pserver(self.pserver1_ep) trainer = self.get_trainer() @@ -291,7 +301,7 @@ class TestL2Decay(TranspilerTest): sgd_optimizer.minimize(avg_cost) return - def test_transpiler(self): + def transpiler_test_impl(self): pserver, startup = self.get_pserver(self.pserver1_ep) trainer = self.get_trainer() @@ -326,7 +336,7 @@ class TestL2DecayWithPiecewise(TranspilerTest): sgd_optimizer.minimize(avg_cost) return - def test_transpiler(self): + def transpiler_test_impl(self): pserver, startup = self.get_pserver(self.pserver1_ep) trainer = self.get_trainer() @@ -350,5 +360,181 @@ class TestL2DecayWithPiecewise(TranspilerTest): ["sum", "scale", "scale", "elementwise_add", "momentum"]) +class TestDistLookupTableBase(TranspilerTest): + def network_with_table(self, is_sparse, is_distributed): + def emb_pool(ids): + table_size = 1000 + emb_size = 64 + emb = fluid.layers.embedding( + input=ids, + size=[table_size, emb_size], + dtype='float32', + param_attr='shared_w', # share parameter + is_sparse=is_sparse, + is_distributed=is_distributed) + pool = fluid.layers.sequence_pool(input=emb, pool_type='average') + return pool + + title_ids = fluid.layers.data( + name='title_ids', shape=[1], dtype='int64', lod_level=1) + brand_ids = fluid.layers.data( + name='brand_ids', shape=[1], dtype='int64', lod_level=1) + title_emb = emb_pool(title_ids) + brand_emb = emb_pool(brand_ids) + fc0 = fluid.layers.concat(input=[title_emb, brand_emb], axis=1) + predict = fluid.layers.fc(input=fc0, + size=2, + act=None, + param_attr=fluid.ParamAttr(name='fc_w'), + bias_attr=fluid.ParamAttr(name='fc_b')) + + label = fluid.layers.data(name='label', shape=[1], dtype='int64') + cost = fluid.layers.cross_entropy(input=predict, label=label) + avg_cost = fluid.layers.mean(cost) + optimizer = fluid.optimizer.Adam(learning_rate=0.003) + optimizer.minimize(avg_cost) + + +class TestLocalLookupTable(TestDistLookupTableBase): + def net_conf(self): + self.network_with_table(is_sparse=True, is_distributed=False) + + def transpiler_test_impl(self): + pserver1, startup1 = self.get_pserver(self.pserver1_ep) + + self.assertEqual(len(pserver1.blocks), 3) + # 0 listen_and_serv + # 1 optimize for fc_w or fc_b adam + self.assertEqual([op.type for op in pserver1.blocks[1].ops], + ["sum", "scale", "adam", "scale", "scale"]) + # 2 optimize for table adam + # NOTE: if param is not selected rows, the grad will scaled to grad / trainer_num + self.assertEqual([op.type for op in pserver1.blocks[2].ops], + ["sum", "adam", "scale", "scale"]) + + trainer = self.get_trainer() + self.assertEqual(len(trainer.blocks), 1) + ops = [ + 'lookup_table', 'sequence_pool', 'lookup_table', 'sequence_pool', + 'concat', 'mul', 'elementwise_add', 'cross_entropy', 'mean', + 'fill_constant', 'mean_grad', 'cross_entropy_grad', + 'elementwise_add_grad', 'send', 'mul_grad', 'send', 'concat_grad', + 'sequence_pool_grad', 'lookup_table_grad', 'sequence_pool_grad', + 'lookup_table_grad', 'sum', 'split_selected_rows', 'send', + 'send_barrier', 'recv', 'recv', 'recv', 'fetch_barrier', 'concat' + ] + self.assertEqual([op.type for op in trainer.blocks[0].ops], ops) + + +class TestDistLookupTable(TestDistLookupTableBase): + def net_conf(self): + self.network_with_table(is_sparse=True, is_distributed=True) + + def transpiler_test_impl(self): + pserver1, startup1 = self.get_pserver(self.pserver1_ep) + + self.assertEqual(len(pserver1.blocks), 6) + # 0 listen_and_serv + # 1 optimize for fc_w or fc_b adam + self.assertEqual([op.type for op in pserver1.blocks[1].ops], + ["sum", "scale", "adam", "scale", "scale"]) + # 2 optimize for table sgd + self.assertEqual([op.type for op in pserver1.blocks[2].ops], + ["sum", "sgd"]) + # 3 prefetch -> lookup_sparse_table for data0 + self.assertEqual([op.type for op in pserver1.blocks[3].ops], + ["lookup_sparse_table"]) + # 4 prefetch -> lookup_sparse_table for data1 + self.assertEqual([op.type for op in pserver1.blocks[4].ops], + ["lookup_sparse_table"]) + # 5 save table + self.assertEqual([op.type for op in pserver1.blocks[5].ops], ["save"]) + + trainer = self.get_trainer() + self.assertEqual(len(trainer.blocks), 1) + ops = [ + 'split_ids', 'prefetch', 'merge_ids', 'sequence_pool', 'split_ids', + 'prefetch', 'merge_ids', 'sequence_pool', 'concat', 'mul', + 'elementwise_add', 'cross_entropy', 'mean', 'fill_constant', + 'mean_grad', 'cross_entropy_grad', 'elementwise_add_grad', 'send', + 'mul_grad', 'send', 'concat_grad', 'sequence_pool_grad', + 'lookup_table_grad', 'sequence_pool_grad', 'lookup_table_grad', + 'sum', 'split_ids', 'send', 'send_barrier', 'recv', 'recv', + 'fetch_barrier' + ] + self.assertEqual([op.type for op in trainer.blocks[0].ops], ops) + + +class TestAsyncLocalLookupTable(TestDistLookupTableBase): + def net_conf(self): + self.network_with_table(is_sparse=True, is_distributed=False) + + def transpiler_test_impl(self): + config = fluid.DistributeTranspilerConfig() + pserver1, startup1 = self.get_pserver(self.pserver1_ep, config, False) + + self.assertEqual(len(pserver1.blocks), 3) + # 0 listen_and_serv + # 1 optimize for fc_w or fc_b adam + self.assertEqual([op.type for op in pserver1.blocks[1].ops], + ["adam", "scale", "scale"]) + # 2 optimize for table adam + # NOTE: if param is not selected rows, the grad will scaled to grad / trainer_num + self.assertEqual([op.type for op in pserver1.blocks[2].ops], + ["adam", "scale", "scale"]) + + trainer = self.get_trainer(config) + self.assertEqual(len(trainer.blocks), 1) + ops = [ + 'lookup_table', 'sequence_pool', 'lookup_table', 'sequence_pool', + 'concat', 'mul', 'elementwise_add', 'cross_entropy', 'mean', + 'fill_constant', 'mean_grad', 'cross_entropy_grad', + 'elementwise_add_grad', 'send', 'mul_grad', 'send', 'concat_grad', + 'sequence_pool_grad', 'lookup_table_grad', 'sequence_pool_grad', + 'lookup_table_grad', 'sum', 'split_selected_rows', 'send', 'recv', + 'recv', 'recv', 'concat' + ] + self.assertEqual([op.type for op in trainer.blocks[0].ops], ops) + + +class TestAsyncDistLookupTable(TestDistLookupTableBase): + def net_conf(self): + self.network_with_table(is_sparse=True, is_distributed=True) + + def transpiler_test_impl(self): + config = fluid.DistributeTranspilerConfig() + + pserver1, startup1 = self.get_pserver(self.pserver1_ep, config, False) + + self.assertEqual(len(pserver1.blocks), 6) + # 0 listen_and_serv + # 1 optimize for fc_w or fc_b adam + self.assertEqual([op.type for op in pserver1.blocks[1].ops], + ["adam", "scale", "scale"]) + # 2 optimize for table sgd + self.assertEqual([op.type for op in pserver1.blocks[2].ops], ["sgd"]) + # 3 prefetch -> lookup_sparse_table for data0 + self.assertEqual([op.type for op in pserver1.blocks[3].ops], + ["lookup_sparse_table"]) + # 4 prefetch -> lookup_sparse_table for data1 + self.assertEqual([op.type for op in pserver1.blocks[4].ops], + ["lookup_sparse_table"]) + # 5 save table + self.assertEqual([op.type for op in pserver1.blocks[5].ops], ["save"]) + + trainer = self.get_trainer(config) + self.assertEqual(len(trainer.blocks), 1) + ops = [ + 'split_ids', 'prefetch', 'merge_ids', 'sequence_pool', 'split_ids', + 'prefetch', 'merge_ids', 'sequence_pool', 'concat', 'mul', + 'elementwise_add', 'cross_entropy', 'mean', 'fill_constant', + 'mean_grad', 'cross_entropy_grad', 'elementwise_add_grad', 'send', + 'mul_grad', 'send', 'concat_grad', 'sequence_pool_grad', + 'lookup_table_grad', 'sequence_pool_grad', 'lookup_table_grad', + 'sum', 'split_ids', 'send', 'recv', 'recv' + ] + self.assertEqual([op.type for op in trainer.blocks[0].ops], ops) + + if __name__ == "__main__": unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_dist_word2vec.py b/python/paddle/fluid/tests/unittests/test_dist_word2vec.py index 712fd5849d80b1915ae3b2ae5108bedee8d88a2c..4bb3998f891959f8270dc4b25821f23f1e6195e0 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_word2vec.py +++ b/python/paddle/fluid/tests/unittests/test_dist_word2vec.py @@ -183,12 +183,12 @@ class TestDistMnist(unittest.TestCase): exec_strategy=exec_strategy) feed_var_list = [ - var for var in trainer_prog.global_block().vars.itervalues() + var for var in trainer_prog.global_block().vars.values() if var.is_data ] feeder = fluid.DataFeeder(feed_var_list, place) - for pass_id in xrange(10): + for pass_id in range(10): for batch_id, data in enumerate(train_reader()): avg_loss_np = train_exe.run(feed=feeder.feed(data), fetch_list=[avg_cost.name]) diff --git a/python/paddle/fluid/tests/unittests/test_dyn_rnn.py b/python/paddle/fluid/tests/unittests/test_dyn_rnn.py index 4448de8839d7ad4ad1f70ecdc4ac94da1e619adb..fdc6adc93bc2488d4faffed61fde5d54bbbbfd57 100644 --- a/python/paddle/fluid/tests/unittests/test_dyn_rnn.py +++ b/python/paddle/fluid/tests/unittests/test_dyn_rnn.py @@ -135,7 +135,7 @@ class TestDynRNN(unittest.TestCase): loss_0 = exe.run(main_program, feed=feeder.feed(data), fetch_list=[loss])[0] - for _ in xrange(100): + for _ in range(100): val = exe.run(main_program, feed=feeder.feed(data), fetch_list=[loss])[0] diff --git a/python/paddle/fluid/tests/unittests/test_dynrnn_gradient_check.py b/python/paddle/fluid/tests/unittests/test_dynrnn_gradient_check.py index 0f289af284773caf8515f9cbdd38e0d4481e4e44..7756885166c88eadb77c2c6d56aab767015abc51 100644 --- a/python/paddle/fluid/tests/unittests/test_dynrnn_gradient_check.py +++ b/python/paddle/fluid/tests/unittests/test_dynrnn_gradient_check.py @@ -61,13 +61,13 @@ class BaseRNN(object): self.num_seq = num_seq self.inputs = collections.defaultdict(list) - for _ in xrange(num_seq): + for _ in range(num_seq): seq_len = random.randint(1, max_seq_len - 1) for iname in ins: ishape = ins[iname].get('shape', None) idtype = ins[iname].get('dtype', 'float32') lst = [] - for _ in xrange(seq_len): + for _ in range(seq_len): lst.append(numpy.random.random(size=ishape).astype(idtype)) self.inputs[iname].append(lst) @@ -96,16 +96,16 @@ class BaseRNN(object): for out in self.outputs: retv[out] = [] - for seq_id in xrange(self.num_seq): + for seq_id in range(self.num_seq): for mname in self.mems: self.mems[mname].reset() for out in self.outputs: self.outputs[out].next_sequence() - iname0 = self.inputs.keys()[0] + iname0 = list(self.inputs.keys())[0] seq_len = len(self.inputs[iname0][seq_id]) - for step_id in xrange(seq_len): + for step_id in range(seq_len): xargs = dict() for iname in self.inputs: @@ -138,7 +138,7 @@ class BaseRNN(object): for iname in self.inputs: lod = [] np_flatten = [] - for seq_id in xrange(len(self.inputs[iname])): + for seq_id in range(len(self.inputs[iname])): seq_len = len(self.inputs[iname][seq_id]) lod.append(seq_len) np_flatten.extend(self.inputs[iname][seq_id]) @@ -159,8 +159,8 @@ class BaseRNN(object): " which is not matrix") g = numpy.zeros(shape=p.shape, dtype=p.dtype) - for i in xrange(p.shape[0]): - for j in xrange(p.shape[1]): + for i in range(p.shape[0]): + for j in range(p.shape[1]): o = p[i][j] p[i][j] += delta pos = self._exe_mean_out_() @@ -184,7 +184,7 @@ class BaseRNN(object): if len(item.shape) != 1: raise ValueError("Not support") - for i in xrange(len(item)): + for i in range(len(item)): o = item[i] item[i] += delta pos = self._exe_mean_out_() @@ -198,14 +198,14 @@ class BaseRNN(object): if not return_one_tensor: return grad - for i in xrange(len(grad)): + for i in range(len(grad)): grad[i] = numpy.concatenate(grad[i]) grad = numpy.concatenate(grad) return grad def _exe_mean_out_(self): outs = self.exe() - return numpy.array([o.mean() for o in outs.itervalues()]).mean() + return numpy.array([o.mean() for o in outs.values()]).mean() class SeedFixedTestCase(unittest.TestCase): @@ -274,13 +274,14 @@ class TestSimpleMul(SeedFixedTestCase): cpu = fluid.CPUPlace() exe = fluid.Executor(cpu) - out, w_g, i_g = map(numpy.array, - exe.run(feed=py_rnn.to_feed(cpu), - fetch_list=[ - out, self.PARAM_NAME + "@GRAD", - self.DATA_NAME + "@GRAD" - ], - return_numpy=False)) + out, w_g, i_g = list( + map(numpy.array, + exe.run(feed=py_rnn.to_feed(cpu), + fetch_list=[ + out, self.PARAM_NAME + "@GRAD", self.DATA_NAME + + "@GRAD" + ], + return_numpy=False))) out_by_python = py_rnn.exe()[self.OUT_NAME] self.assertTrue(numpy.allclose(out, out_by_python)) w_g_num = py_rnn.get_numeric_gradient_of_param(self.PARAM_NAME) @@ -351,14 +352,15 @@ class TestSimpleMulWithMemory(SeedFixedTestCase): cpu = fluid.CPUPlace() exe = fluid.Executor(cpu) feed = py_rnn.to_feed(cpu) - last_np, w_g, i_g = map(numpy.array, - exe.run(feed=feed, - fetch_list=[ - last, self.PARAM_NAME + "@GRAD", - self.DATA_NAME + "@GRAD" - ], - return_numpy=False)) - last_by_py, = py_rnn.exe().values() + last_np, w_g, i_g = list( + map(numpy.array, + exe.run(feed=feed, + fetch_list=[ + last, self.PARAM_NAME + "@GRAD", self.DATA_NAME + + "@GRAD" + ], + return_numpy=False))) + last_by_py, = list(py_rnn.exe().values()) w_g_num = py_rnn.get_numeric_gradient_of_param(self.PARAM_NAME) self.assertTrue(numpy.allclose(last_np, last_by_py)) diff --git a/python/paddle/fluid/tests/unittests/test_dynrnn_static_input.py b/python/paddle/fluid/tests/unittests/test_dynrnn_static_input.py index 31af1245720405ee067a0acf3575e3ae86372c13..d182889a970fb178dec4976aebbd79d05dc3e91e 100644 --- a/python/paddle/fluid/tests/unittests/test_dynrnn_static_input.py +++ b/python/paddle/fluid/tests/unittests/test_dynrnn_static_input.py @@ -67,7 +67,7 @@ class TestDyRnnStaticInput(unittest.TestCase): def _lodtensor_to_ndarray(self, lod_tensor): dims = lod_tensor.shape() ndarray = np.zeros(shape=dims).astype('float32') - for i in xrange(np.product(dims)): + for i in range(np.product(dims)): ndarray.ravel()[i] = lod_tensor._get_float_element(i) return ndarray, lod_tensor.recursive_sequence_lengths() @@ -114,7 +114,7 @@ class TestDyRnnStaticInput(unittest.TestCase): shape=[1], dtype='int64', value=0) step_idx.stop_gradient = True - for i in xrange(self._max_sequence_len): + for i in range(self._max_sequence_len): step_out = fluid.layers.array_read(static_input_out_array, step_idx) step_out.stop_gradient = True @@ -140,27 +140,27 @@ class TestDyRnnStaticInput(unittest.TestCase): static_lod = self.static_input_tensor.recursive_sequence_lengths() static_sliced = [] cur_offset = 0 - for i in xrange(len(static_lod[0])): + for i in range(len(static_lod[0])): static_sliced.append(self.static_input_data[cur_offset:( cur_offset + static_lod[0][i])]) cur_offset += static_lod[0][i] static_seq_len = static_lod[0] static_reordered = [] - for i in xrange(len(x_sorted_indices)): + for i in range(len(x_sorted_indices)): static_reordered.extend(static_sliced[x_sorted_indices[i]].tolist()) static_seq_len_reordered = [ static_seq_len[x_sorted_indices[i]] - for i in xrange(len(x_sorted_indices)) + for i in range(len(x_sorted_indices)) ] static_step_outs = [] static_step_lods = [] - for i in xrange(self._max_sequence_len): + for i in range(self._max_sequence_len): end = len(x_seq_len) - bisect.bisect_left(x_seq_len_sorted, i + 1) lod = [] total_len = 0 - for i in xrange(end): + for i in range(end): lod.append(static_seq_len_reordered[i]) total_len += lod[-1] static_step_lods.append([lod]) @@ -174,7 +174,7 @@ class TestDyRnnStaticInput(unittest.TestCase): static_step_outs = self.build_graph(only_forward=True) self.exe.run(framework.default_startup_program()) expected_outs, expected_lods = self.get_expected_static_step_outs() - for i in xrange(self._max_sequence_len): + for i in range(self._max_sequence_len): step_out, lod = self.fetch_value(static_step_outs[i]) self.assertTrue(np.allclose(step_out, expected_outs[i])) self.assertTrue(np.allclose(lod, expected_lods[i])) @@ -189,7 +189,7 @@ class TestDyRnnStaticInput(unittest.TestCase): numeric_gradients = np.zeros(shape=static_input_shape).astype('float32') # calculate numeric gradients tensor_size = np.product(static_input_shape) - for i in xrange(tensor_size): + for i in range(tensor_size): origin = self.static_input_tensor._get_float_element(i) x_pos = origin + self._delta self.static_input_tensor._set_float_element(i, x_pos) diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_gradient_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_gradient_op.py index c6f45381af8ac64d117eb27325f25763fbf6cae7..6f350044892a4ba2a985b5bc2328ab1fc20c5504 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_gradient_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_gradient_op.py @@ -26,7 +26,7 @@ class TestElementWiseAddOp(unittest.TestCase): def test_with_place(place): out_grad = np.random.random_sample(self.x.shape).astype(np.float32) x_grad = out_grad - sum_axis = range(0, len(self.x.shape)) + sum_axis = list(range(0, len(self.x.shape))) del sum_axis[self.axis] y_grad = np.sum(out_grad, axis=tuple(sum_axis)) diff --git a/python/paddle/fluid/tests/unittests/test_fused_elemwise_activation_op.py b/python/paddle/fluid/tests/unittests/test_fused_elemwise_activation_op.py new file mode 100644 index 0000000000000000000000000000000000000000..ec0a939e9ec21952a6657ea849bb9844bb69cc8d --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_fused_elemwise_activation_op.py @@ -0,0 +1,818 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest +import numpy as np +import paddle.fluid.core as core +from op_test import OpTest + +# scale + add +# TestElementwiseAddOp +# TestFusedOperatorsOp_scalar +# TestFusedOperatorsOp_scalar2 +# TestFusedOperatorsOp_Vector +# TestFusedOperatorsOp_broadcast_0 +# TestFusedOperatorsOp_broadcast_1 +# TestFusedOperatorsOp_broadcast_2 +# TestFusedOperatorsOp_broadcast_3 +# TestFusedOperatorsOp_broadcast_4 +# TestFusedOperatorsOp_rowwise_add_0 +# TestFusedOperatorsOp_rowwise_add_1 +# TestFusedOperatorsOp_channelwise_add + + +class TestElementwiseAddOp(OpTest): + def setUp(self): + self.op_type = "fused_elemwise_activation" + self.dtype = np.float32 + self.axis = -1 + + self.init_axis() + self.init_dtype() + self.init_input() + self.init_output() + self.init_attr() + + self.inputs = { + 'X': OpTest.np_dtype_to_fluid_dtype(self.x), + 'Y': OpTest.np_dtype_to_fluid_dtype(self.y) + } + self.outputs = {'Out': self.out} + + def init_input(self): + self.x = np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype) + self.y = np.random.uniform(0.1, 1, [13, 17]).astype(self.dtype) + + def init_output(self): + self.scale = 0.1 + self.out = (self.x + self.y) * self.scale + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'scale': self.scale, + 'functor_list': ["scale", "elementwise_add"] + } + + def init_dtype(self): + pass + + def init_axis(self): + pass + + def test_check_output(self): + self.check_output() + + def test_check_grad_normal(self): + self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.005) + + def test_check_grad_ingore_x(self): + self.check_grad( + ['Y'], 'Out', max_relative_error=0.005, no_grad_set=set("X")) + + def test_check_grad_ingore_y(self): + self.check_grad( + ['X'], 'Out', max_relative_error=0.005, no_grad_set=set('Y')) + + +class TestFusedOperatorsOp_scalar(TestElementwiseAddOp): + def init_input(self): + self.x = np.random.rand(2, 3, 4).astype(self.dtype) + self.y = np.random.rand(1).astype(self.dtype) + + def init_output(self): + self.scale = 0.1 + self.out = (self.x + self.y) * self.scale + + +class TestFusedOperatorsOp_scalar2(TestElementwiseAddOp): + def init_input(self): + self.x = np.random.rand(2, 3, 4).astype(self.dtype) + self.y = np.random.rand(1, 1).astype(self.dtype) + + def init_output(self): + self.scale = 0.1 + self.out = (self.x + self.y) * self.scale + + +class TestFusedOperatorsOp_Vector(TestElementwiseAddOp): + def init_input(self): + self.x = np.random.random((32, )).astype(self.dtype) + self.y = np.random.random((32, )).astype(self.dtype) + + def init_output(self): + self.scale = 0.1 + self.out = (self.x + self.y) * self.scale + + +class TestFusedOperatorsOp_broadcast_0(TestElementwiseAddOp): + def init_input(self): + self.x = np.random.rand(2, 3, 4).astype(self.dtype) + self.y = np.random.rand(2).astype(self.dtype) + + def init_axis(self): + self.axis = 0 + + def init_output(self): + self.scale = 0.1 + self.out = (self.x + self.y.reshape(2, 1, 1)) * self.scale + + +class TestFusedOperatorsOp_broadcast_1(TestElementwiseAddOp): + def init_input(self): + self.x = np.random.rand(2, 3, 4).astype(self.dtype) + self.y = np.random.rand(3).astype(self.dtype) + + def init_axis(self): + self.axis = 1 + + def init_output(self): + self.scale = 0.1 + self.out = (self.x + self.y.reshape(1, 3, 1)) * self.scale + + +class TestFusedOperatorsOp_broadcast_2(TestElementwiseAddOp): + def init_input(self): + self.x = np.random.rand(2, 3, 4).astype(self.dtype) + self.y = np.random.rand(4).astype(self.dtype) + + def init_output(self): + self.scale = 0.1 + self.out = (self.x + self.y.reshape(1, 1, 4)) * self.scale + + +class TestFusedOperatorsOp_broadcast_3(TestElementwiseAddOp): + def init_input(self): + self.x = np.random.rand(2, 3, 4, 5).astype(self.dtype) + self.y = np.random.rand(3, 4).astype(self.dtype) + + def init_axis(self): + self.axis = 1 + + def init_output(self): + self.scale = 0.1 + self.out = (self.x + self.y.reshape(1, 3, 4, 1)) * self.scale + + +class TestFusedOperatorsOp_broadcast_4(TestElementwiseAddOp): + def init_input(self): + self.x = np.random.rand(2, 3, 4, 5).astype(self.dtype) + self.y = np.random.rand(2, 1).astype(self.dtype) + + def init_axis(self): + self.axis = 0 + + def init_output(self): + self.scale = 0.1 + self.out = (self.x + self.y.reshape(2, 1, 1, 1)) * self.scale + + +class TestFusedOperatorsOp_rowwise_add_0(TestElementwiseAddOp): + def init_input(self): + self.x = np.random.rand(2, 3, 4).astype(self.dtype) + self.y = np.random.rand(3, 4).astype(self.dtype) + + def init_axis(self): + self.axis = 1 + + def init_output(self): + self.scale = 0.1 + self.out = (self.x + self.y.reshape(1, 3, 4)) * self.scale + + +class TestFusedOperatorsOp_rowwise_add_1(TestElementwiseAddOp): + def init_input(self): + self.x = np.random.rand(2, 1).astype(self.dtype) + self.y = np.random.rand(1).astype(self.dtype) + + def init_axis(self): + self.axis = 1 + + def init_output(self): + self.scale = 0.1 + self.out = (self.x + self.y.reshape(1, 1)) * self.scale + + +class TestFusedOperatorsOp_channelwise_add(TestElementwiseAddOp): + def init_input(self): + self.x = np.random.rand(3, 20, 20).astype(self.dtype) + self.y = np.random.rand(3, 1, 1).astype(self.dtype) + + def init_axis(self): + self.axis = -1 + + def init_output(self): + self.scale = 0.1 + self.out = (self.x + self.y) * self.scale + + +# add + scale +# TestElementwiseAddOp_f_add_scale +# TestFusedOperatorsOp_scalar_f_add_scale +# TestFusedOperatorsOp_scalar2_f_add_scale +# TestFusedOperatorsOp_Vector_f_add_scale +# TestFusedOperatorsOp_broadcast_0_f_add_scale +# TestFusedOperatorsOp_broadcast_1_f_add_scale +# TestFusedOperatorsOp_broadcast_2_f_add_scale +# TestFusedOperatorsOp_broadcast_3_f_add_scale +# TestFusedOperatorsOp_broadcast_4_f_add_scale +# TestFusedOperatorsOp_rowwise_add_0_f_add_scale +# TestFusedOperatorsOp_rowwise_add_1_f_add_scale +# TestFusedOperatorsOp_channelwise_add_f_add_scale + + +class TestFusedOperatorsOp_f_add_scale(TestElementwiseAddOp): + def init_output(self): + self.scale = 0.1 + self.out = self.x + self.y * self.scale + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'scale': self.scale, + 'functor_list': ["elementwise_add", "scale"] + } + + +class TestFusedOperatorsOp_scalar_f_add_scale(TestFusedOperatorsOp_scalar): + def init_output(self): + self.scale = 0.1 + self.out = self.x + self.y * self.scale + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'scale': self.scale, + 'functor_list': ["elementwise_add", "scale"] + } + + +class TestFusedOperatorsOp_scalar2_f_add_scale(TestFusedOperatorsOp_scalar2): + def init_output(self): + self.scale = 0.1 + self.out = self.x + self.y * self.scale + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'scale': self.scale, + 'functor_list': ["elementwise_add", "scale"] + } + + +class TestFusedOperatorsOp_Vector_f_add_scale(TestFusedOperatorsOp_Vector): + def init_output(self): + self.scale = 0.1 + self.out = self.x + self.y * self.scale + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'scale': self.scale, + 'functor_list': ["elementwise_add", "scale"] + } + + +class TestFusedOperatorsOp_broadcast_0_f_add_scale( + TestFusedOperatorsOp_broadcast_0): + def init_axis(self): + self.axis = 0 + + def init_output(self): + self.scale = 0.1 + self.out = self.x + self.y.reshape(2, 1, 1) * self.scale + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'scale': self.scale, + 'functor_list': ["elementwise_add", "scale"] + } + + +class TestFusedOperatorsOp_broadcast_1_f_add_scale( + TestFusedOperatorsOp_broadcast_1): + def init_axis(self): + self.axis = 1 + + def init_output(self): + self.scale = 0.1 + self.out = self.x + self.y.reshape(1, 3, 1) * self.scale + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'scale': self.scale, + 'functor_list': ["elementwise_add", "scale"] + } + + +class TestFusedOperatorsOp_broadcast_2_f_add_scale( + TestFusedOperatorsOp_broadcast_2): + def init_output(self): + self.scale = 0.1 + self.out = self.x + self.y.reshape(1, 1, 4) * self.scale + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'scale': self.scale, + 'functor_list': ["elementwise_add", "scale"] + } + + +class TestFusedOperatorsOp_broadcast_3_f_add_scale( + TestFusedOperatorsOp_broadcast_3): + def init_axis(self): + self.axis = 1 + + def init_output(self): + self.scale = 0.1 + self.out = self.x + self.y.reshape(1, 3, 4, 1) * self.scale + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'scale': self.scale, + 'functor_list': ["elementwise_add", "scale"] + } + + +class TestFusedOperatorsOp_broadcast_4_f_add_scale( + TestFusedOperatorsOp_broadcast_4): + def init_axis(self): + self.axis = 0 + + def init_output(self): + self.scale = 0.2 + self.out = self.x + self.y.reshape(2, 1, 1, 1) * self.scale + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'scale': self.scale, + 'functor_list': ["elementwise_add", "scale"] + } + + +class TestFusedOperatorsOp_rowwise_add_0_f_add_scale( + TestFusedOperatorsOp_rowwise_add_0): + def init_axis(self): + self.axis = 1 + + def init_output(self): + self.scale = 0.1 + self.out = self.x + self.y.reshape(1, 3, 4) * self.scale + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'scale': self.scale, + 'functor_list': ["elementwise_add", "scale"] + } + + +class TestFusedOperatorsOp_rowwise_add_1_f_add_scale( + TestFusedOperatorsOp_rowwise_add_1): + def init_axis(self): + self.axis = 1 + + def init_output(self): + self.scale = 0.2 + self.out = self.x + self.y.reshape(1, 1) * self.scale + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'scale': self.scale, + 'functor_list': ["elementwise_add", "scale"] + } + + +class TestFusedOperatorsOp_channelwise_add_f_add_scale( + TestFusedOperatorsOp_channelwise_add): + def init_axis(self): + self.axis = -1 + + def init_output(self): + self.scale = 0.2 + self.out = self.x + self.y * self.scale + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'scale': self.scale, + 'functor_list': ["elementwise_add", "scale"] + } + + +# add + relu +# TestElementwiseAddOp_f_add_relu +# TestFusedOperatorsOp_scalar_f_add_relu +# TestFusedOperatorsOp_scalar2_f_add_relu +# TestFusedOperatorsOp_Vector_f_add_relu +# TestFusedOperatorsOp_broadcast_0_f_add_relu +# TestFusedOperatorsOp_broadcast_1_f_add_relu +# TestFusedOperatorsOp_broadcast_2_f_add_relu +# TestFusedOperatorsOp_broadcast_3_f_add_relu +# TestFusedOperatorsOp_broadcast_4_f_add_relu +# TestFusedOperatorsOp_rowwise_add_0_f_add_relu +# TestFusedOperatorsOp_rowwise_add_1_f_add_relu +# TestFusedOperatorsOp_channelwise_add_f_add_relu + + +class TestFusedOperatorsOp_f_add_relu(TestElementwiseAddOp): + def init_output(self): + # Copy from test_activation_op.py + # Because we set delta = 0.005 in calculating numeric gradient, + # if x is too small, such as 0.002, x_neg will be -0.003 + # x_pos will be 0.007, so the numeric gradient is inaccurate. + # we should avoid this + self.y[np.abs(self.y) < 0.005] = 0.02 + self.out = self.x + np.maximum(self.y, 0) + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'functor_list': ["elementwise_add", "relu"] + } + + +class TestFusedOperatorsOp_scalar_f_add_relu(TestFusedOperatorsOp_scalar): + def init_output(self): + self.y[np.abs(self.y) < 0.005] = 0.02 + self.out = self.x + np.maximum(self.y, 0) + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'functor_list': ["elementwise_add", "relu"] + } + + +class TestFusedOperatorsOp_scalar2_f_add_relu(TestFusedOperatorsOp_scalar2): + def init_output(self): + self.y[np.abs(self.y) < 0.005] = 0.02 + self.out = self.x + np.maximum(self.y, 0) + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'functor_list': ["elementwise_add", "relu"] + } + + +class TestFusedOperatorsOp_Vector_f_add_relu(TestFusedOperatorsOp_Vector): + def init_output(self): + self.y[np.abs(self.y) < 0.005] = 0.02 + self.out = self.x + np.maximum(self.y, 0) + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'functor_list': ["elementwise_add", "relu"] + } + + +class TestFusedOperatorsOp_broadcast_0_f_add_relu( + TestFusedOperatorsOp_broadcast_0): + def init_axis(self): + self.axis = 0 + + def init_output(self): + self.y[np.abs(self.y) < 0.005] = 0.02 + self.out = self.x + np.maximum(self.y.reshape(2, 1, 1), 0) + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'functor_list': ["elementwise_add", "relu"] + } + + +class TestFusedOperatorsOp_broadcast_1_f_add_relu( + TestFusedOperatorsOp_broadcast_1): + def init_axis(self): + self.axis = 1 + + def init_output(self): + self.y[np.abs(self.y) < 0.005] = 0.02 + self.out = self.x + np.maximum(self.y.reshape(1, 3, 1), 0) + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'functor_list': ["elementwise_add", "relu"] + } + + +class TestFusedOperatorsOp_broadcast_2_f_add_relu( + TestFusedOperatorsOp_broadcast_2): + def init_output(self): + self.y[np.abs(self.y) < 0.005] = 0.02 + self.out = self.x + np.maximum(self.y.reshape(1, 1, 4), 0) + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'functor_list': ["elementwise_add", "relu"] + } + + +class TestFusedOperatorsOp_broadcast_3_f_add_relu( + TestFusedOperatorsOp_broadcast_3): + def init_axis(self): + self.axis = 1 + + def init_output(self): + self.y[np.abs(self.y) < 0.005] = 0.02 + self.out = self.x + np.maximum(self.y.reshape(1, 3, 4, 1), 0) + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'functor_list': ["elementwise_add", "relu"] + } + + +class TestFusedOperatorsOp_broadcast_4_f_add_relu( + TestFusedOperatorsOp_broadcast_4): + def init_axis(self): + self.axis = 0 + + def init_output(self): + self.y[np.abs(self.y) < 0.005] = 0.02 + self.out = self.x + np.maximum(self.y.reshape(2, 1, 1, 1), 0) + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'functor_list': ["elementwise_add", "relu"] + } + + +class TestFusedOperatorsOp_rowwise_add_0_f_add_relu( + TestFusedOperatorsOp_rowwise_add_0): + def init_axis(self): + self.axis = 1 + + def init_output(self): + self.y[np.abs(self.y) < 0.005] = 0.02 + self.out = self.x + np.maximum(self.y.reshape(1, 3, 4), 0) + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'functor_list': ["elementwise_add", "relu"] + } + + +class TestFusedOperatorsOp_rowwise_add_1_f_add_relu( + TestFusedOperatorsOp_rowwise_add_1): + def init_axis(self): + self.axis = 1 + + def init_output(self): + self.y[np.abs(self.y) < 0.005] = 0.02 + self.out = self.x + np.maximum(self.y.reshape(1, 1), 0) + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'functor_list': ["elementwise_add", "relu"] + } + + +class TestFusedOperatorsOp_channelwise_add_f_add_relu( + TestFusedOperatorsOp_channelwise_add): + def init_axis(self): + self.axis = -1 + + def init_output(self): + self.y[np.abs(self.y) < 0.005] = 0.02 + self.out = self.x + np.maximum(self.y, 0) + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'functor_list': ["elementwise_add", "relu"] + } + + +# relu + add +# TestElementwiseAddOp_f_relu_add +# TestFusedOperatorsOp_scalar_f_relu_add +# TestFusedOperatorsOp_scalar2_f_relu_add +# TestFusedOperatorsOp_Vector_f_relu_add +# TestFusedOperatorsOp_broadcast_0_f_relu_add +# TestFusedOperatorsOp_broadcast_1_f_relu_add +# TestFusedOperatorsOp_broadcast_2_f_relu_add +# TestFusedOperatorsOp_broadcast_3_f_relu_add +# TestFusedOperatorsOp_broadcast_4_f_relu_add +# TestFusedOperatorsOp_rowwise_add_0_f_relu_add +# TestFusedOperatorsOp_rowwise_add_1_f_relu_add +# TestFusedOperatorsOp_channelwise_add_f_relu_add + + +class TestFusedOperatorsOp_f_relu_add(TestElementwiseAddOp): + def init_output(self): + # Copy from test_activation_op.py + # Because we set delta = 0.005 in calculating numeric gradient, + # if x is too small, such as 0.002, x_neg will be -0.003 + # x_pos will be 0.007, so the numeric gradient is inaccurate. + # we should avoid this + self.out = self.x + self.y + self.out = np.maximum(self.out, 0) + self.out[np.abs(self.out) < 0.005] = 0.02 + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'functor_list': ["relu", "elementwise_add"] + } + + +class TestFusedOperatorsOp_scalar_f_relu_add(TestFusedOperatorsOp_scalar): + def init_output(self): + self.out = self.x + self.y + self.out = np.maximum(self.out, 0) + self.out[np.abs(self.out) < 0.005] = 0.02 + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'functor_list': ["relu", "elementwise_add"] + } + + +class TestFusedOperatorsOp_scalar2_f_relu_add(TestFusedOperatorsOp_scalar2): + def init_output(self): + self.out = self.x + self.y + self.out = np.maximum(self.out, 0) + self.out[np.abs(self.out) < 0.005] = 0.02 + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'functor_list': ["relu", "elementwise_add"] + } + + +class TestFusedOperatorsOp_Vector_f_relu_add(TestFusedOperatorsOp_Vector): + def init_output(self): + self.out = self.x + self.y + self.out = np.maximum(self.out, 0) + self.out[np.abs(self.out) < 0.005] = 0.02 + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'functor_list': ["relu", "elementwise_add"] + } + + +class TestFusedOperatorsOp_broadcast_0_f_relu_add( + TestFusedOperatorsOp_broadcast_0): + def init_axis(self): + self.axis = 0 + + def init_output(self): + self.out = self.x + self.y.reshape(2, 1, 1) + self.out = np.maximum(self.out, 0) + self.out[np.abs(self.out) < 0.005] = 0.02 + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'functor_list': ["relu", "elementwise_add"] + } + + +class TestFusedOperatorsOp_broadcast_1_f_relu_add( + TestFusedOperatorsOp_broadcast_1): + def init_axis(self): + self.axis = 1 + + def init_output(self): + self.out = self.x + self.y.reshape(1, 3, 1) + self.out = np.maximum(self.out, 0) + self.out[np.abs(self.out) < 0.005] = 0.02 + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'functor_list': ["relu", "elementwise_add"] + } + + +class TestFusedOperatorsOp_broadcast_2_f_relu_add( + TestFusedOperatorsOp_broadcast_2): + def init_output(self): + self.out = self.x + self.y.reshape(1, 1, 4) + self.out = np.maximum(self.out, 0) + self.out[np.abs(self.out) < 0.005] = 0.02 + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'functor_list': ["relu", "elementwise_add"] + } + + +class TestFusedOperatorsOp_broadcast_3_f_relu_add( + TestFusedOperatorsOp_broadcast_3): + def init_axis(self): + self.axis = 1 + + def init_output(self): + self.out = self.x + self.y.reshape(1, 3, 4, 1) + self.out = np.maximum(self.out, 0) + self.out[np.abs(self.out) < 0.005] = 0.02 + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'functor_list': ["relu", "elementwise_add"] + } + + +class TestFusedOperatorsOp_broadcast_4_f_relu_add( + TestFusedOperatorsOp_broadcast_4): + def init_axis(self): + self.axis = 0 + + def init_output(self): + self.out = self.x + self.y.reshape(2, 1, 1, 1) + self.out = np.maximum(self.out, 0) + self.out[np.abs(self.out) < 0.005] = 0.02 + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'functor_list': ["relu", "elementwise_add"] + } + + +class TestFusedOperatorsOp_rowwise_add_0_f_relu_add( + TestFusedOperatorsOp_rowwise_add_0): + def init_axis(self): + self.axis = 1 + + def init_output(self): + self.out = self.x + self.y.reshape(1, 3, 4) + self.out = np.maximum(self.out, 0) + self.out[np.abs(self.out) < 0.005] = 0.02 + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'functor_list': ["relu", "elementwise_add"] + } + + +class TestFusedOperatorsOp_rowwise_add_1_f_relu_add( + TestFusedOperatorsOp_rowwise_add_1): + def init_axis(self): + self.axis = 1 + + def init_output(self): + self.out = self.x + self.y.reshape(1, 1) + self.out = np.maximum(self.out, 0) + self.out[np.abs(self.out) < 0.005] = 0.02 + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'functor_list': ["relu", "elementwise_add"] + } + + +class TestFusedOperatorsOp_channelwise_add_f_relu_add( + TestFusedOperatorsOp_channelwise_add): + def init_axis(self): + self.axis = -1 + + def init_output(self): + self.out = self.x + self.y + self.out = np.maximum(self.out, 0) + self.out[np.abs(self.out) < 0.005] = 0.02 + + def init_attr(self): + self.attrs = { + 'axis': self.axis, + 'functor_list': ["relu", "elementwise_add"] + } + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_gru_op.py b/python/paddle/fluid/tests/unittests/test_gru_op.py index 8fbf1560859aa295fc40b36129d0f0d07d55dd9f..86a2c674d01f45b2b141572c8191d2fba7fa312f 100644 --- a/python/paddle/fluid/tests/unittests/test_gru_op.py +++ b/python/paddle/fluid/tests/unittests/test_gru_op.py @@ -38,7 +38,7 @@ class TestGRUOp(OpTest): for i in range(len(seq_lens)): seq_starts.append(seq_starts[-1] + seq_lens[i]) sorted_seqs = sorted( - range(len(seq_lens)), lambda x, y: seq_lens[y] - seq_lens[x]) + list(range(len(seq_lens))), lambda x, y: seq_lens[y] - seq_lens[x]) num_batch = seq_lens[sorted_seqs[0]] for batch_idx in range(num_batch): idx_in_seq = [] @@ -74,15 +74,16 @@ class TestGRUOp(OpTest): def gru(self): input, lod = self.inputs['Input'] w = self.inputs['Weight'] - b = self.inputs['Bias'] if self.inputs.has_key('Bias') else np.zeros( + b = self.inputs['Bias'] if 'Bias' in self.inputs else np.zeros( (1, self.frame_size * 3)) batch_gate = self.outputs['BatchGate'] batch_reset_hidden_prev = self.outputs['BatchResetHiddenPrev'] batch_hidden = self.outputs['BatchHidden'] hidden = self.outputs['Hidden'] idx_in_seq_list = self.idx_in_seq_list - h_p = self.inputs['H0'][self.sorted_seqs] if self.inputs.has_key( - 'H0') else np.zeros((len(idx_in_seq_list[0]), self.frame_size)) + h_p = self.inputs['H0'][ + self.sorted_seqs] if 'H0' in self.inputs else np.zeros( + (len(idx_in_seq_list[0]), self.frame_size)) num_batch = len(idx_in_seq_list) end_idx = 0 for batch_idx in range(num_batch): diff --git a/python/paddle/fluid/tests/unittests/test_gru_unit_op.py b/python/paddle/fluid/tests/unittests/test_gru_unit_op.py index c56b1eefd3a3dfe1478bd0526fa32077edcac9ba..87a9eba4d97459082cdf1499efeddf24ed51e1b1 100644 --- a/python/paddle/fluid/tests/unittests/test_gru_unit_op.py +++ b/python/paddle/fluid/tests/unittests/test_gru_unit_op.py @@ -76,7 +76,7 @@ class TestGRUUnitOp(OpTest): x = self.inputs['Input'] h_p = self.inputs['HiddenPrev'] w = self.inputs['Weight'] - b = self.inputs['Bias'] if self.inputs.has_key('Bias') else np.zeros( + b = self.inputs['Bias'] if 'Bias' in self.inputs else np.zeros( (1, frame_size * 3)) g = x + np.tile(b, (batch_size, 1)) w_u_r = w.flatten()[:frame_size * frame_size * 2].reshape( diff --git a/python/paddle/fluid/tests/unittests/test_image_classification_layer.py b/python/paddle/fluid/tests/unittests/test_image_classification_layer.py index 6ecfa9ea213fe0cf57e18fa83bbb85c223727d71..23b1ed957ad15bb631cd5160eb48328c76302987 100644 --- a/python/paddle/fluid/tests/unittests/test_image_classification_layer.py +++ b/python/paddle/fluid/tests/unittests/test_image_classification_layer.py @@ -43,7 +43,7 @@ class TestLayer(unittest.TestCase): hidden2 = fluid.layers.fc(input=hidden1, size=128, act='relu') fluid.layers.batch_norm(input=hidden2) - print str(main_program) + print(str(main_program)) def test_dropout_layer(self): main_program = Program() @@ -53,7 +53,7 @@ class TestLayer(unittest.TestCase): name='pixel', shape=[3, 48, 48], dtype='float32') fluid.layers.dropout(x=images, dropout_prob=0.5) - print str(main_program) + print(str(main_program)) def test_img_conv_group(self): main_program = Program() @@ -65,7 +65,7 @@ class TestLayer(unittest.TestCase): conv1 = conv_block(images, 64, 2, [0.3, 0]) conv_block(conv1, 256, 3, [0.4, 0.4, 0]) - print str(main_program) + print(str(main_program)) def test_elementwise_add_with_act(self): main_program = Program() diff --git a/python/paddle/fluid/tests/unittests/test_inference_model_io.py b/python/paddle/fluid/tests/unittests/test_inference_model_io.py index 51460cbb1370f6794e13d18fe099865b4713691f..4cd203155f446df07d2fe6c1d56e0d20f1113679 100644 --- a/python/paddle/fluid/tests/unittests/test_inference_model_io.py +++ b/python/paddle/fluid/tests/unittests/test_inference_model_io.py @@ -48,7 +48,7 @@ class TestBook(unittest.TestCase): exe.run(init_program, feed={}, fetch_list=[]) - for i in xrange(100): + for i in range(100): tensor_x = np.array( [[1, 1], [1, 2], [3, 4], [5, 2]]).astype("float32") tensor_y = np.array([[-2], [-3], [-7], [-7]]).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/test_initializer.py b/python/paddle/fluid/tests/unittests/test_initializer.py index 15a72cb605911dfe957fb927763174521a30a085..b215e379864e919af03591ab2566c08dddbb5743 100644 --- a/python/paddle/fluid/tests/unittests/test_initializer.py +++ b/python/paddle/fluid/tests/unittests/test_initializer.py @@ -27,12 +27,13 @@ class TestConstantInitializer(unittest.TestCase): """ program = framework.Program() block = program.global_block() - block.create_parameter( - dtype="float32", - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.ConstantInitializer()) + for _ in range(2): + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.ConstantInitializer()) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'fill_constant') @@ -43,12 +44,13 @@ class TestConstantInitializer(unittest.TestCase): """ program = framework.Program() block = program.global_block() - block.create_parameter( - dtype="float32", - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.ConstantInitializer(2.3)) + for _ in range(2): + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.ConstantInitializer(2.3)) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'fill_constant') @@ -61,12 +63,13 @@ class TestUniformInitializer(unittest.TestCase): """ program = framework.Program() block = program.global_block() - block.create_parameter( - dtype="float32", - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.UniformInitializer()) + for _ in range(2): + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.UniformInitializer()) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'uniform_random') @@ -80,18 +83,19 @@ class TestUniformInitializer(unittest.TestCase): program = framework.Program() program.random_seed = 123 block = program.global_block() - block.create_parameter( - dtype="float32", - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.UniformInitializer()) - block.create_parameter( - dtype="float32", - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.UniformInitializer(seed=456)) + for _ in range(2): + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param1", + initializer=initializer.UniformInitializer()) + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param2", + initializer=initializer.UniformInitializer(seed=456)) init_op = block.ops[1] self.assertEqual(init_op.attr("seed"), 123) init_op1 = block.ops[0] @@ -102,12 +106,13 @@ class TestUniformInitializer(unittest.TestCase): """ program = framework.Program() block = program.global_block() - block.create_parameter( - dtype="float32", - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.UniformInitializer(-4.2, 3.1, 123)) + for _ in range(2): + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.UniformInitializer(-4.2, 3.1, 123)) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'uniform_random') @@ -115,6 +120,25 @@ class TestUniformInitializer(unittest.TestCase): self.assertAlmostEqual(init_op.attr('max'), 3.1, delta=DELTA) self.assertEqual(init_op.attr('seed'), 123) + def test_uniform_initializer_two_op(self): + """Test uniform initializer with supplied attributes + """ + program = framework.Program() + block = program.global_block() + for i in range(2): + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.UniformInitializer(-4.2, float(i), 123)) + self.assertEqual(len(block.ops), 1) + init_op0 = block.ops[0] + self.assertEqual(init_op0.type, 'uniform_random') + self.assertAlmostEqual(init_op0.attr('min'), -4.2, delta=DELTA) + self.assertAlmostEqual(init_op0.attr('max'), 0.0, delta=DELTA) + self.assertEqual(init_op0.attr('seed'), 123) + class TestNormalInitializer(unittest.TestCase): def test_normal_initializer_default_value(self): @@ -122,12 +146,13 @@ class TestNormalInitializer(unittest.TestCase): """ program = framework.Program() block = program.global_block() - block.create_parameter( - dtype="float32", - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.NormalInitializer()) + for _ in range(2): + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.NormalInitializer()) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'gaussian_random') @@ -140,12 +165,13 @@ class TestNormalInitializer(unittest.TestCase): """ program = framework.Program() block = program.global_block() - block.create_parameter( - dtype="float32", - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.NormalInitializer(2.3, 1.9, 123)) + for _ in range(2): + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.NormalInitializer(2.3, 1.9, 123)) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'gaussian_random') @@ -161,12 +187,13 @@ class TestXavierInitializer(unittest.TestCase): """ program = framework.Program() block = program.global_block() - param = block.create_parameter( - dtype="float32", - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.XavierInitializer()) + for _ in range(2): + param = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.XavierInitializer()) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'uniform_random') @@ -181,12 +208,13 @@ class TestXavierInitializer(unittest.TestCase): """ program = framework.Program() block = program.global_block() - param = block.create_parameter( - dtype="float32", - shape=[5, 10, 15, 20], - lod_level=0, - name="param", - initializer=initializer.XavierInitializer()) + for _ in range(2): + param = block.create_parameter( + dtype="float32", + shape=[5, 10, 15, 20], + lod_level=0, + name="param", + initializer=initializer.XavierInitializer()) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'uniform_random') @@ -203,12 +231,13 @@ class TestXavierInitializer(unittest.TestCase): """ program = framework.Program() block = program.global_block() - param = block.create_parameter( - dtype="float32", - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.XavierInitializer(uniform=False)) + for _ in range(2): + param = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.XavierInitializer(uniform=False)) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'gaussian_random') @@ -223,12 +252,13 @@ class TestXavierInitializer(unittest.TestCase): """ program = framework.Program() block = program.global_block() - param = block.create_parameter( - dtype="float32", - shape=[5, 10, 15, 20], - lod_level=0, - name="param", - initializer=initializer.XavierInitializer(uniform=False)) + for _ in range(2): + param = block.create_parameter( + dtype="float32", + shape=[5, 10, 15, 20], + lod_level=0, + name="param", + initializer=initializer.XavierInitializer(uniform=False)) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'gaussian_random') @@ -244,13 +274,14 @@ class TestXavierInitializer(unittest.TestCase): """ program = framework.Program() block = program.global_block() - block.create_parameter( - dtype="float32", - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.XavierInitializer( - fan_in=12, fan_out=23, seed=134)) + for _ in range(2): + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.XavierInitializer( + fan_in=12, fan_out=23, seed=134)) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'uniform_random') @@ -267,12 +298,13 @@ class TestMSRAInitializer(unittest.TestCase): """ program = framework.Program() block = program.global_block() - param = block.create_parameter( - dtype="float32", - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.MSRAInitializer()) + for _ in range(2): + param = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.MSRAInitializer()) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'uniform_random') @@ -287,12 +319,13 @@ class TestMSRAInitializer(unittest.TestCase): """ program = framework.Program() block = program.global_block() - param = block.create_parameter( - dtype="float32", - shape=[5, 10, 15, 20], - lod_level=0, - name="param", - initializer=initializer.MSRAInitializer()) + for _ in range(2): + param = block.create_parameter( + dtype="float32", + shape=[5, 10, 15, 20], + lod_level=0, + name="param", + initializer=initializer.MSRAInitializer()) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'uniform_random') @@ -308,12 +341,13 @@ class TestMSRAInitializer(unittest.TestCase): """ program = framework.Program() block = program.global_block() - param = block.create_parameter( - dtype="float32", - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.MSRAInitializer(uniform=False)) + for _ in range(2): + param = block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.MSRAInitializer(uniform=False)) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'gaussian_random') @@ -328,12 +362,13 @@ class TestMSRAInitializer(unittest.TestCase): """ program = framework.Program() block = program.global_block() - param = block.create_parameter( - dtype="float32", - shape=[5, 10, 15, 20], - lod_level=0, - name="param", - initializer=initializer.MSRAInitializer(uniform=False)) + for _ in range(2): + param = block.create_parameter( + dtype="float32", + shape=[5, 10, 15, 20], + lod_level=0, + name="param", + initializer=initializer.MSRAInitializer(uniform=False)) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'gaussian_random') @@ -348,13 +383,14 @@ class TestMSRAInitializer(unittest.TestCase): """ program = framework.Program() block = program.global_block() - block.create_parameter( - dtype="float32", - shape=[5, 10], - lod_level=0, - name="param", - initializer=initializer.MSRAInitializer( - fan_in=12, seed=134)) + for _ in range(2): + block.create_parameter( + dtype="float32", + shape=[5, 10], + lod_level=0, + name="param", + initializer=initializer.MSRAInitializer( + fan_in=12, seed=134)) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'uniform_random') @@ -370,12 +406,13 @@ class TestMSRAInitializer(unittest.TestCase): """ program = framework.Program() block = program.global_block() - block.create_parameter( - dtype="float32", - shape=[8, 1, 3, 3], - lod_level=0, - name="param", - initializer=initializer.BilinearInitializer()) + for _ in range(2): + block.create_parameter( + dtype="float32", + shape=[8, 1, 3, 3], + lod_level=0, + name="param", + initializer=initializer.BilinearInitializer()) self.assertEqual(len(block.ops), 1) init_op = block.ops[0] self.assertEqual(init_op.type, 'assign_value') diff --git a/python/paddle/fluid/tests/unittests/test_layer_norm_op.py b/python/paddle/fluid/tests/unittests/test_layer_norm_op.py index 69365db4d104a1b69916a605534eff83e242289f..295887ccd171a3101329eb1255da146914fa9264 100644 --- a/python/paddle/fluid/tests/unittests/test_layer_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_layer_norm_op.py @@ -17,6 +17,7 @@ import numpy as np from operator import mul import paddle.fluid.core as core import paddle.fluid as fluid +from functools import reduce np.random.random(123) diff --git a/python/paddle/fluid/tests/unittests/test_layers.py b/python/paddle/fluid/tests/unittests/test_layers.py index 9e1b47643a554bc14170fc57ac05b21afdb8117a..8f2dac786d03334642714d221710a20833939c7a 100644 --- a/python/paddle/fluid/tests/unittests/test_layers.py +++ b/python/paddle/fluid/tests/unittests/test_layers.py @@ -279,7 +279,7 @@ class TestBook(unittest.TestCase): def test_nce(self): window_size = 5 words = [] - for i in xrange(window_size): + for i in range(window_size): words.append( layers.data( name='word_{0}'.format(i), shape=[1], dtype='int64')) @@ -288,7 +288,7 @@ class TestBook(unittest.TestCase): label_word = int(window_size / 2) + 1 embs = [] - for i in xrange(window_size): + for i in range(window_size): if i == label_word: continue diff --git a/python/paddle/fluid/tests/unittests/test_lod_rank_table.py b/python/paddle/fluid/tests/unittests/test_lod_rank_table.py index 16e85830ffa51ec428951570cc7a038f3d10c873..d53ead381d301e797d5a19784aed49a5d6f99319 100644 --- a/python/paddle/fluid/tests/unittests/test_lod_rank_table.py +++ b/python/paddle/fluid/tests/unittests/test_lod_rank_table.py @@ -36,7 +36,7 @@ class TestLoDRankTable(unittest.TestCase): exe.run(scope=scope, feed={'x': tensor}) var = scope.find_var(rank_table.name) table = var.get_lod_rank_table() - self.assertEqual([(0, 5), (1, 1), (2, 1)], table.items()) + self.assertEqual([(0, 5), (1, 1), (2, 1)], list(table.items())) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_lod_tensor_array.py b/python/paddle/fluid/tests/unittests/test_lod_tensor_array.py index 118c22fbb1ff6be5859ae9e4aed6218b0c77deec..0ac6d9b81df0ecbe9c6560cdb0ab0507c3c2ed18 100644 --- a/python/paddle/fluid/tests/unittests/test_lod_tensor_array.py +++ b/python/paddle/fluid/tests/unittests/test_lod_tensor_array.py @@ -24,7 +24,7 @@ class TestLoDTensorArray(unittest.TestCase): tensor_array = arr.get_lod_tensor_array() self.assertEqual(0, len(tensor_array)) cpu = core.CPUPlace() - for i in xrange(10): + for i in range(10): t = core.LoDTensor() t.set(numpy.array([i], dtype='float32'), cpu) t.set_recursive_sequence_lengths([[1]]) @@ -32,7 +32,7 @@ class TestLoDTensorArray(unittest.TestCase): self.assertEqual(10, len(tensor_array)) - for i in xrange(10): + for i in range(10): t = tensor_array[i] self.assertEqual(numpy.array(t), numpy.array([i], dtype='float32')) self.assertEqual([[1]], t.recursive_sequence_lengths()) diff --git a/python/paddle/fluid/tests/unittests/test_lod_tensor_array_ops.py b/python/paddle/fluid/tests/unittests/test_lod_tensor_array_ops.py index 5a4580116bc7009c73f1de14a265bf2cea5acf9b..9789ff4af648b41a1b53844be89249bd260de61b 100644 --- a/python/paddle/fluid/tests/unittests/test_lod_tensor_array_ops.py +++ b/python/paddle/fluid/tests/unittests/test_lod_tensor_array_ops.py @@ -35,8 +35,10 @@ class TestCPULoDTensorArrayOps(unittest.TestCase): tensor.set( numpy.arange(10).reshape(10, 1).astype('int32'), self.place()) tensor.set_recursive_sequence_lengths([[3, 6, 1]]) - expect = map(lambda x: numpy.array(x).astype('int32'), - [[3, 0, 9], [4, 1], [5, 2], [6], [7], [8]]) + expect = [ + numpy.array(x).astype('int32') + for x in [[3, 0, 9], [4, 1], [5, 2], [6], [7], [8]] + ] self.main( tensor=tensor, expect_array=expect, @@ -48,8 +50,10 @@ class TestCPULoDTensorArrayOps(unittest.TestCase): tensor.set( numpy.arange(10).reshape(10, 1).astype('int32'), self.place()) tensor.set_recursive_sequence_lengths([[3, 6, 0, 1]]) - expect = map(lambda x: numpy.array(x).astype('int32'), - [[3, 0, 9], [4, 1], [5, 2], [6], [7], [8]]) + expect = [ + numpy.array(x).astype('int32') + for x in [[3, 0, 9], [4, 1], [5, 2], [6], [7], [8]] + ] self.main( tensor=tensor, expect_array=expect, @@ -111,8 +115,8 @@ class TestCPULoDTensorArrayOps(unittest.TestCase): expect = [ numpy.array( item, dtype='int32') - for item in [[21, 0, 1, 2, 3, 4, 5, 6, 46, 47, 48, 49], range( - 22, 39) + range(7, 21), range(39, 46)] + for item in [[21, 0, 1, 2, 3, 4, 5, 6, 46, 47, 48, 49], list( + range(22, 39)) + list(range(7, 21)), list(range(39, 46))] ] lod = [[[1, 2, 1], [1, 3, 4, 4]], [[4, 3], [1, 4, 4, 8, 4, 6, 4]], [[2], [6, 1]]] diff --git a/python/paddle/fluid/tests/unittests/test_lookup_table_op.py b/python/paddle/fluid/tests/unittests/test_lookup_table_op.py index ad0d555198c36c12fd1cc39c41d39b24b40f64c3..ac25f432dffd544d4b336983ec868f2431a5b91a 100644 --- a/python/paddle/fluid/tests/unittests/test_lookup_table_op.py +++ b/python/paddle/fluid/tests/unittests/test_lookup_table_op.py @@ -56,7 +56,7 @@ class TestLookupTableOpWithPadding(TestLookupTableOp): ids = np.squeeze(self.inputs['Ids']) padding_idx = np.random.choice(ids, 1)[0] self.outputs['Out'][ids == padding_idx] = np.zeros(31) - self.attrs = {'padding_idx': long(padding_idx)} + self.attrs = {'padding_idx': int(padding_idx)} self.check_output() def test_check_grad(self): diff --git a/python/paddle/fluid/tests/unittests/test_mean_iou.py b/python/paddle/fluid/tests/unittests/test_mean_iou.py index 64d42b693bf11f3cb0153243909db4c0612bf4e7..32b4ee184787cd4cda0fd889f67a609141a3cb27 100644 --- a/python/paddle/fluid/tests/unittests/test_mean_iou.py +++ b/python/paddle/fluid/tests/unittests/test_mean_iou.py @@ -80,7 +80,7 @@ class TestMeanIOUOp(OpTest): 'InCorrects': in_corrects, 'InMeanIou': in_mean_ious } - self.attrs = {'num_classes': long(self.num_classes)} + self.attrs = {'num_classes': int(self.num_classes)} mean_iou, out_wrong, out_correct = compute_mean_iou( predictions, labels, self.num_classes, in_wrongs, in_corrects, in_mean_ious) diff --git a/python/paddle/fluid/tests/unittests/test_memory_usage.py b/python/paddle/fluid/tests/unittests/test_memory_usage.py new file mode 100644 index 0000000000000000000000000000000000000000..f9daf83652e18faab0ab31402b9f5889a0beceaf --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_memory_usage.py @@ -0,0 +1,69 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function +import paddle +import paddle.fluid as fluid +import contextlib +import unittest + + +def train_simulator(test_batch_size=10): + if test_batch_size <= 0: + raise ValueError("batch_size should be a positive integeral value, " + "but got batch_size={}".format(test_batch_size)) + + x = fluid.layers.data(name='x', shape=[13], dtype='float32') + y_predict = fluid.layers.fc(input=x, size=1, act=None) + y = fluid.layers.data(name='y', shape=[1], dtype='float32') + + cost = fluid.layers.square_error_cost(input=y_predict, label=y) + avg_cost = fluid.layers.mean(cost) + + sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.001) + sgd_optimizer.minimize(avg_cost) + + # Calculate memory usage in current network config + lower_usage, upper_usage, unit = fluid.contrib.memory_usage( + fluid.default_main_program(), batch_size=test_batch_size) + + print("memory usage is about %.3f - %.3f %s" % + (lower_usage, upper_usage, unit)) + + +class TestMemoryUsage(unittest.TestCase): + def test_with_unit_B(self): + with self.program_scope_guard(): + train_simulator() + + def test_with_unit_KB(self): + with self.program_scope_guard(): + train_simulator(test_batch_size=1000) + + def test_with_unit_MB(self): + with self.program_scope_guard(): + train_simulator(test_batch_size=100000) + + @contextlib.contextmanager + def program_scope_guard(self): + prog = fluid.Program() + startup_prog = fluid.Program() + scope = fluid.core.Scope() + with fluid.scope_guard(scope): + with fluid.program_guard(prog, startup_prog): + yield + + +if __name__ == '__main__': + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_multiclass_nms_op.py b/python/paddle/fluid/tests/unittests/test_multiclass_nms_op.py index aacd8ae45af10a2b19d2903ab121e9bb4f9de7ff..10cb78a08db0471699bcc0b7323d5346e3af64c7 100644 --- a/python/paddle/fluid/tests/unittests/test_multiclass_nms_op.py +++ b/python/paddle/fluid/tests/unittests/test_multiclass_nms_op.py @@ -112,7 +112,7 @@ def multiclass_nms(boxes, scores, background, score_threshold, nms_threshold, if keep_top_k > -1 and num_det > keep_top_k: score_index = [] - for c, indices in selected_indices.iteritems(): + for c, indices in selected_indices.items(): for idx in indices: score_index.append((scores[c][idx], c, idx)) @@ -143,7 +143,7 @@ def batched_multiclass_nms(boxes, scores, background, score_threshold, lod.append(nmsed_num) if nmsed_num == 0: continue - for c, indices in nmsed_outs.iteritems(): + for c, indices in nmsed_outs.items(): for idx in indices: xmin, ymin, xmax, ymax = boxes[n][idx][:] det_outs.append([c, scores[n][c][idx], xmin, ymin, xmax, ymax]) diff --git a/python/paddle/fluid/tests/unittests/test_nce.py b/python/paddle/fluid/tests/unittests/test_nce.py index 76ecc8ba08ba31798040080a0ae99fe515c28cec..7431a142c53a64e58872390776904ce8f781d6a9 100644 --- a/python/paddle/fluid/tests/unittests/test_nce.py +++ b/python/paddle/fluid/tests/unittests/test_nce.py @@ -66,7 +66,7 @@ class TestNCE(OpTest): self.attrs = { 'num_total_classes': num_classes, 'num_neg_samples': num_neg_samples, - 'custom_neg_classes': range(num_neg_samples) + 'custom_neg_classes': list(range(num_neg_samples)) } self.inputs = { 'Input': input, diff --git a/python/paddle/fluid/tests/unittests/test_one_hot_op.py b/python/paddle/fluid/tests/unittests/test_one_hot_op.py index d13f2b3afde10f9b4e632094fa216d8729069afa..06fccd39ac65ab62ee5618ac19d1a0535b481d06 100644 --- a/python/paddle/fluid/tests/unittests/test_one_hot_op.py +++ b/python/paddle/fluid/tests/unittests/test_one_hot_op.py @@ -28,13 +28,13 @@ class TestOneHotOp(OpTest): depth = 10 dimension = 12 x_lod = [[4, 1, 3, 3]] - x = [np.random.randint(0, depth - 1) for i in xrange(sum(x_lod[0]))] + x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int').reshape([sum(x_lod[0]), 1]) out = np.zeros(shape=(np.product(x.shape[:-1]), depth)).astype('float32') - for i in xrange(np.product(x.shape)): + for i in range(np.product(x.shape)): out[i, x[i]] = 1.0 self.inputs = {'X': (x, x_lod)} @@ -51,13 +51,13 @@ class TestOneHotOp_default_dtype(OpTest): depth = 10 dimension = 12 x_lod = [[4, 1, 3, 3]] - x = [np.random.randint(0, depth - 1) for i in xrange(sum(x_lod[0]))] + x = [np.random.randint(0, depth - 1) for i in range(sum(x_lod[0]))] x = np.array(x).astype('int').reshape([sum(x_lod[0]), 1]) out = np.zeros(shape=(np.product(x.shape[:-1]), depth)).astype('float32') - for i in xrange(np.product(x.shape)): + for i in range(np.product(x.shape)): out[i, x[i]] = 1.0 self.inputs = {'X': (x, x_lod)} @@ -76,7 +76,7 @@ class TestOneHotOp_exception(OpTest): self.dimension = 12 self.x = core.LoDTensor() x_lod = [[4, 1, 3, 3]] - data = [np.random.randint(11, 20) for i in xrange(sum(x_lod[0]))] + data = [np.random.randint(11, 20) for i in range(sum(x_lod[0]))] data = np.array(data).astype('int').reshape([sum(x_lod[0]), 1]) self.x.set(data, self.place) self.x.set_recursive_sequence_lengths(x_lod) diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py index 63fb58c6927fa387b3b19147b9dc9d24bb8e5132..d17e493c36a2ffcba632f5f85c7a1d2e5066dd1c 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py @@ -167,10 +167,10 @@ class TestCRFModel(unittest.TestCase): place=fluid.CPUPlace()) data = train_data() - for i in xrange(10): + for i in range(10): cur_batch = next(data) - print pe.run(feed=feeder.feed(cur_batch), - fetch_list=[avg_cost.name])[0] + print(pe.run(feed=feeder.feed(cur_batch), + fetch_list=[avg_cost.name])[0]) @unittest.skip(reason="CI hangs") def test_update_sparse_parameter_all_reduce(self): diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_feed.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_feed.py index 60d63364d5f403f04519363db5ad3ad136f8a975..a43f2e7c49c02ce779344da44e640cabbf27986c 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_feed.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_feed.py @@ -71,7 +71,7 @@ class TestFetchOp(unittest.TestCase): fetch_list = [] all_vars = main.global_block().vars - for k, v in all_vars.iteritems(): + for k, v in all_vars.items(): if 'tmp' not in k and k[0] is not '_' or v.persistable: fetch_list.append(k) @@ -90,7 +90,7 @@ class TestFetchOp(unittest.TestCase): iters = 3 train_inputs = [] for i in range(iters): - train_inputs.append(tst_reader_iter.next()) + train_inputs.append(next(tst_reader_iter)) os.environ['CPU_NUM'] = str(4) if core.is_compiled_with_cuda(): @@ -133,7 +133,7 @@ class TestFeedParallel(unittest.TestCase): for batch_id, data in enumerate(reader()): loss_np = pe.run(feed=data, fetch_list=[loss.name])[0] - print batch_id, loss_np + print(batch_id, loss_np) if batch_id == 2: break diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_mnist.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_mnist.py index c7a039d2589ef67bd1d3771a2f11084698ba909f..9448d89cd58f4e5cff4bac49821fbc44c5a46246 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_mnist.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_mnist.py @@ -37,7 +37,7 @@ def simple_fc_net(use_feed): reader = fluid.layers.io.double_buffer(reader) img, label = fluid.layers.read_file(reader) hidden = img - for _ in xrange(4): + for _ in range(4): hidden = fluid.layers.fc( hidden, size=200, @@ -64,7 +64,7 @@ def fc_with_batchnorm(use_feed): img, label = fluid.layers.read_file(reader) hidden = img - for _ in xrange(1): + for _ in range(1): hidden = fluid.layers.fc( hidden, size=200, @@ -98,16 +98,13 @@ class TestMNIST(TestParallelExecutorBase): fluid.recordio_writer.convert_reader_to_recordio_file( MNIST_RECORDIO_FILE, reader, feeder) - def _init_data(self, random=True): + def _init_data(self): np.random.seed(5) - if random: - img = np.random.random(size=[32, 784]).astype(np.float32) - else: - img = np.ones(shape=[32, 784], dtype='float32') + img = np.random.random(size=[32, 784]).astype(np.float32) label = np.ones(shape=[32, 1], dtype='int64') return img, label - def _compare_reduce_and_allreduce(self, model, use_cuda, random_data=True): + def _compare_reduce_and_allreduce(self, model, use_cuda): if use_cuda and not core.is_compiled_with_cuda(): return self.check_network_convergence( @@ -115,7 +112,7 @@ class TestMNIST(TestParallelExecutorBase): self.check_network_convergence( model, use_cuda=use_cuda, allow_op_delay=True, use_reduce=True) - img, label = self._init_data(random_data) + img, label = self._init_data() all_reduce_first_loss, all_reduce_last_loss = self.check_network_convergence( model, @@ -131,9 +128,9 @@ class TestMNIST(TestParallelExecutorBase): use_reduce=True) for loss in zip(all_reduce_first_loss, reduce_first_loss): - self.assertAlmostEquals(loss[0], loss[1], delta=1e-6) + self.assertAlmostEqual(loss[0], loss[1], delta=1e-6) for loss in zip(all_reduce_last_loss, reduce_last_loss): - self.assertAlmostEquals(loss[0], loss[1], delta=1e-4) + self.assertAlmostEqual(loss[0], loss[1], delta=1e-4) # simple_fc def check_simple_fc_convergence(self, use_cuda, use_reduce=False): @@ -166,27 +163,27 @@ class TestMNIST(TestParallelExecutorBase): if use_cuda and not core.is_compiled_with_cuda(): return - img, label = self._init_data(random=False) + img, label = self._init_data() single_first_loss, single_last_loss = self.check_network_convergence( method=simple_fc_net, - seed=1000, + seed=1, feed_dict={"image": img, "label": label}, use_cuda=use_cuda, use_parallel_executor=False) parallel_first_loss, parallel_last_loss = self.check_network_convergence( method=simple_fc_net, - seed=1000, + seed=1, feed_dict={"image": img, "label": label}, use_cuda=use_cuda, use_parallel_executor=True) - for p_f in parallel_first_loss: - self.assertAlmostEquals(p_f, single_first_loss[0], delta=1e-6) - for p_l in parallel_last_loss: - self.assertAlmostEquals(p_l, single_last_loss[0], delta=1e-6) + self.assertAlmostEquals( + np.mean(parallel_first_loss), single_first_loss, delta=1e-6) + self.assertAlmostEquals( + np.mean(parallel_last_loss), single_last_loss, delta=1e-6) def test_simple_fc_parallel_accuracy(self): self.check_simple_fc_parallel_accuracy(True) diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext.py index 834e920845f29b153909a971eb5afc4f8a33346e..a28428d8dee201ba105e18684c15d4b4582d989f 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext.py @@ -21,6 +21,19 @@ from parallel_executor_test_base import TestParallelExecutorBase import unittest import math import os +import numpy as np + +# FIXME(zcd): If the neural net has dropout_op, the output of ParallelExecutor +# and Executor is different. Because, for ParallelExecutor, the dropout_op of +# the neural net will be copied N copies(N is the number of device). This will +# lead to the random numbers generated by ParallelExecutor and Executor are different. +# So, if we compare the loss of ParallelExecutor and Executor, we should remove the +# dropout_op. +remove_dropout = False + +# FIXME(zcd): If the neural net has batch_norm, the output of ParallelExecutor +# and Executor is different. +remove_bn = False def squeeze_excitation(input, num_channels, reduction_ratio): @@ -53,7 +66,8 @@ def conv_bn_layer(input, num_filters, filter_size, stride=1, groups=1, groups=groups, act=None, bias_attr=False) - return fluid.layers.batch_norm(input=conv, act=act, momentum=0.1) + return conv if remove_bn else fluid.layers.batch_norm( + input=conv, act=act, momentum=0.1) def shortcut(input, ch_out, stride): @@ -92,13 +106,14 @@ def bottleneck_block(input, num_filters, stride, cardinality, reduction_ratio): return fluid.layers.elementwise_add(x=short, y=scale, act='relu') -def SE_ResNeXt50Small(batch_size=2, use_feed=False): - assert not use_feed, "SE_ResNeXt doesn't support feed yet" +batch_size = 12 +img_shape = [3, 224, 224] + - img = fluid.layers.fill_constant( - shape=[batch_size, 3, 224, 224], dtype='float32', value=0.0) - label = fluid.layers.fill_constant( - shape=[batch_size, 1], dtype='int64', value=0.0) +def SE_ResNeXt50Small(use_feed): + + img = fluid.layers.data(name='image', shape=img_shape, dtype='float32') + label = fluid.layers.data(name='label', shape=[1], dtype='int64') conv = conv_bn_layer( input=img, num_filters=16, filter_size=3, stride=2, act='relu') @@ -127,7 +142,8 @@ def SE_ResNeXt50Small(batch_size=2, use_feed=False): reshape = fluid.layers.reshape( x=conv, shape=[-1, shape[1], shape[2] * shape[3]]) pool = fluid.layers.reduce_mean(input=reshape, dim=2) - dropout = fluid.layers.dropout(x=pool, dropout_prob=0.2) + dropout = pool if remove_dropout else fluid.layers.dropout( + x=pool, dropout_prob=0.2, seed=1) # Classifier layer: prediction = fluid.layers.fc(input=dropout, size=1000, act='softmax') loss = fluid.layers.cross_entropy(input=prediction, label=label) @@ -135,75 +151,135 @@ def SE_ResNeXt50Small(batch_size=2, use_feed=False): return loss -class TestResnet(TestParallelExecutorBase): - def check_resnet_convergence_with_learning_rate_decay(self, - use_cuda=True, - use_reduce=False, - iter=20): +def cosine_decay(learning_rate, step_each_epoch, epochs=120): + """ + Applies cosine decay to the learning rate. + lr = 0.05 * (math.cos(epoch * (math.pi / 120)) + 1) + """ + global_step = _decay_step_counter() - if use_cuda and not core.is_compiled_with_cuda(): - return + with init_on_cpu(): + epoch = ops.floor(global_step / step_each_epoch) + decayed_lr = learning_rate * \ + (ops.cos(epoch * (math.pi / epochs)) + 1)/2 + return decayed_lr - os.environ['CPU_NUM'] = str(4) - def _cosine_decay(learning_rate, step_each_epoch, epochs=120): - """ - Applies cosine decay to the learning rate. - lr = 0.05 * (math.cos(epoch * (math.pi / 120)) + 1) - """ - global_step = _decay_step_counter() +def optimizer(learning_rate=0.01): + optimizer = fluid.optimizer.Momentum( + learning_rate=cosine_decay( + learning_rate=learning_rate, step_each_epoch=2, epochs=1), + momentum=0.9, + regularization=fluid.regularizer.L2Decay(1e-4)) + return optimizer - with init_on_cpu(): - epoch = ops.floor(global_step / step_each_epoch) - decayed_lr = learning_rate * \ - (ops.cos(epoch * (math.pi / epochs)) + 1)/2 - return decayed_lr - def _optimizer(learning_rate=0.01): - optimizer = fluid.optimizer.Momentum( - learning_rate=_cosine_decay( - learning_rate=learning_rate, step_each_epoch=2, epochs=1), - momentum=0.9, - regularization=fluid.regularizer.L2Decay(1e-4)) - return optimizer +class TestResnet(TestParallelExecutorBase): + @classmethod + def setUpClass(cls): + os.environ['CPU_NUM'] = str(4) + global remove_dropout + global remove_bn + remove_dropout = False + remove_bn = False + + def _init_data(self, batch_size=2, random=True): + np.random.seed(5) + if random: + img = np.random.random( + size=[batch_size] + img_shape).astype(np.float32) + else: + img = np.ones(shape=[batch_size] + img_shape, dtype='float32') + label = [np.random.randint(0, 999) for _ in range(batch_size)] + label = np.array(label).astype(np.int64).reshape(-1, 1) + return img, label + + def _compare_reduce_and_allreduce(self, + model, + use_cuda, + iter=20, + delta2=1e-6): + if use_cuda and not core.is_compiled_with_cuda(): + return - import functools + global remove_bn + remove_bn = True - batch_size = 2 + img, label = self._init_data(batch_size=batch_size) + all_reduce_first_loss, all_reduce_last_loss = self.check_network_convergence( + model, + feed_dict={"image": img, + "label": label}, + iter=iter, + batch_size=batch_size, + use_cuda=use_cuda, + use_reduce=False, + optimizer=optimizer) + reduce_first_loss, reduce_last_loss = self.check_network_convergence( + model, + feed_dict={"image": img, + "label": label}, + iter=iter, + batch_size=batch_size, + use_cuda=use_cuda, + use_reduce=True, + optimizer=optimizer) + + for loss in zip(all_reduce_first_loss, reduce_first_loss): + self.assertAlmostEquals(loss[0], loss[1], delta=1e-6) + for loss in zip(all_reduce_last_loss, reduce_last_loss): + self.assertAlmostEquals(loss[0], loss[1], delta=delta2) + + def _check_resnet_convergence(self, + model, + use_cuda=True, + use_reduce=False, + iter=20, + delta2=1e-6): + if use_cuda and not core.is_compiled_with_cuda(): + return + global remove_dropout + global remove_bn + remove_dropout = True + remove_bn = True + + img, label = self._init_data(batch_size=batch_size) single_first_loss, single_last_loss = self.check_network_convergence( - functools.partial( - SE_ResNeXt50Small, batch_size=batch_size), + model, + feed_dict={"image": img, + "label": label}, iter=iter, batch_size=batch_size, use_cuda=use_cuda, use_reduce=use_reduce, - optimizer=_optimizer, + optimizer=optimizer, use_parallel_executor=False) - parallel_first_loss, parallel_last_loss = self.check_network_convergence( - functools.partial( - SE_ResNeXt50Small, batch_size=batch_size), + model, + feed_dict={"image": img, + "label": label}, iter=iter, batch_size=batch_size, use_cuda=use_cuda, use_reduce=use_reduce, - optimizer=_optimizer) + optimizer=optimizer) - for p_f in parallel_first_loss: - self.assertAlmostEquals(p_f, single_first_loss[0], delta=1e-6) - for p_l in parallel_last_loss: - self.assertAlmostEquals(p_l, single_last_loss[0], delta=1e-6) + self.assertAlmostEquals( + np.mean(parallel_first_loss), single_first_loss[0], delta=1e-6) + self.assertAlmostEquals( + np.mean(parallel_last_loss), single_last_loss[0], delta=delta2) def test_seresnext_with_learning_rate_decay(self): - self.check_resnet_convergence_with_learning_rate_decay(True, False) - self.check_resnet_convergence_with_learning_rate_decay( - False, False, iter=5) - - def test_seresnext_with_new_strategy_with_learning_rate_decay(self): - self.check_resnet_convergence_with_learning_rate_decay(True, True) - self.check_resnet_convergence_with_learning_rate_decay( - False, True, iter=5) + self._check_resnet_convergence(model=SE_ResNeXt50Small, use_cuda=True) + self._check_resnet_convergence( + model=SE_ResNeXt50Small, use_cuda=False, iter=2, delta2=1e-3) + + def test_seresnext_with_new_strategy(self): + self._compare_reduce_and_allreduce( + model=SE_ResNeXt50Small, use_cuda=True, delta2=1e-2) + self._compare_reduce_and_allreduce( + model=SE_ResNeXt50Small, use_cuda=False, iter=5) if __name__ == '__main__': diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_test_while_train.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_test_while_train.py index 7688b8495d7f7c60e80f62dae2edd72be9f839d4..fcb5947ff05efd1c48ab9ec129ac9d17255d7020 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_test_while_train.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_test_while_train.py @@ -25,7 +25,7 @@ def simple_fc_net(): img = fluid.layers.data(name='image', shape=[784], dtype='float32') label = fluid.layers.data(name='label', shape=[1], dtype='int64') hidden = img - for _ in xrange(4): + for _ in range(4): hidden = fluid.layers.fc( hidden, size=200, @@ -71,7 +71,7 @@ class ParallelExecutorTestingDuringTraining(unittest.TestCase): share_vars_from=train_exe, build_strategy=build_strategy) - for i in xrange(5): + for i in range(5): test_loss, = test_exe.run([loss.name], feed=feed_dict) train_loss, = train_exe.run([loss.name], feed=feed_dict) diff --git a/python/paddle/fluid/tests/unittests/test_parallel_op.py b/python/paddle/fluid/tests/unittests/test_parallel_op.py index 18309f457704f522457daefdb8464ae5df2ffcfb..c9617e36778740ce9620c3ad495c64c17277fde1 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_op.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_op.py @@ -18,6 +18,7 @@ import paddle.fluid as fluid from paddle.fluid.layers.device import get_places import paddle.fluid.profiler as profiler import numpy +import six class BaseParallelForTest(unittest.TestCase): @@ -25,20 +26,20 @@ class BaseParallelForTest(unittest.TestCase): """ Run the unittest for parallel.for Args: - callback(callable): A callable function returns a generator. There - are two yields in the generator function. The first yield - returns the data layers, and the second yield returns the loss. - The modified data variables will be sent back during the first + callback(callable): A callable function returns a generator. There + are two yields in the generator function. The first yield + returns the data layers, and the second yield returns the loss. + The modified data variables will be sent back during the first yield. feed(dict): The executor feeding dictionary. - fetch(list|basestr): The fetch name lists. + fetch(list|basestr): The fetch name lists. Returns: None Raises: - AssertionError when the computation of cpu, parallel.for in cpu, + AssertionError when the computation of cpu, parallel.for in cpu, gpu, parallel.for in gpu are different. """ @@ -95,14 +96,14 @@ class BaseParallelForTest(unittest.TestCase): """ Run a single test, returns the fetch values Args: - place(Place): the computation place. - use_parallel(bool): Whether use parallel.for or not. + place(Place): the computation place. + use_parallel(bool): Whether use parallel.for or not. Returns: Fetched numpy arrays. """ - if isinstance(fetch, basestring): + if isinstance(fetch, six.string_types): fetch = [fetch] main = fluid.Program() startup = fluid.Program() @@ -124,7 +125,7 @@ class BaseParallelForTest(unittest.TestCase): data = [data] with pd.do(): - ins = map(pd.read_input, data) + ins = list(map(pd.read_input, data)) if len(ins) == 1: ins = ins[0] loss = generator.send(ins) # patch input @@ -156,7 +157,7 @@ class BaseParallelForTest(unittest.TestCase): Returns: None - + Raises: AssertionError diff --git a/python/paddle/fluid/tests/unittests/test_polygon_box_transform.py b/python/paddle/fluid/tests/unittests/test_polygon_box_transform.py index 2105d320665367e3ec1bfd7b3a353a144c91244f..8aff4e87f67bc61a162f09e982cf0a7a61639257 100644 --- a/python/paddle/fluid/tests/unittests/test_polygon_box_transform.py +++ b/python/paddle/fluid/tests/unittests/test_polygon_box_transform.py @@ -23,9 +23,9 @@ def PolygonBoxRestore(input): geo_channels = shape[1] h = shape[2] w = shape[3] - h_indexes = np.array(range(h) * w).reshape( + h_indexes = np.array(list(range(h)) * w).reshape( [w, h]).transpose()[np.newaxis, :] # [1, h, w] - w_indexes = np.array(range(w) * h).reshape( + w_indexes = np.array(list(range(w)) * h).reshape( [h, w])[np.newaxis, :] # [1, h, w] indexes = np.concatenate( (w_indexes, h_indexes))[np.newaxis, :] # [1, 2, h, w] diff --git a/python/paddle/fluid/tests/unittests/test_pool2d_op.py b/python/paddle/fluid/tests/unittests/test_pool2d_op.py index f7e1e8573290766cde0c35816d687e7ba6fa4220..1cf70311b40bc7648b7462e93f201aa33c77b137 100644 --- a/python/paddle/fluid/tests/unittests/test_pool2d_op.py +++ b/python/paddle/fluid/tests/unittests/test_pool2d_op.py @@ -35,8 +35,8 @@ def max_pool2D_forward_naive(x, ) / strides[1] + 1 if ceil_mode else (W - ksize[1] + 2 * paddings[1]) / strides[1] + 1 out = np.zeros((N, C, H_out, W_out)) - for i in xrange(H_out): - for j in xrange(W_out): + for i in range(H_out): + for j in range(W_out): r_start = np.max((i * strides[0] - paddings[0], 0)) r_end = np.min((i * strides[0] + ksize[0] - paddings[0], H)) c_start = np.max((j * strides[1] - paddings[1], 0)) @@ -63,8 +63,8 @@ def avg_pool2D_forward_naive(x, ) / strides[1] + 1 if ceil_mode else (W - ksize[1] + 2 * paddings[1]) / strides[1] + 1 out = np.zeros((N, C, H_out, W_out)) - for i in xrange(H_out): - for j in xrange(W_out): + for i in range(H_out): + for j in range(W_out): r_start = np.max((i * strides[0] - paddings[0], 0)) r_end = np.min((i * strides[0] + ksize[0] - paddings[0], H)) c_start = np.max((j * strides[1] - paddings[1], 0)) diff --git a/python/paddle/fluid/tests/unittests/test_pool3d_op.py b/python/paddle/fluid/tests/unittests/test_pool3d_op.py index 142165f29beeaedfaa660f04424147e06710d192..92c64b37921eafd4c90e247a235f2dacea8fea1e 100644 --- a/python/paddle/fluid/tests/unittests/test_pool3d_op.py +++ b/python/paddle/fluid/tests/unittests/test_pool3d_op.py @@ -38,13 +38,13 @@ def max_pool3D_forward_naive(x, ) / strides[2] + 1 if ceil_mode else (W - ksize[2] + 2 * paddings[2]) / strides[2] + 1 out = np.zeros((N, C, D_out, H_out, W_out)) - for k in xrange(D_out): + for k in range(D_out): d_start = np.max((k * strides[0] - paddings[0], 0)) d_end = np.min((k * strides[0] + ksize[0] - paddings[0], D)) - for i in xrange(H_out): + for i in range(H_out): h_start = np.max((i * strides[0] - paddings[0], 0)) h_end = np.min((i * strides[0] + ksize[0] - paddings[0], H)) - for j in xrange(W_out): + for j in range(W_out): w_start = np.max((j * strides[1] - paddings[1], 0)) w_end = np.min((j * strides[1] + ksize[1] - paddings[1], W)) x_masked = x[:, :, d_start:d_end, h_start:h_end, w_start:w_end] @@ -72,13 +72,13 @@ def avg_pool3D_forward_naive(x, ) / strides[2] + 1 if ceil_mode else (W - ksize[2] + 2 * paddings[2]) / strides[2] + 1 out = np.zeros((N, C, D_out, H_out, W_out)) - for k in xrange(D_out): + for k in range(D_out): d_start = np.max((k * strides[0] - paddings[0], 0)) d_end = np.min((k * strides[0] + ksize[0] - paddings[0], D)) - for i in xrange(H_out): + for i in range(H_out): h_start = np.max((i * strides[0] - paddings[0], 0)) h_end = np.min((i * strides[0] + ksize[0] - paddings[0], H)) - for j in xrange(W_out): + for j in range(W_out): w_start = np.max((j * strides[1] - paddings[1], 0)) w_end = np.min((j * strides[1] + ksize[1] - paddings[1], W)) x_masked = x[:, :, d_start:d_end, h_start:h_end, w_start:w_end] diff --git a/python/paddle/fluid/tests/unittests/test_pool_max_op.py b/python/paddle/fluid/tests/unittests/test_pool_max_op.py index cf9b7639224ef3804b946f729bb6a9cead4aae23..e6a9f6f08cf1445c14494506641b0c3502591c37 100644 --- a/python/paddle/fluid/tests/unittests/test_pool_max_op.py +++ b/python/paddle/fluid/tests/unittests/test_pool_max_op.py @@ -29,21 +29,21 @@ def max_pool3D_forward_naive(x, ksize, strides, paddings, global_pool=False): W_out = (W - ksize[2] + 2 * paddings[2]) / strides[2] + 1 out = np.zeros((N, C, D_out, H_out, W_out)) mask = np.zeros((N, C, D_out, H_out, W_out)) - for k in xrange(D_out): + for k in range(D_out): d_start = np.max((k * strides[0] - paddings[0], 0)) d_end = np.min((k * strides[0] + ksize[0] - paddings[0], D)) - for i in xrange(H_out): + for i in range(H_out): h_start = np.max((i * strides[0] - paddings[0], 0)) h_end = np.min((i * strides[0] + ksize[0] - paddings[0], H)) - for j in xrange(W_out): + for j in range(W_out): w_start = np.max((j * strides[1] - paddings[1], 0)) w_end = np.min((j * strides[1] + ksize[1] - paddings[1], W)) x_masked = x[:, :, d_start:d_end, h_start:h_end, w_start:w_end] out[:, :, k, i, j] = np.max(x_masked, axis=(2, 3, 4)) - for n in xrange(N): - for c in xrange(C): + for n in range(N): + for c in range(C): arr = x_masked[n, c, :, :, :] index = np.where(arr == np.max(arr)) sub_deep = index[0][0] @@ -67,8 +67,8 @@ def max_pool2D_forward_naive(x, ksize, strides, paddings, global_pool=False): W_out = (W - ksize[1] + 2 * paddings[1]) / strides[1] + 1 out = np.zeros((N, C, H_out, W_out)) mask = np.zeros((N, C, H_out, W_out)) - for i in xrange(H_out): - for j in xrange(W_out): + for i in range(H_out): + for j in range(W_out): r_start = np.max((i * strides[0] - paddings[0], 0)) r_end = np.min((i * strides[0] + ksize[0] - paddings[0], H)) c_start = np.max((j * strides[1] - paddings[1], 0)) @@ -77,8 +77,8 @@ def max_pool2D_forward_naive(x, ksize, strides, paddings, global_pool=False): out[:, :, i, j] = np.max(x_masked, axis=(2, 3)) - for n in xrange(N): - for c in xrange(C): + for n in range(N): + for c in range(C): arr = x_masked[n, c, :, :] index = np.where(arr == np.max(arr)) sub_row = index[0][0] diff --git a/python/paddle/fluid/tests/unittests/test_positive_negative_pair_op.py b/python/paddle/fluid/tests/unittests/test_positive_negative_pair_op.py index 091cfc9c72769fefc9c792bfeaa872cb357736b7..8c76393bdaccc0b701b409efebf08fac95aa5f1a 100644 --- a/python/paddle/fluid/tests/unittests/test_positive_negative_pair_op.py +++ b/python/paddle/fluid/tests/unittests/test_positive_negative_pair_op.py @@ -32,7 +32,7 @@ def py_pnpair_op(score, label, query, column=-1, weight=None): # accumulate statistics pos, neg, neu = 0, 0, 0 - for _, ranks in predictions.items(): + for _, ranks in list(predictions.items()): for e1, e2 in itertools.combinations(ranks, 2): s1, s2, l1, l2, w1, w2 = e1[0], e2[0], e1[1], e2[1], e1[2], e2[2] w = (w1 + w2) * 0.5 diff --git a/python/paddle/fluid/tests/unittests/test_precision_recall_op.py b/python/paddle/fluid/tests/unittests/test_precision_recall_op.py index 7830ba29583d369c4b9f6f3077dda1dda1fd1c46..5ae425fee18b9b1baa0b945782268b79d6bb6625 100644 --- a/python/paddle/fluid/tests/unittests/test_precision_recall_op.py +++ b/python/paddle/fluid/tests/unittests/test_precision_recall_op.py @@ -39,19 +39,19 @@ def get_states(idxs, labels, cls_num, weights=None): ins_num = idxs.shape[0] # TP FP TN FN states = np.zeros((cls_num, 4)).astype('float32') - for i in xrange(ins_num): + for i in range(ins_num): w = weights[i] if weights is not None else 1.0 idx = idxs[i][0] label = labels[i][0] if idx == label: states[idx][0] += w - for j in xrange(cls_num): + for j in range(cls_num): states[j][2] += w states[idx][2] -= w else: states[label][3] += w states[idx][1] += w - for j in xrange(cls_num): + for j in range(cls_num): states[j][2] += w states[label][2] -= w states[idx][2] -= w @@ -64,7 +64,7 @@ def compute_metrics(states, cls_num): total_fn_count = 0.0 macro_avg_precision = 0.0 macro_avg_recall = 0.0 - for i in xrange(cls_num): + for i in range(cls_num): total_tp_count += states[i][0] total_fp_count += states[i][1] total_fn_count += states[i][3] @@ -90,9 +90,9 @@ class TestPrecisionRecallOp_0(OpTest): ins_num = 64 cls_num = 10 max_probs = np.random.uniform(0, 1.0, (ins_num, 1)).astype('float32') - idxs = np.random.choice(xrange(cls_num), ins_num).reshape( + idxs = np.random.choice(range(cls_num), ins_num).reshape( (ins_num, 1)).astype('int32') - labels = np.random.choice(xrange(cls_num), ins_num).reshape( + labels = np.random.choice(range(cls_num), ins_num).reshape( (ins_num, 1)).astype('int32') states = get_states(idxs, labels, cls_num) metrics = compute_metrics(states, cls_num) @@ -117,10 +117,10 @@ class TestPrecisionRecallOp_1(OpTest): ins_num = 64 cls_num = 10 max_probs = np.random.uniform(0, 1.0, (ins_num, 1)).astype('float32') - idxs = np.random.choice(xrange(cls_num), ins_num).reshape( + idxs = np.random.choice(range(cls_num), ins_num).reshape( (ins_num, 1)).astype('int32') weights = np.random.uniform(0, 1.0, (ins_num, 1)).astype('float32') - labels = np.random.choice(xrange(cls_num), ins_num).reshape( + labels = np.random.choice(range(cls_num), ins_num).reshape( (ins_num, 1)).astype('int32') states = get_states(idxs, labels, cls_num, weights) @@ -151,10 +151,10 @@ class TestPrecisionRecallOp_2(OpTest): ins_num = 64 cls_num = 10 max_probs = np.random.uniform(0, 1.0, (ins_num, 1)).astype('float32') - idxs = np.random.choice(xrange(cls_num), ins_num).reshape( + idxs = np.random.choice(range(cls_num), ins_num).reshape( (ins_num, 1)).astype('int32') weights = np.random.uniform(0, 1.0, (ins_num, 1)).astype('float32') - labels = np.random.choice(xrange(cls_num), ins_num).reshape( + labels = np.random.choice(range(cls_num), ins_num).reshape( (ins_num, 1)).astype('int32') states = np.random.randint(0, 30, (cls_num, 4)).astype('float32') diff --git a/python/paddle/fluid/tests/unittests/test_protobuf_descs.py b/python/paddle/fluid/tests/unittests/test_protobuf_descs.py index f75a79bfa42405747df9e6f4f4ab743014e303b9..621dd681345d2c185c0644de917b8bcd529f9937 100644 --- a/python/paddle/fluid/tests/unittests/test_protobuf_descs.py +++ b/python/paddle/fluid/tests/unittests/test_protobuf_descs.py @@ -183,7 +183,7 @@ class TestBlockDesc(unittest.TestCase): op2 = block.append_op() op0 = block._prepend_op() all_ops = [] - for idx in xrange(0, block.op_size()): + for idx in range(0, block.op_size()): all_ops.append(block.op(idx)) self.assertEqual(all_ops, [op0, op1, op2]) @@ -205,7 +205,7 @@ class TestBlockDesc(unittest.TestCase): program._sync_with_cpp() all_ops = [] - for idx in xrange(0, block.op_size()): + for idx in range(0, block.op_size()): all_ops.append(block.op(idx)) self.assertEqual(all_ops, [op0, op2]) diff --git a/python/paddle/fluid/tests/unittests/test_reader_reset.py b/python/paddle/fluid/tests/unittests/test_reader_reset.py index d35183647ea57e378f0fe201ef03001122cb329f..3ad85d57485956e0cadb197dadd172516fa15c39 100644 --- a/python/paddle/fluid/tests/unittests/test_reader_reset.py +++ b/python/paddle/fluid/tests/unittests/test_reader_reset.py @@ -21,7 +21,7 @@ import unittest class TestReaderReset(unittest.TestCase): def prepare_data(self): def fake_data_generator(): - for n in xrange(self.total_ins_num): + for n in range(self.total_ins_num): yield np.ones(self.ins_shape) * n, n # Prepare data diff --git a/python/paddle/fluid/tests/unittests/test_recurrent_op.py b/python/paddle/fluid/tests/unittests/test_recurrent_op.py index d6ff18430e319e236f03d5661381e923cc956590..2e22df2beba9d74e28788fb72f6f7f7f2bef534e 100644 --- a/python/paddle/fluid/tests/unittests/test_recurrent_op.py +++ b/python/paddle/fluid/tests/unittests/test_recurrent_op.py @@ -203,12 +203,12 @@ class RecurrentOpTest1(unittest.TestCase): num_grad[idx], ana_grad[idx], rtol=0.1).all()) def check_forward(self): - print 'test recurrent op forward' + print('test recurrent op forward') pd_output = self.forward() py_output = self.py_rnn.forward() - print 'pd_output', pd_output + print('pd_output', pd_output) print - print 'py_output', py_output + print('py_output', py_output) self.assertEqual(pd_output.shape, py_output.shape) self.assertTrue(np.isclose(pd_output, py_output, rtol=0.1).all()) @@ -445,7 +445,7 @@ class RecurrentOpNoMemBootTest(RecurrentOpTest1): self.py_rnn = RecurrentOpNoMemBootTest.PySimpleRNN4(self.input_shape, self.output_shape) self.output = layers.mean(self.create_rnn_op(), **self.p_info) - print self.main_program + print(self.main_program) def create_rnn_op(self): x = layers.data( diff --git a/python/paddle/fluid/tests/unittests/test_seq_conv.py b/python/paddle/fluid/tests/unittests/test_seq_conv.py index 9701d9adef1fd272f2520f66607acded6a8c25c6..1a6e1aad799e77b8e746353bee93680691939d24 100644 --- a/python/paddle/fluid/tests/unittests/test_seq_conv.py +++ b/python/paddle/fluid/tests/unittests/test_seq_conv.py @@ -26,9 +26,9 @@ class TestSeqProject(OpTest): if self.context_length == 1 \ and self.context_start == 0 \ and self.padding_trainable: - print "If context_start is 0 " \ + print("If context_start is 0 " \ "and context_length is 1," \ - " padding_trainable should be false." + " padding_trainable should be false.") return # one level, batch size @@ -212,7 +212,7 @@ class TestSeqProjectCase2(TestSeqProject): self.context_stride = 1 self.input_size = [self.input_row, 23] - idx = range(self.input_size[0]) + idx = list(range(self.input_size[0])) del idx[0] offset_lod = [[0] + np.sort(random.sample(idx, 8)).tolist() + [self.input_size[0]]] diff --git a/python/paddle/fluid/tests/unittests/test_sequence_expand.py b/python/paddle/fluid/tests/unittests/test_sequence_expand.py index 0bbd31814efdff6050733f6876ef64e3fcaaaf76..5ff0dab23e516ae8114b8264492fb2a9d5c0b3f8 100644 --- a/python/paddle/fluid/tests/unittests/test_sequence_expand.py +++ b/python/paddle/fluid/tests/unittests/test_sequence_expand.py @@ -44,7 +44,7 @@ class TestSequenceExpand(OpTest): out_lod = [[]] offset = 0 - for i in xrange(len(y_lod[ref_level])): + for i in range(len(y_lod[ref_level])): repeat_num = y_lod[ref_level][i] x_len = x_idx[i] @@ -55,7 +55,7 @@ class TestSequenceExpand(OpTest): stacked_x_sub = np.vstack((stacked_x_sub, x_sub)) out = np.vstack((out, stacked_x_sub)) if x_lod is not None: - for j in xrange(repeat_num): + for j in range(repeat_num): out_lod[0].append(x_len) offset += x_len diff --git a/python/paddle/fluid/tests/unittests/test_sequence_reshape.py b/python/paddle/fluid/tests/unittests/test_sequence_reshape.py index 68f2e5eba35ed318281d14e397dc6d363bcb4079..39b02ecf6ddb40737c4c1737d652c1a1b744d923 100644 --- a/python/paddle/fluid/tests/unittests/test_sequence_reshape.py +++ b/python/paddle/fluid/tests/unittests/test_sequence_reshape.py @@ -35,7 +35,7 @@ class TestSequenceReshape(OpTest): def compute_output(self, x, x_lod, dimension): x_width = x.shape[1] out_lod = [[]] - for i in xrange(len(x_lod[0])): + for i in range(len(x_lod[0])): seq_len = x_lod[0][i] offset = (seq_len * x_width) / dimension assert int(offset) * dimension == seq_len * x_width diff --git a/python/paddle/fluid/tests/unittests/test_shrink_rnn_memory.py b/python/paddle/fluid/tests/unittests/test_shrink_rnn_memory.py index 6f0e337034d1010880514181654170316fd9db19..a994bf181a74ca71a970da0105fe767f82750a6c 100644 --- a/python/paddle/fluid/tests/unittests/test_shrink_rnn_memory.py +++ b/python/paddle/fluid/tests/unittests/test_shrink_rnn_memory.py @@ -48,7 +48,7 @@ class TestShrinkRNNMemoryBase(unittest.TestCase): def sum_lodtensor(self, tensor): sum_res = 0.0 - for i in xrange(np.product(tensor.shape())): + for i in range(np.product(tensor.shape())): sum_res += tensor._get_float_element(i) return sum_res diff --git a/python/paddle/fluid/tests/unittests/test_split_op.py b/python/paddle/fluid/tests/unittests/test_split_op.py index eb49a53e54f4bdb6bcd6cb1991423970f29997bb..6b67a52e81b978ed78c72629f9177759f8e2c4e1 100644 --- a/python/paddle/fluid/tests/unittests/test_split_op.py +++ b/python/paddle/fluid/tests/unittests/test_split_op.py @@ -26,7 +26,7 @@ class TestSplitOp(OpTest): self.inputs = {'X': x} self.attrs = {'axis': axis, 'sections': [2, 1, 2]} self.outputs = {'Out': [('out%d' % i, out[i]) \ - for i in xrange(len(out))]} + for i in range(len(out))]} def _set_op_type(self): self.op_type = "split" diff --git a/python/paddle/fluid/tests/unittests/test_split_selected_rows_op.py b/python/paddle/fluid/tests/unittests/test_split_selected_rows_op.py index 61040a39ced6dc57d05a10bf0605c80011db45c3..2b261820e04b08234477fc0a9adde95262f99bba 100644 --- a/python/paddle/fluid/tests/unittests/test_split_selected_rows_op.py +++ b/python/paddle/fluid/tests/unittests/test_split_selected_rows_op.py @@ -53,7 +53,7 @@ class TestSpliteSelectedRows(unittest.TestCase): height_sections = [5, 5, 5, 5, 3] # initialize output variables [out0, out1] - outs_name = ["out%d" % i for i in xrange(len(height_sections))] + outs_name = ["out%d" % i for i in range(len(height_sections))] outs = [ scope.var(var_name).get_selected_rows() for var_name in outs_name ] diff --git a/python/paddle/fluid/tests/unittests/test_spp_op.py b/python/paddle/fluid/tests/unittests/test_spp_op.py index f0ab5909df62835b252154709e5ff75ca38235c8..3cbfc2a703f1c4a24674d468cd1152bfa6eb8ad2 100644 --- a/python/paddle/fluid/tests/unittests/test_spp_op.py +++ b/python/paddle/fluid/tests/unittests/test_spp_op.py @@ -26,7 +26,7 @@ class TestSppOp(OpTest): input = np.random.random(self.shape).astype("float32") nsize, csize, hsize, wsize = input.shape out_level_flatten = [] - for i in xrange(self.pyramid_height): + for i in range(self.pyramid_height): bins = np.power(2, i) kernel_size = [0, 0] padding = [0, 0] diff --git a/python/paddle/fluid/tests/unittests/test_top_k_op.py b/python/paddle/fluid/tests/unittests/test_top_k_op.py index cc2fcc5ec0a076679c7dd85a7e8f8da6a170172b..cbc3da550306b9febe8a8fd22e7f71efa572a3d0 100644 --- a/python/paddle/fluid/tests/unittests/test_top_k_op.py +++ b/python/paddle/fluid/tests/unittests/test_top_k_op.py @@ -28,7 +28,7 @@ class TestTopkOp(OpTest): self.inputs = {'X': input} self.attrs = {'k': k} - for rowid in xrange(32): + for rowid in range(32): row = input[rowid] output[rowid] = np.sort(row)[-k:] indices[rowid] = row.argsort()[-k:] @@ -52,7 +52,7 @@ class TestTopkOp3d(OpTest): self.inputs = {'X': input_flat_2d} self.attrs = {'k': k} - for rowid in xrange(64): + for rowid in range(64): row = input_flat_2d[rowid] output[rowid] = np.sort(row)[-k:] indices[rowid] = row.argsort()[-k:] diff --git a/python/paddle/fluid/tests/unittests/test_unpool_op.py b/python/paddle/fluid/tests/unittests/test_unpool_op.py index a97d6dfdda9b79eed3be6302fb2b1c3810f189dc..ecce4cdde2d648fe7d65427e34c77f5f9ad61417 100644 --- a/python/paddle/fluid/tests/unittests/test_unpool_op.py +++ b/python/paddle/fluid/tests/unittests/test_unpool_op.py @@ -22,10 +22,10 @@ def unpool2dmax_forward_naive(input, indices, ksize, strides, paddings): out_hsize = (s2 - 1) * strides[0] - 2 * paddings[0] + ksize[0] out_wsize = (s2 - 1) * strides[1] - 2 * paddings[1] + ksize[1] out = np.zeros((s0, s1, out_hsize, out_wsize)) - for nidx in xrange(s0): - for cidx in xrange(s1): - for h in xrange(s2): - for w in xrange(s3): + for nidx in range(s0): + for cidx in range(s1): + for h in range(s2): + for w in range(s3): index = indices[nidx, cidx, h, w] hidx = (index - index % out_wsize) / out_wsize widx = index % out_wsize @@ -47,16 +47,16 @@ class TestUnpoolOp(OpTest): self.strides[1] + 1 input = np.zeros((nsize, csize, hsize_out, wsize_out)) indices = np.zeros((nsize, csize, hsize_out, wsize_out)) - for i in xrange(hsize_out): - for j in xrange(wsize_out): + for i in range(hsize_out): + for j in range(wsize_out): r_start = np.max((i * self.strides[0] - self.paddings[0], 0)) r_end = np.min((i * self.strides[0] + self.ksize[0] - \ self.paddings[0], hsize)) c_start = np.max((j * self.strides[1] - self.paddings[1], 0)) c_end = np.min((j * self.strides[1] + self.ksize[1] - \ self.paddings[1], wsize)) - for nidx in xrange(nsize): - for cidx in xrange(csize): + for nidx in range(nsize): + for cidx in range(csize): x_masked = pre_input[nidx, cidx, r_start:r_end, \ c_start:c_end] input[nidx, cidx, i, j] = x_masked.max() diff --git a/python/paddle/fluid/tests/unittests/test_while_op.py b/python/paddle/fluid/tests/unittests/test_while_op.py index fe8808bc044684c96fb3382836be32dac1d241f3..790e6afe5f02236b00d9c67b7b25a881e07abace 100644 --- a/python/paddle/fluid/tests/unittests/test_while_op.py +++ b/python/paddle/fluid/tests/unittests/test_while_op.py @@ -66,7 +66,7 @@ class TestWhileOp(unittest.TestCase): exe = Executor(cpu) d = [] - for i in xrange(3): + for i in range(3): d.append(numpy.random.random(size=[10]).astype('float32')) outs = exe.run(feed={'d0': d[0], diff --git a/python/paddle/fluid/tests/unittests/testsuite.py b/python/paddle/fluid/tests/unittests/testsuite.py index 910d9538b009496813f40b82d62eb2b12964a99f..c6e176ca31c57e623addd9594be81c0abdce489b 100644 --- a/python/paddle/fluid/tests/unittests/testsuite.py +++ b/python/paddle/fluid/tests/unittests/testsuite.py @@ -150,7 +150,7 @@ def append_input_output(block, op_proto, np_list, is_input, dtype): def append_loss_ops(block, output_names): - mean_inputs = map(block.var, output_names) + mean_inputs = list(map(block.var, output_names)) # for item in mean_inputs: # print(item) # print("Item", item.dtype) diff --git a/python/paddle/fluid/tests/unittests/transformer_model.py b/python/paddle/fluid/tests/unittests/transformer_model.py index d0eb3fd3724899aad39422983fd3cd0d00ff2a2d..17ab875f6a555c99bbe480a54ea2df3b1f60de2d 100644 --- a/python/paddle/fluid/tests/unittests/transformer_model.py +++ b/python/paddle/fluid/tests/unittests/transformer_model.py @@ -118,8 +118,9 @@ def multi_head_attention(queries, # FIXME(guosheng): Decouple the program desc with batch_size. return layers.reshape( x=trans_x, - shape=map(int, - [batch_size, -1, trans_x.shape[2] * trans_x.shape[3]])) + shape=list( + map(int, [batch_size, -1, trans_x.shape[2] * trans_x.shape[3] + ]))) def scaled_dot_product_attention(q, k, v, attn_bias, d_model, dropout_rate): """ diff --git a/python/paddle/fluid/trainer.py b/python/paddle/fluid/trainer.py index 64049a93cb0a267722de9cd94961b6256551330d..eed9b49ef40b591d5b6481846dab714423f57990 100644 --- a/python/paddle/fluid/trainer.py +++ b/python/paddle/fluid/trainer.py @@ -18,16 +18,15 @@ import errno import shutil import time -import core - -import data_feeder -import executor -import framework -import io +from . import core +from . import data_feeder +from . import executor +from . import framework +from . import io # optimizer is same as the parameter of Trainer.__init__. Rename it to opt_module -import optimizer as opt_module -import parallel_executor -from transpiler import distribute_transpiler +from . import optimizer as opt_module +from . import parallel_executor +from .transpiler import distribute_transpiler __all__ = [ 'Trainer', 'BeginEpochEvent', 'EndEpochEvent', 'BeginStepEvent', @@ -73,7 +72,7 @@ class BeginStepEvent(object): self.step = step_id self.fetch_metrics = True """ - If fetch_metrics is true, the metrics will be fetched at the + If fetch_metrics is true, the metrics will be fetched at the EndStepEvent. Default is True. """ @@ -614,11 +613,12 @@ def build_feed_var_list(program, feed_order): if not isinstance(feed_order, dict): raise TypeError( "The 'feed_order' should be either None, list or dict.") - if not sorted(feed_order.values()) == range(len(feed_order)): + if not sorted(feed_order.values()) == list(range(len(feed_order))): raise ValueError( "The values of 'feed_order' should be a permutation of [0, len(feed_order))" ) - sorted_pair_list = sorted(feed_order.items(), key=lambda item: item[1]) + sorted_pair_list = sorted( + list(feed_order.items()), key=lambda item: item[1]) feed_var_list = [ program.global_block().var(pair[0]) for pair in sorted_pair_list ] @@ -644,14 +644,14 @@ def save_checkpoint(executor, pserver_endpoints=None): """ This function filters out all checkpoint variables from the give - main_program and then saves these variables to the `checkpoint_dir` + main_program and then saves these variables to the `checkpoint_dir` directory. In the training precess, we generally save a checkpoint in each - iteration. So there might be a lot of checkpoints in the - `checkpoint_dir`. To avoid them taking too much disk space, the - `max_num_checkpoints` are introduced to limit the total number of - checkpoints. If the number of existing checkpints is greater than + iteration. So there might be a lot of checkpoints in the + `checkpoint_dir`. To avoid them taking too much disk space, the + `max_num_checkpoints` are introduced to limit the total number of + checkpoints. If the number of existing checkpints is greater than the `max_num_checkpoints`, oldest ones will be scroll deleted. A variable is a checkpoint variable and will be saved if it meets @@ -663,21 +663,21 @@ def save_checkpoint(executor, Args: executor(Executor): The executor to run for save checkpoint. checkpoint_dir(str): The folder where to save checkpoints. - trainer_id(int): currect trainer id, if id is equal to 0, the trainer + trainer_id(int): currect trainer id, if id is equal to 0, the trainer is chief. - trainer_args(dict|None): Current training arguments. Such as 'epoch_id' + trainer_args(dict|None): Current training arguments. Such as 'epoch_id' and 'step_id'. Defaut: None main_program(Program): The program whose checkpoint variables will be saved. - max_num_checkpoints(int): The max number of total number of existing + max_num_checkpoints(int): The max number of total number of existing checkpoints. Default: 3 lookup_table(string|None): the lookup table name, when use distribute lookup table, we can get lookup table name by DistributeTranspiler. - table_name - pserver_endpoints(list|None): the parameter server ip:port list. - when use distribute lookup table, we can get pserver_endpoints by + table_name + pserver_endpoints(list|None): the parameter server ip:port list. + when use distribute lookup table, we can get pserver_endpoints by distribute arguments. Returns: @@ -747,8 +747,8 @@ def load_checkpoint(executor, `checkpoint_dir` directory. In the training precess, we generally save a checkpoint in each - iteration. So there are more than one checkpoint in the - `checkpoint_dir` (each checkpoint has its own sub folder), use + iteration. So there are more than one checkpoint in the + `checkpoint_dir` (each checkpoint has its own sub folder), use `serial` to specify which serial of checkpoint you would like to load. @@ -819,9 +819,9 @@ def load_checkpoint(executor, def clean_checkpoint(checkpoint_dir, delete_dir=False): """ - clean the checkpoint dir, when the train exits normally, + clean the checkpoint dir, when the train exits normally, the trainer will call clean_checkpoint to delete checkpoint directory saved before. - delete_dir only works when the directory is empty, otherwise, OSError is raised. + delete_dir only works when the directory is empty, otherwise, OSError is raised. : param checkpoint_dir : param delete_dir @@ -889,7 +889,7 @@ def _load_persist_vars_without_grad(executor, def _load_lookup_table_vars(executor, dirname, program, pserver_id, table_name): """ - The parameter server will load lookup table's local file in + The parameter server will load lookup table's local file in selectedrows variable. Args: @@ -940,7 +940,7 @@ def _load_lookup_table_vars(executor, dirname, program, pserver_id, table_name): def _save_persist_vars_without_grad(executor, dirname, program): """ This function filters out all checkpoint variables from the give - program and then save these variables to a sub-folder '__model__' of + program and then save these variables to a sub-folder '__model__' of the given directory. A variable is a checkpoint variable if it meets all following @@ -969,7 +969,7 @@ def _save_persist_vars_without_grad(executor, dirname, program): # In this example, `_save_persist_vars_without_grad` function # will first filters out all checkpoint variables in the default - # main program, and then saves these variables to the folder + # main program, and then saves these variables to the folder # "./my_paddle_model/__model__". """ cur_dir = _get_model_dir(dirname) @@ -988,7 +988,7 @@ def _save_pserver_vars_by_notify(executor, dirname, lookup_table, """ This function will send checkpoint notify message from Trainer 0 to all the pservers. - The checkpoint notify message contains lookup table name, + The checkpoint notify message contains lookup table name, the absolute path on pserver to save lookup_table. Args: @@ -996,13 +996,13 @@ def _save_pserver_vars_by_notify(executor, dirname, lookup_table, dirname(str): The folder where to save checkpoints. lookup_table(string): the lookup table name, when use distribute lookup table, we can get lookup table name by DistributeTranspiler. - table_name - ps_endpoint_list(list): the parameter server ip:port list. - when use distribute lookup table, we can get ps_endpoint_list by + table_name + ps_endpoint_list(list): the parameter server ip:port list. + when use distribute lookup table, we can get ps_endpoint_list by distribute arguments. Return: None - + Examples: .. code-block:: python @@ -1013,7 +1013,7 @@ def _save_pserver_vars_by_notify(executor, dirname, lookup_table, ps_endpoints = ["127.0.0.1:6000","127.0.0.1:6001"] _save_pserver_vars_by_notify(executor=exe, - dirname=param_path, lookup_table=table_name, + dirname=param_path, lookup_table=table_name, ps_endpoint_list=ps_endpoints) """ cur_dir = _get_lookuptable_dir(dirname) @@ -1036,7 +1036,7 @@ def _save_trainer_args(dirname, trainer_id, trainer_args): cur_dir = _get_trainer_dir(dirname, trainer_id) - for name, value in trainer_args.iteritems(): + for name, value in list(trainer_args.items()): args_file = os.path.join(cur_dir, name) with open(args_file, 'w') as f: f.write(str(value)) @@ -1045,7 +1045,7 @@ def _save_trainer_args(dirname, trainer_id, trainer_args): def _load_trainer_args(checkpoint_dir, serial, trainer_id, trainer_args): """ - trainer will load some args from it's independent directory, + trainer will load some args from it's independent directory, such as epoch_id and step_id. Args: @@ -1168,10 +1168,10 @@ def _scroll_delete(dirname, max_num_checkpoints=3): serial_num = _get_dir_serial(serial) serial_map[serial_num] = serial - if len(serial_map.keys()) <= max_num_checkpoints: + if len(list(serial_map.keys())) <= max_num_checkpoints: return - serials = serial_map.keys() + serials = list(serial_map.keys()) serials.sort(reverse=True) serials = serials[max_num_checkpoints:] for serial in serials: diff --git a/python/paddle/fluid/transpiler/__init__.py b/python/paddle/fluid/transpiler/__init__.py index eae13b50398f791d4a203b72a0e96f3e87cc2a88..a8622ad54433fff40f68520955f0294e2955577e 100644 --- a/python/paddle/fluid/transpiler/__init__.py +++ b/python/paddle/fluid/transpiler/__init__.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from distribute_transpiler import DistributeTranspiler, DistributeTranspilerConfig -from inference_transpiler import InferenceTranspiler -from memory_optimization_transpiler import memory_optimize, release_memory -from ps_dispatcher import HashName, RoundRobin +from .distribute_transpiler import DistributeTranspiler, DistributeTranspilerConfig +from .inference_transpiler import InferenceTranspiler +from .memory_optimization_transpiler import memory_optimize, release_memory +from .ps_dispatcher import HashName, RoundRobin __all__ = [ "DistributeTranspiler", "InferenceTranspiler", "memory_optimize", diff --git a/python/paddle/fluid/transpiler/details/__init__.py b/python/paddle/fluid/transpiler/details/__init__.py index dc597c33849dc06cc975b245099672f64c3539d3..1bfab1f219f8a2f08a0fb5c0042d87a3ad707dd5 100644 --- a/python/paddle/fluid/transpiler/details/__init__.py +++ b/python/paddle/fluid/transpiler/details/__init__.py @@ -12,5 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -from program_utils import * -from ufind import * +from .program_utils import * +from .ufind import * diff --git a/python/paddle/fluid/transpiler/details/program_utils.py b/python/paddle/fluid/transpiler/details/program_utils.py index 2ca1d4716b103d17117ae3ee958667c3a9747cdf..76d10777f5f9ed6d27d55a640108bd036d8d8bac 100644 --- a/python/paddle/fluid/transpiler/details/program_utils.py +++ b/python/paddle/fluid/transpiler/details/program_utils.py @@ -17,8 +17,8 @@ def delete_ops(block, ops): try: start = list(block.ops).index(ops[0]) end = list(block.ops).index(ops[-1]) - [block._remove_op(start) for _ in xrange(end - start + 1)] - except Exception, e: + [block._remove_op(start) for _ in range(end - start + 1)] + except Exception as e: raise e block.program._sync_with_cpp() diff --git a/python/paddle/fluid/transpiler/distribute_transpiler.py b/python/paddle/fluid/transpiler/distribute_transpiler.py index d4d19799fdb291545117f327d2b9b2c25fbfe5f5..63422a6ec5642fbf6568371b846f3aa022a7afc6 100644 --- a/python/paddle/fluid/transpiler/distribute_transpiler.py +++ b/python/paddle/fluid/transpiler/distribute_transpiler.py @@ -28,18 +28,17 @@ Steps to transpile pserver: 5. add listen_and_serv op """ -from __future__ import print_function - import math import random import numpy as np -from ps_dispatcher import RoundRobin, HashName, PSDispatcher +from .ps_dispatcher import RoundRobin, HashName, PSDispatcher from .. import core, framework from ..framework import Program, default_main_program, \ default_startup_program, Block, \ Parameter, grad_var_name -from details import * +from .details import * +from functools import reduce LOOKUP_TABLE_TYPE = "lookup_table" LOOKUP_TABLE_GRAD_TYPE = "lookup_table_grad" @@ -102,7 +101,7 @@ def slice_variable(var_list, slice_count, min_block_size): block_size += dim1 - remains # update split_count after aligning split_count = int(math.ceil(var_numel / float(block_size))) - for block_id in xrange(split_count): + for block_id in range(split_count): curr_block_size = min(block_size, var_numel - ( (block_id) * block_size)) block = VarBlock(var.name, block_id, curr_block_size) @@ -117,7 +116,7 @@ class DistributeTranspilerConfig(object): try to choose the best method to balance loads for pservers. min_block_size (int): Minimum splitted element number in block. According:https://github.com/PaddlePaddle/Paddle/issues/8638#issuecomment-369912156 - We can use bandwidth effiently when data size is larger than 2MB.If you + We can use bandwidth effiently when data size is larger than 2MB.If you want to change it, please be sure you see the slice_variable function. """ @@ -218,7 +217,7 @@ class DistributeTranspiler(object): # fc_w@GRAD_trainer_0, fc_w@GRAD_trainer_1 --> pserver1 # fc_b@GRAD_trainer_0, fc_b@GRAD_trainer_1 --> pserver2 # shuffle the map will avoid the uneven distribution above - grad_var_mapping_items = self.grad_var_mapping.items() + grad_var_mapping_items = list(self.grad_var_mapping.items()) if not self.config.slice_var_up: random.seed(self.trainer_num) random.shuffle(grad_var_mapping_items) @@ -278,7 +277,7 @@ class DistributeTranspiler(object): self.param_grad_ep_mapping[ep]["grads"].append(send_vars[i]) # step4: Concat the parameters splits together after recv. - for varname, splited_var in self.param_var_mapping.iteritems(): + for varname, splited_var in list(self.param_var_mapping.items()): eps = [] for var in splited_var: index = [v.name for v in recv_vars].index(var.name) @@ -293,16 +292,17 @@ class DistributeTranspiler(object): RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE }) - program.global_block().append_op( - type="fetch_barrier", - inputs={}, - outputs={}, - attrs={ - "endpoints": pserver_endpoints, - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE - }) + if self.sync_mode: + program.global_block().append_op( + type="fetch_barrier", + inputs={}, + outputs={}, + attrs={ + "endpoints": pserver_endpoints, + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE + }) - for varname, splited_var in self.param_var_mapping.iteritems(): + for varname, splited_var in list(self.param_var_mapping.items()): if len(splited_var) <= 1: continue orig_param = program.global_block().vars[varname] @@ -372,7 +372,7 @@ class DistributeTranspiler(object): dtype=v.dtype, shape=v.shape) if self.sync_mode and self.trainer_num > 1: - for trainer_id in xrange(self.trainer_num): + for trainer_id in range(self.trainer_num): var = pserver_program.global_block().create_var( name="%s.trainer_%d" % (orig_var_name, trainer_id), persistable=False, @@ -462,7 +462,7 @@ class DistributeTranspiler(object): per_opt_block = pserver_program.create_block(pre_block_idx) optimize_blocks.append(per_opt_block) # append grad merging ops before clip and weight decay - # cases may like: + # cases may like: # L2Decay op -> clip op -> optimize for _, op in enumerate(self.optimize_ops): # find the origin @GRAD var before clipping @@ -559,7 +559,7 @@ class DistributeTranspiler(object): # 1. create vars in pserver program to startup program pserver_vars = pserver_program.global_block().vars created_var_map = dict() - for _, var in pserver_vars.iteritems(): + for _, var in list(pserver_vars.items()): tmpvar = s_prog.global_block()._clone_variable(var) created_var_map[var.name] = tmpvar @@ -896,8 +896,6 @@ class DistributeTranspiler(object): self.table_name ][0] table_opt_block = pserver_program.create_block(pre_block_idx) - # only support sgd now - assert table_opt_op.type == "sgd" if self.sync_mode: # create grad vars in pserver program @@ -937,11 +935,12 @@ class DistributeTranspiler(object): "LearningRate": [lr_var] } outputs = {"ParamOut": [param_var]} - table_opt_block.append_op( - type=table_opt_op.type, - inputs=inputs, - outputs=outputs, - attrs=table_opt_op.attrs) + # only support sgd now + import logging + logging.warn( + "distribute lookup table only support sgd optimizer, change it's optimizer to sgd instead of " + + table_opt_op.type) + table_opt_block.append_op(type="sgd", inputs=inputs, outputs=outputs) # add table parameter gradient and it's block id to grad_to_block_id grad_to_block_id.append(grad_var.name + ":" + str(table_opt_block.idx)) @@ -992,11 +991,11 @@ class DistributeTranspiler(object): var_mapping = dict() for block_str in block_list: varname, offset, size = block_str.split(":") - if not block_map.has_key(varname): + if varname not in block_map: block_map[varname] = [] - block_map[varname].append((long(offset), long(size))) + block_map[varname].append((int(offset), int(size))) - for varname, splited in block_map.iteritems(): + for varname, splited in list(block_map.items()): orig_var = program.global_block().var(varname) if len(splited) == 1: if self.sync_mode and add_trainer_suffix: @@ -1159,7 +1158,7 @@ class DistributeTranspiler(object): grad_to_block_id.append(merged_var.name + ":" + str(optimize_block.idx)) if self.sync_mode and self.trainer_num > 1: vars2merge = [] - for i in xrange(self.trainer_num): + for i in range(self.trainer_num): per_trainer_name = "%s.trainer_%d" % \ (merged_var_name, i) vars2merge.append(pserver_block.vars[per_trainer_name]) @@ -1207,7 +1206,7 @@ class DistributeTranspiler(object): # learning rate variable has already be created by non-optimize op, # don't create it once again. lr_varname = opt_op.input(key)[0] - if pserver_block.vars.has_key(lr_varname): + if lr_varname in pserver_block.vars: new_inputs[key] = pserver_block.vars[opt_op.input(key)[0]] else: origin_var = origin_program.global_block().vars[lr_varname] @@ -1247,7 +1246,9 @@ class DistributeTranspiler(object): def _is_splited_grad_var(self, var, var_dict): grad_block = None - for _, g in var_dict.iteritems(): + # TODO(minqiyang): replace these items() with six.iteritems() to + # improve memory + for _, g in list(var_dict.items()): if self._orig_varname(g.name) == self._orig_varname(var.name): if g.name.find(".trainer_") == -1: grad_block = g @@ -1257,7 +1258,7 @@ class DistributeTranspiler(object): def _clone_lr_op(self, program, block, op): inputs = self._get_input_map_from_op( self.origin_program.global_block().vars, op) - for key, varlist in inputs.iteritems(): + for key, varlist in list(inputs.items()): if not isinstance(varlist, list): varlist = [varlist] for var in varlist: @@ -1266,7 +1267,7 @@ class DistributeTranspiler(object): outputs = self._get_output_map_from_op( self.origin_program.global_block().vars, op) - for key, varlist in outputs.iteritems(): + for key, varlist in list(outputs.items()): if not isinstance(varlist, list): varlist = [varlist] for var in varlist: @@ -1281,7 +1282,7 @@ class DistributeTranspiler(object): # Append the ops for parameters that do not need to be optimized/updated inputs = self._get_input_map_from_op( self.origin_program.global_block().vars, opt_op) - for key, varlist in inputs.iteritems(): + for key, varlist in list(inputs.items()): if not isinstance(varlist, list): varlist = [varlist] for var in varlist: @@ -1291,7 +1292,7 @@ class DistributeTranspiler(object): var, program.global_block().vars) if grad_block: inputs[key] = grad_block - elif not program.global_block().vars.has_key(var.name): + elif var.name not in program.global_block().vars: program.global_block().create_var( name=var.name, persistable=var.persistable, @@ -1300,7 +1301,7 @@ class DistributeTranspiler(object): outputs = self._get_output_map_from_op( self.origin_program.global_block().vars, opt_op) - for key, varlist in outputs.iteritems(): + for key, varlist in list(outputs.items()): if not isinstance(varlist, list): varlist = [varlist] for var in varlist: @@ -1308,7 +1309,7 @@ class DistributeTranspiler(object): var, program.global_block().vars) if grad_block: outputs[key] = grad_block - elif not program.global_block().vars.has_key(var.name): + elif var.name not in program.global_block().vars: program.global_block()._clone_variable(var) return optimize_block.append_op( @@ -1329,8 +1330,8 @@ class DistributeTranspiler(object): def _create_ufind(self, optimize_ops): # Create a unit find data struct by optimize ops ufind = UnionFind(optimize_ops) - for i in xrange(len(optimize_ops)): - for j in xrange(i, len(optimize_ops)): + for i in range(len(optimize_ops)): + for j in range(i, len(optimize_ops)): op1 = optimize_ops[i] op2 = optimize_ops[j] if self._is_op_connected(op1, op2): diff --git a/python/paddle/fluid/transpiler/inference_transpiler.py b/python/paddle/fluid/transpiler/inference_transpiler.py index f1905f08787da7a58a41d840ea68fb6c07f4028f..142fa5c31d2c558e482001da73fa26d8396c3967 100644 --- a/python/paddle/fluid/transpiler/inference_transpiler.py +++ b/python/paddle/fluid/transpiler/inference_transpiler.py @@ -305,6 +305,6 @@ class InferenceTranspiler(object): args += current_op.output_arg_names args = list(set(args)) # unique the input and output arguments - for var in self.block.vars.keys(): + for var in list(self.block.vars.keys()): if var not in args: self.block._remove_var(var) diff --git a/python/paddle/fluid/transpiler/memory_optimization_transpiler.py b/python/paddle/fluid/transpiler/memory_optimization_transpiler.py index 0ca5cf813b51e200da5edd5830767ad9457acec2..20ba7ed2b0b9df0d0432727ee1f69f61533c402e 100644 --- a/python/paddle/fluid/transpiler/memory_optimization_transpiler.py +++ b/python/paddle/fluid/transpiler/memory_optimization_transpiler.py @@ -16,6 +16,8 @@ from collections import defaultdict from .. import core from ..framework import Program, default_main_program, Parameter from ..backward import _rename_arg_ +from functools import reduce +from six.moves import range dtype_to_size = { core.VarDesc.VarType.FP16: 2, @@ -107,7 +109,7 @@ class ControlFlowGraph(object): # Repeatedly apply liveness updates until the algorithm stablize # on a complete set live input vars and live output vars. while True: - for i in reversed(range(self.op_size)): + for i in reversed(list(range(self.op_size))): live_in[i] = set(self._live_in[i]) live_out[i] = set(self._live_out[i]) for s in self._successors[i]: @@ -172,9 +174,10 @@ class ControlFlowGraph(object): is_forward = i < self._forward_num in_diff, out_diff = self._get_diff(self._live_in[i], self._live_out[i]) - can_optimize = filter( - lambda x: self._check_var_validity(block_desc, x, is_forward), - in_diff) + can_optimize = [ + x for x in in_diff + if self._check_var_validity(block_desc, x, is_forward) + ] if can_optimize: index = i + fwd_id + 1 if is_forward else i - self._forward_num + bwd_id + 1 delete_op = block_desc._insert_op(index) @@ -213,9 +216,10 @@ class ControlFlowGraph(object): block_desc = op.block() is_forward = i < self._forward_num if self.pool: - defs_can_optimize = filter( - lambda x: self._check_var_validity(block_desc, x, is_forward), - self._defs[i]) + defs_can_optimize = [ + x for x in self._defs[i] + if self._check_var_validity(block_desc, x, is_forward) + ] out_pair = [ (x, self._find_var(block_desc, x, is_forward).shape()) for x in defs_can_optimize @@ -261,9 +265,10 @@ class ControlFlowGraph(object): break in_diff, _ = self._get_diff(self._live_in[i], self._live_out[i]) - can_optimize = filter( - lambda x: self._check_var_validity(block_desc, x, is_forward), - in_diff) + can_optimize = [ + x for x in in_diff + if self._check_var_validity(block_desc, x, is_forward) + ] if can_optimize: for var_name in can_optimize: self.pool.append((var_name, self._find_var( diff --git a/python/paddle/fluid/unique_name.py b/python/paddle/fluid/unique_name.py index 776619cd36722e338a9fdd5e13bceeaf3724de2c..b125eba4f83c588fa2fa81a357604a7d8592ea80 100644 --- a/python/paddle/fluid/unique_name.py +++ b/python/paddle/fluid/unique_name.py @@ -14,6 +14,7 @@ import collections import contextlib +import six import sys __all__ = ['generate', 'switch', 'guard'] @@ -67,8 +68,10 @@ def switch(new_generator=None): @contextlib.contextmanager def guard(new_generator=None): - if isinstance(new_generator, basestring): + if isinstance(new_generator, six.string_types): new_generator = UniqueNameGenerator(new_generator) + elif isinstance(new_generator, six.binary_type): + new_generator = UniqueNameGenerator(new_generator.decode()) old = switch(new_generator) yield switch(old) diff --git a/python/paddle/reader/creator.py b/python/paddle/reader/creator.py index 4c905d959fad4e8c1a8826ce8dc60c5fa834514d..c861020225fb6fe0a29653363c2151b20dc8f578 100644 --- a/python/paddle/reader/creator.py +++ b/python/paddle/reader/creator.py @@ -67,11 +67,14 @@ def recordio(paths, buf_size=100): import recordio as rec import paddle.reader.decorator as dec - import cPickle as pickle + import six + import six.moves.cPickle as pickle def reader(): - if isinstance(paths, basestring): + if isinstance(paths, six.string_types): path = paths + elif isinstance(paths, six.binary_type): + path = paths.decode() else: path = ",".join(paths) f = rec.reader(path) diff --git a/python/paddle/reader/decorator.py b/python/paddle/reader/decorator.py index 4b1fe94222d35f8c0e4e4cccc364227a3f9509d0..ce410e61b92e7d3f32fa5bfeb415e4b6c5fa9df6 100644 --- a/python/paddle/reader/decorator.py +++ b/python/paddle/reader/decorator.py @@ -21,6 +21,9 @@ from threading import Thread import subprocess from six.moves.queue import Queue +from six.moves import zip_longest +from six.moves import map +from six.moves import zip import itertools import random import zlib @@ -42,7 +45,7 @@ def map_readers(func, *readers): rs = [] for r in readers: rs.append(r()) - for e in itertools.imap(func, *rs): + for e in map(func, *rs): yield e return reader @@ -148,16 +151,16 @@ def compose(*readers, **kwargs): for r in readers: rs.append(r()) if not check_alignment: - for outputs in itertools.izip(*rs): - yield sum(map(make_tuple, outputs), ()) + for outputs in zip(*rs): + yield sum(list(map(make_tuple, outputs)), ()) else: - for outputs in itertools.izip_longest(*rs): + for outputs in zip_longest(*rs): for o in outputs: if o is None: # None will be not be present if compose is aligned raise ComposeNotAligned( "outputs of readers are not aligned.") - yield sum(map(make_tuple, outputs), ()) + yield sum(list(map(make_tuple, outputs)), ()) return reader @@ -306,7 +309,7 @@ def xmap_readers(mapper, reader, process_num, buffer_size, order=False): args = (in_queue, out_queue, mapper, out_order) if order else ( in_queue, out_queue, mapper) workers = [] - for i in xrange(process_num): + for i in range(process_num): worker = Thread(target=target, args=args) worker.daemon = True workers.append(worker) diff --git a/python/paddle/reader/tests/decorator_test.py b/python/paddle/reader/tests/decorator_test.py index bee24d3b6579db5e99ec66931df201fdf9e1af07..537df489b9738864933b3a7922d178701db3d19f 100644 --- a/python/paddle/reader/tests/decorator_test.py +++ b/python/paddle/reader/tests/decorator_test.py @@ -136,7 +136,7 @@ class TestXmap(unittest.TestCase): reader = paddle.reader.xmap_readers(mapper, reader_creator_10(0), tNum, size, order) - for n in xrange(3): + for n in range(3): result = [] for i in reader(): result.append(i) @@ -156,7 +156,7 @@ class TestPipeReader(unittest.TestCase): import tempfile - records = [str(i) for i in xrange(5)] + records = [str(i) for i in range(5)] temp = tempfile.NamedTemporaryFile() try: with open(temp.name, 'w') as f: diff --git a/python/paddle/trainer/PyDataProviderWrapper.py b/python/paddle/trainer/PyDataProviderWrapper.py index 6af250772859811b3c48434ab005e50b435dd320..374976db9f17ad9b1fd33c5d4adf77155336d100 100644 --- a/python/paddle/trainer/PyDataProviderWrapper.py +++ b/python/paddle/trainer/PyDataProviderWrapper.py @@ -42,7 +42,7 @@ except ImportError: try: import cPickle as pickle except ImportError: - import pickle + import six.moves.cPickle as pickle import io diff --git a/python/paddle/trainer_config_helpers/data_sources.py b/python/paddle/trainer_config_helpers/data_sources.py index ab9a2562dcccb394c0b24741ceeb10061e40cb9a..a2a32d848cbc4200397e6a12a3662419102da0a9 100644 --- a/python/paddle/trainer_config_helpers/data_sources.py +++ b/python/paddle/trainer_config_helpers/data_sources.py @@ -20,7 +20,7 @@ from .utils import deprecated try: import cPickle as pickle except ImportError: - import pickle + import six.moves.cPickle as pickle __all__ = ['define_py_data_sources2'] diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index d9787ef42a31b8dfd1836e7a01d5664049cc66b5..ee34c157334b533b9c330b8103424964d7df510b 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -28,7 +28,7 @@ from .default_decorators import * try: import cPickle as pickle except ImportError: - import pickle + import six.moves.cPickle as pickle import copy __all__ = [ diff --git a/python/paddle/v2/dataset/conll05.py b/python/paddle/v2/dataset/conll05.py index 0d544efac9cd20157f87b5cd3b68f97ab5ed2dbc..8312900dc43fdd64cc1a205ab846b6f1deaecf5d 100644 --- a/python/paddle/v2/dataset/conll05.py +++ b/python/paddle/v2/dataset/conll05.py @@ -29,13 +29,13 @@ __all__ = ['test, get_dict', 'get_embedding', 'convert'] DATA_URL = 'http://www.cs.upc.edu/~srlconll/conll05st-tests.tar.gz' DATA_MD5 = '387719152ae52d60422c016e92a742fc' -WORDDICT_URL = 'http://paddlepaddle.bj.bcebos.com/demo/srl_dict_and_embedding/wordDict.txt' +WORDDICT_URL = 'http://paddlemodels.bj.bcebos.com/conll05st%2FwordDict.txt' WORDDICT_MD5 = 'ea7fb7d4c75cc6254716f0177a506baa' -VERBDICT_URL = 'http://paddlepaddle.bj.bcebos.com/demo/srl_dict_and_embedding/verbDict.txt' +VERBDICT_URL = 'http://paddlemodels.bj.bcebos.com/conll05st%2FverbDict.txt' VERBDICT_MD5 = '0d2977293bbb6cbefab5b0f97db1e77c' -TRGDICT_URL = 'http://paddlepaddle.bj.bcebos.com/demo/srl_dict_and_embedding/targetDict.txt' +TRGDICT_URL = 'http://paddlemodels.bj.bcebos.com/conll05st%2FtargetDict.txt' TRGDICT_MD5 = 'd8c7f03ceb5fc2e5a0fa7503a4353751' -EMB_URL = 'http://paddlepaddle.bj.bcebos.com/demo/srl_dict_and_embedding/emb' +EMB_URL = 'http://paddlemodels.bj.bcebos.com/conll05st%2Femb' EMB_MD5 = 'bf436eb0faa1f6f9103017f8be57cdb7' UNK_IDX = 0 diff --git a/python/paddle/v2/dataset/wmt14.py b/python/paddle/v2/dataset/wmt14.py index 5104e29051e4480f3a7eb18421f1b519841b009b..1ec210f265049c8b62cd99cd218f25a9f846ef43 100644 --- a/python/paddle/v2/dataset/wmt14.py +++ b/python/paddle/v2/dataset/wmt14.py @@ -41,7 +41,7 @@ URL_TRAIN = ('http://paddlepaddle.cdn.bcebos.com/demo/' 'wmt_shrinked_data/wmt14.tgz') MD5_TRAIN = '0791583d57d5beb693b9414c5b36798c' # BLEU of this trained model is 26.92 -URL_MODEL = 'http://paddlepaddle.bj.bcebos.com/demo/wmt_14/wmt14_model.tar.gz' +URL_MODEL = 'http://paddlemodels.bj.bcebos.com/wmt%2Fwmt14.tgz' MD5_MODEL = '0cb4a5366189b6acba876491c8724fa3' START = "" diff --git a/tools/test_runner.py b/tools/test_runner.py index 9dc750b89058cd73355a2f7984d577252c03526d..2d6a3cf8a97a3bbaa69b66f5343c54b750624329 100644 --- a/tools/test_runner.py +++ b/tools/test_runner.py @@ -12,19 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function import unittest import os import sys import paddle.fluid as fluid import importlib -import cStringIO +from six.moves import cStringIO def main(): sys.path.append(os.getcwd()) some_test_failed = False for module_name in sys.argv[1:]: - buffer = cStringIO.StringIO() + buffer = cStringIO() main = fluid.Program() startup = fluid.Program() scope = fluid.core.Scope() @@ -37,8 +38,11 @@ def main(): res = unittest.TextTestRunner(stream=buffer).run(tests) if not res.wasSuccessful(): some_test_failed = True - print >> sys.stderr, module_name, 'failed\n', buffer.getvalue( - ) + print( + module_name, + 'failed\n', + buffer.getvalue(), + file=sys.stderr) if some_test_failed: exit(1)