diff --git a/.gitignore b/.gitignore index 9e3a0b499f9f42856429f3a42bef313ea3df3699..b92bb9cc129659fa502b4a9b55548992412e5429 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ python/paddle/v2/fluid/tests/book/image_classification_resnet.inference.model/ python/paddle/v2/fluid/tests/book/image_classification_vgg.inference.model/ python/paddle/v2/fluid/tests/book/label_semantic_roles.inference.model/ *.DS_Store +*.vs build/ build_doc/ *.user @@ -15,6 +16,7 @@ build_doc/ .cproject .pydevproject .settings/ +CMakeSettings.json Makefile .test_env/ third_party/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 920c20d6f813c14df8e02593b1c4a5a13cc11ef0..68447727118a91a2a8c0d06404353c7ccb734c6d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -204,11 +204,12 @@ include(external/snappy) # download snappy include(external/snappystream) include(external/threadpool) -set(WITH_ANAKIN OFF CACHE STRING "Disable Anakin first, will add it later." FORCE) if(WITH_GPU) include(cuda) include(tensorrt) include(external/anakin) +elseif() + set(WITH_ANAKIN OFF CACHE STRING "Anakin is used in GPU only now." FORCE) endif() include(cudnn) # set cudnn libraries, must before configure diff --git a/cmake/configure.cmake b/cmake/configure.cmake index c35096e09b5685ee30f20648e7dd461f71e0b1c4..d14162e0a662afe63152bfc2132e5dfd54f5a86c 100644 --- a/cmake/configure.cmake +++ b/cmake/configure.cmake @@ -56,6 +56,10 @@ if(NOT CMAKE_CROSSCOMPILING) set(SIMD_FLAG ${SSE3_FLAG}) endif() endif() +if(UNIX AND NOT APPLE) + # except apple from nix*Os family + set(LINUX TRUE) +endif(UNIX AND NOT APPLE) if(NOT WITH_GOLANG) add_definitions(-DPADDLE_WITHOUT_GOLANG) @@ -104,6 +108,10 @@ if(WITH_GPU) if(${CUDNN_MAJOR_VERSION} VERSION_LESS 7) message(FATAL_ERROR "Anakin needs CUDNN >= 7.0 to compile") endif() + set(ENV{CUDNN_INCLUDE_DIR} ${CUDNN_INCLUDE_DIR}) + set(ENV{CUDNN_LIBRARY} ${CUDNN_LIBRARY}) + message(STATUS "cudnn include header is ${CUDNN_INCLUDE_DIR}/cudnn.h") + message(STATUS "cudnn library is ${CUDNN_LIBRARY}") endif() elseif(WITH_AMD_GPU) add_definitions(-DPADDLE_WITH_HIP) diff --git a/cmake/external/anakin.cmake b/cmake/external/anakin.cmake index 403873a51017b9bb7c888bed77d89ca9b15b68d2..855897394a602736f3a4fa1132c634f8cb537468 100644 --- a/cmake/external/anakin.cmake +++ b/cmake/external/anakin.cmake @@ -35,9 +35,10 @@ set(ANAKIN_COMPILE_EXTRA_FLAGS ExternalProject_Add( extern_anakin ${EXTERNAL_PROJECT_LOG_ARGS} - # TODO(luotao): use PaddlePaddle/Anakin later + DEPENDS ${MKLML_PROJECT} + # Anakin codes error on Intel(R) Xeon(R) Gold 5117 CPU, temporary do not compile avx512 related code. GIT_REPOSITORY "https://github.com/luotao1/Anakin" - GIT_TAG "3957ae9263eaa0b1986758dac60a88852afb09be" + GIT_TAG "bcf17aabe7921ceb7bce591244b4f9dce7dba5c8" PREFIX ${ANAKIN_SOURCE_DIR} UPDATE_COMMAND "" CMAKE_ARGS -DUSE_GPU_PLACE=YES diff --git a/doc/fluid/new_docs/advanced_usage/deploy/native_infer.rst b/doc/fluid/new_docs/advanced_usage/deploy/native_infer.rst index e1eee3f818796e895362caab10846cf59b557162..3571f81326a9f9ae31a8327c3e288e601f248e4b 100644 --- a/doc/fluid/new_docs/advanced_usage/deploy/native_infer.rst +++ b/doc/fluid/new_docs/advanced_usage/deploy/native_infer.rst @@ -4,7 +4,7 @@ Paddle 预测 API 为了更简单方便的预测部署，Fluid 提供了一套高层 API 用来隐藏底层不同的优化实现。 -`预测库相关代码 `__ +`预测库相关代码 `__ 包括 - 头文件 ``paddle_inference_api.h`` 定义了所有的接口 @@ -104,5 +104,5 @@ engine ------------ - `inference - demos `__ -- `复杂单线程/多线程例子 `__ + demos `__ +- `复杂单线程/多线程例子 `__ diff --git a/paddle/fluid/API.spec b/paddle/fluid/API.spec index 075634a306344f9f391ce7c5b4e2a0d2a8fdd037..a3606f386cee534368631c0315eea052f980f9cf 100644 --- a/paddle/fluid/API.spec +++ b/paddle/fluid/API.spec @@ -55,7 +55,7 @@ paddle.fluid.Inferencer.__init__ ArgSpec(args=['self', 'infer_func', 'param_path paddle.fluid.Inferencer.infer ArgSpec(args=['self', 'inputs', 'return_numpy'], varargs=None, keywords=None, defaults=(True,)) paddle.fluid.DistributeTranspiler.__init__ ArgSpec(args=['self', 'config'], varargs=None, keywords=None, defaults=(None,)) paddle.fluid.DistributeTranspiler.get_pserver_program ArgSpec(args=['self', 'endpoint'], varargs=None, keywords=None, defaults=None) -paddle.fluid.DistributeTranspiler.get_startup_program ArgSpec(args=['self', 'endpoint', 'pserver_program'], varargs=None, keywords=None, defaults=None) +paddle.fluid.DistributeTranspiler.get_startup_program ArgSpec(args=['self', 'endpoint', 'pserver_program', 'startup_program'], varargs=None, keywords=None, defaults=(None,)) paddle.fluid.DistributeTranspiler.get_trainer_program ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None) paddle.fluid.DistributeTranspiler.transpile ArgSpec(args=['self', 'trainer_id', 'program', 'pservers', 'trainers', 'sync_mode'], varargs=None, keywords=None, defaults=(None, '127.0.0.1:6174', 1, True)) paddle.fluid.InferenceTranspiler.__init__ @@ -155,10 +155,12 @@ paddle.fluid.layers.resize_bilinear ArgSpec(args=['input', 'out_shape', 'scale', paddle.fluid.layers.gather ArgSpec(args=['input', 'index'], varargs=None, keywords=None, defaults=None) paddle.fluid.layers.random_crop ArgSpec(args=['x', 'shape', 'seed'], varargs=None, keywords=None, defaults=(None,)) paddle.fluid.layers.mean_iou ArgSpec(args=['input', 'label', 'num_classes'], varargs=None, keywords=None, defaults=None) -paddle.fluid.layers.relu ArgSpec(args=['x'], varargs=None, keywords=None, defaults=None) -paddle.fluid.layers.log ArgSpec(args=['x'], varargs=None, keywords=None, defaults=None) +paddle.fluid.layers.relu ArgSpec(args=['x', 'name'], varargs=None, keywords=None, defaults=(None,)) +paddle.fluid.layers.log ArgSpec(args=['x', 'name'], varargs=None, keywords=None, defaults=(None,)) paddle.fluid.layers.crop ArgSpec(args=['x', 'shape', 'offsets', 'name'], varargs=None, keywords=None, defaults=(None, None, None)) paddle.fluid.layers.rank_loss ArgSpec(args=['label', 'left', 'right', 'name'], varargs=None, keywords=None, defaults=(None,)) +paddle.fluid.layers.prelu ArgSpec(args=['x', 'mode', 'param_attr', 'name'], varargs=None, keywords=None, defaults=(None, None)) +paddle.fluid.layers.flatten ArgSpec(args=['x', 'axis', 'name'], varargs=None, keywords=None, defaults=(1, None)) paddle.fluid.layers.data ArgSpec(args=['name', 'shape', 'append_batch_size', 'dtype', 'lod_level', 'type', 'stop_gradient'], varargs=None, keywords=None, defaults=(True, 'float32', 0, VarType.LOD_TENSOR, True)) paddle.fluid.layers.open_recordio_file ArgSpec(args=['filename', 'shapes', 'lod_levels', 'dtypes', 'pass_num', 'for_parallel'], varargs=None, keywords=None, defaults=(1, True)) paddle.fluid.layers.open_files ArgSpec(args=['filenames', 'shapes', 'lod_levels', 'dtypes', 'thread_num', 'buffer_size', 'pass_num', 'is_test'], varargs=None, keywords=None, defaults=(None, None, 1, None)) @@ -327,7 +329,7 @@ paddle.fluid.contrib.BeamSearchDecoder.update_array ArgSpec(args=['self', 'array paddle.fluid.contrib.memory_usage ArgSpec(args=['program', 'batch_size'], varargs=None, keywords=None, defaults=None) paddle.fluid.transpiler.DistributeTranspiler.__init__ ArgSpec(args=['self', 'config'], varargs=None, keywords=None, defaults=(None,)) paddle.fluid.transpiler.DistributeTranspiler.get_pserver_program ArgSpec(args=['self', 'endpoint'], varargs=None, keywords=None, defaults=None) -paddle.fluid.transpiler.DistributeTranspiler.get_startup_program ArgSpec(args=['self', 'endpoint', 'pserver_program'], varargs=None, keywords=None, defaults=None) +paddle.fluid.transpiler.DistributeTranspiler.get_startup_program ArgSpec(args=['self', 'endpoint', 'pserver_program', 'startup_program'], varargs=None, keywords=None, defaults=(None,)) paddle.fluid.transpiler.DistributeTranspiler.get_trainer_program ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None) paddle.fluid.transpiler.DistributeTranspiler.transpile ArgSpec(args=['self', 'trainer_id', 'program', 'pservers', 'trainers', 'sync_mode'], varargs=None, keywords=None, defaults=(None, '127.0.0.1:6174', 1, True)) paddle.fluid.transpiler.InferenceTranspiler.__init__ diff --git a/paddle/fluid/framework/attribute.h b/paddle/fluid/framework/attribute.h index 8428bf8e3392f68c9d1e2553f4d017cb620bb9f3..14ca3e96209ed17f12e87fda8506806514698977 100644 --- a/paddle/fluid/framework/attribute.h +++ b/paddle/fluid/framework/attribute.h @@ -128,7 +128,8 @@ struct ExtractAttribute { attr_value = &boost::get(attr); } catch (boost::bad_get& bad_get) { PADDLE_THROW("Cannot get attribute %s by type %s, its type is %s", - attr_name_, typeid(T).name(), attr.type().name()); + attr_name_, paddle::platform::demangle(typeid(T).name()), + paddle::platform::demangle(attr.type().name())); } return attr_value; } @@ -160,7 +161,7 @@ struct ExtractAttribute { attr_value = &boost::get(attr); } catch (boost::bad_get& bad_get) { PADDLE_THROW("Cannot get attribute %s by type bool, its type is %s", - attr_name_, attr.type().name()); + attr_name_, paddle::platform::demangle(attr.type().name())); } return attr_value; } @@ -186,7 +187,7 @@ struct ExtractAttribute { attr_value = &boost::get(attr); } catch (boost::bad_get& bad_get) { PADDLE_THROW("Cannot get attribute %s by type int64_t, its type is %s", - attr_name_, attr.type().name()); + attr_name_, paddle::platform::demangle(attr.type().name())); } return attr_value; } diff --git a/paddle/fluid/framework/ir/CMakeLists.txt b/paddle/fluid/framework/ir/CMakeLists.txt index bf7d76a8a6e173e648cea5aaba9b7202d787173b..923a7083d4f30b646bbab03d79992b275aa2b403 100644 --- a/paddle/fluid/framework/ir/CMakeLists.txt +++ b/paddle/fluid/framework/ir/CMakeLists.txt @@ -3,7 +3,10 @@ cc_library(graph SRCS graph.cc DEPS node) cc_library(graph_helper SRCS graph_helper.cc DEPS graph) cc_library(pass SRCS pass.cc DEPS graph node graph_helper) cc_library(graph_viz_pass SRCS graph_viz_pass.cc DEPS graph pass graph_helper) +cc_library(graph_traits SRCS graph_traits.cc DEPS graph) +cc_library(graph_pattern_detecter SRCS graph_pattern_detecter.cc DEPS graph graph_helper graph_traits) cc_test(pass_test SRCS pass_test.cc DEPS graph pass graph_helper) cc_test(graph_test SRCS graph_test.cc DEPS graph graph_helper op_registry) cc_test(graph_helper_test SRCS graph_helper_test.cc DEPS graph graph_helper op_registry) +cc_test(test_graph_pattern_detecter SRCS graph_pattern_detecter_tester.cc DEPS graph_pattern_detecter) diff --git a/paddle/fluid/framework/ir/graph_pattern_detecter.cc b/paddle/fluid/framework/ir/graph_pattern_detecter.cc new file mode 100644 index 0000000000000000000000000000000000000000..dcc4382792d6a21dc10c37d2173d8cf1989219a5 --- /dev/null +++ b/paddle/fluid/framework/ir/graph_pattern_detecter.cc @@ -0,0 +1,187 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +#include "paddle/fluid/framework/ir/graph_helper.h" +#include "paddle/fluid/framework/ir/graph_pattern_detecter.h" +#include "paddle/fluid/framework/ir/graph_traits.h" +#include "paddle/fluid/platform/enforce.h" + +namespace paddle { +namespace framework { +namespace ir { + +PDNode* PDPattern::NewNode(PDNode::teller_t&& teller, const std::string& name) { + nodes_.emplace_back(new PDNode(std::move(teller), name)); + auto* cur = nodes_.back().get(); + return cur; +} + +void PDPattern::AddEdge(PDNode* a, PDNode* b) { + PADDLE_ENFORCE(a); + PADDLE_ENFORCE(b); + PADDLE_ENFORCE(a != b, "can't connect to the same nodes."); + edges_.emplace_back(a, b); +} + +void GraphPatternDetecter::operator()(Graph* graph, + GraphPatternDetecter::handle_t handler) { + if (!MarkPDNodesInGraph(*graph)) return; + auto subgraphs = DetectPatterns(); + UniquePatterns(&subgraphs); + RemoveOverlappedMatch(&subgraphs); + + for (auto& g : subgraphs) { + handler(g, graph); + } +} + +bool GraphPatternDetecter::MarkPDNodesInGraph(const ir::Graph& graph) { + if (graph.Nodes().empty()) return false; + + for (auto& node : GraphTraits::DFS(graph)) { + for (const auto& pdnode : pattern_.nodes()) { + if (pdnode->Tell(&node)) { + pdnodes2nodes_[pdnode.get()].insert(&node); + } + } + } + return !pdnodes2nodes_.empty(); +} + +struct HitGroup { + std::unordered_map roles; + + bool Match(Node* node, PDNode* pat) { + return !roles.count(pat) || roles.at(pat) == node; + } + + void Register(Node* node, PDNode* pat) { roles[pat] = node; } +}; + +// Tell whether Node a links to b. +bool IsNodesLink(Node* a, Node* b) { + for (auto* node : a->outputs) { + if (b == node) { + return true; + } + } + return false; +} + +std::vector +GraphPatternDetecter::DetectPatterns() { + // Init empty subgraphs. + std::vector result; + std::vector init_groups; + PADDLE_ENFORCE(!pattern_.edges().empty(), "At least one edge is needed"); + auto* first_pnode = pattern_.edges().front().first; + if (!pdnodes2nodes_.count(first_pnode)) return result; + for (auto* node : pdnodes2nodes_[first_pnode]) { + HitGroup group; + group.roles[first_pnode] = node; + init_groups.emplace_back(group); + } + + int step = 0; + std::array, 2> bi_records; + bi_records[0] = std::move(init_groups); + + // Extend a PDNode to subgraphs by deducing the connection relations defined + // in edges of PDNodes. + for (const auto& edge : pattern_.edges()) { + // Each role has two PDNodes, which indicates two roles. + // Detect two Nodes that can match these two roles and they are connected. + auto& pre_groups = bi_records[step % 2]; + auto& cur_groups = bi_records[1 - (step++ % 2)]; + cur_groups.clear(); + // source -> target + for (Node* source : pdnodes2nodes_[edge.first]) { + for (Node* target : pdnodes2nodes_[edge.second]) { + // TODO(Superjomn) add some prune strategies. + for (const auto& group : pre_groups) { + HitGroup new_group = group; + if (IsNodesLink(source, target) && + new_group.Match(source, edge.first)) { + new_group.Register(source, edge.first); + if (new_group.Match(target, edge.second)) { + new_group.Register(target, edge.second); + cur_groups.push_back(new_group); + // TODO(Superjomn) need to unique + } + } + } + } + } + } + + for (auto& group : bi_records[step % 2]) { + GraphPatternDetecter::subgraph_t subgraph; + for (auto& role : group.roles) { + subgraph.emplace(role.first, role.second); + } + result.emplace_back(subgraph); + } + return result; +} + +void GraphPatternDetecter::UniquePatterns( + std::vector* subgraphs) { + if (subgraphs->empty()) return; + std::vector result; + + std::unordered_set set; + for (auto& g : *subgraphs) { + size_t key = 0; + for (auto& item : g) { + key ^= std::hash{}(item.first); + key ^= std::hash{}(item.second); + } + if (!set.count(key)) { + result.emplace_back(g); + set.insert(key); + } + } + *subgraphs = result; +} + +void GraphPatternDetecter::RemoveOverlappedMatch( + std::vector* subgraphs) { + std::vector result; + std::unordered_set node_set; + + for (const auto& subgraph : *subgraphs) { + bool valid = true; + for (auto& item : subgraph) { + if (node_set.count(item.second)) { + valid = false; + break; + } + } + if (valid) { + for (auto& item : subgraph) { + node_set.insert(item.second); + } + result.push_back(subgraph); + } + } + *subgraphs = result; +} + +} // namespace ir +} // namespace framework +} // namespace paddle diff --git a/paddle/fluid/framework/ir/graph_pattern_detecter.h b/paddle/fluid/framework/ir/graph_pattern_detecter.h new file mode 100644 index 0000000000000000000000000000000000000000..1778bf00000f60e5cf8b2a585bf7e5dae0a582eb --- /dev/null +++ b/paddle/fluid/framework/ir/graph_pattern_detecter.h @@ -0,0 +1,181 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#ifdef PADDLE_WITH_TESTING +#include +#endif + +#include +#include "paddle/fluid/framework/ir/graph.h" +#include "paddle/fluid/framework/ir/node.h" + +namespace paddle { +namespace framework { +namespace ir { + +// Some basic torminolygies: +// - PDPattern: a pattern defined as a data flow graph. +// - PDNode: the node in the pattern, each PDNode represents an `ir::Node` +// that meets some conditions defined in `PDNode.teller`. +// - A pattern is defined with PDNodes with edges. + +// Pattern detector node. This node helps to build a pattern. +struct PDNode { + // tell whether an ir::Node* is a candidation for a PDNode. + using teller_t = std::function; + + PDNode(teller_t&& teller, const std::string& name = "") + : teller_(teller), name_(name) { + PADDLE_ENFORCE(teller_ != nullptr, "invalid teller functer is set."); + } + + PDNode(PDNode&& other) = default; + + std::vector inlinks; + std::vector outlinks; + + bool Tell(Node* node) const { + PADDLE_ENFORCE(teller_ != nullptr, "teller should be set for a PDNode"); + return teller_(node); + } + + const std::string& name() const { return name_; } + + PDNode(const PDNode&) = delete; + PDNode& operator=(const PDNode&) = delete; + + private: + teller_t teller_; + std::string name_; +}; + +/* + * A pattern in a graph, which defined with PDNode and edges. Most graph + * patterns can be divided into PDNodes and link relations between them. + * + * For example, the FC fusion need to filter the MUL and ELEMENTWISE_ADD + * operators from the computation graph, the MUL's output should have only one + * consumer which is the ELEMENTWISE_ADD. + * This pattern can be defined as with the following pseudo codes + * + * // Create two operator PDNodes. + * MUL = PDPattern.NewNode() + * ELE = PDPattern.NewNode() + * // Create the variable PDNodes. + * MUL_out = PDPattern.NewNode() + * // Add teller to define some rules that help to filter the target Nodes. + * MUL.teller = lambda(node): node->IsOp() && node->Op()->Type == "mul"; + * ELE.teller = lambda(node): \ + * node->IsOp() && node->Op()->Type == "elementwise_add"; + * MUL_out.teller = lambda(node): node->IsVar() && (MUL in node->inputs) + * && (ELE in node->outputs) + * + * One can add more specific tellers for PDNodes or edges, both the Operator + * and Variable Nodes can be ruled in PDNode.teller. + * + * PDPattern can record the general patterns, such as the pattern represents + * - Op in CPU -> Op in GPU -> Op in CPU, to findout the IO abnormal place. + * - Ops whose inputs and outputs share the same variables + */ +class PDPattern { + public: + using edge_t = std::pair; + + void AddEdge(PDNode* a, PDNode* b); + + PDNode* NewNode(PDNode::teller_t&& teller, const std::string& name = ""); + + const std::vector>& nodes() const { return nodes_; } + const std::vector& edges() const { return edges_; } + + private: +#ifdef PADDLE_WITH_TESTING + FRIEND_TEST(PDPattern, AddEdge); + FRIEND_TEST(PDPattern, NewNode); +#endif + + std::vector> nodes_; + std::vector edges_; +}; + +/* + * GraphPatternDetecter helps to detect the specific patterns in the graph. + * Input a pattern, output a list of the matched subgraphs/nodes. + * This helper can be used to support fuse(conv+batchnorm => batchnorm e.g.). + * + * The algorithm has three phases: + * 1. Mark the nodes that match the defined PDNodes in a PDPattern, + * 2. Extend a PDNode to subgraphs by deducing the connection relation defined + * in PAPattern(the edges), + * 3. Get the filtered subgraphs and treat them with a pre-defined handler. + * + * Usage: + * // Create a detector + * GraphPatternDetecter detector; + * // Define the detector's pattern, by adding PDNode and define the edges. + * auto* node0 = detector.mutable_pattern().AddNode(...) + * auto* node1 = detector.mutable_pattern().AddNode(...) + * node0->teller = some lambda. + * node1->teller = some lambda. + * detector.mutable_pattern().AddEdge(node0, node1); + * // Create an handler, to define the behavior of treating the filtered + * // subgraphs that comply with the patterns. + * GraphPatternDetecter::handle_t handler = some labmda + * // Execute the detector. + * detector(&graph, handler); + */ +class GraphPatternDetecter { + public: + using subgraph_t = std::unordered_map; + + // Operate on the detected pattern. + using handle_t = + std::function; + + void operator()(Graph* graph, handle_t handler); + + const PDPattern& pattern() const { return pattern_; } + PDPattern* mutable_pattern() { return &pattern_; } + + private: + // Mark the nodes that fits the pattern. + bool MarkPDNodesInGraph(const ir::Graph& graph); + + // Detect all the pattern and output the hit records. + std::vector DetectPatterns(); + + // Remove duplicate patterns. + void UniquePatterns(std::vector* subgraphs); + + // Remove overlapped match subgraphs, when overlapped, keep the previous one. + void RemoveOverlappedMatch(std::vector* subgraphs); + +#ifdef PADDLE_WITH_TESTING + FRIEND_TEST(GraphPatternDetecter, MarkPDNodesInGraph); + FRIEND_TEST(GraphPatternDetecter, DetectPatterns); +#endif + + private: + using hit_rcd_t = + std::pair; + PDPattern pattern_; + std::vector marked_records_; + std::unordered_map> pdnodes2nodes_; +}; + +} // namespace ir +} // namespace framework +} // namespace paddle diff --git a/paddle/fluid/framework/ir/graph_pattern_detecter_tester.cc b/paddle/fluid/framework/ir/graph_pattern_detecter_tester.cc new file mode 100644 index 0000000000000000000000000000000000000000..993c885a810fe80a170ed190b892b148d85e8b5f --- /dev/null +++ b/paddle/fluid/framework/ir/graph_pattern_detecter_tester.cc @@ -0,0 +1,172 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/framework/ir/graph_pattern_detecter.h" + +#include + +namespace paddle { +namespace framework { +namespace ir { + +void BuildGraph(Graph* g) { + ir::Node* o1 = g->CreateEmptyNode("op1", Node::Type::kOperation); + ir::Node* o2 = g->CreateEmptyNode("op2", Node::Type::kOperation); + ir::Node* o3 = g->CreateEmptyNode("op3", Node::Type::kOperation); + ir::Node* o4 = g->CreateEmptyNode("op4", Node::Type::kOperation); + ir::Node* o5 = g->CreateEmptyNode("op5", Node::Type::kOperation); + ir::Node* v1 = g->CreateEmptyNode("var1", Node::Type::kVariable); + ir::Node* v2 = g->CreateEmptyNode("var2", Node::Type::kVariable); + ir::Node* v3 = g->CreateEmptyNode("var3", Node::Type::kVariable); + ir::Node* v4 = g->CreateEmptyNode("var4", Node::Type::kVariable); + + // o1->v1->o2 + o1->outputs.push_back(v1); + o2->inputs.push_back(v1); + v1->inputs.push_back(o1); + v1->outputs.push_back(o2); + // o2->v2->o3 + // o2->v2->o4 + o2->outputs.push_back(v2); + o3->inputs.push_back(v2); + o4->inputs.push_back(v2); + v2->inputs.push_back(o2); + v2->outputs.push_back(o3); + v2->outputs.push_back(o4); + // o2->v3->o5 + o2->outputs.push_back(v3); + o5->inputs.push_back(v3); + v3->inputs.push_back(o2); + v3->outputs.push_back(o5); + // o3-v4->o5 + o3->outputs.push_back(v4); + o5->inputs.push_back(v4); + v4->inputs.push_back(o3); + v4->outputs.push_back(o5); +} + +TEST(PDPattern, NewNode) { + PDPattern x; + auto* n = x.NewNode([](Node* x) { return true; }); + ASSERT_TRUE(n); + ASSERT_EQ(x.nodes_.size(), 1UL); +} + +TEST(PDPattern, AddEdge) { + PDPattern x; + auto* a = x.NewNode([](Node* x) { return true; }); + auto* b = x.NewNode([](Node* x) { return true; }); + ASSERT_TRUE(a); + ASSERT_TRUE(b); + x.AddEdge(a, b); + ASSERT_EQ(x.nodes_.size(), 2UL); + ASSERT_EQ(x.edges_.size(), 1UL); + ASSERT_EQ(x.edges_.front().first, a); + ASSERT_EQ(x.edges_.front().second, b); + + ASSERT_EQ(x.nodes().size(), 2UL); + ASSERT_EQ(x.edges().size(), 1UL); + ASSERT_EQ(x.edges().front().first, a); + ASSERT_EQ(x.edges().front().second, b); +} + +TEST(GraphPatternDetecter, MarkPDNodesInGraph) { + GraphPatternDetecter x; + // mark o2, o3, v2 + + // The pattern is a graph: + // o2(a node named o2) -> v2(a node named v2) + // v2 -> o3(a node named o3) + auto* o2 = x.pattern_.NewNode([](Node* node) { + // The teller can be any condition, such as op type, or variable's shape. + return node && node->Name() == "op2" && node->IsOp(); + }); + auto* o3 = x.pattern_.NewNode([](Node* node) { + // The teller can be any condition, such as op type, or variable's shape. + return node && node->Name() == "op3" && node->IsOp(); + }); + auto* v2 = x.pattern_.NewNode([](Node* node) { + // The teller can be any condition, such as op type, or variable's shape. + return node && node->Name() == "var2" && node->IsVar(); + }); + + ASSERT_FALSE(o2->Tell(nullptr)); + ASSERT_FALSE(o3->Tell(nullptr)); + ASSERT_FALSE(v2->Tell(nullptr)); + + x.pattern_.AddEdge(o2, v2); + x.pattern_.AddEdge(v2, o3); + + ASSERT_EQ(x.pattern_.edges().size(), 2UL); + ASSERT_EQ(x.pattern_.edges()[0].first, o2); + ASSERT_EQ(x.pattern_.edges()[0].second, v2); + ASSERT_EQ(x.pattern_.edges()[1].first, v2); + ASSERT_EQ(x.pattern_.edges()[1].second, o3); + + ProgramDesc program; + Graph graph(program); + BuildGraph(&graph); + + x.MarkPDNodesInGraph(graph); + + ASSERT_EQ(x.pdnodes2nodes_.size(), 3UL); + + auto subgraphs = x.DetectPatterns(); + ASSERT_EQ(subgraphs.size(), 1UL); +} + +TEST(GraphPatternDetecter, MultiSubgraph) { + ProgramDesc program; + Graph graph(program); + BuildGraph(&graph); + + GraphPatternDetecter x; + + // The pattern is a graph: + // op -> var + auto* any_op = x.mutable_pattern()->NewNode( + [](Node* node) { + return node->IsOp() && (node->Name() == "op2" || node->Name() == "op3"); + }, + "OP0"); + auto* any_var = x.mutable_pattern()->NewNode( + [](Node* node) { return node->IsVar(); }, "VAR"); + auto* any_op1 = x.mutable_pattern()->NewNode( + [](Node* node) { return node->IsOp(); }, "OP1"); + + x.mutable_pattern()->AddEdge(any_op, any_var); + x.mutable_pattern()->AddEdge(any_var, any_op1); + + int count = 0; + GraphPatternDetecter::handle_t handle = [&]( + const GraphPatternDetecter::subgraph_t& s, Graph* g) { + LOG(INFO) << "Detect " << s.at(any_op)->Name() << " -> " + << s.at(any_var)->Name() << " -> " << s.at(any_op1)->Name(); + count++; + }; + + x(&graph, handle); + + // 1. Detect op3 -> var4 -> op5 + // 2. Detect op2 -> var2 -> op3 + // 3. Detect op2 -> var2 -> op4 + // 4. Detect op2 -> var3 -> op5 + // But 2 and 3 and 4 overlapped, so keep 2, so the final choices are 1 and 2 + ASSERT_GE(count, 1UL); + ASSERT_LE(count, 2UL); +} + +} // namespace ir +} // namespace framework +} // namespace paddle diff --git a/paddle/fluid/framework/ir/graph_traits.cc b/paddle/fluid/framework/ir/graph_traits.cc new file mode 100644 index 0000000000000000000000000000000000000000..8f548913e4e1d9d5bc5bdace8b92db9065cf3b5e --- /dev/null +++ b/paddle/fluid/framework/ir/graph_traits.cc @@ -0,0 +1,69 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/framework/ir/graph_traits.h" + +namespace paddle { +namespace framework { +namespace ir { + +// +// NodesDFSIterator +// +NodesDFSIterator::NodesDFSIterator(const std::vector &source) { + for (auto *x : source) stack_.push(x); +} + +NodesDFSIterator::NodesDFSIterator(NodesDFSIterator &&other) noexcept + : stack_(std::move(other.stack_)), + visited_(std::move(other.visited_)) {} + +NodesDFSIterator::NodesDFSIterator(const NodesDFSIterator &other) + : stack_(other.stack_), visited_(other.visited_) {} + +Node &NodesDFSIterator::operator*() { + PADDLE_ENFORCE(!stack_.empty()); + return *stack_.top(); +} + +NodesDFSIterator &NodesDFSIterator::operator++() { + PADDLE_ENFORCE(!stack_.empty(), "the iterator exceeds range"); + visited_.insert(stack_.top()); + auto *cur = stack_.top(); + stack_.pop(); + for (auto *x : cur->outputs) { + if (!visited_.count(x)) { + stack_.push(x); + } + } + return *this; +} +bool NodesDFSIterator::operator==(const NodesDFSIterator &other) { + if (stack_.empty()) return other.stack_.empty(); + if ((!stack_.empty()) && (!other.stack_.empty())) { + return stack_.top() == other.stack_.top(); + } + return false; +} + +NodesDFSIterator &NodesDFSIterator::operator=(const NodesDFSIterator &other) { + stack_ = other.stack_; + visited_ = other.visited_; + return *this; +} +Node *NodesDFSIterator::operator->() { return stack_.top(); } + +} // namespace ir +} // namespace framework +} // namespace paddle diff --git a/paddle/fluid/framework/ir/graph_traits.h b/paddle/fluid/framework/ir/graph_traits.h new file mode 100644 index 0000000000000000000000000000000000000000..edbe45acb98326ee3bf1d86495832ec8469b634e --- /dev/null +++ b/paddle/fluid/framework/ir/graph_traits.h @@ -0,0 +1,90 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include "paddle/fluid/framework/ir/graph.h" +#include "paddle/fluid/framework/ir/node.h" + +namespace paddle { +namespace framework { +namespace ir { + +template +class iterator_range { + IteratorT begin_, end_; + + public: + template + explicit iterator_range(Container &&c) : begin_(c.begin()), end_(c.end()) {} + + iterator_range(const IteratorT &begin, const IteratorT &end) + : begin_(begin), end_(end) {} + + const IteratorT &begin() const { return begin_; } + const IteratorT &end() const { return end_; } +}; + +// DFS iterator on nodes. +struct NodesDFSIterator + : public std::iterator { + NodesDFSIterator() = default; + explicit NodesDFSIterator(const std::vector &source); + NodesDFSIterator(NodesDFSIterator &&other) noexcept; + NodesDFSIterator(const NodesDFSIterator &other); + + Node &operator*(); + NodesDFSIterator &operator++(); + // TODO(Superjomn) current implementation just compare the first + // element, need to compare the graph and all the elements in the queue and + // set. + NodesDFSIterator &operator=(const NodesDFSIterator &other); + bool operator==(const NodesDFSIterator &other); + bool operator!=(const NodesDFSIterator &other) { return !(*this == other); } + Node *operator->(); + + private: + std::stack stack_; + std::unordered_set visited_; +}; + +/* + * GraphTraits contains some graph traversal algorithms. + * + * Usage: + * + */ +struct GraphTraits { + static iterator_range DFS(const Graph &g) { + auto start_points = ExtractStartPoints(g); + NodesDFSIterator x(start_points); + return iterator_range(NodesDFSIterator(start_points), + NodesDFSIterator()); + } + + private: + // The nodes those have no input will be treated as start points. + static std::vector ExtractStartPoints(const Graph &g) { + std::vector result; + for (auto *node : g.Nodes()) { + if (node->inputs.empty()) { + result.push_back(node); + } + } + return result; + } +}; + +} // namespace ir +} // namespace framework +} // namespace paddle diff --git a/paddle/fluid/framework/ir/node.h b/paddle/fluid/framework/ir/node.h index b3138fccee86fb274abe72007961fc1c982b1e96..9c0765ab8ce16733ac021aefc8c7b2bb779319f3 100644 --- a/paddle/fluid/framework/ir/node.h +++ b/paddle/fluid/framework/ir/node.h @@ -58,6 +58,9 @@ class Node { return op_desc_; } + bool IsOp() const { return type_ == Type::kOperation; } + bool IsVar() const { return type_ == Type::kVariable; } + std::vector inputs; std::vector outputs; diff --git a/paddle/fluid/framework/op_desc.cc b/paddle/fluid/framework/op_desc.cc index 03f7e71c03b8dd75d2a47cb4c6d1ef1a71792cf3..122dc161b41246e5f08bd0ae8b763489e9ee22f9 100644 --- a/paddle/fluid/framework/op_desc.cc +++ b/paddle/fluid/framework/op_desc.cc @@ -202,6 +202,52 @@ std::vector OpDesc::AttrNames() const { } void OpDesc::SetAttr(const std::string &name, const Attribute &v) { + // NOTICE(minqiyang): pybind11 will take the empty list in python as + // the std::vector type in C++; so we have to change the attr's type + // here if we meet this issue + proto::AttrType attr_type = static_cast(v.which() - 1); + if (attr_type == proto::AttrType::INTS && + boost::get>(v).size() == 0u) { + // Find current attr via attr name and set the correct attribute value + const proto::OpProto::Attr &attr = GetProtoAttr(name); + switch (attr.type()) { + case proto::AttrType::BOOLEANS: { + VLOG(11) << "SetAttr: " << Type() << ", " << name + << " from INTS to BOOLEANS"; + this->attrs_[name] = std::vector(); + break; + } + case proto::AttrType::INTS: { + VLOG(11) << "SetAttr: " << Type() << ", " << name + << " from INTS to INTS"; + this->attrs_[name] = std::vector(); + break; + } + case proto::AttrType::FLOATS: { + VLOG(11) << "SetAttr: " << Type() << ", " << name + << " from INTS to FLOATS"; + this->attrs_[name] = std::vector(); + break; + } + case proto::AttrType::STRINGS: { + VLOG(11) << "SetAttr: " << Type() << ", " << name + << " from INTS to STRINGS"; + this->attrs_[name] = std::vector(); + break; + } + case proto::AttrType::BLOCKS: { + VLOG(11) << "SetAttr: " << Type() << ", " << name + << " from INTS to BLOCKS"; + this->SetBlocksAttr(name, std::vector()); + return; + } + default: + PADDLE_THROW("Wrong attr type %d", attr.type()); + } + need_update_ = true; + return; + } + this->attrs_[name] = v; need_update_ = true; } @@ -229,6 +275,19 @@ Attribute OpDesc::GetAttr(const std::string &name) const { return it->second; } +const proto::OpProto::Attr &OpDesc::GetProtoAttr( + const std::string &name) const { + const proto::OpProto &proto = OpInfoMap::Instance().Get(Type()).Proto(); + for (int i = 0; i != proto.attrs_size(); ++i) { + const proto::OpProto::Attr &attr = proto.attrs(i); + if (attr.name() == name) { + return attr; + } + } + + PADDLE_THROW("Attribute %s is not found in proto %s", name, proto.type()); +} + Attribute OpDesc::GetNullableAttr(const std::string &name) const { auto it = attrs_.find(name); if (it != attrs_.end()) { diff --git a/paddle/fluid/framework/op_desc.h b/paddle/fluid/framework/op_desc.h index b77d84125a23b81c3de4123bea6f0e09cd6d1e90..2422392e24d864dc3e7973ab35e038ecf2c0392a 100644 --- a/paddle/fluid/framework/op_desc.h +++ b/paddle/fluid/framework/op_desc.h @@ -81,6 +81,8 @@ class OpDesc { Attribute GetAttr(const std::string &name) const; + const proto::OpProto::Attr &GetProtoAttr(const std::string &name) const; + Attribute GetNullableAttr(const std::string &name) const; int GetBlockAttrId(const std::string &name) const; diff --git a/paddle/fluid/framework/rw_lock.h b/paddle/fluid/framework/rw_lock.h new file mode 100644 index 0000000000000000000000000000000000000000..2a4009b765e35c6cef8dc9ffb3e04a50f0dc2b38 --- /dev/null +++ b/paddle/fluid/framework/rw_lock.h @@ -0,0 +1,46 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include + +namespace paddle { +namespace framework { + +struct RWLock { + RWLock() { pthread_rwlock_init(&lock_, nullptr); } + + ~RWLock() { pthread_rwlock_destroy(&lock_); } + + void RDLock() { + PADDLE_ENFORCE_EQ(pthread_rwlock_rdlock(&lock_), 0, + "acquire read lock failed"); + } + + void WRLock() { + PADDLE_ENFORCE_EQ(pthread_rwlock_wrlock(&lock_), 0, + "acquire write lock failed"); + } + + void UNLock() { + PADDLE_ENFORCE_EQ(pthread_rwlock_unlock(&lock_), 0, "unlock failed"); + } + + private: + pthread_rwlock_t lock_; +}; + +} // namespace framework +} // namespace paddle diff --git a/paddle/fluid/framework/selected_rows.cc b/paddle/fluid/framework/selected_rows.cc index 06ed87e7e8a2d5324b48a466b05207042ec1b7fa..c202b0a5be1f891b8ae7b11e1f6e0ce02fcba588 100644 --- a/paddle/fluid/framework/selected_rows.cc +++ b/paddle/fluid/framework/selected_rows.cc @@ -120,66 +120,76 @@ bool SelectedRows::HasKey(int64_t key) const { : true; } -std::vector> SelectedRows::Get( - const std::vector& keys, framework::Tensor* value) const { +int64_t SelectedRows::AutoGrownIndex(int64_t key, bool auto_grown) { + rwlock_->RDLock(); + auto iter = id_to_index_.find(key); + if (iter == id_to_index_.end()) { + rwlock_->UNLock(); + if (!auto_grown) { + PADDLE_THROW("key %d not found", key); + } + rwlock_->WRLock(); + auto map_size = id_to_index_.size(); + auto vector_size = rows_.size(); + if (map_size != vector_size) { + rwlock_->UNLock(); + PADDLE_THROW( + "id_to_index_ size %d should have the same size with rows_ %d", + map_size, vector_size); + } + auto write_iter = id_to_index_.find(key); + if (write_iter == id_to_index_.end()) { + size_t row_num = rows_.size(); + if (row_num == value_->dims()[0]) { + rwlock_->UNLock(); + PADDLE_THROW("selected rows is full, then length exceed %d", row_num); + } + // key logic to put a key into id_to_index_ + rows_.push_back(key); + auto index = static_cast(rows_.size() - 1); + id_to_index_[key] = index; + rwlock_->UNLock(); + return index; + } else { + auto index = write_iter->second; + rwlock_->UNLock(); + return index; + } + } else { + auto index = iter->second; + rwlock_->UNLock(); + return index; + } +} + +void SelectedRows::SyncIndex() { + rwlock_->WRLock(); + id_to_index_.clear(); + for (size_t i = 0; i < rows_.size(); ++i) { + id_to_index_[rows_[i]] = i; + } + rwlock_->UNLock(); +} + +void SelectedRows::Get(const framework::Tensor& ids, framework::Tensor* value, + bool auto_grown) { PADDLE_ENFORCE(value->IsInitialized(), "The value tensor should be initialized."); - std::vector> non_keys_pair; - if (keys.empty()) { + if (ids.numel() == 0) { VLOG(3) << "keys is empty, please check data!"; } else { int64_t value_width = value_->numel() / value_->dims()[0]; PADDLE_ENFORCE_EQ(value_width, value->numel() / value->dims()[0], "output tensor should have the same shape with table " "except the dims[0]."); - - for (size_t i = 0; i < keys.size(); ++i) { - int64_t index = Index(keys[i]); - if (index == -1) { - non_keys_pair.push_back( - std::make_pair(keys[i], static_cast(i))); - } else { - framework::VisitDataType( - framework::ToDataType(value_->type()), - TensorCopyVisitor(value, i * value_width, *value_.get(), - index * value_width, value_width)); - } + for (size_t i = 0; i < ids.numel(); ++i) { + int64_t index = AutoGrownIndex(ids.data()[i], auto_grown); + framework::VisitDataType( + framework::ToDataType(value_->type()), + TensorCopyVisitor(value, i * value_width, *value_.get(), + index * value_width, value_width)); } } - return non_keys_pair; -} - -bool SelectedRows::Set(int64_t key, const framework::Tensor& value) { - PADDLE_ENFORCE(value.IsInitialized(), "The value should be initialized."); - if (value_->IsInitialized()) { - PADDLE_ENFORCE_EQ( - value.type(), value_->type(), - "The type of the value should be same with the original value"); - } - PADDLE_ENFORCE_EQ(value.dims()[0], static_cast(1), - "The first dim of value should be 1."); - std::lock_guard lock(*auto_grown_mutex_.get()); - auto index = Index(key); - bool is_new_key = false; - if (index == -1) { - rows_.push_back(key); - index = rows_.size() - 1; - is_new_key = true; - // whether need to resize the table - if (static_cast(rows_.size()) > value_->dims()[0]) { - auto dims = value_->dims(); - dims[0] = (dims[0] + 1) << 1; - framework::VisitDataType(framework::ToDataType(value.type()), - ReAllocateVisitor(dims, value_.get())); - } - } - - framework::VisitDataType( - framework::ToDataType(value.type()), - TensorCopyVisitor(value_.get(), - index * value_->numel() / value_->dims()[0], value, - static_cast(0), value.numel())); - return is_new_key; } } // namespace framework diff --git a/paddle/fluid/framework/selected_rows.h b/paddle/fluid/framework/selected_rows.h index 7160670ddd204c20021ea87cdd67ee4721d03451..daf5e95304fb84eaba26a30c45414d5021e7ffcb 100644 --- a/paddle/fluid/framework/selected_rows.h +++ b/paddle/fluid/framework/selected_rows.h @@ -17,10 +17,12 @@ limitations under the License. */ #include #include #include // NOLINT +#include #include #include #include "paddle/fluid/framework/lod_tensor.h" +#include "paddle/fluid/framework/rw_lock.h" #include "paddle/fluid/framework/tensor.h" #include "paddle/fluid/memory/memcpy.h" @@ -48,13 +50,13 @@ class SelectedRows { SelectedRows(const std::vector& rows, const int64_t& height) : rows_(rows), height_(height) { value_.reset(new Tensor()); - auto_grown_mutex_.reset(new std::mutex); + rwlock_.reset(new RWLock); } SelectedRows() { height_ = 0; value_.reset(new Tensor()); - auto_grown_mutex_.reset(new std::mutex); + rwlock_.reset(new RWLock); } platform::Place place() const { return value_->place(); } @@ -74,47 +76,51 @@ class SelectedRows { void set_rows(const Vector& rows) { rows_ = rows; } /* - * @brief wheter has the specified key in the table. + * @brief Get the index of key in rows + * + * @return -1 if the key does not exists. + */ + int64_t Index(int64_t key) const { + auto it = std::find(rows_.begin(), rows_.end(), key); + if (it == rows_.end()) { + PADDLE_THROW("id %s not in table", key); + } + return static_cast(std::distance(rows_.begin(), it)); + } + + /* + * @brief whether has the specified key in the table. * * @return true if the key is exists. */ bool HasKey(int64_t key) const; /* - * @brief Get value by the key list, if the + * @brief Get value by the key list. + * Note!!! this interface is only used when selected_rows is used as + * parameters + * for distribute lookup table. * * @return a list of pair which contains the non-exists key and the index in * the value */ - std::vector> Get(const std::vector& keys, - framework::Tensor* value) const; + void Get(const framework::Tensor& ids, framework::Tensor* value, + bool auto_grown = false); /* - * @brief Set a key-value pair into the table. - * This function will double the value memory if it's not engouth. + * @brief Get the index of the key from id_to_index_ map. If the key not + * exist, + * add the key into id_to_index_. * - * @note: - * 1. The first dim of the value should be 1 - * 2. The value should be initialized and the data type - * should be the same with the table. - * - * @return true if the key is a new one, otherwise false + * Note!!! this interface is only used when selected_rows is used as + * parameters + * for distribute lookup table. * + * @return index of the key. */ - bool Set(int64_t key, const Tensor& value); + int64_t AutoGrownIndex(int64_t key, bool auto_grown); - /* - * @brief Get the index of key in rows - * - * @return -1 if the key does not exists. - */ - int64_t Index(int64_t key) const { - auto it = std::find(rows_.begin(), rows_.end(), key); - if (it == rows_.end()) { - return static_cast(-1); - } - return static_cast(std::distance(rows_.begin(), it)); - } + void SyncIndex(); DDim GetCompleteDims() const { std::vector dims = vectorize(value_->dims()); @@ -127,9 +133,10 @@ class SelectedRows { // SelectedRows are simply concated when adding together. Until a // SelectedRows add a Tensor, will the duplicate rows be handled. Vector rows_; + std::unordered_map id_to_index_; std::unique_ptr value_{nullptr}; int64_t height_; - std::unique_ptr auto_grown_mutex_{nullptr}; + std::unique_ptr rwlock_{nullptr}; }; /* diff --git a/paddle/fluid/framework/selected_rows_test.cc b/paddle/fluid/framework/selected_rows_test.cc index eefcaa5672c5a3debf162f5c8eda653408dcf221..5ca864cfdf7176850dd31dd42ef3306061a742cf 100644 --- a/paddle/fluid/framework/selected_rows_test.cc +++ b/paddle/fluid/framework/selected_rows_test.cc @@ -9,8 +9,11 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#include "paddle/fluid/framework/selected_rows.h" +#include +#include // NOLINT + #include "gtest/gtest.h" +#include "paddle/fluid/framework/selected_rows.h" namespace paddle { namespace framework { @@ -59,39 +62,129 @@ TEST_F(SelectedRowsTester, SerializeAndDeseralize) { ASSERT_EQ(selected_rows_->GetCompleteDims(), dst_tensor.GetCompleteDims()); } -TEST_F(SelectedRowsTester, SparseTable) { +TEST(SelectedRows, SparseTable) { platform::CPUPlace cpu; SelectedRows table; + + int64_t table_size = 100; + int64_t embedding_width = 8; // initialize a sparse table - table.mutable_value()->Resize(framework::make_ddim({1, 100})); - table.mutable_value()->mutable_data(cpu); - table.mutable_rows()->push_back(1); + table.mutable_value()->Resize( + framework::make_ddim({table_size, embedding_width})); + auto* data = table.mutable_value()->mutable_data(cpu); + for (int64_t i = 0; i < table_size; ++i) { + for (int64_t j = 0; j < embedding_width; ++j) { + data[i * embedding_width + j] = static_cast(i); + } + } + ASSERT_EQ(table.AutoGrownIndex(10, true), 0); + ASSERT_EQ(table.AutoGrownIndex(8, true), 1); + ASSERT_EQ(table.AutoGrownIndex(8, true), 1); + ASSERT_EQ(table.AutoGrownIndex(6, true), 2); + ASSERT_TRUE(table.HasKey(10)); + ASSERT_TRUE(table.HasKey(8)); + ASSERT_TRUE(table.HasKey(6)); + ASSERT_EQ(table.rows().size(), 3); + + framework::Tensor ids; + ids.Resize(framework::make_ddim({4})); + auto* ids_data = ids.mutable_data(cpu); + ids_data[0] = static_cast(6); + ids_data[1] = static_cast(6); + ids_data[2] = static_cast(8); + ids_data[3] = static_cast(10); - int64_t key = 10000; - int64_t non_key = 999; - framework::Tensor value; - value.Resize(framework::make_ddim({1, 100})); - auto ptr = value.mutable_data(cpu); - ptr[0] = static_cast(10); + framework::Tensor get_value; + auto* value_data = get_value.mutable_data( + framework::make_ddim({4, embedding_width}), cpu); + table.Get(ids, &get_value); - ASSERT_EQ(table.rows().size(), static_cast(1)); - ASSERT_EQ(table.HasKey(key), false); + for (int j = 0; j < embedding_width; ++j) { + ASSERT_EQ(value_data[0 * embedding_width + j], 2); + } + for (int j = 0; j < embedding_width; ++j) { + ASSERT_EQ(value_data[1 * embedding_width + j], 2); + } + for (int j = 0; j < embedding_width; ++j) { + ASSERT_EQ(value_data[2 * embedding_width + j], 1); + } + for (int j = 0; j < embedding_width; ++j) { + ASSERT_EQ(value_data[3 * embedding_width + j], 0); + } +} - table.Set(key, value); +void f1(SelectedRows* table, int table_size) { + for (int i = 1000000; i > 0; --i) { + auto id = i % table_size; + int64_t index1 = table->AutoGrownIndex(id, true); + int64_t index2 = table->AutoGrownIndex(id, false); + int64_t index3 = table->AutoGrownIndex(id, true); + ASSERT_EQ(index1, index2); + ASSERT_EQ(index2, index3); + } +} - ASSERT_EQ(table.rows().size(), static_cast(2)); - ASSERT_EQ(table.HasKey(key), true); - // check re-allocate - ASSERT_EQ(table.value().dims()[0], static_cast(4)); +void f2(SelectedRows* table, int table_size) { + for (int i = 0; i < 1000000; ++i) { + auto id = i % table_size; + int64_t index1 = table->AutoGrownIndex(id, true); + int64_t index2 = table->AutoGrownIndex(id, false); + int64_t index3 = table->AutoGrownIndex(id, true); + ASSERT_EQ(index1, index2); + ASSERT_EQ(index2, index3); + } +} - framework::Tensor get_value; - get_value.mutable_data(framework::make_ddim({2, 100}), cpu); - std::vector keys({non_key, key}); - auto non_key_pairs = table.Get(keys, &get_value); +void f3(SelectedRows* table, int table_size) { + clock_t t1 = clock(); + for (int i = 100000; i > 0; --i) { + auto id1 = table->AutoGrownIndex(i % table_size, true); + auto id2 = table->Index(i % table_size); + ASSERT_EQ(id1, id2); + } + clock_t t2 = clock(); + std::cout << "f3 run time:" << t2 - t1 << std::endl; +} + +void f4(SelectedRows* table, int table_size) { + clock_t t1 = clock(); + for (int i = 0; i < 100000; ++i) { + auto id1 = table->AutoGrownIndex(i % table_size, true); + auto id2 = table->Index(i % table_size); + ASSERT_EQ(id1, id2); + } + clock_t t2 = clock(); + std::cout << "f4 run time:" << t2 - t1 << std::endl; +} + +TEST(SelectedRows, MultiThreadAutoIndex) { + platform::CPUPlace cpu; + SelectedRows table; + + int64_t table_size = 100000; + int64_t embedding_width = 8; + // initialize a sparse table + table.mutable_value()->Resize( + framework::make_ddim({table_size, embedding_width})); + auto* data = table.mutable_value()->mutable_data(cpu); + for (int64_t i = 0; i < table_size; ++i) { + for (int64_t j = 0; j < embedding_width; ++j) { + data[i * embedding_width + j] = static_cast(i); + } + } - ASSERT_EQ(get_value.data()[100], static_cast(10)); - ASSERT_EQ(non_key_pairs.size(), static_cast(1)); - ASSERT_EQ(non_key_pairs[0].first, non_key); + std::thread t1(f1, &table, table_size); + std::thread t11(f1, &table, table_size); + std::thread t2(f2, &table, table_size); + std::thread t22(f2, &table, table_size); + t1.join(); + t11.join(); + t2.join(); + t22.join(); + std::thread t3(f3, &table, table_size); + std::thread t4(f4, &table, table_size); + t3.join(); + t4.join(); } } // namespace framework diff --git a/paddle/fluid/framework/threadpool.cc b/paddle/fluid/framework/threadpool.cc index f26f212d4d5793b88fd1e6d782cdf983bf341879..18cdca3a658a6a89e6ab637a7f38825756acfea8 100644 --- a/paddle/fluid/framework/threadpool.cc +++ b/paddle/fluid/framework/threadpool.cc @@ -20,6 +20,9 @@ DEFINE_int32(io_threadpool_size, 100, "number of threads used for doing IO, default 100"); +DEFINE_int32(dist_threadpool_size, 0, + "number of threads used for distributed executed."); + namespace paddle { namespace framework { @@ -35,6 +38,10 @@ void ThreadPool::Init() { if (threadpool_.get() == nullptr) { // TODO(Yancey1989): specify the max threads number int num_threads = std::thread::hardware_concurrency(); + if (FLAGS_dist_threadpool_size > 0) { + num_threads = FLAGS_dist_threadpool_size; + VLOG(1) << "set dist_threadpool_size to " << num_threads; + } PADDLE_ENFORCE_GT(num_threads, 0); threadpool_.reset(new ThreadPool(num_threads)); } diff --git a/paddle/fluid/inference/api/CMakeLists.txt b/paddle/fluid/inference/api/CMakeLists.txt index 83867e0a2c198f265cb36be3a71546796dfbec67..a72e27d651d0591815a9d93354d2aea8aa216de6 100644 --- a/paddle/fluid/inference/api/CMakeLists.txt +++ b/paddle/fluid/inference/api/CMakeLists.txt @@ -60,7 +60,7 @@ cc_library(paddle_inference_tensorrt_subgraph_engine inference_api_test(test_api_tensorrt_subgraph_engine SRC api_tensorrt_subgraph_engine_tester.cc ARGS test_word2vec) endif() -if (WITH_ANAKIN) # only needed in CI +if (WITH_ANAKIN AND WITH_GPU) # only needed in CI # compile the libinference_anakin_api.a and anakin.so. nv_library(inference_anakin_api SRCS api.cc api_anakin_engine.cc DEPS anakin_shared anakin_saber) #nv_library(inference_anakin_api_shared SHARED SRCS api.cc api_anakin_engine.cc DEPS anakin) diff --git a/paddle/fluid/operators/.flatten_op.cc.swp b/paddle/fluid/operators/.flatten_op.cc.swp deleted file mode 100644 index 3395b6074b6a4c684a97674af702ca8b91dc85e9..0000000000000000000000000000000000000000 Binary files a/paddle/fluid/operators/.flatten_op.cc.swp and /dev/null differ diff --git a/paddle/fluid/operators/CMakeLists.txt b/paddle/fluid/operators/CMakeLists.txt index ff0e989464e76b0f7cb163bd95997b82303036d6..e8b5dec9d49f5613cec92441d19ab7dc1a1ad90c 100644 --- a/paddle/fluid/operators/CMakeLists.txt +++ b/paddle/fluid/operators/CMakeLists.txt @@ -170,6 +170,9 @@ function(op_library TARGET) file(APPEND ${pybind_file} "USE_OP(fake_dequantize_max_abs);\n") elseif(${TARGET} STREQUAL "tensorrt_engine_op") message(STATUS "Pybind skips [tensorrt_engine_op], for this OP is only used in inference") + elseif(${TARGET} STREQUAL "fc") + # HACK: fc only have mkldnn and cpu, which would mismatch the cpu only condition + file(APPEND ${pybind_file} "USE_CPU_ONLY_OP(${TARGET});\n") else() file(APPEND ${pybind_file} "USE_OP(${TARGET});\n") endif() @@ -300,12 +303,6 @@ op_library(channel_recv_op DEPS concurrency) list(REMOVE_ITEM GENERAL_OPS ${DEPS_OPS}) -# The fully connected layer is deleted when the WITH_MKLDNN flag is OFF -# Because the fully connected layer has only one MKLDNN's operator -if(NOT WITH_MKLDNN) - list(REMOVE_ITEM GENERAL_OPS fc_op) -endif(NOT WITH_MKLDNN) - foreach(src ${GENERAL_OPS}) op_library(${src}) endforeach() diff --git a/paddle/fluid/operators/distributed/rpc_server_test.cc b/paddle/fluid/operators/distributed/rpc_server_test.cc index b50830c362d3f6ecf38affbfa6a1ffe2ed77e125..d6176e1443d2a441af7878e5efe99796d486bb7a 100644 --- a/paddle/fluid/operators/distributed/rpc_server_test.cc +++ b/paddle/fluid/operators/distributed/rpc_server_test.cc @@ -78,10 +78,9 @@ void InitTensorsOnServer(framework::Scope* scope, platform::CPUPlace* place, int64_t rows_numel) { CreateVarsOnScope(scope, place); auto w = scope->Var("w")->GetMutable(); - auto rows = w->mutable_rows(); - for (int64_t i = 0; i < rows_numel; ++i) rows->push_back(i); auto w_value = w->mutable_value(); w_value->Resize({rows_numel, 10}); + for (int64_t i = 0; i < rows_numel; ++i) w->AutoGrownIndex(i, true); auto ptr = w_value->mutable_data(*place); diff --git a/paddle/fluid/operators/distributed/variable_response.cc b/paddle/fluid/operators/distributed/variable_response.cc index 466bce18af7cf97014a7b1ba64df68eab193c7c8..8e38b3713f28b045e9214db68aec50f0ba6c06f6 100644 --- a/paddle/fluid/operators/distributed/variable_response.cc +++ b/paddle/fluid/operators/distributed/variable_response.cc @@ -190,12 +190,15 @@ bool VariableResponse::ProcSerializedField( #endif } + VLOG(7) << "ProcSerializedField:" << meta_.varname() + << ", type:" << meta_.type() << std::endl; framework::DDim dims = GetDims(meta_.dims()); if (meta_.type() == sendrecv::LOD_TENSOR) { PADDLE_ENFORCE(meta_.lod_size() >= 0, "lod info should be got first!"); if (!CopyLodTensorData(input, *dev_ctx_, dims, num_bytes)) { return false; } + return true; } @@ -206,7 +209,9 @@ bool VariableResponse::ProcSerializedField( return true; } - return true; + PADDLE_ENFORCE("not supported var types:", meta_.varname(), meta_.type()); + + return false; } }; // namespace distributed diff --git a/paddle/fluid/operators/fc_mkldnn_op.cc b/paddle/fluid/operators/fc_mkldnn_op.cc index 99fa659a351249a4a93f71700e1c646465861aba..e595f1a627cfefbb91b070b898046cf135dc4988 100644 --- a/paddle/fluid/operators/fc_mkldnn_op.cc +++ b/paddle/fluid/operators/fc_mkldnn_op.cc @@ -125,13 +125,16 @@ class FCMKLDNNOpKernel : public paddle::framework::OpKernel { auto input = ctx.Input("Input"); auto w = ctx.Input("W"); + auto bias = ctx.Input("Bias"); PADDLE_ENFORCE(input->dims().size() == 2 || input->dims().size() == 4, "Input must be with 2 or 4 dimensions, i.e. NCHW"); + // TODO(intel friends): the native weight format is io, + // but the mkldnn weight format is oihw, which may need be transposed. PADDLE_ENFORCE(w->dims().size() == 2 || w->dims().size() == 4, "Weights must be with 2 or 4 dimensions, i.e. OI or OIHW"); - bool with_bias = ctx.Attr("bias_attr"); + bool with_bias = bias != nullptr; MKLDNNMD md(input, w, with_bias); std::shared_ptr pd = @@ -154,6 +157,7 @@ class FCMKLDNNOpKernel : public paddle::framework::OpKernel { auto dst_memory = mem.dst(output_data); auto src_memory = mem.src(input_data); auto weights_memory = mem.weights(w_data); + // TODO(intel friends): bias memory should also be obtain from bias->data() auto bias_memory = mem.bias(); auto forward = with_bias ? mkldnn::inner_product_forward( @@ -216,7 +220,8 @@ class FCMKLDNNGradOpKernel : public paddle::framework::OpKernel { const Tensor* out_grad = ctx.Input(framework::GradVarName("Out")); const T* out_grad_data = out_grad->data(); - bool with_bias = ctx.Attr("bias_attr"); + auto bias = ctx.Input("Bias"); + bool with_bias = bias != nullptr; MKLDNNMD md(input, w, with_bias); MKLDNNMemory mem(&md, mkldnn_engine); diff --git a/paddle/fluid/operators/fc_op.cc b/paddle/fluid/operators/fc_op.cc index a9ae1396db8d7dab0364779e506d5c0a3e2ff6ed..099ca52c8e945a0e93c2f13adb612158c67397cf 100644 --- a/paddle/fluid/operators/fc_op.cc +++ b/paddle/fluid/operators/fc_op.cc @@ -14,6 +14,9 @@ limitations under the License. */ #include "paddle/fluid/operators/fc_op.h" #include +#include "paddle/fluid/operators/math/blas.h" + +DECLARE_int32(paddle_num_threads); namespace paddle { namespace operators { @@ -25,16 +28,24 @@ void FCOp::InferShape(framework::InferShapeContext* ctx) const { "Out(Output) of Fully Connected should not be null."); PADDLE_ENFORCE(ctx->HasInput("W"), "W(Input) of Fully Connected should not be null."); - + // NCHW auto in_dims = ctx->GetInputDim("Input"); + // IO, I=C*H*W auto w_dims = ctx->GetInputDim("W"); std::vector output_shape({in_dims[0], w_dims[1]}); + if (ctx->HasInput("Bias")) { + auto bias_dims = ctx->GetInputDim("Bias"); + PADDLE_ENFORCE_EQ(bias_dims[0], 1, "The shape of Bias must be [1, dim]."); + PADDLE_ENFORCE_EQ(bias_dims[1], w_dims[1], + "The shape of Bias must be [1, dim]."); + } PADDLE_ENFORCE(in_dims.size() == 2 || in_dims.size() == 4, "Fully Connected input should be 2-D or 4-D tensor."); - - PADDLE_ENFORCE(w_dims.size() == 2 || w_dims.size() == 4, - "Fully Connected input should be 2-D or 4-D tensor."); + PADDLE_ENFORCE_EQ(w_dims.size(), 2UL, + "Fully Connected input should be 2-D tensor."); + PADDLE_ENFORCE_EQ(framework::product(in_dims) / in_dims[0], w_dims[0], + "Fully Connected input and weigth size do not match."); ctx->SetOutputDim("Out", framework::make_ddim(output_shape)); ctx->ShareLoD("Input", "Out"); @@ -42,9 +53,12 @@ void FCOp::InferShape(framework::InferShapeContext* ctx) const { framework::OpKernelType FCOp::GetExpectedKernelType( const framework::ExecutionContext& ctx) const { - framework::LibraryType library{framework::LibraryType::kMKLDNN}; - framework::DataLayout layout{framework::DataLayout::kMKLDNN}; - + framework::LibraryType library = framework::LibraryType::kPlain; + framework::DataLayout layout = framework::DataLayout::kAnyLayout; + if (ctx.Attr("use_mkldnn")) { + library = framework::LibraryType::kMKLDNN; + layout = framework::DataLayout::kMKLDNN; + } return framework::OpKernelType( framework::ToDataType(ctx.Input("Input")->type()), ctx.GetPlace(), layout, library); @@ -60,27 +74,39 @@ void FCOpGrad::InferShape(framework::InferShapeContext* ctx) const { if (ctx->HasOutput(framework::GradVarName("W"))) { ctx->SetOutputDim(framework::GradVarName("W"), w_dims); } + + if (ctx->HasInput("Bias")) { + PADDLE_ENFORCE(ctx->HasOutput(framework::GradVarName("Bias")), + "Should have bias grad"); + auto bias_dims = ctx->GetInputDim("Bias"); + ctx->SetOutputDim(framework::GradVarName("Bias"), bias_dims); + } } framework::OpKernelType FCOpGrad::GetExpectedKernelType( const framework::ExecutionContext& ctx) const { - framework::LibraryType library{framework::LibraryType::kMKLDNN}; - framework::DataLayout layout{framework::DataLayout::kMKLDNN}; - + framework::LibraryType library = framework::LibraryType::kPlain; + framework::DataLayout layout = framework::DataLayout::kAnyLayout; + if (ctx.Attr("use_mkldnn")) { + library = framework::LibraryType::kMKLDNN; + layout = framework::DataLayout::kMKLDNN; + } return framework::OpKernelType( framework::ToDataType(ctx.Input("Input")->type()), ctx.GetPlace(), layout, library); } void FCOpMaker::Make() { - AddInput("Input", "(Tensor) The input tensor of fully connected operator. "); - AddInput("W", "(Tensor), The second input tensor of fc op."); + AddInput("Input", + "(Tensor), The input tensor of fully connected operator with format " + "(NCHW). "); + AddInput("W", "(Tensor), The weight fc op with shape (I, O)."); + AddInput("Bias", "(Tensor, optional) Bias vector with shape (1 x O") + .AsDispensable(); AddOutput("Out", "(Tensor) The output tensor of fully connected operator. "); AddAttr("use_mkldnn", "(bool, default false) Only used in mkldnn kernel") .SetDefault(false); - AddAttr("bias_attr", "(bool, default false) Only used in mkldnn kernel") - .SetDefault(false); AddComment(R"DOC( Fully Connected Operator. @@ -94,9 +120,47 @@ void FCOpMaker::Make() { )DOC"); } +template +class FCOpKernel : public framework::OpKernel { + public: + void Compute(const paddle::framework::ExecutionContext& ctx) const override { + PADDLE_ENFORCE(platform::is_cpu_place(ctx.GetPlace()), + "It must use CPUPlace."); + auto input = ctx.Input("Input"); + auto w = ctx.Input("W"); + auto bias = ctx.Input("Bias"); + auto output = ctx.Output("Out"); + auto in_dims = input->dims(); + auto w_dims = w->dims(); + + auto& dev_ctx = ctx.template device_context(); + auto blas = math::GetBlas(dev_ctx); + const T* input_data = input->data(); + const T* w_data = w->data(); + T* output_data = output->mutable_data(ctx.GetPlace()); + + blas.GEMM(CblasNoTrans, CblasNoTrans, in_dims[0], w_dims[1], w_dims[0], + static_cast(1), input_data, w_data, static_cast(0), + output_data); + + if (bias) { + const T* bias_data = bias->data(); +#ifdef PADDLE_WITH_MKLML +#pragma omp parallel for if (FLAGS_paddle_num_threads > 1) +#endif + for (int bs = 0; bs < in_dims[0]; bs++) { + blas.AXPY(w_dims[1], static_cast(1), bias_data, + output_data + bs * w_dims[1]); + } + } + } +}; + } // namespace operators } // namespace paddle -REGISTER_OPERATOR(fc, paddle::operators::FCOp, paddle::operators::FCOpMaker, +namespace ops = paddle::operators; +REGISTER_OPERATOR(fc, ops::FCOp, ops::FCOpMaker, paddle::framework::DefaultGradOpDescMaker); -REGISTER_OPERATOR(fc_grad, paddle::operators::FCOpGrad); +REGISTER_OPERATOR(fc_grad, ops::FCOpGrad); +REGISTER_OP_CPU_KERNEL(fc, ops::FCOpKernel, ops::FCOpKernel); diff --git a/paddle/fluid/operators/gru_op.cc b/paddle/fluid/operators/gru_op.cc index 5c746878823b3dcde2573feec00d3d9dac5ceab8..087f903a8bba9a4bfcd7eaabd7098555442a904e 100644 --- a/paddle/fluid/operators/gru_op.cc +++ b/paddle/fluid/operators/gru_op.cc @@ -14,6 +14,11 @@ limitations under the License. */ #include "paddle/fluid/operators/gru_op.h" #include +#include "paddle/fluid/operators/math/blas.h" +#include "paddle/fluid/operators/math/detail/gru_cpu_kernel.h" +#include "paddle/fluid/operators/math/detail/gru_kernel.h" + +DECLARE_int32(paddle_num_threads); namespace paddle { namespace operators { @@ -211,6 +216,158 @@ class GRUGradOp : public framework::OperatorWithKernel { } }; +template +class GRUCPUKernel : public framework::OpKernel { + public: + void BatchCompute(const framework::ExecutionContext& context) const { + using DeviceContext = paddle::platform::CPUDeviceContext; + auto* input = context.Input("Input"); + auto* h0 = context.Input("H0"); + auto* weight = context.Input("Weight"); + const T* weight_data = weight->data(); + auto* bias = context.Input("Bias"); + auto* batch_gate = context.Output("BatchGate"); + batch_gate->mutable_data(context.GetPlace()); + auto* batch_reset_hidden_prev = + context.Output("BatchResetHiddenPrev"); + batch_reset_hidden_prev->mutable_data(context.GetPlace()); + auto* batch_hidden = context.Output("BatchHidden"); + batch_hidden->mutable_data(context.GetPlace()); + auto* hidden = context.Output("Hidden"); + hidden->mutable_data(context.GetPlace()); + + auto hidden_dims = hidden->dims(); + + bool is_reverse = context.Attr("is_reverse"); + math::LoDTensor2BatchFunctor to_batch; + auto& dev_ctx = context.template device_context(); + to_batch(dev_ctx, *input, batch_gate, true, is_reverse); + + if (bias) { + math::RowwiseAdd add_bias; + add_bias(dev_ctx, *batch_gate, *bias, batch_gate); + } + + int frame_size = hidden_dims[1]; + math::GRUMetaValue gru_value; + gru_value.gate_weight = const_cast(weight_data); + gru_value.state_weight = + const_cast(weight_data + 2 * frame_size * frame_size); + Tensor ordered_h0; + + framework::Vector order(batch_gate->lod()[2]); + + if (h0) { + // Since the batch computing for GRU reorders the input sequences + // according to their length. The initialized cell state also needs + // to reorder. + ReorderInitState( + context.template device_context(), *h0, order, + &ordered_h0, true); + gru_value.prev_out_value = ordered_h0.data(); + } else { + gru_value.prev_out_value = nullptr; + } + auto batch_starts = batch_gate->lod()[0]; + size_t seq_len = batch_starts.size() - 1; + auto active_node = math::detail::GetActivationType( + context.Attr("activation")); + auto active_gate = math::detail::GetActivationType( + context.Attr("gate_activation")); + +#ifdef PADDLE_WITH_MKLML + // use MKL packed to speedup GEMM + if (FLAGS_paddle_num_threads >= 4) { + auto blas = math::GetBlas(dev_ctx); + T* packed_gate = blas.GEMM_ALLOC(CblasBMatrix, 1 /*height of C*/, + frame_size * 2 /*width of weight*/, + frame_size /*height of height*/); + PADDLE_ENFORCE(packed_gate); + blas.GEMM_PACK(CblasBMatrix, CblasNoTrans, 1 /*cur bs?*/, frame_size * 2, + frame_size, T(1.0), gru_value.gate_weight, frame_size * 2, + packed_gate); + T* packed_state = blas.GEMM_ALLOC(CblasBMatrix, 1 /*height of C*/, + frame_size /*width of weight*/, + frame_size /*height of height*/); + PADDLE_ENFORCE(packed_state); + blas.GEMM_PACK(CblasBMatrix, CblasNoTrans, 1 /*cur bs?*/, frame_size, + frame_size, T(1.0), gru_value.state_weight, frame_size, + packed_state); + for (size_t n = 0; n < seq_len; n++) { + int bstart = static_cast(batch_starts[n]); + int bend = static_cast(batch_starts[n + 1]); + int cur_batch_size = bend - bstart; + + Tensor gate_t = batch_gate->Slice(bstart, bend); + Tensor reset_hidden_prev_t = + batch_reset_hidden_prev->Slice(bstart, bend); + Tensor hidden_t = batch_hidden->Slice(bstart, bend); + gru_value.output_value = hidden_t.data(); + gru_value.gate_value = gate_t.data(); + gru_value.reset_output_value = reset_hidden_prev_t.data(); + + if (gru_value.prev_out_value) { + blas.GEMM_COMPUTE( + CblasNoTrans, CblasPacked, cur_batch_size, frame_size * 2, + frame_size, gru_value.prev_out_value, frame_size, packed_gate, + frame_size * 2, T(1), gru_value.gate_value, frame_size * 3); + } + + math::detail::forward_reset_output( + math::detail::forward::gru_resetOutput(), gru_value, frame_size, + cur_batch_size, active_gate); + + if (gru_value.prev_out_value) { + blas.GEMM_COMPUTE( + CblasNoTrans, CblasPacked, cur_batch_size, frame_size, frame_size, + gru_value.reset_output_value, frame_size, packed_state, + frame_size, T(1), gru_value.gate_value + frame_size * 2, + frame_size * 3); + } + + math::detail::forward_final_output( + math::detail::forward::gru_finalOutput(), gru_value, frame_size, + cur_batch_size, active_node); + + gru_value.prev_out_value = gru_value.output_value; + } + + blas.GEMM_FREE(packed_gate); + blas.GEMM_FREE(packed_state); + } else { +#endif + for (size_t n = 0; n < seq_len; n++) { + int bstart = static_cast(batch_starts[n]); + int bend = static_cast(batch_starts[n + 1]); + int cur_batch_size = bend - bstart; + + Tensor gate_t = batch_gate->Slice(bstart, bend); + Tensor reset_hidden_prev_t = + batch_reset_hidden_prev->Slice(bstart, bend); + Tensor hidden_t = batch_hidden->Slice(bstart, bend); + gru_value.output_value = hidden_t.data(); + gru_value.gate_value = gate_t.data(); + gru_value.reset_output_value = reset_hidden_prev_t.data(); + + math::GRUUnitFunctor::compute( + dev_ctx, gru_value, frame_size, cur_batch_size, active_node, + active_gate); + + gru_value.prev_out_value = gru_value.output_value; + } +#ifdef PADDLE_WITH_MKLML + } +#endif + math::Batch2LoDTensorFunctor to_seq; + batch_hidden->set_lod(batch_gate->lod()); + to_seq(dev_ctx, *batch_hidden, hidden); + } + + void Compute(const framework::ExecutionContext& context) const override { + BatchCompute(context); + } +}; + } // namespace operators } // namespace paddle @@ -218,9 +375,8 @@ namespace ops = paddle::operators; REGISTER_OPERATOR(gru, ops::GRUOp, ops::GRUOpMaker, paddle::framework::DefaultGradOpDescMaker); REGISTER_OPERATOR(gru_grad, ops::GRUGradOp); -REGISTER_OP_CPU_KERNEL( - gru, ops::GRUKernel, - ops::GRUKernel); +REGISTER_OP_CPU_KERNEL(gru, ops::GRUCPUKernel, + ops::GRUCPUKernel); REGISTER_OP_CPU_KERNEL( gru_grad, ops::GRUGradKernel, ops::GRUGradKernel); diff --git a/paddle/fluid/operators/gru_op.cu.cc b/paddle/fluid/operators/gru_op.cu.cc index baf455a840314d1ab94eb8e0a2e5c660ba4202da..55721c283dd18c2f9642563a9ce1eabfce16fd7b 100644 --- a/paddle/fluid/operators/gru_op.cu.cc +++ b/paddle/fluid/operators/gru_op.cu.cc @@ -14,6 +14,96 @@ limitations under the License. */ #include "paddle/fluid/operators/gru_op.h" +namespace paddle { +namespace operators { + +template +class GRUKernel : public framework::OpKernel { + public: + void BatchCompute(const framework::ExecutionContext& context) const { + auto* input = context.Input("Input"); + auto* h0 = context.Input("H0"); + auto* weight = context.Input("Weight"); + const T* weight_data = weight->data(); + auto* bias = context.Input("Bias"); + auto* batch_gate = context.Output("BatchGate"); + batch_gate->mutable_data(context.GetPlace()); + auto* batch_reset_hidden_prev = + context.Output("BatchResetHiddenPrev"); + batch_reset_hidden_prev->mutable_data(context.GetPlace()); + auto* batch_hidden = context.Output("BatchHidden"); + batch_hidden->mutable_data(context.GetPlace()); + auto* hidden = context.Output("Hidden"); + hidden->mutable_data(context.GetPlace()); + + auto hidden_dims = hidden->dims(); + + bool is_reverse = context.Attr("is_reverse"); + math::LoDTensor2BatchFunctor to_batch; + auto& dev_ctx = context.template device_context(); + to_batch(dev_ctx, *input, batch_gate, true, is_reverse); + + if (bias) { + math::RowwiseAdd add_bias; + add_bias(dev_ctx, *batch_gate, *bias, batch_gate); + } + + int frame_size = hidden_dims[1]; + math::GRUMetaValue gru_value; + gru_value.gate_weight = const_cast(weight_data); + gru_value.state_weight = + const_cast(weight_data + 2 * frame_size * frame_size); + Tensor ordered_h0; + + framework::Vector order(batch_gate->lod()[2]); + + if (h0) { + // Since the batch computing for GRU reorders the input sequences + // according to their length. The initialized cell state also needs + // to reorder. + ReorderInitState( + context.template device_context(), *h0, order, + &ordered_h0, true); + gru_value.prev_out_value = ordered_h0.data(); + } else { + gru_value.prev_out_value = nullptr; + } + auto batch_starts = batch_gate->lod()[0]; + size_t num_batch = batch_starts.size() - 1; + auto active_node = math::detail::GetActivationType( + context.Attr("activation")); + auto active_gate = math::detail::GetActivationType( + context.Attr("gate_activation")); + for (size_t n = 0; n < num_batch; n++) { + int bstart = static_cast(batch_starts[n]); + int bend = static_cast(batch_starts[n + 1]); + int cur_batch_size = bend - bstart; + + Tensor gate_t = batch_gate->Slice(bstart, bend); + Tensor reset_hidden_prev_t = batch_reset_hidden_prev->Slice(bstart, bend); + Tensor hidden_t = batch_hidden->Slice(bstart, bend); + gru_value.output_value = hidden_t.data(); + gru_value.gate_value = gate_t.data(); + gru_value.reset_output_value = reset_hidden_prev_t.data(); + math::GRUUnitFunctor::compute( + dev_ctx, gru_value, frame_size, cur_batch_size, active_node, + active_gate); + gru_value.prev_out_value = gru_value.output_value; + } + + math::Batch2LoDTensorFunctor to_seq; + batch_hidden->set_lod(batch_gate->lod()); + to_seq(dev_ctx, *batch_hidden, hidden); + } + + void Compute(const framework::ExecutionContext& context) const override { + BatchCompute(context); + } +}; + +} // namespace operators +} // namespace paddle + namespace ops = paddle::operators; REGISTER_OP_CUDA_KERNEL( gru, ops::GRUKernel, diff --git a/paddle/fluid/operators/gru_op.h b/paddle/fluid/operators/gru_op.h index 3b0d93e54b72910de1429ddf41eb6b0fe9646942..0b551e8046be16c95f7d6b10b68b32a9af594f73 100644 --- a/paddle/fluid/operators/gru_op.h +++ b/paddle/fluid/operators/gru_op.h @@ -37,90 +37,6 @@ inline void ReorderInitState(const DeviceContext& ctx, row_shuffle(ctx, src, index_lod, dst, indexed_src); } -template -class GRUKernel : public framework::OpKernel { - public: - void BatchCompute(const framework::ExecutionContext& context) const { - auto* input = context.Input("Input"); - auto* h0 = context.Input("H0"); - auto* weight = context.Input("Weight"); - const T* weight_data = weight->data(); - auto* bias = context.Input("Bias"); - auto* batch_gate = context.Output("BatchGate"); - batch_gate->mutable_data(context.GetPlace()); - auto* batch_reset_hidden_prev = - context.Output("BatchResetHiddenPrev"); - batch_reset_hidden_prev->mutable_data(context.GetPlace()); - auto* batch_hidden = context.Output("BatchHidden"); - batch_hidden->mutable_data(context.GetPlace()); - auto* hidden = context.Output("Hidden"); - hidden->mutable_data(context.GetPlace()); - - auto hidden_dims = hidden->dims(); - - bool is_reverse = context.Attr("is_reverse"); - math::LoDTensor2BatchFunctor to_batch; - auto& dev_ctx = context.template device_context(); - to_batch(dev_ctx, *input, batch_gate, true, is_reverse); - - if (bias) { - math::RowwiseAdd add_bias; - add_bias(dev_ctx, *batch_gate, *bias, batch_gate); - } - - int frame_size = hidden_dims[1]; - math::GRUMetaValue gru_value; - gru_value.gate_weight = const_cast(weight_data); - gru_value.state_weight = - const_cast(weight_data + 2 * frame_size * frame_size); - Tensor ordered_h0; - - framework::Vector order(batch_gate->lod()[2]); - - if (h0) { - // Since the batch computing for GRU reorders the input sequences - // according to their length. The initialized cell state also needs - // to reorder. - ReorderInitState( - context.template device_context(), *h0, order, - &ordered_h0, true); - gru_value.prev_out_value = ordered_h0.data(); - } else { - gru_value.prev_out_value = nullptr; - } - auto batch_starts = batch_gate->lod()[0]; - size_t num_batch = batch_starts.size() - 1; - auto active_node = math::detail::GetActivationType( - context.Attr("activation")); - auto active_gate = math::detail::GetActivationType( - context.Attr("gate_activation")); - for (size_t n = 0; n < num_batch; n++) { - int bstart = static_cast(batch_starts[n]); - int bend = static_cast(batch_starts[n + 1]); - int cur_batch_size = bend - bstart; - - Tensor gate_t = batch_gate->Slice(bstart, bend); - Tensor reset_hidden_prev_t = batch_reset_hidden_prev->Slice(bstart, bend); - Tensor hidden_t = batch_hidden->Slice(bstart, bend); - gru_value.output_value = hidden_t.data(); - gru_value.gate_value = gate_t.data(); - gru_value.reset_output_value = reset_hidden_prev_t.data(); - math::GRUUnitFunctor::compute( - dev_ctx, gru_value, frame_size, cur_batch_size, active_node, - active_gate); - gru_value.prev_out_value = gru_value.output_value; - } - - math::Batch2LoDTensorFunctor to_seq; - batch_hidden->set_lod(batch_gate->lod()); - to_seq(dev_ctx, *batch_hidden, hidden); - } - - void Compute(const framework::ExecutionContext& context) const override { - BatchCompute(context); - } -}; - template class GRUGradKernel : public framework::OpKernel { public: diff --git a/paddle/fluid/operators/listen_and_serv_op.cc b/paddle/fluid/operators/listen_and_serv_op.cc index b1948076969e7a651179b97b107b85beea175280..f196e18fe122af9536230752096a2d90de8ab527 100644 --- a/paddle/fluid/operators/listen_and_serv_op.cc +++ b/paddle/fluid/operators/listen_and_serv_op.cc @@ -123,8 +123,11 @@ void ListenAndServOp::RunSyncLoop( optimize_prepared.begin(), std::shared_ptr(nullptr)); + // Trainers will get all parameters from pserver in the + // startup program, so we will wait RequestGet first + rpc_service_->SetCond(distributed::kRequestGet); + rpc_service_->WaitBarrier(distributed::kRequestGet); rpc_service_->ResetBarrierCounter(); - while (true) { rpc_service_->Profiler().OneStep(); // Get from multiple trainers, we don't care about the order in which diff --git a/paddle/fluid/operators/lookup_sparse_table_op.cc b/paddle/fluid/operators/lookup_sparse_table_op.cc index 2ce11e712fb1a8aa9748313ec7cf4e895a931465..de3f0990e109cacd49c4d888bbc1f797fb196e01 100644 --- a/paddle/fluid/operators/lookup_sparse_table_op.cc +++ b/paddle/fluid/operators/lookup_sparse_table_op.cc @@ -17,7 +17,6 @@ limitations under the License. */ #include "paddle/fluid/framework/data_type.h" #include "paddle/fluid/framework/op_registry.h" #include "paddle/fluid/operators/math/math_function.h" -#include "paddle/fluid/platform/device_context.h" namespace paddle { namespace operators { @@ -46,10 +45,6 @@ class LookupSparseTableOp : public framework::OperatorBase { auto out_var = scope.FindVar(Output("Out")); auto w_var = scope.FindVar(Input("W")); auto ids_var = scope.FindVar(Input("Ids")); - unsigned int seed = static_cast(Attr("seed")); - float min = Attr("min"); - float max = Attr("max"); - bool auto_grown_table = Attr("auto_grown_table"); PADDLE_ENFORCE(out_var->IsType(), "The type of Out var should be LodTensor."); @@ -60,46 +55,17 @@ class LookupSparseTableOp : public framework::OperatorBase { auto &ids_t = ids_var->Get(); auto out_t = out_var->GetMutable(); auto w_t = w_var->GetMutable(); - std::vector keys; - keys.resize(ids_t.numel()); - for (int64_t i = 0; i < ids_t.numel(); ++i) { - keys[i] = ids_t.data()[i]; - } // TODO(Yancey1989): support CUDA Place for the sparse table platform::CPUPlace cpu; auto out_shape = w_t->value().dims(); - out_shape[0] = keys.size(); + out_shape[0] = ids_t.numel(); out_t->Resize(out_shape); out_t->mutable_data(cpu, w_t->value().type()); PADDLE_ENFORCE_EQ(framework::ToDataType(w_t->value().type()), framework::proto::VarType::FP32, "The sparse table only support FP32"); - auto non_keys_pair = w_t->Get(keys, out_t); - if (!auto_grown_table) { - PADDLE_ENFORCE_EQ(non_keys_pair.size(), static_cast(0), - "there is some keys does exists in the sparse table."); - } - auto value_shape = w_t->value().dims(); - value_shape[0] = 1; - for (const auto &it : non_keys_pair) { - const auto key = it.first; - const auto index = it.second; - framework::Tensor value; - value.Resize(value_shape); - auto data = value.mutable_data(cpu); - - std::minstd_rand engine; - engine.seed(seed); - std::uniform_real_distribution dist(min, max); - int64_t size = value.numel(); - for (int64_t i = 0; i < size; ++i) { - data[i] = dist(engine); - } - w_t->Set(key, value); - memory::Copy(cpu, out_t->mutable_data(cpu) + index * value.numel(), - cpu, value.data(), value.numel() * sizeof(float)); - } + w_t->Get(ids_t, out_t, true); } }; @@ -121,21 +87,6 @@ class LookupSparseTableOpMaker : public framework::OpProtoAndCheckerMaker { "Otherwise the given value indicates padding the output " "with zeros whenever lookup encounters it in Ids.") .SetDefault(kNoPadding); - AddAttr("min", - "(float, default -1.0) " - "Minimum value of uniform random") - .SetDefault(-1.0f); - AddAttr("max", - "(float, default 1.0) " - "Maximum value of uniform random") - .SetDefault(1.0f); - AddAttr("seed", - "(int, default 0) " - "Random seed used for generating samples. " - "0 means use a seed generated by the system." - "Note that if seed is not 0, this operator will always " - "generate the same random numbers every time.") - .SetDefault(0); AddAttr("auto_grown_table", "(bool default false)" "Whether create new value if for nonexistent key.") diff --git a/paddle/fluid/operators/math/blas.h b/paddle/fluid/operators/math/blas.h index 70f88f24f682e05972ca73ef7b50f96be50d1ef4..2558154e0b39a4281bfaa59ba75867589d73be5d 100644 --- a/paddle/fluid/operators/math/blas.h +++ b/paddle/fluid/operators/math/blas.h @@ -90,6 +90,25 @@ class Blas { void GEMM(bool transA, bool transB, int M, int N, int K, T alpha, const T* A, int lda, const T* B, int ldb, T beta, T* C, int ldc) const; +#ifdef PADDLE_WITH_MKLML + template + T* GEMM_ALLOC(const CBLAS_IDENTIFIER id, const int M, const int N, + const int K) const; + + template + void GEMM_PACK(const CBLAS_IDENTIFIER id, const CBLAS_TRANSPOSE trans, int M, + int N, int K, const T alpha, const T* src, const int ld, + T* dst) const; + + template + void GEMM_COMPUTE(int transA, int transB, int M, int N, int K, const T* A, + const int lda, const T* B, const int ldb, T beta, T* C, + const int ldc) const; + + template + void GEMM_FREE(T* data) const; +#endif + template void MatMul(const framework::Tensor& mat_a, bool trans_a, const framework::Tensor& mat_b, bool trans_b, T alpha, @@ -146,6 +165,28 @@ class BlasT : private Blas { Base()->template GEMM(args...); } +#ifdef PADDLE_WITH_MKLML + template + T* GEMM_ALLOC(ARGS... args) const { + return Base()->template GEMM_ALLOC(args...); + } + + template + void GEMM_PACK(ARGS... args) const { + Base()->template GEMM_PACK(args...); + } + + template + void GEMM_COMPUTE(ARGS... args) const { + Base()->template GEMM_COMPUTE(args...); + } + + template + void GEMM_FREE(ARGS... args) const { + Base()->template GEMM_FREE(args...); + } +#endif + template void MatMul(ARGS... args) const { Base()->template MatMul(args...); diff --git a/paddle/fluid/operators/math/blas_impl.h b/paddle/fluid/operators/math/blas_impl.h index a0802ef90ca7e30a2b22d187cb9092163518d8e9..bf3382107960dfd8b52f94b421b49022dcb6d291 100644 --- a/paddle/fluid/operators/math/blas_impl.h +++ b/paddle/fluid/operators/math/blas_impl.h @@ -31,6 +31,26 @@ struct CBlas { platform::dynload::cblas_sgemm(args...); } + template + static float *GEMM_ALLOC(ARGS... args) { + return platform::dynload::cblas_sgemm_alloc(args...); + } + + template + static void GEMM_PACK(ARGS... args) { + platform::dynload::cblas_sgemm_pack(args...); + } + + template + static void GEMM_COMPUTE(ARGS... args) { + platform::dynload::cblas_sgemm_compute(args...); + } + + template + static void GEMM_FREE(ARGS... args) { + platform::dynload::cblas_sgemm_free(args...); + } + #ifdef PADDLE_WITH_LIBXSMM template static void SMM_GEMM(ARGS... args) { @@ -71,6 +91,26 @@ struct CBlas { platform::dynload::cblas_dgemm(args...); } + template + static double *GEMM_ALLOC(ARGS... args) { + return platform::dynload::cblas_dgemm_alloc(args...); + } + + template + static void GEMM_PACK(ARGS... args) { + platform::dynload::cblas_dgemm_pack(args...); + } + + template + static void GEMM_COMPUTE(ARGS... args) { + platform::dynload::cblas_dgemm_compute(args...); + } + + template + static void GEMM_FREE(ARGS... args) { + platform::dynload::cblas_dgemm_free(args...); + } + #ifdef PADDLE_WITH_LIBXSMM template static void SMM_GEMM(ARGS... args) { @@ -224,6 +264,41 @@ inline void GEMM_WARP(CBLAS_ORDER order, CBLAS_TRANSPOSE transA, beta, C, ldc); } +#ifdef PADDLE_WITH_MKLML +template <> +template +T *Blas::GEMM_ALLOC(const CBLAS_IDENTIFIER id, + const int M, const int N, + const int K) const { + return CBlas::GEMM_ALLOC(id, M, N, K); +} + +template <> +template +void Blas::GEMM_PACK(const CBLAS_IDENTIFIER id, + const CBLAS_TRANSPOSE trans, + int M, int N, int K, + const T alpha, const T *src, + const int ld, T *dst) const { + CBlas::GEMM_PACK(CblasRowMajor, id, trans, M, N, K, alpha, src, ld, dst); +} + +template <> +template +void Blas::GEMM_COMPUTE( + int transA, int transB, int M, int N, int K, const T *A, const int lda, + const T *B, const int ldb, T beta, T *C, const int ldc) const { + CBlas::GEMM_COMPUTE(CblasRowMajor, transA, transB, M, N, K, A, lda, B, ldb, + beta, C, ldc); +} + +template <> +template +void Blas::GEMM_FREE(T *data) const { + CBlas::GEMM_FREE(data); +} +#endif + template <> template void Blas::GEMM(CBLAS_TRANSPOSE transA, diff --git a/paddle/fluid/operators/prelu_op.cc b/paddle/fluid/operators/prelu_op.cc index db040509bc08c3f6ad031c5b97c93574e31337e0..23d9ea88f6701f9f9e5e02948e996878a849ddd6 100644 --- a/paddle/fluid/operators/prelu_op.cc +++ b/paddle/fluid/operators/prelu_op.cc @@ -1,11 +1,8 @@ /* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. - Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,14 +23,40 @@ class PReluOp : public framework::OperatorWithKernel { : OperatorWithKernel(type, inputs, outputs, attrs) {} void InferShape(framework::InferShapeContext *ctx) const override { + std::string mode = ctx->Attrs().Get("mode"); + + auto x_dim = ctx->GetInputDim("X"); PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) should not be null"); PADDLE_ENFORCE(ctx->HasInput("Alpha"), "Input(Alpha) should not be null"); - PADDLE_ENFORCE(product(ctx->GetInputDim("Alpha")) == 1, - "Size of weight Alpha must be one."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), "Output(Out) should not be null"); - ctx->SetOutputDim("Out", ctx->GetInputDim("X")); + if (mode == "all") { + PADDLE_ENFORCE(product(ctx->GetInputDim("Alpha")) == 1, + "For mode 'all', size of weight Alpha must be one."); + } else if (mode == "channel") { + PADDLE_ENFORCE(product(ctx->GetInputDim("Alpha")) == x_dim[1], + "For channel-wise mode, size of weight Alpha must be " + "equal to the number of channels, should be %d", + x_dim[1]); + } else if (mode == "element") { + PADDLE_ENFORCE(product(ctx->GetInputDim("Alpha")) == product(x_dim), + "For element-wise mode, size of weight Alpha must be " + "equal to the number of input, should be %d", + product(x_dim)); + } else { + PADDLE_THROW("Unkown mode %s", mode); + } + ctx->SetOutputDim("Out", x_dim); ctx->ShareLoD("X", /*->*/ "Out"); } + + protected: + framework::OpKernelType GetExpectedKernelType( + const framework::ExecutionContext &ctx) const override { + return framework::OpKernelType( + framework::ToDataType(ctx.Input("X")->type()), + platform::CPUPlace()); + } }; class PReluOpMaker : public framework::OpProtoAndCheckerMaker { @@ -44,9 +67,7 @@ class PReluOpMaker : public framework::OpProtoAndCheckerMaker { AddOutput("Out", "The output tensor of prelu operator."); AddComment(R"DOC( PRelu Operator. - The equation is: - $$ f(x) = \begin{cases} @@ -54,11 +75,15 @@ f(x) = x, \qquad \text{if} \ x >= 0 \end{cases} $$ - The input `X` can carry the LoD (Level of Details) information, or not. And the output shares the LoD information with input `X`. - +There are modes: + all: all elements share same weight + channel: elements in a channel share same weight + element: each element has a weight )DOC"); + AddAttr("mode", "The mode for inputs to share weights.") + .SetDefault("all"); } }; @@ -71,9 +96,23 @@ class PReluGradOp : public framework::OperatorWithKernel { PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) must not be null."); PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")), "Input(Out@GRAD) should not be null"); - ctx->SetOutputDim(framework::GradVarName("X"), ctx->GetInputDim("X")); - ctx->SetOutputDim(framework::GradVarName("Alpha"), - ctx->GetInputDim("Alpha")); + auto x_grad_name = framework::GradVarName("X"); + auto alpha_grad_name = framework::GradVarName("Alpha"); + + if (ctx->HasOutput(x_grad_name)) { + ctx->SetOutputDim(x_grad_name, ctx->GetInputDim("X")); + } + if (ctx->HasOutput(alpha_grad_name)) { + ctx->SetOutputDim(alpha_grad_name, ctx->GetInputDim("Alpha")); + } + } + + protected: + framework::OpKernelType GetExpectedKernelType( + const framework::ExecutionContext &ctx) const override { + return framework::OpKernelType( + framework::ToDataType(ctx.Input("X")->type()), + platform::CPUPlace()); } }; diff --git a/paddle/fluid/operators/prelu_op.cu b/paddle/fluid/operators/prelu_op.cu deleted file mode 100644 index 37d934a29046be04a1721b7330c813f663f61aed..0000000000000000000000000000000000000000 --- a/paddle/fluid/operators/prelu_op.cu +++ /dev/null @@ -1,22 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#include "paddle/fluid/operators/prelu_op.h" - -REGISTER_OP_CUDA_KERNEL( - prelu, - paddle::operators::PReluKernel); -REGISTER_OP_CUDA_KERNEL(prelu_grad, - paddle::operators::PReluGradKernel< - paddle::platform::CUDADeviceContext, float>); diff --git a/paddle/fluid/operators/prelu_op.h b/paddle/fluid/operators/prelu_op.h index a6197d354833a2f4173003ad2a970c487ad9a65b..f9076cbc678534fd5490fa0d7adeac0e50909a39 100644 --- a/paddle/fluid/operators/prelu_op.h +++ b/paddle/fluid/operators/prelu_op.h @@ -1,11 +1,8 @@ /* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. - Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -13,32 +10,16 @@ See the License for the specific language governing permissions and limitations under the License. */ #pragma once +#include #include "paddle/fluid/framework/eigen.h" #include "paddle/fluid/framework/op_registry.h" #include "paddle/fluid/platform/transform.h" - namespace paddle { namespace operators { using Tensor = framework::Tensor; using platform::Transform; -template -class PReluFunctor { - public: - explicit PReluFunctor(const T* alpha) : alpha_(alpha) {} - - HOSTDEVICE T operator()(const T& x) const { - if (x > 0) - return x; - else - return x * (*alpha_); - } - - private: - const T* alpha_; -}; - template class PReluKernel : public framework::OpKernel { public: @@ -50,53 +31,93 @@ class PReluKernel : public framework::OpKernel { const T* x_ptr = x->data(); T* o_ptr = out->mutable_data(context.GetPlace()); - auto* alpha_ptr = alpha->data(); + const T* alpha_ptr = alpha->data(); + std::string mode = context.Attr("mode"); int numel = x->numel(); - - Transform trans; - trans(context.template device_context(), x_ptr, - x_ptr + numel, o_ptr, PReluFunctor(alpha_ptr)); - } -}; - -template -class PReluGradFunctor { - public: - explicit PReluGradFunctor(const T* alpha) : alpha_(alpha) {} - - HOSTDEVICE T operator()(const T& out, const T& dout) const { - if (out > 0) - return dout; - else - return dout * (*alpha_); + auto dim = x->dims(); + int index = 0; + int i = 0; + int temp = 0; + if (mode == "channel") { + for (i = 0; i < numel; i++) { + temp = numel / (dim[0] * dim[1]); + index = (i / temp) % dim[1]; + o_ptr[i] = x_ptr[i] > 0 ? x_ptr[i] : alpha_ptr[index] * x_ptr[i]; + } + } else if (mode == "element") { + for (i = 0; i < numel; i++) { + o_ptr[i] = x_ptr[i] > 0 ? x_ptr[i] : alpha_ptr[i] * x_ptr[i]; + } + } else { + for (i = 0; i < numel; i++) { + o_ptr[i] = x_ptr[i] > 0 ? x_ptr[i] : alpha_ptr[0] * x_ptr[i]; + } + } } - - private: - const T* alpha_; }; template class PReluGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { + auto* x = context.Input("X"); auto* dx = context.Output(framework::GradVarName("X")); auto* dout = context.Input(framework::GradVarName("Out")); - + auto* dalpha = context.Output(framework::GradVarName("Alpha")); auto* out = context.Input("Out"); auto* alpha = context.Input("Alpha"); - auto* alpha_ptr = alpha->data(); - - T* dx_ptr = dx->mutable_data(context.GetPlace()); + const T* alpha_ptr = alpha->data(); + const T* x_ptr = x->data(); const T* dout_ptr = dout->data(); const T* out_ptr = out->data(); - int numel = dx->numel(); - - Transform trans; - trans(context.template device_context(), out_ptr, - out_ptr + numel, dout_ptr, dx_ptr, PReluGradFunctor(alpha_ptr)); - - // TODO(Zhuoyuan): add dalpha upgrade when GPU kernels ready + std::string mode = context.Attr("mode"); + int numel = x->numel(); + auto dim = x->dims(); + int index = 0; + int i = 0; + int temp = 0; + if (dx) { + T* dx_ptr = dx->mutable_data(context.GetPlace()); + if (mode == "channel") { + for (i = 0; i < numel; i++) { + temp = numel / (dim[0] * dim[1]); + index = (i / temp) % dim[1]; + dx_ptr[i] = + out_ptr[i] > 0 ? dout_ptr[i] : alpha_ptr[index] * dout_ptr[i]; + } + } else if (mode == "element") { + for (i = 0; i < numel; i++) { + dx_ptr[i] = out_ptr[i] > 0 ? dout_ptr[i] : alpha_ptr[i] * dout_ptr[i]; + } + } else { + for (i = 0; i < numel; i++) { + dx_ptr[i] = out_ptr[i] > 0 ? dout_ptr[i] : alpha_ptr[0] * dout_ptr[i]; + } + } + } + + index = 0; + if (dalpha) { + T* dalpha_ptr = dalpha->mutable_data(context.GetPlace()); + if (mode == "channel") { + for (i = 0; i < numel; i++) { + temp = numel / (dim[0] * dim[1]); + index = (i / temp) % dim[1]; + dalpha_ptr[index] += out_ptr[i] > 0 ? 0 : x_ptr[i] * dout_ptr[i]; + } + } else if (mode == "element") { + for (i = 0; i < numel; i++) { + dalpha_ptr[i] += out_ptr[i] > 0 ? 0 : x_ptr[i] * dout_ptr[i]; + } + } else { + for (i = 0; i < numel; i++) { + dalpha_ptr[0] += out_ptr[i] > 0 ? 0 : x_ptr[i] * dout_ptr[i]; + } + } + } + + // TODO(Guanzhong): add GPU kernels } }; diff --git a/paddle/fluid/operators/sampling_id_op.cc b/paddle/fluid/operators/sampling_id_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..724463c95c4a29fb5c00fe791b389d3908771640 --- /dev/null +++ b/paddle/fluid/operators/sampling_id_op.cc @@ -0,0 +1,76 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/operators/sampling_id_op.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; + +class SamplingIdOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), + "Input(X) of SamplingIdOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of SamplingIdOp should not be null."); + PADDLE_ENFORCE( + ctx->Attrs().Get("min") < ctx->Attrs().Get("max"), + "min must less then max"); + + auto input_dims = ctx->GetInputDim("X"); + PADDLE_ENFORCE(input_dims.size() == 2, + "Input(X, Filter) should be 2-D tensor."); + + framework::DDim dims = input_dims; + ctx->SetOutputDim("Out", dims); + ctx->ShareLoD("X", "Out"); + } +}; + +class SamplingIdOpMaker : public framework::OpProtoAndCheckerMaker { + public: + void Make() override { + AddInput("X", + "The input tensor of softmax. " + "2-D with shape [batch_size, input_feature_dimensions]."); + AddOutput("Out", "SamplingId data tensor."); + AddComment(R"DOC( +SamplingId Operator. +A layer for sampling id from multinomial distribution from the + input. Sampling one id for one sample.)DOC"); + AddAttr("min", "Minimum value of random. [default 0.0].") + .SetDefault(0.0f); + AddAttr("max", "Maximun value of random. [default 1.0].") + .SetDefault(1.0f); + AddAttr("seed", + "Random seed used for the random number engine. " + "0 means use a seed generated by the system." + "Note that if seed is not 0, this operator will always " + "generate the same random numbers every time. [default 0].") + .SetDefault(0); + } +}; +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OPERATOR(sampling_id, ops::SamplingIdOp, ops::SamplingIdOpMaker, + paddle::framework::EmptyGradOpMaker); + +REGISTER_OP_CPU_KERNEL(sampling_id, paddle::operators::SamplingIdKernel, + paddle::operators::SamplingIdKernel); diff --git a/paddle/fluid/operators/sampling_id_op.cu b/paddle/fluid/operators/sampling_id_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..a4f0470314d00b5e370fd478736b54579c88448c --- /dev/null +++ b/paddle/fluid/operators/sampling_id_op.cu @@ -0,0 +1,19 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/fluid/operators/sampling_id_op.h" + +namespace ops = paddle::operators; +REGISTER_OP_CUDA_KERNEL(sampling_id, paddle::operators::SamplingIdKernel, + paddle::operators::SamplingIdKernel); diff --git a/paddle/fluid/operators/sampling_id_op.h b/paddle/fluid/operators/sampling_id_op.h new file mode 100644 index 0000000000000000000000000000000000000000..f730a9746da56ca82090122193ec54efb774483e --- /dev/null +++ b/paddle/fluid/operators/sampling_id_op.h @@ -0,0 +1,80 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "paddle/fluid/framework/op_registry.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; + +template +class SamplingIdKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + const Tensor* input = context.Input("X"); + const int batch_size = static_cast(input->dims()[0]); + const int width = static_cast(input->dims()[1]); + + PADDLE_ENFORCE_GE(batch_size, 0, + "batch_size(dims[0]) must be nonnegative."); + PADDLE_ENFORCE_GE(width, 0, "width(dims[1]) must be nonnegative."); + + std::vector ins_vector; + framework::TensorToVector(*input, context.device_context(), &ins_vector); + + unsigned int seed = static_cast(context.Attr("seed")); + std::minstd_rand engine; + if (seed == 0) { + seed = std::random_device()(); + } + engine.seed(seed); + std::uniform_real_distribution dist( + static_cast(context.Attr("min")), + static_cast(context.Attr("max"))); + + std::vector ids(batch_size); + for (size_t i = 0; i < batch_size; ++i) { + T r = dist(engine); + int idx = width - 1; + for (int j = 0; j < width; ++j) { + if ((r -= ins_vector[i * width + j]) < 0) { + idx = j; + break; + } + } + ids[i] = ins_vector[i * width + idx]; + } + + std::vector out_dim; + out_dim.push_back(static_cast(batch_size)); + + Tensor* output = context.Output("Out"); + output->Resize(framework::make_ddim(out_dim)); + output->mutable_data(context.GetPlace()); + framework::TensorFromVector(ids, context.device_context(), output); + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/fluid/operators/scatter_op.h b/paddle/fluid/operators/scatter_op.h index d29947b55e751a3e7993f765198364f4debe2472..181bb1af5cce7fad228e61e1a76ed66a9bd61b3e 100644 --- a/paddle/fluid/operators/scatter_op.h +++ b/paddle/fluid/operators/scatter_op.h @@ -35,7 +35,7 @@ class ScatterOpKernel : public framework::OpKernel { auto *Out = ctx.Output("Out"); // In place output: Out = X, Out[Ids] += Updates - Out->ShareDataWith(*X); + framework::TensorCopySync(*X, ctx.GetPlace(), Out); // Apply ScatterUpdate: Out[index] += Updates[:] ScatterAssign(ctx.device_context(), *Updates, *Ids, Out); } @@ -53,7 +53,7 @@ class ScatterGradientOpKernel : public framework::OpKernel { auto *dOut = ctx.Input(framework::GradVarName("Out")); // In place gradient: dX = dO - dX->ShareDataWith(*dOut); + framework::TensorCopySync(*dOut, ctx.GetPlace(), dX); dUpdates->mutable_data(ctx.GetPlace()); // Gradient by Gather: dUpdates += dO[Ids] CPUGather(ctx.device_context(), *dOut, *Ids, dUpdates); diff --git a/paddle/fluid/operators/sgd_op.h b/paddle/fluid/operators/sgd_op.h index 2685ce217ee0f0d3e89f3751e96218dcd19bead4..d8b0165b2a89b04bd55671a37d96ee4ba275b2eb 100644 --- a/paddle/fluid/operators/sgd_op.h +++ b/paddle/fluid/operators/sgd_op.h @@ -111,7 +111,7 @@ class SGDOpKernel : public framework::OpKernel { for (size_t i = 0; i < grad.rows().size(); i++) { PADDLE_ENFORCE(grad.rows()[i] < grad.height(), "Input rows index should less than height"); - int64_t id_index = param.Index(grad.rows()[i]); + int64_t id_index = param_out->AutoGrownIndex(grad.rows()[i], false); PADDLE_ENFORCE_GE(id_index, static_cast(0), "id should be in the table"); for (int64_t j = 0; j < grad_row_width; j++) { diff --git a/paddle/fluid/operators/shape_op.cc b/paddle/fluid/operators/shape_op.cc index b44d5f898013a5d27467bd80118c29a886d5e8b3..1be9fe47af71d31ce2e0eba807ea4a43601f8aca 100644 --- a/paddle/fluid/operators/shape_op.cc +++ b/paddle/fluid/operators/shape_op.cc @@ -38,7 +38,7 @@ class ShapeOpMaker : public framework::OpProtoAndCheckerMaker { AddInput("Input", "(Tensor), The input tensor."); AddOutput("Out", "(Tensor), The shape of input tensor, the data type of the shape" - " is int64_t, will be on the same device with the input Tensor."); + " is int32_t, will be on the same device with the input Tensor."); AddComment(R"DOC( Shape Operator @@ -53,5 +53,5 @@ Get the shape of input tensor. Only support CPU input Tensor now. namespace ops = paddle::operators; REGISTER_OPERATOR(shape, ops::ShapeOp, ops::ShapeOpMaker, paddle::framework::EmptyGradOpMaker); -REGISTER_OP_CPU_KERNEL(shape, ops::ShapeKernel, ops::ShapeKernel, +REGISTER_OP_CPU_KERNEL(shape, ops::ShapeKernel, ops::ShapeKernel, ops::ShapeKernel, ops::ShapeKernel); diff --git a/paddle/fluid/operators/shape_op.cu b/paddle/fluid/operators/shape_op.cu index 7736a2a1e13cfa5d445411b3efac7669a7bf23a2..d8fa9515abf807ab4ae3c47e8e1b1cf7f30440a8 100644 --- a/paddle/fluid/operators/shape_op.cu +++ b/paddle/fluid/operators/shape_op.cu @@ -15,6 +15,6 @@ limitations under the License. */ #include "paddle/fluid/operators/shape_op.h" REGISTER_OP_CUDA_KERNEL(shape, paddle::operators::ShapeKernel, - paddle::operators::ShapeKernel, + paddle::operators::ShapeKernel, paddle::operators::ShapeKernel, paddle::operators::ShapeKernel); diff --git a/paddle/fluid/operators/shape_op.h b/paddle/fluid/operators/shape_op.h index 3be86b66a538e7b38a5d59095fee7e7636364bce..0d510a505583c55e26a26bfc6e5d6192899b3d9e 100644 --- a/paddle/fluid/operators/shape_op.h +++ b/paddle/fluid/operators/shape_op.h @@ -27,7 +27,7 @@ class ShapeKernel : public framework::OpKernel { void Compute(const framework::ExecutionContext& ctx) const override { auto* in_t = ctx.Input("Input"); auto* out_t = ctx.Output("Out"); - auto out_data = out_t->mutable_data(platform::CPUPlace()); + auto out_data = out_t->mutable_data(platform::CPUPlace()); auto in_dims = in_t->dims(); for (int i = 0; i < in_dims.size(); ++i) { out_data[i] = in_dims[i]; diff --git a/paddle/fluid/operators/uniform_random_op.cc b/paddle/fluid/operators/uniform_random_op.cc index edd1baa4ace4e246190afcd12b0716f1dd38e243..5248767c2eeb9388c26d203e64f8b2c68ffe0865 100644 --- a/paddle/fluid/operators/uniform_random_op.cc +++ b/paddle/fluid/operators/uniform_random_op.cc @@ -30,8 +30,10 @@ class CPUUniformRandomKernel : public framework::OpKernel { tensor = out_var->GetMutable(); } else if (out_var->IsType()) { auto shape = ctx.Attr>("shape"); - tensor = out_var->GetMutable()->mutable_value(); + auto* selected_rows = out_var->GetMutable(); + tensor = selected_rows->mutable_value(); tensor->Resize(framework::make_ddim(shape)); + selected_rows->mutable_rows()->reserve(shape[0]); } else { PADDLE_THROW( "uniform_random_op's output only" diff --git a/paddle/fluid/platform/dynload/mklml.h b/paddle/fluid/platform/dynload/mklml.h index 17acefe8cde01809572e4c86cbdccfed9a477a51..9e7a616094e184695de521aa035257bde4170a91 100644 --- a/paddle/fluid/platform/dynload/mklml.h +++ b/paddle/fluid/platform/dynload/mklml.h @@ -60,6 +60,14 @@ extern void* mklml_dso_handle; __macro(cblas_dgemm_batch); \ __macro(vsAdd); \ __macro(vdAdd); \ + __macro(cblas_sgemm_alloc); \ + __macro(cblas_sgemm_pack); \ + __macro(cblas_sgemm_compute); \ + __macro(cblas_sgemm_free); \ + __macro(cblas_dgemm_alloc); \ + __macro(cblas_dgemm_pack); \ + __macro(cblas_dgemm_compute); \ + __macro(cblas_dgemm_free); \ __macro(MKL_Set_Num_Threads) MKLML_ROUTINE_EACH(DECLARE_DYNAMIC_LOAD_MKLML_WRAP); diff --git a/paddle/fluid/platform/enforce.h b/paddle/fluid/platform/enforce.h index 566485cd3c383640047d97f40b452735e8c8c171..81b5359b40589d898bda0dfa71afb6f51385354b 100644 --- a/paddle/fluid/platform/enforce.h +++ b/paddle/fluid/platform/enforce.h @@ -263,7 +263,8 @@ inline void throw_on_error(T e) { * PADDLE_ENFORCE_EQ(a, b); * * will raise an expression described as follows: - * "enforce a == b failed, 1 != 2" with detailed stack information. + * "Enforce failed. Expected input a == b, but received a(1) != b(2)." + * with detailed stack information. * * extra messages is also supported, for example: * PADDLE_ENFORCE(a, b, "some simple enforce failed between %d numbers", 2) @@ -292,9 +293,10 @@ inline void throw_on_error(T e) { #define __PADDLE_BINARY_COMPARE(__VAL0, __VAL1, __CMP, __INV_CMP, ...) \ do { \ if (UNLIKELY(!((__VAL0)__CMP(__VAL1)))) { \ - PADDLE_THROW("enforce %s " #__CMP " %s failed, %s " #__INV_CMP \ - " %s\n%s", \ - #__VAL0, #__VAL1, paddle::string::to_string(__VAL0), \ + PADDLE_THROW("Enforce failed. Expected %s " #__CMP \ + " %s, but received %s:%s " #__INV_CMP " %s:%s.\n%s", \ + #__VAL0, #__VAL1, #__VAL0, \ + paddle::string::to_string(__VAL0), #__VAL1, \ paddle::string::to_string(__VAL1), \ paddle::string::Sprintf("" __VA_ARGS__)); \ } \ diff --git a/paddle/fluid/platform/enforce_test.cc b/paddle/fluid/platform/enforce_test.cc index 0e8684581a93f076b1a077cc52e966d3c88cf078..d52182965552e9ec945cb7d0b421d8addcb758e9 100644 --- a/paddle/fluid/platform/enforce_test.cc +++ b/paddle/fluid/platform/enforce_test.cc @@ -54,7 +54,9 @@ TEST(ENFORCE_EQ, NO_EXTRA_MSG_FAIL) { PADDLE_ENFORCE_EQ(a, 1 + 3); } catch (paddle::platform::EnforceNotMet error) { caught_exception = true; - HasPrefix(StringPiece(error.what()), "enforce a == 1 + 3 failed, 2 != 4"); + HasPrefix( + StringPiece(error.what()), + "Enforce failed. Expected a == 1 + 3, but received a:2 != 1 + 3:4."); } EXPECT_TRUE(caught_exception); } @@ -67,7 +69,8 @@ TEST(ENFORCE_EQ, EXTRA_MSG_FAIL) { } catch (paddle::platform::EnforceNotMet error) { caught_exception = true; HasPrefix(StringPiece(error.what()), - "enforce a == 1 + 3 failed, 2 != 4\ntheir size not match"); + "Enforce failed. Expected a == 1 + 3, but received a:2 != 1 + " + "3:4.\ntheir size not match"); } EXPECT_TRUE(caught_exception); } @@ -84,8 +87,9 @@ TEST(ENFORCE_NE, FAIL) { PADDLE_ENFORCE_NE(1.0, 1UL); } catch (paddle::platform::EnforceNotMet error) { caught_exception = true; - EXPECT_TRUE(HasPrefix(StringPiece(error.what()), - "enforce 1.0 != 1UL failed, 1 == 1")) + EXPECT_TRUE(HasPrefix( + StringPiece(error.what()), + "Enforce failed. Expected 1.0 != 1UL, but received 1.0:1 == 1UL:1.")) << error.what() << " does not have expected prefix"; } EXPECT_TRUE(caught_exception); @@ -98,8 +102,9 @@ TEST(ENFORCE_GT, FAIL) { PADDLE_ENFORCE_GT(1, 2UL); } catch (paddle::platform::EnforceNotMet error) { caught_exception = true; - EXPECT_TRUE( - HasPrefix(StringPiece(error.what()), "enforce 1 > 2UL failed, 1 <= 2")); + EXPECT_TRUE(HasPrefix( + StringPiece(error.what()), + "Enforce failed. Expected 1 > 2UL, but received 1:1 <= 2UL:2.")); } EXPECT_TRUE(caught_exception); } @@ -116,8 +121,9 @@ TEST(ENFORCE_GE, FAIL) { PADDLE_ENFORCE_GE(1, 2UL); } catch (paddle::platform::EnforceNotMet error) { caught_exception = true; - EXPECT_TRUE( - HasPrefix(StringPiece(error.what()), "enforce 1 >= 2UL failed, 1 < 2")); + EXPECT_TRUE(HasPrefix( + StringPiece(error.what()), + "Enforce failed. Expected 1 >= 2UL, but received 1:1 < 2UL:2.")); } EXPECT_TRUE(caught_exception); } @@ -135,8 +141,9 @@ TEST(ENFORCE_LE, FAIL) { PADDLE_ENFORCE_GT(1, 2UL); } catch (paddle::platform::EnforceNotMet error) { caught_exception = true; - EXPECT_TRUE( - HasPrefix(StringPiece(error.what()), "enforce 1 > 2UL failed, 1 <= 2")); + EXPECT_TRUE(HasPrefix( + StringPiece(error.what()), + "Enforce failed. Expected 1 > 2UL, but received 1:1 <= 2UL:2.")); } EXPECT_TRUE(caught_exception); } @@ -153,7 +160,8 @@ TEST(ENFORCE_LT, FAIL) { } catch (paddle::platform::EnforceNotMet error) { caught_exception = true; EXPECT_TRUE(HasPrefix(StringPiece(error.what()), - "enforce 1UL < 0.12 failed, 1 >= 0.12")); + "Enforce failed. Expected 1UL < 0.12, but " + "received 1UL:1 >= 0.12:0.12.")); } EXPECT_TRUE(caught_exception); } diff --git a/paddle/fluid/platform/gpu_info.cc b/paddle/fluid/platform/gpu_info.cc index 4cee93f3a4224cb97327254cd1679021d197a1b1..126636d879213b1c8f242db8fbdf6a358a1d2da9 100644 --- a/paddle/fluid/platform/gpu_info.cc +++ b/paddle/fluid/platform/gpu_info.cc @@ -116,7 +116,8 @@ size_t GpuMaxChunkSize() { size_t allocating = static_cast(FLAGS_fraction_of_gpu_memory_to_use * (total - reserving)); - PADDLE_ENFORCE_LE(allocating, available); + PADDLE_ENFORCE_LE(allocating, available, + "Insufficient GPU memory to allocation."); return allocating; } diff --git a/paddle/fluid/pybind/protobuf.cc b/paddle/fluid/pybind/protobuf.cc index be623703c2480774bb04a6bc0c5b00b699d7bb16..c2137ec6d7df24251432a4dfb8fffc3d3f77194e 100644 --- a/paddle/fluid/pybind/protobuf.cc +++ b/paddle/fluid/pybind/protobuf.cc @@ -205,12 +205,7 @@ void BindBlockDesc(pybind11::module *m) { void BindVarDsec(pybind11::module *m) { pybind11::class_ var_desc(*m, "VarDesc", ""); var_desc - .def("name", - [](pd::VarDesc &self) { - pybind11::bytes name = self.Name(); - return name; - }, - pybind11::return_value_policy::reference) + .def("name", &pd::VarDesc::Name, pybind11::return_value_policy::reference) .def("set_name", &pd::VarDesc::SetName) .def("set_shape", &pd::VarDesc::SetShape) .def("set_shapes", &pd::VarDesc::SetShapes) diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index 7127bb38f6ddf8a55c1741d1f0ef18c8d9067fba..6c58478b0dd0941ab4bf4d573a3c813059650ba8 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -54,6 +54,8 @@ limitations under the License. */ #include "paddle/fluid/platform/gpu_info.h" #endif +#include "pybind11/stl.h" + // disable auto conversion to list in Python PYBIND11_MAKE_OPAQUE(paddle::framework::LoDTensorArray); @@ -247,6 +249,7 @@ PYBIND11_PLUGIN(core) { self.set_rows(new_rows); #endif }) + .def("sync_index", [](SelectedRows &instance) { instance.SyncIndex(); }) .def("rows", [](SelectedRows &self) { auto rows = self.rows(); std::vector new_rows; diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 25900811509aee8b37fdaf09cf902ea2ae3eee57..9cdcb87df5dd1669066c204c86c269973df506f1 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -97,10 +97,11 @@ if(APPLE) if(NOT INSTALL_NAME_TOOL_EXECUTABLE) message(FATAL_ERROR "install_name_tool not found, please check.\n") endif() -else(APPLE) +endif() +if(LINUX) find_program(PATCHELF_EXECUTABLE patchelf) if(NOT PATCHELF_EXECUTABLE) message(FATAL_ERROR "patchelf not found, please install it.\n" "For Ubuntu, the command is: apt-get install -y patchelf.") endif() -endif(APPLE) +endif(LINUX) diff --git a/python/paddle/__init__.py b/python/paddle/__init__.py index 241a07a35297e85763781a42696fd727733459a3..53746afdb25b34b69f89fe0927c877ace62d7d55 100644 --- a/python/paddle/__init__.py +++ b/python/paddle/__init__.py @@ -24,4 +24,5 @@ except ImportError: import paddle.reader import paddle.dataset import paddle.batch +import paddle.compat batch = batch.batch diff --git a/python/paddle/compat.py b/python/paddle/compat.py new file mode 100644 index 0000000000000000000000000000000000000000..50726b6fa1bbbde68a590c86db9344b8f02f79f2 --- /dev/null +++ b/python/paddle/compat.py @@ -0,0 +1,237 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import six +import math + +__all__ = [ + 'long_type', + 'to_text', + 'to_bytes', + 'round', + 'floor_division', + 'get_exception_message', +] + +if six.PY2: + int_type = int + long_type = long +else: + int_type = int + long_type = int + + +# str and bytes related functions +def to_text(obj, encoding='utf-8', inplace=False): + """ + All string in PaddlePaddle should be represented as a literal string. + This function will convert object to a literal string without any encoding. + Especially, if the object type is a list or set container, we will iterate + all items in the object and convert them to literal string. + + In Python3: + Decode the bytes type object to str type with specific encoding + + In Python2: + Decode the str type object to unicode type with specific encoding + + Args: + obj(unicode|str|bytes|list|set) : The object to be decoded. + encoding(str) : The encoding format to decode a string + inplace(bool) : If we change the original object or we create a new one + + Returns: + Decoded result of obj + """ + if obj is None: + return obj + + if isinstance(obj, list): + if inplace: + for i in six.moves.xrange(len(obj)): + obj[i] = _to_text(obj[i], encoding) + return obj + else: + return [_to_text(item, encoding) for item in obj] + elif isinstance(obj, set): + if inplace: + for item in obj: + obj.remove(item) + obj.add(_to_text(item, encoding)) + return obj + else: + return set([_to_text(item, encoding) for item in obj]) + else: + return _to_text(obj, encoding) + + +def _to_text(obj, encoding): + """ + In Python3: + Decode the bytes type object to str type with specific encoding + + In Python2: + Decode the str type object to unicode type with specific encoding, + or we just return the unicode string of object + + Args: + obj(unicode|str|bytes) : The object to be decoded. + encoding(str) : The encoding format + + Returns: + decoded result of obj + """ + if obj is None: + return obj + + if isinstance(obj, six.binary_type): + return obj.decode(encoding) + elif isinstance(obj, six.text_type): + return obj + else: + return six.u(obj) + + +def to_bytes(obj, encoding='utf-8', inplace=False): + """ + All string in PaddlePaddle should be represented as a literal string. + This function will convert object to a bytes with specific encoding. + Especially, if the object type is a list or set container, we will iterate + all items in the object and convert them to bytes. + + In Python3: + Encode the str type object to bytes type with specific encoding + + In Python2: + Encode the unicode type object to str type with specific encoding, + or we just return the 8-bit string of object + + Args: + obj(unicode|str|bytes|list|set) : The object to be encoded. + encoding(str) : The encoding format to encode a string + inplace(bool) : If we change the original object or we create a new one + + Returns: + Decoded result of obj + """ + if obj is None: + return obj + + if isinstance(obj, list): + if inplace: + for i in six.moves.xrange(len(obj)): + obj[i] = _to_bytes(obj[i], encoding) + return obj + else: + return [_to_bytes(item, encoding) for item in obj] + elif isinstance(obj, set): + if inplace: + for item in obj: + obj.remove(item) + obj.add(_to_bytes(item, encoding)) + return obj + else: + return set([_to_bytes(item, encoding) for item in obj]) + else: + return _to_bytes(obj, encoding) + + +def _to_bytes(obj, encoding): + """ + In Python3: + Encode the str type object to bytes type with specific encoding + + In Python2: + Encode the unicode type object to str type with specific encoding, + or we just return the 8-bit string of object + + Args: + obj(unicode|str|bytes) : The object to be encoded. + encoding(str) : The encoding format + + Returns: + encoded result of obj + """ + if obj is None: + return obj + + assert encoding is not None + if isinstance(obj, six.text_type): + return obj.encode(encoding) + elif isinstance(obj, six.binary_type): + return obj + else: + return six.b(obj) + + +# math related functions +def round(x, d=0): + """ + Compatible round which act the same behaviour in Python3. + + Args: + x(float) : The number to round halfway. + + Returns: + round result of x + """ + if six.PY3: + # The official walkaround of round in Python3 is incorrect + # we implement accroding this answer: https://www.techforgeek.info/round_python.html + if x > 0.0: + p = 10**d + return float(math.floor((x * p) + math.copysign(0.5, x))) / p + elif x < 0.0: + p = 10**d + return float(math.ceil((x * p) + math.copysign(0.5, x))) / p + else: + return math.copysign(0.0, x) + else: + import __builtin__ + return __builtin__.round(x, d) + + +def floor_division(x, y): + """ + Compatible division which act the same behaviour in Python3 and Python2, + whose result will be a int value of floor(x / y) in Python3 and value of + (x / y) in Python2. + + Args: + x(int|float) : The number to divide. + y(int|float) : The number to be divided + + Returns: + division result of x // y + """ + return x // y + + +# exception related functions +def get_exception_message(exc): + """ + Get the error message of a specific exception + + Args: + exec(Exception) : The exception to get error message. + + Returns: + the error message of exec + """ + assert exc is not None + + if six.PY2: + return exc.message + else: + return str(exc) diff --git a/python/paddle/dataset/cifar.py b/python/paddle/dataset/cifar.py index f6b4ff8fbd0f83b1d652d37c1b2d04efd3c73cbb..b83fa78c4c65357407b7f884f8c3fe8ef0ccaba8 100644 --- a/python/paddle/dataset/cifar.py +++ b/python/paddle/dataset/cifar.py @@ -28,11 +28,13 @@ images per class. """ +from __future__ import print_function + import itertools import numpy import paddle.dataset.common import tarfile -from six.moves import zip +import six from six.moves import cPickle as pickle __all__ = ['train100', 'test100', 'train10', 'test10', 'convert'] @@ -46,10 +48,11 @@ CIFAR100_MD5 = 'eb9058c3a382ffc7106e4002c42a8d85' def reader_creator(filename, sub_name, cycle=False): def read_batch(batch): - data = batch['data'] - labels = batch.get('labels', batch.get('fine_labels', None)) + data = batch[six.b('data')] + labels = batch.get( + six.b('labels'), batch.get(six.b('fine_labels'), None)) assert labels is not None - for sample, label in zip(data, labels): + for sample, label in six.moves.zip(data, labels): yield (sample / 255.0).astype(numpy.float32), int(label) def reader(): @@ -59,7 +62,11 @@ def reader_creator(filename, sub_name, cycle=False): while True: for name in names: - batch = pickle.load(f.extractfile(name)) + if six.PY2: + batch = pickle.load(f.extractfile(name)) + else: + batch = pickle.load( + f.extractfile(name), encoding='bytes') for item in read_batch(batch): yield item if not cycle: diff --git a/python/paddle/dataset/common.py b/python/paddle/dataset/common.py index 6195cc50df338e83bea1f4ad416529464636a33e..1d7ff582c86a40c8c2086e0de16e89d69c94da60 100644 --- a/python/paddle/dataset/common.py +++ b/python/paddle/dataset/common.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import requests import hashlib import os @@ -85,10 +87,10 @@ def download(url, module_name, md5sum, save_name=None): total_length = r.headers.get('content-length') if total_length is None: - with open(filename, 'w') as f: + with open(filename, 'wb') as f: shutil.copyfileobj(r.raw, f) else: - with open(filename, 'w') as f: + with open(filename, 'wb') as f: dl = 0 total_length = int(total_length) for data in r.iter_content(chunk_size=4096): diff --git a/python/paddle/dataset/conll05.py b/python/paddle/dataset/conll05.py index a97c95d067b876a87f0aa19b2ddd0702a848bd4a..55cfd92721e95d66f1cf38e2f77d9bb6b9e17d7a 100644 --- a/python/paddle/dataset/conll05.py +++ b/python/paddle/dataset/conll05.py @@ -20,15 +20,18 @@ dataset. And a pre-trained word vector model based on Wikipedia corpus is used to initialize SRL model. """ +from __future__ import print_function + import tarfile import gzip import itertools import paddle.dataset.common -from six.moves import zip +import paddle.compat as cpt +from six.moves import zip, range __all__ = ['test, get_dict', 'get_embedding', 'convert'] -DATA_URL = 'http://www.cs.upc.edu/~srlconll/conll05st-tests.tar.gz' +DATA_URL = 'http://paddlemodels.bj.bcebos.com/conll05st/conll05st-tests.tar.gz' DATA_MD5 = '387719152ae52d60422c016e92a742fc' WORDDICT_URL = 'http://paddlemodels.bj.bcebos.com/conll05st%2FwordDict.txt' WORDDICT_MD5 = 'ea7fb7d4c75cc6254716f0177a506baa' @@ -89,8 +92,8 @@ def corpus_reader(data_path, words_name, props_name): labels = [] one_seg = [] for word, label in zip(words_file, props_file): - word = word.strip() - label = label.strip().split() + word = cpt.to_text(word.strip()) + label = cpt.to_text(label.strip().split()) if len(label) == 0: # end of sentence for i in range(len(one_seg[0])): diff --git a/python/paddle/dataset/flowers.py b/python/paddle/dataset/flowers.py index 914dae348bc94d061072543aa14aba2219f4b52d..aa73bbaf7024ec873d9e921205536f12e097ff32 100644 --- a/python/paddle/dataset/flowers.py +++ b/python/paddle/dataset/flowers.py @@ -28,6 +28,9 @@ Graphics and Image Processing (2008) http://www.robots.ox.ac.uk/~vgg/publications/papers/nilsback08.{pdf,ps.gz}. """ + +from __future__ import print_function + import itertools import functools from .common import download @@ -116,7 +119,7 @@ def reader_creator(data_file, for file in open(file_list): file = file.strip() batch = None - with open(file, 'r') as f: + with open(file, 'rb') as f: batch = pickle.load(f) data = batch['data'] labels = batch['label'] diff --git a/python/paddle/dataset/image.py b/python/paddle/dataset/image.py index 3b3d89c93c48d611dccf6f14958c310a6cac1a7b..1cd50bd1802095db07e5618f37b0d42d11e94760 100644 --- a/python/paddle/dataset/image.py +++ b/python/paddle/dataset/image.py @@ -29,10 +29,18 @@ the image layout as follows. formats can be used for training. Noted that, the format should be keep consistent between the training and inference peroid. """ + +from __future__ import print_function + import numpy as np try: import cv2 except ImportError: + import sys + sys.stderr.write( + '''Warning with paddle image module: opencv-python should be imported, + or paddle image module could NOT work; please install opencv-python first.''' + ) cv2 = None import os import tarfile @@ -56,7 +64,7 @@ def batch_images_from_tar(data_file, :type data_file: string :param dataset_name: 'train','test' or 'valid' :type dataset_name: string - :param img2label: a dic with image file name as key + :param img2label: a dic with image file name as key and image's label as value :type img2label: dic :param num_per_batch: image number per batch file @@ -88,7 +96,7 @@ def batch_images_from_tar(data_file, output['data'] = data pickle.dump( output, - open('%s/batch_%d' % (out_path, file_id), 'w'), + open('%s/batch_%d' % (out_path, file_id), 'wb'), protocol=pickle.HIGHEST_PROTOCOL) file_id += 1 data = [] @@ -99,7 +107,7 @@ def batch_images_from_tar(data_file, output['data'] = data pickle.dump( output, - open('%s/batch_%d' % (out_path, file_id), 'w'), + open('%s/batch_%d' % (out_path, file_id), 'wb'), protocol=pickle.HIGHEST_PROTOCOL) with open(meta_file, 'a') as meta: @@ -113,7 +121,7 @@ def load_image_bytes(bytes, is_color=True): Load an color or gray image from bytes array. Example usage: - + .. code-block:: python with open('cat.jpg') as f: @@ -126,6 +134,8 @@ def load_image_bytes(bytes, is_color=True): load and return a gray image. :type is_color: bool """ + assert cv2 is not None + flag = 1 if is_color else 0 file_bytes = np.asarray(bytearray(bytes), dtype=np.uint8) img = cv2.imdecode(file_bytes, flag) @@ -137,7 +147,7 @@ def load_image(file, is_color=True): Load an color or gray image from the file path. Example usage: - + .. code-block:: python im = load_image('cat.jpg') @@ -149,6 +159,8 @@ def load_image(file, is_color=True): load and return a gray image. :type is_color: bool """ + assert cv2 is not None + # cv2.IMAGE_COLOR for OpenCV3 # cv2.CV_LOAD_IMAGE_COLOR for older OpenCV Version # cv2.IMAGE_GRAYSCALE for OpenCV3 @@ -161,27 +173,29 @@ def load_image(file, is_color=True): def resize_short(im, size): - """ + """ Resize an image so that the length of shorter edge is size. Example usage: - + .. code-block:: python im = load_image('cat.jpg') im = resize_short(im, 256) - + :param im: the input image with HWC layout. :type im: ndarray :param size: the shorter edge size of image after resizing. :type size: int """ + assert cv2 is not None + h, w = im.shape[:2] h_new, w_new = size, size if h > w: - h_new = size * h / w + h_new = size * h // w else: - w_new = size * w / h + w_new = size * w // h im = cv2.resize(im, (h_new, w_new), interpolation=cv2.INTER_CUBIC) return im @@ -193,17 +207,17 @@ def to_chw(im, order=(2, 0, 1)): according the order (2,0,1). Example usage: - + .. code-block:: python im = load_image('cat.jpg') im = resize_short(im, 256) im = to_chw(im) - + :param im: the input image with HWC layout. :type im: ndarray :param order: the transposed order. - :type order: tuple|list + :type order: tuple|list """ assert len(im.shape) == len(order) im = im.transpose(order) @@ -215,11 +229,11 @@ def center_crop(im, size, is_color=True): Crop the center of image with size. Example usage: - + .. code-block:: python im = center_crop(im, 224) - + :param im: the input image with HWC layout. :type im: ndarray :param size: the cropping size. @@ -228,8 +242,8 @@ def center_crop(im, size, is_color=True): :type is_color: bool """ h, w = im.shape[:2] - h_start = (h - size) / 2 - w_start = (w - size) / 2 + h_start = (h - size) // 2 + w_start = (w - size) // 2 h_end, w_end = h_start + size, w_start + size if is_color: im = im[h_start:h_end, w_start:w_end, :] @@ -243,11 +257,11 @@ def random_crop(im, size, is_color=True): Randomly crop input image with size. Example usage: - + .. code-block:: python im = random_crop(im, 224) - + :param im: the input image with HWC layout. :type im: ndarray :param size: the cropping size. @@ -272,11 +286,11 @@ def left_right_flip(im, is_color=True): Return the flipped image. Example usage: - + .. code-block:: python im = left_right_flip(im) - + :param im: input image with HWC layout or HW layout for gray image :type im: ndarray :param is_color: whether input image is color or not @@ -299,7 +313,7 @@ def simple_transform(im, resizing, croping and flipping. Example usage: - + .. code-block:: python im = simple_transform(im, 256, 224, True) @@ -314,7 +328,7 @@ def simple_transform(im, :type is_train: bool :param is_color: whether the image is color or not. :type is_color: bool - :param mean: the mean values, which can be element-wise mean values or + :param mean: the mean values, which can be element-wise mean values or mean values per channel. :type mean: numpy array | list """ @@ -332,7 +346,7 @@ def simple_transform(im, im = im.astype('float32') if mean is not None: mean = np.array(mean, dtype=np.float32) - # mean value, may be one value per channel + # mean value, may be one value per channel if mean.ndim == 1 and is_color: mean = mean[:, np.newaxis, np.newaxis] elif mean.ndim == 1: @@ -357,7 +371,7 @@ def load_and_transform(filename, for the transform operations. Example usage: - + .. code-block:: python im = load_and_transform('cat.jpg', 256, 224, True) @@ -372,7 +386,7 @@ def load_and_transform(filename, :type is_train: bool :param is_color: whether the image is color or not. :type is_color: bool - :param mean: the mean values, which can be element-wise mean values or + :param mean: the mean values, which can be element-wise mean values or mean values per channel. :type mean: numpy array | list """ diff --git a/python/paddle/dataset/imdb.py b/python/paddle/dataset/imdb.py index e7fe4e0b7e5832c2bc7ca1307725936a70292c39..fd92523a947689a71b6f9371a3ef4838eb9d194d 100644 --- a/python/paddle/dataset/imdb.py +++ b/python/paddle/dataset/imdb.py @@ -20,11 +20,14 @@ of 25,000 highly polar movie reviews for training, and 25,000 for testing. Besides, this module also provides API for building dictionary. """ +from __future__ import print_function + import paddle.dataset.common import collections import tarfile import re import string +import six __all__ = ['build_dict', 'train', 'test', 'convert'] @@ -42,13 +45,14 @@ def tokenize(pattern): # sequential access of member files, other than # tarfile.extractfile, which does random access and might # destroy hard disks. - tf = next(tarf) + tf = tarf.next() while tf != None: if bool(pattern.match(tf.name)): # newline and punctuations removal and ad-hoc tokenization. - yield tarf.extractfile(tf).read().rstrip("\n\r").translate( - None, string.punctuation).lower().split() - tf = next(tarf) + yield tarf.extractfile(tf).read().rstrip(six.b( + "\n\r")).translate( + None, six.b(string.punctuation)).lower().split() + tf = tarf.next() def build_dict(pattern, cutoff): @@ -62,11 +66,11 @@ def build_dict(pattern, cutoff): word_freq[word] += 1 # Not sure if we should prune less-frequent words here. - word_freq = [x for x in list(word_freq.items()) if x[1] > cutoff] + word_freq = [x for x in six.iteritems(word_freq) if x[1] > cutoff] dictionary = sorted(word_freq, key=lambda x: (-x[1], x[0])) words, _ = list(zip(*dictionary)) - word_idx = dict(list(zip(words, list(range(len(words)))))) + word_idx = dict(list(zip(words, six.moves.range(len(words))))) word_idx[''] = len(words) return word_idx diff --git a/python/paddle/dataset/imikolov.py b/python/paddle/dataset/imikolov.py index bc007c9d3c8e2f1e4ff091f7c2c93eacbbe8d0e0..8eecb75231de450282fa4838aca5b293cc2101d1 100644 --- a/python/paddle/dataset/imikolov.py +++ b/python/paddle/dataset/imikolov.py @@ -14,13 +14,17 @@ """ imikolov's simple dataset. -This module will download dataset from +This module will download dataset from http://www.fit.vutbr.cz/~imikolov/rnnlm/ and parse training set and test set into paddle reader creators. """ + +from __future__ import print_function + import paddle.dataset.common import collections import tarfile +import six __all__ = ['train', 'test', 'build_dict', 'convert'] @@ -64,11 +68,13 @@ def build_dict(min_word_freq=50): # remove for now, since we will set it as last index del word_freq[''] - word_freq = [x for x in list(word_freq.items()) if x[1] > min_word_freq] + word_freq = [ + x for x in six.iteritems(word_freq) if x[1] > min_word_freq + ] word_freq_sorted = sorted(word_freq, key=lambda x: (-x[1], x[0])) words, _ = list(zip(*word_freq_sorted)) - word_idx = dict(list(zip(words, list(range(len(words)))))) + word_idx = dict(list(zip(words, six.moves.range(len(words))))) word_idx[''] = len(words) return word_idx @@ -89,7 +95,7 @@ def reader_creator(filename, word_idx, n, data_type): l = [''] + l.strip().split() + [''] if len(l) >= n: l = [word_idx.get(w, UNK) for w in l] - for i in range(n, len(l) + 1): + for i in six.moves.range(n, len(l) + 1): yield tuple(l[i - n:i]) elif DataType.SEQ == data_type: l = l.strip().split() diff --git a/python/paddle/dataset/mnist.py b/python/paddle/dataset/mnist.py index ffa9008c80129b80b3807dbab37bc198e59cf5a2..3038747bf8d03cf82613f1b329b7eeb778a0b9f2 100644 --- a/python/paddle/dataset/mnist.py +++ b/python/paddle/dataset/mnist.py @@ -17,10 +17,16 @@ MNIST dataset. This module will download dataset from http://yann.lecun.com/exdb/mnist/ and parse training set and test set into paddle reader creators. """ + +from __future__ import print_function + import paddle.dataset.common import subprocess import numpy import platform +import six +import tempfile +from six.moves import range __all__ = ['train', 'test', 'convert'] URL_PREFIX = 'http://yann.lecun.com/exdb/mnist/' @@ -45,23 +51,28 @@ def reader_creator(image_filename, label_filename, buffer_size): # According to http://stackoverflow.com/a/38061619/724872, we # cannot use standard package gzip here. - m = subprocess.Popen([zcat_cmd, image_filename], stdout=subprocess.PIPE) - m.stdout.read(16) # skip some magic bytes + tmp_image_file = tempfile.TemporaryFile(prefix='paddle_dataset') + m = subprocess.Popen( + [zcat_cmd, image_filename], stdout=tmp_image_file).communicate() + tmp_image_file.seek(16) # skip some magic bytes - l = subprocess.Popen([zcat_cmd, label_filename], stdout=subprocess.PIPE) - l.stdout.read(8) # skip some magic bytes + # Python3 will not take stdout as file + tmp_label_file = tempfile.TemporaryFile(prefix='paddle_dataset') + l = subprocess.Popen( + [zcat_cmd, label_filename], stdout=tmp_label_file).communicate() + tmp_label_file.seek(8) # skip some magic bytes try: # reader could be break. while True: labels = numpy.fromfile( - l.stdout, 'ubyte', count=buffer_size).astype("int") + tmp_label_file, 'ubyte', count=buffer_size).astype("int") if labels.size != buffer_size: break # numpy.fromfile returns empty slice after EOF. images = numpy.fromfile( - m.stdout, 'ubyte', count=buffer_size * 28 * 28).reshape( - (buffer_size, 28 * 28)).astype('float32') + tmp_image_file, 'ubyte', count=buffer_size * 28 * + 28).reshape((buffer_size, 28 * 28)).astype('float32') images = images / 255.0 * 2.0 - 1.0 diff --git a/python/paddle/dataset/movielens.py b/python/paddle/dataset/movielens.py index 056ec2178607329dd6daa1764820c2312bbaed59..c98e0019f7ab5fc2723e8df919257a59af7c9e5d 100644 --- a/python/paddle/dataset/movielens.py +++ b/python/paddle/dataset/movielens.py @@ -22,11 +22,15 @@ set and test set into paddle reader creators. """ +from __future__ import print_function + import zipfile import paddle.dataset.common import re import random import functools +import six +import paddle.compat as cpt __all__ = [ 'train', 'test', 'get_movie_title_dict', 'max_movie_id', 'max_user_id', @@ -112,6 +116,7 @@ def __initialize_meta_info__(): categories_set = set() with package.open('ml-1m/movies.dat') as movie_file: for i, line in enumerate(movie_file): + line = cpt.to_text(line, encoding='latin') movie_id, title, categories = line.strip().split('::') categories = categories.split('|') for c in categories: @@ -136,6 +141,7 @@ def __initialize_meta_info__(): USER_INFO = dict() with package.open('ml-1m/users.dat') as user_file: for line in user_file: + line = cpt.to_text(line, encoding='latin') uid, gender, age, job, _ = line.strip().split("::") USER_INFO[int(uid)] = UserInfo( index=uid, gender=gender, age=age, job_id=job) @@ -148,6 +154,7 @@ def __reader__(rand_seed=0, test_ratio=0.1, is_test=False): with zipfile.ZipFile(file=fn) as package: with package.open('ml-1m/ratings.dat') as rating: for line in rating: + line = cpt.to_text(line, encoding='latin') if (rand.random() < test_ratio) == is_test: uid, mov_id, rating, _ = line.strip().split("::") uid = int(uid) @@ -187,7 +194,7 @@ def max_movie_id(): Get the maximum value of movie id. """ __initialize_meta_info__() - return reduce(__max_index_info__, list(MOVIE_INFO.values())).index + return six.moves.reduce(__max_index_info__, list(MOVIE_INFO.values())).index def max_user_id(): @@ -195,7 +202,7 @@ def max_user_id(): Get the maximum value of user id. """ __initialize_meta_info__() - return reduce(__max_index_info__, list(USER_INFO.values())).index + return six.moves.reduce(__max_index_info__, list(USER_INFO.values())).index def __max_job_id_impl__(a, b): @@ -210,7 +217,8 @@ def max_job_id(): Get the maximum value of job id. """ __initialize_meta_info__() - return reduce(__max_job_id_impl__, list(USER_INFO.values())).job_id + return six.moves.reduce(__max_job_id_impl__, + list(USER_INFO.values())).job_id def movie_categories(): diff --git a/python/paddle/dataset/mq2007.py b/python/paddle/dataset/mq2007.py index cc4d088316dfd490dc9d6b247c66c2495cedf2c3..d5740f30c898d5704636e1de9b2e1137d12e3c35 100644 --- a/python/paddle/dataset/mq2007.py +++ b/python/paddle/dataset/mq2007.py @@ -23,6 +23,8 @@ http://research.microsoft.com/en-us/um/beijing/projects/letor/LETOR4.0/Data/MQ20 """ +from __future__ import print_function + import os import functools import rarfile diff --git a/python/paddle/dataset/sentiment.py b/python/paddle/dataset/sentiment.py index 953ada057bc114ebbfe39011d2fd3b5b7a2b0d37..22d867beea25c97efcbcb6f61ca2b7a7777f9c5c 100644 --- a/python/paddle/dataset/sentiment.py +++ b/python/paddle/dataset/sentiment.py @@ -20,6 +20,9 @@ The script fetch and preprocess movie_reviews data set that provided by NLTK TODO(yuyang18): Complete dataset. """ +from __future__ import print_function + +import six import collections from itertools import chain @@ -64,7 +67,7 @@ def get_word_dict(): for field in movie_reviews.fileids(category): for words in movie_reviews.words(field): word_freq_dict[words] += 1 - words_sort_list = list(word_freq_dict.items()) + words_sort_list = six.iteritems(word_freq_dict) words_sort_list.sort(cmp=lambda a, b: b[1] - a[1]) for index, word in enumerate(words_sort_list): words_freq_sorted.append((word[0], index)) diff --git a/python/paddle/dataset/tests/cifar_test.py b/python/paddle/dataset/tests/cifar_test.py index 839125b09dd5c6432e3572374a7345a77a43f7cf..8e514f0fd9a18a7d512430111a8a11b942950d20 100644 --- a/python/paddle/dataset/tests/cifar_test.py +++ b/python/paddle/dataset/tests/cifar_test.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.dataset.cifar import unittest diff --git a/python/paddle/dataset/tests/common_test.py b/python/paddle/dataset/tests/common_test.py index 777cd06a19726f8ad73774c958c8cb512808a3aa..0ce7d83f374f8c09f68527473418de8ce84c36b1 100644 --- a/python/paddle/dataset/tests/common_test.py +++ b/python/paddle/dataset/tests/common_test.py @@ -12,10 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.dataset.common import unittest import tempfile import glob +from six.moves import range class TestCommon(unittest.TestCase): diff --git a/python/paddle/dataset/tests/flowers_test.py b/python/paddle/dataset/tests/flowers_test.py index 06260fd796ce0271b7cec2f42a8a5a255a02dc24..06a0a7761cfa10ca3211297d176e3e909332e271 100644 --- a/python/paddle/dataset/tests/flowers_test.py +++ b/python/paddle/dataset/tests/flowers_test.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.dataset.flowers import unittest diff --git a/python/paddle/dataset/tests/imdb_test.py b/python/paddle/dataset/tests/imdb_test.py index 539da049449cd273db0a9e260851ed40e1be0f04..415947e3477f2e5b9979588528f7cb6f799acf6a 100644 --- a/python/paddle/dataset/tests/imdb_test.py +++ b/python/paddle/dataset/tests/imdb_test.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.dataset.imdb import unittest import re diff --git a/python/paddle/dataset/tests/imikolov_test.py b/python/paddle/dataset/tests/imikolov_test.py index 50f50d947d221686d6308a6ed44cbcff3b10c6f5..1f78a5dd4d1a09c3192bc8c144c5a78c8a214f3a 100644 --- a/python/paddle/dataset/tests/imikolov_test.py +++ b/python/paddle/dataset/tests/imikolov_test.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.dataset.imikolov import unittest diff --git a/python/paddle/dataset/tests/mnist_test.py b/python/paddle/dataset/tests/mnist_test.py index 8ada19d3f2ee13e194d08e19a4b86b558c69a0a7..fbb5d926494e38283e78ec15381530e50f32915d 100644 --- a/python/paddle/dataset/tests/mnist_test.py +++ b/python/paddle/dataset/tests/mnist_test.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.dataset.mnist import unittest diff --git a/python/paddle/dataset/tests/mq2007_test.py b/python/paddle/dataset/tests/mq2007_test.py index fba388724a8e84591df7150b41f8ea39a850fc31..ee0897e88f0d7ad089b7f7b68d31d04d96fa3e9d 100644 --- a/python/paddle/dataset/tests/mq2007_test.py +++ b/python/paddle/dataset/tests/mq2007_test.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.dataset.mq2007 import unittest diff --git a/python/paddle/dataset/tests/test_image.py b/python/paddle/dataset/tests/test_image.py index 8bd56607ae1998935a3b3aaa0e3279515c2a540c..32d2eb17ae673e72bbee2fc3bb5e3b05f1b20074 100644 --- a/python/paddle/dataset/tests/test_image.py +++ b/python/paddle/dataset/tests/test_image.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np diff --git a/python/paddle/dataset/tests/test_sentiment.py b/python/paddle/dataset/tests/test_sentiment.py index 37326517f7b39fb74c694684eb8a547d5f021946..bb9830132e987370022df3192060de3e908a2e85 100644 --- a/python/paddle/dataset/tests/test_sentiment.py +++ b/python/paddle/dataset/tests/test_sentiment.py @@ -15,6 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import nltk import paddle.dataset.sentiment as st diff --git a/python/paddle/dataset/tests/voc2012_test.py b/python/paddle/dataset/tests/voc2012_test.py index 0d285461a8ae8a9cc69fbec0dcf5efc106b594f0..cddeb91cab2c0f90567f28f8258156e2bb654abc 100644 --- a/python/paddle/dataset/tests/voc2012_test.py +++ b/python/paddle/dataset/tests/voc2012_test.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.dataset.voc2012 import unittest diff --git a/python/paddle/dataset/tests/wmt16_test.py b/python/paddle/dataset/tests/wmt16_test.py index 8b949d8bf5212d51016a33da322095bde2038200..be121bb10121967590c9e136e9a1964a133e934b 100644 --- a/python/paddle/dataset/tests/wmt16_test.py +++ b/python/paddle/dataset/tests/wmt16_test.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.dataset.wmt16 import unittest diff --git a/python/paddle/dataset/uci_housing.py b/python/paddle/dataset/uci_housing.py index 410ca7af0d6d1dc26acbf92fce5e49fce7d3a3bb..f87fdcc4f0f3c42a92bcff5ddcd532c3108565b1 100644 --- a/python/paddle/dataset/uci_housing.py +++ b/python/paddle/dataset/uci_housing.py @@ -19,9 +19,10 @@ https://archive.ics.uci.edu/ml/machine-learning-databases/housing/ and parse training set and test set into paddle reader creators. """ -import os +from __future__ import print_function import numpy as np +import six import tempfile import tarfile import os @@ -70,11 +71,11 @@ def load_data(filename, feature_num=14, ratio=0.8): return data = np.fromfile(filename, sep=' ') - data = data.reshape(data.shape[0] / feature_num, feature_num) + data = data.reshape(data.shape[0] // feature_num, feature_num) maximums, minimums, avgs = data.max(axis=0), data.min(axis=0), data.sum( axis=0) / data.shape[0] feature_range(maximums[:-1], minimums[:-1]) - for i in range(feature_num - 1): + for i in six.moves.range(feature_num - 1): data[:, i] = (data[:, i] - avgs[i]) / (maximums[i] - minimums[i]) offset = int(data.shape[0] * ratio) UCI_TRAIN_DATA = data[:offset] @@ -137,7 +138,7 @@ def predict_reader(): It returns just one tuple data to do inference. :return: one tuple data - :rtype: tuple + :rtype: tuple """ global UCI_TEST_DATA load_data(paddle.dataset.common.download(URL, 'uci_housing', MD5)) diff --git a/python/paddle/dataset/voc2012.py b/python/paddle/dataset/voc2012.py index 9c945574dbcc15f5cee370206ed7e70ba8ab5014..50688937654ae72b77e1439f21a0d7c847d5e135 100644 --- a/python/paddle/dataset/voc2012.py +++ b/python/paddle/dataset/voc2012.py @@ -19,6 +19,8 @@ to training/test sets has been maintained. The total number of images with segmentation has been increased from 7,062 to 9,993. """ +from __future__ import print_function + import tarfile import io import numpy as np diff --git a/python/paddle/dataset/wmt14.py b/python/paddle/dataset/wmt14.py index 7504474591fa486428d0310f10387818c4cf0300..f8c1a33574e642b21feb6843d115b7f4205ef250 100644 --- a/python/paddle/dataset/wmt14.py +++ b/python/paddle/dataset/wmt14.py @@ -19,10 +19,15 @@ http://paddlepaddle.cdn.bcebos.com/demo/wmt_shrinked_data/wmt14.tgz and parse training set and test set into paddle reader creators. """ + +from __future__ import print_function + +import six import tarfile import gzip import paddle.dataset.common +import paddle.compat as cpt __all__ = [ 'train', @@ -53,7 +58,7 @@ def __read_to_dict(tar_file, dict_size): out_dict = dict() for line_count, line in enumerate(fd): if line_count < size: - out_dict[line.strip()] = line_count + out_dict[cpt.to_text(line.strip())] = line_count else: break return out_dict @@ -84,7 +89,7 @@ def reader_creator(tar_file, file_name, dict_size): ] for name in names: for line in f.extractfile(name): - line_split = line.strip().split('\t') + line_split = line.strip().split(six.b('\t')) if len(line_split) != 2: continue src_seq = line_split[0] # one source sequence @@ -153,8 +158,8 @@ def get_dict(dict_size, reverse=True): tar_file = paddle.dataset.common.download(URL_TRAIN, 'wmt14', MD5_TRAIN) src_dict, trg_dict = __read_to_dict(tar_file, dict_size) if reverse: - src_dict = {v: k for k, v in list(src_dict.items())} - trg_dict = {v: k for k, v in list(trg_dict.items())} + src_dict = {v: k for k, v in six.iteritems(src_dict)} + trg_dict = {v: k for k, v in six.iteritems(trg_dict)} return src_dict, trg_dict diff --git a/python/paddle/dataset/wmt16.py b/python/paddle/dataset/wmt16.py index 4e3c466c38e402cc574e93ef3a5935edf8f9dd3b..f30dcd518ea6c0c685d027ede3ad6e0a1cb0c82c 100644 --- a/python/paddle/dataset/wmt16.py +++ b/python/paddle/dataset/wmt16.py @@ -28,12 +28,16 @@ Multi30K: Multilingual English-German Image Descriptions. } """ +from __future__ import print_function + import os +import six import tarfile import gzip from collections import defaultdict import paddle.dataset.common +import paddle.compat as cpt __all__ = [ "train", @@ -60,7 +64,7 @@ def __build_dict(tar_file, dict_size, save_path, lang): word_dict = defaultdict(int) with tarfile.open(tar_file, mode="r") as f: for line in f.extractfile("wmt16/train"): - line_split = line.strip().split("\t") + line_split = line.strip().split(six.b("\t")) if len(line_split) != 2: continue sen = line_split[0] if lang == "en" else line_split[1] for w in sen.split(): @@ -70,8 +74,7 @@ def __build_dict(tar_file, dict_size, save_path, lang): fout.write("%s\n%s\n%s\n" % (START_MARK, END_MARK, UNK_MARK)) for idx, word in enumerate( sorted( - iter(list(word_dict.items())), - key=lambda x: x[1], + six.iteritems(word_dict), key=lambda x: x[1], reverse=True)): if idx + 3 == dict_size: break fout.write("%s\n" % (word[0])) @@ -81,16 +84,16 @@ def __load_dict(tar_file, dict_size, lang, reverse=False): dict_path = os.path.join(paddle.dataset.common.DATA_HOME, "wmt16/%s_%d.dict" % (lang, dict_size)) if not os.path.exists(dict_path) or ( - len(open(dict_path, "r").readlines()) != dict_size): + len(open(dict_path, "rb").readlines()) != dict_size): __build_dict(tar_file, dict_size, dict_path, lang) word_dict = {} - with open(dict_path, "r") as fdict: + with open(dict_path, "rb") as fdict: for idx, line in enumerate(fdict): if reverse: - word_dict[idx] = line.strip() + word_dict[idx] = cpt.to_text(line.strip()) else: - word_dict[line.strip()] = idx + word_dict[cpt.to_text(line.strip())] = idx return word_dict @@ -120,7 +123,7 @@ def reader_creator(tar_file, file_name, src_dict_size, trg_dict_size, src_lang): with tarfile.open(tar_file, mode="r") as f: for line in f.extractfile(file_name): - line_split = line.strip().split("\t") + line_split = line.strip().split(six.b("\t")) if len(line_split) != 2: continue src_words = line_split[src_col].split() diff --git a/python/paddle/fluid/__init__.py b/python/paddle/fluid/__init__.py index 1ae05dec8de6b9cbc9568f0f4d437833be520f8d..9aac3c7fc16ae1ded2700662764895385b043130 100644 --- a/python/paddle/fluid/__init__.py +++ b/python/paddle/fluid/__init__.py @@ -122,7 +122,7 @@ def __bootstrap__(): 'use_pinned_memory', 'check_nan_inf', 'benchmark', 'warpctc_dir', 'eager_delete_scope', 'use_mkldnn', 'initial_cpu_memory_in_mb', 'init_allocated_mem', 'free_idle_memory', 'paddle_num_threads', - 'cpu_deterministic' + "dist_threadpool_size", 'cpu_deterministic' ] if core.is_compiled_with_dist(): read_env_flags.append('rpc_deadline') diff --git a/python/paddle/fluid/average.py b/python/paddle/fluid/average.py index 358e24df31bb517604481bb48b9180e579f8460d..42cd3b36420ef5a17a9a7d981978ba8869809936 100644 --- a/python/paddle/fluid/average.py +++ b/python/paddle/fluid/average.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import warnings """ diff --git a/python/paddle/fluid/backward.py b/python/paddle/fluid/backward.py index fd6a76dd0cfa347328d87093884e5cd324395497..a415cdbeaaae2a3bb4a137744205e3fe7366a78f 100644 --- a/python/paddle/fluid/backward.py +++ b/python/paddle/fluid/backward.py @@ -12,11 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from paddle.fluid import framework as framework from . import core import collections import copy import six +from .. import compat as cpt from . import unique_name __all__ = ['append_backward'] @@ -45,13 +48,13 @@ def _create_op_desc_(op_type, inputs, outputs, attrs): """ op_desc = core.OpDesc() op_desc.set_type(op_type) - for para, args in list(inputs.items()): + for para, args in six.iteritems(inputs): op_desc.set_input( para, list( map(lambda arg: arg.decode() if isinstance(arg, six.binary_type) else arg, args))) - for para, args in list(outputs.items()): + for para, args in six.iteritems(outputs): op_desc.set_output( para, list( @@ -63,7 +66,7 @@ def _create_op_desc_(op_type, inputs, outputs, attrs): if op_role_attr_name not in attrs: attrs[ op_role_attr_name] = core.op_proto_and_checker_maker.OpRole.Backward - for name, val in list(attrs.items()): + for name, val in six.iteritems(attrs): if isinstance(val, framework.Block): op_desc.set_block_attr(name, val.desc) else: @@ -75,10 +78,10 @@ def _infer_var_data_type_(grad_var_name, block): """ Infer the data type of given grad variable """ - grad_var = block.desc.find_var(grad_var_name.encode("ascii")) - fwd_name = _strip_grad_suffix_(grad_var_name.encode("ascii")) - if block.desc.has_var_recursive(fwd_name): - fwd_var = block.desc.find_var_recursive(fwd_name.encode("ascii")) + grad_var = block.desc.find_var(cpt.to_bytes(grad_var_name)) + fwd_name = _strip_grad_suffix_(grad_var_name) + if block.desc.has_var_recursive(cpt.to_bytes(fwd_name)): + fwd_var = block.desc.find_var_recursive(cpt.to_bytes(fwd_name)) grad_var.set_dtype(fwd_var.dtype()) else: grad_var.set_dtype(core.VarDesc.VarType.FP32) @@ -102,8 +105,10 @@ def _some_in_set_(cands, s): """ if len(cands) == 0: return False - for c in cands: - if c in s: + literal_set = cpt.to_text(s) + literal_cands = cpt.to_text(cands) + for c in literal_cands: + if c in literal_set: return True return False @@ -114,9 +119,8 @@ def _strip_grad_suffix_(name): e.g. x@GRAD ==> x y@GRAD@RENAME@1 ==> y """ - if isinstance(name, six.text_type): - name = name.encode() - pos = name.find(six.b(core.grad_var_suffix())) + name = cpt.to_text(name) + pos = name.find(core.grad_var_suffix()) return name[:pos] if pos != -1 else name @@ -125,9 +129,7 @@ def _append_grad_suffix_(name): Append grad suffix to the given variable name e.g. x ==> x@GRAD """ - if isinstance(name, six.text_type): - name = name.encode() - return name + six.b(core.grad_var_suffix()) + return cpt.to_text(name) + core.grad_var_suffix() def _addup_repetitive_outputs_(op_descs): @@ -187,7 +189,7 @@ def _addup_repetitive_outputs_(op_descs): op_desc.set_output(param_name, arg_names) renamed_vars[var_name].append(new_name) - for var_name, inputs in list(renamed_vars.items()): + for var_name, inputs in six.iteritems(renamed_vars): if len(inputs) > 1: pending_sum_ops.append( (_create_op_desc_("sum", {"X": inputs}, {"Out": [var_name]}, @@ -243,7 +245,7 @@ from .proto import framework_pb2 def serialize_op_decs(op_desc): protostr = op_desc.serialize_to_string() - proto = framework_pb2.OpDesc.FromString(str(protostr)) + proto = framework_pb2.OpDesc.FromString(six.binary_type(protostr)) return proto.__str__() @@ -364,7 +366,7 @@ def _append_backward_ops_(block, # Getting op's corresponding grad_op grad_op_desc, op_grad_to_var = core.get_grad_op_desc( - op.desc, no_grad_dict[block.idx], grad_sub_block_list) + op.desc, cpt.to_text(no_grad_dict[block.idx]), grad_sub_block_list) grad_op_descs.extend(grad_op_desc) grad_to_var.update(op_grad_to_var) @@ -411,11 +413,10 @@ def _append_backward_vars_(block, start_op_idx, grad_to_var, grad_info_map): new_vars = set() # create new gradient variables for grad_var_name in op_desc.output_arg_names(): - grad_var_name = grad_var_name.encode("ascii") - if block.desc.has_var_recursive( - grad_var_name) or grad_var_name == core.empty_var_name(): + if block.desc.has_var_recursive(cpt.to_bytes( + grad_var_name)) or grad_var_name == core.empty_var_name(): continue - block.desc.var(grad_var_name) + block.desc.var(cpt.to_bytes(grad_var_name)) new_vars.add(grad_var_name) if grad_var_name not in grad_to_var: continue @@ -445,7 +446,7 @@ def _rename_grad_(block, start_op_idx, grad_to_var, target_grad_map): op_desc.rename_output(name, new_name) var_map[name] = new_name - for g, ng in list(var_map.items()): + for g, ng in six.iteritems(var_map): if g in grad_to_var: grad_to_var[ng] = grad_to_var[g] grad_to_var.pop(g) @@ -595,11 +596,12 @@ def append_backward(loss, parameter_list=None, no_grad_set=None, parameters = parameter_list else: params = program.global_block().all_parameters() + program.global_block().iter_parameters() parameters = [param.name for param in params] params_and_grads = [] for param in parameters: - if param not in grad_info_map: + if cpt.to_text(param) not in grad_info_map: continue grad_info = grad_info_map[param] grad_block = grad_info[1] diff --git a/python/paddle/fluid/clip.py b/python/paddle/fluid/clip.py index 4b0a792f784fffcce3f911d3e7448b472d39f8e1..ba7ba3b5e983bfbaa82fc752f4821e8a934dfb8c 100644 --- a/python/paddle/fluid/clip.py +++ b/python/paddle/fluid/clip.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import copy import six diff --git a/python/paddle/fluid/concurrency.py b/python/paddle/fluid/concurrency.py index 676a52a917dd1f9700ec38de32932938ec339be5..b4a06f23a6f2713b665bdd42919925e4a0475a82 100644 --- a/python/paddle/fluid/concurrency.py +++ b/python/paddle/fluid/concurrency.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from .layers.control_flow import BlockGuard, equal from .framework import Operator from .layer_helper import LayerHelper, unique_name diff --git a/python/paddle/fluid/contrib/__init__.py b/python/paddle/fluid/contrib/__init__.py index 58f2da1c3ba2f84602e7a18c7b1c78d1f0d2ede1..5607f11932bbe6aff548be316dc39b4636e079f4 100644 --- a/python/paddle/fluid/contrib/__init__.py +++ b/python/paddle/fluid/contrib/__init__.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from . import decoder from .decoder import * from . import memory_usage_calc diff --git a/python/paddle/fluid/contrib/decoder/__init__.py b/python/paddle/fluid/contrib/decoder/__init__.py index 6343c1543d206f82e605c5c986fa91d70c467113..9f973fd3c9af60a0c9a2ba5225a616671545436b 100644 --- a/python/paddle/fluid/contrib/decoder/__init__.py +++ b/python/paddle/fluid/contrib/decoder/__init__.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from . import beam_search_decoder from .beam_search_decoder import * diff --git a/python/paddle/fluid/contrib/decoder/beam_search_decoder.py b/python/paddle/fluid/contrib/decoder/beam_search_decoder.py index d268a948f7a2cf038a419c95521b81088ed8215f..f2b7ac8375af25beed562b8279b6044f11c09d44 100644 --- a/python/paddle/fluid/contrib/decoder/beam_search_decoder.py +++ b/python/paddle/fluid/contrib/decoder/beam_search_decoder.py @@ -20,6 +20,8 @@ without using the low level API such as while ops. This API is still under active development and may change drastically. """ +from __future__ import print_function + import contextlib import numpy as np import six diff --git a/python/paddle/fluid/contrib/memory_usage_calc.py b/python/paddle/fluid/contrib/memory_usage_calc.py index 5da846edb63c28efd791fdfac4046cfa56c24181..09721e430b7e5bb6b9891d5272ca54475baf6157 100644 --- a/python/paddle/fluid/contrib/memory_usage_calc.py +++ b/python/paddle/fluid/contrib/memory_usage_calc.py @@ -14,12 +14,16 @@ """ This module privides a memory usage calculate function for user. The purpose of this API is to allow users to estimate memory usage of -a program under a special batch size, then user can set appropriate -batch size to fully utilize a GPU. +a program under a special batch size, then user can set appropriate +batch size to fully utilize a GPU. This API is still under active development and may change drastically. """ +from __future__ import print_function + +import six + from .. import core from ..framework import Program, Variable @@ -45,15 +49,15 @@ def memory_usage(program, batch_size): Args: program(Program): The current Program. - batch_size(int): The current input data batch_size. - + batch_size(int): The current input data batch_size. + Returns: min_total_memory(float): the estimate memory usage lower bound. max_total_memory(float): the estimate memory usage upper bound. unit_str(string): the unit of estimate usage result. - + Examples: - + >>> import paddle.fluid as fluid >>> lower_usage, upper_usage, unit = fluid.contrib.memory_usage( fluid.default_main_program(), batch_size=10) @@ -72,7 +76,7 @@ def memory_usage(program, batch_size): # Get the var_name list of first block and calculate total_memory = 0.0 - for var in program.global_block().vars.itervalues(): + for var in six.itervalues(program.global_block().vars): data_count = 1 for x in var.shape: if x == -1: @@ -81,10 +85,10 @@ def memory_usage(program, batch_size): data_count *= x var_memory = data_count * dtype_to_size[var.dtype] if DEBUG: - print "%s memory usage: %d" % (var.name, var_memory) + print("%s memory usage: %d" % (var.name, var_memory)) total_memory += var_memory if DEBUG: - print "total memory usage: %.2f" % (total_memory) + print("total memory usage: %.2f" % (total_memory)) # Convert appropriate unit unit_str = "B" diff --git a/python/paddle/fluid/data_feeder.py b/python/paddle/fluid/data_feeder.py index 9452cf0e2a3a2eddb761149466bfc1ee3d23dfd9..631bbfe1fe59ddd9cd315fb64ca32e1e125b0e8d 100644 --- a/python/paddle/fluid/data_feeder.py +++ b/python/paddle/fluid/data_feeder.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from . import core import numpy import os diff --git a/python/paddle/fluid/debugger.py b/python/paddle/fluid/debugger.py index b7a92cf044900acdd41ede378dd68aa2d9c6b2dc..63060a77d1abdfd4060648bfabe25709afcfeb8d 100644 --- a/python/paddle/fluid/debugger.py +++ b/python/paddle/fluid/debugger.py @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import sys +import six import re from .graphviz import GraphPreviewGenerator from .proto import framework_pb2 @@ -225,7 +228,7 @@ def draw_block_graphviz(block, highlights=None, path="./temp.dot"): graph = GraphPreviewGenerator("some graph") # collect parameters and args protostr = block.desc.serialize_to_string() - desc = framework_pb2.BlockDesc.FromString(str(protostr)) + desc = framework_pb2.BlockDesc.FromString(six.binary_type(protostr)) def need_highlight(name): if highlights is None: return False diff --git a/python/paddle/fluid/default_scope_funcs.py b/python/paddle/fluid/default_scope_funcs.py index f8faf6942524612ccc63713240bb289eeeaf75eb..a5b2c84dfe6f2650b4a2ee4465f723812e5d4a01 100644 --- a/python/paddle/fluid/default_scope_funcs.py +++ b/python/paddle/fluid/default_scope_funcs.py @@ -26,6 +26,8 @@ A `scoped_function` will take a `function` as input. That function will be invoked in a new local scope. """ +from __future__ import print_function + import paddle.fluid.core import threading diff --git a/python/paddle/fluid/evaluator.py b/python/paddle/fluid/evaluator.py index c0671cce9a1f169f02ba03a839c45b6e4df2c47a..7a82038ff78b17b2ddfd7b47320d41a7de9a2b8a 100644 --- a/python/paddle/fluid/evaluator.py +++ b/python/paddle/fluid/evaluator.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import warnings import numpy as np diff --git a/python/paddle/fluid/executor.py b/python/paddle/fluid/executor.py index e24b9faae24084ccc743a5b5126db9667089e128..288951cd7cd32155f136125fb817c35dd2ec6444 100644 --- a/python/paddle/fluid/executor.py +++ b/python/paddle/fluid/executor.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import contextlib import six @@ -320,8 +322,9 @@ class Executor(object): # append fetch_operators if not has_fetch_operators(global_block, fetch_list, fetch_var_name): for i, var in enumerate(fetch_list): - assert isinstance(var, Variable) or isinstance(var, str), ( - "Wrong type for fetch_list[%s]: %s" % (i, type(var))) + assert isinstance(var, Variable) or isinstance( + var, six.string_types), ( + "Wrong type for fetch_list[%s]: %s" % (i, type(var))) global_block.append_op( type='fetch', inputs={'X': [var]}, @@ -346,7 +349,7 @@ class Executor(object): def _fetch_data(self, fetch_list, fetch_var_name, scope): outs = [ core.get_fetch_variable(scope, fetch_var_name, i) - for i in range(len(fetch_list)) + for i in six.moves.range(len(fetch_list)) ] return outs diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py index 2e6b45eb7ac03923da4d8485e736925691b0b580..bb5b4d53606667c707137780624163d11d09d208 100644 --- a/python/paddle/fluid/framework.py +++ b/python/paddle/fluid/framework.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import collections import contextlib import re @@ -19,6 +21,7 @@ import six import numpy as np +from .. import compat as cpt from .proto import framework_pb2 try: from . import core @@ -27,7 +30,7 @@ except ImportError as e: """NOTE: You may need to run \"export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH\" if you encounters \"libmkldnn.so not found\" errors. If you have python installed in other directory, replace \"/usr/local/lib\" with your own - directory. The original error is: \n""" + e.message) + directory. The original error is: \n""" + cpt.get_exception_message(e)) except Exception as e: raise e from . import unique_name @@ -87,7 +90,7 @@ def convert_np_dtype_to_dtype_(np_dtype): elif dtype == np.uint8: return core.VarDesc.VarType.UINT8 else: - raise ValueError("Not supported numpy dtype " + six.binary_type(dtype)) + raise ValueError("Not supported numpy dtype %s" % dtype) def dtype_is_floating(dtype): @@ -198,11 +201,11 @@ class Variable(object): if name is None: name = unique_name.generate('_generated_var') is_new_var = False - name = name if isinstance(name, six.binary_type) else name.encode() - self.desc = self.block.desc.find_var(name) + name = cpt.to_text(name) + self.desc = self.block.desc.find_var(cpt.to_bytes(name)) if self.desc is None: - self.desc = self.block.desc.var(name) + self.desc = self.block.desc.var(cpt.to_bytes(name)) is_new_var = True if is_new_var: @@ -325,7 +328,7 @@ class Variable(object): @property def name(self): - return self.desc.name() + return cpt.to_text(self.desc.name()) @name.setter def name(self, new_name): @@ -531,14 +534,7 @@ class Operator(object): elif isinstance(arg, six.binary_type): in_arg_names.append(arg.decode()) else: - if isinstance(arg.name, six.string_types): - in_arg_names.append(arg.name) - elif isinstance(arg.name, six.binary_type): - in_arg_names.append(arg.name.decode()) - else: - raise TypeError( - "arguments require unicode, str or bytes, but get %s instead." - % (type(arg.name))) + in_arg_names.append(cpt.to_text(arg.name)) self.desc.set_input(in_proto.name, in_arg_names) else: self.desc.set_input(in_proto.name, []) @@ -567,14 +563,7 @@ class Operator(object): (out_proto.name, len(out_args))) out_arg_names = [] for arg in out_args: - if isinstance(arg.name, six.string_types): - out_arg_names.append(arg.name) - elif isinstance(arg.name, six.binary_type): - out_arg_names.append(arg.name.decode()) - else: - raise TypeError( - "arguments require unicode, str or bytes, but get %s instead." - % (type(arg.name))) + out_arg_names.append(cpt.to_text(arg.name)) arg.op = self self.desc.set_output(out_proto.name, out_arg_names) @@ -970,10 +959,9 @@ class Block(object): Variable: the Variable with the giving name. """ if not isinstance(name, six.string_types): - if not isinstance(name, six.binary_type): - raise TypeError( - "var require string as parameter, but get %s instead." % - (type(name))) + raise TypeError( + "var require string as parameter, but get %s instead." % + (type(name))) v = self.vars.get(name, None) if v is None: raise ValueError("var %s not in this block" % name) @@ -1024,7 +1012,7 @@ class Block(object): return list(self.iter_parameters()) def iter_parameters(self): - return (item[1] for item in list(self.vars.items()) + return (item[1] for item in six.iteritems(self.vars) if isinstance(item[1], Parameter)) def create_var(self, *args, **kwargs): @@ -1052,6 +1040,9 @@ class Block(object): Returns: Variable: the Variable with the giving name. """ + name = cpt.to_text(name) + new_name = cpt.to_text(new_name) + if not self.has_var(name): raise ValueError("var %s is not in current block" % name) v = self.var(name) @@ -1070,9 +1061,9 @@ class Block(object): else: raise ValueError("unsupported var type: %s", type(v)) orig_var_type = v.type - self.desc._rename_var(name, new_name) + self.desc._rename_var(cpt.to_bytes(name), cpt.to_bytes(new_name)) # NOTE: v is destroyed by C++ after calling _rename_var. - d = self.desc.find_var(new_name) + d = self.desc.find_var(cpt.to_bytes(new_name)) if var_type == "Parameter": var = Parameter( self, @@ -1103,7 +1094,7 @@ class Block(object): def _remove_var(self, name): self._sync_with_cpp() - self.desc._remove_var(name) + self.desc._remove_var(cpt.to_bytes(name)) del self.vars[name] def create_parameter(self, *args, **kwargs): @@ -1205,7 +1196,7 @@ class Block(object): # sync variables removed from c++ end for var in list(self.vars.keys()): - if not self.desc.find_var(var): + if not self.desc.find_var(cpt.to_bytes(var)): self.vars.pop(var) # sync operators from cpp @@ -1583,7 +1574,9 @@ class Program(object): p.current_block_idx = self.current_block_idx p._seed = self._seed p.desc = core.ProgramDesc(self.desc) - p.blocks = [Block(p, i) for i in xrange(self.desc.num_blocks())] + p.blocks = [ + Block(p, i) for i in six.moves.range(self.desc.num_blocks()) + ] p._current_role = self._current_role p._op_role_var = self._op_role_var @@ -1639,7 +1632,9 @@ class Program(object): targets_idx.append([t.block.idx, t.idx]) res = Program() res.desc = core.prune(self.desc, targets_idx) - res.blocks = [Block(res, i) for i in range(res.desc.num_blocks())] + res.blocks = [ + Block(res, i) for i in six.moves.range(res.desc.num_blocks()) + ] res._sync_with_cpp() return res @@ -1682,16 +1677,18 @@ class Program(object): root_block._remove_op(0, read_op_idx + 1) for var in root_block.all_vars(): if var.type() == core.VarDesc.VarType.READER: - root_block._remove_var(var.name()) + root_block._remove_var(cpt.to_bytes(var.name())) # change all `is_test` attributes to True - for i in range(res.desc.num_blocks()): + for i in six.moves.range(res.desc.num_blocks()): block = res.desc.block(i) - for j in range(block.op_size()): + for j in six.moves.range(block.op_size()): op = block.op(j) if op.has_attr('is_test'): op.set_attr('is_test', True) - res.blocks = [Block(res, i) for i in range(res.desc.num_blocks())] + res.blocks = [ + Block(res, i) for i in six.moves.range(res.desc.num_blocks()) + ] res._sync_with_cpp() return res @@ -1711,7 +1708,7 @@ class Program(object): """ p = Program() p.desc = core.ProgramDesc(binary_str) - p.blocks = [Block(p, i) for i in range(p.desc.num_blocks())] + p.blocks = [Block(p, i) for i in six.moves.range(p.desc.num_blocks())] p._sync_with_cpp() return p diff --git a/python/paddle/fluid/graphviz.py b/python/paddle/fluid/graphviz.py index ba67bf5ae6fe44ea23414d444a270c436c195326..2b18d854d18bcbebce2a0eb30b8690db49d9d246 100644 --- a/python/paddle/fluid/graphviz.py +++ b/python/paddle/fluid/graphviz.py @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import os import random import six +import functools import subprocess import logging @@ -105,8 +108,9 @@ class Graph(object): def _rank_repr(self): ranks = sorted( - list(self.rank_groups.items()), - cmp=lambda a, b: a[1].priority > b[1].priority) + six.iteritems(self.rank_groups), + key=functools.cmp_to_key( + lambda a, b: a[1].priority > b[1].priority)) repr = [] for x in ranks: repr.append(str(x[1])) @@ -149,7 +153,7 @@ class Node(object): name=self.name, label=self.label, extra=',' + ','.join("%s=%s" % (key, crepr(value)) - for key, value in list(self.attrs.items())) + for key, value in six.iteritems(self.attrs)) if self.attrs else "") return reprs @@ -173,7 +177,7 @@ class Edge(object): target=self.target.name, extra="" if not self.attrs else "[" + ','.join("{}={}".format(attr[0], crepr(attr[1])) - for attr in list(self.attrs.items())) + "]") + for attr in six.iteritems(self.attrs)) + "]") return repr diff --git a/python/paddle/fluid/inferencer.py b/python/paddle/fluid/inferencer.py index ff382d8b832b4b2bc6779dbb28d3fd95c8a0984e..3d2ef566173f81b29a6d8ea79cff00991a4ef3c4 100644 --- a/python/paddle/fluid/inferencer.py +++ b/python/paddle/fluid/inferencer.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import contextlib from . import core diff --git a/python/paddle/fluid/initializer.py b/python/paddle/fluid/initializer.py index 3f740dd7c539973c5bb2b260120729e9563eedd2..bd46ed8e50c9344d471578eb0f89b7e214d62722 100644 --- a/python/paddle/fluid/initializer.py +++ b/python/paddle/fluid/initializer.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from . import framework import numpy as np import contextlib -from .framework import convert_np_dtype_to_dtype_ from .core import VarDesc __all__ = [ diff --git a/python/paddle/fluid/io.py b/python/paddle/fluid/io.py index 049c0e40e6b5538fea6f45e64fffa5fd920322d6..4d4fb32c8c3eab26f0a26a504bfb16115db89096 100644 --- a/python/paddle/fluid/io.py +++ b/python/paddle/fluid/io.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import os import errno import time @@ -607,25 +609,15 @@ def save_inference_model(dirname, # "./infer_model". """ - if isinstance(feeded_var_names, six.binary_type): + if isinstance(feeded_var_names, six.string_types): feeded_var_names = [feeded_var_names] - elif isinstance(feeded_var_names, six.text_type): - feeded_var_names = [feeded_var_names.encode()] else: if len(feeded_var_names) > 0: # TODO(paddle-dev): polish these code blocks if not (bool(feeded_var_names) and all( - isinstance(name, six.binary_type) + isinstance(name, six.string_types) for name in feeded_var_names)): - if not (all( - isinstance(name, six.text_type) - for name in feeded_var_names)): - raise ValueError( - "'feed_var_names' should be a list of str.") - else: - feeded_var_names = [ - name.encode() for name in feeded_var_names - ] + raise ValueError("'feed_var_names' should be a list of str.") if isinstance(target_vars, Variable): target_vars = [target_vars] diff --git a/python/paddle/fluid/layer_helper.py b/python/paddle/fluid/layer_helper.py index 0c2b1eb795860373220eb254612161f7dc816ffd..bd9727b6ac0208b199091db00bd0fd5fae74d53b 100644 --- a/python/paddle/fluid/layer_helper.py +++ b/python/paddle/fluid/layer_helper.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import copy import itertools import six @@ -85,7 +87,7 @@ class LayerHelper(object): raise ValueError("parameter number mismatch") elif len(param_attr) == 1 and length != 1: tmp = [None] * length - for i in range(length): + for i in six.moves.range(length): tmp[i] = copy.deepcopy(param_attr[0]) param_attr = tmp return param_attr diff --git a/python/paddle/fluid/layers/__init__.py b/python/paddle/fluid/layers/__init__.py index a48e360463456ab7e00534dc0684aa153c8205cd..a2a808777ddc499570eb9ef92175787a14cf77ca 100644 --- a/python/paddle/fluid/layers/__init__.py +++ b/python/paddle/fluid/layers/__init__.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from . import ops from .ops import * from . import nn diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index 9fb7b4d0cad67db2d2d4b56e43d8837b8160cdb0..173567a0a374ac4453025b67b047950936df2055 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import contextlib from .layer_function_generator import autodoc, templatedoc @@ -22,6 +24,7 @@ from ..initializer import force_init_on_cpu from .ops import logical_and, logical_not, logical_or import numpy import warnings +import six from functools import reduce __all__ = [ @@ -602,7 +605,7 @@ class StaticRNN(object): boot_memories = [] pre_memories = [] memories = [] - for _, mem in list(self.memories.items()): + for _, mem in six.iteritems(self.memories): boot_memories.append(mem.init) pre_memories.append(mem.pre_mem.name) mem_var = rnn_block.var(mem.mem.name) diff --git a/python/paddle/fluid/layers/detection.py b/python/paddle/fluid/layers/detection.py index 0800c02d9ec7dc51d4f22d3b04f3ae90315ebad8..72071478845df444ce72ce946787b2d0ce5f0d23 100644 --- a/python/paddle/fluid/layers/detection.py +++ b/python/paddle/fluid/layers/detection.py @@ -15,12 +15,18 @@ All layers just related to the detection neural network. """ +from __future__ import print_function + from .layer_function_generator import generate_layer_fn from .layer_function_generator import autodoc, templatedoc from ..layer_helper import LayerHelper from . import tensor from . import nn +from . import ops +from ... import compat as cpt import math +import six +import numpy from functools import reduce __all__ = [ @@ -102,7 +108,7 @@ def rpn_target_assign(loc, examples. Returns: - tuple: + tuple: A tuple(predicted_scores, predicted_location, target_label, target_bbox) is returned. The predicted_scores and predicted_location is the predicted result of the RPN. @@ -113,7 +119,7 @@ def rpn_target_assign(loc, anchors. The predicted_scores is a 2D Tensor with shape [F + B, 1], and the shape of target_label is same as the shape of the predicted_scores, B is the number of the background - anchors, the F and B is depends on the input of this operator. + anchors, the F and B is depends on the input of this operator. Examples: .. code-block:: python @@ -230,8 +236,8 @@ def detection_output(loc, nms_eta(float): The parameter for adaptive NMS. Returns: - Variable: - + Variable: + The detection outputs is a LoDTensor with shape [No, 6]. Each row has six values: [label, confidence, xmin, ymin, xmax, ymax]. `No` is the total number of detections in this mini-batch. For each @@ -264,10 +270,11 @@ def detection_output(loc, prior_box_var=prior_box_var, target_box=loc, code_type='decode_center_size') - old_shape = scores.shape - scores = nn.reshape(x=scores, shape=(-1, old_shape[-1])) + compile_shape = scores.shape + run_shape = ops.shape(scores) + scores = nn.flatten(x=scores, axis=2) scores = nn.softmax(input=scores) - scores = nn.reshape(x=scores, shape=old_shape) + scores = nn.reshape(x=scores, shape=compile_shape, actual_shape=run_shape) scores = nn.transpose(scores, perm=[0, 2, 1]) scores.stop_gradient = True nmsed_outs = helper.create_tmp_variable(dtype=decoded_box.dtype) @@ -501,7 +508,7 @@ def target_assign(input, Assumed that the row offset for each instance in `neg_indices` is called neg_lod, for i-th instance and each `id` of neg_indices in this instance: - + .. code-block:: text out[i][id][0 : K] = {mismatch_value, mismatch_value, ...} @@ -519,11 +526,11 @@ def target_assign(input, mismatch_value (float32): Fill this value to the mismatched location. Returns: - tuple: - A tuple(out, out_weight) is returned. out is a 3D Tensor with - shape [N, P, K], N and P is the same as they are in - `neg_indices`, K is the same as it in input of X. If - `match_indices[i][j]`. out_weight is the weight for output with + tuple: + A tuple(out, out_weight) is returned. out is a 3D Tensor with + shape [N, P, K], N and P is the same as they are in + `neg_indices`, K is the same as it in input of X. If + `match_indices[i][j]`. out_weight is the weight for output with the shape of [N, P, 1]. Examples: @@ -677,9 +684,10 @@ def ssd_loss(location, raise ValueError("Only support mining_type == max_negative now.") num, num_prior, num_class = confidence.shape + conf_shape = ops.shape(confidence) def __reshape_to_2d(var): - return nn.reshape(x=var, shape=[-1, var.shape[-1]]) + return nn.flatten(x=var, axis=2) # 1. Find matched boundding box by prior box. # 1.1 Compute IOU similarity between ground-truth boxes and prior boxes. @@ -690,7 +698,8 @@ def ssd_loss(location, # 2. Compute confidence for mining hard examples # 2.1. Get the target label based on matched indices - gt_label = nn.reshape(x=gt_label, shape=gt_label.shape + (1, )) + gt_label = nn.reshape( + x=gt_label, shape=(len(gt_label.shape) - 1) * (0, ) + (-1, 1)) gt_label.stop_gradient = True target_label, _ = target_assign( gt_label, matched_indices, mismatch_value=background_label) @@ -701,9 +710,12 @@ def ssd_loss(location, target_label = __reshape_to_2d(target_label) target_label.stop_gradient = True conf_loss = nn.softmax_with_cross_entropy(confidence, target_label) - # 3. Mining hard examples - conf_loss = nn.reshape(x=conf_loss, shape=(num, num_prior)) + conf_loss = nn.reshape( + x=conf_loss, + shape=(num, num_prior), + actual_shape=ops.slice( + conf_shape, axes=[0], starts=[0], ends=[2])) conf_loss.stop_gradient = True neg_indices = helper.create_tmp_variable(dtype='int32') dtype = matched_indices.dtype @@ -772,7 +784,11 @@ def ssd_loss(location, # 5.3 Compute overall weighted loss. loss = conf_loss_weight * conf_loss + loc_loss_weight * loc_loss # reshape to [N, Np], N is the batch size and Np is the prior box number. - loss = nn.reshape(x=loss, shape=[-1, num_prior]) + loss = nn.reshape( + x=loss, + shape=(num, num_prior), + actual_shape=ops.slice( + conf_shape, axes=[0], starts=[0], ends=[2])) loss = nn.reduce_sum(loss, dim=1, keep_dim=True) if normalize: normalizer = nn.reduce_sum(target_loc_weight) @@ -822,7 +838,7 @@ def prior_box(input, offset(float): Prior boxes center offset. Default: 0.5 name(str): Name of the prior box op. Default: None. min_max_aspect_ratios_order(bool): If set True, the output prior box is - in order of [min, max, aspect_ratios], which is consistent with + in order of [min, max, aspect_ratios], which is consistent with Caffe. Please note, this order affects the weights order of convolution layer followed by and does not affect the final detection results. Default: False. @@ -965,7 +981,7 @@ def multi_box_head(inputs, stride(int|list|tuple): The stride of conv2d. Default:1, name(str): Name of the prior box layer. Default: None. min_max_aspect_ratios_order(bool): If set True, the output prior box is - in order of [min, max, aspect_ratios], which is consistent with + in order of [min, max, aspect_ratios], which is consistent with Caffe. Please note, this order affects the weights order of convolution layer followed by and does not affect the fininal detection results. Default: False. @@ -1005,13 +1021,7 @@ def multi_box_head(inputs, """ def _reshape_with_axis_(input, axis=1): - if not (axis > 0 and axis < len(input.shape)): - raise ValueError("The axis should be smaller than " - "the arity of input and bigger than 0.") - new_shape = [ - -1, reduce(lambda x, y: x * y, input.shape[axis:len(input.shape)]) - ] - out = nn.reshape(x=input, shape=new_shape) + out = nn.flatten(x=input, axis=axis) return out def _is_list_or_tuple_(data): @@ -1033,7 +1043,7 @@ def multi_box_head(inputs, min_sizes = [] max_sizes = [] step = int(math.floor(((max_ratio - min_ratio)) / (num_layer - 2))) - for ratio in range(min_ratio, max_ratio + 1, step): + for ratio in six.moves.range(min_ratio, max_ratio + 1, step): min_sizes.append(base_size * ratio / 100.) max_sizes.append(base_size * (ratio + step) / 100.) min_sizes = [base_size * .10] + min_sizes @@ -1101,11 +1111,13 @@ def multi_box_head(inputs, stride=stride) mbox_loc = nn.transpose(mbox_loc, perm=[0, 2, 3, 1]) - new_shape = [ - mbox_loc.shape[0], - mbox_loc.shape[1] * mbox_loc.shape[2] * mbox_loc.shape[3] / 4, 4 + compile_shape = [ + mbox_loc.shape[0], cpt.floor_division( + mbox_loc.shape[1] * mbox_loc.shape[2] * mbox_loc.shape[3], 4), 4 ] - mbox_loc_flatten = nn.reshape(mbox_loc, shape=new_shape) + run_shape = tensor.assign(numpy.array([0, -1, 4]).astype("int32")) + mbox_loc_flatten = nn.reshape( + mbox_loc, shape=compile_shape, actual_shape=run_shape) mbox_locs.append(mbox_loc_flatten) # get conf @@ -1117,11 +1129,16 @@ def multi_box_head(inputs, padding=pad, stride=stride) conf_loc = nn.transpose(conf_loc, perm=[0, 2, 3, 1]) - new_shape = [ - conf_loc.shape[0], conf_loc.shape[1] * conf_loc.shape[2] * - conf_loc.shape[3] / num_classes, num_classes + new_shape = [0, -1, num_classes] + compile_shape = [ + conf_loc.shape[0], + cpt.floor_division(conf_loc.shape[1] * conf_loc.shape[2] * + conf_loc.shape[3], num_classes), num_classes ] - conf_loc_flatten = nn.reshape(conf_loc, shape=new_shape) + run_shape = tensor.assign( + numpy.array([0, -1, num_classes]).astype("int32")) + conf_loc_flatten = nn.reshape( + conf_loc, shape=compile_shape, actual_shape=run_shape) mbox_confs.append(conf_loc_flatten) if len(box_results) == 1: diff --git a/python/paddle/fluid/layers/device.py b/python/paddle/fluid/layers/device.py index bb1fb7fd571a56acf367e663af0cf9431211bcea..43ebd160de3fd3d2a491a3ec1fbe0e4085fbd0b1 100644 --- a/python/paddle/fluid/layers/device.py +++ b/python/paddle/fluid/layers/device.py @@ -15,6 +15,8 @@ All util layers. """ +from __future__ import print_function + from .layer_function_generator import autodoc from ..framework import unique_name from ..layer_helper import LayerHelper diff --git a/python/paddle/fluid/layers/io.py b/python/paddle/fluid/layers/io.py index 327ae309816344a0bcebfe70ffb59a00eab1d86f..21a295a0982cbc51947a063beee542c13494024d 100644 --- a/python/paddle/fluid/layers/io.py +++ b/python/paddle/fluid/layers/io.py @@ -11,8 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import contextlib import multiprocessing +import six import threading from ..data_feeder import DataFeeder @@ -69,7 +72,7 @@ def data(name, """ helper = LayerHelper('data', **locals()) shape = list(shape) - for i in range(len(shape)): + for i in six.moves.range(len(shape)): if shape[i] is None: shape[i] = -1 append_batch_size = False @@ -674,7 +677,7 @@ def py_reader(capacity, def __tensor_provider__(): for slots in paddle_reader(): - yield [slots[str(idx)] for idx in xrange(counter)] + yield [slots[str(idx)] for idx in six.moves.xrange(counter)] __set_tensor_provider__(__tensor_provider__) @@ -750,7 +753,7 @@ def open_files(filenames, else: buffer_size = int(buffer_size) - if isinstance(filenames, basestring): + if isinstance(filenames, six.string_types): filenames = [filenames] dtypes = [convert_np_dtype_to_dtype_(dt) for dt in dtypes] shape_concat = [] @@ -1005,7 +1008,7 @@ class Preprocessor(object): source_lod_levels = self.underlying_reader.desc.lod_levels() self.source_var_names = [ unique_name("preprocessor_source") - for _ in range(len(source_shapes)) + for _ in six.moves.range(len(source_shapes)) ] source_vars = [] for var_name, shape, dtype, lod_level in zip( diff --git a/python/paddle/fluid/layers/layer_function_generator.py b/python/paddle/fluid/layers/layer_function_generator.py index c0d72620b1ddb183f43ebce766688518b5a737ac..8963d74de014d69c590276d5ff7080111f614230 100644 --- a/python/paddle/fluid/layers/layer_function_generator.py +++ b/python/paddle/fluid/layers/layer_function_generator.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import re import functools import warnings diff --git a/python/paddle/fluid/layers/learning_rate_scheduler.py b/python/paddle/fluid/layers/learning_rate_scheduler.py index daf91a40f7ad7935d355a287819ad1dbcdd84eb8..be368007dd7061ba7fc97414dbadfce00d158776 100644 --- a/python/paddle/fluid/layers/learning_rate_scheduler.py +++ b/python/paddle/fluid/layers/learning_rate_scheduler.py @@ -20,6 +20,8 @@ User can also implement their own learning_rate_decay strategy according to this module. """ +from __future__ import print_function + from . import control_flow from . import nn from . import ops @@ -72,10 +74,10 @@ def noam_decay(d_model, warmup_steps): def exponential_decay(learning_rate, decay_steps, decay_rate, staircase=False): """ - Applies exponential decay to the learning rate. + Applies exponential decay to the learning rate. - When training a model, it is often recommended to lower the learning rate as the - training progresses. By using this function, the learning rate will be decayed by + When training a model, it is often recommended to lower the learning rate as the + training progresses. By using this function, the learning rate will be decayed by 'decay_rate' every 'decay_steps' steps. >>> if staircase == True: @@ -148,8 +150,8 @@ def inverse_time_decay(learning_rate, decay_steps, decay_rate, staircase=False): """ Applies inverse time decay to the initial learning rate. - When training a model, it is often recommended to lower the learning rate as the - training progresses. By using this function, an inverse decay function will be + When training a model, it is often recommended to lower the learning rate as the + training progresses. By using this function, an inverse decay function will be applied to the initial learning rate. >>> if staircase == True: diff --git a/python/paddle/fluid/layers/math_op_patch.py b/python/paddle/fluid/layers/math_op_patch.py index 0e10a91d25877984396f9bcf9aae6438707eeab1..a458cebfb194a068d040a8919fd4abcb4b4bea80 100644 --- a/python/paddle/fluid/layers/math_op_patch.py +++ b/python/paddle/fluid/layers/math_op_patch.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from ..framework import Variable, unique_name from .layer_function_generator import OpProtoHolder from ..initializer import force_init_on_cpu diff --git a/python/paddle/fluid/layers/metric_op.py b/python/paddle/fluid/layers/metric_op.py index 49bae1e8af768d93294120e1d13ef0242313aa3c..2c3bdd77e1fa1c86baa3a288caab4ad4324e2ef2 100644 --- a/python/paddle/fluid/layers/metric_op.py +++ b/python/paddle/fluid/layers/metric_op.py @@ -15,6 +15,8 @@ All layers just related to metric. """ +from __future__ import print_function + import warnings from ..layer_helper import LayerHelper from ..initializer import Normal, Constant @@ -81,9 +83,9 @@ def auc(input, label, curve='ROC', num_thresholds=200, topk=1): **Area Under the Curve (AUC) Layer** This implementation computes the AUC according to forward output and label. - It is used very widely in binary classification evaluation. + It is used very widely in binary classification evaluation. - Note: If input label contains values other than 0 and 1, it will be cast + Note: If input label contains values other than 0 and 1, it will be cast to `bool`. Find the relevant definitions `here `_. @@ -93,14 +95,14 @@ def auc(input, label, curve='ROC', num_thresholds=200, topk=1): 2. PR: Precision Recall Args: - input(Variable): A floating-point 2D Variable, values are in the range - [0, 1]. Each row is sorted in descending order. This - input should be the output of topk. Typically, this + input(Variable): A floating-point 2D Variable, values are in the range + [0, 1]. Each row is sorted in descending order. This + input should be the output of topk. Typically, this Variable indicates the probability of each label. - label(Variable): A 2D int Variable indicating the label of the training + label(Variable): A 2D int Variable indicating the label of the training data. The height is batch size and width is always 1. curve(str): Curve type, can be 'ROC' or 'PR'. Default 'ROC'. - num_thresholds(int): The number of thresholds to use when discretizing + num_thresholds(int): The number of thresholds to use when discretizing the roc curve. Default 200. topk(int): only topk number of prediction output will be used for auc. @@ -109,7 +111,7 @@ def auc(input, label, curve='ROC', num_thresholds=200, topk=1): Examples: .. code-block:: python - + # network is a binary classification model and label the ground truth prediction = network(image, is_infer=True) auc_out=fluid.layers.auc(input=prediction, label=label) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 0960b54123d36b0ddcced6841384399cab816f48..bd2b950cffe646128501772899fea2e09d9bdce3 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -11,24 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -# Copyright (c ) 2018 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. """ All layers just related to the neural network. """ +from __future__ import print_function + from ..layer_helper import LayerHelper from ..initializer import Normal, Constant from ..framework import Variable @@ -112,6 +100,8 @@ __all__ = [ 'log', 'crop', 'rank_loss', + 'prelu', + 'flatten', ] @@ -360,7 +350,7 @@ def dynamic_lstm(input, """ helper = LayerHelper('lstm', **locals()) - size = size / 4 + size = size // 4 weight = helper.create_parameter( attr=helper.param_attr, shape=[size, 4 * size], dtype=dtype) bias_size = [1, 7 * size] @@ -550,7 +540,7 @@ def dynamic_lstmp(input, """ helper = LayerHelper('lstmp', **locals()) - size = size / 4 + size = size // 4 weight = helper.create_parameter( attr=helper.param_attr, shape=[proj_size, 4 * size], dtype=dtype) proj_weight = helper.create_parameter( @@ -778,7 +768,7 @@ def gru_unit(input, helper = LayerHelper('gru_unit', **locals()) dtype = helper.input_dtype() - size = size / 3 + size = size // 3 # create weight weight = helper.create_parameter( @@ -1262,7 +1252,7 @@ def sequence_conv(input, outputs={"Out": pre_bias}, attrs={ 'contextStride': filter_stride, - 'contextStart': -int(filter_size / 2), + 'contextStart': -int(filter_size // 2), 'contextLength': filter_size }) pre_act = helper.append_bias_op(pre_bias) @@ -1318,15 +1308,15 @@ def sequence_softmax(input, param_attr=None, bias_attr=None, use_cudnn=True): def softmax(input, param_attr=None, bias_attr=None, use_cudnn=True, name=None): """ - The input of the softmax operator is a tensor of any rank. The output tensor + The input of the softmax operator is a tensor of any rank. The output tensor has the same shape as the input. - The input tensor will first be logically flattened to a 2-D matrix. The matrix's - second dimension(row length) is as same as the last dimension of the input - tensor, and the first dimension(column length) is the product of all other - dimensions of the input tensor. For each row of the matrix, the softmax operator - squashes the K-dimensional(K is the width of the matrix, which is also the size - of the input tensor's last dimension) vector of arbitrary real values to a + The input tensor will first be logically flattened to a 2-D matrix. The matrix's + second dimension(row length) is as same as the last dimension of the input + tensor, and the first dimension(column length) is the product of all other + dimensions of the input tensor. For each row of the matrix, the softmax operator + squashes the K-dimensional(K is the width of the matrix, which is also the size + of the input tensor's last dimension) vector of arbitrary real values to a K-dimensional vector of real values in the range [0, 1] that add up to 1. It computes the exponential of the given dimension and the sum of exponential @@ -1494,7 +1484,7 @@ def conv2d(input, else: if num_channels % groups != 0: raise ValueError("num_channels must be divisible by groups.") - num_filter_channels = num_channels / groups + num_filter_channels = num_channels // groups filter_size = utils.convert_to_list(filter_size, 2, 'filter_size') stride = utils.convert_to_list(stride, 2, 'stride') @@ -1505,7 +1495,7 @@ def conv2d(input, raise ValueError("use_cudnn should be True or False") input_shape = input.shape - filter_shape = [num_filters, num_filter_channels] + filter_size + filter_shape = [num_filters, int(num_filter_channels)] + filter_size def _get_default_param_initializer(): std = (2.0 / (filter_size[0]**2 * num_channels))**0.5 @@ -1656,7 +1646,7 @@ def conv3d(input, else: if num_channels % groups != 0: raise ValueError("num_channels must be divisible by groups.") - num_filter_channels = num_channels / groups + num_filter_channels = num_channels // groups filter_size = utils.convert_to_list(filter_size, 3, 'filter_size') stride = utils.convert_to_list(stride, 3, 'stride') @@ -2391,16 +2381,16 @@ def conv2d_transpose(input, w_in = input.shape[3] filter_size_h = (output_size[0] - (h_in - 1) * stride[0] + 2 * - padding[0] - 1) / dilation[0] + 1 + padding[0] - 1) // dilation[0] + 1 filter_size_w = (output_size[1] - (w_in - 1) * stride[1] + 2 * - padding[1] - 1) / dilation[1] + 1 + padding[1] - 1) // dilation[1] + 1 filter_size = [filter_size_h, filter_size_w] else: filter_size = utils.convert_to_list(filter_size, 2, 'conv2d_transpose.filter_size') groups = 1 if groups is None else groups - filter_shape = [input_channel, num_filters / groups] + filter_size + filter_shape = [input_channel, num_filters // groups] + filter_size img_filter = helper.create_parameter( dtype=input.dtype, shape=filter_shape, attr=helper.param_attr) @@ -2558,18 +2548,18 @@ def conv3d_transpose(input, w_in = input.shape[4] filter_size_d = (output_size[0] - (d_in - 1) * stride[0] + 2 * - padding[0] - 1) / dilation[0] + 1 + padding[0] - 1) // dilation[0] + 1 filter_size_h = (output_size[1] - (h_in - 1) * stride[1] + 2 * - padding[1] - 1) / dilation[1] + 1 + padding[1] - 1) // dilation[1] + 1 filter_size_w = (output_size[2] - (w_in - 1) * stride[2] + 2 * - padding[2] - 1) / dilation[2] + 1 + padding[2] - 1) // dilation[2] + 1 filter_size = [filter_size_d, filter_size_h, filter_size_w] else: filter_size = utils.convert_to_list(filter_size, 3, 'conv3d_transpose.filter_size') groups = 1 if groups is None else groups - filter_shape = [input_channel, num_filters / groups] + filter_size + filter_shape = [input_channel, num_filters // groups] + filter_size img_filter = helper.create_parameter( dtype=input.dtype, shape=filter_shape, attr=helper.param_attr) @@ -2676,15 +2666,15 @@ def beam_search(pre_ids, Refer to `Beam search `_ for more details. - - This layer does the search in beams for one time step. Specifically, it + + This layer does the search in beams for one time step. Specifically, it selects the top-K candidate word ids of current step from :attr:`ids` according to their :attr:`scores` for all source sentences, where K is :attr:`beam_size` and :attr:`ids, scores` are predicted results from the computation cell. Additionally, :attr:`pre_ids` and :attr:`pre_scores` are the output of beam_search at previous step, they are needed for special use to handle ended candidate translations. - + Note that the :attr:`scores` passed in should be accumulated scores, and length penalty should be done with extra operators before calculating the accumulated scores if needed, also suggest finding top-K before it and @@ -3885,7 +3875,7 @@ def nce(input, def hsigmoid(input, label, num_classes, param_attr=None, bias_attr=None): """ The hierarchical sigmoid operator is used to accelerate the training - process of language model. This operator organizes the classes into a + process of language model. This operator organizes the classes into a complete binary tree, each leaf node represents a class(a word) and each internal node acts as a binary classifier. For each word there's a unique path from root to it's leaf node, hsigmoid calculate the cost for each @@ -3895,9 +3885,9 @@ def hsigmoid(input, label, num_classes, param_attr=None, bias_attr=None): Refer to `Hierarchical Probabilistic Neural Network Language Model `_ - + Args: - input (Variable): The input tensor variable with shape + input (Variable): The input tensor variable with shape :math:`[N \\times D]`, where :math:`N` is the size of mini-batch, and :math:`D` is the feature size. label (Variable): The tensor variable contains labels of training data. @@ -3905,7 +3895,7 @@ def hsigmoid(input, label, num_classes, param_attr=None, bias_attr=None): num_classes: (int), The number of classes, must not be less than 2. param_attr (ParamAttr|list of ParamAttr, default None): The parameter attribute for learnable parameters/weights of this layer. - bias_attr (ParamAttr|list of ParamAttr, default None): The parameter + bias_attr (ParamAttr|list of ParamAttr, default None): The parameter attribute for the bias of this layer. If it is set to False, no bias will be applied. @@ -5088,7 +5078,7 @@ def random_crop(x, shape, seed=None): return out -def log(x): +def log(x, name=None): """ Calculates the natural log of the given input tensor, element-wise. @@ -5098,6 +5088,8 @@ def log(x): Args: x (Variable): Input tensor. + name (str|None, default None): A name for this layer If set None, + the layer will be named automatically. Returns: Variable: The natural log of the input tensor computed element-wise. @@ -5115,7 +5107,7 @@ def log(x): return out -def relu(x): +def relu(x, name=None): """ Relu takes one input data (Tensor) and produces one output data (Tensor) where the rectified linear function, y = max(0, x), is applied to @@ -5127,6 +5119,8 @@ def relu(x): Args: x (Variable): The input tensor. + name (str|None, default None): A name for this layer If set None, + the layer will be named automatically. Returns: Variable: The output tensor with the same shape as input. @@ -5300,23 +5294,23 @@ def rank_loss(label, left, right, name=None): is a pairwise ranking model with a training sample consisting of a pair of documents, A and B. Label P indicates whether A is ranked higher than B or not: - + P = {0, 1} or {0, 0.5, 1}, where 0.5 means that there is no information about the rank of the input pair. - + Rank loss layer takes three inputs: left (o_i), right (o_j) and label (P_{i,j}). The inputs respectively represent RankNet's output scores for documents A and B and the value of label P. The following equation computes rank loss C_{i,j} from the inputs: - + $$ C_{i,j} = -\tilde{P_{ij}} * o_{i,j} + \log(1 + e^{o_{i,j}}) \\ o_{i,j} = o_i - o_j \\ \tilde{P_{i,j}} = \left \{0, 0.5, 1 \right \} \ or \ \left \{0, 1 \right \} $$ - - Rank loss layer takes batch inputs with size batch_size (batch_size >= 1). - + + Rank loss layer takes batch inputs with size batch_size (batch_size >= 1). + Args: label (Variable): Indicats whether A ranked higher than B or not. left (Variable): RankNet's output score for doc A. @@ -5361,3 +5355,123 @@ def rank_loss(label, left, right, name=None): "Right": right}, outputs={'Out': out}) return out + + +def prelu(x, mode, param_attr=None, name=None): + """ + Equation: + + y = \max(0, x) + alpha \min(0, x) + + Args: + x (Variable): The input tensor. + param_attr(ParamAttr|None): The parameter attribute for the learnable + weight (alpha). + mode (string): The mode for weight sharing + all: all elements share same weight + channel:elements in a channel share same weight + element:each element has a weight + name(str|None): A name for this layer(optional). If set None, the layer + will be named automatically. + + Returns: + Variable: The output tensor with the same shape as input. + + Examples: + + .. code-block:: python + + x = fluid.layers.data(name="x", shape=[10,10], dtype="float32") + mode = 'channel' + output = fluid.layers.prelu(x,mode) + """ + helper = LayerHelper('prelu', **locals()) + if mode not in ['all', 'channel', 'element']: + raise ValueError('mode should be one of all, channel, element.') + alpha_shape = [1] + if mode == 'channel': + alpha_shape = [1, x.shape[1], 1, 1] + elif mode == 'element': + alpha_shape = x.shape + dtype = helper.input_dtype(input_param_name='x') + alpha = helper.create_parameter( + attr=param_attr, + shape=alpha_shape, + dtype='float32', + is_bias=False, + default_initializer=Constant(1.0)) + out = helper.create_tmp_variable(dtype) + helper.append_op( + type="prelu", + inputs={"X": x, + 'Alpha': alpha}, + attrs={"mode": mode}, + outputs={"Out": out}) + return out + + +def flatten(x, axis=1, name=None): + """ + **Flatten layer** + Flattens the input tensor into a 2D matrix. + + Examples: + Case 1: + Given + X.shape = (3, 100, 100, 4) + and + axis = 2 + We get: + Out.shape = (3 * 100, 4 * 100) + + Case 2: + Given + X.shape = (3, 100, 100, 4) + and + axis = 0 + We get: + Out.shape = (1, 3 * 100 * 100 * 4) + + Args: + x (Variable): A tensor of rank >= axis. + axis (int): Indicate up to which input dimensions (exclusive) should + be flattened to the outer dimension of the output. + The value for axis must be in the range [0, R], where R + is the rank of the input tensor. When axis = 0, the shape + of the output tensor is (1, (d_0 X d_1 ... d_n), where the + shape of the input tensor is (d_0, d_1, ... d_n). + name(str|None): A name for this layer(optional). If set None, the layer + will be named automatically. + + Returns: + Variable: A 2D tensor with the contents of the input tensor, with input + dimensions up to axis flattened to the outer dimension of + the output and remaining input dimensions flattened into the + inner dimension of the output. + + Raises: + ValueError: If x is not a variable. + ValueError: If axis is not in range [0, rank(x)]. + + Examples: + + .. code-block:: python + + x = fluid.layers.data(name="x", shape=[4, 4, 3], dtype="float32") + out = fluid.layers.flatten(x=x, axis=2) + """ + helper = LayerHelper('flatten', **locals()) + + if not (isinstance(x, Variable)): + raise ValueError("The input x should be a Variable") + + if not (isinstance(axis, int)) or axis > len(x.shape) or axis < 0: + raise ValueError("The axis should be a int, and in range [0, rank(x)]") + + out = helper.create_tmp_variable(x.dtype) + helper.append_op( + type='flatten', + inputs={"X": x}, + outputs={'Out': out}, + attrs={"axis": axis}) + return out diff --git a/python/paddle/fluid/layers/ops.py b/python/paddle/fluid/layers/ops.py index f70c7f2258ce588444cf46d6c8affc4c9555203e..cc4a7de163ac2e52e43131a021160051423e040c 100644 --- a/python/paddle/fluid/layers/ops.py +++ b/python/paddle/fluid/layers/ops.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function from .layer_function_generator import generate_layer_fn __activations__ = [ diff --git a/python/paddle/fluid/layers/tensor.py b/python/paddle/fluid/layers/tensor.py index b93d721c12cb6ead044dc790f2f2af8a61a63b60..04e71497aa762e390c4123c0bf3d7f111a772dd4 100644 --- a/python/paddle/fluid/layers/tensor.py +++ b/python/paddle/fluid/layers/tensor.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from ..layer_helper import LayerHelper from ..param_attr import ParamAttr from ..framework import convert_np_dtype_to_dtype_ diff --git a/python/paddle/fluid/layers/utils.py b/python/paddle/fluid/layers/utils.py index 49ec3088831dff415e042e1b0a632f63106eb07b..5688f04ab2382f5731e69c60225765a2094bba8c 100644 --- a/python/paddle/fluid/layers/utils.py +++ b/python/paddle/fluid/layers/utils.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import numpy as np diff --git a/python/paddle/fluid/lod_tensor.py b/python/paddle/fluid/lod_tensor.py index 53c33616f55be5f5ef7068a6e94418e17d739e3c..a9de09f31f4ed04ba1aa003e85b25fc5a91557e4 100644 --- a/python/paddle/fluid/lod_tensor.py +++ b/python/paddle/fluid/lod_tensor.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from . import core import numpy as np diff --git a/python/paddle/fluid/metrics.py b/python/paddle/fluid/metrics.py index cd8934522755691217a99a2cca271badda55368e..592cb23eb9319658f8542ed5bc6ab3e95cfdb118 100644 --- a/python/paddle/fluid/metrics.py +++ b/python/paddle/fluid/metrics.py @@ -14,11 +14,15 @@ """ Fluid Metrics -The metrics are accomplished via Python natively. +The metrics are accomplished via Python natively. """ + +from __future__ import print_function + import numpy as np import copy import warnings +import six __all__ = [ 'MetricBase', @@ -79,10 +83,10 @@ class MetricBase(object): """ states = { attr: value - for attr, value in list(self.__dict__.items()) + for attr, value in six.iteritems(self.__dict__) if not attr.startswith("_") } - for attr, value in list(states.items()): + for attr, value in six.iteritems(states): if isinstance(value, int): setattr(self, attr, 0) elif isinstance(value, float): @@ -105,7 +109,7 @@ class MetricBase(object): """ states = { attr: value - for attr, value in list(self.__dict__.items()) + for attr, value in six.iteritems(self.__dict__) if not attr.startswith("_") } config = {} @@ -141,10 +145,10 @@ class CompositeMetric(MetricBase): """ Composite multiple metrics in one instance. for example, merge F1, accuracy, recall into one Metric. - + Examples: .. code-block:: python - + labels = fluid.layers.data(name="data", shape=[1], dtype="int32") data = fluid.layers.data(name="data", shape=[32, 32], dtype="int32") pred = fluid.layers.fc(input=data, size=1000, act="tanh") diff --git a/python/paddle/fluid/net_drawer.py b/python/paddle/fluid/net_drawer.py index 623a7d3fd05567a26bb6923550f597a0e1e27e32..0b61c23d07e95acf7b4564753f748e7fb497e73e 100644 --- a/python/paddle/fluid/net_drawer.py +++ b/python/paddle/fluid/net_drawer.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import argparse import json import logging diff --git a/python/paddle/fluid/nets.py b/python/paddle/fluid/nets.py index 08480671d8a5c50bbec97930c451cbcdc241e1fe..051fe84364639ca6028326c0cb02b204a02531af 100644 --- a/python/paddle/fluid/nets.py +++ b/python/paddle/fluid/nets.py @@ -11,6 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function +import six from . import layers __all__ = [ @@ -210,7 +213,7 @@ def img_conv_group(input, conv_with_batchnorm = __extend_list__(conv_with_batchnorm) conv_batchnorm_drop_rate = __extend_list__(conv_batchnorm_drop_rate) - for i in range(len(conv_num_filter)): + for i in six.moves.range(len(conv_num_filter)): local_conv_act = conv_act if conv_with_batchnorm[i]: local_conv_act = None diff --git a/python/paddle/fluid/op.py b/python/paddle/fluid/op.py index 93f021a360ac61f64e769d057df188d79f6f2bb6..667db10d3ebdd24ddd9efbe2310ebb331e268ee2 100644 --- a/python/paddle/fluid/op.py +++ b/python/paddle/fluid/op.py @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + +import numpy as np import six import paddle.fluid.core as core @@ -99,6 +102,8 @@ class OpDescCreationMethod(object): new_attr = op_desc.attrs.add() new_attr.name = attr.name new_attr.type = attr.type + if isinstance(user_defined_attr, np.ndarray): + user_defined_attr = user_defined_attr.tolist() if attr.type == framework_pb2.INT: new_attr.i = user_defined_attr elif attr.type == framework_pb2.FLOAT: diff --git a/python/paddle/fluid/optimizer.py b/python/paddle/fluid/optimizer.py index a07325f46a2892222c2d1dcd74aa7cb01f6760a1..031ddd09a0b27b050b6ac651e4d8c46854092b2f 100644 --- a/python/paddle/fluid/optimizer.py +++ b/python/paddle/fluid/optimizer.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import re from collections import defaultdict from paddle.fluid.framework import Program, Variable diff --git a/python/paddle/fluid/parallel_executor.py b/python/paddle/fluid/parallel_executor.py index 2a3555ebdde4d54f63bb420218896560c1b40ffd..a7765c9591f0bd653c08036c46a36131906a758f 100644 --- a/python/paddle/fluid/parallel_executor.py +++ b/python/paddle/fluid/parallel_executor.py @@ -17,8 +17,10 @@ import multiprocessing from . import core from . import framework from . import executor +from .. import compat as cpt import warnings import sys +import six import os __all__ = ['ParallelExecutor', 'ExecutionStrategy', 'BuildStrategy'] @@ -95,7 +97,7 @@ class ParallelExecutor(object): self._places = [] self._act_places = [] if use_cuda: - for i in range(core.get_cuda_device_count()): + for i in six.moves.range(core.get_cuda_device_count()): p = core.Place() self._act_places.append(core.CUDAPlace(i)) p.set_place(self._act_places[-1]) @@ -103,7 +105,7 @@ class ParallelExecutor(object): else: cpu_num = int( os.environ.get('CPU_NUM', multiprocessing.cpu_count())) - for i in range(cpu_num): + for i in six.moves.range(cpu_num): p = core.Place() self._act_places.append(core.CPUPlace()) p.set_place(self._act_places[-1]) @@ -153,11 +155,13 @@ class ParallelExecutor(object): self.executor = core.ParallelExecutor( self._places, set([ - p.name for p in main.global_block().iter_parameters() + cpt.to_text(p.name) + for p in main.global_block().iter_parameters() if not p.stop_gradient ]), - set(self.persistable_vars), main.desc, loss_name - if loss_name else '', scope, local_scopes, exec_strategy, + set(cpt.to_text(var) for var in self.persistable_vars), main.desc, + cpt.to_text(loss_name) + if loss_name else six.u(''), scope, local_scopes, exec_strategy, build_strategy, num_trainers, trainer_id) self.scope = scope diff --git a/python/paddle/fluid/param_attr.py b/python/paddle/fluid/param_attr.py index afae577656c8970338f3b02208fcb4c738628ab6..f0be794327f51cbbc4202b8b7b401b712b6d66a3 100644 --- a/python/paddle/fluid/param_attr.py +++ b/python/paddle/fluid/param_attr.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import six from .initializer import Initializer, Xavier, Constant diff --git a/python/paddle/fluid/profiler.py b/python/paddle/fluid/profiler.py index 01983a830351b018770e6358f604781ffaae5800..e05885f5f5bfc169828c1c6e723dffff098c3c2e 100644 --- a/python/paddle/fluid/profiler.py +++ b/python/paddle/fluid/profiler.py @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from . import core from contextlib import contextmanager import os +import six __all__ = [ 'cuda_profiler', 'reset_profiler', 'profiler', 'start_profiler', @@ -88,7 +91,7 @@ def cuda_profiler(output_file, output_mode=None, config=None): config = NVPROF_CONFIG if config is None else config config_file = 'nvprof_config_file' with open(config_file, 'wb') as fp: - fp.writelines(["%s\n" % item for item in config]) + fp.writelines([six.b("%s\n" % item) for item in config]) core.nvprof_init(output_file, output_mode, config_file) # Enables profiler collection by the active CUDA profiling tool. core.nvprof_start() diff --git a/python/paddle/fluid/recordio_writer.py b/python/paddle/fluid/recordio_writer.py index 93b38ad3fa37bd4bff04c529cd5518a8138e55ea..a69c0c29d4675d3e6b9b2a2d766b8be9935092cf 100644 --- a/python/paddle/fluid/recordio_writer.py +++ b/python/paddle/fluid/recordio_writer.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import os import contextlib from . import core diff --git a/python/paddle/fluid/regularizer.py b/python/paddle/fluid/regularizer.py index 6eaac4432d4df1288f37607a01484434542f1138..da38626111a6767e1a76a35d6d1375ccc1283de4 100644 --- a/python/paddle/fluid/regularizer.py +++ b/python/paddle/fluid/regularizer.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from . import framework from . import core diff --git a/python/paddle/fluid/tests/book/high-level-api/fit_a_line/test_fit_a_line.py b/python/paddle/fluid/tests/book/high-level-api/fit_a_line/test_fit_a_line.py index 36a1a223cfd7c69aff3e8648da990d23e4e75202..f6017a455df7e8bd197ef2563a759f843b5e7c73 100644 --- a/python/paddle/fluid/tests/book/high-level-api/fit_a_line/test_fit_a_line.py +++ b/python/paddle/fluid/tests/book/high-level-api/fit_a_line/test_fit_a_line.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle import paddle.fluid as fluid import contextlib diff --git a/python/paddle/fluid/tests/book/high-level-api/image_classification/cifar10_small_test_set.py b/python/paddle/fluid/tests/book/high-level-api/image_classification/cifar10_small_test_set.py index 9e4c384d92943227c2d68da829e6019e649a35fb..48c0f3d3611547308b5d4460748d3aab765f5805 100644 --- a/python/paddle/fluid/tests/book/high-level-api/image_classification/cifar10_small_test_set.py +++ b/python/paddle/fluid/tests/book/high-level-api/image_classification/cifar10_small_test_set.py @@ -28,12 +28,14 @@ images per class. """ +from __future__ import print_function + import itertools import numpy -import paddle.v2.dataset.common +import paddle.dataset.common import tarfile +import six from six.moves import cPickle as pickle -from six.moves import zip __all__ = ['train10'] @@ -44,20 +46,25 @@ CIFAR10_MD5 = 'c58f30108f718f92721af3b95e74349a' def reader_creator(filename, sub_name, batch_size=None): def read_batch(batch): - data = batch['data'] - labels = batch.get('labels', batch.get('fine_labels', None)) + data = batch[six.b('data')] + labels = batch.get( + six.b('labels'), batch.get(six.b('fine_labels'), None)) assert labels is not None - for sample, label in zip(data, labels): + for sample, label in six.moves.zip(data, labels): yield (sample / 255.0).astype(numpy.float32), int(label) def reader(): with tarfile.open(filename, mode='r') as f: - names = (each_item.name for each_item in f - if sub_name in each_item.name) + names = [ + each_item.name for each_item in f if sub_name in each_item.name + ] batch_count = 0 for name in names: - batch = pickle.load(f.extractfile(name)) + if six.PY2: + batch = pickle.load(f.extractfile(name)) + else: + batch = pickle.load(f.extractfile(name), encoding='bytes') for item in read_batch(batch): if isinstance(batch_size, int) and batch_count > batch_size: break @@ -78,6 +85,6 @@ def train10(batch_size=None): :rtype: callable """ return reader_creator( - paddle.v2.dataset.common.download(CIFAR10_URL, 'cifar', CIFAR10_MD5), + paddle.dataset.common.download(CIFAR10_URL, 'cifar', CIFAR10_MD5), 'data_batch', batch_size=batch_size) diff --git a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_resnet.py b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_resnet.py index a1f62db093904b617f0e37dc20d586ccea7eacd2..be494a0d340c62fb35afbf97fba38eff08a965e6 100644 --- a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_resnet.py +++ b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_resnet.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle import paddle.fluid as fluid import numpy @@ -55,7 +57,7 @@ def resnet_cifar10(input, depth=32): return tmp assert (depth - 2) % 6 == 0 - n = (depth - 2) / 6 + n = (depth - 2) // 6 conv1 = conv_bn_layer( input=input, ch_out=16, filter_size=3, stride=1, padding=1) res1 = layer_warp(basicblock, conv1, 16, 16, n, 1) diff --git a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py index 8429551765740e7db0eda82ce0b17cff129359b0..dbc7bc06c93157f271c79e85b6925468e861e57f 100644 --- a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py +++ b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle import paddle.fluid as fluid import numpy diff --git a/python/paddle/fluid/tests/book/high-level-api/label_semantic_roles/test_label_semantic_roles_newapi.py b/python/paddle/fluid/tests/book/high-level-api/label_semantic_roles/test_label_semantic_roles_newapi.py index e3602e2d5643c233b2575d1adb7f181127f60287..ec4e1c768c7f2a2421ac409a2eecc0100c086a6a 100755 --- a/python/paddle/fluid/tests/book/high-level-api/label_semantic_roles/test_label_semantic_roles_newapi.py +++ b/python/paddle/fluid/tests/book/high-level-api/label_semantic_roles/test_label_semantic_roles_newapi.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle import paddle.fluid as fluid import numpy as np diff --git a/python/paddle/fluid/tests/book/high-level-api/machine_translation/test_machine_translation.py b/python/paddle/fluid/tests/book/high-level-api/machine_translation/test_machine_translation.py index 6fb0c85a8be2b4560ea1fdb32f01146a9206ee78..560f1189581f631dc6a3470cf8f22f902ca26f26 100644 --- a/python/paddle/fluid/tests/book/high-level-api/machine_translation/test_machine_translation.py +++ b/python/paddle/fluid/tests/book/high-level-api/machine_translation/test_machine_translation.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import contextlib import numpy as np diff --git a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_conv.py b/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_conv.py index 898807db6f343cbefcc877e0f03ed6c5b82dd669..187bef1b0c1a614fbca88ef22097831d7bd5cd7f 100644 --- a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_conv.py +++ b/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_conv.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import argparse import paddle.fluid as fluid import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py b/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py index 6dd64be315159f1835244fa027e578434e6cb038..b95e7db122adbb1414da1691926c920b963fd6fe 100644 --- a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py +++ b/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import argparse import paddle.fluid as fluid import paddle diff --git a/python/paddle/fluid/tests/book/high-level-api/recommender_system/test_recommender_system_newapi.py b/python/paddle/fluid/tests/book/high-level-api/recommender_system/test_recommender_system_newapi.py index 60f3d8e105209938360487d963b0328d95e7b1f0..9e2767783bb6748cfc8f95567627068d7532a8c8 100644 --- a/python/paddle/fluid/tests/book/high-level-api/recommender_system/test_recommender_system_newapi.py +++ b/python/paddle/fluid/tests/book/high-level-api/recommender_system/test_recommender_system_newapi.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import math import sys import numpy as np diff --git a/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_conv.py b/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_conv.py index 24e65d1bd54cff7ad64453a3a61f50351d32ef08..097c2a468fca558106aba2f24c332256189d9076 100644 --- a/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_conv.py +++ b/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_conv.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle import paddle.fluid as fluid from functools import partial diff --git a/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_dynamic_rnn.py b/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_dynamic_rnn.py index b3b1505a0fad07144f3f53c22abd5553054d8c51..5f74cd142590abb93f8846bc831a9f5e3dd2f311 100644 --- a/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_dynamic_rnn.py +++ b/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_dynamic_rnn.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle import paddle.fluid as fluid from functools import partial diff --git a/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_stacked_lstm.py b/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_stacked_lstm.py index 25f99ff0fd2d1050bb62338a6bf87aa29f913fb6..284a6ca168636377699c287236c491352566909b 100644 --- a/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_stacked_lstm.py +++ b/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_stacked_lstm.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle import paddle.fluid as fluid from functools import partial diff --git a/python/paddle/fluid/tests/book/high-level-api/word2vec/test_word2vec_new_api.py b/python/paddle/fluid/tests/book/high-level-api/word2vec/test_word2vec_new_api.py index 02e65cf56c4d1bd262831320befd2edc735c0d1c..1c7cf3199a07c3f65d967eda70a481b1bd1b1638 100644 --- a/python/paddle/fluid/tests/book/high-level-api/word2vec/test_word2vec_new_api.py +++ b/python/paddle/fluid/tests/book/high-level-api/word2vec/test_word2vec_new_api.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle import paddle.fluid as fluid import numpy as np diff --git a/python/paddle/fluid/tests/book/notest_understand_sentiment.py b/python/paddle/fluid/tests/book/notest_understand_sentiment.py index ce6342c2dad0b33e57d0ea90fc6ef1660ae4e68b..82f1c6615f3c4ca54bf5e979b55082022cd4da9f 100644 --- a/python/paddle/fluid/tests/book/notest_understand_sentiment.py +++ b/python/paddle/fluid/tests/book/notest_understand_sentiment.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from paddle.fluid.layers.device import get_places import unittest import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/book/test_fit_a_line.py b/python/paddle/fluid/tests/book/test_fit_a_line.py index 37b64fa94a9aad7042e153e414ed29de3142db5a..334294ab485cf203aa0ccf680a53010322d3af3b 100644 --- a/python/paddle/fluid/tests/book/test_fit_a_line.py +++ b/python/paddle/fluid/tests/book/test_fit_a_line.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle import paddle.fluid as fluid import contextlib diff --git a/python/paddle/fluid/tests/book/test_image_classification.py b/python/paddle/fluid/tests/book/test_image_classification.py index de6fe5f140a86545e3291db165af824739a814ef..9fe361425c128590da910128beaccb3336f8ba57 100644 --- a/python/paddle/fluid/tests/book/test_image_classification.py +++ b/python/paddle/fluid/tests/book/test_image_classification.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle import paddle.fluid as fluid import contextlib @@ -60,7 +62,7 @@ def resnet_cifar10(input, depth=32): return tmp assert (depth - 2) % 6 == 0 - n = (depth - 2) / 6 + n = (depth - 2) // 6 conv1 = conv_bn_layer( input=input, ch_out=16, filter_size=3, stride=1, padding=1) res1 = layer_warp(basicblock, conv1, 16, 16, n, 1) diff --git a/python/paddle/fluid/tests/book/test_label_semantic_roles.py b/python/paddle/fluid/tests/book/test_label_semantic_roles.py index b7ac911cafdc751f38c7f66bc48263a17a84dc08..f63387a90617dc4e9b7c9ee7caa2d01595237a03 100644 --- a/python/paddle/fluid/tests/book/test_label_semantic_roles.py +++ b/python/paddle/fluid/tests/book/test_label_semantic_roles.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import contextlib import math import numpy as np diff --git a/python/paddle/fluid/tests/book/test_machine_translation.py b/python/paddle/fluid/tests/book/test_machine_translation.py index 462faad3e1cb7108f3bd6934017efe25fb9a4276..5e241aaa32727686b84a0354a11d5a92f9576a90 100644 --- a/python/paddle/fluid/tests/book/test_machine_translation.py +++ b/python/paddle/fluid/tests/book/test_machine_translation.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import contextlib import numpy as np diff --git a/python/paddle/fluid/tests/book/test_recognize_digits.py b/python/paddle/fluid/tests/book/test_recognize_digits.py index 3e5f76d12d41d016c995e5c85feda3c1847e356f..da216d0cc4a2867cb169240d28235b6db747a818 100644 --- a/python/paddle/fluid/tests/book/test_recognize_digits.py +++ b/python/paddle/fluid/tests/book/test_recognize_digits.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid.core as core import math import os diff --git a/python/paddle/fluid/tests/book/test_recommender_system.py b/python/paddle/fluid/tests/book/test_recommender_system.py index b30c8771fcf267260c4c5aa7076bedc89e3b7e8b..cf8c48f34697d789d3d81d4d94f90a7169657baf 100644 --- a/python/paddle/fluid/tests/book/test_recommender_system.py +++ b/python/paddle/fluid/tests/book/test_recommender_system.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import math import sys import os diff --git a/python/paddle/fluid/tests/book/test_rnn_encoder_decoder.py b/python/paddle/fluid/tests/book/test_rnn_encoder_decoder.py index 2e79be2bd0fc7a368df86e188b7fa616055bb3e7..91c8705aa4c88dbfeea45e15c368459ba5b5ac1f 100644 --- a/python/paddle/fluid/tests/book/test_rnn_encoder_decoder.py +++ b/python/paddle/fluid/tests/book/test_rnn_encoder_decoder.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import paddle import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/book/test_word2vec.py b/python/paddle/fluid/tests/book/test_word2vec.py index e761e05795313da23a9d984263ac2e202939b1e7..fe063eb4629dbe06dc65ce98c6c01858db901f03 100644 --- a/python/paddle/fluid/tests/book/test_word2vec.py +++ b/python/paddle/fluid/tests/book/test_word2vec.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle import paddle.fluid as fluid from paddle.fluid.layers.device import get_places diff --git a/python/paddle/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py b/python/paddle/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py index ccc62b442f62fa9fa175de031b0732febe38ee9a..f530f8f4882a23df18c141b51560cb618fce86b5 100644 --- a/python/paddle/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py +++ b/python/paddle/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import math import sys diff --git a/python/paddle/fluid/tests/book_memory_optimization/test_memopt_image_classification_train.py b/python/paddle/fluid/tests/book_memory_optimization/test_memopt_image_classification_train.py index b2a59d27da9b3348b581d51a68d769bbf3b90d35..3951e7b8ca649b63eea4b311f6205a6c7d761804 100644 --- a/python/paddle/fluid/tests/book_memory_optimization/test_memopt_image_classification_train.py +++ b/python/paddle/fluid/tests/book_memory_optimization/test_memopt_image_classification_train.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import sys import paddle @@ -56,7 +58,7 @@ def resnet_cifar10(input, depth=32): return tmp assert (depth - 2) % 6 == 0 - n = (depth - 2) / 6 + n = (depth - 2) // 6 conv1 = conv_bn_layer( input=input, ch_out=16, filter_size=3, stride=1, padding=1) res1 = layer_warp(basicblock, conv1, 16, 16, n, 1) diff --git a/python/paddle/fluid/tests/book_memory_optimization/test_memopt_machine_translation.py b/python/paddle/fluid/tests/book_memory_optimization/test_memopt_machine_translation.py index 323ddfb6911fdd57b32344933373189370005126..1ad51936b5b8f7c5149452d6033754a570c72654 100644 --- a/python/paddle/fluid/tests/book_memory_optimization/test_memopt_machine_translation.py +++ b/python/paddle/fluid/tests/book_memory_optimization/test_memopt_machine_translation.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import paddle import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/demo/fc_gan.py b/python/paddle/fluid/tests/demo/fc_gan.py index 3d92f50f0adeca79adefc291cdfba6a012fc2118..bd77779ce6ab5cf19e3e5ace3e51e39734b27c10 100644 --- a/python/paddle/fluid/tests/demo/fc_gan.py +++ b/python/paddle/fluid/tests/demo/fc_gan.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import errno import math import os diff --git a/python/paddle/fluid/tests/demo/file_reader/convert_data_to_recordio.py b/python/paddle/fluid/tests/demo/file_reader/convert_data_to_recordio.py index a00325d79be2eba4d7f770b5316c5857952fe272..45a104ec9625eacfcb87ea6eae619e3d71410da9 100644 --- a/python/paddle/fluid/tests/demo/file_reader/convert_data_to_recordio.py +++ b/python/paddle/fluid/tests/demo/file_reader/convert_data_to_recordio.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import sys import paddle.fluid as fluid import paddle.v2 as paddle diff --git a/python/paddle/fluid/tests/demo/file_reader/train.py b/python/paddle/fluid/tests/demo/file_reader/train.py index bc3a6dc81d24afec66ed1489aead1cff79a59bca..5f5d2848da42e18f2a142faae0c89352344d8cee 100644 --- a/python/paddle/fluid/tests/demo/file_reader/train.py +++ b/python/paddle/fluid/tests/demo/file_reader/train.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid as fluid import numpy import sys diff --git a/python/paddle/fluid/tests/demo/pyreader.py b/python/paddle/fluid/tests/demo/pyreader.py index 82065401935036ca346fa395c033f0f57100f01b..ec61e0ebae4feb1a2177da916b77b2ba2d3981b9 100644 --- a/python/paddle/fluid/tests/demo/pyreader.py +++ b/python/paddle/fluid/tests/demo/pyreader.py @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy +import six import paddle import paddle.dataset.mnist as mnist @@ -31,7 +34,7 @@ def network(is_train): hidden = img - for i in xrange(2): + for i in six.moves.xrange(2): hidden = fluid.layers.fc(input=hidden, size=100, act='tanh') hidden = fluid.layers.dropout( hidden, dropout_prob=0.5, is_test=not is_train) @@ -74,7 +77,7 @@ def main(): test_reader.decorate_paddle_reader(paddle.batch(mnist.test(), 512)) - for epoch_id in xrange(10): + for epoch_id in six.moves.xrange(10): train_reader.start() try: while True: diff --git a/python/paddle/fluid/tests/no_test_concurrency.py b/python/paddle/fluid/tests/no_test_concurrency.py index 3bc0c9808e2345b610dea79abc56cfb0065ea46f..b5d7676f4a2cb085c6900cd0bd0644afa2b2afd5 100644 --- a/python/paddle/fluid/tests/no_test_concurrency.py +++ b/python/paddle/fluid/tests/no_test_concurrency.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/notest_concurrency.py b/python/paddle/fluid/tests/notest_concurrency.py index 77107f8b36f31c1f494b0ade218ee047ef7eb7c6..fd9da4cce0ea51c53b4b01e7c3dc2a2ed1eeb089 100644 --- a/python/paddle/fluid/tests/notest_concurrency.py +++ b/python/paddle/fluid/tests/notest_concurrency.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/test_beam_search_decoder.py b/python/paddle/fluid/tests/test_beam_search_decoder.py index 8bf750940d570cdad5e110168afc5f632202e869..fe8a9daa3bea4b99bb42edc78538685c5ce11fe3 100644 --- a/python/paddle/fluid/tests/test_beam_search_decoder.py +++ b/python/paddle/fluid/tests/test_beam_search_decoder.py @@ -15,6 +15,8 @@ A simple machine translation demo using beam search decoder. """ +from __future__ import print_function + import contextlib import numpy as np import paddle diff --git a/python/paddle/fluid/tests/test_cpp_reader.py b/python/paddle/fluid/tests/test_cpp_reader.py index 6cc291dfcffdd7083f498389834e37bd06ca4572..b2a5253b9500bb504c651b2ab684206133199ada 100644 --- a/python/paddle/fluid/tests/test_cpp_reader.py +++ b/python/paddle/fluid/tests/test_cpp_reader.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle import paddle.fluid as fluid import numpy as np diff --git a/python/paddle/fluid/tests/test_data_feeder.py b/python/paddle/fluid/tests/test_data_feeder.py index 30b7a634a2b978df85d6432854ef12285460be44..01de564aa438e5f14a5c578f7bbbfb475155ca55 100644 --- a/python/paddle/fluid/tests/test_data_feeder.py +++ b/python/paddle/fluid/tests/test_data_feeder.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid as fluid import unittest diff --git a/python/paddle/fluid/tests/test_detection.py b/python/paddle/fluid/tests/test_detection.py index fd45abd0a77cb54a3ca8e60cf80a1efe9f9d2060..1467e72caac26a3ea2a0c770d665141988696630 100644 --- a/python/paddle/fluid/tests/test_detection.py +++ b/python/paddle/fluid/tests/test_detection.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid as fluid import paddle.fluid.layers as layers from paddle.fluid.framework import Program, program_guard diff --git a/python/paddle/fluid/tests/test_error_clip.py b/python/paddle/fluid/tests/test_error_clip.py index e8edd7fbbb31b1a6ecbf2a25a7d39e7b3f66363a..3c977afc7c813908fbe2dfb7445d9ca183cf2231 100644 --- a/python/paddle/fluid/tests/test_error_clip.py +++ b/python/paddle/fluid/tests/test_error_clip.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import paddle import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/test_gradient_clip.py b/python/paddle/fluid/tests/test_gradient_clip.py index d530601f13be6810a8a99b13c92faf584df568f9..266687fcd092dfdeec9343e2592f4c22b683d588 100644 --- a/python/paddle/fluid/tests/test_gradient_clip.py +++ b/python/paddle/fluid/tests/test_gradient_clip.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import paddle import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/test_if_else_op.py b/python/paddle/fluid/tests/test_if_else_op.py index 082f64c146f65eee4be0757d07495c33764fa841..10918a985f80f9f20629180b0359ce73f448d33b 100644 --- a/python/paddle/fluid/tests/test_if_else_op.py +++ b/python/paddle/fluid/tests/test_if_else_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle import paddle.fluid.layers as layers from paddle.fluid.framework import Program, program_guard diff --git a/python/paddle/fluid/tests/test_lod_tensor.py b/python/paddle/fluid/tests/test_lod_tensor.py index f7a9dd4129027417a06a6c25ff9a801fff259c5e..722b5f07b04f9374db3f262f5134347fe753ba19 100644 --- a/python/paddle/fluid/tests/test_lod_tensor.py +++ b/python/paddle/fluid/tests/test_lod_tensor.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid as fluid from paddle.fluid.lod_tensor import create_lod_tensor, create_random_int_lodtensor import numpy as np diff --git a/python/paddle/fluid/tests/test_python_operator_overriding.py b/python/paddle/fluid/tests/test_python_operator_overriding.py index b5ac97eac559e8c52a8949cfd63fc8671ba52514..5f92c437ec726f510d9194d23f1a01a5478827d6 100644 --- a/python/paddle/fluid/tests/test_python_operator_overriding.py +++ b/python/paddle/fluid/tests/test_python_operator_overriding.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/CMakeLists.txt b/python/paddle/fluid/tests/unittests/CMakeLists.txt index a6a911721dfa31e5fb8d57645071af42adc968be..e7dd85ef5c3641be04261dc5d4166fa8452b4200 100644 --- a/python/paddle/fluid/tests/unittests/CMakeLists.txt +++ b/python/paddle/fluid/tests/unittests/CMakeLists.txt @@ -59,8 +59,8 @@ py_test_modules(test_warpctc_op MODULES test_warpctc_op ENVS FLAGS_warpctc_dir=$ if(WITH_DISTRIBUTE) py_test_modules(test_dist_train MODULES test_dist_train SERIAL) set_tests_properties(test_listen_and_serv_op PROPERTIES TIMEOUT 20) - set_tests_properties(test_dist_mnist PROPERTIES TIMEOUT 180) - set_tests_properties(test_dist_word2vec PROPERTIES TIMEOUT 180) + set_tests_properties(test_dist_mnist PROPERTIES TIMEOUT 200) + set_tests_properties(test_dist_word2vec PROPERTIES TIMEOUT 200) endif() py_test_modules(test_parallel_executor_crf MODULES test_parallel_executor_crf SERIAL) py_test_modules(test_parallel_executor_fetch_feed MODULES test_parallel_executor_fetch_feed SERIAL) diff --git a/python/paddle/fluid/tests/unittests/benchmark.py b/python/paddle/fluid/tests/unittests/benchmark.py index b98a92dcbe5626c6cca93b3f5894302399793bf9..9ea95f3e8700274977eda4ca113a6468c631584c 100644 --- a/python/paddle/fluid/tests/unittests/benchmark.py +++ b/python/paddle/fluid/tests/unittests/benchmark.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import unittest import time @@ -54,7 +56,7 @@ class BenchmarkSuite(OpTest): def _get_input_names(self): inputs = [] - for name, value in list(self.inputs.items()): + for name, value in six.iteritems(self.inputs): if isinstance(value, list): inputs.extend([sub_name for sub_name, _ in value]) inputs.append(name) @@ -62,7 +64,7 @@ class BenchmarkSuite(OpTest): def _get_output_names(self): outputs = [] - for var_name, var in list(self.outputs.items()): + for var_name, var in six.iteritems(self.outputs): if isinstance(var, list): for sub_var_name, sub_var in var: outputs.append(sub_var_name) diff --git a/python/paddle/fluid/tests/unittests/benchmark_sum_op.py b/python/paddle/fluid/tests/unittests/benchmark_sum_op.py index 91a5f1bca4441d80489a02eb9283928e38321826..0e7338b839e2a7f5808e7a752e9ca6389622c2cb 100644 --- a/python/paddle/fluid/tests/unittests/benchmark_sum_op.py +++ b/python/paddle/fluid/tests/unittests/benchmark_sum_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/decorators.py b/python/paddle/fluid/tests/unittests/decorators.py index d1165e2a9199454dbcc1fda411afad20449bcc92..1a5f4540cf033b4d3244537cc5016ee06f341464 100644 --- a/python/paddle/fluid/tests/unittests/decorators.py +++ b/python/paddle/fluid/tests/unittests/decorators.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid as fluid __all__ = ['many_times', 'prog_scope'] diff --git a/python/paddle/fluid/tests/unittests/dist_mnist.py b/python/paddle/fluid/tests/unittests/dist_mnist.py index 8f5ba33f7cbf5286edc4503c219fd3cdff60c517..722b3e159abf4737b2bb43c7b84e23a3618cda12 100644 --- a/python/paddle/fluid/tests/unittests/dist_mnist.py +++ b/python/paddle/fluid/tests/unittests/dist_mnist.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import argparse import time diff --git a/python/paddle/fluid/tests/unittests/dist_se_resnext.py b/python/paddle/fluid/tests/unittests/dist_se_resnext.py index d576a173ce2546119ede49128ef69d240c7cf482..b0ee6ff9f5941b090e663dac0122cfb575f2f442 100644 --- a/python/paddle/fluid/tests/unittests/dist_se_resnext.py +++ b/python/paddle/fluid/tests/unittests/dist_se_resnext.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import argparse import six @@ -173,7 +175,7 @@ class SE_ResNeXt(): num_filters=num_filters, filter_size=filter_size, stride=stride, - padding=(filter_size - 1) / 2, + padding=(filter_size - 1) // 2, groups=groups, act=None, # avoid pserver CPU init differs from GPU @@ -187,7 +189,7 @@ class SE_ResNeXt(): input=input, pool_size=0, pool_type='avg', global_pooling=True) stdv = 1.0 / math.sqrt(pool.shape[1] * 1.0) squeeze = fluid.layers.fc(input=pool, - size=num_channels / reduction_ratio, + size=num_channels // reduction_ratio, act='relu') stdv = 1.0 / math.sqrt(squeeze.shape[1] * 1.0) excitation = fluid.layers.fc(input=squeeze, diff --git a/python/paddle/fluid/tests/unittests/dist_transformer.py b/python/paddle/fluid/tests/unittests/dist_transformer.py index ee8020a73546cb9037e9dc4be589c62bb1b6b937..ab4c5c3f368333ac42781aebd495579b5c26f388 100644 --- a/python/paddle/fluid/tests/unittests/dist_transformer.py +++ b/python/paddle/fluid/tests/unittests/dist_transformer.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import argparse import time @@ -22,6 +24,7 @@ import paddle.fluid as fluid from paddle.fluid import core import os import sys +import six import transformer_model import paddle.dataset.wmt16 as wmt16 @@ -159,6 +162,7 @@ def get_model(): avg_cost = transformer(use_feed=False) optimizer = fluid.optimizer.Adam() optimizer.minimize(avg_cost) + fluid.memory_optimize(fluid.default_main_program()) return avg_cost @@ -222,7 +226,7 @@ class DistTransformer2x2(object): first_loss, = exe.run(fetch_list=[avg_cost.name]) print(first_loss) - for i in xrange(5): + for i in six.moves.xrange(5): _ = exe.run(fetch_list=[avg_cost.name]) last_loss, = exe.run(fetch_list=[avg_cost.name]) print(last_loss) diff --git a/python/paddle/fluid/tests/unittests/dist_word2vec.py b/python/paddle/fluid/tests/unittests/dist_word2vec.py index 54a70f4adb4a9bb24e3c618a7fe71f42a376609b..0ad994a258c04cabc807823b7d2a8ae8bb62ab2c 100644 --- a/python/paddle/fluid/tests/unittests/dist_word2vec.py +++ b/python/paddle/fluid/tests/unittests/dist_word2vec.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import argparse import time diff --git a/python/paddle/fluid/tests/unittests/op_test.py b/python/paddle/fluid/tests/unittests/op_test.py index b27d773f09d9a6daad5a10b65e683f4e11881de1..972e44c9528a29417d9689dcb2408b9381346f31 100644 --- a/python/paddle/fluid/tests/unittests/op_test.py +++ b/python/paddle/fluid/tests/unittests/op_test.py @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import random +import six import time import itertools import collections @@ -26,15 +29,13 @@ from paddle.fluid.op import Operator from paddle.fluid.executor import Executor from paddle.fluid.framework import Program, OpProtoHolder, Variable from testsuite import create_op, set_input, append_input_output, append_loss_ops -from functools import reduce -from six.moves import zip def randomize_probability(batch_size, class_num, dtype='float32'): prob = np.random.uniform( 0.1, 1.0, size=(batch_size, class_num)).astype(dtype) prob_sum = prob.sum(axis=1) - for i in range(len(prob)): + for i in six.moves.xrange(len(prob)): prob[i] /= prob_sum[i] return prob @@ -51,7 +52,7 @@ def get_numeric_gradient(place, set_input(scope, op, inputs, place) def product(dim): - return reduce(lambda a, b: a * b, dim, 1) + return six.moves.reduce(lambda a, b: a * b, dim, 1) def get_output(): sum = [] @@ -103,7 +104,7 @@ def get_numeric_gradient(place, # we only compute gradient of one element each time. # we use a for loop to compute the gradient of every element. - for i in range(tensor_size): + for i in six.moves.xrange(tensor_size): if in_place: set_input(scope, op, inputs, place) @@ -161,7 +162,7 @@ class OpTest(unittest.TestCase): assert isinstance( numpy_dict, dict), "self.inputs, self.outputs must be numpy_dict" - for var_name, var_value in numpy_dict.items(): + for var_name, var_value in six.iteritems(numpy_dict): if isinstance(var_value, (np.ndarray, np.generic)): self.try_call_once(var_value.dtype) elif isinstance(var_value, (list, tuple)): @@ -225,7 +226,7 @@ class OpTest(unittest.TestCase): def _get_io_vars(self, block, numpy_inputs): inputs = {} - for name, value in numpy_inputs.items(): + for name, value in six.iteritems(numpy_inputs): if isinstance(value, list): var_list = [ block.var(sub_name) for sub_name, sub_value in value @@ -268,7 +269,7 @@ class OpTest(unittest.TestCase): # if the fetch_list is customized by user, we use it directly. # if not, fill the fetch_list by the user configured outputs in test. if len(fetch_list) == 0: - for var_name, var in outputs.items(): + for var_name, var in six.iteritems(outputs): if isinstance(var, list): for v in var: fetch_list.append(v) @@ -366,12 +367,13 @@ class OpTest(unittest.TestCase): for place in places: outs = self.calc_output(place) outs = [np.array(out) for out in outs] + outs.sort(key=len) checker(outs) def __assert_is_close(self, numeric_grads, analytic_grads, names, max_relative_error, msg_prefix): - for a, b, name in zip(numeric_grads, analytic_grads, names): + for a, b, name in six.moves.zip(numeric_grads, analytic_grads, names): abs_a = np.abs(a) abs_a[abs_a < 1e-3] = 1 diff --git a/python/paddle/fluid/tests/unittests/parallel_executor_test_base.py b/python/paddle/fluid/tests/unittests/parallel_executor_test_base.py index 67c35e9de7e83699bf30ca946856bb907152cbdd..9be53c4609d5027dd4e482ab936a135420a346ae 100644 --- a/python/paddle/fluid/tests/unittests/parallel_executor_test_base.py +++ b/python/paddle/fluid/tests/unittests/parallel_executor_test_base.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import multiprocessing import os import unittest diff --git a/python/paddle/fluid/tests/unittests/test_accuracy_op.py b/python/paddle/fluid/tests/unittests/test_accuracy_op.py index db1861fd10e371ebe631a16380af591875886769..1b2b53f2d4ce91ae7b5b191ed770b5338f0948c8 100644 --- a/python/paddle/fluid/tests/unittests/test_accuracy_op.py +++ b/python/paddle/fluid/tests/unittests/test_accuracy_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_activation_mkldnn_op.py b/python/paddle/fluid/tests/unittests/test_activation_mkldnn_op.py index 7d554c2276c9acd710d14c8f8b32c802e3e17515..611d0dd076b827b0f528f2e3a31182cc4939d1f1 100644 --- a/python/paddle/fluid/tests/unittests/test_activation_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/test_activation_mkldnn_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_activation_op.py b/python/paddle/fluid/tests/unittests/test_activation_op.py index 34f9cf0620fd1351111e93e16ed5f7e765d7078b..30651c1326328180592520447e597aa722146a42 100644 --- a/python/paddle/fluid/tests/unittests/test_activation_op.py +++ b/python/paddle/fluid/tests/unittests/test_activation_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_adadelta_op.py b/python/paddle/fluid/tests/unittests/test_adadelta_op.py index 1b892e64c7654a1a3905672813452650885790a5..969a7da3b71b69296f3313342adbf989c60edb50 100644 --- a/python/paddle/fluid/tests/unittests/test_adadelta_op.py +++ b/python/paddle/fluid/tests/unittests/test_adadelta_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_adagrad_op.py b/python/paddle/fluid/tests/unittests/test_adagrad_op.py index 2f0ea79f4d6afe91ee7e0d747f3d8f4884d8f9ee..fc3b7ce2fd87afc22030bcca55236fb949c1f129 100644 --- a/python/paddle/fluid/tests/unittests/test_adagrad_op.py +++ b/python/paddle/fluid/tests/unittests/test_adagrad_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_adam_op.py b/python/paddle/fluid/tests/unittests/test_adam_op.py index fa4b39879c0ede569b6802502b2c71a93b163373..5318d2f9766ce671925be614feef57d679270b19 100644 --- a/python/paddle/fluid/tests/unittests/test_adam_op.py +++ b/python/paddle/fluid/tests/unittests/test_adam_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_adamax_op.py b/python/paddle/fluid/tests/unittests/test_adamax_op.py index 8099beefa583d152715334e83f0c6e8e4a3e7a0d..a6d1be7616c73019cd8f66dcf0c108cd58ec600b 100644 --- a/python/paddle/fluid/tests/unittests/test_adamax_op.py +++ b/python/paddle/fluid/tests/unittests/test_adamax_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_anchor_generator_op.py b/python/paddle/fluid/tests/unittests/test_anchor_generator_op.py index 9c7d5d41f0c512a9fb609dce304c1eed929d28b5..d31eaa0114c3b035add3e6ca792696b5cafb9690 100644 --- a/python/paddle/fluid/tests/unittests/test_anchor_generator_op.py +++ b/python/paddle/fluid/tests/unittests/test_anchor_generator_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import sys diff --git a/python/paddle/fluid/tests/unittests/test_arg_min_max_op.py b/python/paddle/fluid/tests/unittests/test_arg_min_max_op.py index e04412f809cdd75d07d28a60f0c2f19041a684f6..0712e102b30fc72c7f8b62eb9230e7f4ab615ef0 100644 --- a/python/paddle/fluid/tests/unittests/test_arg_min_max_op.py +++ b/python/paddle/fluid/tests/unittests/test_arg_min_max_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_argsort_op.py b/python/paddle/fluid/tests/unittests/test_argsort_op.py index b29a102a3880406156481fdac54ca7043d3415db..7bc6f2599d617b192908da9b57d0cd715019bd71 100644 --- a/python/paddle/fluid/tests/unittests/test_argsort_op.py +++ b/python/paddle/fluid/tests/unittests/test_argsort_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_array_read_write_op.py b/python/paddle/fluid/tests/unittests/test_array_read_write_op.py index 0000fb0958a129e9e1098de1fad888c503cfbdc5..b86d0bc43a9f84988f2b1b27f7aeffce46a46bd9 100644 --- a/python/paddle/fluid/tests/unittests/test_array_read_write_op.py +++ b/python/paddle/fluid/tests/unittests/test_array_read_write_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.core as core import paddle.fluid.layers as layers diff --git a/python/paddle/fluid/tests/unittests/test_assign_op.py b/python/paddle/fluid/tests/unittests/test_assign_op.py index e93c02bd3ee9f710cbb9bff4e195dfc3caabe422..ba2eecfaf197ea63c187e77ae7ae8cf34873d66b 100644 --- a/python/paddle/fluid/tests/unittests/test_assign_op.py +++ b/python/paddle/fluid/tests/unittests/test_assign_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import op_test import numpy import unittest diff --git a/python/paddle/fluid/tests/unittests/test_assign_value_op.py b/python/paddle/fluid/tests/unittests/test_assign_value_op.py index 02f2e6eddc80fcce4ca5a444cff82db355c085ca..5a9d8efef1f3e5a9e116720c2ffe32c2ef0a082f 100644 --- a/python/paddle/fluid/tests/unittests/test_assign_value_op.py +++ b/python/paddle/fluid/tests/unittests/test_assign_value_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid as fluid import paddle.fluid.layers as layers import op_test diff --git a/python/paddle/fluid/tests/unittests/test_auc_op.py b/python/paddle/fluid/tests/unittests/test_auc_op.py index 6580c70ca68c4ba24919f03d071f6f88fb68953c..5393a17e674a3cad6d705a1ff7a45320e644af94 100644 --- a/python/paddle/fluid/tests/unittests/test_auc_op.py +++ b/python/paddle/fluid/tests/unittests/test_auc_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_batch_norm_mkldnn_op.py b/python/paddle/fluid/tests/unittests/test_batch_norm_mkldnn_op.py index 18fa5461590134d2032a29e40699109c12092c6d..1286cee8dc1855c1b1695da46ae0b5222c065114 100644 --- a/python/paddle/fluid/tests/unittests/test_batch_norm_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/test_batch_norm_mkldnn_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_batch_norm_op.py b/python/paddle/fluid/tests/unittests/test_batch_norm_op.py index f805fdc35f624bf6e9d94d66839dcb2a0143a29b..80261eff4e747f87658bc7c9114c21bee511df09 100644 --- a/python/paddle/fluid/tests/unittests/test_batch_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_batch_norm_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_beam_search_decode_op.py b/python/paddle/fluid/tests/unittests/test_beam_search_decode_op.py index 4a3ac2a31e072eb1a15af31f558cf9f626a7ac53..51eee41ab2d4d1113426991c63bee949cca15ad4 100644 --- a/python/paddle/fluid/tests/unittests/test_beam_search_decode_op.py +++ b/python/paddle/fluid/tests/unittests/test_beam_search_decode_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_beam_search_op.py b/python/paddle/fluid/tests/unittests/test_beam_search_op.py index e8283fc9422d93af5735aaec1a165b46ac1ef78e..c28dda4b53ce5d394ff11222e5df8d257b4e80da 100644 --- a/python/paddle/fluid/tests/unittests/test_beam_search_op.py +++ b/python/paddle/fluid/tests/unittests/test_beam_search_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import logging from paddle.fluid.op import Operator, DynamicRecurrentOp import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_bilinear_interp_op.py b/python/paddle/fluid/tests/unittests/test_bilinear_interp_op.py index b04f25ef874cc6204211a4f5f5991a0ec8c473dd..bed847c3c168c906a89c32631b2a8f0ba2e6e7be 100644 --- a/python/paddle/fluid/tests/unittests/test_bilinear_interp_op.py +++ b/python/paddle/fluid/tests/unittests/test_bilinear_interp_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_bilinear_tensor_product_op.py b/python/paddle/fluid/tests/unittests/test_bilinear_tensor_product_op.py index d20a11e27eaac12534ea33c398247adb8db01d4b..46831119c5fee938780ec8fdb9d0cdb3b63a473d 100644 --- a/python/paddle/fluid/tests/unittests/test_bilinear_tensor_product_op.py +++ b/python/paddle/fluid/tests/unittests/test_bilinear_tensor_product_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_bipartite_match_op.py b/python/paddle/fluid/tests/unittests/test_bipartite_match_op.py index ceeca25b74d85ed2874d672e402e3186c4ce7d47..5cc8e2ba15d260b988ee66a5711aed42ca04c10b 100644 --- a/python/paddle/fluid/tests/unittests/test_bipartite_match_op.py +++ b/python/paddle/fluid/tests/unittests/test_bipartite_match_op.py @@ -11,6 +11,8 @@ #WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #See the License for the specific language governing permissions and #limitations under the License. + +from __future__ import print_function import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_box_coder_op.py b/python/paddle/fluid/tests/unittests/test_box_coder_op.py index 4ce9a4783e2332b6882164a70e1462c6a6d31bef..2511c5c22e012babdeb71a71d3546456ea2ceaf3 100644 --- a/python/paddle/fluid/tests/unittests/test_box_coder_op.py +++ b/python/paddle/fluid/tests/unittests/test_box_coder_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import sys diff --git a/python/paddle/fluid/tests/unittests/test_calc_gradient.py b/python/paddle/fluid/tests/unittests/test_calc_gradient.py index 7f2a9e6971ed933463216e38498d48ab132a1a37..4120a18b72f87c7e750a0fb68780292b58e3a7f4 100644 --- a/python/paddle/fluid/tests/unittests/test_calc_gradient.py +++ b/python/paddle/fluid/tests/unittests/test_calc_gradient.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/unittests/test_cast_op.py b/python/paddle/fluid/tests/unittests/test_cast_op.py index b8d3ed3aa3eb0e47e79f46cdf681a3b9cca46036..71a2ccb6da47588d84c263105560626435ac461a 100644 --- a/python/paddle/fluid/tests/unittests/test_cast_op.py +++ b/python/paddle/fluid/tests/unittests/test_cast_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import op_test import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_chunk_eval_op.py b/python/paddle/fluid/tests/unittests/test_chunk_eval_op.py index 354110f1f96f6b4aad1a4866c8d1337dec3acd16..48eb8e9f7585d41d541ac3645e9a50dc79058de7 100644 --- a/python/paddle/fluid/tests/unittests/test_chunk_eval_op.py +++ b/python/paddle/fluid/tests/unittests/test_chunk_eval_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_clip_by_norm_op.py b/python/paddle/fluid/tests/unittests/test_clip_by_norm_op.py index 129958fa2818418dccce91683a9424e6324c6ac2..6103c3aafc0bb154194314830c5c8c5d89460cfe 100644 --- a/python/paddle/fluid/tests/unittests/test_clip_by_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_clip_by_norm_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_clip_op.py b/python/paddle/fluid/tests/unittests/test_clip_op.py index 3df80c8ec8fa39c0aaa2b8726fe3b37aef488442..32677bdb4c897b4e20f8fb166b080ac6e6a221b7 100644 --- a/python/paddle/fluid/tests/unittests/test_clip_op.py +++ b/python/paddle/fluid/tests/unittests/test_clip_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_compare_op.py b/python/paddle/fluid/tests/unittests/test_compare_op.py index 405afebae85eaae6f6af0012058ad58c8bb69a2f..437ad35538a5fa380f950fd3b71e334276214ec7 100644 --- a/python/paddle/fluid/tests/unittests/test_compare_op.py +++ b/python/paddle/fluid/tests/unittests/test_compare_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import op_test import unittest import numpy diff --git a/python/paddle/fluid/tests/unittests/test_compat.py b/python/paddle/fluid/tests/unittests/test_compat.py new file mode 100644 index 0000000000000000000000000000000000000000..1c2c46f99a82875b917a330d6ec76062222420de --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_compat.py @@ -0,0 +1,505 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import unittest +import paddle.compat as cpt +import six + + +class TestCompatible(unittest.TestCase): + def test_type(self): + if six.PY2: + self.assertEqual(cpt.int_type, int) + self.assertEqual(cpt.long_type, long) + else: + self.assertEqual(cpt.int_type, int) + self.assertEqual(cpt.long_type, int) + + def test_to_text(self): + # Only support python2.x and python3.x now + self.assertTrue(six.PY2 | six.PY3) + + if six.PY2: + # check None + self.assertIsNone(cpt.to_text(None)) + + # check all string related types + self.assertTrue(isinstance(cpt.to_text(str("")), unicode)) + self.assertTrue(isinstance(cpt.to_text(str("123")), unicode)) + self.assertTrue(isinstance(cpt.to_text(b""), unicode)) + self.assertTrue(isinstance(cpt.to_text(b""), unicode)) + self.assertTrue(isinstance(cpt.to_text(u""), unicode)) + self.assertTrue(isinstance(cpt.to_text(u""), unicode)) + + self.assertEqual(u"", cpt.to_text(str(""))) + self.assertEqual(u"123", cpt.to_text(str("123"))) + self.assertEqual(u"", cpt.to_text(b"")) + self.assertEqual(u"123", cpt.to_text(b"123")) + self.assertEqual(u"", cpt.to_text(u"")) + self.assertEqual(u"123", cpt.to_text(u"123")) + + # check list types, not inplace + l = [""] + l2 = cpt.to_text(l) + self.assertTrue(isinstance(l2, list)) + self.assertFalse(l is l2) + self.assertEqual(l, l2) + self.assertEqual([u""], l2) + l = ["", "123"] + l2 = cpt.to_text(l) + self.assertTrue(isinstance(l2, list)) + self.assertFalse(l is l2) + self.assertEqual(l, l2) + self.assertEqual([u"", u"123"], l2) + l = ["", b'123', u"321"] + l2 = cpt.to_text(l) + self.assertTrue(isinstance(l2, list)) + self.assertFalse(l is l2) + self.assertEqual(l, l2) + self.assertEqual([u"", u"123", u"321"], l2) + for i in l2: + self.assertTrue(isinstance(i, unicode)) + + # check list types, inplace + l = [""] + l2 = cpt.to_text(l, inplace=True) + self.assertTrue(isinstance(l2, list)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual([u""], l2) + l = ["", "123"] + l2 = cpt.to_text(l, inplace=True) + self.assertTrue(isinstance(l2, list)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual([u"", u"123"], l2) + l = ["", b"123", u"321"] + l2 = cpt.to_text(l, inplace=True) + self.assertTrue(isinstance(l2, list)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual([u"", u"123", u"321"], l2) + + # check set types, not inplace + l = set("") + l2 = cpt.to_text(l, inplace=False) + self.assertTrue(isinstance(l2, set)) + self.assertFalse(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set(u""), l2) + l = set([b"", b"123"]) + l2 = cpt.to_text(l, inplace=False) + self.assertTrue(isinstance(l2, set)) + self.assertFalse(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set([u"", u"123"]), l2) + l = set(["", b"123", u"321"]) + l2 = cpt.to_text(l, inplace=False) + self.assertTrue(isinstance(l2, set)) + self.assertFalse(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set([u"", u"123", u"321"]), l2) + for i in l2: + self.assertTrue(isinstance(i, unicode)) + + # check set types, inplace + l = set("") + l2 = cpt.to_text(l, inplace=True) + self.assertTrue(isinstance(l2, set)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set(u""), l2) + l = set([b"", b"123"]) + l2 = cpt.to_text(l, inplace=True) + self.assertTrue(isinstance(l2, set)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set([u"", u"123"]), l2) + l = set(["", b"123", u"321"]) + l2 = cpt.to_text(l, inplace=True) + self.assertTrue(isinstance(l2, set)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set([u"", u"123", u"321"]), l2) + + elif six.PY3: + self.assertIsNone(cpt.to_text(None)) + + self.assertTrue(isinstance(cpt.to_text(str("")), str)) + self.assertTrue(isinstance(cpt.to_text(str("123")), str)) + self.assertTrue(isinstance(cpt.to_text(b""), str)) + self.assertTrue(isinstance(cpt.to_text(b""), str)) + self.assertTrue(isinstance(cpt.to_text(u""), str)) + self.assertTrue(isinstance(cpt.to_text(u""), str)) + + self.assertEqual("", cpt.to_text(str(""))) + self.assertEqual("123", cpt.to_text(str("123"))) + self.assertEqual("", cpt.to_text(b"")) + self.assertEqual("123", cpt.to_text(b"123")) + self.assertEqual("", cpt.to_text(u"")) + self.assertEqual("123", cpt.to_text(u"123")) + + # check list types, not inplace + l = [""] + l2 = cpt.to_text(l) + self.assertTrue(isinstance(l2, list)) + self.assertFalse(l is l2) + self.assertEqual(l, l2) + self.assertEqual([""], l2) + l = ["", "123"] + l2 = cpt.to_text(l) + self.assertTrue(isinstance(l2, list)) + self.assertFalse(l is l2) + self.assertEqual(l, l2) + self.assertEqual(["", "123"], l2) + l = ["", b"123", u"321"] + l2 = cpt.to_text(l) + self.assertTrue(isinstance(l2, list)) + self.assertFalse(l is l2) + self.assertNotEqual(l, l2) + self.assertEqual(["", "123", "321"], l2) + + # check list types, inplace + l = [""] + l2 = cpt.to_text(l, inplace=True) + self.assertTrue(isinstance(l2, list)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual([""], l2) + l = ["", b"123"] + l2 = cpt.to_text(l, inplace=True) + self.assertTrue(isinstance(l2, list)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual(["", "123"], l2) + l = ["", b"123", u"321"] + l2 = cpt.to_text(l, inplace=True) + self.assertTrue(isinstance(l2, list)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual(["", "123", "321"], l2) + for i in l2: + self.assertTrue(isinstance(i, str)) + + # check set types, not inplace + l = set("") + l2 = cpt.to_text(l, inplace=False) + self.assertTrue(isinstance(l2, set)) + self.assertFalse(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set(""), l2) + l = set([b"", b"123"]) + l2 = cpt.to_text(l, inplace=False) + self.assertTrue(isinstance(l2, set)) + self.assertFalse(l is l2) + self.assertNotEqual(l, l2) + self.assertEqual(set(["", "123"]), l2) + l = set(["", b"123", u"321"]) + l2 = cpt.to_text(l, inplace=False) + self.assertTrue(isinstance(l2, set)) + self.assertFalse(l is l2) + self.assertNotEqual(l, l2) + self.assertEqual(set(["", "123", "321"]), l2) + + # check set types, inplace + l = set("") + l2 = cpt.to_text(l, inplace=True) + self.assertTrue(isinstance(l2, set)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set(""), l2) + l = set([b"", b"123"]) + l2 = cpt.to_text(l, inplace=True) + self.assertTrue(isinstance(l2, set)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set(["", "123"]), l2) + l = set(["", b"123", u"321"]) + l2 = cpt.to_text(l, inplace=True) + self.assertTrue(isinstance(l2, set)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set(["", "123", "321"]), l2) + for i in l2: + self.assertTrue(isinstance(i, str)) + + def test_to_bytes(self): + # Only support python2.x and python3.x now + self.assertTrue(six.PY2 | six.PY3) + + if six.PY2: + # check None + self.assertIsNone(cpt.to_bytes(None)) + + # check all string related types + self.assertTrue(isinstance(cpt.to_bytes(str("")), bytes)) + self.assertTrue(isinstance(cpt.to_bytes(str("123")), bytes)) + self.assertTrue(isinstance(cpt.to_bytes(b""), bytes)) + self.assertTrue(isinstance(cpt.to_bytes(b""), bytes)) + self.assertTrue(isinstance(cpt.to_bytes(u""), bytes)) + self.assertTrue(isinstance(cpt.to_bytes(u""), bytes)) + + self.assertEqual(b"", cpt.to_bytes(str(""))) + self.assertEqual(b"123", cpt.to_bytes(str("123"))) + self.assertEqual(b"", cpt.to_bytes(b"")) + self.assertEqual(b"123", cpt.to_bytes(b"123")) + self.assertEqual(b"", cpt.to_bytes(u"")) + self.assertEqual(b"123", cpt.to_bytes(u"123")) + + # check list types, not inplace + l = [""] + l2 = cpt.to_bytes(l) + self.assertTrue(isinstance(l2, list)) + self.assertFalse(l is l2) + self.assertEqual(l, l2) + self.assertEqual([b""], l2) + l = ["", "123"] + l2 = cpt.to_bytes(l) + self.assertTrue(isinstance(l2, list)) + self.assertFalse(l is l2) + self.assertEqual(l, l2) + self.assertEqual([b"", b"123"], l2) + l = ["", b'123', u"321"] + l2 = cpt.to_bytes(l) + self.assertTrue(isinstance(l2, list)) + self.assertFalse(l is l2) + self.assertEqual(l, l2) + self.assertEqual([b"", b"123", b"321"], l2) + for i in l2: + self.assertTrue(isinstance(i, bytes)) + + # check list types, inplace + l = [""] + l2 = cpt.to_bytes(l, inplace=True) + self.assertTrue(isinstance(l2, list)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual([b""], l2) + l = ["", "123"] + l2 = cpt.to_bytes(l, inplace=True) + self.assertTrue(isinstance(l2, list)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual([b"", b"123"], l2) + l = ["", b"123", u"321"] + l2 = cpt.to_bytes(l, inplace=True) + self.assertTrue(isinstance(l2, list)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual([b"", b"123", b"321"], l2) + + # check set types, not inplace + l = set("") + l2 = cpt.to_bytes(l, inplace=False) + self.assertTrue(isinstance(l2, set)) + self.assertFalse(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set(b""), l2) + l = set([b"", b"123"]) + l2 = cpt.to_bytes(l, inplace=False) + self.assertTrue(isinstance(l2, set)) + self.assertFalse(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set([b"", b"123"]), l2) + l = set(["", b"123", u"321"]) + l2 = cpt.to_bytes(l, inplace=False) + self.assertTrue(isinstance(l2, set)) + self.assertFalse(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set([b"", b"123", b"321"]), l2) + for i in l2: + self.assertTrue(isinstance(i, bytes)) + + # check set types, inplace + l = set("") + l2 = cpt.to_bytes(l, inplace=True) + self.assertTrue(isinstance(l2, set)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set(b""), l2) + l = set([b"", b"123"]) + l2 = cpt.to_bytes(l, inplace=True) + self.assertTrue(isinstance(l2, set)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set([b"", b"123"]), l2) + l = set(["", b"123", u"321"]) + l2 = cpt.to_bytes(l, inplace=True) + self.assertTrue(isinstance(l2, set)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set([b"", b"123", b"321"]), l2) + + elif six.PY3: + self.assertIsNone(cpt.to_bytes(None)) + + self.assertTrue(isinstance(cpt.to_bytes(str("")), bytes)) + self.assertTrue(isinstance(cpt.to_bytes(str("123")), bytes)) + self.assertTrue(isinstance(cpt.to_bytes(b""), bytes)) + self.assertTrue(isinstance(cpt.to_bytes(b""), bytes)) + self.assertTrue(isinstance(cpt.to_bytes(u""), bytes)) + self.assertTrue(isinstance(cpt.to_bytes(u""), bytes)) + + self.assertEqual(b"", cpt.to_bytes(str(""))) + self.assertEqual(b"123", cpt.to_bytes(str("123"))) + self.assertEqual(b"", cpt.to_bytes(b"")) + self.assertEqual(b"123", cpt.to_bytes(b"123")) + self.assertEqual(b"", cpt.to_bytes(u"")) + self.assertEqual(b"123", cpt.to_bytes(u"123")) + + # check list types, not inplace + l = [""] + l2 = cpt.to_bytes(l) + self.assertTrue(isinstance(l2, list)) + self.assertFalse(l is l2) + self.assertNotEqual(l, l2) + self.assertEqual([b""], l2) + l = ["", "123"] + l2 = cpt.to_bytes(l) + self.assertTrue(isinstance(l2, list)) + self.assertFalse(l is l2) + self.assertNotEqual(l, l2) + self.assertEqual([b"", b"123"], l2) + l = ["", b"123", u"321"] + l2 = cpt.to_bytes(l) + self.assertTrue(isinstance(l2, list)) + self.assertFalse(l is l2) + self.assertNotEqual(l, l2) + self.assertEqual([b"", b"123", b"321"], l2) + + # check list types, inplace + l = [""] + l2 = cpt.to_bytes(l, inplace=True) + self.assertTrue(isinstance(l2, list)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual([b""], l2) + l = ["", b"123"] + l2 = cpt.to_bytes(l, inplace=True) + self.assertTrue(isinstance(l2, list)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual([b"", b"123"], l2) + l = ["", b"123", u"321"] + l2 = cpt.to_bytes(l, inplace=True) + self.assertTrue(isinstance(l2, list)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual([b"", b"123", b"321"], l2) + for i in l2: + self.assertTrue(isinstance(i, bytes)) + + # check set types, not inplace + l = set([""]) + l2 = cpt.to_bytes(l, inplace=False) + self.assertTrue(isinstance(l2, set)) + self.assertFalse(l is l2) + self.assertNotEqual(l, l2) + self.assertEqual(set([b""]), l2) + l = set([u"", u"123"]) + l2 = cpt.to_bytes(l, inplace=False) + self.assertTrue(isinstance(l2, set)) + self.assertFalse(l is l2) + self.assertNotEqual(l, l2) + self.assertEqual(set([b"", b"123"]), l2) + l = set(["", b"123", u"321"]) + l2 = cpt.to_bytes(l, inplace=False) + self.assertTrue(isinstance(l2, set)) + self.assertFalse(l is l2) + self.assertNotEqual(l, l2) + self.assertEqual(set([b"", b"123", b"321"]), l2) + + # check set types, inplace + l = set("") + l2 = cpt.to_bytes(l, inplace=True) + self.assertTrue(isinstance(l2, set)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set(b""), l2) + l = set([u"", u"123"]) + l2 = cpt.to_bytes(l, inplace=True) + self.assertTrue(isinstance(l2, set)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set([b"", b"123"]), l2) + l = set(["", b"123", u"321"]) + l2 = cpt.to_bytes(l, inplace=True) + self.assertTrue(isinstance(l2, set)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set([b"", b"123", b"321"]), l2) + for i in l2: + self.assertTrue(isinstance(i, bytes)) + + def test_round(self): + self.assertEqual(3.0, cpt.round(3.4)) + self.assertEqual(4.0, cpt.round(3.5)) + self.assertEqual(0.0, cpt.round(0.1)) + self.assertEqual(0.0, cpt.round(0.0)) + self.assertEqual(-0.0, cpt.round(-0.0)) + self.assertEqual(-0.0, cpt.round(-0.1)) + self.assertEqual(-3.0, cpt.round(-3.4)) + self.assertEqual(-4.0, cpt.round(-3.5)) + self.assertEqual(5.0, cpt.round(5)) + self.assertRaises(TypeError, cpt.round, None) + + def test_floor_division(self): + self.assertEqual(0.0, cpt.floor_division(3, 4)) + self.assertEqual(1.0, cpt.floor_division(4, 3)) + self.assertEqual(2.0, cpt.floor_division(6, 3)) + self.assertEqual(-2.0, cpt.floor_division(-4, 3)) + self.assertEqual(-2.0, cpt.floor_division(-6, 3)) + self.assertRaises(ZeroDivisionError, cpt.floor_division, 3, 0) + self.assertRaises(TypeError, cpt.floor_division, None, None) + + def test_get_exception_message(self): + exception_message = "test_message" + self.assertRaises(AssertionError, cpt.get_exception_message, None) + if six.PY2: + self.assertRaises(AttributeError, cpt.get_exception_message, + exception_message) + try: + raise RuntimeError(exception_message) + except Exception as e: + self.assertEqual(exception_message, + cpt.get_exception_message(e)) + self.assertIsNotNone(e) + + try: + raise Exception(exception_message) + except Exception as e: + self.assertEqual(exception_message, + cpt.get_exception_message(e)) + self.assertIsNotNone(e) + + if six.PY3: + try: + raise RuntimeError(exception_message) + except Exception as e: + self.assertEqual(exception_message, + cpt.get_exception_message(e)) + self.assertIsNotNone(e) + + try: + raise Exception(exception_message) + except Exception as e: + self.assertEqual(exception_message, + cpt.get_exception_message(e)) + self.assertIsNotNone(e) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_concat_op.py b/python/paddle/fluid/tests/unittests/test_concat_op.py index e9f3c45dc40b3333fe7304f8e4313d156bd5374c..436ab7d49f4cafcd30366ae57c40d49e6f7d614f 100644 --- a/python/paddle/fluid/tests/unittests/test_concat_op.py +++ b/python/paddle/fluid/tests/unittests/test_concat_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_conditional_block.py b/python/paddle/fluid/tests/unittests/test_conditional_block.py index 77869a1242e08d348bfb1031b8f5b1ab5c81d868..5b2b71d050c42b4fea84bab89824d3f5c164b36e 100644 --- a/python/paddle/fluid/tests/unittests/test_conditional_block.py +++ b/python/paddle/fluid/tests/unittests/test_conditional_block.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.layers as layers import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_const_value.py b/python/paddle/fluid/tests/unittests/test_const_value.py index 58ac6fa0a9a30a08a831111513777cca59062724..0b2431d7726e845da33f6bcf9c74058788dd9654 100644 --- a/python/paddle/fluid/tests/unittests/test_const_value.py +++ b/python/paddle/fluid/tests/unittests/test_const_value.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.framework as framework diff --git a/python/paddle/fluid/tests/unittests/test_conv2d_mkldnn_op.py b/python/paddle/fluid/tests/unittests/test_conv2d_mkldnn_op.py index d0de7ad52c8a851c16cbbbf544d479f696dee136..1902a9869807ba7ce3f9828c124256cc6752857e 100644 --- a/python/paddle/fluid/tests/unittests/test_conv2d_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv2d_mkldnn_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest from test_conv2d_op import TestConv2dOp, TestWithPad, TestWithStride diff --git a/python/paddle/fluid/tests/unittests/test_conv2d_op.py b/python/paddle/fluid/tests/unittests/test_conv2d_op.py index bb1cd87d615fa341b7244e9f3e113b9fb4765ac2..6a2732e9399aa5a93f4c47eb73bfd23dba608c3d 100644 --- a/python/paddle/fluid/tests/unittests/test_conv2d_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv2d_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np @@ -24,12 +26,12 @@ def conv2d_forward_naive(input, filter, group, conv_param): out_c, f_c, f_h, f_w = filter.shape assert f_c * group == in_c assert np.mod(out_c, group) == 0 - sub_out_c = out_c / group + sub_out_c = out_c // group stride, pad, dilation = conv_param['stride'], conv_param['pad'], conv_param[ 'dilation'] - out_h = 1 + (in_h + 2 * pad[0] - (dilation[0] * (f_h - 1) + 1)) / stride[0] - out_w = 1 + (in_w + 2 * pad[1] - (dilation[1] * (f_w - 1) + 1)) / stride[1] + out_h = 1 + (in_h + 2 * pad[0] - (dilation[0] * (f_h - 1) + 1)) // stride[0] + out_w = 1 + (in_w + 2 * pad[1] - (dilation[1] * (f_w - 1) + 1)) // stride[1] out = np.zeros((in_n, out_c, out_h, out_w)) d_bolck_h = (dilation[0] * (f_h - 1) + 1) @@ -138,7 +140,7 @@ class TestConv2dOp(OpTest): self.stride = [1, 1] self.input_size = [2, 3, 5, 5] # NCHW assert np.mod(self.input_size[1], self.groups) == 0 - f_c = self.input_size[1] / self.groups + f_c = self.input_size[1] // self.groups self.filter_size = [6, f_c, 3, 3] def init_dilation(self): @@ -157,7 +159,7 @@ class TestWithPad(TestConv2dOp): self.stride = [1, 1] self.input_size = [2, 3, 5, 5] # NCHW assert np.mod(self.input_size[1], self.groups) == 0 - f_c = self.input_size[1] / self.groups + f_c = self.input_size[1] // self.groups self.filter_size = [6, f_c, 3, 3] @@ -167,7 +169,7 @@ class TestWithStride(TestConv2dOp): self.stride = [2, 2] self.input_size = [2, 3, 6, 6] # NCHW assert np.mod(self.input_size[1], self.groups) == 0 - f_c = self.input_size[1] / self.groups + f_c = self.input_size[1] // self.groups self.filter_size = [6, f_c, 3, 3] @@ -182,7 +184,7 @@ class TestWith1x1(TestConv2dOp): self.stride = [1, 1] self.input_size = [2, 3, 5, 5] # NCHW assert np.mod(self.input_size[1], self.groups) == 0 - f_c = self.input_size[1] / self.groups + f_c = self.input_size[1] // self.groups self.filter_size = [6, f_c, 1, 1] def init_group(self): @@ -195,7 +197,7 @@ class TestWithDilation(TestConv2dOp): self.stride = [1, 1] self.input_size = [2, 3, 10, 10] # NCHW assert np.mod(self.input_size[1], self.groups) == 0 - f_c = self.input_size[1] / self.groups + f_c = self.input_size[1] // self.groups self.filter_size = [6, f_c, 3, 3] def init_dilation(self): @@ -211,7 +213,7 @@ class TestWithInput1x1Filter1x1(TestConv2dOp): self.stride = [1, 1] self.input_size = [2, 3, 1, 1] # NCHW assert np.mod(self.input_size[1], self.groups) == 0 - f_c = self.input_size[1] / self.groups + f_c = self.input_size[1] // self.groups self.filter_size = [6, f_c, 1, 1] def init_group(self): @@ -328,7 +330,7 @@ class TestDepthwiseConv(TestConv2dOp): self.input_size = [2, 3, 5, 5] # NCHW self.groups = 3 assert np.mod(self.input_size[1], self.groups) == 0 - f_c = self.input_size[1] / self.groups + f_c = self.input_size[1] // self.groups self.filter_size = [6, f_c, 3, 3] self.op_type = "depthwise_conv2d" @@ -340,7 +342,7 @@ class TestDepthwiseConv2(TestConv2dOp): self.input_size = [2, 3, 5, 5] # NCHW self.groups = 3 assert np.mod(self.input_size[1], self.groups) == 0 - f_c = self.input_size[1] / self.groups + f_c = self.input_size[1] // self.groups self.filter_size = [6, f_c, 3, 3] self.op_type = "depthwise_conv2d" diff --git a/python/paddle/fluid/tests/unittests/test_conv2d_transpose_op.py b/python/paddle/fluid/tests/unittests/test_conv2d_transpose_op.py index af6cd99b0d7e6b0a2dfd4fc1d33e8390017a5906..2a320e735bd7db5dc138f8263ba1b5cb115ba197 100644 --- a/python/paddle/fluid/tests/unittests/test_conv2d_transpose_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv2d_transpose_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np @@ -25,7 +27,7 @@ def conv2dtranspose_forward_naive(input_, filter_, attrs): groups = attrs['groups'] assert in_c == f_c out_c = f_out_c * groups - sub_in_c = in_c / groups + sub_in_c = in_c // groups stride, pad, dilations = attrs['strides'], attrs['paddings'], attrs[ 'dilations'] @@ -258,7 +260,7 @@ class TestDepthwiseConvTranspose(TestConv2dTransposeOp): self.input_size = [2, 8, 16, 16] # NCHW self.groups = 8 assert np.mod(self.input_size[1], self.groups) == 0 - f_c = self.input_size[1] / self.groups + f_c = self.input_size[1] // self.groups self.filter_size = [self.input_size[1], f_c, 4, 4] self.op_type = "depthwise_conv2d_transpose" diff --git a/python/paddle/fluid/tests/unittests/test_conv3d_op.py b/python/paddle/fluid/tests/unittests/test_conv3d_op.py index dd4ef7cc94ea1e8de5fe4775408389907d47d0d6..ddaf99fe061205f0f2e4c592c9e28e27e657c16a 100644 --- a/python/paddle/fluid/tests/unittests/test_conv3d_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv3d_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np @@ -24,14 +26,14 @@ def conv3d_forward_naive(input, filter, group, conv_param): out_c, f_c, f_d, f_h, f_w = filter.shape assert f_c * group == in_c assert np.mod(out_c, group) == 0 - sub_out_c = out_c / group + sub_out_c = out_c // group stride, pad, dilation = conv_param['stride'], conv_param['pad'], conv_param[ 'dilations'] - out_d = 1 + (in_d + 2 * pad[0] - (dilation[0] * (f_d - 1) + 1)) / stride[0] - out_h = 1 + (in_h + 2 * pad[1] - (dilation[1] * (f_h - 1) + 1)) / stride[1] - out_w = 1 + (in_w + 2 * pad[2] - (dilation[2] * (f_w - 1) + 1)) / stride[2] + out_d = 1 + (in_d + 2 * pad[0] - (dilation[0] * (f_d - 1) + 1)) // stride[0] + out_h = 1 + (in_h + 2 * pad[1] - (dilation[1] * (f_h - 1) + 1)) // stride[1] + out_w = 1 + (in_w + 2 * pad[2] - (dilation[2] * (f_w - 1) + 1)) // stride[2] out = np.zeros((in_n, out_c, out_d, out_h, out_w)) @@ -166,7 +168,7 @@ class TestConv3dOp(OpTest): self.stride = [1, 1, 1] self.input_size = [2, 3, 4, 4, 4] # NCDHW assert np.mod(self.input_size[1], self.groups) == 0 - f_c = self.input_size[1] / self.groups + f_c = self.input_size[1] // self.groups self.filter_size = [6, f_c, 3, 3, 3] def init_dilation(self): @@ -185,7 +187,7 @@ class TestCase1(TestConv3dOp): self.stride = [1, 1, 1] self.input_size = [2, 3, 4, 4, 4] # NCDHW assert np.mod(self.input_size[1], self.groups) == 0 - f_c = self.input_size[1] / self.groups + f_c = self.input_size[1] // self.groups self.filter_size = [6, f_c, 3, 3, 3] @@ -205,7 +207,7 @@ class TestWith1x1(TestConv3dOp): self.stride = [1, 1, 1] self.input_size = [2, 3, 4, 4, 4] # NCHW assert np.mod(self.input_size[1], self.groups) == 0 - f_c = self.input_size[1] / self.groups + f_c = self.input_size[1] // self.groups self.filter_size = [6, f_c, 1, 1, 1] def init_dilation(self): @@ -221,7 +223,7 @@ class TestWithInput1x1Filter1x1(TestConv3dOp): self.stride = [1, 1, 1] self.input_size = [2, 3, 1, 1, 1] # NCHW assert np.mod(self.input_size[1], self.groups) == 0 - f_c = self.input_size[1] / self.groups + f_c = self.input_size[1] // self.groups self.filter_size = [6, f_c, 1, 1, 1] def init_dilation(self): @@ -237,7 +239,7 @@ class TestWithDilation(TestConv3dOp): self.stride = [1, 1, 1] self.input_size = [2, 3, 6, 6, 6] # NCDHW assert np.mod(self.input_size[1], self.groups) == 0 - f_c = self.input_size[1] / self.groups + f_c = self.input_size[1] // self.groups self.filter_size = [6, f_c, 2, 2, 2] def init_dilation(self): diff --git a/python/paddle/fluid/tests/unittests/test_conv3d_transpose_op.py b/python/paddle/fluid/tests/unittests/test_conv3d_transpose_op.py index 300fa5e8bde001e0f66c5f924a81c30add99aead..8d9075961cbec32bc34fcf0c92cfbb7e6c00d886 100644 --- a/python/paddle/fluid/tests/unittests/test_conv3d_transpose_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv3d_transpose_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np @@ -25,7 +27,7 @@ def conv3dtranspose_forward_naive(input_, filter_, attrs): groups = attrs['groups'] assert in_c == f_c out_c = f_out_c * groups - sub_in_c = in_c / groups + sub_in_c = in_c // groups stride, pad, dilations = attrs['strides'], attrs['paddings'], attrs[ 'dilations'] diff --git a/python/paddle/fluid/tests/unittests/test_conv_shift_op.py b/python/paddle/fluid/tests/unittests/test_conv_shift_op.py index 9fdb7baa90d2184c3c439e76b6bb5f0668f5f9ee..b7364e869e7420e610363eafcc4964b825e57326 100644 --- a/python/paddle/fluid/tests/unittests/test_conv_shift_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv_shift_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest @@ -21,7 +23,7 @@ def conv_shift_forward(x, y): out = np.zeros_like(x) M = x.shape[1] N = y.shape[1] - y_half_width = (N - 1) / 2 + y_half_width = (N - 1) // 2 for i in range(M): for j in range(N): out[:, i] += x[:, (i + j + M - y_half_width) % M] * y[:, j] diff --git a/python/paddle/fluid/tests/unittests/test_cos_sim_op.py b/python/paddle/fluid/tests/unittests/test_cos_sim_op.py index 1b27cd57670e6c9db5eae6b226989a5c772866ce..3c3fd6d4d71503ccc3678ca69d55bcc8536c8c6a 100644 --- a/python/paddle/fluid/tests/unittests/test_cos_sim_op.py +++ b/python/paddle/fluid/tests/unittests/test_cos_sim_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_create_op_doc_string.py b/python/paddle/fluid/tests/unittests/test_create_op_doc_string.py index 07c89eefc32fab37ce093e91d96fbe4471ecddc6..fd34c8fc9390b69afd93229b56aa9189da2a8b28 100644 --- a/python/paddle/fluid/tests/unittests/test_create_op_doc_string.py +++ b/python/paddle/fluid/tests/unittests/test_create_op_doc_string.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.layers as layers diff --git a/python/paddle/fluid/tests/unittests/test_crf_decoding_op.py b/python/paddle/fluid/tests/unittests/test_crf_decoding_op.py index 122b076c2d3e3a69f52a2c335e2bc89707b4fa9b..51bd1300e61d58c934a40abf81ab8f137e44910f 100644 --- a/python/paddle/fluid/tests/unittests/test_crf_decoding_op.py +++ b/python/paddle/fluid/tests/unittests/test_crf_decoding_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import random import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_crop_op.py b/python/paddle/fluid/tests/unittests/test_crop_op.py index 4016089c01644f0389855ab114360f90c50a1bbe..d7bcfba8deab1b73e4cbab8a27f9eeef9a37d29b 100644 --- a/python/paddle/fluid/tests/unittests/test_crop_op.py +++ b/python/paddle/fluid/tests/unittests/test_crop_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_cross_entropy_op.py b/python/paddle/fluid/tests/unittests/test_cross_entropy_op.py index 86ac159323a5f9f6149ce5ed4437402eb885c6bc..fa367f95fc9c65dd782d53a2799cacadf74dcfd2 100644 --- a/python/paddle/fluid/tests/unittests/test_cross_entropy_op.py +++ b/python/paddle/fluid/tests/unittests/test_cross_entropy_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest, randomize_probability diff --git a/python/paddle/fluid/tests/unittests/test_ctc_align.py b/python/paddle/fluid/tests/unittests/test_ctc_align.py index 131b4076f45ae25b45bb3f64da07a5c3aacc43d5..5f17d2d407cca9a4c95d919d05a3a03b784d1942 100644 --- a/python/paddle/fluid/tests/unittests/test_ctc_align.py +++ b/python/paddle/fluid/tests/unittests/test_ctc_align.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import sys import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_cumsum_op.py b/python/paddle/fluid/tests/unittests/test_cumsum_op.py index 04e7f0b94510987a1872c2d625ac4d29a3c6feba..13a4eacece8a211513d6537db0d09b80c238178e 100644 --- a/python/paddle/fluid/tests/unittests/test_cumsum_op.py +++ b/python/paddle/fluid/tests/unittests/test_cumsum_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_data_balance.py b/python/paddle/fluid/tests/unittests/test_data_balance.py index 951282e8bab5018204c0d31caa10f8f84a8f3d6c..e39eedd282daf6bbe0603a22c357e06c95c086b6 100644 --- a/python/paddle/fluid/tests/unittests/test_data_balance.py +++ b/python/paddle/fluid/tests/unittests/test_data_balance.py @@ -12,9 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid -import paddle.v2 as paddle +import paddle import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_debugger.py b/python/paddle/fluid/tests/unittests/test_debugger.py index 870952f2f916dcdec5991ac5c10d2da3a7ab18a8..f4c9466d63a201ba9a5e77515ae64a33bedc5b23 100644 --- a/python/paddle/fluid/tests/unittests/test_debugger.py +++ b/python/paddle/fluid/tests/unittests/test_debugger.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_decayed_adagrad_op.py b/python/paddle/fluid/tests/unittests/test_decayed_adagrad_op.py index 84c44d4817366518a2cbc3f0a777ab32b67f3d11..a664a1529f4de1f372241319b57fad6b0ba8b8a2 100644 --- a/python/paddle/fluid/tests/unittests/test_decayed_adagrad_op.py +++ b/python/paddle/fluid/tests/unittests/test_decayed_adagrad_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_default_scope_funcs.py b/python/paddle/fluid/tests/unittests/test_default_scope_funcs.py index 868bcca881a65dad7d0ecabb1e388818cdd0997e..01a7b6824885b32e922a8eb34f5d8117ee3e584f 100644 --- a/python/paddle/fluid/tests/unittests/test_default_scope_funcs.py +++ b/python/paddle/fluid/tests/unittests/test_default_scope_funcs.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from paddle.fluid.default_scope_funcs import * import unittest diff --git a/python/paddle/fluid/tests/unittests/test_desc_clone.py b/python/paddle/fluid/tests/unittests/test_desc_clone.py index 8603d3a5b3b5d368fe87b8dcf9dc7363f95caf86..88d44e453c7976f5e0fbda2c0871dfabd4bb30aa 100644 --- a/python/paddle/fluid/tests/unittests/test_desc_clone.py +++ b/python/paddle/fluid/tests/unittests/test_desc_clone.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import argparse import time diff --git a/python/paddle/fluid/tests/unittests/test_detection_map_op.py b/python/paddle/fluid/tests/unittests/test_detection_map_op.py index 8b66d1b270980a18fd1bbd068917e982a450ad6f..f6eb8f2c6d8b94f92e24ff789c91efb53a645a46 100644 --- a/python/paddle/fluid/tests/unittests/test_detection_map_op.py +++ b/python/paddle/fluid/tests/unittests/test_detection_map_op.py @@ -12,8 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np +import six import sys import collections import math @@ -176,7 +179,7 @@ class TestDetectionMAPOp(OpTest): true_pos[label].append([score, tp]) false_pos[label].append([score, fp]) - for (label, label_pos_num) in list(label_count.items()): + for (label, label_pos_num) in six.iteritems(label_count): if label_pos_num == 0 or label not in true_pos: continue label_true_pos = true_pos[label] label_false_pos = false_pos[label] diff --git a/python/paddle/fluid/tests/unittests/test_dist_base.py b/python/paddle/fluid/tests/unittests/test_dist_base.py index 4379463aca4443eb7a886ce78446440cc59f3b30..4c71181d0d736bd1a8796b2d38ed1667557e3db8 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_base.py +++ b/python/paddle/fluid/tests/unittests/test_dist_base.py @@ -11,11 +11,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import time import unittest import os import sys +import six import signal import subprocess import six @@ -123,6 +126,9 @@ def runtime_main(test_class): model.run_trainer(p, endpoints, trainer_id, trainers, is_dist) +import paddle.compat as cpt + + class TestDistBase(unittest.TestCase): def setUp(self): self._trainers = 2 @@ -209,7 +215,7 @@ class TestDistBase(unittest.TestCase): local_proc.wait() out, err = local_proc.communicate() - local_ret = out + local_ret = cpt.to_text(out) sys.stderr.write('local_loss: %s\n' % local_ret) sys.stderr.write('local_stderr: %s\n' % err) @@ -256,7 +262,7 @@ class TestDistBase(unittest.TestCase): tr1_proc.wait() out, err = tr0_proc.communicate() sys.stderr.write('dist_stderr: %s\n' % err) - loss_data0 = out + loss_data0 = cpt.to_text(out) sys.stderr.write('dist_loss: %s\n' % loss_data0) lines = loss_data0.split("\n") dist_first_loss = eval(lines[0].replace(" ", ","))[0] diff --git a/python/paddle/fluid/tests/unittests/test_dist_mnist.py b/python/paddle/fluid/tests/unittests/test_dist_mnist.py index b3ccec9a7d65de57778a1f013465d41a5a267676..4ec68d411b0f0e9ae89b107914e8fd844a19228b 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_mnist.py +++ b/python/paddle/fluid/tests/unittests/test_dist_mnist.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import unittest from test_dist_base import TestDistBase diff --git a/python/paddle/fluid/tests/unittests/test_dist_se_resnext.py b/python/paddle/fluid/tests/unittests/test_dist_se_resnext.py index a33a338fc11e4301a8ec0eb565686d62b547b7f7..16525f6fdb60a90a44a628fb0648f4130218c102 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_se_resnext.py +++ b/python/paddle/fluid/tests/unittests/test_dist_se_resnext.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import unittest from test_dist_base import TestDistBase diff --git a/python/paddle/fluid/tests/unittests/test_dist_train.py b/python/paddle/fluid/tests/unittests/test_dist_train.py index aab8969a96ff69d1a306506337a0e009f14758b9..9581abdf394d738470d32ae609838832077ee519 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_train.py +++ b/python/paddle/fluid/tests/unittests/test_dist_train.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import os import time import unittest @@ -26,6 +28,12 @@ from paddle.fluid.layers.io import ListenAndServ from paddle.fluid.layers.io import Recv from paddle.fluid.layers.io import Send +from paddle.fluid import core + +RPC_OP_ROLE_ATTR_NAME = op_role_attr_name = core.op_proto_and_checker_maker.kOpRoleAttrName( +) +RPC_OP_ROLE_ATTR_VALUE = core.op_proto_and_checker_maker.OpRole.RPC + class TestSendOp(unittest.TestCase): def test_send(self): @@ -89,18 +97,29 @@ class TestSendOp(unittest.TestCase): def init_client(self, place, port): main = fluid.Program() with fluid.program_guard(main): + main.global_block().append_op( + type="fetch_barrier", + inputs={}, + outputs={}, + attrs={ + "endpoints": ["127.0.0.1:{0}".format(port)], + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE + }) + x = layers.data( shape=[32, 32], dtype='float32', name='X', append_batch_size=False) fluid.initializer.Constant(value=2.3)(x, main.global_block()) + get_var = main.global_block().create_var( name="scale_0.tmp_0", # server side var dtype="float32", persistable=False, shape=[32, 32]) fluid.initializer.Constant(value=2.3)(get_var, main.global_block()) + Send("127.0.0.1:%d" % port, [x]) o = Recv("127.0.0.1:%d" % port, [get_var]) diff --git a/python/paddle/fluid/tests/unittests/test_dist_transformer.py b/python/paddle/fluid/tests/unittests/test_dist_transformer.py index 68cd35d751dbce7eef9919dc8678fc0dd117757b..313207ff9ce054f81322224cb6ceafaaf25bbedf 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_transformer.py +++ b/python/paddle/fluid/tests/unittests/test_dist_transformer.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest from test_dist_base import TestDistBase diff --git a/python/paddle/fluid/tests/unittests/test_dist_transpiler.py b/python/paddle/fluid/tests/unittests/test_dist_transpiler.py index b88fd2707d58ffefad6ad85c9880b64725d7e421..2528f82ff88f7303a6858764e5d2ea12e3ea1df4 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_transpiler.py +++ b/python/paddle/fluid/tests/unittests/test_dist_transpiler.py @@ -12,10 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + +import math + import unittest import paddle.fluid as fluid from paddle.fluid.transpiler.distribute_transpiler import delete_ops import traceback +import collections class TranspilerTest(unittest.TestCase): @@ -46,14 +51,24 @@ class TranspilerTest(unittest.TestCase): def get_main_program(self): main = fluid.Program() + main.random_seed = 1 with fluid.program_guard(main): self.net_conf() self.origin_prog = main.clone() return main - def get_trainer(self, config=None, sync_mode=True): - t = self._transpiler_instance(config, sync_mode) - return t.get_trainer_program() + def get_trainer(self, config=None): + src = fluid.default_startup_program().clone() + + t = self._transpiler_instance(config) + + trainer_main = t.get_trainer_program() + trainer_startup = fluid.default_startup_program() + + assert (src.num_blocks == 1) + assert (trainer_startup.num_blocks == src.num_blocks) + + return trainer_main, trainer_startup def get_pserver(self, ep, config=None, sync_mode=True): t = self._transpiler_instance(config, sync_mode) @@ -89,7 +104,21 @@ class TestBasicModel(TranspilerTest): pserver, startup = self.get_pserver(self.pserver1_ep) pserver2, startup2 = self.get_pserver(self.pserver2_ep) - trainer = self.get_trainer() + trainer, trainer_startup = self.get_trainer() + + # splited var blocks should be in startup program + self.assertTrue("fc_w.block0" in trainer_startup.global_block().vars) + self.assertTrue("fc_w.block1" in trainer_startup.global_block().vars) + self.assertTrue("fc_w" in trainer_startup.global_block().vars) + self.assertTrue("fc_b" in trainer_startup.global_block().vars) + self.assertTrue("fc_w@GRAD" not in trainer_startup.global_block().vars) + self.assertTrue("fc_b@GRAD" not in trainer_startup.global_block().vars) + + src = [op.type for op in trainer_startup.global_block().ops] + dst = ['fill_constant', 'fill_constant', 'uniform_random', 'recv', 'recv', \ + 'fetch_barrier', 'concat'] + + self.assertEqual(src, dst) self.assertEqual([op.type for op in trainer.global_block().ops], [ 'mul', 'elementwise_add', 'elementwise_sub', 'square', 'mean', @@ -140,7 +169,7 @@ class TestBasicModelWithLargeBlockSize(TranspilerTest): pserver, startup = self.get_pserver(self.pserver1_ep, config) pserver2, startup2 = self.get_pserver(self.pserver2_ep, config) - trainer = self.get_trainer(config) + trainer, _ = self.get_trainer(config) self.assertEqual([op.type for op in trainer.global_block().ops], [ 'mul', 'elementwise_add', 'elementwise_sub', 'square', 'mean', @@ -224,7 +253,7 @@ class TestLRDecay(TranspilerTest): def transpiler_test_impl(self): pserver, startup = self.get_pserver(self.pserver1_ep) - trainer = self.get_trainer() + trainer, _ = self.get_trainer() self.assertEqual(len(pserver.blocks), 4) lr_decay_ops = [op.type for op in pserver.blocks[1].ops] @@ -254,7 +283,7 @@ class TestLRDecayConditional(TranspilerTest): def transpiler_test_impl(self): pserver, startup = self.get_pserver(self.pserver1_ep) - trainer = self.get_trainer() + trainer, _ = self.get_trainer() serv_op = pserver.blocks[0].ops[0] sub_blocks = [] @@ -303,7 +332,7 @@ class TestL2Decay(TranspilerTest): def transpiler_test_impl(self): pserver, startup = self.get_pserver(self.pserver1_ep) - trainer = self.get_trainer() + trainer, _ = self.get_trainer() self.assertEqual(len(pserver.blocks), 3) self.assertEqual([op.type for op in pserver.blocks[1].ops], @@ -338,7 +367,7 @@ class TestL2DecayWithPiecewise(TranspilerTest): def transpiler_test_impl(self): pserver, startup = self.get_pserver(self.pserver1_ep) - trainer = self.get_trainer() + trainer, _ = self.get_trainer() self.assertEqual(len(pserver.blocks), 9) self.assertEqual([op.type for op in pserver.blocks[1].ops], [ @@ -362,12 +391,13 @@ class TestL2DecayWithPiecewise(TranspilerTest): class TestDistLookupTableBase(TranspilerTest): def network_with_table(self, is_sparse, is_distributed): + self.table_size = 1000 + self.emb_size = 64 + def emb_pool(ids): - table_size = 1000 - emb_size = 64 emb = fluid.layers.embedding( input=ids, - size=[table_size, emb_size], + size=[self.table_size, self.emb_size], dtype='float32', param_attr='shared_w', # share parameter is_sparse=is_sparse, @@ -412,7 +442,7 @@ class TestLocalLookupTable(TestDistLookupTableBase): self.assertEqual([op.type for op in pserver1.blocks[2].ops], ["sum", "adam", "scale", "scale"]) - trainer = self.get_trainer() + trainer, _ = self.get_trainer() self.assertEqual(len(trainer.blocks), 1) ops = [ 'lookup_table', 'sequence_pool', 'lookup_table', 'sequence_pool', @@ -450,7 +480,7 @@ class TestDistLookupTable(TestDistLookupTableBase): # 5 save table self.assertEqual([op.type for op in pserver1.blocks[5].ops], ["save"]) - trainer = self.get_trainer() + trainer, _ = self.get_trainer() self.assertEqual(len(trainer.blocks), 1) ops = [ 'split_ids', 'prefetch', 'merge_ids', 'sequence_pool', 'split_ids', @@ -483,7 +513,7 @@ class TestAsyncLocalLookupTable(TestDistLookupTableBase): self.assertEqual([op.type for op in pserver1.blocks[2].ops], ["adam", "scale", "scale"]) - trainer = self.get_trainer(config) + trainer, _ = self.get_trainer(config) self.assertEqual(len(trainer.blocks), 1) ops = [ 'lookup_table', 'sequence_pool', 'lookup_table', 'sequence_pool', @@ -522,7 +552,7 @@ class TestAsyncDistLookupTable(TestDistLookupTableBase): # 5 save table self.assertEqual([op.type for op in pserver1.blocks[5].ops], ["save"]) - trainer = self.get_trainer(config) + trainer, _ = self.get_trainer(config) self.assertEqual(len(trainer.blocks), 1) ops = [ 'split_ids', 'prefetch', 'merge_ids', 'sequence_pool', 'split_ids', @@ -536,17 +566,20 @@ class TestAsyncDistLookupTable(TestDistLookupTableBase): self.assertEqual([op.type for op in trainer.blocks[0].ops], ops) -class TestDistArgsInProgram(TestDistLookupTableBase): +class TestDistLookupTableSliceSize(TestDistLookupTableBase): def net_conf(self): self.network_with_table(is_sparse=True, is_distributed=True) def transpiler_test_impl(self): config = fluid.DistributeTranspilerConfig() - pserver1, _ = self.get_pserver(self.pserver1_ep, config, False) - - self.assertTrue(pserver1._is_chief) - self.assertTrue(pserver1._is_distributed) - self.assertEqual(pserver1._distributed_lookup_table) + pserver1, startup1 = self.get_pserver(self.pserver1_ep, config) + + self.assertTrue(self.transpiler.has_distributed_lookup_table) + lookup_table_var = pserver1.global_block().vars[ + self.transpiler.table_name] + row_size = lookup_table_var.shape[0] + calc_row_size = int(math.ceil(self.table_size / self.pservers)) + self.assertEqual(row_size, calc_row_size) class TestRMSPropOptimizer(TranspilerTest): diff --git a/python/paddle/fluid/tests/unittests/test_dist_word2vec.py b/python/paddle/fluid/tests/unittests/test_dist_word2vec.py index 543d0f9dc2c9b8cdcfaaaa14a7a4f197d210d951..e43992c488d35d1b3f670e13650d420b0498eeec 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_word2vec.py +++ b/python/paddle/fluid/tests/unittests/test_dist_word2vec.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import unittest from test_dist_base import TestDistBase diff --git a/python/paddle/fluid/tests/unittests/test_dropout_op.py b/python/paddle/fluid/tests/unittests/test_dropout_op.py index eaa3435a86462236a99489749abe877648677053..0296bc2af4e0b79478c34b4cceab32b5a8a50f2f 100644 --- a/python/paddle/fluid/tests/unittests/test_dropout_op.py +++ b/python/paddle/fluid/tests/unittests/test_dropout_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_dyn_rnn.py b/python/paddle/fluid/tests/unittests/test_dyn_rnn.py index fdc6adc93bc2488d4faffed61fde5d54bbbbfd57..d84dab1499a267ca081c2e8ea2856c7c4bb627cb 100644 --- a/python/paddle/fluid/tests/unittests/test_dyn_rnn.py +++ b/python/paddle/fluid/tests/unittests/test_dyn_rnn.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid as fluid import paddle import unittest diff --git a/python/paddle/fluid/tests/unittests/test_dynrnn_gradient_check.py b/python/paddle/fluid/tests/unittests/test_dynrnn_gradient_check.py index 7756885166c88eadb77c2c6d56aab767015abc51..9d635f36fe83d041bb57df0759da1481f66bbaa2 100644 --- a/python/paddle/fluid/tests/unittests/test_dynrnn_gradient_check.py +++ b/python/paddle/fluid/tests/unittests/test_dynrnn_gradient_check.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy import random import collections diff --git a/python/paddle/fluid/tests/unittests/test_dynrnn_static_input.py b/python/paddle/fluid/tests/unittests/test_dynrnn_static_input.py index d182889a970fb178dec4976aebbd79d05dc3e91e..b4359fc69ae18b45774af0d2b20c1540bd99da5c 100644 --- a/python/paddle/fluid/tests/unittests/test_dynrnn_static_input.py +++ b/python/paddle/fluid/tests/unittests/test_dynrnn_static_input.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_edit_distance_op.py b/python/paddle/fluid/tests/unittests/test_edit_distance_op.py index 816562621b4fc749f3c6b0eca8ee3c5850ef1ba9..4d03523025d357e453848f3016ffee890b5d46ec 100644 --- a/python/paddle/fluid/tests/unittests/test_edit_distance_op.py +++ b/python/paddle/fluid/tests/unittests/test_edit_distance_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_add_mkldnn_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_add_mkldnn_op.py index bcdbfc8e527d0dc9a95eddaf040f8035207b6c20..d85cc1f856df8eaa73cef318b48a292042488edf 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_add_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_add_mkldnn_op.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import unittest import numpy as np import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_add_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_add_op.py index fb9a496126f0b6efcad73590c78efe5a47b88cd6..5aec5d8e38aba39e6aba9a8f19637587c2f12544 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_add_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_add_op.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import unittest import numpy as np import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_div_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_div_op.py index bfe022af6dac711e76678b79005d4cfff90c2a2b..cadaf1df53af0af56afa8c3631b0f5ce390f318c 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_div_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_div_op.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_gradient_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_gradient_op.py index 6f350044892a4ba2a985b5bc2328ab1fc20c5504..9f452ffde74ee18d14f155fb5ed53fee57f12f49 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_gradient_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_gradient_op.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_max_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_max_op.py index b6cd18a579520f921feed48cc86d8027f6a7bd1e..43c58710ba50c27077942643b84b7642eaf57710 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_max_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_max_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_min_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_min_op.py index 92099724fe65050b62c738f4e6c269a0ca3f4ef1..45c861e2c3df9f14f9886091012d6cca69944454 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_min_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_min_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_mul_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_mul_op.py index 2742bb21d95ed3363e82a70b0172cc787878abd9..775c2253ab3b27708b745b85fc007fcb504d1aed 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_mul_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_mul_op.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_pow_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_pow_op.py index a3fd18669c556701b2586e88ddbb89ca79549a86..7bf642f03f480b1eeec68298f9d453deb1fa2ac3 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_pow_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_pow_op.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_sub_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_sub_op.py index 1854232194963bcbe302010320a30d85747eea96..6cb88a8bb1cad7a58ca175cfc14298c959e3bad6 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_sub_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_sub_op.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_exception.py b/python/paddle/fluid/tests/unittests/test_exception.py index bb7c0f88f6027807394e15aa6803da2ddc22f4e2..798ed53cddade22e986cae65109b6c6ac7a291b6 100644 --- a/python/paddle/fluid/tests/unittests/test_exception.py +++ b/python/paddle/fluid/tests/unittests/test_exception.py @@ -12,19 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + +import paddle.compat as cpt import paddle.fluid.core as core import unittest class TestException(unittest.TestCase): def test_exception(self): - ex = None + exception = None try: core.__unittest_throw_exception__() except core.EnforceNotMet as ex: - self.assertIn("test exception", ex.message) + self.assertIn("test exception", cpt.get_exception_message(ex)) + exception = ex - self.assertIsNotNone(ex) + self.assertIsNotNone(exception) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_executor_and_mul.py b/python/paddle/fluid/tests/unittests/test_executor_and_mul.py index e1272c1d6dd7131b55ecf33fa0de0fc78a3ac5a7..b1f89eca6e58aec41b5863f4c885d5c6231a72f4 100644 --- a/python/paddle/fluid/tests/unittests/test_executor_and_mul.py +++ b/python/paddle/fluid/tests/unittests/test_executor_and_mul.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy diff --git a/python/paddle/fluid/tests/unittests/test_expand_op.py b/python/paddle/fluid/tests/unittests/test_expand_op.py index a91e3aef5a18a681f3baf405da2beebb8c85360c..67a8d8f0721c2c75b432d68d64be8fc1035ffc74 100644 --- a/python/paddle/fluid/tests/unittests/test_expand_op.py +++ b/python/paddle/fluid/tests/unittests/test_expand_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_extract_rows_op.py b/python/paddle/fluid/tests/unittests/test_extract_rows_op.py index 6a41c44fe655b18626bdb727745dae032babe8ad..8629bcf0f2e3c37aefdbf79b203176a43e0c3a7e 100644 --- a/python/paddle/fluid/tests/unittests/test_extract_rows_op.py +++ b/python/paddle/fluid/tests/unittests/test_extract_rows_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_fake_dequantize_op.py b/python/paddle/fluid/tests/unittests/test_fake_dequantize_op.py index 026ac2112b2d78644b3315b9cab8019ca27e9714..d84ebed3fac67db323392494c701cf2a51b28305 100644 --- a/python/paddle/fluid/tests/unittests/test_fake_dequantize_op.py +++ b/python/paddle/fluid/tests/unittests/test_fake_dequantize_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import math diff --git a/python/paddle/fluid/tests/unittests/test_fake_quantize_op.py b/python/paddle/fluid/tests/unittests/test_fake_quantize_op.py index 6c6aa9d3bb656740c528c728efafc6a47e8bff91..cc0494774a5f2f24faaae65f193fc3ff9270d9ac 100644 --- a/python/paddle/fluid/tests/unittests/test_fake_quantize_op.py +++ b/python/paddle/fluid/tests/unittests/test_fake_quantize_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_fc_mkldnn_op.py b/python/paddle/fluid/tests/unittests/test_fc_mkldnn_op.py index 3f547f3c484bf034a87823a75d946ef130a5cb70..45951a34d6f61a242cb2dc004d6801a6c1c9dd92 100644 --- a/python/paddle/fluid/tests/unittests/test_fc_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/test_fc_mkldnn_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest @@ -22,6 +24,7 @@ def fully_connected_naive(input, weights, bias_data=None): w_h, w_c = weights.shape x_data = np.reshape(input, [in_n, in_c * in_h * in_w]) + # this transpose should be implemented at C code w_data = np.transpose(np.reshape(weights, (w_c, in_c * in_h * in_w))) result = None @@ -43,15 +46,11 @@ class TestFCMKLDNNOp(OpTest): def setUp(self): self.op_type = "fc" self.use_mkldnn = True - self.with_bias = True self.matrix = MatrixGenerate(1, 10, 15, 3, 3) self.inputs = {'Input': self.matrix.input, 'W': self.matrix.weights} - self.attrs = { - 'use_mkldnn': self.use_mkldnn, - 'with_bias': self.with_bias - } + self.attrs = {'use_mkldnn': self.use_mkldnn, } self.outputs = { 'Out': fully_connected_naive(self.matrix.input, self.matrix.weights) @@ -85,13 +84,11 @@ class TestFCMKLDNNOp3(TestFCMKLDNNOp): class TestFCMKLDNNOp4(TestFCMKLDNNOp): def init_op_type(self): - self.with_bias = False self.matrix = MatrixGenerate(2, 32, 48, 2, 2) class TestFCMKLDNNOp4(TestFCMKLDNNOp): def init_op_type(self): - self.with_bias = False self.matrix = MatrixGenerate(2, 32, 1000, 6, 6) diff --git a/python/paddle/fluid/tests/unittests/test_fc_op.py b/python/paddle/fluid/tests/unittests/test_fc_op.py new file mode 100644 index 0000000000000000000000000000000000000000..2bb920710a9b10f3a8159bad3b33dd15ffbada19 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_fc_op.py @@ -0,0 +1,90 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import numpy as np +from op_test import OpTest + + +def fc_refer(matrix, with_bias): + in_n, in_c, in_h, in_w = matrix.input.shape + w_i, w_o = matrix.weights.shape + + x_data = np.reshape(matrix.input, [in_n, in_c * in_h * in_w]) + w_data = np.reshape(matrix.weights, [w_i, w_o]) + b_data = np.reshape(matrix.bias, [1, w_o]) + result = None + + if with_bias: + result = np.dot(x_data, w_data) + b_data + else: + result = np.dot(x_data, w_data) + + return result + + +class MatrixGenerate: + def __init__(self, mb, ic, oc, h, w): + self.input = np.random.random((mb, ic, h, w)).astype("float32") + self.weights = np.random.random((ic * h * w, oc)).astype("float32") + self.bias = np.random.random((1, oc)).astype("float32") + + +class TestFCOp(OpTest): + def setUp(self): + self.op_type = "fc" + self.matrix = MatrixGenerate(1, 10, 15, 3, 3) + + self.with_bias = True + if self.with_bias: + self.inputs = { + 'Input': self.matrix.input, + 'W': self.matrix.weights, + 'Bias': self.matrix.bias + } + else: + self.inputs = {'Input': self.matrix.input, 'W': self.matrix.weights} + + self.attrs = {'use_mkldnn': False} + + self.outputs = {'Out': fc_refer(self.matrix, self.with_bias)} + + def test_check_output(self): + self.check_output() + + +class TestFCOpBiasBoth(TestFCOp): + def init_shapes(self, mb, ic, oc, h, w): + for with_bias in {True, False}: + self.with_bias = with_bias + self.matrix = MatrixGenerate(mb, ic, oc, h, w) + + +class TestFCOp1(TestFCOpBiasBoth): + def init_op_type(self): + self.init_shapes(2, 8, 10, 1, 1) + + +class TestFCOp2(TestFCOpBiasBoth): + def init_op_type(self): + self.init_shapes(4, 5, 6, 2, 2) + + +class TestFCOp4(TestFCOpBiasBoth): + def init_op_type(self): + self.init_shapes(1, 32, 64, 3, 3) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_feed_fetch_method.py b/python/paddle/fluid/tests/unittests/test_feed_fetch_method.py index 8b9da843115409c65055927d317867d1290c8f0e..b823d397e9530362f5fee417278e36477d65f6f5 100644 --- a/python/paddle/fluid/tests/unittests/test_feed_fetch_method.py +++ b/python/paddle/fluid/tests/unittests/test_feed_fetch_method.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid.core as core import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_fetch_var.py b/python/paddle/fluid/tests/unittests/test_fetch_var.py index e6f37f0b4ca781e4ec83a00f8f2605ef02716bd7..de339d821b1329662469c26eacd234b74a102e13 100644 --- a/python/paddle/fluid/tests/unittests/test_fetch_var.py +++ b/python/paddle/fluid/tests/unittests/test_fetch_var.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid as fluid import paddle.fluid.layers as layers import op_test diff --git a/python/paddle/fluid/tests/unittests/test_fill_constant_batch_size_like_op.py b/python/paddle/fluid/tests/unittests/test_fill_constant_batch_size_like_op.py index 0c75cf33f5f208d11081a6802910c25553b8c4ec..fdc8a118e56f4473da5ed60169daebec14c7c33c 100644 --- a/python/paddle/fluid/tests/unittests/test_fill_constant_batch_size_like_op.py +++ b/python/paddle/fluid/tests/unittests/test_fill_constant_batch_size_like_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_fill_constant_op.py b/python/paddle/fluid/tests/unittests/test_fill_constant_op.py index 5e2ddb218af8fcf4f686260296b57519ec7486b9..44fb1d047dff48d2554c0bf637afbfda725e0a02 100644 --- a/python/paddle/fluid/tests/unittests/test_fill_constant_op.py +++ b/python/paddle/fluid/tests/unittests/test_fill_constant_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_fill_op.py b/python/paddle/fluid/tests/unittests/test_fill_op.py index 762d29199e2127415ed7daabca63edcdbae3344f..b734ee05b3f2291d7a79f1550946bf6546ada6e0 100644 --- a/python/paddle/fluid/tests/unittests/test_fill_op.py +++ b/python/paddle/fluid/tests/unittests/test_fill_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_fill_zeros_like_op.py b/python/paddle/fluid/tests/unittests/test_fill_zeros_like_op.py index c9b3e4ba138f425fd2991bf637d2e32be3eb5168..eec73d0beb39c49f535a03532e536092001c8445 100644 --- a/python/paddle/fluid/tests/unittests/test_fill_zeros_like_op.py +++ b/python/paddle/fluid/tests/unittests/test_fill_zeros_like_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_flatten_op.py b/python/paddle/fluid/tests/unittests/test_flatten_op.py index f8692ce2ea66ef61c63bc41e77df050398ac63fd..17b01e03124e8007c51107b414c628d4bfc49c79 100644 --- a/python/paddle/fluid/tests/unittests/test_flatten_op.py +++ b/python/paddle/fluid/tests/unittests/test_flatten_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_framework_debug_str.py b/python/paddle/fluid/tests/unittests/test_framework_debug_str.py index c906c74afe66b05e2ca0e1122677e2dc738351b8..72f43e56ccbe04f56cfd5a655fb57c58369039bb 100644 --- a/python/paddle/fluid/tests/unittests/test_framework_debug_str.py +++ b/python/paddle/fluid/tests/unittests/test_framework_debug_str.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest from paddle.fluid.framework import Program diff --git a/python/paddle/fluid/tests/unittests/test_ftrl_op.py b/python/paddle/fluid/tests/unittests/test_ftrl_op.py index 5f7581391afb2bd57cb25695f3c0d4db8573c80c..a6390b054f06184831c289fe9556216ae213be7c 100644 --- a/python/paddle/fluid/tests/unittests/test_ftrl_op.py +++ b/python/paddle/fluid/tests/unittests/test_ftrl_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_fused_elemwise_activation_op.py b/python/paddle/fluid/tests/unittests/test_fused_elemwise_activation_op.py index ec0a939e9ec21952a6657ea849bb9844bb69cc8d..97e1b9061afb738dd9e5f8b3b6a9c9a123c6aac6 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_elemwise_activation_op.py +++ b/python/paddle/fluid/tests/unittests/test_fused_elemwise_activation_op.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import unittest import numpy as np import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_gather_op.py b/python/paddle/fluid/tests/unittests/test_gather_op.py index 4ae90864806204197c52bbbdc5516f141afd4613..bd5785aa55af241fe42a1ae2c550dbdb980f42e2 100644 --- a/python/paddle/fluid/tests/unittests/test_gather_op.py +++ b/python/paddle/fluid/tests/unittests/test_gather_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_gaussian_random_batch_size_like_op.py b/python/paddle/fluid/tests/unittests/test_gaussian_random_batch_size_like_op.py index 1398166a74e714e0e902532166cde5d94ccae5f6..9a0631fa26a3e93c5c2115fd03a37de3fac46ce5 100644 --- a/python/paddle/fluid/tests/unittests/test_gaussian_random_batch_size_like_op.py +++ b/python/paddle/fluid/tests/unittests/test_gaussian_random_batch_size_like_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_gaussian_random_mkldnn_op.py b/python/paddle/fluid/tests/unittests/test_gaussian_random_mkldnn_op.py index 3ae877a60818744f852d3af9a02ffebf5e2affc8..9777ec390656d3f6166bf9f5de7bbad8b6bd786d 100644 --- a/python/paddle/fluid/tests/unittests/test_gaussian_random_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/test_gaussian_random_mkldnn_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest from test_gaussian_random_op import TestGaussianRandomOp diff --git a/python/paddle/fluid/tests/unittests/test_gaussian_random_op.py b/python/paddle/fluid/tests/unittests/test_gaussian_random_op.py index 8481500fd78f0ccf34f09c66bec27e195b9aada3..496aa4111056591efce14549011d66f9ae49713a 100644 --- a/python/paddle/fluid/tests/unittests/test_gaussian_random_op.py +++ b/python/paddle/fluid/tests/unittests/test_gaussian_random_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy diff --git a/python/paddle/fluid/tests/unittests/test_get_places_op.py b/python/paddle/fluid/tests/unittests/test_get_places_op.py index 964423e2d2638224244b4ca774d8eee08f3ec989..441666a97b16a320692d6a15363f61156e52242b 100644 --- a/python/paddle/fluid/tests/unittests/test_get_places_op.py +++ b/python/paddle/fluid/tests/unittests/test_get_places_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid as fluid from paddle.fluid.layers.device import get_places import decorators diff --git a/python/paddle/fluid/tests/unittests/test_gru_op.py b/python/paddle/fluid/tests/unittests/test_gru_op.py index 86a2c674d01f45b2b141572c8191d2fba7fa312f..001fd7efb159e60bdf3cd0698d85dea90ad71616 100644 --- a/python/paddle/fluid/tests/unittests/test_gru_op.py +++ b/python/paddle/fluid/tests/unittests/test_gru_op.py @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import math +import functools from op_test import OpTest from test_lstm_op import identity, sigmoid, tanh, relu @@ -38,7 +41,8 @@ class TestGRUOp(OpTest): for i in range(len(seq_lens)): seq_starts.append(seq_starts[-1] + seq_lens[i]) sorted_seqs = sorted( - list(range(len(seq_lens))), lambda x, y: seq_lens[y] - seq_lens[x]) + list(range(len(seq_lens))), + key=functools.cmp_to_key(lambda x, y: seq_lens[y] - seq_lens[x])) num_batch = seq_lens[sorted_seqs[0]] for batch_idx in range(num_batch): idx_in_seq = [] diff --git a/python/paddle/fluid/tests/unittests/test_gru_unit_op.py b/python/paddle/fluid/tests/unittests/test_gru_unit_op.py index 87a9eba4d97459082cdf1499efeddf24ed51e1b1..b5a66fdf086f2abc0c9a8af663241b9eda739407 100644 --- a/python/paddle/fluid/tests/unittests/test_gru_unit_op.py +++ b/python/paddle/fluid/tests/unittests/test_gru_unit_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import math import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_hinge_loss_op.py b/python/paddle/fluid/tests/unittests/test_hinge_loss_op.py index 70586c6be3da415fbccf4114615e6f7e08de0f0f..1eb441e2c52905c2b60104de5e04037714b34648 100644 --- a/python/paddle/fluid/tests/unittests/test_hinge_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_hinge_loss_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_hsigmoid_op.py b/python/paddle/fluid/tests/unittests/test_hsigmoid_op.py index daa5da8d95129af0305b326832a557daeb4c5c9c..6948ae30023a75d4735db1c78466e89e28640c9e 100644 --- a/python/paddle/fluid/tests/unittests/test_hsigmoid_op.py +++ b/python/paddle/fluid/tests/unittests/test_hsigmoid_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import math diff --git a/python/paddle/fluid/tests/unittests/test_huber_loss_op.py b/python/paddle/fluid/tests/unittests/test_huber_loss_op.py index a8d0a77625598e9d929a993db46ba95b0e07527a..0055ef0052fe126b268cf7a17a8307224cced99a 100644 --- a/python/paddle/fluid/tests/unittests/test_huber_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_huber_loss_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_im2sequence_op.py b/python/paddle/fluid/tests/unittests/test_im2sequence_op.py index 13bc5768740ece00bbe285a0b47d82bb8a42d2c7..833e46483c2532e283fd672dc56cb93941f5b4ba 100644 --- a/python/paddle/fluid/tests/unittests/test_im2sequence_op.py +++ b/python/paddle/fluid/tests/unittests/test_im2sequence_op.py @@ -11,6 +11,8 @@ #WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #See the License for the specific language governing permissions and #limitations under the License. + +from __future__ import print_function import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_image_classification_layer.py b/python/paddle/fluid/tests/unittests/test_image_classification_layer.py index 23b1ed957ad15bb631cd5160eb48328c76302987..405637969af6fb515a24ecb077e470279c3ffc24 100644 --- a/python/paddle/fluid/tests/unittests/test_image_classification_layer.py +++ b/python/paddle/fluid/tests/unittests/test_image_classification_layer.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/unittests/test_infer_shape.py b/python/paddle/fluid/tests/unittests/test_infer_shape.py index 699a2d42467b7ac0dcf1939bde744ad2fcb29c97..a3d700aad8236fea7bb0e6d043323ad3bd0851f2 100644 --- a/python/paddle/fluid/tests/unittests/test_infer_shape.py +++ b/python/paddle/fluid/tests/unittests/test_infer_shape.py @@ -12,8 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest +import six import paddle.fluid.core as core @@ -27,14 +30,14 @@ class TestInferShape(unittest.TestCase): shape = [10, 20] # prepare input/output - x1 = block.var("x1") + x1 = block.var(six.b("x1")) x1.set_type(core.VarDesc.VarType.LOD_TENSOR) x1.set_shape(shape) - x2 = block.var("x2") + x2 = block.var(six.b("x2")) x2.set_type(core.VarDesc.VarType.LOD_TENSOR) x2.set_shape(shape) - out = block.var("out") + out = block.var(six.b("out")) out.set_type(core.VarDesc.VarType.LOD_TENSOR) # prepare the operator @@ -57,14 +60,14 @@ class TestInferShape(unittest.TestCase): y_shape = [20, 30] # prepare input/output - x1 = block.var("x") + x1 = block.var(six.b("x")) x1.set_type(core.VarDesc.VarType.LOD_TENSOR) x1.set_shape(x_shape) - x2 = block.var("y") + x2 = block.var(six.b("y")) x2.set_type(core.VarDesc.VarType.LOD_TENSOR) x2.set_shape(y_shape) - out = block.var("out") + out = block.var(six.b("out")) out.set_type(core.VarDesc.VarType.LOD_TENSOR) # prepare the operator diff --git a/python/paddle/fluid/tests/unittests/test_inference_model_io.py b/python/paddle/fluid/tests/unittests/test_inference_model_io.py index 4cd203155f446df07d2fe6c1d56e0d20f1113679..9962702f69644b7aef7d868f086abb390441f617 100644 --- a/python/paddle/fluid/tests/unittests/test_inference_model_io.py +++ b/python/paddle/fluid/tests/unittests/test_inference_model_io.py @@ -12,8 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest +import six import numpy as np import paddle.fluid.core as core @@ -48,7 +51,7 @@ class TestBook(unittest.TestCase): exe.run(init_program, feed={}, fetch_list=[]) - for i in range(100): + for i in six.moves.xrange(100): tensor_x = np.array( [[1, 1], [1, 2], [3, 4], [5, 2]]).astype("float32") tensor_y = np.array([[-2], [-3], [-7], [-7]]).astype("float32") @@ -64,7 +67,7 @@ class TestBook(unittest.TestCase): 'y': tensor_y}, fetch_list=[avg_cost])[0] - reload(executor) # reload to build a new scope + six.moves.reload_module(executor) # reload to build a new scope exe = executor.Executor(place) [infer_prog, feed_var_names, fetch_vars] = load_inference_model( diff --git a/python/paddle/fluid/tests/unittests/test_initializer.py b/python/paddle/fluid/tests/unittests/test_initializer.py index b215e379864e919af03591ab2566c08dddbb5743..ab7183f88df809e584ca50ba16221bfdfe1376a9 100644 --- a/python/paddle/fluid/tests/unittests/test_initializer.py +++ b/python/paddle/fluid/tests/unittests/test_initializer.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import unittest diff --git a/python/paddle/fluid/tests/unittests/test_iou_similarity_op.py b/python/paddle/fluid/tests/unittests/test_iou_similarity_op.py index eff4212d91e609a7ef531280bbd3cf3671a59830..7c1808cf998e84c22c46df68ef07259c1a021c19 100644 --- a/python/paddle/fluid/tests/unittests/test_iou_similarity_op.py +++ b/python/paddle/fluid/tests/unittests/test_iou_similarity_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import numpy.random as random diff --git a/python/paddle/fluid/tests/unittests/test_is_empty_op.py b/python/paddle/fluid/tests/unittests/test_is_empty_op.py index 11121d9b65351eab639b7618fac0e54714cf4680..26d607718aec0bdffa00b9b4bca06ec6c0196217 100644 --- a/python/paddle/fluid/tests/unittests/test_is_empty_op.py +++ b/python/paddle/fluid/tests/unittests/test_is_empty_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_l1_norm_op.py b/python/paddle/fluid/tests/unittests/test_l1_norm_op.py index fa5b18a16f7a3e734ff8bb4f53240e8a9ce8fd8f..4e24a78ee54dfb1fb0e4f97317642cfaffe9436e 100644 --- a/python/paddle/fluid/tests/unittests/test_l1_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_l1_norm_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import unittest from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_label_smooth_op.py b/python/paddle/fluid/tests/unittests/test_label_smooth_op.py index ca21289a0d48a123aed90bc557ccc732702b47f1..62d385bc52cfb3a9fe15a82096ff33abc1bcc552 100644 --- a/python/paddle/fluid/tests/unittests/test_label_smooth_op.py +++ b/python/paddle/fluid/tests/unittests/test_label_smooth_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_layer_norm_op.py b/python/paddle/fluid/tests/unittests/test_layer_norm_op.py index 295887ccd171a3101329eb1255da146914fa9264..fb6c43136ff82af55d1fcc2969cf4a07ae081204 100644 --- a/python/paddle/fluid/tests/unittests/test_layer_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_layer_norm_op.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_layers.py b/python/paddle/fluid/tests/unittests/test_layers.py index 8f2dac786d03334642714d221710a20833939c7a..e833a7db482db3eb69ef8596c468c535dee02370 100644 --- a/python/paddle/fluid/tests/unittests/test_layers.py +++ b/python/paddle/fluid/tests/unittests/test_layers.py @@ -21,6 +21,7 @@ import paddle.fluid.nets as nets from paddle.fluid.framework import Program, program_guard, default_main_program from paddle.fluid.param_attr import ParamAttr import decorators +from paddle.fluid.initializer import Constant class TestBook(unittest.TestCase): @@ -158,7 +159,7 @@ class TestBook(unittest.TestCase): input=crf_decode, label=label, chunk_scheme="IOB", - num_chunk_types=(label_dict_len - 1) / 2) + num_chunk_types=(label_dict_len - 1) // 2) self.assertFalse(crf is None) self.assertFalse(crf_decode is None) @@ -285,7 +286,7 @@ class TestBook(unittest.TestCase): name='word_{0}'.format(i), shape=[1], dtype='int64')) dict_size = 10000 - label_word = int(window_size / 2) + 1 + label_word = int(window_size // 2) + 1 embs = [] for i in range(window_size): @@ -465,6 +466,17 @@ class TestBook(unittest.TestCase): self.assertIsNotNone(out) print(str(program)) + def test_flatten(self): + program = Program() + with program_guard(program): + x = layers.data( + name='x', + append_batch_size=False, + shape=[4, 4, 3], + dtype="float32") + out = layers.flatten(x, axis=1, name="flatten") + self.assertIsNotNone(out) + def test_shape(self): program = Program() with program_guard(program): @@ -474,6 +486,20 @@ class TestBook(unittest.TestCase): self.assertIsNotNone(out) print(str(program)) + def test_prelu(self): + program = Program() + with program_guard(program): + input = layers.data( + name="input", shape=[5, 200, 100, 100], dtype="float32") + mode = 'channel' + out = layers.prelu( + input, + mode, + param_attr=ParamAttr(initializer=Constant(1.0)), + name='prelu') + self.assertIsNotNone(out) + print(str(program)) + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_learning_rate_scheduler.py b/python/paddle/fluid/tests/unittests/test_learning_rate_scheduler.py index e628195e7265ec564bd64a212c4a35fdff495063..0d3e6d73e0149fe633b8f1de9041068c2e3bb293 100644 --- a/python/paddle/fluid/tests/unittests/test_learning_rate_scheduler.py +++ b/python/paddle/fluid/tests/unittests/test_learning_rate_scheduler.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import copy import math import unittest diff --git a/python/paddle/fluid/tests/unittests/test_linear_chain_crf_op.py b/python/paddle/fluid/tests/unittests/test_linear_chain_crf_op.py index 696d0ab4fa81a409a2bf0d6f6f23779ec26eb6d2..6e31e9204e95d98fcf69ed84a46d6cf3d94c808a 100644 --- a/python/paddle/fluid/tests/unittests/test_linear_chain_crf_op.py +++ b/python/paddle/fluid/tests/unittests/test_linear_chain_crf_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import random import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_listen_and_serv_op.py b/python/paddle/fluid/tests/unittests/test_listen_and_serv_op.py index 1cdc69501043d120b9e3cc8ccda3a1212d205886..48b52a5412eb99fbc7a5c8534a766ede4954e849 100644 --- a/python/paddle/fluid/tests/unittests/test_listen_and_serv_op.py +++ b/python/paddle/fluid/tests/unittests/test_listen_and_serv_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle import paddle.fluid as fluid import os diff --git a/python/paddle/fluid/tests/unittests/test_lod_array_length_op.py b/python/paddle/fluid/tests/unittests/test_lod_array_length_op.py index d8b4e40662568f580ccff0257512cb8809488f17..15485df5ac440f2ff666ca27ef8e8bcc5df866c0 100644 --- a/python/paddle/fluid/tests/unittests/test_lod_array_length_op.py +++ b/python/paddle/fluid/tests/unittests/test_lod_array_length_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.layers as layers from paddle.fluid.executor import Executor diff --git a/python/paddle/fluid/tests/unittests/test_lod_rank_table.py b/python/paddle/fluid/tests/unittests/test_lod_rank_table.py index d53ead381d301e797d5a19784aed49a5d6f99319..865ca118d55f82c66d44f4e3d553baafa0c14c3a 100644 --- a/python/paddle/fluid/tests/unittests/test_lod_rank_table.py +++ b/python/paddle/fluid/tests/unittests/test_lod_rank_table.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from paddle.fluid.layers import data from paddle.fluid.layers.control_flow import lod_rank_table from paddle.fluid.executor import Executor diff --git a/python/paddle/fluid/tests/unittests/test_lod_reset_op.py b/python/paddle/fluid/tests/unittests/test_lod_reset_op.py index 77905c4b96499c855fd5c5e704b8051ccdb7a323..31f364a42f624c8662a5ae087b003ca0304ae419 100644 --- a/python/paddle/fluid/tests/unittests/test_lod_reset_op.py +++ b/python/paddle/fluid/tests/unittests/test_lod_reset_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_lod_tensor_array.py b/python/paddle/fluid/tests/unittests/test_lod_tensor_array.py index 0ac6d9b81df0ecbe9c6560cdb0ab0507c3c2ed18..6ad27de9a0e42d1a15ec4a17804c7c0f7ebf5d94 100644 --- a/python/paddle/fluid/tests/unittests/test_lod_tensor_array.py +++ b/python/paddle/fluid/tests/unittests/test_lod_tensor_array.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.core as core import numpy diff --git a/python/paddle/fluid/tests/unittests/test_lod_tensor_array_ops.py b/python/paddle/fluid/tests/unittests/test_lod_tensor_array_ops.py index 9789ff4af648b41a1b53844be89249bd260de61b..6a78ef5078a738efa2ae39ea23645fedaecce63b 100644 --- a/python/paddle/fluid/tests/unittests/test_lod_tensor_array_ops.py +++ b/python/paddle/fluid/tests/unittests/test_lod_tensor_array_ops.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.core as core import numpy diff --git a/python/paddle/fluid/tests/unittests/test_log_loss_op.py b/python/paddle/fluid/tests/unittests/test_log_loss_op.py index d3980b8db93ca517d16b6f782ba800ce839c3f45..784f4f648d52bdf4f2357f4454d790a8d53288f3 100644 --- a/python/paddle/fluid/tests/unittests/test_log_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_log_loss_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_logical_op.py b/python/paddle/fluid/tests/unittests/test_logical_op.py index 1d7dfe60f200459705a48664c1a5b22d2a5888d2..521851a3d57a4a3e8b2c8e1639325cc6c88fdd84 100644 --- a/python/paddle/fluid/tests/unittests/test_logical_op.py +++ b/python/paddle/fluid/tests/unittests/test_logical_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import op_test import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_lookup_sparse_table_op.py b/python/paddle/fluid/tests/unittests/test_lookup_sparse_table_op.py index aa9eae1e882f55ef51f38e158317a1a9aeed641c..11e5d8b536fb65b66c954991bf815241774702ec 100644 --- a/python/paddle/fluid/tests/unittests/test_lookup_sparse_table_op.py +++ b/python/paddle/fluid/tests/unittests/test_lookup_sparse_table_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest @@ -19,36 +21,27 @@ import paddle.fluid.core as core from paddle.fluid.op import Operator -def output_hist(out): - hist, _ = np.histogram(out, range=(-5, 10)) - hist = hist.astype("float32") - hist /= float(out.size) - prob = 0.1 * np.ones((10)) - return hist, prob - - class TestLookupSpraseTable(OpTest): def check_with_place(self, place): scope = core.Scope() - # create and initialize Id Variable - ids = scope.var("Ids").get_tensor() - ids_array = np.array([0, 2, 3, 5, 100]).astype("int64") - ids.set(ids_array, place) - # create and initialize W Variable - rows = [0, 1, 2, 3, 4, 5, 6] - row_numel = 10000 + table_size = 10000 + row_numel = 8 w_selected_rows = scope.var('W').get_selected_rows() - w_selected_rows.set_height(len(rows)) - w_selected_rows.set_rows(rows) - w_array = np.ones((len(rows), row_numel)).astype("float32") - for i in range(len(rows)): + w_selected_rows.set_height(table_size) + w_array = np.ones((table_size, row_numel)).astype("float32") + for i in range(table_size): w_array[i] *= i w_tensor = w_selected_rows.get_tensor() w_tensor.set(w_array, place) + # create and initialize Id Variable + ids = scope.var("Ids").get_tensor() + ids_array1 = np.array([0, 2, 3, 2, 5, 0, 100]).astype("int64") + ids.set(ids_array1, place) + # create Out Variable out_tensor = scope.var('Out').get_tensor() @@ -64,16 +57,28 @@ class TestLookupSpraseTable(OpTest): lookup_table.run(scope, place) # get result from Out - result_array = np.array(out_tensor) + result_array1 = np.array(out_tensor) # all(): return True if all elements of the iterable are true (or if the iterable is empty) - for idx, row in enumerate(ids_array[:-2]): - assert (row == result_array[idx]).all() + assert (result_array1[0] == w_array[0]).all() + assert (result_array1[1] == w_array[1]).all() + assert (result_array1[2] == w_array[2]).all() + assert (result_array1[3] == w_array[1]).all() + assert (result_array1[4] == w_array[3]).all() + assert (result_array1[5] == w_array[0]).all() + assert (result_array1[6] == w_array[4]).all() + + # create and initialize Id Variable + ids = scope.var("Ids").get_tensor() + ids_array2 = np.array([4, 2, 3, 7, 100000]).astype("int64") + ids.set(ids_array2, place) + lookup_table.run(scope, place) - # check the random value - hist, prob = output_hist(result_array[-1]) - self.assertTrue( - np.allclose( - hist, prob, rtol=0, atol=0.01), "hist: " + str(hist)) + result_array2 = np.array(out_tensor) + assert (result_array2[0] == w_array[5]).all() + assert (result_array2[1] == w_array[1]).all() + assert (result_array2[2] == w_array[2]).all() + assert (result_array2[3] == w_array[6]).all() + assert (result_array2[4] == w_array[7]).all() def test_w_is_selected_rows(self): places = [core.CPUPlace()] diff --git a/python/paddle/fluid/tests/unittests/test_lookup_table_op.py b/python/paddle/fluid/tests/unittests/test_lookup_table_op.py index ac25f432dffd544d4b336983ec868f2431a5b91a..4990ee898d81089735f6db4ee4ad6758944e311a 100644 --- a/python/paddle/fluid/tests/unittests/test_lookup_table_op.py +++ b/python/paddle/fluid/tests/unittests/test_lookup_table_op.py @@ -12,11 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest import paddle.fluid.core as core from paddle.fluid.op import Operator +import paddle.compat as cpt class TestLookupTableOp(OpTest): @@ -71,7 +74,7 @@ class TestLookupTableOpWithTensorIdsAndPadding(TestLookupTableOpWithTensorIds): flatten_idx = ids.flatten() padding_idx = np.random.choice(flatten_idx, 1)[0] self.outputs['Out'][np.squeeze(ids == padding_idx)] = np.zeros(31) - self.attrs = {'padding_idx': long(padding_idx)} + self.attrs = {'padding_idx': cpt.long_type(padding_idx)} self.check_output() def test_check_grad(self): diff --git a/python/paddle/fluid/tests/unittests/test_lrn_mkldnn_op.py b/python/paddle/fluid/tests/unittests/test_lrn_mkldnn_op.py index 966a16dc870c041b9deb140bed57d907cf305fd8..f6bb2ab7a696c40cb61dd5b38ca702b577fe7ea2 100644 --- a/python/paddle/fluid/tests/unittests/test_lrn_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/test_lrn_mkldnn_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest from test_lrn_op import TestLRNOp diff --git a/python/paddle/fluid/tests/unittests/test_lrn_op.py b/python/paddle/fluid/tests/unittests/test_lrn_op.py index eaff45cbb2a58798e9d55149510bec72eea370cd..bb91f26bbb53de454a6d037af4c9d96262866ce3 100644 --- a/python/paddle/fluid/tests/unittests/test_lrn_op.py +++ b/python/paddle/fluid/tests/unittests/test_lrn_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest @@ -34,7 +36,7 @@ class TestLRNOp(OpTest): return x + 1 def get_out(self): - start = -(self.n - 1) / 2 + start = -(self.n - 1) // 2 end = start + self.n mid = np.empty((self.N, self.C, self.H, self.W)).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/test_lstm_op.py b/python/paddle/fluid/tests/unittests/test_lstm_op.py index 705a24bd8f39a55e0a352944d961f8d33aaf96ff..76a24123fc7d51231bf24a3d1a6930186c94a5db 100644 --- a/python/paddle/fluid/tests/unittests/test_lstm_op.py +++ b/python/paddle/fluid/tests/unittests/test_lstm_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_lstm_unit_op.py b/python/paddle/fluid/tests/unittests/test_lstm_unit_op.py index e343265874f99afcd8201fa997932e2613fffc4c..eaa6b774c4d3e7add555c34f887e86dc847583b2 100644 --- a/python/paddle/fluid/tests/unittests/test_lstm_unit_op.py +++ b/python/paddle/fluid/tests/unittests/test_lstm_unit_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_lstmp_op.py b/python/paddle/fluid/tests/unittests/test_lstmp_op.py index ed2262da4bc727657c2e65d69cb1922891e17b09..9c3ec45515ffe0a07541fd9cfb7e92b079264071 100644 --- a/python/paddle/fluid/tests/unittests/test_lstmp_op.py +++ b/python/paddle/fluid/tests/unittests/test_lstmp_op.py @@ -11,6 +11,8 @@ #WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #See the License for the specific language governing permissions and #limitations under the License. + +from __future__ import print_function import unittest import numpy as np import test_lstm_op as LstmTest diff --git a/python/paddle/fluid/tests/unittests/test_margin_rank_loss_op.py b/python/paddle/fluid/tests/unittests/test_margin_rank_loss_op.py index 97c112487fd9193ab77d18945585a554e5fbcdf8..4a7e952436bd46c92c6256b4ec2d0652cfa38959 100644 --- a/python/paddle/fluid/tests/unittests/test_margin_rank_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_margin_rank_loss_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_math_op_patch.py b/python/paddle/fluid/tests/unittests/test_math_op_patch.py index 852a80261e02f5ed19e7fbe608d490be1f7798a9..b25d40a3a15e259878222ee5482cd842543b63d6 100644 --- a/python/paddle/fluid/tests/unittests/test_math_op_patch.py +++ b/python/paddle/fluid/tests/unittests/test_math_op_patch.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import decorators import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/unittests/test_matmul_op.py b/python/paddle/fluid/tests/unittests/test_matmul_op.py index cae2c8fa87d9857de8f26cf4962d9370eca66243..abf10437d83268a6a84a1c62399eb02cd3b1d663 100644 --- a/python/paddle/fluid/tests/unittests/test_matmul_op.py +++ b/python/paddle/fluid/tests/unittests/test_matmul_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_maxout_op.py b/python/paddle/fluid/tests/unittests/test_maxout_op.py index f5ddf72516bf8adb41698d9b2e22c7de74a3fad9..d588b22fe2607a6041359d420ebba757d8a632d6 100644 --- a/python/paddle/fluid/tests/unittests/test_maxout_op.py +++ b/python/paddle/fluid/tests/unittests/test_maxout_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest @@ -19,7 +21,7 @@ from op_test import OpTest def maxout_forward_naive(input, groups): s0, s1, s2, s3 = input.shape - return np.ndarray([s0, s1 / groups, groups, s2, s3], \ + return np.ndarray([s0, s1 // groups, groups, s2, s3], \ buffer = input, dtype=input.dtype).max(axis=(2)) diff --git a/python/paddle/fluid/tests/unittests/test_mean_iou.py b/python/paddle/fluid/tests/unittests/test_mean_iou.py index 32b4ee184787cd4cda0fd889f67a609141a3cb27..03e94483178e83adad9886cd7df2107581360dd1 100644 --- a/python/paddle/fluid/tests/unittests/test_mean_iou.py +++ b/python/paddle/fluid/tests/unittests/test_mean_iou.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from __future__ import division import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_mean_op.py b/python/paddle/fluid/tests/unittests/test_mean_op.py index 15472a8fc4716f218b0eddf17589634c565130b1..ff338f0e0037307e81a92eed804096c9a2a87361 100644 --- a/python/paddle/fluid/tests/unittests/test_mean_op.py +++ b/python/paddle/fluid/tests/unittests/test_mean_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_memory_usage.py b/python/paddle/fluid/tests/unittests/test_memory_usage.py index f9daf83652e18faab0ab31402b9f5889a0beceaf..4cdb5b5d9f7f020c4eb9a3b3a804c074d7ddbb35 100644 --- a/python/paddle/fluid/tests/unittests/test_memory_usage.py +++ b/python/paddle/fluid/tests/unittests/test_memory_usage.py @@ -34,7 +34,7 @@ def train_simulator(test_batch_size=10): sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.001) sgd_optimizer.minimize(avg_cost) - # Calculate memory usage in current network config + # Calculate memory usage in current network config lower_usage, upper_usage, unit = fluid.contrib.memory_usage( fluid.default_main_program(), batch_size=test_batch_size) diff --git a/python/paddle/fluid/tests/unittests/test_merge_ids_op.py b/python/paddle/fluid/tests/unittests/test_merge_ids_op.py index f209bdf30faffc0b2c7932b7b10f384d6d61a831..26ce7024117162e8bad403a9d8b8518c27578c83 100644 --- a/python/paddle/fluid/tests/unittests/test_merge_ids_op.py +++ b/python/paddle/fluid/tests/unittests/test_merge_ids_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_mine_hard_examples_op.py b/python/paddle/fluid/tests/unittests/test_mine_hard_examples_op.py index 54ee85c1a7a539fe9517f32adb35ab99b5ae2a07..4e5cc91268c5df4be3de3c04a82ef65b33cf4d20 100644 --- a/python/paddle/fluid/tests/unittests/test_mine_hard_examples_op.py +++ b/python/paddle/fluid/tests/unittests/test_mine_hard_examples_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import sys diff --git a/python/paddle/fluid/tests/unittests/test_minus_op.py b/python/paddle/fluid/tests/unittests/test_minus_op.py index ee32bd49925e266cfd5beb51496355e111a3d0d2..54253b17b967871b03628023c5a9fdb339af1828 100644 --- a/python/paddle/fluid/tests/unittests/test_minus_op.py +++ b/python/paddle/fluid/tests/unittests/test_minus_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_modified_huber_loss_op.py b/python/paddle/fluid/tests/unittests/test_modified_huber_loss_op.py index 62035efe8ec3809a7672e58697de4304426338d7..02fecfe47ec3fbff085b0a7f24316e5d0f6cd814 100644 --- a/python/paddle/fluid/tests/unittests/test_modified_huber_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_modified_huber_loss_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_momentum_op.py b/python/paddle/fluid/tests/unittests/test_momentum_op.py index c75d3bd276a5b494090c1aa1fea0bb4f2c067173..7137fd0fdb7c503492107da684b95989037eb872 100644 --- a/python/paddle/fluid/tests/unittests/test_momentum_op.py +++ b/python/paddle/fluid/tests/unittests/test_momentum_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_mul_op.py b/python/paddle/fluid/tests/unittests/test_mul_op.py index bbc782c1bce302df68ab30013f3a7667e51ed479..fca4ffa88b79ebfad009b436d440e86ddceaaed7 100644 --- a/python/paddle/fluid/tests/unittests/test_mul_op.py +++ b/python/paddle/fluid/tests/unittests/test_mul_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_multi_file_reader.py b/python/paddle/fluid/tests/unittests/test_multi_file_reader.py index cb0ea96ff69ce32b0bb1b49f0318c353aa08d388..09788868ccb926f56c2f622b5caf695670fd17f8 100644 --- a/python/paddle/fluid/tests/unittests/test_multi_file_reader.py +++ b/python/paddle/fluid/tests/unittests/test_multi_file_reader.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/unittests/test_multi_pass_reader.py b/python/paddle/fluid/tests/unittests/test_multi_pass_reader.py index 7fc9f550440d3d0e1a8182a69f5692b3df0aa258..4fae11e928dc7e066799a8936bada0e252afaa42 100644 --- a/python/paddle/fluid/tests/unittests/test_multi_pass_reader.py +++ b/python/paddle/fluid/tests/unittests/test_multi_pass_reader.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/unittests/test_multiclass_nms_op.py b/python/paddle/fluid/tests/unittests/test_multiclass_nms_op.py index 10cb78a08db0471699bcc0b7323d5346e3af64c7..df0562dcc79cbb960136c19d2b3f243cf2e09782 100644 --- a/python/paddle/fluid/tests/unittests/test_multiclass_nms_op.py +++ b/python/paddle/fluid/tests/unittests/test_multiclass_nms_op.py @@ -11,6 +11,8 @@ #WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #See the License for the specific language governing permissions and #limitations under the License. + +from __future__ import print_function import unittest import numpy as np import copy diff --git a/python/paddle/fluid/tests/unittests/test_multihead_attention.py b/python/paddle/fluid/tests/unittests/test_multihead_attention.py index 80c3c67967e970a7182c008b6cfd138aff044167..f60da862ac091ca1eefccfe2834201d1c79e2def 100644 --- a/python/paddle/fluid/tests/unittests/test_multihead_attention.py +++ b/python/paddle/fluid/tests/unittests/test_multihead_attention.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_multiplex_op.py b/python/paddle/fluid/tests/unittests/test_multiplex_op.py index 03cad8b43bac053f246156b8c57df9fceab20dcd..1567a74808aa37e5e18bbe583cc1d8987b31cd58 100644 --- a/python/paddle/fluid/tests/unittests/test_multiplex_op.py +++ b/python/paddle/fluid/tests/unittests/test_multiplex_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_nce.py b/python/paddle/fluid/tests/unittests/test_nce.py index 7431a142c53a64e58872390776904ce8f781d6a9..0745bd274f73715b6fdec236819b8d89827e1346 100644 --- a/python/paddle/fluid/tests/unittests/test_nce.py +++ b/python/paddle/fluid/tests/unittests/test_nce.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_network_with_dtype.py b/python/paddle/fluid/tests/unittests/test_network_with_dtype.py index d4835dd18405fc7a0d508a780a734922e0abd12c..60dcf195daf61d76a2e6d6f764fa216270804f55 100644 --- a/python/paddle/fluid/tests/unittests/test_network_with_dtype.py +++ b/python/paddle/fluid/tests/unittests/test_network_with_dtype.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_norm_op.py b/python/paddle/fluid/tests/unittests/test_norm_op.py index 108a665f37f5cd652ec83f784a56ca52e6b49fe8..22bc45ff1ea0efea0ec766a6a9e819cdd81b0866 100644 --- a/python/paddle/fluid/tests/unittests/test_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_norm_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_normalization_wrapper.py b/python/paddle/fluid/tests/unittests/test_normalization_wrapper.py index 198c68866d399023c51c2a43b588aa8ec49c3c9a..24fdcf8c88417244e981194e63bd77a2fdbd179d 100644 --- a/python/paddle/fluid/tests/unittests/test_normalization_wrapper.py +++ b/python/paddle/fluid/tests/unittests/test_normalization_wrapper.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_nvprof.py b/python/paddle/fluid/tests/unittests/test_nvprof.py index 226e5e5d1131b1f33cfbbfefec536e6974f85b36..da943d64da6cfc64d121b7373f7c067c1cff731c 100644 --- a/python/paddle/fluid/tests/unittests/test_nvprof.py +++ b/python/paddle/fluid/tests/unittests/test_nvprof.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import os import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_one_hot_op.py b/python/paddle/fluid/tests/unittests/test_one_hot_op.py index 06fccd39ac65ab62ee5618ac19d1a0535b481d06..7afdae804a65b9fb05a521a1b08ce0bfb21d721f 100644 --- a/python/paddle/fluid/tests/unittests/test_one_hot_op.py +++ b/python/paddle/fluid/tests/unittests/test_one_hot_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import math diff --git a/python/paddle/fluid/tests/unittests/test_op_support_gpu.py b/python/paddle/fluid/tests/unittests/test_op_support_gpu.py index 5fafb8280e19cca46e5bf687494c07200ca53153..e203fccd03f86077c51e176456c1c313ac14a9ee 100644 --- a/python/paddle/fluid/tests/unittests/test_op_support_gpu.py +++ b/python/paddle/fluid/tests/unittests/test_op_support_gpu.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_operator.py b/python/paddle/fluid/tests/unittests/test_operator.py index 5e418fe6ac2d62948762290a65686207d017275c..544fca8cecd0a2b94a5aec40b9442f86036fd4d2 100644 --- a/python/paddle/fluid/tests/unittests/test_operator.py +++ b/python/paddle/fluid/tests/unittests/test_operator.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.op as op diff --git a/python/paddle/fluid/tests/unittests/test_operator_desc.py b/python/paddle/fluid/tests/unittests/test_operator_desc.py index c098a5a0cb0364f9ec93c95c1ef50912e574b3d9..6d01955993324498de42462b7f85ef6f8e444505 100644 --- a/python/paddle/fluid/tests/unittests/test_operator_desc.py +++ b/python/paddle/fluid/tests/unittests/test_operator_desc.py @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.core as core +import paddle.compat as cpt from paddle.fluid.framework import Program, default_startup_program @@ -29,14 +32,15 @@ class TestOperator(unittest.TestCase): self.assertFail() except ValueError as v_err: self.assertEqual( - v_err.message, + cpt.get_exception_message(v_err), "`type` to initilized an Operator can not be None.") try: block.append_op(type="no_such_op") self.assertFail() except ValueError as a_err: - self.assertEqual(a_err.message, - "Operator \"no_such_op\" has not been registered.") + self.assertEqual( + cpt.get_exception_message(a_err), + "Operator \"no_such_op\" has not been registered.") def test_op_desc_creation(self): program = Program() diff --git a/python/paddle/fluid/tests/unittests/test_optimizer.py b/python/paddle/fluid/tests/unittests/test_optimizer.py index 18921d727f94a85b69259c07273f09c3e19390c6..4374d198f2f869afab5fb76fdcb43e3c445f7689 100644 --- a/python/paddle/fluid/tests/unittests/test_optimizer.py +++ b/python/paddle/fluid/tests/unittests/test_optimizer.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.framework as framework diff --git a/python/paddle/fluid/tests/unittests/test_pad_op.py b/python/paddle/fluid/tests/unittests/test_pad_op.py index 300f3ffcb8d1a6152b1e03f1356582c02bc4b2a3..58e56ca1a4dbdc48765a36e1a64b9a2ec8cf9025 100644 --- a/python/paddle/fluid/tests/unittests/test_pad_op.py +++ b/python/paddle/fluid/tests/unittests/test_pad_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py index d17e493c36a2ffcba632f5f85c7a1d2e5066dd1c..6d6917300cb66afcc8a0c509986a0f26be8b1f09 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.dataset.conll05 as conll05 import paddle.fluid as fluid import unittest diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_feed.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_feed.py index a43f2e7c49c02ce779344da44e640cabbf27986c..372ef748b2e704fd3858c382e048e51448ed3bd5 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_feed.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_feed.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.dataset.flowers as flowers import math import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_mnist.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_mnist.py index 9448d89cd58f4e5cff4bac49821fbc44c5a46246..893acd763fb21fc5040b8a24700eb947a9fe37c6 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_mnist.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_mnist.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from parallel_executor_test_base import TestParallelExecutorBase import paddle.fluid as fluid import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext.py index a28428d8dee201ba105e18684c15d4b4582d989f..cc2d692e18430eb48e6e800106eab0c3739d3f53 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid as fluid import paddle.fluid.layers.ops as ops from paddle.fluid.initializer import init_on_cpu @@ -46,7 +48,7 @@ def squeeze_excitation(input, num_channels, reduction_ratio): pool = fluid.layers.reduce_mean(input=reshape, dim=2) squeeze = fluid.layers.fc(input=pool, - size=num_channels / reduction_ratio, + size=num_channels // reduction_ratio, act='relu') excitation = fluid.layers.fc(input=squeeze, size=num_channels, @@ -62,7 +64,7 @@ def conv_bn_layer(input, num_filters, filter_size, stride=1, groups=1, num_filters=num_filters, filter_size=filter_size, stride=stride, - padding=(filter_size - 1) / 2, + padding=(filter_size - 1) // 2, groups=groups, act=None, bias_attr=False) diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_test_while_train.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_test_while_train.py index fcb5947ff05efd1c48ab9ec129ac9d17255d7020..f5a0ba624698b49e0d323e6f830be23a4148392b 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_test_while_train.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_test_while_train.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid as fluid import paddle.fluid.core as core import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_transformer.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_transformer.py index 8203d5d1fce0950130ab71db40fb306f73c41bd4..5ad922725a0b692e28552737a99b745ed09ddbd5 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_transformer.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_transformer.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid as fluid import transformer_model import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_parallel_op.py b/python/paddle/fluid/tests/unittests/test_parallel_op.py index c9617e36778740ce9620c3ad495c64c17277fde1..d7b9af8bac67ef89cc1ae59ccf002c2c488f3436 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_op.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/unittests/test_parameter.py b/python/paddle/fluid/tests/unittests/test_parameter.py index e09865074e8aa9345fd9cc84e1f19eaf0436142f..df42e6cb9a050b76099b4a53fdd08d2852284d1f 100644 --- a/python/paddle/fluid/tests/unittests/test_parameter.py +++ b/python/paddle/fluid/tests/unittests/test_parameter.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest from paddle.fluid.framework import default_main_program import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_polygon_box_transform.py b/python/paddle/fluid/tests/unittests/test_polygon_box_transform.py index 8aff4e87f67bc61a162f09e982cf0a7a61639257..dfedf8190f75ec26532f281338f076ca0c7d83af 100644 --- a/python/paddle/fluid/tests/unittests/test_polygon_box_transform.py +++ b/python/paddle/fluid/tests/unittests/test_polygon_box_transform.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_pool2d_mkldnn_op.py b/python/paddle/fluid/tests/unittests/test_pool2d_mkldnn_op.py index 003ebba18b26198427d9f313596ae85656ac24fa..14d7ed9057d622b136056e1b5bbbe57f9a04d5d7 100644 --- a/python/paddle/fluid/tests/unittests/test_pool2d_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/test_pool2d_mkldnn_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest from test_pool2d_op import TestPool2d_Op, TestCase1, TestCase2, TestCase3, TestCase4, TestCase5 diff --git a/python/paddle/fluid/tests/unittests/test_pool2d_op.py b/python/paddle/fluid/tests/unittests/test_pool2d_op.py index 1cf70311b40bc7648b7462e93f201aa33c77b137..26969bd5230afdac83a943d2dc21094a0972d60a 100644 --- a/python/paddle/fluid/tests/unittests/test_pool2d_op.py +++ b/python/paddle/fluid/tests/unittests/test_pool2d_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np @@ -29,11 +31,11 @@ def max_pool2D_forward_naive(x, if global_pool == 1: ksize = [H, W] H_out = (H - ksize[0] + 2 * paddings[0] + strides[0] - 1 - ) / strides[0] + 1 if ceil_mode else (H - ksize[0] + 2 * - paddings[0]) / strides[0] + 1 + ) // strides[0] + 1 if ceil_mode else ( + H - ksize[0] + 2 * paddings[0]) // strides[0] + 1 W_out = (W - ksize[1] + 2 * paddings[1] + strides[1] - 1 - ) / strides[1] + 1 if ceil_mode else (W - ksize[1] + 2 * - paddings[1]) / strides[1] + 1 + ) // strides[1] + 1 if ceil_mode else ( + W - ksize[1] + 2 * paddings[1]) // strides[1] + 1 out = np.zeros((N, C, H_out, W_out)) for i in range(H_out): for j in range(W_out): @@ -57,11 +59,11 @@ def avg_pool2D_forward_naive(x, if global_pool == 1: ksize = [H, W] H_out = (H - ksize[0] + 2 * paddings[0] + strides[0] - 1 - ) / strides[0] + 1 if ceil_mode else (H - ksize[0] + 2 * - paddings[0]) / strides[0] + 1 + ) // strides[0] + 1 if ceil_mode else ( + H - ksize[0] + 2 * paddings[0]) // strides[0] + 1 W_out = (W - ksize[1] + 2 * paddings[1] + strides[1] - 1 - ) / strides[1] + 1 if ceil_mode else (W - ksize[1] + 2 * - paddings[1]) / strides[1] + 1 + ) // strides[1] + 1 if ceil_mode else ( + W - ksize[1] + 2 * paddings[1]) // strides[1] + 1 out = np.zeros((N, C, H_out, W_out)) for i in range(H_out): for j in range(W_out): diff --git a/python/paddle/fluid/tests/unittests/test_pool3d_op.py b/python/paddle/fluid/tests/unittests/test_pool3d_op.py index 92c64b37921eafd4c90e247a235f2dacea8fea1e..77045c1307baead3711d58ed368dfa5f2acc3699 100644 --- a/python/paddle/fluid/tests/unittests/test_pool3d_op.py +++ b/python/paddle/fluid/tests/unittests/test_pool3d_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np @@ -29,14 +31,14 @@ def max_pool3D_forward_naive(x, if global_pool == 1: ksize = [D, H, W] D_out = (D - ksize[0] + 2 * paddings[0] + strides[0] - 1 - ) / strides[0] + 1 if ceil_mode else (H - ksize[0] + 2 * - paddings[0]) / strides[0] + 1 + ) // strides[0] + 1 if ceil_mode else ( + H - ksize[0] + 2 * paddings[0]) // strides[0] + 1 H_out = (H - ksize[1] + 2 * paddings[1] + strides[1] - 1 - ) / strides[1] + 1 if ceil_mode else (W - ksize[1] + 2 * - paddings[1]) / strides[1] + 1 + ) // strides[1] + 1 if ceil_mode else ( + W - ksize[1] + 2 * paddings[1]) // strides[1] + 1 W_out = (W - ksize[2] + 2 * paddings[2] + strides[2] - 1 - ) / strides[2] + 1 if ceil_mode else (W - ksize[2] + 2 * - paddings[2]) / strides[2] + 1 + ) // strides[2] + 1 if ceil_mode else ( + W - ksize[2] + 2 * paddings[2]) // strides[2] + 1 out = np.zeros((N, C, D_out, H_out, W_out)) for k in range(D_out): d_start = np.max((k * strides[0] - paddings[0], 0)) @@ -63,14 +65,14 @@ def avg_pool3D_forward_naive(x, if global_pool == 1: ksize = [D, H, W] D_out = (D - ksize[0] + 2 * paddings[0] + strides[0] - 1 - ) / strides[0] + 1 if ceil_mode else (H - ksize[0] + 2 * - paddings[0]) / strides[0] + 1 + ) // strides[0] + 1 if ceil_mode else ( + H - ksize[0] + 2 * paddings[0]) // strides[0] + 1 H_out = (H - ksize[1] + 2 * paddings[1] + strides[1] - 1 - ) / strides[1] + 1 if ceil_mode else (W - ksize[1] + 2 * - paddings[1]) / strides[1] + 1 + ) // strides[1] + 1 if ceil_mode else ( + W - ksize[1] + 2 * paddings[1]) // strides[1] + 1 W_out = (W - ksize[2] + 2 * paddings[2] + strides[2] - 1 - ) / strides[2] + 1 if ceil_mode else (W - ksize[2] + 2 * - paddings[2]) / strides[2] + 1 + ) // strides[2] + 1 if ceil_mode else ( + W - ksize[2] + 2 * paddings[2]) // strides[2] + 1 out = np.zeros((N, C, D_out, H_out, W_out)) for k in range(D_out): d_start = np.max((k * strides[0] - paddings[0], 0)) diff --git a/python/paddle/fluid/tests/unittests/test_pool_max_op.py b/python/paddle/fluid/tests/unittests/test_pool_max_op.py index e6a9f6f08cf1445c14494506641b0c3502591c37..488ff431d4f2ef76ce0c9486d8c307b4e01b5544 100644 --- a/python/paddle/fluid/tests/unittests/test_pool_max_op.py +++ b/python/paddle/fluid/tests/unittests/test_pool_max_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest @@ -24,9 +26,9 @@ def max_pool3D_forward_naive(x, ksize, strides, paddings, global_pool=False): ksize = [D, H, W] paddings = [0, 0, 0] - D_out = (D - ksize[0] + 2 * paddings[0]) / strides[0] + 1 - H_out = (H - ksize[1] + 2 * paddings[1]) / strides[1] + 1 - W_out = (W - ksize[2] + 2 * paddings[2]) / strides[2] + 1 + D_out = (D - ksize[0] + 2 * paddings[0]) // strides[0] + 1 + H_out = (H - ksize[1] + 2 * paddings[1]) // strides[1] + 1 + W_out = (W - ksize[2] + 2 * paddings[2]) // strides[2] + 1 out = np.zeros((N, C, D_out, H_out, W_out)) mask = np.zeros((N, C, D_out, H_out, W_out)) for k in range(D_out): @@ -63,8 +65,8 @@ def max_pool2D_forward_naive(x, ksize, strides, paddings, global_pool=False): ksize = [H, W] paddings = [0, 0] - H_out = (H - ksize[0] + 2 * paddings[0]) / strides[0] + 1 - W_out = (W - ksize[1] + 2 * paddings[1]) / strides[1] + 1 + H_out = (H - ksize[0] + 2 * paddings[0]) // strides[0] + 1 + W_out = (W - ksize[1] + 2 * paddings[1]) // strides[1] + 1 out = np.zeros((N, C, H_out, W_out)) mask = np.zeros((N, C, H_out, W_out)) for i in range(H_out): diff --git a/python/paddle/fluid/tests/unittests/test_positive_negative_pair_op.py b/python/paddle/fluid/tests/unittests/test_positive_negative_pair_op.py index 8c76393bdaccc0b701b409efebf08fac95aa5f1a..afe8d212d6ec218c3799780849c377e46a44bd6c 100644 --- a/python/paddle/fluid/tests/unittests/test_positive_negative_pair_op.py +++ b/python/paddle/fluid/tests/unittests/test_positive_negative_pair_op.py @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import itertools import numpy as np +import six from op_test import OpTest @@ -32,7 +35,7 @@ def py_pnpair_op(score, label, query, column=-1, weight=None): # accumulate statistics pos, neg, neu = 0, 0, 0 - for _, ranks in list(predictions.items()): + for _, ranks in six.iteritems(predictions): for e1, e2 in itertools.combinations(ranks, 2): s1, s2, l1, l2, w1, w2 = e1[0], e2[0], e1[1], e2[1], e1[2], e2[2] w = (w1 + w2) * 0.5 diff --git a/python/paddle/fluid/tests/unittests/test_precision_recall_op.py b/python/paddle/fluid/tests/unittests/test_precision_recall_op.py index 5ae425fee18b9b1baa0b945782268b79d6bb6625..645637625959f214db3875bc58e4c593c27ae8f6 100644 --- a/python/paddle/fluid/tests/unittests/test_precision_recall_op.py +++ b/python/paddle/fluid/tests/unittests/test_precision_recall_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_prelu_op.py b/python/paddle/fluid/tests/unittests/test_prelu_op.py index ae19a553bb826002c562c15ee07759391d51b4d8..979be5af3bdc24b1a2fc115198eeab53469a91c0 100644 --- a/python/paddle/fluid/tests/unittests/test_prelu_op.py +++ b/python/paddle/fluid/tests/unittests/test_prelu_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest @@ -20,30 +22,58 @@ from op_test import OpTest class PReluTest(OpTest): def setUp(self): self.op_type = "prelu" - x_np = np.random.normal(size=(10, 10)).astype("float32") - - for pos, val in np.ndenumerate(x_np): - # Since zero point in prelu is not differentiable, avoid randomize - # zero. - while abs(val) < 1e-3: - x_np[pos] = np.random.normal() - val = x_np[pos] - - x_np_sign = np.sign(x_np) - x_np = x_np_sign * np.maximum(x_np, .005) - alpha_np = np.array([.1], dtype="float32") - self.inputs = {'X': x_np, 'Alpha': alpha_np} + self.initTestCase() + x_np = np.random.normal(size=(3, 5, 5, 10)).astype("float32") + + # Since zero point in prelu is not differentiable, avoid randomize + # zero. + x_np[np.abs(x_np) < 0.005] = 0.02 + + if self.attrs == {'mode': "all"}: + alpha_np = np.random.rand(1).astype("float32") + self.inputs = {'X': x_np, 'Alpha': alpha_np} + elif self.attrs == {'mode': "channel"}: + alpha_np = np.random.rand(1, x_np.shape[1], 1, 1).astype("float32") + self.inputs = {'X': x_np, 'Alpha': alpha_np} + else: + alpha_np = np.random.rand(*x_np.shape).astype("float32") + self.inputs = {'X': x_np, 'Alpha': alpha_np} + out_np = np.maximum(self.inputs['X'], 0.) out_np = out_np + np.minimum(self.inputs['X'], 0.) * self.inputs['Alpha'] assert out_np is not self.inputs['X'] self.outputs = {'Out': out_np} + def initTestCase(self): + self.attrs = {'mode': "channel"} + def test_check_output(self): self.check_output() def test_check_grad(self): - self.check_grad(['X'], 'Out') + self.check_grad(['X', 'Alpha'], 'Out') + + def test_check_grad_ignore_x(self): + self.check_grad(['Alpha'], 'Out', no_grad_set=set('X')) + + def test_check_grad_ignore_alpha(self): + self.check_grad(['X'], 'Out', no_grad_set=set('Alpha')) + + +class TestCase1(PReluTest): + def initTestCase(self): + self.attrs = {'mode': "all"} + + +class TestCase2(PReluTest): + def initTestCase(self): + self.attrs = {'mode': "channel"} + + +class TestCase3(PReluTest): + def initTestCase(self): + self.attrs = {'mode': "element"} if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_preprocessor.py b/python/paddle/fluid/tests/unittests/test_preprocessor.py index cbf1a7e0c50a87cd43507ffdb94109873cf4e5d9..98e609b76982650c9d18f87c3c0637056cc40899 100644 --- a/python/paddle/fluid/tests/unittests/test_preprocessor.py +++ b/python/paddle/fluid/tests/unittests/test_preprocessor.py @@ -12,12 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np +import paddle import paddle.fluid as fluid -import paddle.v2 as paddle -import paddle.v2.dataset.mnist as mnist +import paddle.dataset.mnist as mnist class TestPreprocessor(unittest.TestCase): diff --git a/python/paddle/fluid/tests/unittests/test_print_op.py b/python/paddle/fluid/tests/unittests/test_print_op.py index b461c5c9401d74ef8dcf4afc84dc0ea6920a2419..ac682d6181cfcc5a064a51a736b03d493c37b780 100644 --- a/python/paddle/fluid/tests/unittests/test_print_op.py +++ b/python/paddle/fluid/tests/unittests/test_print_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.core as core from paddle.fluid.executor import Executor diff --git a/python/paddle/fluid/tests/unittests/test_prior_box_op.py b/python/paddle/fluid/tests/unittests/test_prior_box_op.py index e15554737b9f3fa36382dde15ded928271679538..7381b74af71051f8b993ba6d116b5282dd9b84e1 100644 --- a/python/paddle/fluid/tests/unittests/test_prior_box_op.py +++ b/python/paddle/fluid/tests/unittests/test_prior_box_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import sys diff --git a/python/paddle/fluid/tests/unittests/test_profiler.py b/python/paddle/fluid/tests/unittests/test_profiler.py index 9f8d33f9bbfc78b6f1a0c089b34b2f41d561c640..38a7c913bf07a32494089addf17068017d24d628 100644 --- a/python/paddle/fluid/tests/unittests/test_profiler.py +++ b/python/paddle/fluid/tests/unittests/test_profiler.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import os import numpy as np @@ -93,7 +95,7 @@ class TestProfiler(unittest.TestCase): "profiler is enabled only with GPU") def test_all_profiler(self): self.net_profiler('All', '/tmp/profile_out') - with open('/tmp/profile_out', 'r') as f: + with open('/tmp/profile_out', 'rb') as f: self.assertGreater(len(f.read()), 0) diff --git a/python/paddle/fluid/tests/unittests/test_protobuf.py b/python/paddle/fluid/tests/unittests/test_protobuf.py index c3f1fa80185bfc4afc3ed715d736bcba092629d8..7b80927c48d02e83a9bfaac572c81a6a95a69c8c 100644 --- a/python/paddle/fluid/tests/unittests/test_protobuf.py +++ b/python/paddle/fluid/tests/unittests/test_protobuf.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid.proto.framework_pb2 as framework_pb2 import unittest diff --git a/python/paddle/fluid/tests/unittests/test_protobuf_descs.py b/python/paddle/fluid/tests/unittests/test_protobuf_descs.py index 9853fb4e9a89944bfdf2954e3d3d86fef92ac93c..d24b5cbd06ddf9f332c1369ebd513bef27b77e14 100644 --- a/python/paddle/fluid/tests/unittests/test_protobuf_descs.py +++ b/python/paddle/fluid/tests/unittests/test_protobuf_descs.py @@ -12,8 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.core as core +import paddle.compat as cpt from paddle.fluid.framework import Program @@ -108,7 +111,7 @@ class TestVarDesc(unittest.TestCase): def test_shape(self): program_desc = core.ProgramDesc() block = program_desc.block(0) - var = block.var('my_var') + var = block.var(cpt.to_bytes('my_var')) var.set_type(core.VarDesc.VarType.SELECTED_ROWS) src_shape = [3, 2, 10, 8] var.set_shape(src_shape) @@ -119,7 +122,7 @@ class TestVarDesc(unittest.TestCase): def test_multiple_shape(self): program_desc = core.ProgramDesc() block = program_desc.block(0) - var = block.var('my_reader') + var = block.var(cpt.to_bytes('my_reader')) var.set_type(core.VarDesc.VarType.READER) src_shapes = [[2, 3, 3], [4, 5], [6, 7, 8, 9]] var.set_shapes(src_shapes) @@ -130,7 +133,7 @@ class TestVarDesc(unittest.TestCase): def test_dtype(self): program_desc = core.ProgramDesc() block = program_desc.block(0) - var = block.var('my_var') + var = block.var(cpt.to_bytes('my_var')) var.set_type(core.VarDesc.VarType.LOD_TENSOR) var.set_dtype(core.VarDesc.VarType.INT32) self.assertEqual(core.VarDesc.VarType.INT32, var.dtype()) @@ -139,7 +142,7 @@ class TestVarDesc(unittest.TestCase): def test_multiple_dtype(self): program_desc = core.ProgramDesc() block = program_desc.block(0) - var = block.var('my_reader') + var = block.var(cpt.to_bytes('my_reader')) var.set_type(core.VarDesc.VarType.READER) src_types = [ core.VarDesc.VarType.INT32, core.VarDesc.VarType.FP64, @@ -152,7 +155,7 @@ class TestVarDesc(unittest.TestCase): def test_multiple_lod_level(self): program_desc = core.ProgramDesc() block = program_desc.block(0) - var = block.var('my_reader') + var = block.var(cpt.to_bytes('my_reader')) var.set_type(core.VarDesc.VarType.READER) src_types = [3, 1, 2] var.set_lod_levels(src_types) @@ -166,12 +169,12 @@ class TestBlockDesc(unittest.TestCase): self.assertIsNotNone(program_desc) block = program_desc.block(0) self.assertIsNotNone(block) - var1 = block.var("var1") - var2 = block.var("var2") - var3 = block.var("var3") + var1 = block.var(cpt.to_bytes("var1")) + var2 = block.var(cpt.to_bytes("var2")) + var3 = block.var(cpt.to_bytes("var3")) all_vars = block.all_vars() self.assertEqual(set(all_vars), {var1, var2, var3}) - var2_re = block.find_var("var2") + var2_re = block.find_var(cpt.to_bytes("var2")) self.assertEqual(var2_re, var2) def test_add_op(self): diff --git a/python/paddle/fluid/tests/unittests/test_proximal_adagrad_op.py b/python/paddle/fluid/tests/unittests/test_proximal_adagrad_op.py index 3c2689585061af5a11a247a01b87b432dcd86e13..57e96f1fa34fa94f5e095d088016655f24b58d0c 100644 --- a/python/paddle/fluid/tests/unittests/test_proximal_adagrad_op.py +++ b/python/paddle/fluid/tests/unittests/test_proximal_adagrad_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_proximal_gd_op.py b/python/paddle/fluid/tests/unittests/test_proximal_gd_op.py index 137594b9a08e13bf6c3f3779356c209596f9ba8e..067502baecc73cc84a6aa8ab78a9afbcc191c49a 100644 --- a/python/paddle/fluid/tests/unittests/test_proximal_gd_op.py +++ b/python/paddle/fluid/tests/unittests/test_proximal_gd_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_py_reader_push_pop.py b/python/paddle/fluid/tests/unittests/test_py_reader_push_pop.py index f9bda5e4701f693f41fe7041ba0f5ec80b6fc31c..3efe5aac8848b8230f42f4f3905eefc517c0fa5e 100644 --- a/python/paddle/fluid/tests/unittests/test_py_reader_push_pop.py +++ b/python/paddle/fluid/tests/unittests/test_py_reader_push_pop.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_py_reader_using_executor.py b/python/paddle/fluid/tests/unittests/test_py_reader_using_executor.py index 9a379bdbaa7e278879117a8cdc2dddb335a10ca1..931cac409f26fce4ecca18c4b0cfcca2e675046f 100644 --- a/python/paddle/fluid/tests/unittests/test_py_reader_using_executor.py +++ b/python/paddle/fluid/tests/unittests/test_py_reader_using_executor.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_random_crop_op.py b/python/paddle/fluid/tests/unittests/test_random_crop_op.py index 1c708d0386da4028f1f3d177d0a3fd494c077c6e..f29dddff7a28ed041908741007361224624e436a 100644 --- a/python/paddle/fluid/tests/unittests/test_random_crop_op.py +++ b/python/paddle/fluid/tests/unittests/test_random_crop_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import paddle.fluid.core as core @@ -21,11 +23,12 @@ from op_test import OpTest class TestRandomCropOp(OpTest): def setUp(self): to_crop = np.array([[[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]] * - 5).astype("float32") + 5).astype(np.int32) self.possible_res = [ - np.array([[1, 2, 3], [5, 6, 7]]), np.array([[2, 3, 4], [6, 7, 8]]), - np.array([[5, 6, 7], [9, 10, 11]]), - np.array([[6, 7, 8], [10, 11, 12]]) + np.array([[1, 2, 3], [5, 6, 7]]).astype(np.int32), + np.array([[2, 3, 4], [6, 7, 8]]).astype(np.int32), + np.array([[5, 6, 7], [9, 10, 11]]).astype(np.int32), + np.array([[6, 7, 8], [10, 11, 12]]).astype(np.int32) ] self.op_type = "random_crop" self.inputs = {'X': to_crop, 'Seed': np.array([10])} diff --git a/python/paddle/fluid/tests/unittests/test_rank_loss_op.py b/python/paddle/fluid/tests/unittests/test_rank_loss_op.py index 7eba1e2077e25325d537f01f043ed1afa372800c..c9fa24b103deb50aa896403e09b11e891fb62c6d 100644 --- a/python/paddle/fluid/tests/unittests/test_rank_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_rank_loss_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_reader_reset.py b/python/paddle/fluid/tests/unittests/test_reader_reset.py index 3ad85d57485956e0cadb197dadd172516fa15c39..8ad11d76f683d556f05cafc3251acc942efef72f 100644 --- a/python/paddle/fluid/tests/unittests/test_reader_reset.py +++ b/python/paddle/fluid/tests/unittests/test_reader_reset.py @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid as fluid -import paddle.v2 as paddle +import paddle import numpy as np import unittest diff --git a/python/paddle/fluid/tests/unittests/test_recordio_reader.py b/python/paddle/fluid/tests/unittests/test_recordio_reader.py index 69a522e273db017ac55b408276b4a28f5f907c42..c5210bb2085bc386df43cd0d20292d7b308a1093 100644 --- a/python/paddle/fluid/tests/unittests/test_recordio_reader.py +++ b/python/paddle/fluid/tests/unittests/test_recordio_reader.py @@ -12,11 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid -import paddle.v2 as paddle -import paddle.v2.dataset.mnist as mnist +import paddle +import paddle.dataset.mnist as mnist class TestRecordIO(unittest.TestCase): diff --git a/python/paddle/fluid/tests/unittests/test_recurrent_op.py b/python/paddle/fluid/tests/unittests/test_recurrent_op.py index 2e22df2beba9d74e28788fb72f6f7f7f2bef534e..6dfc85e301a2eda66bade09a8b6dd0004155f385 100644 --- a/python/paddle/fluid/tests/unittests/test_recurrent_op.py +++ b/python/paddle/fluid/tests/unittests/test_recurrent_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.layers as layers diff --git a/python/paddle/fluid/tests/unittests/test_reduce_op.py b/python/paddle/fluid/tests/unittests/test_reduce_op.py index 06d116601bf733986ccf9c725340456ab1258be2..328f0f0011381b77cccb8b2d9b266aa53b259473 100644 --- a/python/paddle/fluid/tests/unittests/test_reduce_op.py +++ b/python/paddle/fluid/tests/unittests/test_reduce_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_registry.py b/python/paddle/fluid/tests/unittests/test_registry.py index a361c4624e3e2efa817e8137ff31133997a0a1fb..7381bb61eb4630cb67bc306fde211704e9580af4 100644 --- a/python/paddle/fluid/tests/unittests/test_registry.py +++ b/python/paddle/fluid/tests/unittests/test_registry.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import unittest import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/unittests/test_regularizer.py b/python/paddle/fluid/tests/unittests/test_regularizer.py index 9b1c4ceada52322b3f1fdc4ab2e90a2c089ee67e..6727335c6059161d235a64a1b90d36b84004f9b3 100644 --- a/python/paddle/fluid/tests/unittests/test_regularizer.py +++ b/python/paddle/fluid/tests/unittests/test_regularizer.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.framework as framework diff --git a/python/paddle/fluid/tests/unittests/test_reorder_lod_tensor.py b/python/paddle/fluid/tests/unittests/test_reorder_lod_tensor.py index 6e1cd56b3e309fc014dc981a1e3aa841159fca15..28c8c4699adbc108c05e4a500815752e2ec24c61 100644 --- a/python/paddle/fluid/tests/unittests/test_reorder_lod_tensor.py +++ b/python/paddle/fluid/tests/unittests/test_reorder_lod_tensor.py @@ -12,11 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid import paddle.fluid.core as core from paddle.fluid.layers.control_flow import lod_rank_table import numpy +import functools class TestReorderLoDTensor(unittest.TestCase): @@ -101,7 +104,8 @@ class TestReorderLoDTensor(unittest.TestCase): rank_table = [] # list of (index, length) for i in range(len(ref_lod)): rank_table.append((i, ref_lod[i])) - rank_table = sorted(rank_table, lambda x, y: y[1] - x[1]) + rank_table = sorted( + rank_table, key=functools.cmp_to_key(lambda x, y: y[1] - x[1])) # compute the input sequence info according to input_lod input_value, input_lod = self.data[self.data_desc[0][0]] diff --git a/python/paddle/fluid/tests/unittests/test_reshape_op.py b/python/paddle/fluid/tests/unittests/test_reshape_op.py index 2f5558578ac2a002a83c2a7e027ec5a96d8b4414..1de35dc35b0176b77eb2d9b25cd6ee4e645e56c3 100644 --- a/python/paddle/fluid/tests/unittests/test_reshape_op.py +++ b/python/paddle/fluid/tests/unittests/test_reshape_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_reverse_op.py b/python/paddle/fluid/tests/unittests/test_reverse_op.py index f845575a02869f08299d76b5600074598ca27f6c..e83f548c228c7c045ff795e882738ea56e3f2d24 100644 --- a/python/paddle/fluid/tests/unittests/test_reverse_op.py +++ b/python/paddle/fluid/tests/unittests/test_reverse_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_rmsprop_op.py b/python/paddle/fluid/tests/unittests/test_rmsprop_op.py index 0d84a5853ead45b84de9383dd8749992d2f91440..3d4623c74d9a307b12ab6d72ad0b4d2dae938720 100644 --- a/python/paddle/fluid/tests/unittests/test_rmsprop_op.py +++ b/python/paddle/fluid/tests/unittests/test_rmsprop_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_rnn_memory_helper_op.py b/python/paddle/fluid/tests/unittests/test_rnn_memory_helper_op.py index 178606f05961263df5ef0398064a1fd135fbe784..9bfec8e9bdd8c4667fb19f3dd479b759d6dd665b 100644 --- a/python/paddle/fluid/tests/unittests/test_rnn_memory_helper_op.py +++ b/python/paddle/fluid/tests/unittests/test_rnn_memory_helper_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest from paddle.fluid.framework import Program diff --git a/python/paddle/fluid/tests/unittests/test_roi_pool_op.py b/python/paddle/fluid/tests/unittests/test_roi_pool_op.py index df5684ab173a4889dd7b693f9246bafd12e0345f..ed7f467835f32242a9650f226b4a5ad9d6d87af4 100644 --- a/python/paddle/fluid/tests/unittests/test_roi_pool_op.py +++ b/python/paddle/fluid/tests/unittests/test_roi_pool_op.py @@ -12,10 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import math import sys +import paddle.compat as cpt from op_test import OpTest @@ -59,10 +62,10 @@ class TestROIPoolOp(OpTest): for i in range(self.rois_num): roi = self.rois[i] roi_batch_id = roi[0] - roi_start_w = int(round(roi[1] * self.spatial_scale)) - roi_start_h = int(round(roi[2] * self.spatial_scale)) - roi_end_w = int(round(roi[3] * self.spatial_scale)) - roi_end_h = int(round(roi[4] * self.spatial_scale)) + roi_start_w = int(cpt.round(roi[1] * self.spatial_scale)) + roi_start_h = int(cpt.round(roi[2] * self.spatial_scale)) + roi_end_w = int(cpt.round(roi[3] * self.spatial_scale)) + roi_end_h = int(cpt.round(roi[4] * self.spatial_scale)) roi_height = int(max(roi_end_h - roi_start_h + 1, 1)) roi_width = int(max(roi_end_w - roi_start_w + 1, 1)) @@ -97,8 +100,8 @@ class TestROIPoolOp(OpTest): for w in range(wstart, wend): if x_i[c, h, w] > out_data[i, c, ph, pw]: out_data[i, c, ph, pw] = x_i[c, h, w] - argmax_data[i, c, ph, pw] = h * \ - self.width + w + argmax_data[i, c, ph, + pw] = h * self.width + w self.outs = out_data.astype('float32') self.argmaxes = argmax_data.astype('int64') @@ -110,14 +113,14 @@ class TestROIPoolOp(OpTest): self.rois_lod[0].append(bno + 1) for i in range(bno + 1): x1 = np.random.random_integers( - 0, self.width / self.spatial_scale - self.pooled_width) + 0, self.width // self.spatial_scale - self.pooled_width) y1 = np.random.random_integers( - 0, self.height / self.spatial_scale - self.pooled_height) + 0, self.height // self.spatial_scale - self.pooled_height) x2 = np.random.random_integers(x1 + self.pooled_width, - self.width / self.spatial_scale) - y2 = np.random.random_integers(y1 + self.pooled_height, - self.height / self.spatial_scale) + self.width // self.spatial_scale) + y2 = np.random.random_integers( + y1 + self.pooled_height, self.height // self.spatial_scale) roi = [bno, x1, y1, x2, y2] rois.append(roi) diff --git a/python/paddle/fluid/tests/unittests/test_row_conv_op.py b/python/paddle/fluid/tests/unittests/test_row_conv_op.py index 07dcd108689ae6069e30fe22029258d192215549..2f13f067ef313685227c7de9a49fae8640ca6b32 100644 --- a/python/paddle/fluid/tests/unittests/test_row_conv_op.py +++ b/python/paddle/fluid/tests/unittests/test_row_conv_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_rpn_target_assign_op.py b/python/paddle/fluid/tests/unittests/test_rpn_target_assign_op.py index df6e0faaca6fd007b39a8f358d964055e149a025..08c462d9036cacab81dab7c9ea16664c9159479f 100644 --- a/python/paddle/fluid/tests/unittests/test_rpn_target_assign_op.py +++ b/python/paddle/fluid/tests/unittests/test_rpn_target_assign_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_sampling_id_op.py b/python/paddle/fluid/tests/unittests/test_sampling_id_op.py new file mode 100644 index 0000000000000000000000000000000000000000..708265b4576809b1f4157d54989c6138c6e5a2b0 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_sampling_id_op.py @@ -0,0 +1,51 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import numpy as np +from op_test import OpTest + +import paddle.fluid.core as core +from paddle.fluid.op import Operator + + +class TestSamplingIdOp(OpTest): + def setUp(self): + self.op_type = "sampling_id" + self.use_mkldnn = False + self.init_kernel_type() + self.X = np.random.random((8, 4)).astype('float32') + self.inputs = {"X": self.X} + self.Y = np.random.random(8).astype('float32') + self.outputs = {'Out': self.Y} + self.attrs = {'max': 1.0, 'min': 0.0, 'seed': 1} + + def test_check_output(self): + self.check_output_customized(self.verify_output) + y1 = self.out + self.check_output_customized(self.verify_output) + y2 = self.out + self.assertTrue(np.array_equal(y1, y2)) + self.assertEqual(len(y1), len(self.Y)) + + def verify_output(self, outs): + out = np.array(outs[0]) + self.out = out + + def init_kernel_type(self): + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_scale_op.py b/python/paddle/fluid/tests/unittests/test_scale_op.py index 53f59c399054a96f5b5f07a390e6fa9eeae878ce..0a8a43253d79ba21c7333dd19af05d8adf410289 100644 --- a/python/paddle/fluid/tests/unittests/test_scale_op.py +++ b/python/paddle/fluid/tests/unittests/test_scale_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_scatter_op.py b/python/paddle/fluid/tests/unittests/test_scatter_op.py index fb1728743630b3ea908ae835444eff7fd71b72c8..088996f9d7dee1ea914e36e3342c9a5110001c44 100644 --- a/python/paddle/fluid/tests/unittests/test_scatter_op.py +++ b/python/paddle/fluid/tests/unittests/test_scatter_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_scope.py b/python/paddle/fluid/tests/unittests/test_scope.py index d249a989a9499d01f6ed10d6cdbc6c456a7262c5..45fcbfba6eb7c6fc4e75f6d8228d721c0186ef36 100644 --- a/python/paddle/fluid/tests/unittests/test_scope.py +++ b/python/paddle/fluid/tests/unittests/test_scope.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid.core import unittest diff --git a/python/paddle/fluid/tests/unittests/test_selected_rows.py b/python/paddle/fluid/tests/unittests/test_selected_rows.py index f504a06ffff8cb636498652554fca05e22bb905d..2f34f79b8eafad8e7fdf6b359548747f354b141f 100644 --- a/python/paddle/fluid/tests/unittests/test_selected_rows.py +++ b/python/paddle/fluid/tests/unittests/test_selected_rows.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid.core as core import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_seq_concat_op.py b/python/paddle/fluid/tests/unittests/test_seq_concat_op.py index 11ffa761a690eb1f9f6dc50c45128a99301741db..9d1d139721ad7ee3e29d44c9b3e7c666b78a4556 100644 --- a/python/paddle/fluid/tests/unittests/test_seq_concat_op.py +++ b/python/paddle/fluid/tests/unittests/test_seq_concat_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import sys diff --git a/python/paddle/fluid/tests/unittests/test_seq_conv.py b/python/paddle/fluid/tests/unittests/test_seq_conv.py index 1a6e1aad799e77b8e746353bee93680691939d24..dcc86382e5286f354c4f2e81ead598f12c75b2c1 100644 --- a/python/paddle/fluid/tests/unittests/test_seq_conv.py +++ b/python/paddle/fluid/tests/unittests/test_seq_conv.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import random diff --git a/python/paddle/fluid/tests/unittests/test_seq_pool.py b/python/paddle/fluid/tests/unittests/test_seq_pool.py index 0b3659d7a67956f7546d368346bd102eeedf1d97..66e77714c5d65d51262f76519901032182985ea8 100644 --- a/python/paddle/fluid/tests/unittests/test_seq_pool.py +++ b/python/paddle/fluid/tests/unittests/test_seq_pool.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_sequence_erase_op.py b/python/paddle/fluid/tests/unittests/test_sequence_erase_op.py index 8f0765277ae85af2b17ad96d4fd0c1148c393ff0..92cd5b0cbcd1ab56300158d26850969870e86f2b 100644 --- a/python/paddle/fluid/tests/unittests/test_sequence_erase_op.py +++ b/python/paddle/fluid/tests/unittests/test_sequence_erase_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_sequence_expand.py b/python/paddle/fluid/tests/unittests/test_sequence_expand.py index 5ff0dab23e516ae8114b8264492fb2a9d5c0b3f8..ffd4026dbade2f8f7eace399c52ae0428f3e8d7b 100644 --- a/python/paddle/fluid/tests/unittests/test_sequence_expand.py +++ b/python/paddle/fluid/tests/unittests/test_sequence_expand.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_sequence_reshape.py b/python/paddle/fluid/tests/unittests/test_sequence_reshape.py index 39b02ecf6ddb40737c4c1737d652c1a1b744d923..f11fa6c39c35efc14f8600dd746ab64cc940cd71 100644 --- a/python/paddle/fluid/tests/unittests/test_sequence_reshape.py +++ b/python/paddle/fluid/tests/unittests/test_sequence_reshape.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import math diff --git a/python/paddle/fluid/tests/unittests/test_sequence_slice_op.py b/python/paddle/fluid/tests/unittests/test_sequence_slice_op.py index 313e485d1e3080f2c59c68256cbc5c81aa6558cd..1561490087330c9af3ea3e384bf735eaa268a749 100644 --- a/python/paddle/fluid/tests/unittests/test_sequence_slice_op.py +++ b/python/paddle/fluid/tests/unittests/test_sequence_slice_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import sys diff --git a/python/paddle/fluid/tests/unittests/test_sequence_softmax_op.py b/python/paddle/fluid/tests/unittests/test_sequence_softmax_op.py index c4fc8b74cf80c3596b0af9f7f0434864591195bd..3e00e7d95f63ea652ea1964eb792f9393ffa5994 100644 --- a/python/paddle/fluid/tests/unittests/test_sequence_softmax_op.py +++ b/python/paddle/fluid/tests/unittests/test_sequence_softmax_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_sgd_op.py b/python/paddle/fluid/tests/unittests/test_sgd_op.py index 3126293f9d8e52daa866be5fc1533648a33f3363..b46e4bfb86bd5dc9c74375693328f2506281be3e 100644 --- a/python/paddle/fluid/tests/unittests/test_sgd_op.py +++ b/python/paddle/fluid/tests/unittests/test_sgd_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import paddle.fluid.core as core @@ -124,6 +126,7 @@ class TestSGDOpOptimizeSelectedRows(unittest.TestCase): w_selected_rows = scope.var('Param').get_selected_rows() w_selected_rows.set_height(len(param_rows)) w_selected_rows.set_rows(param_rows) + w_selected_rows.sync_index() w_array = np.ones((len(param_rows), row_width)).astype("float32") for i in range(len(param_rows)): w_array[i] *= i diff --git a/python/paddle/fluid/tests/unittests/test_shape_op.py b/python/paddle/fluid/tests/unittests/test_shape_op.py index a62ee050075cb8c9f8817c142825a89c24bdfedf..02231ea943e1e92a08730e6e9f1aa3cefeb927c0 100644 --- a/python/paddle/fluid/tests/unittests/test_shape_op.py +++ b/python/paddle/fluid/tests/unittests/test_shape_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_shrink_rnn_memory.py b/python/paddle/fluid/tests/unittests/test_shrink_rnn_memory.py index a994bf181a74ca71a970da0105fe767f82750a6c..97f79f9421d498723da4c7992551f1210d3f6003 100644 --- a/python/paddle/fluid/tests/unittests/test_shrink_rnn_memory.py +++ b/python/paddle/fluid/tests/unittests/test_shrink_rnn_memory.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.core as core from paddle.fluid.executor import Executor diff --git a/python/paddle/fluid/tests/unittests/test_sigmoid_cross_entropy_with_logits_op.py b/python/paddle/fluid/tests/unittests/test_sigmoid_cross_entropy_with_logits_op.py index c435796569cd2479c19d70a849991f439bf5292a..97ff203499c0bf223930c904de46e1abdd902799 100644 --- a/python/paddle/fluid/tests/unittests/test_sigmoid_cross_entropy_with_logits_op.py +++ b/python/paddle/fluid/tests/unittests/test_sigmoid_cross_entropy_with_logits_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np from op_test import OpTest from scipy.special import logit diff --git a/python/paddle/fluid/tests/unittests/test_sign_op.py b/python/paddle/fluid/tests/unittests/test_sign_op.py index 087a0c575bfa6bc18cb229ad274b4e1e90210605..85a9d9cae47c2b0942da0e0d962d4512af1566c0 100644 --- a/python/paddle/fluid/tests/unittests/test_sign_op.py +++ b/python/paddle/fluid/tests/unittests/test_sign_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_slice_op.py b/python/paddle/fluid/tests/unittests/test_slice_op.py index 1a48bce3bb7c74551a365fd471f6869b128babac..134df38eea6655857db04dfdc19dd7f7897946f4 100644 --- a/python/paddle/fluid/tests/unittests/test_slice_op.py +++ b/python/paddle/fluid/tests/unittests/test_slice_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_slice_var.py b/python/paddle/fluid/tests/unittests/test_slice_var.py index 82305b23a1a1e2cee8cef6b291d848581fe5b509..fab63b7d5631829feffd26fc1dce2bd338d2036b 100644 --- a/python/paddle/fluid/tests/unittests/test_slice_var.py +++ b/python/paddle/fluid/tests/unittests/test_slice_var.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import math import unittest from paddle.fluid.transpiler.distribute_transpiler import slice_variable diff --git a/python/paddle/fluid/tests/unittests/test_smooth_l1_loss_op.py b/python/paddle/fluid/tests/unittests/test_smooth_l1_loss_op.py index e74664dac4d87c5a7e1e061294e93e8267e3cc17..8ab6833821c75262124b3ae4200a17e457b718d5 100644 --- a/python/paddle/fluid/tests/unittests/test_smooth_l1_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_smooth_l1_loss_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_softmax_op.py b/python/paddle/fluid/tests/unittests/test_softmax_op.py index 70ad05597c4a160cf6a25aeb3c379320cef69c63..d88aa1ae1c9d848eba7a2224d22b5201fc27b857 100644 --- a/python/paddle/fluid/tests/unittests/test_softmax_op.py +++ b/python/paddle/fluid/tests/unittests/test_softmax_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_softmax_with_cross_entropy_op.py b/python/paddle/fluid/tests/unittests/test_softmax_with_cross_entropy_op.py index c0d9fc8f22a7c4f791d80a9cad87d003b5d54299..b7e5ff6d52ad7dde3dd94b3bd660cfca383e1ada 100644 --- a/python/paddle/fluid/tests/unittests/test_softmax_with_cross_entropy_op.py +++ b/python/paddle/fluid/tests/unittests/test_softmax_with_cross_entropy_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_split_and_merge_lod_tensor_op.py b/python/paddle/fluid/tests/unittests/test_split_and_merge_lod_tensor_op.py index ea1146166d34a31efbd859318b411cea50895fe1..5397d5c52158ccfb9ad5703b957ca59d6fa11418 100644 --- a/python/paddle/fluid/tests/unittests/test_split_and_merge_lod_tensor_op.py +++ b/python/paddle/fluid/tests/unittests/test_split_and_merge_lod_tensor_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.core as core import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_split_ids_op.py b/python/paddle/fluid/tests/unittests/test_split_ids_op.py index ca7861309839d183e18c168403881a0b1b5bf309..4c3d0258980fd8595704a65219deb520b96e222e 100644 --- a/python/paddle/fluid/tests/unittests/test_split_ids_op.py +++ b/python/paddle/fluid/tests/unittests/test_split_ids_op.py @@ -12,8 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np +import six from op_test import OpTest import paddle.fluid.core as core from paddle.fluid.op import Operator @@ -59,7 +62,7 @@ class TestSpliteIds(unittest.TestCase): x_tensor = x.get_tensor() x_tensor.set(np_array, place) - outs_name = ["out%d" % i for i in xrange(3)] + outs_name = ["out%d" % i for i in six.moves.xrange(3)] outs = [ scope.var(var_name).get_selected_rows() for var_name in outs_name ] diff --git a/python/paddle/fluid/tests/unittests/test_split_op.py b/python/paddle/fluid/tests/unittests/test_split_op.py index 6b67a52e81b978ed78c72629f9177759f8e2c4e1..3c5dd782f85235c4a2feb5a8ca6d048a012c5e1c 100644 --- a/python/paddle/fluid/tests/unittests/test_split_op.py +++ b/python/paddle/fluid/tests/unittests/test_split_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_split_selected_rows_op.py b/python/paddle/fluid/tests/unittests/test_split_selected_rows_op.py index 2b261820e04b08234477fc0a9adde95262f99bba..41a5ee59ea523b1f6c5015974a12c526e883fa35 100644 --- a/python/paddle/fluid/tests/unittests/test_split_selected_rows_op.py +++ b/python/paddle/fluid/tests/unittests/test_split_selected_rows_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.core as core import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_spp_op.py b/python/paddle/fluid/tests/unittests/test_spp_op.py index 3cbfc2a703f1c4a24674d468cd1152bfa6eb8ad2..a6c2cccd39c9cecb2ae904a1930b44ba18dbbd7e 100644 --- a/python/paddle/fluid/tests/unittests/test_spp_op.py +++ b/python/paddle/fluid/tests/unittests/test_spp_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_squared_l2_distance_op.py b/python/paddle/fluid/tests/unittests/test_squared_l2_distance_op.py index 78bc300ebec1cd34e44343d47376fef05a6d0135..a8bc1004d9bbe91e323db49c0cf0b576f8da306e 100644 --- a/python/paddle/fluid/tests/unittests/test_squared_l2_distance_op.py +++ b/python/paddle/fluid/tests/unittests/test_squared_l2_distance_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_squared_l2_norm_op.py b/python/paddle/fluid/tests/unittests/test_squared_l2_norm_op.py index 609445d52287f421e67a5796f9e50c1fb42c8e49..439bae9510ee84b131050bb6804a3ede2ad6a8b3 100644 --- a/python/paddle/fluid/tests/unittests/test_squared_l2_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_squared_l2_norm_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import unittest from numpy import linalg as LA diff --git a/python/paddle/fluid/tests/unittests/test_squeeze_op.py b/python/paddle/fluid/tests/unittests/test_squeeze_op.py index bca6af2fd5dfadbc48cf1a76cfa6ffd4f1fdfdef..a2a5584459a5d4dc416b9c542a4bb0567982e765 100644 --- a/python/paddle/fluid/tests/unittests/test_squeeze_op.py +++ b/python/paddle/fluid/tests/unittests/test_squeeze_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_sum_mkldnn_op.py b/python/paddle/fluid/tests/unittests/test_sum_mkldnn_op.py index 7956897d68a3fb49d62ba696d0b6400b4f909989..55820f31b81df9f3618d1004f6d21565564efa29 100644 --- a/python/paddle/fluid/tests/unittests/test_sum_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/test_sum_mkldnn_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest from test_sum_op import TestSumOp diff --git a/python/paddle/fluid/tests/unittests/test_sum_op.py b/python/paddle/fluid/tests/unittests/test_sum_op.py index 1d90414e137a70e6265042e24e106fe565802778..9dc93048e631801acd3b21a3bb140ecc4fcd7732 100644 --- a/python/paddle/fluid/tests/unittests/test_sum_op.py +++ b/python/paddle/fluid/tests/unittests/test_sum_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_switch.py b/python/paddle/fluid/tests/unittests/test_switch.py index 528c5cce4bc7262ade196f6a81a57a57089117ec..2a9c07a889ba5fe24fd1c098729a233cb8fbb16f 100644 --- a/python/paddle/fluid/tests/unittests/test_switch.py +++ b/python/paddle/fluid/tests/unittests/test_switch.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_target_assign_op.py b/python/paddle/fluid/tests/unittests/test_target_assign_op.py index bd208897520122b6a5dcf71da325b1b9dba632f6..aec219f80639415a9be55ba18e7940953d0e11b0 100644 --- a/python/paddle/fluid/tests/unittests/test_target_assign_op.py +++ b/python/paddle/fluid/tests/unittests/test_target_assign_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import random diff --git a/python/paddle/fluid/tests/unittests/test_tensor.py b/python/paddle/fluid/tests/unittests/test_tensor.py index 5ccc876ae8e6e20f76c77c1892f4de59d72bffc8..e9d0f8a0193c77da33a8cf128dbf8a1c5087782b 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor.py +++ b/python/paddle/fluid/tests/unittests/test_tensor.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid.core as core import unittest import numpy diff --git a/python/paddle/fluid/tests/unittests/test_top_k_op.py b/python/paddle/fluid/tests/unittests/test_top_k_op.py index cbc3da550306b9febe8a8fd22e7f71efa572a3d0..e54e170f7f1e03db4b63db72edb7395d18130f68 100644 --- a/python/paddle/fluid/tests/unittests/test_top_k_op.py +++ b/python/paddle/fluid/tests/unittests/test_top_k_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_transpose_op.py b/python/paddle/fluid/tests/unittests/test_transpose_op.py index ebd63fbd495354eafe298ad5cc3456a196538a6a..0853f80b82030679d140f7fabdd42557c2374599 100644 --- a/python/paddle/fluid/tests/unittests/test_transpose_op.py +++ b/python/paddle/fluid/tests/unittests/test_transpose_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_uniform_random_batch_size_like_op.py b/python/paddle/fluid/tests/unittests/test_uniform_random_batch_size_like_op.py index e033e86114f17d37c01480fe8350648eb8aa27cb..7b8be24d9da8c15eeb52c0ba207ea780b03254f8 100644 --- a/python/paddle/fluid/tests/unittests/test_uniform_random_batch_size_like_op.py +++ b/python/paddle/fluid/tests/unittests/test_uniform_random_batch_size_like_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_uniform_random_op.py b/python/paddle/fluid/tests/unittests/test_uniform_random_op.py index 346a949b6e7c96b5535f5e65ddbada11e110a0a7..d6a5d68765c53d9d711add64c86575a0db6997e4 100644 --- a/python/paddle/fluid/tests/unittests/test_uniform_random_op.py +++ b/python/paddle/fluid/tests/unittests/test_uniform_random_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_unique_name.py b/python/paddle/fluid/tests/unittests/test_unique_name.py index 49ef335618ca7ca1e8249a61a97ca552dabdb9e8..b8c751b2e9b5a905d9de40fc5f78a02c6ca5e034 100644 --- a/python/paddle/fluid/tests/unittests/test_unique_name.py +++ b/python/paddle/fluid/tests/unittests/test_unique_name.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/unittests/test_unpool_op.py b/python/paddle/fluid/tests/unittests/test_unpool_op.py index ecce4cdde2d648fe7d65427e34c77f5f9ad61417..b0c7c3c8662e217f4e88245f22f6b50e7a48c8b7 100644 --- a/python/paddle/fluid/tests/unittests/test_unpool_op.py +++ b/python/paddle/fluid/tests/unittests/test_unpool_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest @@ -27,7 +29,7 @@ def unpool2dmax_forward_naive(input, indices, ksize, strides, paddings): for h in range(s2): for w in range(s3): index = indices[nidx, cidx, h, w] - hidx = (index - index % out_wsize) / out_wsize + hidx = (index - index % out_wsize) // out_wsize widx = index % out_wsize out[nidx, cidx, int(hidx), int(widx)] = \ input[nidx, cidx, h, w] @@ -41,9 +43,9 @@ class TestUnpoolOp(OpTest): self.init_test_case() pre_input = np.random.random(self.shape).astype("float32") nsize, csize, hsize, wsize = pre_input.shape - hsize_out = (hsize - self.ksize[0] + 2 * self.paddings[0]) / \ + hsize_out = (hsize - self.ksize[0] + 2 * self.paddings[0]) // \ self.strides[0] + 1 - wsize_out = (wsize - self.ksize[1] + 2 * self.paddings[1]) / \ + wsize_out = (wsize - self.ksize[1] + 2 * self.paddings[1]) // \ self.strides[1] + 1 input = np.zeros((nsize, csize, hsize_out, wsize_out)) indices = np.zeros((nsize, csize, hsize_out, wsize_out)) @@ -62,7 +64,7 @@ class TestUnpoolOp(OpTest): input[nidx, cidx, i, j] = x_masked.max() arg = x_masked.argmax() indices[nidx, cidx, i, j] = \ - (r_start + arg / self.ksize[1]) * wsize + \ + (r_start + arg // self.ksize[1]) * wsize + \ c_start + arg % self.ksize[1] output = self.unpool2d_forward_naive(input, indices, self.ksize, \ self.strides, self.paddings).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/test_unsqueeze_op.py b/python/paddle/fluid/tests/unittests/test_unsqueeze_op.py index 7a4aa0a40b5eb494f6027e800ca6b466bbe1c302..5fcabe4c83457a91ff0e9cd5568904698353b62a 100644 --- a/python/paddle/fluid/tests/unittests/test_unsqueeze_op.py +++ b/python/paddle/fluid/tests/unittests/test_unsqueeze_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_variable.py b/python/paddle/fluid/tests/unittests/test_variable.py index 49784e21c461bacadd404bf4a8640ebc4dcb26ca..b0830e130dd9a9037f8dd900a256eea3d05f64b8 100644 --- a/python/paddle/fluid/tests/unittests/test_variable.py +++ b/python/paddle/fluid/tests/unittests/test_variable.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest from paddle.fluid.framework import default_main_program, Program, convert_np_dtype_to_dtype_ import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_version.py b/python/paddle/fluid/tests/unittests/test_version.py index a09c8a759b9461edcf7d5ddbd62d74408d5f292e..42a0e5c802c53ed0e6aad38fb9ab0f64122e87f5 100644 --- a/python/paddle/fluid/tests/unittests/test_version.py +++ b/python/paddle/fluid/tests/unittests/test_version.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import re diff --git a/python/paddle/fluid/tests/unittests/test_warpctc_op.py b/python/paddle/fluid/tests/unittests/test_warpctc_op.py index 9f1aaee472f918da7deb8816a0a4654dafe74a30..5e3aa13546d0c4fdcde4a3d6378d5a1748327814 100644 --- a/python/paddle/fluid/tests/unittests/test_warpctc_op.py +++ b/python/paddle/fluid/tests/unittests/test_warpctc_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import sys import unittest import numpy as np @@ -132,7 +134,7 @@ class CTCForward(object): for k in range(end - start): j = k + start if j & 1 == 1: - label_idx = j / 2 + label_idx = j // 2 label_val = labels_a_sequence[label_idx, 0] fv = self.log_add(forward_vars[i - 1, j], forward_vars[i - 1, j - 1]) diff --git a/python/paddle/fluid/tests/unittests/test_weight_normalization.py b/python/paddle/fluid/tests/unittests/test_weight_normalization.py index 436f9b9f86fb86270e47c8e30c5c0701787ca0f1..e990d8b2498f6a1b62f7a34d329e3ca72a962728 100644 --- a/python/paddle/fluid/tests/unittests/test_weight_normalization.py +++ b/python/paddle/fluid/tests/unittests/test_weight_normalization.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy import collections diff --git a/python/paddle/fluid/tests/unittests/test_while_op.py b/python/paddle/fluid/tests/unittests/test_while_op.py index 790e6afe5f02236b00d9c67b7b25a881e07abace..b75373cf24a7344bf59b3c6fcb9c4c3969be6503 100644 --- a/python/paddle/fluid/tests/unittests/test_while_op.py +++ b/python/paddle/fluid/tests/unittests/test_while_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.layers as layers from paddle.fluid.executor import Executor diff --git a/python/paddle/fluid/tests/unittests/testsuite.py b/python/paddle/fluid/tests/unittests/testsuite.py index c6e176ca31c57e623addd9594be81c0abdce489b..31ae25f02c6fab0d68e83a131ace054d4eb56c12 100644 --- a/python/paddle/fluid/tests/unittests/testsuite.py +++ b/python/paddle/fluid/tests/unittests/testsuite.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/transformer_model.py b/python/paddle/fluid/tests/unittests/transformer_model.py index 868a0248be6833d0e8fed8a26549352562c279c1..f0e74aff6bdfa7d9f0a7f10e64cac4de88009f0a 100644 --- a/python/paddle/fluid/tests/unittests/transformer_model.py +++ b/python/paddle/fluid/tests/unittests/transformer_model.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from functools import partial import numpy as np diff --git a/python/paddle/fluid/trainer.py b/python/paddle/fluid/trainer.py index eed9b49ef40b591d5b6481846dab714423f57990..294308f1877360174c69cc59b0d2037e494985e7 100644 --- a/python/paddle/fluid/trainer.py +++ b/python/paddle/fluid/trainer.py @@ -12,10 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import contextlib import os import errno import shutil +import six import time from . import core @@ -618,7 +621,7 @@ def build_feed_var_list(program, feed_order): "The values of 'feed_order' should be a permutation of [0, len(feed_order))" ) sorted_pair_list = sorted( - list(feed_order.items()), key=lambda item: item[1]) + six.iteritems(feed_order), key=lambda item: item[1]) feed_var_list = [ program.global_block().var(pair[0]) for pair in sorted_pair_list ] @@ -1036,7 +1039,7 @@ def _save_trainer_args(dirname, trainer_id, trainer_args): cur_dir = _get_trainer_dir(dirname, trainer_id) - for name, value in list(trainer_args.items()): + for name, value in six.iteritems(trainer_args): args_file = os.path.join(cur_dir, name) with open(args_file, 'w') as f: f.write(str(value)) diff --git a/python/paddle/fluid/transpiler/__init__.py b/python/paddle/fluid/transpiler/__init__.py index a8622ad54433fff40f68520955f0294e2955577e..8429e2fd7c5141f064c66d8f406889bca1510fe2 100644 --- a/python/paddle/fluid/transpiler/__init__.py +++ b/python/paddle/fluid/transpiler/__init__.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from .distribute_transpiler import DistributeTranspiler, DistributeTranspilerConfig from .inference_transpiler import InferenceTranspiler from .memory_optimization_transpiler import memory_optimize, release_memory diff --git a/python/paddle/fluid/transpiler/details/__init__.py b/python/paddle/fluid/transpiler/details/__init__.py index 1bfab1f219f8a2f08a0fb5c0042d87a3ad707dd5..5e98266a761c7e01bd6668e85e6adeb54103ca80 100644 --- a/python/paddle/fluid/transpiler/details/__init__.py +++ b/python/paddle/fluid/transpiler/details/__init__.py @@ -12,5 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from .program_utils import * from .ufind import * diff --git a/python/paddle/fluid/transpiler/details/program_utils.py b/python/paddle/fluid/transpiler/details/program_utils.py index 76d10777f5f9ed6d27d55a640108bd036d8d8bac..640dbf4bbed58edf746456419af18c75241fa03c 100644 --- a/python/paddle/fluid/transpiler/details/program_utils.py +++ b/python/paddle/fluid/transpiler/details/program_utils.py @@ -12,12 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + +import six + def delete_ops(block, ops): try: start = list(block.ops).index(ops[0]) end = list(block.ops).index(ops[-1]) - [block._remove_op(start) for _ in range(end - start + 1)] + [block._remove_op(start) for _ in six.moves.range(end - start + 1)] except Exception as e: raise e block.program._sync_with_cpp() diff --git a/python/paddle/fluid/transpiler/details/ufind.py b/python/paddle/fluid/transpiler/details/ufind.py index 0e30d0e3f9c5712c494daf17b2b4bcec86f69c23..aa63af7dcf7ac85031fb00ca4c39fb36d7e588b8 100644 --- a/python/paddle/fluid/transpiler/details/ufind.py +++ b/python/paddle/fluid/transpiler/details/ufind.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + class UnionFind(object): """ Union-find data structure. diff --git a/python/paddle/fluid/transpiler/distribute_transpiler.py b/python/paddle/fluid/transpiler/distribute_transpiler.py index 9cc57eae9bed4c138cfadaef20c6c1b9039367bd..1c72ecd1d96525f70c4adf5e88888864cb85a476 100644 --- a/python/paddle/fluid/transpiler/distribute_transpiler.py +++ b/python/paddle/fluid/transpiler/distribute_transpiler.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function """ Steps to transpile trainer: 1. split variable to multiple blocks, aligned by product(dim[1:]) (width). @@ -31,6 +33,7 @@ Steps to transpile pserver: import math import random import numpy as np +import collections from .ps_dispatcher import RoundRobin, HashName, PSDispatcher from .. import core, framework @@ -195,6 +198,9 @@ class DistributeTranspiler(object): if program is None: program = default_main_program() self.origin_program = program + self.origin_startup_program = default_startup_program().clone() + + self.startup_program = default_startup_program() self.trainer_num = trainers self.sync_mode = sync_mode self.trainer_id = trainer_id @@ -212,9 +218,10 @@ class DistributeTranspiler(object): self.origin_program._distributed_lookup_table = self.table_name if self.table_name else None # split and create vars, then put splited vars in dicts for later use. + # step 1: split and create vars, then put splited vars in dicts for later use. self._init_splited_vars() - # step 3.1: insert send op to send gradient vars to parameter servers + # step 2: insert send op to send gradient vars to parameter servers ps_dispatcher.reset() send_vars = [] @@ -223,9 +230,10 @@ class DistributeTranspiler(object): # fc_w@GRAD_trainer_0, fc_w@GRAD_trainer_1 --> pserver1 # fc_b@GRAD_trainer_0, fc_b@GRAD_trainer_1 --> pserver2 # shuffle the map will avoid the uneven distribution above - grad_var_mapping_items = list(self.grad_var_mapping.items()) + grad_var_mapping_items = list(six.iteritems(self.grad_var_mapping)) + if not self.config.slice_var_up: - random.seed(self.trainer_num) + random.seed(self.origin_program.random_seed) random.shuffle(grad_var_mapping_items) for orig_varname, splited_vars in grad_var_mapping_items: @@ -271,7 +279,7 @@ class DistributeTranspiler(object): RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE }) - # step 3.2: insert recv op to receive parameters from parameter server + # step 3: insert recv op to receive parameters from parameter server recv_vars = [] for _, var in enumerate(send_vars): recv_vars.append(self.grad_param_mapping[var]) @@ -283,7 +291,7 @@ class DistributeTranspiler(object): self.param_grad_ep_mapping[ep]["grads"].append(send_vars[i]) # step4: Concat the parameters splits together after recv. - for varname, splited_var in list(self.param_var_mapping.items()): + for varname, splited_var in six.iteritems(self.param_var_mapping): eps = [] for var in splited_var: index = [v.name for v in recv_vars].index(var.name) @@ -308,7 +316,7 @@ class DistributeTranspiler(object): RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE }) - for varname, splited_var in list(self.param_var_mapping.items()): + for varname, splited_var in six.iteritems(self.param_var_mapping): if len(splited_var) <= 1: continue orig_param = program.global_block().vars[varname] @@ -318,6 +326,8 @@ class DistributeTranspiler(object): outputs={"Out": [orig_param]}, attrs={"axis": 0}) + self._get_trainer_startup_program(recv_vars=recv_vars, eplist=eplist) + if self.has_distributed_lookup_table: self._replace_lookup_table_op_with_prefetch(program, pserver_endpoints) @@ -334,8 +344,78 @@ class DistributeTranspiler(object): # FIXME(typhoonzero): Also ops like clip_gradient, lrn_decay? delete_ops(self.origin_program.global_block(), self.optimize_ops) self.origin_program.__str__() + return self.origin_program + def _get_trainer_startup_program(self, + recv_vars, + eplist, + startup_program=None): + """ + Get transpiled trainer side startup program. + + Args: + startup_program(Program): Startup program. + + Returns: + Program: trainer side startup program. + """ + if startup_program is None: + startup_program = self.startup_program + + # FIXME(gongwb): delete not need ops. + # note that: some parameter is not trainable and those ops can't be deleted. + + for varname, splited_var in self.param_var_mapping.iteritems(): + # Get the eplist of recv vars + eps = [] + for var in splited_var: + index = [v.name for v in recv_vars].index(var.name) + eps.append(eplist[index]) + + for var in splited_var: + if startup_program.global_block().has_var(var.name): + continue + + startup_program.global_block().create_var( + name=var.name, + persistable=False, + type=var.type, + dtype=var.dtype, + shape=var.shape, + lod_level=var.lod_level) + + op = startup_program.global_block().append_op( + type="recv", + inputs={}, + outputs={"Out": splited_var}, + attrs={ + "epmap": eps, + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE + }) + + startup_program.global_block().append_op( + type="fetch_barrier", + inputs={}, + outputs={}, + attrs={ + "endpoints": self.pserver_endpoints, + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE + }) + + for varname, splited_var in self.param_var_mapping.iteritems(): + #add concat ops to merge splited parameters received from parameter servers. + if len(splited_var) <= 1: + continue + orig_param = startup_program.global_block().vars[varname] + startup_program.global_block().append_op( + type="concat", + inputs={"X": splited_var}, + outputs={"Out": [orig_param]}, + attrs={"axis": 0}) + + return startup_program + def get_pserver_program(self, endpoint): """ Get parameter server side program. @@ -542,7 +622,10 @@ class DistributeTranspiler(object): pserver_program._sync_with_cpp() return pserver_program - def get_startup_program(self, endpoint, pserver_program): + def get_startup_program(self, + endpoint, + pserver_program, + startup_program=None): """ Get startup program for current parameter server. Modify operator input variables if there are variables that @@ -552,12 +635,17 @@ class DistributeTranspiler(object): endpoint (str): current pserver endpoint. pserver_program (Program): call get_pserver_program first and pass the result here. + startup_program (Program): if pass None, will use + default_startup_program Returns: Program: parameter server side startup program. """ s_prog = Program() - orig_s_prog = default_startup_program() + if not startup_program: + orig_s_prog = default_startup_program() + else: + orig_s_prog = startup_program s_prog.random_seed = orig_s_prog.random_seed params = self.param_grad_ep_mapping[endpoint]["params"] @@ -570,24 +658,26 @@ class DistributeTranspiler(object): # 1. create vars in pserver program to startup program pserver_vars = pserver_program.global_block().vars - created_var_map = dict() - for _, var in list(pserver_vars.items()): + created_var_map = collections.OrderedDict() + for _, var in six.iteritems(pserver_vars): tmpvar = s_prog.global_block()._clone_variable(var) created_var_map[var.name] = tmpvar # 2. rename op outputs for op in orig_s_prog.global_block().ops: - new_outputs = dict() + new_outputs = collections.OrderedDict() # do not append startup op if var is not on this pserver op_on_pserver = False - for key in op.output_names: - newname, _ = _get_splited_name_and_shape(op.output(key)[0]) - if newname: - op_on_pserver = True - new_outputs[key] = created_var_map[newname] - elif op.output(key)[0] in pserver_vars: - op_on_pserver = True - new_outputs[key] = pserver_vars[op.output(key)[0]] + # TODO(gongwb): remove this line. + if op.type not in ["recv", "fetch_barrier", "concat"]: + for key in op.output_names: + newname, _ = _get_splited_name_and_shape(op.output(key)[0]) + if newname: + op_on_pserver = True + new_outputs[key] = created_var_map[newname] + elif op.output(key)[0] in pserver_vars: + op_on_pserver = True + new_outputs[key] = pserver_vars[op.output(key)[0]] if op_on_pserver: # most startup program ops have no inputs @@ -743,7 +833,7 @@ class DistributeTranspiler(object): self.origin_program, grad_blocks, add_trainer_suffix=self.trainer_num > 1) - self.grad_param_mapping = dict() + self.grad_param_mapping = collections.OrderedDict() for g, p in zip(grad_blocks, param_blocks): g_name, g_bid, _ = g.split(":") p_name, p_bid, _ = p.split(":") @@ -751,7 +841,7 @@ class DistributeTranspiler(object): self.param_var_mapping[p_name][int(p_bid)] # create mapping of endpoint -> split var to create pserver side program - self.param_grad_ep_mapping = dict() + self.param_grad_ep_mapping = collections.OrderedDict() [ self.param_grad_ep_mapping.update({ ep: { @@ -916,9 +1006,15 @@ class DistributeTranspiler(object): # create table param and grad var in pserver program origin_param_var = self.origin_program.global_block().vars[ self.table_name] + + zero_dim = int( + math.ceil(origin_param_var.shape[0] / len(self.pserver_endpoints))) + table_shape = list(origin_param_var.shape) + table_shape[0] = zero_dim + param_var = pserver_program.global_block().create_var( name=origin_param_var.name, - shape=origin_param_var.shape, + shape=table_shape, dtype=origin_param_var.dtype, type=core.VarDesc.VarType.SELECTED_ROWS, persistable=True) @@ -1020,21 +1116,21 @@ class DistributeTranspiler(object): block_list (list[(varname, block_id, block_size)]): List of gradient blocks. add_trainer_suffix (Bool): Add trainer suffix to new variable's name if set True. Returns: - var_mapping (dict(varname->[new_varname_variable])):A dict mapping + var_mapping (collections.OrderedDict(varname->[new_varname_variable])):A dict mapping from original var name to each var split. """ # varname->[(block_id, current_block_size)] - block_map = dict() + block_map = collections.OrderedDict() - var_mapping = dict() + var_mapping = collections.OrderedDict() for block_str in block_list: varname, offset, size = block_str.split(":") if varname not in block_map: block_map[varname] = [] block_map[varname].append((int(offset), int(size))) - for varname, splited in list(block_map.items()): + for varname, splited in six.iteritems(block_map): orig_var = program.global_block().var(varname) if len(splited) == 1: if self.sync_mode and add_trainer_suffix: @@ -1047,7 +1143,6 @@ class DistributeTranspiler(object): var_mapping[varname] = \ [program.global_block().var(orig_var.name)] continue - var_mapping[varname] = [] orig_shape = orig_var.shape orig_dim1_flatten = 1 @@ -1056,7 +1151,7 @@ class DistributeTranspiler(object): for i, block in enumerate(splited): size = block[1] - rows = size / orig_dim1_flatten + rows = size // orig_dim1_flatten splited_shape = [rows] if len(orig_shape) >= 2: splited_shape.extend(orig_shape[1:]) @@ -1220,7 +1315,7 @@ class DistributeTranspiler(object): grad_to_block_id, origin_program, merged_var): program = optimize_block.program pserver_block = program.global_block() - new_inputs = dict() + new_inputs = collections.OrderedDict() # update param/grad shape first, then other inputs like # moment can use the updated shape @@ -1306,9 +1401,7 @@ class DistributeTranspiler(object): def _is_splited_grad_var(self, var, var_dict): grad_block = None - # TODO(minqiyang): replace these items() with six.iteritems() to - # improve memory - for _, g in list(var_dict.items()): + for _, g in six.iteritems(var_dict): if self._orig_varname(g.name) == self._orig_varname(var.name): if g.name.find(".trainer_") == -1: grad_block = g @@ -1318,7 +1411,7 @@ class DistributeTranspiler(object): def _clone_lr_op(self, program, block, op): inputs = self._get_input_map_from_op( self.origin_program.global_block().vars, op) - for key, varlist in list(inputs.items()): + for key, varlist in six.iteritems(inputs): if not isinstance(varlist, list): varlist = [varlist] for var in varlist: @@ -1327,7 +1420,7 @@ class DistributeTranspiler(object): outputs = self._get_output_map_from_op( self.origin_program.global_block().vars, op) - for key, varlist in list(outputs.items()): + for key, varlist in six.iteritems(outputs): if not isinstance(varlist, list): varlist = [varlist] for var in varlist: @@ -1342,7 +1435,7 @@ class DistributeTranspiler(object): # Append the ops for parameters that do not need to be optimized/updated inputs = self._get_input_map_from_op( self.origin_program.global_block().vars, opt_op) - for key, varlist in list(inputs.items()): + for key, varlist in six.iteritems(inputs): if not isinstance(varlist, list): varlist = [varlist] for var in varlist: @@ -1361,7 +1454,7 @@ class DistributeTranspiler(object): outputs = self._get_output_map_from_op( self.origin_program.global_block().vars, opt_op) - for key, varlist in list(outputs.items()): + for key, varlist in six.iteritems(outputs): if not isinstance(varlist, list): varlist = [varlist] for var in varlist: @@ -1419,7 +1512,7 @@ class DistributeTranspiler(object): def _get_input_map_from_op(self, varmap, op): """Returns a dict from op input name to the vars in varmap.""" - iomap = dict() + iomap = collections.OrderedDict() for key in op.input_names: vars = [] for varname in op.input(key): @@ -1432,7 +1525,7 @@ class DistributeTranspiler(object): def _get_output_map_from_op(self, varmap, op): """Returns a dict from op output name to the vars in varmap.""" - iomap = dict() + iomap = collections.OrderedDict() for key in op.output_names: vars = [] for varname in op.output(key): diff --git a/python/paddle/fluid/transpiler/inference_transpiler.py b/python/paddle/fluid/transpiler/inference_transpiler.py index 87f20bbccf3138585841952efacef5b0a3cbbace..42005839c44c1b3e8b910e9a3932181468659cae 100644 --- a/python/paddle/fluid/transpiler/inference_transpiler.py +++ b/python/paddle/fluid/transpiler/inference_transpiler.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import os import numpy as np from .. import core diff --git a/python/paddle/fluid/transpiler/memory_optimization_transpiler.py b/python/paddle/fluid/transpiler/memory_optimization_transpiler.py index 20ba7ed2b0b9df0d0432727ee1f69f61533c402e..3e58e125de4188144646236f7999c620cd8e9459 100644 --- a/python/paddle/fluid/transpiler/memory_optimization_transpiler.py +++ b/python/paddle/fluid/transpiler/memory_optimization_transpiler.py @@ -12,8 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from collections import defaultdict from .. import core +from ... import compat as cpt from ..framework import Program, default_main_program, Parameter from ..backward import _rename_arg_ from functools import reduce @@ -125,15 +128,15 @@ class ControlFlowGraph(object): def _has_var(self, block_desc, var_name, is_forward): if is_forward: - return block_desc.has_var(str(var_name)) + return block_desc.has_var(cpt.to_bytes(var_name)) else: - return block_desc.has_var_recursive(str(var_name)) + return block_desc.has_var_recursive(cpt.to_bytes(var_name)) def _find_var(self, block_desc, var_name, is_forward): if is_forward: - return block_desc.find_var(str(var_name)) + return block_desc.find_var(cpt.to_bytes(var_name)) else: - return block_desc.find_var_recursive(str(var_name)) + return block_desc.find_var_recursive(cpt.to_bytes(var_name)) def _check_var_validity(self, block_desc, x, is_forward): if str(x) == "@EMPTY@": @@ -258,7 +261,7 @@ class ControlFlowGraph(object): # Rename the var to the cache var already with # memory allocated in order to reuse the memory. _rename_arg_(self._ops, x, cache_var, begin_idx=i) - self._program.block(block_desc.id).var(str( + self._program.block(block_desc.id).var(cpt.to_text( x)).desc = self._find_var(block_desc, cache_var, is_forward) self._update_graph(x, cache_var, begin_idx=i) diff --git a/python/paddle/fluid/transpiler/ps_dispatcher.py b/python/paddle/fluid/transpiler/ps_dispatcher.py index dcffadd531719431f27feb464ed58a65c04770ee..6a6d14a69ba771e192a28951a6df7027741a655a 100644 --- a/python/paddle/fluid/transpiler/ps_dispatcher.py +++ b/python/paddle/fluid/transpiler/ps_dispatcher.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + class PSDispatcher(object): """ diff --git a/python/paddle/fluid/unique_name.py b/python/paddle/fluid/unique_name.py index b125eba4f83c588fa2fa81a357604a7d8592ea80..b9957a699e597898bee75ce0e7283f7224293f0c 100644 --- a/python/paddle/fluid/unique_name.py +++ b/python/paddle/fluid/unique_name.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import collections import contextlib import six diff --git a/python/paddle/reader/decorator.py b/python/paddle/reader/decorator.py index ce410e61b92e7d3f32fa5bfeb415e4b6c5fa9df6..6d7ac876fdf65fe0e85cceb94c311a93d9ea39c2 100644 --- a/python/paddle/reader/decorator.py +++ b/python/paddle/reader/decorator.py @@ -27,6 +27,7 @@ from six.moves import zip import itertools import random import zlib +import paddle.compat as cpt def map_readers(func, *readers): @@ -390,9 +391,9 @@ class PipeReader: buff = self.process.stdout.read(self.bufsize) if buff: if self.file_type == "gzip": - decomp_buff = self.dec.decompress(buff) + decomp_buff = cpt.to_text(self.dec.decompress(buff)) elif self.file_type == "plain": - decomp_buff = buff + decomp_buff = cpt.to_text(buff) else: raise TypeError("file_type %s is not allowed" % self.file_type) diff --git a/python/paddle/reader/tests/creator_test.py b/python/paddle/reader/tests/creator_test.py index c4238c12a74759d52eb09f31ce1126cc93dd3489..d7107610a5dd751cad8f8365aec32c6ba92c53ae 100644 --- a/python/paddle/reader/tests/creator_test.py +++ b/python/paddle/reader/tests/creator_test.py @@ -29,6 +29,7 @@ import os import unittest import numpy as np import paddle.reader.creator +import six class TestNumpyArray(unittest.TestCase): @@ -37,7 +38,7 @@ class TestNumpyArray(unittest.TestCase): x = np.array(l, np.int32) reader = paddle.reader.creator.np_array(x) for idx, e in enumerate(reader()): - self.assertItemsEqual(e, l[idx]) + six.assertCountEqual(self, e, l[idx]) class TestTextFile(unittest.TestCase): diff --git a/python/paddle/utils/dump_config.py b/python/paddle/utils/dump_config.py index d27af7f76246a4c9db9a43c17715506d82031b9c..6a96a0a78fc77c50904ee7822c725c41e646c5e6 100644 --- a/python/paddle/utils/dump_config.py +++ b/python/paddle/utils/dump_config.py @@ -37,9 +37,9 @@ if __name__ == '__main__': assert isinstance(conf, TrainerConfig_pb2.TrainerConfig) if whole_conf: - print conf + print(conf) else: if binary: sys.stdout.write(conf.model_config.SerializeToString()) else: - print conf.model_config + print(conf.model_config) diff --git a/python/paddle/utils/image_multiproc.py b/python/paddle/utils/image_multiproc.py index 3e3e519f76d388eeb477f0014bcbb3e7cd09352a..d1bbda3fd3562efe486377d41a9fb7359bafa4e7 100644 --- a/python/paddle/utils/image_multiproc.py +++ b/python/paddle/utils/image_multiproc.py @@ -15,7 +15,8 @@ import os, sys import numpy as np from PIL import Image -from cStringIO import StringIO +import six +from six.moves import cStringIO as StringIO import multiprocessing import functools import itertools @@ -187,7 +188,8 @@ class PILTransformer(ImageTransformer): return self.transform(im) -def job(is_img_string, transformer, (data, label)): +def job(is_img_string, transformer, data_label_pack): + (data, label) = data_label_pack if is_img_string: return transformer.transform_from_string(data), label else: @@ -208,7 +210,7 @@ class MultiProcessImageTransformer(object): """ Processing image with multi-process. If it is used in PyDataProvider, the simple usage for CNN is as follows: - + .. code-block:: python def hool(settings, is_train, **kwargs): @@ -229,7 +231,7 @@ class MultiProcessImageTransformer(object): @provider(init_hook=hook, pool_size=20480) def process(settings, file_list): with open(file_list, 'r') as fdata: - for line in fdata: + for line in fdata: data_dic = np.load(line.strip()) # load the data batch pickled by Pickle. data = data_dic['data'] labels = data_dic['label'] @@ -249,10 +251,10 @@ class MultiProcessImageTransformer(object): :type channel_swap: tuple or list :param mean: the mean values of image, per-channel mean or element-wise mean. :type mean: array, The dimension is 1 for per-channel mean. - The dimension is 3 for element-wise mean. + The dimension is 3 for element-wise mean. :param is_train: training peroid or testing peroid. :type is_train: bool. - :param is_color: the image is color or gray. + :param is_color: the image is color or gray. :type is_color: bool. :param is_img_string: The input can be the file name of image or image string. :type is_img_string: bool. @@ -273,4 +275,4 @@ class MultiProcessImageTransformer(object): def run(self, data, label): fun = functools.partial(job, self.is_img_string, self.transformer) return self.pool.imap_unordered( - fun, itertools.izip(data, label), chunksize=100 * self.procnum) + fun, six.moves.zip(data, label), chunksize=100 * self.procnum) diff --git a/python/paddle/utils/image_util.py b/python/paddle/utils/image_util.py index d3d79b14405256bbc95c41d805dbee56cb104f5e..a8092349cde8a4cb30873bf819fd5ed96289a945 100644 --- a/python/paddle/utils/image_util.py +++ b/python/paddle/utils/image_util.py @@ -14,7 +14,7 @@ import numpy as np from PIL import Image -from cStringIO import StringIO +from six.moves import cStringIO as StringIO def resize_image(img, target_size): @@ -34,7 +34,7 @@ def flip(im): """ Return the flipped image. Flip an image along the horizontal direction. - im: input image, (H x W x K) ndarrays + im: input image, (H x W x K) ndarrays """ if len(im.shape) == 3: return im[:, :, ::-1] @@ -132,7 +132,7 @@ def load_meta(meta_path, mean_img_size, crop_size, color=True): def load_image(img_path, is_color=True): """ - Load image and return. + Load image and return. img_path: image path. is_color: is color image or not. """ @@ -205,7 +205,7 @@ class ImageTransformer: def set_mean(self, mean): if mean is not None: - # mean value, may be one value per channel + # mean value, may be one value per channel if mean.ndim == 1: mean = mean[:, np.newaxis, np.newaxis] else: diff --git a/python/paddle/utils/make_model_diagram.py b/python/paddle/utils/make_model_diagram.py index 40f99075de7fb2401b3b704afe1eb44dbe6072dd..52759d3ad230c3a5a5488a8bc46a2e8f8fae1025 100644 --- a/python/paddle/utils/make_model_diagram.py +++ b/python/paddle/utils/make_model_diagram.py @@ -15,6 +15,9 @@ # Generate dot diagram file for the given paddle model config # The generated file can be viewed using Graphviz (http://graphviz.org) +from __future__ import print_function + +import six import sys import traceback @@ -61,9 +64,9 @@ def make_diagram_from_proto(model_config, dot_file): name2id[mem.link_name]) return s - print >> f, 'digraph graphname {' - print >> f, 'node [width=0.375,height=0.25];' - for i in xrange(len(model_config.layers)): + print('digraph graphname {', file=f) + print('node [width=0.375,height=0.25];', file=f) + for i in six.moves.xrange(len(model_config.layers)): l = model_config.layers[i] name2id[l.name] = i @@ -71,12 +74,12 @@ def make_diagram_from_proto(model_config, dot_file): for sub_model in model_config.sub_models: if sub_model.name == 'root': continue - print >> f, 'subgraph cluster_%s {' % i - print >> f, 'style=dashed;' + print('subgraph cluster_%s {' % i, file=f) + print('style=dashed;', file=f) label = '%s ' % sub_model.name if sub_model.reversed: label += '<==' - print >> f, 'label = "%s";' % label + print('label = "%s";' % label, file=f) i += 1 submodel_layers.add(sub_model.name) for layer_name in sub_model.layer_names: @@ -84,37 +87,41 @@ def make_diagram_from_proto(model_config, dot_file): lid = name2id[layer_name] layer_config = model_config.layers[lid] label = make_layer_label(layer_config) - print >> f, 'l%s [label="%s", shape=box];' % (lid, label) - print >> f, '}' + print('l%s [label="%s", shape=box];' % (lid, label), file=f) + print('}', file=f) - for i in xrange(len(model_config.layers)): + for i in six.moves.xrange(len(model_config.layers)): l = model_config.layers[i] if l.name not in submodel_layers: label = make_layer_label(l) - print >> f, 'l%s [label="%s", shape=box];' % (i, label) + print('l%s [label="%s", shape=box];' % (i, label), file=f) for sub_model in model_config.sub_models: if sub_model.name == 'root': continue for link in sub_model.in_links: - print >> f, make_link(link) + print(make_link(link), file=f) for link in sub_model.out_links: - print >> f, make_link(link) + print(make_link(link), file=f) for mem in sub_model.memories: - print >> f, make_mem(mem) + print(make_mem(mem), file=f) - for i in xrange(len(model_config.layers)): + for i in six.moves.xrange(len(model_config.layers)): for l in model_config.layers[i].inputs: - print >> f, 'l%s -> l%s [label="%s"];' % ( - name2id[l.input_layer_name], i, l.input_parameter_name) + print( + 'l%s -> l%s [label="%s"];' % (name2id[l.input_layer_name], i, + l.input_parameter_name), + file=f) - print >> f, '}' + print('}', file=f) f.close() def usage(): - print >> sys.stderr, ("Usage: python show_model_diagram.py" + - " CONFIG_FILE DOT_FILE [config_str]") + print( + ("Usage: python show_model_diagram.py" + + " CONFIG_FILE DOT_FILE [config_str]"), + file=sys.stderr) exit(1) diff --git a/python/paddle/utils/merge_model.py b/python/paddle/utils/merge_model.py index 2b100207728a8532e900992f7db4d3910e893dea..b74649e93640c3600636034d58792b8d12dffeda 100644 --- a/python/paddle/utils/merge_model.py +++ b/python/paddle/utils/merge_model.py @@ -70,4 +70,4 @@ def merge_v2_model(net, param_file, output_file): for pname in param_names: params.serialize(pname, f) - print 'Generate %s success!' % (output_file) + print('Generate %s success!' % (output_file)) diff --git a/python/paddle/utils/plotcurve.py b/python/paddle/utils/plotcurve.py index 27bd8157d39632913e2fa3278f3af20ddea61da7..a95e5497e23571e61e5d7652830a99efd7793083 100644 --- a/python/paddle/utils/plotcurve.py +++ b/python/paddle/utils/plotcurve.py @@ -44,6 +44,7 @@ To use this script to generate plot for AvgCost, error: python plotcurve.py -i paddle.INFO -o figure.png AvgCost error """ +import six import sys import matplotlib # the following line is added immediately after import matplotlib @@ -91,7 +92,7 @@ def plot_paddle_curve(keys, inputfile, outputfile, format='png', sys.stderr.write("No data to plot. Exiting!\n") return m = len(keys) + 1 - for i in xrange(1, m): + for i in six.moves.xrange(1, m): pyplot.plot( x[:, 0], x[:, i], diff --git a/python/paddle/utils/predefined_net.py b/python/paddle/utils/predefined_net.py index fa05f981f2b66bf55303a6f7c332c0bc9b112d29..2801f4877c079615239b92be146b3e33df16b37f 100644 --- a/python/paddle/utils/predefined_net.py +++ b/python/paddle/utils/predefined_net.py @@ -13,6 +13,7 @@ # limitations under the License. import numpy as np +import six import os from paddle.trainer.config_parser import * from paddle.utils.preprocess_img import \ @@ -112,7 +113,7 @@ def simple_conv_net(data_conf, is_color=False): num_classes: num of classes. is_color: whether the input images are color. """ - for k, v in data_conf.iteritems(): + for k, v in six.iteritems(data_conf): globals()[k] = v data_input, label_input, num_image_channels = \ image_data_layers(image_size, num_classes, is_color, is_predict) @@ -340,7 +341,7 @@ def small_vgg(data_conf, is_predict=False): num_classes: num of classes. is_color: whether the input images are color. """ - for k, v in data_conf.iteritems(): + for k, v in six.iteritems(data_conf): globals()[k] = v vgg_conv_net(image_size, num_classes, num_layers=[2, 2, 3, 3], diff --git a/python/paddle/utils/preprocess_img.py b/python/paddle/utils/preprocess_img.py index 975f1e9edea161331d37afbc6b5af46286f185bf..a322f7b769a2a32df516a4b8ea04289a7f882ff2 100644 --- a/python/paddle/utils/preprocess_img.py +++ b/python/paddle/utils/preprocess_img.py @@ -17,9 +17,9 @@ import os import random import numpy as np import PIL.Image as Image -import StringIO -import preprocess_util -from image_util import crop_img +from six.moves import cStringIO as StringIO +from . import preprocess_util +from .image_util import crop_img def resize_image(img, target_size): @@ -52,7 +52,7 @@ class DiskImage: def read_image(self): if self.img is None: - print "reading: " + self.path + print("reading: " + self.path) image = resize_image(Image.open(self.path), self.target_size) self.img = image @@ -69,7 +69,7 @@ class DiskImage: convert the image into the paddle batch format. """ self.read_image() - output = StringIO.StringIO() + output = StringIO() self.img.save(output, "jpeg") contents = output.getvalue() return contents @@ -127,7 +127,7 @@ class ImageClassificationDatasetCreater(preprocess_util.DatasetCreater): image_path = items[0] label_name = items[1] if not label_name in label_set: - label_set[label_name] = len(label_set.keys()) + label_set[label_name] = len(list(label_set.keys())) img = DiskImage(path=image_path, target_size=self.target_size) label = preprocess_util.Lablel( label=label_set[label_name], name=label_name) @@ -144,7 +144,7 @@ class ImageClassificationDatasetCreater(preprocess_util.DatasetCreater): return create_dataset_from_list(path) label_set = preprocess_util.get_label_set_from_dir(path) data = [] - for l_name in label_set.keys(): + for l_name in list(label_set.keys()): image_paths = preprocess_util.list_images( os.path.join(path, l_name)) for p in image_paths: diff --git a/python/paddle/utils/preprocess_util.py b/python/paddle/utils/preprocess_util.py index 1d17a488243eb81e46bea3ead686efd021499e22..05b2067d01a2c544d7f5bd68320e79c805282286 100644 --- a/python/paddle/utils/preprocess_util.py +++ b/python/paddle/utils/preprocess_util.py @@ -14,7 +14,7 @@ import os import math -import cPickle as pickle +import six.moves.cPickle as pickle import random import collections @@ -169,7 +169,7 @@ class Dataset: random.shuffle(keyvalue_indices[k]) num_data_per_key_batch = \ - math.ceil(num_per_batch / float(len(keyvalue_indices.keys()))) + math.ceil(num_per_batch / float(len(list(keyvalue_indices.keys())))) if num_data_per_key_batch < 2: raise Exception("The number of data in a batch is too small") @@ -182,8 +182,8 @@ class Dataset: end_idx = int( min(begin_idx + num_data_per_key_batch, len(keyvalue_indices[k]))) - print "begin_idx, end_idx" - print begin_idx, end_idx + print("begin_idx, end_idx") + print(begin_idx, end_idx) for idx in range(begin_idx, end_idx): permuted_data.append(self.data[keyvalue_indices[k][idx]]) keyvalue_readpointer[k] = end_idx @@ -357,6 +357,6 @@ class DatasetCreater(object): data_batcher.create_batches_and_list( self.output_path, self.train_list_name, self.test_list_name, self.label_set_name) - self.num_classes = len(train_label_set.keys()) + self.num_classes = len(list(train_label_set.keys())) self.create_meta_file(train_data) return out_path diff --git a/python/paddle/utils/show_pb.py b/python/paddle/utils/show_pb.py index 20614826d1d01f50a2bb54a840e2c584fb93b247..da7a71a665aea4d93d366e8508f438a9aba88e94 100644 --- a/python/paddle/utils/show_pb.py +++ b/python/paddle/utils/show_pb.py @@ -15,6 +15,8 @@ Show the content of proto buffer data file of PADDLE """ +from __future__ import print_function + import os import sys from google.protobuf.internal.decoder import _DecodeVarint @@ -39,7 +41,7 @@ def read_proto(file, message): def usage(): - print >> sys.stderr, "Usage: python show_pb.py PROTO_DATA_FILE" + print("Usage: python show_pb.py PROTO_DATA_FILE", file=sys.stderr) exit(1) @@ -50,8 +52,8 @@ if __name__ == '__main__': f = open(sys.argv[1]) header = DataFormat.DataHeader() read_proto(f, header) - print header + print(header) sample = DataFormat.DataSample() while read_proto(f, sample): - print sample + print(sample) diff --git a/python/paddle/utils/torch2paddle.py b/python/paddle/utils/torch2paddle.py index 91490111a1144ae25ed6566ff1c83db4f7954d33..398d3aa4e02cc74b7885f7e676937d7fd254bc5e 100644 --- a/python/paddle/utils/torch2paddle.py +++ b/python/paddle/utils/torch2paddle.py @@ -24,7 +24,7 @@ import sys import struct import numpy as np import torchfile -import cPickle as pickle +import six.moves.cPickle as pickle import argparse @@ -48,7 +48,7 @@ def save_net_parameters(layers, params, output_path): biases = params[i * 2 + 1] weight_file = os.path.join(output_path, '_%s.w0' % layers[i]) biases_file = os.path.join(output_path, '_%s.wbias' % layers[i]) - print "Saving for layer %s." % layers[i] + print("Saving for layer %s." % layers[i]) save_layer_parameters(weight_file, [weight]) save_layer_parameters(biases_file, biases) diff --git a/tools/manylinux1/Dockerfile.x64 b/tools/manylinux1/Dockerfile.x64 index 0d59e4c110ff8502acb4dbcda15f855f7652a946..987fc01cfc13bcefbb9d792a6780fdfff158755d 100644 --- a/tools/manylinux1/Dockerfile.x64 +++ b/tools/manylinux1/Dockerfile.x64 @@ -13,7 +13,7 @@ ENV PATH /opt/rh/devtoolset-2/root/usr/bin:$PATH ENV LD_LIBRARY_PATH /opt/rh/devtoolset-2/root/usr/lib64:/opt/rh/devtoolset-2/root/usr/lib:/usr/local/lib64:/usr/local/lib:${LD_LIBRARY_PATH} ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig -RUN yum install -y sqlite-devel zlib-devel openssl-devel pcre-devel vim tk-devel tkinter libtool xz graphviz +RUN yum install -y sqlite-devel zlib-devel openssl-devel pcre-devel vim tk-devel tkinter libtool xz freetype-devel libpng-devel graphviz COPY build_scripts /build_scripts RUN bash build_scripts/build.sh && \ bash build_scripts/install_nccl2.sh && rm -r build_scripts diff --git a/tools/manylinux1/build_scripts/build.sh b/tools/manylinux1/build_scripts/build.sh index 93591fa9ddad8a78df344e1e912a5f1c7e93dfa4..d99d3db2ed7e3a719f044d8117baeb7ac212b74d 100644 --- a/tools/manylinux1/build_scripts/build.sh +++ b/tools/manylinux1/build_scripts/build.sh @@ -105,7 +105,7 @@ curl-config --features rm -rf /usr/local/ssl # Install patchelf (latest with unreleased bug fixes) -curl -sLO https://nipy.bic.berkeley.edu/manylinux/patchelf-0.9njs2.tar.gz +curl -sLO http://nipy.bic.berkeley.edu/manylinux/patchelf-0.9njs2.tar.gz check_sha256sum patchelf-0.9njs2.tar.gz $PATCHELF_HASH tar -xzf patchelf-0.9njs2.tar.gz (cd patchelf-0.9njs2 && ./configure && make && make install) diff --git a/tools/manylinux1/build_scripts/install_nccl2.sh b/tools/manylinux1/build_scripts/install_nccl2.sh index 282c5c290da14bd3c04346ab01fdb48423c23f88..43a99d8287bbaa13ff75d9f25972a6335ae0754a 100644 --- a/tools/manylinux1/build_scripts/install_nccl2.sh +++ b/tools/manylinux1/build_scripts/install_nccl2.sh @@ -21,5 +21,5 @@ for sub_deb in $DEBS; do ar x $sub_deb && tar xf data.tar.xz done mv -f usr/include/nccl.h /usr/local/include/ -mv -f usr/lib/libnccl* /usr/local/lib/ +mv -f usr/lib/x86_64-linux-gnu/libnccl* /usr/local/lib/ rm -rf $DIR diff --git a/tools/test_runner.py b/tools/test_runner.py index 2d6a3cf8a97a3bbaa69b66f5343c54b750624329..9b9f165e7368364bbb0a78d6dcbbe4be0d6bf98b 100644 --- a/tools/test_runner.py +++ b/tools/test_runner.py @@ -13,6 +13,7 @@ # limitations under the License. from __future__ import print_function + import unittest import os import sys