diff --git a/cmake/external/anakin.cmake b/cmake/external/anakin.cmake index 455ef91ac5e9d98def959256d15ae836bcf0befc..855897394a602736f3a4fa1132c634f8cb537468 100644 --- a/cmake/external/anakin.cmake +++ b/cmake/external/anakin.cmake @@ -35,8 +35,10 @@ set(ANAKIN_COMPILE_EXTRA_FLAGS ExternalProject_Add( extern_anakin ${EXTERNAL_PROJECT_LOG_ARGS} - GIT_REPOSITORY "https://github.com/PaddlePaddle/Anakin" - GIT_TAG "04256ba78fa3da0beb74e8036c8efd68c12824d6" + DEPENDS ${MKLML_PROJECT} + # Anakin codes error on Intel(R) Xeon(R) Gold 5117 CPU, temporary do not compile avx512 related code. + GIT_REPOSITORY "https://github.com/luotao1/Anakin" + GIT_TAG "bcf17aabe7921ceb7bce591244b4f9dce7dba5c8" PREFIX ${ANAKIN_SOURCE_DIR} UPDATE_COMMAND "" CMAKE_ARGS -DUSE_GPU_PLACE=YES diff --git a/paddle/fluid/framework/CMakeLists.txt b/paddle/fluid/framework/CMakeLists.txt index 1d62792b80dd002b894da28be9162fc7d3ce054e..fac9f16a89bab311c338475aef7c79015ab466be 100644 --- a/paddle/fluid/framework/CMakeLists.txt +++ b/paddle/fluid/framework/CMakeLists.txt @@ -115,6 +115,8 @@ cc_test(cow_ptr_tests SRCS details/cow_ptr_test.cc) # cc_test(channel_test SRCS channel_test.cc) cc_test(tuple_test SRCS tuple_test.cc ) +cc_test(rw_lock_test SRCS rw_lock_test.cc) + # disable test temporarily. # TODO https://github.com/PaddlePaddle/Paddle/issues/11971 # cc_test(concurrency_test SRCS concurrency_test.cc DEPS go_op channel_close_op channel_create_op diff --git a/paddle/fluid/framework/ir/graph_pattern_detecter.cc b/paddle/fluid/framework/ir/graph_pattern_detecter.cc index f27d9b0509aa4561cfd1e5da3b46a3a085cc888c..dcc4382792d6a21dc10c37d2173d8cf1989219a5 100644 --- a/paddle/fluid/framework/ir/graph_pattern_detecter.cc +++ b/paddle/fluid/framework/ir/graph_pattern_detecter.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include diff --git a/paddle/fluid/framework/ir/graph_traits.h b/paddle/fluid/framework/ir/graph_traits.h index edbe45acb98326ee3bf1d86495832ec8469b634e..f42bab20ed97e372d2da0c4a492a4458ab94e0a0 100644 --- a/paddle/fluid/framework/ir/graph_traits.h +++ b/paddle/fluid/framework/ir/graph_traits.h @@ -12,7 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +#pragma once + #include +#include + #include "paddle/fluid/framework/ir/graph.h" #include "paddle/fluid/framework/ir/node.h" diff --git a/paddle/fluid/framework/op_desc.cc b/paddle/fluid/framework/op_desc.cc index 03f7e71c03b8dd75d2a47cb4c6d1ef1a71792cf3..122dc161b41246e5f08bd0ae8b763489e9ee22f9 100644 --- a/paddle/fluid/framework/op_desc.cc +++ b/paddle/fluid/framework/op_desc.cc @@ -202,6 +202,52 @@ std::vector OpDesc::AttrNames() const { } void OpDesc::SetAttr(const std::string &name, const Attribute &v) { + // NOTICE(minqiyang): pybind11 will take the empty list in python as + // the std::vector type in C++; so we have to change the attr's type + // here if we meet this issue + proto::AttrType attr_type = static_cast(v.which() - 1); + if (attr_type == proto::AttrType::INTS && + boost::get>(v).size() == 0u) { + // Find current attr via attr name and set the correct attribute value + const proto::OpProto::Attr &attr = GetProtoAttr(name); + switch (attr.type()) { + case proto::AttrType::BOOLEANS: { + VLOG(11) << "SetAttr: " << Type() << ", " << name + << " from INTS to BOOLEANS"; + this->attrs_[name] = std::vector(); + break; + } + case proto::AttrType::INTS: { + VLOG(11) << "SetAttr: " << Type() << ", " << name + << " from INTS to INTS"; + this->attrs_[name] = std::vector(); + break; + } + case proto::AttrType::FLOATS: { + VLOG(11) << "SetAttr: " << Type() << ", " << name + << " from INTS to FLOATS"; + this->attrs_[name] = std::vector(); + break; + } + case proto::AttrType::STRINGS: { + VLOG(11) << "SetAttr: " << Type() << ", " << name + << " from INTS to STRINGS"; + this->attrs_[name] = std::vector(); + break; + } + case proto::AttrType::BLOCKS: { + VLOG(11) << "SetAttr: " << Type() << ", " << name + << " from INTS to BLOCKS"; + this->SetBlocksAttr(name, std::vector()); + return; + } + default: + PADDLE_THROW("Wrong attr type %d", attr.type()); + } + need_update_ = true; + return; + } + this->attrs_[name] = v; need_update_ = true; } @@ -229,6 +275,19 @@ Attribute OpDesc::GetAttr(const std::string &name) const { return it->second; } +const proto::OpProto::Attr &OpDesc::GetProtoAttr( + const std::string &name) const { + const proto::OpProto &proto = OpInfoMap::Instance().Get(Type()).Proto(); + for (int i = 0; i != proto.attrs_size(); ++i) { + const proto::OpProto::Attr &attr = proto.attrs(i); + if (attr.name() == name) { + return attr; + } + } + + PADDLE_THROW("Attribute %s is not found in proto %s", name, proto.type()); +} + Attribute OpDesc::GetNullableAttr(const std::string &name) const { auto it = attrs_.find(name); if (it != attrs_.end()) { diff --git a/paddle/fluid/framework/op_desc.h b/paddle/fluid/framework/op_desc.h index b77d84125a23b81c3de4123bea6f0e09cd6d1e90..2422392e24d864dc3e7973ab35e038ecf2c0392a 100644 --- a/paddle/fluid/framework/op_desc.h +++ b/paddle/fluid/framework/op_desc.h @@ -81,6 +81,8 @@ class OpDesc { Attribute GetAttr(const std::string &name) const; + const proto::OpProto::Attr &GetProtoAttr(const std::string &name) const; + Attribute GetNullableAttr(const std::string &name) const; int GetBlockAttrId(const std::string &name) const; diff --git a/paddle/fluid/framework/rw_lock.h b/paddle/fluid/framework/rw_lock.h new file mode 100644 index 0000000000000000000000000000000000000000..1418fb5134fdde2392da912b5f1bd9fc74e58400 --- /dev/null +++ b/paddle/fluid/framework/rw_lock.h @@ -0,0 +1,48 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include + +#include "paddle/fluid/platform/enforce.h" + +namespace paddle { +namespace framework { + +struct RWLock { + RWLock() { pthread_rwlock_init(&lock_, nullptr); } + + ~RWLock() { pthread_rwlock_destroy(&lock_); } + + void RDLock() { + PADDLE_ENFORCE_EQ(pthread_rwlock_rdlock(&lock_), 0, + "acquire read lock failed"); + } + + void WRLock() { + PADDLE_ENFORCE_EQ(pthread_rwlock_wrlock(&lock_), 0, + "acquire write lock failed"); + } + + void UNLock() { + PADDLE_ENFORCE_EQ(pthread_rwlock_unlock(&lock_), 0, "unlock failed"); + } + + private: + pthread_rwlock_t lock_; +}; + +} // namespace framework +} // namespace paddle diff --git a/paddle/fluid/framework/rw_lock_test.cc b/paddle/fluid/framework/rw_lock_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..16f9cbb65229f10912ee90436c3557aaaca169b8 --- /dev/null +++ b/paddle/fluid/framework/rw_lock_test.cc @@ -0,0 +1,81 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/framework/rw_lock.h" +#include +#include // NOLINT +#include // NOLINT +#include + +namespace f = paddle::framework; + +void f1(f::RWLock *lock) { + lock->RDLock(); + lock->UNLock(); +} + +TEST(RWLOCK, read_read) { + f::RWLock lock; + lock.RDLock(); + std::thread t1(f1, &lock); + std::thread t2(f1, &lock); + t1.join(); + t2.join(); + lock.UNLock(); +} + +void f2(f::RWLock *lock, std::vector *result) { + lock->RDLock(); + ASSERT_EQ(result->size(), 0UL); + lock->UNLock(); +} + +void f3(f::RWLock *lock, std::vector *result) { + lock->WRLock(); + result->push_back(1); + lock->UNLock(); +} + +TEST(RWLOCK, read_write) { + f::RWLock lock; + std::vector result; + + lock.RDLock(); + std::thread t1(f2, &lock, &result); + t1.join(); + std::thread t2(f3, &lock, &result); + std::this_thread::sleep_for(std::chrono::seconds(1)); + ASSERT_EQ(result.size(), 0UL); + lock.UNLock(); + t2.join(); + ASSERT_EQ(result.size(), 1UL); +} + +void f4(f::RWLock *lock, std::vector *result) { + lock->RDLock(); + ASSERT_EQ(result->size(), 1UL); + lock->UNLock(); +} + +TEST(RWLOCK, write_read) { + f::RWLock lock; + std::vector result; + + lock.WRLock(); + std::thread t1(f4, &lock, &result); + std::this_thread::sleep_for(std::chrono::seconds(1)); + result.push_back(1); + lock.UNLock(); + t1.join(); +} diff --git a/paddle/fluid/framework/selected_rows.cc b/paddle/fluid/framework/selected_rows.cc index 06ed87e7e8a2d5324b48a466b05207042ec1b7fa..c202b0a5be1f891b8ae7b11e1f6e0ce02fcba588 100644 --- a/paddle/fluid/framework/selected_rows.cc +++ b/paddle/fluid/framework/selected_rows.cc @@ -120,66 +120,76 @@ bool SelectedRows::HasKey(int64_t key) const { : true; } -std::vector> SelectedRows::Get( - const std::vector& keys, framework::Tensor* value) const { +int64_t SelectedRows::AutoGrownIndex(int64_t key, bool auto_grown) { + rwlock_->RDLock(); + auto iter = id_to_index_.find(key); + if (iter == id_to_index_.end()) { + rwlock_->UNLock(); + if (!auto_grown) { + PADDLE_THROW("key %d not found", key); + } + rwlock_->WRLock(); + auto map_size = id_to_index_.size(); + auto vector_size = rows_.size(); + if (map_size != vector_size) { + rwlock_->UNLock(); + PADDLE_THROW( + "id_to_index_ size %d should have the same size with rows_ %d", + map_size, vector_size); + } + auto write_iter = id_to_index_.find(key); + if (write_iter == id_to_index_.end()) { + size_t row_num = rows_.size(); + if (row_num == value_->dims()[0]) { + rwlock_->UNLock(); + PADDLE_THROW("selected rows is full, then length exceed %d", row_num); + } + // key logic to put a key into id_to_index_ + rows_.push_back(key); + auto index = static_cast(rows_.size() - 1); + id_to_index_[key] = index; + rwlock_->UNLock(); + return index; + } else { + auto index = write_iter->second; + rwlock_->UNLock(); + return index; + } + } else { + auto index = iter->second; + rwlock_->UNLock(); + return index; + } +} + +void SelectedRows::SyncIndex() { + rwlock_->WRLock(); + id_to_index_.clear(); + for (size_t i = 0; i < rows_.size(); ++i) { + id_to_index_[rows_[i]] = i; + } + rwlock_->UNLock(); +} + +void SelectedRows::Get(const framework::Tensor& ids, framework::Tensor* value, + bool auto_grown) { PADDLE_ENFORCE(value->IsInitialized(), "The value tensor should be initialized."); - std::vector> non_keys_pair; - if (keys.empty()) { + if (ids.numel() == 0) { VLOG(3) << "keys is empty, please check data!"; } else { int64_t value_width = value_->numel() / value_->dims()[0]; PADDLE_ENFORCE_EQ(value_width, value->numel() / value->dims()[0], "output tensor should have the same shape with table " "except the dims[0]."); - - for (size_t i = 0; i < keys.size(); ++i) { - int64_t index = Index(keys[i]); - if (index == -1) { - non_keys_pair.push_back( - std::make_pair(keys[i], static_cast(i))); - } else { - framework::VisitDataType( - framework::ToDataType(value_->type()), - TensorCopyVisitor(value, i * value_width, *value_.get(), - index * value_width, value_width)); - } + for (size_t i = 0; i < ids.numel(); ++i) { + int64_t index = AutoGrownIndex(ids.data()[i], auto_grown); + framework::VisitDataType( + framework::ToDataType(value_->type()), + TensorCopyVisitor(value, i * value_width, *value_.get(), + index * value_width, value_width)); } } - return non_keys_pair; -} - -bool SelectedRows::Set(int64_t key, const framework::Tensor& value) { - PADDLE_ENFORCE(value.IsInitialized(), "The value should be initialized."); - if (value_->IsInitialized()) { - PADDLE_ENFORCE_EQ( - value.type(), value_->type(), - "The type of the value should be same with the original value"); - } - PADDLE_ENFORCE_EQ(value.dims()[0], static_cast(1), - "The first dim of value should be 1."); - std::lock_guard lock(*auto_grown_mutex_.get()); - auto index = Index(key); - bool is_new_key = false; - if (index == -1) { - rows_.push_back(key); - index = rows_.size() - 1; - is_new_key = true; - // whether need to resize the table - if (static_cast(rows_.size()) > value_->dims()[0]) { - auto dims = value_->dims(); - dims[0] = (dims[0] + 1) << 1; - framework::VisitDataType(framework::ToDataType(value.type()), - ReAllocateVisitor(dims, value_.get())); - } - } - - framework::VisitDataType( - framework::ToDataType(value.type()), - TensorCopyVisitor(value_.get(), - index * value_->numel() / value_->dims()[0], value, - static_cast(0), value.numel())); - return is_new_key; } } // namespace framework diff --git a/paddle/fluid/framework/selected_rows.h b/paddle/fluid/framework/selected_rows.h index 7160670ddd204c20021ea87cdd67ee4721d03451..daf5e95304fb84eaba26a30c45414d5021e7ffcb 100644 --- a/paddle/fluid/framework/selected_rows.h +++ b/paddle/fluid/framework/selected_rows.h @@ -17,10 +17,12 @@ limitations under the License. */ #include #include #include // NOLINT +#include #include #include #include "paddle/fluid/framework/lod_tensor.h" +#include "paddle/fluid/framework/rw_lock.h" #include "paddle/fluid/framework/tensor.h" #include "paddle/fluid/memory/memcpy.h" @@ -48,13 +50,13 @@ class SelectedRows { SelectedRows(const std::vector& rows, const int64_t& height) : rows_(rows), height_(height) { value_.reset(new Tensor()); - auto_grown_mutex_.reset(new std::mutex); + rwlock_.reset(new RWLock); } SelectedRows() { height_ = 0; value_.reset(new Tensor()); - auto_grown_mutex_.reset(new std::mutex); + rwlock_.reset(new RWLock); } platform::Place place() const { return value_->place(); } @@ -74,47 +76,51 @@ class SelectedRows { void set_rows(const Vector& rows) { rows_ = rows; } /* - * @brief wheter has the specified key in the table. + * @brief Get the index of key in rows + * + * @return -1 if the key does not exists. + */ + int64_t Index(int64_t key) const { + auto it = std::find(rows_.begin(), rows_.end(), key); + if (it == rows_.end()) { + PADDLE_THROW("id %s not in table", key); + } + return static_cast(std::distance(rows_.begin(), it)); + } + + /* + * @brief whether has the specified key in the table. * * @return true if the key is exists. */ bool HasKey(int64_t key) const; /* - * @brief Get value by the key list, if the + * @brief Get value by the key list. + * Note!!! this interface is only used when selected_rows is used as + * parameters + * for distribute lookup table. * * @return a list of pair which contains the non-exists key and the index in * the value */ - std::vector> Get(const std::vector& keys, - framework::Tensor* value) const; + void Get(const framework::Tensor& ids, framework::Tensor* value, + bool auto_grown = false); /* - * @brief Set a key-value pair into the table. - * This function will double the value memory if it's not engouth. + * @brief Get the index of the key from id_to_index_ map. If the key not + * exist, + * add the key into id_to_index_. * - * @note: - * 1. The first dim of the value should be 1 - * 2. The value should be initialized and the data type - * should be the same with the table. - * - * @return true if the key is a new one, otherwise false + * Note!!! this interface is only used when selected_rows is used as + * parameters + * for distribute lookup table. * + * @return index of the key. */ - bool Set(int64_t key, const Tensor& value); + int64_t AutoGrownIndex(int64_t key, bool auto_grown); - /* - * @brief Get the index of key in rows - * - * @return -1 if the key does not exists. - */ - int64_t Index(int64_t key) const { - auto it = std::find(rows_.begin(), rows_.end(), key); - if (it == rows_.end()) { - return static_cast(-1); - } - return static_cast(std::distance(rows_.begin(), it)); - } + void SyncIndex(); DDim GetCompleteDims() const { std::vector dims = vectorize(value_->dims()); @@ -127,9 +133,10 @@ class SelectedRows { // SelectedRows are simply concated when adding together. Until a // SelectedRows add a Tensor, will the duplicate rows be handled. Vector rows_; + std::unordered_map id_to_index_; std::unique_ptr value_{nullptr}; int64_t height_; - std::unique_ptr auto_grown_mutex_{nullptr}; + std::unique_ptr rwlock_{nullptr}; }; /* diff --git a/paddle/fluid/framework/selected_rows_test.cc b/paddle/fluid/framework/selected_rows_test.cc index eefcaa5672c5a3debf162f5c8eda653408dcf221..5ca864cfdf7176850dd31dd42ef3306061a742cf 100644 --- a/paddle/fluid/framework/selected_rows_test.cc +++ b/paddle/fluid/framework/selected_rows_test.cc @@ -9,8 +9,11 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#include "paddle/fluid/framework/selected_rows.h" +#include +#include // NOLINT + #include "gtest/gtest.h" +#include "paddle/fluid/framework/selected_rows.h" namespace paddle { namespace framework { @@ -59,39 +62,129 @@ TEST_F(SelectedRowsTester, SerializeAndDeseralize) { ASSERT_EQ(selected_rows_->GetCompleteDims(), dst_tensor.GetCompleteDims()); } -TEST_F(SelectedRowsTester, SparseTable) { +TEST(SelectedRows, SparseTable) { platform::CPUPlace cpu; SelectedRows table; + + int64_t table_size = 100; + int64_t embedding_width = 8; // initialize a sparse table - table.mutable_value()->Resize(framework::make_ddim({1, 100})); - table.mutable_value()->mutable_data(cpu); - table.mutable_rows()->push_back(1); + table.mutable_value()->Resize( + framework::make_ddim({table_size, embedding_width})); + auto* data = table.mutable_value()->mutable_data(cpu); + for (int64_t i = 0; i < table_size; ++i) { + for (int64_t j = 0; j < embedding_width; ++j) { + data[i * embedding_width + j] = static_cast(i); + } + } + ASSERT_EQ(table.AutoGrownIndex(10, true), 0); + ASSERT_EQ(table.AutoGrownIndex(8, true), 1); + ASSERT_EQ(table.AutoGrownIndex(8, true), 1); + ASSERT_EQ(table.AutoGrownIndex(6, true), 2); + ASSERT_TRUE(table.HasKey(10)); + ASSERT_TRUE(table.HasKey(8)); + ASSERT_TRUE(table.HasKey(6)); + ASSERT_EQ(table.rows().size(), 3); + + framework::Tensor ids; + ids.Resize(framework::make_ddim({4})); + auto* ids_data = ids.mutable_data(cpu); + ids_data[0] = static_cast(6); + ids_data[1] = static_cast(6); + ids_data[2] = static_cast(8); + ids_data[3] = static_cast(10); - int64_t key = 10000; - int64_t non_key = 999; - framework::Tensor value; - value.Resize(framework::make_ddim({1, 100})); - auto ptr = value.mutable_data(cpu); - ptr[0] = static_cast(10); + framework::Tensor get_value; + auto* value_data = get_value.mutable_data( + framework::make_ddim({4, embedding_width}), cpu); + table.Get(ids, &get_value); - ASSERT_EQ(table.rows().size(), static_cast(1)); - ASSERT_EQ(table.HasKey(key), false); + for (int j = 0; j < embedding_width; ++j) { + ASSERT_EQ(value_data[0 * embedding_width + j], 2); + } + for (int j = 0; j < embedding_width; ++j) { + ASSERT_EQ(value_data[1 * embedding_width + j], 2); + } + for (int j = 0; j < embedding_width; ++j) { + ASSERT_EQ(value_data[2 * embedding_width + j], 1); + } + for (int j = 0; j < embedding_width; ++j) { + ASSERT_EQ(value_data[3 * embedding_width + j], 0); + } +} - table.Set(key, value); +void f1(SelectedRows* table, int table_size) { + for (int i = 1000000; i > 0; --i) { + auto id = i % table_size; + int64_t index1 = table->AutoGrownIndex(id, true); + int64_t index2 = table->AutoGrownIndex(id, false); + int64_t index3 = table->AutoGrownIndex(id, true); + ASSERT_EQ(index1, index2); + ASSERT_EQ(index2, index3); + } +} - ASSERT_EQ(table.rows().size(), static_cast(2)); - ASSERT_EQ(table.HasKey(key), true); - // check re-allocate - ASSERT_EQ(table.value().dims()[0], static_cast(4)); +void f2(SelectedRows* table, int table_size) { + for (int i = 0; i < 1000000; ++i) { + auto id = i % table_size; + int64_t index1 = table->AutoGrownIndex(id, true); + int64_t index2 = table->AutoGrownIndex(id, false); + int64_t index3 = table->AutoGrownIndex(id, true); + ASSERT_EQ(index1, index2); + ASSERT_EQ(index2, index3); + } +} - framework::Tensor get_value; - get_value.mutable_data(framework::make_ddim({2, 100}), cpu); - std::vector keys({non_key, key}); - auto non_key_pairs = table.Get(keys, &get_value); +void f3(SelectedRows* table, int table_size) { + clock_t t1 = clock(); + for (int i = 100000; i > 0; --i) { + auto id1 = table->AutoGrownIndex(i % table_size, true); + auto id2 = table->Index(i % table_size); + ASSERT_EQ(id1, id2); + } + clock_t t2 = clock(); + std::cout << "f3 run time:" << t2 - t1 << std::endl; +} + +void f4(SelectedRows* table, int table_size) { + clock_t t1 = clock(); + for (int i = 0; i < 100000; ++i) { + auto id1 = table->AutoGrownIndex(i % table_size, true); + auto id2 = table->Index(i % table_size); + ASSERT_EQ(id1, id2); + } + clock_t t2 = clock(); + std::cout << "f4 run time:" << t2 - t1 << std::endl; +} + +TEST(SelectedRows, MultiThreadAutoIndex) { + platform::CPUPlace cpu; + SelectedRows table; + + int64_t table_size = 100000; + int64_t embedding_width = 8; + // initialize a sparse table + table.mutable_value()->Resize( + framework::make_ddim({table_size, embedding_width})); + auto* data = table.mutable_value()->mutable_data(cpu); + for (int64_t i = 0; i < table_size; ++i) { + for (int64_t j = 0; j < embedding_width; ++j) { + data[i * embedding_width + j] = static_cast(i); + } + } - ASSERT_EQ(get_value.data()[100], static_cast(10)); - ASSERT_EQ(non_key_pairs.size(), static_cast(1)); - ASSERT_EQ(non_key_pairs[0].first, non_key); + std::thread t1(f1, &table, table_size); + std::thread t11(f1, &table, table_size); + std::thread t2(f2, &table, table_size); + std::thread t22(f2, &table, table_size); + t1.join(); + t11.join(); + t2.join(); + t22.join(); + std::thread t3(f3, &table, table_size); + std::thread t4(f4, &table, table_size); + t3.join(); + t4.join(); } } // namespace framework diff --git a/paddle/fluid/inference/analysis/CMakeLists.txt b/paddle/fluid/inference/analysis/CMakeLists.txt index 27fe575cb6167a726ff92a8f3d2e47b6f536ba39..b972efe5b0a6942bf0710755bbabbaf863477931 100644 --- a/paddle/fluid/inference/analysis/CMakeLists.txt +++ b/paddle/fluid/inference/analysis/CMakeLists.txt @@ -8,7 +8,7 @@ cc_library(analysis SRCS pass_manager.cc dot.cc node.cc data_flow_graph.cc graph helper.cc model_store_pass.cc DEPS framework_proto proto_desc) -cc_test(test_node SRCS node_tester.cc DEPS analysis) +cc_test(test_node SRCS node_tester.cc DEPS analysis gflags glog gtest) cc_test(test_dot SRCS dot_tester.cc DEPS analysis) cc_binary(inference_analyzer SRCS analyzer_main.cc DEPS analysis) diff --git a/paddle/fluid/inference/analysis/node.cc b/paddle/fluid/inference/analysis/node.cc index f2e918f3ff41d9db0c3ec38561015967bed26f4e..3339b5044df0cf91d00aa9ddad310d4bf263bc3c 100644 --- a/paddle/fluid/inference/analysis/node.cc +++ b/paddle/fluid/inference/analysis/node.cc @@ -20,17 +20,6 @@ namespace paddle { namespace inference { namespace analysis { -template <> -std::string &NodeAttr::As() { - if (data_.empty()) { - type_index_ = std::type_index(typeid(std::string)); - } - PADDLE_ENFORCE_EQ(type_index_, std::type_index(typeid(std::string))); - return data_; -} - -std::string &NodeAttr::String() { return As(); } - std::vector Value::dot_attrs() const { return std::vector({Dot::Attr("style", "filled,rounded"), Dot::Attr("shape", "box"), diff --git a/paddle/fluid/inference/analysis/node.h b/paddle/fluid/inference/analysis/node.h index 47e524bc5c4a6b1324d5f182053129311487522d..fb426fb893d12c017deda74fc05016053fbc6b1c 100644 --- a/paddle/fluid/inference/analysis/node.h +++ b/paddle/fluid/inference/analysis/node.h @@ -29,6 +29,7 @@ limitations under the License. */ #include "paddle/fluid/inference/analysis/device.h" #include "paddle/fluid/inference/analysis/dot.h" #include "paddle/fluid/inference/analysis/helper.h" +#include "paddle/fluid/platform/variant.h" namespace paddle { namespace inference { @@ -38,39 +39,35 @@ class NodeMap; // A helper class to maintain the status from Pass. struct NodeAttr { + using any_t = + boost::variant; // NOTE T should be a primary type or a struct combined by several primary // types. // NOTE the STL containers should not use here. // Some usages // Attr attr; // attr.Bool() = true; - bool &Bool() { return As(); } float &Float() { return As(); } int32_t &Int32() { return As(); } int64_t &Int64() { return As(); } void *&Pointer() { return As(); } - std::string &String(); + std::string &String() { return As(); } private: template T &As() { - // init storage in the first usage. - if (data_.empty()) { - VLOG(4) << "resize data to " << sizeof(T); - type_index_ = std::type_index(typeid(T)); - data_.resize(sizeof(T)); + if (type_index_ == typeid(NodeAttr)) { + type_index_ = typeid(T); + any_data_ = T(); + } else { + PADDLE_ENFORCE(type_index_ == typeid(T), "fetch error type"); } - PADDLE_ENFORCE(framework::IsType(type_index_), - "type not matched, origin is %s, want %s", - DataTypeNamer::Global().repr(type_index_), - DataTypeNamer::Global().repr()); - PADDLE_ENFORCE_EQ(data_.size(), sizeof(T), "Node attr type recast error"); - return *reinterpret_cast(&data_[0]); + return boost::get(any_data_); } private: - std::string data_; + any_t any_data_; std::type_index type_index_{typeid(NodeAttr)}; }; diff --git a/paddle/fluid/inference/analysis/node_tester.cc b/paddle/fluid/inference/analysis/node_tester.cc index ea832a3a7e47758be9b6bd59a4325ddb576ec446..8bbcfff53741772ee3705e2efdf46a1b59ee02ab 100644 --- a/paddle/fluid/inference/analysis/node_tester.cc +++ b/paddle/fluid/inference/analysis/node_tester.cc @@ -20,6 +20,24 @@ namespace paddle { namespace inference { namespace analysis { +TEST(NodeAttr, bool) { + NodeAttr x; + x.Bool() = true; + ASSERT_EQ(x.Bool(), true); +} + +TEST(NodeAttr, int32) { + NodeAttr x; + x.Int32() = 32; + ASSERT_EQ(x.Int32(), 32); +} + +TEST(NodeAttr, string) { + NodeAttr x; + x.String() = "Hello"; + ASSERT_EQ(x.String(), "Hello"); +} + TEST(Node, Attr) { // Node is an abstract class, use Value instead for they share the same Attr // logic. @@ -27,6 +45,9 @@ TEST(Node, Attr) { auto* node = nodes.Create(Node::Type::kValue); node->attr("v0").Int32() = 2008; ASSERT_EQ(node->attr("v0").Int32(), 2008); + + node->attr("str").String() = "hello world"; + ASSERT_EQ(node->attr("str").String(), "hello world"); } } // namespace analysis diff --git a/paddle/fluid/inference/api/demo_ci/run.sh b/paddle/fluid/inference/api/demo_ci/run.sh index 3e829dd726b132844a45427b7b0b39eedf197496..7824ef2649af81a2390ff3bc537eb7c93c70e402 100755 --- a/paddle/fluid/inference/api/demo_ci/run.sh +++ b/paddle/fluid/inference/api/demo_ci/run.sh @@ -13,16 +13,22 @@ else use_gpu_list='false' fi +PREFIX=inference-vis-demos%2F +URL_ROOT=http://paddlemodels.bj.bcebos.com/${PREFIX} + # download vis_demo data function download() { dir_name=$1 mkdir -p $dir_name cd $dir_name - wget -q ${URL_ROOT}$dir_name.tar.gz - tar xzf *.tar.gz + if [[ -e "${PREFIX}${dir_name}.tar.gz" ]]; then + echo "${PREFIX}{dir_name}.tar.gz has been downloaded." + else + wget -q ${URL_ROOT}$dir_name.tar.gz + tar xzf *.tar.gz + fi cd .. } -URL_ROOT=http://paddlemodels.bj.bcebos.com/inference-vis-demos%2F mkdir -p data cd data vis_demo_list='se_resnext50 ocr mobilenet' diff --git a/paddle/fluid/operators/activation_op.cu b/paddle/fluid/operators/activation_op.cu index d3a7ceed466a9b5e4d773f1531d198adff97eac2..27487b396ccf63d962defa6b270063ccb409164e 100644 --- a/paddle/fluid/operators/activation_op.cu +++ b/paddle/fluid/operators/activation_op.cu @@ -26,8 +26,6 @@ namespace plat = paddle::platform; act_type##_grad, ops::ActivationGradKernel>, \ ops::ActivationGradKernel>, \ - ops::ActivationGradKernel>); + ops::grad_functor>); FOR_EACH_KERNEL_FUNCTOR(REGISTER_ACTIVATION_CUDA_KERNEL); diff --git a/paddle/fluid/operators/activation_op.h b/paddle/fluid/operators/activation_op.h index 48f3b5a5bc06fbc211895a1a6d1521cfd97e0086..912415192659dc004f54a76e9cd1a20581d512a6 100644 --- a/paddle/fluid/operators/activation_op.h +++ b/paddle/fluid/operators/activation_op.h @@ -333,7 +333,8 @@ struct SqrtGradFunctor : public BaseActivationFunctor { template void operator()(Device d, X x, Out out, dOut dout, dX dx) const { - dx.device(d) = static_cast(0.5) * dout / out; + const Out out_conj = Eigen::numext::conj(out); + dx.device(d) = static_cast(0.5) * dout / out_conj; } }; @@ -739,7 +740,7 @@ struct PowGradFunctor : public BaseActivationFunctor { typename dX> void operator()(Device d, X x, Out out, dOut dout, dX dx) const { dx.device(d) = dout * static_cast(factor) * - x.pow(static_cast(factor) - static_cast(1)); + x.pow(static_cast(factor - static_cast(1))); } }; @@ -862,11 +863,10 @@ struct SwishGradFunctor : public BaseActivationFunctor { template void operator()(Device d, X x, Out out, dOut dout, dX dx) const { - T b = static_cast(beta); auto temp1 = static_cast(1) / - (static_cast(1) + (static_cast(-b) * x).exp()); - auto temp2 = temp1 * (static_cast(1) - (b * out)); - dx.device(d) = dout * ((b * out) + temp2); + (static_cast(1) + (static_cast(-beta) * x).exp()); + auto temp2 = temp1 * (static_cast(1) - (beta * out)); + dx.device(d) = dout * ((beta * out) + temp2); } }; diff --git a/paddle/fluid/operators/assign_value_op.cu.cc b/paddle/fluid/operators/assign_value_op.cu.cc index 0ff174b3884df63d54d6486b017cc1a15ab23103..08bfde5dc92de9c675e5b9b85f8e65a3bab8631c 100644 --- a/paddle/fluid/operators/assign_value_op.cu.cc +++ b/paddle/fluid/operators/assign_value_op.cu.cc @@ -13,10 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/fluid/operators/assign_value_op.h" -#include "paddle/fluid/platform/float16.h" namespace ops = paddle::operators; -namespace plat = paddle::platform; REGISTER_OP_CUDA_KERNEL(assign_value, ops::AssignValueKernel, - ops::AssignValueKernel, - ops::AssignValueKernel); + ops::AssignValueKernel); diff --git a/paddle/fluid/operators/conv_cudnn_op.cu.cc b/paddle/fluid/operators/conv_cudnn_op.cu.cc index 59bfe8f61d8ebb530ba617006650c0ef9215e2a6..22cbf680c0670552fb014043c69fcadc56863529 100644 --- a/paddle/fluid/operators/conv_cudnn_op.cu.cc +++ b/paddle/fluid/operators/conv_cudnn_op.cu.cc @@ -39,27 +39,6 @@ using ScalingParamType = typename platform::CudnnDataType::ScalingParamType; static constexpr size_t kCONV_CUDNN_WORKSPACE_LIMIT_BYTES = static_cast(1024) * 1024 * 1024; -template -// bool EnableFp16(const T& dummy, const DeviceContext& dev_ctx, -bool EnableFp16(const DeviceContext& dev_ctx, - cudnnConvolutionDescriptor_t cudnn_conv_desc) { -#if CUDA_VERSION >= 9000 && CUDNN_VERSION_MIN(7, 0, 1) - // Tensor core is supported since the volta GPU and - // is only enabled when input and filter data are float16 - if (dev_ctx.GetComputeCapability() >= 70 && - std::type_index(typeid(T)) == - std::type_index(typeid(platform::float16))) { - PADDLE_ENFORCE(platform::dynload::cudnnSetConvolutionMathType( - cudnn_conv_desc, CUDNN_TENSOR_OP_MATH)); - return true; - } else { - PADDLE_ENFORCE(platform::dynload::cudnnSetConvolutionMathType( - cudnn_conv_desc, CUDNN_DEFAULT_MATH)); - } -#endif - return false; -} - template class CUDNNConvOpKernel : public framework::OpKernel { public: @@ -149,14 +128,27 @@ class CUDNNConvOpKernel : public framework::OpKernel { cudnnConvolutionFwdAlgo_t algo; auto& dev_ctx = ctx.template device_context(); auto handle = dev_ctx.cudnn_handle(); - if (EnableFp16(dev_ctx, cudnn_conv_desc)) { + + CUDNN_ENFORCE(platform::dynload::cudnnGetConvolutionForwardAlgorithm( + handle, cudnn_input_desc, cudnn_filter_desc, cudnn_conv_desc, + cudnn_output_desc, CUDNN_CONVOLUTION_FWD_SPECIFY_WORKSPACE_LIMIT, + workspace_size_limit, &algo)); + +#if CUDA_VERSION >= 9000 && CUDNN_VERSION_MIN(7, 0, 1) + // Tensor core is supported since the volta GPU and + // is only enabled when input and filter data are float16 + if (dev_ctx.GetComputeCapability() >= 70 && + std::type_index(typeid(T)) == + std::type_index(typeid(platform::float16))) { + CUDNN_ENFORCE(platform::dynload::cudnnSetConvolutionMathType( + cudnn_conv_desc, CUDNN_TENSOR_OP_MATH)); + // Currently tensor core is only enabled using this algo algo = CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM; } else { - PADDLE_ENFORCE(platform::dynload::cudnnGetConvolutionForwardAlgorithm( - handle, cudnn_input_desc, cudnn_filter_desc, cudnn_conv_desc, - cudnn_output_desc, CUDNN_CONVOLUTION_FWD_SPECIFY_WORKSPACE_LIMIT, - workspace_size_limit, &algo)); + CUDNN_ENFORCE(platform::dynload::cudnnSetConvolutionMathType( + cudnn_conv_desc, CUDNN_DEFAULT_MATH)); } +#endif // get workspace size able to allocate CUDNN_ENFORCE(platform::dynload::cudnnGetConvolutionForwardWorkspaceSize( @@ -296,9 +288,6 @@ class CUDNNConvGradOpKernel : public framework::OpKernel { } else { data_algo = CUDNN_CONVOLUTION_BWD_DATA_ALGO_1; } - if (EnableFp16(dev_ctx, cudnn_conv_desc)) { - data_algo = CUDNN_CONVOLUTION_BWD_DATA_ALGO_1; - } CUDNN_ENFORCE( platform::dynload::cudnnGetConvolutionBackwardDataWorkspaceSize( @@ -318,9 +307,6 @@ class CUDNNConvGradOpKernel : public framework::OpKernel { } else { filter_algo = CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1; } - if (EnableFp16(dev_ctx, cudnn_conv_desc)) { - filter_algo = CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1; - } CUDNN_ENFORCE( platform::dynload::cudnnGetConvolutionBackwardFilterWorkspaceSize( @@ -376,8 +362,7 @@ REGISTER_OP_KERNEL(conv2d, CUDNN, plat::CUDAPlace, paddle::operators::CUDNNConvOpKernel); REGISTER_OP_KERNEL(conv2d_grad, CUDNN, plat::CUDAPlace, paddle::operators::CUDNNConvGradOpKernel, - paddle::operators::CUDNNConvGradOpKernel, - paddle::operators::CUDNNConvGradOpKernel); + paddle::operators::CUDNNConvGradOpKernel); REGISTER_OP_KERNEL(conv3d, CUDNN, plat::CUDAPlace, paddle::operators::CUDNNConvOpKernel, @@ -385,5 +370,4 @@ REGISTER_OP_KERNEL(conv3d, CUDNN, plat::CUDAPlace, paddle::operators::CUDNNConvOpKernel); REGISTER_OP_KERNEL(conv3d_grad, CUDNN, plat::CUDAPlace, paddle::operators::CUDNNConvGradOpKernel, - paddle::operators::CUDNNConvGradOpKernel, - paddle::operators::CUDNNConvGradOpKernel) + paddle::operators::CUDNNConvGradOpKernel); diff --git a/paddle/fluid/operators/cross_entropy_op.cu b/paddle/fluid/operators/cross_entropy_op.cu index 65fd3a5dbc9ffed4c5d1114346fcc0660c183dae..30dbd5bd3d39dd2992c3dd91364003bb7715a2eb 100644 --- a/paddle/fluid/operators/cross_entropy_op.cu +++ b/paddle/fluid/operators/cross_entropy_op.cu @@ -13,16 +13,12 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/fluid/operators/cross_entropy_op.h" -#include "paddle/fluid/platform/float16.h" namespace ops = paddle::operators; -namespace plat = paddle::platform; using CUDACtx = paddle::platform::CUDADeviceContext; REGISTER_OP_CUDA_KERNEL(cross_entropy, ops::CrossEntropyOpKernel, - ops::CrossEntropyOpKernel, - ops::CrossEntropyOpKernel); -REGISTER_OP_CUDA_KERNEL( - cross_entropy_grad, ops::CrossEntropyGradientOpKernel, - ops::CrossEntropyGradientOpKernel, - ops::CrossEntropyGradientOpKernel); + ops::CrossEntropyOpKernel); +REGISTER_OP_CUDA_KERNEL(cross_entropy_grad, + ops::CrossEntropyGradientOpKernel, + ops::CrossEntropyGradientOpKernel); diff --git a/paddle/fluid/operators/distributed/rpc_server_test.cc b/paddle/fluid/operators/distributed/rpc_server_test.cc index b50830c362d3f6ecf38affbfa6a1ffe2ed77e125..d6176e1443d2a441af7878e5efe99796d486bb7a 100644 --- a/paddle/fluid/operators/distributed/rpc_server_test.cc +++ b/paddle/fluid/operators/distributed/rpc_server_test.cc @@ -78,10 +78,9 @@ void InitTensorsOnServer(framework::Scope* scope, platform::CPUPlace* place, int64_t rows_numel) { CreateVarsOnScope(scope, place); auto w = scope->Var("w")->GetMutable(); - auto rows = w->mutable_rows(); - for (int64_t i = 0; i < rows_numel; ++i) rows->push_back(i); auto w_value = w->mutable_value(); w_value->Resize({rows_numel, 10}); + for (int64_t i = 0; i < rows_numel; ++i) w->AutoGrownIndex(i, true); auto ptr = w_value->mutable_data(*place); diff --git a/paddle/fluid/operators/elementwise_add_op.cu b/paddle/fluid/operators/elementwise_add_op.cu index f9f5c66d34fa1d73db00173e493f9953b8579518..dfff518f170b56d180b6883c363effb8dbd677b6 100644 --- a/paddle/fluid/operators/elementwise_add_op.cu +++ b/paddle/fluid/operators/elementwise_add_op.cu @@ -30,5 +30,4 @@ REGISTER_OP_CUDA_KERNEL( ops::ElementwiseAddGradKernel, ops::ElementwiseAddGradKernel, ops::ElementwiseAddGradKernel, - ops::ElementwiseAddGradKernel, - ops::ElementwiseAddGradKernel); + ops::ElementwiseAddGradKernel); diff --git a/paddle/fluid/operators/elementwise_div_op.cu b/paddle/fluid/operators/elementwise_div_op.cu index 4cc7ba0f43c6031bf4a27222a17eca84bad5a668..588d1f7420241ba1697e5141e4e4a2870f2dc87c 100644 --- a/paddle/fluid/operators/elementwise_div_op.cu +++ b/paddle/fluid/operators/elementwise_div_op.cu @@ -14,24 +14,19 @@ limitations under the License. */ #define EIGEN_USE_GPU #include "paddle/fluid/operators/elementwise_div_op.h" -#include "paddle/fluid/platform/float16.h" namespace ops = paddle::operators; -namespace plat = paddle::platform; REGISTER_OP_CUDA_KERNEL( elementwise_div, ops::ElementwiseDivKernel, ops::ElementwiseDivKernel, ops::ElementwiseDivKernel, - ops::ElementwiseDivKernel, - ops::ElementwiseDivKernel); + ops::ElementwiseDivKernel); REGISTER_OP_CUDA_KERNEL( elementwise_div_grad, ops::ElementwiseDivGradKernel, ops::ElementwiseDivGradKernel, ops::ElementwiseDivGradKernel, - ops::ElementwiseDivGradKernel, ops::ElementwiseDivGradKernel); + int64_t>); diff --git a/paddle/fluid/operators/elementwise_mul_op.cu b/paddle/fluid/operators/elementwise_mul_op.cu index 350d43168dea7e88127b0d28d663e680458e1dba..2fb1b4bee689c059625e3dbd59f80c541ace83a0 100644 --- a/paddle/fluid/operators/elementwise_mul_op.cu +++ b/paddle/fluid/operators/elementwise_mul_op.cu @@ -14,25 +14,19 @@ limitations under the License. */ #define EIGEN_USE_GPU #include "paddle/fluid/operators/elementwise_mul_op.h" -#include "paddle/fluid/platform/float16.h" namespace ops = paddle::operators; -namespace plat = paddle::platform; REGISTER_OP_CUDA_KERNEL( elementwise_mul, ops::ElementwiseMulKernel, ops::ElementwiseMulKernel, ops::ElementwiseMulKernel, - ops::ElementwiseMulKernel, - ops::ElementwiseMulKernel); + ops::ElementwiseMulKernel); REGISTER_OP_CUDA_KERNEL( elementwise_mul_grad, ops::ElementwiseMulGradKernel, ops::ElementwiseMulGradKernel, ops::ElementwiseMulGradKernel, - ops::ElementwiseMulGradKernel, ops::ElementwiseMulGradKernel); diff --git a/paddle/fluid/operators/elementwise_op_function.h b/paddle/fluid/operators/elementwise_op_function.h index 7223a972d23119c8ef93fb49bfe42922cc14571d..bc3e95e904f8b6c2cdd2ae6685bf67580178e6b6 100644 --- a/paddle/fluid/operators/elementwise_op_function.h +++ b/paddle/fluid/operators/elementwise_op_function.h @@ -350,7 +350,7 @@ static __global__ void ElemwiseGradBroadcast1CUDAKernel( int j = blockIdx.x; int i = threadIdx.x; int tid = threadIdx.x; - T val(0); + T val = 0; do { int x_offset = i * w + j; @@ -418,7 +418,7 @@ static __global__ void ElemwiseGradBroadcast2CUDAKernel( int tid = threadIdx.x; int j = blockIdx.x; - T val(0); + T val = 0; int ttid = tid; while (true) { diff --git a/paddle/fluid/operators/elementwise_sub_op.cu b/paddle/fluid/operators/elementwise_sub_op.cu index ff3f6f8a2cb542c2fb6b43d539f6413b39250992..8709f686f9af1bf4dacbc2dfc3e2d5dcc1c59b9a 100644 --- a/paddle/fluid/operators/elementwise_sub_op.cu +++ b/paddle/fluid/operators/elementwise_sub_op.cu @@ -14,25 +14,19 @@ limitations under the License. */ #define EIGEN_USE_GPU #include "paddle/fluid/operators/elementwise_sub_op.h" -#include "paddle/fluid/platform/float16.h" namespace ops = paddle::operators; -namespace plat = paddle::platform; REGISTER_OP_CUDA_KERNEL( elementwise_sub, ops::ElementwiseSubKernel, ops::ElementwiseSubKernel, ops::ElementwiseSubKernel, - ops::ElementwiseSubKernel, - ops::ElementwiseSubKernel); + ops::ElementwiseSubKernel); REGISTER_OP_CUDA_KERNEL( elementwise_sub_grad, ops::ElementwiseSubGradKernel, ops::ElementwiseSubGradKernel, ops::ElementwiseSubGradKernel, - ops::ElementwiseSubGradKernel, ops::ElementwiseSubGradKernel); diff --git a/paddle/fluid/operators/fill_constant_op.cc b/paddle/fluid/operators/fill_constant_op.cc index 862249269eaecdac262a691c884ea59f89f54061..130f18dde4f979a6a9925ede9cbf745fcec14d48 100644 --- a/paddle/fluid/operators/fill_constant_op.cc +++ b/paddle/fluid/operators/fill_constant_op.cc @@ -12,28 +12,48 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#include "paddle/fluid/operators/fill_constant_op.h" -#include "paddle/fluid/platform/float16.h" +#include "paddle/fluid/framework/data_type.h" +#include "paddle/fluid/framework/op_registry.h" +#include "paddle/fluid/operators/math/math_function.h" +#include "paddle/fluid/platform/device_context.h" namespace paddle { namespace operators { -class FillConstantOp : public framework::OperatorWithKernel { +class FillConstantInferShape : public framework::InferShapeBase { public: - using framework::OperatorWithKernel::OperatorWithKernel; - - void InferShape(framework::InferShapeContext* ctx) const override { + void operator()(framework::InferShapeContext *ctx) const override { PADDLE_ENFORCE(ctx->HasOutput("Out"), "Output(Out) of FillConstantOp should not be null."); - auto& shape = ctx->Attrs().Get>("shape"); + auto &shape = ctx->Attrs().Get>("shape"); ctx->SetOutputDim("Out", framework::make_ddim(shape)); } +}; + +class FillConstantOp : public framework::OperatorBase { + public: + using framework::OperatorBase::OperatorBase; + + private: + void RunImpl(const framework::Scope &scope, + const platform::Place &dev_place) const override { + auto data_type = + static_cast(Attr("dtype")); + auto value = Attr("value"); + auto force_cpu = Attr("force_cpu"); + auto &out = + *scope.FindVar(Output("Out"))->GetMutable(); + out.Resize(framework::make_ddim(Attr>("shape"))); + if (force_cpu) { + auto cpu = platform::CPUPlace(); + out.mutable_data(cpu, framework::ToTypeIndex(data_type)); + } else { + out.mutable_data(dev_place, framework::ToTypeIndex(data_type)); + } - framework::OpKernelType GetExpectedKernelType( - const framework::ExecutionContext& ctx) const override { - return framework::OpKernelType( - static_cast(ctx.Attr("dtype")), - ctx.device_context()); + platform::DeviceContextPool &pool = platform::DeviceContextPool::Instance(); + auto &dev_ctx = *pool.Get(dev_place); + math::set_constant(dev_ctx, &out, value); } }; @@ -67,11 +87,6 @@ Fill up a variable with specified constant value. } // namespace paddle namespace ops = paddle::operators; -REGISTER_OPERATOR(fill_constant, ops::FillConstantOp, ops::FillConstantOpMaker, +REGISTER_OPERATOR(fill_constant, ops::FillConstantOp, + ops::FillConstantInferShape, ops::FillConstantOpMaker, paddle::framework::EmptyGradOpMaker); -REGISTER_OP_CPU_KERNEL( - fill_constant, - ops::FillConstantOpKernel, - ops::FillConstantOpKernel, - ops::FillConstantOpKernel, - ops::FillConstantOpKernel) diff --git a/paddle/fluid/operators/fill_constant_op.cu.cc b/paddle/fluid/operators/fill_constant_op.cu.cc deleted file mode 100644 index 51ccaefa4338dfa18d26441a59d5fed2b9fa0c39..0000000000000000000000000000000000000000 --- a/paddle/fluid/operators/fill_constant_op.cu.cc +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "paddle/fluid/operators/fill_constant_op.h" -#include "paddle/fluid/platform/float16.h" - -namespace ops = paddle::operators; -REGISTER_OP_CUDA_KERNEL( - fill_constant, - ops::FillConstantOpKernel, - ops::FillConstantOpKernel, - ops::FillConstantOpKernel, - ops::FillConstantOpKernel, - ops::FillConstantOpKernel) diff --git a/paddle/fluid/operators/fill_constant_op.h b/paddle/fluid/operators/fill_constant_op.h deleted file mode 100644 index b2a2a7b2faedf9b94e01ed908ff39749973be1df..0000000000000000000000000000000000000000 --- a/paddle/fluid/operators/fill_constant_op.h +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once -#include - -#include "paddle/fluid/framework/data_type.h" -#include "paddle/fluid/framework/op_registry.h" -#include "paddle/fluid/operators/math/math_function.h" - -namespace paddle { -namespace operators { - -template -class FillConstantOpKernel : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext& ctx) const override { - auto data_type = - static_cast(ctx.Attr("dtype")); - auto value = ctx.Attr("value"); - auto force_cpu = ctx.Attr("force_cpu"); - auto* out = ctx.Output("Out"); - out->Resize(framework::make_ddim(ctx.Attr>("shape"))); - if (force_cpu) { - auto cpu = platform::CPUPlace(); - out->mutable_data(cpu, framework::ToTypeIndex(data_type)); - } else { - out->mutable_data(ctx.GetPlace(), framework::ToTypeIndex(data_type)); - } - - math::set_constant(ctx.template device_context(), out, - value); - } -}; - -} // namespace operators -} // namespace paddle diff --git a/paddle/fluid/operators/fill_op.cc b/paddle/fluid/operators/fill_op.cc index 352a17c927bc70bdd6e4307951f0e0ac3d10ac2d..925dc19061e2196a40411f415eb6e5ad59ab52ff 100644 --- a/paddle/fluid/operators/fill_op.cc +++ b/paddle/fluid/operators/fill_op.cc @@ -16,7 +16,6 @@ limitations under the License. */ #include "paddle/fluid/framework/op_registry.h" #include "paddle/fluid/operators/detail/safe_ref.h" #include "paddle/fluid/platform/device_context.h" -#include "paddle/fluid/platform/float16.h" namespace paddle { namespace operators { @@ -70,6 +69,7 @@ class FillOp : public framework::OperatorBase { framework::VisitDataType( dtype, FillOpVisitor(&tensor, Attr>("value"))); + if (!force_cpu && platform::is_gpu_place(place)) { // Copy tensor to out platform::DeviceContextPool &pool = diff --git a/paddle/fluid/operators/gaussian_random_op.cu b/paddle/fluid/operators/gaussian_random_op.cu index b4907237954ba478197d5ca8bdcbc3e1915e9dcf..7784856417e579fd43f79fa331d46df8af6c36b8 100644 --- a/paddle/fluid/operators/gaussian_random_op.cu +++ b/paddle/fluid/operators/gaussian_random_op.cu @@ -15,7 +15,6 @@ limitations under the License. */ #include #include "paddle/fluid/framework/op_registry.h" #include "paddle/fluid/framework/operator.h" -#include "paddle/fluid/platform/float16.h" namespace paddle { namespace operators { @@ -61,7 +60,6 @@ class GPUGaussianRandomKernel : public framework::OpKernel { } // namespace operators } // namespace paddle -namespace plat = paddle::platform; REGISTER_OP_CUDA_KERNEL(gaussian_random, paddle::operators::GPUGaussianRandomKernel, paddle::operators::GPUGaussianRandomKernel); diff --git a/paddle/fluid/operators/lookup_sparse_table_op.cc b/paddle/fluid/operators/lookup_sparse_table_op.cc index 2ce11e712fb1a8aa9748313ec7cf4e895a931465..de3f0990e109cacd49c4d888bbc1f797fb196e01 100644 --- a/paddle/fluid/operators/lookup_sparse_table_op.cc +++ b/paddle/fluid/operators/lookup_sparse_table_op.cc @@ -17,7 +17,6 @@ limitations under the License. */ #include "paddle/fluid/framework/data_type.h" #include "paddle/fluid/framework/op_registry.h" #include "paddle/fluid/operators/math/math_function.h" -#include "paddle/fluid/platform/device_context.h" namespace paddle { namespace operators { @@ -46,10 +45,6 @@ class LookupSparseTableOp : public framework::OperatorBase { auto out_var = scope.FindVar(Output("Out")); auto w_var = scope.FindVar(Input("W")); auto ids_var = scope.FindVar(Input("Ids")); - unsigned int seed = static_cast(Attr("seed")); - float min = Attr("min"); - float max = Attr("max"); - bool auto_grown_table = Attr("auto_grown_table"); PADDLE_ENFORCE(out_var->IsType(), "The type of Out var should be LodTensor."); @@ -60,46 +55,17 @@ class LookupSparseTableOp : public framework::OperatorBase { auto &ids_t = ids_var->Get(); auto out_t = out_var->GetMutable(); auto w_t = w_var->GetMutable(); - std::vector keys; - keys.resize(ids_t.numel()); - for (int64_t i = 0; i < ids_t.numel(); ++i) { - keys[i] = ids_t.data()[i]; - } // TODO(Yancey1989): support CUDA Place for the sparse table platform::CPUPlace cpu; auto out_shape = w_t->value().dims(); - out_shape[0] = keys.size(); + out_shape[0] = ids_t.numel(); out_t->Resize(out_shape); out_t->mutable_data(cpu, w_t->value().type()); PADDLE_ENFORCE_EQ(framework::ToDataType(w_t->value().type()), framework::proto::VarType::FP32, "The sparse table only support FP32"); - auto non_keys_pair = w_t->Get(keys, out_t); - if (!auto_grown_table) { - PADDLE_ENFORCE_EQ(non_keys_pair.size(), static_cast(0), - "there is some keys does exists in the sparse table."); - } - auto value_shape = w_t->value().dims(); - value_shape[0] = 1; - for (const auto &it : non_keys_pair) { - const auto key = it.first; - const auto index = it.second; - framework::Tensor value; - value.Resize(value_shape); - auto data = value.mutable_data(cpu); - - std::minstd_rand engine; - engine.seed(seed); - std::uniform_real_distribution dist(min, max); - int64_t size = value.numel(); - for (int64_t i = 0; i < size; ++i) { - data[i] = dist(engine); - } - w_t->Set(key, value); - memory::Copy(cpu, out_t->mutable_data(cpu) + index * value.numel(), - cpu, value.data(), value.numel() * sizeof(float)); - } + w_t->Get(ids_t, out_t, true); } }; @@ -121,21 +87,6 @@ class LookupSparseTableOpMaker : public framework::OpProtoAndCheckerMaker { "Otherwise the given value indicates padding the output " "with zeros whenever lookup encounters it in Ids.") .SetDefault(kNoPadding); - AddAttr("min", - "(float, default -1.0) " - "Minimum value of uniform random") - .SetDefault(-1.0f); - AddAttr("max", - "(float, default 1.0) " - "Maximum value of uniform random") - .SetDefault(1.0f); - AddAttr("seed", - "(int, default 0) " - "Random seed used for generating samples. " - "0 means use a seed generated by the system." - "Note that if seed is not 0, this operator will always " - "generate the same random numbers every time.") - .SetDefault(0); AddAttr("auto_grown_table", "(bool default false)" "Whether create new value if for nonexistent key.") diff --git a/paddle/fluid/operators/math/cross_entropy.cu b/paddle/fluid/operators/math/cross_entropy.cu index 58b85abf822741905a4e9547823b6cdbe645d39a..0de58d5fddd84d33f708c4c73e5a19dc2fe8a86b 100644 --- a/paddle/fluid/operators/math/cross_entropy.cu +++ b/paddle/fluid/operators/math/cross_entropy.cu @@ -15,25 +15,11 @@ limitations under the License. */ #include "paddle/fluid/operators/math/cross_entropy.h" #include "paddle/fluid/platform/cuda_device_function.h" #include "paddle/fluid/platform/cuda_primitives.h" -#include "paddle/fluid/platform/float16.h" namespace paddle { namespace operators { namespace math { -template -HOSTDEVICE T log(const T& val) { - return std::log(val); -} - -template <> -HOSTDEVICE platform::float16 log(const platform::float16& val) { - // strage bug, hlog is not exists. - return static_cast(0); - // half tmp = static_cast(val); - // return static_cast(hlog(tmp)); -} - namespace { template __global__ void CrossEntropyKernel(T* Y, const T* X, const int64_t* label, @@ -49,12 +35,12 @@ template __global__ void SoftCrossEntropyKernel(T* Y, const T* X, const T* label, const int class_num) { int tid = threadIdx.x; - T val(0); + T val = 0; int idx = blockIdx.x * class_num + tid; int end = blockIdx.x * class_num + class_num; for (; idx < end; idx += blockDim.x) { - val += math::TolerableValue()(log(X[idx])) * label[idx]; + val += math::TolerableValue()(std::log(X[idx])) * label[idx]; } val = paddle::platform::reduceSum(val, tid, blockDim.x); @@ -98,8 +84,6 @@ class CrossEntropyFunctor { template class CrossEntropyFunctor; template class CrossEntropyFunctor; -template class CrossEntropyFunctor; } // namespace math } // namespace operators } // namespace paddle diff --git a/paddle/fluid/operators/math/cross_entropy.h b/paddle/fluid/operators/math/cross_entropy.h index 2e4e4781c2eee1d9a0fc6760093a424ab3d5eb9d..adc5b3fe47cd3bf524eb56747b6bd51e345a2eb6 100644 --- a/paddle/fluid/operators/math/cross_entropy.h +++ b/paddle/fluid/operators/math/cross_entropy.h @@ -13,10 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. */ #pragma once -#include #include "paddle/fluid/framework/eigen.h" #include "paddle/fluid/framework/tensor.h" -#include "paddle/fluid/platform/float16.h" #include "paddle/fluid/platform/hostdevice.h" namespace paddle { @@ -35,21 +33,6 @@ struct TolerableValue { } }; -// float16 value clip behave different. -using paddle::platform::float16; -using paddle::platform::isfinite; -template <> -struct TolerableValue { - HOSTDEVICE float16 operator()(const float16& x) const { - if (isfinite(x)) - return x; - else if (x > static_cast(0)) - return std::numeric_limits::max(); - else - return std::numeric_limits::min(); - } -}; - template class CrossEntropyFunctor { public: diff --git a/paddle/fluid/operators/math/selected_rows_functor.cu b/paddle/fluid/operators/math/selected_rows_functor.cu index 00dbfc11a239da70ec81e3498d2f4d5e5bf1c63f..a92762c7fea865fad2c7784736cce93a8af21892 100644 --- a/paddle/fluid/operators/math/selected_rows_functor.cu +++ b/paddle/fluid/operators/math/selected_rows_functor.cu @@ -18,7 +18,6 @@ limitations under the License. */ #include "paddle/fluid/operators/math/math_function.h" #include "paddle/fluid/operators/math/selected_rows_functor.h" #include "paddle/fluid/platform/cuda_primitives.h" -#include "paddle/fluid/platform/float16.h" namespace paddle { namespace operators { @@ -77,7 +76,6 @@ struct SelectedRowsAdd { template struct SelectedRowsAdd; template struct SelectedRowsAdd; -template struct SelectedRowsAdd; namespace { template @@ -122,7 +120,7 @@ struct SelectedRowsAddTensor { auto* out_data = output->data(); SetConstant functor; - functor(context, output, static_cast(0)); + functor(context, output, 0.0); const int block_size = 256; dim3 threads(block_size, 1); @@ -140,8 +138,6 @@ struct SelectedRowsAddTensor { template struct SelectedRowsAddTensor; template struct SelectedRowsAddTensor; -template struct SelectedRowsAddTensor; template struct SelectedRowsAddTo { @@ -181,8 +177,6 @@ template struct SelectedRowsAddTo; template struct SelectedRowsAddTo; template struct SelectedRowsAddTo; template struct SelectedRowsAddTo; -template struct SelectedRowsAddTo; namespace { template @@ -235,8 +229,6 @@ template struct SelectedRowsAddToTensor; template struct SelectedRowsAddToTensor; template struct SelectedRowsAddToTensor; template struct SelectedRowsAddToTensor; -template struct SelectedRowsAddToTensor; namespace scatter { @@ -284,7 +276,7 @@ struct MergeAdd { context.GetPlace()); math::SetConstant constant_functor; - constant_functor(context, out.mutable_value(), static_cast(0)); + constant_functor(context, out.mutable_value(), 0.0); auto* out_data = out.mutable_value()->data(); auto* input_data = input.value().data(); @@ -308,7 +300,6 @@ template struct MergeAdd; template struct MergeAdd; template struct MergeAdd; template struct MergeAdd; -template struct MergeAdd; template __global__ void UpdateToTensorKernel(const T* selected_rows, diff --git a/paddle/fluid/operators/math/softmax.cu b/paddle/fluid/operators/math/softmax.cu index 785c4baecbf056d08930f4bb704aec067a2db4a2..3effe776258cb541dbba32f63eda457d917011f4 100644 --- a/paddle/fluid/operators/math/softmax.cu +++ b/paddle/fluid/operators/math/softmax.cu @@ -94,15 +94,12 @@ void SoftmaxGradCUDNNFunctor::operator()( template class SoftmaxCUDNNFunctor; template class SoftmaxCUDNNFunctor; template class SoftmaxCUDNNFunctor; -template class SoftmaxGradCUDNNFunctor; template class SoftmaxGradCUDNNFunctor; template class SoftmaxGradCUDNNFunctor; template class SoftmaxFunctor; template class SoftmaxFunctor; template class SoftmaxFunctor; -template class SoftmaxGradFunctor; template class SoftmaxGradFunctor; template class SoftmaxGradFunctor; diff --git a/paddle/fluid/operators/mean_op.cu b/paddle/fluid/operators/mean_op.cu index 07aa23754f9786c56c0be14c2a71d5290d2cccf7..91e0ab28efc21d4376524c8ecf66b429d51d8847 100644 --- a/paddle/fluid/operators/mean_op.cu +++ b/paddle/fluid/operators/mean_op.cu @@ -12,16 +12,14 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ +#define EIGEN_USE_GPU + #include "paddle/fluid/operators/mean_op.h" -#include "paddle/fluid/platform/float16.h" namespace ops = paddle::operators; -namespace plat = paddle::platform; REGISTER_OP_CUDA_KERNEL( mean, ops::MeanKernel, - ops::MeanKernel, - ops::MeanKernel); + ops::MeanKernel); REGISTER_OP_CUDA_KERNEL( mean_grad, ops::MeanGradKernel, - ops::MeanGradKernel, - ops::MeanGradKernel); + ops::MeanGradKernel); diff --git a/paddle/fluid/operators/mean_op.h b/paddle/fluid/operators/mean_op.h index a41d50ae0b99797800078184f7ffeb366367f493..362e9f9ae8b2f0f77198e3f3939211ae1117b27b 100644 --- a/paddle/fluid/operators/mean_op.h +++ b/paddle/fluid/operators/mean_op.h @@ -55,7 +55,7 @@ class MeanGradKernel : public framework::OpKernel { IG->mutable_data(context.GetPlace()); T ig_size = static_cast(IG->numel()); - Eigen::DSizes bcast(static_cast(ig_size)); + Eigen::DSizes bcast(ig_size); EigenVector::Flatten(*IG).device( *context.template device_context().eigen_device()) = diff --git a/paddle/fluid/operators/mul_op.cu.cc b/paddle/fluid/operators/mul_op.cu.cc index 6c5a83c6a50c463502171f09bbf18e17e43917b5..81f3e42bf412fa4d2cb48405f2f8ee49b6aa0b67 100644 --- a/paddle/fluid/operators/mul_op.cu.cc +++ b/paddle/fluid/operators/mul_op.cu.cc @@ -20,7 +20,6 @@ namespace plat = paddle::platform; REGISTER_OP_CUDA_KERNEL(mul, ops::MulKernel, ops::MulKernel, ops::MulKernel); -REGISTER_OP_CUDA_KERNEL( - mul_grad, ops::MulGradKernel, - ops::MulGradKernel, - ops::MulGradKernel); +REGISTER_OP_CUDA_KERNEL(mul_grad, + ops::MulGradKernel, + ops::MulGradKernel); diff --git a/paddle/fluid/operators/pool_cudnn_op.cu.cc b/paddle/fluid/operators/pool_cudnn_op.cu.cc index 9fdbee818a217842e47c8ab11b84c6d5513ad219..31f083565fddee66aea1485ed71f41b6199f4502 100644 --- a/paddle/fluid/operators/pool_cudnn_op.cu.cc +++ b/paddle/fluid/operators/pool_cudnn_op.cu.cc @@ -174,8 +174,7 @@ REGISTER_OP_KERNEL(pool2d, CUDNN, plat::CUDAPlace, ops::PoolCUDNNOpKernel); REGISTER_OP_KERNEL(pool2d_grad, CUDNN, plat::CUDAPlace, ops::PoolCUDNNGradOpKernel, - ops::PoolCUDNNGradOpKernel, - ops::PoolCUDNNGradOpKernel); + ops::PoolCUDNNGradOpKernel); REGISTER_OP_KERNEL(pool3d, CUDNN, plat::CUDAPlace, ops::PoolCUDNNOpKernel, @@ -183,5 +182,4 @@ REGISTER_OP_KERNEL(pool3d, CUDNN, plat::CUDAPlace, ops::PoolCUDNNOpKernel); REGISTER_OP_KERNEL(pool3d_grad, CUDNN, plat::CUDAPlace, ops::PoolCUDNNGradOpKernel, - ops::PoolCUDNNGradOpKernel, - ops::PoolCUDNNGradOpKernel); + ops::PoolCUDNNGradOpKernel); diff --git a/paddle/fluid/operators/recv_op.cc b/paddle/fluid/operators/recv_op.cc index 4a6ce938a5f337d035b21f562d46daf606236db0..a1f368e8690512cec2db7593aabc0279bbe174eb 100644 --- a/paddle/fluid/operators/recv_op.cc +++ b/paddle/fluid/operators/recv_op.cc @@ -57,6 +57,8 @@ class RecvOp : public framework::OperatorBase { class RecvOpMaker : public framework::OpProtoAndCheckerMaker { public: void Make() { + AddInput("X", "(Any) Dummy inputs, used for control dependency") + .AsDuplicable(); AddOutput("Out", "(Tensor) Variables to get from server.").AsDuplicable(); AddComment(R"DOC( Recv operator diff --git a/paddle/fluid/operators/sampling_id_op.cc b/paddle/fluid/operators/sampling_id_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..724463c95c4a29fb5c00fe791b389d3908771640 --- /dev/null +++ b/paddle/fluid/operators/sampling_id_op.cc @@ -0,0 +1,76 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/operators/sampling_id_op.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; + +class SamplingIdOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + void InferShape(framework::InferShapeContext* ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), + "Input(X) of SamplingIdOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of SamplingIdOp should not be null."); + PADDLE_ENFORCE( + ctx->Attrs().Get("min") < ctx->Attrs().Get("max"), + "min must less then max"); + + auto input_dims = ctx->GetInputDim("X"); + PADDLE_ENFORCE(input_dims.size() == 2, + "Input(X, Filter) should be 2-D tensor."); + + framework::DDim dims = input_dims; + ctx->SetOutputDim("Out", dims); + ctx->ShareLoD("X", "Out"); + } +}; + +class SamplingIdOpMaker : public framework::OpProtoAndCheckerMaker { + public: + void Make() override { + AddInput("X", + "The input tensor of softmax. " + "2-D with shape [batch_size, input_feature_dimensions]."); + AddOutput("Out", "SamplingId data tensor."); + AddComment(R"DOC( +SamplingId Operator. +A layer for sampling id from multinomial distribution from the + input. Sampling one id for one sample.)DOC"); + AddAttr("min", "Minimum value of random. [default 0.0].") + .SetDefault(0.0f); + AddAttr("max", "Maximun value of random. [default 1.0].") + .SetDefault(1.0f); + AddAttr("seed", + "Random seed used for the random number engine. " + "0 means use a seed generated by the system." + "Note that if seed is not 0, this operator will always " + "generate the same random numbers every time. [default 0].") + .SetDefault(0); + } +}; +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OPERATOR(sampling_id, ops::SamplingIdOp, ops::SamplingIdOpMaker, + paddle::framework::EmptyGradOpMaker); + +REGISTER_OP_CPU_KERNEL(sampling_id, paddle::operators::SamplingIdKernel, + paddle::operators::SamplingIdKernel); diff --git a/paddle/fluid/operators/sampling_id_op.cu b/paddle/fluid/operators/sampling_id_op.cu new file mode 100644 index 0000000000000000000000000000000000000000..a4f0470314d00b5e370fd478736b54579c88448c --- /dev/null +++ b/paddle/fluid/operators/sampling_id_op.cu @@ -0,0 +1,19 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/fluid/operators/sampling_id_op.h" + +namespace ops = paddle::operators; +REGISTER_OP_CUDA_KERNEL(sampling_id, paddle::operators::SamplingIdKernel, + paddle::operators::SamplingIdKernel); diff --git a/paddle/fluid/operators/sampling_id_op.h b/paddle/fluid/operators/sampling_id_op.h new file mode 100644 index 0000000000000000000000000000000000000000..f730a9746da56ca82090122193ec54efb774483e --- /dev/null +++ b/paddle/fluid/operators/sampling_id_op.h @@ -0,0 +1,80 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "paddle/fluid/framework/op_registry.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; + +template +class SamplingIdKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + const Tensor* input = context.Input("X"); + const int batch_size = static_cast(input->dims()[0]); + const int width = static_cast(input->dims()[1]); + + PADDLE_ENFORCE_GE(batch_size, 0, + "batch_size(dims[0]) must be nonnegative."); + PADDLE_ENFORCE_GE(width, 0, "width(dims[1]) must be nonnegative."); + + std::vector ins_vector; + framework::TensorToVector(*input, context.device_context(), &ins_vector); + + unsigned int seed = static_cast(context.Attr("seed")); + std::minstd_rand engine; + if (seed == 0) { + seed = std::random_device()(); + } + engine.seed(seed); + std::uniform_real_distribution dist( + static_cast(context.Attr("min")), + static_cast(context.Attr("max"))); + + std::vector ids(batch_size); + for (size_t i = 0; i < batch_size; ++i) { + T r = dist(engine); + int idx = width - 1; + for (int j = 0; j < width; ++j) { + if ((r -= ins_vector[i * width + j]) < 0) { + idx = j; + break; + } + } + ids[i] = ins_vector[i * width + idx]; + } + + std::vector out_dim; + out_dim.push_back(static_cast(batch_size)); + + Tensor* output = context.Output("Out"); + output->Resize(framework::make_ddim(out_dim)); + output->mutable_data(context.GetPlace()); + framework::TensorFromVector(ids, context.device_context(), output); + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/fluid/operators/scale_op.cu b/paddle/fluid/operators/scale_op.cu index d266867046334f95eaaf4b7a9acb3fec20f1e439..04c802da12958a53626f533833c2709110531136 100644 --- a/paddle/fluid/operators/scale_op.cu +++ b/paddle/fluid/operators/scale_op.cu @@ -13,15 +13,11 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/fluid/operators/scale_op.h" -#include "paddle/fluid/platform/float16.h" -namespace plat = paddle::platform; REGISTER_OP_CUDA_KERNEL( scale, paddle::operators::ScaleKernel, paddle::operators::ScaleKernel, paddle::operators::ScaleKernel, paddle::operators::ScaleKernel, - paddle::operators::ScaleKernel); + int64_t>); diff --git a/paddle/fluid/operators/send_barrier_op.cc b/paddle/fluid/operators/send_barrier_op.cc index 1866a86048acbefadcb4d82cd6309cd16f0352d6..14b07649c416ff1b671fc9b5ee4eb956b44570c5 100644 --- a/paddle/fluid/operators/send_barrier_op.cc +++ b/paddle/fluid/operators/send_barrier_op.cc @@ -37,22 +37,19 @@ class SendBarrierOp : public framework::OperatorBase { void RunImpl(const framework::Scope& scope, const platform::Place& place) const override { std::vector eps = Attr>("endpoints"); - bool sync_mode = Attr("sync_mode"); distributed::RPCClient* rpc_client = distributed::RPCClient::GetInstance(); - VLOG(3) << "SendBarrierOp sync_mode:" << sync_mode; + VLOG(3) << "SendBarrierOp sync"; // need to wait before sending send_barrier message PADDLE_ENFORCE(rpc_client->Wait(), "internal error in RPCClient"); - if (sync_mode) { - for (auto& ep : eps) { - VLOG(3) << "send barrier, ep: " << ep; - rpc_client->AsyncSendBatchBarrier(ep); - } - PADDLE_ENFORCE(rpc_client->Wait(), "internal error in RPCClient"); + for (auto& ep : eps) { + VLOG(3) << "send barrier, ep: " << ep; + rpc_client->AsyncSendBatchBarrier(ep); } + PADDLE_ENFORCE(rpc_client->Wait(), "internal error in RPCClient"); } }; @@ -70,7 +67,6 @@ the Parameter Server would knew all variables have been sent. "(string vector, default 127.0.0.1:6164)" "Server endpoints to send variables to.") .SetDefault({"127.0.0.1:6164"}); - AddAttr("sync_mode", "work in sync_mode or not").SetDefault(true); } }; diff --git a/paddle/fluid/operators/send_op.cc b/paddle/fluid/operators/send_op.cc index 3cd42f2d059532b7090e66ce21de8e5cb014adf1..82a70e4bf13247d784371ffdf419c9f792d7f721 100644 --- a/paddle/fluid/operators/send_op.cc +++ b/paddle/fluid/operators/send_op.cc @@ -66,6 +66,8 @@ class SendOpMaker : public framework::OpProtoAndCheckerMaker { void Make() { AddInput("X", "(Tensor, SelectedRows) Input variables to be sent") .AsDuplicable(); + AddOutput("Out", "(Any) Dummy outputs, used for control dependency") + .AsDuplicable(); AddComment(R"DOC( Send operator diff --git a/paddle/fluid/operators/sgd_op.h b/paddle/fluid/operators/sgd_op.h index 2685ce217ee0f0d3e89f3751e96218dcd19bead4..d8b0165b2a89b04bd55671a37d96ee4ba275b2eb 100644 --- a/paddle/fluid/operators/sgd_op.h +++ b/paddle/fluid/operators/sgd_op.h @@ -111,7 +111,7 @@ class SGDOpKernel : public framework::OpKernel { for (size_t i = 0; i < grad.rows().size(); i++) { PADDLE_ENFORCE(grad.rows()[i] < grad.height(), "Input rows index should less than height"); - int64_t id_index = param.Index(grad.rows()[i]); + int64_t id_index = param_out->AutoGrownIndex(grad.rows()[i], false); PADDLE_ENFORCE_GE(id_index, static_cast(0), "id should be in the table"); for (int64_t j = 0; j < grad_row_width; j++) { diff --git a/paddle/fluid/operators/softmax_cudnn_op.cu.cc b/paddle/fluid/operators/softmax_cudnn_op.cu.cc index c2d45c3d2ef82683352afe0e72f0330f7cd753f6..2bdb23e999621b10799b5163f326bc4b66a437e6 100644 --- a/paddle/fluid/operators/softmax_cudnn_op.cu.cc +++ b/paddle/fluid/operators/softmax_cudnn_op.cu.cc @@ -78,5 +78,4 @@ REGISTER_OP_KERNEL(softmax, CUDNN, plat::CUDAPlace, ops::SoftmaxCUDNNKernel, ops::SoftmaxCUDNNKernel); REGISTER_OP_KERNEL(softmax_grad, CUDNN, plat::CUDAPlace, - ops::SoftmaxGradCUDNNKernel, - ops::SoftmaxGradCUDNNKernel); + ops::SoftmaxGradCUDNNKernel); diff --git a/paddle/fluid/operators/softmax_op.cu.cc b/paddle/fluid/operators/softmax_op.cu.cc index 19359b7eef5126d84f0707d39095a74ae4561186..5fb4f011d9b47cebc4a23bcce47eada825263343 100644 --- a/paddle/fluid/operators/softmax_op.cu.cc +++ b/paddle/fluid/operators/softmax_op.cu.cc @@ -23,5 +23,4 @@ REGISTER_OP_CUDA_KERNEL( ops::SoftmaxKernel); REGISTER_OP_CUDA_KERNEL( softmax_grad, ops::SoftmaxGradKernel, - ops::SoftmaxGradKernel, - ops::SoftmaxGradKernel); + ops::SoftmaxGradKernel); diff --git a/paddle/fluid/operators/sum_op.cu b/paddle/fluid/operators/sum_op.cu index db4c2d6c115f04b436db00854ca4b02fea09866b..89bcd1bbc86dc29cb7b98cbef3057a8f98c74555 100644 --- a/paddle/fluid/operators/sum_op.cu +++ b/paddle/fluid/operators/sum_op.cu @@ -11,13 +11,10 @@ limitations under the License. */ #define EIGEN_USE_GPU #include "paddle/fluid/operators/sum_op.h" -#include "paddle/fluid/platform/float16.h" namespace ops = paddle::operators; -namespace plat = paddle::platform; REGISTER_OP_CUDA_KERNEL( sum, ops::SumKernel, ops::SumKernel, ops::SumKernel, - ops::SumKernel, - ops::SumKernel); + ops::SumKernel); diff --git a/paddle/fluid/operators/sum_op.h b/paddle/fluid/operators/sum_op.h index dda6772796c821ffb813e73da0c34370e5339001..49a4afb3a8a19c97e844e66477c6288772ece807 100644 --- a/paddle/fluid/operators/sum_op.h +++ b/paddle/fluid/operators/sum_op.h @@ -46,7 +46,7 @@ class SumKernel : public framework::OpKernel { if (!in_place) { math::SetConstant constant_functor; constant_functor(context.template device_context(), out, - static_cast(0)); + 0.0); } math::SelectedRowsAddToTensor functor; diff --git a/paddle/fluid/operators/top_k_op.cu b/paddle/fluid/operators/top_k_op.cu index 5fc0784f665f9f4a4422ca9b70f7dc6001833a8f..9da8551eb2d7ea66ad434c42b54522432095ce29 100644 --- a/paddle/fluid/operators/top_k_op.cu +++ b/paddle/fluid/operators/top_k_op.cu @@ -11,19 +11,16 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#include #include "paddle/fluid/framework/op_registry.h" #include "paddle/fluid/operators/top_k_op.h" #include "paddle/fluid/platform/assert.h" #include "paddle/fluid/platform/cuda_device_function.h" -#include "paddle/fluid/platform/float16.h" namespace paddle { namespace operators { using Tensor = framework::Tensor; -using paddle::platform::float16; template struct Pair { @@ -35,11 +32,6 @@ struct Pair { id = id; } - __device__ __forceinline__ void clear() { - v = -INFINITY; - id = -1; - } - __device__ __forceinline__ void operator=(const Pair& in) { v = in.v; id = in.id; @@ -61,12 +53,6 @@ struct Pair { int64_t id; }; -template <> -__device__ __forceinline__ void Pair::clear() { - v = platform::raw_uint16_to_float16(0x400); - id = -1; -} - template __device__ __forceinline__ void AddTo(Pair topk[], const Pair& p, int beam_size) { @@ -164,7 +150,7 @@ __device__ __forceinline__ void ThreadGetTopK(Pair topk[], int* beam, if (k < MaxLength - (*beam)) { topk[k] = topk[k + *beam]; } else { - topk[k].clear(); + topk[k].set(-INFINITY, -1); } } if (!(*is_empty)) { @@ -174,7 +160,7 @@ __device__ __forceinline__ void ThreadGetTopK(Pair topk[], int* beam, } *max = topk[MaxLength - 1]; - if ((*max).v == static_cast(-1)) *is_empty = true; + if ((*max).v == -1) *is_empty = true; *beam = 0; } } @@ -195,7 +181,7 @@ __device__ __forceinline__ void ThreadGetTopK(Pair topk[], int* beam, if (k < MaxLength - *beam) { topk[k] = topk[k + *beam]; } else { - topk[k].set(std::numeric_limits::min(), -1); + topk[k].set(-INFINITY, -1); } } if (!(*is_empty)) { @@ -287,7 +273,7 @@ __global__ void KeMatrixTopK(T* output, int output_stride, int64_t* indices, bool firststep = true; for (int k = 0; k < MaxLength; k++) { - topk[k].clear(); + topk[k].set(-INFINITY, -1); } while (k) { ThreadGetTopK(topk, &beam, k, @@ -339,7 +325,5 @@ class TopkOpCUDAKernel : public framework::OpKernel { } // namespace operators } // namespace paddle -REGISTER_OP_CUDA_KERNEL( - top_k, paddle::operators::TopkOpCUDAKernel, - paddle::operators::TopkOpCUDAKernel, - paddle::operators::TopkOpCUDAKernel); +REGISTER_OP_CUDA_KERNEL(top_k, paddle::operators::TopkOpCUDAKernel, + paddle::operators::TopkOpCUDAKernel); diff --git a/paddle/fluid/operators/uniform_random_op.cc b/paddle/fluid/operators/uniform_random_op.cc index edd1baa4ace4e246190afcd12b0716f1dd38e243..5248767c2eeb9388c26d203e64f8b2c68ffe0865 100644 --- a/paddle/fluid/operators/uniform_random_op.cc +++ b/paddle/fluid/operators/uniform_random_op.cc @@ -30,8 +30,10 @@ class CPUUniformRandomKernel : public framework::OpKernel { tensor = out_var->GetMutable(); } else if (out_var->IsType()) { auto shape = ctx.Attr>("shape"); - tensor = out_var->GetMutable()->mutable_value(); + auto* selected_rows = out_var->GetMutable(); + tensor = selected_rows->mutable_value(); tensor->Resize(framework::make_ddim(shape)); + selected_rows->mutable_rows()->reserve(shape[0]); } else { PADDLE_THROW( "uniform_random_op's output only" diff --git a/paddle/fluid/operators/uniform_random_op.cu b/paddle/fluid/operators/uniform_random_op.cu index 2b8039a0c1bea07402435958608ea035ba862c90..e1c7323a30233f4ec4f60e46aa6088ee6d8601b7 100644 --- a/paddle/fluid/operators/uniform_random_op.cu +++ b/paddle/fluid/operators/uniform_random_op.cu @@ -11,14 +11,10 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#include #include #include -#include "paddle/fluid/framework/eigen.h" #include "paddle/fluid/framework/op_registry.h" #include "paddle/fluid/framework/operator.h" -#include "paddle/fluid/platform/float16.h" -#include "paddle/fluid/platform/transform.h" namespace paddle { namespace operators { @@ -40,11 +36,6 @@ struct UniformGenerator { } }; -template -struct CastFunctor { - HOSTDEVICE V operator()(const T& a) { return static_cast(a); } -}; - // It seems that Eigen::Tensor::random in GPU will SEGFAULT. // Use std::random and thrust::random(thrust is a std library in CUDA) to // implement uniform random. @@ -75,50 +66,18 @@ class GPUUniformRandomKernel : public framework::OpKernel { T max = static_cast(context.Attr("max")); thrust::counting_iterator index_sequence_begin(0); int64_t size = tensor->numel(); - if (out_var->IsType() && - std::type_index(typeid(T)) == - std::type_index(typeid(platform::float16))) { - framework::Tensor master_copy_tensor; - master_copy_tensor.Resize(tensor->dims()); - float* master_copy_tensor_data = - master_copy_tensor.mutable_data(context.GetPlace()); - thrust::transform(index_sequence_begin, index_sequence_begin + size, - thrust::device_ptr(master_copy_tensor_data), - UniformGenerator(static_cast(min), - static_cast(max), seed)); - platform::Transform trans; - auto* in_begin = master_copy_tensor.data(); - auto* in_end = in_begin + master_copy_tensor.numel(); - auto* out_begin = tensor->mutable_data(context.GetPlace()); - trans(context.template device_context(), - in_begin, in_end, out_begin, CastFunctor()); - } else { - thrust::transform(index_sequence_begin, index_sequence_begin + size, - thrust::device_ptr(data), - UniformGenerator(min, max, seed)); - } - if (VLOG_IS_ON(5)) { - framework::Tensor cpu_tensor; - framework::TensorCopySync(*tensor, platform::CPUPlace(), &cpu_tensor); - auto& dev_ctx = - *platform::DeviceContextPool::Instance().Get(context.GetPlace()); - dev_ctx.Wait(); - auto x = framework::EigenVector::Flatten(cpu_tensor); - VLOG(5) << "The Uniform output " << x; - } + thrust::transform(index_sequence_begin, index_sequence_begin + size, + thrust::device_ptr(data), + UniformGenerator(min, max, seed)); } }; } // namespace operators } // namespace paddle -namespace plat = paddle::platform; -REGISTER_OP_CUDA_KERNEL( - uniform_random, paddle::operators::GPUUniformRandomKernel, - paddle::operators::GPUUniformRandomKernel, - paddle::operators::GPUUniformRandomKernel); -REGISTER_OP_CUDA_KERNEL( - uniform_random_batch_size_like, - paddle::operators::GPUUniformRandomKernel, - paddle::operators::GPUUniformRandomKernel, - paddle::operators::GPUUniformRandomKernel); +REGISTER_OP_CUDA_KERNEL(uniform_random, + paddle::operators::GPUUniformRandomKernel, + paddle::operators::GPUUniformRandomKernel); +REGISTER_OP_CUDA_KERNEL(uniform_random_batch_size_like, + paddle::operators::GPUUniformRandomKernel, + paddle::operators::GPUUniformRandomKernel); diff --git a/paddle/fluid/platform/cuda_device_function.h b/paddle/fluid/platform/cuda_device_function.h index 23457ff5fe1ec27094113ba0dde26adc64c716b5..9f504d14a8da116648483c0f64cb511b46e6a97e 100644 --- a/paddle/fluid/platform/cuda_device_function.h +++ b/paddle/fluid/platform/cuda_device_function.h @@ -36,7 +36,7 @@ __forceinline__ __device__ T CudaShuffleDownSync(unsigned mask, T val, #if CUDA_VERSION < 9000 return __shfl_down(val, delta, width); #else - return __shfl_down_sync(mask, val, delta, width); + return __shfl_down_sync(mask, val, static_cast(delta), width); #endif } @@ -46,9 +46,16 @@ template <> __forceinline__ __device__ float16 CudaShuffleDownSync(unsigned mask, float16 val, int delta, int width) { - half tmp = static_cast(val); - __shfl_down(tmp, static_cast(delta), width); - return float16(tmp); + return float16( + __shfl_down(static_cast(val), static_cast(delta), width)); +} +#else +template <> +__forceinline__ __device__ float16 CudaShuffleDownSync(unsigned mask, + float16 val, int delta, + int width) { + return float16(__shfl_down_sync(mask, static_cast(val), + static_cast(delta), width)); } #endif diff --git a/paddle/fluid/platform/cuda_helper_test.cu b/paddle/fluid/platform/cuda_helper_test.cu index ca5ca1caeb23f01c047feeccf9c276b2dcd1cb68..ee45afab93d079374aefe366425502890854c28d 100644 --- a/paddle/fluid/platform/cuda_helper_test.cu +++ b/paddle/fluid/platform/cuda_helper_test.cu @@ -13,6 +13,7 @@ // limitations under the License. #include +#include #include #include @@ -123,7 +124,7 @@ void TestUnalign(size_t num, const int shift_bit) { cudaMemcpy(out, d_in2, array_size, cudaMemcpyDeviceToHost); cudaDeviceSynchronize(); for (size_t i = 0; i < num / 2; ++i) { - // NOTE(dzhwinter): the float16 add has small underflow/overflow + // NOTE(dzhwinter): the float16 add has small truncate error. // so we use EXPECT_NEAR to check the result. EXPECT_NEAR(static_cast(out[i]), static_cast(AddFunctor()(r_in1[i], r_in2[i])), @@ -151,3 +152,83 @@ TEST(CudaAtomic, float16Unalign) { TestUnalign(static_cast(1024), /*shift_bit*/ 3); TestUnalign(static_cast(1024 * 1024), /*shift_bit*/ 3); } + +// https://devblogs.nvidia.com/faster-parallel-reductions-kepler/ +template +static __forceinline__ __device__ T WarpReduceSum(T val) { + unsigned mask = 0u; + CREATE_SHFL_MASK(mask, true); + for (int offset = warpSize / 2; offset > 0; offset /= 2) { + val += paddle::platform::CudaShuffleDownSync(mask, val, offset); + } + return val; +} + +template +__forceinline__ __device__ T BlockReduce(T val) { + static __shared__ T shared[32]; // Shared mem for 32 partial sums + int lane = threadIdx.x % warpSize; + int wid = threadIdx.x / warpSize; + + val = WarpReduceSum(val); // Each warp performs partial reduction + + if (lane == 0) shared[wid] = val; // Write reduced value to shared memory + + __syncthreads(); // Wait for all partial reductions + + // read from shared memory only if that warp existed + val = + (threadIdx.x < blockDim.x / warpSize) ? shared[lane] : static_cast(0); + + if (wid == 0) val = WarpReduceSum(val); // Final reduce within first warp + + return val; +} + +template +__global__ void DeviceReduceSum(T* in, T* out, size_t N) { + T sum(0); + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < N; + i += blockDim.x * gridDim.x) { + sum += in[i]; + } + sum = BlockReduce(sum); + __syncthreads(); + if (threadIdx.x == 0) out[blockIdx.x] = sum; +} + +template +void TestReduce(size_t num, float atol = 0.01) { + T* in1; + T *d_in1, *d_in2; + size_t size = sizeof(T) * num; + cudaMalloc(reinterpret_cast(&d_in1), size); + cudaMalloc(reinterpret_cast(&d_in2), sizeof(T)); + in1 = reinterpret_cast(malloc(size)); + std::minstd_rand engine; + std::uniform_real_distribution dist(0.0, 1.0); + for (size_t i = 0; i < num; ++i) { + in1[i] = static_cast(dist(engine)); + } + auto out = std::accumulate(in1, in1 + num, static_cast(0)); + cudaMemcpy(d_in1, in1, size, cudaMemcpyHostToDevice); + cudaDeviceSynchronize(); + DeviceReduceSum<<<1, PADDLE_CUDA_NUM_THREADS>>>(d_in1, d_in2, num); + cudaMemcpy(in1, d_in2, sizeof(T), cudaMemcpyDeviceToHost); + cudaDeviceSynchronize(); + // NOTE(dzhwinter): the float16 add has small underflow/overflow + // so we use EXPECT_NEAR to check the result. + EXPECT_NEAR(static_cast(in1[0]), static_cast(out), atol); + free(in1); + cudaFree(d_in1); + cudaFree(d_in2); +} + +TEST(CudaShuffleSync, float16) { + TestReduce(10); + TestReduce(1000); + + // float16 will overflow or accumulate truncate errors in big size. + TestReduce(10); + TestReduce(100, /*atol error*/ 1.0); +} diff --git a/paddle/fluid/pybind/protobuf.cc b/paddle/fluid/pybind/protobuf.cc index be623703c2480774bb04a6bc0c5b00b699d7bb16..c2137ec6d7df24251432a4dfb8fffc3d3f77194e 100644 --- a/paddle/fluid/pybind/protobuf.cc +++ b/paddle/fluid/pybind/protobuf.cc @@ -205,12 +205,7 @@ void BindBlockDesc(pybind11::module *m) { void BindVarDsec(pybind11::module *m) { pybind11::class_ var_desc(*m, "VarDesc", ""); var_desc - .def("name", - [](pd::VarDesc &self) { - pybind11::bytes name = self.Name(); - return name; - }, - pybind11::return_value_policy::reference) + .def("name", &pd::VarDesc::Name, pybind11::return_value_policy::reference) .def("set_name", &pd::VarDesc::SetName) .def("set_shape", &pd::VarDesc::SetShape) .def("set_shapes", &pd::VarDesc::SetShapes) diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index 7127bb38f6ddf8a55c1741d1f0ef18c8d9067fba..6c58478b0dd0941ab4bf4d573a3c813059650ba8 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -54,6 +54,8 @@ limitations under the License. */ #include "paddle/fluid/platform/gpu_info.h" #endif +#include "pybind11/stl.h" + // disable auto conversion to list in Python PYBIND11_MAKE_OPAQUE(paddle::framework::LoDTensorArray); @@ -247,6 +249,7 @@ PYBIND11_PLUGIN(core) { self.set_rows(new_rows); #endif }) + .def("sync_index", [](SelectedRows &instance) { instance.SyncIndex(); }) .def("rows", [](SelectedRows &self) { auto rows = self.rows(); std::vector new_rows; diff --git a/paddle/scripts/submit_local.sh.in b/paddle/scripts/submit_local.sh.in index 1283de9d957a46b848c7bb6caf9c5f49398468e2..622a2d51049d164b6e8423e4054081f40f190cb9 100755 --- a/paddle/scripts/submit_local.sh.in +++ b/paddle/scripts/submit_local.sh.in @@ -54,7 +54,7 @@ function cpu_config() { if [ $platform == "Linux" ]; then ht=`lscpu |grep "per core"|awk -F':' '{print $2}'|xargs` elif [ $platform == "Darwin" ]; then - if [`sysctl -n hw.physicalcpu` -eq `sysctl -n hw.logicalcpu`]; then + if [ `sysctl -n hw.physicalcpu` -eq `sysctl -n hw.logicalcpu` ]; then # HT is OFF ht=1 fi diff --git a/python/paddle/__init__.py b/python/paddle/__init__.py index 241a07a35297e85763781a42696fd727733459a3..53746afdb25b34b69f89fe0927c877ace62d7d55 100644 --- a/python/paddle/__init__.py +++ b/python/paddle/__init__.py @@ -24,4 +24,5 @@ except ImportError: import paddle.reader import paddle.dataset import paddle.batch +import paddle.compat batch = batch.batch diff --git a/python/paddle/compat.py b/python/paddle/compat.py new file mode 100644 index 0000000000000000000000000000000000000000..50726b6fa1bbbde68a590c86db9344b8f02f79f2 --- /dev/null +++ b/python/paddle/compat.py @@ -0,0 +1,237 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import six +import math + +__all__ = [ + 'long_type', + 'to_text', + 'to_bytes', + 'round', + 'floor_division', + 'get_exception_message', +] + +if six.PY2: + int_type = int + long_type = long +else: + int_type = int + long_type = int + + +# str and bytes related functions +def to_text(obj, encoding='utf-8', inplace=False): + """ + All string in PaddlePaddle should be represented as a literal string. + This function will convert object to a literal string without any encoding. + Especially, if the object type is a list or set container, we will iterate + all items in the object and convert them to literal string. + + In Python3: + Decode the bytes type object to str type with specific encoding + + In Python2: + Decode the str type object to unicode type with specific encoding + + Args: + obj(unicode|str|bytes|list|set) : The object to be decoded. + encoding(str) : The encoding format to decode a string + inplace(bool) : If we change the original object or we create a new one + + Returns: + Decoded result of obj + """ + if obj is None: + return obj + + if isinstance(obj, list): + if inplace: + for i in six.moves.xrange(len(obj)): + obj[i] = _to_text(obj[i], encoding) + return obj + else: + return [_to_text(item, encoding) for item in obj] + elif isinstance(obj, set): + if inplace: + for item in obj: + obj.remove(item) + obj.add(_to_text(item, encoding)) + return obj + else: + return set([_to_text(item, encoding) for item in obj]) + else: + return _to_text(obj, encoding) + + +def _to_text(obj, encoding): + """ + In Python3: + Decode the bytes type object to str type with specific encoding + + In Python2: + Decode the str type object to unicode type with specific encoding, + or we just return the unicode string of object + + Args: + obj(unicode|str|bytes) : The object to be decoded. + encoding(str) : The encoding format + + Returns: + decoded result of obj + """ + if obj is None: + return obj + + if isinstance(obj, six.binary_type): + return obj.decode(encoding) + elif isinstance(obj, six.text_type): + return obj + else: + return six.u(obj) + + +def to_bytes(obj, encoding='utf-8', inplace=False): + """ + All string in PaddlePaddle should be represented as a literal string. + This function will convert object to a bytes with specific encoding. + Especially, if the object type is a list or set container, we will iterate + all items in the object and convert them to bytes. + + In Python3: + Encode the str type object to bytes type with specific encoding + + In Python2: + Encode the unicode type object to str type with specific encoding, + or we just return the 8-bit string of object + + Args: + obj(unicode|str|bytes|list|set) : The object to be encoded. + encoding(str) : The encoding format to encode a string + inplace(bool) : If we change the original object or we create a new one + + Returns: + Decoded result of obj + """ + if obj is None: + return obj + + if isinstance(obj, list): + if inplace: + for i in six.moves.xrange(len(obj)): + obj[i] = _to_bytes(obj[i], encoding) + return obj + else: + return [_to_bytes(item, encoding) for item in obj] + elif isinstance(obj, set): + if inplace: + for item in obj: + obj.remove(item) + obj.add(_to_bytes(item, encoding)) + return obj + else: + return set([_to_bytes(item, encoding) for item in obj]) + else: + return _to_bytes(obj, encoding) + + +def _to_bytes(obj, encoding): + """ + In Python3: + Encode the str type object to bytes type with specific encoding + + In Python2: + Encode the unicode type object to str type with specific encoding, + or we just return the 8-bit string of object + + Args: + obj(unicode|str|bytes) : The object to be encoded. + encoding(str) : The encoding format + + Returns: + encoded result of obj + """ + if obj is None: + return obj + + assert encoding is not None + if isinstance(obj, six.text_type): + return obj.encode(encoding) + elif isinstance(obj, six.binary_type): + return obj + else: + return six.b(obj) + + +# math related functions +def round(x, d=0): + """ + Compatible round which act the same behaviour in Python3. + + Args: + x(float) : The number to round halfway. + + Returns: + round result of x + """ + if six.PY3: + # The official walkaround of round in Python3 is incorrect + # we implement accroding this answer: https://www.techforgeek.info/round_python.html + if x > 0.0: + p = 10**d + return float(math.floor((x * p) + math.copysign(0.5, x))) / p + elif x < 0.0: + p = 10**d + return float(math.ceil((x * p) + math.copysign(0.5, x))) / p + else: + return math.copysign(0.0, x) + else: + import __builtin__ + return __builtin__.round(x, d) + + +def floor_division(x, y): + """ + Compatible division which act the same behaviour in Python3 and Python2, + whose result will be a int value of floor(x / y) in Python3 and value of + (x / y) in Python2. + + Args: + x(int|float) : The number to divide. + y(int|float) : The number to be divided + + Returns: + division result of x // y + """ + return x // y + + +# exception related functions +def get_exception_message(exc): + """ + Get the error message of a specific exception + + Args: + exec(Exception) : The exception to get error message. + + Returns: + the error message of exec + """ + assert exc is not None + + if six.PY2: + return exc.message + else: + return str(exc) diff --git a/python/paddle/dataset/cifar.py b/python/paddle/dataset/cifar.py index f6b4ff8fbd0f83b1d652d37c1b2d04efd3c73cbb..b83fa78c4c65357407b7f884f8c3fe8ef0ccaba8 100644 --- a/python/paddle/dataset/cifar.py +++ b/python/paddle/dataset/cifar.py @@ -28,11 +28,13 @@ images per class. """ +from __future__ import print_function + import itertools import numpy import paddle.dataset.common import tarfile -from six.moves import zip +import six from six.moves import cPickle as pickle __all__ = ['train100', 'test100', 'train10', 'test10', 'convert'] @@ -46,10 +48,11 @@ CIFAR100_MD5 = 'eb9058c3a382ffc7106e4002c42a8d85' def reader_creator(filename, sub_name, cycle=False): def read_batch(batch): - data = batch['data'] - labels = batch.get('labels', batch.get('fine_labels', None)) + data = batch[six.b('data')] + labels = batch.get( + six.b('labels'), batch.get(six.b('fine_labels'), None)) assert labels is not None - for sample, label in zip(data, labels): + for sample, label in six.moves.zip(data, labels): yield (sample / 255.0).astype(numpy.float32), int(label) def reader(): @@ -59,7 +62,11 @@ def reader_creator(filename, sub_name, cycle=False): while True: for name in names: - batch = pickle.load(f.extractfile(name)) + if six.PY2: + batch = pickle.load(f.extractfile(name)) + else: + batch = pickle.load( + f.extractfile(name), encoding='bytes') for item in read_batch(batch): yield item if not cycle: diff --git a/python/paddle/dataset/common.py b/python/paddle/dataset/common.py index 6195cc50df338e83bea1f4ad416529464636a33e..1d7ff582c86a40c8c2086e0de16e89d69c94da60 100644 --- a/python/paddle/dataset/common.py +++ b/python/paddle/dataset/common.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import requests import hashlib import os @@ -85,10 +87,10 @@ def download(url, module_name, md5sum, save_name=None): total_length = r.headers.get('content-length') if total_length is None: - with open(filename, 'w') as f: + with open(filename, 'wb') as f: shutil.copyfileobj(r.raw, f) else: - with open(filename, 'w') as f: + with open(filename, 'wb') as f: dl = 0 total_length = int(total_length) for data in r.iter_content(chunk_size=4096): diff --git a/python/paddle/dataset/conll05.py b/python/paddle/dataset/conll05.py index a97c95d067b876a87f0aa19b2ddd0702a848bd4a..55cfd92721e95d66f1cf38e2f77d9bb6b9e17d7a 100644 --- a/python/paddle/dataset/conll05.py +++ b/python/paddle/dataset/conll05.py @@ -20,15 +20,18 @@ dataset. And a pre-trained word vector model based on Wikipedia corpus is used to initialize SRL model. """ +from __future__ import print_function + import tarfile import gzip import itertools import paddle.dataset.common -from six.moves import zip +import paddle.compat as cpt +from six.moves import zip, range __all__ = ['test, get_dict', 'get_embedding', 'convert'] -DATA_URL = 'http://www.cs.upc.edu/~srlconll/conll05st-tests.tar.gz' +DATA_URL = 'http://paddlemodels.bj.bcebos.com/conll05st/conll05st-tests.tar.gz' DATA_MD5 = '387719152ae52d60422c016e92a742fc' WORDDICT_URL = 'http://paddlemodels.bj.bcebos.com/conll05st%2FwordDict.txt' WORDDICT_MD5 = 'ea7fb7d4c75cc6254716f0177a506baa' @@ -89,8 +92,8 @@ def corpus_reader(data_path, words_name, props_name): labels = [] one_seg = [] for word, label in zip(words_file, props_file): - word = word.strip() - label = label.strip().split() + word = cpt.to_text(word.strip()) + label = cpt.to_text(label.strip().split()) if len(label) == 0: # end of sentence for i in range(len(one_seg[0])): diff --git a/python/paddle/dataset/flowers.py b/python/paddle/dataset/flowers.py index 914dae348bc94d061072543aa14aba2219f4b52d..aa73bbaf7024ec873d9e921205536f12e097ff32 100644 --- a/python/paddle/dataset/flowers.py +++ b/python/paddle/dataset/flowers.py @@ -28,6 +28,9 @@ Graphics and Image Processing (2008) http://www.robots.ox.ac.uk/~vgg/publications/papers/nilsback08.{pdf,ps.gz}. """ + +from __future__ import print_function + import itertools import functools from .common import download @@ -116,7 +119,7 @@ def reader_creator(data_file, for file in open(file_list): file = file.strip() batch = None - with open(file, 'r') as f: + with open(file, 'rb') as f: batch = pickle.load(f) data = batch['data'] labels = batch['label'] diff --git a/python/paddle/dataset/image.py b/python/paddle/dataset/image.py index 3b3d89c93c48d611dccf6f14958c310a6cac1a7b..1cd50bd1802095db07e5618f37b0d42d11e94760 100644 --- a/python/paddle/dataset/image.py +++ b/python/paddle/dataset/image.py @@ -29,10 +29,18 @@ the image layout as follows. formats can be used for training. Noted that, the format should be keep consistent between the training and inference peroid. """ + +from __future__ import print_function + import numpy as np try: import cv2 except ImportError: + import sys + sys.stderr.write( + '''Warning with paddle image module: opencv-python should be imported, + or paddle image module could NOT work; please install opencv-python first.''' + ) cv2 = None import os import tarfile @@ -56,7 +64,7 @@ def batch_images_from_tar(data_file, :type data_file: string :param dataset_name: 'train','test' or 'valid' :type dataset_name: string - :param img2label: a dic with image file name as key + :param img2label: a dic with image file name as key and image's label as value :type img2label: dic :param num_per_batch: image number per batch file @@ -88,7 +96,7 @@ def batch_images_from_tar(data_file, output['data'] = data pickle.dump( output, - open('%s/batch_%d' % (out_path, file_id), 'w'), + open('%s/batch_%d' % (out_path, file_id), 'wb'), protocol=pickle.HIGHEST_PROTOCOL) file_id += 1 data = [] @@ -99,7 +107,7 @@ def batch_images_from_tar(data_file, output['data'] = data pickle.dump( output, - open('%s/batch_%d' % (out_path, file_id), 'w'), + open('%s/batch_%d' % (out_path, file_id), 'wb'), protocol=pickle.HIGHEST_PROTOCOL) with open(meta_file, 'a') as meta: @@ -113,7 +121,7 @@ def load_image_bytes(bytes, is_color=True): Load an color or gray image from bytes array. Example usage: - + .. code-block:: python with open('cat.jpg') as f: @@ -126,6 +134,8 @@ def load_image_bytes(bytes, is_color=True): load and return a gray image. :type is_color: bool """ + assert cv2 is not None + flag = 1 if is_color else 0 file_bytes = np.asarray(bytearray(bytes), dtype=np.uint8) img = cv2.imdecode(file_bytes, flag) @@ -137,7 +147,7 @@ def load_image(file, is_color=True): Load an color or gray image from the file path. Example usage: - + .. code-block:: python im = load_image('cat.jpg') @@ -149,6 +159,8 @@ def load_image(file, is_color=True): load and return a gray image. :type is_color: bool """ + assert cv2 is not None + # cv2.IMAGE_COLOR for OpenCV3 # cv2.CV_LOAD_IMAGE_COLOR for older OpenCV Version # cv2.IMAGE_GRAYSCALE for OpenCV3 @@ -161,27 +173,29 @@ def load_image(file, is_color=True): def resize_short(im, size): - """ + """ Resize an image so that the length of shorter edge is size. Example usage: - + .. code-block:: python im = load_image('cat.jpg') im = resize_short(im, 256) - + :param im: the input image with HWC layout. :type im: ndarray :param size: the shorter edge size of image after resizing. :type size: int """ + assert cv2 is not None + h, w = im.shape[:2] h_new, w_new = size, size if h > w: - h_new = size * h / w + h_new = size * h // w else: - w_new = size * w / h + w_new = size * w // h im = cv2.resize(im, (h_new, w_new), interpolation=cv2.INTER_CUBIC) return im @@ -193,17 +207,17 @@ def to_chw(im, order=(2, 0, 1)): according the order (2,0,1). Example usage: - + .. code-block:: python im = load_image('cat.jpg') im = resize_short(im, 256) im = to_chw(im) - + :param im: the input image with HWC layout. :type im: ndarray :param order: the transposed order. - :type order: tuple|list + :type order: tuple|list """ assert len(im.shape) == len(order) im = im.transpose(order) @@ -215,11 +229,11 @@ def center_crop(im, size, is_color=True): Crop the center of image with size. Example usage: - + .. code-block:: python im = center_crop(im, 224) - + :param im: the input image with HWC layout. :type im: ndarray :param size: the cropping size. @@ -228,8 +242,8 @@ def center_crop(im, size, is_color=True): :type is_color: bool """ h, w = im.shape[:2] - h_start = (h - size) / 2 - w_start = (w - size) / 2 + h_start = (h - size) // 2 + w_start = (w - size) // 2 h_end, w_end = h_start + size, w_start + size if is_color: im = im[h_start:h_end, w_start:w_end, :] @@ -243,11 +257,11 @@ def random_crop(im, size, is_color=True): Randomly crop input image with size. Example usage: - + .. code-block:: python im = random_crop(im, 224) - + :param im: the input image with HWC layout. :type im: ndarray :param size: the cropping size. @@ -272,11 +286,11 @@ def left_right_flip(im, is_color=True): Return the flipped image. Example usage: - + .. code-block:: python im = left_right_flip(im) - + :param im: input image with HWC layout or HW layout for gray image :type im: ndarray :param is_color: whether input image is color or not @@ -299,7 +313,7 @@ def simple_transform(im, resizing, croping and flipping. Example usage: - + .. code-block:: python im = simple_transform(im, 256, 224, True) @@ -314,7 +328,7 @@ def simple_transform(im, :type is_train: bool :param is_color: whether the image is color or not. :type is_color: bool - :param mean: the mean values, which can be element-wise mean values or + :param mean: the mean values, which can be element-wise mean values or mean values per channel. :type mean: numpy array | list """ @@ -332,7 +346,7 @@ def simple_transform(im, im = im.astype('float32') if mean is not None: mean = np.array(mean, dtype=np.float32) - # mean value, may be one value per channel + # mean value, may be one value per channel if mean.ndim == 1 and is_color: mean = mean[:, np.newaxis, np.newaxis] elif mean.ndim == 1: @@ -357,7 +371,7 @@ def load_and_transform(filename, for the transform operations. Example usage: - + .. code-block:: python im = load_and_transform('cat.jpg', 256, 224, True) @@ -372,7 +386,7 @@ def load_and_transform(filename, :type is_train: bool :param is_color: whether the image is color or not. :type is_color: bool - :param mean: the mean values, which can be element-wise mean values or + :param mean: the mean values, which can be element-wise mean values or mean values per channel. :type mean: numpy array | list """ diff --git a/python/paddle/dataset/imdb.py b/python/paddle/dataset/imdb.py index e7fe4e0b7e5832c2bc7ca1307725936a70292c39..fd92523a947689a71b6f9371a3ef4838eb9d194d 100644 --- a/python/paddle/dataset/imdb.py +++ b/python/paddle/dataset/imdb.py @@ -20,11 +20,14 @@ of 25,000 highly polar movie reviews for training, and 25,000 for testing. Besides, this module also provides API for building dictionary. """ +from __future__ import print_function + import paddle.dataset.common import collections import tarfile import re import string +import six __all__ = ['build_dict', 'train', 'test', 'convert'] @@ -42,13 +45,14 @@ def tokenize(pattern): # sequential access of member files, other than # tarfile.extractfile, which does random access and might # destroy hard disks. - tf = next(tarf) + tf = tarf.next() while tf != None: if bool(pattern.match(tf.name)): # newline and punctuations removal and ad-hoc tokenization. - yield tarf.extractfile(tf).read().rstrip("\n\r").translate( - None, string.punctuation).lower().split() - tf = next(tarf) + yield tarf.extractfile(tf).read().rstrip(six.b( + "\n\r")).translate( + None, six.b(string.punctuation)).lower().split() + tf = tarf.next() def build_dict(pattern, cutoff): @@ -62,11 +66,11 @@ def build_dict(pattern, cutoff): word_freq[word] += 1 # Not sure if we should prune less-frequent words here. - word_freq = [x for x in list(word_freq.items()) if x[1] > cutoff] + word_freq = [x for x in six.iteritems(word_freq) if x[1] > cutoff] dictionary = sorted(word_freq, key=lambda x: (-x[1], x[0])) words, _ = list(zip(*dictionary)) - word_idx = dict(list(zip(words, list(range(len(words)))))) + word_idx = dict(list(zip(words, six.moves.range(len(words))))) word_idx[''] = len(words) return word_idx diff --git a/python/paddle/dataset/imikolov.py b/python/paddle/dataset/imikolov.py index bc007c9d3c8e2f1e4ff091f7c2c93eacbbe8d0e0..8eecb75231de450282fa4838aca5b293cc2101d1 100644 --- a/python/paddle/dataset/imikolov.py +++ b/python/paddle/dataset/imikolov.py @@ -14,13 +14,17 @@ """ imikolov's simple dataset. -This module will download dataset from +This module will download dataset from http://www.fit.vutbr.cz/~imikolov/rnnlm/ and parse training set and test set into paddle reader creators. """ + +from __future__ import print_function + import paddle.dataset.common import collections import tarfile +import six __all__ = ['train', 'test', 'build_dict', 'convert'] @@ -64,11 +68,13 @@ def build_dict(min_word_freq=50): # remove for now, since we will set it as last index del word_freq[''] - word_freq = [x for x in list(word_freq.items()) if x[1] > min_word_freq] + word_freq = [ + x for x in six.iteritems(word_freq) if x[1] > min_word_freq + ] word_freq_sorted = sorted(word_freq, key=lambda x: (-x[1], x[0])) words, _ = list(zip(*word_freq_sorted)) - word_idx = dict(list(zip(words, list(range(len(words)))))) + word_idx = dict(list(zip(words, six.moves.range(len(words))))) word_idx[''] = len(words) return word_idx @@ -89,7 +95,7 @@ def reader_creator(filename, word_idx, n, data_type): l = [''] + l.strip().split() + [''] if len(l) >= n: l = [word_idx.get(w, UNK) for w in l] - for i in range(n, len(l) + 1): + for i in six.moves.range(n, len(l) + 1): yield tuple(l[i - n:i]) elif DataType.SEQ == data_type: l = l.strip().split() diff --git a/python/paddle/dataset/mnist.py b/python/paddle/dataset/mnist.py index ffa9008c80129b80b3807dbab37bc198e59cf5a2..38addd0cfd9bd0afde7eefc57f2111b717b7e636 100644 --- a/python/paddle/dataset/mnist.py +++ b/python/paddle/dataset/mnist.py @@ -17,10 +17,15 @@ MNIST dataset. This module will download dataset from http://yann.lecun.com/exdb/mnist/ and parse training set and test set into paddle reader creators. """ + +from __future__ import print_function + import paddle.dataset.common import subprocess import numpy import platform +import tempfile +from six.moves import range __all__ = ['train', 'test', 'convert'] URL_PREFIX = 'http://yann.lecun.com/exdb/mnist/' @@ -45,23 +50,28 @@ def reader_creator(image_filename, label_filename, buffer_size): # According to http://stackoverflow.com/a/38061619/724872, we # cannot use standard package gzip here. - m = subprocess.Popen([zcat_cmd, image_filename], stdout=subprocess.PIPE) - m.stdout.read(16) # skip some magic bytes + tmp_image_file = tempfile.TemporaryFile(prefix='paddle_dataset') + m = subprocess.Popen( + [zcat_cmd, image_filename], stdout=tmp_image_file).communicate() + tmp_image_file.seek(16) # skip some magic bytes - l = subprocess.Popen([zcat_cmd, label_filename], stdout=subprocess.PIPE) - l.stdout.read(8) # skip some magic bytes + # Python3 will not take stdout as file + tmp_label_file = tempfile.TemporaryFile(prefix='paddle_dataset') + l = subprocess.Popen( + [zcat_cmd, label_filename], stdout=tmp_label_file).communicate() + tmp_label_file.seek(8) # skip some magic bytes try: # reader could be break. while True: labels = numpy.fromfile( - l.stdout, 'ubyte', count=buffer_size).astype("int") + tmp_label_file, 'ubyte', count=buffer_size).astype("int") if labels.size != buffer_size: break # numpy.fromfile returns empty slice after EOF. images = numpy.fromfile( - m.stdout, 'ubyte', count=buffer_size * 28 * 28).reshape( - (buffer_size, 28 * 28)).astype('float32') + tmp_image_file, 'ubyte', count=buffer_size * 28 * + 28).reshape((buffer_size, 28 * 28)).astype('float32') images = images / 255.0 * 2.0 - 1.0 diff --git a/python/paddle/dataset/movielens.py b/python/paddle/dataset/movielens.py index 056ec2178607329dd6daa1764820c2312bbaed59..c98e0019f7ab5fc2723e8df919257a59af7c9e5d 100644 --- a/python/paddle/dataset/movielens.py +++ b/python/paddle/dataset/movielens.py @@ -22,11 +22,15 @@ set and test set into paddle reader creators. """ +from __future__ import print_function + import zipfile import paddle.dataset.common import re import random import functools +import six +import paddle.compat as cpt __all__ = [ 'train', 'test', 'get_movie_title_dict', 'max_movie_id', 'max_user_id', @@ -112,6 +116,7 @@ def __initialize_meta_info__(): categories_set = set() with package.open('ml-1m/movies.dat') as movie_file: for i, line in enumerate(movie_file): + line = cpt.to_text(line, encoding='latin') movie_id, title, categories = line.strip().split('::') categories = categories.split('|') for c in categories: @@ -136,6 +141,7 @@ def __initialize_meta_info__(): USER_INFO = dict() with package.open('ml-1m/users.dat') as user_file: for line in user_file: + line = cpt.to_text(line, encoding='latin') uid, gender, age, job, _ = line.strip().split("::") USER_INFO[int(uid)] = UserInfo( index=uid, gender=gender, age=age, job_id=job) @@ -148,6 +154,7 @@ def __reader__(rand_seed=0, test_ratio=0.1, is_test=False): with zipfile.ZipFile(file=fn) as package: with package.open('ml-1m/ratings.dat') as rating: for line in rating: + line = cpt.to_text(line, encoding='latin') if (rand.random() < test_ratio) == is_test: uid, mov_id, rating, _ = line.strip().split("::") uid = int(uid) @@ -187,7 +194,7 @@ def max_movie_id(): Get the maximum value of movie id. """ __initialize_meta_info__() - return reduce(__max_index_info__, list(MOVIE_INFO.values())).index + return six.moves.reduce(__max_index_info__, list(MOVIE_INFO.values())).index def max_user_id(): @@ -195,7 +202,7 @@ def max_user_id(): Get the maximum value of user id. """ __initialize_meta_info__() - return reduce(__max_index_info__, list(USER_INFO.values())).index + return six.moves.reduce(__max_index_info__, list(USER_INFO.values())).index def __max_job_id_impl__(a, b): @@ -210,7 +217,8 @@ def max_job_id(): Get the maximum value of job id. """ __initialize_meta_info__() - return reduce(__max_job_id_impl__, list(USER_INFO.values())).job_id + return six.moves.reduce(__max_job_id_impl__, + list(USER_INFO.values())).job_id def movie_categories(): diff --git a/python/paddle/dataset/mq2007.py b/python/paddle/dataset/mq2007.py index cc4d088316dfd490dc9d6b247c66c2495cedf2c3..d5740f30c898d5704636e1de9b2e1137d12e3c35 100644 --- a/python/paddle/dataset/mq2007.py +++ b/python/paddle/dataset/mq2007.py @@ -23,6 +23,8 @@ http://research.microsoft.com/en-us/um/beijing/projects/letor/LETOR4.0/Data/MQ20 """ +from __future__ import print_function + import os import functools import rarfile diff --git a/python/paddle/dataset/sentiment.py b/python/paddle/dataset/sentiment.py index 953ada057bc114ebbfe39011d2fd3b5b7a2b0d37..22d867beea25c97efcbcb6f61ca2b7a7777f9c5c 100644 --- a/python/paddle/dataset/sentiment.py +++ b/python/paddle/dataset/sentiment.py @@ -20,6 +20,9 @@ The script fetch and preprocess movie_reviews data set that provided by NLTK TODO(yuyang18): Complete dataset. """ +from __future__ import print_function + +import six import collections from itertools import chain @@ -64,7 +67,7 @@ def get_word_dict(): for field in movie_reviews.fileids(category): for words in movie_reviews.words(field): word_freq_dict[words] += 1 - words_sort_list = list(word_freq_dict.items()) + words_sort_list = six.iteritems(word_freq_dict) words_sort_list.sort(cmp=lambda a, b: b[1] - a[1]) for index, word in enumerate(words_sort_list): words_freq_sorted.append((word[0], index)) diff --git a/python/paddle/dataset/tests/cifar_test.py b/python/paddle/dataset/tests/cifar_test.py index 839125b09dd5c6432e3572374a7345a77a43f7cf..8e514f0fd9a18a7d512430111a8a11b942950d20 100644 --- a/python/paddle/dataset/tests/cifar_test.py +++ b/python/paddle/dataset/tests/cifar_test.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.dataset.cifar import unittest diff --git a/python/paddle/dataset/tests/common_test.py b/python/paddle/dataset/tests/common_test.py index 777cd06a19726f8ad73774c958c8cb512808a3aa..0ce7d83f374f8c09f68527473418de8ce84c36b1 100644 --- a/python/paddle/dataset/tests/common_test.py +++ b/python/paddle/dataset/tests/common_test.py @@ -12,10 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.dataset.common import unittest import tempfile import glob +from six.moves import range class TestCommon(unittest.TestCase): diff --git a/python/paddle/dataset/tests/flowers_test.py b/python/paddle/dataset/tests/flowers_test.py index 06260fd796ce0271b7cec2f42a8a5a255a02dc24..06a0a7761cfa10ca3211297d176e3e909332e271 100644 --- a/python/paddle/dataset/tests/flowers_test.py +++ b/python/paddle/dataset/tests/flowers_test.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.dataset.flowers import unittest diff --git a/python/paddle/dataset/tests/imdb_test.py b/python/paddle/dataset/tests/imdb_test.py index 539da049449cd273db0a9e260851ed40e1be0f04..415947e3477f2e5b9979588528f7cb6f799acf6a 100644 --- a/python/paddle/dataset/tests/imdb_test.py +++ b/python/paddle/dataset/tests/imdb_test.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.dataset.imdb import unittest import re diff --git a/python/paddle/dataset/tests/imikolov_test.py b/python/paddle/dataset/tests/imikolov_test.py index 50f50d947d221686d6308a6ed44cbcff3b10c6f5..1f78a5dd4d1a09c3192bc8c144c5a78c8a214f3a 100644 --- a/python/paddle/dataset/tests/imikolov_test.py +++ b/python/paddle/dataset/tests/imikolov_test.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.dataset.imikolov import unittest diff --git a/python/paddle/dataset/tests/mnist_test.py b/python/paddle/dataset/tests/mnist_test.py index 8ada19d3f2ee13e194d08e19a4b86b558c69a0a7..fbb5d926494e38283e78ec15381530e50f32915d 100644 --- a/python/paddle/dataset/tests/mnist_test.py +++ b/python/paddle/dataset/tests/mnist_test.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.dataset.mnist import unittest diff --git a/python/paddle/dataset/tests/mq2007_test.py b/python/paddle/dataset/tests/mq2007_test.py index fba388724a8e84591df7150b41f8ea39a850fc31..ee0897e88f0d7ad089b7f7b68d31d04d96fa3e9d 100644 --- a/python/paddle/dataset/tests/mq2007_test.py +++ b/python/paddle/dataset/tests/mq2007_test.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.dataset.mq2007 import unittest diff --git a/python/paddle/dataset/tests/test_image.py b/python/paddle/dataset/tests/test_image.py index 8bd56607ae1998935a3b3aaa0e3279515c2a540c..32d2eb17ae673e72bbee2fc3bb5e3b05f1b20074 100644 --- a/python/paddle/dataset/tests/test_image.py +++ b/python/paddle/dataset/tests/test_image.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np diff --git a/python/paddle/dataset/tests/test_sentiment.py b/python/paddle/dataset/tests/test_sentiment.py index 37326517f7b39fb74c694684eb8a547d5f021946..bb9830132e987370022df3192060de3e908a2e85 100644 --- a/python/paddle/dataset/tests/test_sentiment.py +++ b/python/paddle/dataset/tests/test_sentiment.py @@ -15,6 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import nltk import paddle.dataset.sentiment as st diff --git a/python/paddle/dataset/tests/voc2012_test.py b/python/paddle/dataset/tests/voc2012_test.py index 0d285461a8ae8a9cc69fbec0dcf5efc106b594f0..cddeb91cab2c0f90567f28f8258156e2bb654abc 100644 --- a/python/paddle/dataset/tests/voc2012_test.py +++ b/python/paddle/dataset/tests/voc2012_test.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.dataset.voc2012 import unittest diff --git a/python/paddle/dataset/tests/wmt16_test.py b/python/paddle/dataset/tests/wmt16_test.py index 8b949d8bf5212d51016a33da322095bde2038200..be121bb10121967590c9e136e9a1964a133e934b 100644 --- a/python/paddle/dataset/tests/wmt16_test.py +++ b/python/paddle/dataset/tests/wmt16_test.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.dataset.wmt16 import unittest diff --git a/python/paddle/dataset/uci_housing.py b/python/paddle/dataset/uci_housing.py index 410ca7af0d6d1dc26acbf92fce5e49fce7d3a3bb..f87fdcc4f0f3c42a92bcff5ddcd532c3108565b1 100644 --- a/python/paddle/dataset/uci_housing.py +++ b/python/paddle/dataset/uci_housing.py @@ -19,9 +19,10 @@ https://archive.ics.uci.edu/ml/machine-learning-databases/housing/ and parse training set and test set into paddle reader creators. """ -import os +from __future__ import print_function import numpy as np +import six import tempfile import tarfile import os @@ -70,11 +71,11 @@ def load_data(filename, feature_num=14, ratio=0.8): return data = np.fromfile(filename, sep=' ') - data = data.reshape(data.shape[0] / feature_num, feature_num) + data = data.reshape(data.shape[0] // feature_num, feature_num) maximums, minimums, avgs = data.max(axis=0), data.min(axis=0), data.sum( axis=0) / data.shape[0] feature_range(maximums[:-1], minimums[:-1]) - for i in range(feature_num - 1): + for i in six.moves.range(feature_num - 1): data[:, i] = (data[:, i] - avgs[i]) / (maximums[i] - minimums[i]) offset = int(data.shape[0] * ratio) UCI_TRAIN_DATA = data[:offset] @@ -137,7 +138,7 @@ def predict_reader(): It returns just one tuple data to do inference. :return: one tuple data - :rtype: tuple + :rtype: tuple """ global UCI_TEST_DATA load_data(paddle.dataset.common.download(URL, 'uci_housing', MD5)) diff --git a/python/paddle/dataset/voc2012.py b/python/paddle/dataset/voc2012.py index 9c945574dbcc15f5cee370206ed7e70ba8ab5014..50688937654ae72b77e1439f21a0d7c847d5e135 100644 --- a/python/paddle/dataset/voc2012.py +++ b/python/paddle/dataset/voc2012.py @@ -19,6 +19,8 @@ to training/test sets has been maintained. The total number of images with segmentation has been increased from 7,062 to 9,993. """ +from __future__ import print_function + import tarfile import io import numpy as np diff --git a/python/paddle/dataset/wmt14.py b/python/paddle/dataset/wmt14.py index 7504474591fa486428d0310f10387818c4cf0300..f8c1a33574e642b21feb6843d115b7f4205ef250 100644 --- a/python/paddle/dataset/wmt14.py +++ b/python/paddle/dataset/wmt14.py @@ -19,10 +19,15 @@ http://paddlepaddle.cdn.bcebos.com/demo/wmt_shrinked_data/wmt14.tgz and parse training set and test set into paddle reader creators. """ + +from __future__ import print_function + +import six import tarfile import gzip import paddle.dataset.common +import paddle.compat as cpt __all__ = [ 'train', @@ -53,7 +58,7 @@ def __read_to_dict(tar_file, dict_size): out_dict = dict() for line_count, line in enumerate(fd): if line_count < size: - out_dict[line.strip()] = line_count + out_dict[cpt.to_text(line.strip())] = line_count else: break return out_dict @@ -84,7 +89,7 @@ def reader_creator(tar_file, file_name, dict_size): ] for name in names: for line in f.extractfile(name): - line_split = line.strip().split('\t') + line_split = line.strip().split(six.b('\t')) if len(line_split) != 2: continue src_seq = line_split[0] # one source sequence @@ -153,8 +158,8 @@ def get_dict(dict_size, reverse=True): tar_file = paddle.dataset.common.download(URL_TRAIN, 'wmt14', MD5_TRAIN) src_dict, trg_dict = __read_to_dict(tar_file, dict_size) if reverse: - src_dict = {v: k for k, v in list(src_dict.items())} - trg_dict = {v: k for k, v in list(trg_dict.items())} + src_dict = {v: k for k, v in six.iteritems(src_dict)} + trg_dict = {v: k for k, v in six.iteritems(trg_dict)} return src_dict, trg_dict diff --git a/python/paddle/dataset/wmt16.py b/python/paddle/dataset/wmt16.py index 4e3c466c38e402cc574e93ef3a5935edf8f9dd3b..f30dcd518ea6c0c685d027ede3ad6e0a1cb0c82c 100644 --- a/python/paddle/dataset/wmt16.py +++ b/python/paddle/dataset/wmt16.py @@ -28,12 +28,16 @@ Multi30K: Multilingual English-German Image Descriptions. } """ +from __future__ import print_function + import os +import six import tarfile import gzip from collections import defaultdict import paddle.dataset.common +import paddle.compat as cpt __all__ = [ "train", @@ -60,7 +64,7 @@ def __build_dict(tar_file, dict_size, save_path, lang): word_dict = defaultdict(int) with tarfile.open(tar_file, mode="r") as f: for line in f.extractfile("wmt16/train"): - line_split = line.strip().split("\t") + line_split = line.strip().split(six.b("\t")) if len(line_split) != 2: continue sen = line_split[0] if lang == "en" else line_split[1] for w in sen.split(): @@ -70,8 +74,7 @@ def __build_dict(tar_file, dict_size, save_path, lang): fout.write("%s\n%s\n%s\n" % (START_MARK, END_MARK, UNK_MARK)) for idx, word in enumerate( sorted( - iter(list(word_dict.items())), - key=lambda x: x[1], + six.iteritems(word_dict), key=lambda x: x[1], reverse=True)): if idx + 3 == dict_size: break fout.write("%s\n" % (word[0])) @@ -81,16 +84,16 @@ def __load_dict(tar_file, dict_size, lang, reverse=False): dict_path = os.path.join(paddle.dataset.common.DATA_HOME, "wmt16/%s_%d.dict" % (lang, dict_size)) if not os.path.exists(dict_path) or ( - len(open(dict_path, "r").readlines()) != dict_size): + len(open(dict_path, "rb").readlines()) != dict_size): __build_dict(tar_file, dict_size, dict_path, lang) word_dict = {} - with open(dict_path, "r") as fdict: + with open(dict_path, "rb") as fdict: for idx, line in enumerate(fdict): if reverse: - word_dict[idx] = line.strip() + word_dict[idx] = cpt.to_text(line.strip()) else: - word_dict[line.strip()] = idx + word_dict[cpt.to_text(line.strip())] = idx return word_dict @@ -120,7 +123,7 @@ def reader_creator(tar_file, file_name, src_dict_size, trg_dict_size, src_lang): with tarfile.open(tar_file, mode="r") as f: for line in f.extractfile(file_name): - line_split = line.strip().split("\t") + line_split = line.strip().split(six.b("\t")) if len(line_split) != 2: continue src_words = line_split[src_col].split() diff --git a/python/paddle/fluid/average.py b/python/paddle/fluid/average.py index 358e24df31bb517604481bb48b9180e579f8460d..42cd3b36420ef5a17a9a7d981978ba8869809936 100644 --- a/python/paddle/fluid/average.py +++ b/python/paddle/fluid/average.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import warnings """ diff --git a/python/paddle/fluid/backward.py b/python/paddle/fluid/backward.py index fd6a76dd0cfa347328d87093884e5cd324395497..a415cdbeaaae2a3bb4a137744205e3fe7366a78f 100644 --- a/python/paddle/fluid/backward.py +++ b/python/paddle/fluid/backward.py @@ -12,11 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from paddle.fluid import framework as framework from . import core import collections import copy import six +from .. import compat as cpt from . import unique_name __all__ = ['append_backward'] @@ -45,13 +48,13 @@ def _create_op_desc_(op_type, inputs, outputs, attrs): """ op_desc = core.OpDesc() op_desc.set_type(op_type) - for para, args in list(inputs.items()): + for para, args in six.iteritems(inputs): op_desc.set_input( para, list( map(lambda arg: arg.decode() if isinstance(arg, six.binary_type) else arg, args))) - for para, args in list(outputs.items()): + for para, args in six.iteritems(outputs): op_desc.set_output( para, list( @@ -63,7 +66,7 @@ def _create_op_desc_(op_type, inputs, outputs, attrs): if op_role_attr_name not in attrs: attrs[ op_role_attr_name] = core.op_proto_and_checker_maker.OpRole.Backward - for name, val in list(attrs.items()): + for name, val in six.iteritems(attrs): if isinstance(val, framework.Block): op_desc.set_block_attr(name, val.desc) else: @@ -75,10 +78,10 @@ def _infer_var_data_type_(grad_var_name, block): """ Infer the data type of given grad variable """ - grad_var = block.desc.find_var(grad_var_name.encode("ascii")) - fwd_name = _strip_grad_suffix_(grad_var_name.encode("ascii")) - if block.desc.has_var_recursive(fwd_name): - fwd_var = block.desc.find_var_recursive(fwd_name.encode("ascii")) + grad_var = block.desc.find_var(cpt.to_bytes(grad_var_name)) + fwd_name = _strip_grad_suffix_(grad_var_name) + if block.desc.has_var_recursive(cpt.to_bytes(fwd_name)): + fwd_var = block.desc.find_var_recursive(cpt.to_bytes(fwd_name)) grad_var.set_dtype(fwd_var.dtype()) else: grad_var.set_dtype(core.VarDesc.VarType.FP32) @@ -102,8 +105,10 @@ def _some_in_set_(cands, s): """ if len(cands) == 0: return False - for c in cands: - if c in s: + literal_set = cpt.to_text(s) + literal_cands = cpt.to_text(cands) + for c in literal_cands: + if c in literal_set: return True return False @@ -114,9 +119,8 @@ def _strip_grad_suffix_(name): e.g. x@GRAD ==> x y@GRAD@RENAME@1 ==> y """ - if isinstance(name, six.text_type): - name = name.encode() - pos = name.find(six.b(core.grad_var_suffix())) + name = cpt.to_text(name) + pos = name.find(core.grad_var_suffix()) return name[:pos] if pos != -1 else name @@ -125,9 +129,7 @@ def _append_grad_suffix_(name): Append grad suffix to the given variable name e.g. x ==> x@GRAD """ - if isinstance(name, six.text_type): - name = name.encode() - return name + six.b(core.grad_var_suffix()) + return cpt.to_text(name) + core.grad_var_suffix() def _addup_repetitive_outputs_(op_descs): @@ -187,7 +189,7 @@ def _addup_repetitive_outputs_(op_descs): op_desc.set_output(param_name, arg_names) renamed_vars[var_name].append(new_name) - for var_name, inputs in list(renamed_vars.items()): + for var_name, inputs in six.iteritems(renamed_vars): if len(inputs) > 1: pending_sum_ops.append( (_create_op_desc_("sum", {"X": inputs}, {"Out": [var_name]}, @@ -243,7 +245,7 @@ from .proto import framework_pb2 def serialize_op_decs(op_desc): protostr = op_desc.serialize_to_string() - proto = framework_pb2.OpDesc.FromString(str(protostr)) + proto = framework_pb2.OpDesc.FromString(six.binary_type(protostr)) return proto.__str__() @@ -364,7 +366,7 @@ def _append_backward_ops_(block, # Getting op's corresponding grad_op grad_op_desc, op_grad_to_var = core.get_grad_op_desc( - op.desc, no_grad_dict[block.idx], grad_sub_block_list) + op.desc, cpt.to_text(no_grad_dict[block.idx]), grad_sub_block_list) grad_op_descs.extend(grad_op_desc) grad_to_var.update(op_grad_to_var) @@ -411,11 +413,10 @@ def _append_backward_vars_(block, start_op_idx, grad_to_var, grad_info_map): new_vars = set() # create new gradient variables for grad_var_name in op_desc.output_arg_names(): - grad_var_name = grad_var_name.encode("ascii") - if block.desc.has_var_recursive( - grad_var_name) or grad_var_name == core.empty_var_name(): + if block.desc.has_var_recursive(cpt.to_bytes( + grad_var_name)) or grad_var_name == core.empty_var_name(): continue - block.desc.var(grad_var_name) + block.desc.var(cpt.to_bytes(grad_var_name)) new_vars.add(grad_var_name) if grad_var_name not in grad_to_var: continue @@ -445,7 +446,7 @@ def _rename_grad_(block, start_op_idx, grad_to_var, target_grad_map): op_desc.rename_output(name, new_name) var_map[name] = new_name - for g, ng in list(var_map.items()): + for g, ng in six.iteritems(var_map): if g in grad_to_var: grad_to_var[ng] = grad_to_var[g] grad_to_var.pop(g) @@ -595,11 +596,12 @@ def append_backward(loss, parameter_list=None, no_grad_set=None, parameters = parameter_list else: params = program.global_block().all_parameters() + program.global_block().iter_parameters() parameters = [param.name for param in params] params_and_grads = [] for param in parameters: - if param not in grad_info_map: + if cpt.to_text(param) not in grad_info_map: continue grad_info = grad_info_map[param] grad_block = grad_info[1] diff --git a/python/paddle/fluid/clip.py b/python/paddle/fluid/clip.py index 4b0a792f784fffcce3f911d3e7448b472d39f8e1..ba7ba3b5e983bfbaa82fc752f4821e8a934dfb8c 100644 --- a/python/paddle/fluid/clip.py +++ b/python/paddle/fluid/clip.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import copy import six diff --git a/python/paddle/fluid/concurrency.py b/python/paddle/fluid/concurrency.py index 676a52a917dd1f9700ec38de32932938ec339be5..b4a06f23a6f2713b665bdd42919925e4a0475a82 100644 --- a/python/paddle/fluid/concurrency.py +++ b/python/paddle/fluid/concurrency.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from .layers.control_flow import BlockGuard, equal from .framework import Operator from .layer_helper import LayerHelper, unique_name diff --git a/python/paddle/fluid/contrib/__init__.py b/python/paddle/fluid/contrib/__init__.py index 58f2da1c3ba2f84602e7a18c7b1c78d1f0d2ede1..5607f11932bbe6aff548be316dc39b4636e079f4 100644 --- a/python/paddle/fluid/contrib/__init__.py +++ b/python/paddle/fluid/contrib/__init__.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from . import decoder from .decoder import * from . import memory_usage_calc diff --git a/python/paddle/fluid/contrib/decoder/__init__.py b/python/paddle/fluid/contrib/decoder/__init__.py index 6343c1543d206f82e605c5c986fa91d70c467113..9f973fd3c9af60a0c9a2ba5225a616671545436b 100644 --- a/python/paddle/fluid/contrib/decoder/__init__.py +++ b/python/paddle/fluid/contrib/decoder/__init__.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from . import beam_search_decoder from .beam_search_decoder import * diff --git a/python/paddle/fluid/contrib/decoder/beam_search_decoder.py b/python/paddle/fluid/contrib/decoder/beam_search_decoder.py index d268a948f7a2cf038a419c95521b81088ed8215f..f2b7ac8375af25beed562b8279b6044f11c09d44 100644 --- a/python/paddle/fluid/contrib/decoder/beam_search_decoder.py +++ b/python/paddle/fluid/contrib/decoder/beam_search_decoder.py @@ -20,6 +20,8 @@ without using the low level API such as while ops. This API is still under active development and may change drastically. """ +from __future__ import print_function + import contextlib import numpy as np import six diff --git a/python/paddle/fluid/contrib/memory_usage_calc.py b/python/paddle/fluid/contrib/memory_usage_calc.py index 5da846edb63c28efd791fdfac4046cfa56c24181..09721e430b7e5bb6b9891d5272ca54475baf6157 100644 --- a/python/paddle/fluid/contrib/memory_usage_calc.py +++ b/python/paddle/fluid/contrib/memory_usage_calc.py @@ -14,12 +14,16 @@ """ This module privides a memory usage calculate function for user. The purpose of this API is to allow users to estimate memory usage of -a program under a special batch size, then user can set appropriate -batch size to fully utilize a GPU. +a program under a special batch size, then user can set appropriate +batch size to fully utilize a GPU. This API is still under active development and may change drastically. """ +from __future__ import print_function + +import six + from .. import core from ..framework import Program, Variable @@ -45,15 +49,15 @@ def memory_usage(program, batch_size): Args: program(Program): The current Program. - batch_size(int): The current input data batch_size. - + batch_size(int): The current input data batch_size. + Returns: min_total_memory(float): the estimate memory usage lower bound. max_total_memory(float): the estimate memory usage upper bound. unit_str(string): the unit of estimate usage result. - + Examples: - + >>> import paddle.fluid as fluid >>> lower_usage, upper_usage, unit = fluid.contrib.memory_usage( fluid.default_main_program(), batch_size=10) @@ -72,7 +76,7 @@ def memory_usage(program, batch_size): # Get the var_name list of first block and calculate total_memory = 0.0 - for var in program.global_block().vars.itervalues(): + for var in six.itervalues(program.global_block().vars): data_count = 1 for x in var.shape: if x == -1: @@ -81,10 +85,10 @@ def memory_usage(program, batch_size): data_count *= x var_memory = data_count * dtype_to_size[var.dtype] if DEBUG: - print "%s memory usage: %d" % (var.name, var_memory) + print("%s memory usage: %d" % (var.name, var_memory)) total_memory += var_memory if DEBUG: - print "total memory usage: %.2f" % (total_memory) + print("total memory usage: %.2f" % (total_memory)) # Convert appropriate unit unit_str = "B" diff --git a/python/paddle/fluid/data_feeder.py b/python/paddle/fluid/data_feeder.py index 9452cf0e2a3a2eddb761149466bfc1ee3d23dfd9..631bbfe1fe59ddd9cd315fb64ca32e1e125b0e8d 100644 --- a/python/paddle/fluid/data_feeder.py +++ b/python/paddle/fluid/data_feeder.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from . import core import numpy import os diff --git a/python/paddle/fluid/debugger.py b/python/paddle/fluid/debugger.py index b7a92cf044900acdd41ede378dd68aa2d9c6b2dc..63060a77d1abdfd4060648bfabe25709afcfeb8d 100644 --- a/python/paddle/fluid/debugger.py +++ b/python/paddle/fluid/debugger.py @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import sys +import six import re from .graphviz import GraphPreviewGenerator from .proto import framework_pb2 @@ -225,7 +228,7 @@ def draw_block_graphviz(block, highlights=None, path="./temp.dot"): graph = GraphPreviewGenerator("some graph") # collect parameters and args protostr = block.desc.serialize_to_string() - desc = framework_pb2.BlockDesc.FromString(str(protostr)) + desc = framework_pb2.BlockDesc.FromString(six.binary_type(protostr)) def need_highlight(name): if highlights is None: return False diff --git a/python/paddle/fluid/default_scope_funcs.py b/python/paddle/fluid/default_scope_funcs.py index f8faf6942524612ccc63713240bb289eeeaf75eb..a5b2c84dfe6f2650b4a2ee4465f723812e5d4a01 100644 --- a/python/paddle/fluid/default_scope_funcs.py +++ b/python/paddle/fluid/default_scope_funcs.py @@ -26,6 +26,8 @@ A `scoped_function` will take a `function` as input. That function will be invoked in a new local scope. """ +from __future__ import print_function + import paddle.fluid.core import threading diff --git a/python/paddle/fluid/evaluator.py b/python/paddle/fluid/evaluator.py index c0671cce9a1f169f02ba03a839c45b6e4df2c47a..7a82038ff78b17b2ddfd7b47320d41a7de9a2b8a 100644 --- a/python/paddle/fluid/evaluator.py +++ b/python/paddle/fluid/evaluator.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import warnings import numpy as np diff --git a/python/paddle/fluid/executor.py b/python/paddle/fluid/executor.py index e24b9faae24084ccc743a5b5126db9667089e128..288951cd7cd32155f136125fb817c35dd2ec6444 100644 --- a/python/paddle/fluid/executor.py +++ b/python/paddle/fluid/executor.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import contextlib import six @@ -320,8 +322,9 @@ class Executor(object): # append fetch_operators if not has_fetch_operators(global_block, fetch_list, fetch_var_name): for i, var in enumerate(fetch_list): - assert isinstance(var, Variable) or isinstance(var, str), ( - "Wrong type for fetch_list[%s]: %s" % (i, type(var))) + assert isinstance(var, Variable) or isinstance( + var, six.string_types), ( + "Wrong type for fetch_list[%s]: %s" % (i, type(var))) global_block.append_op( type='fetch', inputs={'X': [var]}, @@ -346,7 +349,7 @@ class Executor(object): def _fetch_data(self, fetch_list, fetch_var_name, scope): outs = [ core.get_fetch_variable(scope, fetch_var_name, i) - for i in range(len(fetch_list)) + for i in six.moves.range(len(fetch_list)) ] return outs diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py index 45b3abb88c9431f52705bb62df2c32779dd0cf9d..2377ac5f929eb21449689240da3061152a0541f9 100644 --- a/python/paddle/fluid/framework.py +++ b/python/paddle/fluid/framework.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import collections import contextlib import re @@ -19,6 +21,7 @@ import six import numpy as np +from .. import compat as cpt from .proto import framework_pb2 try: from . import core @@ -27,7 +30,7 @@ except ImportError as e: """NOTE: You may need to run \"export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH\" if you encounters \"libmkldnn.so not found\" errors. If you have python installed in other directory, replace \"/usr/local/lib\" with your own - directory. The original error is: \n""" + e.message) + directory. The original error is: \n""" + cpt.get_exception_message(e)) except Exception as e: raise e from . import unique_name @@ -87,7 +90,7 @@ def convert_np_dtype_to_dtype_(np_dtype): elif dtype == np.uint8: return core.VarDesc.VarType.UINT8 else: - raise ValueError("Not supported numpy dtype " + six.binary_type(dtype)) + raise ValueError("Not supported numpy dtype %s" % dtype) def dtype_is_floating(dtype): @@ -198,11 +201,11 @@ class Variable(object): if name is None: name = unique_name.generate('_generated_var') is_new_var = False - name = name if isinstance(name, six.binary_type) else name.encode() - self.desc = self.block.desc.find_var(name) + name = cpt.to_text(name) + self.desc = self.block.desc.find_var(cpt.to_bytes(name)) if self.desc is None: - self.desc = self.block.desc.var(name) + self.desc = self.block.desc.var(cpt.to_bytes(name)) is_new_var = True if is_new_var: @@ -325,7 +328,7 @@ class Variable(object): @property def name(self): - return self.desc.name() + return cpt.to_text(self.desc.name()) @name.setter def name(self, new_name): @@ -531,14 +534,7 @@ class Operator(object): elif isinstance(arg, six.binary_type): in_arg_names.append(arg.decode()) else: - if isinstance(arg.name, six.string_types): - in_arg_names.append(arg.name) - elif isinstance(arg.name, six.binary_type): - in_arg_names.append(arg.name.decode()) - else: - raise TypeError( - "arguments require unicode, str or bytes, but get %s instead." - % (type(arg.name))) + in_arg_names.append(cpt.to_text(arg.name)) self.desc.set_input(in_proto.name, in_arg_names) else: self.desc.set_input(in_proto.name, []) @@ -567,14 +563,7 @@ class Operator(object): (out_proto.name, len(out_args))) out_arg_names = [] for arg in out_args: - if isinstance(arg.name, six.string_types): - out_arg_names.append(arg.name) - elif isinstance(arg.name, six.binary_type): - out_arg_names.append(arg.name.decode()) - else: - raise TypeError( - "arguments require unicode, str or bytes, but get %s instead." - % (type(arg.name))) + out_arg_names.append(cpt.to_text(arg.name)) arg.op = self self.desc.set_output(out_proto.name, out_arg_names) @@ -970,10 +959,9 @@ class Block(object): Variable: the Variable with the giving name. """ if not isinstance(name, six.string_types): - if not isinstance(name, six.binary_type): - raise TypeError( - "var require string as parameter, but get %s instead." % - (type(name))) + raise TypeError( + "var require string as parameter, but get %s instead." % + (type(name))) v = self.vars.get(name, None) if v is None: raise ValueError("var %s not in this block" % name) @@ -1024,7 +1012,7 @@ class Block(object): return list(self.iter_parameters()) def iter_parameters(self): - return (item[1] for item in list(self.vars.items()) + return (item[1] for item in six.iteritems(self.vars) if isinstance(item[1], Parameter)) def create_var(self, *args, **kwargs): @@ -1052,6 +1040,9 @@ class Block(object): Returns: Variable: the Variable with the giving name. """ + name = cpt.to_text(name) + new_name = cpt.to_text(new_name) + if not self.has_var(name): raise ValueError("var %s is not in current block" % name) v = self.var(name) @@ -1070,9 +1061,9 @@ class Block(object): else: raise ValueError("unsupported var type: %s", type(v)) orig_var_type = v.type - self.desc._rename_var(name, new_name) + self.desc._rename_var(cpt.to_bytes(name), cpt.to_bytes(new_name)) # NOTE: v is destroyed by C++ after calling _rename_var. - d = self.desc.find_var(new_name) + d = self.desc.find_var(cpt.to_bytes(new_name)) if var_type == "Parameter": var = Parameter( self, @@ -1103,7 +1094,7 @@ class Block(object): def _remove_var(self, name): self._sync_with_cpp() - self.desc._remove_var(name) + self.desc._remove_var(cpt.to_bytes(name)) del self.vars[name] def create_parameter(self, *args, **kwargs): @@ -1205,7 +1196,7 @@ class Block(object): # sync variables removed from c++ end for var in list(self.vars.keys()): - if not self.desc.find_var(var): + if not self.desc.find_var(cpt.to_bytes(var)): self.vars.pop(var) # sync operators from cpp @@ -1576,7 +1567,9 @@ class Program(object): p.current_block_idx = self.current_block_idx p._seed = self._seed p.desc = core.ProgramDesc(self.desc) - p.blocks = [Block(p, i) for i in xrange(self.desc.num_blocks())] + p.blocks = [ + Block(p, i) for i in six.moves.range(self.desc.num_blocks()) + ] p._current_role = self._current_role p._op_role_var = self._op_role_var @@ -1632,7 +1625,9 @@ class Program(object): targets_idx.append([t.block.idx, t.idx]) res = Program() res.desc = core.prune(self.desc, targets_idx) - res.blocks = [Block(res, i) for i in range(res.desc.num_blocks())] + res.blocks = [ + Block(res, i) for i in six.moves.range(res.desc.num_blocks()) + ] res._sync_with_cpp() return res @@ -1675,16 +1670,18 @@ class Program(object): root_block._remove_op(0, read_op_idx + 1) for var in root_block.all_vars(): if var.type() == core.VarDesc.VarType.READER: - root_block._remove_var(var.name()) + root_block._remove_var(cpt.to_bytes(var.name())) # change all `is_test` attributes to True - for i in range(res.desc.num_blocks()): + for i in six.moves.range(res.desc.num_blocks()): block = res.desc.block(i) - for j in range(block.op_size()): + for j in six.moves.range(block.op_size()): op = block.op(j) if op.has_attr('is_test'): op.set_attr('is_test', True) - res.blocks = [Block(res, i) for i in range(res.desc.num_blocks())] + res.blocks = [ + Block(res, i) for i in six.moves.range(res.desc.num_blocks()) + ] res._sync_with_cpp() return res @@ -1704,7 +1701,7 @@ class Program(object): """ p = Program() p.desc = core.ProgramDesc(binary_str) - p.blocks = [Block(p, i) for i in range(p.desc.num_blocks())] + p.blocks = [Block(p, i) for i in six.moves.range(p.desc.num_blocks())] p._sync_with_cpp() return p diff --git a/python/paddle/fluid/graphviz.py b/python/paddle/fluid/graphviz.py index ba67bf5ae6fe44ea23414d444a270c436c195326..2b18d854d18bcbebce2a0eb30b8690db49d9d246 100644 --- a/python/paddle/fluid/graphviz.py +++ b/python/paddle/fluid/graphviz.py @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import os import random import six +import functools import subprocess import logging @@ -105,8 +108,9 @@ class Graph(object): def _rank_repr(self): ranks = sorted( - list(self.rank_groups.items()), - cmp=lambda a, b: a[1].priority > b[1].priority) + six.iteritems(self.rank_groups), + key=functools.cmp_to_key( + lambda a, b: a[1].priority > b[1].priority)) repr = [] for x in ranks: repr.append(str(x[1])) @@ -149,7 +153,7 @@ class Node(object): name=self.name, label=self.label, extra=',' + ','.join("%s=%s" % (key, crepr(value)) - for key, value in list(self.attrs.items())) + for key, value in six.iteritems(self.attrs)) if self.attrs else "") return reprs @@ -173,7 +177,7 @@ class Edge(object): target=self.target.name, extra="" if not self.attrs else "[" + ','.join("{}={}".format(attr[0], crepr(attr[1])) - for attr in list(self.attrs.items())) + "]") + for attr in six.iteritems(self.attrs)) + "]") return repr diff --git a/python/paddle/fluid/inferencer.py b/python/paddle/fluid/inferencer.py index ff382d8b832b4b2bc6779dbb28d3fd95c8a0984e..3d2ef566173f81b29a6d8ea79cff00991a4ef3c4 100644 --- a/python/paddle/fluid/inferencer.py +++ b/python/paddle/fluid/inferencer.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import contextlib from . import core diff --git a/python/paddle/fluid/initializer.py b/python/paddle/fluid/initializer.py index 6dedbae7a6586f862328c7f23d0aea6ba5022614..bd46ed8e50c9344d471578eb0f89b7e214d62722 100644 --- a/python/paddle/fluid/initializer.py +++ b/python/paddle/fluid/initializer.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from . import framework import numpy as np import contextlib diff --git a/python/paddle/fluid/io.py b/python/paddle/fluid/io.py index af734210323913a36f861380dc38a98253aca0a1..6b67128fbfdb10d1abde4ebe7d663a2685bff109 100644 --- a/python/paddle/fluid/io.py +++ b/python/paddle/fluid/io.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import os import errno import time @@ -603,25 +605,15 @@ def save_inference_model(dirname, # "./infer_model". """ - if isinstance(feeded_var_names, six.binary_type): + if isinstance(feeded_var_names, six.string_types): feeded_var_names = [feeded_var_names] - elif isinstance(feeded_var_names, six.text_type): - feeded_var_names = [feeded_var_names.encode()] else: if len(feeded_var_names) > 0: # TODO(paddle-dev): polish these code blocks if not (bool(feeded_var_names) and all( - isinstance(name, six.binary_type) + isinstance(name, six.string_types) for name in feeded_var_names)): - if not (all( - isinstance(name, six.text_type) - for name in feeded_var_names)): - raise ValueError( - "'feed_var_names' should be a list of str.") - else: - feeded_var_names = [ - name.encode() for name in feeded_var_names - ] + raise ValueError("'feed_var_names' should be a list of str.") if isinstance(target_vars, Variable): target_vars = [target_vars] diff --git a/python/paddle/fluid/layer_helper.py b/python/paddle/fluid/layer_helper.py index 0c2b1eb795860373220eb254612161f7dc816ffd..bd9727b6ac0208b199091db00bd0fd5fae74d53b 100644 --- a/python/paddle/fluid/layer_helper.py +++ b/python/paddle/fluid/layer_helper.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import copy import itertools import six @@ -85,7 +87,7 @@ class LayerHelper(object): raise ValueError("parameter number mismatch") elif len(param_attr) == 1 and length != 1: tmp = [None] * length - for i in range(length): + for i in six.moves.range(length): tmp[i] = copy.deepcopy(param_attr[0]) param_attr = tmp return param_attr diff --git a/python/paddle/fluid/layers/__init__.py b/python/paddle/fluid/layers/__init__.py index a48e360463456ab7e00534dc0684aa153c8205cd..a2a808777ddc499570eb9ef92175787a14cf77ca 100644 --- a/python/paddle/fluid/layers/__init__.py +++ b/python/paddle/fluid/layers/__init__.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from . import ops from .ops import * from . import nn diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index 9fb7b4d0cad67db2d2d4b56e43d8837b8160cdb0..173567a0a374ac4453025b67b047950936df2055 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import contextlib from .layer_function_generator import autodoc, templatedoc @@ -22,6 +24,7 @@ from ..initializer import force_init_on_cpu from .ops import logical_and, logical_not, logical_or import numpy import warnings +import six from functools import reduce __all__ = [ @@ -602,7 +605,7 @@ class StaticRNN(object): boot_memories = [] pre_memories = [] memories = [] - for _, mem in list(self.memories.items()): + for _, mem in six.iteritems(self.memories): boot_memories.append(mem.init) pre_memories.append(mem.pre_mem.name) mem_var = rnn_block.var(mem.mem.name) diff --git a/python/paddle/fluid/layers/detection.py b/python/paddle/fluid/layers/detection.py index b996c8368862184f9bc8b177f3b6e43aebdfb007..72071478845df444ce72ce946787b2d0ce5f0d23 100644 --- a/python/paddle/fluid/layers/detection.py +++ b/python/paddle/fluid/layers/detection.py @@ -15,13 +15,17 @@ All layers just related to the detection neural network. """ +from __future__ import print_function + from .layer_function_generator import generate_layer_fn from .layer_function_generator import autodoc, templatedoc from ..layer_helper import LayerHelper from . import tensor from . import nn from . import ops +from ... import compat as cpt import math +import six import numpy from functools import reduce @@ -104,7 +108,7 @@ def rpn_target_assign(loc, examples. Returns: - tuple: + tuple: A tuple(predicted_scores, predicted_location, target_label, target_bbox) is returned. The predicted_scores and predicted_location is the predicted result of the RPN. @@ -115,7 +119,7 @@ def rpn_target_assign(loc, anchors. The predicted_scores is a 2D Tensor with shape [F + B, 1], and the shape of target_label is same as the shape of the predicted_scores, B is the number of the background - anchors, the F and B is depends on the input of this operator. + anchors, the F and B is depends on the input of this operator. Examples: .. code-block:: python @@ -232,8 +236,8 @@ def detection_output(loc, nms_eta(float): The parameter for adaptive NMS. Returns: - Variable: - + Variable: + The detection outputs is a LoDTensor with shape [No, 6]. Each row has six values: [label, confidence, xmin, ymin, xmax, ymax]. `No` is the total number of detections in this mini-batch. For each @@ -504,7 +508,7 @@ def target_assign(input, Assumed that the row offset for each instance in `neg_indices` is called neg_lod, for i-th instance and each `id` of neg_indices in this instance: - + .. code-block:: text out[i][id][0 : K] = {mismatch_value, mismatch_value, ...} @@ -522,11 +526,11 @@ def target_assign(input, mismatch_value (float32): Fill this value to the mismatched location. Returns: - tuple: - A tuple(out, out_weight) is returned. out is a 3D Tensor with - shape [N, P, K], N and P is the same as they are in - `neg_indices`, K is the same as it in input of X. If - `match_indices[i][j]`. out_weight is the weight for output with + tuple: + A tuple(out, out_weight) is returned. out is a 3D Tensor with + shape [N, P, K], N and P is the same as they are in + `neg_indices`, K is the same as it in input of X. If + `match_indices[i][j]`. out_weight is the weight for output with the shape of [N, P, 1]. Examples: @@ -834,7 +838,7 @@ def prior_box(input, offset(float): Prior boxes center offset. Default: 0.5 name(str): Name of the prior box op. Default: None. min_max_aspect_ratios_order(bool): If set True, the output prior box is - in order of [min, max, aspect_ratios], which is consistent with + in order of [min, max, aspect_ratios], which is consistent with Caffe. Please note, this order affects the weights order of convolution layer followed by and does not affect the final detection results. Default: False. @@ -977,7 +981,7 @@ def multi_box_head(inputs, stride(int|list|tuple): The stride of conv2d. Default:1, name(str): Name of the prior box layer. Default: None. min_max_aspect_ratios_order(bool): If set True, the output prior box is - in order of [min, max, aspect_ratios], which is consistent with + in order of [min, max, aspect_ratios], which is consistent with Caffe. Please note, this order affects the weights order of convolution layer followed by and does not affect the fininal detection results. Default: False. @@ -1039,7 +1043,7 @@ def multi_box_head(inputs, min_sizes = [] max_sizes = [] step = int(math.floor(((max_ratio - min_ratio)) / (num_layer - 2))) - for ratio in range(min_ratio, max_ratio + 1, step): + for ratio in six.moves.range(min_ratio, max_ratio + 1, step): min_sizes.append(base_size * ratio / 100.) max_sizes.append(base_size * (ratio + step) / 100.) min_sizes = [base_size * .10] + min_sizes @@ -1108,8 +1112,8 @@ def multi_box_head(inputs, mbox_loc = nn.transpose(mbox_loc, perm=[0, 2, 3, 1]) compile_shape = [ - mbox_loc.shape[0], - mbox_loc.shape[1] * mbox_loc.shape[2] * mbox_loc.shape[3] / 4, 4 + mbox_loc.shape[0], cpt.floor_division( + mbox_loc.shape[1] * mbox_loc.shape[2] * mbox_loc.shape[3], 4), 4 ] run_shape = tensor.assign(numpy.array([0, -1, 4]).astype("int32")) mbox_loc_flatten = nn.reshape( @@ -1127,8 +1131,9 @@ def multi_box_head(inputs, conf_loc = nn.transpose(conf_loc, perm=[0, 2, 3, 1]) new_shape = [0, -1, num_classes] compile_shape = [ - conf_loc.shape[0], conf_loc.shape[1] * conf_loc.shape[2] * - conf_loc.shape[3] / num_classes, num_classes + conf_loc.shape[0], + cpt.floor_division(conf_loc.shape[1] * conf_loc.shape[2] * + conf_loc.shape[3], num_classes), num_classes ] run_shape = tensor.assign( numpy.array([0, -1, num_classes]).astype("int32")) diff --git a/python/paddle/fluid/layers/device.py b/python/paddle/fluid/layers/device.py index bb1fb7fd571a56acf367e663af0cf9431211bcea..43ebd160de3fd3d2a491a3ec1fbe0e4085fbd0b1 100644 --- a/python/paddle/fluid/layers/device.py +++ b/python/paddle/fluid/layers/device.py @@ -15,6 +15,8 @@ All util layers. """ +from __future__ import print_function + from .layer_function_generator import autodoc from ..framework import unique_name from ..layer_helper import LayerHelper diff --git a/python/paddle/fluid/layers/io.py b/python/paddle/fluid/layers/io.py index 327ae309816344a0bcebfe70ffb59a00eab1d86f..b03ee514f50f9a8c1425bd5b1d409b58ed62351a 100644 --- a/python/paddle/fluid/layers/io.py +++ b/python/paddle/fluid/layers/io.py @@ -11,8 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import contextlib import multiprocessing +import six import threading from ..data_feeder import DataFeeder @@ -21,7 +24,7 @@ from .layer_function_generator import templatedoc from .. import core from ..executor import global_scope from ..framework import convert_np_dtype_to_dtype_, default_main_program, \ - default_startup_program, program_guard, Program + default_startup_program, program_guard, Program, Variable from ..layer_helper import LayerHelper from ..unique_name import generate as unique_name @@ -69,7 +72,7 @@ def data(name, """ helper = LayerHelper('data', **locals()) shape = list(shape) - for i in range(len(shape)): + for i in six.moves.range(len(shape)): if shape[i] is None: shape[i] = -1 append_batch_size = False @@ -206,7 +209,7 @@ class ListenAndServ(object): }) -def Send(endpoints, send_vars, sync=True): +def Send(endpoints, send_vars, dummy_output=None, sync=True): """ Send variables to the server side, and get vars from server side when server have finished running server side program. @@ -220,6 +223,13 @@ def Send(endpoints, send_vars, sync=True): """ assert (type(send_vars) == list) + if dummy_output is None: + dummy_output = [] + elif isinstance(dummy_output, Variable): + dummy_output = [dummy_output] + + assert (type(dummy_output) == list) + epmap = endpoints.split(",") endpoints = list(set(epmap)) @@ -229,6 +239,7 @@ def Send(endpoints, send_vars, sync=True): helper.append_op( type="send", inputs={"X": send_vars}, + outputs={"Out": dummy_output}, attrs={ "endpoints": endpoints, "epmap": epmap, @@ -238,7 +249,7 @@ def Send(endpoints, send_vars, sync=True): helper.append_op(type="send_barrier", attrs={"endpoints": endpoints}) -def Recv(endpoints, get_vars, sync=True): +def Recv(endpoints, get_vars, dummy_input=None, sync=True): """ Receive variables from server side @@ -253,13 +264,20 @@ def Recv(endpoints, get_vars, sync=True): """ assert (type(get_vars) == list) + if dummy_input is None: + dummy_input = [] + elif isinstance(dummy_input, Variable): + dummy_input = [dummy_input] + + assert (type(dummy_input) == list) + epmap = endpoints.split(",") endpoints = list(set(epmap)) helper = LayerHelper("Recv", **locals()) helper.append_op( type="recv", - inputs={"X": get_vars}, + inputs={"X": dummy_input}, outputs={"Out": get_vars}, attrs={"endpoints": endpoints, "epmap": epmap}) @@ -674,7 +692,7 @@ def py_reader(capacity, def __tensor_provider__(): for slots in paddle_reader(): - yield [slots[str(idx)] for idx in xrange(counter)] + yield [slots[str(idx)] for idx in six.moves.xrange(counter)] __set_tensor_provider__(__tensor_provider__) @@ -750,7 +768,7 @@ def open_files(filenames, else: buffer_size = int(buffer_size) - if isinstance(filenames, basestring): + if isinstance(filenames, six.string_types): filenames = [filenames] dtypes = [convert_np_dtype_to_dtype_(dt) for dt in dtypes] shape_concat = [] @@ -1005,7 +1023,7 @@ class Preprocessor(object): source_lod_levels = self.underlying_reader.desc.lod_levels() self.source_var_names = [ unique_name("preprocessor_source") - for _ in range(len(source_shapes)) + for _ in six.moves.range(len(source_shapes)) ] source_vars = [] for var_name, shape, dtype, lod_level in zip( diff --git a/python/paddle/fluid/layers/layer_function_generator.py b/python/paddle/fluid/layers/layer_function_generator.py index c0d72620b1ddb183f43ebce766688518b5a737ac..8963d74de014d69c590276d5ff7080111f614230 100644 --- a/python/paddle/fluid/layers/layer_function_generator.py +++ b/python/paddle/fluid/layers/layer_function_generator.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import re import functools import warnings diff --git a/python/paddle/fluid/layers/learning_rate_scheduler.py b/python/paddle/fluid/layers/learning_rate_scheduler.py index daf91a40f7ad7935d355a287819ad1dbcdd84eb8..be368007dd7061ba7fc97414dbadfce00d158776 100644 --- a/python/paddle/fluid/layers/learning_rate_scheduler.py +++ b/python/paddle/fluid/layers/learning_rate_scheduler.py @@ -20,6 +20,8 @@ User can also implement their own learning_rate_decay strategy according to this module. """ +from __future__ import print_function + from . import control_flow from . import nn from . import ops @@ -72,10 +74,10 @@ def noam_decay(d_model, warmup_steps): def exponential_decay(learning_rate, decay_steps, decay_rate, staircase=False): """ - Applies exponential decay to the learning rate. + Applies exponential decay to the learning rate. - When training a model, it is often recommended to lower the learning rate as the - training progresses. By using this function, the learning rate will be decayed by + When training a model, it is often recommended to lower the learning rate as the + training progresses. By using this function, the learning rate will be decayed by 'decay_rate' every 'decay_steps' steps. >>> if staircase == True: @@ -148,8 +150,8 @@ def inverse_time_decay(learning_rate, decay_steps, decay_rate, staircase=False): """ Applies inverse time decay to the initial learning rate. - When training a model, it is often recommended to lower the learning rate as the - training progresses. By using this function, an inverse decay function will be + When training a model, it is often recommended to lower the learning rate as the + training progresses. By using this function, an inverse decay function will be applied to the initial learning rate. >>> if staircase == True: diff --git a/python/paddle/fluid/layers/math_op_patch.py b/python/paddle/fluid/layers/math_op_patch.py index 0e10a91d25877984396f9bcf9aae6438707eeab1..a458cebfb194a068d040a8919fd4abcb4b4bea80 100644 --- a/python/paddle/fluid/layers/math_op_patch.py +++ b/python/paddle/fluid/layers/math_op_patch.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from ..framework import Variable, unique_name from .layer_function_generator import OpProtoHolder from ..initializer import force_init_on_cpu diff --git a/python/paddle/fluid/layers/metric_op.py b/python/paddle/fluid/layers/metric_op.py index 49bae1e8af768d93294120e1d13ef0242313aa3c..2c3bdd77e1fa1c86baa3a288caab4ad4324e2ef2 100644 --- a/python/paddle/fluid/layers/metric_op.py +++ b/python/paddle/fluid/layers/metric_op.py @@ -15,6 +15,8 @@ All layers just related to metric. """ +from __future__ import print_function + import warnings from ..layer_helper import LayerHelper from ..initializer import Normal, Constant @@ -81,9 +83,9 @@ def auc(input, label, curve='ROC', num_thresholds=200, topk=1): **Area Under the Curve (AUC) Layer** This implementation computes the AUC according to forward output and label. - It is used very widely in binary classification evaluation. + It is used very widely in binary classification evaluation. - Note: If input label contains values other than 0 and 1, it will be cast + Note: If input label contains values other than 0 and 1, it will be cast to `bool`. Find the relevant definitions `here `_. @@ -93,14 +95,14 @@ def auc(input, label, curve='ROC', num_thresholds=200, topk=1): 2. PR: Precision Recall Args: - input(Variable): A floating-point 2D Variable, values are in the range - [0, 1]. Each row is sorted in descending order. This - input should be the output of topk. Typically, this + input(Variable): A floating-point 2D Variable, values are in the range + [0, 1]. Each row is sorted in descending order. This + input should be the output of topk. Typically, this Variable indicates the probability of each label. - label(Variable): A 2D int Variable indicating the label of the training + label(Variable): A 2D int Variable indicating the label of the training data. The height is batch size and width is always 1. curve(str): Curve type, can be 'ROC' or 'PR'. Default 'ROC'. - num_thresholds(int): The number of thresholds to use when discretizing + num_thresholds(int): The number of thresholds to use when discretizing the roc curve. Default 200. topk(int): only topk number of prediction output will be used for auc. @@ -109,7 +111,7 @@ def auc(input, label, curve='ROC', num_thresholds=200, topk=1): Examples: .. code-block:: python - + # network is a binary classification model and label the ground truth prediction = network(image, is_infer=True) auc_out=fluid.layers.auc(input=prediction, label=label) diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index be852b67119182cc817495b5e993c872cb9a88bf..bd2b950cffe646128501772899fea2e09d9bdce3 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -11,24 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -# Copyright (c ) 2018 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. """ All layers just related to the neural network. """ +from __future__ import print_function + from ..layer_helper import LayerHelper from ..initializer import Normal, Constant from ..framework import Variable @@ -362,7 +350,7 @@ def dynamic_lstm(input, """ helper = LayerHelper('lstm', **locals()) - size = size / 4 + size = size // 4 weight = helper.create_parameter( attr=helper.param_attr, shape=[size, 4 * size], dtype=dtype) bias_size = [1, 7 * size] @@ -552,7 +540,7 @@ def dynamic_lstmp(input, """ helper = LayerHelper('lstmp', **locals()) - size = size / 4 + size = size // 4 weight = helper.create_parameter( attr=helper.param_attr, shape=[proj_size, 4 * size], dtype=dtype) proj_weight = helper.create_parameter( @@ -780,7 +768,7 @@ def gru_unit(input, helper = LayerHelper('gru_unit', **locals()) dtype = helper.input_dtype() - size = size / 3 + size = size // 3 # create weight weight = helper.create_parameter( @@ -1264,7 +1252,7 @@ def sequence_conv(input, outputs={"Out": pre_bias}, attrs={ 'contextStride': filter_stride, - 'contextStart': -int(filter_size / 2), + 'contextStart': -int(filter_size // 2), 'contextLength': filter_size }) pre_act = helper.append_bias_op(pre_bias) @@ -1320,15 +1308,15 @@ def sequence_softmax(input, param_attr=None, bias_attr=None, use_cudnn=True): def softmax(input, param_attr=None, bias_attr=None, use_cudnn=True, name=None): """ - The input of the softmax operator is a tensor of any rank. The output tensor + The input of the softmax operator is a tensor of any rank. The output tensor has the same shape as the input. - The input tensor will first be logically flattened to a 2-D matrix. The matrix's - second dimension(row length) is as same as the last dimension of the input - tensor, and the first dimension(column length) is the product of all other - dimensions of the input tensor. For each row of the matrix, the softmax operator - squashes the K-dimensional(K is the width of the matrix, which is also the size - of the input tensor's last dimension) vector of arbitrary real values to a + The input tensor will first be logically flattened to a 2-D matrix. The matrix's + second dimension(row length) is as same as the last dimension of the input + tensor, and the first dimension(column length) is the product of all other + dimensions of the input tensor. For each row of the matrix, the softmax operator + squashes the K-dimensional(K is the width of the matrix, which is also the size + of the input tensor's last dimension) vector of arbitrary real values to a K-dimensional vector of real values in the range [0, 1] that add up to 1. It computes the exponential of the given dimension and the sum of exponential @@ -1496,7 +1484,7 @@ def conv2d(input, else: if num_channels % groups != 0: raise ValueError("num_channels must be divisible by groups.") - num_filter_channels = num_channels / groups + num_filter_channels = num_channels // groups filter_size = utils.convert_to_list(filter_size, 2, 'filter_size') stride = utils.convert_to_list(stride, 2, 'stride') @@ -1507,7 +1495,7 @@ def conv2d(input, raise ValueError("use_cudnn should be True or False") input_shape = input.shape - filter_shape = [num_filters, num_filter_channels] + filter_size + filter_shape = [num_filters, int(num_filter_channels)] + filter_size def _get_default_param_initializer(): std = (2.0 / (filter_size[0]**2 * num_channels))**0.5 @@ -1658,7 +1646,7 @@ def conv3d(input, else: if num_channels % groups != 0: raise ValueError("num_channels must be divisible by groups.") - num_filter_channels = num_channels / groups + num_filter_channels = num_channels // groups filter_size = utils.convert_to_list(filter_size, 3, 'filter_size') stride = utils.convert_to_list(stride, 3, 'stride') @@ -2393,16 +2381,16 @@ def conv2d_transpose(input, w_in = input.shape[3] filter_size_h = (output_size[0] - (h_in - 1) * stride[0] + 2 * - padding[0] - 1) / dilation[0] + 1 + padding[0] - 1) // dilation[0] + 1 filter_size_w = (output_size[1] - (w_in - 1) * stride[1] + 2 * - padding[1] - 1) / dilation[1] + 1 + padding[1] - 1) // dilation[1] + 1 filter_size = [filter_size_h, filter_size_w] else: filter_size = utils.convert_to_list(filter_size, 2, 'conv2d_transpose.filter_size') groups = 1 if groups is None else groups - filter_shape = [input_channel, num_filters / groups] + filter_size + filter_shape = [input_channel, num_filters // groups] + filter_size img_filter = helper.create_parameter( dtype=input.dtype, shape=filter_shape, attr=helper.param_attr) @@ -2560,18 +2548,18 @@ def conv3d_transpose(input, w_in = input.shape[4] filter_size_d = (output_size[0] - (d_in - 1) * stride[0] + 2 * - padding[0] - 1) / dilation[0] + 1 + padding[0] - 1) // dilation[0] + 1 filter_size_h = (output_size[1] - (h_in - 1) * stride[1] + 2 * - padding[1] - 1) / dilation[1] + 1 + padding[1] - 1) // dilation[1] + 1 filter_size_w = (output_size[2] - (w_in - 1) * stride[2] + 2 * - padding[2] - 1) / dilation[2] + 1 + padding[2] - 1) // dilation[2] + 1 filter_size = [filter_size_d, filter_size_h, filter_size_w] else: filter_size = utils.convert_to_list(filter_size, 3, 'conv3d_transpose.filter_size') groups = 1 if groups is None else groups - filter_shape = [input_channel, num_filters / groups] + filter_size + filter_shape = [input_channel, num_filters // groups] + filter_size img_filter = helper.create_parameter( dtype=input.dtype, shape=filter_shape, attr=helper.param_attr) @@ -2678,15 +2666,15 @@ def beam_search(pre_ids, Refer to `Beam search `_ for more details. - - This layer does the search in beams for one time step. Specifically, it + + This layer does the search in beams for one time step. Specifically, it selects the top-K candidate word ids of current step from :attr:`ids` according to their :attr:`scores` for all source sentences, where K is :attr:`beam_size` and :attr:`ids, scores` are predicted results from the computation cell. Additionally, :attr:`pre_ids` and :attr:`pre_scores` are the output of beam_search at previous step, they are needed for special use to handle ended candidate translations. - + Note that the :attr:`scores` passed in should be accumulated scores, and length penalty should be done with extra operators before calculating the accumulated scores if needed, also suggest finding top-K before it and @@ -3887,7 +3875,7 @@ def nce(input, def hsigmoid(input, label, num_classes, param_attr=None, bias_attr=None): """ The hierarchical sigmoid operator is used to accelerate the training - process of language model. This operator organizes the classes into a + process of language model. This operator organizes the classes into a complete binary tree, each leaf node represents a class(a word) and each internal node acts as a binary classifier. For each word there's a unique path from root to it's leaf node, hsigmoid calculate the cost for each @@ -3897,9 +3885,9 @@ def hsigmoid(input, label, num_classes, param_attr=None, bias_attr=None): Refer to `Hierarchical Probabilistic Neural Network Language Model `_ - + Args: - input (Variable): The input tensor variable with shape + input (Variable): The input tensor variable with shape :math:`[N \\times D]`, where :math:`N` is the size of mini-batch, and :math:`D` is the feature size. label (Variable): The tensor variable contains labels of training data. @@ -3907,7 +3895,7 @@ def hsigmoid(input, label, num_classes, param_attr=None, bias_attr=None): num_classes: (int), The number of classes, must not be less than 2. param_attr (ParamAttr|list of ParamAttr, default None): The parameter attribute for learnable parameters/weights of this layer. - bias_attr (ParamAttr|list of ParamAttr, default None): The parameter + bias_attr (ParamAttr|list of ParamAttr, default None): The parameter attribute for the bias of this layer. If it is set to False, no bias will be applied. @@ -5306,23 +5294,23 @@ def rank_loss(label, left, right, name=None): is a pairwise ranking model with a training sample consisting of a pair of documents, A and B. Label P indicates whether A is ranked higher than B or not: - + P = {0, 1} or {0, 0.5, 1}, where 0.5 means that there is no information about the rank of the input pair. - + Rank loss layer takes three inputs: left (o_i), right (o_j) and label (P_{i,j}). The inputs respectively represent RankNet's output scores for documents A and B and the value of label P. The following equation computes rank loss C_{i,j} from the inputs: - + $$ C_{i,j} = -\tilde{P_{ij}} * o_{i,j} + \log(1 + e^{o_{i,j}}) \\ o_{i,j} = o_i - o_j \\ \tilde{P_{i,j}} = \left \{0, 0.5, 1 \right \} \ or \ \left \{0, 1 \right \} $$ - - Rank loss layer takes batch inputs with size batch_size (batch_size >= 1). - + + Rank loss layer takes batch inputs with size batch_size (batch_size >= 1). + Args: label (Variable): Indicats whether A ranked higher than B or not. left (Variable): RankNet's output score for doc A. @@ -5435,7 +5423,7 @@ def flatten(x, axis=1, name=None): axis = 2 We get: Out.shape = (3 * 100, 4 * 100) - + Case 2: Given X.shape = (3, 100, 100, 4) @@ -5446,8 +5434,8 @@ def flatten(x, axis=1, name=None): Args: x (Variable): A tensor of rank >= axis. - axis (int): Indicate up to which input dimensions (exclusive) should - be flattened to the outer dimension of the output. + axis (int): Indicate up to which input dimensions (exclusive) should + be flattened to the outer dimension of the output. The value for axis must be in the range [0, R], where R is the rank of the input tensor. When axis = 0, the shape of the output tensor is (1, (d_0 X d_1 ... d_n), where the @@ -5463,7 +5451,7 @@ def flatten(x, axis=1, name=None): Raises: ValueError: If x is not a variable. - ValueError: If axis is not in range [0, rank(x)]. + ValueError: If axis is not in range [0, rank(x)]. Examples: diff --git a/python/paddle/fluid/layers/ops.py b/python/paddle/fluid/layers/ops.py index f70c7f2258ce588444cf46d6c8affc4c9555203e..cc4a7de163ac2e52e43131a021160051423e040c 100644 --- a/python/paddle/fluid/layers/ops.py +++ b/python/paddle/fluid/layers/ops.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function from .layer_function_generator import generate_layer_fn __activations__ = [ diff --git a/python/paddle/fluid/layers/tensor.py b/python/paddle/fluid/layers/tensor.py index b93d721c12cb6ead044dc790f2f2af8a61a63b60..04e71497aa762e390c4123c0bf3d7f111a772dd4 100644 --- a/python/paddle/fluid/layers/tensor.py +++ b/python/paddle/fluid/layers/tensor.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from ..layer_helper import LayerHelper from ..param_attr import ParamAttr from ..framework import convert_np_dtype_to_dtype_ diff --git a/python/paddle/fluid/layers/utils.py b/python/paddle/fluid/layers/utils.py index 49ec3088831dff415e042e1b0a632f63106eb07b..5688f04ab2382f5731e69c60225765a2094bba8c 100644 --- a/python/paddle/fluid/layers/utils.py +++ b/python/paddle/fluid/layers/utils.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import numpy as np diff --git a/python/paddle/fluid/lod_tensor.py b/python/paddle/fluid/lod_tensor.py index 53c33616f55be5f5ef7068a6e94418e17d739e3c..a9de09f31f4ed04ba1aa003e85b25fc5a91557e4 100644 --- a/python/paddle/fluid/lod_tensor.py +++ b/python/paddle/fluid/lod_tensor.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from . import core import numpy as np diff --git a/python/paddle/fluid/metrics.py b/python/paddle/fluid/metrics.py index cd8934522755691217a99a2cca271badda55368e..592cb23eb9319658f8542ed5bc6ab3e95cfdb118 100644 --- a/python/paddle/fluid/metrics.py +++ b/python/paddle/fluid/metrics.py @@ -14,11 +14,15 @@ """ Fluid Metrics -The metrics are accomplished via Python natively. +The metrics are accomplished via Python natively. """ + +from __future__ import print_function + import numpy as np import copy import warnings +import six __all__ = [ 'MetricBase', @@ -79,10 +83,10 @@ class MetricBase(object): """ states = { attr: value - for attr, value in list(self.__dict__.items()) + for attr, value in six.iteritems(self.__dict__) if not attr.startswith("_") } - for attr, value in list(states.items()): + for attr, value in six.iteritems(states): if isinstance(value, int): setattr(self, attr, 0) elif isinstance(value, float): @@ -105,7 +109,7 @@ class MetricBase(object): """ states = { attr: value - for attr, value in list(self.__dict__.items()) + for attr, value in six.iteritems(self.__dict__) if not attr.startswith("_") } config = {} @@ -141,10 +145,10 @@ class CompositeMetric(MetricBase): """ Composite multiple metrics in one instance. for example, merge F1, accuracy, recall into one Metric. - + Examples: .. code-block:: python - + labels = fluid.layers.data(name="data", shape=[1], dtype="int32") data = fluid.layers.data(name="data", shape=[32, 32], dtype="int32") pred = fluid.layers.fc(input=data, size=1000, act="tanh") diff --git a/python/paddle/fluid/net_drawer.py b/python/paddle/fluid/net_drawer.py index 623a7d3fd05567a26bb6923550f597a0e1e27e32..0b61c23d07e95acf7b4564753f748e7fb497e73e 100644 --- a/python/paddle/fluid/net_drawer.py +++ b/python/paddle/fluid/net_drawer.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import argparse import json import logging diff --git a/python/paddle/fluid/nets.py b/python/paddle/fluid/nets.py index 08480671d8a5c50bbec97930c451cbcdc241e1fe..051fe84364639ca6028326c0cb02b204a02531af 100644 --- a/python/paddle/fluid/nets.py +++ b/python/paddle/fluid/nets.py @@ -11,6 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function +import six from . import layers __all__ = [ @@ -210,7 +213,7 @@ def img_conv_group(input, conv_with_batchnorm = __extend_list__(conv_with_batchnorm) conv_batchnorm_drop_rate = __extend_list__(conv_batchnorm_drop_rate) - for i in range(len(conv_num_filter)): + for i in six.moves.range(len(conv_num_filter)): local_conv_act = conv_act if conv_with_batchnorm[i]: local_conv_act = None diff --git a/python/paddle/fluid/op.py b/python/paddle/fluid/op.py index 93f021a360ac61f64e769d057df188d79f6f2bb6..667db10d3ebdd24ddd9efbe2310ebb331e268ee2 100644 --- a/python/paddle/fluid/op.py +++ b/python/paddle/fluid/op.py @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + +import numpy as np import six import paddle.fluid.core as core @@ -99,6 +102,8 @@ class OpDescCreationMethod(object): new_attr = op_desc.attrs.add() new_attr.name = attr.name new_attr.type = attr.type + if isinstance(user_defined_attr, np.ndarray): + user_defined_attr = user_defined_attr.tolist() if attr.type == framework_pb2.INT: new_attr.i = user_defined_attr elif attr.type == framework_pb2.FLOAT: diff --git a/python/paddle/fluid/optimizer.py b/python/paddle/fluid/optimizer.py index a07325f46a2892222c2d1dcd74aa7cb01f6760a1..031ddd09a0b27b050b6ac651e4d8c46854092b2f 100644 --- a/python/paddle/fluid/optimizer.py +++ b/python/paddle/fluid/optimizer.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import re from collections import defaultdict from paddle.fluid.framework import Program, Variable diff --git a/python/paddle/fluid/parallel_executor.py b/python/paddle/fluid/parallel_executor.py index 2a3555ebdde4d54f63bb420218896560c1b40ffd..a7765c9591f0bd653c08036c46a36131906a758f 100644 --- a/python/paddle/fluid/parallel_executor.py +++ b/python/paddle/fluid/parallel_executor.py @@ -17,8 +17,10 @@ import multiprocessing from . import core from . import framework from . import executor +from .. import compat as cpt import warnings import sys +import six import os __all__ = ['ParallelExecutor', 'ExecutionStrategy', 'BuildStrategy'] @@ -95,7 +97,7 @@ class ParallelExecutor(object): self._places = [] self._act_places = [] if use_cuda: - for i in range(core.get_cuda_device_count()): + for i in six.moves.range(core.get_cuda_device_count()): p = core.Place() self._act_places.append(core.CUDAPlace(i)) p.set_place(self._act_places[-1]) @@ -103,7 +105,7 @@ class ParallelExecutor(object): else: cpu_num = int( os.environ.get('CPU_NUM', multiprocessing.cpu_count())) - for i in range(cpu_num): + for i in six.moves.range(cpu_num): p = core.Place() self._act_places.append(core.CPUPlace()) p.set_place(self._act_places[-1]) @@ -153,11 +155,13 @@ class ParallelExecutor(object): self.executor = core.ParallelExecutor( self._places, set([ - p.name for p in main.global_block().iter_parameters() + cpt.to_text(p.name) + for p in main.global_block().iter_parameters() if not p.stop_gradient ]), - set(self.persistable_vars), main.desc, loss_name - if loss_name else '', scope, local_scopes, exec_strategy, + set(cpt.to_text(var) for var in self.persistable_vars), main.desc, + cpt.to_text(loss_name) + if loss_name else six.u(''), scope, local_scopes, exec_strategy, build_strategy, num_trainers, trainer_id) self.scope = scope diff --git a/python/paddle/fluid/param_attr.py b/python/paddle/fluid/param_attr.py index afae577656c8970338f3b02208fcb4c738628ab6..f0be794327f51cbbc4202b8b7b401b712b6d66a3 100644 --- a/python/paddle/fluid/param_attr.py +++ b/python/paddle/fluid/param_attr.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import six from .initializer import Initializer, Xavier, Constant diff --git a/python/paddle/fluid/profiler.py b/python/paddle/fluid/profiler.py index 01983a830351b018770e6358f604781ffaae5800..e05885f5f5bfc169828c1c6e723dffff098c3c2e 100644 --- a/python/paddle/fluid/profiler.py +++ b/python/paddle/fluid/profiler.py @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from . import core from contextlib import contextmanager import os +import six __all__ = [ 'cuda_profiler', 'reset_profiler', 'profiler', 'start_profiler', @@ -88,7 +91,7 @@ def cuda_profiler(output_file, output_mode=None, config=None): config = NVPROF_CONFIG if config is None else config config_file = 'nvprof_config_file' with open(config_file, 'wb') as fp: - fp.writelines(["%s\n" % item for item in config]) + fp.writelines([six.b("%s\n" % item) for item in config]) core.nvprof_init(output_file, output_mode, config_file) # Enables profiler collection by the active CUDA profiling tool. core.nvprof_start() diff --git a/python/paddle/fluid/recordio_writer.py b/python/paddle/fluid/recordio_writer.py index 93b38ad3fa37bd4bff04c529cd5518a8138e55ea..a69c0c29d4675d3e6b9b2a2d766b8be9935092cf 100644 --- a/python/paddle/fluid/recordio_writer.py +++ b/python/paddle/fluid/recordio_writer.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import os import contextlib from . import core diff --git a/python/paddle/fluid/regularizer.py b/python/paddle/fluid/regularizer.py index 6eaac4432d4df1288f37607a01484434542f1138..da38626111a6767e1a76a35d6d1375ccc1283de4 100644 --- a/python/paddle/fluid/regularizer.py +++ b/python/paddle/fluid/regularizer.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from . import framework from . import core diff --git a/python/paddle/fluid/tests/book/high-level-api/fit_a_line/test_fit_a_line.py b/python/paddle/fluid/tests/book/high-level-api/fit_a_line/test_fit_a_line.py index 36a1a223cfd7c69aff3e8648da990d23e4e75202..f6017a455df7e8bd197ef2563a759f843b5e7c73 100644 --- a/python/paddle/fluid/tests/book/high-level-api/fit_a_line/test_fit_a_line.py +++ b/python/paddle/fluid/tests/book/high-level-api/fit_a_line/test_fit_a_line.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle import paddle.fluid as fluid import contextlib diff --git a/python/paddle/fluid/tests/book/high-level-api/image_classification/cifar10_small_test_set.py b/python/paddle/fluid/tests/book/high-level-api/image_classification/cifar10_small_test_set.py index 9e4c384d92943227c2d68da829e6019e649a35fb..48c0f3d3611547308b5d4460748d3aab765f5805 100644 --- a/python/paddle/fluid/tests/book/high-level-api/image_classification/cifar10_small_test_set.py +++ b/python/paddle/fluid/tests/book/high-level-api/image_classification/cifar10_small_test_set.py @@ -28,12 +28,14 @@ images per class. """ +from __future__ import print_function + import itertools import numpy -import paddle.v2.dataset.common +import paddle.dataset.common import tarfile +import six from six.moves import cPickle as pickle -from six.moves import zip __all__ = ['train10'] @@ -44,20 +46,25 @@ CIFAR10_MD5 = 'c58f30108f718f92721af3b95e74349a' def reader_creator(filename, sub_name, batch_size=None): def read_batch(batch): - data = batch['data'] - labels = batch.get('labels', batch.get('fine_labels', None)) + data = batch[six.b('data')] + labels = batch.get( + six.b('labels'), batch.get(six.b('fine_labels'), None)) assert labels is not None - for sample, label in zip(data, labels): + for sample, label in six.moves.zip(data, labels): yield (sample / 255.0).astype(numpy.float32), int(label) def reader(): with tarfile.open(filename, mode='r') as f: - names = (each_item.name for each_item in f - if sub_name in each_item.name) + names = [ + each_item.name for each_item in f if sub_name in each_item.name + ] batch_count = 0 for name in names: - batch = pickle.load(f.extractfile(name)) + if six.PY2: + batch = pickle.load(f.extractfile(name)) + else: + batch = pickle.load(f.extractfile(name), encoding='bytes') for item in read_batch(batch): if isinstance(batch_size, int) and batch_count > batch_size: break @@ -78,6 +85,6 @@ def train10(batch_size=None): :rtype: callable """ return reader_creator( - paddle.v2.dataset.common.download(CIFAR10_URL, 'cifar', CIFAR10_MD5), + paddle.dataset.common.download(CIFAR10_URL, 'cifar', CIFAR10_MD5), 'data_batch', batch_size=batch_size) diff --git a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_resnet.py b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_resnet.py index a1f62db093904b617f0e37dc20d586ccea7eacd2..be494a0d340c62fb35afbf97fba38eff08a965e6 100644 --- a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_resnet.py +++ b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_resnet.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle import paddle.fluid as fluid import numpy @@ -55,7 +57,7 @@ def resnet_cifar10(input, depth=32): return tmp assert (depth - 2) % 6 == 0 - n = (depth - 2) / 6 + n = (depth - 2) // 6 conv1 = conv_bn_layer( input=input, ch_out=16, filter_size=3, stride=1, padding=1) res1 = layer_warp(basicblock, conv1, 16, 16, n, 1) diff --git a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py index 8429551765740e7db0eda82ce0b17cff129359b0..dbc7bc06c93157f271c79e85b6925468e861e57f 100644 --- a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py +++ b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle import paddle.fluid as fluid import numpy diff --git a/python/paddle/fluid/tests/book/high-level-api/label_semantic_roles/test_label_semantic_roles_newapi.py b/python/paddle/fluid/tests/book/high-level-api/label_semantic_roles/test_label_semantic_roles_newapi.py index e3602e2d5643c233b2575d1adb7f181127f60287..ec4e1c768c7f2a2421ac409a2eecc0100c086a6a 100755 --- a/python/paddle/fluid/tests/book/high-level-api/label_semantic_roles/test_label_semantic_roles_newapi.py +++ b/python/paddle/fluid/tests/book/high-level-api/label_semantic_roles/test_label_semantic_roles_newapi.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle import paddle.fluid as fluid import numpy as np diff --git a/python/paddle/fluid/tests/book/high-level-api/machine_translation/test_machine_translation.py b/python/paddle/fluid/tests/book/high-level-api/machine_translation/test_machine_translation.py index 6fb0c85a8be2b4560ea1fdb32f01146a9206ee78..560f1189581f631dc6a3470cf8f22f902ca26f26 100644 --- a/python/paddle/fluid/tests/book/high-level-api/machine_translation/test_machine_translation.py +++ b/python/paddle/fluid/tests/book/high-level-api/machine_translation/test_machine_translation.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import contextlib import numpy as np diff --git a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_conv.py b/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_conv.py index 898807db6f343cbefcc877e0f03ed6c5b82dd669..187bef1b0c1a614fbca88ef22097831d7bd5cd7f 100644 --- a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_conv.py +++ b/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_conv.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import argparse import paddle.fluid as fluid import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py b/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py index 6dd64be315159f1835244fa027e578434e6cb038..b95e7db122adbb1414da1691926c920b963fd6fe 100644 --- a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py +++ b/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import argparse import paddle.fluid as fluid import paddle diff --git a/python/paddle/fluid/tests/book/high-level-api/recommender_system/test_recommender_system_newapi.py b/python/paddle/fluid/tests/book/high-level-api/recommender_system/test_recommender_system_newapi.py index 60f3d8e105209938360487d963b0328d95e7b1f0..9e2767783bb6748cfc8f95567627068d7532a8c8 100644 --- a/python/paddle/fluid/tests/book/high-level-api/recommender_system/test_recommender_system_newapi.py +++ b/python/paddle/fluid/tests/book/high-level-api/recommender_system/test_recommender_system_newapi.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import math import sys import numpy as np diff --git a/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_conv.py b/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_conv.py index 24e65d1bd54cff7ad64453a3a61f50351d32ef08..097c2a468fca558106aba2f24c332256189d9076 100644 --- a/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_conv.py +++ b/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_conv.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle import paddle.fluid as fluid from functools import partial diff --git a/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_dynamic_rnn.py b/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_dynamic_rnn.py index b3b1505a0fad07144f3f53c22abd5553054d8c51..5f74cd142590abb93f8846bc831a9f5e3dd2f311 100644 --- a/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_dynamic_rnn.py +++ b/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_dynamic_rnn.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle import paddle.fluid as fluid from functools import partial diff --git a/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_stacked_lstm.py b/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_stacked_lstm.py index 25f99ff0fd2d1050bb62338a6bf87aa29f913fb6..284a6ca168636377699c287236c491352566909b 100644 --- a/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_stacked_lstm.py +++ b/python/paddle/fluid/tests/book/high-level-api/understand_sentiment/test_understand_sentiment_stacked_lstm.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle import paddle.fluid as fluid from functools import partial diff --git a/python/paddle/fluid/tests/book/high-level-api/word2vec/test_word2vec_new_api.py b/python/paddle/fluid/tests/book/high-level-api/word2vec/test_word2vec_new_api.py index 02e65cf56c4d1bd262831320befd2edc735c0d1c..1c7cf3199a07c3f65d967eda70a481b1bd1b1638 100644 --- a/python/paddle/fluid/tests/book/high-level-api/word2vec/test_word2vec_new_api.py +++ b/python/paddle/fluid/tests/book/high-level-api/word2vec/test_word2vec_new_api.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle import paddle.fluid as fluid import numpy as np diff --git a/python/paddle/fluid/tests/book/notest_understand_sentiment.py b/python/paddle/fluid/tests/book/notest_understand_sentiment.py index ce6342c2dad0b33e57d0ea90fc6ef1660ae4e68b..82f1c6615f3c4ca54bf5e979b55082022cd4da9f 100644 --- a/python/paddle/fluid/tests/book/notest_understand_sentiment.py +++ b/python/paddle/fluid/tests/book/notest_understand_sentiment.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from paddle.fluid.layers.device import get_places import unittest import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/book/test_fit_a_line.py b/python/paddle/fluid/tests/book/test_fit_a_line.py index 37b64fa94a9aad7042e153e414ed29de3142db5a..334294ab485cf203aa0ccf680a53010322d3af3b 100644 --- a/python/paddle/fluid/tests/book/test_fit_a_line.py +++ b/python/paddle/fluid/tests/book/test_fit_a_line.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle import paddle.fluid as fluid import contextlib diff --git a/python/paddle/fluid/tests/book/test_image_classification.py b/python/paddle/fluid/tests/book/test_image_classification.py index de6fe5f140a86545e3291db165af824739a814ef..9fe361425c128590da910128beaccb3336f8ba57 100644 --- a/python/paddle/fluid/tests/book/test_image_classification.py +++ b/python/paddle/fluid/tests/book/test_image_classification.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle import paddle.fluid as fluid import contextlib @@ -60,7 +62,7 @@ def resnet_cifar10(input, depth=32): return tmp assert (depth - 2) % 6 == 0 - n = (depth - 2) / 6 + n = (depth - 2) // 6 conv1 = conv_bn_layer( input=input, ch_out=16, filter_size=3, stride=1, padding=1) res1 = layer_warp(basicblock, conv1, 16, 16, n, 1) diff --git a/python/paddle/fluid/tests/book/test_label_semantic_roles.py b/python/paddle/fluid/tests/book/test_label_semantic_roles.py index b7ac911cafdc751f38c7f66bc48263a17a84dc08..f63387a90617dc4e9b7c9ee7caa2d01595237a03 100644 --- a/python/paddle/fluid/tests/book/test_label_semantic_roles.py +++ b/python/paddle/fluid/tests/book/test_label_semantic_roles.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import contextlib import math import numpy as np diff --git a/python/paddle/fluid/tests/book/test_machine_translation.py b/python/paddle/fluid/tests/book/test_machine_translation.py index 462faad3e1cb7108f3bd6934017efe25fb9a4276..5e241aaa32727686b84a0354a11d5a92f9576a90 100644 --- a/python/paddle/fluid/tests/book/test_machine_translation.py +++ b/python/paddle/fluid/tests/book/test_machine_translation.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import contextlib import numpy as np diff --git a/python/paddle/fluid/tests/book/test_recognize_digits.py b/python/paddle/fluid/tests/book/test_recognize_digits.py index 3e5f76d12d41d016c995e5c85feda3c1847e356f..da216d0cc4a2867cb169240d28235b6db747a818 100644 --- a/python/paddle/fluid/tests/book/test_recognize_digits.py +++ b/python/paddle/fluid/tests/book/test_recognize_digits.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid.core as core import math import os diff --git a/python/paddle/fluid/tests/book/test_recommender_system.py b/python/paddle/fluid/tests/book/test_recommender_system.py index b30c8771fcf267260c4c5aa7076bedc89e3b7e8b..cf8c48f34697d789d3d81d4d94f90a7169657baf 100644 --- a/python/paddle/fluid/tests/book/test_recommender_system.py +++ b/python/paddle/fluid/tests/book/test_recommender_system.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import math import sys import os diff --git a/python/paddle/fluid/tests/book/test_rnn_encoder_decoder.py b/python/paddle/fluid/tests/book/test_rnn_encoder_decoder.py index 2e79be2bd0fc7a368df86e188b7fa616055bb3e7..91c8705aa4c88dbfeea45e15c368459ba5b5ac1f 100644 --- a/python/paddle/fluid/tests/book/test_rnn_encoder_decoder.py +++ b/python/paddle/fluid/tests/book/test_rnn_encoder_decoder.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import paddle import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/book/test_word2vec.py b/python/paddle/fluid/tests/book/test_word2vec.py index e761e05795313da23a9d984263ac2e202939b1e7..fe063eb4629dbe06dc65ce98c6c01858db901f03 100644 --- a/python/paddle/fluid/tests/book/test_word2vec.py +++ b/python/paddle/fluid/tests/book/test_word2vec.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle import paddle.fluid as fluid from paddle.fluid.layers.device import get_places diff --git a/python/paddle/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py b/python/paddle/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py index ccc62b442f62fa9fa175de031b0732febe38ee9a..f530f8f4882a23df18c141b51560cb618fce86b5 100644 --- a/python/paddle/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py +++ b/python/paddle/fluid/tests/book_memory_optimization/test_memopt_fit_a_line.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import math import sys diff --git a/python/paddle/fluid/tests/book_memory_optimization/test_memopt_image_classification_train.py b/python/paddle/fluid/tests/book_memory_optimization/test_memopt_image_classification_train.py index b2a59d27da9b3348b581d51a68d769bbf3b90d35..3951e7b8ca649b63eea4b311f6205a6c7d761804 100644 --- a/python/paddle/fluid/tests/book_memory_optimization/test_memopt_image_classification_train.py +++ b/python/paddle/fluid/tests/book_memory_optimization/test_memopt_image_classification_train.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import sys import paddle @@ -56,7 +58,7 @@ def resnet_cifar10(input, depth=32): return tmp assert (depth - 2) % 6 == 0 - n = (depth - 2) / 6 + n = (depth - 2) // 6 conv1 = conv_bn_layer( input=input, ch_out=16, filter_size=3, stride=1, padding=1) res1 = layer_warp(basicblock, conv1, 16, 16, n, 1) diff --git a/python/paddle/fluid/tests/book_memory_optimization/test_memopt_machine_translation.py b/python/paddle/fluid/tests/book_memory_optimization/test_memopt_machine_translation.py index 323ddfb6911fdd57b32344933373189370005126..1ad51936b5b8f7c5149452d6033754a570c72654 100644 --- a/python/paddle/fluid/tests/book_memory_optimization/test_memopt_machine_translation.py +++ b/python/paddle/fluid/tests/book_memory_optimization/test_memopt_machine_translation.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import paddle import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/demo/fc_gan.py b/python/paddle/fluid/tests/demo/fc_gan.py index 3d92f50f0adeca79adefc291cdfba6a012fc2118..bd77779ce6ab5cf19e3e5ace3e51e39734b27c10 100644 --- a/python/paddle/fluid/tests/demo/fc_gan.py +++ b/python/paddle/fluid/tests/demo/fc_gan.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import errno import math import os diff --git a/python/paddle/fluid/tests/demo/file_reader/convert_data_to_recordio.py b/python/paddle/fluid/tests/demo/file_reader/convert_data_to_recordio.py index a00325d79be2eba4d7f770b5316c5857952fe272..45a104ec9625eacfcb87ea6eae619e3d71410da9 100644 --- a/python/paddle/fluid/tests/demo/file_reader/convert_data_to_recordio.py +++ b/python/paddle/fluid/tests/demo/file_reader/convert_data_to_recordio.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import sys import paddle.fluid as fluid import paddle.v2 as paddle diff --git a/python/paddle/fluid/tests/demo/file_reader/train.py b/python/paddle/fluid/tests/demo/file_reader/train.py index bc3a6dc81d24afec66ed1489aead1cff79a59bca..5f5d2848da42e18f2a142faae0c89352344d8cee 100644 --- a/python/paddle/fluid/tests/demo/file_reader/train.py +++ b/python/paddle/fluid/tests/demo/file_reader/train.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid as fluid import numpy import sys diff --git a/python/paddle/fluid/tests/demo/pyreader.py b/python/paddle/fluid/tests/demo/pyreader.py index 82065401935036ca346fa395c033f0f57100f01b..ec61e0ebae4feb1a2177da916b77b2ba2d3981b9 100644 --- a/python/paddle/fluid/tests/demo/pyreader.py +++ b/python/paddle/fluid/tests/demo/pyreader.py @@ -12,7 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy +import six import paddle import paddle.dataset.mnist as mnist @@ -31,7 +34,7 @@ def network(is_train): hidden = img - for i in xrange(2): + for i in six.moves.xrange(2): hidden = fluid.layers.fc(input=hidden, size=100, act='tanh') hidden = fluid.layers.dropout( hidden, dropout_prob=0.5, is_test=not is_train) @@ -74,7 +77,7 @@ def main(): test_reader.decorate_paddle_reader(paddle.batch(mnist.test(), 512)) - for epoch_id in xrange(10): + for epoch_id in six.moves.xrange(10): train_reader.start() try: while True: diff --git a/python/paddle/fluid/tests/no_test_concurrency.py b/python/paddle/fluid/tests/no_test_concurrency.py index 3bc0c9808e2345b610dea79abc56cfb0065ea46f..b5d7676f4a2cb085c6900cd0bd0644afa2b2afd5 100644 --- a/python/paddle/fluid/tests/no_test_concurrency.py +++ b/python/paddle/fluid/tests/no_test_concurrency.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/notest_concurrency.py b/python/paddle/fluid/tests/notest_concurrency.py index 77107f8b36f31c1f494b0ade218ee047ef7eb7c6..fd9da4cce0ea51c53b4b01e7c3dc2a2ed1eeb089 100644 --- a/python/paddle/fluid/tests/notest_concurrency.py +++ b/python/paddle/fluid/tests/notest_concurrency.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/test_beam_search_decoder.py b/python/paddle/fluid/tests/test_beam_search_decoder.py index 8bf750940d570cdad5e110168afc5f632202e869..fe8a9daa3bea4b99bb42edc78538685c5ce11fe3 100644 --- a/python/paddle/fluid/tests/test_beam_search_decoder.py +++ b/python/paddle/fluid/tests/test_beam_search_decoder.py @@ -15,6 +15,8 @@ A simple machine translation demo using beam search decoder. """ +from __future__ import print_function + import contextlib import numpy as np import paddle diff --git a/python/paddle/fluid/tests/test_cpp_reader.py b/python/paddle/fluid/tests/test_cpp_reader.py index 6cc291dfcffdd7083f498389834e37bd06ca4572..b2a5253b9500bb504c651b2ab684206133199ada 100644 --- a/python/paddle/fluid/tests/test_cpp_reader.py +++ b/python/paddle/fluid/tests/test_cpp_reader.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle import paddle.fluid as fluid import numpy as np diff --git a/python/paddle/fluid/tests/test_data_feeder.py b/python/paddle/fluid/tests/test_data_feeder.py index 30b7a634a2b978df85d6432854ef12285460be44..01de564aa438e5f14a5c578f7bbbfb475155ca55 100644 --- a/python/paddle/fluid/tests/test_data_feeder.py +++ b/python/paddle/fluid/tests/test_data_feeder.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid as fluid import unittest diff --git a/python/paddle/fluid/tests/test_detection.py b/python/paddle/fluid/tests/test_detection.py index fd45abd0a77cb54a3ca8e60cf80a1efe9f9d2060..1467e72caac26a3ea2a0c770d665141988696630 100644 --- a/python/paddle/fluid/tests/test_detection.py +++ b/python/paddle/fluid/tests/test_detection.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid as fluid import paddle.fluid.layers as layers from paddle.fluid.framework import Program, program_guard diff --git a/python/paddle/fluid/tests/test_error_clip.py b/python/paddle/fluid/tests/test_error_clip.py index e8edd7fbbb31b1a6ecbf2a25a7d39e7b3f66363a..3c977afc7c813908fbe2dfb7445d9ca183cf2231 100644 --- a/python/paddle/fluid/tests/test_error_clip.py +++ b/python/paddle/fluid/tests/test_error_clip.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import paddle import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/test_gradient_clip.py b/python/paddle/fluid/tests/test_gradient_clip.py index d530601f13be6810a8a99b13c92faf584df568f9..266687fcd092dfdeec9343e2592f4c22b683d588 100644 --- a/python/paddle/fluid/tests/test_gradient_clip.py +++ b/python/paddle/fluid/tests/test_gradient_clip.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import paddle import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/test_if_else_op.py b/python/paddle/fluid/tests/test_if_else_op.py index 082f64c146f65eee4be0757d07495c33764fa841..10918a985f80f9f20629180b0359ce73f448d33b 100644 --- a/python/paddle/fluid/tests/test_if_else_op.py +++ b/python/paddle/fluid/tests/test_if_else_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle import paddle.fluid.layers as layers from paddle.fluid.framework import Program, program_guard diff --git a/python/paddle/fluid/tests/test_lod_tensor.py b/python/paddle/fluid/tests/test_lod_tensor.py index f7a9dd4129027417a06a6c25ff9a801fff259c5e..722b5f07b04f9374db3f262f5134347fe753ba19 100644 --- a/python/paddle/fluid/tests/test_lod_tensor.py +++ b/python/paddle/fluid/tests/test_lod_tensor.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid as fluid from paddle.fluid.lod_tensor import create_lod_tensor, create_random_int_lodtensor import numpy as np diff --git a/python/paddle/fluid/tests/test_python_operator_overriding.py b/python/paddle/fluid/tests/test_python_operator_overriding.py index b5ac97eac559e8c52a8949cfd63fc8671ba52514..5f92c437ec726f510d9194d23f1a01a5478827d6 100644 --- a/python/paddle/fluid/tests/test_python_operator_overriding.py +++ b/python/paddle/fluid/tests/test_python_operator_overriding.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/benchmark.py b/python/paddle/fluid/tests/unittests/benchmark.py index b98a92dcbe5626c6cca93b3f5894302399793bf9..9ea95f3e8700274977eda4ca113a6468c631584c 100644 --- a/python/paddle/fluid/tests/unittests/benchmark.py +++ b/python/paddle/fluid/tests/unittests/benchmark.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import unittest import time @@ -54,7 +56,7 @@ class BenchmarkSuite(OpTest): def _get_input_names(self): inputs = [] - for name, value in list(self.inputs.items()): + for name, value in six.iteritems(self.inputs): if isinstance(value, list): inputs.extend([sub_name for sub_name, _ in value]) inputs.append(name) @@ -62,7 +64,7 @@ class BenchmarkSuite(OpTest): def _get_output_names(self): outputs = [] - for var_name, var in list(self.outputs.items()): + for var_name, var in six.iteritems(self.outputs): if isinstance(var, list): for sub_var_name, sub_var in var: outputs.append(sub_var_name) diff --git a/python/paddle/fluid/tests/unittests/benchmark_sum_op.py b/python/paddle/fluid/tests/unittests/benchmark_sum_op.py index 91a5f1bca4441d80489a02eb9283928e38321826..0e7338b839e2a7f5808e7a752e9ca6389622c2cb 100644 --- a/python/paddle/fluid/tests/unittests/benchmark_sum_op.py +++ b/python/paddle/fluid/tests/unittests/benchmark_sum_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/decorators.py b/python/paddle/fluid/tests/unittests/decorators.py index d1165e2a9199454dbcc1fda411afad20449bcc92..1a5f4540cf033b4d3244537cc5016ee06f341464 100644 --- a/python/paddle/fluid/tests/unittests/decorators.py +++ b/python/paddle/fluid/tests/unittests/decorators.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid as fluid __all__ = ['many_times', 'prog_scope'] diff --git a/python/paddle/fluid/tests/unittests/dist_mnist.py b/python/paddle/fluid/tests/unittests/dist_mnist.py index 8f5ba33f7cbf5286edc4503c219fd3cdff60c517..722b3e159abf4737b2bb43c7b84e23a3618cda12 100644 --- a/python/paddle/fluid/tests/unittests/dist_mnist.py +++ b/python/paddle/fluid/tests/unittests/dist_mnist.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import argparse import time diff --git a/python/paddle/fluid/tests/unittests/dist_se_resnext.py b/python/paddle/fluid/tests/unittests/dist_se_resnext.py index d576a173ce2546119ede49128ef69d240c7cf482..1307ba4e4ad11ef01094c44068d916ff2d442f78 100644 --- a/python/paddle/fluid/tests/unittests/dist_se_resnext.py +++ b/python/paddle/fluid/tests/unittests/dist_se_resnext.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import argparse -import six import time import math @@ -173,7 +174,7 @@ class SE_ResNeXt(): num_filters=num_filters, filter_size=filter_size, stride=stride, - padding=(filter_size - 1) / 2, + padding=(filter_size - 1) // 2, groups=groups, act=None, # avoid pserver CPU init differs from GPU @@ -187,7 +188,7 @@ class SE_ResNeXt(): input=input, pool_size=0, pool_type='avg', global_pooling=True) stdv = 1.0 / math.sqrt(pool.shape[1] * 1.0) squeeze = fluid.layers.fc(input=pool, - size=num_channels / reduction_ratio, + size=num_channels // reduction_ratio, act='relu') stdv = 1.0 / math.sqrt(squeeze.shape[1] * 1.0) excitation = fluid.layers.fc(input=squeeze, diff --git a/python/paddle/fluid/tests/unittests/dist_transformer.py b/python/paddle/fluid/tests/unittests/dist_transformer.py index ee8020a73546cb9037e9dc4be589c62bb1b6b937..ab4c5c3f368333ac42781aebd495579b5c26f388 100644 --- a/python/paddle/fluid/tests/unittests/dist_transformer.py +++ b/python/paddle/fluid/tests/unittests/dist_transformer.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import argparse import time @@ -22,6 +24,7 @@ import paddle.fluid as fluid from paddle.fluid import core import os import sys +import six import transformer_model import paddle.dataset.wmt16 as wmt16 @@ -159,6 +162,7 @@ def get_model(): avg_cost = transformer(use_feed=False) optimizer = fluid.optimizer.Adam() optimizer.minimize(avg_cost) + fluid.memory_optimize(fluid.default_main_program()) return avg_cost @@ -222,7 +226,7 @@ class DistTransformer2x2(object): first_loss, = exe.run(fetch_list=[avg_cost.name]) print(first_loss) - for i in xrange(5): + for i in six.moves.xrange(5): _ = exe.run(fetch_list=[avg_cost.name]) last_loss, = exe.run(fetch_list=[avg_cost.name]) print(last_loss) diff --git a/python/paddle/fluid/tests/unittests/dist_word2vec.py b/python/paddle/fluid/tests/unittests/dist_word2vec.py index 54a70f4adb4a9bb24e3c618a7fe71f42a376609b..0ad994a258c04cabc807823b7d2a8ae8bb62ab2c 100644 --- a/python/paddle/fluid/tests/unittests/dist_word2vec.py +++ b/python/paddle/fluid/tests/unittests/dist_word2vec.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import argparse import time diff --git a/python/paddle/fluid/tests/unittests/op_test.py b/python/paddle/fluid/tests/unittests/op_test.py index b27d773f09d9a6daad5a10b65e683f4e11881de1..972e44c9528a29417d9689dcb2408b9381346f31 100644 --- a/python/paddle/fluid/tests/unittests/op_test.py +++ b/python/paddle/fluid/tests/unittests/op_test.py @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import random +import six import time import itertools import collections @@ -26,15 +29,13 @@ from paddle.fluid.op import Operator from paddle.fluid.executor import Executor from paddle.fluid.framework import Program, OpProtoHolder, Variable from testsuite import create_op, set_input, append_input_output, append_loss_ops -from functools import reduce -from six.moves import zip def randomize_probability(batch_size, class_num, dtype='float32'): prob = np.random.uniform( 0.1, 1.0, size=(batch_size, class_num)).astype(dtype) prob_sum = prob.sum(axis=1) - for i in range(len(prob)): + for i in six.moves.xrange(len(prob)): prob[i] /= prob_sum[i] return prob @@ -51,7 +52,7 @@ def get_numeric_gradient(place, set_input(scope, op, inputs, place) def product(dim): - return reduce(lambda a, b: a * b, dim, 1) + return six.moves.reduce(lambda a, b: a * b, dim, 1) def get_output(): sum = [] @@ -103,7 +104,7 @@ def get_numeric_gradient(place, # we only compute gradient of one element each time. # we use a for loop to compute the gradient of every element. - for i in range(tensor_size): + for i in six.moves.xrange(tensor_size): if in_place: set_input(scope, op, inputs, place) @@ -161,7 +162,7 @@ class OpTest(unittest.TestCase): assert isinstance( numpy_dict, dict), "self.inputs, self.outputs must be numpy_dict" - for var_name, var_value in numpy_dict.items(): + for var_name, var_value in six.iteritems(numpy_dict): if isinstance(var_value, (np.ndarray, np.generic)): self.try_call_once(var_value.dtype) elif isinstance(var_value, (list, tuple)): @@ -225,7 +226,7 @@ class OpTest(unittest.TestCase): def _get_io_vars(self, block, numpy_inputs): inputs = {} - for name, value in numpy_inputs.items(): + for name, value in six.iteritems(numpy_inputs): if isinstance(value, list): var_list = [ block.var(sub_name) for sub_name, sub_value in value @@ -268,7 +269,7 @@ class OpTest(unittest.TestCase): # if the fetch_list is customized by user, we use it directly. # if not, fill the fetch_list by the user configured outputs in test. if len(fetch_list) == 0: - for var_name, var in outputs.items(): + for var_name, var in six.iteritems(outputs): if isinstance(var, list): for v in var: fetch_list.append(v) @@ -366,12 +367,13 @@ class OpTest(unittest.TestCase): for place in places: outs = self.calc_output(place) outs = [np.array(out) for out in outs] + outs.sort(key=len) checker(outs) def __assert_is_close(self, numeric_grads, analytic_grads, names, max_relative_error, msg_prefix): - for a, b, name in zip(numeric_grads, analytic_grads, names): + for a, b, name in six.moves.zip(numeric_grads, analytic_grads, names): abs_a = np.abs(a) abs_a[abs_a < 1e-3] = 1 diff --git a/python/paddle/fluid/tests/unittests/parallel_executor_test_base.py b/python/paddle/fluid/tests/unittests/parallel_executor_test_base.py index 67c35e9de7e83699bf30ca946856bb907152cbdd..9be53c4609d5027dd4e482ab936a135420a346ae 100644 --- a/python/paddle/fluid/tests/unittests/parallel_executor_test_base.py +++ b/python/paddle/fluid/tests/unittests/parallel_executor_test_base.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import multiprocessing import os import unittest diff --git a/python/paddle/fluid/tests/unittests/test_accuracy_op.py b/python/paddle/fluid/tests/unittests/test_accuracy_op.py index db1861fd10e371ebe631a16380af591875886769..1b2b53f2d4ce91ae7b5b191ed770b5338f0948c8 100644 --- a/python/paddle/fluid/tests/unittests/test_accuracy_op.py +++ b/python/paddle/fluid/tests/unittests/test_accuracy_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_activation_mkldnn_op.py b/python/paddle/fluid/tests/unittests/test_activation_mkldnn_op.py index 7d554c2276c9acd710d14c8f8b32c802e3e17515..611d0dd076b827b0f528f2e3a31182cc4939d1f1 100644 --- a/python/paddle/fluid/tests/unittests/test_activation_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/test_activation_mkldnn_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_activation_op.py b/python/paddle/fluid/tests/unittests/test_activation_op.py index 34f9cf0620fd1351111e93e16ed5f7e765d7078b..30651c1326328180592520447e597aa722146a42 100644 --- a/python/paddle/fluid/tests/unittests/test_activation_op.py +++ b/python/paddle/fluid/tests/unittests/test_activation_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_adadelta_op.py b/python/paddle/fluid/tests/unittests/test_adadelta_op.py index 1b892e64c7654a1a3905672813452650885790a5..969a7da3b71b69296f3313342adbf989c60edb50 100644 --- a/python/paddle/fluid/tests/unittests/test_adadelta_op.py +++ b/python/paddle/fluid/tests/unittests/test_adadelta_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_adagrad_op.py b/python/paddle/fluid/tests/unittests/test_adagrad_op.py index 2f0ea79f4d6afe91ee7e0d747f3d8f4884d8f9ee..fc3b7ce2fd87afc22030bcca55236fb949c1f129 100644 --- a/python/paddle/fluid/tests/unittests/test_adagrad_op.py +++ b/python/paddle/fluid/tests/unittests/test_adagrad_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_adam_op.py b/python/paddle/fluid/tests/unittests/test_adam_op.py index fa4b39879c0ede569b6802502b2c71a93b163373..5318d2f9766ce671925be614feef57d679270b19 100644 --- a/python/paddle/fluid/tests/unittests/test_adam_op.py +++ b/python/paddle/fluid/tests/unittests/test_adam_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_adamax_op.py b/python/paddle/fluid/tests/unittests/test_adamax_op.py index 8099beefa583d152715334e83f0c6e8e4a3e7a0d..a6d1be7616c73019cd8f66dcf0c108cd58ec600b 100644 --- a/python/paddle/fluid/tests/unittests/test_adamax_op.py +++ b/python/paddle/fluid/tests/unittests/test_adamax_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_anchor_generator_op.py b/python/paddle/fluid/tests/unittests/test_anchor_generator_op.py index 9c7d5d41f0c512a9fb609dce304c1eed929d28b5..d31eaa0114c3b035add3e6ca792696b5cafb9690 100644 --- a/python/paddle/fluid/tests/unittests/test_anchor_generator_op.py +++ b/python/paddle/fluid/tests/unittests/test_anchor_generator_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import sys diff --git a/python/paddle/fluid/tests/unittests/test_arg_min_max_op.py b/python/paddle/fluid/tests/unittests/test_arg_min_max_op.py index e04412f809cdd75d07d28a60f0c2f19041a684f6..0712e102b30fc72c7f8b62eb9230e7f4ab615ef0 100644 --- a/python/paddle/fluid/tests/unittests/test_arg_min_max_op.py +++ b/python/paddle/fluid/tests/unittests/test_arg_min_max_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_argsort_op.py b/python/paddle/fluid/tests/unittests/test_argsort_op.py index b29a102a3880406156481fdac54ca7043d3415db..7bc6f2599d617b192908da9b57d0cd715019bd71 100644 --- a/python/paddle/fluid/tests/unittests/test_argsort_op.py +++ b/python/paddle/fluid/tests/unittests/test_argsort_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_array_read_write_op.py b/python/paddle/fluid/tests/unittests/test_array_read_write_op.py index 0000fb0958a129e9e1098de1fad888c503cfbdc5..b86d0bc43a9f84988f2b1b27f7aeffce46a46bd9 100644 --- a/python/paddle/fluid/tests/unittests/test_array_read_write_op.py +++ b/python/paddle/fluid/tests/unittests/test_array_read_write_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.core as core import paddle.fluid.layers as layers diff --git a/python/paddle/fluid/tests/unittests/test_assign_op.py b/python/paddle/fluid/tests/unittests/test_assign_op.py index e93c02bd3ee9f710cbb9bff4e195dfc3caabe422..ba2eecfaf197ea63c187e77ae7ae8cf34873d66b 100644 --- a/python/paddle/fluid/tests/unittests/test_assign_op.py +++ b/python/paddle/fluid/tests/unittests/test_assign_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import op_test import numpy import unittest diff --git a/python/paddle/fluid/tests/unittests/test_assign_value_op.py b/python/paddle/fluid/tests/unittests/test_assign_value_op.py index 02f2e6eddc80fcce4ca5a444cff82db355c085ca..5a9d8efef1f3e5a9e116720c2ffe32c2ef0a082f 100644 --- a/python/paddle/fluid/tests/unittests/test_assign_value_op.py +++ b/python/paddle/fluid/tests/unittests/test_assign_value_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid as fluid import paddle.fluid.layers as layers import op_test diff --git a/python/paddle/fluid/tests/unittests/test_auc_op.py b/python/paddle/fluid/tests/unittests/test_auc_op.py index 6580c70ca68c4ba24919f03d071f6f88fb68953c..5393a17e674a3cad6d705a1ff7a45320e644af94 100644 --- a/python/paddle/fluid/tests/unittests/test_auc_op.py +++ b/python/paddle/fluid/tests/unittests/test_auc_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_batch_norm_mkldnn_op.py b/python/paddle/fluid/tests/unittests/test_batch_norm_mkldnn_op.py index 18fa5461590134d2032a29e40699109c12092c6d..1286cee8dc1855c1b1695da46ae0b5222c065114 100644 --- a/python/paddle/fluid/tests/unittests/test_batch_norm_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/test_batch_norm_mkldnn_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_batch_norm_op.py b/python/paddle/fluid/tests/unittests/test_batch_norm_op.py index f805fdc35f624bf6e9d94d66839dcb2a0143a29b..80261eff4e747f87658bc7c9114c21bee511df09 100644 --- a/python/paddle/fluid/tests/unittests/test_batch_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_batch_norm_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_beam_search_decode_op.py b/python/paddle/fluid/tests/unittests/test_beam_search_decode_op.py index 4a3ac2a31e072eb1a15af31f558cf9f626a7ac53..51eee41ab2d4d1113426991c63bee949cca15ad4 100644 --- a/python/paddle/fluid/tests/unittests/test_beam_search_decode_op.py +++ b/python/paddle/fluid/tests/unittests/test_beam_search_decode_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_beam_search_op.py b/python/paddle/fluid/tests/unittests/test_beam_search_op.py index e8283fc9422d93af5735aaec1a165b46ac1ef78e..c28dda4b53ce5d394ff11222e5df8d257b4e80da 100644 --- a/python/paddle/fluid/tests/unittests/test_beam_search_op.py +++ b/python/paddle/fluid/tests/unittests/test_beam_search_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import logging from paddle.fluid.op import Operator, DynamicRecurrentOp import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_bilinear_interp_op.py b/python/paddle/fluid/tests/unittests/test_bilinear_interp_op.py index b04f25ef874cc6204211a4f5f5991a0ec8c473dd..bed847c3c168c906a89c32631b2a8f0ba2e6e7be 100644 --- a/python/paddle/fluid/tests/unittests/test_bilinear_interp_op.py +++ b/python/paddle/fluid/tests/unittests/test_bilinear_interp_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_bilinear_tensor_product_op.py b/python/paddle/fluid/tests/unittests/test_bilinear_tensor_product_op.py index d20a11e27eaac12534ea33c398247adb8db01d4b..46831119c5fee938780ec8fdb9d0cdb3b63a473d 100644 --- a/python/paddle/fluid/tests/unittests/test_bilinear_tensor_product_op.py +++ b/python/paddle/fluid/tests/unittests/test_bilinear_tensor_product_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_bipartite_match_op.py b/python/paddle/fluid/tests/unittests/test_bipartite_match_op.py index ceeca25b74d85ed2874d672e402e3186c4ce7d47..5cc8e2ba15d260b988ee66a5711aed42ca04c10b 100644 --- a/python/paddle/fluid/tests/unittests/test_bipartite_match_op.py +++ b/python/paddle/fluid/tests/unittests/test_bipartite_match_op.py @@ -11,6 +11,8 @@ #WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #See the License for the specific language governing permissions and #limitations under the License. + +from __future__ import print_function import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_box_coder_op.py b/python/paddle/fluid/tests/unittests/test_box_coder_op.py index 4ce9a4783e2332b6882164a70e1462c6a6d31bef..2511c5c22e012babdeb71a71d3546456ea2ceaf3 100644 --- a/python/paddle/fluid/tests/unittests/test_box_coder_op.py +++ b/python/paddle/fluid/tests/unittests/test_box_coder_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import sys diff --git a/python/paddle/fluid/tests/unittests/test_calc_gradient.py b/python/paddle/fluid/tests/unittests/test_calc_gradient.py index 7f2a9e6971ed933463216e38498d48ab132a1a37..4120a18b72f87c7e750a0fb68780292b58e3a7f4 100644 --- a/python/paddle/fluid/tests/unittests/test_calc_gradient.py +++ b/python/paddle/fluid/tests/unittests/test_calc_gradient.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/unittests/test_cast_op.py b/python/paddle/fluid/tests/unittests/test_cast_op.py index b8d3ed3aa3eb0e47e79f46cdf681a3b9cca46036..71a2ccb6da47588d84c263105560626435ac461a 100644 --- a/python/paddle/fluid/tests/unittests/test_cast_op.py +++ b/python/paddle/fluid/tests/unittests/test_cast_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import op_test import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_chunk_eval_op.py b/python/paddle/fluid/tests/unittests/test_chunk_eval_op.py index 354110f1f96f6b4aad1a4866c8d1337dec3acd16..48eb8e9f7585d41d541ac3645e9a50dc79058de7 100644 --- a/python/paddle/fluid/tests/unittests/test_chunk_eval_op.py +++ b/python/paddle/fluid/tests/unittests/test_chunk_eval_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_clip_by_norm_op.py b/python/paddle/fluid/tests/unittests/test_clip_by_norm_op.py index 129958fa2818418dccce91683a9424e6324c6ac2..6103c3aafc0bb154194314830c5c8c5d89460cfe 100644 --- a/python/paddle/fluid/tests/unittests/test_clip_by_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_clip_by_norm_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_clip_op.py b/python/paddle/fluid/tests/unittests/test_clip_op.py index 3df80c8ec8fa39c0aaa2b8726fe3b37aef488442..32677bdb4c897b4e20f8fb166b080ac6e6a221b7 100644 --- a/python/paddle/fluid/tests/unittests/test_clip_op.py +++ b/python/paddle/fluid/tests/unittests/test_clip_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_compare_op.py b/python/paddle/fluid/tests/unittests/test_compare_op.py index 405afebae85eaae6f6af0012058ad58c8bb69a2f..437ad35538a5fa380f950fd3b71e334276214ec7 100644 --- a/python/paddle/fluid/tests/unittests/test_compare_op.py +++ b/python/paddle/fluid/tests/unittests/test_compare_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import op_test import unittest import numpy diff --git a/python/paddle/fluid/tests/unittests/test_compat.py b/python/paddle/fluid/tests/unittests/test_compat.py new file mode 100644 index 0000000000000000000000000000000000000000..1c2c46f99a82875b917a330d6ec76062222420de --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_compat.py @@ -0,0 +1,505 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import unittest +import paddle.compat as cpt +import six + + +class TestCompatible(unittest.TestCase): + def test_type(self): + if six.PY2: + self.assertEqual(cpt.int_type, int) + self.assertEqual(cpt.long_type, long) + else: + self.assertEqual(cpt.int_type, int) + self.assertEqual(cpt.long_type, int) + + def test_to_text(self): + # Only support python2.x and python3.x now + self.assertTrue(six.PY2 | six.PY3) + + if six.PY2: + # check None + self.assertIsNone(cpt.to_text(None)) + + # check all string related types + self.assertTrue(isinstance(cpt.to_text(str("")), unicode)) + self.assertTrue(isinstance(cpt.to_text(str("123")), unicode)) + self.assertTrue(isinstance(cpt.to_text(b""), unicode)) + self.assertTrue(isinstance(cpt.to_text(b""), unicode)) + self.assertTrue(isinstance(cpt.to_text(u""), unicode)) + self.assertTrue(isinstance(cpt.to_text(u""), unicode)) + + self.assertEqual(u"", cpt.to_text(str(""))) + self.assertEqual(u"123", cpt.to_text(str("123"))) + self.assertEqual(u"", cpt.to_text(b"")) + self.assertEqual(u"123", cpt.to_text(b"123")) + self.assertEqual(u"", cpt.to_text(u"")) + self.assertEqual(u"123", cpt.to_text(u"123")) + + # check list types, not inplace + l = [""] + l2 = cpt.to_text(l) + self.assertTrue(isinstance(l2, list)) + self.assertFalse(l is l2) + self.assertEqual(l, l2) + self.assertEqual([u""], l2) + l = ["", "123"] + l2 = cpt.to_text(l) + self.assertTrue(isinstance(l2, list)) + self.assertFalse(l is l2) + self.assertEqual(l, l2) + self.assertEqual([u"", u"123"], l2) + l = ["", b'123', u"321"] + l2 = cpt.to_text(l) + self.assertTrue(isinstance(l2, list)) + self.assertFalse(l is l2) + self.assertEqual(l, l2) + self.assertEqual([u"", u"123", u"321"], l2) + for i in l2: + self.assertTrue(isinstance(i, unicode)) + + # check list types, inplace + l = [""] + l2 = cpt.to_text(l, inplace=True) + self.assertTrue(isinstance(l2, list)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual([u""], l2) + l = ["", "123"] + l2 = cpt.to_text(l, inplace=True) + self.assertTrue(isinstance(l2, list)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual([u"", u"123"], l2) + l = ["", b"123", u"321"] + l2 = cpt.to_text(l, inplace=True) + self.assertTrue(isinstance(l2, list)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual([u"", u"123", u"321"], l2) + + # check set types, not inplace + l = set("") + l2 = cpt.to_text(l, inplace=False) + self.assertTrue(isinstance(l2, set)) + self.assertFalse(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set(u""), l2) + l = set([b"", b"123"]) + l2 = cpt.to_text(l, inplace=False) + self.assertTrue(isinstance(l2, set)) + self.assertFalse(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set([u"", u"123"]), l2) + l = set(["", b"123", u"321"]) + l2 = cpt.to_text(l, inplace=False) + self.assertTrue(isinstance(l2, set)) + self.assertFalse(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set([u"", u"123", u"321"]), l2) + for i in l2: + self.assertTrue(isinstance(i, unicode)) + + # check set types, inplace + l = set("") + l2 = cpt.to_text(l, inplace=True) + self.assertTrue(isinstance(l2, set)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set(u""), l2) + l = set([b"", b"123"]) + l2 = cpt.to_text(l, inplace=True) + self.assertTrue(isinstance(l2, set)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set([u"", u"123"]), l2) + l = set(["", b"123", u"321"]) + l2 = cpt.to_text(l, inplace=True) + self.assertTrue(isinstance(l2, set)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set([u"", u"123", u"321"]), l2) + + elif six.PY3: + self.assertIsNone(cpt.to_text(None)) + + self.assertTrue(isinstance(cpt.to_text(str("")), str)) + self.assertTrue(isinstance(cpt.to_text(str("123")), str)) + self.assertTrue(isinstance(cpt.to_text(b""), str)) + self.assertTrue(isinstance(cpt.to_text(b""), str)) + self.assertTrue(isinstance(cpt.to_text(u""), str)) + self.assertTrue(isinstance(cpt.to_text(u""), str)) + + self.assertEqual("", cpt.to_text(str(""))) + self.assertEqual("123", cpt.to_text(str("123"))) + self.assertEqual("", cpt.to_text(b"")) + self.assertEqual("123", cpt.to_text(b"123")) + self.assertEqual("", cpt.to_text(u"")) + self.assertEqual("123", cpt.to_text(u"123")) + + # check list types, not inplace + l = [""] + l2 = cpt.to_text(l) + self.assertTrue(isinstance(l2, list)) + self.assertFalse(l is l2) + self.assertEqual(l, l2) + self.assertEqual([""], l2) + l = ["", "123"] + l2 = cpt.to_text(l) + self.assertTrue(isinstance(l2, list)) + self.assertFalse(l is l2) + self.assertEqual(l, l2) + self.assertEqual(["", "123"], l2) + l = ["", b"123", u"321"] + l2 = cpt.to_text(l) + self.assertTrue(isinstance(l2, list)) + self.assertFalse(l is l2) + self.assertNotEqual(l, l2) + self.assertEqual(["", "123", "321"], l2) + + # check list types, inplace + l = [""] + l2 = cpt.to_text(l, inplace=True) + self.assertTrue(isinstance(l2, list)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual([""], l2) + l = ["", b"123"] + l2 = cpt.to_text(l, inplace=True) + self.assertTrue(isinstance(l2, list)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual(["", "123"], l2) + l = ["", b"123", u"321"] + l2 = cpt.to_text(l, inplace=True) + self.assertTrue(isinstance(l2, list)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual(["", "123", "321"], l2) + for i in l2: + self.assertTrue(isinstance(i, str)) + + # check set types, not inplace + l = set("") + l2 = cpt.to_text(l, inplace=False) + self.assertTrue(isinstance(l2, set)) + self.assertFalse(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set(""), l2) + l = set([b"", b"123"]) + l2 = cpt.to_text(l, inplace=False) + self.assertTrue(isinstance(l2, set)) + self.assertFalse(l is l2) + self.assertNotEqual(l, l2) + self.assertEqual(set(["", "123"]), l2) + l = set(["", b"123", u"321"]) + l2 = cpt.to_text(l, inplace=False) + self.assertTrue(isinstance(l2, set)) + self.assertFalse(l is l2) + self.assertNotEqual(l, l2) + self.assertEqual(set(["", "123", "321"]), l2) + + # check set types, inplace + l = set("") + l2 = cpt.to_text(l, inplace=True) + self.assertTrue(isinstance(l2, set)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set(""), l2) + l = set([b"", b"123"]) + l2 = cpt.to_text(l, inplace=True) + self.assertTrue(isinstance(l2, set)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set(["", "123"]), l2) + l = set(["", b"123", u"321"]) + l2 = cpt.to_text(l, inplace=True) + self.assertTrue(isinstance(l2, set)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set(["", "123", "321"]), l2) + for i in l2: + self.assertTrue(isinstance(i, str)) + + def test_to_bytes(self): + # Only support python2.x and python3.x now + self.assertTrue(six.PY2 | six.PY3) + + if six.PY2: + # check None + self.assertIsNone(cpt.to_bytes(None)) + + # check all string related types + self.assertTrue(isinstance(cpt.to_bytes(str("")), bytes)) + self.assertTrue(isinstance(cpt.to_bytes(str("123")), bytes)) + self.assertTrue(isinstance(cpt.to_bytes(b""), bytes)) + self.assertTrue(isinstance(cpt.to_bytes(b""), bytes)) + self.assertTrue(isinstance(cpt.to_bytes(u""), bytes)) + self.assertTrue(isinstance(cpt.to_bytes(u""), bytes)) + + self.assertEqual(b"", cpt.to_bytes(str(""))) + self.assertEqual(b"123", cpt.to_bytes(str("123"))) + self.assertEqual(b"", cpt.to_bytes(b"")) + self.assertEqual(b"123", cpt.to_bytes(b"123")) + self.assertEqual(b"", cpt.to_bytes(u"")) + self.assertEqual(b"123", cpt.to_bytes(u"123")) + + # check list types, not inplace + l = [""] + l2 = cpt.to_bytes(l) + self.assertTrue(isinstance(l2, list)) + self.assertFalse(l is l2) + self.assertEqual(l, l2) + self.assertEqual([b""], l2) + l = ["", "123"] + l2 = cpt.to_bytes(l) + self.assertTrue(isinstance(l2, list)) + self.assertFalse(l is l2) + self.assertEqual(l, l2) + self.assertEqual([b"", b"123"], l2) + l = ["", b'123', u"321"] + l2 = cpt.to_bytes(l) + self.assertTrue(isinstance(l2, list)) + self.assertFalse(l is l2) + self.assertEqual(l, l2) + self.assertEqual([b"", b"123", b"321"], l2) + for i in l2: + self.assertTrue(isinstance(i, bytes)) + + # check list types, inplace + l = [""] + l2 = cpt.to_bytes(l, inplace=True) + self.assertTrue(isinstance(l2, list)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual([b""], l2) + l = ["", "123"] + l2 = cpt.to_bytes(l, inplace=True) + self.assertTrue(isinstance(l2, list)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual([b"", b"123"], l2) + l = ["", b"123", u"321"] + l2 = cpt.to_bytes(l, inplace=True) + self.assertTrue(isinstance(l2, list)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual([b"", b"123", b"321"], l2) + + # check set types, not inplace + l = set("") + l2 = cpt.to_bytes(l, inplace=False) + self.assertTrue(isinstance(l2, set)) + self.assertFalse(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set(b""), l2) + l = set([b"", b"123"]) + l2 = cpt.to_bytes(l, inplace=False) + self.assertTrue(isinstance(l2, set)) + self.assertFalse(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set([b"", b"123"]), l2) + l = set(["", b"123", u"321"]) + l2 = cpt.to_bytes(l, inplace=False) + self.assertTrue(isinstance(l2, set)) + self.assertFalse(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set([b"", b"123", b"321"]), l2) + for i in l2: + self.assertTrue(isinstance(i, bytes)) + + # check set types, inplace + l = set("") + l2 = cpt.to_bytes(l, inplace=True) + self.assertTrue(isinstance(l2, set)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set(b""), l2) + l = set([b"", b"123"]) + l2 = cpt.to_bytes(l, inplace=True) + self.assertTrue(isinstance(l2, set)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set([b"", b"123"]), l2) + l = set(["", b"123", u"321"]) + l2 = cpt.to_bytes(l, inplace=True) + self.assertTrue(isinstance(l2, set)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set([b"", b"123", b"321"]), l2) + + elif six.PY3: + self.assertIsNone(cpt.to_bytes(None)) + + self.assertTrue(isinstance(cpt.to_bytes(str("")), bytes)) + self.assertTrue(isinstance(cpt.to_bytes(str("123")), bytes)) + self.assertTrue(isinstance(cpt.to_bytes(b""), bytes)) + self.assertTrue(isinstance(cpt.to_bytes(b""), bytes)) + self.assertTrue(isinstance(cpt.to_bytes(u""), bytes)) + self.assertTrue(isinstance(cpt.to_bytes(u""), bytes)) + + self.assertEqual(b"", cpt.to_bytes(str(""))) + self.assertEqual(b"123", cpt.to_bytes(str("123"))) + self.assertEqual(b"", cpt.to_bytes(b"")) + self.assertEqual(b"123", cpt.to_bytes(b"123")) + self.assertEqual(b"", cpt.to_bytes(u"")) + self.assertEqual(b"123", cpt.to_bytes(u"123")) + + # check list types, not inplace + l = [""] + l2 = cpt.to_bytes(l) + self.assertTrue(isinstance(l2, list)) + self.assertFalse(l is l2) + self.assertNotEqual(l, l2) + self.assertEqual([b""], l2) + l = ["", "123"] + l2 = cpt.to_bytes(l) + self.assertTrue(isinstance(l2, list)) + self.assertFalse(l is l2) + self.assertNotEqual(l, l2) + self.assertEqual([b"", b"123"], l2) + l = ["", b"123", u"321"] + l2 = cpt.to_bytes(l) + self.assertTrue(isinstance(l2, list)) + self.assertFalse(l is l2) + self.assertNotEqual(l, l2) + self.assertEqual([b"", b"123", b"321"], l2) + + # check list types, inplace + l = [""] + l2 = cpt.to_bytes(l, inplace=True) + self.assertTrue(isinstance(l2, list)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual([b""], l2) + l = ["", b"123"] + l2 = cpt.to_bytes(l, inplace=True) + self.assertTrue(isinstance(l2, list)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual([b"", b"123"], l2) + l = ["", b"123", u"321"] + l2 = cpt.to_bytes(l, inplace=True) + self.assertTrue(isinstance(l2, list)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual([b"", b"123", b"321"], l2) + for i in l2: + self.assertTrue(isinstance(i, bytes)) + + # check set types, not inplace + l = set([""]) + l2 = cpt.to_bytes(l, inplace=False) + self.assertTrue(isinstance(l2, set)) + self.assertFalse(l is l2) + self.assertNotEqual(l, l2) + self.assertEqual(set([b""]), l2) + l = set([u"", u"123"]) + l2 = cpt.to_bytes(l, inplace=False) + self.assertTrue(isinstance(l2, set)) + self.assertFalse(l is l2) + self.assertNotEqual(l, l2) + self.assertEqual(set([b"", b"123"]), l2) + l = set(["", b"123", u"321"]) + l2 = cpt.to_bytes(l, inplace=False) + self.assertTrue(isinstance(l2, set)) + self.assertFalse(l is l2) + self.assertNotEqual(l, l2) + self.assertEqual(set([b"", b"123", b"321"]), l2) + + # check set types, inplace + l = set("") + l2 = cpt.to_bytes(l, inplace=True) + self.assertTrue(isinstance(l2, set)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set(b""), l2) + l = set([u"", u"123"]) + l2 = cpt.to_bytes(l, inplace=True) + self.assertTrue(isinstance(l2, set)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set([b"", b"123"]), l2) + l = set(["", b"123", u"321"]) + l2 = cpt.to_bytes(l, inplace=True) + self.assertTrue(isinstance(l2, set)) + self.assertTrue(l is l2) + self.assertEqual(l, l2) + self.assertEqual(set([b"", b"123", b"321"]), l2) + for i in l2: + self.assertTrue(isinstance(i, bytes)) + + def test_round(self): + self.assertEqual(3.0, cpt.round(3.4)) + self.assertEqual(4.0, cpt.round(3.5)) + self.assertEqual(0.0, cpt.round(0.1)) + self.assertEqual(0.0, cpt.round(0.0)) + self.assertEqual(-0.0, cpt.round(-0.0)) + self.assertEqual(-0.0, cpt.round(-0.1)) + self.assertEqual(-3.0, cpt.round(-3.4)) + self.assertEqual(-4.0, cpt.round(-3.5)) + self.assertEqual(5.0, cpt.round(5)) + self.assertRaises(TypeError, cpt.round, None) + + def test_floor_division(self): + self.assertEqual(0.0, cpt.floor_division(3, 4)) + self.assertEqual(1.0, cpt.floor_division(4, 3)) + self.assertEqual(2.0, cpt.floor_division(6, 3)) + self.assertEqual(-2.0, cpt.floor_division(-4, 3)) + self.assertEqual(-2.0, cpt.floor_division(-6, 3)) + self.assertRaises(ZeroDivisionError, cpt.floor_division, 3, 0) + self.assertRaises(TypeError, cpt.floor_division, None, None) + + def test_get_exception_message(self): + exception_message = "test_message" + self.assertRaises(AssertionError, cpt.get_exception_message, None) + if six.PY2: + self.assertRaises(AttributeError, cpt.get_exception_message, + exception_message) + try: + raise RuntimeError(exception_message) + except Exception as e: + self.assertEqual(exception_message, + cpt.get_exception_message(e)) + self.assertIsNotNone(e) + + try: + raise Exception(exception_message) + except Exception as e: + self.assertEqual(exception_message, + cpt.get_exception_message(e)) + self.assertIsNotNone(e) + + if six.PY3: + try: + raise RuntimeError(exception_message) + except Exception as e: + self.assertEqual(exception_message, + cpt.get_exception_message(e)) + self.assertIsNotNone(e) + + try: + raise Exception(exception_message) + except Exception as e: + self.assertEqual(exception_message, + cpt.get_exception_message(e)) + self.assertIsNotNone(e) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_concat_op.py b/python/paddle/fluid/tests/unittests/test_concat_op.py index e9f3c45dc40b3333fe7304f8e4313d156bd5374c..436ab7d49f4cafcd30366ae57c40d49e6f7d614f 100644 --- a/python/paddle/fluid/tests/unittests/test_concat_op.py +++ b/python/paddle/fluid/tests/unittests/test_concat_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_conditional_block.py b/python/paddle/fluid/tests/unittests/test_conditional_block.py index 77869a1242e08d348bfb1031b8f5b1ab5c81d868..5b2b71d050c42b4fea84bab89824d3f5c164b36e 100644 --- a/python/paddle/fluid/tests/unittests/test_conditional_block.py +++ b/python/paddle/fluid/tests/unittests/test_conditional_block.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.layers as layers import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_const_value.py b/python/paddle/fluid/tests/unittests/test_const_value.py index 58ac6fa0a9a30a08a831111513777cca59062724..0b2431d7726e845da33f6bcf9c74058788dd9654 100644 --- a/python/paddle/fluid/tests/unittests/test_const_value.py +++ b/python/paddle/fluid/tests/unittests/test_const_value.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.framework as framework diff --git a/python/paddle/fluid/tests/unittests/test_conv2d_mkldnn_op.py b/python/paddle/fluid/tests/unittests/test_conv2d_mkldnn_op.py index d0de7ad52c8a851c16cbbbf544d479f696dee136..1902a9869807ba7ce3f9828c124256cc6752857e 100644 --- a/python/paddle/fluid/tests/unittests/test_conv2d_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv2d_mkldnn_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest from test_conv2d_op import TestConv2dOp, TestWithPad, TestWithStride diff --git a/python/paddle/fluid/tests/unittests/test_conv2d_op.py b/python/paddle/fluid/tests/unittests/test_conv2d_op.py index bb1cd87d615fa341b7244e9f3e113b9fb4765ac2..6a2732e9399aa5a93f4c47eb73bfd23dba608c3d 100644 --- a/python/paddle/fluid/tests/unittests/test_conv2d_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv2d_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np @@ -24,12 +26,12 @@ def conv2d_forward_naive(input, filter, group, conv_param): out_c, f_c, f_h, f_w = filter.shape assert f_c * group == in_c assert np.mod(out_c, group) == 0 - sub_out_c = out_c / group + sub_out_c = out_c // group stride, pad, dilation = conv_param['stride'], conv_param['pad'], conv_param[ 'dilation'] - out_h = 1 + (in_h + 2 * pad[0] - (dilation[0] * (f_h - 1) + 1)) / stride[0] - out_w = 1 + (in_w + 2 * pad[1] - (dilation[1] * (f_w - 1) + 1)) / stride[1] + out_h = 1 + (in_h + 2 * pad[0] - (dilation[0] * (f_h - 1) + 1)) // stride[0] + out_w = 1 + (in_w + 2 * pad[1] - (dilation[1] * (f_w - 1) + 1)) // stride[1] out = np.zeros((in_n, out_c, out_h, out_w)) d_bolck_h = (dilation[0] * (f_h - 1) + 1) @@ -138,7 +140,7 @@ class TestConv2dOp(OpTest): self.stride = [1, 1] self.input_size = [2, 3, 5, 5] # NCHW assert np.mod(self.input_size[1], self.groups) == 0 - f_c = self.input_size[1] / self.groups + f_c = self.input_size[1] // self.groups self.filter_size = [6, f_c, 3, 3] def init_dilation(self): @@ -157,7 +159,7 @@ class TestWithPad(TestConv2dOp): self.stride = [1, 1] self.input_size = [2, 3, 5, 5] # NCHW assert np.mod(self.input_size[1], self.groups) == 0 - f_c = self.input_size[1] / self.groups + f_c = self.input_size[1] // self.groups self.filter_size = [6, f_c, 3, 3] @@ -167,7 +169,7 @@ class TestWithStride(TestConv2dOp): self.stride = [2, 2] self.input_size = [2, 3, 6, 6] # NCHW assert np.mod(self.input_size[1], self.groups) == 0 - f_c = self.input_size[1] / self.groups + f_c = self.input_size[1] // self.groups self.filter_size = [6, f_c, 3, 3] @@ -182,7 +184,7 @@ class TestWith1x1(TestConv2dOp): self.stride = [1, 1] self.input_size = [2, 3, 5, 5] # NCHW assert np.mod(self.input_size[1], self.groups) == 0 - f_c = self.input_size[1] / self.groups + f_c = self.input_size[1] // self.groups self.filter_size = [6, f_c, 1, 1] def init_group(self): @@ -195,7 +197,7 @@ class TestWithDilation(TestConv2dOp): self.stride = [1, 1] self.input_size = [2, 3, 10, 10] # NCHW assert np.mod(self.input_size[1], self.groups) == 0 - f_c = self.input_size[1] / self.groups + f_c = self.input_size[1] // self.groups self.filter_size = [6, f_c, 3, 3] def init_dilation(self): @@ -211,7 +213,7 @@ class TestWithInput1x1Filter1x1(TestConv2dOp): self.stride = [1, 1] self.input_size = [2, 3, 1, 1] # NCHW assert np.mod(self.input_size[1], self.groups) == 0 - f_c = self.input_size[1] / self.groups + f_c = self.input_size[1] // self.groups self.filter_size = [6, f_c, 1, 1] def init_group(self): @@ -328,7 +330,7 @@ class TestDepthwiseConv(TestConv2dOp): self.input_size = [2, 3, 5, 5] # NCHW self.groups = 3 assert np.mod(self.input_size[1], self.groups) == 0 - f_c = self.input_size[1] / self.groups + f_c = self.input_size[1] // self.groups self.filter_size = [6, f_c, 3, 3] self.op_type = "depthwise_conv2d" @@ -340,7 +342,7 @@ class TestDepthwiseConv2(TestConv2dOp): self.input_size = [2, 3, 5, 5] # NCHW self.groups = 3 assert np.mod(self.input_size[1], self.groups) == 0 - f_c = self.input_size[1] / self.groups + f_c = self.input_size[1] // self.groups self.filter_size = [6, f_c, 3, 3] self.op_type = "depthwise_conv2d" diff --git a/python/paddle/fluid/tests/unittests/test_conv2d_transpose_op.py b/python/paddle/fluid/tests/unittests/test_conv2d_transpose_op.py index af6cd99b0d7e6b0a2dfd4fc1d33e8390017a5906..2a320e735bd7db5dc138f8263ba1b5cb115ba197 100644 --- a/python/paddle/fluid/tests/unittests/test_conv2d_transpose_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv2d_transpose_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np @@ -25,7 +27,7 @@ def conv2dtranspose_forward_naive(input_, filter_, attrs): groups = attrs['groups'] assert in_c == f_c out_c = f_out_c * groups - sub_in_c = in_c / groups + sub_in_c = in_c // groups stride, pad, dilations = attrs['strides'], attrs['paddings'], attrs[ 'dilations'] @@ -258,7 +260,7 @@ class TestDepthwiseConvTranspose(TestConv2dTransposeOp): self.input_size = [2, 8, 16, 16] # NCHW self.groups = 8 assert np.mod(self.input_size[1], self.groups) == 0 - f_c = self.input_size[1] / self.groups + f_c = self.input_size[1] // self.groups self.filter_size = [self.input_size[1], f_c, 4, 4] self.op_type = "depthwise_conv2d_transpose" diff --git a/python/paddle/fluid/tests/unittests/test_conv3d_op.py b/python/paddle/fluid/tests/unittests/test_conv3d_op.py index dd4ef7cc94ea1e8de5fe4775408389907d47d0d6..ddaf99fe061205f0f2e4c592c9e28e27e657c16a 100644 --- a/python/paddle/fluid/tests/unittests/test_conv3d_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv3d_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np @@ -24,14 +26,14 @@ def conv3d_forward_naive(input, filter, group, conv_param): out_c, f_c, f_d, f_h, f_w = filter.shape assert f_c * group == in_c assert np.mod(out_c, group) == 0 - sub_out_c = out_c / group + sub_out_c = out_c // group stride, pad, dilation = conv_param['stride'], conv_param['pad'], conv_param[ 'dilations'] - out_d = 1 + (in_d + 2 * pad[0] - (dilation[0] * (f_d - 1) + 1)) / stride[0] - out_h = 1 + (in_h + 2 * pad[1] - (dilation[1] * (f_h - 1) + 1)) / stride[1] - out_w = 1 + (in_w + 2 * pad[2] - (dilation[2] * (f_w - 1) + 1)) / stride[2] + out_d = 1 + (in_d + 2 * pad[0] - (dilation[0] * (f_d - 1) + 1)) // stride[0] + out_h = 1 + (in_h + 2 * pad[1] - (dilation[1] * (f_h - 1) + 1)) // stride[1] + out_w = 1 + (in_w + 2 * pad[2] - (dilation[2] * (f_w - 1) + 1)) // stride[2] out = np.zeros((in_n, out_c, out_d, out_h, out_w)) @@ -166,7 +168,7 @@ class TestConv3dOp(OpTest): self.stride = [1, 1, 1] self.input_size = [2, 3, 4, 4, 4] # NCDHW assert np.mod(self.input_size[1], self.groups) == 0 - f_c = self.input_size[1] / self.groups + f_c = self.input_size[1] // self.groups self.filter_size = [6, f_c, 3, 3, 3] def init_dilation(self): @@ -185,7 +187,7 @@ class TestCase1(TestConv3dOp): self.stride = [1, 1, 1] self.input_size = [2, 3, 4, 4, 4] # NCDHW assert np.mod(self.input_size[1], self.groups) == 0 - f_c = self.input_size[1] / self.groups + f_c = self.input_size[1] // self.groups self.filter_size = [6, f_c, 3, 3, 3] @@ -205,7 +207,7 @@ class TestWith1x1(TestConv3dOp): self.stride = [1, 1, 1] self.input_size = [2, 3, 4, 4, 4] # NCHW assert np.mod(self.input_size[1], self.groups) == 0 - f_c = self.input_size[1] / self.groups + f_c = self.input_size[1] // self.groups self.filter_size = [6, f_c, 1, 1, 1] def init_dilation(self): @@ -221,7 +223,7 @@ class TestWithInput1x1Filter1x1(TestConv3dOp): self.stride = [1, 1, 1] self.input_size = [2, 3, 1, 1, 1] # NCHW assert np.mod(self.input_size[1], self.groups) == 0 - f_c = self.input_size[1] / self.groups + f_c = self.input_size[1] // self.groups self.filter_size = [6, f_c, 1, 1, 1] def init_dilation(self): @@ -237,7 +239,7 @@ class TestWithDilation(TestConv3dOp): self.stride = [1, 1, 1] self.input_size = [2, 3, 6, 6, 6] # NCDHW assert np.mod(self.input_size[1], self.groups) == 0 - f_c = self.input_size[1] / self.groups + f_c = self.input_size[1] // self.groups self.filter_size = [6, f_c, 2, 2, 2] def init_dilation(self): diff --git a/python/paddle/fluid/tests/unittests/test_conv3d_transpose_op.py b/python/paddle/fluid/tests/unittests/test_conv3d_transpose_op.py index 300fa5e8bde001e0f66c5f924a81c30add99aead..8d9075961cbec32bc34fcf0c92cfbb7e6c00d886 100644 --- a/python/paddle/fluid/tests/unittests/test_conv3d_transpose_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv3d_transpose_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np @@ -25,7 +27,7 @@ def conv3dtranspose_forward_naive(input_, filter_, attrs): groups = attrs['groups'] assert in_c == f_c out_c = f_out_c * groups - sub_in_c = in_c / groups + sub_in_c = in_c // groups stride, pad, dilations = attrs['strides'], attrs['paddings'], attrs[ 'dilations'] diff --git a/python/paddle/fluid/tests/unittests/test_conv_shift_op.py b/python/paddle/fluid/tests/unittests/test_conv_shift_op.py index 9fdb7baa90d2184c3c439e76b6bb5f0668f5f9ee..b7364e869e7420e610363eafcc4964b825e57326 100644 --- a/python/paddle/fluid/tests/unittests/test_conv_shift_op.py +++ b/python/paddle/fluid/tests/unittests/test_conv_shift_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest @@ -21,7 +23,7 @@ def conv_shift_forward(x, y): out = np.zeros_like(x) M = x.shape[1] N = y.shape[1] - y_half_width = (N - 1) / 2 + y_half_width = (N - 1) // 2 for i in range(M): for j in range(N): out[:, i] += x[:, (i + j + M - y_half_width) % M] * y[:, j] diff --git a/python/paddle/fluid/tests/unittests/test_cos_sim_op.py b/python/paddle/fluid/tests/unittests/test_cos_sim_op.py index 1b27cd57670e6c9db5eae6b226989a5c772866ce..3c3fd6d4d71503ccc3678ca69d55bcc8536c8c6a 100644 --- a/python/paddle/fluid/tests/unittests/test_cos_sim_op.py +++ b/python/paddle/fluid/tests/unittests/test_cos_sim_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_create_op_doc_string.py b/python/paddle/fluid/tests/unittests/test_create_op_doc_string.py index 07c89eefc32fab37ce093e91d96fbe4471ecddc6..fd34c8fc9390b69afd93229b56aa9189da2a8b28 100644 --- a/python/paddle/fluid/tests/unittests/test_create_op_doc_string.py +++ b/python/paddle/fluid/tests/unittests/test_create_op_doc_string.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.layers as layers diff --git a/python/paddle/fluid/tests/unittests/test_crf_decoding_op.py b/python/paddle/fluid/tests/unittests/test_crf_decoding_op.py index 122b076c2d3e3a69f52a2c335e2bc89707b4fa9b..51bd1300e61d58c934a40abf81ab8f137e44910f 100644 --- a/python/paddle/fluid/tests/unittests/test_crf_decoding_op.py +++ b/python/paddle/fluid/tests/unittests/test_crf_decoding_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import random import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_crop_op.py b/python/paddle/fluid/tests/unittests/test_crop_op.py index 4016089c01644f0389855ab114360f90c50a1bbe..d7bcfba8deab1b73e4cbab8a27f9eeef9a37d29b 100644 --- a/python/paddle/fluid/tests/unittests/test_crop_op.py +++ b/python/paddle/fluid/tests/unittests/test_crop_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_cross_entropy_op.py b/python/paddle/fluid/tests/unittests/test_cross_entropy_op.py index 86ac159323a5f9f6149ce5ed4437402eb885c6bc..fa367f95fc9c65dd782d53a2799cacadf74dcfd2 100644 --- a/python/paddle/fluid/tests/unittests/test_cross_entropy_op.py +++ b/python/paddle/fluid/tests/unittests/test_cross_entropy_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest, randomize_probability diff --git a/python/paddle/fluid/tests/unittests/test_ctc_align.py b/python/paddle/fluid/tests/unittests/test_ctc_align.py index 131b4076f45ae25b45bb3f64da07a5c3aacc43d5..5f17d2d407cca9a4c95d919d05a3a03b784d1942 100644 --- a/python/paddle/fluid/tests/unittests/test_ctc_align.py +++ b/python/paddle/fluid/tests/unittests/test_ctc_align.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import sys import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_cumsum_op.py b/python/paddle/fluid/tests/unittests/test_cumsum_op.py index 04e7f0b94510987a1872c2d625ac4d29a3c6feba..13a4eacece8a211513d6537db0d09b80c238178e 100644 --- a/python/paddle/fluid/tests/unittests/test_cumsum_op.py +++ b/python/paddle/fluid/tests/unittests/test_cumsum_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_data_balance.py b/python/paddle/fluid/tests/unittests/test_data_balance.py index 951282e8bab5018204c0d31caa10f8f84a8f3d6c..e39eedd282daf6bbe0603a22c357e06c95c086b6 100644 --- a/python/paddle/fluid/tests/unittests/test_data_balance.py +++ b/python/paddle/fluid/tests/unittests/test_data_balance.py @@ -12,9 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid -import paddle.v2 as paddle +import paddle import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_debugger.py b/python/paddle/fluid/tests/unittests/test_debugger.py index 870952f2f916dcdec5991ac5c10d2da3a7ab18a8..f4c9466d63a201ba9a5e77515ae64a33bedc5b23 100644 --- a/python/paddle/fluid/tests/unittests/test_debugger.py +++ b/python/paddle/fluid/tests/unittests/test_debugger.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_decayed_adagrad_op.py b/python/paddle/fluid/tests/unittests/test_decayed_adagrad_op.py index 84c44d4817366518a2cbc3f0a777ab32b67f3d11..a664a1529f4de1f372241319b57fad6b0ba8b8a2 100644 --- a/python/paddle/fluid/tests/unittests/test_decayed_adagrad_op.py +++ b/python/paddle/fluid/tests/unittests/test_decayed_adagrad_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_default_scope_funcs.py b/python/paddle/fluid/tests/unittests/test_default_scope_funcs.py index 868bcca881a65dad7d0ecabb1e388818cdd0997e..01a7b6824885b32e922a8eb34f5d8117ee3e584f 100644 --- a/python/paddle/fluid/tests/unittests/test_default_scope_funcs.py +++ b/python/paddle/fluid/tests/unittests/test_default_scope_funcs.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from paddle.fluid.default_scope_funcs import * import unittest diff --git a/python/paddle/fluid/tests/unittests/test_desc_clone.py b/python/paddle/fluid/tests/unittests/test_desc_clone.py index 8603d3a5b3b5d368fe87b8dcf9dc7363f95caf86..88d44e453c7976f5e0fbda2c0871dfabd4bb30aa 100644 --- a/python/paddle/fluid/tests/unittests/test_desc_clone.py +++ b/python/paddle/fluid/tests/unittests/test_desc_clone.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import argparse import time diff --git a/python/paddle/fluid/tests/unittests/test_detection_map_op.py b/python/paddle/fluid/tests/unittests/test_detection_map_op.py index 8b66d1b270980a18fd1bbd068917e982a450ad6f..f6eb8f2c6d8b94f92e24ff789c91efb53a645a46 100644 --- a/python/paddle/fluid/tests/unittests/test_detection_map_op.py +++ b/python/paddle/fluid/tests/unittests/test_detection_map_op.py @@ -12,8 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np +import six import sys import collections import math @@ -176,7 +179,7 @@ class TestDetectionMAPOp(OpTest): true_pos[label].append([score, tp]) false_pos[label].append([score, fp]) - for (label, label_pos_num) in list(label_count.items()): + for (label, label_pos_num) in six.iteritems(label_count): if label_pos_num == 0 or label not in true_pos: continue label_true_pos = true_pos[label] label_false_pos = false_pos[label] diff --git a/python/paddle/fluid/tests/unittests/test_dist_base.py b/python/paddle/fluid/tests/unittests/test_dist_base.py index 4379463aca4443eb7a886ce78446440cc59f3b30..4c71181d0d736bd1a8796b2d38ed1667557e3db8 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_base.py +++ b/python/paddle/fluid/tests/unittests/test_dist_base.py @@ -11,11 +11,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import time import unittest import os import sys +import six import signal import subprocess import six @@ -123,6 +126,9 @@ def runtime_main(test_class): model.run_trainer(p, endpoints, trainer_id, trainers, is_dist) +import paddle.compat as cpt + + class TestDistBase(unittest.TestCase): def setUp(self): self._trainers = 2 @@ -209,7 +215,7 @@ class TestDistBase(unittest.TestCase): local_proc.wait() out, err = local_proc.communicate() - local_ret = out + local_ret = cpt.to_text(out) sys.stderr.write('local_loss: %s\n' % local_ret) sys.stderr.write('local_stderr: %s\n' % err) @@ -256,7 +262,7 @@ class TestDistBase(unittest.TestCase): tr1_proc.wait() out, err = tr0_proc.communicate() sys.stderr.write('dist_stderr: %s\n' % err) - loss_data0 = out + loss_data0 = cpt.to_text(out) sys.stderr.write('dist_loss: %s\n' % loss_data0) lines = loss_data0.split("\n") dist_first_loss = eval(lines[0].replace(" ", ","))[0] diff --git a/python/paddle/fluid/tests/unittests/test_dist_mnist.py b/python/paddle/fluid/tests/unittests/test_dist_mnist.py index b3ccec9a7d65de57778a1f013465d41a5a267676..4ec68d411b0f0e9ae89b107914e8fd844a19228b 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_mnist.py +++ b/python/paddle/fluid/tests/unittests/test_dist_mnist.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import unittest from test_dist_base import TestDistBase diff --git a/python/paddle/fluid/tests/unittests/test_dist_se_resnext.py b/python/paddle/fluid/tests/unittests/test_dist_se_resnext.py index a33a338fc11e4301a8ec0eb565686d62b547b7f7..16525f6fdb60a90a44a628fb0648f4130218c102 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_se_resnext.py +++ b/python/paddle/fluid/tests/unittests/test_dist_se_resnext.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import unittest from test_dist_base import TestDistBase diff --git a/python/paddle/fluid/tests/unittests/test_dist_train.py b/python/paddle/fluid/tests/unittests/test_dist_train.py index 55aa923f5ab229bc8e9a0b891e0ac9c2ec49d31b..9581abdf394d738470d32ae609838832077ee519 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_train.py +++ b/python/paddle/fluid/tests/unittests/test_dist_train.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import os import time import unittest diff --git a/python/paddle/fluid/tests/unittests/test_dist_transformer.py b/python/paddle/fluid/tests/unittests/test_dist_transformer.py index 68cd35d751dbce7eef9919dc8678fc0dd117757b..313207ff9ce054f81322224cb6ceafaaf25bbedf 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_transformer.py +++ b/python/paddle/fluid/tests/unittests/test_dist_transformer.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest from test_dist_base import TestDistBase diff --git a/python/paddle/fluid/tests/unittests/test_dist_transpiler.py b/python/paddle/fluid/tests/unittests/test_dist_transpiler.py index 124abf4ccde98d565b3286c72793c91fd26bb71c..03083c9960636886ffa49137b9e9d4a71fbd72fe 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_transpiler.py +++ b/python/paddle/fluid/tests/unittests/test_dist_transpiler.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import math import unittest @@ -49,6 +51,7 @@ class TranspilerTest(unittest.TestCase): def get_main_program(self): main = fluid.Program() + main.random_seed = 1 with fluid.program_guard(main): self.net_conf() self.origin_prog = main.clone() diff --git a/python/paddle/fluid/tests/unittests/test_dist_word2vec.py b/python/paddle/fluid/tests/unittests/test_dist_word2vec.py index 543d0f9dc2c9b8cdcfaaaa14a7a4f197d210d951..e43992c488d35d1b3f670e13650d420b0498eeec 100644 --- a/python/paddle/fluid/tests/unittests/test_dist_word2vec.py +++ b/python/paddle/fluid/tests/unittests/test_dist_word2vec.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import unittest from test_dist_base import TestDistBase diff --git a/python/paddle/fluid/tests/unittests/test_dropout_op.py b/python/paddle/fluid/tests/unittests/test_dropout_op.py index eaa3435a86462236a99489749abe877648677053..0296bc2af4e0b79478c34b4cceab32b5a8a50f2f 100644 --- a/python/paddle/fluid/tests/unittests/test_dropout_op.py +++ b/python/paddle/fluid/tests/unittests/test_dropout_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_dyn_rnn.py b/python/paddle/fluid/tests/unittests/test_dyn_rnn.py index fdc6adc93bc2488d4faffed61fde5d54bbbbfd57..d84dab1499a267ca081c2e8ea2856c7c4bb627cb 100644 --- a/python/paddle/fluid/tests/unittests/test_dyn_rnn.py +++ b/python/paddle/fluid/tests/unittests/test_dyn_rnn.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid as fluid import paddle import unittest diff --git a/python/paddle/fluid/tests/unittests/test_dynrnn_gradient_check.py b/python/paddle/fluid/tests/unittests/test_dynrnn_gradient_check.py index 7756885166c88eadb77c2c6d56aab767015abc51..9d635f36fe83d041bb57df0759da1481f66bbaa2 100644 --- a/python/paddle/fluid/tests/unittests/test_dynrnn_gradient_check.py +++ b/python/paddle/fluid/tests/unittests/test_dynrnn_gradient_check.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy import random import collections diff --git a/python/paddle/fluid/tests/unittests/test_dynrnn_static_input.py b/python/paddle/fluid/tests/unittests/test_dynrnn_static_input.py index d182889a970fb178dec4976aebbd79d05dc3e91e..b4359fc69ae18b45774af0d2b20c1540bd99da5c 100644 --- a/python/paddle/fluid/tests/unittests/test_dynrnn_static_input.py +++ b/python/paddle/fluid/tests/unittests/test_dynrnn_static_input.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_edit_distance_op.py b/python/paddle/fluid/tests/unittests/test_edit_distance_op.py index 816562621b4fc749f3c6b0eca8ee3c5850ef1ba9..4d03523025d357e453848f3016ffee890b5d46ec 100644 --- a/python/paddle/fluid/tests/unittests/test_edit_distance_op.py +++ b/python/paddle/fluid/tests/unittests/test_edit_distance_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_add_mkldnn_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_add_mkldnn_op.py index bcdbfc8e527d0dc9a95eddaf040f8035207b6c20..d85cc1f856df8eaa73cef318b48a292042488edf 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_add_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_add_mkldnn_op.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import unittest import numpy as np import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_add_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_add_op.py index fb9a496126f0b6efcad73590c78efe5a47b88cd6..5aec5d8e38aba39e6aba9a8f19637587c2f12544 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_add_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_add_op.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import unittest import numpy as np import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_div_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_div_op.py index bfe022af6dac711e76678b79005d4cfff90c2a2b..cadaf1df53af0af56afa8c3631b0f5ce390f318c 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_div_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_div_op.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_gradient_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_gradient_op.py index 6f350044892a4ba2a985b5bc2328ab1fc20c5504..9f452ffde74ee18d14f155fb5ed53fee57f12f49 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_gradient_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_gradient_op.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_max_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_max_op.py index b6cd18a579520f921feed48cc86d8027f6a7bd1e..43c58710ba50c27077942643b84b7642eaf57710 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_max_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_max_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_min_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_min_op.py index 92099724fe65050b62c738f4e6c269a0ca3f4ef1..45c861e2c3df9f14f9886091012d6cca69944454 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_min_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_min_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_mul_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_mul_op.py index 2742bb21d95ed3363e82a70b0172cc787878abd9..775c2253ab3b27708b745b85fc007fcb504d1aed 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_mul_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_mul_op.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_pow_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_pow_op.py index a3fd18669c556701b2586e88ddbb89ca79549a86..7bf642f03f480b1eeec68298f9d453deb1fa2ac3 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_pow_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_pow_op.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_elementwise_sub_op.py b/python/paddle/fluid/tests/unittests/test_elementwise_sub_op.py index 1854232194963bcbe302010320a30d85747eea96..6cb88a8bb1cad7a58ca175cfc14298c959e3bad6 100644 --- a/python/paddle/fluid/tests/unittests/test_elementwise_sub_op.py +++ b/python/paddle/fluid/tests/unittests/test_elementwise_sub_op.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_exception.py b/python/paddle/fluid/tests/unittests/test_exception.py index bb7c0f88f6027807394e15aa6803da2ddc22f4e2..798ed53cddade22e986cae65109b6c6ac7a291b6 100644 --- a/python/paddle/fluid/tests/unittests/test_exception.py +++ b/python/paddle/fluid/tests/unittests/test_exception.py @@ -12,19 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + +import paddle.compat as cpt import paddle.fluid.core as core import unittest class TestException(unittest.TestCase): def test_exception(self): - ex = None + exception = None try: core.__unittest_throw_exception__() except core.EnforceNotMet as ex: - self.assertIn("test exception", ex.message) + self.assertIn("test exception", cpt.get_exception_message(ex)) + exception = ex - self.assertIsNotNone(ex) + self.assertIsNotNone(exception) if __name__ == "__main__": diff --git a/python/paddle/fluid/tests/unittests/test_executor_and_mul.py b/python/paddle/fluid/tests/unittests/test_executor_and_mul.py index e1272c1d6dd7131b55ecf33fa0de0fc78a3ac5a7..b1f89eca6e58aec41b5863f4c885d5c6231a72f4 100644 --- a/python/paddle/fluid/tests/unittests/test_executor_and_mul.py +++ b/python/paddle/fluid/tests/unittests/test_executor_and_mul.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy diff --git a/python/paddle/fluid/tests/unittests/test_expand_op.py b/python/paddle/fluid/tests/unittests/test_expand_op.py index a91e3aef5a18a681f3baf405da2beebb8c85360c..67a8d8f0721c2c75b432d68d64be8fc1035ffc74 100644 --- a/python/paddle/fluid/tests/unittests/test_expand_op.py +++ b/python/paddle/fluid/tests/unittests/test_expand_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_extract_rows_op.py b/python/paddle/fluid/tests/unittests/test_extract_rows_op.py index 6a41c44fe655b18626bdb727745dae032babe8ad..8629bcf0f2e3c37aefdbf79b203176a43e0c3a7e 100644 --- a/python/paddle/fluid/tests/unittests/test_extract_rows_op.py +++ b/python/paddle/fluid/tests/unittests/test_extract_rows_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_fake_dequantize_op.py b/python/paddle/fluid/tests/unittests/test_fake_dequantize_op.py index 026ac2112b2d78644b3315b9cab8019ca27e9714..d84ebed3fac67db323392494c701cf2a51b28305 100644 --- a/python/paddle/fluid/tests/unittests/test_fake_dequantize_op.py +++ b/python/paddle/fluid/tests/unittests/test_fake_dequantize_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import math diff --git a/python/paddle/fluid/tests/unittests/test_fake_quantize_op.py b/python/paddle/fluid/tests/unittests/test_fake_quantize_op.py index 6c6aa9d3bb656740c528c728efafc6a47e8bff91..cc0494774a5f2f24faaae65f193fc3ff9270d9ac 100644 --- a/python/paddle/fluid/tests/unittests/test_fake_quantize_op.py +++ b/python/paddle/fluid/tests/unittests/test_fake_quantize_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_fc_mkldnn_op.py b/python/paddle/fluid/tests/unittests/test_fc_mkldnn_op.py index 099e6e60642e9637f8f3648696e844c667e1c406..45951a34d6f61a242cb2dc004d6801a6c1c9dd92 100644 --- a/python/paddle/fluid/tests/unittests/test_fc_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/test_fc_mkldnn_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_feed_fetch_method.py b/python/paddle/fluid/tests/unittests/test_feed_fetch_method.py index 8b9da843115409c65055927d317867d1290c8f0e..b823d397e9530362f5fee417278e36477d65f6f5 100644 --- a/python/paddle/fluid/tests/unittests/test_feed_fetch_method.py +++ b/python/paddle/fluid/tests/unittests/test_feed_fetch_method.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid.core as core import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_fetch_var.py b/python/paddle/fluid/tests/unittests/test_fetch_var.py index e6f37f0b4ca781e4ec83a00f8f2605ef02716bd7..de339d821b1329662469c26eacd234b74a102e13 100644 --- a/python/paddle/fluid/tests/unittests/test_fetch_var.py +++ b/python/paddle/fluid/tests/unittests/test_fetch_var.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid as fluid import paddle.fluid.layers as layers import op_test diff --git a/python/paddle/fluid/tests/unittests/test_fill_constant_batch_size_like_op.py b/python/paddle/fluid/tests/unittests/test_fill_constant_batch_size_like_op.py index 0c75cf33f5f208d11081a6802910c25553b8c4ec..fdc8a118e56f4473da5ed60169daebec14c7c33c 100644 --- a/python/paddle/fluid/tests/unittests/test_fill_constant_batch_size_like_op.py +++ b/python/paddle/fluid/tests/unittests/test_fill_constant_batch_size_like_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_fill_constant_op.py b/python/paddle/fluid/tests/unittests/test_fill_constant_op.py index 5e2ddb218af8fcf4f686260296b57519ec7486b9..44fb1d047dff48d2554c0bf637afbfda725e0a02 100644 --- a/python/paddle/fluid/tests/unittests/test_fill_constant_op.py +++ b/python/paddle/fluid/tests/unittests/test_fill_constant_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_fill_op.py b/python/paddle/fluid/tests/unittests/test_fill_op.py index 762d29199e2127415ed7daabca63edcdbae3344f..b734ee05b3f2291d7a79f1550946bf6546ada6e0 100644 --- a/python/paddle/fluid/tests/unittests/test_fill_op.py +++ b/python/paddle/fluid/tests/unittests/test_fill_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_fill_zeros_like_op.py b/python/paddle/fluid/tests/unittests/test_fill_zeros_like_op.py index c9b3e4ba138f425fd2991bf637d2e32be3eb5168..eec73d0beb39c49f535a03532e536092001c8445 100644 --- a/python/paddle/fluid/tests/unittests/test_fill_zeros_like_op.py +++ b/python/paddle/fluid/tests/unittests/test_fill_zeros_like_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_flatten_op.py b/python/paddle/fluid/tests/unittests/test_flatten_op.py index f8692ce2ea66ef61c63bc41e77df050398ac63fd..17b01e03124e8007c51107b414c628d4bfc49c79 100644 --- a/python/paddle/fluid/tests/unittests/test_flatten_op.py +++ b/python/paddle/fluid/tests/unittests/test_flatten_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_framework_debug_str.py b/python/paddle/fluid/tests/unittests/test_framework_debug_str.py index c906c74afe66b05e2ca0e1122677e2dc738351b8..72f43e56ccbe04f56cfd5a655fb57c58369039bb 100644 --- a/python/paddle/fluid/tests/unittests/test_framework_debug_str.py +++ b/python/paddle/fluid/tests/unittests/test_framework_debug_str.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest from paddle.fluid.framework import Program diff --git a/python/paddle/fluid/tests/unittests/test_ftrl_op.py b/python/paddle/fluid/tests/unittests/test_ftrl_op.py index 5f7581391afb2bd57cb25695f3c0d4db8573c80c..a6390b054f06184831c289fe9556216ae213be7c 100644 --- a/python/paddle/fluid/tests/unittests/test_ftrl_op.py +++ b/python/paddle/fluid/tests/unittests/test_ftrl_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_fused_elemwise_activation_op.py b/python/paddle/fluid/tests/unittests/test_fused_elemwise_activation_op.py index ec0a939e9ec21952a6657ea849bb9844bb69cc8d..97e1b9061afb738dd9e5f8b3b6a9c9a123c6aac6 100644 --- a/python/paddle/fluid/tests/unittests/test_fused_elemwise_activation_op.py +++ b/python/paddle/fluid/tests/unittests/test_fused_elemwise_activation_op.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import unittest import numpy as np import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_gather_op.py b/python/paddle/fluid/tests/unittests/test_gather_op.py index 4ae90864806204197c52bbbdc5516f141afd4613..bd5785aa55af241fe42a1ae2c550dbdb980f42e2 100644 --- a/python/paddle/fluid/tests/unittests/test_gather_op.py +++ b/python/paddle/fluid/tests/unittests/test_gather_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_gaussian_random_batch_size_like_op.py b/python/paddle/fluid/tests/unittests/test_gaussian_random_batch_size_like_op.py index 1398166a74e714e0e902532166cde5d94ccae5f6..9a0631fa26a3e93c5c2115fd03a37de3fac46ce5 100644 --- a/python/paddle/fluid/tests/unittests/test_gaussian_random_batch_size_like_op.py +++ b/python/paddle/fluid/tests/unittests/test_gaussian_random_batch_size_like_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_gaussian_random_mkldnn_op.py b/python/paddle/fluid/tests/unittests/test_gaussian_random_mkldnn_op.py index 3ae877a60818744f852d3af9a02ffebf5e2affc8..9777ec390656d3f6166bf9f5de7bbad8b6bd786d 100644 --- a/python/paddle/fluid/tests/unittests/test_gaussian_random_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/test_gaussian_random_mkldnn_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest from test_gaussian_random_op import TestGaussianRandomOp diff --git a/python/paddle/fluid/tests/unittests/test_gaussian_random_op.py b/python/paddle/fluid/tests/unittests/test_gaussian_random_op.py index 8481500fd78f0ccf34f09c66bec27e195b9aada3..496aa4111056591efce14549011d66f9ae49713a 100644 --- a/python/paddle/fluid/tests/unittests/test_gaussian_random_op.py +++ b/python/paddle/fluid/tests/unittests/test_gaussian_random_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy diff --git a/python/paddle/fluid/tests/unittests/test_get_places_op.py b/python/paddle/fluid/tests/unittests/test_get_places_op.py index 964423e2d2638224244b4ca774d8eee08f3ec989..441666a97b16a320692d6a15363f61156e52242b 100644 --- a/python/paddle/fluid/tests/unittests/test_get_places_op.py +++ b/python/paddle/fluid/tests/unittests/test_get_places_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid as fluid from paddle.fluid.layers.device import get_places import decorators diff --git a/python/paddle/fluid/tests/unittests/test_gru_op.py b/python/paddle/fluid/tests/unittests/test_gru_op.py index 86a2c674d01f45b2b141572c8191d2fba7fa312f..001fd7efb159e60bdf3cd0698d85dea90ad71616 100644 --- a/python/paddle/fluid/tests/unittests/test_gru_op.py +++ b/python/paddle/fluid/tests/unittests/test_gru_op.py @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import math +import functools from op_test import OpTest from test_lstm_op import identity, sigmoid, tanh, relu @@ -38,7 +41,8 @@ class TestGRUOp(OpTest): for i in range(len(seq_lens)): seq_starts.append(seq_starts[-1] + seq_lens[i]) sorted_seqs = sorted( - list(range(len(seq_lens))), lambda x, y: seq_lens[y] - seq_lens[x]) + list(range(len(seq_lens))), + key=functools.cmp_to_key(lambda x, y: seq_lens[y] - seq_lens[x])) num_batch = seq_lens[sorted_seqs[0]] for batch_idx in range(num_batch): idx_in_seq = [] diff --git a/python/paddle/fluid/tests/unittests/test_gru_unit_op.py b/python/paddle/fluid/tests/unittests/test_gru_unit_op.py index 87a9eba4d97459082cdf1499efeddf24ed51e1b1..b5a66fdf086f2abc0c9a8af663241b9eda739407 100644 --- a/python/paddle/fluid/tests/unittests/test_gru_unit_op.py +++ b/python/paddle/fluid/tests/unittests/test_gru_unit_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import math import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_hinge_loss_op.py b/python/paddle/fluid/tests/unittests/test_hinge_loss_op.py index 70586c6be3da415fbccf4114615e6f7e08de0f0f..1eb441e2c52905c2b60104de5e04037714b34648 100644 --- a/python/paddle/fluid/tests/unittests/test_hinge_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_hinge_loss_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_hsigmoid_op.py b/python/paddle/fluid/tests/unittests/test_hsigmoid_op.py index daa5da8d95129af0305b326832a557daeb4c5c9c..6948ae30023a75d4735db1c78466e89e28640c9e 100644 --- a/python/paddle/fluid/tests/unittests/test_hsigmoid_op.py +++ b/python/paddle/fluid/tests/unittests/test_hsigmoid_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import math diff --git a/python/paddle/fluid/tests/unittests/test_huber_loss_op.py b/python/paddle/fluid/tests/unittests/test_huber_loss_op.py index a8d0a77625598e9d929a993db46ba95b0e07527a..0055ef0052fe126b268cf7a17a8307224cced99a 100644 --- a/python/paddle/fluid/tests/unittests/test_huber_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_huber_loss_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_im2sequence_op.py b/python/paddle/fluid/tests/unittests/test_im2sequence_op.py index 13bc5768740ece00bbe285a0b47d82bb8a42d2c7..833e46483c2532e283fd672dc56cb93941f5b4ba 100644 --- a/python/paddle/fluid/tests/unittests/test_im2sequence_op.py +++ b/python/paddle/fluid/tests/unittests/test_im2sequence_op.py @@ -11,6 +11,8 @@ #WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #See the License for the specific language governing permissions and #limitations under the License. + +from __future__ import print_function import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_image_classification_layer.py b/python/paddle/fluid/tests/unittests/test_image_classification_layer.py index 23b1ed957ad15bb631cd5160eb48328c76302987..405637969af6fb515a24ecb077e470279c3ffc24 100644 --- a/python/paddle/fluid/tests/unittests/test_image_classification_layer.py +++ b/python/paddle/fluid/tests/unittests/test_image_classification_layer.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/unittests/test_infer_shape.py b/python/paddle/fluid/tests/unittests/test_infer_shape.py index 699a2d42467b7ac0dcf1939bde744ad2fcb29c97..a3d700aad8236fea7bb0e6d043323ad3bd0851f2 100644 --- a/python/paddle/fluid/tests/unittests/test_infer_shape.py +++ b/python/paddle/fluid/tests/unittests/test_infer_shape.py @@ -12,8 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest +import six import paddle.fluid.core as core @@ -27,14 +30,14 @@ class TestInferShape(unittest.TestCase): shape = [10, 20] # prepare input/output - x1 = block.var("x1") + x1 = block.var(six.b("x1")) x1.set_type(core.VarDesc.VarType.LOD_TENSOR) x1.set_shape(shape) - x2 = block.var("x2") + x2 = block.var(six.b("x2")) x2.set_type(core.VarDesc.VarType.LOD_TENSOR) x2.set_shape(shape) - out = block.var("out") + out = block.var(six.b("out")) out.set_type(core.VarDesc.VarType.LOD_TENSOR) # prepare the operator @@ -57,14 +60,14 @@ class TestInferShape(unittest.TestCase): y_shape = [20, 30] # prepare input/output - x1 = block.var("x") + x1 = block.var(six.b("x")) x1.set_type(core.VarDesc.VarType.LOD_TENSOR) x1.set_shape(x_shape) - x2 = block.var("y") + x2 = block.var(six.b("y")) x2.set_type(core.VarDesc.VarType.LOD_TENSOR) x2.set_shape(y_shape) - out = block.var("out") + out = block.var(six.b("out")) out.set_type(core.VarDesc.VarType.LOD_TENSOR) # prepare the operator diff --git a/python/paddle/fluid/tests/unittests/test_inference_model_io.py b/python/paddle/fluid/tests/unittests/test_inference_model_io.py index 4cd203155f446df07d2fe6c1d56e0d20f1113679..9962702f69644b7aef7d868f086abb390441f617 100644 --- a/python/paddle/fluid/tests/unittests/test_inference_model_io.py +++ b/python/paddle/fluid/tests/unittests/test_inference_model_io.py @@ -12,8 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest +import six import numpy as np import paddle.fluid.core as core @@ -48,7 +51,7 @@ class TestBook(unittest.TestCase): exe.run(init_program, feed={}, fetch_list=[]) - for i in range(100): + for i in six.moves.xrange(100): tensor_x = np.array( [[1, 1], [1, 2], [3, 4], [5, 2]]).astype("float32") tensor_y = np.array([[-2], [-3], [-7], [-7]]).astype("float32") @@ -64,7 +67,7 @@ class TestBook(unittest.TestCase): 'y': tensor_y}, fetch_list=[avg_cost])[0] - reload(executor) # reload to build a new scope + six.moves.reload_module(executor) # reload to build a new scope exe = executor.Executor(place) [infer_prog, feed_var_names, fetch_vars] = load_inference_model( diff --git a/python/paddle/fluid/tests/unittests/test_initializer.py b/python/paddle/fluid/tests/unittests/test_initializer.py index b215e379864e919af03591ab2566c08dddbb5743..ab7183f88df809e584ca50ba16221bfdfe1376a9 100644 --- a/python/paddle/fluid/tests/unittests/test_initializer.py +++ b/python/paddle/fluid/tests/unittests/test_initializer.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import unittest diff --git a/python/paddle/fluid/tests/unittests/test_iou_similarity_op.py b/python/paddle/fluid/tests/unittests/test_iou_similarity_op.py index eff4212d91e609a7ef531280bbd3cf3671a59830..7c1808cf998e84c22c46df68ef07259c1a021c19 100644 --- a/python/paddle/fluid/tests/unittests/test_iou_similarity_op.py +++ b/python/paddle/fluid/tests/unittests/test_iou_similarity_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import numpy.random as random diff --git a/python/paddle/fluid/tests/unittests/test_is_empty_op.py b/python/paddle/fluid/tests/unittests/test_is_empty_op.py index 11121d9b65351eab639b7618fac0e54714cf4680..26d607718aec0bdffa00b9b4bca06ec6c0196217 100644 --- a/python/paddle/fluid/tests/unittests/test_is_empty_op.py +++ b/python/paddle/fluid/tests/unittests/test_is_empty_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_l1_norm_op.py b/python/paddle/fluid/tests/unittests/test_l1_norm_op.py index fa5b18a16f7a3e734ff8bb4f53240e8a9ce8fd8f..4e24a78ee54dfb1fb0e4f97317642cfaffe9436e 100644 --- a/python/paddle/fluid/tests/unittests/test_l1_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_l1_norm_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import unittest from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_label_smooth_op.py b/python/paddle/fluid/tests/unittests/test_label_smooth_op.py index ca21289a0d48a123aed90bc557ccc732702b47f1..62d385bc52cfb3a9fe15a82096ff33abc1bcc552 100644 --- a/python/paddle/fluid/tests/unittests/test_label_smooth_op.py +++ b/python/paddle/fluid/tests/unittests/test_label_smooth_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_layer_norm_op.py b/python/paddle/fluid/tests/unittests/test_layer_norm_op.py index 295887ccd171a3101329eb1255da146914fa9264..fb6c43136ff82af55d1fcc2969cf4a07ae081204 100644 --- a/python/paddle/fluid/tests/unittests/test_layer_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_layer_norm_op.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_layers.py b/python/paddle/fluid/tests/unittests/test_layers.py index 07fd0575d333dacf309620a883e4052c6126739f..e833a7db482db3eb69ef8596c468c535dee02370 100644 --- a/python/paddle/fluid/tests/unittests/test_layers.py +++ b/python/paddle/fluid/tests/unittests/test_layers.py @@ -159,7 +159,7 @@ class TestBook(unittest.TestCase): input=crf_decode, label=label, chunk_scheme="IOB", - num_chunk_types=(label_dict_len - 1) / 2) + num_chunk_types=(label_dict_len - 1) // 2) self.assertFalse(crf is None) self.assertFalse(crf_decode is None) @@ -286,7 +286,7 @@ class TestBook(unittest.TestCase): name='word_{0}'.format(i), shape=[1], dtype='int64')) dict_size = 10000 - label_word = int(window_size / 2) + 1 + label_word = int(window_size // 2) + 1 embs = [] for i in range(window_size): diff --git a/python/paddle/fluid/tests/unittests/test_learning_rate_scheduler.py b/python/paddle/fluid/tests/unittests/test_learning_rate_scheduler.py index e628195e7265ec564bd64a212c4a35fdff495063..0d3e6d73e0149fe633b8f1de9041068c2e3bb293 100644 --- a/python/paddle/fluid/tests/unittests/test_learning_rate_scheduler.py +++ b/python/paddle/fluid/tests/unittests/test_learning_rate_scheduler.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import copy import math import unittest diff --git a/python/paddle/fluid/tests/unittests/test_linear_chain_crf_op.py b/python/paddle/fluid/tests/unittests/test_linear_chain_crf_op.py index 696d0ab4fa81a409a2bf0d6f6f23779ec26eb6d2..6e31e9204e95d98fcf69ed84a46d6cf3d94c808a 100644 --- a/python/paddle/fluid/tests/unittests/test_linear_chain_crf_op.py +++ b/python/paddle/fluid/tests/unittests/test_linear_chain_crf_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import random import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_listen_and_serv_op.py b/python/paddle/fluid/tests/unittests/test_listen_and_serv_op.py index 1cdc69501043d120b9e3cc8ccda3a1212d205886..48b52a5412eb99fbc7a5c8534a766ede4954e849 100644 --- a/python/paddle/fluid/tests/unittests/test_listen_and_serv_op.py +++ b/python/paddle/fluid/tests/unittests/test_listen_and_serv_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle import paddle.fluid as fluid import os diff --git a/python/paddle/fluid/tests/unittests/test_lod_array_length_op.py b/python/paddle/fluid/tests/unittests/test_lod_array_length_op.py index d8b4e40662568f580ccff0257512cb8809488f17..15485df5ac440f2ff666ca27ef8e8bcc5df866c0 100644 --- a/python/paddle/fluid/tests/unittests/test_lod_array_length_op.py +++ b/python/paddle/fluid/tests/unittests/test_lod_array_length_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.layers as layers from paddle.fluid.executor import Executor diff --git a/python/paddle/fluid/tests/unittests/test_lod_rank_table.py b/python/paddle/fluid/tests/unittests/test_lod_rank_table.py index d53ead381d301e797d5a19784aed49a5d6f99319..865ca118d55f82c66d44f4e3d553baafa0c14c3a 100644 --- a/python/paddle/fluid/tests/unittests/test_lod_rank_table.py +++ b/python/paddle/fluid/tests/unittests/test_lod_rank_table.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from paddle.fluid.layers import data from paddle.fluid.layers.control_flow import lod_rank_table from paddle.fluid.executor import Executor diff --git a/python/paddle/fluid/tests/unittests/test_lod_reset_op.py b/python/paddle/fluid/tests/unittests/test_lod_reset_op.py index 77905c4b96499c855fd5c5e704b8051ccdb7a323..31f364a42f624c8662a5ae087b003ca0304ae419 100644 --- a/python/paddle/fluid/tests/unittests/test_lod_reset_op.py +++ b/python/paddle/fluid/tests/unittests/test_lod_reset_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_lod_tensor_array.py b/python/paddle/fluid/tests/unittests/test_lod_tensor_array.py index 0ac6d9b81df0ecbe9c6560cdb0ab0507c3c2ed18..6ad27de9a0e42d1a15ec4a17804c7c0f7ebf5d94 100644 --- a/python/paddle/fluid/tests/unittests/test_lod_tensor_array.py +++ b/python/paddle/fluid/tests/unittests/test_lod_tensor_array.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.core as core import numpy diff --git a/python/paddle/fluid/tests/unittests/test_lod_tensor_array_ops.py b/python/paddle/fluid/tests/unittests/test_lod_tensor_array_ops.py index 9789ff4af648b41a1b53844be89249bd260de61b..6a78ef5078a738efa2ae39ea23645fedaecce63b 100644 --- a/python/paddle/fluid/tests/unittests/test_lod_tensor_array_ops.py +++ b/python/paddle/fluid/tests/unittests/test_lod_tensor_array_ops.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.core as core import numpy diff --git a/python/paddle/fluid/tests/unittests/test_log_loss_op.py b/python/paddle/fluid/tests/unittests/test_log_loss_op.py index d3980b8db93ca517d16b6f782ba800ce839c3f45..784f4f648d52bdf4f2357f4454d790a8d53288f3 100644 --- a/python/paddle/fluid/tests/unittests/test_log_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_log_loss_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_logical_op.py b/python/paddle/fluid/tests/unittests/test_logical_op.py index 1d7dfe60f200459705a48664c1a5b22d2a5888d2..521851a3d57a4a3e8b2c8e1639325cc6c88fdd84 100644 --- a/python/paddle/fluid/tests/unittests/test_logical_op.py +++ b/python/paddle/fluid/tests/unittests/test_logical_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import op_test import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_lookup_sparse_table_op.py b/python/paddle/fluid/tests/unittests/test_lookup_sparse_table_op.py index aa9eae1e882f55ef51f38e158317a1a9aeed641c..11e5d8b536fb65b66c954991bf815241774702ec 100644 --- a/python/paddle/fluid/tests/unittests/test_lookup_sparse_table_op.py +++ b/python/paddle/fluid/tests/unittests/test_lookup_sparse_table_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest @@ -19,36 +21,27 @@ import paddle.fluid.core as core from paddle.fluid.op import Operator -def output_hist(out): - hist, _ = np.histogram(out, range=(-5, 10)) - hist = hist.astype("float32") - hist /= float(out.size) - prob = 0.1 * np.ones((10)) - return hist, prob - - class TestLookupSpraseTable(OpTest): def check_with_place(self, place): scope = core.Scope() - # create and initialize Id Variable - ids = scope.var("Ids").get_tensor() - ids_array = np.array([0, 2, 3, 5, 100]).astype("int64") - ids.set(ids_array, place) - # create and initialize W Variable - rows = [0, 1, 2, 3, 4, 5, 6] - row_numel = 10000 + table_size = 10000 + row_numel = 8 w_selected_rows = scope.var('W').get_selected_rows() - w_selected_rows.set_height(len(rows)) - w_selected_rows.set_rows(rows) - w_array = np.ones((len(rows), row_numel)).astype("float32") - for i in range(len(rows)): + w_selected_rows.set_height(table_size) + w_array = np.ones((table_size, row_numel)).astype("float32") + for i in range(table_size): w_array[i] *= i w_tensor = w_selected_rows.get_tensor() w_tensor.set(w_array, place) + # create and initialize Id Variable + ids = scope.var("Ids").get_tensor() + ids_array1 = np.array([0, 2, 3, 2, 5, 0, 100]).astype("int64") + ids.set(ids_array1, place) + # create Out Variable out_tensor = scope.var('Out').get_tensor() @@ -64,16 +57,28 @@ class TestLookupSpraseTable(OpTest): lookup_table.run(scope, place) # get result from Out - result_array = np.array(out_tensor) + result_array1 = np.array(out_tensor) # all(): return True if all elements of the iterable are true (or if the iterable is empty) - for idx, row in enumerate(ids_array[:-2]): - assert (row == result_array[idx]).all() + assert (result_array1[0] == w_array[0]).all() + assert (result_array1[1] == w_array[1]).all() + assert (result_array1[2] == w_array[2]).all() + assert (result_array1[3] == w_array[1]).all() + assert (result_array1[4] == w_array[3]).all() + assert (result_array1[5] == w_array[0]).all() + assert (result_array1[6] == w_array[4]).all() + + # create and initialize Id Variable + ids = scope.var("Ids").get_tensor() + ids_array2 = np.array([4, 2, 3, 7, 100000]).astype("int64") + ids.set(ids_array2, place) + lookup_table.run(scope, place) - # check the random value - hist, prob = output_hist(result_array[-1]) - self.assertTrue( - np.allclose( - hist, prob, rtol=0, atol=0.01), "hist: " + str(hist)) + result_array2 = np.array(out_tensor) + assert (result_array2[0] == w_array[5]).all() + assert (result_array2[1] == w_array[1]).all() + assert (result_array2[2] == w_array[2]).all() + assert (result_array2[3] == w_array[6]).all() + assert (result_array2[4] == w_array[7]).all() def test_w_is_selected_rows(self): places = [core.CPUPlace()] diff --git a/python/paddle/fluid/tests/unittests/test_lookup_table_op.py b/python/paddle/fluid/tests/unittests/test_lookup_table_op.py index ac25f432dffd544d4b336983ec868f2431a5b91a..4990ee898d81089735f6db4ee4ad6758944e311a 100644 --- a/python/paddle/fluid/tests/unittests/test_lookup_table_op.py +++ b/python/paddle/fluid/tests/unittests/test_lookup_table_op.py @@ -12,11 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest import paddle.fluid.core as core from paddle.fluid.op import Operator +import paddle.compat as cpt class TestLookupTableOp(OpTest): @@ -71,7 +74,7 @@ class TestLookupTableOpWithTensorIdsAndPadding(TestLookupTableOpWithTensorIds): flatten_idx = ids.flatten() padding_idx = np.random.choice(flatten_idx, 1)[0] self.outputs['Out'][np.squeeze(ids == padding_idx)] = np.zeros(31) - self.attrs = {'padding_idx': long(padding_idx)} + self.attrs = {'padding_idx': cpt.long_type(padding_idx)} self.check_output() def test_check_grad(self): diff --git a/python/paddle/fluid/tests/unittests/test_lrn_mkldnn_op.py b/python/paddle/fluid/tests/unittests/test_lrn_mkldnn_op.py index 966a16dc870c041b9deb140bed57d907cf305fd8..f6bb2ab7a696c40cb61dd5b38ca702b577fe7ea2 100644 --- a/python/paddle/fluid/tests/unittests/test_lrn_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/test_lrn_mkldnn_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest from test_lrn_op import TestLRNOp diff --git a/python/paddle/fluid/tests/unittests/test_lrn_op.py b/python/paddle/fluid/tests/unittests/test_lrn_op.py index eaff45cbb2a58798e9d55149510bec72eea370cd..bb91f26bbb53de454a6d037af4c9d96262866ce3 100644 --- a/python/paddle/fluid/tests/unittests/test_lrn_op.py +++ b/python/paddle/fluid/tests/unittests/test_lrn_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest @@ -34,7 +36,7 @@ class TestLRNOp(OpTest): return x + 1 def get_out(self): - start = -(self.n - 1) / 2 + start = -(self.n - 1) // 2 end = start + self.n mid = np.empty((self.N, self.C, self.H, self.W)).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/test_lstm_op.py b/python/paddle/fluid/tests/unittests/test_lstm_op.py index 705a24bd8f39a55e0a352944d961f8d33aaf96ff..76a24123fc7d51231bf24a3d1a6930186c94a5db 100644 --- a/python/paddle/fluid/tests/unittests/test_lstm_op.py +++ b/python/paddle/fluid/tests/unittests/test_lstm_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_lstm_unit_op.py b/python/paddle/fluid/tests/unittests/test_lstm_unit_op.py index e343265874f99afcd8201fa997932e2613fffc4c..eaa6b774c4d3e7add555c34f887e86dc847583b2 100644 --- a/python/paddle/fluid/tests/unittests/test_lstm_unit_op.py +++ b/python/paddle/fluid/tests/unittests/test_lstm_unit_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_lstmp_op.py b/python/paddle/fluid/tests/unittests/test_lstmp_op.py index ed2262da4bc727657c2e65d69cb1922891e17b09..9c3ec45515ffe0a07541fd9cfb7e92b079264071 100644 --- a/python/paddle/fluid/tests/unittests/test_lstmp_op.py +++ b/python/paddle/fluid/tests/unittests/test_lstmp_op.py @@ -11,6 +11,8 @@ #WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #See the License for the specific language governing permissions and #limitations under the License. + +from __future__ import print_function import unittest import numpy as np import test_lstm_op as LstmTest diff --git a/python/paddle/fluid/tests/unittests/test_margin_rank_loss_op.py b/python/paddle/fluid/tests/unittests/test_margin_rank_loss_op.py index 97c112487fd9193ab77d18945585a554e5fbcdf8..4a7e952436bd46c92c6256b4ec2d0652cfa38959 100644 --- a/python/paddle/fluid/tests/unittests/test_margin_rank_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_margin_rank_loss_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_math_op_patch.py b/python/paddle/fluid/tests/unittests/test_math_op_patch.py index 852a80261e02f5ed19e7fbe608d490be1f7798a9..b25d40a3a15e259878222ee5482cd842543b63d6 100644 --- a/python/paddle/fluid/tests/unittests/test_math_op_patch.py +++ b/python/paddle/fluid/tests/unittests/test_math_op_patch.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import decorators import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/unittests/test_matmul_op.py b/python/paddle/fluid/tests/unittests/test_matmul_op.py index cae2c8fa87d9857de8f26cf4962d9370eca66243..abf10437d83268a6a84a1c62399eb02cd3b1d663 100644 --- a/python/paddle/fluid/tests/unittests/test_matmul_op.py +++ b/python/paddle/fluid/tests/unittests/test_matmul_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_maxout_op.py b/python/paddle/fluid/tests/unittests/test_maxout_op.py index f5ddf72516bf8adb41698d9b2e22c7de74a3fad9..d588b22fe2607a6041359d420ebba757d8a632d6 100644 --- a/python/paddle/fluid/tests/unittests/test_maxout_op.py +++ b/python/paddle/fluid/tests/unittests/test_maxout_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest @@ -19,7 +21,7 @@ from op_test import OpTest def maxout_forward_naive(input, groups): s0, s1, s2, s3 = input.shape - return np.ndarray([s0, s1 / groups, groups, s2, s3], \ + return np.ndarray([s0, s1 // groups, groups, s2, s3], \ buffer = input, dtype=input.dtype).max(axis=(2)) diff --git a/python/paddle/fluid/tests/unittests/test_mean_iou.py b/python/paddle/fluid/tests/unittests/test_mean_iou.py index 32b4ee184787cd4cda0fd889f67a609141a3cb27..03e94483178e83adad9886cd7df2107581360dd1 100644 --- a/python/paddle/fluid/tests/unittests/test_mean_iou.py +++ b/python/paddle/fluid/tests/unittests/test_mean_iou.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from __future__ import division import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_mean_op.py b/python/paddle/fluid/tests/unittests/test_mean_op.py index 15472a8fc4716f218b0eddf17589634c565130b1..ff338f0e0037307e81a92eed804096c9a2a87361 100644 --- a/python/paddle/fluid/tests/unittests/test_mean_op.py +++ b/python/paddle/fluid/tests/unittests/test_mean_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_memory_usage.py b/python/paddle/fluid/tests/unittests/test_memory_usage.py index f9daf83652e18faab0ab31402b9f5889a0beceaf..4cdb5b5d9f7f020c4eb9a3b3a804c074d7ddbb35 100644 --- a/python/paddle/fluid/tests/unittests/test_memory_usage.py +++ b/python/paddle/fluid/tests/unittests/test_memory_usage.py @@ -34,7 +34,7 @@ def train_simulator(test_batch_size=10): sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.001) sgd_optimizer.minimize(avg_cost) - # Calculate memory usage in current network config + # Calculate memory usage in current network config lower_usage, upper_usage, unit = fluid.contrib.memory_usage( fluid.default_main_program(), batch_size=test_batch_size) diff --git a/python/paddle/fluid/tests/unittests/test_merge_ids_op.py b/python/paddle/fluid/tests/unittests/test_merge_ids_op.py index f209bdf30faffc0b2c7932b7b10f384d6d61a831..26ce7024117162e8bad403a9d8b8518c27578c83 100644 --- a/python/paddle/fluid/tests/unittests/test_merge_ids_op.py +++ b/python/paddle/fluid/tests/unittests/test_merge_ids_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_mine_hard_examples_op.py b/python/paddle/fluid/tests/unittests/test_mine_hard_examples_op.py index 54ee85c1a7a539fe9517f32adb35ab99b5ae2a07..4e5cc91268c5df4be3de3c04a82ef65b33cf4d20 100644 --- a/python/paddle/fluid/tests/unittests/test_mine_hard_examples_op.py +++ b/python/paddle/fluid/tests/unittests/test_mine_hard_examples_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import sys diff --git a/python/paddle/fluid/tests/unittests/test_minus_op.py b/python/paddle/fluid/tests/unittests/test_minus_op.py index ee32bd49925e266cfd5beb51496355e111a3d0d2..54253b17b967871b03628023c5a9fdb339af1828 100644 --- a/python/paddle/fluid/tests/unittests/test_minus_op.py +++ b/python/paddle/fluid/tests/unittests/test_minus_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_modified_huber_loss_op.py b/python/paddle/fluid/tests/unittests/test_modified_huber_loss_op.py index 62035efe8ec3809a7672e58697de4304426338d7..02fecfe47ec3fbff085b0a7f24316e5d0f6cd814 100644 --- a/python/paddle/fluid/tests/unittests/test_modified_huber_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_modified_huber_loss_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_momentum_op.py b/python/paddle/fluid/tests/unittests/test_momentum_op.py index c75d3bd276a5b494090c1aa1fea0bb4f2c067173..7137fd0fdb7c503492107da684b95989037eb872 100644 --- a/python/paddle/fluid/tests/unittests/test_momentum_op.py +++ b/python/paddle/fluid/tests/unittests/test_momentum_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_mul_op.py b/python/paddle/fluid/tests/unittests/test_mul_op.py index bbc782c1bce302df68ab30013f3a7667e51ed479..fca4ffa88b79ebfad009b436d440e86ddceaaed7 100644 --- a/python/paddle/fluid/tests/unittests/test_mul_op.py +++ b/python/paddle/fluid/tests/unittests/test_mul_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_multi_file_reader.py b/python/paddle/fluid/tests/unittests/test_multi_file_reader.py index cb0ea96ff69ce32b0bb1b49f0318c353aa08d388..09788868ccb926f56c2f622b5caf695670fd17f8 100644 --- a/python/paddle/fluid/tests/unittests/test_multi_file_reader.py +++ b/python/paddle/fluid/tests/unittests/test_multi_file_reader.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/unittests/test_multi_pass_reader.py b/python/paddle/fluid/tests/unittests/test_multi_pass_reader.py index 7fc9f550440d3d0e1a8182a69f5692b3df0aa258..4fae11e928dc7e066799a8936bada0e252afaa42 100644 --- a/python/paddle/fluid/tests/unittests/test_multi_pass_reader.py +++ b/python/paddle/fluid/tests/unittests/test_multi_pass_reader.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/unittests/test_multiclass_nms_op.py b/python/paddle/fluid/tests/unittests/test_multiclass_nms_op.py index 10cb78a08db0471699bcc0b7323d5346e3af64c7..df0562dcc79cbb960136c19d2b3f243cf2e09782 100644 --- a/python/paddle/fluid/tests/unittests/test_multiclass_nms_op.py +++ b/python/paddle/fluid/tests/unittests/test_multiclass_nms_op.py @@ -11,6 +11,8 @@ #WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #See the License for the specific language governing permissions and #limitations under the License. + +from __future__ import print_function import unittest import numpy as np import copy diff --git a/python/paddle/fluid/tests/unittests/test_multihead_attention.py b/python/paddle/fluid/tests/unittests/test_multihead_attention.py index 80c3c67967e970a7182c008b6cfd138aff044167..f60da862ac091ca1eefccfe2834201d1c79e2def 100644 --- a/python/paddle/fluid/tests/unittests/test_multihead_attention.py +++ b/python/paddle/fluid/tests/unittests/test_multihead_attention.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_multiplex_op.py b/python/paddle/fluid/tests/unittests/test_multiplex_op.py index 03cad8b43bac053f246156b8c57df9fceab20dcd..1567a74808aa37e5e18bbe583cc1d8987b31cd58 100644 --- a/python/paddle/fluid/tests/unittests/test_multiplex_op.py +++ b/python/paddle/fluid/tests/unittests/test_multiplex_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_nce.py b/python/paddle/fluid/tests/unittests/test_nce.py index 7431a142c53a64e58872390776904ce8f781d6a9..0745bd274f73715b6fdec236819b8d89827e1346 100644 --- a/python/paddle/fluid/tests/unittests/test_nce.py +++ b/python/paddle/fluid/tests/unittests/test_nce.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_network_with_dtype.py b/python/paddle/fluid/tests/unittests/test_network_with_dtype.py index d4835dd18405fc7a0d508a780a734922e0abd12c..60dcf195daf61d76a2e6d6f764fa216270804f55 100644 --- a/python/paddle/fluid/tests/unittests/test_network_with_dtype.py +++ b/python/paddle/fluid/tests/unittests/test_network_with_dtype.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_norm_op.py b/python/paddle/fluid/tests/unittests/test_norm_op.py index 108a665f37f5cd652ec83f784a56ca52e6b49fe8..22bc45ff1ea0efea0ec766a6a9e819cdd81b0866 100644 --- a/python/paddle/fluid/tests/unittests/test_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_norm_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_normalization_wrapper.py b/python/paddle/fluid/tests/unittests/test_normalization_wrapper.py index 198c68866d399023c51c2a43b588aa8ec49c3c9a..24fdcf8c88417244e981194e63bd77a2fdbd179d 100644 --- a/python/paddle/fluid/tests/unittests/test_normalization_wrapper.py +++ b/python/paddle/fluid/tests/unittests/test_normalization_wrapper.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_nvprof.py b/python/paddle/fluid/tests/unittests/test_nvprof.py index 226e5e5d1131b1f33cfbbfefec536e6974f85b36..da943d64da6cfc64d121b7373f7c067c1cff731c 100644 --- a/python/paddle/fluid/tests/unittests/test_nvprof.py +++ b/python/paddle/fluid/tests/unittests/test_nvprof.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import os import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_one_hot_op.py b/python/paddle/fluid/tests/unittests/test_one_hot_op.py index 06fccd39ac65ab62ee5618ac19d1a0535b481d06..7afdae804a65b9fb05a521a1b08ce0bfb21d721f 100644 --- a/python/paddle/fluid/tests/unittests/test_one_hot_op.py +++ b/python/paddle/fluid/tests/unittests/test_one_hot_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import math diff --git a/python/paddle/fluid/tests/unittests/test_op_support_gpu.py b/python/paddle/fluid/tests/unittests/test_op_support_gpu.py index 5fafb8280e19cca46e5bf687494c07200ca53153..e203fccd03f86077c51e176456c1c313ac14a9ee 100644 --- a/python/paddle/fluid/tests/unittests/test_op_support_gpu.py +++ b/python/paddle/fluid/tests/unittests/test_op_support_gpu.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_operator.py b/python/paddle/fluid/tests/unittests/test_operator.py index 5e418fe6ac2d62948762290a65686207d017275c..544fca8cecd0a2b94a5aec40b9442f86036fd4d2 100644 --- a/python/paddle/fluid/tests/unittests/test_operator.py +++ b/python/paddle/fluid/tests/unittests/test_operator.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.op as op diff --git a/python/paddle/fluid/tests/unittests/test_operator_desc.py b/python/paddle/fluid/tests/unittests/test_operator_desc.py index c098a5a0cb0364f9ec93c95c1ef50912e574b3d9..6d01955993324498de42462b7f85ef6f8e444505 100644 --- a/python/paddle/fluid/tests/unittests/test_operator_desc.py +++ b/python/paddle/fluid/tests/unittests/test_operator_desc.py @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.core as core +import paddle.compat as cpt from paddle.fluid.framework import Program, default_startup_program @@ -29,14 +32,15 @@ class TestOperator(unittest.TestCase): self.assertFail() except ValueError as v_err: self.assertEqual( - v_err.message, + cpt.get_exception_message(v_err), "`type` to initilized an Operator can not be None.") try: block.append_op(type="no_such_op") self.assertFail() except ValueError as a_err: - self.assertEqual(a_err.message, - "Operator \"no_such_op\" has not been registered.") + self.assertEqual( + cpt.get_exception_message(a_err), + "Operator \"no_such_op\" has not been registered.") def test_op_desc_creation(self): program = Program() diff --git a/python/paddle/fluid/tests/unittests/test_optimizer.py b/python/paddle/fluid/tests/unittests/test_optimizer.py index 18921d727f94a85b69259c07273f09c3e19390c6..4374d198f2f869afab5fb76fdcb43e3c445f7689 100644 --- a/python/paddle/fluid/tests/unittests/test_optimizer.py +++ b/python/paddle/fluid/tests/unittests/test_optimizer.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.framework as framework diff --git a/python/paddle/fluid/tests/unittests/test_pad_op.py b/python/paddle/fluid/tests/unittests/test_pad_op.py index 300f3ffcb8d1a6152b1e03f1356582c02bc4b2a3..58e56ca1a4dbdc48765a36e1a64b9a2ec8cf9025 100644 --- a/python/paddle/fluid/tests/unittests/test_pad_op.py +++ b/python/paddle/fluid/tests/unittests/test_pad_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py index d17e493c36a2ffcba632f5f85c7a1d2e5066dd1c..6d6917300cb66afcc8a0c509986a0f26be8b1f09 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_crf.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.dataset.conll05 as conll05 import paddle.fluid as fluid import unittest diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_feed.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_feed.py index a43f2e7c49c02ce779344da44e640cabbf27986c..372ef748b2e704fd3858c382e048e51448ed3bd5 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_feed.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_fetch_feed.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.dataset.flowers as flowers import math import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_mnist.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_mnist.py index 9448d89cd58f4e5cff4bac49821fbc44c5a46246..893acd763fb21fc5040b8a24700eb947a9fe37c6 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_mnist.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_mnist.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from parallel_executor_test_base import TestParallelExecutorBase import paddle.fluid as fluid import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext.py index a28428d8dee201ba105e18684c15d4b4582d989f..cc2d692e18430eb48e6e800106eab0c3739d3f53 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_seresnext.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid as fluid import paddle.fluid.layers.ops as ops from paddle.fluid.initializer import init_on_cpu @@ -46,7 +48,7 @@ def squeeze_excitation(input, num_channels, reduction_ratio): pool = fluid.layers.reduce_mean(input=reshape, dim=2) squeeze = fluid.layers.fc(input=pool, - size=num_channels / reduction_ratio, + size=num_channels // reduction_ratio, act='relu') excitation = fluid.layers.fc(input=squeeze, size=num_channels, @@ -62,7 +64,7 @@ def conv_bn_layer(input, num_filters, filter_size, stride=1, groups=1, num_filters=num_filters, filter_size=filter_size, stride=stride, - padding=(filter_size - 1) / 2, + padding=(filter_size - 1) // 2, groups=groups, act=None, bias_attr=False) diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_test_while_train.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_test_while_train.py index fcb5947ff05efd1c48ab9ec129ac9d17255d7020..f5a0ba624698b49e0d323e6f830be23a4148392b 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_test_while_train.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_test_while_train.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid as fluid import paddle.fluid.core as core import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_parallel_executor_transformer.py b/python/paddle/fluid/tests/unittests/test_parallel_executor_transformer.py index 8203d5d1fce0950130ab71db40fb306f73c41bd4..5ad922725a0b692e28552737a99b745ed09ddbd5 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_executor_transformer.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_executor_transformer.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid as fluid import transformer_model import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_parallel_op.py b/python/paddle/fluid/tests/unittests/test_parallel_op.py index c9617e36778740ce9620c3ad495c64c17277fde1..d7b9af8bac67ef89cc1ae59ccf002c2c488f3436 100644 --- a/python/paddle/fluid/tests/unittests/test_parallel_op.py +++ b/python/paddle/fluid/tests/unittests/test_parallel_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/unittests/test_parameter.py b/python/paddle/fluid/tests/unittests/test_parameter.py index e09865074e8aa9345fd9cc84e1f19eaf0436142f..df42e6cb9a050b76099b4a53fdd08d2852284d1f 100644 --- a/python/paddle/fluid/tests/unittests/test_parameter.py +++ b/python/paddle/fluid/tests/unittests/test_parameter.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest from paddle.fluid.framework import default_main_program import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_polygon_box_transform.py b/python/paddle/fluid/tests/unittests/test_polygon_box_transform.py index 8aff4e87f67bc61a162f09e982cf0a7a61639257..dfedf8190f75ec26532f281338f076ca0c7d83af 100644 --- a/python/paddle/fluid/tests/unittests/test_polygon_box_transform.py +++ b/python/paddle/fluid/tests/unittests/test_polygon_box_transform.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_pool2d_mkldnn_op.py b/python/paddle/fluid/tests/unittests/test_pool2d_mkldnn_op.py index 003ebba18b26198427d9f313596ae85656ac24fa..14d7ed9057d622b136056e1b5bbbe57f9a04d5d7 100644 --- a/python/paddle/fluid/tests/unittests/test_pool2d_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/test_pool2d_mkldnn_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest from test_pool2d_op import TestPool2d_Op, TestCase1, TestCase2, TestCase3, TestCase4, TestCase5 diff --git a/python/paddle/fluid/tests/unittests/test_pool2d_op.py b/python/paddle/fluid/tests/unittests/test_pool2d_op.py index 1cf70311b40bc7648b7462e93f201aa33c77b137..26969bd5230afdac83a943d2dc21094a0972d60a 100644 --- a/python/paddle/fluid/tests/unittests/test_pool2d_op.py +++ b/python/paddle/fluid/tests/unittests/test_pool2d_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np @@ -29,11 +31,11 @@ def max_pool2D_forward_naive(x, if global_pool == 1: ksize = [H, W] H_out = (H - ksize[0] + 2 * paddings[0] + strides[0] - 1 - ) / strides[0] + 1 if ceil_mode else (H - ksize[0] + 2 * - paddings[0]) / strides[0] + 1 + ) // strides[0] + 1 if ceil_mode else ( + H - ksize[0] + 2 * paddings[0]) // strides[0] + 1 W_out = (W - ksize[1] + 2 * paddings[1] + strides[1] - 1 - ) / strides[1] + 1 if ceil_mode else (W - ksize[1] + 2 * - paddings[1]) / strides[1] + 1 + ) // strides[1] + 1 if ceil_mode else ( + W - ksize[1] + 2 * paddings[1]) // strides[1] + 1 out = np.zeros((N, C, H_out, W_out)) for i in range(H_out): for j in range(W_out): @@ -57,11 +59,11 @@ def avg_pool2D_forward_naive(x, if global_pool == 1: ksize = [H, W] H_out = (H - ksize[0] + 2 * paddings[0] + strides[0] - 1 - ) / strides[0] + 1 if ceil_mode else (H - ksize[0] + 2 * - paddings[0]) / strides[0] + 1 + ) // strides[0] + 1 if ceil_mode else ( + H - ksize[0] + 2 * paddings[0]) // strides[0] + 1 W_out = (W - ksize[1] + 2 * paddings[1] + strides[1] - 1 - ) / strides[1] + 1 if ceil_mode else (W - ksize[1] + 2 * - paddings[1]) / strides[1] + 1 + ) // strides[1] + 1 if ceil_mode else ( + W - ksize[1] + 2 * paddings[1]) // strides[1] + 1 out = np.zeros((N, C, H_out, W_out)) for i in range(H_out): for j in range(W_out): diff --git a/python/paddle/fluid/tests/unittests/test_pool3d_op.py b/python/paddle/fluid/tests/unittests/test_pool3d_op.py index 92c64b37921eafd4c90e247a235f2dacea8fea1e..77045c1307baead3711d58ed368dfa5f2acc3699 100644 --- a/python/paddle/fluid/tests/unittests/test_pool3d_op.py +++ b/python/paddle/fluid/tests/unittests/test_pool3d_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np @@ -29,14 +31,14 @@ def max_pool3D_forward_naive(x, if global_pool == 1: ksize = [D, H, W] D_out = (D - ksize[0] + 2 * paddings[0] + strides[0] - 1 - ) / strides[0] + 1 if ceil_mode else (H - ksize[0] + 2 * - paddings[0]) / strides[0] + 1 + ) // strides[0] + 1 if ceil_mode else ( + H - ksize[0] + 2 * paddings[0]) // strides[0] + 1 H_out = (H - ksize[1] + 2 * paddings[1] + strides[1] - 1 - ) / strides[1] + 1 if ceil_mode else (W - ksize[1] + 2 * - paddings[1]) / strides[1] + 1 + ) // strides[1] + 1 if ceil_mode else ( + W - ksize[1] + 2 * paddings[1]) // strides[1] + 1 W_out = (W - ksize[2] + 2 * paddings[2] + strides[2] - 1 - ) / strides[2] + 1 if ceil_mode else (W - ksize[2] + 2 * - paddings[2]) / strides[2] + 1 + ) // strides[2] + 1 if ceil_mode else ( + W - ksize[2] + 2 * paddings[2]) // strides[2] + 1 out = np.zeros((N, C, D_out, H_out, W_out)) for k in range(D_out): d_start = np.max((k * strides[0] - paddings[0], 0)) @@ -63,14 +65,14 @@ def avg_pool3D_forward_naive(x, if global_pool == 1: ksize = [D, H, W] D_out = (D - ksize[0] + 2 * paddings[0] + strides[0] - 1 - ) / strides[0] + 1 if ceil_mode else (H - ksize[0] + 2 * - paddings[0]) / strides[0] + 1 + ) // strides[0] + 1 if ceil_mode else ( + H - ksize[0] + 2 * paddings[0]) // strides[0] + 1 H_out = (H - ksize[1] + 2 * paddings[1] + strides[1] - 1 - ) / strides[1] + 1 if ceil_mode else (W - ksize[1] + 2 * - paddings[1]) / strides[1] + 1 + ) // strides[1] + 1 if ceil_mode else ( + W - ksize[1] + 2 * paddings[1]) // strides[1] + 1 W_out = (W - ksize[2] + 2 * paddings[2] + strides[2] - 1 - ) / strides[2] + 1 if ceil_mode else (W - ksize[2] + 2 * - paddings[2]) / strides[2] + 1 + ) // strides[2] + 1 if ceil_mode else ( + W - ksize[2] + 2 * paddings[2]) // strides[2] + 1 out = np.zeros((N, C, D_out, H_out, W_out)) for k in range(D_out): d_start = np.max((k * strides[0] - paddings[0], 0)) diff --git a/python/paddle/fluid/tests/unittests/test_pool_max_op.py b/python/paddle/fluid/tests/unittests/test_pool_max_op.py index e6a9f6f08cf1445c14494506641b0c3502591c37..488ff431d4f2ef76ce0c9486d8c307b4e01b5544 100644 --- a/python/paddle/fluid/tests/unittests/test_pool_max_op.py +++ b/python/paddle/fluid/tests/unittests/test_pool_max_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest @@ -24,9 +26,9 @@ def max_pool3D_forward_naive(x, ksize, strides, paddings, global_pool=False): ksize = [D, H, W] paddings = [0, 0, 0] - D_out = (D - ksize[0] + 2 * paddings[0]) / strides[0] + 1 - H_out = (H - ksize[1] + 2 * paddings[1]) / strides[1] + 1 - W_out = (W - ksize[2] + 2 * paddings[2]) / strides[2] + 1 + D_out = (D - ksize[0] + 2 * paddings[0]) // strides[0] + 1 + H_out = (H - ksize[1] + 2 * paddings[1]) // strides[1] + 1 + W_out = (W - ksize[2] + 2 * paddings[2]) // strides[2] + 1 out = np.zeros((N, C, D_out, H_out, W_out)) mask = np.zeros((N, C, D_out, H_out, W_out)) for k in range(D_out): @@ -63,8 +65,8 @@ def max_pool2D_forward_naive(x, ksize, strides, paddings, global_pool=False): ksize = [H, W] paddings = [0, 0] - H_out = (H - ksize[0] + 2 * paddings[0]) / strides[0] + 1 - W_out = (W - ksize[1] + 2 * paddings[1]) / strides[1] + 1 + H_out = (H - ksize[0] + 2 * paddings[0]) // strides[0] + 1 + W_out = (W - ksize[1] + 2 * paddings[1]) // strides[1] + 1 out = np.zeros((N, C, H_out, W_out)) mask = np.zeros((N, C, H_out, W_out)) for i in range(H_out): diff --git a/python/paddle/fluid/tests/unittests/test_positive_negative_pair_op.py b/python/paddle/fluid/tests/unittests/test_positive_negative_pair_op.py index 8c76393bdaccc0b701b409efebf08fac95aa5f1a..afe8d212d6ec218c3799780849c377e46a44bd6c 100644 --- a/python/paddle/fluid/tests/unittests/test_positive_negative_pair_op.py +++ b/python/paddle/fluid/tests/unittests/test_positive_negative_pair_op.py @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import itertools import numpy as np +import six from op_test import OpTest @@ -32,7 +35,7 @@ def py_pnpair_op(score, label, query, column=-1, weight=None): # accumulate statistics pos, neg, neu = 0, 0, 0 - for _, ranks in list(predictions.items()): + for _, ranks in six.iteritems(predictions): for e1, e2 in itertools.combinations(ranks, 2): s1, s2, l1, l2, w1, w2 = e1[0], e2[0], e1[1], e2[1], e1[2], e2[2] w = (w1 + w2) * 0.5 diff --git a/python/paddle/fluid/tests/unittests/test_precision_recall_op.py b/python/paddle/fluid/tests/unittests/test_precision_recall_op.py index 5ae425fee18b9b1baa0b945782268b79d6bb6625..645637625959f214db3875bc58e4c593c27ae8f6 100644 --- a/python/paddle/fluid/tests/unittests/test_precision_recall_op.py +++ b/python/paddle/fluid/tests/unittests/test_precision_recall_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_prelu_op.py b/python/paddle/fluid/tests/unittests/test_prelu_op.py index cb7de3fc93c0379ea50c88044876d6a8ee617a69..979be5af3bdc24b1a2fc115198eeab53469a91c0 100644 --- a/python/paddle/fluid/tests/unittests/test_prelu_op.py +++ b/python/paddle/fluid/tests/unittests/test_prelu_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_preprocessor.py b/python/paddle/fluid/tests/unittests/test_preprocessor.py index cbf1a7e0c50a87cd43507ffdb94109873cf4e5d9..98e609b76982650c9d18f87c3c0637056cc40899 100644 --- a/python/paddle/fluid/tests/unittests/test_preprocessor.py +++ b/python/paddle/fluid/tests/unittests/test_preprocessor.py @@ -12,12 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np +import paddle import paddle.fluid as fluid -import paddle.v2 as paddle -import paddle.v2.dataset.mnist as mnist +import paddle.dataset.mnist as mnist class TestPreprocessor(unittest.TestCase): diff --git a/python/paddle/fluid/tests/unittests/test_print_op.py b/python/paddle/fluid/tests/unittests/test_print_op.py index b461c5c9401d74ef8dcf4afc84dc0ea6920a2419..ac682d6181cfcc5a064a51a736b03d493c37b780 100644 --- a/python/paddle/fluid/tests/unittests/test_print_op.py +++ b/python/paddle/fluid/tests/unittests/test_print_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.core as core from paddle.fluid.executor import Executor diff --git a/python/paddle/fluid/tests/unittests/test_prior_box_op.py b/python/paddle/fluid/tests/unittests/test_prior_box_op.py index e15554737b9f3fa36382dde15ded928271679538..7381b74af71051f8b993ba6d116b5282dd9b84e1 100644 --- a/python/paddle/fluid/tests/unittests/test_prior_box_op.py +++ b/python/paddle/fluid/tests/unittests/test_prior_box_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import sys diff --git a/python/paddle/fluid/tests/unittests/test_profiler.py b/python/paddle/fluid/tests/unittests/test_profiler.py index 9f8d33f9bbfc78b6f1a0c089b34b2f41d561c640..38a7c913bf07a32494089addf17068017d24d628 100644 --- a/python/paddle/fluid/tests/unittests/test_profiler.py +++ b/python/paddle/fluid/tests/unittests/test_profiler.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import os import numpy as np @@ -93,7 +95,7 @@ class TestProfiler(unittest.TestCase): "profiler is enabled only with GPU") def test_all_profiler(self): self.net_profiler('All', '/tmp/profile_out') - with open('/tmp/profile_out', 'r') as f: + with open('/tmp/profile_out', 'rb') as f: self.assertGreater(len(f.read()), 0) diff --git a/python/paddle/fluid/tests/unittests/test_protobuf.py b/python/paddle/fluid/tests/unittests/test_protobuf.py index c3f1fa80185bfc4afc3ed715d736bcba092629d8..7b80927c48d02e83a9bfaac572c81a6a95a69c8c 100644 --- a/python/paddle/fluid/tests/unittests/test_protobuf.py +++ b/python/paddle/fluid/tests/unittests/test_protobuf.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid.proto.framework_pb2 as framework_pb2 import unittest diff --git a/python/paddle/fluid/tests/unittests/test_protobuf_descs.py b/python/paddle/fluid/tests/unittests/test_protobuf_descs.py index 9853fb4e9a89944bfdf2954e3d3d86fef92ac93c..d24b5cbd06ddf9f332c1369ebd513bef27b77e14 100644 --- a/python/paddle/fluid/tests/unittests/test_protobuf_descs.py +++ b/python/paddle/fluid/tests/unittests/test_protobuf_descs.py @@ -12,8 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.core as core +import paddle.compat as cpt from paddle.fluid.framework import Program @@ -108,7 +111,7 @@ class TestVarDesc(unittest.TestCase): def test_shape(self): program_desc = core.ProgramDesc() block = program_desc.block(0) - var = block.var('my_var') + var = block.var(cpt.to_bytes('my_var')) var.set_type(core.VarDesc.VarType.SELECTED_ROWS) src_shape = [3, 2, 10, 8] var.set_shape(src_shape) @@ -119,7 +122,7 @@ class TestVarDesc(unittest.TestCase): def test_multiple_shape(self): program_desc = core.ProgramDesc() block = program_desc.block(0) - var = block.var('my_reader') + var = block.var(cpt.to_bytes('my_reader')) var.set_type(core.VarDesc.VarType.READER) src_shapes = [[2, 3, 3], [4, 5], [6, 7, 8, 9]] var.set_shapes(src_shapes) @@ -130,7 +133,7 @@ class TestVarDesc(unittest.TestCase): def test_dtype(self): program_desc = core.ProgramDesc() block = program_desc.block(0) - var = block.var('my_var') + var = block.var(cpt.to_bytes('my_var')) var.set_type(core.VarDesc.VarType.LOD_TENSOR) var.set_dtype(core.VarDesc.VarType.INT32) self.assertEqual(core.VarDesc.VarType.INT32, var.dtype()) @@ -139,7 +142,7 @@ class TestVarDesc(unittest.TestCase): def test_multiple_dtype(self): program_desc = core.ProgramDesc() block = program_desc.block(0) - var = block.var('my_reader') + var = block.var(cpt.to_bytes('my_reader')) var.set_type(core.VarDesc.VarType.READER) src_types = [ core.VarDesc.VarType.INT32, core.VarDesc.VarType.FP64, @@ -152,7 +155,7 @@ class TestVarDesc(unittest.TestCase): def test_multiple_lod_level(self): program_desc = core.ProgramDesc() block = program_desc.block(0) - var = block.var('my_reader') + var = block.var(cpt.to_bytes('my_reader')) var.set_type(core.VarDesc.VarType.READER) src_types = [3, 1, 2] var.set_lod_levels(src_types) @@ -166,12 +169,12 @@ class TestBlockDesc(unittest.TestCase): self.assertIsNotNone(program_desc) block = program_desc.block(0) self.assertIsNotNone(block) - var1 = block.var("var1") - var2 = block.var("var2") - var3 = block.var("var3") + var1 = block.var(cpt.to_bytes("var1")) + var2 = block.var(cpt.to_bytes("var2")) + var3 = block.var(cpt.to_bytes("var3")) all_vars = block.all_vars() self.assertEqual(set(all_vars), {var1, var2, var3}) - var2_re = block.find_var("var2") + var2_re = block.find_var(cpt.to_bytes("var2")) self.assertEqual(var2_re, var2) def test_add_op(self): diff --git a/python/paddle/fluid/tests/unittests/test_proximal_adagrad_op.py b/python/paddle/fluid/tests/unittests/test_proximal_adagrad_op.py index 3c2689585061af5a11a247a01b87b432dcd86e13..57e96f1fa34fa94f5e095d088016655f24b58d0c 100644 --- a/python/paddle/fluid/tests/unittests/test_proximal_adagrad_op.py +++ b/python/paddle/fluid/tests/unittests/test_proximal_adagrad_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_proximal_gd_op.py b/python/paddle/fluid/tests/unittests/test_proximal_gd_op.py index 137594b9a08e13bf6c3f3779356c209596f9ba8e..067502baecc73cc84a6aa8ab78a9afbcc191c49a 100644 --- a/python/paddle/fluid/tests/unittests/test_proximal_gd_op.py +++ b/python/paddle/fluid/tests/unittests/test_proximal_gd_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_py_reader_push_pop.py b/python/paddle/fluid/tests/unittests/test_py_reader_push_pop.py index f9bda5e4701f693f41fe7041ba0f5ec80b6fc31c..3efe5aac8848b8230f42f4f3905eefc517c0fa5e 100644 --- a/python/paddle/fluid/tests/unittests/test_py_reader_push_pop.py +++ b/python/paddle/fluid/tests/unittests/test_py_reader_push_pop.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_py_reader_using_executor.py b/python/paddle/fluid/tests/unittests/test_py_reader_using_executor.py index 9a379bdbaa7e278879117a8cdc2dddb335a10ca1..931cac409f26fce4ecca18c4b0cfcca2e675046f 100644 --- a/python/paddle/fluid/tests/unittests/test_py_reader_using_executor.py +++ b/python/paddle/fluid/tests/unittests/test_py_reader_using_executor.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_random_crop_op.py b/python/paddle/fluid/tests/unittests/test_random_crop_op.py index 1c708d0386da4028f1f3d177d0a3fd494c077c6e..f29dddff7a28ed041908741007361224624e436a 100644 --- a/python/paddle/fluid/tests/unittests/test_random_crop_op.py +++ b/python/paddle/fluid/tests/unittests/test_random_crop_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import paddle.fluid.core as core @@ -21,11 +23,12 @@ from op_test import OpTest class TestRandomCropOp(OpTest): def setUp(self): to_crop = np.array([[[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]]] * - 5).astype("float32") + 5).astype(np.int32) self.possible_res = [ - np.array([[1, 2, 3], [5, 6, 7]]), np.array([[2, 3, 4], [6, 7, 8]]), - np.array([[5, 6, 7], [9, 10, 11]]), - np.array([[6, 7, 8], [10, 11, 12]]) + np.array([[1, 2, 3], [5, 6, 7]]).astype(np.int32), + np.array([[2, 3, 4], [6, 7, 8]]).astype(np.int32), + np.array([[5, 6, 7], [9, 10, 11]]).astype(np.int32), + np.array([[6, 7, 8], [10, 11, 12]]).astype(np.int32) ] self.op_type = "random_crop" self.inputs = {'X': to_crop, 'Seed': np.array([10])} diff --git a/python/paddle/fluid/tests/unittests/test_rank_loss_op.py b/python/paddle/fluid/tests/unittests/test_rank_loss_op.py index 7eba1e2077e25325d537f01f043ed1afa372800c..c9fa24b103deb50aa896403e09b11e891fb62c6d 100644 --- a/python/paddle/fluid/tests/unittests/test_rank_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_rank_loss_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_reader_reset.py b/python/paddle/fluid/tests/unittests/test_reader_reset.py index 3ad85d57485956e0cadb197dadd172516fa15c39..8ad11d76f683d556f05cafc3251acc942efef72f 100644 --- a/python/paddle/fluid/tests/unittests/test_reader_reset.py +++ b/python/paddle/fluid/tests/unittests/test_reader_reset.py @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid as fluid -import paddle.v2 as paddle +import paddle import numpy as np import unittest diff --git a/python/paddle/fluid/tests/unittests/test_recordio_reader.py b/python/paddle/fluid/tests/unittests/test_recordio_reader.py index 69a522e273db017ac55b408276b4a28f5f907c42..c5210bb2085bc386df43cd0d20292d7b308a1093 100644 --- a/python/paddle/fluid/tests/unittests/test_recordio_reader.py +++ b/python/paddle/fluid/tests/unittests/test_recordio_reader.py @@ -12,11 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid -import paddle.v2 as paddle -import paddle.v2.dataset.mnist as mnist +import paddle +import paddle.dataset.mnist as mnist class TestRecordIO(unittest.TestCase): diff --git a/python/paddle/fluid/tests/unittests/test_recurrent_op.py b/python/paddle/fluid/tests/unittests/test_recurrent_op.py index 2e22df2beba9d74e28788fb72f6f7f7f2bef534e..6dfc85e301a2eda66bade09a8b6dd0004155f385 100644 --- a/python/paddle/fluid/tests/unittests/test_recurrent_op.py +++ b/python/paddle/fluid/tests/unittests/test_recurrent_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.layers as layers diff --git a/python/paddle/fluid/tests/unittests/test_reduce_op.py b/python/paddle/fluid/tests/unittests/test_reduce_op.py index 06d116601bf733986ccf9c725340456ab1258be2..328f0f0011381b77cccb8b2d9b266aa53b259473 100644 --- a/python/paddle/fluid/tests/unittests/test_reduce_op.py +++ b/python/paddle/fluid/tests/unittests/test_reduce_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_registry.py b/python/paddle/fluid/tests/unittests/test_registry.py index a361c4624e3e2efa817e8137ff31133997a0a1fb..7381bb61eb4630cb67bc306fde211704e9580af4 100644 --- a/python/paddle/fluid/tests/unittests/test_registry.py +++ b/python/paddle/fluid/tests/unittests/test_registry.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function import unittest import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/unittests/test_regularizer.py b/python/paddle/fluid/tests/unittests/test_regularizer.py index 9b1c4ceada52322b3f1fdc4ab2e90a2c089ee67e..6727335c6059161d235a64a1b90d36b84004f9b3 100644 --- a/python/paddle/fluid/tests/unittests/test_regularizer.py +++ b/python/paddle/fluid/tests/unittests/test_regularizer.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.framework as framework diff --git a/python/paddle/fluid/tests/unittests/test_reorder_lod_tensor.py b/python/paddle/fluid/tests/unittests/test_reorder_lod_tensor.py index 6e1cd56b3e309fc014dc981a1e3aa841159fca15..28c8c4699adbc108c05e4a500815752e2ec24c61 100644 --- a/python/paddle/fluid/tests/unittests/test_reorder_lod_tensor.py +++ b/python/paddle/fluid/tests/unittests/test_reorder_lod_tensor.py @@ -12,11 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid import paddle.fluid.core as core from paddle.fluid.layers.control_flow import lod_rank_table import numpy +import functools class TestReorderLoDTensor(unittest.TestCase): @@ -101,7 +104,8 @@ class TestReorderLoDTensor(unittest.TestCase): rank_table = [] # list of (index, length) for i in range(len(ref_lod)): rank_table.append((i, ref_lod[i])) - rank_table = sorted(rank_table, lambda x, y: y[1] - x[1]) + rank_table = sorted( + rank_table, key=functools.cmp_to_key(lambda x, y: y[1] - x[1])) # compute the input sequence info according to input_lod input_value, input_lod = self.data[self.data_desc[0][0]] diff --git a/python/paddle/fluid/tests/unittests/test_reshape_op.py b/python/paddle/fluid/tests/unittests/test_reshape_op.py index 2f5558578ac2a002a83c2a7e027ec5a96d8b4414..1de35dc35b0176b77eb2d9b25cd6ee4e645e56c3 100644 --- a/python/paddle/fluid/tests/unittests/test_reshape_op.py +++ b/python/paddle/fluid/tests/unittests/test_reshape_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_reverse_op.py b/python/paddle/fluid/tests/unittests/test_reverse_op.py index f845575a02869f08299d76b5600074598ca27f6c..e83f548c228c7c045ff795e882738ea56e3f2d24 100644 --- a/python/paddle/fluid/tests/unittests/test_reverse_op.py +++ b/python/paddle/fluid/tests/unittests/test_reverse_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_rmsprop_op.py b/python/paddle/fluid/tests/unittests/test_rmsprop_op.py index 0d84a5853ead45b84de9383dd8749992d2f91440..3d4623c74d9a307b12ab6d72ad0b4d2dae938720 100644 --- a/python/paddle/fluid/tests/unittests/test_rmsprop_op.py +++ b/python/paddle/fluid/tests/unittests/test_rmsprop_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_rnn_memory_helper_op.py b/python/paddle/fluid/tests/unittests/test_rnn_memory_helper_op.py index 178606f05961263df5ef0398064a1fd135fbe784..9bfec8e9bdd8c4667fb19f3dd479b759d6dd665b 100644 --- a/python/paddle/fluid/tests/unittests/test_rnn_memory_helper_op.py +++ b/python/paddle/fluid/tests/unittests/test_rnn_memory_helper_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest from paddle.fluid.framework import Program diff --git a/python/paddle/fluid/tests/unittests/test_roi_pool_op.py b/python/paddle/fluid/tests/unittests/test_roi_pool_op.py index df5684ab173a4889dd7b693f9246bafd12e0345f..ed7f467835f32242a9650f226b4a5ad9d6d87af4 100644 --- a/python/paddle/fluid/tests/unittests/test_roi_pool_op.py +++ b/python/paddle/fluid/tests/unittests/test_roi_pool_op.py @@ -12,10 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import math import sys +import paddle.compat as cpt from op_test import OpTest @@ -59,10 +62,10 @@ class TestROIPoolOp(OpTest): for i in range(self.rois_num): roi = self.rois[i] roi_batch_id = roi[0] - roi_start_w = int(round(roi[1] * self.spatial_scale)) - roi_start_h = int(round(roi[2] * self.spatial_scale)) - roi_end_w = int(round(roi[3] * self.spatial_scale)) - roi_end_h = int(round(roi[4] * self.spatial_scale)) + roi_start_w = int(cpt.round(roi[1] * self.spatial_scale)) + roi_start_h = int(cpt.round(roi[2] * self.spatial_scale)) + roi_end_w = int(cpt.round(roi[3] * self.spatial_scale)) + roi_end_h = int(cpt.round(roi[4] * self.spatial_scale)) roi_height = int(max(roi_end_h - roi_start_h + 1, 1)) roi_width = int(max(roi_end_w - roi_start_w + 1, 1)) @@ -97,8 +100,8 @@ class TestROIPoolOp(OpTest): for w in range(wstart, wend): if x_i[c, h, w] > out_data[i, c, ph, pw]: out_data[i, c, ph, pw] = x_i[c, h, w] - argmax_data[i, c, ph, pw] = h * \ - self.width + w + argmax_data[i, c, ph, + pw] = h * self.width + w self.outs = out_data.astype('float32') self.argmaxes = argmax_data.astype('int64') @@ -110,14 +113,14 @@ class TestROIPoolOp(OpTest): self.rois_lod[0].append(bno + 1) for i in range(bno + 1): x1 = np.random.random_integers( - 0, self.width / self.spatial_scale - self.pooled_width) + 0, self.width // self.spatial_scale - self.pooled_width) y1 = np.random.random_integers( - 0, self.height / self.spatial_scale - self.pooled_height) + 0, self.height // self.spatial_scale - self.pooled_height) x2 = np.random.random_integers(x1 + self.pooled_width, - self.width / self.spatial_scale) - y2 = np.random.random_integers(y1 + self.pooled_height, - self.height / self.spatial_scale) + self.width // self.spatial_scale) + y2 = np.random.random_integers( + y1 + self.pooled_height, self.height // self.spatial_scale) roi = [bno, x1, y1, x2, y2] rois.append(roi) diff --git a/python/paddle/fluid/tests/unittests/test_row_conv_op.py b/python/paddle/fluid/tests/unittests/test_row_conv_op.py index 07dcd108689ae6069e30fe22029258d192215549..2f13f067ef313685227c7de9a49fae8640ca6b32 100644 --- a/python/paddle/fluid/tests/unittests/test_row_conv_op.py +++ b/python/paddle/fluid/tests/unittests/test_row_conv_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_rpn_target_assign_op.py b/python/paddle/fluid/tests/unittests/test_rpn_target_assign_op.py index df6e0faaca6fd007b39a8f358d964055e149a025..08c462d9036cacab81dab7c9ea16664c9159479f 100644 --- a/python/paddle/fluid/tests/unittests/test_rpn_target_assign_op.py +++ b/python/paddle/fluid/tests/unittests/test_rpn_target_assign_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_sampling_id_op.py b/python/paddle/fluid/tests/unittests/test_sampling_id_op.py new file mode 100644 index 0000000000000000000000000000000000000000..708265b4576809b1f4157d54989c6138c6e5a2b0 --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_sampling_id_op.py @@ -0,0 +1,51 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import numpy as np +from op_test import OpTest + +import paddle.fluid.core as core +from paddle.fluid.op import Operator + + +class TestSamplingIdOp(OpTest): + def setUp(self): + self.op_type = "sampling_id" + self.use_mkldnn = False + self.init_kernel_type() + self.X = np.random.random((8, 4)).astype('float32') + self.inputs = {"X": self.X} + self.Y = np.random.random(8).astype('float32') + self.outputs = {'Out': self.Y} + self.attrs = {'max': 1.0, 'min': 0.0, 'seed': 1} + + def test_check_output(self): + self.check_output_customized(self.verify_output) + y1 = self.out + self.check_output_customized(self.verify_output) + y2 = self.out + self.assertTrue(np.array_equal(y1, y2)) + self.assertEqual(len(y1), len(self.Y)) + + def verify_output(self, outs): + out = np.array(outs[0]) + self.out = out + + def init_kernel_type(self): + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/python/paddle/fluid/tests/unittests/test_scale_op.py b/python/paddle/fluid/tests/unittests/test_scale_op.py index 53f59c399054a96f5b5f07a390e6fa9eeae878ce..0a8a43253d79ba21c7333dd19af05d8adf410289 100644 --- a/python/paddle/fluid/tests/unittests/test_scale_op.py +++ b/python/paddle/fluid/tests/unittests/test_scale_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_scatter_op.py b/python/paddle/fluid/tests/unittests/test_scatter_op.py index fb1728743630b3ea908ae835444eff7fd71b72c8..088996f9d7dee1ea914e36e3342c9a5110001c44 100644 --- a/python/paddle/fluid/tests/unittests/test_scatter_op.py +++ b/python/paddle/fluid/tests/unittests/test_scatter_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_scope.py b/python/paddle/fluid/tests/unittests/test_scope.py index d249a989a9499d01f6ed10d6cdbc6c456a7262c5..45fcbfba6eb7c6fc4e75f6d8228d721c0186ef36 100644 --- a/python/paddle/fluid/tests/unittests/test_scope.py +++ b/python/paddle/fluid/tests/unittests/test_scope.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid.core import unittest diff --git a/python/paddle/fluid/tests/unittests/test_selected_rows.py b/python/paddle/fluid/tests/unittests/test_selected_rows.py index f504a06ffff8cb636498652554fca05e22bb905d..2f34f79b8eafad8e7fdf6b359548747f354b141f 100644 --- a/python/paddle/fluid/tests/unittests/test_selected_rows.py +++ b/python/paddle/fluid/tests/unittests/test_selected_rows.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid.core as core import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_seq_concat_op.py b/python/paddle/fluid/tests/unittests/test_seq_concat_op.py index 11ffa761a690eb1f9f6dc50c45128a99301741db..9d1d139721ad7ee3e29d44c9b3e7c666b78a4556 100644 --- a/python/paddle/fluid/tests/unittests/test_seq_concat_op.py +++ b/python/paddle/fluid/tests/unittests/test_seq_concat_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import sys diff --git a/python/paddle/fluid/tests/unittests/test_seq_conv.py b/python/paddle/fluid/tests/unittests/test_seq_conv.py index 1a6e1aad799e77b8e746353bee93680691939d24..dcc86382e5286f354c4f2e81ead598f12c75b2c1 100644 --- a/python/paddle/fluid/tests/unittests/test_seq_conv.py +++ b/python/paddle/fluid/tests/unittests/test_seq_conv.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import random diff --git a/python/paddle/fluid/tests/unittests/test_seq_pool.py b/python/paddle/fluid/tests/unittests/test_seq_pool.py index 0b3659d7a67956f7546d368346bd102eeedf1d97..66e77714c5d65d51262f76519901032182985ea8 100644 --- a/python/paddle/fluid/tests/unittests/test_seq_pool.py +++ b/python/paddle/fluid/tests/unittests/test_seq_pool.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_sequence_erase_op.py b/python/paddle/fluid/tests/unittests/test_sequence_erase_op.py index 8f0765277ae85af2b17ad96d4fd0c1148c393ff0..92cd5b0cbcd1ab56300158d26850969870e86f2b 100644 --- a/python/paddle/fluid/tests/unittests/test_sequence_erase_op.py +++ b/python/paddle/fluid/tests/unittests/test_sequence_erase_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_sequence_expand.py b/python/paddle/fluid/tests/unittests/test_sequence_expand.py index 5ff0dab23e516ae8114b8264492fb2a9d5c0b3f8..ffd4026dbade2f8f7eace399c52ae0428f3e8d7b 100644 --- a/python/paddle/fluid/tests/unittests/test_sequence_expand.py +++ b/python/paddle/fluid/tests/unittests/test_sequence_expand.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_sequence_reshape.py b/python/paddle/fluid/tests/unittests/test_sequence_reshape.py index 39b02ecf6ddb40737c4c1737d652c1a1b744d923..f11fa6c39c35efc14f8600dd746ab64cc940cd71 100644 --- a/python/paddle/fluid/tests/unittests/test_sequence_reshape.py +++ b/python/paddle/fluid/tests/unittests/test_sequence_reshape.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import math diff --git a/python/paddle/fluid/tests/unittests/test_sequence_slice_op.py b/python/paddle/fluid/tests/unittests/test_sequence_slice_op.py index 313e485d1e3080f2c59c68256cbc5c81aa6558cd..1561490087330c9af3ea3e384bf735eaa268a749 100644 --- a/python/paddle/fluid/tests/unittests/test_sequence_slice_op.py +++ b/python/paddle/fluid/tests/unittests/test_sequence_slice_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import sys diff --git a/python/paddle/fluid/tests/unittests/test_sequence_softmax_op.py b/python/paddle/fluid/tests/unittests/test_sequence_softmax_op.py index c4fc8b74cf80c3596b0af9f7f0434864591195bd..3e00e7d95f63ea652ea1964eb792f9393ffa5994 100644 --- a/python/paddle/fluid/tests/unittests/test_sequence_softmax_op.py +++ b/python/paddle/fluid/tests/unittests/test_sequence_softmax_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_sgd_op.py b/python/paddle/fluid/tests/unittests/test_sgd_op.py index 3126293f9d8e52daa866be5fc1533648a33f3363..b46e4bfb86bd5dc9c74375693328f2506281be3e 100644 --- a/python/paddle/fluid/tests/unittests/test_sgd_op.py +++ b/python/paddle/fluid/tests/unittests/test_sgd_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import paddle.fluid.core as core @@ -124,6 +126,7 @@ class TestSGDOpOptimizeSelectedRows(unittest.TestCase): w_selected_rows = scope.var('Param').get_selected_rows() w_selected_rows.set_height(len(param_rows)) w_selected_rows.set_rows(param_rows) + w_selected_rows.sync_index() w_array = np.ones((len(param_rows), row_width)).astype("float32") for i in range(len(param_rows)): w_array[i] *= i diff --git a/python/paddle/fluid/tests/unittests/test_shape_op.py b/python/paddle/fluid/tests/unittests/test_shape_op.py index a62ee050075cb8c9f8817c142825a89c24bdfedf..02231ea943e1e92a08730e6e9f1aa3cefeb927c0 100644 --- a/python/paddle/fluid/tests/unittests/test_shape_op.py +++ b/python/paddle/fluid/tests/unittests/test_shape_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_shrink_rnn_memory.py b/python/paddle/fluid/tests/unittests/test_shrink_rnn_memory.py index a994bf181a74ca71a970da0105fe767f82750a6c..97f79f9421d498723da4c7992551f1210d3f6003 100644 --- a/python/paddle/fluid/tests/unittests/test_shrink_rnn_memory.py +++ b/python/paddle/fluid/tests/unittests/test_shrink_rnn_memory.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.core as core from paddle.fluid.executor import Executor diff --git a/python/paddle/fluid/tests/unittests/test_sigmoid_cross_entropy_with_logits_op.py b/python/paddle/fluid/tests/unittests/test_sigmoid_cross_entropy_with_logits_op.py index c435796569cd2479c19d70a849991f439bf5292a..97ff203499c0bf223930c904de46e1abdd902799 100644 --- a/python/paddle/fluid/tests/unittests/test_sigmoid_cross_entropy_with_logits_op.py +++ b/python/paddle/fluid/tests/unittests/test_sigmoid_cross_entropy_with_logits_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np from op_test import OpTest from scipy.special import logit diff --git a/python/paddle/fluid/tests/unittests/test_sign_op.py b/python/paddle/fluid/tests/unittests/test_sign_op.py index 087a0c575bfa6bc18cb229ad274b4e1e90210605..85a9d9cae47c2b0942da0e0d962d4512af1566c0 100644 --- a/python/paddle/fluid/tests/unittests/test_sign_op.py +++ b/python/paddle/fluid/tests/unittests/test_sign_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_slice_op.py b/python/paddle/fluid/tests/unittests/test_slice_op.py index 1a48bce3bb7c74551a365fd471f6869b128babac..134df38eea6655857db04dfdc19dd7f7897946f4 100644 --- a/python/paddle/fluid/tests/unittests/test_slice_op.py +++ b/python/paddle/fluid/tests/unittests/test_slice_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_slice_var.py b/python/paddle/fluid/tests/unittests/test_slice_var.py index 82305b23a1a1e2cee8cef6b291d848581fe5b509..fab63b7d5631829feffd26fc1dce2bd338d2036b 100644 --- a/python/paddle/fluid/tests/unittests/test_slice_var.py +++ b/python/paddle/fluid/tests/unittests/test_slice_var.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import math import unittest from paddle.fluid.transpiler.distribute_transpiler import slice_variable diff --git a/python/paddle/fluid/tests/unittests/test_smooth_l1_loss_op.py b/python/paddle/fluid/tests/unittests/test_smooth_l1_loss_op.py index e74664dac4d87c5a7e1e061294e93e8267e3cc17..8ab6833821c75262124b3ae4200a17e457b718d5 100644 --- a/python/paddle/fluid/tests/unittests/test_smooth_l1_loss_op.py +++ b/python/paddle/fluid/tests/unittests/test_smooth_l1_loss_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_softmax_op.py b/python/paddle/fluid/tests/unittests/test_softmax_op.py index 70ad05597c4a160cf6a25aeb3c379320cef69c63..d88aa1ae1c9d848eba7a2224d22b5201fc27b857 100644 --- a/python/paddle/fluid/tests/unittests/test_softmax_op.py +++ b/python/paddle/fluid/tests/unittests/test_softmax_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_softmax_with_cross_entropy_op.py b/python/paddle/fluid/tests/unittests/test_softmax_with_cross_entropy_op.py index c0d9fc8f22a7c4f791d80a9cad87d003b5d54299..b7e5ff6d52ad7dde3dd94b3bd660cfca383e1ada 100644 --- a/python/paddle/fluid/tests/unittests/test_softmax_with_cross_entropy_op.py +++ b/python/paddle/fluid/tests/unittests/test_softmax_with_cross_entropy_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_split_and_merge_lod_tensor_op.py b/python/paddle/fluid/tests/unittests/test_split_and_merge_lod_tensor_op.py index ea1146166d34a31efbd859318b411cea50895fe1..5397d5c52158ccfb9ad5703b957ca59d6fa11418 100644 --- a/python/paddle/fluid/tests/unittests/test_split_and_merge_lod_tensor_op.py +++ b/python/paddle/fluid/tests/unittests/test_split_and_merge_lod_tensor_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.core as core import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_split_ids_op.py b/python/paddle/fluid/tests/unittests/test_split_ids_op.py index ca7861309839d183e18c168403881a0b1b5bf309..4c3d0258980fd8595704a65219deb520b96e222e 100644 --- a/python/paddle/fluid/tests/unittests/test_split_ids_op.py +++ b/python/paddle/fluid/tests/unittests/test_split_ids_op.py @@ -12,8 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np +import six from op_test import OpTest import paddle.fluid.core as core from paddle.fluid.op import Operator @@ -59,7 +62,7 @@ class TestSpliteIds(unittest.TestCase): x_tensor = x.get_tensor() x_tensor.set(np_array, place) - outs_name = ["out%d" % i for i in xrange(3)] + outs_name = ["out%d" % i for i in six.moves.xrange(3)] outs = [ scope.var(var_name).get_selected_rows() for var_name in outs_name ] diff --git a/python/paddle/fluid/tests/unittests/test_split_op.py b/python/paddle/fluid/tests/unittests/test_split_op.py index 6b67a52e81b978ed78c72629f9177759f8e2c4e1..3c5dd782f85235c4a2feb5a8ca6d048a012c5e1c 100644 --- a/python/paddle/fluid/tests/unittests/test_split_op.py +++ b/python/paddle/fluid/tests/unittests/test_split_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_split_selected_rows_op.py b/python/paddle/fluid/tests/unittests/test_split_selected_rows_op.py index 2b261820e04b08234477fc0a9adde95262f99bba..41a5ee59ea523b1f6c5015974a12c526e883fa35 100644 --- a/python/paddle/fluid/tests/unittests/test_split_selected_rows_op.py +++ b/python/paddle/fluid/tests/unittests/test_split_selected_rows_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.core as core import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_spp_op.py b/python/paddle/fluid/tests/unittests/test_spp_op.py index 3cbfc2a703f1c4a24674d468cd1152bfa6eb8ad2..a6c2cccd39c9cecb2ae904a1930b44ba18dbbd7e 100644 --- a/python/paddle/fluid/tests/unittests/test_spp_op.py +++ b/python/paddle/fluid/tests/unittests/test_spp_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_squared_l2_distance_op.py b/python/paddle/fluid/tests/unittests/test_squared_l2_distance_op.py index 78bc300ebec1cd34e44343d47376fef05a6d0135..a8bc1004d9bbe91e323db49c0cf0b576f8da306e 100644 --- a/python/paddle/fluid/tests/unittests/test_squared_l2_distance_op.py +++ b/python/paddle/fluid/tests/unittests/test_squared_l2_distance_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_squared_l2_norm_op.py b/python/paddle/fluid/tests/unittests/test_squared_l2_norm_op.py index 609445d52287f421e67a5796f9e50c1fb42c8e49..439bae9510ee84b131050bb6804a3ede2ad6a8b3 100644 --- a/python/paddle/fluid/tests/unittests/test_squared_l2_norm_op.py +++ b/python/paddle/fluid/tests/unittests/test_squared_l2_norm_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import unittest from numpy import linalg as LA diff --git a/python/paddle/fluid/tests/unittests/test_squeeze_op.py b/python/paddle/fluid/tests/unittests/test_squeeze_op.py index bca6af2fd5dfadbc48cf1a76cfa6ffd4f1fdfdef..a2a5584459a5d4dc416b9c542a4bb0567982e765 100644 --- a/python/paddle/fluid/tests/unittests/test_squeeze_op.py +++ b/python/paddle/fluid/tests/unittests/test_squeeze_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_sum_mkldnn_op.py b/python/paddle/fluid/tests/unittests/test_sum_mkldnn_op.py index 7956897d68a3fb49d62ba696d0b6400b4f909989..55820f31b81df9f3618d1004f6d21565564efa29 100644 --- a/python/paddle/fluid/tests/unittests/test_sum_mkldnn_op.py +++ b/python/paddle/fluid/tests/unittests/test_sum_mkldnn_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest from test_sum_op import TestSumOp diff --git a/python/paddle/fluid/tests/unittests/test_sum_op.py b/python/paddle/fluid/tests/unittests/test_sum_op.py index 1d90414e137a70e6265042e24e106fe565802778..9dc93048e631801acd3b21a3bb140ecc4fcd7732 100644 --- a/python/paddle/fluid/tests/unittests/test_sum_op.py +++ b/python/paddle/fluid/tests/unittests/test_sum_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_switch.py b/python/paddle/fluid/tests/unittests/test_switch.py index 528c5cce4bc7262ade196f6a81a57a57089117ec..2a9c07a889ba5fe24fd1c098729a233cb8fbb16f 100644 --- a/python/paddle/fluid/tests/unittests/test_switch.py +++ b/python/paddle/fluid/tests/unittests/test_switch.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_target_assign_op.py b/python/paddle/fluid/tests/unittests/test_target_assign_op.py index bd208897520122b6a5dcf71da325b1b9dba632f6..aec219f80639415a9be55ba18e7940953d0e11b0 100644 --- a/python/paddle/fluid/tests/unittests/test_target_assign_op.py +++ b/python/paddle/fluid/tests/unittests/test_target_assign_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np import random diff --git a/python/paddle/fluid/tests/unittests/test_tensor.py b/python/paddle/fluid/tests/unittests/test_tensor.py index 5ccc876ae8e6e20f76c77c1892f4de59d72bffc8..e9d0f8a0193c77da33a8cf128dbf8a1c5087782b 100644 --- a/python/paddle/fluid/tests/unittests/test_tensor.py +++ b/python/paddle/fluid/tests/unittests/test_tensor.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import paddle.fluid.core as core import unittest import numpy diff --git a/python/paddle/fluid/tests/unittests/test_top_k_op.py b/python/paddle/fluid/tests/unittests/test_top_k_op.py index cbc3da550306b9febe8a8fd22e7f71efa572a3d0..e54e170f7f1e03db4b63db72edb7395d18130f68 100644 --- a/python/paddle/fluid/tests/unittests/test_top_k_op.py +++ b/python/paddle/fluid/tests/unittests/test_top_k_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_transpose_op.py b/python/paddle/fluid/tests/unittests/test_transpose_op.py index ebd63fbd495354eafe298ad5cc3456a196538a6a..0853f80b82030679d140f7fabdd42557c2374599 100644 --- a/python/paddle/fluid/tests/unittests/test_transpose_op.py +++ b/python/paddle/fluid/tests/unittests/test_transpose_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_uniform_random_batch_size_like_op.py b/python/paddle/fluid/tests/unittests/test_uniform_random_batch_size_like_op.py index e033e86114f17d37c01480fe8350648eb8aa27cb..7b8be24d9da8c15eeb52c0ba207ea780b03254f8 100644 --- a/python/paddle/fluid/tests/unittests/test_uniform_random_batch_size_like_op.py +++ b/python/paddle/fluid/tests/unittests/test_uniform_random_batch_size_like_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_uniform_random_op.py b/python/paddle/fluid/tests/unittests/test_uniform_random_op.py index 346a949b6e7c96b5535f5e65ddbada11e110a0a7..d6a5d68765c53d9d711add64c86575a0db6997e4 100644 --- a/python/paddle/fluid/tests/unittests/test_uniform_random_op.py +++ b/python/paddle/fluid/tests/unittests/test_uniform_random_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest diff --git a/python/paddle/fluid/tests/unittests/test_unique_name.py b/python/paddle/fluid/tests/unittests/test_unique_name.py index 49ef335618ca7ca1e8249a61a97ca552dabdb9e8..b8c751b2e9b5a905d9de40fc5f78a02c6ca5e034 100644 --- a/python/paddle/fluid/tests/unittests/test_unique_name.py +++ b/python/paddle/fluid/tests/unittests/test_unique_name.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid as fluid diff --git a/python/paddle/fluid/tests/unittests/test_unpool_op.py b/python/paddle/fluid/tests/unittests/test_unpool_op.py index ecce4cdde2d648fe7d65427e34c77f5f9ad61417..b0c7c3c8662e217f4e88245f22f6b50e7a48c8b7 100644 --- a/python/paddle/fluid/tests/unittests/test_unpool_op.py +++ b/python/paddle/fluid/tests/unittests/test_unpool_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np from op_test import OpTest @@ -27,7 +29,7 @@ def unpool2dmax_forward_naive(input, indices, ksize, strides, paddings): for h in range(s2): for w in range(s3): index = indices[nidx, cidx, h, w] - hidx = (index - index % out_wsize) / out_wsize + hidx = (index - index % out_wsize) // out_wsize widx = index % out_wsize out[nidx, cidx, int(hidx), int(widx)] = \ input[nidx, cidx, h, w] @@ -41,9 +43,9 @@ class TestUnpoolOp(OpTest): self.init_test_case() pre_input = np.random.random(self.shape).astype("float32") nsize, csize, hsize, wsize = pre_input.shape - hsize_out = (hsize - self.ksize[0] + 2 * self.paddings[0]) / \ + hsize_out = (hsize - self.ksize[0] + 2 * self.paddings[0]) // \ self.strides[0] + 1 - wsize_out = (wsize - self.ksize[1] + 2 * self.paddings[1]) / \ + wsize_out = (wsize - self.ksize[1] + 2 * self.paddings[1]) // \ self.strides[1] + 1 input = np.zeros((nsize, csize, hsize_out, wsize_out)) indices = np.zeros((nsize, csize, hsize_out, wsize_out)) @@ -62,7 +64,7 @@ class TestUnpoolOp(OpTest): input[nidx, cidx, i, j] = x_masked.max() arg = x_masked.argmax() indices[nidx, cidx, i, j] = \ - (r_start + arg / self.ksize[1]) * wsize + \ + (r_start + arg // self.ksize[1]) * wsize + \ c_start + arg % self.ksize[1] output = self.unpool2d_forward_naive(input, indices, self.ksize, \ self.strides, self.paddings).astype("float32") diff --git a/python/paddle/fluid/tests/unittests/test_unsqueeze_op.py b/python/paddle/fluid/tests/unittests/test_unsqueeze_op.py index 7a4aa0a40b5eb494f6027e800ca6b466bbe1c302..5fcabe4c83457a91ff0e9cd5568904698353b62a 100644 --- a/python/paddle/fluid/tests/unittests/test_unsqueeze_op.py +++ b/python/paddle/fluid/tests/unittests/test_unsqueeze_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy as np diff --git a/python/paddle/fluid/tests/unittests/test_variable.py b/python/paddle/fluid/tests/unittests/test_variable.py index 49784e21c461bacadd404bf4a8640ebc4dcb26ca..b0830e130dd9a9037f8dd900a256eea3d05f64b8 100644 --- a/python/paddle/fluid/tests/unittests/test_variable.py +++ b/python/paddle/fluid/tests/unittests/test_variable.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest from paddle.fluid.framework import default_main_program, Program, convert_np_dtype_to_dtype_ import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/test_version.py b/python/paddle/fluid/tests/unittests/test_version.py index a09c8a759b9461edcf7d5ddbd62d74408d5f292e..42a0e5c802c53ed0e6aad38fb9ab0f64122e87f5 100644 --- a/python/paddle/fluid/tests/unittests/test_version.py +++ b/python/paddle/fluid/tests/unittests/test_version.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import re diff --git a/python/paddle/fluid/tests/unittests/test_warpctc_op.py b/python/paddle/fluid/tests/unittests/test_warpctc_op.py index 9f1aaee472f918da7deb8816a0a4654dafe74a30..5e3aa13546d0c4fdcde4a3d6378d5a1748327814 100644 --- a/python/paddle/fluid/tests/unittests/test_warpctc_op.py +++ b/python/paddle/fluid/tests/unittests/test_warpctc_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import sys import unittest import numpy as np @@ -132,7 +134,7 @@ class CTCForward(object): for k in range(end - start): j = k + start if j & 1 == 1: - label_idx = j / 2 + label_idx = j // 2 label_val = labels_a_sequence[label_idx, 0] fv = self.log_add(forward_vars[i - 1, j], forward_vars[i - 1, j - 1]) diff --git a/python/paddle/fluid/tests/unittests/test_weight_normalization.py b/python/paddle/fluid/tests/unittests/test_weight_normalization.py index 436f9b9f86fb86270e47c8e30c5c0701787ca0f1..e990d8b2498f6a1b62f7a34d329e3ca72a962728 100644 --- a/python/paddle/fluid/tests/unittests/test_weight_normalization.py +++ b/python/paddle/fluid/tests/unittests/test_weight_normalization.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import numpy import collections diff --git a/python/paddle/fluid/tests/unittests/test_while_op.py b/python/paddle/fluid/tests/unittests/test_while_op.py index 790e6afe5f02236b00d9c67b7b25a881e07abace..b75373cf24a7344bf59b3c6fcb9c4c3969be6503 100644 --- a/python/paddle/fluid/tests/unittests/test_while_op.py +++ b/python/paddle/fluid/tests/unittests/test_while_op.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import unittest import paddle.fluid.layers as layers from paddle.fluid.executor import Executor diff --git a/python/paddle/fluid/tests/unittests/testsuite.py b/python/paddle/fluid/tests/unittests/testsuite.py index c6e176ca31c57e623addd9594be81c0abdce489b..31ae25f02c6fab0d68e83a131ace054d4eb56c12 100644 --- a/python/paddle/fluid/tests/unittests/testsuite.py +++ b/python/paddle/fluid/tests/unittests/testsuite.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import numpy as np import paddle.fluid.core as core diff --git a/python/paddle/fluid/tests/unittests/transformer_model.py b/python/paddle/fluid/tests/unittests/transformer_model.py index 868a0248be6833d0e8fed8a26549352562c279c1..f0e74aff6bdfa7d9f0a7f10e64cac4de88009f0a 100644 --- a/python/paddle/fluid/tests/unittests/transformer_model.py +++ b/python/paddle/fluid/tests/unittests/transformer_model.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from functools import partial import numpy as np diff --git a/python/paddle/fluid/trainer.py b/python/paddle/fluid/trainer.py index eed9b49ef40b591d5b6481846dab714423f57990..294308f1877360174c69cc59b0d2037e494985e7 100644 --- a/python/paddle/fluid/trainer.py +++ b/python/paddle/fluid/trainer.py @@ -12,10 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import contextlib import os import errno import shutil +import six import time from . import core @@ -618,7 +621,7 @@ def build_feed_var_list(program, feed_order): "The values of 'feed_order' should be a permutation of [0, len(feed_order))" ) sorted_pair_list = sorted( - list(feed_order.items()), key=lambda item: item[1]) + six.iteritems(feed_order), key=lambda item: item[1]) feed_var_list = [ program.global_block().var(pair[0]) for pair in sorted_pair_list ] @@ -1036,7 +1039,7 @@ def _save_trainer_args(dirname, trainer_id, trainer_args): cur_dir = _get_trainer_dir(dirname, trainer_id) - for name, value in list(trainer_args.items()): + for name, value in six.iteritems(trainer_args): args_file = os.path.join(cur_dir, name) with open(args_file, 'w') as f: f.write(str(value)) diff --git a/python/paddle/fluid/transpiler/__init__.py b/python/paddle/fluid/transpiler/__init__.py index a8622ad54433fff40f68520955f0294e2955577e..8429e2fd7c5141f064c66d8f406889bca1510fe2 100644 --- a/python/paddle/fluid/transpiler/__init__.py +++ b/python/paddle/fluid/transpiler/__init__.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from .distribute_transpiler import DistributeTranspiler, DistributeTranspilerConfig from .inference_transpiler import InferenceTranspiler from .memory_optimization_transpiler import memory_optimize, release_memory diff --git a/python/paddle/fluid/transpiler/details/__init__.py b/python/paddle/fluid/transpiler/details/__init__.py index 1bfab1f219f8a2f08a0fb5c0042d87a3ad707dd5..5e98266a761c7e01bd6668e85e6adeb54103ca80 100644 --- a/python/paddle/fluid/transpiler/details/__init__.py +++ b/python/paddle/fluid/transpiler/details/__init__.py @@ -12,5 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from .program_utils import * from .ufind import * diff --git a/python/paddle/fluid/transpiler/details/program_utils.py b/python/paddle/fluid/transpiler/details/program_utils.py index 76d10777f5f9ed6d27d55a640108bd036d8d8bac..640dbf4bbed58edf746456419af18c75241fa03c 100644 --- a/python/paddle/fluid/transpiler/details/program_utils.py +++ b/python/paddle/fluid/transpiler/details/program_utils.py @@ -12,12 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + +import six + def delete_ops(block, ops): try: start = list(block.ops).index(ops[0]) end = list(block.ops).index(ops[-1]) - [block._remove_op(start) for _ in range(end - start + 1)] + [block._remove_op(start) for _ in six.moves.range(end - start + 1)] except Exception as e: raise e block.program._sync_with_cpp() diff --git a/python/paddle/fluid/transpiler/details/ufind.py b/python/paddle/fluid/transpiler/details/ufind.py index 0e30d0e3f9c5712c494daf17b2b4bcec86f69c23..aa63af7dcf7ac85031fb00ca4c39fb36d7e588b8 100644 --- a/python/paddle/fluid/transpiler/details/ufind.py +++ b/python/paddle/fluid/transpiler/details/ufind.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + class UnionFind(object): """ Union-find data structure. diff --git a/python/paddle/fluid/transpiler/distribute_transpiler.py b/python/paddle/fluid/transpiler/distribute_transpiler.py index ce4709f23b752cc061f3b767a262f82378b86707..57bc2e8a0ba173bb1273a5183340d0b618f0d73c 100644 --- a/python/paddle/fluid/transpiler/distribute_transpiler.py +++ b/python/paddle/fluid/transpiler/distribute_transpiler.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from __future__ import print_function """ Steps to transpile trainer: 1. split variable to multiple blocks, aligned by product(dim[1:]) (width). @@ -31,6 +33,8 @@ Steps to transpile pserver: import math import random import numpy as np +import collections +import six from .ps_dispatcher import RoundRobin, HashName, PSDispatcher from .. import core, framework @@ -207,6 +211,9 @@ class DistributeTranspiler(object): ps_dispatcher = self.config.split_method(self.pserver_endpoints) self.has_distributed_lookup_table = self._has_distributed_lookup_table() + self.param_name_to_grad_name = dict() + for param_var, grad_var in self.params_grads: + self.param_name_to_grad_name[param_var.name] = grad_var.name # step 1: split and create vars, then put splited vars in dicts for later use. self._init_splited_vars() @@ -220,39 +227,45 @@ class DistributeTranspiler(object): # fc_w@GRAD_trainer_0, fc_w@GRAD_trainer_1 --> pserver1 # fc_b@GRAD_trainer_0, fc_b@GRAD_trainer_1 --> pserver2 # shuffle the map will avoid the uneven distribution above - grad_var_mapping_items = list(self.grad_var_mapping.items()) + grad_var_mapping_items = list(six.iteritems(self.grad_var_mapping)) + if not self.config.slice_var_up: - random.seed(self.trainer_num) + random.seed(self.origin_program.random_seed) random.shuffle(grad_var_mapping_items) - for orig_varname, splited_vars in grad_var_mapping_items: + grad_name_to_send_dummy_out = dict() + for grad_varname, splited_vars in grad_var_mapping_items: eplist = ps_dispatcher.dispatch(splited_vars) if not self.config.slice_var_up: assert (len(splited_vars) == 1) + splited_grad_varname = grad_varname if len(splited_vars) == 1: - orig_varname = splited_vars[0].name + splited_grad_varname = splited_vars[0].name index = find_op_by_output_arg(program.global_block(), - orig_varname) + splited_grad_varname) elif len(splited_vars) > 1: - orig_var = program.global_block().vars[orig_varname] + orig_var = program.global_block().vars[splited_grad_varname] index = find_op_by_output_arg(program.global_block(), - orig_varname) + splited_grad_varname) self._insert_split_op(program, orig_var, index, splited_vars) index += 1 else: AssertionError("Can not insert the send op by original " - "variable name :", orig_varname) + "variable name :", splited_grad_varname) + dummy_output = program.global_block().create_var() + grad_name_to_send_dummy_out[grad_varname] = dummy_output program.global_block()._insert_op( index=index + 1, type="send", inputs={"X": splited_vars}, - outputs={}, + outputs={"Out": dummy_output}, attrs={ "epmap": eplist, - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + "sync_mode": not self.sync_mode, }) for _, var in enumerate(splited_vars): send_vars.append(var) @@ -264,7 +277,6 @@ class DistributeTranspiler(object): outputs={}, attrs={ "endpoints": pserver_endpoints, - "sync_mode": self.sync_mode, RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE }) @@ -280,19 +292,21 @@ class DistributeTranspiler(object): self.param_grad_ep_mapping[ep]["grads"].append(send_vars[i]) # step4: Concat the parameters splits together after recv. - for varname, splited_var in list(self.param_var_mapping.items()): + for param_varname, splited_var in six.iteritems(self.param_var_mapping): eps = [] for var in splited_var: index = [v.name for v in recv_vars].index(var.name) eps.append(eplist[index]) - + grad_send_dummy_out = grad_name_to_send_dummy_out[ + self.param_name_to_grad_name[param_varname]] program.global_block().append_op( type="recv", - inputs={}, + inputs={"X": [grad_send_dummy_out]}, outputs={"Out": splited_var}, attrs={ "epmap": eps, - RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE + RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE, + "sync_mode": not self.sync_mode }) if self.sync_mode: @@ -305,10 +319,10 @@ class DistributeTranspiler(object): RPC_OP_ROLE_ATTR_NAME: RPC_OP_ROLE_ATTR_VALUE }) - for varname, splited_var in list(self.param_var_mapping.items()): + for param_varname, splited_var in six.iteritems(self.param_var_mapping): if len(splited_var) <= 1: continue - orig_param = program.global_block().vars[varname] + orig_param = program.global_block().vars[param_varname] program.global_block().append_op( type="concat", inputs={"X": splited_var}, @@ -376,7 +390,7 @@ class DistributeTranspiler(object): op = startup_program.global_block().append_op( type="recv", - inputs={}, + inputs={"X": []}, outputs={"Out": splited_var}, attrs={ "epmap": eps, @@ -641,14 +655,14 @@ class DistributeTranspiler(object): # 1. create vars in pserver program to startup program pserver_vars = pserver_program.global_block().vars - created_var_map = dict() - for _, var in list(pserver_vars.items()): + created_var_map = collections.OrderedDict() + for _, var in six.iteritems(pserver_vars): tmpvar = s_prog.global_block()._clone_variable(var) created_var_map[var.name] = tmpvar # 2. rename op outputs for op in orig_s_prog.global_block().ops: - new_outputs = dict() + new_outputs = collections.OrderedDict() # do not append startup op if var is not on this pserver op_on_pserver = False # TODO(gongwb): remove this line. @@ -782,22 +796,24 @@ class DistributeTranspiler(object): self.config.min_block_size) assert (len(grad_blocks) == len(param_blocks)) - # origin_varname -> [splited_var] + # origin_param_name -> [splited_param_vars] self.param_var_mapping = self._create_vars_from_blocklist( self.origin_program, param_blocks) + # origin_grad_name -> [splited_grad_vars] self.grad_var_mapping = self._create_vars_from_blocklist( self.origin_program, grad_blocks, add_trainer_suffix=self.trainer_num > 1) - self.grad_param_mapping = dict() + # dict(grad_splited_var -> param_splited_var) + self.grad_param_mapping = collections.OrderedDict() for g, p in zip(grad_blocks, param_blocks): g_name, g_bid, _ = g.split(":") p_name, p_bid, _ = p.split(":") self.grad_param_mapping[self.grad_var_mapping[g_name][int(g_bid)]] = \ - self.param_var_mapping[p_name][int(p_bid)] + self.param_var_mapping[p_name][int(p_bid)] # create mapping of endpoint -> split var to create pserver side program - self.param_grad_ep_mapping = dict() + self.param_grad_ep_mapping = collections.OrderedDict() [ self.param_grad_ep_mapping.update({ ep: { @@ -915,7 +931,7 @@ class DistributeTranspiler(object): index=op_index + 2, type="send", inputs={'X': self.trainer_side_table_grad_list}, - outputs={}, + outputs={'Out': []}, attrs={ "sync_mode": True, "epmap": pserver_endpoints, @@ -1072,21 +1088,21 @@ class DistributeTranspiler(object): block_list (list[(varname, block_id, block_size)]): List of gradient blocks. add_trainer_suffix (Bool): Add trainer suffix to new variable's name if set True. Returns: - var_mapping (dict(varname->[new_varname_variable])):A dict mapping + var_mapping (collections.OrderedDict(varname->[new_varname_variable])):A dict mapping from original var name to each var split. """ # varname->[(block_id, current_block_size)] - block_map = dict() + block_map = collections.OrderedDict() - var_mapping = dict() + var_mapping = collections.OrderedDict() for block_str in block_list: varname, offset, size = block_str.split(":") if varname not in block_map: block_map[varname] = [] block_map[varname].append((int(offset), int(size))) - for varname, splited in list(block_map.items()): + for varname, splited in six.iteritems(block_map): orig_var = program.global_block().var(varname) if len(splited) == 1: if self.sync_mode and add_trainer_suffix: @@ -1107,7 +1123,7 @@ class DistributeTranspiler(object): for i, block in enumerate(splited): size = block[1] - rows = size / orig_dim1_flatten + rows = size // orig_dim1_flatten splited_shape = [rows] if len(orig_shape) >= 2: splited_shape.extend(orig_shape[1:]) @@ -1271,7 +1287,7 @@ class DistributeTranspiler(object): grad_to_block_id, origin_program, merged_var): program = optimize_block.program pserver_block = program.global_block() - new_inputs = dict() + new_inputs = collections.OrderedDict() # update param/grad shape first, then other inputs like # moment can use the updated shape @@ -1357,9 +1373,7 @@ class DistributeTranspiler(object): def _is_splited_grad_var(self, var, var_dict): grad_block = None - # TODO(minqiyang): replace these items() with six.iteritems() to - # improve memory - for _, g in list(var_dict.items()): + for _, g in six.iteritems(var_dict): if self._orig_varname(g.name) == self._orig_varname(var.name): if g.name.find(".trainer_") == -1: grad_block = g @@ -1369,7 +1383,7 @@ class DistributeTranspiler(object): def _clone_lr_op(self, program, block, op): inputs = self._get_input_map_from_op( self.origin_program.global_block().vars, op) - for key, varlist in list(inputs.items()): + for key, varlist in six.iteritems(inputs): if not isinstance(varlist, list): varlist = [varlist] for var in varlist: @@ -1378,7 +1392,7 @@ class DistributeTranspiler(object): outputs = self._get_output_map_from_op( self.origin_program.global_block().vars, op) - for key, varlist in list(outputs.items()): + for key, varlist in six.iteritems(outputs): if not isinstance(varlist, list): varlist = [varlist] for var in varlist: @@ -1393,7 +1407,7 @@ class DistributeTranspiler(object): # Append the ops for parameters that do not need to be optimized/updated inputs = self._get_input_map_from_op( self.origin_program.global_block().vars, opt_op) - for key, varlist in list(inputs.items()): + for key, varlist in six.iteritems(inputs): if not isinstance(varlist, list): varlist = [varlist] for var in varlist: @@ -1412,7 +1426,7 @@ class DistributeTranspiler(object): outputs = self._get_output_map_from_op( self.origin_program.global_block().vars, opt_op) - for key, varlist in list(outputs.items()): + for key, varlist in six.iteritems(outputs): if not isinstance(varlist, list): varlist = [varlist] for var in varlist: @@ -1470,7 +1484,7 @@ class DistributeTranspiler(object): def _get_input_map_from_op(self, varmap, op): """Returns a dict from op input name to the vars in varmap.""" - iomap = dict() + iomap = collections.OrderedDict() for key in op.input_names: vars = [] for varname in op.input(key): @@ -1483,7 +1497,7 @@ class DistributeTranspiler(object): def _get_output_map_from_op(self, varmap, op): """Returns a dict from op output name to the vars in varmap.""" - iomap = dict() + iomap = collections.OrderedDict() for key in op.output_names: vars = [] for varname in op.output(key): diff --git a/python/paddle/fluid/transpiler/inference_transpiler.py b/python/paddle/fluid/transpiler/inference_transpiler.py index 87f20bbccf3138585841952efacef5b0a3cbbace..42005839c44c1b3e8b910e9a3932181468659cae 100644 --- a/python/paddle/fluid/transpiler/inference_transpiler.py +++ b/python/paddle/fluid/transpiler/inference_transpiler.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import os import numpy as np from .. import core diff --git a/python/paddle/fluid/transpiler/memory_optimization_transpiler.py b/python/paddle/fluid/transpiler/memory_optimization_transpiler.py index 20ba7ed2b0b9df0d0432727ee1f69f61533c402e..3e58e125de4188144646236f7999c620cd8e9459 100644 --- a/python/paddle/fluid/transpiler/memory_optimization_transpiler.py +++ b/python/paddle/fluid/transpiler/memory_optimization_transpiler.py @@ -12,8 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + from collections import defaultdict from .. import core +from ... import compat as cpt from ..framework import Program, default_main_program, Parameter from ..backward import _rename_arg_ from functools import reduce @@ -125,15 +128,15 @@ class ControlFlowGraph(object): def _has_var(self, block_desc, var_name, is_forward): if is_forward: - return block_desc.has_var(str(var_name)) + return block_desc.has_var(cpt.to_bytes(var_name)) else: - return block_desc.has_var_recursive(str(var_name)) + return block_desc.has_var_recursive(cpt.to_bytes(var_name)) def _find_var(self, block_desc, var_name, is_forward): if is_forward: - return block_desc.find_var(str(var_name)) + return block_desc.find_var(cpt.to_bytes(var_name)) else: - return block_desc.find_var_recursive(str(var_name)) + return block_desc.find_var_recursive(cpt.to_bytes(var_name)) def _check_var_validity(self, block_desc, x, is_forward): if str(x) == "@EMPTY@": @@ -258,7 +261,7 @@ class ControlFlowGraph(object): # Rename the var to the cache var already with # memory allocated in order to reuse the memory. _rename_arg_(self._ops, x, cache_var, begin_idx=i) - self._program.block(block_desc.id).var(str( + self._program.block(block_desc.id).var(cpt.to_text( x)).desc = self._find_var(block_desc, cache_var, is_forward) self._update_graph(x, cache_var, begin_idx=i) diff --git a/python/paddle/fluid/transpiler/ps_dispatcher.py b/python/paddle/fluid/transpiler/ps_dispatcher.py index dcffadd531719431f27feb464ed58a65c04770ee..6a6d14a69ba771e192a28951a6df7027741a655a 100644 --- a/python/paddle/fluid/transpiler/ps_dispatcher.py +++ b/python/paddle/fluid/transpiler/ps_dispatcher.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + class PSDispatcher(object): """ diff --git a/python/paddle/fluid/unique_name.py b/python/paddle/fluid/unique_name.py index b125eba4f83c588fa2fa81a357604a7d8592ea80..b9957a699e597898bee75ce0e7283f7224293f0c 100644 --- a/python/paddle/fluid/unique_name.py +++ b/python/paddle/fluid/unique_name.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import collections import contextlib import six diff --git a/python/paddle/reader/decorator.py b/python/paddle/reader/decorator.py index ce410e61b92e7d3f32fa5bfeb415e4b6c5fa9df6..6d7ac876fdf65fe0e85cceb94c311a93d9ea39c2 100644 --- a/python/paddle/reader/decorator.py +++ b/python/paddle/reader/decorator.py @@ -27,6 +27,7 @@ from six.moves import zip import itertools import random import zlib +import paddle.compat as cpt def map_readers(func, *readers): @@ -390,9 +391,9 @@ class PipeReader: buff = self.process.stdout.read(self.bufsize) if buff: if self.file_type == "gzip": - decomp_buff = self.dec.decompress(buff) + decomp_buff = cpt.to_text(self.dec.decompress(buff)) elif self.file_type == "plain": - decomp_buff = buff + decomp_buff = cpt.to_text(buff) else: raise TypeError("file_type %s is not allowed" % self.file_type) diff --git a/python/paddle/reader/tests/creator_test.py b/python/paddle/reader/tests/creator_test.py index c4238c12a74759d52eb09f31ce1126cc93dd3489..d7107610a5dd751cad8f8365aec32c6ba92c53ae 100644 --- a/python/paddle/reader/tests/creator_test.py +++ b/python/paddle/reader/tests/creator_test.py @@ -29,6 +29,7 @@ import os import unittest import numpy as np import paddle.reader.creator +import six class TestNumpyArray(unittest.TestCase): @@ -37,7 +38,7 @@ class TestNumpyArray(unittest.TestCase): x = np.array(l, np.int32) reader = paddle.reader.creator.np_array(x) for idx, e in enumerate(reader()): - self.assertItemsEqual(e, l[idx]) + six.assertCountEqual(self, e, l[idx]) class TestTextFile(unittest.TestCase): diff --git a/python/paddle/utils/dump_config.py b/python/paddle/utils/dump_config.py index d27af7f76246a4c9db9a43c17715506d82031b9c..6a96a0a78fc77c50904ee7822c725c41e646c5e6 100644 --- a/python/paddle/utils/dump_config.py +++ b/python/paddle/utils/dump_config.py @@ -37,9 +37,9 @@ if __name__ == '__main__': assert isinstance(conf, TrainerConfig_pb2.TrainerConfig) if whole_conf: - print conf + print(conf) else: if binary: sys.stdout.write(conf.model_config.SerializeToString()) else: - print conf.model_config + print(conf.model_config) diff --git a/python/paddle/utils/image_multiproc.py b/python/paddle/utils/image_multiproc.py index 3e3e519f76d388eeb477f0014bcbb3e7cd09352a..d1bbda3fd3562efe486377d41a9fb7359bafa4e7 100644 --- a/python/paddle/utils/image_multiproc.py +++ b/python/paddle/utils/image_multiproc.py @@ -15,7 +15,8 @@ import os, sys import numpy as np from PIL import Image -from cStringIO import StringIO +import six +from six.moves import cStringIO as StringIO import multiprocessing import functools import itertools @@ -187,7 +188,8 @@ class PILTransformer(ImageTransformer): return self.transform(im) -def job(is_img_string, transformer, (data, label)): +def job(is_img_string, transformer, data_label_pack): + (data, label) = data_label_pack if is_img_string: return transformer.transform_from_string(data), label else: @@ -208,7 +210,7 @@ class MultiProcessImageTransformer(object): """ Processing image with multi-process. If it is used in PyDataProvider, the simple usage for CNN is as follows: - + .. code-block:: python def hool(settings, is_train, **kwargs): @@ -229,7 +231,7 @@ class MultiProcessImageTransformer(object): @provider(init_hook=hook, pool_size=20480) def process(settings, file_list): with open(file_list, 'r') as fdata: - for line in fdata: + for line in fdata: data_dic = np.load(line.strip()) # load the data batch pickled by Pickle. data = data_dic['data'] labels = data_dic['label'] @@ -249,10 +251,10 @@ class MultiProcessImageTransformer(object): :type channel_swap: tuple or list :param mean: the mean values of image, per-channel mean or element-wise mean. :type mean: array, The dimension is 1 for per-channel mean. - The dimension is 3 for element-wise mean. + The dimension is 3 for element-wise mean. :param is_train: training peroid or testing peroid. :type is_train: bool. - :param is_color: the image is color or gray. + :param is_color: the image is color or gray. :type is_color: bool. :param is_img_string: The input can be the file name of image or image string. :type is_img_string: bool. @@ -273,4 +275,4 @@ class MultiProcessImageTransformer(object): def run(self, data, label): fun = functools.partial(job, self.is_img_string, self.transformer) return self.pool.imap_unordered( - fun, itertools.izip(data, label), chunksize=100 * self.procnum) + fun, six.moves.zip(data, label), chunksize=100 * self.procnum) diff --git a/python/paddle/utils/image_util.py b/python/paddle/utils/image_util.py index d3d79b14405256bbc95c41d805dbee56cb104f5e..a8092349cde8a4cb30873bf819fd5ed96289a945 100644 --- a/python/paddle/utils/image_util.py +++ b/python/paddle/utils/image_util.py @@ -14,7 +14,7 @@ import numpy as np from PIL import Image -from cStringIO import StringIO +from six.moves import cStringIO as StringIO def resize_image(img, target_size): @@ -34,7 +34,7 @@ def flip(im): """ Return the flipped image. Flip an image along the horizontal direction. - im: input image, (H x W x K) ndarrays + im: input image, (H x W x K) ndarrays """ if len(im.shape) == 3: return im[:, :, ::-1] @@ -132,7 +132,7 @@ def load_meta(meta_path, mean_img_size, crop_size, color=True): def load_image(img_path, is_color=True): """ - Load image and return. + Load image and return. img_path: image path. is_color: is color image or not. """ @@ -205,7 +205,7 @@ class ImageTransformer: def set_mean(self, mean): if mean is not None: - # mean value, may be one value per channel + # mean value, may be one value per channel if mean.ndim == 1: mean = mean[:, np.newaxis, np.newaxis] else: diff --git a/python/paddle/utils/make_model_diagram.py b/python/paddle/utils/make_model_diagram.py index 40f99075de7fb2401b3b704afe1eb44dbe6072dd..52759d3ad230c3a5a5488a8bc46a2e8f8fae1025 100644 --- a/python/paddle/utils/make_model_diagram.py +++ b/python/paddle/utils/make_model_diagram.py @@ -15,6 +15,9 @@ # Generate dot diagram file for the given paddle model config # The generated file can be viewed using Graphviz (http://graphviz.org) +from __future__ import print_function + +import six import sys import traceback @@ -61,9 +64,9 @@ def make_diagram_from_proto(model_config, dot_file): name2id[mem.link_name]) return s - print >> f, 'digraph graphname {' - print >> f, 'node [width=0.375,height=0.25];' - for i in xrange(len(model_config.layers)): + print('digraph graphname {', file=f) + print('node [width=0.375,height=0.25];', file=f) + for i in six.moves.xrange(len(model_config.layers)): l = model_config.layers[i] name2id[l.name] = i @@ -71,12 +74,12 @@ def make_diagram_from_proto(model_config, dot_file): for sub_model in model_config.sub_models: if sub_model.name == 'root': continue - print >> f, 'subgraph cluster_%s {' % i - print >> f, 'style=dashed;' + print('subgraph cluster_%s {' % i, file=f) + print('style=dashed;', file=f) label = '%s ' % sub_model.name if sub_model.reversed: label += '<==' - print >> f, 'label = "%s";' % label + print('label = "%s";' % label, file=f) i += 1 submodel_layers.add(sub_model.name) for layer_name in sub_model.layer_names: @@ -84,37 +87,41 @@ def make_diagram_from_proto(model_config, dot_file): lid = name2id[layer_name] layer_config = model_config.layers[lid] label = make_layer_label(layer_config) - print >> f, 'l%s [label="%s", shape=box];' % (lid, label) - print >> f, '}' + print('l%s [label="%s", shape=box];' % (lid, label), file=f) + print('}', file=f) - for i in xrange(len(model_config.layers)): + for i in six.moves.xrange(len(model_config.layers)): l = model_config.layers[i] if l.name not in submodel_layers: label = make_layer_label(l) - print >> f, 'l%s [label="%s", shape=box];' % (i, label) + print('l%s [label="%s", shape=box];' % (i, label), file=f) for sub_model in model_config.sub_models: if sub_model.name == 'root': continue for link in sub_model.in_links: - print >> f, make_link(link) + print(make_link(link), file=f) for link in sub_model.out_links: - print >> f, make_link(link) + print(make_link(link), file=f) for mem in sub_model.memories: - print >> f, make_mem(mem) + print(make_mem(mem), file=f) - for i in xrange(len(model_config.layers)): + for i in six.moves.xrange(len(model_config.layers)): for l in model_config.layers[i].inputs: - print >> f, 'l%s -> l%s [label="%s"];' % ( - name2id[l.input_layer_name], i, l.input_parameter_name) + print( + 'l%s -> l%s [label="%s"];' % (name2id[l.input_layer_name], i, + l.input_parameter_name), + file=f) - print >> f, '}' + print('}', file=f) f.close() def usage(): - print >> sys.stderr, ("Usage: python show_model_diagram.py" + - " CONFIG_FILE DOT_FILE [config_str]") + print( + ("Usage: python show_model_diagram.py" + + " CONFIG_FILE DOT_FILE [config_str]"), + file=sys.stderr) exit(1) diff --git a/python/paddle/utils/merge_model.py b/python/paddle/utils/merge_model.py index 2b100207728a8532e900992f7db4d3910e893dea..b74649e93640c3600636034d58792b8d12dffeda 100644 --- a/python/paddle/utils/merge_model.py +++ b/python/paddle/utils/merge_model.py @@ -70,4 +70,4 @@ def merge_v2_model(net, param_file, output_file): for pname in param_names: params.serialize(pname, f) - print 'Generate %s success!' % (output_file) + print('Generate %s success!' % (output_file)) diff --git a/python/paddle/utils/plotcurve.py b/python/paddle/utils/plotcurve.py index 27bd8157d39632913e2fa3278f3af20ddea61da7..a95e5497e23571e61e5d7652830a99efd7793083 100644 --- a/python/paddle/utils/plotcurve.py +++ b/python/paddle/utils/plotcurve.py @@ -44,6 +44,7 @@ To use this script to generate plot for AvgCost, error: python plotcurve.py -i paddle.INFO -o figure.png AvgCost error """ +import six import sys import matplotlib # the following line is added immediately after import matplotlib @@ -91,7 +92,7 @@ def plot_paddle_curve(keys, inputfile, outputfile, format='png', sys.stderr.write("No data to plot. Exiting!\n") return m = len(keys) + 1 - for i in xrange(1, m): + for i in six.moves.xrange(1, m): pyplot.plot( x[:, 0], x[:, i], diff --git a/python/paddle/utils/predefined_net.py b/python/paddle/utils/predefined_net.py index fa05f981f2b66bf55303a6f7c332c0bc9b112d29..2801f4877c079615239b92be146b3e33df16b37f 100644 --- a/python/paddle/utils/predefined_net.py +++ b/python/paddle/utils/predefined_net.py @@ -13,6 +13,7 @@ # limitations under the License. import numpy as np +import six import os from paddle.trainer.config_parser import * from paddle.utils.preprocess_img import \ @@ -112,7 +113,7 @@ def simple_conv_net(data_conf, is_color=False): num_classes: num of classes. is_color: whether the input images are color. """ - for k, v in data_conf.iteritems(): + for k, v in six.iteritems(data_conf): globals()[k] = v data_input, label_input, num_image_channels = \ image_data_layers(image_size, num_classes, is_color, is_predict) @@ -340,7 +341,7 @@ def small_vgg(data_conf, is_predict=False): num_classes: num of classes. is_color: whether the input images are color. """ - for k, v in data_conf.iteritems(): + for k, v in six.iteritems(data_conf): globals()[k] = v vgg_conv_net(image_size, num_classes, num_layers=[2, 2, 3, 3], diff --git a/python/paddle/utils/preprocess_img.py b/python/paddle/utils/preprocess_img.py index 975f1e9edea161331d37afbc6b5af46286f185bf..a322f7b769a2a32df516a4b8ea04289a7f882ff2 100644 --- a/python/paddle/utils/preprocess_img.py +++ b/python/paddle/utils/preprocess_img.py @@ -17,9 +17,9 @@ import os import random import numpy as np import PIL.Image as Image -import StringIO -import preprocess_util -from image_util import crop_img +from six.moves import cStringIO as StringIO +from . import preprocess_util +from .image_util import crop_img def resize_image(img, target_size): @@ -52,7 +52,7 @@ class DiskImage: def read_image(self): if self.img is None: - print "reading: " + self.path + print("reading: " + self.path) image = resize_image(Image.open(self.path), self.target_size) self.img = image @@ -69,7 +69,7 @@ class DiskImage: convert the image into the paddle batch format. """ self.read_image() - output = StringIO.StringIO() + output = StringIO() self.img.save(output, "jpeg") contents = output.getvalue() return contents @@ -127,7 +127,7 @@ class ImageClassificationDatasetCreater(preprocess_util.DatasetCreater): image_path = items[0] label_name = items[1] if not label_name in label_set: - label_set[label_name] = len(label_set.keys()) + label_set[label_name] = len(list(label_set.keys())) img = DiskImage(path=image_path, target_size=self.target_size) label = preprocess_util.Lablel( label=label_set[label_name], name=label_name) @@ -144,7 +144,7 @@ class ImageClassificationDatasetCreater(preprocess_util.DatasetCreater): return create_dataset_from_list(path) label_set = preprocess_util.get_label_set_from_dir(path) data = [] - for l_name in label_set.keys(): + for l_name in list(label_set.keys()): image_paths = preprocess_util.list_images( os.path.join(path, l_name)) for p in image_paths: diff --git a/python/paddle/utils/preprocess_util.py b/python/paddle/utils/preprocess_util.py index 1d17a488243eb81e46bea3ead686efd021499e22..05b2067d01a2c544d7f5bd68320e79c805282286 100644 --- a/python/paddle/utils/preprocess_util.py +++ b/python/paddle/utils/preprocess_util.py @@ -14,7 +14,7 @@ import os import math -import cPickle as pickle +import six.moves.cPickle as pickle import random import collections @@ -169,7 +169,7 @@ class Dataset: random.shuffle(keyvalue_indices[k]) num_data_per_key_batch = \ - math.ceil(num_per_batch / float(len(keyvalue_indices.keys()))) + math.ceil(num_per_batch / float(len(list(keyvalue_indices.keys())))) if num_data_per_key_batch < 2: raise Exception("The number of data in a batch is too small") @@ -182,8 +182,8 @@ class Dataset: end_idx = int( min(begin_idx + num_data_per_key_batch, len(keyvalue_indices[k]))) - print "begin_idx, end_idx" - print begin_idx, end_idx + print("begin_idx, end_idx") + print(begin_idx, end_idx) for idx in range(begin_idx, end_idx): permuted_data.append(self.data[keyvalue_indices[k][idx]]) keyvalue_readpointer[k] = end_idx @@ -357,6 +357,6 @@ class DatasetCreater(object): data_batcher.create_batches_and_list( self.output_path, self.train_list_name, self.test_list_name, self.label_set_name) - self.num_classes = len(train_label_set.keys()) + self.num_classes = len(list(train_label_set.keys())) self.create_meta_file(train_data) return out_path diff --git a/python/paddle/utils/show_pb.py b/python/paddle/utils/show_pb.py index 20614826d1d01f50a2bb54a840e2c584fb93b247..da7a71a665aea4d93d366e8508f438a9aba88e94 100644 --- a/python/paddle/utils/show_pb.py +++ b/python/paddle/utils/show_pb.py @@ -15,6 +15,8 @@ Show the content of proto buffer data file of PADDLE """ +from __future__ import print_function + import os import sys from google.protobuf.internal.decoder import _DecodeVarint @@ -39,7 +41,7 @@ def read_proto(file, message): def usage(): - print >> sys.stderr, "Usage: python show_pb.py PROTO_DATA_FILE" + print("Usage: python show_pb.py PROTO_DATA_FILE", file=sys.stderr) exit(1) @@ -50,8 +52,8 @@ if __name__ == '__main__': f = open(sys.argv[1]) header = DataFormat.DataHeader() read_proto(f, header) - print header + print(header) sample = DataFormat.DataSample() while read_proto(f, sample): - print sample + print(sample) diff --git a/python/paddle/utils/torch2paddle.py b/python/paddle/utils/torch2paddle.py index 91490111a1144ae25ed6566ff1c83db4f7954d33..398d3aa4e02cc74b7885f7e676937d7fd254bc5e 100644 --- a/python/paddle/utils/torch2paddle.py +++ b/python/paddle/utils/torch2paddle.py @@ -24,7 +24,7 @@ import sys import struct import numpy as np import torchfile -import cPickle as pickle +import six.moves.cPickle as pickle import argparse @@ -48,7 +48,7 @@ def save_net_parameters(layers, params, output_path): biases = params[i * 2 + 1] weight_file = os.path.join(output_path, '_%s.w0' % layers[i]) biases_file = os.path.join(output_path, '_%s.wbias' % layers[i]) - print "Saving for layer %s." % layers[i] + print("Saving for layer %s." % layers[i]) save_layer_parameters(weight_file, [weight]) save_layer_parameters(biases_file, biases) diff --git a/tools/manylinux1/build_scripts/build.sh b/tools/manylinux1/build_scripts/build.sh index 93591fa9ddad8a78df344e1e912a5f1c7e93dfa4..eb4b477dcb538f7ba17cfc54057a97c9669a6916 100644 --- a/tools/manylinux1/build_scripts/build.sh +++ b/tools/manylinux1/build_scripts/build.sh @@ -28,7 +28,7 @@ AUTOCONF_HASH=954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969 PYTHON_COMPILE_DEPS="zlib-devel bzip2-devel ncurses-devel sqlite-devel readline-devel tk-devel gdbm-devel db4-devel libpcap-devel xz-devel" # Libraries that are allowed as part of the manylinux1 profile -MANYLINUX1_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel ncurses-devel" +MANYLINUX1_DEPS="glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel ncurses-devel freetype-devel libpng-devel" # Get build utilities MY_DIR=$(dirname "${BASH_SOURCE[0]}") @@ -105,7 +105,7 @@ curl-config --features rm -rf /usr/local/ssl # Install patchelf (latest with unreleased bug fixes) -curl -sLO https://nipy.bic.berkeley.edu/manylinux/patchelf-0.9njs2.tar.gz +curl -sLO http://nipy.bic.berkeley.edu/manylinux/patchelf-0.9njs2.tar.gz check_sha256sum patchelf-0.9njs2.tar.gz $PATCHELF_HASH tar -xzf patchelf-0.9njs2.tar.gz (cd patchelf-0.9njs2 && ./configure && make && make install) diff --git a/tools/manylinux1/build_scripts/install_nccl2.sh b/tools/manylinux1/build_scripts/install_nccl2.sh index 282c5c290da14bd3c04346ab01fdb48423c23f88..43a99d8287bbaa13ff75d9f25972a6335ae0754a 100644 --- a/tools/manylinux1/build_scripts/install_nccl2.sh +++ b/tools/manylinux1/build_scripts/install_nccl2.sh @@ -21,5 +21,5 @@ for sub_deb in $DEBS; do ar x $sub_deb && tar xf data.tar.xz done mv -f usr/include/nccl.h /usr/local/include/ -mv -f usr/lib/libnccl* /usr/local/lib/ +mv -f usr/lib/x86_64-linux-gnu/libnccl* /usr/local/lib/ rm -rf $DIR diff --git a/tools/test_runner.py b/tools/test_runner.py index 2d6a3cf8a97a3bbaa69b66f5343c54b750624329..9b9f165e7368364bbb0a78d6dcbbe4be0d6bf98b 100644 --- a/tools/test_runner.py +++ b/tools/test_runner.py @@ -13,6 +13,7 @@ # limitations under the License. from __future__ import print_function + import unittest import os import sys