diff --git a/paddle/fluid/pybind/CMakeLists.txt b/paddle/fluid/pybind/CMakeLists.txt index ada69ea4a425f70dc085ad9046bb6b930136803d..787925d9f8800b49de5b8b642304605ef4087d1e 100644 --- a/paddle/fluid/pybind/CMakeLists.txt +++ b/paddle/fluid/pybind/CMakeLists.txt @@ -15,4 +15,6 @@ if(WITH_PYTHON) target_link_libraries(paddle_pybind rt) endif(NOT APPLE AND NOT ANDROID) endif(WITH_AMD_GPU) + + cc_test(tensor_py_test SRCS tensor_py_test.cc DEPS python) endif(WITH_PYTHON) diff --git a/paddle/fluid/pybind/const_value.cc b/paddle/fluid/pybind/const_value.cc index 6657b25ed2443c1ac9cb0a09098968d3181fc6ba..3f28e616494ad1322708ad6403aaf50b22d724e6 100644 --- a/paddle/fluid/pybind/const_value.cc +++ b/paddle/fluid/pybind/const_value.cc @@ -12,17 +12,17 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#include "const_value.h" +#include "paddle/fluid/pybind/const_value.h" #include "paddle/fluid/framework/operator.h" namespace paddle { namespace pybind { -void BindConstValue(pybind11::module& m) { - m.def("kEmptyVarName", [] { return framework::kEmptyVarName; }); - m.def("kTempVarName", [] { return framework::kTempVarName; }); - m.def("kGradVarSuffix", [] { return framework::kGradVarSuffix; }); - m.def("kZeroVarSuffix", [] { return framework::kZeroVarSuffix; }); +void BindConstValue(pybind11::module* m) { + m->def("kEmptyVarName", [] { return framework::kEmptyVarName; }); + m->def("kTempVarName", [] { return framework::kTempVarName; }); + m->def("kGradVarSuffix", [] { return framework::kGradVarSuffix; }); + m->def("kZeroVarSuffix", [] { return framework::kZeroVarSuffix; }); } } // namespace pybind diff --git a/paddle/fluid/pybind/const_value.h b/paddle/fluid/pybind/const_value.h index 79e71e039dea6585aaf8193f1417c6ab3fbf6f76..2fab3160d1d95af7f6a49c472c2e211c19e67cac 100644 --- a/paddle/fluid/pybind/const_value.h +++ b/paddle/fluid/pybind/const_value.h @@ -11,16 +11,17 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #pragma once + #include + #include "paddle/fluid/platform/enforce.h" #include "pybind11/pybind11.h" -namespace py = pybind11; - namespace paddle { namespace pybind { -extern void BindConstValue(pybind11::module& m); + +void BindConstValue(pybind11::module* m); + } // namespace pybind } // namespace paddle diff --git a/paddle/fluid/pybind/exception.cc b/paddle/fluid/pybind/exception.cc index 4bd3ecf728dedaf74a554f77b114065f2d515786..08a2f185e117718d07ba984f76dfe5bf8229c33c 100644 --- a/paddle/fluid/pybind/exception.cc +++ b/paddle/fluid/pybind/exception.cc @@ -17,8 +17,8 @@ limitations under the License. */ namespace paddle { namespace pybind { -void BindException(pybind11::module& m) { - static pybind11::exception exc(m, "EnforceNotMet"); +void BindException(pybind11::module* m) { + static pybind11::exception exc(*m, "EnforceNotMet"); pybind11::register_exception_translator([](std::exception_ptr p) { try { if (p) std::rethrow_exception(p); @@ -27,7 +27,8 @@ void BindException(pybind11::module& m) { } }); - m.def("__unittest_throw_exception__", [] { PADDLE_THROW("test exception"); }); + m->def("__unittest_throw_exception__", + [] { PADDLE_THROW("test exception"); }); } } // namespace pybind diff --git a/paddle/fluid/pybind/exception.h b/paddle/fluid/pybind/exception.h index bc6b0c067978959d4cdafec51db9574927b34b21..5e054267361f2c62b3ad36581be0ad17ce0718de 100644 --- a/paddle/fluid/pybind/exception.h +++ b/paddle/fluid/pybind/exception.h @@ -11,14 +11,17 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #pragma once + #include + #include "paddle/fluid/platform/enforce.h" #include "pybind11/pybind11.h" + namespace paddle { namespace pybind { -extern void BindException(pybind11::module& m); +void BindException(pybind11::module* m); + } // namespace pybind } // namespace paddle diff --git a/paddle/fluid/pybind/protobuf.cc b/paddle/fluid/pybind/protobuf.cc index 985984983a2239f6961bf519bae27fcbb9e7d6d3..2fe829036386086075a7f6ad0b9348a9e8c5e85a 100644 --- a/paddle/fluid/pybind/protobuf.cc +++ b/paddle/fluid/pybind/protobuf.cc @@ -11,12 +11,13 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #include "paddle/fluid/pybind/protobuf.h" + #include #include #include #include + #include "paddle/fluid/framework/backward.h" #include "paddle/fluid/framework/block_desc.h" #include "paddle/fluid/framework/op_desc.h" @@ -97,10 +98,11 @@ struct type_caster> namespace paddle { namespace pybind { -using namespace paddle::framework; // NOLINT +namespace pd = paddle::framework; template -static py::bytes SerializeMessage(T &self) { // NOLINT +static pybind11::bytes SerializeMessage( + T &self) { // NOLINT due to pybind11 convention. // Check IsInitialized in Python std::string retv; PADDLE_ENFORCE(self.Proto()->SerializePartialToString(&retv), @@ -109,24 +111,24 @@ static py::bytes SerializeMessage(T &self) { // NOLINT } // Bind Methods -void BindProgramDesc(py::module &m) { // NOLINT - py::class_(m, "ProgramDesc", "") - .def(py::init<>()) +void BindProgramDesc(pybind11::module *m) { + pybind11::class_(*m, "ProgramDesc", "") + .def(pybind11::init<>()) .def("__init__", - [](ProgramDesc &self, const ProgramDesc &other) { - new (&self) ProgramDesc(other); + [](pd::ProgramDesc &self, const pd::ProgramDesc &other) { + new (&self) pd::ProgramDesc(other); }) .def("__init__", - [](ProgramDesc &self, const py::bytes &binary_str) { + [](pd::ProgramDesc &self, const pybind11::bytes &binary_str) { std::string str(binary_str); - new (&self) ProgramDesc(str); + new (&self) pd::ProgramDesc(str); }) - .def("append_block", &ProgramDesc::AppendBlock, - py::return_value_policy::reference) + .def("append_block", &pd::ProgramDesc::AppendBlock, + pybind11::return_value_policy::reference) .def("append_backward", - [](ProgramDesc &program_desc, const VarDesc &target, + [](pd::ProgramDesc &program_desc, const pd::VarDesc &target, const std::unordered_set &no_grad_vars) { - ParamGradInfoMap param_grad_map = + pd::ParamGradInfoMap param_grad_map = AppendBackward(program_desc, target, no_grad_vars); std::unordered_map< std::string, std::tuple) + .def("block", &pd::ProgramDesc::MutableBlock, + pybind11::return_value_policy::reference) + .def("num_blocks", &pd::ProgramDesc::Size) + .def("serialize_to_string", SerializeMessage) .def("parse_from_string", - [](ProgramDesc &program_desc, const std::string &data) { - proto::ProgramDesc *desc = program_desc.Proto(); + [](pd::ProgramDesc &program_desc, const std::string &data) { + pd::proto::ProgramDesc *desc = program_desc.Proto(); PADDLE_ENFORCE(desc->ParseFromString(data), "Fail to parse ProgramDesc from string. This could " "be a bug of Paddle."); }); } -void BindBlockDesc(py::module &m) { // NOLINT - py::class_(m, "BlockDesc", "") - .def_property_readonly("id", &BlockDesc::ID) - .def_property_readonly("parent", &BlockDesc::Parent) - .def("get_forward_block_idx", &BlockDesc::ForwardBlockID) - .def("set_forward_block_idx", &BlockDesc::SetForwardBlockID) - .def("append_op", &BlockDesc::AppendOp, - py::return_value_policy::reference) - .def("prepend_op", &BlockDesc::PrependOp, - py::return_value_policy::reference) - .def("insert_op", &BlockDesc::InsertOp, - py::return_value_policy::reference) - .def("remove_op", &BlockDesc::RemoveOp) +void BindBlockDesc(pybind11::module *m) { + pybind11::class_(*m, "BlockDesc", "") + .def_property_readonly("id", &pd::BlockDesc::ID) + .def_property_readonly("parent", &pd::BlockDesc::Parent) + .def("get_forward_block_idx", &pd::BlockDesc::ForwardBlockID) + .def("set_forward_block_idx", &pd::BlockDesc::SetForwardBlockID) + .def("append_op", &pd::BlockDesc::AppendOp, + pybind11::return_value_policy::reference) + .def("prepend_op", &pd::BlockDesc::PrependOp, + pybind11::return_value_policy::reference) + .def("insert_op", &pd::BlockDesc::InsertOp, + pybind11::return_value_policy::reference) + .def("remove_op", &pd::BlockDesc::RemoveOp) .def("var", - [](BlockDesc &self, py::bytes byte_name) { + [](pd::BlockDesc &self, pybind11::bytes byte_name) { std::string name = byte_name; return self.Var(name); }, - py::return_value_policy::reference) + pybind11::return_value_policy::reference) .def("has_var", - [](BlockDesc &self, py::bytes byte_name) { + [](pd::BlockDesc &self, pybind11::bytes byte_name) { std::string name = byte_name; return self.HasVar(name); }, - py::return_value_policy::reference) + pybind11::return_value_policy::reference) .def("rename_var", - [](BlockDesc &self, const py::bytes &byte_name, - const py::bytes &byte_name_new) { + [](pd::BlockDesc &self, const pybind11::bytes &byte_name, + const pybind11::bytes &byte_name_new) { std::string name = byte_name; std::string new_name = byte_name_new; self.RenameVar(name, new_name); }) .def("has_var_recursive", - [](BlockDesc &self, py::bytes byte_name) { + [](pd::BlockDesc &self, pybind11::bytes byte_name) { std::string name = byte_name; return self.HasVarRecursive(name); }) .def("find_var", - [](BlockDesc &self, py::bytes byte_name) { + [](pd::BlockDesc &self, pybind11::bytes byte_name) { std::string name = byte_name; return self.FindVar(name); }, - py::return_value_policy::reference) + pybind11::return_value_policy::reference) .def("find_var_recursive", - [](BlockDesc &self, py::bytes byte_name) { + [](pd::BlockDesc &self, pybind11::bytes byte_name) { std::string name = byte_name; return self.FindVarRecursive(name); }, - py::return_value_policy::reference) + pybind11::return_value_policy::reference) .def("remove_var", - [](BlockDesc &self, py::bytes byte_name) { + [](pd::BlockDesc &self, pybind11::bytes byte_name) { std::string name = byte_name; return self.RemoveVar(name); }, - py::return_value_policy::reference) - .def("all_vars", &BlockDesc::AllVars, py::return_value_policy::reference) - .def("op_size", &BlockDesc::OpSize) - .def("op", &BlockDesc::Op, py::return_value_policy::reference) - .def("serialize_to_string", SerializeMessage); + pybind11::return_value_policy::reference) + .def("all_vars", &pd::BlockDesc::AllVars, + pybind11::return_value_policy::reference) + .def("op_size", &pd::BlockDesc::OpSize) + .def("op", &pd::BlockDesc::Op, pybind11::return_value_policy::reference) + .def("serialize_to_string", SerializeMessage); } -void BindVarDsec(py::module &m) { // NOLINT - py::class_ var_desc(m, "VarDesc", ""); +void BindVarDsec(pybind11::module *m) { + pybind11::class_ var_desc(*m, "VarDesc", ""); var_desc .def("name", - [](VarDesc &self) { - py::bytes name = self.Name(); + [](pd::VarDesc &self) { + pybind11::bytes name = self.Name(); return name; }, - py::return_value_policy::reference) - .def("set_name", &VarDesc::SetName) - .def("set_shape", &VarDesc::SetShape) - .def("set_shapes", &VarDesc::SetShapes) - .def("set_dtype", &VarDesc::SetDataType) - .def("set_dtypes", &VarDesc::SetDataTypes) - .def("set_capacity", &VarDesc::SetCapacity) - .def("shape", &VarDesc::GetShape, py::return_value_policy::reference) - .def("shapes", &VarDesc::GetShapes, py::return_value_policy::reference) - .def("dtype", &VarDesc::GetDataType, py::return_value_policy::reference) - .def("dtypes", &VarDesc::GetDataTypes, py::return_value_policy::reference) - .def("lod_level", &VarDesc::GetLoDLevel) - .def("lod_levels", &VarDesc::GetLoDLevels, - py::return_value_policy::reference) - .def("set_lod_level", &VarDesc::SetLoDLevel) - .def("set_lod_levels", &VarDesc::SetLoDLevels) - .def("type", &VarDesc::GetType) - .def("set_type", &VarDesc::SetType) - .def("serialize_to_string", SerializeMessage) - .def("persistable", &VarDesc::Persistable) - .def("set_persistable", &VarDesc::SetPersistable); + pybind11::return_value_policy::reference) + .def("set_name", &pd::VarDesc::SetName) + .def("set_shape", &pd::VarDesc::SetShape) + .def("set_shapes", &pd::VarDesc::SetShapes) + .def("set_dtype", &pd::VarDesc::SetDataType) + .def("set_dtypes", &pd::VarDesc::SetDataTypes) + .def("set_capacity", &pd::VarDesc::SetCapacity) + .def("shape", &pd::VarDesc::GetShape, + pybind11::return_value_policy::reference) + .def("shapes", &pd::VarDesc::GetShapes, + pybind11::return_value_policy::reference) + .def("dtype", &pd::VarDesc::GetDataType, + pybind11::return_value_policy::reference) + .def("dtypes", &pd::VarDesc::GetDataTypes, + pybind11::return_value_policy::reference) + .def("lod_level", &pd::VarDesc::GetLoDLevel) + .def("lod_levels", &pd::VarDesc::GetLoDLevels, + pybind11::return_value_policy::reference) + .def("set_lod_level", &pd::VarDesc::SetLoDLevel) + .def("set_lod_levels", &pd::VarDesc::SetLoDLevels) + .def("type", &pd::VarDesc::GetType) + .def("set_type", &pd::VarDesc::SetType) + .def("serialize_to_string", SerializeMessage) + .def("persistable", &pd::VarDesc::Persistable) + .def("set_persistable", &pd::VarDesc::SetPersistable); - py::enum_(var_desc, "VarType", "") - .value("BOOL", proto::VarType::BOOL) - .value("INT16", proto::VarType::INT16) - .value("INT32", proto::VarType::INT32) - .value("INT64", proto::VarType::INT64) - .value("FP16", proto::VarType::FP16) - .value("FP32", proto::VarType::FP32) - .value("FP64", proto::VarType::FP64) - .value("LOD_TENSOR", proto::VarType::LOD_TENSOR) - .value("SELECTED_ROWS", proto::VarType::SELECTED_ROWS) - .value("FEED_MINIBATCH", proto::VarType::FEED_MINIBATCH) - .value("FETCH_LIST", proto::VarType::FETCH_LIST) - .value("STEP_SCOPES", proto::VarType::STEP_SCOPES) - .value("LOD_RANK_TABLE", proto::VarType::LOD_RANK_TABLE) - .value("LOD_TENSOR_ARRAY", proto::VarType::LOD_TENSOR_ARRAY) - .value("CHANNEL", proto::VarType::CHANNEL) - .value("PLACE_LIST", proto::VarType::PLACE_LIST) - .value("READER", proto::VarType::READER) - .value("RAW", proto::VarType::RAW); + pybind11::enum_(var_desc, "VarType", "") + .value("BOOL", pd::proto::VarType::BOOL) + .value("INT16", pd::proto::VarType::INT16) + .value("INT32", pd::proto::VarType::INT32) + .value("INT64", pd::proto::VarType::INT64) + .value("FP16", pd::proto::VarType::FP16) + .value("FP32", pd::proto::VarType::FP32) + .value("FP64", pd::proto::VarType::FP64) + .value("LOD_TENSOR", pd::proto::VarType::LOD_TENSOR) + .value("SELECTED_ROWS", pd::proto::VarType::SELECTED_ROWS) + .value("FEED_MINIBATCH", pd::proto::VarType::FEED_MINIBATCH) + .value("FETCH_LIST", pd::proto::VarType::FETCH_LIST) + .value("STEP_SCOPES", pd::proto::VarType::STEP_SCOPES) + .value("LOD_RANK_TABLE", pd::proto::VarType::LOD_RANK_TABLE) + .value("LOD_TENSOR_ARRAY", pd::proto::VarType::LOD_TENSOR_ARRAY) + .value("CHANNEL", pd::proto::VarType::CHANNEL) + .value("PLACE_LIST", pd::proto::VarType::PLACE_LIST) + .value("READER", pd::proto::VarType::READER) + .value("RAW", pd::proto::VarType::RAW); } -void BindOpDesc(py::module &m) { // NOLINT - py::enum_(m, "AttrType", "") - .value("INT", proto::AttrType::INT) - .value("INTS", proto::AttrType::INTS) - .value("FLOAT", proto::AttrType::FLOAT) - .value("FLOATS", proto::AttrType::FLOATS) - .value("STRING", proto::AttrType::STRING) - .value("STRINGS", proto::AttrType::STRINGS) - .value("BOOL", proto::AttrType::BOOLEAN) - .value("BOOLS", proto::AttrType::BOOLEANS) - .value("BLOCK", proto::AttrType::BLOCK); +void BindOpDesc(pybind11::module *m) { + pybind11::enum_(*m, "AttrType", "") + .value("INT", pd::proto::AttrType::INT) + .value("INTS", pd::proto::AttrType::INTS) + .value("FLOAT", pd::proto::AttrType::FLOAT) + .value("FLOATS", pd::proto::AttrType::FLOATS) + .value("STRING", pd::proto::AttrType::STRING) + .value("STRINGS", pd::proto::AttrType::STRINGS) + .value("BOOL", pd::proto::AttrType::BOOLEAN) + .value("BOOLS", pd::proto::AttrType::BOOLEANS) + .value("BLOCK", pd::proto::AttrType::BLOCK); - py::class_ op_desc(m, "OpDesc", ""); + pybind11::class_ op_desc(*m, "OpDesc", ""); op_desc - .def("__init__", [](OpDesc &self) { new (&self) OpDesc(); }, - py::return_value_policy::reference) - .def("copy_from", &OpDesc::CopyFrom) - .def("type", &OpDesc::Type) - .def("set_type", &OpDesc::SetType) - .def("input", &OpDesc::Input) - .def("input_names", &OpDesc::InputNames) - .def("output", &OpDesc::Output) - .def("output_names", &OpDesc::OutputNames) - .def("set_input", &OpDesc::SetInput) - .def("set_output", &OpDesc::SetOutput) - .def("input_arg_names", &OpDesc::InputArgumentNames) - .def("output_arg_names", &OpDesc::OutputArgumentNames) - .def("rename_input", &OpDesc::RenameInput) - .def("rename_output", &OpDesc::RenameOutput) - .def("has_attr", &OpDesc::HasAttr) - .def("attr_type", &OpDesc::GetAttrType) - .def("attr_names", &OpDesc::AttrNames) - .def("set_attr", &OpDesc::SetAttr) - .def("attr", &OpDesc::GetAttr) - .def("set_block_attr", &OpDesc::SetBlockAttr) + .def("__init__", [](pd::OpDesc &self) { new (&self) pd::OpDesc(); }, + pybind11::return_value_policy::reference) + .def("copy_from", &pd::OpDesc::CopyFrom) + .def("type", &pd::OpDesc::Type) + .def("set_type", &pd::OpDesc::SetType) + .def("input", &pd::OpDesc::Input) + .def("input_names", &pd::OpDesc::InputNames) + .def("output", &pd::OpDesc::Output) + .def("output_names", &pd::OpDesc::OutputNames) + .def("set_input", &pd::OpDesc::SetInput) + .def("set_output", &pd::OpDesc::SetOutput) + .def("input_arg_names", &pd::OpDesc::InputArgumentNames) + .def("output_arg_names", &pd::OpDesc::OutputArgumentNames) + .def("rename_input", &pd::OpDesc::RenameInput) + .def("rename_output", &pd::OpDesc::RenameOutput) + .def("has_attr", &pd::OpDesc::HasAttr) + .def("attr_type", &pd::OpDesc::GetAttrType) + .def("attr_names", &pd::OpDesc::AttrNames) + .def("set_attr", &pd::OpDesc::SetAttr) + .def("attr", &pd::OpDesc::GetAttr) + .def("set_block_attr", &pd::OpDesc::SetBlockAttr) .def("set_serialized_attr", - [](OpDesc &self, const std::string &name, - const py::bytes &seriralized) { + [](pd::OpDesc &self, const std::string &name, + const pybind11::bytes &seriralized) { std::string ser(seriralized); self.SetAttr(name, ser); }) - .def("block_attr", &OpDesc::GetBlockAttr) - .def("check_attrs", &OpDesc::CheckAttrs) - .def("infer_shape", &OpDesc::InferShape) - .def("infer_var_type", &OpDesc::InferVarType) - .def("serialize_to_string", SerializeMessage) - .def("block", &OpDesc::Block, py::return_value_policy::reference); + .def("block_attr", &pd::OpDesc::GetBlockAttr) + .def("check_attrs", &pd::OpDesc::CheckAttrs) + .def("infer_shape", &pd::OpDesc::InferShape) + .def("infer_var_type", &pd::OpDesc::InferVarType) + .def("serialize_to_string", SerializeMessage) + .def("block", &pd::OpDesc::Block, + pybind11::return_value_policy::reference); } } // namespace pybind diff --git a/paddle/fluid/pybind/protobuf.h b/paddle/fluid/pybind/protobuf.h index d0dc8936b3df50ca12315f113fbb36b0f98bb53f..e7370672a88fcf9238cc88c6aae65c6ee643746b 100644 --- a/paddle/fluid/pybind/protobuf.h +++ b/paddle/fluid/pybind/protobuf.h @@ -11,25 +11,25 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #pragma once #include + #include #include + #include "paddle/fluid/platform/variant.h" #include "pybind11/numpy.h" #include "pybind11/pybind11.h" #include "pybind11/stl.h" -namespace py = pybind11; - namespace paddle { namespace pybind { -void BindProgramDesc(py::module& m); -void BindBlockDesc(py::module& m); -void BindVarDsec(py::module& m); -void BindOpDesc(py::module& m); +void BindProgramDesc(pybind11::module* m); +void BindBlockDesc(pybind11::module* m); +void BindVarDsec(pybind11::module* m); +void BindOpDesc(pybind11::module* m); + } // namespace pybind } // namespace paddle diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index 9bc3ff51287b714e350326013f59db4faf028b4a..748ad75a99ea4955730327a10ae8468a107fed0a 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -74,7 +74,7 @@ PYBIND11_PLUGIN(core) { // not cause namespace pollution. using namespace paddle::framework; // NOLINT - BindException(m); + BindException(&m); py::class_(m, "Tensor", py::buffer_protocol()) .def_buffer( @@ -478,11 +478,11 @@ All parameter, weight, gradient are variables in Paddle. m.def("set_feed_variable", framework::SetFeedVariable); m.def("get_fetch_variable", framework::GetFetchVariable); - BindProgramDesc(m); - BindBlockDesc(m); - BindVarDsec(m); - BindOpDesc(m); - BindConstValue(m); + BindProgramDesc(&m); + BindBlockDesc(&m); + BindVarDsec(&m); + BindOpDesc(&m); + BindConstValue(&m); py::class_(m, "LodRankTable") .def("items", [](framework::LoDRankTable &table) { @@ -553,7 +553,7 @@ All parameter, weight, gradient are variables in Paddle. }) .def("run", &ParallelExecutor::Run); - BindRecordIOWriter(m); + BindRecordIOWriter(&m); return m.ptr(); } } // namespace pybind diff --git a/paddle/fluid/pybind/recordio.cc b/paddle/fluid/pybind/recordio.cc index 16f8bfb1a2e3a840670594d3cc2970e690dce891..0644d91425af1a1ac9363b1dec9e317689331fcb 100644 --- a/paddle/fluid/pybind/recordio.cc +++ b/paddle/fluid/pybind/recordio.cc @@ -13,13 +13,19 @@ // limitations under the License. #include "paddle/fluid/pybind/recordio.h" + #include +#include +#include + #include "paddle/fluid/framework/lod_tensor.h" #include "paddle/fluid/recordio/writer.h" namespace paddle { namespace pybind { +namespace { + class RecordIOWriter { public: RecordIOWriter(const std::string& filename, recordio::Compressor compressor, @@ -49,8 +55,10 @@ class RecordIOWriter { recordio::Writer writer_; }; -void BindRecordIOWriter(py::module& m) { - py::class_ writer(m, "RecordIOWriter", ""); +} // namespace + +void BindRecordIOWriter(py::module* m) { + py::class_ writer(*m, "RecordIOWriter", ""); py::enum_(writer, "Compressor", "") .value("Snappy", recordio::Compressor::kSnappy) .value("NoCompress", recordio::Compressor::kNoCompress); diff --git a/paddle/fluid/pybind/recordio.h b/paddle/fluid/pybind/recordio.h index 60e6a9e8595614b38375fca8c13d520739af9aaf..2555f9b719af8f73fbac10d92b890afd99fac290 100644 --- a/paddle/fluid/pybind/recordio.h +++ b/paddle/fluid/pybind/recordio.h @@ -21,6 +21,7 @@ namespace py = pybind11; namespace paddle { namespace pybind { -extern void BindRecordIOWriter(py::module& m); +void BindRecordIOWriter(py::module* m); + } // namespace pybind } // namespace paddle diff --git a/paddle/fluid/pybind/tensor_py.h b/paddle/fluid/pybind/tensor_py.h index 868966433e8c729c260e7b6e2101ced3814185c4..fbe953b2d8f12ca529f3daa01cc8e2fe8875a416 100644 --- a/paddle/fluid/pybind/tensor_py.h +++ b/paddle/fluid/pybind/tensor_py.h @@ -23,12 +23,8 @@ limitations under the License. */ #include "pybind11/numpy.h" #include "pybind11/pybind11.h" -namespace py = pybind11; - namespace paddle { - namespace pybind { - namespace details { template @@ -36,16 +32,16 @@ struct CastToPyBufferImpl; template struct CastToPyBufferImpl { - py::buffer_info operator()(framework::Tensor &tensor) { + pybind11::buffer_info operator()(const framework::Tensor &tensor) { PADDLE_THROW("This type of tensor cannot be expose to Python"); - return py::buffer_info(); + return pybind11::buffer_info(); } }; template struct CastToPyBufferImpl { using CUR_TYPE = typename std::tuple_element>::type; - py::buffer_info operator()(framework::Tensor &tensor) { + pybind11::buffer_info operator()(const framework::Tensor &tensor) { if (std::type_index(typeid(CUR_TYPE)) == tensor.type()) { auto dim_vec = framework::vectorize(tensor.dims()); std::vector dims_outside; @@ -84,15 +80,15 @@ struct CastToPyBufferImpl { if (std::type_index(typeid(CUR_TYPE)) == std::type_index(typeid(platform::float16))) { - return py::buffer_info(dst_tensor.data(), sizeof(CUR_TYPE), - "e", /* np.dtype('e') == np.float16 */ - (size_t)framework::arity(dst_tensor.dims()), - dims_outside, strides); + return pybind11::buffer_info( + dst_tensor.data(), sizeof(CUR_TYPE), + "e", /* np.dtype('e') == np.float16 */ + (size_t)framework::arity(dst_tensor.dims()), dims_outside, strides); } else { - return py::buffer_info(dst_tensor.data(), sizeof(CUR_TYPE), - py::format_descriptor::format(), - (size_t)framework::arity(dst_tensor.dims()), - dims_outside, strides); + return pybind11::buffer_info( + dst_tensor.data(), sizeof(CUR_TYPE), + pybind11::format_descriptor::format(), + (size_t)framework::arity(dst_tensor.dims()), dims_outside, strides); } } else { constexpr bool less = I + 1 < std::tuple_size>::value; @@ -103,7 +99,7 @@ struct CastToPyBufferImpl { } // namespace details -inline py::buffer_info CastToPyBuffer(framework::Tensor &tensor) { +inline pybind11::buffer_info CastToPyBuffer(const framework::Tensor &tensor) { auto buffer_info = details::CastToPyBufferImpl()(tensor); @@ -111,7 +107,7 @@ inline py::buffer_info CastToPyBuffer(framework::Tensor &tensor) { } template -T TensorGetElement(framework::Tensor &self, size_t offset) { +T TensorGetElement(const framework::Tensor &self, size_t offset) { if (platform::is_cpu_place(self.place())) { return self.data()[offset]; } else { @@ -123,31 +119,32 @@ T TensorGetElement(framework::Tensor &self, size_t offset) { // TODO(dzhwinter) : fix the redundent Tensor allocate and free template -void TensorSetElement(framework::Tensor &self, size_t offset, T elem) { - if (platform::is_gpu_place(self.place())) { +void TensorSetElement(framework::Tensor *self, size_t offset, T elem) { + if (platform::is_gpu_place(self->place())) { std::shared_ptr dst(new framework::Tensor); - framework::TensorCopy(self, platform::CPUPlace(), dst.get()); + framework::TensorCopy(*self, platform::CPUPlace(), dst.get()); dst->data()[offset] = elem; - framework::TensorCopy(*dst.get(), self.place(), &self); + framework::TensorCopy(*dst.get(), self->place(), self); - } else if (platform::is_cpu_place(self.place())) { - self.data()[offset] = elem; + } else if (platform::is_cpu_place(self->place())) { + self->data()[offset] = elem; } } template void PyCPUTensorSetFromArray( - framework::Tensor &self, - py::array_t array, - paddle::platform::CPUPlace &place) { + framework::Tensor *self, + pybind11::array_t + array, + paddle::platform::CPUPlace place) { std::vector dims; dims.reserve(array.ndim()); for (size_t i = 0; i < array.ndim(); ++i) { dims.push_back(static_cast(array.shape()[i])); } - self.Resize(framework::make_ddim(dims)); - auto *dst = self.mutable_data(place); + self->Resize(framework::make_ddim(dims)); + auto *dst = self->mutable_data(place); std::memcpy(dst, array.data(), sizeof(T) * array.size()); } @@ -155,34 +152,37 @@ template <> // This following specialization maps uint16_t in the parameter type to // platform::float16. void PyCPUTensorSetFromArray( - framework::Tensor &self, - py::array_t array, - paddle::platform::CPUPlace &place) { + framework::Tensor *self, + pybind11::array_t + array, + paddle::platform::CPUPlace place) { std::vector dims; dims.reserve(array.ndim()); for (size_t i = 0; i < array.ndim(); ++i) { dims.push_back(static_cast(array.shape()[i])); } - self.Resize(framework::make_ddim(dims)); - auto *dst = self.mutable_data(place); + self->Resize(framework::make_ddim(dims)); + auto *dst = self->mutable_data(place); std::memcpy(dst, array.data(), sizeof(uint16_t) * array.size()); } #ifdef PADDLE_WITH_CUDA template void PyCUDATensorSetFromArray( - framework::Tensor &self, - py::array_t array, - paddle::platform::CUDAPlace &place) { + framework::Tensor *self, + pybind11::array_t + array, + paddle::platform::CUDAPlace place) { std::vector dims; dims.reserve(array.ndim()); for (size_t i = 0; i < array.ndim(); ++i) { dims.push_back(static_cast(array.shape()[i])); } - self.Resize(framework::make_ddim(dims)); - auto *dst = self.mutable_data(place); + self->Resize(framework::make_ddim(dims)); + auto *dst = self->mutable_data(place); platform::DeviceContextPool &pool = platform::DeviceContextPool::Instance(); auto dev_ctx = @@ -195,17 +195,19 @@ template <> // This following specialization maps uint16_t in the parameter type to // platform::float16. void PyCUDATensorSetFromArray( - framework::Tensor &self, - py::array_t array, - paddle::platform::CUDAPlace &place) { + framework::Tensor *self, + pybind11::array_t + array, + paddle::platform::CUDAPlace place) { std::vector dims; dims.reserve(array.ndim()); for (size_t i = 0; i < array.ndim(); ++i) { dims.push_back(static_cast(array.shape()[i])); } - self.Resize(framework::make_ddim(dims)); - auto *dst = self.mutable_data(place); + self->Resize(framework::make_ddim(dims)); + auto *dst = self->mutable_data(place); platform::DeviceContextPool &pool = platform::DeviceContextPool::Instance(); auto dev_ctx = @@ -217,8 +219,9 @@ void PyCUDATensorSetFromArray( template void PyCUDAPinnedTensorSetFromArray( - framework::Tensor &self, - py::array_t array, + framework::Tensor *self, + pybind11::array_t + array, const paddle::platform::CUDAPinnedPlace &place) { std::vector dims; dims.reserve(array.ndim()); @@ -226,8 +229,8 @@ void PyCUDAPinnedTensorSetFromArray( dims.push_back(static_cast(array.shape()[i])); } - self.Resize(framework::make_ddim(dims)); - auto *dst = self.mutable_data(place); + self->Resize(framework::make_ddim(dims)); + auto *dst = self->mutable_data(place); std::memcpy(dst, array.data(), sizeof(T) * array.size()); } @@ -235,8 +238,10 @@ template <> // This following specialization maps uint16_t in the parameter type to // platform::float16. void PyCUDAPinnedTensorSetFromArray( - framework::Tensor &self, - py::array_t array, + framework::Tensor *self, + pybind11::array_t + array, const paddle::platform::CUDAPinnedPlace &place) { std::vector dims; dims.reserve(array.ndim()); @@ -244,8 +249,8 @@ void PyCUDAPinnedTensorSetFromArray( dims.push_back(static_cast(array.shape()[i])); } - self.Resize(framework::make_ddim(dims)); - auto *dst = self.mutable_data(place); + self->Resize(framework::make_ddim(dims)); + auto *dst = self->mutable_data(place); std::memcpy(dst, array.data(), sizeof(uint16_t) * array.size()); } #endif diff --git a/paddle/fluid/pybind/tensor_py_test.cc b/paddle/fluid/pybind/tensor_py_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..1a0ae1d65833b1097bf69befe05884cab1317a89 --- /dev/null +++ b/paddle/fluid/pybind/tensor_py_test.cc @@ -0,0 +1,44 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/pybind/tensor_py.h" + +#include + +#include "gtest/gtest.h" +#include "paddle/fluid/framework/tensor.h" + +TEST(TensorPy, CastToPyBufferImpl) { + typedef int ElemType; + + paddle::framework::Tensor t; + auto d = paddle::framework::make_ddim({1, 2, 3}); + int* p = t.mutable_data(d, paddle::platform::CPUPlace()); + for (int i = 0; i < paddle::framework::product(d); ++i) { + p[i] = i; + } + + pybind11::buffer_info bi = paddle::pybind::CastToPyBuffer(t); + EXPECT_EQ(bi.itemsize, static_cast(sizeof(ElemType))); + EXPECT_EQ(bi.size, static_cast(paddle::framework::product(d))); + EXPECT_EQ(bi.ndim, static_cast(3)); // 3-dimensional as d. + EXPECT_EQ(bi.shape.size(), 3U); // as Dim d. + EXPECT_EQ(bi.shape[0], static_cast(1)); + EXPECT_EQ(bi.shape[1], static_cast(2)); + EXPECT_EQ(bi.shape[2], static_cast(3)); + EXPECT_EQ(bi.strides.size(), 3U); // 3-dimensional as d. + EXPECT_EQ(bi.strides[2], static_cast(sizeof(ElemType))); + EXPECT_EQ(bi.strides[1], static_cast(sizeof(ElemType) * 3)); + EXPECT_EQ(bi.strides[0], static_cast(sizeof(ElemType) * 2 * 3)); +}