From a89c7ffa94bc26a879b8978273219980648aaec4 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 18 Jul 2017 11:57:31 +0800 Subject: [PATCH] Make Tensor <--> Numpy interactive in tensor.h * Follow review comments to seperate Tensor Numpy interactive methods in tensor.h. * Simplify logic for `CastToPyBufferImpl`, make it as one struct and in details namespace. * Remove `Scope` expose in Python, since it currently is useless. * Remove some debug functions. --- paddle/pybind/pybind.cc | 118 +----------------- paddle/pybind/tensor.h | 91 ++++++++++++++ .../paddle/v2/framework/tests/CMakeLists.txt | 3 +- 3 files changed, 97 insertions(+), 115 deletions(-) create mode 100644 paddle/pybind/tensor.h diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc index 8222323e36a..e3dc3e718ca 100644 --- a/paddle/pybind/pybind.cc +++ b/paddle/pybind/pybind.cc @@ -15,6 +15,7 @@ limitations under the License. */ #include #include #include +#include #include #include #include @@ -26,125 +27,14 @@ namespace pd = paddle::framework; USE_OP(add_two); -struct PlaceDebugString : public boost::static_visitor { - std::string operator()(const paddle::platform::GPUPlace& place) const { - return "GPU(" + std::to_string(place.device) + ")"; - } - - std::string operator()(const paddle::platform::CPUPlace& place) const { - return "CPU"; - } -}; - -template -struct TensorToPyBuffer { - pd::Tensor& self_; - explicit TensorToPyBuffer(pd::Tensor& self) : self_(self) {} - - bool CanCast() const { return std::type_index(typeid(T)) == self_.type(); } - - py::buffer_info Cast() const { - auto dim_vec = pd::vectorize(self_.dims()); - std::vector dims_outside; - std::vector strides; - dims_outside.resize(dim_vec.size()); - strides.resize(dim_vec.size()); - - size_t prod = 1; - for (size_t i = dim_vec.size(); i != 0; --i) { - dims_outside[i - 1] = (size_t)dim_vec[i - 1]; - strides[i - 1] = sizeof(float) * prod; - prod *= dims_outside[i - 1]; - } - - return py::buffer_info(self_.mutable_data(self_.place()), - sizeof(T), - py::format_descriptor::format(), - (size_t)pd::arity(self_.dims()), - dims_outside, - strides); - } -}; - -template -struct CastToPyBufferImpl; - -template -struct CastToPyBufferImpl { - py::buffer_info operator()(pd::Tensor& tensor) { - PADDLE_THROW("This type of tensor cannot be expose to Python"); - return py::buffer_info(); - } -}; - -template -struct CastToPyBufferImpl { - using CUR_TYPE = typename std::tuple_element>::type; - py::buffer_info operator()(pd::Tensor& tensor) { - TensorToPyBuffer cast_object(tensor); - if (cast_object.CanCast()) { - return cast_object.Cast(); - } else { - constexpr bool less = I + 1 < std::tuple_size>::value; - return CastToPyBufferImpl()(tensor); - } - } -}; - -template -std::ostream& operator<<(std::ostream& os, const std::vector& vec) { - for (size_t i = 0; i < vec.size(); ++i) { - os << vec[i]; - if (i + 1 != vec.size()) { - os << ", "; - } - } - return os; -} - -py::buffer_info CastToPyBuffer(pd::Tensor& tensor) { - auto buffer_info = CastToPyBufferImpl()(tensor); - return buffer_info; -} - -template -void PyTensorSet( - pd::Tensor& self, - py::array_t array) { - std::vector dims; - dims.reserve(array.ndim()); - for (size_t i = 0; i < array.ndim(); ++i) { - dims.push_back((int)array.shape()[i]); - } - - self.set_dims(pd::make_ddim(dims)); - auto* dst = self.mutable_data(paddle::platform::CPUPlace()); - std::memcpy(dst, array.data(), sizeof(T) * array.size()); -} - PYBIND11_PLUGIN(core) { py::module m("core", "C++ core of Paddle Paddle"); - py::class_( - m, "Place", R"DOC(Device Place Class.)DOC") - .def("__str__", - [](const paddle::platform::Place& self) { - return boost::apply_visitor(PlaceDebugString(), self); - }) - .def("is_gpu", - [](const paddle::platform::Place& self) { - return paddle::platform::is_gpu_place(self); - }) - .def("is_cpu", [](const paddle::platform::Place& self) { - return paddle::platform::is_cpu_place(self); - }); - py::class_(m, "Tensor", py::buffer_protocol()) - .def("get_place", &pd::Tensor::place) .def_buffer([](pd::Tensor& self) -> py::buffer_info { PADDLE_ENFORCE(paddle::platform::is_cpu_place(self.place()), "Only CPU tensor can cast to numpy array"); - return CastToPyBuffer(self); + return paddle::pybind::CastToPyBuffer(self); }) .def("get_dims", [](const pd::Tensor& self) { return pd::vectorize(self.dims()); }) @@ -160,8 +50,8 @@ PYBIND11_PLUGIN(core) { [](pd::Tensor& self) { self.mutable_data(paddle::platform::CPUPlace()); }) - .def("set", PyTensorSet) - .def("set", PyTensorSet); + .def("set", paddle::pybind::PyTensorSetFromArray) + .def("set", paddle::pybind::PyTensorSetFromArray); py::class_(m, "Variable", R"DOC(Variable Class. diff --git a/paddle/pybind/tensor.h b/paddle/pybind/tensor.h new file mode 100644 index 00000000000..ef07144ad44 --- /dev/null +++ b/paddle/pybind/tensor.h @@ -0,0 +1,91 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include +#include +#include + +namespace py = pybind11; + +namespace paddle { + +namespace pybind { + +namespace details { + +template +struct CastToPyBufferImpl; + +template +struct CastToPyBufferImpl { + py::buffer_info operator()(framework::Tensor &tensor) { + PADDLE_THROW("This type of tensor cannot be expose to Python"); + return py::buffer_info(); + } +}; + +template +struct CastToPyBufferImpl { + using CUR_TYPE = typename std::tuple_element>::type; + py::buffer_info operator()(framework::Tensor &tensor) { + if (std::type_index(typeid(CUR_TYPE)) == tensor.type()) { + auto dim_vec = framework::vectorize(tensor.dims()); + std::vector dims_outside; + std::vector strides; + dims_outside.resize(dim_vec.size()); + strides.resize(dim_vec.size()); + + size_t prod = 1; + for (size_t i = dim_vec.size(); i != 0; --i) { + dims_outside[i - 1] = (size_t)dim_vec[i - 1]; + strides[i - 1] = sizeof(CUR_TYPE) * prod; + prod *= dims_outside[i - 1]; + } + + return py::buffer_info(tensor.mutable_data(tensor.place()), + sizeof(CUR_TYPE), + py::format_descriptor::format(), + (size_t)framework::arity(tensor.dims()), + dims_outside, + strides); + } else { + constexpr bool less = I + 1 < std::tuple_size>::value; + return CastToPyBufferImpl()(tensor); + } + } +}; +} // namespace details +inline py::buffer_info CastToPyBuffer(framework::Tensor &tensor) { + auto buffer_info = details::CastToPyBufferImpl()(tensor); + return buffer_info; +} + +template +void PyTensorSetFromArray( + framework::Tensor &self, + py::array_t array) { + std::vector dims; + dims.reserve(array.ndim()); + for (size_t i = 0; i < array.ndim(); ++i) { + dims.push_back((int)array.shape()[i]); + } + + self.set_dims(framework::make_ddim(dims)); + auto *dst = self.mutable_data(paddle::platform::CPUPlace()); + std::memcpy(dst, array.data(), sizeof(T) * array.size()); +} + +} // namespace pybind +} // namespace paddle diff --git a/python/paddle/v2/framework/tests/CMakeLists.txt b/python/paddle/v2/framework/tests/CMakeLists.txt index 86fc60f26ae..4ce2bef6fcc 100644 --- a/python/paddle/v2/framework/tests/CMakeLists.txt +++ b/python/paddle/v2/framework/tests/CMakeLists.txt @@ -1,2 +1,3 @@ add_python_test(test_framework test_protobuf.py test_scope.py - test_default_scope_funcs.py test_op_creation_methods.py) + test_default_scope_funcs.py test_op_creation_methods.py + test_tensor.py) -- GitLab