Merge pull request #4457 from Canpio/dev_move_desc_to_framework

move Protobuf desc to framework

paddle/framework/CMakeLists.txt

@@ -19,6 +19,7 @@ cc_test(scope_test SRCS scope_test.cc DEPS scope)
 proto_library(framework_proto SRCS framework.proto)
 
 cc_library(attribute SRCS attribute.cc DEPS framework_proto)
+cc_library(proto_desc SRCS var_desc.cc op_desc.cc block_desc.cc program_desc.cc DEPS attribute)
 cc_library(op_proto_maker SRCS op_proto_maker.cc DEPS framework_proto attribute)
 cc_test(op_proto_maker_test SRCS op_proto_maker_test.cc DEPS op_proto_maker)
 cc_library(op_info SRCS op_info.cc DEPS attribute framework_proto)

paddle/framework/block_desc.cc

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/framework/block_desc.h"
+#include "paddle/framework/program_desc.h"
+
+namespace paddle {
+namespace framework {
+
+VarDescBind *BlockDescBind::NewVar(const std::string &name) {
+  need_update_ = true;
+  auto it = vars_.find(name);
+  PADDLE_ENFORCE(it == vars_.end(), "Duplicated variable %s", name);
+  auto var = new VarDescBind(name);
+  vars_[name].reset(var);
+  return var;
+}
+
+VarDescBind *BlockDescBind::Var(const std::string &name) const {
+  auto it = vars_.find(name);
+  PADDLE_ENFORCE(it != vars_.end(),
+                 "Can not find variable %s in current block.", name);
+  return it->second.get();
+}
+
+std::vector<VarDescBind *> BlockDescBind::AllVars() const {
+  std::vector<VarDescBind *> res;
+  for (const auto &p : vars_) {
+    res.push_back(p.second.get());
+  }
+  return res;
+}
+
+OpDescBind *BlockDescBind::AppendOp() {
+  need_update_ = true;
+  ops_.emplace_back(new OpDescBind());
+  return ops_.back().get();
+}
+
+OpDescBind *BlockDescBind::PrependOp() {
+  need_update_ = true;
+  ops_.emplace_front(new OpDescBind());
+  return ops_.front().get();
+}
+
+std::vector<OpDescBind *> BlockDescBind::AllOps() const {
+  std::vector<OpDescBind *> res;
+  for (const auto &op : ops_) {
+    res.push_back(op.get());
+  }
+  return res;
+}
+
+void BlockDescBind::Sync() {
+  if (need_update_) {
+    auto &op_field = *this->desc_->mutable_ops();
+    op_field.Clear();
+    op_field.Reserve(static_cast<int>(ops_.size()));
+    for (auto &op_desc : ops_) {
+      op_field.AddAllocated(op_desc->Proto());
+    }
+    need_update_ = false;
+  }
+}
+
+BlockDescBind *BlockDescBind::ParentBlock() const {
+  if (this->desc_->parent_idx() == -1) {
+    return nullptr;
+  }
+  return prog_->Block(static_cast<size_t>(this->desc_->parent_idx()));
+}
+
+void OpDescBind::SetBlockAttr(const std::string &name, BlockDescBind &block) {
+  BlockDesc *desc = block.RawPtr();
+  this->attrs_[name] = desc;
+}
+}  // namespace framework
+}  // namespace paddle

paddle/framework/block_desc.h

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include <deque>
+#include <unordered_map>
+#include <vector>
+#include "paddle/framework/op_desc.h"
+#include "paddle/framework/var_desc.h"
+
+namespace paddle {
+namespace framework {
+
+class ProgramDescBind;
+
+// Each Protobuf Message, we provide a XXXBind class. In that class, we optimize
+// read/write speed. Only when we want the protobuf message, the local changes
+// will be synchronized (by `Sync` method).
+
+class BlockDescBind {
+ public:
+  BlockDescBind(ProgramDescBind *prog, BlockDesc *desc)
+      : prog_(prog), desc_(desc), need_update_(false) {}
+
+  BlockDescBind(const BlockDescBind &o) = delete;
+  BlockDescBind &operator=(const BlockDescBind &o) = delete;
+
+  int32_t ID() const { return desc_->idx(); }
+
+  int32_t Parent() const { return desc_->parent_idx(); }
+
+  VarDescBind *NewVar(const std::string &name_bytes);
+
+  VarDescBind *Var(const std::string &name_bytes) const;
+
+  std::vector<VarDescBind *> AllVars() const;
+
+  BlockDescBind *ParentBlock() const;
+
+  OpDescBind *AppendOp();
+
+  OpDescBind *PrependOp();
+
+  std::vector<OpDescBind *> AllOps() const;
+
+  void Sync();
+
+  BlockDesc *RawPtr() { return desc_; }
+
+ private:
+  ProgramDescBind *prog_;  // not_own
+  BlockDesc *desc_;        // not_own
+  bool need_update_;
+
+  std::deque<std::unique_ptr<OpDescBind>> ops_;
+  std::unordered_map<std::string, std::unique_ptr<VarDescBind>> vars_;
+};
+}  // namespace framework
+}  // namespace paddle

paddle/framework/op_desc.cc

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/framework/op_desc.h"
+#include "paddle/framework/block_desc.h"
+
+namespace paddle {
+namespace framework {
+
+OpDesc *OpDescBind::Proto() {
+  Sync();
+  return &op_desc_;
+}
+
+const std::vector<std::string> &OpDescBind::Input(
+    const std::string &name) const {
+  auto it = inputs_.find(name);
+  PADDLE_ENFORCE(it != inputs_.end(), "Input %s cannot be found in Op %s", name,
+                 Type());
+  return it->second;
+}
+
+std::vector<std::string> OpDescBind::InputNames() const {
+  std::vector<std::string> retv;
+  retv.reserve(this->inputs_.size());
+  for (auto &ipt : this->inputs_) {
+    retv.push_back(ipt.first);
+  }
+  return retv;
+}
+
+void OpDescBind::SetInput(const std::string &param_name,
+                          const std::vector<std::string> &args) {
+  need_update_ = true;
+  inputs_[param_name] = args;
+}
+
+const std::vector<std::string> &OpDescBind::Output(
+    const std::string &name) const {
+  auto it = outputs_.find(name);
+  PADDLE_ENFORCE(it != outputs_.end(), "Output %s cannot be found in Op %s",
+                 name, Type());
+  return it->second;
+}
+
+std::vector<std::string> OpDescBind::OutputNames() const {
+  std::vector<std::string> retv;
+  retv.reserve(this->outputs_.size());
+  for (auto &ipt : this->outputs_) {
+    retv.push_back(ipt.first);
+  }
+  return retv;
+}
+
+void OpDescBind::SetOutput(const std::string &param_name,
+                           const std::vector<std::string> &args) {
+  need_update_ = true;
+  this->outputs_[param_name] = args;
+}
+
+AttrType OpDescBind::GetAttrType(const std::string &name) const {
+  auto it = attrs_.find(name);
+  PADDLE_ENFORCE(it != attrs_.end(), "Attribute %s is not found", name);
+  return static_cast<AttrType>(it->second.which() - 1);
+}
+
+std::vector<std::string> OpDescBind::AttrNames() const {
+  std::vector<std::string> retv;
+  retv.reserve(attrs_.size());
+  for (auto &attr : attrs_) {
+    retv.push_back(attr.first);
+  }
+  return retv;
+}
+
+void OpDescBind::SetAttr(const std::string &name, const Attribute &v) {
+  this->attrs_[name] = v;
+  need_update_ = true;
+}
+
+Attribute OpDescBind::GetAttr(const std::string &name) const {
+  auto it = attrs_.find(name);
+  PADDLE_ENFORCE(it != attrs_.end(), "Attribute %s is not found", name);
+  return it->second;
+}
+
+int OpDescBind::GetBlockAttr(const std::string &name) const {
+  auto it = attrs_.find(name);
+  PADDLE_ENFORCE(it != attrs_.end(), "Attribute %s is not found", name);
+  return boost::get<BlockDesc *>(it->second)->idx();
+}
+
+void OpDescBind::Sync() {
+  if (need_update_) {
+    this->op_desc_.mutable_inputs()->Clear();
+    for (auto &ipt : inputs_) {
+      auto *input = op_desc_.add_inputs();
+      input->set_parameter(ipt.first);
+      VectorToRepeated(ipt.second, input->mutable_arguments());
+    }
+
+    this->op_desc_.mutable_outputs()->Clear();
+    for (auto &opt : outputs_) {
+      auto *output = op_desc_.add_outputs();
+      output->set_parameter(opt.first);
+      VectorToRepeated(opt.second, output->mutable_arguments());
+    }
+
+    this->op_desc_.mutable_attrs()->Clear();
+    for (auto &attr : attrs_) {
+      auto *attr_desc = op_desc_.add_attrs();
+      attr_desc->set_name(attr.first);
+      attr_desc->set_type(
+          static_cast<framework::AttrType>(attr.second.which() - 1));
+      boost::apply_visitor(SetAttrDescVisitor(attr_desc), attr.second);
+    }
+
+    need_update_ = false;
+  }
+}
+}  // namespace framework
+}  // namespace paddle

paddle/framework/op_desc.h

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include <unordered_map>
+#include <vector>
+#include "paddle/framework/attribute.h"
+#include "paddle/framework/var_desc.h"
+
+namespace paddle {
+namespace framework {
+
+class BlockDescBind;
+
+class OpDescBind {
+ public:
+  OpDesc *Proto();
+
+  std::string Type() const { return op_desc_.type(); }
+
+  void SetType(const std::string &type) { op_desc_.set_type(type); }
+
+  const std::vector<std::string> &Input(const std::string &name) const;
+
+  std::vector<std::string> InputNames() const;
+
+  void SetInput(const std::string &param_name,
+                const std::vector<std::string> &args);
+
+  const std::vector<std::string> &Output(const std::string &name) const;
+
+  std::vector<std::string> OutputNames() const;
+
+  void SetOutput(const std::string &param_name,
+                 const std::vector<std::string> &args);
+
+  std::string DebugString() { return this->Proto()->DebugString(); }
+
+  bool HasAttr(const std::string &name) const {
+    return attrs_.find(name) != attrs_.end();
+  }
+
+  AttrType GetAttrType(const std::string &name) const;
+
+  std::vector<std::string> AttrNames() const;
+
+  void SetAttr(const std::string &name, const Attribute &v);
+
+  void SetBlockAttr(const std::string &name, BlockDescBind &block);
+
+  Attribute GetAttr(const std::string &name) const;
+
+  int GetBlockAttr(const std::string &name) const;
+
+ private:
+  struct SetAttrDescVisitor : public boost::static_visitor<void> {
+    explicit SetAttrDescVisitor(OpDesc::Attr *attr) : attr_(attr) {}
+    mutable OpDesc::Attr *attr_;
+    void operator()(int v) const { attr_->set_i(v); }
+    void operator()(float v) const { attr_->set_f(v); }
+    void operator()(const std::string &v) const { attr_->set_s(v); }
+    void operator()(bool b) const { attr_->set_b(b); }
+
+    void operator()(const std::vector<int> &v) const {
+      VectorToRepeated(v, attr_->mutable_ints());
+    }
+    void operator()(const std::vector<float> &v) const {
+      VectorToRepeated(v, attr_->mutable_floats());
+    }
+    void operator()(const std::vector<std::string> &v) const {
+      VectorToRepeated(v, attr_->mutable_strings());
+    }
+    void operator()(const std::vector<bool> &v) const {
+      VectorToRepeated(v, attr_->mutable_bools());
+    }
+    void operator()(BlockDesc *desc) const {
+      attr_->set_block_idx(desc->idx());
+    }
+    void operator()(boost::blank) const { PADDLE_THROW("Unexpected branch"); }
+  };
+
+  void Sync();
+
+  OpDesc op_desc_;
+  std::unordered_map<std::string, std::vector<std::string>> inputs_;
+  std::unordered_map<std::string, std::vector<std::string>> outputs_;
+  std::unordered_map<std::string, Attribute> attrs_;
+
+  // need_update_ indicate there some local changes not be synchronized. If
+  // local changes should be synchronized, need_update_ should be set to true.
+  bool need_update_{false};
+};
+}  // namespace framework
+}  // namespace paddle

paddle/framework/program_desc.cc

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/framework/program_desc.h"
+#include "paddle/framework/block_desc.h"
+
+namespace paddle {
+namespace framework {
+
+using ProgDescMap =
+    std::unordered_map<ProgramDesc *, std::unique_ptr<ProgramDescBind>>;
+static ProgDescMap *g_bind_map = nullptr;
+
+ProgramDescBind &ProgramDescBind::Instance(ProgramDesc *prog) {
+  if (g_bind_map == nullptr) {
+    g_bind_map = new ProgDescMap();
+  }
+  auto &map = *g_bind_map;
+  auto &ptr = map[prog];
+
+  if (ptr == nullptr) {
+    ptr.reset(new ProgramDescBind(prog));
+  }
+  return *ptr;
+}
+
+BlockDescBind *ProgramDescBind::AppendBlock(const BlockDescBind &parent) {
+  auto *b = prog_->add_blocks();
+  b->set_parent_idx(parent.ID());
+  b->set_idx(prog_->blocks_size() - 1);
+  blocks_.emplace_back(new BlockDescBind(this, b));
+  return blocks_.back().get();
+}
+
+ProgramDesc *ProgramDescBind::Proto() {
+  for (auto &block : blocks_) {
+    block->Sync();
+  }
+  return prog_;
+}
+
+ProgramDescBind::ProgramDescBind(ProgramDesc *prog) {
+  prog_ = prog;
+  for (auto &block : *prog->mutable_blocks()) {
+    blocks_.emplace_back(new BlockDescBind(this, &block));
+  }
+}
+}  // namespace framework
+}  // namespace paddle

paddle/framework/program_desc.h

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include <vector>
+#include "paddle/framework/framework.pb.h"
+
+namespace paddle {
+namespace framework {
+
+class BlockDescBind;
+
+class ProgramDescBind {
+ public:
+  static ProgramDescBind &Instance(ProgramDesc *prog);
+
+  ProgramDescBind(const ProgramDescBind &o) = delete;
+  ProgramDescBind &operator=(const ProgramDescBind &o) = delete;
+
+  BlockDescBind *AppendBlock(const BlockDescBind &parent);
+
+  BlockDescBind *Block(size_t idx) { return blocks_[idx].get(); }
+
+  std::string DebugString() { return Proto()->DebugString(); }
+
+  size_t Size() const { return blocks_.size(); }
+
+  ProgramDesc *Proto();
+
+ private:
+  explicit ProgramDescBind(ProgramDesc *prog);
+
+  // Not owned
+  ProgramDesc *prog_;
+
+  std::vector<std::unique_ptr<BlockDescBind>> blocks_;
+};
+}  // namespace framework
+}  // namespace paddle

paddle/framework/var_desc.cc

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/framework/var_desc.h"
+
+namespace paddle {
+namespace framework {
+
+void VarDescBind::SetShape(const std::vector<int64_t> &dims) {
+  VectorToRepeated(dims, desc_.mutable_lod_tensor()->mutable_dims());
+}
+
+void VarDescBind::SetDataType(DataType data_type) {
+  desc_.mutable_lod_tensor()->set_data_type(data_type);
+}
+
+std::vector<int64_t> VarDescBind::Shape() const {
+  return RepeatedToVector(desc_.lod_tensor().dims());
+}
+
+DataType VarDescBind::GetDataType() const {
+  return desc_.lod_tensor().data_type();
+}
+}  // namespace framework
+}  // namespace paddle

paddle/framework/var_desc.h

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include <vector>
+#include "paddle/framework/framework.pb.h"
+
+namespace paddle {
+namespace framework {
+
+// convert between std::vector and protobuf repeated.
+template <typename T>
+inline std::vector<T> RepeatedToVector(
+    const google::protobuf::RepeatedField<T> &repeated_field) {
+  std::vector<T> ret;
+  ret.reserve(repeated_field.size());
+  std::copy(repeated_field.begin(), repeated_field.end(),
+            std::back_inserter(ret));
+  return ret;
+}
+
+template <typename T, typename RepeatedField>
+inline void VectorToRepeated(const std::vector<T> &vec,
+                             RepeatedField *repeated_field) {
+  repeated_field->Reserve(vec.size());
+  for (const auto &elem : vec) {
+    *repeated_field->Add() = elem;
+  }
+}
+
+// Specialize vector<bool>.
+template <typename RepeatedField>
+inline void VectorToRepeated(const std::vector<bool> &vec,
+                             RepeatedField *repeated_field) {
+  repeated_field->Reserve(vec.size());
+  for (auto elem : vec) {
+    *repeated_field->Add() = elem;
+  }
+}
+
+class VarDescBind {
+ public:
+  explicit VarDescBind(const std::string &name) { desc_.set_name(name); }
+
+  VarDesc *Proto() { return &desc_; }
+
+  std::string Name() const { return desc_.name(); }
+
+  void SetShape(const std::vector<int64_t> &dims);
+
+  void SetDataType(DataType data_type);
+
+  std::vector<int64_t> Shape() const;
+
+  DataType GetDataType() const;
+
+ private:
+  VarDesc desc_;
+};
+}  // namespace framework
+}  // namespace paddle

paddle/pybind/CMakeLists.txt

 if(WITH_PYTHON)
   cc_library(paddle_pybind SHARED
     SRCS pybind.cc exception.cc protobuf.cc
-    DEPS pybind python backward
+    DEPS pybind python backward proto_desc
     ${GLOB_OP_LIB})
 endif(WITH_PYTHON)

paddle/pybind/protobuf.cc

@@ -15,7 +15,10 @@ limitations under the License. */
 #include "paddle/pybind/protobuf.h"
 #include <deque>
 #include <iostream>
-#include "paddle/framework/attribute.h"
+#include "paddle/framework/block_desc.h"
+#include "paddle/framework/op_desc.h"
+#include "paddle/framework/program_desc.h"
+#include "paddle/framework/var_desc.h"
 
 // Cast boost::variant for PyBind.
 // Copy from
@@ -93,383 +96,6 @@ namespace pybind {
 
 using namespace paddle::framework;  // NOLINT
 
-// convert between std::vector and protobuf repeated.
-template <typename T>
-inline std::vector<T> RepeatedToVector(
-    const google::protobuf::RepeatedField<T> &repeated_field) {
-  std::vector<T> ret;
-  ret.reserve(repeated_field.size());
-  std::copy(repeated_field.begin(), repeated_field.end(),
-            std::back_inserter(ret));
-  return ret;
-}
-
-template <typename T, typename RepeatedField>
-inline void VectorToRepeated(const std::vector<T> &vec,
-                             RepeatedField *repeated_field) {
-  repeated_field->Reserve(vec.size());
-  for (const auto &elem : vec) {
-    *repeated_field->Add() = elem;
-  }
-}
-
-// Specialize vector<bool>.
-template <typename RepeatedField>
-inline void VectorToRepeated(const std::vector<bool> &vec,
-                             RepeatedField *repeated_field) {
-  repeated_field->Reserve(vec.size());
-  for (auto elem : vec) {
-    *repeated_field->Add() = elem;
-  }
-}
-
-class ProgramDescBind;
-class OpDescBind;
-class BlockDescBind;
-class VarDescBind;
-
-// Each Protobuf Message, we provide a XXXBind class. In that class, we optimize
-// read/write speed. Only when we want the protobuf message, the local changes
-// will be synchronized (by `Sync` method).
-class VarDescBind {
- public:
-  explicit VarDescBind(const std::string &name) { desc_.set_name(name); }
-
-  VarDesc *Proto() { return &desc_; }
-
-  py::bytes Name() const { return desc_.name(); }
-
-  void SetShape(const std::vector<int64_t> &dims) {
-    VectorToRepeated(dims, desc_.mutable_lod_tensor()->mutable_dims());
-  }
-
-  void SetDataType(framework::DataType data_type) {
-    desc_.mutable_lod_tensor()->set_data_type(data_type);
-  }
-
-  std::vector<int64_t> Shape() const {
-    return RepeatedToVector(desc_.lod_tensor().dims());
-  }
-
-  framework::DataType DataType() const {
-    return desc_.lod_tensor().data_type();
-  }
-
- private:
-  VarDesc desc_;
-};
-
-class OpDescBind {
- public:
-  OpDesc *Proto() {
-    Sync();
-    return &op_desc_;
-  }
-
-  std::string Type() const { return op_desc_.type(); }
-
-  void SetType(const std::string &type) { op_desc_.set_type(type); }
-
-  const std::vector<std::string> &Input(const std::string &name) const {
-    auto it = inputs_.find(name);
-    PADDLE_ENFORCE(it != inputs_.end(), "Input %s cannot be found in Op %s",
-                   name, Type());
-    return it->second;
-  }
-
-  std::vector<std::string> InputNames() const {
-    std::vector<std::string> retv;
-    retv.reserve(this->inputs_.size());
-    for (auto &ipt : this->inputs_) {
-      retv.push_back(ipt.first);
-    }
-    return retv;
-  }
-
-  void SetInput(const std::string &param_name,
-                const std::vector<std::string> &args) {
-    need_update_ = true;
-    inputs_[param_name] = args;
-  }
-
-  const std::vector<std::string> &Output(const std::string &name) const {
-    auto it = outputs_.find(name);
-    PADDLE_ENFORCE(it != outputs_.end(), "Output %s cannot be found in Op %s",
-                   name, Type());
-    return it->second;
-  }
-
-  std::vector<std::string> OutputNames() const {
-    std::vector<std::string> retv;
-    retv.reserve(this->outputs_.size());
-    for (auto &ipt : this->outputs_) {
-      retv.push_back(ipt.first);
-    }
-    return retv;
-  }
-
-  void SetOutput(const std::string &param_name,
-                 const std::vector<std::string> &args) {
-    need_update_ = true;
-    this->outputs_[param_name] = args;
-  }
-
-  std::string DebugString() { return this->Proto()->DebugString(); }
-
-  bool HasAttr(const std::string &name) const {
-    return attrs_.find(name) != attrs_.end();
-  }
-
-  framework::AttrType GetAttrType(const std::string &name) const {
-    auto it = attrs_.find(name);
-    PADDLE_ENFORCE(it != attrs_.end(), "Attribute %s is not found", name);
-    return static_cast<framework::AttrType>(it->second.which() - 1);
-  }
-
-  std::vector<std::string> AttrNames() const {
-    std::vector<std::string> retv;
-    retv.reserve(attrs_.size());
-    for (auto &attr : attrs_) {
-      retv.push_back(attr.first);
-    }
-    return retv;
-  }
-
-  void SetAttr(const std::string &name, const Attribute &v) {
-    this->attrs_[name] = v;
-    need_update_ = true;
-  }
-
-  void SetBlockAttr(const std::string &name, BlockDescBind &block);
-
-  Attribute GetAttr(const std::string &name) const {
-    auto it = attrs_.find(name);
-    PADDLE_ENFORCE(it != attrs_.end(), "Attribute %s is not found", name);
-    return it->second;
-  }
-
-  int GetBlockAttr(const std::string &name) const {
-    auto it = attrs_.find(name);
-    PADDLE_ENFORCE(it != attrs_.end(), "Attribute %s is not found", name);
-    return boost::get<BlockDesc *>(it->second)->idx();
-  }
-
- private:
-  struct SetAttrDescVisitor : public boost::static_visitor<void> {
-    explicit SetAttrDescVisitor(OpDesc::Attr *attr) : attr_(attr) {}
-    mutable OpDesc::Attr *attr_;
-    void operator()(int v) const { attr_->set_i(v); }
-    void operator()(float v) const { attr_->set_f(v); }
-    void operator()(const std::string &v) const { attr_->set_s(v); }
-    void operator()(bool b) const { attr_->set_b(b); }
-
-    void operator()(const std::vector<int> &v) const {
-      VectorToRepeated(v, attr_->mutable_ints());
-    }
-    void operator()(const std::vector<float> &v) const {
-      VectorToRepeated(v, attr_->mutable_floats());
-    }
-    void operator()(const std::vector<std::string> &v) const {
-      VectorToRepeated(v, attr_->mutable_strings());
-    }
-    void operator()(const std::vector<bool> &v) const {
-      VectorToRepeated(v, attr_->mutable_bools());
-    }
-    void operator()(BlockDesc *desc) const {
-      attr_->set_block_idx(desc->idx());
-    }
-    void operator()(boost::blank) const { PADDLE_THROW("Unexpected branch"); }
-  };
-
-  void Sync() {
-    if (need_update_) {
-      this->op_desc_.mutable_inputs()->Clear();
-      for (auto &ipt : inputs_) {
-        auto *input = op_desc_.add_inputs();
-        input->set_parameter(ipt.first);
-        VectorToRepeated(ipt.second, input->mutable_arguments());
-      }
-
-      this->op_desc_.mutable_outputs()->Clear();
-      for (auto &opt : outputs_) {
-        auto *output = op_desc_.add_outputs();
-        output->set_parameter(opt.first);
-        VectorToRepeated(opt.second, output->mutable_arguments());
-      }
-
-      this->op_desc_.mutable_attrs()->Clear();
-      for (auto &attr : attrs_) {
-        auto *attr_desc = op_desc_.add_attrs();
-        attr_desc->set_name(attr.first);
-        attr_desc->set_type(
-            static_cast<framework::AttrType>(attr.second.which() - 1));
-        boost::apply_visitor(SetAttrDescVisitor(attr_desc), attr.second);
-      }
-
-      need_update_ = false;
-    }
-  }
-
-  OpDesc op_desc_;
-  std::unordered_map<std::string, std::vector<std::string>> inputs_;
-  std::unordered_map<std::string, std::vector<std::string>> outputs_;
-  std::unordered_map<std::string, Attribute> attrs_;
-
-  // need_update_ indicate there some local changes not be synchronized. If
-  // local changes should be synchronized, need_update_ should be set to true.
-  bool need_update_{false};
-};
-
-class BlockDescBind {
- public:
-  BlockDescBind(ProgramDescBind *prog, BlockDesc *desc)
-      : prog_(prog), desc_(desc), need_update_(false) {}
-
-  BlockDescBind(const BlockDescBind &o) = delete;
-  BlockDescBind &operator=(const BlockDescBind &o) = delete;
-
-  int32_t ID() const { return desc_->idx(); }
-
-  int32_t Parent() const { return desc_->parent_idx(); }
-
-  VarDescBind *NewVar(py::bytes name_bytes) {
-    std::string name = name_bytes;
-    need_update_ = true;
-    auto it = vars_.find(name);
-    PADDLE_ENFORCE(it == vars_.end(), "Duplicated variable %s", name);
-    auto var = new VarDescBind(name);
-    vars_[name].reset(var);
-    return var;
-  }
-
-  VarDescBind *Var(py::bytes name_bytes) const {
-    std::string name = name_bytes;
-    auto it = vars_.find(name);
-    PADDLE_ENFORCE(it != vars_.end(),
-                   "Can not find variable %s in current block.", name);
-    return it->second.get();
-  }
-
-  std::vector<VarDescBind *> AllVars() const {
-    std::vector<VarDescBind *> res;
-    for (const auto &p : vars_) {
-      res.push_back(p.second.get());
-    }
-    return res;
-  }
-
-  BlockDescBind *ParentBlock() const;
-
-  OpDescBind *AppendOp() {
-    need_update_ = true;
-    ops_.emplace_back(new OpDescBind());
-    return ops_.back().get();
-  }
-
-  OpDescBind *PrependOp() {
-    need_update_ = true;
-    ops_.emplace_front(new OpDescBind());
-    return ops_.front().get();
-  }
-
-  std::vector<OpDescBind *> AllOps() const {
-    std::vector<OpDescBind *> res;
-    for (const auto &op : ops_) {
-      res.push_back(op.get());
-    }
-    return res;
-  }
-
-  void Sync() {
-    if (need_update_) {
-      auto &op_field = *this->desc_->mutable_ops();
-      op_field.Clear();
-      op_field.Reserve(static_cast<int>(ops_.size()));
-      for (auto &op_desc : ops_) {
-        op_field.AddAllocated(op_desc->Proto());
-      }
-      need_update_ = false;
-    }
-  }
-
-  BlockDesc *RawPtr() { return desc_; }
-
- private:
-  ProgramDescBind *prog_;  // not_own
-  BlockDesc *desc_;        // not_own
-  bool need_update_;
-
-  std::deque<std::unique_ptr<OpDescBind>> ops_;
-  std::unordered_map<std::string, std::unique_ptr<VarDescBind>> vars_;
-};
-
-using ProgDescMap =
-    std::unordered_map<ProgramDesc *, std::unique_ptr<ProgramDescBind>>;
-static ProgDescMap *g_bind_map = nullptr;
-
-class ProgramDescBind {
- public:
-  static ProgramDescBind &Instance(ProgramDesc *prog) {
-    if (g_bind_map == nullptr) {
-      g_bind_map = new ProgDescMap();
-    }
-    auto &map = *g_bind_map;
-    auto &ptr = map[prog];
-
-    if (ptr == nullptr) {
-      ptr.reset(new ProgramDescBind(prog));
-    }
-    return *ptr;
-  }
-  ProgramDescBind(const ProgramDescBind &o) = delete;
-  ProgramDescBind &operator=(const ProgramDescBind &o) = delete;
-
-  BlockDescBind *AppendBlock(const BlockDescBind &parent) {
-    auto *b = prog_->add_blocks();
-    b->set_parent_idx(parent.ID());
-    b->set_idx(prog_->blocks_size() - 1);
-    blocks_.emplace_back(new BlockDescBind(this, b));
-    return blocks_.back().get();
-  }
-
-  BlockDescBind *Block(size_t idx) { return blocks_[idx].get(); }
-
-  std::string DebugString() { return Proto()->DebugString(); }
-
-  size_t Size() const { return blocks_.size(); }
-
-  ProgramDesc *Proto() {
-    for (auto &block : blocks_) {
-      block->Sync();
-    }
-    return prog_;
-  }
-
- private:
-  explicit ProgramDescBind(ProgramDesc *prog) : prog_(prog) {
-    for (auto &block : *prog->mutable_blocks()) {
-      blocks_.emplace_back(new BlockDescBind(this, &block));
-    }
-  }
-
-  // Not owned
-  ProgramDesc *prog_;
-
-  std::vector<std::unique_ptr<BlockDescBind>> blocks_;
-};
-
-BlockDescBind *BlockDescBind::ParentBlock() const {
-  if (this->desc_->parent_idx() == -1) {
-    return nullptr;
-  }
-  return prog_->Block(static_cast<size_t>(this->desc_->parent_idx()));
-}
-
-void OpDescBind::SetBlockAttr(const std::string &name, BlockDescBind &block) {
-  BlockDesc *desc = block.RawPtr();
-  this->attrs_[name] = desc;
-}
-
 // Bind Methods
 void BindProgramDesc(py::module &m) {
   py::class_<ProgramDescBind>(m, "ProgramDesc", "")
@@ -503,9 +129,18 @@ void BindBlockDesc(py::module &m) {
            py::return_value_policy::reference)
       .def("prepend_op", &BlockDescBind::PrependOp,
            py::return_value_policy::reference)
-      .def("new_var", &BlockDescBind::NewVar,
+      .def("new_var",
+           [](BlockDescBind &self, py::bytes byte_name) {
+             std::string name = byte_name;
+             return self.NewVar(name);
+           },
+           py::return_value_policy::reference)
+      .def("var",
+           [](BlockDescBind &self, py::bytes byte_name) {
+             std::string name = byte_name;
+             return self.Var(name);
+           },
            py::return_value_policy::reference)
-      .def("var", &BlockDescBind::Var, py::return_value_policy::reference)
       .def("all_vars", &BlockDescBind::AllVars,
            py::return_value_policy::reference)
       .def("all_ops", &BlockDescBind::AllOps,
@@ -513,7 +148,7 @@ void BindBlockDesc(py::module &m) {
 }
 
 void BindVarDsec(py::module &m) {
-  py::enum_<framework::DataType>(m, "DataType", "")
+  py::enum_<DataType>(m, "DataType", "")
       .value("BOOL", DataType::BOOL)
       .value("INT16", DataType::INT16)
       .value("INT32", DataType::INT32)
@@ -523,15 +158,20 @@ void BindVarDsec(py::module &m) {
       .value("FP64", DataType::FP64);
 
   py::class_<VarDescBind>(m, "VarDesc", "")
-      .def("name", &VarDescBind::Name, py::return_value_policy::reference)
+      .def("name",
+           [](const VarDescBind &self) {
+             py::bytes name = self.Name();
+             return name;
+           },
+           py::return_value_policy::reference)
       .def("set_shape", &VarDescBind::SetShape)
       .def("set_data_type", &VarDescBind::SetDataType)
       .def("shape", &VarDescBind::Shape, py::return_value_policy::reference)
-      .def("data_type", &VarDescBind::DataType);
+      .def("data_type", &VarDescBind::GetDataType);
 }
 
 void BindOpDesc(py::module &m) {
-  py::enum_<framework::AttrType>(m, "AttrType", "")
+  py::enum_<AttrType>(m, "AttrType", "")
       .value("INT", AttrType::INT)
       .value("INTS", AttrType::INTS)
       .value("FLOAT", AttrType::FLOAT)

paddle/pybind/protobuf.h

@@ -17,7 +17,6 @@ limitations under the License. */
 #include <Python.h>
 #include <fstream>
 #include <vector>
-#include "paddle/framework/op_registry.h"
 #include "pybind11/numpy.h"
 #include "pybind11/pybind11.h"
 #include "pybind11/stl.h"