Move proto desc to framework

paddle/framework/CMakeLists.txt

@@ -18,6 +18,8 @@ cc_test(scope_test SRCS scope_test.cc DEPS scope)
 
 proto_library(framework_proto SRCS framework.proto)
 
+cc_library(var_desc SRCS var_desc.cc DEPS framework_proto)
+
 cc_library(attribute SRCS attribute.cc DEPS framework_proto)
 cc_library(op_proto_maker SRCS op_proto_maker.cc DEPS framework_proto attribute)
 cc_test(op_proto_maker_test SRCS op_proto_maker_test.cc DEPS op_proto_maker)

paddle/framework/block_desc.cc

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/framework/block_desc.h"
+#include "paddle/framework/op_desc.h"
+#include "paddle/framework/var_desc.h"
+
+namespace paddle {
+namespace framework {
+
+VarDescBind *BlockDescBind::NewVar(const std::string &name) {
+  need_update_ = true;
+  auto it = vars_.find(name);
+  PADDLE_ENFORCE(it == vars_.end(), "Duplicated variable %s", name);
+  auto var = new VarDescBind(name);
+  vars_[name].reset(var);
+  return var;
+}
+
+VarDescBind *BlockDescBind::Var(const std::string &name) const {
+  auto it = vars_.find(name);
+  PADDLE_ENFORCE(it != vars_.end(),
+                 "Can not find variable %s in current block.", name);
+  return it->second.get();
+}
+
+std::vector<VarDescBind *> BlockDescBind::AllVars() const {
+  std::vector<VarDescBind *> res;
+  for (const auto &p : vars_) {
+    res.push_back(p.second.get());
+  }
+  return res;
+}
+
+OpDescBind *BlockDescBind::AppendOp() {
+  need_update_ = true;
+  ops_.emplace_back(new OpDescBind());
+  return ops_.back().get();
+}
+
+OpDescBind *BlockDescBind::PrependOp() {
+  need_update_ = true;
+  ops_.emplace_front(new OpDescBind());
+  return ops_.front().get();
+}
+
+std::vector<OpDescBind *> BlockDescBind::AllOps() const {
+  std::vector<OpDescBind *> res;
+  for (const auto &op : ops_) {
+    res.push_back(op.get());
+  }
+  return res;
+}
+
+void BlockDescBind::Sync() {
+  if (need_update_) {
+    auto &op_field = *this->desc_->mutable_ops();
+    op_field.Clear();
+    op_field.Reserve(static_cast<int>(ops_.size()));
+    for (auto &op_desc : ops_) {
+      op_field.AddAllocated(op_desc->Proto());
+    }
+    need_update_ = false;
+  }
+}
+
+BlockDescBind *BlockDescBind::ParentBlock() const {
+  if (this->desc_->parent_idx() == -1) {
+    return nullptr;
+  }
+  return prog_->Block(static_cast<size_t>(this->desc_->parent_idx()));
+}
+
+void OpDescBind::SetBlockAttr(const std::string &name, BlockDescBind &block) {
+  BlockDesc *desc = block.RawPtr();
+  this->attrs_[name] = desc;
+}
+}
+}
\ No newline at end of file

paddle/framework/block_desc.h

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include <unordered_map>
+#include <vector>
+#include "paddle/framework/framework.pb.h"
+
+namespace paddle {
+namespace framework {
+
+class ProgramDescBind;
+class OpDescBind;
+class VarDescBind;
+
+// Each Protobuf Message, we provide a XXXBind class. In that class, we optimize
+// read/write speed. Only when we want the protobuf message, the local changes
+// will be synchronized (by `Sync` method).
+
+class BlockDescBind {
+ public:
+  BlockDescBind(ProgramDescBind *prog, BlockDesc *desc)
+      : prog_(prog), desc_(desc), need_update_(false) {}
+
+  BlockDescBind(const BlockDescBind &o) = delete;
+  BlockDescBind &operator=(const BlockDescBind &o) = delete;
+
+  int32_t ID() const { return desc_->idx(); }
+
+  int32_t Parent() const { return desc_->parent_idx(); }
+
+  VarDescBind *NewVar(const std::string &name_bytes);
+
+  VarDescBind *Var(const std::string &name_bytes) const;
+
+  std::vector<VarDescBind *> AllVars() const;
+
+  BlockDescBind *ParentBlock() const;
+
+  OpDescBind *AppendOp();
+
+  OpDescBind *PrependOp();
+
+  std::vector<OpDescBind *> AllOps() const;
+
+  void Sync();
+
+  BlockDesc *RawPtr() { return desc_; }
+
+ private:
+  ProgramDescBind *prog_;  // not_own
+  BlockDesc *desc_;        // not_own
+  bool need_update_;
+
+  std::deque<std::unique_ptr<OpDescBind>> ops_;
+  std::unordered_map<std::string, std::unique_ptr<VarDescBind>> vars_;
+};
+}
+}

paddle/framework/op_desc.cc

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/framework/op_desc.h"
+#include "paddle/frameword/block_desc.h"
+
+namespace paddle {
+namespace framework {
+
+OpDesc *OpDescBind::Proto() {
+  Sync();
+  return &op_desc_;
+}
+
+const std::vector<std::string> &OpDescBind::Input(
+    const std::string &name) const {
+  auto it = inputs_.find(name);
+  PADDLE_ENFORCE(it != inputs_.end(), "Input %s cannot be found in Op %s", name,
+                 Type());
+  return it->second;
+}
+
+std::vector<std::string> OpDescBind::InputNames() const {
+  std::vector<std::string> retv;
+  retv.reserve(this->inputs_.size());
+  for (auto &ipt : this->inputs_) {
+    retv.push_back(ipt.first);
+  }
+  return retv;
+}
+
+void OpDescBind::SetInput(const std::string &param_name,
+                          const std::vector<std::string> &args) {
+  need_update_ = true;
+  inputs_[param_name] = args;
+}
+
+const std::vector<std::string> &OpDescBind::Output(
+    const std::string &name) const {
+  auto it = outputs_.find(name);
+  PADDLE_ENFORCE(it != outputs_.end(), "Output %s cannot be found in Op %s",
+                 name, Type());
+  return it->second;
+}
+
+std::vector<std::string> OpDescBind::OutputNames() const {
+  std::vector<std::string> retv;
+  retv.reserve(this->outputs_.size());
+  for (auto &ipt : this->outputs_) {
+    retv.push_back(ipt.first);
+  }
+  return retv;
+}
+
+void OpDescBind::SetOutput(const std::string &param_name,
+                           const std::vector<std::string> &args) {
+  need_update_ = true;
+  this->outputs_[param_name] = args;
+}
+
+AttrType OpDescBind::GetAttrType(const std::string &name) const {
+  auto it = attrs_.find(name);
+  PADDLE_ENFORCE(it != attrs_.end(), "Attribute %s is not found", name);
+  return static_cast<AttrType>(it->second.which() - 1);
+}
+
+std::vector<std::string> OpDescBind::AttrNames() const {
+  std::vector<std::string> retv;
+  retv.reserve(attrs_.size());
+  for (auto &attr : attrs_) {
+    retv.push_back(attr.first);
+  }
+  return retv;
+}
+
+void OpDescBind::SetAttr(const std::string &name, const Attribute &v) {
+  this->attrs_[name] = v;
+  need_update_ = true;
+}
+
+Attribute OpDescBind::GetAttr(const std::string &name) const {
+  auto it = attrs_.find(name);
+  PADDLE_ENFORCE(it != attrs_.end(), "Attribute %s is not found", name);
+  return it->second;
+}
+
+int OpDescBind::GetBlockAttr(const std::string &name) const {
+  auto it = attrs_.find(name);
+  PADDLE_ENFORCE(it != attrs_.end(), "Attribute %s is not found", name);
+  return boost::get<BlockDesc *>(it->second)->idx();
+}
+
+void OpDescBind::Sync() {
+  if (need_update_) {
+    this->op_desc_.mutable_inputs()->Clear();
+    for (auto &ipt : inputs_) {
+      auto *input = op_desc_.add_inputs();
+      input->set_parameter(ipt.first);
+      VectorToRepeated(ipt.second, input->mutable_arguments());
+    }
+
+    this->op_desc_.mutable_outputs()->Clear();
+    for (auto &opt : outputs_) {
+      auto *output = op_desc_.add_outputs();
+      output->set_parameter(opt.first);
+      VectorToRepeated(opt.second, output->mutable_arguments());
+    }
+
+    this->op_desc_.mutable_attrs()->Clear();
+    for (auto &attr : attrs_) {
+      auto *attr_desc = op_desc_.add_attrs();
+      attr_desc->set_name(attr.first);
+      attr_desc->set_type(
+          static_cast<framework::AttrType>(attr.second.which() - 1));
+      boost::apply_visitor(SetAttrDescVisitor(attr_desc), attr.second);
+    }
+
+    need_update_ = false;
+  }
+}
+}
+}
\ No newline at end of file

paddle/framework/op_desc.h

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include <unordered_map>
+#include <vector>
+#include "paddle/framework/attribute.h"
+#include "paddle/framework/var_desc.h"
+
+namespace paddle {
+namespace framework {
+
+class BlockDescBind;
+
+class OpDescBind {
+ public:
+  OpDesc *Proto();
+
+  std::string Type() const { return op_desc_.type(); }
+
+  void SetType(const std::string &type) { op_desc_.set_type(type); }
+
+  const std::vector<std::string> &Input(const std::string &name) const;
+
+  std::vector<std::string> InputNames() const;
+
+  void SetInput(const std::string &param_name,
+                const std::vector<std::string> &args);
+
+  const std::vector<std::string> &Output(const std::string &name) const;
+
+  std::vector<std::string> OutputNames() const;
+
+  void SetOutput(const std::string &param_name,
+                 const std::vector<std::string> &args);
+
+  std::string DebugString() { return this->Proto()->DebugString(); }
+
+  bool HasAttr(const std::string &name) const {
+    return attrs_.find(name) != attrs_.end();
+  }
+
+  AttrType GetAttrType(const std::string &name) const;
+
+  std::vector<std::string> AttrNames() const;
+
+  void SetAttr(const std::string &name, const Attribute &v);
+
+  void SetBlockAttr(const std::string &name, BlockDescBind &block);
+
+  Attribute GetAttr(const std::string &name) const;
+
+  int GetBlockAttr(const std::string &name) const;
+
+ private:
+  struct SetAttrDescVisitor : public boost::static_visitor<void> {
+    explicit SetAttrDescVisitor(OpDesc::Attr *attr) : attr_(attr) {}
+    mutable OpDesc::Attr *attr_;
+    void operator()(int v) const { attr_->set_i(v); }
+    void operator()(float v) const { attr_->set_f(v); }
+    void operator()(const std::string &v) const { attr_->set_s(v); }
+    void operator()(bool b) const { attr_->set_b(b); }
+
+    void operator()(const std::vector<int> &v) const {
+      VectorToRepeated(v, attr_->mutable_ints());
+    }
+    void operator()(const std::vector<float> &v) const {
+      VectorToRepeated(v, attr_->mutable_floats());
+    }
+    void operator()(const std::vector<std::string> &v) const {
+      VectorToRepeated(v, attr_->mutable_strings());
+    }
+    void operator()(const std::vector<bool> &v) const {
+      VectorToRepeated(v, attr_->mutable_bools());
+    }
+    void operator()(BlockDesc *desc) const {
+      attr_->set_block_idx(desc->idx());
+    }
+    void operator()(boost::blank) const { PADDLE_THROW("Unexpected branch"); }
+  };
+
+  void Sync();
+
+  OpDesc op_desc_;
+  std::unordered_map<std::string, std::vector<std::string>> inputs_;
+  std::unordered_map<std::string, std::vector<std::string>> outputs_;
+  std::unordered_map<std::string, Attribute> attrs_;
+
+  // need_update_ indicate there some local changes not be synchronized. If
+  // local changes should be synchronized, need_update_ should be set to true.
+  bool need_update_{false};
+};
+}
+}

paddle/framework/program_desc.cc

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/farmework/block_desc.h"
+#include "paddle/framework/programe_desc.h"
+
+namespace paddle {
+namespace framework {
+
+using ProgDescMap =
+    std::unordered_map<ProgramDesc *, std::unique_ptr<ProgramDescBind>>;
+static ProgDescMap *g_bind_map = nullptr;
+
+ProgramDescBind &ProgramDescBind::Instance(ProgramDesc *prog) {
+  if (g_bind_map == nullptr) {
+    g_bind_map = new ProgDescMap();
+  }
+  auto &map = *g_bind_map;
+  auto &ptr = map[prog];
+
+  if (ptr == nullptr) {
+    ptr.reset(new ProgramDescBind(prog));
+  }
+  return *ptr;
+}
+
+BlockDescBind *ProgramDescBind::AppendBlock(const BlockDescBind &parent) {
+  auto *b = prog_->add_blocks();
+  b->set_parent_idx(parent.ID());
+  b->set_idx(prog_->blocks_size() - 1);
+  blocks_.emplace_back(new BlockDescBind(this, b));
+  return blocks_.back().get();
+}
+
+ProgramDesc *ProgramDescBind::Proto() {
+  for (auto &block : blocks_) {
+    block->Sync();
+  }
+  return prog_;
+}
+
+ProgramDescBind::ProgramDescBind(ProgramDesc *prog) {
+  prog_ = prog;
+  for (auto &block : *prog->mutable_blocks()) {
+    blocks_.emplace_back(new BlockDescBind(this, &block));
+  }
+}
+}
+}
\ No newline at end of file

paddle/framework/programe_desc.h

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include <vector>
+#include "paddle/framework/framework.pb.h"
+
+namespace paddle {
+namespace framework {
+
+class BlockDescBind;
+
+class ProgramDescBind {
+ public:
+  static ProgramDescBind &Instance(ProgramDesc *prog);
+
+  ProgramDescBind(const ProgramDescBind &o) = delete;
+  ProgramDescBind &operator=(const ProgramDescBind &o) = delete;
+
+  BlockDescBind *AppendBlock(const BlockDescBind &parent);
+
+  BlockDescBind *Block(size_t idx) { return blocks_[idx].get(); }
+
+  std::string DebugString() { return Proto()->DebugString(); }
+
+  size_t Size() const { return blocks_.size(); }
+
+  ProgramDesc *Proto();
+
+ private:
+  explicit ProgramDescBind(ProgramDesc *prog);
+
+  // Not owned
+  ProgramDesc *prog_;
+
+  std::vector<std::unique_ptr<BlockDescBind>> blocks_;
+};
+}
+}

paddle/framework/var_desc.cc

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/framework/var_desc.h"
+
+namespace paddle {
+namespace framework {
+
+void VarDescBind::SetShape(const std::vector<int64_t> &dims) {
+  VectorToRepeated(dims, desc_.mutable_lod_tensor()->mutable_dims());
+}
+
+void VarDescBind::SetDataType(enum DataType data_type) {
+  desc_.mutable_lod_tensor()->set_data_type(data_type);
+}
+
+std::vector<int64_t> VarDescBind::Shape() const {
+  return RepeatedToVector(desc_.lod_tensor().dims());
+}
+
+DataType VarDescBind::DataType() const {
+  return desc_.lod_tensor().data_type();
+}
+}
+}
\ No newline at end of file

paddle/framework/var_desc.h

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include <vector>
+#include "paddle/framework/framework.pb.h"
+
+namespace paddle {
+namespace framework {
+
+// convert between std::vector and protobuf repeated.
+template <typename T>
+inline std::vector<T> RepeatedToVector(
+    const google::protobuf::RepeatedField<T> &repeated_field) {
+  std::vector<T> ret;
+  ret.reserve(repeated_field.size());
+  std::copy(repeated_field.begin(), repeated_field.end(),
+            std::back_inserter(ret));
+  return ret;
+}
+
+template <typename T, typename RepeatedField>
+inline void VectorToRepeated(const std::vector<T> &vec,
+                             RepeatedField *repeated_field) {
+  repeated_field->Reserve(vec.size());
+  for (const auto &elem : vec) {
+    *repeated_field->Add() = elem;
+  }
+}
+
+// Specialize vector<bool>.
+template <typename RepeatedField>
+inline void VectorToRepeated(const std::vector<bool> &vec,
+                             RepeatedField *repeated_field) {
+  repeated_field->Reserve(vec.size());
+  for (auto elem : vec) {
+    *repeated_field->Add() = elem;
+  }
+}
+
+class VarDescBind {
+ public:
+  explicit VarDescBind(const std::string &name) { desc_.set_name(name); }
+
+  VarDesc *Proto() { return &desc_; }
+
+  std::string Name() const { return desc_.name(); }
+
+  void SetShape(const std::vector<int64_t> &dims);
+
+  void SetDataType(DataType data_type);
+
+  std::vector<int64_t> Shape() const;
+
+  DataType DataType() const;
+
+ private:
+  VarDesc desc_;
+};
+}
+}

paddle/pybind/protobuf.cc

@@ -15,7 +15,10 @@ limitations under the License. */
 #include "paddle/pybind/protobuf.h"
 #include <deque>
 #include <iostream>
-#include "paddle/framework/attribute.h"
+#include "paddle/framework/block_desc.h"
+#include "paddle/framework/op_desc.h"
+#include "paddle/framework/program_desc.h"
+#include "paddle/framework/var_desc.h"
 
 // Cast boost::variant for PyBind.
 // Copy from
@@ -91,424 +94,56 @@ struct type_caster<boost::variant<Args...>>
 namespace paddle {
 namespace pybind {
 
-using namespace paddle::framework;  // NOLINT
-
-// convert between std::vector and protobuf repeated.
-template <typename T>
-inline std::vector<T> RepeatedToVector(
-    const google::protobuf::RepeatedField<T> &repeated_field) {
-  std::vector<T> ret;
-  ret.reserve(repeated_field.size());
-  std::copy(repeated_field.begin(), repeated_field.end(),
-            std::back_inserter(ret));
-  return ret;
-}
-
-template <typename T, typename RepeatedField>
-inline void VectorToRepeated(const std::vector<T> &vec,
-                             RepeatedField *repeated_field) {
-  repeated_field->Reserve(vec.size());
-  for (const auto &elem : vec) {
-    *repeated_field->Add() = elem;
-  }
-}
-
-// Specialize vector<bool>.
-template <typename RepeatedField>
-inline void VectorToRepeated(const std::vector<bool> &vec,
-                             RepeatedField *repeated_field) {
-  repeated_field->Reserve(vec.size());
-  for (auto elem : vec) {
-    *repeated_field->Add() = elem;
-  }
-}
-
-class ProgramDescBind;
-class OpDescBind;
-class BlockDescBind;
-class VarDescBind;
-
-// Each Protobuf Message, we provide a XXXBind class. In that class, we optimize
-// read/write speed. Only when we want the protobuf message, the local changes
-// will be synchronized (by `Sync` method).
-class VarDescBind {
- public:
-  explicit VarDescBind(const std::string &name) { desc_.set_name(name); }
-
-  VarDesc *Proto() { return &desc_; }
-
-  py::bytes Name() const { return desc_.name(); }
-
-  void SetShape(const std::vector<int64_t> &dims) {
-    VectorToRepeated(dims, desc_.mutable_lod_tensor()->mutable_dims());
-  }
-
-  void SetDataType(framework::DataType data_type) {
-    desc_.mutable_lod_tensor()->set_data_type(data_type);
-  }
-
-  std::vector<int64_t> Shape() const {
-    return RepeatedToVector(desc_.lod_tensor().dims());
-  }
-
-  framework::DataType DataType() const {
-    return desc_.lod_tensor().data_type();
-  }
-
- private:
-  VarDesc desc_;
-};
-
-class OpDescBind {
- public:
-  OpDesc *Proto() {
-    Sync();
-    return &op_desc_;
-  }
-
-  std::string Type() const { return op_desc_.type(); }
-
-  void SetType(const std::string &type) { op_desc_.set_type(type); }
-
-  const std::vector<std::string> &Input(const std::string &name) const {
-    auto it = inputs_.find(name);
-    PADDLE_ENFORCE(it != inputs_.end(), "Input %s cannot be found in Op %s",
-                   name, Type());
-    return it->second;
-  }
-
-  std::vector<std::string> InputNames() const {
-    std::vector<std::string> retv;
-    retv.reserve(this->inputs_.size());
-    for (auto &ipt : this->inputs_) {
-      retv.push_back(ipt.first);
-    }
-    return retv;
-  }
-
-  void SetInput(const std::string &param_name,
-                const std::vector<std::string> &args) {
-    need_update_ = true;
-    inputs_[param_name] = args;
-  }
-
-  const std::vector<std::string> &Output(const std::string &name) const {
-    auto it = outputs_.find(name);
-    PADDLE_ENFORCE(it != outputs_.end(), "Output %s cannot be found in Op %s",
-                   name, Type());
-    return it->second;
-  }
-
-  std::vector<std::string> OutputNames() const {
-    std::vector<std::string> retv;
-    retv.reserve(this->outputs_.size());
-    for (auto &ipt : this->outputs_) {
-      retv.push_back(ipt.first);
-    }
-    return retv;
-  }
-
-  void SetOutput(const std::string &param_name,
-                 const std::vector<std::string> &args) {
-    need_update_ = true;
-    this->outputs_[param_name] = args;
-  }
-
-  std::string DebugString() { return this->Proto()->DebugString(); }
-
-  bool HasAttr(const std::string &name) const {
-    return attrs_.find(name) != attrs_.end();
-  }
-
-  framework::AttrType GetAttrType(const std::string &name) const {
-    auto it = attrs_.find(name);
-    PADDLE_ENFORCE(it != attrs_.end(), "Attribute %s is not found", name);
-    return static_cast<framework::AttrType>(it->second.which() - 1);
-  }
-
-  std::vector<std::string> AttrNames() const {
-    std::vector<std::string> retv;
-    retv.reserve(attrs_.size());
-    for (auto &attr : attrs_) {
-      retv.push_back(attr.first);
-    }
-    return retv;
-  }
-
-  void SetAttr(const std::string &name, const Attribute &v) {
-    this->attrs_[name] = v;
-    need_update_ = true;
-  }
-
-  void SetBlockAttr(const std::string &name, BlockDescBind &block);
-
-  Attribute GetAttr(const std::string &name) const {
-    auto it = attrs_.find(name);
-    PADDLE_ENFORCE(it != attrs_.end(), "Attribute %s is not found", name);
-    return it->second;
-  }
-
-  int GetBlockAttr(const std::string &name) const {
-    auto it = attrs_.find(name);
-    PADDLE_ENFORCE(it != attrs_.end(), "Attribute %s is not found", name);
-    return boost::get<BlockDesc *>(it->second)->idx();
-  }
-
- private:
-  struct SetAttrDescVisitor : public boost::static_visitor<void> {
-    explicit SetAttrDescVisitor(OpDesc::Attr *attr) : attr_(attr) {}
-    mutable OpDesc::Attr *attr_;
-    void operator()(int v) const { attr_->set_i(v); }
-    void operator()(float v) const { attr_->set_f(v); }
-    void operator()(const std::string &v) const { attr_->set_s(v); }
-    void operator()(bool b) const { attr_->set_b(b); }
-
-    void operator()(const std::vector<int> &v) const {
-      VectorToRepeated(v, attr_->mutable_ints());
-    }
-    void operator()(const std::vector<float> &v) const {
-      VectorToRepeated(v, attr_->mutable_floats());
-    }
-    void operator()(const std::vector<std::string> &v) const {
-      VectorToRepeated(v, attr_->mutable_strings());
-    }
-    void operator()(const std::vector<bool> &v) const {
-      VectorToRepeated(v, attr_->mutable_bools());
-    }
-    void operator()(BlockDesc *desc) const {
-      attr_->set_block_idx(desc->idx());
-    }
-    void operator()(boost::blank) const { PADDLE_THROW("Unexpected branch"); }
-  };
-
-  void Sync() {
-    if (need_update_) {
-      this->op_desc_.mutable_inputs()->Clear();
-      for (auto &ipt : inputs_) {
-        auto *input = op_desc_.add_inputs();
-        input->set_parameter(ipt.first);
-        VectorToRepeated(ipt.second, input->mutable_arguments());
-      }
-
-      this->op_desc_.mutable_outputs()->Clear();
-      for (auto &opt : outputs_) {
-        auto *output = op_desc_.add_outputs();
-        output->set_parameter(opt.first);
-        VectorToRepeated(opt.second, output->mutable_arguments());
-      }
-
-      this->op_desc_.mutable_attrs()->Clear();
-      for (auto &attr : attrs_) {
-        auto *attr_desc = op_desc_.add_attrs();
-        attr_desc->set_name(attr.first);
-        attr_desc->set_type(
-            static_cast<framework::AttrType>(attr.second.which() - 1));
-        boost::apply_visitor(SetAttrDescVisitor(attr_desc), attr.second);
-      }
-
-      need_update_ = false;
-    }
-  }
-
-  OpDesc op_desc_;
-  std::unordered_map<std::string, std::vector<std::string>> inputs_;
-  std::unordered_map<std::string, std::vector<std::string>> outputs_;
-  std::unordered_map<std::string, Attribute> attrs_;
-
-  // need_update_ indicate there some local changes not be synchronized. If
-  // local changes should be synchronized, need_update_ should be set to true.
-  bool need_update_{false};
-};
-
-class BlockDescBind {
- public:
-  BlockDescBind(ProgramDescBind *prog, BlockDesc *desc)
-      : prog_(prog), desc_(desc), need_update_(false) {}
-
-  BlockDescBind(const BlockDescBind &o) = delete;
-  BlockDescBind &operator=(const BlockDescBind &o) = delete;
-
-  int32_t ID() const { return desc_->idx(); }
-
-  int32_t Parent() const { return desc_->parent_idx(); }
-
-  VarDescBind *NewVar(py::bytes name_bytes) {
-    std::string name = name_bytes;
-    need_update_ = true;
-    auto it = vars_.find(name);
-    PADDLE_ENFORCE(it == vars_.end(), "Duplicated variable %s", name);
-    auto var = new VarDescBind(name);
-    vars_[name].reset(var);
-    return var;
-  }
-
-  VarDescBind *Var(py::bytes name_bytes) const {
-    std::string name = name_bytes;
-    auto it = vars_.find(name);
-    PADDLE_ENFORCE(it != vars_.end(),
-                   "Can not find variable %s in current block.", name);
-    return it->second.get();
-  }
-
-  std::vector<VarDescBind *> AllVars() const {
-    std::vector<VarDescBind *> res;
-    for (const auto &p : vars_) {
-      res.push_back(p.second.get());
-    }
-    return res;
-  }
-
-  BlockDescBind *ParentBlock() const;
-
-  OpDescBind *AppendOp() {
-    need_update_ = true;
-    ops_.emplace_back(new OpDescBind());
-    return ops_.back().get();
-  }
-
-  OpDescBind *PrependOp() {
-    need_update_ = true;
-    ops_.emplace_front(new OpDescBind());
-    return ops_.front().get();
-  }
-
-  std::vector<OpDescBind *> AllOps() const {
-    std::vector<OpDescBind *> res;
-    for (const auto &op : ops_) {
-      res.push_back(op.get());
-    }
-    return res;
-  }
-
-  void Sync() {
-    if (need_update_) {
-      auto &op_field = *this->desc_->mutable_ops();
-      op_field.Clear();
-      op_field.Reserve(static_cast<int>(ops_.size()));
-      for (auto &op_desc : ops_) {
-        op_field.AddAllocated(op_desc->Proto());
-      }
-      need_update_ = false;
-    }
-  }
-
-  BlockDesc *RawPtr() { return desc_; }
-
- private:
-  ProgramDescBind *prog_;  // not_own
-  BlockDesc *desc_;        // not_own
-  bool need_update_;
-
-  std::deque<std::unique_ptr<OpDescBind>> ops_;
-  std::unordered_map<std::string, std::unique_ptr<VarDescBind>> vars_;
-};
-
-using ProgDescMap =
-    std::unordered_map<ProgramDesc *, std::unique_ptr<ProgramDescBind>>;
-static ProgDescMap *g_bind_map = nullptr;
-
-class ProgramDescBind {
- public:
-  static ProgramDescBind &Instance(ProgramDesc *prog) {
-    if (g_bind_map == nullptr) {
-      g_bind_map = new ProgDescMap();
-    }
-    auto &map = *g_bind_map;
-    auto &ptr = map[prog];
-
-    if (ptr == nullptr) {
-      ptr.reset(new ProgramDescBind(prog));
-    }
-    return *ptr;
-  }
-  ProgramDescBind(const ProgramDescBind &o) = delete;
-  ProgramDescBind &operator=(const ProgramDescBind &o) = delete;
-
-  BlockDescBind *AppendBlock(const BlockDescBind &parent) {
-    auto *b = prog_->add_blocks();
-    b->set_parent_idx(parent.ID());
-    b->set_idx(prog_->blocks_size() - 1);
-    blocks_.emplace_back(new BlockDescBind(this, b));
-    return blocks_.back().get();
-  }
-
-  BlockDescBind *Block(size_t idx) { return blocks_[idx].get(); }
-
-  std::string DebugString() { return Proto()->DebugString(); }
-
-  size_t Size() const { return blocks_.size(); }
-
-  ProgramDesc *Proto() {
-    for (auto &block : blocks_) {
-      block->Sync();
-    }
-    return prog_;
-  }
-
- private:
-  explicit ProgramDescBind(ProgramDesc *prog) : prog_(prog) {
-    for (auto &block : *prog->mutable_blocks()) {
-      blocks_.emplace_back(new BlockDescBind(this, &block));
-    }
-  }
-
-  // Not owned
-  ProgramDesc *prog_;
-
-  std::vector<std::unique_ptr<BlockDescBind>> blocks_;
-};
-
-BlockDescBind *BlockDescBind::ParentBlock() const {
-  if (this->desc_->parent_idx() == -1) {
-    return nullptr;
-  }
-  return prog_->Block(static_cast<size_t>(this->desc_->parent_idx()));
-}
-
-void OpDescBind::SetBlockAttr(const std::string &name, BlockDescBind &block) {
-  BlockDesc *desc = block.RawPtr();
-  this->attrs_[name] = desc;
-}
-
 // Bind Methods
 void BindProgramDesc(py::module &m) {
-  py::class_<ProgramDescBind>(m, "ProgramDesc", "")
-      .def_static("instance",
-                  []() -> ProgramDescBind * {
-                    return &ProgramDescBind::Instance(&GetProgramDesc());
-                  },
-                  py::return_value_policy::reference)
+  py::class_<framework::ProgramDescBind>(m, "ProgramDesc", "")
+      .def_static(
+          "instance",
+          []() -> framework::ProgramDescBind * {
+            return &framework::ProgramDescBind::Instance(&GetProgramDesc());
+          },
+          py::return_value_policy::reference)
       .def_static("__create_program_desc__",
-                  []() -> ProgramDescBind * {
+                  []() -> framework::ProgramDescBind * {
                     // Only used for unit-test
                     auto *prog_desc = new ProgramDesc;
                     auto *block = prog_desc->mutable_blocks()->Add();
                     block->set_idx(0);
                     block->set_parent_idx(-1);
-                    return &ProgramDescBind::Instance(prog_desc);
+                    return &framework::ProgramDescBind::Instance(prog_desc);
                   },
                   py::return_value_policy::reference)
-      .def("append_block", &ProgramDescBind::AppendBlock,
+      .def("append_block", &framework::ProgramDescBind::AppendBlock,
            py::return_value_policy::reference)
-      .def("block", &ProgramDescBind::Block, py::return_value_policy::reference)
-      .def("__str__", &ProgramDescBind::DebugString)
-      .def("num_blocks", &ProgramDescBind::Size);
+      .def("block", &framework::ProgramDescBind::Block,
+           py::return_value_policy::reference)
+      .def("__str__", &framework::ProgramDescBind::DebugString)
+      .def("num_blocks", &framework::ProgramDescBind::Size);
 }
 
 void BindBlockDesc(py::module &m) {
-  py::class_<BlockDescBind>(m, "BlockDesc", "")
-      .def_property_readonly("id", &BlockDescBind::ID)
-      .def_property_readonly("parent", &BlockDescBind::Parent)
-      .def("append_op", &BlockDescBind::AppendOp,
+  py::class_<framework::BlockDescBind>(m, "BlockDesc", "")
+      .def_property_readonly("id", &framework::BlockDescBind::ID)
+      .def_property_readonly("parent", &framework::BlockDescBind::Parent)
+      .def("append_op", &framework::BlockDescBind::AppendOp,
+           py::return_value_policy::reference)
+      .def("prepend_op", &framework::BlockDescBind::PrependOp,
            py::return_value_policy::reference)
-      .def("prepend_op", &BlockDescBind::PrependOp,
+      .def("new_var",
+           [](framework::BlockDescBind &self, py::bytes byte_name) {
+             std::string name = byte_name;
+             return self.NewVar(name);
+           },
            py::return_value_policy::reference)
-      .def("new_var", &BlockDescBind::NewVar,
+      .def("var",
+           [](framework::BlockDescBind &self, py::bytes byte_name) {
+             std::string name = byte_name;
+             return self.Var(name);
+           },
            py::return_value_policy::reference)
-      .def("var", &BlockDescBind::Var, py::return_value_policy::reference)
-      .def("all_vars", &BlockDescBind::AllVars,
+      .def("all_vars", &framework::BlockDescBind::AllVars,
            py::return_value_policy::reference)
-      .def("all_ops", &BlockDescBind::AllOps,
+      .def("all_ops", &framework::BlockDescBind::AllOps,
            py::return_value_policy::reference);
 }
 
@@ -522,12 +157,18 @@ void BindVarDsec(py::module &m) {
       .value("FP32", DataType::FP32)
       .value("FP64", DataType::FP64);
 
-  py::class_<VarDescBind>(m, "VarDesc", "")
-      .def("name", &VarDescBind::Name, py::return_value_policy::reference)
-      .def("set_shape", &VarDescBind::SetShape)
-      .def("set_data_type", &VarDescBind::SetDataType)
-      .def("shape", &VarDescBind::Shape, py::return_value_policy::reference)
-      .def("data_type", &VarDescBind::DataType);
+  py::class_<framework::VarDescBind>(m, "VarDesc", "")
+      .def("name",
+           [](const framework::framework::VarDescBind &self) {
+             py::bytes name = self.Name();
+             return name;
+           },
+           py::return_value_policy::reference)
+      .def("set_shape", &framework::VarDescBind::SetShape)
+      .def("set_data_type", &framework::VarDescBind::SetDataType)
+      .def("shape", &framework::VarDescBind::Shape,
+           py::return_value_policy::reference)
+      .def("data_type", &framework::VarDescBind::DataType);
 }
 
 void BindOpDesc(py::module &m) {
@@ -542,24 +183,24 @@ void BindOpDesc(py::module &m) {
       .value("BOOLS", AttrType::BOOLEANS)
       .value("BLOCK", AttrType::BLOCK);
 
-  py::class_<OpDescBind> op_desc(m, "OpDesc", "");
-  op_desc.def("type", &OpDescBind::Type)
-      .def("set_type", &OpDescBind::SetType)
-      .def("input", &OpDescBind::Input)
-      .def("input_names", &OpDescBind::InputNames)
-      .def("set_input", &OpDescBind::SetInput)
-      .def("output", &OpDescBind::Output)
-      .def("output_names", &OpDescBind::OutputNames)
-      .def("set_output", &OpDescBind::SetOutput)
-      .def("__str__", &OpDescBind::DebugString)
-      .def("__repr__", &OpDescBind::DebugString)
-      .def("has_attr", &OpDescBind::HasAttr)
-      .def("attr_type", &OpDescBind::GetAttrType)
-      .def("attr_names", &OpDescBind::AttrNames)
-      .def("set_attr", &OpDescBind::SetAttr)
-      .def("attr", &OpDescBind::GetAttr)
-      .def("set_block_attr", &OpDescBind::SetBlockAttr)
-      .def("get_block_attr", &OpDescBind::GetBlockAttr);
+  py::class_<framework::OpDescBind> op_desc(m, "OpDesc", "");
+  op_desc.def("type", &framework::OpDescBind::Type)
+      .def("set_type", &framework::OpDescBind::SetType)
+      .def("input", &framework::OpDescBind::Input)
+      .def("input_names", &framework::OpDescBind::InputNames)
+      .def("set_input", &framework::OpDescBind::SetInput)
+      .def("output", &framework::OpDescBind::Output)
+      .def("output_names", &framework::OpDescBind::OutputNames)
+      .def("set_output", &framework::OpDescBind::SetOutput)
+      .def("__str__", &framework::OpDescBind::DebugString)
+      .def("__repr__", &framework::OpDescBind::DebugString)
+      .def("has_attr", &framework::OpDescBind::HasAttr)
+      .def("attr_type", &framework::OpDescBind::GetAttrType)
+      .def("attr_names", &framework::OpDescBind::AttrNames)
+      .def("set_attr", &framework::OpDescBind::SetAttr)
+      .def("attr", &framework::OpDescBind::GetAttr)
+      .def("set_block_attr", &framework::OpDescBind::SetBlockAttr)
+      .def("get_block_attr", &framework::OpDescBind::GetBlockAttr);
 }
 
 }  // namespace pybind

paddle/pybind/protobuf.h

@@ -17,7 +17,6 @@ limitations under the License. */
 #include <Python.h>
 #include <fstream>
 #include <vector>
-#include "paddle/framework/op_registry.h"
 #include "pybind11/numpy.h"
 #include "pybind11/pybind11.h"
 #include "pybind11/stl.h"