From 8a3a01ee564a516c7d1b44df3b28040b5b69974f Mon Sep 17 00:00:00 2001 From: liuruilong Date: Sat, 26 May 2018 21:05:00 +0800 Subject: [PATCH] replace protobuf with protobuf-c --- src/framework/attribute.h | 78 ++++++++ src/framework/program/block_desc.cpp | 13 +- src/framework/program/block_desc.h | 6 +- src/framework/program/op_desc.cpp | 32 +-- src/framework/program/op_desc.h | 4 +- src/framework/program/program_desc.cpp | 24 ++- src/framework/program/program_desc.h | 5 +- src/framework/program/tensor_desc.cpp | 5 + src/framework/program/tensor_desc.h | 75 ++++++++ src/framework/program/var_desc.cpp | 1 - src/framework/program/var_desc.h | 151 +++++++++++---- src/io.cpp | 257 ++++++++----------------- src/io.h | 2 +- test/executor_for_test.h | 1 + 14 files changed, 398 insertions(+), 256 deletions(-) create mode 100644 src/framework/program/tensor_desc.cpp create mode 100644 src/framework/program/tensor_desc.h diff --git a/src/framework/attribute.h b/src/framework/attribute.h index bf6b084fba..c0c19e1dfb 100644 --- a/src/framework/attribute.h +++ b/src/framework/attribute.h @@ -15,8 +15,10 @@ limitations under the License. */ #pragma once #include "common/log.h" +#include "common/enforce.h" #include "common/variant.h" #include "framework/framework.pb.h" +#include "framework/framework.pb-c.h" namespace paddle_mobile { namespace framework { @@ -89,6 +91,82 @@ class Attribute { return attr; } + /* + * PADDLE_MOBILE__FRAMEWORK__PROTO__ATTR_TYPE__INT = 0, + PADDLE_MOBILE__FRAMEWORK__PROTO__ATTR_TYPE__FLOAT = 1, + PADDLE_MOBILE__FRAMEWORK__PROTO__ATTR_TYPE__STRING = 2, + PADDLE_MOBILE__FRAMEWORK__PROTO__ATTR_TYPE__INTS = 3, + PADDLE_MOBILE__FRAMEWORK__PROTO__ATTR_TYPE__FLOATS = 4, + PADDLE_MOBILE__FRAMEWORK__PROTO__ATTR_TYPE__STRINGS = 5, + PADDLE_MOBILE__FRAMEWORK__PROTO__ATTR_TYPE__BOOLEAN = 6, + PADDLE_MOBILE__FRAMEWORK__PROTO__ATTR_TYPE__BOOLEANS = 7, + PADDLE_MOBILE__FRAMEWORK__PROTO__ATTR_TYPE__BLOCK = 8, + PADDLE_MOBILE__FRAMEWORK__PROTO__ATTR_TYPE__LONG = 9 + PROTOBUF_C__FORCE_ENUM_TO_BE_INT_SIZE(PADDLE_MOBILE__FRAMEWORK__PROTO__ATTR_TYPE) + * + * */ + static Attribute GetAttrValue(PaddleMobile__Framework__Proto__OpDesc__Attr *attr_desc) { + // std::cout << "begin get attr value" << std::endl; + Attribute attr; + switch (attr_desc->type) { + case PADDLE_MOBILE__FRAMEWORK__PROTO__ATTR_TYPE__BOOLEAN: { + attr.Set(attr_desc->b); + break; + } + case PADDLE_MOBILE__FRAMEWORK__PROTO__ATTR_TYPE__INT: { + attr.Set(attr_desc->i); + break; + } + case PADDLE_MOBILE__FRAMEWORK__PROTO__ATTR_TYPE__FLOAT: { + attr.Set(attr_desc->f); + break; + } + case PADDLE_MOBILE__FRAMEWORK__PROTO__ATTR_TYPE__STRING: { + attr.Set(std::string(attr_desc->s)); + break; + } + case PADDLE_MOBILE__FRAMEWORK__PROTO__ATTR_TYPE__BOOLEANS: { + std::vector val(attr_desc->n_bools); + for (int i = 0; i < attr_desc->n_bools; ++i) { + val[i] = attr_desc->bools[i]; + } + attr.Set>(val); + break; + } + case PADDLE_MOBILE__FRAMEWORK__PROTO__ATTR_TYPE__INTS: { + std::vector val(attr_desc->n_ints); + for (int i = 0; i < attr_desc->n_ints; ++i) { + val[i] = attr_desc->ints[i]; + } + attr.Set>(val); + break; + } + case PADDLE_MOBILE__FRAMEWORK__PROTO__ATTR_TYPE__FLOATS: { + std::vector val(attr_desc->n_floats); + for (int i = 0; i < attr_desc->n_floats; ++i) { + val[i] = attr_desc->floats[i]; + } + attr.Set>(val); + break; + } + case PADDLE_MOBILE__FRAMEWORK__PROTO__ATTR_TYPE__STRINGS: { + std::vector val(attr_desc->n_strings); + for (int i = 0; i < attr_desc->n_strings; ++i) { + val[i] = attr_desc->strings[i]; + } + attr.Set>(val); + break; + } + case PADDLE_MOBILE__FRAMEWORK__PROTO__ATTR_TYPE__LONG: { + attr.Set(attr_desc->l); + break; + } + default: + PADDLE_MOBILE_THROW_EXCEPTION("attr type not support"); + } + return attr; + } + Attribute() {} template Attribute &Set(Args &&... args) { diff --git a/src/framework/program/block_desc.cpp b/src/framework/program/block_desc.cpp index 471cd0eda0..db522d0a51 100644 --- a/src/framework/program/block_desc.cpp +++ b/src/framework/program/block_desc.cpp @@ -33,14 +33,17 @@ std::vector> BlockDesc::Ops() const { return res; } -BlockDesc::BlockDesc(const proto::BlockDesc &desc) - : index_(desc.idx()), parent_index_(desc.parent_idx()) { - for (const proto::VarDesc &var_desc : desc.vars()) { - vars_[var_desc.name()].reset(new VarDesc(var_desc)); +BlockDesc::BlockDesc(PaddleMobile__Framework__Proto__BlockDesc *desc): index_(desc->idx), parent_index_(desc->idx) { + for (int i = 0; i < desc->n_vars; ++i) { + PaddleMobile__Framework__Proto__VarDesc *var_desc = desc->vars[i]; + vars_[std::string(var_desc->name)] = std::shared_ptr(new VarDesc(var_desc)); } - for (const proto::OpDesc &op_desc : desc.ops()) { + + for (int j = 0; j < desc->n_ops; ++j) { + PaddleMobile__Framework__Proto__OpDesc *op_desc = desc->ops[j]; ops_.emplace_back(new framework::OpDesc(op_desc)); } + } } // namespace framework diff --git a/src/framework/program/block_desc.h b/src/framework/program/block_desc.h index 4b0fb8b865..27c37eba09 100644 --- a/src/framework/program/block_desc.h +++ b/src/framework/program/block_desc.h @@ -15,9 +15,10 @@ limitations under the License. */ #pragma once #include "framework/framework.pb.h" -#include "framework/paddle_mobile_object.h" +#include "framework/framework.pb-c.h" #include "framework/program/op_desc.h" #include "framework/program/var_desc.h" +#include "framework/paddle_mobile_object.h" namespace paddle_mobile { namespace framework { @@ -26,8 +27,7 @@ class BlockDesc : PaddleMobileObject { public: friend class Node; friend class ProgramOptimize; - BlockDesc(const proto::BlockDesc &desc); - + BlockDesc(PaddleMobile__Framework__Proto__BlockDesc *desc); BlockDesc(const BlockDesc &block_desc) : index_(block_desc.index_), parent_index_(block_desc.parent_index_) { for (auto &op_desc : block_desc.ops_) { diff --git a/src/framework/program/op_desc.cpp b/src/framework/program/op_desc.cpp index 5bd8a0fc26..c8cbe3b608 100644 --- a/src/framework/program/op_desc.cpp +++ b/src/framework/program/op_desc.cpp @@ -20,28 +20,28 @@ limitations under the License. */ namespace paddle_mobile { namespace framework { -OpDesc::OpDesc(const proto::OpDesc &desc) : type_(desc.type()) { - for (int i = 0; i < desc.inputs_size(); ++i) { - const proto::OpDesc::Var &var = desc.inputs(i); - std::vector &args = inputs_[var.parameter()]; - int arg_size = var.arguments_size(); - for (int j = 0; j < arg_size; ++j) { - args.push_back(var.arguments(j)); +OpDesc::OpDesc(PaddleMobile__Framework__Proto__OpDesc *desc) { + this->type_ = std::string(desc->type); + for (int i = 0; i < desc->n_inputs; ++i) { + PaddleMobile__Framework__Proto__OpDesc__Var *var = desc->inputs[i]; + std::vector &args = inputs_[std::string(var->parameter)]; + for (int j = 0; j < var->n_arguments; ++j) { + args.emplace_back(std::string(var->arguments[j])); } } - for (int i = 0; i < desc.outputs_size(); ++i) { - const proto::OpDesc::Var &var = desc.outputs(i); - std::vector &args = outputs_[var.parameter()]; - int arg_size = var.arguments_size(); - for (int j = 0; j < arg_size; ++j) { - args.push_back(var.arguments(j)); + for (int i = 0; i < desc->n_outputs; ++i) { + PaddleMobile__Framework__Proto__OpDesc__Var *var = desc->outputs[i]; + std::vector &args = outputs_[std::string(var->parameter)]; + for (int j = 0; j < var->n_arguments; ++j) { + args.emplace_back(std::string(var->arguments[j])); } } - for (const proto::OpDesc::Attr &attr : desc.attrs()) { - std::string attr_name = attr.name(); - if (attr.type() != proto::AttrType::BLOCK) { + for (int k = 0; k < desc->n_attrs; ++k) { + PaddleMobile__Framework__Proto__OpDesc__Attr *attr = desc->attrs[k]; + std::string attr_name(attr->name); + if (attr->type != PADDLE_MOBILE__FRAMEWORK__PROTO__ATTR_TYPE__BLOCK) { attrs_[attr_name] = Attribute::GetAttrValue(attr); } } diff --git a/src/framework/program/op_desc.h b/src/framework/program/op_desc.h index df32c9f64a..e5b0665ee5 100644 --- a/src/framework/program/op_desc.h +++ b/src/framework/program/op_desc.h @@ -20,6 +20,7 @@ limitations under the License. */ #include "common/log.h" #include "common/type_define.h" #include "framework/framework.pb.h" +#include "framework/framework.pb-c.h" #include "framework/paddle_mobile_object.h" namespace paddle_mobile { @@ -30,8 +31,7 @@ class OpDesc : PaddleMobileObject { friend class ProgramOptimize; friend class FusionOpMatcher; friend class Node; - explicit OpDesc(const proto::OpDesc &desc); - + explicit OpDesc(PaddleMobile__Framework__Proto__OpDesc *op_desc); OpDesc(const OpDesc &op_desc) : type_(op_desc.type_) { this->inputs_ = op_desc.inputs_; this->outputs_ = op_desc.outputs_; diff --git a/src/framework/program/program_desc.cpp b/src/framework/program/program_desc.cpp index 13d3ba6588..efcd7e8b14 100644 --- a/src/framework/program/program_desc.cpp +++ b/src/framework/program/program_desc.cpp @@ -16,14 +16,14 @@ limitations under the License. */ #include #include "program_desc.h" +#include "framework/program/tensor_desc.h" namespace paddle_mobile { namespace framework { -ProgramDesc::ProgramDesc(const proto::ProgramDesc &desc) { - for (auto &block_desc : desc.blocks()) { - // new framework::BlockDesc(block_desc) - blocks_.emplace_back(std::make_shared(block_desc)); +ProgramDesc::ProgramDesc(PaddleMobile__Framework__Proto__ProgramDesc *desc) { + for (int i = 0; i < desc->n_blocks; ++i) { + blocks_.emplace_back(std::make_shared(desc->blocks[i])); } } @@ -55,6 +55,22 @@ void ProgramDesc::Description(std::string header) { LOG(kLOG_DEBUG3) << "argument - " << attr.second; } } + + for (const auto &var_desc : block->Vars()) { + if (var_desc->Type() == VARTYPE_TYPE_LOD_TENSOR) { + LOG(kLOG_DEBUG1) << "var name: " << var_desc->Name(); + + const TensorDesc &tensor_desc = var_desc->Tensor_desc(); + + LOG(kLOG_DEBUG2) << "in var tensor desc dims size: " + << tensor_desc.Dims().size(); + for (int l = 0; l < tensor_desc.Dims().size(); ++l) { + LOG(kLOG_DEBUG3) << "var tensor desc dim " << l + << " value: " << tensor_desc.Dims()[l]; + } + } + } + } #endif } diff --git a/src/framework/program/program_desc.h b/src/framework/program/program_desc.h index 1bcd5b562c..6aa7dd44ce 100644 --- a/src/framework/program/program_desc.h +++ b/src/framework/program/program_desc.h @@ -17,6 +17,7 @@ limitations under the License. */ #include #include "common/types.h" +#include "framework/framework.pb-c.h" #include "framework/paddle_mobile_object.h" #include "framework/program/block_desc.h" @@ -27,8 +28,10 @@ class ProgramDesc : PaddleMobileObject { public: friend class Node; friend class ProgramOptimize; - explicit ProgramDesc(const proto::ProgramDesc &desc); + explicit ProgramDesc(PaddleMobile__Framework__Proto__ProgramDesc *desc); + std::shared_ptr Block(size_t idx); + const std::vector> &Blocks() { return blocks_; } ProgramDesc(const ProgramDesc &program_desc) { for (auto &block : program_desc.blocks_) { diff --git a/src/framework/program/tensor_desc.cpp b/src/framework/program/tensor_desc.cpp new file mode 100644 index 0000000000..94bd424890 --- /dev/null +++ b/src/framework/program/tensor_desc.cpp @@ -0,0 +1,5 @@ +// +// Created by liuRuiLong on 2018/5/26. +// + +#include "tensor_desc.h" diff --git a/src/framework/program/tensor_desc.h b/src/framework/program/tensor_desc.h new file mode 100644 index 0000000000..e948273226 --- /dev/null +++ b/src/framework/program/tensor_desc.h @@ -0,0 +1,75 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include + +#include "framework/framework.pb-c.h" + +namespace paddle_mobile { +namespace framework { + +enum VarType_Type{ + VARTYPE_TYPE_BOOL = 0, + VARTYPE_TYPE_INT16 = 1, + VARTYPE_TYPE_INT32 = 2, + VARTYPE_TYPE_INT64 = 3, + VARTYPE_TYPE_FP16 = 4, + VARTYPE_TYPE_FP32 = 5, + VARTYPE_TYPE_FP64 = 6, + VARTYPE_TYPE_LOD_TENSOR = 7, + VARTYPE_TYPE_SELECTED_ROWS = 8, + VARTYPE_TYPE_FEED_MINIBATCH = 9, + VARTYPE_TYPE_FETCH_LIST = 10, + VARTYPE_TYPE_STEP_SCOPES = 11, + VARTYPE_TYPE_STEP_LOD_RANK_TABLE = 12, + VARTYPE_TYPE_STEP_LOD_TENSOR_ARRAY = 13, + VARTYPE_TYPE_STEP_PLACE_LIST = 14, + VARTYPE_TYPE_READER = 15, + VARTYPE_TYPE_CHANNEL = 16, + VARTYPE_TYPE_RAW = 17, + VARTYPE_TYPE_TUPLE = 18 +}; + +class TensorDesc { + public: + TensorDesc() = default; + TensorDesc(const TensorDesc &desc) { + this->dims_ = desc.dims_; + this->data_type_ = desc.data_type_; + } + + TensorDesc(PaddleMobile__Framework__Proto__VarType__TensorDesc *desc) { + for (int i = 0; i < desc->n_dims; ++i) { + int64_t d = desc->dims[i]; + dims_.emplace_back(d); + } + data_type_ = (VarType_Type)desc->data_type; + } + + std::vector Dims() const { + return dims_; + }; + VarType_Type DataType() const { + return data_type_; + } + private: + std::vector dims_; + VarType_Type data_type_; + +}; + +} +} diff --git a/src/framework/program/var_desc.cpp b/src/framework/program/var_desc.cpp index d5919ba277..fde8fddcf3 100644 --- a/src/framework/program/var_desc.cpp +++ b/src/framework/program/var_desc.cpp @@ -18,7 +18,6 @@ namespace paddle_mobile { namespace framework { -VarDesc::VarDesc(const proto::VarDesc &desc) : desc_(desc) {} } // namespace framework } // namespace paddle_mobile diff --git a/src/framework/program/var_desc.h b/src/framework/program/var_desc.h index 827e7ac00c..6c0b42ba24 100644 --- a/src/framework/program/var_desc.h +++ b/src/framework/program/var_desc.h @@ -15,71 +15,140 @@ limitations under the License. */ #pragma once #include "framework/framework.pb.h" +#include "framework/framework.pb-c.h" +#include "framework/program/tensor_desc.h" #include "framework/paddle_mobile_object.h" namespace paddle_mobile { namespace framework { -class VarDesc { - public: - VarDesc(const proto::VarDesc &desc); +/* - VarDesc(const VarDesc &var_desc) : desc_(var_desc.desc_) {} +PADDLE_MOBILE__FRAMEWORK__PROTO__VAR_TYPE__TYPE__BOOL = 0, + PADDLE_MOBILE__FRAMEWORK__PROTO__VAR_TYPE__TYPE__INT16 = 1, + PADDLE_MOBILE__FRAMEWORK__PROTO__VAR_TYPE__TYPE__INT32 = 2, + PADDLE_MOBILE__FRAMEWORK__PROTO__VAR_TYPE__TYPE__INT64 = 3, + PADDLE_MOBILE__FRAMEWORK__PROTO__VAR_TYPE__TYPE__FP16 = 4, + PADDLE_MOBILE__FRAMEWORK__PROTO__VAR_TYPE__TYPE__FP32 = 5, + PADDLE_MOBILE__FRAMEWORK__PROTO__VAR_TYPE__TYPE__FP64 = 6, - std::string Name() const { return desc_.name(); } + PADDLE_MOBILE__FRAMEWORK__PROTO__VAR_TYPE__TYPE__LOD_TENSOR = 7, + PADDLE_MOBILE__FRAMEWORK__PROTO__VAR_TYPE__TYPE__SELECTED_ROWS = 8, + PADDLE_MOBILE__FRAMEWORK__PROTO__VAR_TYPE__TYPE__FEED_MINIBATCH = 9, + PADDLE_MOBILE__FRAMEWORK__PROTO__VAR_TYPE__TYPE__FETCH_LIST = 10, + PADDLE_MOBILE__FRAMEWORK__PROTO__VAR_TYPE__TYPE__STEP_SCOPES = 11, + PADDLE_MOBILE__FRAMEWORK__PROTO__VAR_TYPE__TYPE__LOD_RANK_TABLE = 12, + PADDLE_MOBILE__FRAMEWORK__PROTO__VAR_TYPE__TYPE__LOD_TENSOR_ARRAY = 13, + PADDLE_MOBILE__FRAMEWORK__PROTO__VAR_TYPE__TYPE__PLACE_LIST = 14, + PADDLE_MOBILE__FRAMEWORK__PROTO__VAR_TYPE__TYPE__READER = 15, + PADDLE_MOBILE__FRAMEWORK__PROTO__VAR_TYPE__TYPE__CHANNEL = 16, - proto::VarType::Type GetType() const { return desc_.type().type(); } + PADDLE_MOBILE__FRAMEWORK__PROTO__VAR_TYPE__TYPE__RAW = 17, + PADDLE_MOBILE__FRAMEWORK__PROTO__VAR_TYPE__TYPE__TUPLE = 18 - bool Persistable() const { return desc_.persistable(); } - const proto::VarType::ChannelDesc &channel_desc() const { - switch (desc_.type().type()) { - case proto::VarType::CHANNEL: - return desc_.type().channel(); - default: - break; - } + */ + + +class VarDesc { + public: + VarDesc(const VarDesc &var_desc) { + this->data_type_ = var_desc.data_type_; + this->name_ = var_desc.name_; + this->persistable_ = var_desc.persistable_; + this->tensor_desc_ = var_desc.tensor_desc_; + this->type_ = var_desc.type_; + /* + * + * std::string name_; + bool persistable_; + TensorDesc tensor_desc_; + VarType_Type type_; + VarType_Type data_type_; + * */ + } + VarDesc(PaddleMobile__Framework__Proto__VarDesc *desc) { + type_ = (VarType_Type)desc->type->type; + name_ = std::string(desc->name); + persistable_ = (bool)desc->persistable; + + switch (type_) { + case VARTYPE_TYPE_SELECTED_ROWS: + tensor_desc_ = TensorDesc(desc->type->selected_rows); + break; + case VARTYPE_TYPE_LOD_TENSOR: + tensor_desc_ = TensorDesc(desc->type->lod_tensor->tensor); + break; + case VARTYPE_TYPE_STEP_LOD_TENSOR_ARRAY: + desc->type->tensor_array->tensor->data_type; + tensor_desc_ = TensorDesc(desc->type->tensor_array->tensor); - const proto::VarType::TensorDesc &tensor_desc() const { - switch (desc_.type().type()) { - case proto::VarType::SELECTED_ROWS: - return desc_.type().selected_rows(); - case proto::VarType::LOD_TENSOR: - return desc_.type().lod_tensor().tensor(); - case proto::VarType::LOD_TENSOR_ARRAY: - return desc_.type().tensor_array().tensor(); + break; default: break; } - } - - proto::VarType::Type GetDataType() const { - switch (desc_.type().type()) { - case proto::VarType::CHANNEL: - return channel_desc().data_type(); + switch (type_) { + case VARTYPE_TYPE_CHANNEL: + data_type_ = (VarType_Type)desc->type->channel->data_type; break; default: - return tensor_desc().data_type(); + data_type_ = tensor_desc_.DataType(); + break; + } - } - template - std::vector RepeatedToVector( - const google::protobuf::RepeatedField &repeated_field) const { - std::vector ret; - ret.reserve(repeated_field.size()); - std::copy(repeated_field.begin(), repeated_field.end(), - std::back_inserter(ret)); - return ret; } + std::string Name() const { return name_; } + + VarType_Type Type() const { return type_; } + + bool Persistable() const { return persistable_; } - std::vector GetShape() const { - return this->RepeatedToVector(tensor_desc().dims()); + const TensorDesc &Tensor_desc() const { + return tensor_desc_; } + // const proto::VarType::ChannelDesc &channel_desc() const { +// switch (desc_.type().type()) { +// case proto::VarType::CHANNEL: +// return desc_.type().channel(); +// default: +// break; +// } +// } + +// proto::VarType::Type GetDataType() const { +// switch (desc_.type().type()) { +// case proto::VarType::CHANNEL: +// return channel_desc().data_type(); +// break; +// default: +// return tensor_desc().data_type(); +// } +// } + +// template +// std::vector RepeatedToVector( +// const google::protobuf::RepeatedField &repeated_field) const { +// std::vector ret; +// ret.reserve(repeated_field.size()); +// std::copy(repeated_field.begin(), repeated_field.end(), +// std::back_inserter(ret)); +// return ret; +// } + +// std::vector GetShape() const { +// return this->RepeatedToVector(tensor_desc().dims()); +// } + private: - proto::VarDesc desc_; + std::string name_; + bool persistable_; + TensorDesc tensor_desc_; + VarType_Type type_; + VarType_Type data_type_; + }; } // namespace framework diff --git a/src/io.cpp b/src/io.cpp index eaab40b519..85a2342dc4 100644 --- a/src/io.cpp +++ b/src/io.cpp @@ -16,14 +16,16 @@ limitations under the License. */ #include #include -#include "common/enforce.h" #include "common/log.h" -#include "framework/framework.pb.h" -#include "framework/lod_tensor.h" -#include "framework/operator.h" -#include "framework/program/program_desc.h" +#include "common/enforce.h" #include "framework/scope.h" #include "framework/tensor.h" +#include "framework/operator.h" +#include "framework/lod_tensor.h" +#include "framework/framework.pb.h" +#include "framework/framework.pb-c.h" +#include "framework/program/var_desc.h" +#include "framework/program/program_desc.h" namespace paddle_mobile { @@ -39,12 +41,37 @@ void ReadBinaryFile(const std::string &filename, std::string *contents) { fin.close(); } +static size_t ReadBuffer (const char *file_name, uint8_t **out) { + printf("%s \n", file_name); + FILE *fp; + fp = fopen(file_name, "rb"); + PADDLE_MOBILE_ENFORCE(fp != NULL, "open failed !"); + + fseek(fp, 0, SEEK_END); + size_t size = ftell(fp); + rewind(fp); + + DLOG << "model size: " << size; + + *out = (uint8_t *)malloc(size); + + size_t cur_len = 0; + size_t nread; + while ((nread=fread(*out + cur_len, 1, size - cur_len, fp)) != 0) { + cur_len += nread; + } + fclose(fp); + return cur_len; +} + template -void Loader::LoadVar(framework::LoDTensor *tensor, +void Loader::LoadVar(framework::Variable *variable, const framework::VarDesc &var_desc, const std::string &file_path) { + auto tensor = variable->GetMutable(); std::ifstream is(file_path); PADDLE_MOBILE_ENFORCE(is.is_open(), "open file: %s failed", file_path.c_str()); + std::fpos pos; pos = is.tellg(); // save current position is.seekg(0, std::ios::end); @@ -81,39 +108,44 @@ void Loader::LoadVar(framework::LoDTensor *tensor, std::unique_ptr buf(new char[size]); is.read(reinterpret_cast(buf.get()), size); - framework::proto::VarType::TensorDesc desc; - desc.ParseFromArray(buf.get(), size); + const framework::TensorDesc &desc = var_desc.Tensor_desc(); + +// framework::TensorDesc &tensor_desc = variable-> +// PaddleMobile__Framework__Proto__ProgramDesc *c_program; +// uint8_t *proto_buf = NULL; +// size_t read_size = ReadBuffer(file_path.c_str(), &proto_buf); +// c_program = paddle_mobile__framework__proto__program_desc__unpack(NULL, read_size, buf); + +// paddle_mobile__framework__proto__var_type__tensor_desc__init() + int memory_size = 1; - for (auto l : desc.dims()) { + for (auto l : desc.Dims()) { memory_size *= l; } - std::vector dims; - dims.reserve(static_cast(desc.dims().size())); - std::copy(desc.dims().begin(), desc.dims().end(), std::back_inserter(dims)); - tensor->Resize(framework::make_ddim(dims)); + tensor->Resize(framework::make_ddim(desc.Dims())); void *memory = tensor; int type_size = 0; - switch (desc.data_type()) { - case framework::proto::VarType::FP16: + switch (desc.DataType()) { + case framework::VARTYPE_TYPE_FP16: type_size = 2; break; - case framework::proto::VarType::FP32: + case framework::VARTYPE_TYPE_FP32: type_size = 4; memory = tensor->mutable_data(); break; - case framework::proto::VarType::FP64: + case framework::VARTYPE_TYPE_FP64: type_size = 8; break; - case framework::proto::VarType::INT32: + case framework::VARTYPE_TYPE_INT32: type_size = 4; break; - case framework::proto::VarType::INT64: + case framework::VARTYPE_TYPE_INT64: type_size = 8; break; - case framework::proto::VarType::BOOL: + case framework::VARTYPE_TYPE_BOOL: type_size = 1; break; default: @@ -128,13 +160,20 @@ template const framework::Program Loader::Load( const std::string &dirname) { std::string model_filename = dirname + "/__model__"; - std::string program_desc_str; - ReadBinaryFile(model_filename, &program_desc_str); - framework::proto::ProgramDesc program_desc_proto; - program_desc_proto.ParseFromString(program_desc_str); + PaddleMobile__Framework__Proto__ProgramDesc *c_program; + uint8_t *buf = NULL; + size_t read_size = ReadBuffer(model_filename.c_str(), &buf); + + PADDLE_MOBILE_ENFORCE(buf != NULL, "read from __model__ is null"); + + c_program = paddle_mobile__framework__proto__program_desc__unpack(NULL, read_size, buf); + + PADDLE_MOBILE_ENFORCE(c_program != NULL, "program is null"); + + DLOG << "n_ops: " << (*c_program->blocks)->n_ops; std::shared_ptr originProgramDesc = - std::make_shared(program_desc_proto); + std::make_shared(c_program); framework::Program program; program.model_path = dirname; @@ -148,172 +187,26 @@ const framework::Program Loader::Load( for (const auto &block : originProgramDesc->Blocks()) { for (int i = 0; i < block->Vars().size(); ++i) { std::shared_ptr var_desc = block->Vars()[i]; +// DLOG << "var name-- " << var_desc->Name(); auto var = scope->Var(var_desc->Name()); - if (var_desc->GetType() == framework::proto::VarType::LOD_TENSOR) { + + if (var_desc->Type() == framework::VARTYPE_TYPE_LOD_TENSOR) { if (var_desc->Persistable() && - var_desc->GetType() != framework::proto::VarType::FEED_MINIBATCH && - var_desc->GetType() != framework::proto::VarType::FETCH_LIST) { - auto tensor = var->GetMutable(); - // to load - LoadVar(tensor, dirname + "/" + var_desc->Name()); + var_desc->Type() != framework::VARTYPE_TYPE_FEED_MINIBATCH && + var_desc->Type() != framework::VARTYPE_TYPE_FETCH_LIST) { +// DLOG << "to load var "; + LoadVar(var, *var_desc, dirname + "/" + var_desc->Name()); } + } else { // TODO(codeWorm): some. } } } -#ifdef PADDLE_MOBILE_DEBUG - for (const auto &block : program_desc_proto.blocks()) { - LOG(kLOG_DEBUG) << "block: " << block.idx(); - for (int j = 0; j < block.ops().size(); ++j) { - framework::proto::OpDesc op = block.ops()[j]; - LOG(kLOG_DEBUG1) << "op: " << op.type(); - for (int m = 0; m < op.inputs_size(); ++m) { - const framework::proto::OpDesc::Var &var = op.inputs(m); - LOG(kLOG_DEBUG2) << "input parameter: " << var.parameter(); - for (const auto &n : var.arguments()) { - LOG(kLOG_DEBUG3) << "argument - " << n; - } - } - - for (int y = 0; y < op.outputs_size(); ++y) { - const framework::proto::OpDesc::Var &var = op.outputs(y); - LOG(kLOG_DEBUG2) << "out parameter: " << var.parameter(); - for (const auto &z : var.arguments()) { - LOG(kLOG_DEBUG3) << "argument - " << z; - } - } - - for (const auto &attr : op.attrs()) { - LOG(kLOG_DEBUG2) << "attr name: " << attr.name(); - - switch (attr.type()) { - case framework::proto::AttrType::BOOLEAN: - LOG(kLOG_DEBUG3) << "boolen: " << attr.b(); - break; - case framework::proto::AttrType::INT: - LOG(kLOG_DEBUG3) << "int: " << attr.i(); - break; - case framework::proto::AttrType::FLOAT: - LOG(kLOG_DEBUG3) << "float: " << attr.f(); - case framework::proto::AttrType::STRING: - LOG(kLOG_DEBUG3) << "string: " << attr.s(); - case framework::proto::AttrType::BOOLEANS: - for (int y = 0; y < attr.bools_size(); ++y) { - LOG(kLOG_DEBUG3) << "bools: " << attr.bools(y); - } - case framework::proto::AttrType::LONG: - LOG(kLOG_DEBUG3) << "long: " << attr.l(); - case framework::proto::AttrType::FLOATS: - for (int y = 0; y < attr.floats_size(); ++y) { - LOG(kLOG_DEBUG3) << "floats: " << attr.floats(y); - } - case framework::proto::AttrType::INTS: - for (int y = 0; y < attr.ints_size(); ++y) { - LOG(kLOG_DEBUG3) << "ints: " << attr.ints(y); - } - case framework::proto::AttrType::STRINGS: - for (int y = 0; y < attr.strings_size(); ++y) { - LOG(kLOG_DEBUG3) << "strings: " << attr.strings(y); - } - case framework::proto::BLOCK: - break; - } - } - } - - for (const auto &var : block.vars()) { - if (var.type().type() == framework::proto::VarType::LOD_TENSOR) { - LOG(kLOG_DEBUG1) << "var name: " << var.name(); - const framework::proto::VarType::TensorDesc &tensor_desc = - var.type().lod_tensor().tensor(); - LOG(kLOG_DEBUG2) << "in var tensor desc dims size: " - << tensor_desc.dims().size(); - for (int l = 0; l < tensor_desc.dims().size(); ++l) { - LOG(kLOG_DEBUG3) << "var tensor desc dim " << l - << " value: " << tensor_desc.dims()[l]; - } - } - - if (var.persistable() && - var.type().type() != framework::proto::VarType::FEED_MINIBATCH && - var.type().type() != framework::proto::VarType::FETCH_LIST) { - std::string file_path = dirname + "/" + var.name(); - std::ifstream is(file_path); - PADDLE_MOBILE_ENFORCE(is.is_open(), "open file: %s failed", - file_path.c_str()); - std::fpos pos; - pos = is.tellg(); // save current position - is.seekg(0, std::ios::end); - is.seekg(pos); // restore saved position - - // 1. version - uint32_t version; - is.read(reinterpret_cast(&version), sizeof(version)); - - // 2 Lod information - uint64_t lod_level; - is.read(reinterpret_cast(&lod_level), sizeof(lod_level)); - for (uint64_t i = 0; i < lod_level; ++i) { - uint64_t size; - is.read(reinterpret_cast(&size), sizeof(size)); - std::vector tmp(size / sizeof(size_t)); - is.read(reinterpret_cast(tmp.data()), - static_cast(size)); - for (int j = 0; j < tmp.size(); ++j) { - } - } - - is.read(reinterpret_cast(&version), sizeof(version)); - - int32_t size; - is.read(reinterpret_cast(&size), sizeof(size)); - std::unique_ptr buf(new char[size]); - is.read(reinterpret_cast(buf.get()), size); - - framework::proto::VarType::TensorDesc desc; - desc.ParseFromArray(buf.get(), size); - - int memory_size = 1; - for (long long l : desc.dims()) { - memory_size *= l; - } - - int type_size = 0; - switch (desc.data_type()) { - case framework::proto::VarType::FP16: - type_size = 2; - break; - case framework::proto::VarType::FP32: - type_size = 4; - break; - case framework::proto::VarType::FP64: - type_size = 8; - break; - case framework::proto::VarType::INT32: - type_size = 4; - break; - case framework::proto::VarType::INT64: - type_size = 8; - break; - case framework::proto::VarType::BOOL: - type_size = 1; - break; - default: - break; - } - - void *memory = malloc(memory_size * type_size); - is.read(static_cast(memory), memory_size * type_size); - is.close(); - } else { - // TODO - } - } - } + originProgramDesc->Description("program: "); -#endif + paddle_mobile__framework__proto__program_desc__free_unpacked(c_program, NULL); return program; } @@ -440,7 +333,7 @@ void Executor::InitMemory() { auto tensor = var->template GetMutable(); LoadMemory(tensor, program_.model_path + "/" + var_desc->Name()); } else { - if (var_desc->GetType() == framework::proto::VarType::LOD_TENSOR) { + if (var_desc->Type() == framework::VARTYPE_TYPE_LOD_TENSOR) { auto tensor = var->template GetMutable(); tensor->template mutable_data(); } diff --git a/src/io.h b/src/io.h index f58c7d312c..ce4272439b 100644 --- a/src/io.h +++ b/src/io.h @@ -33,7 +33,7 @@ class Loader : PaddleMobileObject { const framework::Program Load(const std::string &dirname); private: - void LoadVar(framework::LoDTensor *tensor, const std::string &file_path); + void LoadVar(framework::Variable *variable, const framework::VarDesc &var_desc, const std::string &file_path); }; template diff --git a/test/executor_for_test.h b/test/executor_for_test.h index 27866e1ac5..33bd62cd24 100644 --- a/test/executor_for_test.h +++ b/test/executor_for_test.h @@ -55,6 +55,7 @@ class Executor4Test : public Executor { for (std::shared_ptr block_desc : blocks) { std::vector> ops = block_desc->Ops(); for (std::shared_ptr op : ops) { + if (op->Type() == op_type) { std::shared_ptr op_ptr = std::make_shared( op->Type(), op->GetInputs(), op->GetOutputs(), op->GetAttrMap(), -- GitLab