diff --git a/paddle/fluid/framework/CMakeLists.txt b/paddle/fluid/framework/CMakeLists.txt index cc7938b2ac07f11ceb7f33a2e37380d1e2ed2072..d998109df21f585bc4905e00e59fe07247fd3f5e 100644 --- a/paddle/fluid/framework/CMakeLists.txt +++ b/paddle/fluid/framework/CMakeLists.txt @@ -56,9 +56,9 @@ else() cc_test(mixed_vector_test SRCS mixed_vector_test.cc DEPS place memory device_context tensor) endif() if (NOT WIN32) -cc_library(lod_tensor SRCS lod_tensor.cc DEPS ddim place tensor framework_proto recordio) +cc_library(lod_tensor SRCS lod_tensor.cc DEPS ddim place tensor framework_proto recordio version) else() -cc_library(lod_tensor SRCS lod_tensor.cc DEPS ddim place tensor framework_proto) +cc_library(lod_tensor SRCS lod_tensor.cc DEPS ddim place tensor framework_proto version) endif (NOT WIN32) cc_test(lod_tensor_test SRCS lod_tensor_test.cc DEPS lod_tensor memory) @@ -116,7 +116,11 @@ cc_library(operator SRCS operator.cc DEPS op_info device_context tensor scope gl endif(NOT WIN32) cc_test(operator_test SRCS operator_test.cc DEPS operator op_registry device_context) -cc_library(proto_desc SRCS var_desc.cc op_desc.cc block_desc.cc program_desc.cc DEPS shape_inference op_info operator glog) + +cc_library(version SRCS version.cc) +cc_test(version_test SRCS version_test.cc DEPS version) + +cc_library(proto_desc SRCS var_desc.cc op_desc.cc block_desc.cc program_desc.cc DEPS shape_inference op_info operator glog version) cc_library(op_registry SRCS op_registry.cc DEPS op_proto_maker op_info operator glog proto_desc) nv_test(op_registry_test SRCS op_registry_test.cc DEPS op_registry) diff --git a/paddle/fluid/framework/framework.proto b/paddle/fluid/framework/framework.proto index c6588435819a982166cf2d2368a82b4402fdc2bc..460401df5473f8650f450a2bd247a703d91b6048 100644 --- a/paddle/fluid/framework/framework.proto +++ b/paddle/fluid/framework/framework.proto @@ -16,6 +16,13 @@ syntax = "proto2"; option optimize_for = LITE_RUNTIME; package paddle.framework.proto; +// Any incompatible changes to ProgramDesc and its dependencies should +// raise the version defined version.h. +// +// Serailization and Deserialization codes should be modified in a way +// that supports old versions following the version and compatibility policy. +message Version { optional int64 version = 1 [ default = 0 ]; } + enum AttrType { INT = 0; FLOAT = 1; @@ -180,4 +187,8 @@ message BlockDesc { // for more details. // TODO(panyx0718): A model can have multiple programs. Need a // way to distinguish them. Maybe ID or name? -message ProgramDesc { repeated BlockDesc blocks = 1; } +message ProgramDesc { + repeated BlockDesc blocks = 1; + + optional Version version = 2; +} diff --git a/paddle/fluid/framework/lod_tensor.cc b/paddle/fluid/framework/lod_tensor.cc index adeb26e4e78693eb9760ec1e12e4b71ba3115d5b..1e7da9a69c7cbf8c13306656599a759515802b76 100644 --- a/paddle/fluid/framework/lod_tensor.cc +++ b/paddle/fluid/framework/lod_tensor.cc @@ -21,6 +21,7 @@ limitations under the License. */ #include "paddle/fluid/framework/framework.pb.h" #include "paddle/fluid/framework/lod_tensor.h" #include "paddle/fluid/framework/var_type.h" +#include "paddle/fluid/framework/version.h" #include "paddle/fluid/memory/memcpy.h" #include "paddle/fluid/memory/memory.h" @@ -251,8 +252,8 @@ void AppendLoD(LoD *lod, const LoD &lod_length) { void SerializeToStream(std::ostream &os, const LoDTensor &tensor, const platform::DeviceContext &dev_ctx) { { // the 1st field, uint32_t version for LoDTensor - constexpr uint32_t version = 0; - os.write(reinterpret_cast(&version), sizeof(version)); + os.write(reinterpret_cast(&kCurTensorVersion), + sizeof(kCurTensorVersion)); } { // the 2st field, LoD information @@ -281,6 +282,8 @@ void DeserializeFromStream(std::istream &is, LoDTensor *tensor, // the 1st field, unit32_t version for LoDTensor uint32_t version; is.read(reinterpret_cast(&version), sizeof(version)); + PADDLE_ENFORCE(framework::IsTensorVersionSupported(version), + "tensor version %u is not supported.", version); PADDLE_ENFORCE_EQ(version, 0U, "Only version 0 is supported"); } { diff --git a/paddle/fluid/framework/program_desc.cc b/paddle/fluid/framework/program_desc.cc index a63944eaee6132c1082947fddcad4e0d72e26df1..589905828f7793c614c0fe12259e9ba5ab11ceac 100644 --- a/paddle/fluid/framework/program_desc.cc +++ b/paddle/fluid/framework/program_desc.cc @@ -15,6 +15,7 @@ limitations under the License. */ #include "paddle/fluid/framework/program_desc.h" #include "paddle/fluid/framework/block_desc.h" #include "paddle/fluid/framework/feed_fetch_type.h" +#include "paddle/fluid/framework/version.h" namespace paddle { namespace framework { @@ -38,7 +39,10 @@ proto::ProgramDesc *ProgramDesc::Proto() { return &desc_; } +int64_t ProgramDesc::Version() const { return desc_.version().version(); } + ProgramDesc::ProgramDesc() { + desc_.mutable_version()->set_version(kCurProgramVersion); auto *block = desc_.mutable_blocks()->Add(); block->set_idx(kRootBlockIndex); block->set_parent_idx(kNoneBlockIndex); diff --git a/paddle/fluid/framework/program_desc.h b/paddle/fluid/framework/program_desc.h index a0e81cade18c0ca5eb1b98fee8325ae2d917d1a2..2ec0e9d7a0969d44f88c7407bfb8cd4646530147 100644 --- a/paddle/fluid/framework/program_desc.h +++ b/paddle/fluid/framework/program_desc.h @@ -57,6 +57,8 @@ class ProgramDesc { proto::ProgramDesc *Proto(); + int64_t Version() const; + // The output variable of feed_op is referenced as feed_target. // This function is used to collect the output variable's name of all // feed_ops. diff --git a/paddle/fluid/framework/version.cc b/paddle/fluid/framework/version.cc new file mode 100644 index 0000000000000000000000000000000000000000..81c0392bf3cc7378cec06a9de3ae81f2b221ecec --- /dev/null +++ b/paddle/fluid/framework/version.cc @@ -0,0 +1,36 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/framework/version.h" +#include + +namespace paddle { +namespace framework { +bool IsProgramVersionSupported(int64_t version) { + static int num_supported = + sizeof(kSupportedProgramVersion) / sizeof(kSupportedProgramVersion[0]); + return std::find(kSupportedProgramVersion, + kSupportedProgramVersion + num_supported, + version) != kSupportedProgramVersion + num_supported; +} + +bool IsTensorVersionSupported(uint32_t version) { + static int num_supported = + sizeof(kSupportedTensorVersion) / sizeof(kSupportedTensorVersion[0]); + return std::find(kSupportedTensorVersion, + kSupportedTensorVersion + num_supported, + version) != kSupportedTensorVersion + num_supported; +} +} // namespace framework +} // namespace paddle diff --git a/paddle/fluid/framework/version.h b/paddle/fluid/framework/version.h new file mode 100644 index 0000000000000000000000000000000000000000..9945bc58c69df8456ff3d1aa0c777970bdbdbf98 --- /dev/null +++ b/paddle/fluid/framework/version.h @@ -0,0 +1,47 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include + +#pragma once + +namespace paddle { +namespace framework { + +// Note: +// Program and Tensor that pass the IsXXXVersionSupported should +// be supported by the current codes. Otherwise, it's a compatibility +// bug. + +// The program version the current codes generate. +constexpr int64_t kCurProgramVersion = 0; + +// The program version that was generated by previous or current codes +// and supported by current codes. +constexpr int64_t kSupportedProgramVersion[] = {0}; + +// Due to historical reasons, tensor version use uint32_t. +// The tensor version the current codes generate. +constexpr uint32_t kCurTensorVersion = 0; + +// The tensor version that was generated by previous or current codes +// and supported by current codes. +constexpr uint32_t kSupportedTensorVersion[] = {0}; + +bool IsProgramVersionSupported(int64_t version); + +bool IsTensorVersionSupported(uint32_t version); + +} // namespace framework +} // namespace paddle diff --git a/paddle/fluid/framework/version_test.cc b/paddle/fluid/framework/version_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..e8c5f256000522af976bbf487741a586f1abc439 --- /dev/null +++ b/paddle/fluid/framework/version_test.cc @@ -0,0 +1,30 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/framework/version.h" +#include "gtest/gtest.h" + +namespace paddle { +namespace framework { +TEST(Version, Basic) { + EXPECT_TRUE(IsProgramVersionSupported(0)); + EXPECT_FALSE(IsProgramVersionSupported(1)); + EXPECT_FALSE(IsProgramVersionSupported(-1)); + + EXPECT_TRUE(IsTensorVersionSupported(0)); + EXPECT_FALSE(IsTensorVersionSupported(1)); + EXPECT_FALSE(IsTensorVersionSupported(-1)); +} +} // namespace framework +} // namespace paddle diff --git a/paddle/fluid/inference/io.cc b/paddle/fluid/inference/io.cc index cef7b2a7e3a29da05628d7540f5545dc9adda27e..e246a06fd079d837ac321197914c9f70b528f2c8 100644 --- a/paddle/fluid/inference/io.cc +++ b/paddle/fluid/inference/io.cc @@ -20,6 +20,7 @@ limitations under the License. */ #include "paddle/fluid/framework/block_desc.h" #include "paddle/fluid/framework/feed_fetch_type.h" #include "paddle/fluid/framework/op_registry.h" +#include "paddle/fluid/framework/version.h" #include "paddle/fluid/platform/cpu_helper.h" #include "paddle/fluid/pybind/pybind.h" @@ -124,6 +125,9 @@ std::unique_ptr Load(framework::Executor* executor, std::unique_ptr main_program( new framework::ProgramDesc(program_desc_str)); + PADDLE_ENFORCE(framework::IsProgramVersionSupported(main_program->Version()), + "model version %ld is not supported.", + main_program->Version()); LoadPersistables(executor, scope, *main_program, dirname, ""); return main_program; @@ -138,6 +142,9 @@ std::unique_ptr Load( std::unique_ptr main_program( new framework::ProgramDesc(program_desc_str)); + PADDLE_ENFORCE(framework::IsProgramVersionSupported(main_program->Version()), + "model version %ld is not supported.", + main_program->Version()); LoadPersistables(executor, scope, *main_program, "", param_filename); return main_program; diff --git a/paddle/fluid/pybind/protobuf.cc b/paddle/fluid/pybind/protobuf.cc index f21f8d23f99c27529b2ed1995c92fd4eee4a5807..67501186d150171728194f23bc02d2c014848dd7 100644 --- a/paddle/fluid/pybind/protobuf.cc +++ b/paddle/fluid/pybind/protobuf.cc @@ -137,7 +137,10 @@ void BindProgramDesc(pybind11::module *m) { PADDLE_ENFORCE(desc->ParseFromString(data), "Fail to parse ProgramDesc from string. This could " "be a bug of Paddle."); - }); + }) + .def("_version", [](pd::ProgramDesc &self) -> int64_t { + return self.Proto()->version().version(); + }); } void BindBlockDesc(pybind11::module *m) { diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index 5b20b87174e42f4dfdd22214e8f9dd20c7296374..20fc08e21dadc12be8903476df374abb5caecf61 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -33,6 +33,7 @@ limitations under the License. */ #include "paddle/fluid/framework/prune.h" #include "paddle/fluid/framework/reader.h" #include "paddle/fluid/framework/selected_rows.h" +#include "paddle/fluid/framework/version.h" #include "paddle/fluid/operators/activation_op.h" #include "paddle/fluid/operators/reader/lod_tensor_blocking_queue.h" #include "paddle/fluid/platform/enforce.h" @@ -530,6 +531,8 @@ All parameter, weight, gradient are variables in Paddle. m.def("set_feed_variable", framework::SetFeedVariable); m.def("get_fetch_variable", framework::GetFetchVariable); + m.def("_is_program_version_supported", IsProgramVersionSupported); + BindProgramDesc(&m); BindBlockDesc(&m); BindVarDsec(&m); diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py index b0e0d27ff7a0c603523065d34169b1b73eabdac3..8892606486ee97bb085e642e89fce872e5ba1f7e 100644 --- a/python/paddle/fluid/framework.py +++ b/python/paddle/fluid/framework.py @@ -1564,6 +1564,9 @@ class Program(object): """ return self.desc + def _version(self): + return self.desc._version() + def clone(self, for_test=False): """ Create a new, duplicated program. diff --git a/python/paddle/fluid/io.py b/python/paddle/fluid/io.py index 5c4ec99c533829240ac0bcc9647acb870f3412f8..656fafa0cb54d70e0eba8ec2bef21488c50d8d94 100644 --- a/python/paddle/fluid/io.py +++ b/python/paddle/fluid/io.py @@ -750,6 +750,10 @@ def load_inference_model(dirname, program_desc_str = f.read() program = Program.parse_from_string(program_desc_str) + if not core._is_program_version_supported(program._version()): + raise ValueError("Unsupported program version: %d\n" % + program._version()) + # Binary data also need versioning. load_persistables(executor, dirname, program, params_filename) if pserver_endpoints: